diff --git a/nativelink-proto/com/github/trace_machina/nativelink/remote_execution/worker_api.proto b/nativelink-proto/com/github/trace_machina/nativelink/remote_execution/worker_api.proto index 3c5e25982..e73512971 100644 --- a/nativelink-proto/com/github/trace_machina/nativelink/remote_execution/worker_api.proto +++ b/nativelink-proto/com/github/trace_machina/nativelink/remote_execution/worker_api.proto @@ -125,7 +125,17 @@ message ActionResourceUsage { /// The worker ID that observed the resource usage. string worker_id = 4; - reserved 5; // NextId. + /// Total user-mode CPU time consumed by the action's process group in + /// nanoseconds, when the platform sampler collects it. Zero when + /// unavailable. + uint64 cpu_user_ns = 5; + + /// Total kernel-mode CPU time consumed by the action's process group in + /// nanoseconds, when the platform sampler collects it. Zero when + /// unavailable. + uint64 cpu_system_ns = 6; + + reserved 7; // NextId. } /// Result sent back from the server when a node connects. diff --git a/nativelink-proto/genproto/com.github.trace_machina.nativelink.remote_execution.pb.rs b/nativelink-proto/genproto/com.github.trace_machina.nativelink.remote_execution.pb.rs index 7eba7cf68..80ee5193a 100644 --- a/nativelink-proto/genproto/com.github.trace_machina.nativelink.remote_execution.pb.rs +++ b/nativelink-proto/genproto/com.github.trace_machina.nativelink.remote_execution.pb.rs @@ -104,6 +104,16 @@ pub struct ActionResourceUsage { /// / The worker ID that observed the resource usage. #[prost(string, tag = "4")] pub worker_id: ::prost::alloc::string::String, + /// / Total user-mode CPU time consumed by the action's process group in + /// / nanoseconds, when the platform sampler collects it. Zero when + /// / unavailable. + #[prost(uint64, tag = "5")] + pub cpu_user_ns: u64, + /// / Total kernel-mode CPU time consumed by the action's process group in + /// / nanoseconds, when the platform sampler collects it. Zero when + /// / unavailable. + #[prost(uint64, tag = "6")] + pub cpu_system_ns: u64, } /// / Result sent back from the server when a node connects. #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] diff --git a/nativelink-worker/src/running_actions_manager.rs b/nativelink-worker/src/running_actions_manager.rs index f0e5000fc..f34771801 100644 --- a/nativelink-worker/src/running_actions_manager.rs +++ b/nativelink-worker/src/running_actions_manager.rs @@ -99,37 +99,53 @@ const REQUIRES_WORKER_PROTOCOL_PROPERTY: &str = "requires-worker-protocol"; const DEFAULT_HISTORICAL_RESULTS_STRATEGY: UploadCacheResultsStrategy = UploadCacheResultsStrategy::FailuresOnly; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] const RESOURCE_USAGE_SAMPLE_INTERVAL: Duration = Duration::from_millis(250); -#[cfg(target_os = "linux")] +/// Resource usage aggregated by the platform sampler over one action's +/// process group. CPU totals are collected on macOS only today; Linux +/// reports memory only. +#[cfg(any(target_os = "linux", target_os = "macos"))] +#[derive(Clone, Copy, Default)] +struct SampledResourceUsage { + peak_memory_kb: u64, + cpu_user_ns: u64, + cpu_system_ns: u64, +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] struct ActionResourceUsageSampler { stop_tx: watch::Sender, - handle: tokio::task::JoinHandle, + handle: tokio::task::JoinHandle, } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn start_action_resource_usage_sampler(pgid: u32) -> ActionResourceUsageSampler { let (stop_tx, stop_rx) = watch::channel(false); let handle = background_spawn!( "action_resource_usage_sampler", - sample_action_peak_memory_kb(pgid, stop_rx) + sample_action_resource_usage(pgid, stop_rx) ); ActionResourceUsageSampler { stop_tx, handle } } -#[cfg(target_os = "linux")] -async fn finish_action_resource_usage_sampler(sampler: ActionResourceUsageSampler) -> Option { +#[cfg(any(target_os = "linux", target_os = "macos"))] +async fn finish_action_resource_usage_sampler( + sampler: ActionResourceUsageSampler, +) -> Option { let _ = sampler.stop_tx.send(true); sampler.handle.await.ok() } #[cfg(target_os = "linux")] -async fn sample_action_peak_memory_kb(pgid: u32, mut stop_rx: watch::Receiver) -> u64 { - let mut peak_memory_kb = 0; +async fn sample_action_resource_usage( + pgid: u32, + mut stop_rx: watch::Receiver, +) -> SampledResourceUsage { + let mut usage = SampledResourceUsage::default(); loop { if let Some(memory_kb) = sample_process_group_memory_kb(pgid) { - peak_memory_kb = peak_memory_kb.max(memory_kb); + usage.peak_memory_kb = usage.peak_memory_kb.max(memory_kb); } else if !Path::new(&format!("/proc/{pgid}")).exists() { // The group leader has been reaped and no member process remains, // so the action is finished. @@ -144,7 +160,7 @@ async fn sample_action_peak_memory_kb(pgid: u32, mut stop_rx: watch::Receiver { if changed.is_ok() && *stop_rx.borrow() { if let Some(memory_kb) = sample_process_group_memory_kb(pgid) { - peak_memory_kb = peak_memory_kb.max(memory_kb); + usage.peak_memory_kb = usage.peak_memory_kb.max(memory_kb); } break; } @@ -152,7 +168,136 @@ async fn sample_action_peak_memory_kb(pgid: u32, mut stop_rx: watch::Receiver {} } } - peak_memory_kb + usage +} + +/// Samples the action's process group via `proc_listpids` and +/// `proc_pid_rusage`, mirroring the Linux `/proc` sampler: peak resident +/// memory is the maximum over per-tick sums, and CPU totals are the sum of +/// the last-seen cumulative user/system time per member pid (cumulative per +/// process, so members that exit between ticks retain their last observation; +/// up to one sample interval of final CPU per process can be missed). +#[cfg(target_os = "macos")] +async fn sample_action_resource_usage( + pgid: u32, + mut stop_rx: watch::Receiver, +) -> SampledResourceUsage { + let mut usage = SampledResourceUsage::default(); + let mut cpu_ns_by_pid: HashMap = HashMap::new(); + loop { + let members = sample_process_group_usage(pgid, &mut usage, &mut cpu_ns_by_pid); + if members == 0 { + // No member process remains, so the action is finished. + break; + } + + if *stop_rx.borrow() { + break; + } + + tokio::select! { + changed = stop_rx.changed() => { + if changed.is_ok() && *stop_rx.borrow() { + sample_process_group_usage(pgid, &mut usage, &mut cpu_ns_by_pid); + break; + } + } + () = tokio::time::sleep(RESOURCE_USAGE_SAMPLE_INTERVAL) => {} + } + } + usage.cpu_user_ns = cpu_ns_by_pid.values().map(|&(user_ns, _)| user_ns).sum(); + usage.cpu_system_ns = cpu_ns_by_pid + .values() + .map(|&(_, system_ns)| system_ns) + .sum(); + usage +} + +/// Converts mach absolute time units (the unit of `rusage_info` time fields +/// on Apple Silicon; 1:1 with nanoseconds on Intel) to nanoseconds. +#[cfg(target_os = "macos")] +fn mach_ticks_to_ns(ticks: u64) -> u64 { + static TIMEBASE: std::sync::OnceLock<(u32, u32)> = std::sync::OnceLock::new(); + let &(numer, denom) = TIMEBASE.get_or_init(|| { + let mut info = libc::mach_timebase_info { numer: 0, denom: 0 }; + // SAFETY: `mach_timebase_info` fills the passed struct. + if unsafe { libc::mach_timebase_info(&raw mut info) } != 0 || info.denom == 0 { + return (1, 1); + } + (info.numer, info.denom) + }); + u64::try_from(u128::from(ticks) * u128::from(numer) / u128::from(denom)).unwrap_or(u64::MAX) +} + +/// Sums resident memory and records cumulative CPU time for every process in +/// the action's process group. Returns the number of member processes seen. +#[cfg(target_os = "macos")] +fn sample_process_group_usage( + pgid: u32, + usage: &mut SampledResourceUsage, + cpu_ns_by_pid: &mut HashMap, +) -> usize { + /// From `libproc.h` (stable public API); not exposed by the libc crate. + const PROC_PGRP_ONLY: u32 = 2; + const MAX_GROUP_PIDS: usize = 4096; + + let mut pids = [0i32; MAX_GROUP_PIDS]; + let buffer_size_bytes = i32::try_from(size_of_val(&pids)).unwrap_or(i32::MAX); + // SAFETY: `proc_listpids` writes at most `buffer_size_bytes` bytes of + // pids into the buffer and returns the number of bytes written. + let bytes_written = unsafe { + libc::proc_listpids( + PROC_PGRP_ONLY, + pgid, + pids.as_mut_ptr().cast::(), + buffer_size_bytes, + ) + }; + let Ok(bytes_written) = usize::try_from(bytes_written) else { + return 0; + }; + let pid_count = (bytes_written / size_of::()).min(MAX_GROUP_PIDS); + + let mut total_rss_bytes: u64 = 0; + let mut members = 0; + for &member_pid in &pids[..pid_count] { + if member_pid <= 0 { + continue; + } + let mut info = core::mem::MaybeUninit::::zeroed(); + // SAFETY: the `RUSAGE_INFO_V2` flavor fills exactly a + // `rusage_info_v2`. + let result = unsafe { + libc::proc_pid_rusage( + member_pid, + libc::RUSAGE_INFO_V2, + info.as_mut_ptr().cast::(), + ) + }; + if result != 0 { + // The process exited between listing and sampling. + continue; + } + // SAFETY: `proc_pid_rusage` returned success, so `info` is + // initialized. + let info = unsafe { info.assume_init() }; + total_rss_bytes = total_rss_bytes.saturating_add(info.ri_resident_size); + // Times are cumulative mach absolute time units; keep the latest + // observation per pid so CPU of already-exited members is retained + // in the totals. + cpu_ns_by_pid.insert( + member_pid, + ( + mach_ticks_to_ns(info.ri_user_time), + mach_ticks_to_ns(info.ri_system_time), + ), + ); + members += 1; + } + if members > 0 { + usage.peak_memory_kb = usage.peak_memory_kb.max(total_rss_bytes / 1024); + } + members } /// Sums the resident memory of every process in the action's process group. @@ -218,6 +363,18 @@ pub fn parse_pgid_from_stat(stat: &str) -> Option { after_comm.split_whitespace().nth(2)?.parse().ok() } +/// Sends `SIGKILL` to every process in the action's process group so helper +/// processes spawned by the action cannot outlive it on timeout or kill. +/// `ESRCH` (the group is already gone) is expected and ignored. +#[cfg(target_os = "macos")] +fn kill_process_group(pgid: u32) { + let Ok(pgid) = i32::try_from(pgid) else { + return; + }; + // SAFETY: `kill(2)` with a negated pgid signals the whole process group. + let _ = unsafe { libc::kill(-pgid, libc::SIGKILL) }; +} + /// Valid string reasons for a failure. /// Note: If these change, the documentation should be updated. #[derive(Debug, Deserialize)] @@ -1480,15 +1637,20 @@ impl RunningActionImpl { }); } } - - // Run the action as its own process-group leader (pgid == child - // pid). The resource-usage sampler attributes memory by process - // group, so this keeps the whole action together — including - // processes reparented to the worker when an intermediate shell - // exits — and never conflates it with the worker's own group. - command_builder.process_group(0); } + // Run the action as its own process-group leader (pgid == child + // pid). The resource-usage sampler attributes usage by process + // group, so this keeps the whole action together — including + // processes reparented to the worker when an intermediate shell + // exits — and never conflates it with the worker's own group, and + // kill paths can signal the entire tree via `kill(-pgid)`. On macOS + // this maps to posix_spawn's `POSIX_SPAWN_SETPGROUP` attribute (the + // standard library only falls back to fork/exec for uid/gid/ + // `pre_exec`/groups/chroot), so the fast spawn path is preserved. + #[cfg(unix)] + command_builder.process_group(0); + let mut child_process = command_builder .spawn() .err_tip(|| format!("Could not execute command {args:?}"))?; @@ -1512,9 +1674,13 @@ impl RunningActionImpl { child_process, ); - #[cfg(target_os = "linux")] - let mut maybe_resource_usage_sampler = - child_process.id().map(start_action_resource_usage_sampler); + // The spawned child is its own process-group leader, so its pid is + // the pgid of the whole action tree. + #[cfg(any(target_os = "linux", target_os = "macos"))] + let maybe_pgid = child_process.id(); + + #[cfg(any(target_os = "linux", target_os = "macos"))] + let mut maybe_resource_usage_sampler = maybe_pgid.map(start_action_resource_usage_sampler); let mut child_process_guard = guard(child_process, |mut child_process| { let result: Result, std::io::Error> = @@ -1530,6 +1696,10 @@ impl RunningActionImpl { ); background_spawn!("running_actions_manager_kill_child_process", async move { drop(child_process.kill().await); + #[cfg(target_os = "macos")] + if let Some(pgid) = maybe_pgid { + kill_process_group(pgid); + } }); } } @@ -1576,6 +1746,10 @@ impl RunningActionImpl { "Could not kill process in RunningActionsManager for action timeout", ); } + #[cfg(target_os = "macos")] + if let Some(pgid) = maybe_pgid { + kill_process_group(pgid); + } { let joined_command = args.join(OsStr::new(" ")); let command = joined_command.to_string_lossy(); @@ -1639,21 +1813,26 @@ impl RunningActionImpl { exit_code }); - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] let resource_usage = match maybe_resource_usage_sampler.take() { Some(sampler) => finish_action_resource_usage_sampler(sampler) .await - .and_then(|peak_memory_kb| { - (peak_memory_kb > 0).then_some(ActionResourceUsage { - peak_memory_kb, - sampled: true, - operation_id: String::new(), - worker_id: String::new(), - }) + .and_then(|usage| { + (usage.peak_memory_kb > 0 + || usage.cpu_user_ns > 0 + || usage.cpu_system_ns > 0) + .then_some(ActionResourceUsage { + peak_memory_kb: usage.peak_memory_kb, + sampled: true, + operation_id: String::new(), + worker_id: String::new(), + cpu_user_ns: usage.cpu_user_ns, + cpu_system_ns: usage.cpu_system_ns, + }) }), None => None, }; - #[cfg(not(target_os = "linux"))] + #[cfg(not(any(target_os = "linux", target_os = "macos")))] let resource_usage = None; info!(?args, "Command complete"); @@ -1688,6 +1867,10 @@ impl RunningActionImpl { "Could not kill process", ); } + #[cfg(target_os = "macos")] + if let Some(pgid) = maybe_pgid { + kill_process_group(pgid); + } { let mut state = self.state.lock(); state.error = Error::merge_option(state.error.take(), Some(Error::new( diff --git a/nativelink-worker/tests/running_actions_manager_test.rs b/nativelink-worker/tests/running_actions_manager_test.rs index cd3711feb..07dfe00e7 100644 --- a/nativelink-worker/tests/running_actions_manager_test.rs +++ b/nativelink-worker/tests/running_actions_manager_test.rs @@ -4088,13 +4088,15 @@ exit 1 count=0 while IFS= read -r request; do if ! pwd -P >/dev/null 2>&1; then - printf '{"exitCode":1,"output":"worker cwd was removed"}\n' + printf '{"exitCode":1,"output":"worker cwd was removed"} +' continue fi count=$((count + 1)) sandbox_dir=$(printf '%s' "$request" | sed -n 's/.*"sandboxDir":"\([^"]*\)".*/\1/p') printf '%s' "$count" > "$sandbox_dir/count.txt" - printf '{"exitCode":0,"output":"count=%s"}\n' "$count" + printf '{"exitCode":0,"output":"count=%s"} +' "$count" done "#; let worker_script_bytes = Bytes::from(worker_script); @@ -5120,4 +5122,247 @@ done assert_eq!(parse_pgid_from_stat("123 (only) S"), None); // too few fields assert_eq!(parse_pgid_from_stat(""), None); } + + /// A killed action must not leave helper processes behind: the kill path + /// signals the entire process group, so a grandchild that would survive a + /// leader-only `SIGKILL` (it reparents to init but keeps its pgid) must + /// also die. + #[cfg(target_os = "macos")] + #[nativelink_test] + async fn kill_reaps_entire_process_group() -> Result<(), Box> { + const WORKER_ID: &str = "foo_worker_id"; + + let (_, _, cas_store, ac_store) = setup_stores().await?; + let root_action_directory = make_temp_path("root_action_directory"); + fs::create_dir_all(&root_action_directory).await?; + + let running_actions_manager = + Arc::new(RunningActionsManagerImpl::new(RunningActionsManagerArgs { + root_action_directory: root_action_directory.clone(), + execution_configuration: ExecutionConfiguration::default(), + cas_store: cas_store.clone(), + ac_store: Some(Store::new(ac_store.clone())), + historical_store: Store::new(cas_store.clone()), + upload_action_result_config: &UploadActionResultConfig { + upload_ac_results_strategy: UploadCacheResultsStrategy::Never, + ..Default::default() + }, + max_action_timeout: Duration::MAX, + max_upload_timeout: Duration::from_secs(DEFAULT_MAX_UPLOAD_TIMEOUT), + max_cleanup_wait: Duration::from_secs(DEFAULT_MAX_CLEANUP_WAIT), + max_cleanup_backoff: Duration::from_millis(DEFAULT_MAX_CLEANUP_BACKOFF), + timeout_handled_externally: false, + directory_cache: None, + })?); + + // A unique sleep duration doubles as a pgrep-able marker for the + // grandchild process. + let marker_seconds = format!("9{}.271828", std::process::id() % 1000); + let process_started_file = { + let tmp_dir = make_temp_path("process_started_dir"); + fs::create_dir_all(&tmp_dir).await.unwrap(); + format!("{tmp_dir}/process_started") + }; + let command = Command { + arguments: vec![ + "sh".to_string(), + "-c".to_string(), + format!("sleep {marker_seconds} & touch {process_started_file} && sleep 24h"), + ], + output_paths: vec![], + working_directory: ".".to_string(), + environment_variables: vec![EnvironmentVariable { + name: "PATH".to_string(), + value: env::var("PATH").unwrap(), + }], + ..Default::default() + }; + let command_digest = serialize_and_upload_message( + &command, + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + let input_root_digest = serialize_and_upload_message( + &Directory::default(), + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + let action = Action { + command_digest: Some(command_digest.into()), + input_root_digest: Some(input_root_digest.into()), + ..Default::default() + }; + let action_digest = serialize_and_upload_message( + &action, + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + + let running_action_impl = running_actions_manager + .clone() + .create_and_add_action( + WORKER_ID.to_string(), + StartExecute { + execute_request: Some(ExecuteRequest { + action_digest: Some(action_digest.into()), + ..Default::default() + }), + operation_id: OperationId::default().to_string(), + queued_timestamp: Some(make_system_time(1000).into()), + platform: action.platform.clone(), + worker_id: WORKER_ID.to_string(), + }, + ) + .await?; + + let run_action_fut = run_action(running_action_impl); + tokio::pin!(run_action_fut); + + // Wait until the leader has spawned its children. + loop { + assert_eq!(futures::poll!(&mut run_action_fut), Poll::Pending); + tokio::task::yield_now().await; + match fs::metadata(&process_started_file).await { + Ok(_) => break, + Err(err) => { + assert_eq!(err.code, Code::NotFound, "Unknown error {err:?}"); + tokio::time::sleep(Duration::from_millis(1)).await; + } + } + } + + drop(futures::join!(run_action_fut, running_actions_manager.kill_all()).0); + + // The grandchild must be reaped by the group kill; poll briefly since + // signal delivery is asynchronous. + let marker = format!("sleep {marker_seconds}"); + let mut grandchild_alive = true; + for _ in 0..300 { + let output = tokio::process::Command::new("pgrep") + .args(["-f", &marker]) + .output() + .await?; + if !output.status.success() { + grandchild_alive = false; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!( + !grandchild_alive, + "Expected the grandchild '{marker}' to be killed with the process group" + ); + Ok(()) + } + + /// The macOS resource-usage sampler must report both peak memory and CPU + /// time for an action that burns CPU in a helper process. + #[cfg(target_os = "macos")] + #[nativelink_test] + async fn resource_usage_reports_cpu_and_memory() -> Result<(), Box> { + const WORKER_ID: &str = "foo_worker_id"; + + let (_, _, cas_store, ac_store) = setup_stores().await?; + let root_action_directory = make_temp_path("root_action_directory"); + fs::create_dir_all(&root_action_directory).await?; + + let running_actions_manager = + Arc::new(RunningActionsManagerImpl::new(RunningActionsManagerArgs { + root_action_directory: root_action_directory.clone(), + execution_configuration: ExecutionConfiguration::default(), + cas_store: cas_store.clone(), + ac_store: Some(Store::new(ac_store.clone())), + historical_store: Store::new(cas_store.clone()), + upload_action_result_config: &UploadActionResultConfig { + upload_ac_results_strategy: UploadCacheResultsStrategy::Never, + ..Default::default() + }, + max_action_timeout: Duration::MAX, + max_upload_timeout: Duration::from_secs(DEFAULT_MAX_UPLOAD_TIMEOUT), + max_cleanup_wait: Duration::from_secs(DEFAULT_MAX_CLEANUP_WAIT), + max_cleanup_backoff: Duration::from_millis(DEFAULT_MAX_CLEANUP_BACKOFF), + timeout_handled_externally: false, + directory_cache: None, + })?); + + // Burn CPU in a helper for ~0.7s so the 250ms sampler observes it. + let command = Command { + arguments: vec![ + "sh".to_string(), + "-c".to_string(), + "yes > /dev/null & Y=$!; sleep 0.7; kill $Y 2>/dev/null; true".to_string(), + ], + output_paths: vec![], + working_directory: ".".to_string(), + environment_variables: vec![EnvironmentVariable { + name: "PATH".to_string(), + value: env::var("PATH").unwrap(), + }], + ..Default::default() + }; + let command_digest = serialize_and_upload_message( + &command, + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + let input_root_digest = serialize_and_upload_message( + &Directory::default(), + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + let action = Action { + command_digest: Some(command_digest.into()), + input_root_digest: Some(input_root_digest.into()), + ..Default::default() + }; + let action_digest = serialize_and_upload_message( + &action, + cas_store.as_pin(), + &mut DigestHasherFunc::Sha256.hasher(), + ) + .await?; + + let running_action = running_actions_manager + .clone() + .create_and_add_action( + WORKER_ID.to_string(), + StartExecute { + execute_request: Some(ExecuteRequest { + action_digest: Some(action_digest.into()), + ..Default::default() + }), + operation_id: OperationId::default().to_string(), + queued_timestamp: Some(make_system_time(1000).into()), + platform: action.platform.clone(), + worker_id: WORKER_ID.to_string(), + }, + ) + .await? + .prepare_action() + .and_then(RunningAction::execute) + .and_then(RunningAction::upload_results) + .await?; + + let resource_usage = running_action + .resource_usage() + .expect("expected sampled resource usage on macOS"); + running_action.cleanup().await?; + + assert!(resource_usage.sampled, "usage must be marked sampled"); + assert!( + resource_usage.peak_memory_kb > 0, + "expected nonzero peak memory, got {resource_usage:?}" + ); + let total_cpu_ns = resource_usage.cpu_user_ns + resource_usage.cpu_system_ns; + assert!( + (10_000_000..120_000_000_000).contains(&total_cpu_ns), + "expected plausible CPU time (10ms..120s), got {total_cpu_ns}ns ({resource_usage:?})" + ); + Ok(()) + } } diff --git a/typos.toml b/typos.toml index d6efd4c10..a2aaf5c52 100644 --- a/typos.toml +++ b/typos.toml @@ -3,6 +3,8 @@ conly = "conly" # in-toto attestation format used by SLSA provenance (`*.intoto.jsonl`) intoto = "intoto" +# mach_timebase_info's numerator field on macOS +numer = "numer" [default] # Old wrong spelling support