Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions src/config/loading/config_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,32 +20,45 @@ pub struct ConfigBuilderLoader {

impl ConfigBuilderLoader {
/// Sets whether to interpolate environment variables in the config.
#[must_use]
pub const fn interpolate_env(mut self, interpolate: bool) -> Self {
self.interpolate_env = interpolate;
self
}

/// Sets the secrets map for secret interpolation.
#[must_use]
pub fn secrets(mut self, secrets: HashMap<String, String>) -> Self {
self.secrets = secrets;
self
}

/// Sets whether to allow empty configuration.
#[must_use]
pub const fn allow_empty(mut self, allow_empty: bool) -> Self {
self.builder.allow_empty = allow_empty;
self
}

/// Builds the ConfigBuilderLoader and loads configuration from the specified paths.
/// Builds the `ConfigBuilderLoader` and loads configuration from the specified paths.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
pub fn load_from_paths(
self,
config_paths: &[super::ConfigPath],
) -> Result<ConfigBuilder, Vec<String>> {
super::loader_from_paths(self, config_paths)
}

/// Builds the ConfigBuilderLoader and loads configuration from an input reader.
/// Builds the `ConfigBuilderLoader` and loads configuration from an input reader.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
pub fn load_from_input<R: Read>(
self,
input: R,
Expand Down Expand Up @@ -113,7 +126,7 @@ impl Process for ConfigBuilderLoader {
None => {
self.builder.append(deserialize_config_map(map)?)?;
}
};
}

Ok(())
}
Expand Down
23 changes: 20 additions & 3 deletions src/config/loading/interpolation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,15 @@ use serde_json::Value;
use super::representation::ConfigMap;

/// Interpolates environment variables in string leaves without changing keys or value types.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::implicit_hasher,
reason = "Keep the current collection type and API bounds during the lint rollout."
)]
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
pub fn interpolate_config_map_with_env_vars(
map: &ConfigMap,
vars: &HashMap<String, String>,
Expand Down Expand Up @@ -73,6 +82,15 @@ pub static ENVIRONMENT_VARIABLE_INTERPOLATION_REGEX: LazyLock<Regex> = LazyLock:
});

/// Result<interpolated config, errors>
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::implicit_hasher,
reason = "Keep the current collection type and API bounds during the lint rollout."
)]
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
pub fn interpolate(input: &str, vars: &HashMap<String, String>) -> Result<String, Vec<String>> {
let mut errors = Vec::new();

Expand All @@ -83,7 +101,7 @@ pub fn interpolate(input: &str, vars: &HashMap<String, String>) -> Result<String
caps.get(1)
.or_else(|| caps.get(2))
.map(|m| m.as_str())
.map(|name| {
.map_or("$", |name| {
// Get the value and check for newlines (LF or CR)
let val = vars.get(name).and_then(|v| {
if v.contains(['\n', '\r']) {
Expand Down Expand Up @@ -125,7 +143,6 @@ pub fn interpolate(input: &str, vars: &HashMap<String, String>) -> Result<String
}),
}
})
.unwrap_or("$")
.to_string()
})
.into_owned();
Expand Down Expand Up @@ -188,7 +205,7 @@ mod test {
("FOOBAR".into(), "cats".into()),
// Java commonly uses .s in env var names
("FOO.BAR".into(), "turtles".into()),
("EMPTY".into(), "".into()),
("EMPTY".into(), String::new()),
]
.into_iter()
.collect();
Expand Down
35 changes: 31 additions & 4 deletions src/config/loading/loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ impl ComponentHint {

/// Joins a component sub-folder to a provided path, for traversal. Since `Self` is a
/// `Copy`, this is more efficient to pass by value than ref.
#[must_use]
pub fn join_path(self, path: &Path) -> PathBuf {
path.join(self.as_component_field())
}
Expand All @@ -49,7 +50,10 @@ impl ComponentHint {
pub(super) mod process {
use std::io::Read;

use super::*;
use super::{
ComponentHint, ConfigMap, Format, Path, Value, component_name, deserialize_config,
merge_with_value, open_file, read_dir,
};

/// This trait contains methods that deserialize files/folders. There are a few methods
/// in here with subtly different names that can be hidden from public view, hence why
Expand All @@ -71,6 +75,15 @@ pub(super) mod process {

/// Helper method used by other methods to recursively handle file/dir loading, merging
/// values against a provided configuration map.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::manual_let_else,
reason = "Keep the existing branching and control flow during the lint rollout."
)]
#[allow(
clippy::unnecessary_debug_formatting,
reason = "Preserve the existing diagnostic text and escaping behavior."
)]
fn load_dir_into(
&mut self,
path: &Path,
Expand All @@ -94,8 +107,7 @@ pub(super) mod process {
if !entry
.file_name()
.and_then(|name| name.to_str())
.map(|name| name.starts_with('.'))
.unwrap_or(false)
.is_some_and(|name| name.starts_with('.'))
{
folders.push(entry);
}
Expand All @@ -106,7 +118,7 @@ pub(super) mod process {
"Could not read entry in config dir: {path:?}, {err}."
));
}
};
}
}

for entry in files {
Expand Down Expand Up @@ -217,6 +229,11 @@ where
/// Consumes Self, and returns the final, deserialized `T`.
fn take(self) -> T;

// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
fn load_from_str<R: std::io::Read>(
&mut self,
input: R,
Expand All @@ -230,6 +247,11 @@ where

/// Deserializes a file with the provided format, and makes the result available via `take`.
/// Returns a vector of non-fatal warnings on success, or a vector of error strings on failure.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
fn load_from_file(&mut self, path: &Path, format: Format) -> Result<(), Vec<String>> {
if let Some((_, map)) = self.load_file(path, format)? {
self.merge(map, None)?;
Expand All @@ -241,6 +263,11 @@ where

/// Deserializes a dir with the provided format, and makes the result available via `take`.
/// Returns a vector of non-fatal warnings on success, or a vector of error strings on failure.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
fn load_from_dir(&mut self, path: &Path) -> Result<(), Vec<String>> {
// Iterator containing component-specific sub-folders to attempt traversing into.
let hints = [
Expand Down
73 changes: 59 additions & 14 deletions src/config/loading/mod.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
#![warn(clippy::pedantic)]

mod config_builder;
pub(crate) mod interpolation;
mod loader;
Expand Down Expand Up @@ -34,6 +36,11 @@ static ALLOW_ENV_VAR_INTERPOLATION: OnceLock<bool> = OnceLock::new();

/// Sets whether environment variable interpolation is enabled for the process.
/// Must be called exactly once at startup before any config loading.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_panics_doc,
reason = "Audit and document the existing panic conditions separately from lint enforcement."
)]
pub fn set_env_var_interpolation(allow: bool) {
ALLOW_ENV_VAR_INTERPOLATION
.set(allow)
Expand All @@ -56,7 +63,7 @@ pub(super) fn component_name<P: AsRef<Path> + Debug>(path: P) -> Result<String,
path.as_ref()
.file_stem()
.and_then(|name| name.to_str())
.map(|name| name.to_string())
.map(std::string::ToString::to_string)
.ok_or_else(|| vec![format!("Couldn't get component name for file: {path:?}")])
}

Expand Down Expand Up @@ -91,11 +98,16 @@ pub fn merge_path_lists(

/// Expand a list of paths (potentially containing glob patterns) into real
/// config paths, replacing it with the default paths when empty.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_panics_doc,
reason = "Audit and document the existing panic conditions separately from lint enforcement."
)]
pub fn process_paths(config_paths: &[ConfigPath]) -> Option<Vec<ConfigPath>> {
let starting_paths = if !config_paths.is_empty() {
config_paths.to_owned()
} else {
let starting_paths = if config_paths.is_empty() {
default_config_paths()
} else {
config_paths.to_owned()
};

let mut paths = Vec::new();
Expand Down Expand Up @@ -125,7 +137,7 @@ pub fn process_paths(config_paths: &[ConfigPath]) -> Option<Vec<ConfigPath>> {
}
ConfigPath::Dir(_) => {
for path in matches {
paths.push(ConfigPath::Dir(path))
paths.push(ConfigPath::Dir(path));
}
}
}
Expand All @@ -143,6 +155,11 @@ pub fn process_paths(config_paths: &[ConfigPath]) -> Option<Vec<ConfigPath>> {
Some(paths)
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
pub fn load_from_paths(config_paths: &[ConfigPath]) -> Result<Config, Vec<String>> {
let builder = ConfigBuilderLoader::default().load_from_paths(config_paths)?;
let (config, build_warnings) = builder.build_with_warnings()?;
Expand All @@ -157,6 +174,11 @@ pub fn load_from_paths(config_paths: &[ConfigPath]) -> Result<Config, Vec<String
/// Loads a configuration from paths. Handle secret replacement and if a provider is present
/// in the builder, the config is used as bootstrapping for a remote source. Otherwise,
/// provider instantiation is skipped.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
pub async fn load_from_paths_with_provider_and_secrets(
config_paths: &[ConfigPath],
signal_handler: &mut signal::SignalHandler,
Expand Down Expand Up @@ -196,6 +218,11 @@ pub(crate) async fn load_builder_from_paths_with_secrets(
.load_from_paths(config_paths)
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
pub async fn load_from_str_with_secrets(
input: &str,
format: Format,
Expand Down Expand Up @@ -240,7 +267,7 @@ where
L: Loader<T> + Process,
R: std::io::Read,
{
loader.load_from_str(input, format).map(|_| loader.take())
loader.load_from_str(input, format).map(|()| loader.take())
}

/// Iterators over `ConfigPaths`, and processes a file/dir according to a provided `Loader`.
Expand All @@ -265,14 +292,12 @@ where
) {
Ok(()) => {}
Err(errs) => errors.extend(errs),
};
}
ConfigPath::Dir(path) => {
match loader.load_from_dir(path) {
Ok(()) => {}
Err(errs) => errors.extend(errs),
};
}
}
ConfigPath::Dir(path) => match loader.load_from_dir(path) {
Ok(()) => {}
Err(errs) => errors.extend(errs),
},
}
}

Expand All @@ -284,12 +309,22 @@ where
}

/// Uses `SourceLoader` to process `ConfigPaths`, deserializing to a JSON object.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
pub fn load_source_from_paths(
config_paths: &[ConfigPath],
) -> Result<serde_json::Map<String, serde_json::Value>, Vec<String>> {
loader_from_paths(SourceLoader::new(), config_paths)
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
pub fn load_from_str(input: &str, format: Format) -> Result<Config, Vec<String>> {
let builder = load_from_inputs(std::iter::once((input.as_bytes(), format)))?;
let (config, build_warnings) = builder.build_with_warnings()?;
Expand All @@ -310,7 +345,7 @@ fn load_from_inputs(
for (input, format) in inputs {
if let Err(errs) = load(input, format).and_then(|n| config.append(n)) {
// TODO: add back paths
errors.extend(errs.iter().map(|e| e.to_string()));
errors.extend(errs.iter().cloned());
}
}

Expand All @@ -321,6 +356,11 @@ fn load_from_inputs(
}
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
pub fn prepare_input<R: std::io::Read>(
mut input: R,
interpolate_env: bool,
Expand Down Expand Up @@ -349,6 +389,11 @@ pub fn prepare_input<R: std::io::Read>(
}
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::missing_errors_doc,
reason = "Audit and document the existing error contracts separately from lint enforcement."
)]
pub fn load<R: std::io::Read, T>(input: R, format: Format) -> Result<T, Vec<String>>
where
T: serde::de::DeserializeOwned,
Expand Down
5 changes: 5 additions & 0 deletions src/config/loading/representation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,11 @@ mod tests {
_count: u64,
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::unnecessary_wraps,
reason = "Preserve the existing return type and caller contracts during the lint rollout."
)]
fn default_optional_value() -> Option<String> {
Some("default".to_string())
}
Expand Down
Loading
Loading