Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions src/config/loading/config_builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,32 +20,39 @@ pub struct ConfigBuilderLoader {

impl ConfigBuilderLoader {
/// Sets whether to interpolate environment variables in the config.
#[must_use]
pub const fn interpolate_env(mut self, interpolate: bool) -> Self {
self.interpolate_env = interpolate;
self
}

/// Sets the secrets map for secret interpolation.
#[must_use]
pub fn secrets(mut self, secrets: HashMap<String, String>) -> Self {
self.secrets = secrets;
self
}

/// Sets whether to allow empty configuration.
#[must_use]
pub const fn allow_empty(mut self, allow_empty: bool) -> Self {
self.builder.allow_empty = allow_empty;
self
}

/// Builds the ConfigBuilderLoader and loads configuration from the specified paths.
/// Builds the `ConfigBuilderLoader` and loads configuration from the specified paths.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
pub fn load_from_paths(
self,
config_paths: &[super::ConfigPath],
) -> Result<ConfigBuilder, Vec<String>> {
super::loader_from_paths(self, config_paths)
}

/// Builds the ConfigBuilderLoader and loads configuration from an input reader.
/// Builds the `ConfigBuilderLoader` and loads configuration from an input reader.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
pub fn load_from_input<R: Read>(
self,
input: R,
Expand Down Expand Up @@ -113,7 +120,7 @@ impl Process for ConfigBuilderLoader {
None => {
self.builder.append(deserialize_config_map(map)?)?;
}
};
}

Ok(())
}
Expand Down
17 changes: 14 additions & 3 deletions src/config/loading/interpolation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ use serde_json::Value;
use super::representation::ConfigMap;

/// Interpolates environment variables in string leaves without changing keys or value types.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::implicit_hasher,
reason = "Preserve collection type and API bounds"
)]
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
pub fn interpolate_config_map_with_env_vars(
map: &ConfigMap,
vars: &HashMap<String, String>,
Expand Down Expand Up @@ -73,6 +79,12 @@ pub static ENVIRONMENT_VARIABLE_INTERPOLATION_REGEX: LazyLock<Regex> = LazyLock:
});

/// Result<interpolated config, errors>
// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::implicit_hasher,
reason = "Preserve collection type and API bounds"
)]
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
pub fn interpolate(input: &str, vars: &HashMap<String, String>) -> Result<String, Vec<String>> {
let mut errors = Vec::new();

Expand All @@ -83,7 +95,7 @@ pub fn interpolate(input: &str, vars: &HashMap<String, String>) -> Result<String
caps.get(1)
.or_else(|| caps.get(2))
.map(|m| m.as_str())
.map(|name| {
.map_or("$", |name| {
// Get the value and check for newlines (LF or CR)
let val = vars.get(name).and_then(|v| {
if v.contains(['\n', '\r']) {
Expand Down Expand Up @@ -125,7 +137,6 @@ pub fn interpolate(input: &str, vars: &HashMap<String, String>) -> Result<String
}),
}
})
.unwrap_or("$")
.to_string()
})
.into_owned();
Expand Down Expand Up @@ -188,7 +199,7 @@ mod test {
("FOOBAR".into(), "cats".into()),
// Java commonly uses .s in env var names
("FOO.BAR".into(), "turtles".into()),
("EMPTY".into(), "".into()),
("EMPTY".into(), String::new()),
]
.into_iter()
.collect();
Expand Down
23 changes: 19 additions & 4 deletions src/config/loading/loader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ impl ComponentHint {

/// Joins a component sub-folder to a provided path, for traversal. Since `Self` is a
/// `Copy`, this is more efficient to pass by value than ref.
#[must_use]
pub fn join_path(self, path: &Path) -> PathBuf {
path.join(self.as_component_field())
}
Expand All @@ -49,7 +50,10 @@ impl ComponentHint {
pub(super) mod process {
use std::io::Read;

use super::*;
use super::{
ComponentHint, ConfigMap, Format, Path, Value, component_name, deserialize_config,
merge_with_value, open_file, read_dir,
};

/// This trait contains methods that deserialize files/folders. There are a few methods
/// in here with subtly different names that can be hidden from public view, hence why
Expand All @@ -71,6 +75,12 @@ pub(super) mod process {

/// Helper method used by other methods to recursively handle file/dir loading, merging
/// values against a provided configuration map.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::manual_let_else, reason = "Preserve existing control flow")]
#[allow(
clippy::unnecessary_debug_formatting,
reason = "Preserve diagnostic text and escaping"
)]
fn load_dir_into(
&mut self,
path: &Path,
Expand All @@ -94,8 +104,7 @@ pub(super) mod process {
if !entry
.file_name()
.and_then(|name| name.to_str())
.map(|name| name.starts_with('.'))
.unwrap_or(false)
.is_some_and(|name| name.starts_with('.'))
{
folders.push(entry);
}
Expand All @@ -106,7 +115,7 @@ pub(super) mod process {
"Could not read entry in config dir: {path:?}, {err}."
));
}
};
}
}

for entry in files {
Expand Down Expand Up @@ -217,6 +226,8 @@ where
/// Consumes Self, and returns the final, deserialized `T`.
fn take(self) -> T;

// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
fn load_from_str<R: std::io::Read>(
&mut self,
input: R,
Expand All @@ -230,6 +241,8 @@ where

/// Deserializes a file with the provided format, and makes the result available via `take`.
/// Returns a vector of non-fatal warnings on success, or a vector of error strings on failure.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
fn load_from_file(&mut self, path: &Path, format: Format) -> Result<(), Vec<String>> {
if let Some((_, map)) = self.load_file(path, format)? {
self.merge(map, None)?;
Expand All @@ -241,6 +254,8 @@ where

/// Deserializes a dir with the provided format, and makes the result available via `take`.
/// Returns a vector of non-fatal warnings on success, or a vector of error strings on failure.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
fn load_from_dir(&mut self, path: &Path) -> Result<(), Vec<String>> {
// Iterator containing component-specific sub-folders to attempt traversing into.
let hints = [
Expand Down
46 changes: 32 additions & 14 deletions src/config/loading/mod.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
#![warn(clippy::pedantic)]

mod config_builder;
pub(crate) mod interpolation;
mod loader;
Expand Down Expand Up @@ -34,6 +36,8 @@ static ALLOW_ENV_VAR_INTERPOLATION: OnceLock<bool> = OnceLock::new();

/// Sets whether environment variable interpolation is enabled for the process.
/// Must be called exactly once at startup before any config loading.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_panics_doc, reason = "Panic documentation deferred")]
pub fn set_env_var_interpolation(allow: bool) {
ALLOW_ENV_VAR_INTERPOLATION
.set(allow)
Expand All @@ -56,7 +60,7 @@ pub(super) fn component_name<P: AsRef<Path> + Debug>(path: P) -> Result<String,
path.as_ref()
.file_stem()
.and_then(|name| name.to_str())
.map(|name| name.to_string())
.map(std::string::ToString::to_string)
.ok_or_else(|| vec![format!("Couldn't get component name for file: {path:?}")])
}

Expand Down Expand Up @@ -91,11 +95,13 @@ pub fn merge_path_lists(

/// Expand a list of paths (potentially containing glob patterns) into real
/// config paths, replacing it with the default paths when empty.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_panics_doc, reason = "Panic documentation deferred")]
pub fn process_paths(config_paths: &[ConfigPath]) -> Option<Vec<ConfigPath>> {
let starting_paths = if !config_paths.is_empty() {
config_paths.to_owned()
} else {
let starting_paths = if config_paths.is_empty() {
default_config_paths()
} else {
config_paths.to_owned()
};

let mut paths = Vec::new();
Expand Down Expand Up @@ -125,7 +131,7 @@ pub fn process_paths(config_paths: &[ConfigPath]) -> Option<Vec<ConfigPath>> {
}
ConfigPath::Dir(_) => {
for path in matches {
paths.push(ConfigPath::Dir(path))
paths.push(ConfigPath::Dir(path));
}
}
}
Expand All @@ -143,6 +149,8 @@ pub fn process_paths(config_paths: &[ConfigPath]) -> Option<Vec<ConfigPath>> {
Some(paths)
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
pub fn load_from_paths(config_paths: &[ConfigPath]) -> Result<Config, Vec<String>> {
let builder = ConfigBuilderLoader::default().load_from_paths(config_paths)?;
let (config, build_warnings) = builder.build_with_warnings()?;
Expand All @@ -157,6 +165,8 @@ pub fn load_from_paths(config_paths: &[ConfigPath]) -> Result<Config, Vec<String
/// Loads a configuration from paths. Handle secret replacement and if a provider is present
/// in the builder, the config is used as bootstrapping for a remote source. Otherwise,
/// provider instantiation is skipped.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
pub async fn load_from_paths_with_provider_and_secrets(
config_paths: &[ConfigPath],
signal_handler: &mut signal::SignalHandler,
Expand Down Expand Up @@ -196,6 +206,8 @@ pub(crate) async fn load_builder_from_paths_with_secrets(
.load_from_paths(config_paths)
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
pub async fn load_from_str_with_secrets(
input: &str,
format: Format,
Expand Down Expand Up @@ -240,7 +252,7 @@ where
L: Loader<T> + Process,
R: std::io::Read,
{
loader.load_from_str(input, format).map(|_| loader.take())
loader.load_from_str(input, format).map(|()| loader.take())
}

/// Iterators over `ConfigPaths`, and processes a file/dir according to a provided `Loader`.
Expand All @@ -265,14 +277,12 @@ where
) {
Ok(()) => {}
Err(errs) => errors.extend(errs),
};
}
ConfigPath::Dir(path) => {
match loader.load_from_dir(path) {
Ok(()) => {}
Err(errs) => errors.extend(errs),
};
}
}
ConfigPath::Dir(path) => match loader.load_from_dir(path) {
Ok(()) => {}
Err(errs) => errors.extend(errs),
},
}
}

Expand All @@ -284,12 +294,16 @@ where
}

/// Uses `SourceLoader` to process `ConfigPaths`, deserializing to a JSON object.
// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
pub fn load_source_from_paths(
config_paths: &[ConfigPath],
) -> Result<serde_json::Map<String, serde_json::Value>, Vec<String>> {
loader_from_paths(SourceLoader::new(), config_paths)
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
pub fn load_from_str(input: &str, format: Format) -> Result<Config, Vec<String>> {
let builder = load_from_inputs(std::iter::once((input.as_bytes(), format)))?;
let (config, build_warnings) = builder.build_with_warnings()?;
Expand All @@ -310,7 +324,7 @@ fn load_from_inputs(
for (input, format) in inputs {
if let Err(errs) = load(input, format).and_then(|n| config.append(n)) {
// TODO: add back paths
errors.extend(errs.iter().map(|e| e.to_string()));
errors.extend(errs.iter().cloned());
}
}

Expand All @@ -321,6 +335,8 @@ fn load_from_inputs(
}
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
pub fn prepare_input<R: std::io::Read>(
mut input: R,
interpolate_env: bool,
Expand Down Expand Up @@ -349,6 +365,8 @@ pub fn prepare_input<R: std::io::Read>(
}
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(clippy::missing_errors_doc, reason = "Error documentation deferred")]
pub fn load<R: std::io::Read, T>(input: R, format: Format) -> Result<T, Vec<String>>
where
T: serde::de::DeserializeOwned,
Expand Down
5 changes: 5 additions & 0 deletions src/config/loading/representation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,11 @@ mod tests {
_count: u64,
}

// https://github.com/vectordotdev/vector/issues/23659
#[allow(
clippy::unnecessary_wraps,
reason = "Preserve return type and caller contracts"
)]
fn default_optional_value() -> Option<String> {
Some("default".to_string())
}
Expand Down
Loading
Loading