feat(config)!: parse configuration before interpolation

changelog.d/parse_first_interpolation.breaking.md

@@ -0,0 +1,52 @@
+# Configuration is parsed before interpolation
+
+## Summary
+
+Vector now parses configuration files before substituting environment variables and
+`SECRET[backend.key]` references. Substitution applies only to string values, followed
+by conversion to the field's declared type. Quotes, newlines, and other configuration
+syntax in substituted values cannot add keys, components, or array elements.
+
+This removes structural interpolation, deprecated in 0.57.0. Environment-variable
+interpolation remains disabled by default and still requires
+`--dangerously-allow-env-var-interpolation` or
+`VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION=true`.
+
+## Migration
+
+Configuration must be valid YAML, TOML, or JSON before substitution. Quote placeholders
+in TOML and JSON values, including numeric and boolean fields. For example, change
+`count = ${MY_COUNT}` to `count = "${MY_COUNT}"`. The same applies to secret references.
+
+YAML string placeholders remain supported, including in numeric fields:
+
+```yaml
+sources:
+  demo:
+    type: demo_logs
+    format: json
+    count: "${MY_COUNT}"
+```
+
+Replace interpolated map keys, component names, and table headers with literal names.
+Placeholders in keys and comments are no longer substituted or collected as secrets.
+Values cannot expand into configuration fragments or multiple array elements. For
+example, replace a comma-separated `${VECTOR_INPUTS}` expansion with an explicit list:
+
+```yaml
+inputs:
+  - "${FIRST_INPUT}"
+  - "${SECOND_INPUT}"
+```
+
+Environment-variable and secret values are now inserted as literal strings. Remove
+escaping that was needed only for the outer configuration format. Multiline environment
+values are supported within a string field, but cannot inject configuration blocks.
+Escaping required by an embedded language, such as VRL, is still your responsibility.
+
+For configurations that rely on structural environment-variable expansion, preprocess
+the file with a tool such as `envsubst`, inspect the resulting configuration, and pass
+that static file to Vector. `envsubst` does not resolve `SECRET[...]` references or
+Vector's extended default/required-variable syntax.
+
+authors: pront

lib/codecs/src/decoding/framing/character_delimited.rs

@@ -1,5 +1,6 @@
 use bytes::{Buf, Bytes, BytesMut};
 use memchr::memchr;
+use serde_with::serde_as;
 use tokio_util::codec::Decoder;
 use tracing::{trace, warn};
 use vector_config::configurable_component;
@@ -54,12 +55,13 @@ impl CharacterDelimitedDecoderConfig {
 }
 
 /// Options for building a `CharacterDelimitedDecoder`.
+#[serde_as]
 #[configurable_component]
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub struct CharacterDelimitedDecoderOptions {
     /// The character that delimits byte sequences.
     #[configurable(metadata(docs::type_override = "ascii_char"))]
-    #[serde(with = "vector_core::serde::ascii_char")]
+    #[serde_as(as = "vector_core::serde::ascii_char::AsciiChar")]
     pub delimiter: u8,
 
     /// The maximum length of the byte buffer.
@@ -244,6 +246,28 @@ mod tests {
 
     use super::*;
 
+    #[test]
+    fn delimiter_schema_matches_its_ascii_string_representation() {
+        let schema = serde_json::to_value(
+            vector_config::schema::generate_root_schema::<CharacterDelimitedDecoderOptions>()
+                .unwrap(),
+        )
+        .unwrap();
+        assert_eq!(schema["properties"]["delimiter"]["type"], "string");
+
+        let input = serde_json::json!({"delimiter": "1"});
+        let options: CharacterDelimitedDecoderOptions =
+            serde_json::from_value(input.clone()).unwrap();
+        assert_eq!(options.delimiter, b'1');
+        assert_eq!(serde_json::to_value(options).unwrap(), input);
+        assert!(
+            serde_json::from_value::<CharacterDelimitedDecoderOptions>(
+                serde_json::json!({"delimiter": 1})
+            )
+            .is_err()
+        );
+    }
+
     #[test]
     fn decode() {
         let mut codec = CharacterDelimitedDecoder::new(b'\n');

lib/codecs/src/encoding/format/csv.rs

@@ -2,6 +2,7 @@ use bytes::BytesMut;
 use chrono::SecondsFormat;
 use csv_core::{WriteResult, Writer, WriterBuilder};
 use lookup::lookup_v2::ConfigTargetPath;
+use serde_with::serde_as;
 use tokio_util::codec::Encoder;
 use vector_config_macros::configurable_component;
 use vector_core::{
@@ -81,14 +82,15 @@ impl CsvSerializerConfig {
 }
 
 /// Config used to build a `CsvSerializer`.
+#[serde_as]
 #[configurable_component]
 #[derive(Debug, Clone)]
 pub struct CsvSerializerOptions {
     /// The field delimiter to use when writing CSV.
     #[configurable(metadata(docs::type_override = "ascii_char"))]
+    #[serde_as(as = "vector_core::serde::ascii_char::AsciiChar")]
     #[serde(
         default = "default_delimiter",
-        with = "vector_core::serde::ascii_char",
         skip_serializing_if = "vector_core::serde::is_default"
     )]
     pub delimiter: u8,
@@ -110,18 +112,18 @@ pub struct CsvSerializerOptions {
     ///
     /// To use this, `double_quotes` needs to be disabled as well; otherwise, this setting is ignored.
     #[configurable(metadata(docs::type_override = "ascii_char"))]
+    #[serde_as(as = "vector_core::serde::ascii_char::AsciiChar")]
     #[serde(
         default = "default_escape",
-        with = "vector_core::serde::ascii_char",
         skip_serializing_if = "vector_core::serde::is_default"
     )]
     pub escape: u8,
 
     /// The quote character to use when writing CSV.
     #[configurable(metadata(docs::type_override = "ascii_char"))]
+    #[serde_as(as = "vector_core::serde::ascii_char::AsciiChar")]
     #[serde(
         default = "default_escape",
-        with = "vector_core::serde::ascii_char",
         skip_serializing_if = "vector_core::serde::is_default"
     )]
     quote: u8,
@@ -323,6 +325,35 @@ mod tests {
 
     use super::*;
 
+    #[test]
+    fn ascii_field_schemas_match_their_serialized_values_and_defaults() {
+        let schema = serde_json::to_value(
+            vector_config::schema::generate_root_schema::<CsvSerializerOptions>().unwrap(),
+        )
+        .unwrap();
+        for (field, default) in [("delimiter", ","), ("escape", "\""), ("quote", "\"")] {
+            assert_eq!(schema["properties"][field]["type"], "string", "{field}");
+            assert_eq!(schema["properties"][field]["default"], default, "{field}");
+        }
+
+        for field in ["delimiter", "escape", "quote"] {
+            let mut input = serde_json::json!({"fields": ["message"]});
+            input[field] = serde_json::json!("1");
+            let options: CsvSerializerOptions = serde_json::from_value(input.clone()).unwrap();
+            assert_eq!(
+                serde_json::to_value(options).unwrap()[field],
+                "1",
+                "{field}"
+            );
+
+            input[field] = serde_json::json!(1);
+            assert!(
+                serde_json::from_value::<CsvSerializerOptions>(input).is_err(),
+                "{field}"
+            );
+        }
+    }
+
     fn make_event_with_fields(field_data: Vec<(&str, &str)>) -> (Vec<ConfigTargetPath>, Event) {
         let mut fields: Vec<ConfigTargetPath> = std::vec::Vec::new();
         let mut tree = ObjectMap::new();

lib/codecs/src/encoding/framing/character_delimited.rs

@@ -1,4 +1,5 @@
 use bytes::{BufMut, BytesMut};
+use serde_with::serde_as;
 use tokio_util::codec::Encoder;
 use vector_config::configurable_component;
 
@@ -29,12 +30,13 @@ impl CharacterDelimitedEncoderConfig {
 }
 
 /// Configuration for character-delimited framing.
+#[serde_as]
 #[configurable_component]
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct CharacterDelimitedEncoderOptions {
     /// The ASCII (7-bit) character that delimits byte sequences.
     #[configurable(metadata(docs::type_override = "ascii_char"))]
-    #[serde(with = "vector_core::serde::ascii_char")]
+    #[serde_as(as = "vector_core::serde::ascii_char::AsciiChar")]
     pub delimiter: u8,
 }
 
@@ -66,6 +68,28 @@ impl Encoder<()> for CharacterDelimitedEncoder {
 mod tests {
     use super::*;
 
+    #[test]
+    fn delimiter_schema_matches_its_ascii_string_representation() {
+        let schema = serde_json::to_value(
+            vector_config::schema::generate_root_schema::<CharacterDelimitedEncoderOptions>()
+                .unwrap(),
+        )
+        .unwrap();
+        assert_eq!(schema["properties"]["delimiter"]["type"], "string");
+
+        let input = serde_json::json!({"delimiter": "1"});
+        let options: CharacterDelimitedEncoderOptions =
+            serde_json::from_value(input.clone()).unwrap();
+        assert_eq!(options.delimiter, b'1');
+        assert_eq!(serde_json::to_value(options).unwrap(), input);
+        assert!(
+            serde_json::from_value::<CharacterDelimitedEncoderOptions>(
+                serde_json::json!({"delimiter": 1})
+            )
+            .is_err()
+        );
+    }
+
     #[test]
     fn encode() {
         let mut codec = CharacterDelimitedEncoder::new(b'\n');

lib/vector-config-common/src/constants.rs

@@ -29,6 +29,8 @@ pub const DOCS_META_TYPE_UNIT: &str = "docs::type_unit";
 pub const LOGICAL_NAME: &str = "logical_name";
 pub const METADATA: &str = "_metadata";
 pub const SERDE_ALIASES: &str = "serde::aliases";
+/// The custom deserializer requires a string; do not stringify other scalar types.
+pub const SERDE_STRING_ONLY: &str = "serde::string_only";
 pub const SERDE_VARIANT_ALIASES: &str = "serde::variant_aliases";
 
 /// Well-known component types.