From 2872d81fafac341272bf8e1cadd9b503fac715d4 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 15 Jul 2025 06:24:49 +0300 Subject: [PATCH 01/12] Enabling failure store for log data streams --- .../datastreams/LogsDataStreamIT.java | 68 ++++++++++++++++ .../data_stream/260_logs_failure_store.yml | 81 +++++++++++++++++++ .../src/main/resources/logs@settings.json | 5 ++ 3 files changed, 154 insertions(+) create mode 100644 modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 9a3d3bd2fc6e4..d553fa4303540 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -16,6 +16,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -740,6 +741,73 @@ public void testIgnoreDynamicBeyondLimit() throws Exception { assertThat(ignored.stream().filter(i -> i.startsWith("field") == false).toList(), empty()); } + @SuppressWarnings("unchecked") + public void testFailureStoreWithInvalidFieldType() throws Exception { + String dataStreamName = "logs-app-with-failure-store"; + createDataStream(client, dataStreamName); + + indexDoc(client, dataStreamName, """ + { + "@timestamp": "2023-11-30T12:00:00Z", + "message": "This is a valid message" + } + """); + + // invalid document (message as an object instead of string) + indexDoc(client, dataStreamName, """ + { + "@timestamp": "2023-11-30T12:01:00Z", + "message": { + "nested": "This should fail because message should be a string" + } + } + """); + + refreshAllIndices(); + + Request dsInfoRequest = new Request("GET", "/_data_stream/" + dataStreamName); + Map dsInfoResponse = entityAsMap(client.performRequest(dsInfoRequest)); + List> dataStreams = (List>) dsInfoResponse.get("data_streams"); + Map dataStream = dataStreams.getFirst(); + Map failureStoreInfo = (Map) dataStream.get("failure_store"); + assertNotNull(failureStoreInfo); + assertThat(failureStoreInfo.get("enabled"), is(true)); + List> failureIndices = (List>) failureStoreInfo.get("indices"); + + assertThat(failureIndices, not(empty())); + String failureIndex = (String) failureIndices.getFirst().get("index_name"); + assertThat(failureIndex, matchesRegex("\\.fs-" + dataStreamName + "-.*")); + + // query the failure store index + Request failureStoreQuery = new Request("GET", "/" + failureIndex + "/_search"); + failureStoreQuery.setJsonEntity(""" + { + "query": { + "match_all": {} + } + } + """); + Map failureStoreResponse = entityAsMap(client.performRequest(failureStoreQuery)); + Map hits = (Map) failureStoreResponse.get("hits"); + List> hitsList = (List>) hits.get("hits"); + + // Verify the failed document is in the failure store + assertThat(hitsList.size(), is(1)); + Map failedDoc = (Map) hitsList.getFirst().get("_source"); + Map document = (Map) failedDoc.get("document"); + assertNotNull(document); + Map source = (Map) document.get("source"); + assertNotNull(source); + Map message = (Map) source.get("message"); + assertNotNull(message); + assertThat(message.get("nested"), equalTo("This should fail because message should be a string")); + Map error = (Map) failedDoc.get("error"); + assertNotNull(error); + assertEquals("document_parsing_exception", error.get("type")); + String errorMessage = (String) error.get("message"); + assertThat(errorMessage, containsString("failed to parse field [message] of type [match_only_text] in document with id")); + } + @Override protected String indexTemplateName() { return "logs"; diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml new file mode 100644 index 0000000000000..cd69f40ccbcf6 --- /dev/null +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml @@ -0,0 +1,81 @@ +--- +setup: + - requires: + cluster_features: [ "gte_v9.1.99" ] + reason: "failure store became enabled by default for log data streams in 9.2.0" + + - do: + indices.create_data_stream: + name: logs-app-default +--- +teardown: + - do: + indices.delete_data_stream: + name: logs-app-default + ignore: 404 + +--- +"Test logs-*-* data streams have failure store enabled by default": + # index a valid document (string message) + - do: + index: + index: logs-app-default + refresh: true + body: + '@timestamp': '2023-01-01T12:00:00Z' + host: + name: 'server-01' + severity: 'INFO' + message: "Application started successfully" + - match: { result: created } + + - do: + indices.get_data_stream: + name: logs-app-default + - match: { data_streams.0.name: logs-app-default } + - length: { data_streams.0.indices: 1 } + - set: { data_streams.0.indices.0.index_name: idx0name } + - match: { data_streams.0.failure_store.enabled: true } + - length: { data_streams.0.failure_store.indices: 0 } + + # index a document with (object message, causing a mapping conflict) + - do: + index: + index: logs-app-default + refresh: true + body: + '@timestamp': '2023-01-01T12:01:00Z' + host: + name: 'server-02' + severity: 'ERROR' + message: + struct: + value: 42 + - match: { result: 'created' } + - match: { failure_store: used} + + - do: + indices.get_data_stream: + name: logs-app-default + - length: { data_streams.0.failure_store.indices: 1 } + - set: { data_streams.0.failure_store.indices.0.index_name: fs0name } + + - do: + search: + index: $idx0name + body: + query: + match_all: {} + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.severity: "INFO" } + - match: { hits.hits.0._source.message: "Application started successfully" } + + - do: + search: + index: $fs0name + body: + query: + match_all: {} + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.document.source.message.struct.value: 42 } + - match: { hits.hits.0._source.error.type: "document_parsing_exception" } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json index ca2659b8d8dea..898e1b88cc632 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json @@ -14,6 +14,11 @@ }, "default_pipeline": "logs@default-pipeline" } + }, + "data_stream_options": { + "failure_store": { + "enabled": true + } } }, "_meta": { From 7c53ce757a48190299437389ec2698f78cafdcea Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 15 Jul 2025 06:39:20 +0300 Subject: [PATCH 02/12] Update docs/changelog/131261.yaml --- docs/changelog/131261.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/131261.yaml diff --git a/docs/changelog/131261.yaml b/docs/changelog/131261.yaml new file mode 100644 index 0000000000000..18795d0b9e8d1 --- /dev/null +++ b/docs/changelog/131261.yaml @@ -0,0 +1,6 @@ +pr: 131261 +summary: Enable failure store for log data streams +area: Data streams +type: feature +issues: + - 131105 From 011f8629b7cc5399b09825493bc95b645a0fd568 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 15 Jul 2025 07:50:37 +0300 Subject: [PATCH 03/12] Adjusting tests --- .../xpack/core/DataStreamRestIT.java | 16 ++++++++-------- .../rest-api-spec/test/stack/10_basic.yml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java index c4c8d0c460afc..3c380fa546c8e 100644 --- a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java +++ b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java @@ -70,20 +70,20 @@ public void testDSXpackUsage() throws Exception { assertThat(failureStoreStats.get("effectively_enabled_count"), equalTo(0)); assertThat(failureStoreStats.get("failure_indices_count"), equalTo(0)); assertBusy(() -> { - Map logsTemplate = (Map) ((List) getLocation("/_index_template/logs").get("index_templates")).get(0); - assertThat(logsTemplate, notNullValue()); - assertThat(logsTemplate.get("name"), equalTo("logs")); - assertThat(((Map) logsTemplate.get("index_template")).get("data_stream"), notNullValue()); + Map syntheticsTemplate = (Map) ((List) getLocation("/_index_template/synthetics").get("index_templates")).get(0); + assertThat(syntheticsTemplate, notNullValue()); + assertThat(syntheticsTemplate.get("name"), equalTo("synthetics")); + assertThat(((Map) syntheticsTemplate.get("index_template")).get("data_stream"), notNullValue()); }); putFailureStoreTemplate(); // Create a data stream - Request indexRequest = new Request("POST", "/logs-mysql-default/_doc"); + Request indexRequest = new Request("POST", "/synthetics-myapp-default/_doc"); indexRequest.setJsonEntity("{\"@timestamp\": \"2020-01-01\"}"); client().performRequest(indexRequest); // Roll over the data stream - Request rollover = new Request("POST", "/logs-mysql-default/_rollover"); + Request rollover = new Request("POST", "/synthetics-myapp-default/_rollover"); client().performRequest(rollover); // Create failure store data stream @@ -105,10 +105,10 @@ public void testDSXpackUsage() throws Exception { assertThat(failureStoreStats.get("effectively_enabled_count"), equalTo(1)); assertThat(failureStoreStats.get("failure_indices_count"), equalTo(1)); - // Enable the failure store for logs-mysql-default using the cluster setting... + // Enable the failure store for synthetics-myapp-default using the cluster setting... updateClusterSettings( Settings.builder() - .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "logs-mysql-default") + .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "synthetics-myapp-default") .build() ); // ...and assert that it counts towards effectively_enabled_count but not explicitly_enabled_count: diff --git a/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml b/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml index 3b8bfa9dfb7c8..8d6bf14e419a0 100644 --- a/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml +++ b/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml @@ -276,7 +276,6 @@ setup: data_stream.namespace: "namespace1" - do: - catch: bad_request index: index: logs-dataset0-namespace1 body: @@ -284,6 +283,7 @@ setup: data_stream.type: "metrics" data_stream.dataset: "dataset0" data_stream.namespace: "namespace1" + - match: { failure_store: used } - do: catch: bad_request From cac9c821f2c91b272af991e222ce4f4207d737af Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Wed, 16 Jul 2025 10:15:41 +0300 Subject: [PATCH 04/12] Update docs/changelog/131261.yaml Co-authored-by: Lee Hinman --- docs/changelog/131261.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/131261.yaml b/docs/changelog/131261.yaml index 18795d0b9e8d1..a45dffc1db366 100644 --- a/docs/changelog/131261.yaml +++ b/docs/changelog/131261.yaml @@ -1,5 +1,5 @@ pr: 131261 -summary: Enable failure store for log data streams +summary: Enable failure store for log-*-* data streams area: Data streams type: feature issues: From 71a4898feef86aeb15f002a5325c87e19f065bda Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Wed, 16 Jul 2025 10:18:59 +0300 Subject: [PATCH 05/12] Bump StackTemplateRegistry#REGISTRY_VERSION --- .../org/elasticsearch/xpack/stack/StackTemplateRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index a2f8dd91176e2..bd1178937b463 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -37,7 +37,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { // The stack template registry version. This number must be incremented when we make changes // to built-in templates. - public static final int REGISTRY_VERSION = 16; + public static final int REGISTRY_VERSION = 17; public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version"; public static final Setting STACK_TEMPLATES_ENABLED = Setting.boolSetting( From af5a001ae08da86a86568c8a31f6a452c1729714 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Wed, 16 Jul 2025 10:29:47 +0300 Subject: [PATCH 06/12] Replace explicit indices with API conventions in yaml tests --- .../test/data_stream/260_logs_failure_store.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml index cd69f40ccbcf6..e769529a71974 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml @@ -34,7 +34,6 @@ teardown: name: logs-app-default - match: { data_streams.0.name: logs-app-default } - length: { data_streams.0.indices: 1 } - - set: { data_streams.0.indices.0.index_name: idx0name } - match: { data_streams.0.failure_store.enabled: true } - length: { data_streams.0.failure_store.indices: 0 } @@ -58,11 +57,10 @@ teardown: indices.get_data_stream: name: logs-app-default - length: { data_streams.0.failure_store.indices: 1 } - - set: { data_streams.0.failure_store.indices.0.index_name: fs0name } - do: search: - index: $idx0name + index: logs-app-default::data body: query: match_all: {} @@ -72,7 +70,7 @@ teardown: - do: search: - index: $fs0name + index: logs-app-default::failures body: query: match_all: {} From 781436d2938954fcba613b7c30d5ec6b8195c4d1 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Thu, 24 Jul 2025 19:42:48 +0300 Subject: [PATCH 07/12] Update docs/changelog/131261.yaml --- docs/changelog/131261.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog/131261.yaml b/docs/changelog/131261.yaml index a45dffc1db366..a57b8b2370612 100644 --- a/docs/changelog/131261.yaml +++ b/docs/changelog/131261.yaml @@ -4,3 +4,7 @@ area: Data streams type: feature issues: - 131105 +highlight: + title: Enable failure store for log data streams + body: "Closes #131105 Now let's see which tests this breaks 🙂" + notable: true From 0e3647d89edc136a22aae0d9d8254802884167f9 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Sun, 27 Jul 2025 05:49:55 +0300 Subject: [PATCH 08/12] Update 131261.yaml --- docs/changelog/131261.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/changelog/131261.yaml b/docs/changelog/131261.yaml index a57b8b2370612..b20bcf4f41393 100644 --- a/docs/changelog/131261.yaml +++ b/docs/changelog/131261.yaml @@ -6,5 +6,4 @@ issues: - 131105 highlight: title: Enable failure store for log data streams - body: "Closes #131105 Now let's see which tests this breaks 🙂" notable: true From 789ca17d6c8ea43481b52a727e74c55da06dbe7c Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Sun, 27 Jul 2025 06:12:17 +0300 Subject: [PATCH 09/12] Fixing release notes body --- docs/changelog/131261.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog/131261.yaml b/docs/changelog/131261.yaml index b20bcf4f41393..4da739c149a4c 100644 --- a/docs/changelog/131261.yaml +++ b/docs/changelog/131261.yaml @@ -6,4 +6,8 @@ issues: - 131105 highlight: title: Enable failure store for log data streams + body: |- + The failure store is now enabled for `log-*-*` data streams. This means that such data streams will now store invalid documents in a + dedicated failure index instead of rejecting them. Clients that previously received a `400 Bad Request` response when indexing invalid + documents will now receive a `201 Created` response, that would also contain the marker: `failure_store: used`. notable: true From a7eb8d2c69d31194c6fe7e5fb0071340fed9694b Mon Sep 17 00:00:00 2001 From: Luca Wintergerst Date: Mon, 28 Jul 2025 08:43:27 +0200 Subject: [PATCH 10/12] Update 131261.yaml tweak release notes --- docs/changelog/131261.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/changelog/131261.yaml b/docs/changelog/131261.yaml index 4da739c149a4c..fcd3c98576eb9 100644 --- a/docs/changelog/131261.yaml +++ b/docs/changelog/131261.yaml @@ -1,13 +1,13 @@ pr: 131261 -summary: Enable failure store for log-*-* data streams +summary: Enable Failure Store for new logs-*-* data streams area: Data streams type: feature issues: - 131105 highlight: - title: Enable failure store for log data streams + title: Enable Failure Store for new logs data streams body: |- - The failure store is now enabled for `log-*-*` data streams. This means that such data streams will now store invalid documents in a - dedicated failure index instead of rejecting them. Clients that previously received a `400 Bad Request` response when indexing invalid - documents will now receive a `201 Created` response, that would also contain the marker: `failure_store: used`. + The [Failure Store](https://www.elastic.co/docs/manage-data/data-store/data-streams/failure-store) is now enabled by default for new logs data streams matching the pattern `logs-*-*`. This means that such data streams will now store invalid documents in a + dedicated failure index instead of rejecting them, allowing better visibility and control over data quality issues without loosing data. This can be [enabled manually](https://www.elastic.co/docs/manage-data/data-store/data-streams/failure-store#set-up-failure-store-existing) for existing data streams. + Note: With the failure store enabled, the http response code clients receive when indexing invalid documents will change from `400 Bad Request` to `201 Created`, with an additional response attribute `"failure_store" : "used"`. notable: true From 10af65e2a1ce4dfc5c301a96769184a1b3845cd7 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 29 Jul 2025 10:20:44 +0300 Subject: [PATCH 11/12] Using cluster feature instead of synthetic version for yaml test --- .../elasticsearch/datastreams/DataStreamFeatures.java | 9 ++++++++- .../test/data_stream/260_logs_failure_store.yml | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java index c7033b0d560ad..38f03557ca9c7 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java @@ -28,6 +28,8 @@ public class DataStreamFeatures implements FeatureSpecification { public static final NodeFeature LOGS_STREAM_FEATURE = new NodeFeature("logs_stream"); + public static final NodeFeature FAILURE_STORE_IN_LOG_DATA_STREAMS = new NodeFeature("logs_data_streams.failure_store.enabled"); + @Override public Set getFeatures() { return Set.of(DataStream.DATA_STREAM_FAILURE_STORE_FEATURE); @@ -35,6 +37,11 @@ public Set getFeatures() { @Override public Set getTestFeatures() { - return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX, LOGS_STREAM_FEATURE); + return Set.of( + DATA_STREAM_FAILURE_STORE_TSDB_FIX, + DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX, + LOGS_STREAM_FEATURE, + FAILURE_STORE_IN_LOG_DATA_STREAMS + ); } } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml index e769529a71974..21dd18e8cb28d 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml @@ -1,7 +1,7 @@ --- setup: - requires: - cluster_features: [ "gte_v9.1.99" ] + cluster_features: [ "logs_data_streams.failure_store.enabled" ] reason: "failure store became enabled by default for log data streams in 9.2.0" - do: From 595c6b36b85c32774c6b7e9859ec37c6348d1b4f Mon Sep 17 00:00:00 2001 From: Luca Wintergerst Date: Tue, 29 Jul 2025 10:12:56 +0200 Subject: [PATCH 12/12] add proper docs links instead of static links --- docs/changelog/131261.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog/131261.yaml b/docs/changelog/131261.yaml index fcd3c98576eb9..c9aa986d612b3 100644 --- a/docs/changelog/131261.yaml +++ b/docs/changelog/131261.yaml @@ -7,7 +7,7 @@ issues: highlight: title: Enable Failure Store for new logs data streams body: |- - The [Failure Store](https://www.elastic.co/docs/manage-data/data-store/data-streams/failure-store) is now enabled by default for new logs data streams matching the pattern `logs-*-*`. This means that such data streams will now store invalid documents in a - dedicated failure index instead of rejecting them, allowing better visibility and control over data quality issues without loosing data. This can be [enabled manually](https://www.elastic.co/docs/manage-data/data-store/data-streams/failure-store#set-up-failure-store-existing) for existing data streams. + The [Failure Store](docs-content://manage-data/data-store/data-streams/failure-store.md) is now enabled by default for new logs data streams matching the pattern `logs-*-*`. This means that such data streams will now store invalid documents in a + dedicated failure index instead of rejecting them, allowing better visibility and control over data quality issues without loosing data. This can be [enabled manually](docs-content://manage-data/data-store/data-streams/failure-store.md#set-up-failure-store-existing) for existing data streams. Note: With the failure store enabled, the http response code clients receive when indexing invalid documents will change from `400 Bad Request` to `201 Created`, with an additional response attribute `"failure_store" : "used"`. notable: true