diff --git a/config/_default/menus/main.en.yaml b/config/_default/menus/main.en.yaml index 8c2b14598b2eb..a625c24dd5fb8 100644 --- a/config/_default/menus/main.en.yaml +++ b/config/_default/menus/main.en.yaml @@ -5161,26 +5161,31 @@ menu: parent: observability_pipelines_log_volume_control identifier: observability_pipelines_log_volume_control_logstash weight: 1019 + - name: Socket (TCP or UDP) + url: observability_pipelines/set_up_pipelines/log_volume_control/socket/ + parent: observability_pipelines_log_volume_control + identifier: observability_pipelines_log_volume_control_socket + weight: 1020 - name: Splunk HTTP Event Collector url: observability_pipelines/set_up_pipelines/log_volume_control/splunk_hec/ parent: observability_pipelines_log_volume_control identifier: observability_pipelines_log_volume_control_splunk_hec - weight: 1020 + weight: 1021 - name: Splunk Forwarders (TCP) url: observability_pipelines/set_up_pipelines/log_volume_control/splunk_tcp/ parent: observability_pipelines_log_volume_control identifier: observability_pipelines_log_volume_control_splunk_tcp - weight: 1021 + weight: 1022 - name: Sumo Logic Hosted Collector url: observability_pipelines/set_up_pipelines/log_volume_control/sumo_logic_hosted_collector/ parent: observability_pipelines_log_volume_control identifier: observability_pipelines_log_volume_control_sumo_logic_hosted_collector - weight: 1022 + weight: 1023 - name: Syslog url: observability_pipelines/set_up_pipelines/log_volume_control/syslog/ parent: observability_pipelines_log_volume_control identifier: observability_pipelines_log_volume_control_syslog - weight: 1023 + weight: 1024 - name: Dual Ship Logs url: observability_pipelines/set_up_pipelines/dual_ship_logs/ parent: observability_pipelines_set_up_pipelines @@ -5231,26 +5236,31 @@ menu: parent: observability_pipelines_dual_ship_logs identifier: observability_pipelines_dual_ship_logs_logstash weight: 2019 + - name: Socket (TCP or UDP) + url: observability_pipelines/set_up_pipelines/dual_ship_logs/socket/ + parent: observability_pipelines_dual_ship_logs + identifier: observability_pipelines_dual_ship_logs_socket + weight: 2020 - name: Splunk HTTP Event Collector url: observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_hec/ parent: observability_pipelines_dual_ship_logs identifier: observability_pipelines_dual_ship_logs_splunk_hec - weight: 2020 + weight: 2021 - name: Splunk Forwarders (TCP) url: observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_tcp/ parent: observability_pipelines_dual_ship_logs identifier: observability_pipelines_dual_ship_logs_splunk_tcp - weight: 2021 + weight: 2022 - name: Sumo Logic Hosted Collector url: observability_pipelines/set_up_pipelines/dual_ship_logs/sumo_logic_hosted_collector/ parent: observability_pipelines_dual_ship_logs identifier: observability_pipelines_dual_ship_logs_sumo_logic_hosted_collector - weight: 2022 + weight: 2023 - name: Syslog url: observability_pipelines/set_up_pipelines/dual_ship_logs/syslog/ parent: observability_pipelines_dual_ship_logs identifier: observability_pipelines_dual_ship_logs_syslog - weight: 2021 + weight: 2024 - name: Archive Logs url: observability_pipelines/set_up_pipelines/archive_logs/ parent: observability_pipelines_set_up_pipelines @@ -5301,26 +5311,31 @@ menu: parent: observability_pipelines_archive_logs identifier: observability_pipelines_archive_logs_logstash weight: 3019 + - name: Socket (TCP or UDP) + url: observability_pipelines/set_up_pipelines/archive_logs/socket/ + parent: observability_pipelines_archive_logs + identifier: observability_pipelines_archive_logs_socket + weight: 3020 - name: Splunk HTTP Event Collector url: observability_pipelines/set_up_pipelines/archive_logs/splunk_hec/ parent: observability_pipelines_archive_logs identifier: observability_pipelines_archive_logs_splunk_hec - weight: 3020 + weight: 3021 - name: Splunk Forwarders (TCP) url: observability_pipelines/set_up_pipelines/archive_logs/splunk_tcp/ parent: observability_pipelines_archive_logs identifier: observability_pipelines_archive_logs_splunk_tcp - weight: 3021 + weight: 3022 - name: Sumo Logic Hosted Collector url: observability_pipelines/set_up_pipelines/archive_logs/sumo_logic_hosted_collector/ parent: observability_pipelines_archive_logs identifier: observability_pipelines_archive_logs_sumo_logic_hosted_collector - weight: 3022 + weight: 3023 - name: Syslog url: observability_pipelines/set_up_pipelines/archive_logs/syslog/ parent: observability_pipelines_archive_logs identifier: observability_pipelines_archive_logs_syslog - weight: 3023 + weight: 3024 - name: Split Logs url: observability_pipelines/set_up_pipelines/split_logs/ parent: observability_pipelines_set_up_pipelines @@ -5371,26 +5386,31 @@ menu: parent: observability_pipelines_split_logs identifier: observability_pipelines_split_logs_logstash weight: 4019 + - name: Socket (TCP or UDP) + url: observability_pipelines/set_up_pipelines/split_logs/socket/ + parent: observability_pipelines_split_logs + identifier: observability_pipelines_split_logs_socket + weight: 4020 - name: Splunk HTTP Event Collector url: observability_pipelines/set_up_pipelines/split_logs/splunk_hec/ parent: observability_pipelines_split_logs identifier: observability_pipelines_split_logs_splunk_hec - weight: 4020 + weight: 4021 - name: Splunk Forwarders (TCP) url: observability_pipelines/set_up_pipelines/split_logs/splunk_tcp/ parent: observability_pipelines_split_logs identifier: observability_pipelines_split_logs_splunk_tcp - weight: 4021 + weight: 4022 - name: Sumo Logic Hosted Collector url: observability_pipelines/set_up_pipelines/split_logs/sumo_logic_hosted_collector/ parent: observability_pipelines_split_logs identifier: observability_pipelines_split_logs_sumo_logic_hosted_collector - weight: 4022 + weight: 4023 - name: Syslog url: observability_pipelines/set_up_pipelines/split_logs/syslog/ parent: observability_pipelines_split_logs identifier: observability_pipelines_split_logs_syslog - weight: 4023 + weight: 4024 - name: Sensitive Data Redaction url: observability_pipelines/set_up_pipelines/sensitive_data_redaction/ parent: observability_pipelines_set_up_pipelines @@ -5441,26 +5461,31 @@ menu: parent: observability_pipelines_sensitive_data_redaction identifier: observability_pipelines_sensitive_data_redaction_logstash weight: 5019 + - name: Socket (TCP or UDP) + url: observability_pipelines/set_up_pipelines/sensitive_data_redaction/socket/ + parent: observability_pipelines_sensitive_data_redaction + identifier: observability_pipelines_sensitive_data_redaction_socket + weight: 5020 - name: Splunk HTTP Event Collector url: observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_hec/ parent: observability_pipelines_sensitive_data_redaction identifier: observability_pipelines_sensitive_data_redaction_splunk_hec - weight: 5020 + weight: 5021 - name: Splunk Forwarders (TCP) url: observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_tcp/ parent: observability_pipelines_sensitive_data_redaction identifier: observability_pipelines_sensitive_data_redaction_splunk_tcp - weight: 5021 + weight: 5022 - name: Sumo Logic Hosted Collector url: observability_pipelines/set_up_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector/ parent: observability_pipelines_sensitive_data_redaction identifier: observability_pipelines_sensitive_data_redaction_sumo_logic_hosted_collector - weight: 5022 + weight: 5023 - name: Syslog url: observability_pipelines/set_up_pipelines/sensitive_data_redaction/syslog/ parent: observability_pipelines_sensitive_data_redaction identifier: observability_pipelines_sensitive_data_redaction_syslog - weight: 5023 + weight: 5024 - name: Log Enrichment url: observability_pipelines/set_up_pipelines/log_enrichment/ parent: observability_pipelines_set_up_pipelines @@ -5511,26 +5536,31 @@ menu: parent: observability_pipelines_log_enrichment identifier: observability_pipelines_log_enrichment_logstash weight: 6019 + - name: Socket (TCP or UDP) + url: observability_pipelines/set_up_pipelines/log_enrichment/socket/ + parent: observability_pipelines_log_enrichment + identifier: observability_pipelines_log_enrichment_socket + weight: 6020 - name: Splunk HTTP Event Collector url: observability_pipelines/set_up_pipelines/log_enrichment/splunk_hec/ parent: observability_pipelines_log_enrichment identifier: observability_pipelines_log_enrichment_splunk_hec - weight: 6020 + weight: 6021 - name: Splunk Forwarders (TCP) url: observability_pipelines/set_up_pipelines/log_enrichment/splunk_tcp/ parent: observability_pipelines_log_enrichment identifier: observability_pipelines_log_enrichment_splunk_tcp - weight: 6021 + weight: 6022 - name: Sumo Logic Hosted Collector url: observability_pipelines/set_up_pipelines/log_enrichment/sumo_logic_hosted_collector/ parent: observability_pipelines_log_enrichment identifier: observability_pipelines_log_enrichment_sumo_logic_hosted_collector - weight: 6022 + weight: 6023 - name: Syslog url: observability_pipelines/set_up_pipelines/log_enrichment/syslog/ parent: observability_pipelines_log_enrichment identifier: observability_pipelines_log_enrichment_syslog - weight: 6023 + weight: 6024 - name: Generate Metrics identifier: observability_pipelines_generate_metrics url: /observability_pipelines/set_up_pipelines/generate_metrics/ @@ -5581,26 +5611,31 @@ menu: parent: observability_pipelines_generate_metrics identifier: observability_pipelines_generate_metrics_logstash weight: 7019 + - name: Socket (TCP or UDP) + url: observability_pipelines/set_up_pipelines/generate_metrics/socket/ + parent: observability_pipelines_generate_metrics + identifier: observability_pipelines_generate_metrics_socket + weight: 7020 - name: Splunk HTTP Event Collector url: observability_pipelines/set_up_pipelines/generate_metrics/splunk_hec/ parent: observability_pipelines_generate_metrics identifier: observability_pipelines_generate_metrics_splunk_hec - weight: 7020 + weight: 7021 - name: Splunk Forwarders (TCP) url: observability_pipelines/set_up_pipelines/generate_metrics/splunk_tcp/ parent: observability_pipelines_generate_metrics identifier: observability_pipelines_generate_metrics_splunk_tcp - weight: 7021 + weight: 7022 - name: Sumo Logic Hosted Collector url: observability_pipelines/set_up_pipelines/generate_metrics/sumo_logic_hosted_collector/ parent: observability_pipelines_generate_metrics identifier: observability_pipelines_generate_metrics_sumo_logic_hosted_collector - weight: 7022 + weight: 7023 - name: Syslog url: observability_pipelines/set_up_pipelines/generate_metrics/syslog/ parent: observability_pipelines_generate_metrics identifier: observability_pipelines_generate_metrics_syslog - weight: 7023 + weight: 7024 - name: Run Multiple Pipelines on a Host identifier: observability_run_multiple_pipelines_on_a_host url: /observability_pipelines/set_up_pipelines/run_multiple_pipelines_on_a_host/ @@ -5706,26 +5741,31 @@ menu: parent: observability_pipelines_sources identifier: observability_pipelines_sources_logstash weight: 810 + - name: Socket + url: observability_pipelines/sources/socket/ + parent: observability_pipelines_sources + identifier: observability_pipelines_sources_socket + weight: 811 - name: Splunk HEC url: observability_pipelines/sources/splunk_hec/ parent: observability_pipelines_sources identifier: observability_pipelines_sources_splunk_hec - weight: 811 + weight: 812 - name: Splunk TCP url: observability_pipelines/sources/splunk_tcp/ parent: observability_pipelines_sources identifier: observability_pipelines_sources_splunk_tcp - weight: 812 + weight: 813 - name: Sumo Logic Hosted Collector url: observability_pipelines/sources/sumo_logic/ parent: observability_pipelines_sources identifier: observability_pipelines_sources_sumo_logic - weight: 813 + weight: 814 - name: Syslog url: observability_pipelines/sources/syslog/ parent: observability_pipelines_sources identifier: observability_pipelines_sources_syslog - weight: 814 + weight: 815 - name: Processors url: observability_pipelines/processors/ parent: observability_pipelines @@ -5816,11 +5856,16 @@ menu: parent: observability_pipelines_processors identifier: observability_pipelines_processors_split_array weight: 917 + - name: Tags Processor + url: observability_pipelines/processors/tags_processor + parent: observability_pipelines_processors + identifier: observability_pipelines_processors_tags_processor + weight: 918 - name: Throttle url: observability_pipelines/processors/throttle parent: observability_pipelines_processors identifier: observability_pipelines_processors_throttle - weight: 918 + weight: 919 - name: Destinations url: observability_pipelines/destinations/ parent: observability_pipelines @@ -5886,21 +5931,31 @@ menu: parent: observability_pipelines_destinations identifier: observability_pipelines_sentinelone weight: 1012 + - name: Socket + url: observability_pipelines/destinations/socket + parent: observability_pipelines_destinations + identifier: observability_pipelines_socket + weight: 1013 - name: Splunk HEC url: observability_pipelines/destinations/splunk_hec parent: observability_pipelines_destinations identifier: observability_pipelines_splunk_hec - weight: 1013 + weight: 1014 - name: Sumo Logic Hosted Collector url: observability_pipelines/destinations/sumo_logic_hosted_collector parent: observability_pipelines_destinations identifier: observability_pipelines_sumo_logic_hosted_collector - weight: 1014 + weight: 1015 - name: Syslog url: observability_pipelines/destinations/syslog parent: observability_pipelines_destinations identifier: observability_pipelines_syslog - weight: 1015 + weight: 1016 + - name: Syslog + url: observability_pipelines/destinations/syslog + parent: observability_pipelines_destinations + identifier: observability_pipelines_syslog + weight: 1017 - name: Environment Variables url: observability_pipelines/environment_variables/ parent: observability_pipelines diff --git a/content/en/observability_pipelines/destinations/socket.md b/content/en/observability_pipelines/destinations/socket.md new file mode 100644 index 0000000000000..62020a5c6f78b --- /dev/null +++ b/content/en/observability_pipelines/destinations/socket.md @@ -0,0 +1,27 @@ +--- +title: Socket Destination +disable_toc: false +--- + +Use Observability Pipelines' Socket destination to send logs to a socket endpoint. + +## Setup + +Set up the Socket destination and its environment variables when you [set up a pipeline][1]. The information below is configured in the pipelines UI. + +### Set up the destination + +{{% observability_pipelines/destination_settings/socket %}} + +### Set the environment variables + +{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/socket %}} + +### How the destination works + +#### Event batching + +The Socket destination does not batch events. + +[1]: https://app.datadoghq.com/observability-pipelines +[2]: /observability_pipelines/destinations/#event-batching \ No newline at end of file diff --git a/content/en/observability_pipelines/environment_variables.md b/content/en/observability_pipelines/environment_variables.md index 5ee95d3541257..8fe79e7d5f299 100644 --- a/content/en/observability_pipelines/environment_variables.md +++ b/content/en/observability_pipelines/environment_variables.md @@ -43,6 +43,10 @@ Some Observability Pipelines components require setting up environment variables ### Logstash {{% observability_pipelines/configure_existing_pipelines/source_env_vars/logstash %}} +### Socket + +{{% observability_pipelines/configure_existing_pipelines/source_env_vars/socket %}} + ### Splunk HEC {{% observability_pipelines/configure_existing_pipelines/source_env_vars/splunk_hec %}} @@ -100,6 +104,9 @@ Some Observability Pipelines components require setting up environment variables ### SentinelOne {{% observability_pipelines/configure_existing_pipelines/destination_env_vars/sentinelone %}} +### Socket +{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/socket %}} + ### Splunk HEC {{% observability_pipelines/configure_existing_pipelines/destination_env_vars/splunk_hec %}} diff --git a/content/en/observability_pipelines/processors/tags_processor.md b/content/en/observability_pipelines/processors/tags_processor.md new file mode 100644 index 0000000000000..df2f4387147f4 --- /dev/null +++ b/content/en/observability_pipelines/processors/tags_processor.md @@ -0,0 +1,8 @@ +--- +title: Tags Processor +disable_toc: false +--- + +{{% observability_pipelines/processors/tags_processor %}} + +{{% observability_pipelines/processors/filter_syntax %}} \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/_index.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/_index.md index adf1d7ebbadc3..d9e39fb69dccc 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/_index.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/_index.md @@ -28,6 +28,7 @@ Select a source to get started: - [Logstash][6] - [Splunk HTTP Event Collector (HEC)][7] - [Splunk Heavy or Universal Forwarders (TCP)][8] +- [Socket (TCP or UDP)][14] - [Sumo Logic Hosted Collector][9] - [rsylsog or syslog-ng][10] @@ -44,3 +45,4 @@ Select a source to get started: [11]: /observability_pipelines/set_up_pipelines/archive_logs/amazon_s3 [12]: /observability_pipelines/set_up_pipelines/archive_logs/amazon_data_firehose [13]: /observability_pipelines/set_up_pipelines/archive_logs/kafka +[14]: /observability_pipelines/set_up_pipelines/archive_logs/socket diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_data_firehose.md index 378ac735d371c..ae7f4d45273ae 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_data_firehose.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_data_firehose.md @@ -161,6 +161,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -301,6 +306,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -388,6 +398,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_s3.md index 50cfd40d0ab7d..d87502ab2b713 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_s3.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/amazon_s3.md @@ -160,6 +160,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -300,6 +305,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -387,6 +397,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/datadog_agent.md index a8603b0c8f497..c8c40db031ad9 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/datadog_agent.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/datadog_agent.md @@ -164,6 +164,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -303,6 +308,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -390,6 +400,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/fluent.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/fluent.md index 370266fbc5f49..e13098033c2cb 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/fluent.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/fluent.md @@ -165,6 +165,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -305,6 +310,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -393,6 +403,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/google_pubsub.md index b094ec6ea7f8d..446cfe2b2ae58 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/google_pubsub.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/google_pubsub.md @@ -162,6 +162,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -302,6 +307,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -388,6 +398,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_client.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_client.md index 3625aecb7a5e6..57215e89f8027 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_client.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_client.md @@ -163,6 +163,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -303,6 +308,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -391,6 +401,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_server.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_server.md index d7b6ed972cf5f..117aa42186fb5 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_server.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/http_server.md @@ -160,6 +160,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -300,6 +305,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -387,6 +397,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/kafka.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/kafka.md index c413b032f0960..6be0ba7708fe6 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/kafka.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/kafka.md @@ -161,6 +161,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -301,6 +306,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -391,6 +401,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/logstash.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/logstash.md index 66f9261bca3c1..e76713de81505 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/logstash.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/logstash.md @@ -161,6 +161,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -301,6 +306,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -388,6 +398,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/socket.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/socket.md new file mode 100644 index 0000000000000..a2239c283f94e --- /dev/null +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/socket.md @@ -0,0 +1,443 @@ +--- +title: Archive Logs for the Socket Source (TCP or UDP) +disable_toc: false +--- + +## Overview + +Send your logs over a socket connection to the Observability Pipelines Worker to format your logs into a Datadog-rehydratable format before routing them to Datadog Log Archives. + +{{% observability_pipelines/use_case_images/archive_logs %}} + +This document walks you through the following: +1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines +1. [Configuring a Log Archive](#configure-a-log-archive) +1. [Setting up Observability Pipelines](#set-up-observability-pipelines) +1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker) + +## Prerequisites + +{{% observability_pipelines/prerequisites/socket %}} + +## Configure Log Archives + +If you already have a Datadog Log Archive configured for Observability Pipelines, skip to [Set up Observability Pipelines](#set-up-observability-pipelines). + +You need to have the Datadog integration for your cloud provider installed to set up Datadog Log Archive. See the [AWS integration][1], [Google Cloud Platform][2], and [Azure integration][3] documentation for more information. + +Select the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h4" %}} +{{% observability_pipelines/configure_log_archive/amazon_s3/instructions %}} + +{{< tabs >}} +{{% tab "Docker" %}} + +{{% observability_pipelines/configure_log_archive/amazon_s3/docker %}} + +{{% /tab %}} +{{% tab "Amazon EKS" %}} + +{{% observability_pipelines/configure_log_archive/amazon_s3/amazon_eks %}} + +{{% /tab %}} +{{% tab "Linux (APT)" %}} + +{{% observability_pipelines/configure_log_archive/amazon_s3/linux_apt %}} + +{{% /tab %}} +{{% tab "Linux (RPM)" %}} + +{{% observability_pipelines/configure_log_archive/amazon_s3/linux_rpm %}} + +{{% /tab %}} +{{< /tabs >}} + +{{% observability_pipelines/configure_log_archive/amazon_s3/connect_s3_to_datadog_log_archives %}} + +{{% /collapse-content %}} + +{{% collapse-content title="Google Cloud Storage" level="h4" %}} + +{{% observability_pipelines/configure_log_archive/google_cloud_storage/instructions %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h4" %}} + +{{% observability_pipelines/configure_log_archive/azure_storage/instructions %}} + +{{% /collapse-content %}} + +## Set up Observability Pipelines + +1. Navigate to [Observability Pipelines][4]. +1. Select the **Archive Logs** template to create a new pipeline. +1. Select the **Socket** source. + +### Set up the source + +{{% observability_pipelines/source_settings/socket %}} + +### Set up the destinations + +Enter the following information based on your selected logs destinations. + +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_settings/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_settings/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_settings/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_note %}} + +Follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_settings/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_settings/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_settings/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_settings/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_settings/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_settings/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_settings/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_settings/syslog %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add additional destinations + +{{% observability_pipelines/multiple_destinations %}} + +### Set up processors + +{{% observability_pipelines/processors/intro %}} + +{{% observability_pipelines/processors/filter_syntax %}} + +{{% observability_pipelines/processors/add_processors %}} + +{{< tabs >}} +{{% tab "Add env vars" %}} + +{{% observability_pipelines/processors/add_env_vars %}} + +{{% /tab %}} +{{% tab "Add hostname" %}} + +{{% observability_pipelines/processors/add_hostname %}} + +{{% /tab %}} +{{% tab "Custom Processor" %}} + +{{% observability_pipelines/processors/custom_processor %}} + +{{% /tab %}} +{{% tab "Dedupe" %}} + +{{% observability_pipelines/processors/dedupe %}} + +{{% /tab %}} +{{% tab "Edit fields" %}} + +{{% observability_pipelines/processors/remap %}} + +{{% /tab %}} +{{% tab "Enrichment table" %}} + +{{% observability_pipelines/processors/enrichment_table %}} + +{{% /tab %}} +{{% tab "Filter" %}} + +{{% observability_pipelines/processors/filter %}} + +{{% /tab %}} +{{% tab "Generate metrics" %}} + +{{% observability_pipelines/processors/generate_metrics %}} + +{{% /tab %}} +{{% tab "Grok Parser" %}} + +{{% observability_pipelines/processors/grok_parser %}} + +{{% /tab %}} +{{% tab "Parse JSON" %}} + +{{% observability_pipelines/processors/parse_json %}} + +{{% /tab %}} +{{% tab "Parse XML" %}} + +{{% observability_pipelines/processors/parse_xml %}} + +{{% /tab %}} +{{% tab "Quota" %}} + +{{% observability_pipelines/processors/quota %}} + +{{% /tab %}} +{{% tab "Reduce" %}} + +{{% observability_pipelines/processors/reduce %}} + +{{% /tab %}} +{{% tab "Remap to OCSF" %}} + +{{% observability_pipelines/processors/remap_ocsf %}} + +{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} + +{{% /collapse-content %}} +{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} + +{{% /collapse-content %}} + +{{% observability_pipelines/processors/filter_syntax %}} + +{{% /tab %}} +{{% tab "Sample" %}} + +{{% observability_pipelines/processors/sample %}} + +{{% /tab %}} +{{% tab "Sensitive Data Scanner" %}} + +{{% observability_pipelines/processors/sensitive_data_scanner %}} + +{{% collapse-content title="Add rules from the library" level="h5" %}} + +{{% observability_pipelines/processors/sds_library_rules %}} + +{{% /collapse-content %}} +{{% collapse-content title="Add a custom rule" level="h5" %}} + +{{% observability_pipelines/processors/sds_custom_rules %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Split array" %}} + +{{% observability_pipelines/processors/split_array %}} + +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + +{{% /tab %}} +{{% tab "Throttle" %}} + +{{% observability_pipelines/processors/throttle %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add another set of processors and destinations + +{{% observability_pipelines/multiple_processors %}} + +### Install the Observability Pipelines Worker +1. Select your platform in the **Choose your installation platform** dropdown menu. +1. TKTK +1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_env_vars/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_env_vars/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_env_vars/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_env_vars/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_env_vars/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_env_vars/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_env_vars/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_env_vars/syslog %}} + +{{% /tab %}} +{{< /tabs >}} + +1. Follow the instructions for your environment to install the Worker. +{{< tabs >}} +{{% tab "Docker" %}} + +{{% observability_pipelines/install_worker/docker %}} + +{{% /tab %}} +{{% tab "Kubernetes" %}} + +{{% observability_pipelines/install_worker/kubernetes %}} + +{{% /tab %}} +{{% tab "Linux (APT)" %}} + +{{% observability_pipelines/install_worker/linux_apt %}} + +{{% /tab %}} +{{% tab "Linux (RPM)" %}} + +{{% observability_pipelines/install_worker/linux_rpm %}} + +{{% /tab %}} +{{% tab "CloudFormation" %}} + +{{% observability_pipelines/install_worker/cloudformation %}} + +{{% /tab %}} +{{< /tabs >}} + +## Send logs to the Observability Pipelines Worker + +{{% observability_pipelines/log_source_configuration/socket %}} + +[1]: /integrations/amazon_web_services/#setup +[2]: /integrations/google_cloud_platform/#setup +[3]: /integrations/azure/#setup +[4]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_hec.md index 1e591ff9c6db8..7b73624ea0ebf 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_hec.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_hec.md @@ -163,6 +163,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -303,6 +308,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -390,6 +400,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_tcp.md index 780e5ada98860..3b4ddb2550f95 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_tcp.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/splunk_tcp.md @@ -163,6 +163,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -303,6 +308,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -390,6 +400,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/sumo_logic_hosted_collector.md index cf4a053b34a38..169f2e6392ba3 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/sumo_logic_hosted_collector.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/sumo_logic_hosted_collector.md @@ -163,6 +163,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -303,6 +308,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -390,6 +400,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/archive_logs/syslog.md b/content/en/observability_pipelines/set_up_pipelines/archive_logs/syslog.md index d1924b815de4b..54ad862d4d7ef 100644 --- a/content/en/observability_pipelines/set_up_pipelines/archive_logs/syslog.md +++ b/content/en/observability_pipelines/set_up_pipelines/archive_logs/syslog.md @@ -164,6 +164,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -304,6 +309,11 @@ Follow the instructions for the cloud provider you are using to archive your log {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -392,6 +402,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/_index.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/_index.md index c318cd6e05e95..3ef7b3c1d4b57 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/_index.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/_index.md @@ -24,6 +24,7 @@ Select a source to get started: - [Logstash][6] - [Splunk HTTP Event Collector (HEC)][7] - [Splunk Heavy or Universal Forwarders (TCP)][8] +- [Socket (TCP or UDP)][14] - [Sumo Logic Hosted Collector][9] - [rsyslog or syslog-ng][10] @@ -40,3 +41,4 @@ Select a source to get started: [11]: /observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_s3 [12]: /observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_data_firehose [13]: /observability_pipelines/set_up_pipelines/dual_ship_logs/kafka +[14]: /observability_pipelines/set_up_pipelines/dual_ship_logs/socket diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_data_firehose.md index a4e724b5e4dae..716afd4ef281d 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_data_firehose.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_data_firehose.md @@ -114,6 +114,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -254,6 +259,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -341,6 +351,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_s3.md index ae75d3671700f..094477db3a6ba 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_s3.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/amazon_s3.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -339,6 +349,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/datadog_agent.md index 597999e59819b..7fe7720e657f1 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/datadog_agent.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/datadog_agent.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/fluent.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/fluent.md index ead73fd2e7b07..c0024d6d192e1 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/fluent.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/fluent.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/google_pubsub.md index ea8f9373a319e..0f64804c40a05 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/google_pubsub.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/google_pubsub.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -338,6 +348,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_client.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_client.md index b2266680dc9b9..af63d9aecbb03 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_client.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_client.md @@ -114,6 +114,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -254,6 +259,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_server.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_server.md index 7d0ce71ff32aa..d356b3283f552 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_server.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/http_server.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -339,6 +349,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/kafka.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/kafka.md index 2310ab53cc48f..47283ff37a2c0 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/kafka.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/kafka.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/logstash.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/logstash.md index ae1c7da5aa3ac..43d4ddcdf4acd 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/logstash.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/logstash.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/socket.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/socket.md new file mode 100644 index 0000000000000..414ef0c1574a7 --- /dev/null +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/socket.md @@ -0,0 +1,388 @@ +--- +title: Dual Ship Logs for the Socket Source (TCP or UDP) +disable_toc: false +--- + +## Overview + +Send your logs over a socket connection to the Observability Pipelines Worker so that you can aggregate and process those logs before routing them to various applications. + +{{% observability_pipelines/use_case_images/dual_ship_logs %}} + +This document walks you through the following: +1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines +1. [Setting up Observability Pipelines](#set-up-observability-pipelines) +1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-socket) + +## Prerequisites + +{{% observability_pipelines/prerequisites/socket %}} + +## Set up Observability Pipelines + +1. Navigate to [Observability Pipelines][1]. +1. Select the **Dual Ship Logs** template to create a new pipeline. +1. Select the **Socket** source. + +### Set up the source + +{{% observability_pipelines/source_settings/socket %}} + +### Set up the destinations + +Enter the following information based on your selected logs destinations. + +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_settings/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_settings/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_settings/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_note %}} + +Follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_settings/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_settings/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_settings/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_settings/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_settings/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_settings/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_settings/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_settings/syslog %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add additional destinations + +{{% observability_pipelines/multiple_destinations %}} + +### Set up processors + +{{% observability_pipelines/processors/intro %}} + +{{% observability_pipelines/processors/filter_syntax %}} + +{{% observability_pipelines/processors/add_processors %}} + +{{< tabs >}} +{{% tab "Add env vars" %}} + +{{% observability_pipelines/processors/add_env_vars %}} + +{{% /tab %}} +{{% tab "Add hostname" %}} + +{{% observability_pipelines/processors/add_hostname %}} + +{{% /tab %}} +{{% tab "Custom Processor" %}} + +{{% observability_pipelines/processors/custom_processor %}} + +{{% /tab %}} +{{% tab "Dedupe" %}} + +{{% observability_pipelines/processors/dedupe %}} + +{{% /tab %}} +{{% tab "Edit fields" %}} + +{{% observability_pipelines/processors/remap %}} + +{{% /tab %}} +{{% tab "Enrichment table" %}} + +{{% observability_pipelines/processors/enrichment_table %}} + +{{% /tab %}} +{{% tab "Filter" %}} + +{{% observability_pipelines/processors/filter %}} + +{{% /tab %}} +{{% tab "Generate metrics" %}} + +{{% observability_pipelines/processors/generate_metrics %}} + +{{% /tab %}} +{{% tab "Grok Parser" %}} + +{{% observability_pipelines/processors/grok_parser %}} + +{{% /tab %}} +{{% tab "Parse JSON" %}} + +{{% observability_pipelines/processors/parse_json %}} + +{{% /tab %}} +{{% tab "Parse XML" %}} + +{{% observability_pipelines/processors/parse_xml %}} + +{{% /tab %}} +{{% tab "Quota" %}} + +{{% observability_pipelines/processors/quota %}} + +{{% /tab %}} +{{% tab "Reduce" %}} + +{{% observability_pipelines/processors/reduce %}} + +{{% /tab %}} +{{% tab "Remap to OCSF" %}} + +{{% observability_pipelines/processors/remap_ocsf %}} + +{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} + +{{% /collapse-content %}} + +{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Sample" %}} + +{{% observability_pipelines/processors/sample %}} + +{{% /tab %}} +{{% tab "Sensitive Data Scanner" %}} + +{{% observability_pipelines/processors/sensitive_data_scanner %}} + +{{% collapse-content title="Add rules from the library" level="h5" %}} + +{{% observability_pipelines/processors/sds_library_rules %}} + +{{% /collapse-content %}} +{{% collapse-content title="Add a custom rule" level="h5" %}} + +{{% observability_pipelines/processors/sds_custom_rules %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Split array" %}} + +{{% observability_pipelines/processors/split_array %}} + +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + +{{% /tab %}} +{{% tab "Throttle" %}} + +{{% observability_pipelines/processors/throttle %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add another set of processors and destinations + +{{% observability_pipelines/multiple_processors %}} + +### Install the Observability Pipelines Worker +1. Select your platform in the **Choose your installation platform** dropdown menu. +1. TKTK +1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_env_vars/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_env_vars/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_env_vars/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_env_vars/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_env_vars/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_env_vars/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_env_vars/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_env_vars/syslog %}} + +{{% /tab %}} +{{< /tabs >}} +1. Follow the instructions for your environment to install the Worker. +{{< tabs >}} +{{% tab "Docker" %}} + +{{% observability_pipelines/install_worker/docker %}} + +{{% /tab %}} +{{% tab "Kubernetes" %}} + +{{% observability_pipelines/install_worker/kubernetes %}} + +{{% /tab %}} +{{% tab "Linux (APT)" %}} + +{{% observability_pipelines/install_worker/linux_apt %}} + +{{% /tab %}} +{{% tab "Linux (RPM)" %}} + +{{% observability_pipelines/install_worker/linux_rpm %}} + +{{% /tab %}} +{{% tab "CloudFormation" %}} + +{{% observability_pipelines/install_worker/cloudformation %}} + +{{% /tab %}} +{{< /tabs >}} + +## Send logs to the Observability Pipelines Worker over Socket + +{{% observability_pipelines/log_source_configuration/socket %}} + +[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_hec.md index eed8e692e1739..51fd5e3e12b0d 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_hec.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_hec.md @@ -123,6 +123,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -263,6 +268,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -350,6 +360,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_tcp.md index 81b964c44e5ba..38415a8212504 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_tcp.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/splunk_tcp.md @@ -121,6 +121,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -261,6 +266,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -348,6 +358,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/sumo_logic_hosted_collector.md index 63b5a696bdb6a..fd2ffd9b27a67 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/sumo_logic_hosted_collector.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/sumo_logic_hosted_collector.md @@ -121,6 +121,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -261,6 +266,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -348,6 +358,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/syslog.md b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/syslog.md index bb4e23568a438..88958c681266c 100644 --- a/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/syslog.md +++ b/content/en/observability_pipelines/set_up_pipelines/dual_ship_logs/syslog.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/_index.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/_index.md index 3d98b2691dab9..9619bad5af6a6 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/_index.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/_index.md @@ -15,6 +15,38 @@ Some log sources, such as firewalls and network appliances, generate a large vol {{% observability_pipelines/use_case_images/generate_metrics %}} +Select a source to get started: + +- [Amazon Data Firehose][12] +- [Amazon S3][11] +- [Datadog Agent][1] +- [Fluentd or Fluent Bit][2] +- [Google Pub/Sub][3] +- [HTTP Client][4] +- [HTTP Server][5] +- [Kafka][13] +- [Logstash][6] +- [Socket][14] +- [Splunk HTTP Event Collector (HEC)][7] +- [Splunk Heavy or Universal Forwarders (TCP)][8] +- [Sumo Logic Hosted Collector][9] +- [rsyslog or syslog-ng][10] + +[1]: /observability_pipelines/set_up_pipelines/generate_metrics/datadog_agent +[2]: /observability_pipelines/set_up_pipelines/generate_metrics/fluent +[3]: /observability_pipelines/set_up_pipelines/generate_metrics/google_pubsub +[4]: /observability_pipelines/set_up_pipelines/generate_metrics/http_client +[5]: /observability_pipelines/set_up_pipelines/generate_metrics/http_server +[6]: /observability_pipelines/set_up_pipelines/generate_metrics/logstash +[7]: /observability_pipelines/set_up_pipelines/generate_metrics/splunk_hec +[8]: /observability_pipelines/set_up_pipelines/generate_metrics/splunk_tcp +[9]: /observability_pipelines/set_up_pipelines/generate_metrics/sumo_logic_hosted_collector +[10]: /observability_pipelines/set_up_pipelines/generate_metrics/syslog +[11]: /observability_pipelines/set_up_pipelines/generate_metrics/amazon_s3 +[12]: /observability_pipelines/set_up_pipelines/generate_metrics/amazon_data_firehose +[13]: /observability_pipelines/set_up_pipelines/generate_metrics/kafka +[14]: /observability_pipelines/set_up_pipelines/generate_metrics/socket + ## Metrics types {{% observability_pipelines/metrics_types %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_data_firehose.md index 75130deec7213..25ada4515c91f 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_data_firehose.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_data_firehose.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_s3.md index 8bb2c304d3e70..7f77e6de6e98d 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_s3.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/amazon_s3.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -339,6 +349,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/datadog_agent.md index 16b4aa2ee041a..d4ef050afbba5 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/datadog_agent.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/datadog_agent.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/fluent.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/fluent.md index 3de19bef090c9..0760a7f7558f4 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/fluent.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/fluent.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -341,6 +351,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/google_pubsub.md index 6fb1b245a8e68..f857262f9c7fa 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/google_pubsub.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/google_pubsub.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -338,6 +348,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_client.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_client.md index 261839ce9353b..21c99f988a610 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_client.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_client.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_server.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_server.md index 0313c0f2f7071..11943b2694c96 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_server.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/http_server.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -339,6 +349,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/kafka.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/kafka.md index 9c10eec27333a..e01415243de06 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/kafka.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/kafka.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/logstash.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/logstash.md index c17d60fb5e0e8..ea6c361bb33e7 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/logstash.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/logstash.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/socket.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/socket.md new file mode 100644 index 0000000000000..23d279f11bc77 --- /dev/null +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/socket.md @@ -0,0 +1,388 @@ +--- +title: Generate Metrics for the Socket Source (TCP or UDP) +disable_toc: false +--- + +## Overview + +Send logs over a socket connection to the Observability Pipelines Worker so that you can generate metrics from those logs. + +{{% observability_pipelines/use_case_images/generate_metrics %}} + +This document walks you through the following: +1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines +1. [Setting up Observability Pipelines](#set-up-observability-pipelines) +1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-socket) + +## Prerequisites + +{{% observability_pipelines/prerequisites/socket %}} + +## Set up Observability Pipelines + +1. Navigate to [Observability Pipelines][1]. +1. Select the **Generate Metrics** template to create a new pipeline. +1. Select the **Socket** source. + +### Set up the source + +{{% observability_pipelines/source_settings/socket %}} + +### Set up the destinations + +Enter the following information based on your selected logs destinations. + +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_settings/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_settings/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_settings/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_note %}} + +Follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_settings/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_settings/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_settings/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_settings/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_settings/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_settings/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_settings/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_settings/syslog %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add additional destinations + +{{% observability_pipelines/multiple_destinations %}} + +### Set up processors + +{{% observability_pipelines/processors/intro %}} + +{{% observability_pipelines/processors/filter_syntax %}} + +{{% observability_pipelines/processors/add_processors %}} + +{{< tabs >}} +{{% tab "Add env vars" %}} + +{{% observability_pipelines/processors/add_env_vars %}} + +{{% /tab %}} +{{% tab "Add hostname" %}} + +{{% observability_pipelines/processors/add_hostname %}} + +{{% /tab %}} +{{% tab "Custom Processor" %}} + +{{% observability_pipelines/processors/custom_processor %}} + +{{% /tab %}} +{{% tab "Dedupe" %}} + +{{% observability_pipelines/processors/dedupe %}} + +{{% /tab %}} +{{% tab "Edit fields" %}} + +{{% observability_pipelines/processors/remap %}} + +{{% /tab %}} +{{% tab "Enrichment table" %}} + +{{% observability_pipelines/processors/enrichment_table %}} + +{{% /tab %}} +{{% tab "Filter" %}} + +{{% observability_pipelines/processors/filter %}} + +{{% /tab %}} +{{% tab "Generate metrics" %}} + +{{% observability_pipelines/processors/generate_metrics %}} + +{{% /tab %}} +{{% tab "Grok Parser" %}} + +{{% observability_pipelines/processors/grok_parser %}} + +{{% /tab %}} +{{% tab "Parse JSON" %}} + +{{% observability_pipelines/processors/parse_json %}} + +{{% /tab %}} +{{% tab "Parse XML" %}} + +{{% observability_pipelines/processors/parse_xml %}} + +{{% /tab %}} +{{% tab "Quota" %}} + +{{% observability_pipelines/processors/quota %}} + +{{% /tab %}} +{{% tab "Reduce" %}} + +{{% observability_pipelines/processors/reduce %}} + +{{% /tab %}} +{{% tab "Remap to OCSF" %}} + +{{% observability_pipelines/processors/remap_ocsf %}} + +{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} + +{{% /collapse-content %}} + +{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Sample" %}} + +{{% observability_pipelines/processors/sample %}} + +{{% /tab %}} +{{% tab "Sensitive Data Scanner" %}} + +{{% observability_pipelines/processors/sensitive_data_scanner %}} + +{{% collapse-content title="Add rules from the library" level="h5" %}} + +{{% observability_pipelines/processors/sds_library_rules %}} + +{{% /collapse-content %}} +{{% collapse-content title="Add a custom rule" level="h5" %}} + +{{% observability_pipelines/processors/sds_custom_rules %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Split array" %}} + +{{% observability_pipelines/processors/split_array %}} + +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + +{{% /tab %}} +{{% tab "Throttle" %}} + +{{% observability_pipelines/processors/throttle %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add another set of processors and destinations + +{{% observability_pipelines/multiple_processors %}} + +### Install the Observability Pipelines Worker +1. Select your platform in the **Choose your installation platform** dropdown menu. +1. TKTK +1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_env_vars/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_env_vars/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_env_vars/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_env_vars/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_env_vars/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_env_vars/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_env_vars/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_env_vars/syslog %}} + +{{% /tab %}} +{{< /tabs >}} +1. Follow the instructions for your environment to install the Worker. +{{< tabs >}} +{{% tab "Docker" %}} + +{{% observability_pipelines/install_worker/docker %}} + +{{% /tab %}} +{{% tab "Kubernetes" %}} + +{{% observability_pipelines/install_worker/kubernetes %}} + +{{% /tab %}} +{{% tab "Linux (APT)" %}} + +{{% observability_pipelines/install_worker/linux_apt %}} + +{{% /tab %}} +{{% tab "Linux (RPM)" %}} + +{{% observability_pipelines/install_worker/linux_rpm %}} + +{{% /tab %}} +{{% tab "CloudFormation" %}} + +{{% observability_pipelines/install_worker/cloudformation %}} + +{{% /tab %}} +{{< /tabs >}} + +## Send logs to the Observability Pipelines Worker over Socket + +{{% observability_pipelines/log_source_configuration/socket %}} + +[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_hec.md index 967a65f785d34..fb0e743aea4ed 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_hec.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_hec.md @@ -119,6 +119,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -259,6 +264,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -346,6 +356,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_tcp.md index 826d09a72990e..1e1b115ce52c8 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_tcp.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/splunk_tcp.md @@ -119,6 +119,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -259,6 +264,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -346,6 +356,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/sumo_logic_hosted_collector.md index bf0043f879877..f2da49cb26661 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/sumo_logic_hosted_collector.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/sumo_logic_hosted_collector.md @@ -119,6 +119,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -259,6 +264,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -346,6 +356,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/syslog.md b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/syslog.md index 38e54260c411c..23f15401beeb2 100644 --- a/content/en/observability_pipelines/set_up_pipelines/generate_metrics/syslog.md +++ b/content/en/observability_pipelines/set_up_pipelines/generate_metrics/syslog.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -341,6 +351,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/_index.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/_index.md index d228303253e57..7ae81b1a011cf 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/_index.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/_index.md @@ -29,6 +29,7 @@ Select a source to get started: - [Logstash][6] - [Splunk HTTP Event Collector (HEC)][7] - [Splunk Heavy or Universal Forwarders (TCP)][8] +- [Socket (TCP or UDP)][14] - [Sumo Logic Hosted Collector][9] - [rsyslog or syslog-ng][10] @@ -45,3 +46,4 @@ Select a source to get started: [11]: /observability_pipelines/set_up_pipelines/log_enrichment/amazon_s3 [12]: /observability_pipelines/set_up_pipelines/log_enrichment/amazon_data_firehose [13]: /observability_pipelines/set_up_pipelines/log_enrichment/kafka +[14]: /observability_pipelines/set_up_pipelines/log_enrichment/socket diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_data_firehose.md index 5a4aedc90f76c..cb63ac4df5216 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_data_firehose.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_data_firehose.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_s3.md index 72c95c364f977..c3a0ad6043a51 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_s3.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/amazon_s3.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -339,6 +349,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/datadog_agent.md index cf68bea80fefd..5b3af63334dc0 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/datadog_agent.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/datadog_agent.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/fluent.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/fluent.md index 6b04c63506612..42341b6ba3846 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/fluent.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/fluent.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/google_pubsub.md index 54a2b83aeeccd..42fc9b68aa002 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/google_pubsub.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/google_pubsub.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -338,6 +348,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_client.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_client.md index 5149b90ce59e9..1d5696607d80f 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_client.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_client.md @@ -114,6 +114,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -254,6 +259,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_server.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_server.md index c6a54a0427f41..408a2bd3758ef 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_server.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/http_server.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/kafka.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/kafka.md index ed5fee4d26358..ddb72891d5dc8 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/kafka.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/kafka.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/logstash.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/logstash.md index 3069daf7093e7..c454cd2c9a47b 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/logstash.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/logstash.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/socket.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/socket.md new file mode 100644 index 0000000000000..51dd297117023 --- /dev/null +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/socket.md @@ -0,0 +1,388 @@ +--- +title: Log Enrichment for the Socket Source (TCP or UDP) +disable_toc: false +--- + +## Overview + +Send logs over a socket connection to the Observability Pipelines Worker so you can enrich and transform your logs before routing them to their destination. + +{{% observability_pipelines/use_case_images/log_enrichment %}} + +This document walks you through the following: +1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines +1. [Setting up Observability Pipelines](#set-up-observability-pipelines) +1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-socket) + +## Prerequisites + +{{% observability_pipelines/prerequisites/socket %}} + +## Set up Observability Pipelines + +1. Navigate to [Observability Pipelines][1]. +1. Select the **Log Enrichment** template to create a new pipeline. +1. Select the **Socket** source. + +### Set up the source + +{{% observability_pipelines/source_settings/socket %}} + +### Set up the destinations + +Enter the following information based on your selected logs destinations. + +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_settings/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_settings/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_settings/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_note %}} + +Follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_settings/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_settings/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_settings/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_settings/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_settings/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_settings/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_settings/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_settings/syslog %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add additional destinations + +{{% observability_pipelines/multiple_destinations %}} + +### Set up processors + +{{% observability_pipelines/processors/intro %}} + +{{% observability_pipelines/processors/filter_syntax %}} + +{{% observability_pipelines/processors/add_processors %}} + +{{< tabs >}} +{{% tab "Add env vars" %}} + +{{% observability_pipelines/processors/add_env_vars %}} + +{{% /tab %}} +{{% tab "Add hostname" %}} + +{{% observability_pipelines/processors/add_hostname %}} + +{{% /tab %}} +{{% tab "Custom Processor" %}} + +{{% observability_pipelines/processors/custom_processor %}} + +{{% /tab %}} +{{% tab "Dedupe" %}} + +{{% observability_pipelines/processors/dedupe %}} + +{{% /tab %}} +{{% tab "Edit fields" %}} + +{{% observability_pipelines/processors/remap %}} + +{{% /tab %}} +{{% tab "Enrichment table" %}} + +{{% observability_pipelines/processors/enrichment_table %}} + +{{% /tab %}} +{{% tab "Filter" %}} + +{{% observability_pipelines/processors/filter %}} + +{{% /tab %}} +{{% tab "Generate metrics" %}} + +{{% observability_pipelines/processors/generate_metrics %}} + +{{% /tab %}} +{{% tab "Grok Parser" %}} + +{{% observability_pipelines/processors/grok_parser %}} + +{{% /tab %}} +{{% tab "Parse JSON" %}} + +{{% observability_pipelines/processors/parse_json %}} + +{{% /tab %}} +{{% tab "Parse XML" %}} + +{{% observability_pipelines/processors/parse_xml %}} + +{{% /tab %}} +{{% tab "Quota" %}} + +{{% observability_pipelines/processors/quota %}} + +{{% /tab %}} +{{% tab "Reduce" %}} + +{{% observability_pipelines/processors/reduce %}} + +{{% /tab %}} +{{% tab "Remap to OCSF" %}} + +{{% observability_pipelines/processors/remap_ocsf %}} + +{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} + +{{% /collapse-content %}} + +{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Sample" %}} + +{{% observability_pipelines/processors/sample %}} + +{{% /tab %}} +{{% tab "Sensitive Data Scanner" %}} + +{{% observability_pipelines/processors/sensitive_data_scanner %}} + +{{% collapse-content title="Add rules from the library" level="h5" %}} + +{{% observability_pipelines/processors/sds_library_rules %}} + +{{% /collapse-content %}} +{{% collapse-content title="Add a custom rule" level="h5" %}} + +{{% observability_pipelines/processors/sds_custom_rules %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Split array" %}} + +{{% observability_pipelines/processors/split_array %}} + +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + +{{% /tab %}} +{{% tab "Throttle" %}} + +{{% observability_pipelines/processors/throttle %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add another set of processors and destinations + +{{% observability_pipelines/multiple_processors %}} + +### Install the Observability Pipelines Worker +1. Select your platform in the **Choose your installation platform** dropdown menu. +1. TKTK +1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_env_vars/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_env_vars/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_env_vars/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_env_vars/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_env_vars/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_env_vars/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_env_vars/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_env_vars/syslog %}} + +{{% /tab %}} +{{< /tabs >}} +1. Follow the instructions for your environment to install the Worker. +{{< tabs >}} +{{% tab "Docker" %}} + +{{% observability_pipelines/install_worker/docker %}} + +{{% /tab %}} +{{% tab "Kubernetes" %}} + +{{% observability_pipelines/install_worker/kubernetes %}} + +{{% /tab %}} +{{% tab "Linux (APT)" %}} + +{{% observability_pipelines/install_worker/linux_apt %}} + +{{% /tab %}} +{{% tab "Linux (RPM)" %}} + +{{% observability_pipelines/install_worker/linux_rpm %}} + +{{% /tab %}} +{{% tab "CloudFormation" %}} + +{{% observability_pipelines/install_worker/cloudformation %}} + +{{% /tab %}} +{{< /tabs >}} + +## Send logs to the Observability Pipelines Worker over Socket + +{{% observability_pipelines/log_source_configuration/socket %}} + +[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_hec.md index 00d741904ef3f..404ff98c00645 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_hec.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_hec.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_tcp.md index 5bddd3c8042c6..0faa737b8762c 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_tcp.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/splunk_tcp.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/sumo_logic_hosted_collector.md index 0246ea7c2d59e..10fac833fc76e 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/sumo_logic_hosted_collector.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/sumo_logic_hosted_collector.md @@ -111,6 +111,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -251,6 +256,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -338,6 +348,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/syslog.md b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/syslog.md index c538867dfe20f..0a10d81e8f166 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_enrichment/syslog.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_enrichment/syslog.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/_index.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/_index.md index d3e2e9ca4da8a..5ea30f986fd25 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/_index.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/_index.md @@ -34,6 +34,7 @@ Select a log source to get started: - [Logstash][6] - [Splunk HTTP Event Collector (HEC)][7] - [Splunk Heavy or Universal Forwarders (TCP)][8] +- [Socket (TCP or UDP)][14] - [Sumo Logic Hosted Collector][9] - [rsyslog or syslog-ng][10] @@ -54,3 +55,4 @@ Select a log source to get started: [11]: /observability_pipelines/set_up_pipelines/log_volume_control/amazon_s3 [12]: /observability_pipelines/set_up_pipelines/log_volume_control/amazon_data_firehose [13]: /observability_pipelines/set_up_pipelines/log_volume_control/kafka +[14]: /observability_pipelines/set_up_pipelines/log_volume_control/socket diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_data_firehose.md index 3335441cf3ba1..6fb34e9533470 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_data_firehose.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_data_firehose.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_s3.md index 549d9b7e36870..8aac91bbac759 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_s3.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/amazon_s3.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -339,6 +349,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/datadog_agent.md index b53d2c315bf5d..91a2b7ea0b05d 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/datadog_agent.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/datadog_agent.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/fluent.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/fluent.md index a7c21ae4db565..3b9881373ceaa 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/fluent.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/fluent.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/google_pubsub.md index fda0dad04695c..d3b7d006fec6a 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/google_pubsub.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/google_pubsub.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -338,6 +348,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_client.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_client.md index ad78fd4227a8b..66d03ecb64dd3 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_client.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_client.md @@ -114,6 +114,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -254,6 +259,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_server.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_server.md index 18587876624e9..cb728855e17e0 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_server.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/http_server.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -339,6 +349,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/kafka.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/kafka.md index 54411a250dfca..713be44b7a694 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/kafka.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/kafka.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/logstash.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/logstash.md index 5f9e82764e9b7..e31ee479fe540 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/logstash.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/logstash.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/socket.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/socket.md new file mode 100644 index 0000000000000..8d9c28ad76c23 --- /dev/null +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/socket.md @@ -0,0 +1,388 @@ +--- +title: Log Volume Control for the Socket Source (TCP or UDP) +disable_toc: false +--- + +## Overview + +Send your logs over a socket connection to the Observability Pipelines Worker so that you only route useful logs to your destinations. + +{{% observability_pipelines/use_case_images/log_volume_control %}} + +This document walks you through the following: +1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines +1. [Setting up Observability Pipelines](#set-up-observability-pipelines) +1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-socket) + +## Prerequisites + +{{% observability_pipelines/prerequisites/socket %}} + +## Set up Observability Pipelines + +1. Navigate to [Observability Pipelines][1]. +1. Select the **Log Volume Control** template to create a new pipeline. +1. Select the **Socket** source. + +### Set up the source + +{{% observability_pipelines/source_settings/socket %}} + +### Set up the destinations + +Enter the following information based on your selected logs destinations. + +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_settings/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_settings/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_settings/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_note %}} + +Follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_settings/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_settings/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_settings/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_settings/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_settings/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_settings/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_settings/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_settings/syslog %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add additional destinations + +{{% observability_pipelines/multiple_destinations %}} + +### Set up processors + +{{% observability_pipelines/processors/intro %}} + +{{% observability_pipelines/processors/filter_syntax %}} + +{{% observability_pipelines/processors/add_processors %}} + +{{< tabs >}} +{{% tab "Add env vars" %}} + +{{% observability_pipelines/processors/add_env_vars %}} + +{{% /tab %}} +{{% tab "Add hostname" %}} + +{{% observability_pipelines/processors/add_hostname %}} + +{{% /tab %}} +{{% tab "Custom Processor" %}} + +{{% observability_pipelines/processors/custom_processor %}} + +{{% /tab %}} +{{% tab "Dedupe" %}} + +{{% observability_pipelines/processors/dedupe %}} + +{{% /tab %}} +{{% tab "Edit fields" %}} + +{{% observability_pipelines/processors/remap %}} + +{{% /tab %}} +{{% tab "Enrichment table" %}} + +{{% observability_pipelines/processors/enrichment_table %}} + +{{% /tab %}} +{{% tab "Filter" %}} + +{{% observability_pipelines/processors/filter %}} + +{{% /tab %}} +{{% tab "Generate metrics" %}} + +{{% observability_pipelines/processors/generate_metrics %}} + +{{% /tab %}} +{{% tab "Grok Parser" %}} + +{{% observability_pipelines/processors/grok_parser %}} + +{{% /tab %}} +{{% tab "Parse JSON" %}} + +{{% observability_pipelines/processors/parse_json %}} + +{{% /tab %}} +{{% tab "Parse XML" %}} + +{{% observability_pipelines/processors/parse_xml %}} + +{{% /tab %}} +{{% tab "Quota" %}} + +{{% observability_pipelines/processors/quota %}} + +{{% /tab %}} +{{% tab "Reduce" %}} + +{{% observability_pipelines/processors/reduce %}} + +{{% /tab %}} +{{% tab "Remap to OCSF" %}} + +{{% observability_pipelines/processors/remap_ocsf %}} + +{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} + +{{% /collapse-content %}} + +{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Sample" %}} + +{{% observability_pipelines/processors/sample %}} + +{{% /tab %}} +{{% tab "Sensitive Data Scanner" %}} + +{{% observability_pipelines/processors/sensitive_data_scanner %}} + +{{% collapse-content title="Add rules from the library" level="h5" %}} + +{{% observability_pipelines/processors/sds_library_rules %}} + +{{% /collapse-content %}} +{{% collapse-content title="Add a custom rule" level="h5" %}} + +{{% observability_pipelines/processors/sds_custom_rules %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Split array" %}} + +{{% observability_pipelines/processors/split_array %}} + +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + +{{% /tab %}} +{{% tab "Throttle" %}} + +{{% observability_pipelines/processors/throttle %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add another set of processors and destinations + +{{% observability_pipelines/multiple_processors %}} + +### Install the Observability Pipelines Worker +1. Select your platform in the **Choose your installation platform** dropdown menu. +1. TKTK +1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_env_vars/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_env_vars/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_env_vars/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_env_vars/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_env_vars/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_env_vars/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_env_vars/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_env_vars/syslog %}} + +{{% /tab %}} +{{< /tabs >}} +1. Follow the instructions for your environment to install the Worker. +{{< tabs >}} +{{% tab "Docker" %}} + +{{% observability_pipelines/install_worker/docker %}} + +{{% /tab %}} +{{% tab "Kubernetes" %}} + +{{% observability_pipelines/install_worker/kubernetes %}} + +{{% /tab %}} +{{% tab "Linux (APT)" %}} + +{{% observability_pipelines/install_worker/linux_apt %}} + +{{% /tab %}} +{{% tab "Linux (RPM)" %}} + +{{% observability_pipelines/install_worker/linux_rpm %}} + +{{% /tab %}} +{{% tab "CloudFormation" %}} + +{{% observability_pipelines/install_worker/cloudformation %}} + +{{% /tab %}} +{{< /tabs >}} + +## Send logs to the Observability Pipelines Worker over Socket + +{{% observability_pipelines/log_source_configuration/socket %}} + +[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_hec.md index 887c649b9a2d3..d8e41439631ef 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_hec.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_hec.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_tcp.md index cb8f95776c925..ce5f522ce3262 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_tcp.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/splunk_tcp.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/sumo_logic_hosted_collector.md index 57ccf064226c2..f86ffff43145b 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/sumo_logic_hosted_collector.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/sumo_logic_hosted_collector.md @@ -111,6 +111,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -251,6 +256,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -338,6 +348,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/syslog.md b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/syslog.md index c514231780f3a..4d0400352448e 100644 --- a/content/en/observability_pipelines/set_up_pipelines/log_volume_control/syslog.md +++ b/content/en/observability_pipelines/set_up_pipelines/log_volume_control/syslog.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/_index.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/_index.md index deea9ef4eba79..0a7836bba85ad 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/_index.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/_index.md @@ -26,6 +26,7 @@ Select a log source to get started: - [Logstash][6] - [Splunk HTTP Event Collector (HEC)][7] - [Splunk Heavy or Universal Forwarders (TCP)][8] +- [Socket (TCP or UDP)][14] - [Sumo Logic Hosted Collector][9] - [rsyslog or syslog-ng][10] @@ -42,3 +43,4 @@ Select a log source to get started: [11]: /observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_s3 [12]: /observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_data_firehose [13]: /observability_pipelines/set_up_pipelines/sensitive_data_redaction/kafka +[14]: /observability_pipelines/set_up_pipelines/sensitive_data_redaction/socket diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_data_firehose.md index 9bd0a63f7b6ae..6c3f84ea70d95 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_data_firehose.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_data_firehose.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_s3.md index 08c91d3aa48d8..1a0224d4c0679 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_s3.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/amazon_s3.md @@ -114,6 +114,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -254,6 +259,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -341,6 +351,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/datadog_agent.md index b970ed186f368..e9145b22efb91 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/datadog_agent.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/datadog_agent.md @@ -117,6 +117,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -257,6 +262,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -344,6 +354,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/fluent.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/fluent.md index daab1b701c8c2..ed1b89795ffb6 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/fluent.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/fluent.md @@ -117,6 +117,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -257,6 +262,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -345,6 +355,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/google_pubsub.md index 91ee95a7bc44e..ca62b00b4921c 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/google_pubsub.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/google_pubsub.md @@ -114,6 +114,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -254,6 +259,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_client.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_client.md index 5b551780e1c02..91587863808b8 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_client.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_client.md @@ -116,6 +116,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -256,6 +261,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -344,6 +354,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_server.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_server.md index 1f82476b9ae2a..fbbb60a2bc2f6 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_server.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/http_server.md @@ -114,6 +114,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -254,6 +259,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -341,6 +351,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/kafka.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/kafka.md index ac31922212b99..f41507e46dcbf 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/kafka.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/kafka.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -345,6 +355,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/logstash.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/logstash.md index ade8173e8a713..b9f9c62d0de75 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/logstash.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/logstash.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/socket.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/socket.md new file mode 100644 index 0000000000000..00edabbb42971 --- /dev/null +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/socket.md @@ -0,0 +1,390 @@ +--- +title: Sensitive Data Redaction for the Socket Source (TCP or UDP) +disable_toc: false +--- + +## Overview + +Sensitive data, such as credit card numbers, bank routing numbers, and API keys, can be revealed unintentionally in your logs, which can expose your organization to financial and privacy risks. + +Use Observability Pipelines to identify, tag, and optionally redact or hash sensitive information before routing logs to different destinations and outside of your infrastructure. You can use out-of-the-box scanning rules to detect common patterns such as email addresses, credit card numbers, API keys, authorization tokens, and more. You can also create custom scanning rules using regex patterns to match sensitive information. + +{{% observability_pipelines/use_case_images/sensitive_data_redaction %}} + +This document walks you through the following: +1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines +1. [Setting up Observability Pipelines](#set-up-observability-pipelines) +1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-socket) + +## Prerequisites + +{{% observability_pipelines/prerequisites/socket %}} + +## Set up Observability Pipelines + +1. Navigate to [Observability Pipelines][1]. +1. Select the **Sensitive Data Redactions** template to create a new pipeline. +1. Select the **Socket** source. + +### Set up the source + +{{% observability_pipelines/source_settings/socket %}} + +### Set up the destinations + +Enter the following information based on your selected logs destinations. + +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_settings/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_settings/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_settings/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_note %}} + +Follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_settings/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_settings/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_settings/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_settings/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_settings/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_settings/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_settings/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_settings/syslog %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add additional destinations + +{{% observability_pipelines/multiple_destinations %}} + +### Set up processors + +{{% observability_pipelines/processors/intro %}} + +{{% observability_pipelines/processors/filter_syntax %}} + +{{% observability_pipelines/processors/add_processors_sds %}} + +{{< tabs >}} +{{% tab "Add env vars" %}} + +{{% observability_pipelines/processors/add_env_vars %}} + +{{% /tab %}} +{{% tab "Add hostname" %}} + +{{% observability_pipelines/processors/add_hostname %}} + +{{% /tab %}} +{{% tab "Custom Processor" %}} + +{{% observability_pipelines/processors/custom_processor %}} + +{{% /tab %}} +{{% tab "Dedupe" %}} + +{{% observability_pipelines/processors/dedupe %}} + +{{% /tab %}} +{{% tab "Edit fields" %}} + +{{% observability_pipelines/processors/remap %}} + +{{% /tab %}} +{{% tab "Enrichment table" %}} + +{{% observability_pipelines/processors/enrichment_table %}} + +{{% /tab %}} +{{% tab "Filter" %}} + +{{% observability_pipelines/processors/filter %}} + +{{% /tab %}} +{{% tab "Generate metrics" %}} + +{{% observability_pipelines/processors/generate_metrics %}} + +{{% /tab %}} +{{% tab "Grok Parser" %}} + +{{% observability_pipelines/processors/grok_parser %}} + +{{% /tab %}} +{{% tab "Parse JSON" %}} + +{{% observability_pipelines/processors/parse_json %}} + +{{% /tab %}} +{{% tab "Parse XML" %}} + +{{% observability_pipelines/processors/parse_xml %}} + +{{% /tab %}} +{{% tab "Quota" %}} + +{{% observability_pipelines/processors/quota %}} + +{{% /tab %}} +{{% tab "Reduce" %}} + +{{% observability_pipelines/processors/reduce %}} + +{{% /tab %}} +{{% tab "Remap to OCSF" %}} + +{{% observability_pipelines/processors/remap_ocsf %}} + +{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} + +{{% /collapse-content %}} + +{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Sample" %}} + +{{% observability_pipelines/processors/sample %}} + +{{% /tab %}} +{{% tab "Sensitive Data Scanner" %}} + +{{% observability_pipelines/processors/sensitive_data_scanner %}} + +{{% collapse-content title="Add rules from the library" level="h5" %}} + +{{% observability_pipelines/processors/sds_library_rules %}} + +{{% /collapse-content %}} +{{% collapse-content title="Add a custom rule" level="h5" %}} + +{{% observability_pipelines/processors/sds_custom_rules %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Split array" %}} + +{{% observability_pipelines/processors/split_array %}} + +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + +{{% /tab %}} +{{% tab "Throttle" %}} + +{{% observability_pipelines/processors/throttle %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add another set of processors and destinations + +{{% observability_pipelines/multiple_processors %}} + +### Install the Observability Pipelines Worker +1. Select your platform in the **Choose your installation platform** dropdown menu. +1. TKTK +1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_env_vars/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_env_vars/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_env_vars/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_env_vars/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_env_vars/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_env_vars/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_env_vars/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_env_vars/syslog %}} + +{{% /tab %}} +{{< /tabs >}} +1. Follow the instructions for your environment to install the Worker. +{{< tabs >}} +{{% tab "Docker" %}} + +{{% observability_pipelines/install_worker/docker %}} + +{{% /tab %}} +{{% tab "Kubernetes" %}} + +{{% observability_pipelines/install_worker/kubernetes %}} + +{{% /tab %}} +{{% tab "Linux (APT)" %}} + +{{% observability_pipelines/install_worker/linux_apt %}} + +{{% /tab %}} +{{% tab "Linux (RPM)" %}} + +{{% observability_pipelines/install_worker/linux_rpm %}} + +{{% /tab %}} +{{% tab "CloudFormation" %}} + +{{% observability_pipelines/install_worker/cloudformation %}} + +{{% /tab %}} +{{< /tabs >}} + +## Send logs to the Observability Pipelines Worker over Socket + +{{% observability_pipelines/log_source_configuration/socket %}} + +[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_hec.md index 79f46a45218a1..babd4c0b01f36 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_hec.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_hec.md @@ -117,6 +117,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -257,6 +262,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -344,6 +354,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_tcp.md index 43a848bb2f038..06073ce0d3a51 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_tcp.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/splunk_tcp.md @@ -117,6 +117,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -257,6 +262,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -344,6 +354,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector.md index 30109df93315e..751f3019bb304 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/sumo_logic_hosted_collector.md @@ -119,6 +119,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -259,6 +264,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -346,6 +356,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/syslog.md b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/syslog.md index afd0932dc3c9a..16490e3277229 100644 --- a/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/syslog.md +++ b/content/en/observability_pipelines/set_up_pipelines/sensitive_data_redaction/syslog.md @@ -117,6 +117,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -257,6 +262,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -345,6 +355,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/_index.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/_index.md index e187f236ac561..4ab0c18367ba3 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/_index.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/_index.md @@ -24,6 +24,7 @@ Select your log source to get started: - [Logstash][6] - [Splunk HTTP Event Collector (HEC)][7] - [Splunk Heavy or Universal Forwarders (TCP)][8] +- [Socket (TCP or UDP)][14] - [Sumo Logic Hosted Collector][9] - [rsyslog or syslog-ng][10] @@ -39,4 +40,5 @@ Select your log source to get started: [10]: /observability_pipelines/split_logs/syslog [11]: /observability_pipelines/set_up_pipelines/split_logs/amazon_s3 [12]: /observability_pipelines/set_up_pipelines/split_logs/amazon_data_firehose -[13]: /observability_pipelines/set_up_pipelines/split_logs/kafka \ No newline at end of file +[13]: /observability_pipelines/set_up_pipelines/split_logs/kafka +[14]: /observability_pipelines/set_up_pipelines/split_logs/socket \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_data_firehose.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_data_firehose.md index 480bdedac2471..7716779cc747f 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_data_firehose.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_data_firehose.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_s3.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_s3.md index bf57110eb8556..3622ec04284dc 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_s3.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/amazon_s3.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -339,6 +349,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/datadog_agent.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/datadog_agent.md index 9322f72de7f8e..dacc144fe588b 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/datadog_agent.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/datadog_agent.md @@ -128,6 +128,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -268,6 +273,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -355,6 +365,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/fluent.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/fluent.md index 3cd09cebb9a8b..277318a657f4c 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/fluent.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/fluent.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/google_pubsub.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/google_pubsub.md index 598b7a5566d6e..bd7135e418f81 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/google_pubsub.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/google_pubsub.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -338,6 +348,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/http_client.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/http_client.md index feacd9b4071a6..1846635c57083 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/http_client.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/http_client.md @@ -114,6 +114,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -254,6 +259,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -342,6 +352,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/http_server.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/http_server.md index aa47bc0fb4667..a62cfe6b0dbcc 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/http_server.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/http_server.md @@ -112,6 +112,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -252,6 +257,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -339,6 +349,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/kafka.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/kafka.md index fc9ecda9f2b8f..d781cd6bfad9c 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/kafka.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/kafka.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/logstash.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/logstash.md index 8fec2d4f1c941..a7f9ff53900ef 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/logstash.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/logstash.md @@ -113,6 +113,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -253,6 +258,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -340,6 +350,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/socket.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/socket.md new file mode 100644 index 0000000000000..0d6bc6e4ccfdf --- /dev/null +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/socket.md @@ -0,0 +1,388 @@ +--- +title: Split Logs for the Socket Source (TCP or UDP) +disable_toc: false +--- + +## Overview + +Send logs over a socket connection to the Observability Pipelines Worker and then process and route them to different destinations based on your use case. + +{{% observability_pipelines/use_case_images/split_logs %}} + +This document walks you through the following: +1. The [prerequisites](#prerequisites) needed to set up Observability Pipelines +1. [Setting up Observability Pipelines](#set-up-observability-pipelines) +1. [Sending logs to the Observability Pipelines Worker](#send-logs-to-the-observability-pipelines-worker-over-socket) + +## Prerequisites + +{{% observability_pipelines/prerequisites/socket %}} + +## Set up Observability Pipelines + +1. Navigate to [Observability Pipelines][1]. +1. Select the **Split Logs** template to create a new pipeline. +1. Select the **Socket** source. + +### Set up the source + +{{% observability_pipelines/source_settings/socket %}} + +### Set up the destinations + +Enter the following information based on your selected logs destinations. + +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_settings/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_settings/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_settings/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_settings/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_note %}} + +Follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_settings/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_settings/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_settings/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_settings/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_settings/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_settings/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_settings/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_settings/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_settings/syslog %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add additional destinations + +{{% observability_pipelines/multiple_destinations %}} + +### Set up processors + +{{% observability_pipelines/processors/intro %}} + +{{% observability_pipelines/processors/filter_syntax %}} + +{{% observability_pipelines/processors/add_processors %}} + +{{< tabs >}} +{{% tab "Add env vars" %}} + +{{% observability_pipelines/processors/add_env_vars %}} + +{{% /tab %}} +{{% tab "Add hostname" %}} + +{{% observability_pipelines/processors/add_hostname %}} + +{{% /tab %}} +{{% tab "Custom Processor" %}} + +{{% observability_pipelines/processors/custom_processor %}} + +{{% /tab %}} +{{% tab "Dedupe" %}} + +{{% observability_pipelines/processors/dedupe %}} + +{{% /tab %}} +{{% tab "Edit fields" %}} + +{{% observability_pipelines/processors/remap %}} + +{{% /tab %}} +{{% tab "Enrichment table" %}} + +{{% observability_pipelines/processors/enrichment_table %}} + +{{% /tab %}} +{{% tab "Filter" %}} + +{{% observability_pipelines/processors/filter %}} + +{{% /tab %}} +{{% tab "Generate metrics" %}} + +{{% observability_pipelines/processors/generate_metrics %}} + +{{% /tab %}} +{{% tab "Grok Parser" %}} + +{{% observability_pipelines/processors/grok_parser %}} + +{{% /tab %}} +{{% tab "Parse JSON" %}} + +{{% observability_pipelines/processors/parse_json %}} + +{{% /tab %}} +{{% tab "Parse XML" %}} + +{{% observability_pipelines/processors/parse_xml %}} + +{{% /tab %}} +{{% tab "Quota" %}} + +{{% observability_pipelines/processors/quota %}} + +{{% /tab %}} +{{% tab "Reduce" %}} + +{{% observability_pipelines/processors/reduce %}} + +{{% /tab %}} +{{% tab "Remap to OCSF" %}} + +{{% observability_pipelines/processors/remap_ocsf %}} + +{{% collapse-content title="Library mapping" level="h5" expanded=false id="library_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_library_mapping %}} + +{{% /collapse-content %}} + +{{% collapse-content title="Custom mapping" level="h5" expanded=false id="custom_mapping" %}} + +{{% observability_pipelines/processors/remap_ocsf_custom_mapping %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Sample" %}} + +{{% observability_pipelines/processors/sample %}} + +{{% /tab %}} +{{% tab "Sensitive Data Scanner" %}} + +{{% observability_pipelines/processors/sensitive_data_scanner %}} + +{{% collapse-content title="Add rules from the library" level="h5" %}} + +{{% observability_pipelines/processors/sds_library_rules %}} + +{{% /collapse-content %}} +{{% collapse-content title="Add a custom rule" level="h5" %}} + +{{% observability_pipelines/processors/sds_custom_rules %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Split array" %}} + +{{% observability_pipelines/processors/split_array %}} + +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + +{{% /tab %}} +{{% tab "Throttle" %}} + +{{% observability_pipelines/processors/throttle %}} + +{{% /tab %}} +{{< /tabs >}} + +#### Add another set of processors and destinations + +{{% observability_pipelines/multiple_processors %}} + +### Install the Observability Pipelines Worker +1. Select your platform in the **Choose your installation platform** dropdown menu. +1. TKTK +1. Provide the environment variables for each of your selected destinations. See [Prerequisites](#prerequisites) for more information. +{{< tabs >}} +{{% tab "Amazon OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/amazon_opensearch %}} + +{{% /tab %}} +{{% tab "Chronicle" %}} + +{{% observability_pipelines/destination_env_vars/chronicle %}} + +{{% /tab %}} +{{% tab "CrowdStrike NG-SIEM" %}} + +{{% observability_pipelines/destination_env_vars/crowdstrike_ng_siem %}} + +{{% /tab %}} +{{% tab "Datadog" %}} + +{{% observability_pipelines/destination_env_vars/datadog %}} + +{{% /tab %}} +{{% tab "Datadog Archives" %}} + +For the Datadog Archives destination, follow the instructions for the cloud provider you are using to archive your logs. + +{{% collapse-content title="Amazon S3" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_amazon_s3 %}} + +{{% /collapse-content %}} +{{% collapse-content title="Google Cloud Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_google_cloud_storage %}} + +{{% /collapse-content %}} +{{% collapse-content title="Azure Storage" level="h5" %}} + +{{% observability_pipelines/destination_env_vars/datadog_archives_azure_storage %}} + +{{% /collapse-content %}} + +{{% /tab %}} +{{% tab "Elasticsearch" %}} + +{{% observability_pipelines/destination_env_vars/elasticsearch %}} + +{{% /tab %}} +{{% tab "Microsoft Sentinel" %}} + +{{% observability_pipelines/destination_env_vars/microsoft_sentinel %}} + +{{% /tab %}} +{{% tab "New Relic" %}} + +{{% observability_pipelines/destination_env_vars/new_relic %}} + +{{% /tab %}} +{{% tab "OpenSearch" %}} + +{{% observability_pipelines/destination_env_vars/opensearch %}} + +{{% /tab %}} +{{% tab "SentinelOne" %}} + +{{% observability_pipelines/destination_env_vars/sentinelone %}} + +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + +{{% /tab %}} +{{% tab "Splunk HEC" %}} + +{{% observability_pipelines/destination_env_vars/splunk_hec %}} + +{{% /tab %}} +{{% tab "Sumo Logic" %}} + +{{% observability_pipelines/destination_env_vars/sumo_logic %}} + +{{% /tab %}} +{{% tab "Syslog" %}} + +{{% observability_pipelines/destination_env_vars/syslog %}} + +{{% /tab %}} +{{< /tabs >}} +1. Follow the instructions for your environment to install the Worker. +{{< tabs >}} +{{% tab "Docker" %}} + +{{% observability_pipelines/install_worker/docker %}} + +{{% /tab %}} +{{% tab "Kubernetes" %}} + +{{% observability_pipelines/install_worker/kubernetes %}} + +{{% /tab %}} +{{% tab "Linux (APT)" %}} + +{{% observability_pipelines/install_worker/linux_apt %}} + +{{% /tab %}} +{{% tab "Linux (RPM)" %}} + +{{% observability_pipelines/install_worker/linux_rpm %}} + +{{% /tab %}} +{{% tab "CloudFormation" %}} + +{{% observability_pipelines/install_worker/cloudformation %}} + +{{% /tab %}} +{{< /tabs >}} + +## Send logs to the Observability Pipelines Worker over Socket + +{{% observability_pipelines/log_source_configuration/socket %}} + +[1]: https://app.datadoghq.com/observability-pipelines \ No newline at end of file diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_hec.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_hec.md index 6210a3200816e..629a42c0f0e4f 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_hec.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_hec.md @@ -121,6 +121,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -261,6 +266,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -348,6 +358,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_tcp.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_tcp.md index bb3af7ccd67db..ab349c4cb8308 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_tcp.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/splunk_tcp.md @@ -121,6 +121,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -261,6 +266,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -348,6 +358,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/sumo_logic_hosted_collector.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/sumo_logic_hosted_collector.md index 482d9f83d621b..73cdc8cff12df 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/sumo_logic_hosted_collector.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/sumo_logic_hosted_collector.md @@ -121,6 +121,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -261,6 +266,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -348,6 +358,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/set_up_pipelines/split_logs/syslog.md b/content/en/observability_pipelines/set_up_pipelines/split_logs/syslog.md index 6bed110aed1d8..c8986054cb3c0 100644 --- a/content/en/observability_pipelines/set_up_pipelines/split_logs/syslog.md +++ b/content/en/observability_pipelines/set_up_pipelines/split_logs/syslog.md @@ -115,6 +115,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/destination_settings/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_settings/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -255,6 +260,11 @@ To set up the destination, follow the instructions for the cloud provider you ar {{% observability_pipelines/processors/split_array %}} +{{% /tab %}} +{{% tab "Tags Processor" %}} + +{{% observability_pipelines/processors/tags_processor %}} + {{% /tab %}} {{% tab "Throttle" %}} @@ -343,6 +353,11 @@ For the Datadog Archives destination, follow the instructions for the cloud prov {{% observability_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/content/en/observability_pipelines/sources/socket.md b/content/en/observability_pipelines/sources/socket.md new file mode 100644 index 0000000000000..1f207f3df9145 --- /dev/null +++ b/content/en/observability_pipelines/sources/socket.md @@ -0,0 +1,20 @@ +--- +title: Socket Source +disable_toc: false +--- + +Use Observability Pipelines’ Socket source to send logs to the Worker over a socket connection (TCP or UDP). Select and set up this source when you [set up a pipeline][1]. + +## Prerequisites + +{{% observability_pipelines/prerequisites/socket %}} + +## Set up the source in the pipeline UI + +{{% observability_pipelines/source_settings/socket %}} + +## Send logs to the Observability Pipelines Worker over a socket connection + +{{% observability_pipelines/log_source_configuration/socket %}} + +[1]: /observability_pipelines/set_up_pipelines/ \ No newline at end of file diff --git a/content/en/observability_pipelines/update_existing_pipelines.md b/content/en/observability_pipelines/update_existing_pipelines.md index 239a311d10265..5f040e52b2149 100644 --- a/content/en/observability_pipelines/update_existing_pipelines.md +++ b/content/en/observability_pipelines/update_existing_pipelines.md @@ -68,6 +68,11 @@ On the Worker installation page: {{% observability_pipelines/configure_existing_pipelines/source_env_vars/logstash %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/configure_existing_pipelines/source_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} @@ -150,6 +155,11 @@ On the Worker installation page: {{% observability_pipelines/configure_existing_pipelines/destination_env_vars/sentinelone %}} +{{% /tab %}} +{{% tab "Socket" %}} + +{{% observability_pipelines/configure_existing_pipelines/destination_env_vars/socket %}} + {{% /tab %}} {{% tab "Splunk HEC" %}} diff --git a/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/destination_env_vars/socket.md b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/destination_env_vars/socket.md new file mode 100644 index 0000000000000..dd8352deb24cc --- /dev/null +++ b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/destination_env_vars/socket.md @@ -0,0 +1,3 @@ +- Socket address: + - The Observability Pipelines Worker sends processed logs to this address. + - Stored as the environment variable `DD_OP_DESTINATION_SOCKET_ADDRESS`. \ No newline at end of file diff --git a/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/source_env_vars/socket.md b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/source_env_vars/socket.md new file mode 100644 index 0000000000000..7221b4d732336 --- /dev/null +++ b/layouts/shortcodes/observability_pipelines/configure_existing_pipelines/source_env_vars/socket.md @@ -0,0 +1 @@ +REUSE INSTRUCTIONS \ No newline at end of file diff --git a/layouts/shortcodes/observability_pipelines/destination_batching.en.md b/layouts/shortcodes/observability_pipelines/destination_batching.en.md index 713b3e2cd6e42..6dad67419472e 100644 --- a/layouts/shortcodes/observability_pipelines/destination_batching.en.md +++ b/layouts/shortcodes/observability_pipelines/destination_batching.en.md @@ -13,7 +13,9 @@ | New Relic | 100 | 1,000,000 | 1 | | OpenSearch | None | 10,000,000 | 1 | | SentinelOne | None | 1,000,000 | 1 | +| Socket* | N/A | N/A | N/A | | Splunk HTTP Event Collector (HEC) | None | 1,000,000 | 1 | | Sumo Logic Hosted Collecter | None | 10,000,000 | 1 | +| Syslog* | N/A | N/A | N/A | -**Note**: The rsyslog and syslog-ng destinations do not batch events. \ No newline at end of file +*Destination does not batch events. \ No newline at end of file diff --git a/layouts/shortcodes/observability_pipelines/destination_env_vars/socket.md b/layouts/shortcodes/observability_pipelines/destination_env_vars/socket.md new file mode 100644 index 0000000000000..a16b29a74c506 --- /dev/null +++ b/layouts/shortcodes/observability_pipelines/destination_env_vars/socket.md @@ -0,0 +1 @@ +Enter the socket destination address, such as `92.12.333.224:5000` or `https://somehost:5000`. The address must include a port. \ No newline at end of file diff --git a/layouts/shortcodes/observability_pipelines/destination_settings/socket.md b/layouts/shortcodes/observability_pipelines/destination_settings/socket.md new file mode 100644 index 0000000000000..a15ecaf1c3bb7 --- /dev/null +++ b/layouts/shortcodes/observability_pipelines/destination_settings/socket.md @@ -0,0 +1,6 @@ +1. In the **Mode** dropdown menu, select the socket type to use. +1. In the **Encoding** dropdown menu, select whether you want to encode your pipeline's output in `JSON` or `Raw message`. +1. Optionally, toggle the switch to enable TLS. If you enable TLS, the following certificate and key files are required: + - `Server Certificate Path`: The path to the certificate file that has been signed by your Certificate Authority (CA) Root File in DER or PEM (X.509). + - `CA Certificate Path`: The path to the certificate file that is your Certificate Authority (CA) Root File in DER or PEM (X.509). + - `Private Key Path`: The path to the `.key` private key file that belongs to your Server Certificate Path in DER or PEM (PKCS#8) format. diff --git a/layouts/shortcodes/observability_pipelines/log_source_configuration/socket.md b/layouts/shortcodes/observability_pipelines/log_source_configuration/socket.md new file mode 100644 index 0000000000000..7221b4d732336 --- /dev/null +++ b/layouts/shortcodes/observability_pipelines/log_source_configuration/socket.md @@ -0,0 +1 @@ +REUSE INSTRUCTIONS \ No newline at end of file diff --git a/layouts/shortcodes/observability_pipelines/prerequisites/socket.md b/layouts/shortcodes/observability_pipelines/prerequisites/socket.md new file mode 100644 index 0000000000000..7221b4d732336 --- /dev/null +++ b/layouts/shortcodes/observability_pipelines/prerequisites/socket.md @@ -0,0 +1 @@ +REUSE INSTRUCTIONS \ No newline at end of file diff --git a/layouts/shortcodes/observability_pipelines/processors/tags_processor.md b/layouts/shortcodes/observability_pipelines/processors/tags_processor.md new file mode 100644 index 0000000000000..526c7737b9892 --- /dev/null +++ b/layouts/shortcodes/observability_pipelines/processors/tags_processor.md @@ -0,0 +1,30 @@ +Use this processor to exclude or include specific tags in the Datadog tags (`ddtags`) array for logs coming from the Datadog Agent. Tags that are excluded or not included are dropped and may reduce your outbound log volume. + +To set up the processor: + +1. Define a filter query. Only logs that match the specified filter query are processed. All logs, regardless of whether they match the filter query, are sent to the next step in the pipeline. +1. Enter the path to the array. Use the path notation `.` to match subfields. See the [Path notation example](#path-notation-example) below. +1. Optionally, input a Datadog tags array for the **Configure tags** section. For example: `["a:b", "c:d"]` or `["a:b", "c:d", "e"]`. +1. In the **Configure tags** section, select whether you want to **Exclude tags** or **Include tags**. In the dropdown menu, select the tag keys that you want to exclude or include. **Note**: Wild cards are supported. + +##### Path notation example + +For the following message structure: + +```json +{ + "outer_key": { + "inner_key": "inner_value", + "a": { + "double_inner_key": "double_inner_value", + "b": "b value" + }, + "c": "c value" + }, + "d": "d value" +} +``` + +- Use `outer_key.inner_key` to refer to the key with the value `inner_value`. +- Use `outer_key.inner_key.double_inner_key` to refer to the key with the value `double_inner_value`. + diff --git a/layouts/shortcodes/observability_pipelines/source_settings/socket.md b/layouts/shortcodes/observability_pipelines/source_settings/socket.md new file mode 100644 index 0000000000000..7221b4d732336 --- /dev/null +++ b/layouts/shortcodes/observability_pipelines/source_settings/socket.md @@ -0,0 +1 @@ +REUSE INSTRUCTIONS \ No newline at end of file