From 7f57e9d5388156df1fa3f111e820022060cd16fc Mon Sep 17 00:00:00 2001 From: Gustavo Lopes Date: Tue, 3 Jun 2025 17:49:55 +0100 Subject: [PATCH] Support DD_APM_TRACING_ENABLED=false --- CMakeLists.txt | 2 +- dd-trace-cpp | 2 +- src/common/headers.cpp | 66 +++- src/common/headers.h | 27 +- src/datadog_conf.h | 4 + src/datadog_context.cpp | 3 +- src/datadog_directive.h | 1 + src/ngx_header_writer.h | 9 +- src/ngx_http_datadog_module.cpp | 40 ++- src/request_tracing.cpp | 12 - src/security/context.cpp | 18 +- src/security/context.h | 12 +- src/security/library.cpp | 12 + src/security/library.h | 2 + src/tracing/directives.h | 8 + src/tracing_library.cpp | 4 + .../configuration/conf/apm_tracing_off.conf | 27 ++ test/cases/configuration/conf/waf.json | 65 ++++ .../configuration/test_apm_tracing_enabled.py | 333 ++++++++++++++++++ 19 files changed, 597 insertions(+), 50 deletions(-) create mode 100644 test/cases/configuration/conf/apm_tracing_off.conf create mode 100644 test/cases/configuration/conf/waf.json create mode 100644 test/cases/configuration/test_apm_tracing_enabled.py diff --git a/CMakeLists.txt b/CMakeLists.txt index df2eeff0..0276e54c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) cmake_policy(SET CMP0068 NEW) cmake_policy(SET CMP0135 NEW) -set(NGINX_DATADOG_VERSION 1.6.1) +set(NGINX_DATADOG_VERSION 1.7.0) project(ngx_http_datadog_module VERSION ${NGINX_DATADOG_VERSION}) option(NGINX_DATADOG_ASM_ENABLED "Build with libddwaf" ON) diff --git a/dd-trace-cpp b/dd-trace-cpp index 83958bcd..749f18be 160000 --- a/dd-trace-cpp +++ b/dd-trace-cpp @@ -1 +1 @@ -Subproject commit 83958bcd5f0519fddd1bbdadadf8c94f290ac388 +Subproject commit 749f18bea4e74780e13b718f4196fb5605db9e07 diff --git a/src/common/headers.cpp b/src/common/headers.cpp index 8de5a5b0..ad74d353 100644 --- a/src/common/headers.cpp +++ b/src/common/headers.cpp @@ -1,13 +1,24 @@ #include "headers.h" +#include + #include "string_util.h" -namespace datadog::common { +extern "C" { +#include +} -ngx_table_elt_t *search_header(ngx_list_t &headers, std::string_view key) { - ngx_list_part_t *part = &headers.part; - auto *h = static_cast(part->elts); +namespace { + +template +auto search_header_impl(ngx_list_t &headers, std::string_view key) { + auto key_lc = std::unique_ptr{new u_char[key.size()]}; + std::transform(key.begin(), key.end(), key_lc.get(), + datadog::nginx::to_lower); + ngx_uint_t key_hash = ngx_hash_key(key_lc.get(), key.size()); + ngx_list_part_t *part = &headers.part; + ngx_table_elt_t *h = static_cast(part->elts); for (std::size_t i = 0;; i++) { if (i >= part->nelts) { if (part->next == nullptr) { @@ -19,19 +30,42 @@ ngx_table_elt_t *search_header(ngx_list_t &headers, std::string_view key) { i = 0; } - if (key.size() != h[i].key.len || - ngx_strncasecmp((u_char *)key.data(), h[i].key.data, key.size()) != 0) { + if (h[i].hash != key_hash || key.size() != h[i].key.len || + memcmp(key_lc.get(), h[i].lowcase_key, key.size()) != 0) { continue; } - return &h[i]; + if constexpr (delete_header) { + part->nelts--; + if (i < part->nelts) { + memmove(&h[i], &h[i + 1], (part->nelts - i) * sizeof(*h)); + } + return true; + } else { + return &h[i]; + } + } + + if constexpr (delete_header) { + return false; + } else { + return static_cast(nullptr); } +} +} // namespace + +namespace datadog::common { - return nullptr; +ngx_table_elt_t *search_req_header(ngx_list_t &headers, std::string_view key) { + return search_header_impl(headers, key); } -bool add_header(ngx_pool_t &pool, ngx_list_t &headers, std::string_view key, - std::string_view value) { +bool delete_req_header(ngx_list_t &headers, std::string_view key) { + return search_header_impl(headers, key); +} + +bool add_req_header(ngx_pool_t &pool, ngx_list_t &headers, std::string_view key, + std::string_view value) { if (headers.last == nullptr) { // Certainly a bad request (4xx). No need to add HTTP headers. return false; @@ -44,13 +78,6 @@ bool add_header(ngx_pool_t &pool, ngx_list_t &headers, std::string_view key, const auto key_size = key.size(); - // This trick tells ngx_http_header_module to reflect the header value - // in the actual response. Otherwise the header will be ignored and client - // will never see it. To date the value must be just non zero. - // Source: - // - h->hash = 1; - // HTTP proxy module expects the header to has a lowercased key value // Instead of allocating twice the same key, `h->key` and `h->lowcase_key` // use the same data. @@ -61,6 +88,11 @@ bool add_header(ngx_pool_t &pool, ngx_list_t &headers, std::string_view key, } h->lowcase_key = h->key.data; + // In request headers, the hash should be calculated from the lowercase key. + // See ngx_http_parse_header_line in ngx_http_parse.c + // Response headers OTOH use either 1 or 0, with 0 meaning "skip this header". + h->hash = ngx_hash_key(h->lowcase_key, key.size()); + h->value = nginx::to_ngx_str(&pool, value); return true; } diff --git a/src/common/headers.h b/src/common/headers.h index aad4c3dc..bea604a1 100644 --- a/src/common/headers.h +++ b/src/common/headers.h @@ -9,7 +9,8 @@ extern "C" { namespace datadog::common { -/// Searches through an NGINX header list to find a header with a matching key. +/// Searches through an NGINX request header list to find a header with a +/// matching key. /// /// @param headers /// A reference to an NGINX-style list (`ngx_list_t`) containing @@ -23,9 +24,25 @@ namespace datadog::common { /// A pointer to the matching `ngx_table_elt_t` header element if found, /// or `nullptr` if no header with the given key exists in the list. //// -ngx_table_elt_t *search_header(ngx_list_t &headers, std::string_view key); +ngx_table_elt_t *search_req_header(ngx_list_t &headers, std::string_view key); -/// Adds a new HTTP header to an NGINX-style header list. +/// Deletes a request header with the specified key from a NGINX-request header +/// list. +/// +/// @param headers +/// A reference to an NGINX-style list (`ngx_list_t`) containing +/// `ngx_table_elt_t` elements, typically representing HTTP headers. +/// +/// @param key +/// A string view representing the name of the header to delete. +/// The comparison is case-insensitive. +/// +/// @return +/// `true` if a header with the given key was found and deleted; +/// `false` if no header with the given key exists in the list. +bool delete_req_header(ngx_list_t &headers, std::string_view key); + +/// Adds a new HTTP request header to an NGINX-style header list. /// /// @param pool /// A reference to the NGINX memory pool (`ngx_pool_t`) used for allocating @@ -47,7 +64,7 @@ ngx_table_elt_t *search_header(ngx_list_t &headers, std::string_view key); /// @return /// `true` if the header was successfully added to the list; /// `false` if memory allocation failed or the list could not be updated. -bool add_header(ngx_pool_t &pool, ngx_list_t &headers, std::string_view key, - std::string_view value); +bool add_req_header(ngx_pool_t &pool, ngx_list_t &headers, std::string_view key, + std::string_view value); } // namespace datadog::common diff --git a/src/datadog_conf.h b/src/datadog_conf.h index 99a82de8..7f4e89c4 100644 --- a/src/datadog_conf.h +++ b/src/datadog_conf.h @@ -57,6 +57,10 @@ struct sampling_rule_t { }; struct datadog_main_conf_t { + // DD_APM_TRACING_ENABLED + // Whether we discard almost all traces not setting _dd.p.ts + ngx_flag_t apm_tracing_enabled{NGX_CONF_UNSET}; + std::unordered_map tags; // `are_propagation_styles_locked` is whether the tracer's propagation styles // have been set, either by an explicit `datadog_propagation_styles` diff --git a/src/datadog_context.cpp b/src/datadog_context.cpp index 87eb1519..6720a6d9 100644 --- a/src/datadog_context.cpp +++ b/src/datadog_context.cpp @@ -22,7 +22,8 @@ DatadogContext::DatadogContext(ngx_http_request_t *request, datadog_loc_conf_t *loc_conf) #ifdef WITH_WAF : sec_ctx_{security::Context::maybe_create( - *loc_conf, security::Library::max_saved_output_data())} + security::Library::max_saved_output_data(), + security::Library::apm_tracing_enabled())} #endif { if (loc_conf->enable_tracing) { diff --git a/src/datadog_directive.h b/src/datadog_directive.h index 828d6efa..3aa9ab9a 100644 --- a/src/datadog_directive.h +++ b/src/datadog_directive.h @@ -7,6 +7,7 @@ extern "C" { #include } +#include #include #include "string_util.h" diff --git a/src/ngx_header_writer.h b/src/ngx_header_writer.h index 8ab92847..7aed53d8 100644 --- a/src/ngx_header_writer.h +++ b/src/ngx_header_writer.h @@ -24,12 +24,17 @@ class NgxHeaderWriter : public datadog::tracing::DictWriter { : request_(request), pool_(request_->pool) {} void set(std::string_view key, std::string_view value) override { + if (value.empty()) { + common::delete_req_header(request_->headers_in.headers, key); + return; + } + ngx_table_elt_t *h = - common::search_header(request_->headers_in.headers, key); + common::search_req_header(request_->headers_in.headers, key); if (h != nullptr) { h->value = to_ngx_str(pool_, value); } else { - common::add_header(*pool_, request_->headers_in.headers, key, value); + common::add_req_header(*pool_, request_->headers_in.headers, key, value); } } }; diff --git a/src/ngx_http_datadog_module.cpp b/src/ngx_http_datadog_module.cpp index efc17984..ca80fe80 100644 --- a/src/ngx_http_datadog_module.cpp +++ b/src/ngx_http_datadog_module.cpp @@ -3,14 +3,14 @@ #include #include #include -#include #include -#include #include #include +#include "datadog/injection_options.h" #include "datadog_conf.h" #include "datadog_conf_handler.h" +#include "datadog_context.h" #include "datadog_directive.h" #include "datadog_handler.h" #include "datadog_variable.h" @@ -28,6 +28,7 @@ #include "rum/config.h" #endif #include "common/variable.h" +#include "ngx_header_writer.h" #include "string_util.h" #include "tracing_library.h" #include "version.h" @@ -278,6 +279,8 @@ static ngx_int_t datadog_master_process_post_config( return NGX_OK; } +static ngx_int_t on_precontent_phase(ngx_http_request_t *request) noexcept; + static ngx_int_t datadog_module_init(ngx_conf_t *cf) noexcept { ngx_http_next_header_filter = ngx_http_top_header_filter; ngx_http_top_header_filter = on_header_filter; @@ -313,6 +316,11 @@ static ngx_int_t datadog_module_init(ngx_conf_t *cf) noexcept { } #endif + if (set_handler(cf->log, core_main_config, NGX_HTTP_PRECONTENT_PHASE, + on_precontent_phase) != NGX_OK) { + return NGX_ERROR; + } + // Add default span tags. const auto tags = TracingLibrary::default_tags(); if (!tags.empty()) { @@ -493,3 +501,31 @@ static char *merge_datadog_loc_conf(ngx_conf_t *cf, void *parent, return NGX_CONF_OK; } + +static ngx_int_t on_precontent_phase(ngx_http_request_t *request) noexcept { + auto *ctx = get_datadog_context(request); + if (!ctx) { + return NGX_DECLINED; + } + + // inject headers in the precontent phase into the request headers + // These headers will be copied by ngx_http_proxy_create_request on the + // content phase into the outgoing request headers (probably) + RequestTracing &tracing = ctx->single_trace(); + dd::Span &span = tracing.active_span(); + span.set_tag("span.kind", "client"); + + datadog::tracing::InjectionOptions opts{}; +#ifdef WITH_WAF + if (auto sec_ctx = ctx->get_security_context()) { + if (sec_ctx->has_matches()) { + opts.trace_source = {'0', '2'}; + } + } +#endif + + NgxHeaderWriter writer(request); + span.inject(writer, opts); + + return NGX_DECLINED; +} diff --git a/src/request_tracing.cpp b/src/request_tracing.cpp index 687ccc0d..f1eef46d 100644 --- a/src/request_tracing.cpp +++ b/src/request_tracing.cpp @@ -240,12 +240,6 @@ RequestTracing::RequestTracing(ngx_http_request_t *request, // We care about sampling rules for the request span only, because it's the // only span that could be the root span. set_sample_rate_tag(request_, loc_conf_, *request_span_); - - // Inject the active span - NgxHeaderWriter writer(request_); - auto &span = active_span(); - span.set_tag("span.kind", "client"); - span.inject(writer); } void RequestTracing::on_change_block(ngx_http_core_loc_conf_t *core_loc_conf, @@ -275,12 +269,6 @@ void RequestTracing::on_change_block(ngx_http_core_loc_conf_t *core_loc_conf, // We care about sampling rules for the request span only, because it's the // only span that could be the root span. set_sample_rate_tag(request_, loc_conf_, *request_span_); - - // Inject the active span - NgxHeaderWriter writer(request_); - auto &span = active_span(); - span.set_tag("span.kind", "client"); - span.inject(writer); } dd::Span &RequestTracing::active_span() { diff --git a/src/security/context.cpp b/src/security/context.cpp index 1684a90d..5c4c8eef 100644 --- a/src/security/context.cpp +++ b/src/security/context.cpp @@ -197,8 +197,11 @@ auto catch_exceptions(std::string_view name, const ngx_http_request_t &req, namespace datadog::nginx::security { -Context::Context(std::shared_ptr handle) - : stage_{new std::atomic{}}, waf_handle_{std::move(handle)} { +Context::Context(std::shared_ptr handle, + bool apm_tracing_enabled) + : stage_{new std::atomic{}}, + waf_handle_{std::move(handle)}, + apm_tracing_enabled_{apm_tracing_enabled} { if (!waf_handle_) { return; } @@ -210,13 +213,14 @@ Context::Context(std::shared_ptr handle) } std::unique_ptr Context::maybe_create( - datadog_loc_conf_t &loc_conf, - std::optional max_saved_output_data) { + std::optional max_saved_output_data, + bool apm_tracing_enabled) { std::shared_ptr handle = Library::get_handle(); if (!handle) { return {}; } - auto res = std::unique_ptr{new Context{std::move(handle)}}; + auto res = std::unique_ptr{ + new Context{std::move(handle), apm_tracing_enabled}}; if (max_saved_output_data) { res->max_saved_output_data_ = *max_saved_output_data; } @@ -1786,6 +1790,10 @@ void Context::report_matches(ngx_http_request_t &request, dd::Span &span) { report_match(request, span.trace_segment(), span, results_); results_.clear(); + + if (!apm_tracing_enabled_) { + span.set_tag("_dd.p.ts"sv, "02"sv); + } } void Context::report_client_ip(dd::Span &span) const { diff --git a/src/security/context.h b/src/security/context.h index 5a41880e..7d30e992 100644 --- a/src/security/context.h +++ b/src/security/context.h @@ -52,13 +52,14 @@ struct OwnedDdwafContext }; class Context { - Context(std::shared_ptr waf_handle); + Context(std::shared_ptr waf_handle, + bool apm_tracing_enabled); public: // returns a new context or an empty unique_ptr if the waf is not active static std::unique_ptr maybe_create( - datadog_loc_conf_t &loc_conf, - std::optional max_saved_output_data); + std::optional max_saved_output_data, + bool apm_tracing_enabled); ngx_int_t request_body_filter(ngx_http_request_t &request, ngx_chain_t *chain, dd::Span &span) noexcept; @@ -87,6 +88,8 @@ class Context { std::optional run_waf_end(ngx_http_request_t &request, dd::Span &span); + bool has_matches() const noexcept; + private: bool do_on_request_start(ngx_http_request_t &request, dd::Span &span); ngx_int_t do_request_body_filter(ngx_http_request_t &request, @@ -96,7 +99,6 @@ class Context { ngx_chain_t *chain, dd::Span &span); void do_on_main_log_request(ngx_http_request_t &request, dd::Span &span); - bool has_matches() const noexcept; void report_matches(ngx_http_request_t &request, dd::Span &span); void report_client_ip(dd::Span &span) const; @@ -198,6 +200,8 @@ class Context { static inline constexpr std::size_t kDefaultMaxSavedOutputData = 256 * 1024; std::size_t max_saved_output_data_{kDefaultMaxSavedOutputData}; + bool apm_tracing_enabled_; + struct FilterCtx { ngx_chain_t *out; // the buffered request or response body ngx_chain_t **out_latest{&out}; diff --git a/src/security/library.cpp b/src/security/library.cpp index 79d98cdd..e79666a5 100644 --- a/src/security/library.cpp +++ b/src/security/library.cpp @@ -3,6 +3,8 @@ #include #include +#include "global_tracer.h" + extern "C" { #include #include @@ -790,4 +792,14 @@ std::vector Library::environment_variable_names() { std::optional Library::max_saved_output_data() { return config_settings_->get_max_saved_output_data(); }; + +bool Library::apm_tracing_enabled() noexcept { + // get global tracer + dd::Tracer *tracer = global_tracer(); + if (!tracer) { + return false; + } + // check if it's enabled + return tracer->is_apm_tracing_enabled(); +} } // namespace datadog::nginx::security diff --git a/src/security/library.h b/src/security/library.h index 51dd32fa..833530b9 100644 --- a/src/security/library.h +++ b/src/security/library.h @@ -50,6 +50,8 @@ class Library { static std::optional max_saved_output_data(); + static bool apm_tracing_enabled() noexcept; + protected: static std::atomic active_; // NOLINT static std::unique_ptr config_settings_; // NOLINT diff --git a/src/tracing/directives.h b/src/tracing/directives.h index 467e392e..f0b7fd78 100644 --- a/src/tracing/directives.h +++ b/src/tracing/directives.h @@ -51,6 +51,14 @@ constexpr datadog::nginx::directive tracing_directives[] = { offsetof(datadog_loc_conf_t, enable_tracing), nullptr, }, + { + "datadog_apm_tracing_enabled", + NGX_HTTP_MAIN_CONF | NGX_CONF_TAKE1, + ngx_conf_set_flag_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(datadog_main_conf_t, apm_tracing_enabled), + nullptr, + }, { "datadog_trace_locations", anywhere | NGX_CONF_TAKE1, diff --git a/src/tracing_library.cpp b/src/tracing_library.cpp index 9ae8789b..0f36cfb3 100644 --- a/src/tracing_library.cpp +++ b/src/tracing_library.cpp @@ -55,6 +55,10 @@ dd::Expected TracingLibrary::make_tracer( config.integration_version = NGINX_VERSION; config.service = "nginx"; + if (nginx_conf.apm_tracing_enabled != NGX_CONF_UNSET) { + config.apm_tracing_enabled = {nginx_conf.apm_tracing_enabled == 1}; + } + if (!nginx_conf.propagation_styles.empty()) { config.injection_styles = config.extraction_styles = nginx_conf.propagation_styles; diff --git a/test/cases/configuration/conf/apm_tracing_off.conf b/test/cases/configuration/conf/apm_tracing_off.conf new file mode 100644 index 00000000..64afaf4c --- /dev/null +++ b/test/cases/configuration/conf/apm_tracing_off.conf @@ -0,0 +1,27 @@ +thread_pool waf_thread_pool threads=2 max_queue=5; + +load_module /datadog-tests/ngx_http_datadog_module.so; + +events { + worker_connections 1024; +} + +http { + datadog_agent_url http://agent:8126; + datadog_appsec_enabled on; + datadog_appsec_ruleset_file /tmp/waf.json; + datadog_appsec_waf_timeout 2s; + datadog_waf_thread_pool_name waf_thread_pool; + datadog_apm_tracing_enabled off; + + server { + listen 80; + location / { + return 200 "apm_tracing_off"; + } + + location /http { + proxy_pass http://http:8080; + } + } +} diff --git a/test/cases/configuration/conf/waf.json b/test/cases/configuration/conf/waf.json new file mode 100644 index 00000000..77a059c7 --- /dev/null +++ b/test/cases/configuration/conf/waf.json @@ -0,0 +1,65 @@ +{ + "version": "2.1", + "metadata": { + "rules_version": "1.2.6" + }, + "rules": [ + { + "id": "block_default", + "name": "Block with default action", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + }, + { + "address": "server.request.body" + } + ], + "regex": "^block_default$" + }, + "operator": "match_regex" + } + ], + "on_match": [ + "block" + ] + }, + { + "id": "no_block", + "name": "No block", + "tags": { + "type": "security_scanner", + "category": "attack_attempt" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.headers.no_cookies", + "key_path": [ + "user-agent" + ] + }, + { + "address": "server.request.body" + } + ], + "regex": "^no_block$" + }, + "operator": "match_regex" + } + ] + } + ] +} diff --git a/test/cases/configuration/test_apm_tracing_enabled.py b/test/cases/configuration/test_apm_tracing_enabled.py new file mode 100644 index 00000000..51a67b7b --- /dev/null +++ b/test/cases/configuration/test_apm_tracing_enabled.py @@ -0,0 +1,333 @@ +from .. import formats +from .. import case +from pathlib import Path +import json + + +class TestApmTracingEnabled(case.TestCase): + requires_waf = True + config_setup_done = False + + # Constants for distributed tracing tests + DISTRIBUTED_TRACE_ID_DEC = 7337010839542040699 + DISTRIBUTED_PARENT_ID_DEC = 2356450358339785194 + INJECTED_SAMPLING_PRIORITY_USER_DROP = -1 + INJECTED_SAMPLING_PRIORITY_AUTO_KEEP = 1 + INJECTED_SAMPLING_PRIORITY_USER_KEEP = 2 + X_DATADOG_ORIGIN_RUM = "rum" + + def setUp(self): + super().setUp() + # avoid reconfiguration (cuts time almost in half) + if not TestApmTracingEnabled.config_setup_done: + waf_path = Path(__file__).parent / './conf/waf.json' + waf_text = waf_path.read_text() + self.orch.nginx_replace_file('/tmp/waf.json', waf_text) + + conf_path = Path(__file__).parent / "conf" / "apm_tracing_off.conf" + conf_text = conf_path.read_text() + + status, log_lines = self.orch.nginx_replace_config( + conf_text, conf_path.name) + self.assertEqual(0, status, log_lines) + + # clear any previous agent data + self.orch.sync_service('agent') + + TestApmTracingEnabled.config_setup_done = True + + def get_traces(self, on_chunk): + self.orch.reload_nginx() + log_lines = self.orch.sync_service('agent') + # Find the trace that came from nginx, and pass its chunks (groups of + # spans) to the callback. + found_nginx_trace = False + for line in log_lines: + trace = formats.parse_trace(line) + if trace is None: + continue + for chunk in trace: + if chunk[0]['service'] != 'nginx': + continue + found_nginx_trace = True + on_chunk(chunk) + + return found_nginx_trace + + def test_apm_tracing_off_no_waf(self): + for _ in range(2): + status, _, body = self.orch.send_nginx_http_request("/") + self.assertEqual(200, status) + self.assertEqual(body, "apm_tracing_off") + + is_first = True + + def on_chunk(chunk): + nonlocal is_first + self.assertEqual(len(chunk), 1, "Expected one span in the trace") + span = chunk[0] + self.assertNotIn("_dd.p.ts", span["meta"]) + self.assertNotIn("_dd.p.dm", span["metrics"]) + + if is_first: + self.assertEqual(span["metrics"]["_dd.apm.enabled"], 0) + self.assertEqual(span["metrics"]["_sampling_priority_v1"], + 2) # USER_KEEP + is_first = False + else: + self.assertEqual(span["metrics"]["_dd.apm.enabled"], 0) + self.assertEqual(span["metrics"]["_sampling_priority_v1"], + -1) # drop + + self.assertTrue(self.get_traces(on_chunk)) + + def test_apm_tracing_off_waf(self): + for _ in range(2): + status, _, body = self.orch.send_nginx_http_request( + "/http/", headers={'User-agent': 'no_block'}) + self.assertEqual(200, status) + + response = json.loads(body) + forwarded_headers = response["headers"] + + self.assertEqual(forwarded_headers["x-datadog-sampling-priority"], + "2") + + self.assertIn("x-datadog-tags", forwarded_headers, + "Missing x-datadog-tags header") + tags = forwarded_headers["x-datadog-tags"] + self.assertIn("_dd.p.ts=02", tags, + f"Missing _dd.p.ts=02 in x-datadog-tags: {tags}") + self.assertIn("_dd.p.dm=-5", tags, + f"Missing _dd.p.dm=-5 in x-datadog-tags: {tags}") + + def on_chunk(chunk): + self.assertEqual(len(chunk), 1, "Expected one span in the trace") + span = chunk[0] + + self.assertEqual(span["metrics"]["_dd.apm.enabled"], 0) + self.assertEqual(span["metrics"]["_sampling_priority_v1"], + 2) # USER_KEEP + self.assertEqual(span["meta"]["_dd.p.ts"], "02") + self.assertEqual(span["meta"]["_dd.p.dm"], "-4") + + self.assertTrue(self.get_traces(on_chunk)) + + def test_distributed_tracing_no_waf(self): + headers = { + 'x-datadog-trace-id': + str(self.DISTRIBUTED_TRACE_ID_DEC), + 'x-datadog-parent-id': + str(self.DISTRIBUTED_PARENT_ID_DEC), + 'x-datadog-sampling-priority': + str(self.INJECTED_SAMPLING_PRIORITY_USER_KEEP), + 'x-datadog-origin': + self.X_DATADOG_ORIGIN_RUM, + } + request_count = 0 + for _ in range(2): + request_count += 1 + status, _, body = self.orch.send_nginx_http_request( + "/http", headers=headers) + self.assertEqual(200, status) + + # For the first request, assert the propagation header values + if request_count == 1: + response = json.loads(body) + forwarded_headers = response["headers"] + + # Assert that the propagation headers match the expected format + # The trace ID should remain the same, but parent ID should be different (new span) + self.assertIn("x-datadog-trace-id", forwarded_headers, + "Missing x-datadog-trace-id header") + self.assertEqual( + forwarded_headers["x-datadog-trace-id"], + str(self.DISTRIBUTED_TRACE_ID_DEC), + f"x-datadog-trace-id mismatch: expected {self.DISTRIBUTED_TRACE_ID_DEC}, got {forwarded_headers['x-datadog-trace-id']}" + ) + + self.assertIn("x-datadog-parent-id", forwarded_headers, + "Missing x-datadog-parent-id header") + self.assertNotEqual( + forwarded_headers["x-datadog-parent-id"], + str(self.DISTRIBUTED_PARENT_ID_DEC), + f"x-datadog-parent-id should be different from original {self.DISTRIBUTED_PARENT_ID_DEC}, but got {forwarded_headers['x-datadog-parent-id']}" + ) + + expected_headers = { + "x-datadog-sampling-priority": + str(self.INJECTED_SAMPLING_PRIORITY_USER_KEEP), + "x-datadog-origin": + self.X_DATADOG_ORIGIN_RUM, + } + + for header_name, expected_value in expected_headers.items(): + self.assertIn(header_name, forwarded_headers) + self.assertEqual(forwarded_headers[header_name], + expected_value) + + # For the second request, assert that there are no propagation headers + if request_count == 2: + # Parse the response to check that no propagation headers are present + response = json.loads(body) + forwarded_headers = response.get("headers", {}) + + # Assert that propagation headers are NOT present on the second request + propagation_headers = [ + "x-datadog-trace-id", "x-datadog-parent-id", + "x-datadog-sampling-priority", "x-datadog-origin", + "x-datadog-tags", "traceparent", "tracestate" + ] + + for header_name in propagation_headers: + self.assertNotIn(header_name, forwarded_headers) + + span_count = 0 + + def on_chunk(chunk): + nonlocal span_count + span_count += 1 + + self.assertEqual(len(chunk), 1, "Expected one span in the trace") + span = chunk[0] + + self.assertEqual(span["trace_id"], self.DISTRIBUTED_TRACE_ID_DEC) + self.assertEqual(span["parent_id"], self.DISTRIBUTED_PARENT_ID_DEC) + self.assertEqual(span["metrics"]["_dd.apm.enabled"], 0) + + if span_count == 1: + self.assertEqual( + span["metrics"]["_sampling_priority_v1"], + int(self.INJECTED_SAMPLING_PRIORITY_USER_KEEP)) + self.assertNotIn("_dd.p.ts", span.get("meta", {})) + self.assertNotEqual(span["meta"].get("_dd.p.dm", ""), "-4") + else: + self.assertEqual( + span["metrics"]["_sampling_priority_v1"], + int(self.INJECTED_SAMPLING_PRIORITY_USER_DROP)) + + self.assertEqual( + span.get("meta", {}).get("_dd.origin"), + self.X_DATADOG_ORIGIN_RUM) + + self.assertTrue( + self.get_traces(on_chunk), + "Failed to find traces for distributed_tracing_no_waf") + self.assertEqual( + span_count, 2, + "Expected to process two spans for distributed_tracing_no_waf") + + def test_distributed_tracing_waf(self): + headers = { + 'x-datadog-trace-id': self.DISTRIBUTED_TRACE_ID_DEC, + 'x-datadog-parent-id': self.DISTRIBUTED_PARENT_ID_DEC, + 'x-datadog-sampling-priority': + self.INJECTED_SAMPLING_PRIORITY_AUTO_KEEP, + 'x-datadog-origin': self.X_DATADOG_ORIGIN_RUM, + 'x-datadog-tags': '_dd.p.ts=02', + } + for _ in range(2): + status, _, body = self.orch.send_nginx_http_request( + "/http", headers=headers) + self.assertEqual(200, status) + + # Parse the response to check propagation headers + response = json.loads(body) + forwarded_headers = response["headers"] + + # Assert that the propagation headers match the expected format + # The trace ID should remain the same, but parent ID should be different (new span) + self.assertIn("x-datadog-trace-id", forwarded_headers, + "Missing x-datadog-trace-id header") + self.assertEqual( + forwarded_headers["x-datadog-trace-id"], + str(self.DISTRIBUTED_TRACE_ID_DEC), + f"x-datadog-trace-id mismatch: expected {self.DISTRIBUTED_TRACE_ID_DEC}, got {forwarded_headers['x-datadog-trace-id']}" + ) + + self.assertIn("x-datadog-parent-id", forwarded_headers, + "Missing x-datadog-parent-id header") + self.assertNotEqual( + forwarded_headers["x-datadog-parent-id"], + str(self.DISTRIBUTED_PARENT_ID_DEC), + f"x-datadog-parent-id should be different from original {self.DISTRIBUTED_PARENT_ID_DEC}, but got {forwarded_headers['x-datadog-parent-id']}" + ) + + expected_headers = { + "x-datadog-sampling-priority": "2", + "x-datadog-origin": "rum", + "x-datadog-tags": "_dd.p.ts=02,_dd.p.dm=-5", + } + + for header_name, expected_value in expected_headers.items(): + self.assertIn(header_name, forwarded_headers, + f"Missing header: {header_name}") + self.assertEqual( + forwarded_headers[header_name], expected_value, + f"Header {header_name} mismatch: expected {expected_value}, got {forwarded_headers[header_name]}" + ) + + # Check traceparent format (trace ID should match, span ID should be different) + self.assertIn("traceparent", forwarded_headers, + "Missing traceparent header") + traceparent = forwarded_headers["traceparent"] + traceparent_parts = traceparent.split("-") + self.assertEqual(len(traceparent_parts), 4, + f"Invalid traceparent format: {traceparent}") + self.assertEqual( + traceparent_parts[0], "00", + f"Invalid traceparent version: {traceparent_parts[0]}") + # Convert decimal trace ID to hex and pad to 32 chars + expected_trace_id_hex = f"{self.DISTRIBUTED_TRACE_ID_DEC:032x}" + self.assertEqual( + traceparent_parts[1], expected_trace_id_hex, + f"traceparent trace ID mismatch: expected {expected_trace_id_hex}, got {traceparent_parts[1]}" + ) + # Span ID should be different from the original parent ID + original_parent_id_hex = f"{self.DISTRIBUTED_PARENT_ID_DEC:016x}" + self.assertNotEqual( + traceparent_parts[2], original_parent_id_hex, + f"traceparent span ID should be different from original {original_parent_id_hex}, but got {traceparent_parts[2]}" + ) + self.assertEqual( + traceparent_parts[3], "01", + f"Invalid traceparent flags: {traceparent_parts[3]}") + + # Check tracestate format + self.assertIn("tracestate", forwarded_headers, + "Missing tracestate header") + tracestate = forwarded_headers["tracestate"] + # The tracestate should contain the new span ID (same as in traceparent) + expected_span_id = traceparent_parts[2] + expected_tracestate = f"dd=s:2;p:{expected_span_id};o:rum;t.ts:02;t.dm:-5" + self.assertEqual( + tracestate, expected_tracestate, + f"tracestate mismatch: expected {expected_tracestate}, got {tracestate}" + ) + + span_count = 0 + + def on_chunk(chunk): + nonlocal span_count + span_count += 1 + + self.assertEqual(len(chunk), 1, "Expected one span in the trace") + span = chunk[0] + + self.assertEqual(span["trace_id"], self.DISTRIBUTED_TRACE_ID_DEC) + self.assertEqual(span["parent_id"], self.DISTRIBUTED_PARENT_ID_DEC) + + self.assertEqual(span["metrics"]["_dd.apm.enabled"], 0) + self.assertEqual(span["metrics"]["_sampling_priority_v1"], 2) + + self.assertEqual(span["meta"]["_dd.p.ts"], "02") + + self.assertEqual( + span.get("meta", {}).get("_dd.origin"), + self.X_DATADOG_ORIGIN_RUM) + + self.assertTrue(self.get_traces(on_chunk), + "Failed to find traces for distributed_tracing_waf") + self.assertEqual( + span_count, 2, + "Expected to process two spans for distributed_tracing_waf")