Merge pull request #2046 from DataDog/emmett.butler/http-header-tags-…

…server-support

support http header tags tracing in parametric test server and enable its test

manifests/cpp.yml

@@ -102,6 +102,7 @@ tests/:
       Test_DsmSQS: missing_feature
   parametric/:
     test_dynamic_configuration.py:
+      TestDynamicConfigHeaderTags: missing_feature
       TestDynamicConfigTracingEnabled: missing_feature
       TestDynamicConfigV1: missing_feature
       TestDynamicConfigV2: missing_feature

manifests/dotnet.yml

@@ -219,6 +219,7 @@ tests/:
       Test_DsmSQS: missing_feature
   parametric/:
     test_dynamic_configuration.py:
+      TestDynamicConfigHeaderTags: missing_feature
       TestDynamicConfigTracingEnabled: missing_feature
       TestDynamicConfigV1: v2.33.0
       TestDynamicConfigV2: v2.44.0

manifests/golang.yml

@@ -344,6 +344,7 @@ tests/:
       Test_DsmSQS: missing_feature
   parametric/:
     test_dynamic_configuration.py:
+      TestDynamicConfigHeaderTags: missing_feature
       TestDynamicConfigTracingEnabled: missing_feature
       TestDynamicConfigV1: v1.59.0
       TestDynamicConfigV2: v1.59.0

manifests/java.yml

@@ -892,6 +892,7 @@ tests/:
       Test_Sql: bug (Endpoint is probably improperly implemented on weblog)
   parametric/:
     test_dynamic_configuration.py:
+      TestDynamicConfigHeaderTags: missing_feature
       TestDynamicConfigTracingEnabled: missing_feature
       TestDynamicConfigV1: v1.17.0
       TestDynamicConfigV2: missing_feature

manifests/nodejs.yml

@@ -347,6 +347,7 @@ tests/:
         '*': missing_feature
   parametric/:
     test_dynamic_configuration.py:
+      TestDynamicConfigHeaderTags: missing_feature
       TestDynamicConfigTracingEnabled: missing_feature
       TestDynamicConfigV1: v4.11.0
       TestDynamicConfigV2: v4.23.0

manifests/php.yml

@@ -191,6 +191,7 @@ tests/:
       Test_DsmSQS: missing_feature
   parametric/:
     test_dynamic_configuration.py:
+      TestDynamicConfigHeaderTags: missing_feature
       TestDynamicConfigTracingEnabled: missing_feature
       TestDynamicConfigV1: missing_feature
       TestDynamicConfigV2: missing_feature

manifests/python.yml

@@ -494,6 +494,7 @@ tests/:
         flask-poc: v1.20.3
   parametric/:
     test_dynamic_configuration.py:
+      TestDynamicConfigHeaderTags: v2.6.0
       TestDynamicConfigTracingEnabled: missing_feature
       TestDynamicConfigV1: missing_feature
       TestDynamicConfigV2: missing_feature

manifests/ruby.yml

@@ -231,6 +231,7 @@ tests/:
       Test_DsmSQS: missing_feature
   parametric/:
     test_dynamic_configuration.py:
+      TestDynamicConfigHeaderTags: missing_feature
       TestDynamicConfigTracingEnabled: missing_feature
       TestDynamicConfigV1: v1.13.0
       TestDynamicConfigV2: missing_feature

tests/parametric/test_dynamic_configuration.py

@@ -97,6 +97,78 @@ def assert_sampling_rate(trace: List[Dict], rate: float):
 ENV_SAMPLING_RULE_RATE = 0.55
 
 
+@scenarios.parametric
+@features.dynamic_configuration
+class TestDynamicConfigHeaderTags:
+    @parametrize(
+        "library_env",
+        [
+            {
+                **DEFAULT_ENVVARS,
+                "DD_TRACE_HEADER_TAGS": "X-Test-Header:test_header_env,X-Test-Header-2:test_header_env2,Content-Length:content_length_env",
+            },
+        ],
+    )
+    def test_tracing_client_http_header_tags(
+        self, library_env, test_agent, test_library, test_agent_hostname, test_agent_port
+    ):
+        """Ensure the tracing http header tags can be set via RC.
+
+        Testing is done using a http client request RPC and asserting the span tags.
+
+        Requests are made to the test agent.
+        """
+
+        # Test without RC.
+        test_library.http_client_request(
+            method="GET",
+            url=f"http://{test_agent_hostname}:{test_agent_port}",
+            headers=[("X-Test-Header", "test-value"), ("X-Test-Header-2", "test-value-2"), ("Content-Length", "35"),],
+        )
+        trace = test_agent.wait_for_num_traces(num=1, clear=True)
+        assert trace[0][0]["meta"]["test_header_env"] == "test-value"
+        assert trace[0][0]["meta"]["test_header_env2"] == "test-value-2"
+        assert int(trace[0][0]["meta"]["content_length_env"]) > 0
+
+        # Set and test with RC.
+        set_and_wait_rc(
+            test_agent,
+            config_overrides={
+                "tracing_header_tags": [
+                    {"header": "X-Test-Header", "tag_name": "test_header_rc"},
+                    {"header": "X-Test-Header-2", "tag_name": "test_header_rc2"},
+                    {"header": "Content-Length", "tag_name": ""},
+                ]
+            },
+        )
+        test_library.http_client_request(
+            method="GET",
+            url=f"http://{test_agent_hostname}:{test_agent_port}",
+            headers=[("X-Test-Header", "test-value"), ("X-Test-Header-2", "test-value-2"), ("Content-Length", "0")],
+        )
+        trace = test_agent.wait_for_num_traces(num=1, clear=True)
+        assert trace[0][0]["meta"]["test_header_rc"] == "test-value"
+        assert trace[0][0]["meta"]["test_header_rc2"] == "test-value-2"
+        assert trace[0][0]["meta"]["http.request.headers.content-length"] == "0"
+        assert (
+            trace[0][0]["meta"]["http.response.headers.content-length"] == "14"
+        ), "response content-length header tag value matches the header value set by the server"
+        assert "test_header_env" not in trace[0][0]["meta"]
+        assert "test_header_env2" not in trace[0][0]["meta"]
+
+        # Unset RC.
+        set_and_wait_rc(test_agent, config_overrides={})
+        test_library.http_client_request(
+            method="GET",
+            url=f"http://{test_agent_hostname}:{test_agent_port}",
+            headers=[("X-Test-Header", "test-value"), ("X-Test-Header-2", "test-value-2"), ("Content-Length", "35"),],
+        )
+        trace = test_agent.wait_for_num_traces(num=1, clear=True)
+        assert trace[0][0]["meta"]["test_header_env"] == "test-value"
+        assert trace[0][0]["meta"]["test_header_env2"] == "test-value-2"
+        assert int(trace[0][0]["meta"]["content_length_env"]) > 0
+
+
 @scenarios.parametric
 @features.dynamic_configuration
 class TestDynamicConfigTracingEnabled:
@@ -146,7 +218,6 @@ class TestDynamicConfigV1:
     v1 includes support for:
         - tracing_sampling_rate
         - log_injection_enabled
-        - tracing_header_tags
     """
 
     @parametrize("library_env", [{"DD_TELEMETRY_HEARTBEAT_INTERVAL": "0.1"}])
@@ -310,75 +381,6 @@ def test_log_injection_enabled(self, library_env, test_agent, test_library):
         cfg_state = set_and_wait_rc(test_agent, config_overrides={"tracing_sample_rate": None})
         assert cfg_state["apply_state"] == 2
 
-    @missing_feature(
-        context.library in ["java", "dotnet", "python", "golang", "nodejs"], reason="RPC not implemented yet"
-    )
-    @parametrize(
-        "library_env",
-        [
-            {
-                **DEFAULT_ENVVARS,
-                "DD_TRACE_HEADER_TAGS": "X-Test-Header:test_header_env,X-Test-Header-2:test_header_env2,Content-Length:content_length_env",
-            },
-        ],
-    )
-    def test_tracing_client_http_header_tags(
-        self, library_env, test_agent, test_library, test_agent_hostname, test_agent_port
-    ):
-        """Ensure the tracing http header tags can be set via RC.
-
-        Testing is done using a http client request RPC and asserting the span tags.
-
-        Requests are made to the test agent.
-        """
-
-        # Test without RC.
-        test_library.http_client_request(
-            method="GET",
-            url=f"http://{test_agent_hostname}:{test_agent_port}",
-            headers=[("X-Test-Header", "test-value"), ("X-Test-Header-2", "test-value-2"), ("Content-Length", "35"),],
-        )
-        trace = test_agent.wait_for_num_traces(num=1, clear=True)
-        assert trace[0][0]["meta"]["test_header_env"] == "test-value"
-        assert trace[0][0]["meta"]["test_header_env2"] == "test-value-2"
-        assert int(trace[0][0]["meta"]["content_length_env"]) > 0
-
-        # Set and test with RC.
-        set_and_wait_rc(
-            test_agent,
-            config_overrides={
-                "tracing_header_tags": [
-                    {"header": "X-Test-Header", "tag_name": "test_header_rc",},
-                    {"header": "X-Test-Header-2", "tag_name": "test_header_rc2",},
-                    {"header": "Content-Length", "tag_name": "",},
-                ]
-            },
-        )
-        test_library.http_client_request(
-            method="GET",
-            url=f"http://{test_agent_hostname}:{test_agent_port}",
-            headers=[("X-Test-Header", "test-value"), ("X-Test-Header-2", "test-value-2"), ("Content-Length", "0")],
-        )
-        trace = test_agent.wait_for_num_traces(num=1, clear=True)
-        assert trace[0][0]["meta"]["test_header_rc"] == "test-value"
-        assert trace[0][0]["meta"]["test_header_rc2"] == "test-value-2"
-        assert trace[0][0]["meta"]["http.request.headers.content-length"] == "0"
-        assert trace[0][0]["meta"]["http.response.headers.content-length"] == "14"
-        assert "test_header_env" not in trace[0][0]["meta"]
-        assert "test_header_env2" not in trace[0][0]["meta"]
-
-        # Unset RC.
-        set_and_wait_rc(test_agent, config_overrides={"tracing_header_tags": None})
-        test_library.http_client_request(
-            method="GET",
-            url=f"http://{test_agent_hostname}:{test_agent_port}",
-            headers=[("X-Test-Header", "test-value"), ("X-Test-Header-2", "test-value-2"), ("Content-Length", "35"),],
-        )
-        trace = test_agent.wait_for_num_traces(num=1, clear=True)
-        assert trace[0][0]["meta"]["test_header_env"] == "test-value"
-        assert trace[0][0]["meta"]["test_header_env2"] == "test-value-2"
-        assert int(trace[0][0]["meta"]["content_length_env"]) > 0
-
 
 @rfc("https://docs.google.com/document/d/1V4ZBsTsRPv8pAVG5WCmONvl33Hy3gWdsulkYsE4UZgU/edit")
 @scenarios.parametric

utils/build/docker/python/parametric/apm_test_client/server.py

@@ -21,6 +21,8 @@
 
 import ddtrace
 from ddtrace import Span
+from ddtrace import config
+from ddtrace.contrib.trace_utils import set_http_meta
 from ddtrace.context import Context
 from ddtrace.constants import ERROR_MSG
 from ddtrace.constants import ERROR_STACK
@@ -234,6 +236,36 @@ def trace_span_add_link(args: TraceSpanAddLinksArgs) -> TraceSpanAddLinkReturn:
     return TraceSpanAddLinkReturn()
 
 
+class HttpClientRequestArgs(BaseModel):
+    method: str
+    url: str
+    headers: List[Tuple[str, str]]
+    body: str
+
+
+class HttpClientRequestReturn(BaseModel):
+    pass
+
+
+@app.post("/http/client/request")
+def http_client_request(args: HttpClientRequestArgs) -> HttpClientRequestReturn:
+    """Creates and finishes a span similar to the ones created during HTTP request/response cycles"""
+    # falcon config doesn't really matter here - any config object with http header tracing enabled will work
+    integration_config = config.falcon
+    request_headers = {k: v for k, v in args.headers}
+    response_headers = {"Content-Length": "14"}
+    with ddtrace.tracer.trace("fake-request") as request_span:
+        set_http_meta(
+            request_span, integration_config, request_headers=request_headers, response_headers=response_headers
+        )
+        spans[request_span.span_id] = request_span
+    # this cache invalidation happens in most web frameworks as a side effect of their multithread design
+    # it's made explicit here to allow test expectations to be precise
+    config.http._reset()
+    config._header_tag_name.invalidate()
+    return HttpClientRequestReturn()
+
+
 class OtelStartSpanArgs(BaseModel):
     name: str
     parent_id: int

utils/parametric/_library_client.py

@@ -241,13 +241,12 @@ def otel_flush(self, timeout: int) -> bool:
         resp = self._session.post(self._url("/trace/otel/flush"), json={"seconds": timeout}).json()
         return resp["success"]
 
-    # TODO: test and implement this endpoint for test_dynamic_configuration tests
-    # def http_client_request(self, method: str, url: str, headers: List[Tuple[str, str]], body: bytes) -> int:
-    #     resp = self._session.post(
-    #         self._url("/http/client/request"),
-    #         json={"method": method, "url": url, "headers": headers or [], "body": body.decode()},
-    #     ).json()
-    #     return resp
+    def http_client_request(self, method: str, url: str, headers: List[Tuple[str, str]], body: bytes) -> int:
+        resp = self._session.post(
+            self._url("/http/client/request"),
+            json={"method": method, "url": url, "headers": headers or [], "body": body.decode()},
+        ).json()
+        return resp
 
 
 class _TestSpan: