chore: Introduce new datadog-agent -> vrl -> blackhole soak (#9849)

* Introduce new datadog-agent -> vrl -> blackhole soak

This commit introduces a new soak that does not output, intending to test only
VRL performance. This is in service to the VRL performance work being done by
@StephenWakely.

Closes #9831
Closes #9832
REF #9515

Signed-off-by: Brian L. Troutwine <brian@troutwine.us>

* add soak to workflow

Signed-off-by: Brian L. Troutwine <brian@troutwine.us>

* adjust naming scheme

Signed-off-by: Brian L. Troutwine <brian@troutwine.us>

* update with master

Signed-off-by: Brian L. Troutwine <brian@troutwine.us>

* removing del because it's not well supported

Signed-off-by: Brian L. Troutwine <brian@troutwine.us>

.github/workflows/soak.yml

@@ -215,14 +215,7 @@ jobs:
     runs-on: [self-hosted, linux, x64, soak] # could be general if we move away from miller
     needs: [soak-baseline, soak-comparison]
     strategy:
-      matrix:
-        target:
-          - datadog_agent_remap_datadog_logs
-          - syslog_humio_logs
-          - syslog_log2metric_humio_metrics
-          - syslog_loki
-          - syslog_regex_logs2metric_ddmetrics
-          - syslog_splunk_hec_logs
+      matrix: ${{ fromJson(needs.compute-test-plan.outputs.matrix) }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2

soaks/tests/datadog_agent_remap_blackhole/README.md

@@ -0,0 +1,9 @@
+# Datadog Agent -> Remap -> Blackhole
+
+This soak tests Datadog agent source feeding into the blackhole sink through a
+non-trivial remap transform. It is a straight pipe otherwise.
+
+## Method
+
+Lading `http_gen` is used to generate log load into vector. There is no sink
+outside of vector.

soaks/tests/datadog_agent_remap_blackhole/terraform/http_gen.toml

@@ -0,0 +1,12 @@
+worker_threads = 2
+prometheus_addr = "0.0.0.0:9090"
+
+[targets.vector]
+target_uri = "http://vector:8282/v1/input"
+bytes_per_second = "500 Mb"
+parallel_connections = 10
+method.type = "Post"
+method.variant = "DatadogLog"
+method.maximum_prebuild_cache_size_bytes = "256 Mb"
+[targets.vector.headers]
+dd-api-key = "DEADBEEF"

soaks/tests/datadog_agent_remap_blackhole/terraform/main.tf

@@ -0,0 +1,50 @@
+# Set up the providers needed to run this soak and other terraform related
+# business. Here we only require 'kubernetes' to interact with the soak
+# minikube.
+terraform {
+  required_providers {
+    kubernetes = {
+      version = "~> 2.5.0"
+      source  = "hashicorp/kubernetes"
+    }
+  }
+}
+
+# Rig the kubernetes provider to communicate with minikube. The details of
+# adjusting `~/.kube/config` are addressed by the soak control scripts.
+provider "kubernetes" {
+  config_path = "~/.kube/config"
+}
+
+# Setup background monitoring details. These are needed by the soak control to
+# understand what vector et al's running behavior is.
+module "monitoring" {
+  source        = "../../../common/terraform/modules/monitoring"
+  type         = var.type
+  vector_image = var.vector_image
+}
+
+# Setup the soak pieces
+#
+# This soak config sets up a vector soak with lading/http-gen feeding into vector,
+# lading/http-blackhole receiving.
+resource "kubernetes_namespace" "soak" {
+  metadata {
+    name = "soak"
+  }
+}
+
+module "vector" {
+  source       = "../../../common/terraform/modules/vector"
+  type         = var.type
+  vector_image = var.vector_image
+  test_name    = "datadog_agent_remap_blackhole"
+  vector-toml  = file("${path.module}/vector.toml")
+  namespace    = kubernetes_namespace.soak.metadata[0].name
+}
+module "http-gen" {
+  source        = "../../../common/terraform/modules/lading_http_gen"
+  type          = var.type
+  http-gen-toml = file("${path.module}/http_gen.toml")
+  namespace     = kubernetes_namespace.soak.metadata[0].name
+}

soaks/tests/datadog_agent_remap_blackhole/terraform/variables.tf

@@ -0,0 +1,9 @@
+variable "type" {
+  description = "The type of the vector install, whether 'baseline' or 'comparison'"
+  type        = string
+}
+
+variable "vector_image" {
+  description = "The image of vector to use in this investigation"
+  type        = string
+}

soaks/tests/datadog_agent_remap_blackhole/terraform/vector.toml

@@ -0,0 +1,48 @@
+data_dir = "/var/lib/vector"
+
+##
+## Sources
+##
+
+[sources.internal_metrics]
+type = "internal_metrics"
+
+[sources.datadog_agent]
+type = "datadog_agent"
+acknowledgements = false
+address = "0.0.0.0:8282"
+
+##
+## Transforms
+##
+
+[transforms.remap]
+type = "remap"
+inputs = ["datadog_agent"]
+source = '''
+.hostname = "vector"
+
+if .status == "warning" {
+  .thing = upcase(.hostname)
+} else if .status == "notice" {
+  .thung = downcase(.hostname)
+} else {
+  .nong = upcase(.hostname)
+}
+
+.matches = { "name": .message, "num": "2" }
+.origin, .err = .hostname + "/" + .matches.name + "/" + .matches.num
+'''
+
+##
+## Sinks
+##
+
+[sinks.prometheus]
+type = "prometheus_exporter"
+inputs = ["internal_metrics"]
+address = "0.0.0.0:9090"
+
+[sinks.blackhole]
+type = "blackhole"
+inputs = ["remap"]