diff --git a/soaks/datadog_agent_vrl_blackhole/README.md b/soaks/datadog_agent_vrl_blackhole/README.md new file mode 100644 index 0000000000000..c7d497e309a86 --- /dev/null +++ b/soaks/datadog_agent_vrl_blackhole/README.md @@ -0,0 +1,9 @@ +# Datadog Agent -> Remap -> Blackhole + +This soak tests Datadog agent source feeding into the blackhole sink through a +non-trivial remap transform. It is a straight pipe otherwise. + +## Method + +Lading `http_gen` is used to generate log load into vector. There is no sink +outside of vector. diff --git a/soaks/datadog_agent_vrl_blackhole/terraform/.terraform.lock.hcl b/soaks/datadog_agent_vrl_blackhole/terraform/.terraform.lock.hcl new file mode 100644 index 0000000000000..ada2708243dce --- /dev/null +++ b/soaks/datadog_agent_vrl_blackhole/terraform/.terraform.lock.hcl @@ -0,0 +1,38 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.5.1" + constraints = "~> 2.5.0" + hashes = [ + "h1:wg+6NFJUKrwccLZQPkJTgdRMgh709ZX+sBuJ9IlJoNU=", + "zh:05f42363a1bdf1858bfdb56dc55509a066a640d05ef62e0fddcf7cc4d97a0dfa", + "zh:2b974bebbc7823f759159fcf1efe0717930b7756f643ff1551e2d633a6651d10", + "zh:329817c534ae807ae88030f78e5c1ad1f124e9d5d7ae2830229a6d18ba7dd22e", + "zh:3c41ea252ded2788d7c47d613e9c1a6543d61e67905f9b1ebc8be23df556e844", + "zh:3fdff88596baf4f76dc64e016a5f7ba455b8b0b46b45e124e03074746f6a4d1b", + "zh:480819de824ab56074c4fcb9ebc5d7c7d44dda5fe6e9a3bc0b858b795f74ad41", + "zh:498fa4b9a5e73afe7560fe4540553a4e1e6a5691acc9384f0ebee32af0afe698", + "zh:6dfb17f21cc3dfd5096a1cf0d01603527213b127dade411f79c524d2bddc78b9", + "zh:b5132f132af0b0927d9cad126909d375aad2c11abed896a345529f1a2680eda8", + "zh:c0fc51da9dff350d29d3887660e8d390b2402ee415c190e17dbcf271d2234c16", + "zh:dfb1c8dd3409c7b09e2ffeeeadfa41cad076cc06b0d95a303ee096c01e9e68cf", + ] +} + +provider "registry.terraform.io/hashicorp/template" { + version = "2.2.0" + hashes = [ + "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", + "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", + "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", + "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", + "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", + "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", + "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", + "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", + "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", + "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", + "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", + ] +} diff --git a/soaks/datadog_agent_vrl_blackhole/terraform/http_gen.toml b/soaks/datadog_agent_vrl_blackhole/terraform/http_gen.toml new file mode 100644 index 0000000000000..fb06efcc7077a --- /dev/null +++ b/soaks/datadog_agent_vrl_blackhole/terraform/http_gen.toml @@ -0,0 +1,12 @@ +worker_threads = 2 +prometheus_addr = "0.0.0.0:9090" + +[targets.vector] +target_uri = "http://vector:8282/v1/input" +bytes_per_second = "500 Mb" +parallel_connections = 10 +method.type = "Post" +method.variant = "DatadogLog" +method.maximum_prebuild_cache_size_bytes = "256 Mb" +[targets.vector.headers] +dd-api-key = "DEADBEEF" diff --git a/soaks/datadog_agent_vrl_blackhole/terraform/main.tf b/soaks/datadog_agent_vrl_blackhole/terraform/main.tf new file mode 100644 index 0000000000000..0bd5643178841 --- /dev/null +++ b/soaks/datadog_agent_vrl_blackhole/terraform/main.tf @@ -0,0 +1,50 @@ +# Set up the providers needed to run this soak and other terraform related +# business. Here we only require 'kubernetes' to interact with the soak +# minikube. +terraform { + required_providers { + kubernetes = { + version = "~> 2.5.0" + source = "hashicorp/kubernetes" + } + } +} + +# Rig the kubernetes provider to communicate with minikube. The details of +# adjusting `~/.kube/config` are addressed by the soak control scripts. +provider "kubernetes" { + config_path = "~/.kube/config" +} + +# Setup background monitoring details. These are needed by the soak control to +# understand what vector et al's running behavior is. +module "monitoring" { + source = "../../common/terraform/modules/monitoring" + type = var.type + vector_image = var.vector_image +} + +# Setup the soak pieces +# +# This soak config sets up a vector soak with lading/http-gen feeding into vector, +# lading/http-blackhole receiving. +resource "kubernetes_namespace" "soak" { + metadata { + name = "soak" + } +} + +module "vector" { + source = "../../common/terraform/modules/vector" + type = var.type + vector_image = var.vector_image + test_name = "datadog_agent_remap_datadog_logs" + vector-toml = file("${path.module}/vector.toml") + namespace = kubernetes_namespace.soak.metadata[0].name +} +module "http-gen" { + source = "../../common/terraform/modules/lading_http_gen" + type = var.type + http-gen-toml = file("${path.module}/http_gen.toml") + namespace = kubernetes_namespace.soak.metadata[0].name +} diff --git a/soaks/datadog_agent_vrl_blackhole/terraform/variables.tf b/soaks/datadog_agent_vrl_blackhole/terraform/variables.tf new file mode 100644 index 0000000000000..2fe029d08c139 --- /dev/null +++ b/soaks/datadog_agent_vrl_blackhole/terraform/variables.tf @@ -0,0 +1,9 @@ +variable "type" { + description = "The type of the vector install, whether 'baseline' or 'comparison'" + type = string +} + +variable "vector_image" { + description = "The image of vector to use in this investigation" + type = string +} diff --git a/soaks/datadog_agent_vrl_blackhole/terraform/vector.toml b/soaks/datadog_agent_vrl_blackhole/terraform/vector.toml new file mode 100644 index 0000000000000..34e054ecb6347 --- /dev/null +++ b/soaks/datadog_agent_vrl_blackhole/terraform/vector.toml @@ -0,0 +1,49 @@ +data_dir = "/var/lib/vector" + +## +## Sources +## + +[sources.internal_metrics] +type = "internal_metrics" + +[sources.datadog_agent] +type = "datadog_agent" +acknowledgements = false +address = "0.0.0.0:8282" + +## +## Transforms +## + +[transforms.remap] +type = "remap" +inputs = ["datadog_agent"] +source = ''' +.hostname = "vector" + +if .status == "warning" { + .thing = upcase(.hostname) +} else if .status == "notice" { + .thung = downcase(.hostname) +} else { + .nong = upcase(.hostname) +} + +.matches = { "name": .message, "num": "2" } +.origin, .err = .hostname + "/" + .matches.name + "/" + .matches.num +del(.hostname) +''' + +## +## Sinks +## + +[sinks.prometheus] +type = "prometheus_exporter" +inputs = ["internal_metrics"] +address = "0.0.0.0:9090" + +[sinks.blackhole] +type = "blackhole" +inputs = ["remap"]