enhancement(vdev): Load compose files and inject network block

scripts/integration/amqp/compose.yaml

@@ -5,7 +5,3 @@ services:
     image: docker.io/rabbitmq:${CONFIG_VERSION}
     ports:
     - 5672:5672
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/aws/compose.yaml

@@ -14,7 +14,3 @@ services:
     volumes:
     - $DOCKER_SOCKET:/var/run/docker.sock
     - $HOME/.aws/:/home/.aws/
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/azure/compose.yaml

@@ -6,7 +6,3 @@ services:
     command: azurite --blobHost 0.0.0.0 --loose
     volumes:
     - /var/run:/var/run
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/chronicle/compose.yaml

@@ -10,7 +10,3 @@ services:
     command:
     - -p
     - /public.pem
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/clickhouse/compose.yaml

@@ -3,7 +3,3 @@ version: '3'
 services:
   clickhouse:
     image: docker.io/yandex/clickhouse-server:${CONFIG_VERSION}
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/databend/compose.yaml

@@ -19,7 +19,3 @@ services:
       - minio
     healthcheck:
       test: "curl -f localhost:8080/v1/health || exit 1"
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/datadog-agent/compose.yaml

@@ -26,7 +26,3 @@ services:
     - DD_CMD_PORT=5002
     - DD_USE_DOGSTATSD=false
     - DD_HOSTNAME=datadog-trace-agent
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/datadog-traces/compose.yaml

@@ -29,7 +29,3 @@ services:
     - DD_APM_MAX_MEMORY=0
     - DD_APM_MAX_CPU_PERCENT=0
     - DD_HOSTNAME=datadog-trace-agent-to-vector
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/dnstap/compose.yaml

@@ -11,9 +11,5 @@ services:
       - dnstap-sockets:/bind2/etc/bind/socket
       - dnstap-sockets:/bind3/etc/bind/socket
 
-networks:
-  default:
-    name: ${VECTOR_NETWORK}
-
 volumes:
   dnstap-sockets: {}

scripts/integration/elasticsearch/compose.yaml

@@ -25,7 +25,3 @@ services:
     - ES_JAVA_OPTS=-Xms400m -Xmx400m
     volumes:
     - ../../../tests/data/ca:/usr/share/elasticsearch/config/certs:ro
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/eventstoredb/compose.yaml

@@ -6,7 +6,3 @@ services:
     command: --insecure --stats-period-sec=1
     volumes:
     - ../../../tests/data:/etc/vector:ro
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/gcp/compose.yaml

@@ -6,7 +6,3 @@ services:
     environment:
     - PUBSUB_PROJECT1=testproject,topic1:subscription1
     - PUBSUB_PROJECT2=sourceproject,topic2:subscription2
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/http-client/compose.yaml

@@ -29,7 +29,3 @@ services:
     - ../../../tests/data/http-client/serve:/data
     - ../../../tests/data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem
     - ../../../tests/data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/humio/compose.yaml

@@ -3,7 +3,3 @@ version: '3'
 services:
   humio:
     image: docker.io/humio/humio:${CONFIG_VERSION}
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/influxdb/compose.yaml

@@ -19,7 +19,3 @@ services:
     command: influxd --reporting-disabled
     environment:
     - INFLUXDB_REPORTING_DISABLED=true
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/kafka/compose.yaml

@@ -33,7 +33,3 @@ services:
     volumes:
     - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro
     - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/logstash/compose.yaml

@@ -12,7 +12,3 @@ services:
     - /dev/null:/usr/share/logstash/pipeline/logstash.yml
     - ../../../tests/data/host.docker.internal.crt:/tmp/logstash.crt
     - ../../../tests/data/logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/loki/compose.yaml

@@ -4,7 +4,3 @@ services:
   loki:
     image: docker.io/grafana/loki:${CONFIG_VERSION}
     command: -config.file=/etc/loki/local-config.yaml -auth.enabled=true
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/mongodb/compose.yaml

@@ -30,7 +30,3 @@ services:
     - MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017
     - MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor
     - MONGODB_REPLICA_SET_KEY=vector
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/nats/compose.yaml

@@ -43,7 +43,3 @@ services:
     - /usr/share/nats/config/nats-jwt.conf
     volumes:
     - ../../../tests/data/nats:/usr/share/nats/config
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/nginx/compose.yaml

@@ -22,6 +22,4 @@ services:
     - proxy
 
 networks:
-  default:
-    name: ${VECTOR_NETWORK}
   proxy: {}

scripts/integration/opentelemetry/compose.yaml

@@ -5,7 +5,3 @@ services:
     image: docker.io/otel/opentelemetry-collector-contrib:${CONFIG_VERSION}
     volumes:
     - ../../../tests/data/opentelemetry/config.yaml:/etc/otelcol-contrib/config.yaml
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/postgres/compose.yaml

@@ -12,9 +12,5 @@ services:
     - ../../../tests/data/postgres-init.sh:/postgres-init.sh:ro
     - ../../../tests/data/ca:/certs:ro
 
-networks:
-  default:
-    name: ${VECTOR_NETWORK}
-
 volumes:
   socket: {}

scripts/integration/prometheus/compose.yaml

@@ -21,7 +21,3 @@ services:
     command: --config.file=/etc/vector/prometheus.yaml
     volumes:
     - ../../../tests/data:/etc/vector:ro
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/pulsar/compose.yaml

@@ -6,7 +6,3 @@ services:
     command: bin/pulsar standalone
     ports:
     - 6650:6650
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/redis/compose.yaml

@@ -3,7 +3,3 @@ version: '3'
 services:
   redis:
     image: docker.io/redis:${CONFIG_VERSION}
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/shutdown/compose.yaml

@@ -31,7 +31,3 @@ services:
     volumes:
     - ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro
     - ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/splunk/compose.yaml

@@ -13,7 +13,3 @@ services:
     - 8000:8000
     - 8088:8088
     - 8089:8089
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

scripts/integration/webhdfs/compose.yaml

@@ -30,7 +30,3 @@ services:
       interval: 5s
       timeout: 5s
       retries: 3
-
-networks:
-  default:
-    name: ${VECTOR_NETWORK}

vdev/Cargo.toml

@@ -35,5 +35,5 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0.95"
 serde_yaml = "0.9.19"
 sha2 = "0.10.6"
-tempfile = "3.4.0"
+tempfile = "3.5.0"
 toml = { version = "0.7.2", default-features = false, features = ["parse"] }

vdev/src/testing/config.rs

@@ -5,7 +5,7 @@ use std::{env, fs};
 use anyhow::{bail, Context, Result};
 use hashlink::LinkedHashMap;
 use itertools::{self, Itertools};
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use serde_yaml::Value;
 
 use crate::{app, util};
@@ -37,16 +37,66 @@ impl RustToolchainConfig {
     }
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct ComposeConfig {
     pub services: BTreeMap<String, ComposeService>,
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
     pub volumes: BTreeMap<String, Value>,
+    #[serde(default)]
+    pub networks: BTreeMap<String, BTreeMap<String, String>>,
 }
 
-#[derive(Debug, Deserialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct ComposeService {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub image: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub hostname: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub container_name: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub build: Option<BuildConfig>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub command: Option<Command>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub ports: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub env_file: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub volumes: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub environment: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub depends_on: Option<Vec<String>>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub healthcheck: Option<HealthCheck>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct HealthCheck {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub test: Option<Command>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub interval: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub timeout: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub retries: Option<u32>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub start_period: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct BuildConfig {
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub context: Option<String>,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+#[serde(untagged)]
+pub enum Command {
+    Single(String),
+    Multiple(Vec<String>),
 }
 
 impl ComposeConfig {