Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

enhancement(vdev): Load compose files and inject network block #17025

Merged
merged 22 commits into from
Apr 7, 2023
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
9e6de82
enhancement(vdev): Load compose files and inject network block
jonathanpv Mar 31, 2023
fc15fbd
remove temp file after compose.stop() is called
jonathanpv Mar 31, 2023
537049b
Make config service for serialization include more fields
jonathanpv Apr 2, 2023
06433ab
Add compose support for webdhfs
jonathanpv Apr 2, 2023
1209010
remove renaming setting from serde annotations
jonathanpv Apr 2, 2023
7b7a346
Use from_iter() in network injection
jonathanpv Apr 2, 2023
1562ad7
utilize tempfile for creation of the rewritten compose file
jonathanpv Apr 3, 2023
a9df591
chore(vdev): Make network non optional in Compose struct
jonathanpv Apr 3, 2023
e1bef44
Merge via cherry-pick network name PR, cargo fmt
jonathanpv Apr 3, 2023
317da52
Remove unneeded comments
jonathanpv Apr 3, 2023
76a9b97
Reintroduce path in Compose struct, clean up comments
jonathanpv Apr 3, 2023
0dd377a
Merge branch 'master' into jonathanpv/vdev-running-state
jonathanpv Apr 4, 2023
54aacd0
add cargo.toml
jonathanpv Apr 4, 2023
1a64739
Utilize tempfile_in() to fix relative file path resolution of compose…
jonathanpv Apr 4, 2023
cc03e3c
Restore proxy network on nginx integration test
jonathanpv Apr 4, 2023
ba1f01f
Make Compose run command command use the original file path if enviro…
jonathanpv Apr 4, 2023
e69623b
Conditional compile for Unix, prevent windows CI failure
jonathanpv Apr 5, 2023
6a7e226
Condtional compilation fix
jonathanpv Apr 5, 2023
612e1e0
Using Value for fields not used
jonathanpv Apr 5, 2023
eacf027
Remove conditional compilation flags to ensure temp_file created on W…
jonathanpv Apr 5, 2023
cd1e267
Allow deadcode for windows
Apr 6, 2023
4b0260e
Remove unneeded to_string()
jonathanpv Apr 6, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions scripts/integration/amqp/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,3 @@ services:
image: docker.io/rabbitmq:${CONFIG_VERSION}
ports:
- 5672:5672

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/aws/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,3 @@ services:
volumes:
- $DOCKER_SOCKET:/var/run/docker.sock
- $HOME/.aws/:/home/.aws/

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/azure/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,3 @@ services:
command: azurite --blobHost 0.0.0.0 --loose
volumes:
- /var/run:/var/run

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/chronicle/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,3 @@ services:
command:
- -p
- /public.pem

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/clickhouse/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,3 @@ version: '3'
services:
clickhouse:
image: docker.io/yandex/clickhouse-server:${CONFIG_VERSION}

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/databend/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,3 @@ services:
- minio
healthcheck:
test: "curl -f localhost:8080/v1/health || exit 1"

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/datadog-agent/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,3 @@ services:
- DD_CMD_PORT=5002
- DD_USE_DOGSTATSD=false
- DD_HOSTNAME=datadog-trace-agent

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/datadog-traces/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,3 @@ services:
- DD_APM_MAX_MEMORY=0
- DD_APM_MAX_CPU_PERCENT=0
- DD_HOSTNAME=datadog-trace-agent-to-vector

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/dnstap/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,5 @@ services:
- dnstap-sockets:/bind2/etc/bind/socket
- dnstap-sockets:/bind3/etc/bind/socket

networks:
default:
name: ${VECTOR_NETWORK}

volumes:
dnstap-sockets: {}
4 changes: 0 additions & 4 deletions scripts/integration/elasticsearch/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,3 @@ services:
- ES_JAVA_OPTS=-Xms400m -Xmx400m
volumes:
- ../../../tests/data/ca:/usr/share/elasticsearch/config/certs:ro

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/eventstoredb/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,3 @@ services:
command: --insecure --stats-period-sec=1
volumes:
- ../../../tests/data:/etc/vector:ro

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/gcp/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,3 @@ services:
environment:
- PUBSUB_PROJECT1=testproject,topic1:subscription1
- PUBSUB_PROJECT2=sourceproject,topic2:subscription2

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/http-client/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,3 @@ services:
- ../../../tests/data/http-client/serve:/data
- ../../../tests/data/ca/intermediate_server/certs/dufs-https-chain.cert.pem:/certs/ca.cert.pem
- ../../../tests/data/ca/intermediate_server/private/dufs-https.key.pem:/certs/ca.key.pem

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/humio/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,3 @@ version: '3'
services:
humio:
image: docker.io/humio/humio:${CONFIG_VERSION}

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/influxdb/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,3 @@ services:
command: influxd --reporting-disabled
environment:
- INFLUXDB_REPORTING_DISABLED=true

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/kafka/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,3 @@ services:
volumes:
- ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro
- ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/logstash/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,3 @@ services:
- /dev/null:/usr/share/logstash/pipeline/logstash.yml
- ../../../tests/data/host.docker.internal.crt:/tmp/logstash.crt
- ../../../tests/data/logstash/logstash.conf:/usr/share/logstash/pipeline/logstash.conf

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/loki/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,3 @@ services:
loki:
image: docker.io/grafana/loki:${CONFIG_VERSION}
command: -config.file=/etc/loki/local-config.yaml -auth.enabled=true

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/mongodb/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,3 @@ services:
- MONGODB_INITIAL_PRIMARY_PORT_NUMBER=27017
- MONGODB_INITIAL_PRIMARY_ROOT_PASSWORD=toor
- MONGODB_REPLICA_SET_KEY=vector

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/nats/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,3 @@ services:
- /usr/share/nats/config/nats-jwt.conf
volumes:
- ../../../tests/data/nats:/usr/share/nats/config

networks:
default:
name: ${VECTOR_NETWORK}
5 changes: 0 additions & 5 deletions scripts/integration/nginx/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,3 @@ services:
- ../../../tests/data/nginx/:/etc/nginx:ro
networks:
- proxy

networks:
default:
name: ${VECTOR_NETWORK}
proxy: {}
4 changes: 0 additions & 4 deletions scripts/integration/opentelemetry/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,3 @@ services:
image: docker.io/otel/opentelemetry-collector-contrib:${CONFIG_VERSION}
volumes:
- ../../../tests/data/opentelemetry/config.yaml:/etc/otelcol-contrib/config.yaml

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/postgres/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,5 @@ services:
- ../../../tests/data/postgres-init.sh:/postgres-init.sh:ro
- ../../../tests/data/ca:/certs:ro

networks:
default:
name: ${VECTOR_NETWORK}

volumes:
socket: {}
4 changes: 0 additions & 4 deletions scripts/integration/prometheus/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,3 @@ services:
command: --config.file=/etc/vector/prometheus.yaml
volumes:
- ../../../tests/data:/etc/vector:ro

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/pulsar/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,3 @@ services:
command: bin/pulsar standalone
ports:
- 6650:6650

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/redis/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,3 @@ version: '3'
services:
redis:
image: docker.io/redis:${CONFIG_VERSION}

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/shutdown/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,3 @@ services:
volumes:
- ../../../tests/data/ca/intermediate_server/private/kafka.p12:/certs/kafka.p12:ro
- ../../../tests/data/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/splunk/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,3 @@ services:
- 8000:8000
- 8088:8088
- 8089:8089

networks:
default:
name: ${VECTOR_NETWORK}
4 changes: 0 additions & 4 deletions scripts/integration/webhdfs/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,3 @@ services:
interval: 5s
timeout: 5s
retries: 3

networks:
default:
name: ${VECTOR_NETWORK}
60 changes: 56 additions & 4 deletions vdev/src/testing/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::{env, fs};
use anyhow::{bail, Context, Result};
use hashlink::LinkedHashMap;
use itertools::{self, Itertools};
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use serde_yaml::Value;

use crate::{app, util};
Expand Down Expand Up @@ -37,16 +37,68 @@ impl RustToolchainConfig {
}
}

#[derive(Debug, Deserialize)]
// Added the networks key, to match the structure of networks eg
// networks: default: name: value: <some value here>
jonathanpv marked this conversation as resolved.
Show resolved Hide resolved
#[derive(Debug, Deserialize, Serialize)]
pub struct ComposeConfig {
pub services: BTreeMap<String, ComposeService>,
#[serde(default)]
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub volumes: BTreeMap<String, Value>,
#[serde(default)]
pub networks: BTreeMap<String, BTreeMap<String, String>>,
}

#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, Serialize)]
pub struct ComposeService {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub image: Option<String>,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was under the assumption that image was required, in what case can it be None?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This might have been an artifact of me being copy pasta happy with that line above and putting it over even fields that are required or guaranteed to be included.

Copy link
Contributor Author

@jonathanpv jonathanpv Apr 7, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah but in this case it was added because cargo vdev integration start dnstap would fail if making the image field a String instead of an option

scripts/integration/dnstap/compose.yaml

version: "3"

services:
  dnstap:
    build:
      context: ../../../tests/data/dnstap
    container_name: vector_dnstap
    hostname: ns.example.com
    volumes:
      - dnstap-sockets:/bind1/etc/bind/socket
      - dnstap-sockets:/bind2/etc/bind/socket
      - dnstap-sockets:/bind3/etc/bind/socket

volumes:
  dnstap-sockets: {}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

noting that image isn't required when defining the test runner service, which builds from the local source.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But it is required to capture and recreate the image parameter already existing in the compose files.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah, yeah I was just adding a case example to Spencer's earlier question, could have quoted that

I was under the assumption that image was required, in what case can it be None?

#[serde(default, skip_serializing_if = "Option::is_none")]
pub hostname: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub container_name: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub build: Option<BuildConfig>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub command: Option<Command>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ports: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub env_file: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub volumes: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub environment: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub depends_on: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub healthcheck: Option<HealthCheck>,
jonathanpv marked this conversation as resolved.
Show resolved Hide resolved
}

#[derive(Debug, Deserialize, Serialize)]
pub struct HealthCheck {
#[serde(rename = "test", skip_serializing_if = "Option::is_none")]
pub test: Option<Command>,
#[serde(rename = "interval", skip_serializing_if = "Option::is_none")]
pub interval: Option<String>,
#[serde(rename = "timeout", skip_serializing_if = "Option::is_none")]
pub timeout: Option<String>,
#[serde(rename = "retries", skip_serializing_if = "Option::is_none")]
pub retries: Option<u32>,
#[serde(rename = "start_period", skip_serializing_if = "Option::is_none")]
pub start_period: Option<String>,
}

#[derive(Debug, Deserialize, Serialize)]
pub struct BuildConfig {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub context: Option<String>
}

#[derive(Debug, Deserialize, Serialize)]
#[serde(untagged)]
pub enum Command {
Single(String),
Multiple(Vec<String>),
}

impl ComposeConfig {
Expand Down
33 changes: 30 additions & 3 deletions vdev/src/testing/integration.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use std::{path::Path, path::PathBuf, process::Command};
use std::collections::BTreeMap;
use std::{fs, path::Path, path::PathBuf, process::Command};

use anyhow::{bail, Context, Result};

Expand Down Expand Up @@ -28,11 +29,28 @@ impl IntegrationTest {
let integration = integration.into();
let environment = environment.into();
let (test_dir, config) = IntegrationTestConfig::load(&integration)?;
// Load thee compose_config
jonathanpv marked this conversation as resolved.
Show resolved Hide resolved
let mut compose_config = ComposeConfig::parse(&test_dir.join("compose.yaml"))?;
let network_name = format!("vector-integration-tests-{integration}");
// Inject the networks block
// TODO: Inject the labels block
compose_config.networks.insert("default".to_string(), {
let mut default_network = BTreeMap::new();
default_network.insert("name".to_string(), network_name.clone());
default_network
jonathanpv marked this conversation as resolved.
Show resolved Hide resolved
});

let temp_compose_path = test_dir.join("compose-temp.yaml");
fs::write(
temp_compose_path,
serde_yaml::to_string(&compose_config)
.with_context(|| "Failed to serialize modified compose.yml".to_string())?,
)?;

let envs_dir = EnvsDir::new(&integration);
let Some(env_config) = config.environments().get(&environment).map(Clone::clone) else {
bail!("Could not find environment named {environment:?}");
};
let network_name = format!("vector-integration-tests-{integration}");
let compose = Compose::new(test_dir, env_config.clone(), Some(network_name.clone()))?;
let runner = IntegrationTestRunner::new(
integration.clone(),
Expand Down Expand Up @@ -128,12 +146,17 @@ impl IntegrationTest {
self.runner.remove()?;
compose.stop()?;
self.envs_dir.remove()?;
std::fs::remove_file(&compose.path)?;
}

Ok(())
}
}

// Question: Should I inject network here or should I keep it in the compose config?
// I am leaning towards keeping it in the compose config. Since that seems to be the
// Compose file direct mapping, but I would like guidance on this. For now I can
// Add to network by reading the compose config.
jonathanpv marked this conversation as resolved.
Show resolved Hide resolved
struct Compose {
path: PathBuf,
test_dir: PathBuf,
Expand All @@ -145,13 +168,17 @@ struct Compose {

impl Compose {
fn new(test_dir: PathBuf, env: Environment, network: Option<String>) -> Result<Option<Self>> {
let path: PathBuf = [&test_dir, Path::new("compose.yaml")].iter().collect();
let path: PathBuf = [&test_dir, Path::new("compose-temp.yaml")].iter().collect();
jonathanpv marked this conversation as resolved.
Show resolved Hide resolved
match path.try_exists() {
Err(error) => Err(error).with_context(|| format!("Could not lookup {path:?}")),
Ok(false) => Ok(None),
Ok(true) => {
#[cfg(unix)]
let config = ComposeConfig::parse(&path)?;
println!(
jonathanpv marked this conversation as resolved.
Show resolved Hide resolved
"[jonathanpv:debugging]\nUsing compose file: {:?}, found in: {:?}",
Fixed Show fixed Hide fixed
Fixed Show fixed Hide fixed
config, path
);
Ok(Some(Self {
path,
test_dir,
Expand Down