enhancement(auth): NATS sink+source authentication and TLS support (#…

…10688) * add tls support for nats source/sink, add support for all nats authenticatation options for nats source/sink, integration tests for nats tls and nats auth Signed-off-by: Collins Huff <collinsrhuff@gmail.com> * fix broken podman setup, remove old print statements Signed-off-by: Collins Huff <collinsrhuff@gmail.com> * add registry to image name so podman doesn't prompt Signed-off-by: Collins Huff <collinsrhuff@gmail.com> * format cue files Signed-off-by: Collins Huff <collinsrhuff@gmail.com> * use snafu context to make error handling more concise, add print statements to asserts in integration tests to report what failed, cleanup NATS docs Signed-off-by: Collins Huff <collinsrhuff@gmail.com> * Update auth options to follow external tagging format Signed-off-by: Jesse Szwedko <jesse@szwedko.me> * clippy Signed-off-by: Jesse Szwedko <jesse@szwedko.me> Co-authored-by: Bruce Guenter <bruce.guenter@datadoghq.com> Co-authored-by: Jesse Szwedko <jesse@szwedko.me>
vectordotdev · Mar 9, 2022 · d06ee2a · d06ee2a
1 parent 5402bb3
commit d06ee2a
Show file tree

Hide file tree

Showing 20 changed files with 1,510 additions and 118 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -251,6 +251,7 @@ md-5 = { version = "0.10", optional = true }
 memchr = { version = "2.4", default-features = false, optional = true }
 mongodb = { version = "2.1.0", default-features = false, features = ["tokio-runtime"], optional = true }
 nats = { version = "0.18.1", default-features = false, optional = true }
+nkeys = { version = "0.2.0", optional = true }
 nom = { version = "7.1.0", default-features = false, optional = true }
 notify = { version = "4.0.17", default-features = false }
 num_cpus = { version = "1.13.1", default-features = false }
@@ -469,7 +470,7 @@ sources-internal_logs = []
 sources-internal_metrics = []
 sources-journald = ["codecs"]
 sources-kafka = ["rdkafka", "codecs"]
-sources-nats = ["nats", "codecs"]
+sources-nats = ["nats", "nkeys", "codecs"]
 sources-logstash = ["listenfd", "tokio-util/net", "sources-utils-tcp-keepalive", "sources-utils-tcp-socket", "sources-utils-tls", "codecs"]
 sources-kubernetes_logs = ["file-source", "kubernetes", "transforms-merge", "transforms-regex_parser"]
 sources-mongodb_metrics = ["mongodb"]
@@ -655,7 +656,7 @@ sinks-influxdb = []
 sinks-kafka = ["rdkafka"]
 sinks-logdna = []
 sinks-loki = []
-sinks-nats = ["nats"]
+sinks-nats = ["nats", "nkeys"]
 sinks-new_relic_logs = ["sinks-http"]
 sinks-new_relic = []
 sinks-papertrail = ["syslog"]

diff --git a/scripts/setup_integration/nats_integration_env.sh b/scripts/setup_integration/nats_integration_env.sh
@@ -14,30 +14,133 @@ then
 fi
 ACTION=$1
 
-#
-# Functions
-#
 
 start_podman () {
-  podman pod create --replace --name vector-test-integration-nats -p 4222:4222
-  podman run -d --pod=vector-test-integration-nats  --name vector_nats \
-	 nats
+  podman pod create --replace --name vector_nats -p 4222:4222
+  podman pod create --replace --name vector_nats_userpass -p 4223:4222
+  podman pod create --replace --name vector_nats_token -p 4224:4222
+  podman pod create --replace --name vector_nats_nkey -p 4225:4222
+  podman pod create --replace --name vector_nats_tls -p 4227:4222
+  podman pod create --replace --name vector_nats_tls_client_cert -p 4228:4222
+  podman pod create --replace --name vector_nats_jwt -p 4229:4222
+
+  podman run -d --pod=vector_nats --name vector_nats_test docker.io/library/nats:latest
+  podman run -d --pod=vector_nats_userpass --name vector_nats_userpass_test docker.io/library/nats:latest \
+      --user natsuser --pass natspass
+  podman run -d --pod=vector_nats_token --name vector_nats_token_test docker.io/library/nats:latest \
+      --auth secret
+  podman run -d --pod=vector_nats_nkey --name vector_nats_nkey_test \
+    -v "$(pwd)"/tests/data:/usr/share/nats/config:ro \
+    docker.io/library/nats:latest -c /usr/share/nats/config/nats-nkey.conf
+
+  podman run -d --pod=vector_nats_tls --name vector_nats_tls_test \
+    -v "$(pwd)"/tests/data:/usr/share/nats/config:ro \
+    docker.io/library/nats:latest -c /usr/share/nats/config/nats-tls.conf
+
+  podman run -d --pod=vector_nats_tls_client_cert --name vector_nats_tls_client_cert_test \
+    -v "$(pwd)"/tests/data:/usr/share/nats/config:ro \
+    docker.io/library/nats:latest -c /usr/share/nats/config/nats-tls-client-cert.conf
+
+  podman run -d --pod=vector_nats_jwt --name vector_nats_jwt_test \
+    -v "$(pwd)"/tests/data:/usr/share/nats/config:ro \
+    docker.io/library/nats:latest -c /usr/share/nats/config/nats-jwt.conf
 }
 
 start_docker () {
   docker network create vector-test-integration-nats
-  docker run -d --network=vector-test-integration-nats -p 4222:4222 --name vector_nats \
-	 nats
+  docker run -d --network=vector-test-integration-nats -p 4222:4222 --name vector_nats nats
+  docker run -d --network=vector-test-integration-nats -p 4223:4222 --name vector_nats_userpass nats \
+    --user natsuser --pass natspass
+  docker run -d --network=vector-test-integration-nats -p 4224:4222 --name vector_nats_token nats \
+    --auth secret
+
+  # The following tls tests use mkcert
+  # https://github.com/FiloSottile/mkcert
+  # See https://docs.nats.io/running-a-nats-service/configuration/securing_nats/tls
+
+
+  # Generate a new NKey with the following command:
+  # $ nk -gen user -pubout
+  # SUANIRXEZUROTXNFN3TJYMT27K7ZZVMD46FRIHF6KXKS4KGNVBS57YAFGY
+  # UD345ZYSUJQD7PNCTWQPINYSO3VH4JBSADBSYUZOBT666DRASFRAWAWT
+  #
+  # The first line of output is the Seed, which is a private key
+  # The second line of output is the User string, which is a public key
+  docker run -d --network=vector-test-integration-nats -p 4225:4222 \
+    -v "$(pwd)"/tests/data:/usr/share/nats/config:ro \
+    --name vector_nats_nkey nats \
+    -c /usr/share/nats/config/nats-nkey.conf
+
+  # First, generate a certificate for the NATS server using the following command
+  # $ mkcert -cert-file server-cert.pem -key-file server-key.pem localhost ::1
+  #
+  # Next, move the mkcert root CA to the correct location
+  # $ mv "$(mkcert -CAROOT)/rootCA.pem" tests/data/mkcert_rootCA.pem
+  docker run -d --network=vector-test-integration-nats -p 4227:4222 \
+    -v "$(pwd)"/tests/data:/usr/share/nats/config:ro \
+    --name vector_nats_tls nats \
+    -c /usr/share/nats/config/nats-tls.conf
+
+  # Generate a client cert using the following command
+  # $ mkcert -client -cert-file nats_client_cert.pem -key-file nats_client_key.pem localhost ::1 email@localhost
+  docker run -d --network=vector-test-integration-nats -p 4228:4222 \
+    -v "$(pwd)"/tests/data:/usr/share/nats/config:ro \
+    --name vector_nats_tls_client_cert nats \
+    -c /usr/share/nats/config/nats-tls-client-cert.conf
+
+  # Follow the instructions here
+  # See https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/jwt/mem_resolver
+  # Then run the following additional commands
+  # $ mv /tmp/server.conf tests/data/nats-jwt.conf
+  # $ cat << EOF >> tests/data/nats-jwt.conf
+  #
+  #tls: {
+  #  cert_file: "/usr/share/nats/config/localhost-mkcert.pem"
+  #  key_file: "/usr/share/nats/config/localhost-mkcert-key.pem"
+  #}
+  #EOF
+  #
+  # $ mv ~/.nkeys/creds/memory/A/TA.creds tests/data/nats.creds
+  # $ cp tests/data/nats.creds tests/data/nats-bad.creds
+  # # edit test/data/nats-bad.creds and change one of the characters in the Seed
+
+  docker run -d --network=vector-test-integration-nats -p 4229:4222 \
+    -v "$(pwd)"/tests/data:/usr/share/nats/config:ro \
+    --name vector_nats_jwt nats \
+    -c /usr/share/nats/config/nats-jwt.conf
 }
 
 stop_podman () {
-  podman rm --force vector_nats 2>/dev/null; true
-  podman pod stop vector-test-integration-nats 2>/dev/null; true
-  podman pod rm --force vector-test-integration-nats 2>/dev/null; true
+  podman pod stop vector_nats_test 2>/dev/null; true
+  podman pod rm --force vector_nats 2>/dev/null; true
+
+  podman pod stop vector_nats_userpass_test 2>/dev/null; true
+  podman pod rm --force vector_nats_userpass 2>/dev/null; true
+
+  podman pod stop vector_nats_token_test 2>/dev/null; true
+  podman pod rm --force vector_nats_token 2>/dev/null; true
+
+  podman pod stop vector_nats_nkey_test 2>/dev/null; true
+  podman pod rm --force vector_nats_nkey 2>/dev/null; true
+
+  podman pod stop vector_nats_tls_test 2>/dev/null; true
+  podman pod rm --force vector_nats_tls 2>/dev/null; true
+
+  podman pod stop vector_nats_tls_client_cert_test 2>/dev/null; true
+  podman pod rm --force vector_nats_tls_client_cert 2>/dev/null; true
+
+  podman pod stop vector_nats_jwt_test 2>/dev/null; true
+  podman pod rm --force vector_nats_jwt 2>/dev/null; true
 }
 
 stop_docker () {
   docker rm --force vector_nats 2>/dev/null; true
+  docker rm --force vector_nats_userpass 2>/dev/null; true
+  docker rm --force vector_nats_token 2>/dev/null; true
+  docker rm --force vector_nats_nkey 2>/dev/null; true
+  docker rm --force vector_nats_tls 2>/dev/null; true
+  docker rm --force vector_nats_tls_client_cert 2>/dev/null; true
+  docker rm --force vector_nats_jwt 2>/dev/null; true
   docker network rm vector-test-integration-nats 2>/dev/null; true
 }
 

diff --git a/src/lib.rs b/src/lib.rs
@@ -63,6 +63,8 @@ pub(crate) mod kafka;
 pub mod kubernetes;
 pub mod line_agg;
 pub mod list;
+#[cfg(any(feature = "sources-nats", feature = "sinks-nats"))]
+pub(crate) mod nats;
 #[allow(unreachable_pub)]
 pub(crate) mod proto;
 pub mod providers;