[INF-93] Remove rsyslog implementation and build Logspout sidecar container

.circleci/config.yml

@@ -635,6 +635,8 @@ jobs:
     parameters:
       repo:
         type: string
+      logspout-tag:
+        type: string
     steps:
       - checkout
       - setup_remote_docker:
@@ -643,10 +645,11 @@ jobs:
           name: Get tag or latest
           command: |
             # order of precendence for IMAGE tag is:
-            # 1. CIRCLE_TAG if defined,
-            # 2. Branch name if 'hotfix' in branch name (branch name passed in via $CIRCLE_BRANCH)
-            # 3. 'latest', which is the default
-            echo "export IMAGE_TAG=`[ $CIRCLE_TAG ] && echo $(echo $CIRCLE_TAG | cut -d@ -f3) || [[ "$CIRCLE_BRANCH" =~ (hotfix) ]] && echo $CIRCLE_BRANCH || echo "latest" `" | tee -a $BASH_ENV
+            # 1. $(head -n1 logging/logspout/Dockerfile) if << parameters.logspout-tag >> defined,
+            # 2. CIRCLE_TAG if defined,
+            # 3. Branch name if 'hotfix' in branch name (branch name passed in via $CIRCLE_BRANCH)
+            # 4. 'latest', which is the default
+            echo "export IMAGE_TAG=`[ << parameters.logspout-tag >> ] && echo $(head -n1 logging/logspout/Dockerfile | cut -f 2 -d ':') || [ $CIRCLE_TAG ] && echo $(echo $CIRCLE_TAG | cut -d@ -f3) || [[ "$CIRCLE_BRANCH" =~ (hotfix) ]] && echo $CIRCLE_BRANCH || echo "latest" `" | tee -a $BASH_ENV
       - run:
           name: Docker login
           command: |
@@ -655,7 +658,8 @@ jobs:
           name: Docker build << parameters.repo >>
           command: |
             cd << parameters.repo >>
-            docker build -t audius/<< parameters.repo >>:$IMAGE_TAG -t audius/<< parameters.repo>>:$(git rev-parse HEAD) --build-arg git_sha=$(git rev-parse HEAD) --build-arg audius_loggly_disable=$audius_loggly_disable --build-arg audius_loggly_token=$audius_loggly_token --build-arg audius_loggly_tags=$audius_loggly_tags --build-arg BUILD_NUM=$CIRCLE_BUILD_NUM .
+            audius_loggly_token_b64=$(echo ${audius_loggly_token} | base64)
+            docker build -t audius/<< parameters.repo >>:$IMAGE_TAG -t audius/<< parameters.repo>>:$(git rev-parse HEAD) --build-arg git_sha=$(git rev-parse HEAD) --build-arg audius_loggly_token=$audius_loggly_token_b64 --build-arg BUILD_NUM=$CIRCLE_BUILD_NUM .
             docker image prune --filter label=prune=true --filter label=build=$CIRCLE_BUILD_NUM --force
       - run:
           name: Docker push << parameters.repo >>
@@ -690,7 +694,7 @@ jobs:
           name: Docker build << parameters.repo >>
           command: |
             cd << parameters.repo >>
-            docker build -t audius/<< parameters.repo >>:$IMAGE_TAG -t audius/<< parameters.repo>>:$(git rev-parse HEAD) --build-arg git_sha=$(git rev-parse HEAD) --build-arg audius_loggly_disable=$audius_loggly_disable --build-arg audius_loggly_token=$audius_loggly_token --build-arg audius_loggly_tags=$audius_loggly_tags .
+            docker build -t audius/<< parameters.repo >>:$IMAGE_TAG -t audius/<< parameters.repo>>:$(git rev-parse HEAD) --build-arg git_sha=$(git rev-parse HEAD) --build-arg audius_loggly_token=$audius_loggly_token .
       - run:
           name: Docker push << parameters.repo >>
           command: |
@@ -737,6 +741,13 @@ workflows:
           name: build-identity-service
           repo: identity-service
 
+      - test-logspout:
+          name: test-logspout
+      - docker-build-and-push:
+          name: build-logspout
+          repo: logspout/logspout
+          logspout-tag: true
+
       - test-solana-programs:
           name: test-solana-programs
       - test-solana-programs-anchor:

creator-node/Dockerfile

@@ -78,14 +78,7 @@ VOLUME "/file_storage"
 ENV storagePath=/file_storage
 
 ARG git_sha
-ARG audius_loggly_disable
-ARG audius_loggly_token
-ARG audius_loggly_tags
-
 ENV GIT_SHA=$git_sha
-ENV logglyDisable=$audius_loggly_disable
-ENV logglyToken=$audius_loggly_token
-ENV logglyTags=$audius_loggly_tags
 
 HEALTHCHECK --interval=5s --timeout=5s --start-period=5m --retries=12 \
     CMD curl -f http://localhost:4000/health_check || exit 1

creator-node/scripts/start.sh

@@ -7,28 +7,6 @@ if [[ "$WAIT_HOSTS" != "" ]]; then
     /usr/bin/wait
 fi
 
-if [[ -z "$logglyDisable" ]]; then
-    if [[ -n "$logglyToken" ]]; then
-        logglyTags=$(echo $logglyTags | python3 -c "print(' '.join(f'tag=\\\\\"{i}\\\\\"' for i in input().split(',')))")
-        mkdir -p /var/spool/rsyslog
-        mkdir -p /etc/rsyslog.d
-        sed -i '1s|^|$MaxMessageSize 64k\n|' /etc/rsyslog.conf
-        cat >/etc/rsyslog.d/22-loggly.conf <<EOF
-\$WorkDirectory /var/spool/rsyslog # where to place spool files
-\$ActionQueueFileName fwdRule1   # unique name prefix for spool files
-\$ActionQueueMaxDiskSpace 1g    # 1gb space limit (use as much as possible)
-\$ActionQueueSaveOnShutdown on   # save messages to disk on shutdown
-\$ActionQueueType LinkedList    # run asynchronously
-\$ActionResumeRetryCount -1    # infinite retries if host is down
-template(name="LogglyFormat" type="string"
- string="<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$logglyToken@41058 $logglyTags \\"$creatorNodeEndpoint\\"] %msg%\n")
-# Send messages to Loggly over TCP using the template.
-action(type="omfwd" protocol="tcp" target="logs-01.loggly.com" port="514" template="LogglyFormat")
-EOF
-        rsyslogd
-    fi
-fi
-
 if [ -z "$redisHost" ]; then
     redis-server --daemonize yes
     export redisHost=localhost
@@ -68,12 +46,12 @@ if [[ "$devMode" == "true" ]]; then
         npm link
         cd ../app
         npm link @audius/libs
-        npx nodemon  --exec 'node --inspect=0.0.0.0:${debuggerPort} --require ts-node/register src/index.ts' --watch src/ --watch ../audius-libs/ | tee >(logger) | ./node_modules/.bin/bunyan
+        npx nodemon  --exec 'node --inspect=0.0.0.0:${debuggerPort} --require ts-node/register src/index.ts' --watch src/ --watch ../audius-libs/ | ./node_modules/.bin/bunyan
     else
-        npx nodemon  --exec 'node --inspect=0.0.0.0:${debuggerPort} --require ts-node/register src/index.ts' --watch src/ | tee >(logger) | ./node_modules/.bin/bunyan
+        npx nodemon  --exec 'node --inspect=0.0.0.0:${debuggerPort} --require ts-node/register src/index.ts' --watch src/ | ./node_modules/.bin/bunyan
     fi
 else
-    node build/src/index.js | tee >(logger)
+    node build/src/index.js
     docker run -d --name watchtower -v /var/run/docker.sock:/var/run/docker.sock containrrr/watchtower --interval 10
 fi
 

discovery-provider/Dockerfile

@@ -95,14 +95,7 @@ COPY --from=contracts /usr/src/app/build/contracts/ ./build/contracts/
 RUN bash scripts/init-db.sh
 
 ARG git_sha
-ARG audius_loggly_disable
-ARG audius_loggly_token
-ARG audius_loggly_tags
-
 ENV GIT_SHA=$git_sha
-ENV audius_loggly_disable=$audius_loggly_disable
-ENV audius_loggly_token=$audius_loggly_token
-ENV audius_loggly_tags=$audius_loggly_tags
 
 ENV PROMETHEUS_MULTIPROC_DIR /prometheus_data
 RUN mkdir -p ${PROMETHEUS_MULTIPROC_DIR}

discovery-provider/scripts/start.sh

@@ -1,66 +1,6 @@
 #!/bin/bash
 set -e
 
-mkdir -p /var/log
-mkdir -p /var/spool/rsyslog
-mkdir -p /etc/rsyslog.d
-
-if [[ -z "$audius_loggly_disable" ]]; then
-    if [[ -n "$audius_loggly_token" ]]; then
-        # use regex to extract domain in url (source: https://stackoverflow.com/a/2506635/8674706)
-        audius_discprov_hostname=$(echo $audius_discprov_url | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')
-
-        # add hostname to loggly tags
-        if [[ "$audius_discprov_hostname" != "" ]]; then
-            if [[ "$audius_loggly_tags" != "" ]]; then
-                audius_loggly_tags="$audius_loggly_tags,$audius_discprov_hostname"
-            else
-                audius_loggly_tags="$audius_discprov_hostname"
-            fi
-        fi
-
-        if [[ "$audius_loggly_tags" != "" ]]; then
-            audius_loggly_tags="tag=\\\"${audius_loggly_tags//,/\\\" tag=\\\"}\\\""
-            # ${audius_loggly_tags//,/\\\" tag=\\\"} replaces , with \" tag=\"
-            # then we add tag=\" to the start and \" to the end
-        fi
-
-        cat >/etc/rsyslog.d/10-loggly.conf <<EOF
-\$WorkDirectory /var/spool/rsyslog  # where to place spool files
-\$ActionQueueFileName loggly        # unique name prefix for spool files
-\$ActionQueueMaxDiskSpace 1g        # 1gb space limit (use as much as possible)
-\$ActionQueueSaveOnShutdown on      # save messages to disk on shutdown
-\$ActionQueueType LinkedList        # run asynchronously
-\$ActionResumeRetryCount -1         # infinite retries if host is down
-
-template(name="LogglyFormat" type="string"
- string="<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$audius_loggly_token@41058 $audius_loggly_tags] %msg%\n")
-
-# Send messages to Loggly over TCP using the template.
-action(type="omfwd" protocol="tcp" target="logs-01.loggly.com" port="514" template="LogglyFormat")
-EOF
-    fi
-fi
-
-cat >/etc/rsyslog.d/20-file.conf <<EOF
-\$WorkDirectory /var/spool/rsyslog  # where to place spool files
-\$ActionQueueFileName file          # unique name prefix for spool files
-\$ActionQueueMaxDiskSpace 1g        # 1gb space limit (use as much as possible)
-\$ActionQueueSaveOnShutdown on      # save messages to disk on shutdown
-\$ActionQueueType LinkedList        # run asynchronously
-\$ActionResumeRetryCount -1         # infinite retries if host is down
-
-\$outchannel server_log,/var/log/discprov-server.log, 52428800,/audius-discovery-provider/scripts/rotate-log.sh
-\$outchannel worker_log,/var/log/discprov-worker.log, 52428800,/audius-discovery-provider/scripts/rotate-log.sh
-\$outchannel beat_log,/var/log/discprov-beat.log, 52428800,/audius-discovery-provider/scripts/rotate-log.sh
-
-if \$programname == 'server' then :omfile:\$server_log
-if \$programname == 'worker' then :omfile:\$worker_log
-if \$programname == 'beat' then   :omfile:\$beat_log
-EOF
-
-rsyslogd
-
 if [ -z "$audius_redis_url" ]; then
     redis-server --daemonize yes
     export audius_redis_url="redis://localhost:6379/00"
@@ -106,20 +46,16 @@ if [[ "$audius_discprov_dev_mode" == "true" ]]; then
         echo "Finished running migrations"
     fi
 
-    # filter tail to server/worker/beat logs with
-    # docker exec -it <container> tail -f /var/log/discprov-server.log
-    # docker exec -it <container> tail -f /var/log/discprov-worker.log
-    # docker exec -it <container> tail -f /var/log/discprov-beat.log
-    ./scripts/dev-server.sh 2>&1 | tee >(logger -t server) &
+    ./scripts/dev-server.sh &
     if [[ "$audius_no_workers" != "true" ]] && [[ "$audius_no_workers" != "1" ]]; then
-        watchmedo auto-restart --directory ./ --pattern=*.py --recursive -- celery -A src.worker.celery worker --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t worker) &
-        celery -A src.worker.celery beat --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t beat) &
+        watchmedo auto-restart --directory ./ --pattern=*.py --recursive -- celery -A src.worker.celery worker --loglevel $audius_discprov_loglevel &
+        celery -A src.worker.celery beat --loglevel $audius_discprov_loglevel &
     fi
 else
-    ./scripts/prod-server.sh 2>&1 | tee >(logger -t server) &
+    ./scripts/prod-server.sh &
     if [[ "$audius_no_workers" != "true" ]] && [[ "$audius_no_workers" != "1" ]]; then
-        celery -A src.worker.celery worker --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t worker) &
-        celery -A src.worker.celery beat --loglevel $audius_discprov_loglevel 2>&1 | tee >(logger -t beat) &
+        celery -A src.worker.celery worker --loglevel $audius_discprov_loglevel &
+        celery -A src.worker.celery beat --loglevel $audius_discprov_loglevel &
     fi
 fi
 

identity-service/Dockerfile

@@ -31,14 +31,7 @@ RUN apk update && \
 EXPOSE 7000
 
 ARG git_sha
-ARG audius_loggly_disable
-ARG audius_loggly_token
-ARG audius_loggly_tags
-
 ENV GIT_SHA=$git_sha
-ENV logglyDisable=$audius_loggly_disable
-ENV logglyToken=$audius_loggly_token
-ENV logglyTags=$audius_loggly_tags
 
 HEALTHCHECK --interval=5s --timeout=5s \
     CMD curl -f http://localhost:7000/health_check || exit 1

identity-service/scripts/start.sh

@@ -2,25 +2,4 @@
 
 /usr/bin/wait
 
-if [[ -z "$logglyDisable" ]]; then
-    if [[ -n "$logglyToken" ]]; then
-        logglyTags=$(echo $logglyTags | python3 -c "print(' '.join(f'tag=\\\\\"{i}\\\\\"' for i in input().split(',')))")
-        mkdir -p /var/spool/rsyslog
-        mkdir -p /etc/rsyslog.d
-        cat >/etc/rsyslog.d/22-loggly.conf <<EOF
-\$WorkDirectory /var/spool/rsyslog # where to place spool files
-\$ActionQueueFileName fwdRule1   # unique name prefix for spool files
-\$ActionQueueMaxDiskSpace 1g    # 1gb space limit (use as much as possible)
-\$ActionQueueSaveOnShutdown on   # save messages to disk on shutdown
-\$ActionQueueType LinkedList    # run asynchronously
-\$ActionResumeRetryCount -1    # infinite retries if host is down
-template(name="LogglyFormat" type="string"
- string="<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$logglyToken@41058 $logglyTags] %msg%\n")
-# Send messages to Loggly over TCP using the template.
-action(type="omfwd" protocol="tcp" target="logs-01.loggly.com" port="514" template="LogglyFormat")
-EOF
-        rsyslogd
-    fi
-fi
-
-node src/index.js | tee >(logger)
+node src/index.js

logging/.gitignore

@@ -0,0 +1,2 @@
+.env
+

logging/logspout/Dockerfile

@@ -0,0 +1,18 @@
+FROM gliderlabs/logspout:v3.2.14
+
+# ignores previously log on startup
+# could be lossy if the container restarts
+# could send double logs upon container restarts if we remove this
+ENV BACKLOG false
+
+ARG git_sha
+ENV GIT_SHA=${git_sha}
+
+ARG audius_loggly_token
+ENV audius_loggly_token ${audius_loggly_token}
+
+ENTRYPOINT []
+CMD audius_loggly_token_64=$(echo ${audius_loggly_token} | base64 -d) \
+    && export SYSLOG_STRUCTURED_DATA="${audius_loggly_token_64}@41058 tag=\"logspout\"" \
+    && echo SYSLOG_STRUCTURED_DATA=${SYSLOG_STRUCTURED_DATA} \
+    && /bin/logspout multiline+syslog+tcp://logs-01.loggly.com:514

logging/logspout/README.md

@@ -0,0 +1,16 @@
+# Build Custom Audius Logstash Sidecar Container
+
+```bash
+# .env contains: audius_loggly_token=xxx
+. .env
+
+LOGSPOUT_VERSION=$(head -n1 Dockerfile | cut -f 2 -d ':')
+[ ${audius_loggly_token} ] \
+    && audius_loggly_token_64=$(echo ${audius_loggly_token} | base64) \
+    && docker build \
+        -t audius/logspout:${LOGSPOUT_VERSION} \
+        --build-arg git_sha=$(git rev-parse HEAD) \
+        --build-arg audius_loggly_token=${audius_loggly_token_64} \
+        . \
+    && docker push audius/logspout:${LOGSPOUT_VERSION}
+```

logging/logspout/build.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+# https://github.com/gliderlabs/logspout/blob/818dd8260e52d2c148280d86170bdf5267b5c637/build.sh
+
+set -e
+apk add --update go build-base git mercurial ca-certificates
+cd /src
+go build -ldflags "-X main.Version=$1" -o /bin/logspout
+apk del go git mercurial build-base
+rm -rf /root/go /var/cache/apk/*
+
+# backwards compatibility
+ln -fs /tmp/docker.sock /var/run/docker.sock
+

logging/logspout/modules.go

@@ -0,0 +1,15 @@
+// https://github.com/gliderlabs/logspout/blob/818dd8260e52d2c148280d86170bdf5267b5c637/modules.go
+
+package main
+
+import (
+	_ "github.com/gliderlabs/logspout/adapters/multiline"
+	_ "github.com/gliderlabs/logspout/adapters/raw"
+	_ "github.com/gliderlabs/logspout/adapters/syslog"
+	_ "github.com/gliderlabs/logspout/healthcheck"
+	_ "github.com/gliderlabs/logspout/httpstream"
+	_ "github.com/gliderlabs/logspout/routesapi"
+	_ "github.com/gliderlabs/logspout/transports/tcp"
+	_ "github.com/gliderlabs/logspout/transports/tls"
+	_ "github.com/gliderlabs/logspout/transports/udp"
+)