From 96ca95123bcacef3b7fe61fc8c5c674beb0e19fd Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Tue, 5 Sep 2023 22:28:37 -0700 Subject: [PATCH] docker build with community server --- docker/README.md | 52 ++++++++++++++- docker/server/Dockerfile | 51 ++++++++++++--- docker/server/bin/startup.sh | 7 ++- docker/server/config/config.properties | 27 ++------ docker/server/nginx/nginx.conf | 50 +++++++++++++++ docker/serverAndUI/Dockerfile | 63 ------------------- docker/serverAndUI/README.md | 10 --- docker/serverAndUI/bin/startup.sh | 36 ----------- .../config/config-local.properties | 33 ---------- docker/serverAndUI/config/config.properties | 35 ----------- docker/serverAndUI/nginx/nginx.conf | 20 ------ 11 files changed, 154 insertions(+), 230 deletions(-) create mode 100644 docker/server/nginx/nginx.conf delete mode 100644 docker/serverAndUI/Dockerfile delete mode 100644 docker/serverAndUI/README.md delete mode 100755 docker/serverAndUI/bin/startup.sh delete mode 100755 docker/serverAndUI/config/config-local.properties delete mode 100755 docker/serverAndUI/config/config.properties delete mode 100644 docker/serverAndUI/nginx/nginx.conf diff --git a/docker/README.md b/docker/README.md index 538ebae353..9dba9ae5e8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1 +1,51 @@ -[Docker Instructions](/docs/docs/gettingstarted/docker.md) \ No newline at end of file +# Conductor Docker Builds + +## Pre-built docker images + +Conductor server with support for the following backend: +1. Redis +2. Postgres +3. Mysql +4. Cassandra + +```shell +docker pull docker.orkes.io/conductor:latest +``` + +### Docker File for Server and UI + +[Docker Image Source for Server with UI](serverAndUI/Dockerfile) + +#### Pre-requisites for building +1. [Docker](https://www.docker.com/) +2. [Node](https://nodejs.org/en) +3. [JDK](https://openjdk.org/) + +### Configuration Guide for Conductor Server +Conductor uses a persistent store for managing state. +The choice of backend is quite flexible and can be configured at runtime using `conductor.db.type` property. + +Refer to the table below for various supported backend and required configurations to enable each of them. + +| Backend | Property | Required Configuration | +|------------|------------------------------------|------------------------| +| postgres | conductor.db.type=postgres | | +| redis | conductor.db.type=redis_standalone | | +| mysql | conductor.db.type=mysql | | +| cassandra | conductor.db.type=cassandra | | + +Conductor using Elasticsearch for indexing the workflow data. +Currently, Elasticsearch 6 and 7 are supported. +We welcome community contributions for other indexing backends. + +**Note:** Docker images use Elasticsearch 7. + +### Recommended Configuration for the server +```properties + +``` + +## Helm Charts +TODO: Link to the helm charts + +## Run Docker Compose Locally diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 80017603ac..0b9e4a8911 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -5,31 +5,66 @@ # =========================================================================================================== # 0. Builder stage # =========================================================================================================== -FROM eclipse-temurin:11-jdk-jammy AS builder +FROM alpine:3.18 AS builder LABEL maintainer="Netflix OSS " -# Copy the project directly onto the image +# =========================================================================================================== +# 0. Build Conductor Server +# =========================================================================================================== + + +# Install dependencies +RUN apk add openjdk11 +RUN apk add git +RUN apk add --update nodejs npm yarn + COPY . /conductor -WORKDIR /conductor +WORKDIR /conductor/ui +RUN yarn install && yarn build +RUN ls -ltr +RUN echo "Done building UI" + +# Checkout the community project +WORKDIR / +RUN mkdir server-build +WORKDIR server-build +RUN ls -ltr + +RUN git clone https://github.com/Netflix/conductor-community.git + +# Copy the project directly onto the image +WORKDIR conductor-community +RUN ls -ltr # Build the server on run RUN ./gradlew build -x test --stacktrace +WORKDIR /server-build +RUN ls -ltr +RUN pwd # =========================================================================================================== # 1. Bin stage # =========================================================================================================== -FROM eclipse-temurin:11-jre-jammy - +FROM alpine:3.18 LABEL maintainer="Netflix OSS " +RUN apk add openjdk11 +RUN apk add nginx + # Make app folders RUN mkdir -p /app/config /app/logs /app/libs # Copy the compiled output to new image -COPY --from=builder /conductor/docker/server/bin /app -COPY --from=builder /conductor/docker/server/config /app/config -COPY --from=builder /conductor/server/build/libs/conductor-server-*-boot.jar /app/libs +COPY docker/server/bin /app +COPY docker/server/config /app/config +COPY --from=builder /server-build/conductor-community/community-server/build/libs/*boot*.jar /app/libs/conductor-server.jar + +# Copy compiled UI assets to nginx www directory +WORKDIR /usr/share/nginx/html +RUN rm -rf ./* +COPY --from=builder /conductor/ui/build . +COPY --from=builder /conductor/docker/server/nginx/nginx.conf /etc/nginx/http.d/default.conf # Copy the files for the server into the app folders RUN chmod +x /app/startup.sh diff --git a/docker/server/bin/startup.sh b/docker/server/bin/startup.sh index 9d1b98cba8..8870aa5d39 100755 --- a/docker/server/bin/startup.sh +++ b/docker/server/bin/startup.sh @@ -15,6 +15,9 @@ # startup.sh - startup script for the server docker image echo "Starting Conductor server" +echo "Running Nginx in background" +# Start nginx as daemon +nginx # Start the server cd /app/libs @@ -25,7 +28,7 @@ export config_file= if [ -z "$CONFIG_PROP" ]; then echo "Using an in-memory instance of conductor"; - export config_file=/app/config/config-local.properties + export config_file=/app/config/config.properties else echo "Using '$CONFIG_PROP'"; export config_file=/app/config/$CONFIG_PROP @@ -33,4 +36,4 @@ fi echo "Using java options config: $JAVA_OPTS" -java ${JAVA_OPTS} -jar -DCONDUCTOR_CONFIG_FILE=$config_file conductor-server-*-boot.jar 2>&1 | tee -a /app/logs/server.log +java ${JAVA_OPTS} -jar -DCONDUCTOR_CONFIG_FILE=$config_file conductor-server.jar 2>&1 | tee -a /app/logs/server.log diff --git a/docker/server/config/config.properties b/docker/server/config/config.properties index 55124c78c0..b1463a6e5e 100755 --- a/docker/server/config/config.properties +++ b/docker/server/config/config.properties @@ -1,38 +1,21 @@ -# Servers. -conductor.grpc-server.enabled=false - # Database persistence type. -conductor.db.type=dynomite - -# Dynomite Cluster details. -# format is host:port:rack separated by semicolon -conductor.redis.hosts=dyno1:8102:us-east-1c - -# Dynomite cluster name -conductor.redis.clusterName=dyno1 - -# Namespace for the keys stored in Dynomite/Redis +conductor.db.type=redis_standalone +conductor.redis.hosts=localhost:6379:us-east-1c +conductor.redis-lock.serverAddress=redis://localhost:6379 +conductor.redis.taskDefCacheRefreshInterval=1 conductor.redis.workflowNamespacePrefix=conductor - -# Namespace prefix for the dyno queues conductor.redis.queueNamespacePrefix=conductor_queues -# No. of threads allocated to dyno-queues (optional) -queues.dynomite.threads=10 # By default with dynomite, we want the repairservice enabled conductor.app.workflowRepairServiceEnabled=true -# Non-quorum port used to connect to local redis. Used by dyno-queues. -# When using redis directly, set this to the same port as redis server -# For Dynomite, this is 22122 by default or the local redis-server port used by Dynomite. -conductor.redis.queuesNonQuorumPort=22122 # Elastic search instance indexing is enabled. conductor.indexing.enabled=true # Transport address to elasticsearch -conductor.elasticsearch.url=http://es:9200 +conductor.elasticsearch.url=http://localhost:9200 # Name of the elasticsearch cluster conductor.elasticsearch.indexName=conductor diff --git a/docker/server/nginx/nginx.conf b/docker/server/nginx/nginx.conf new file mode 100644 index 0000000000..3af603b55c --- /dev/null +++ b/docker/server/nginx/nginx.conf @@ -0,0 +1,50 @@ +server { + listen 5000; + server_name conductor; + server_tokens off; + + location / { + add_header Referrer-Policy "strict-origin"; + add_header X-Frame-Options "SAMEORIGIN"; + add_header X-Content-Type-Options "nosniff"; + add_header Content-Security-Policy "script-src 'self' 'unsafe-inline' 'unsafe-eval' assets.orkes.io *.googletagmanager.com *.pendo.io https://cdn.jsdelivr.net; worker-src 'self' 'unsafe-inline' 'unsafe-eval' data: blob:;"; + add_header Permissions-Policy "accelerometer=(), autoplay=(), camera=(), cross-origin-isolated=(), display-capture=(), document-domain=(), encrypted-media=(), fullscreen=(), geolocation=(), gyroscope=(), keyboard-map=(), magnetometer=(), microphone=(), midi=(), payment=(), picture-in-picture=(), publickey-credentials-get=(), screen-wake-lock=(), sync-xhr=(), usb=(), xr-spatial-tracking=(), clipboard-read=(self), clipboard-write=(self), gamepad=(), hid=(), idle-detection=(), serial=(), window-placement=(self)"; + # This would be the directory where your React app's static files are stored at + root /usr/share/nginx/html; + try_files $uri /index.html; + } + + location /api { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://localhost:8080/api; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /actuator { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://localhost:8080/actuator; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + + location /swagger-ui { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-NginX-Proxy true; + proxy_pass http://localhost:8080/swagger-ui; + proxy_ssl_session_reuse off; + proxy_set_header Host $http_host; + proxy_cache_bypass $http_upgrade; + proxy_redirect off; + } + +} \ No newline at end of file diff --git a/docker/serverAndUI/Dockerfile b/docker/serverAndUI/Dockerfile deleted file mode 100644 index 1722e07269..0000000000 --- a/docker/serverAndUI/Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ -# -# conductor:serverAndUI - Combined Netflix conductor server & UI -# -# =========================================================================================================== -# 0. Builder stage -# =========================================================================================================== -FROM openjdk:11-jdk AS builder -LABEL maintainer="Netflix OSS " - -# Install Node -SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - \ - && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \ - && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \ - && apt-get update -qq \ - && apt-get install -qq --no-install-recommends \ - build-essential \ - nodejs \ - yarn \ - && apt-get upgrade -qq \ - && rm -rf /var/lib/apt/lists/* - -# Copy the project onto the builder image -COPY . /conductor - -# Build the server -WORKDIR /conductor -RUN ./gradlew build -x test - -# Build the client -WORKDIR /conductor/ui -RUN yarn install && yarn build - -# =========================================================================================================== -# 1. Bin stage -# =========================================================================================================== - -FROM nginx:alpine -RUN apk add openjdk11-jre - -LABEL maintainer="Netflix OSS " - -# Make app folders -RUN mkdir -p /app/config /app/logs /app/libs - -# Copy the compiled output to new image -COPY --from=builder /conductor/docker/serverAndUI/bin /app -COPY --from=builder /conductor/docker/serverAndUI/config /app/config -COPY --from=builder /conductor/server/build/libs/conductor-server-*-boot.jar /app/libs - -# Copy compiled UI assets to nginx www directory -WORKDIR /usr/share/nginx/html -RUN rm -rf ./* -COPY --from=builder /conductor/ui/build . -COPY --from=builder /conductor/docker/serverAndUI/nginx/nginx.conf /etc/nginx/conf.d/default.conf - -# Copy the files for the server into the app folders -RUN chmod +x /app/startup.sh - -HEALTHCHECK --interval=60s --timeout=30s --retries=10 CMD curl -I -XGET http://localhost:8080/health || exit 1 - -CMD [ "/app/startup.sh" ] -ENTRYPOINT [ "/bin/sh"] diff --git a/docker/serverAndUI/README.md b/docker/serverAndUI/README.md deleted file mode 100644 index 275d74add6..0000000000 --- a/docker/serverAndUI/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Docker -## Conductor server and UI -This Dockerfile create the conductor:serverAndUI image - -## Building the image -`docker build -t conductor:serverAndUI .` - -## Running the conductor server - - Standalone server (interal DB): `docker run -p 8080:8080 -p 80:5000 -d -t conductor:serverAndUI` - - Server (external DB required): `docker run -p 8080:8080 -p 80:5000 -d -t -e "CONFIG_PROP=config.properties" conductor:serverAndUI` diff --git a/docker/serverAndUI/bin/startup.sh b/docker/serverAndUI/bin/startup.sh deleted file mode 100755 index 0070cd0b9d..0000000000 --- a/docker/serverAndUI/bin/startup.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# -# Copyright 2021 Netflix, Inc. -#

-# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -#

-# http://www.apache.org/licenses/LICENSE-2.0 -#

-# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -# specific language governing permissions and limitations under the License. -# - -echo "Starting Conductor Server and UI" -echo "Running Nginx in background" -# Start nginx as daemon -nginx - -# Start the server -cd /app/libs -echo "Property file: $CONFIG_PROP" -echo $CONFIG_PROP -export config_file= - -if [ -z "$CONFIG_PROP" ]; - then - echo "Using an in-memory instance of conductor"; - export config_file=/app/config/config-local.properties - else - echo "Using '$CONFIG_PROP'"; - export config_file=/app/config/$CONFIG_PROP -fi - -nohup java -jar -DCONDUCTOR_CONFIG_FILE=$config_file conductor-server-*-boot.jar 1>&2 > /app/logs/server.log diff --git a/docker/serverAndUI/config/config-local.properties b/docker/serverAndUI/config/config-local.properties deleted file mode 100755 index d725130e89..0000000000 --- a/docker/serverAndUI/config/config-local.properties +++ /dev/null @@ -1,33 +0,0 @@ -# Database persistence type. -conductor.db.type=memory - -# Dynomite Cluster details. -# format is host:port:rack separated by semicolon -conductor.redis.hosts=dyno1:8102:us-east-1c - -# Namespace for the keys stored in Dynomite/Redis -conductor.redis.workflowNamespacePrefix=conductor - -# Namespace prefix for the dyno queues -conductor.redis.queueNamespacePrefix=conductor_queues - -# No. of threads allocated to dyno-queues (optional) -queues.dynomite.threads=10 - -# By default with dynomite, we want the repairservice enabled -conductor.app.workflowRepairServiceEnabled=true - - -# Non-quorum port used to connect to local redis. Used by dyno-queues. -# When using redis directly, set this to the same port as redis server -# For Dynomite, this is 22122 by default or the local redis-server port used by Dynomite. -conductor.redis.queuesNonQuorumPort=22122 - -# Transport address to elasticsearch -conductor.elasticsearch.url=localhost:9300 - -# Name of the elasticsearch cluster -conductor.elasticsearch.indexName=conductor - -# Load sample kitchen sink workflow -loadSample=true diff --git a/docker/serverAndUI/config/config.properties b/docker/serverAndUI/config/config.properties deleted file mode 100755 index c596c6f10f..0000000000 --- a/docker/serverAndUI/config/config.properties +++ /dev/null @@ -1,35 +0,0 @@ -# Database persistence model. -conductor.db.type=dynomite - -# Dynomite Cluster details. -# format is host:port:rack separated by semicolon -conductor.redis.hosts=dyno1:8102:us-east-1c - -# Dynomite cluster name -conductor.redis.clusterName=dyno1 - -# Namespace for the keys stored in Dynomite/Redis -conductor.redis.workflowNamespacePrefix=conductor - -# Namespace prefix for the dyno queues -conductor.redis.queueNamespacePrefix=conductor_queues - -# No. of threads allocated to dyno-queues (optional) -queues.dynomite.threads=10 - -# By default with dynomite, we want the repairservice enabled -conductor.app.workflowRepairServiceEnabled=true - -# Non-quorum port used to connect to local redis. Used by dyno-queues. -# When using redis directly, set this to the same port as redis server -# For Dynomite, this is 22122 by default or the local redis-server port used by Dynomite. -conductor.redis.queuesNonQuorumPort=22122 - -# Transport address to elasticsearch -conductor.elasticsearch.url=es:9300 - -# Name of the elasticsearch cluster -conductor.elasticsearch.indexName=conductor - -# Load sample kitchen sink workflow -loadSample=true diff --git a/docker/serverAndUI/nginx/nginx.conf b/docker/serverAndUI/nginx/nginx.conf deleted file mode 100644 index 74e0ec2e61..0000000000 --- a/docker/serverAndUI/nginx/nginx.conf +++ /dev/null @@ -1,20 +0,0 @@ -server { - listen 5000; - server_name conductor; - location / { - # This would be the directory where your React app's static files are stored at - root /usr/share/nginx/html; - try_files $uri /index.html; - } - - location /api { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-NginX-Proxy true; - proxy_pass http://localhost:8080/api; - proxy_ssl_session_reuse off; - proxy_set_header Host $http_host; - proxy_cache_bypass $http_upgrade; - proxy_redirect off; - } -} \ No newline at end of file