Squashed commit of the following:

commit 7809a8a Author: Viren Baraiya <virenx@gmail.com> Date: Sun Sep 24 20:10:37 2023 -0700 update readme commit f127ac0 Author: Viren Baraiya <virenx@gmail.com> Date: Sat Sep 23 15:26:03 2023 -0700 fixes commit 39caae3 Author: Viren Baraiya <virenx@gmail.com> Date: Sat Sep 23 12:06:29 2023 -0700 Update README.md commit cc21acd Author: Viren Baraiya <virenx@gmail.com> Date: Sat Sep 23 12:05:40 2023 -0700 Update README.md commit 5739ea4 Author: Viren Baraiya <virenx@gmail.com> Date: Sat Sep 23 12:02:58 2023 -0700 fixes and readme update commit 67279e9 Author: Viren Baraiya <virenx@gmail.com> Date: Sat Sep 23 11:56:22 2023 -0700 update comments commit aae0a92 Author: Viren Baraiya <virenx@gmail.com> Date: Sat Sep 23 11:30:17 2023 -0700 Remove local commit abb08a7 Author: Viren Baraiya <virenx@gmail.com> Date: Sat Sep 9 20:55:06 2023 -0700 docker compose changes commit 22967e3 Author: Viren Baraiya <virenx@gmail.com> Date: Sat Sep 9 14:52:21 2023 -0700 build fixes commit 373f594 Author: Viren Baraiya <virenx@gmail.com> Date: Sat Sep 9 00:57:05 2023 -0700 docker changes commit 06a4076 Author: Viren Baraiya <virenx@gmail.com> Date: Wed Sep 6 21:57:12 2023 -0700 local container for conductor commit b01460b Author: Viren Baraiya <virenx@gmail.com> Date: Tue Sep 5 23:38:21 2023 -0700 removed old compose commit fd23d22 Author: Viren Baraiya <virenx@gmail.com> Date: Tue Sep 5 23:34:48 2023 -0700 restore compose files commit a616536 Merge: 3bf2cfe c56b56a Author: Manan Bhatt <manan16489@gmail.com> Date: Wed Sep 6 11:18:52 2023 +0530 Merge branch 'docker_buils' into docker_changes commit 3bf2cfe Author: Viren Baraiya <virenx@gmail.com> Date: Tue Sep 5 22:40:58 2023 -0700 readme commit 96ca951 Author: Viren Baraiya <virenx@gmail.com> Date: Tue Sep 5 22:28:37 2023 -0700 docker build with community server commit c56b56a Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Thu Aug 31 21:54:26 2023 +0530 changes commit 4a95a56 Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Wed Aug 30 00:08:45 2023 +0530 changes. commit 236fd36 Merge: 2714253 4d99ff5 Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Tue Aug 29 21:40:16 2023 +0530 Merge branch 'main' of https://github.com/Netflix/conductor into docker_buils commit 2714253 Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Tue Aug 29 21:39:15 2023 +0530 revert property changes commit 1c07b5f Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Tue Aug 29 21:37:40 2023 +0530 revert commit 99f6b7c Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Tue Aug 29 21:03:52 2023 +0530 docker builds commit 0cbec91 Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Tue Aug 29 17:51:46 2023 +0530 update doc commit 90691b3 Merge: efe072a 55ef67e Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Tue Aug 29 17:45:31 2023 +0530 Merge branch 'main' into docker_buils commit efe072a Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Tue Aug 29 14:37:31 2023 +0530 changes commit ad28adc Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Mon Aug 28 12:47:21 2023 +0530 docker build changes. commit 55ef67e Merge: 3ae3bc2 a32acec Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Wed Aug 23 14:14:46 2023 +0530 Merge branch 'main' of https://github.com/Netflix/conductor commit 3ae3bc2 Merge: e6ef05f d0a5883 Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Thu Aug 3 00:42:17 2023 +0530 Merge branch 'main' of https://github.com/Netflix/conductor commit e6ef05f Merge: 90fd3fd 878fcd9 Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Wed Jun 14 10:53:06 2023 +0530 Merge branch 'main' of https://github.com/Netflix/conductor commit 90fd3fd Merge: e4e7e0d 3d9b27f Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Wed Apr 12 18:43:53 2023 +0530 Merge remote-tracking branch 'upstream/main' commit e4e7e0d Merge: 8c65ec1 9322716 Author: manan164 <1897158+manan164@users.noreply.github.com> Date: Tue Apr 4 10:32:39 2023 +0530 Merge remote-tracking branch 'origin/loop_performance_improvement' commit 9322716 Author: Manan Bhatt <manan16489@gmail.com> Date: Wed Mar 2 11:32:29 2022 +0530 format plugin commit e1a0844 Author: Manan Bhatt <manan16489@gmail.com> Date: Wed Mar 2 11:21:40 2022 +0530 Wait and check only for current iteration task to get completed. Wait and check only for current iteration task to get completed.
v1r3n · Sep 27, 2023 · dde972c · dde972c
1 parent 5ed77fb
commit dde972c
Show file tree

Hide file tree

Showing 25 changed files with 261 additions and 524 deletions.
diff --git a/docker/README.md b/docker/README.md
@@ -1 +1,51 @@
-[Docker Instructions](/docs/docs/gettingstarted/docker.md)
+
+# Conductor Docker Builds
+
+## Pre-built docker images
+
+Conductor server with support for the following backend:
+1. Redis
+2. Postgres
+3. Mysql
+4. Cassandra
+
+### Docker File for Server and UI
+
+[Docker Image Source for Server with UI](docker/server/DockerFile)
+
+### Configuration Guide for Conductor Server
+Conductor uses a persistent store for managing state.  
+The choice of backend is quite flexible and can be configured at runtime using `conductor.db.type` property.
+
+Refer to the table below for various supported backend and required configurations to enable each of them.
+
+> [!IMPORTANT]
+> 
+> See [config.properties](docker/server/config/config.properties) for the required properties for each of the backends.
+>
+> | Backend    | Property                           |
+> |------------|------------------------------------|
+> | postgres   | conductor.db.type=postgres         |
+> | redis      | conductor.db.type=redis_standalone |
+> | mysql      | conductor.db.type=mysql            |
+> | cassandra  | conductor.db.type=cassandra        |    
+>
+
+Conductor using Elasticsearch for indexing the workflow data.  
+Currently, Elasticsearch 6 and 7 are supported.
+
+We welcome community contributions for other indexing backends.
+
+**Note:** Docker images use Elasticsearch 7.
+
+## Helm Charts
+TODO: Link to the helm charts
+
+## Run Docker Compose Locally
+### Use the docker-compose to bring up the local conductor server.
+
+| Docker Compose                                               | Description                |
+|--------------------------------------------------------------|----------------------------|
+| [docker-compose.yaml](docker-compose.yaml)                   | Redis + Elasticsearch 7    |
+| [docker-compose-postgres.yaml](docker-compose-postgres.yaml) | Postgres + Elasticsearch 7 |
+| [docker-compose-postgres.yaml](docker-compose-mysql.yaml)    | Mysql + Elasticsearch 7    |
diff --git a/docker/docker-compose-dynomite.yaml b/docker/docker-compose-dynomite.yaml
diff --git a/docker/grpc/docker-compose.yaml → docker/docker-compose-mysql.yaml b/docker/grpc/docker-compose.yaml → docker/docker-compose-mysql.yaml
@@ -4,11 +4,12 @@ services:
 
   conductor-server:
     environment:
-      - CONFIG_PROP=config-mysql-grpc.properties
+      - conductor.db.type=redis_standalone
+      - conductor.redis.hosts=rs:6379:us-east-1c
     image: conductor:server
     build:
-      context: ../../
-      dockerfile: docker/server/Dockerfile
+      context: ..
+      dockerfile: server/Dockerfile
     networks:
       - internal
     ports:
@@ -30,8 +31,8 @@ services:
       - WF_SERVER=http://conductor-server:8080
     image: conductor:ui
     build:
-      context: ../../
-      dockerfile: docker/ui/Dockerfile
+      context: ..
+      dockerfile: ui/Dockerfile
     networks:
       - internal
     ports:

diff --git a/docker/docker-compose-postgres.yaml b/docker/docker-compose-postgres.yaml
@@ -13,29 +13,27 @@ services:
       - internal
     ports:
       - 8080:8080
+      - 5000:5000
     healthcheck:
       test: [ "CMD", "curl","-I" ,"-XGET", "http://localhost:8080/health" ]
       interval: 60s
       timeout: 30s
       retries: 12
     links:
-      - elasticsearch:es
-      - redis:rs
-      - postgres:postgresdb
+      - conductor-elasticsearch:es
+      - conductor-postgres:postgresdb
     depends_on:
-      elasticsearch:
+      conductor-elasticsearch:
         condition: service_healthy
-      redis:
-        condition: service_healthy
-      postgres:
+      conductor-postgres:
         condition: service_healthy
     logging:
       driver: "json-file"
       options:
         max-size: "1k"
         max-file: "3"
 
-  postgres:
+  conductor-postgres:
     image: postgres
     environment:
       - POSTGRES_USER=conductor
@@ -45,7 +43,7 @@ services:
     networks:
       - internal
     ports:
-      - 5432:5432
+      - 6432:5432
     healthcheck:
       test: timeout 5 bash -c 'cat < /dev/null > /dev/tcp/localhost/5432'
       interval: 5s
@@ -57,32 +55,20 @@ services:
         max-size: "1k"
         max-file: "3"
 
-  redis:
-    image: redis:6.2.3-alpine
-    volumes:
-      - ./redis.conf:/usr/local/etc/redis/redis.conf
-    networks:
-      - internal
-    ports:
-      - 6379:6379
-    healthcheck:
-      test: [ "CMD", "redis-cli","ping" ]
-
-  elasticsearch:
-    image: elasticsearch:6.8.15
-    container_name: elasticsearch
+  conductor-elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.11
     environment:
       - "ES_JAVA_OPTS=-Xms512m -Xmx1024m"
-      - transport.host=0.0.0.0
-      - discovery.type=single-node
       - xpack.security.enabled=false
+      - discovery.type=single-node
+    volumes:
+      - esdata-conductor:/usr/share/elasticsearch/data
     networks:
       - internal
     ports:
-      - 9200:9200
-      - 9300:9300
+      - 9201:9200
     healthcheck:
-      test: wget http://localhost:9200/ -O /dev/null
+      test: curl http://localhost:9200/_cluster/health -o /dev/null
       interval: 5s
       timeout: 5s
       retries: 12
@@ -95,6 +81,8 @@ services:
 volumes:
   pgdata-conductor:
     driver: local
+  esdata-conductor:
+    driver: local
 
 networks:
   internal:
diff --git a/docker/docker-compose-prometheus.yaml b/docker/docker-compose-prometheus.yaml
diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml
@@ -3,7 +3,7 @@ version: '2.3'
 services:
   conductor-server:
     environment:
-      - CONFIG_PROP=config-local.properties
+      - CONFIG_PROP=/app/config/config-redis.properties
     image: conductor:server
     container_name: conductor-server
     build:
@@ -13,69 +13,51 @@ services:
       - internal
     ports:
       - 8080:8080
+      - 5000:5000
     healthcheck:
       test: ["CMD", "curl","-I" ,"-XGET", "http://localhost:8080/health"]
       interval: 60s
       timeout: 30s
       retries: 12
     links:
-      - elasticsearch:es
-      - redis:rs
+      - conductor-elasticsearch:es
+      - conductor-redis:rs
     depends_on:
-      elasticsearch:
+      conductor-elasticsearch:
         condition: service_healthy
-      redis:
+      conductor-redis:
         condition: service_healthy
     logging:
       driver: "json-file"
       options:
         max-size: "1k"
         max-file: "3"
 
-  conductor-ui:
-    environment:
-      - WF_SERVER=http://conductor-server:8080
-    image: conductor:ui
-    container_name: conductor-ui
-    build:
-      context: ../
-      dockerfile: docker/ui/Dockerfile
-    networks:
-      - internal
-    ports:
-      - 5000:5000
-    links:
-      - conductor-server
-    stdin_open: true
-
-  redis:
+  conductor-redis:
     image: redis:6.2.3-alpine
     volumes:
-      - ./redis.conf:/usr/local/etc/redis/redis.conf
+      - ../server/config/redis.conf:/usr/local/etc/redis/redis.conf
     networks:
       - internal
     ports:
-      - 6379:6379
+      - 7379:6379
     healthcheck:
       test: [ "CMD", "redis-cli","ping" ]
 
-  elasticsearch:
-    image: elasticsearch:6.8.15
-    container_name: elasticsearch
+  conductor-elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.11
     environment:
       - "ES_JAVA_OPTS=-Xms512m -Xmx1024m"
-      - transport.host=0.0.0.0
-      - discovery.type=single-node
       - xpack.security.enabled=false
+      - discovery.type=single-node
     volumes:
       - esdata-conductor:/usr/share/elasticsearch/data
     networks:
       - internal
     ports:
-      - 9200:9200
-      - 9300:9300
+      - 9201:9200
     healthcheck:
-      test: wget http://localhost:9200/ -O /dev/null
+      test: curl http://localhost:9200/_cluster/health -o /dev/null
       interval: 5s
       timeout: 5s
       retries: 12

diff --git a/docker/grpc/Makefile b/docker/grpc/Makefile