Merge branch 'master' into containerize-orchestration-neptune

.gitmodules

@@ -1,6 +1,8 @@
-[submodule "pyfluigi"]
-	path = pyfluigi
-	url = https://github.com/CIDARLAB/pyFluigi.git
 [submodule "primitives-server"]
 	path = primitives-server
-	url = https://github.com/cidarlab/3duf.git
+	url = https://github.com/cidarlab/3duf
+	branch = primitives-server
+[submodule "pyfluigi"]
+	path = pyfluigi
+	url = https://github.com/cidarlab/pyfluigi
+	branch = master

README.md

@@ -24,3 +24,14 @@ TBA Rest of the guide...
 ### Contact
 
 Reach out to [@rkrishnasanka](https://github.com/rkrishnasanka) for more details.
+
+
+### Running individual services
+
+```bash
+docker run -i containername:dev --env-file=myenvfile
+```
+
+```bash
+docker exec -ti <container name> /bin/bash
+```

docker-compose.yml

@@ -22,54 +22,82 @@ services:
         - "9000:9000"
         - "9001:9001"
       volumes:
-        - mino_data:/data/s3
+        - minio_data_container:/data/s3
       environment:
-        - MINIO_ACCESS_KEY=minio
-        - MINIO_SECRET_KEY=minio123
+        MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID}
+        MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY}
       command: server /data/s3 --console-address ":9001"
       restart: unless-stopped
+      networks:
+        - main_network
 
     broker:
       image: redis:latest
       container_name: realtime-comm-server
       ports:
         - "6379:6379"
+      restart: unless-stopped
+      networks:
+        - main_network
 
-    backend_database:
+    backend-database:
       image: mongo:latest
       environment:
-        AUTH: "yes"
-        MONGO_INITDB_ROOT_USERNAME: root
-        MONGO_INITDB_ROOT_PASSWORD: rootpassword
+        MONGO_INITDB_ROOT_USERNAME: ${MONGO_INITDB_ROOT_USERNAME}
+        MONGO_INITDB_ROOT_PASSWORD: ${MONGO_INITDB_ROOT_PASSWORD}
       ports:
         - 27017:27017
       volumes:
         - mongodb_data_container:/data/db
+      restart: unless-stopped
+      networks:
+        - main_network
 
     worker:
       build: 
         context: .
         dockerfile: ./job-runner/Dockerfile
-      container_name: demo_worker
-      command: celery --app=simple worker --loglevel=info
+      container_name: async-worker
+      command: python /runner/job_runner/server.py #celery --app=simple worker --loglevel=info
       volumes:
         - .:/usr/src
       environment:
-        - CELERY_BROKER_URL=redis://broker:6379
+        MONGO_INITDB_NAME: ${MONGO_INITDB_NAME}
+        MONGO_HOST: ${MONGO_HOST}
+        MONGO_PORT: ${MONGO_PORT}
+        MONGO_INITDB_ROOT_USERNAME: ${MONGO_INITDB_ROOT_USERNAME}
+        MONGO_INITDB_ROOT_PASSWORD: ${MONGO_INITDB_ROOT_PASSWORD}
+        CELERY_BROKER_URL: ${CELERY_BROKER_URL}
+        AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+        AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+        AWS_S3_BUCKET_NAME: ${AWS_S3_BUCKET_NAME}
+        AWS_ENDPOINT_URL: ${AWS_ENDPOINT_URL}
+        SOCKETIO_REDIS_HOST: ${SOCKETIO_REDIS_HOST}
+        SOCKETIO_REDIS_PORT: ${SOCKETIO_REDIS_PORT}
         # - CELERY_RESULT_BACKEND=db+postgresql://dbc:dbc@backend:5432/celery
       depends_on:
         - broker
-        - backend_database
+        - backend-database
         - s3local
+      restart: unless-stopped
+      networks:
+        - main_network
 
     primitives-server:
       build:
         context: ./primitives-server
         dockerfile: ./primitives-server.Dockerfile
       restart: unless-stopped
       ports:
-        - "6060:5555"
-
+        - 6060:6060
+      command: npm run start
+      networks:
+        - main_network
+
 volumes:
   mongodb_data_container:
-  mino_data:
+  minio_data_container:
+
+networks:
+  main_network:
+    driver: bridge

job-runner/Dockerfile

@@ -1,40 +1,39 @@
-FROM --platform=linux/amd64 python:3.8-buster as runner
+FROM --platform=linux/amd64 python:3.8-buster as base
 
 ENV   PYTHONFAULTHANDLER=1 \
   PYTHONUNBUFFERED=1 \
   PYTHONHASHSEED=random \
   PIP_NO_CACHE_DIR=off \
   PIP_DISABLE_PIP_VERSION_CHECK=on \
-  PIP_DEFAULT_TIMEOUT=100
+  PIP_DEFAULT_TIMEOUT=100 \
+  POETRY_VERSION=1.1.14
 
 RUN apt-get update
 RUN apt-get install -y \ 
 software-properties-common make build-essential python3-pip \
 python-dev libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
 libsqlite3-dev curl libffi-dev graphviz libgraphviz-dev \
-libcairo2-dev pkg-config python3-dev \
-graphviz libgraphviz-dev
-#python3-pygraphviz pygraphviz
+libcairo2-dev pkg-config python3-dev python3-pygraphviz apt-utils
+# pygraphviz
 
+RUN pip install "poetry==$POETRY_VERSION"
 
+# Project initialization:
+RUN poetry config virtualenvs.create false
+
 
-
-RUN pip install -U celery
+FROM base as runner
 
 WORKDIR /main
 ADD ./pyfluigi ./pyfluigi
-RUN pip install ./pyfluigi/pylfr
-RUN pip install ./pyfluigi
+WORKDIR /main/pyfluigi
+
+RUN poetry install --no-dev --no-interaction --no-ansi 
 
 WORKDIR /runner/
-# ADD ./job-runner /main/runner
-COPY ./job-runner /runner/
+COPY ./job-runner .
 RUN pip install .
 
-# RUN pip install
-
-
-
 RUN fluigi --help
 
 

job-runner/README.md

@@ -3,3 +3,12 @@
 Async Task Running infrastructure for the fluigi command line tool.
 
 
+
+# Getting Started with Dev
+
+
+## Load the Enviroment Variables
+
+```bash
+set -a; source .env; set +a
+```

job-runner/job_runner/filesystem.py

@@ -0,0 +1,48 @@
+from typing import List
+import uuid
+import boto3
+from pprint import pprint
+from pathlib import Path
+
+from job_runner.setting import AWS_ENDPOINT_URL, AWS_S3_BUCKET_NAME
+
+S3_CLIENT = boto3.client(
+        "s3",
+        endpoint_url=AWS_ENDPOINT_URL
+    )
+
+def upload_file_using_client(file_location: Path) -> str:
+    """ Uploads file to S3 bucket using S3 client object
+
+    Args:
+        file_location (Path): Location of the file to upload
+
+    Returns:
+        str: S3 object name  
+    """
+    bucket_name = AWS_S3_BUCKET_NAME
+    file_name = file_location.name
+    file_path = str(file_location.absolute())
+    s3_object_name = f"{str(uuid.uuid4())}-{file_name}"
+    S3_CLIENT.upload_file(file_path, bucket_name, s3_object_name)
+
+    print(f"Uploaded {file_path} to {bucket_name}/{file_name}")
+
+    return s3_object_name
+
+def download_file_using_client(s3_object_name: str, local_file_name: str, download_directory: Path) -> None:
+    """ Downloads file from S3 bucket using S3 client object
+
+    Args:
+        s3_object_name (str): name of the S3 object to download
+        local_file_name (str): name of the local file to download to
+        download_directory (Path): location of the file to download
+
+    Returns:
+        None: None
+    """
+    local_file_path_location = str(download_directory.joinpath(local_file_name).absolute())
+    S3_CLIENT.download_file(AWS_S3_BUCKET_NAME, s3_object_name, local_file_path_location)
+
+    print(f"Downloaded {s3_object_name} from {AWS_S3_BUCKET_NAME} to {local_file_path_location}")
+

job-runner/job_runner/server.py

@@ -0,0 +1,34 @@
+import imp
+import os
+import time
+import mongoengine
+from celery import Celery
+
+from job_runner.setting import CELERY_BROKER_URL, CELERY_BACKEND_URL, MONGO_HOST, MONGO_PORT, MONGODB_JOBS_DB, MONGODB_PASSWORD, MONGODB_USER
+
+print("Connecting to MongoDB")
+print("MongoDB_USER:", MONGODB_USER)
+print("MongoDB_PASSWORD:", MONGODB_PASSWORD)
+print("MongoDB_PORT:", MONGO_PORT)
+print("MongoDB_HOST:", MONGO_HOST)
+print("MongoDB_JOBS_DB:", MONGODB_JOBS_DB)
+connection = mongoengine.connect(
+    db=MONGODB_JOBS_DB,
+    username=MONGODB_USER,
+    password=MONGODB_PASSWORD, 
+    host=MONGO_HOST, 
+    port=MONGO_PORT
+)
+
+# print("Databases")
+# print(connection.list_database_names())
+
+print("Connecting to Celery")
+celery_app = Celery(
+    "job_runner", 
+    broker=CELERY_BROKER_URL,
+    backend=CELERY_BACKEND_URL,
+    include=["job_runner.tasks"]
+)
+
+celery_app.start(["-A", "job_runner.server" ,"worker", "-E","--concurrency=1", "--loglevel=INFO"])

job-runner/job_runner/setting.py

@@ -0,0 +1,30 @@
+import os
+
+# SOCKETIO REDIS CONFIGURATION
+SOCKETIO_REDIS_HOST = os.getenv('SOCKETIO_REDIS_HOST')
+SOCKETIO_REDIS_PORT = int(os.getenv('SOCKETIO_REDIS_PORT', 6379))
+
+
+# CELERY SETTINGS
+CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL')
+CELERY_ACCEPT_CONTENT = ['json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+
+
+# CELERY MONGO SETTINGS
+MONGODB_USER = os.getenv('MONGO_INITDB_ROOT_USERNAME')
+MONGODB_PASSWORD = os.getenv('MONGO_INITDB_ROOT_PASSWORD')
+MONGODB_JOBS_DB = os.getenv('MONGO_INITDB_NAME')
+MONGO_HOST = os.getenv('MONGO_HOST')
+MONGO_PORT = int(os.getenv('MONGO_PORT', 27017))
+
+CELERY_BACKEND_URL = f'mongodb://{MONGODB_USER}:{MONGODB_PASSWORD}@{MONGO_HOST}:{MONGO_PORT}/{MONGODB_JOBS_DB}'
+
+
+
+# AWS S3 SETTINGS
+AWS_S3_BUCKET_NAME = os.getenv('AWS_S3_BUCKET_NAME')
+AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID')
+AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY')
+AWS_ENDPOINT_URL = os.getenv('AWS_ENDPOINT_URL')