-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into containerize-orchestration-neptune
- Loading branch information
Showing
19 changed files
with
714 additions
and
78 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,8 @@ | ||
[submodule "pyfluigi"] | ||
path = pyfluigi | ||
url = https://github.com/CIDARLAB/pyFluigi.git | ||
[submodule "primitives-server"] | ||
path = primitives-server | ||
url = https://github.com/cidarlab/3duf.git | ||
url = https://github.com/cidarlab/3duf | ||
branch = primitives-server | ||
[submodule "pyfluigi"] | ||
path = pyfluigi | ||
url = https://github.com/cidarlab/pyfluigi | ||
branch = master |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
from typing import List | ||
import uuid | ||
import boto3 | ||
from pprint import pprint | ||
from pathlib import Path | ||
|
||
from job_runner.setting import AWS_ENDPOINT_URL, AWS_S3_BUCKET_NAME | ||
|
||
S3_CLIENT = boto3.client( | ||
"s3", | ||
endpoint_url=AWS_ENDPOINT_URL | ||
) | ||
|
||
def upload_file_using_client(file_location: Path) -> str: | ||
""" Uploads file to S3 bucket using S3 client object | ||
Args: | ||
file_location (Path): Location of the file to upload | ||
Returns: | ||
str: S3 object name | ||
""" | ||
bucket_name = AWS_S3_BUCKET_NAME | ||
file_name = file_location.name | ||
file_path = str(file_location.absolute()) | ||
s3_object_name = f"{str(uuid.uuid4())}-{file_name}" | ||
S3_CLIENT.upload_file(file_path, bucket_name, s3_object_name) | ||
|
||
print(f"Uploaded {file_path} to {bucket_name}/{file_name}") | ||
|
||
return s3_object_name | ||
|
||
def download_file_using_client(s3_object_name: str, local_file_name: str, download_directory: Path) -> None: | ||
""" Downloads file from S3 bucket using S3 client object | ||
Args: | ||
s3_object_name (str): name of the S3 object to download | ||
local_file_name (str): name of the local file to download to | ||
download_directory (Path): location of the file to download | ||
Returns: | ||
None: None | ||
""" | ||
local_file_path_location = str(download_directory.joinpath(local_file_name).absolute()) | ||
S3_CLIENT.download_file(AWS_S3_BUCKET_NAME, s3_object_name, local_file_path_location) | ||
|
||
print(f"Downloaded {s3_object_name} from {AWS_S3_BUCKET_NAME} to {local_file_path_location}") | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
import imp | ||
import os | ||
import time | ||
import mongoengine | ||
from celery import Celery | ||
|
||
from job_runner.setting import CELERY_BROKER_URL, CELERY_BACKEND_URL, MONGO_HOST, MONGO_PORT, MONGODB_JOBS_DB, MONGODB_PASSWORD, MONGODB_USER | ||
|
||
print("Connecting to MongoDB") | ||
print("MongoDB_USER:", MONGODB_USER) | ||
print("MongoDB_PASSWORD:", MONGODB_PASSWORD) | ||
print("MongoDB_PORT:", MONGO_PORT) | ||
print("MongoDB_HOST:", MONGO_HOST) | ||
print("MongoDB_JOBS_DB:", MONGODB_JOBS_DB) | ||
connection = mongoengine.connect( | ||
db=MONGODB_JOBS_DB, | ||
username=MONGODB_USER, | ||
password=MONGODB_PASSWORD, | ||
host=MONGO_HOST, | ||
port=MONGO_PORT | ||
) | ||
|
||
# print("Databases") | ||
# print(connection.list_database_names()) | ||
|
||
print("Connecting to Celery") | ||
celery_app = Celery( | ||
"job_runner", | ||
broker=CELERY_BROKER_URL, | ||
backend=CELERY_BACKEND_URL, | ||
include=["job_runner.tasks"] | ||
) | ||
|
||
celery_app.start(["-A", "job_runner.server" ,"worker", "-E","--concurrency=1", "--loglevel=INFO"]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
import os | ||
|
||
# SOCKETIO REDIS CONFIGURATION | ||
SOCKETIO_REDIS_HOST = os.getenv('SOCKETIO_REDIS_HOST') | ||
SOCKETIO_REDIS_PORT = int(os.getenv('SOCKETIO_REDIS_PORT', 6379)) | ||
|
||
|
||
# CELERY SETTINGS | ||
CELERY_BROKER_URL = os.getenv('CELERY_BROKER_URL') | ||
CELERY_ACCEPT_CONTENT = ['json'] | ||
CELERY_TASK_SERIALIZER = 'json' | ||
CELERY_RESULT_SERIALIZER = 'json' | ||
|
||
|
||
# CELERY MONGO SETTINGS | ||
MONGODB_USER = os.getenv('MONGO_INITDB_ROOT_USERNAME') | ||
MONGODB_PASSWORD = os.getenv('MONGO_INITDB_ROOT_PASSWORD') | ||
MONGODB_JOBS_DB = os.getenv('MONGO_INITDB_NAME') | ||
MONGO_HOST = os.getenv('MONGO_HOST') | ||
MONGO_PORT = int(os.getenv('MONGO_PORT', 27017)) | ||
|
||
CELERY_BACKEND_URL = f'mongodb://{MONGODB_USER}:{MONGODB_PASSWORD}@{MONGO_HOST}:{MONGO_PORT}/{MONGODB_JOBS_DB}' | ||
|
||
|
||
|
||
# AWS S3 SETTINGS | ||
AWS_S3_BUCKET_NAME = os.getenv('AWS_S3_BUCKET_NAME') | ||
AWS_ACCESS_KEY_ID = os.getenv('AWS_ACCESS_KEY_ID') | ||
AWS_SECRET_ACCESS_KEY = os.getenv('AWS_SECRET_ACCESS_KEY') | ||
AWS_ENDPOINT_URL = os.getenv('AWS_ENDPOINT_URL') |
Oops, something went wrong.