From 11e51ebf98279c95e1d4e790327bbd46bee38cf4 Mon Sep 17 00:00:00 2001 From: Eli Fajardo Date: Thu, 7 Mar 2024 11:33:09 -0500 Subject: [PATCH 1/2] fix paths to environment yaml --- conda/environments/examples_cuda-121_arch-x86_64.yaml | 1 + docker/Dockerfile | 1 + docs/README.md | 2 +- docs/source/developer_guide/contributing.md | 2 +- examples/digital_fingerprinting/production/Dockerfile | 5 +---- .../production/morpheus/benchmarks/README.md | 2 +- examples/gnn_fraud_detection_pipeline/README.md | 2 +- examples/llm/agents/README.md | 2 +- examples/llm/completion/README.md | 2 +- .../training-tuning-scripts/fraud-detection-models/README.md | 2 +- morpheus/llm/services/nemo_llm_service.py | 2 +- morpheus/llm/services/openai_chat_service.py | 2 +- tests/benchmarks/README.md | 2 +- tests/conftest.py | 4 ++-- tests/examples/llm/common/conftest.py | 2 +- tests/stages/arxiv/conftest.py | 2 +- 16 files changed, 17 insertions(+), 18 deletions(-) diff --git a/conda/environments/examples_cuda-121_arch-x86_64.yaml b/conda/environments/examples_cuda-121_arch-x86_64.yaml index ba479b45f9..6199cf5583 100644 --- a/conda/environments/examples_cuda-121_arch-x86_64.yaml +++ b/conda/environments/examples_cuda-121_arch-x86_64.yaml @@ -68,4 +68,5 @@ dependencies: - milvus==2.3.5 - nemollm - pymilvus==2.3.6 + - python-logging-loki name: examples_cuda-121_arch-x86_64 diff --git a/docker/Dockerfile b/docker/Dockerfile index db93d08486..e70732770d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -233,6 +233,7 @@ ARG CUDA_MAJOR_VER ARG CUDA_MINOR_VER COPY ${MORPHEUS_ROOT_HOST}/conda/environments/runtime_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml ./conda/environments/runtime.yaml +COPY ${MORPHEUS_ROOT_HOST}/conda/environments/examples_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml ./conda/environments/examples.yaml # Mount Morpheus conda package build in `conda_bld_morpheus` RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target=/opt/conda/conda-bld \ diff --git a/docs/README.md b/docs/README.md index c6ad45283b..28d0d2d0b4 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,7 +22,7 @@ Additional packages required for building the documentation are defined in `./co ## Install Additional Dependencies From the root of the Morpheus repo: ```bash -conda env update --solver=libmamba -n morpheus --file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune +conda env update --solver=libmamba -n morpheus --file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune ``` ## Build Morpheus and Documentation diff --git a/docs/source/developer_guide/contributing.md b/docs/source/developer_guide/contributing.md index 7b38f322f6..7064574855 100644 --- a/docs/source/developer_guide/contributing.md +++ b/docs/source/developer_guide/contributing.md @@ -205,7 +205,7 @@ git submodule update --init --recursive 1. Create the Morpheus Conda environment ```bash - conda env create --solver=libmamba -n morpheus --file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml + conda env create --solver=libmamba -n morpheus --file conda/environments/dev_cuda-121_arch-x86_64.yaml conda activate morpheus ``` diff --git a/examples/digital_fingerprinting/production/Dockerfile b/examples/digital_fingerprinting/production/Dockerfile index d5464c8b32..962154ad53 100644 --- a/examples/digital_fingerprinting/production/Dockerfile +++ b/examples/digital_fingerprinting/production/Dockerfile @@ -26,12 +26,9 @@ SHELL ["/bin/bash", "-c"] # Set the workdir to the DFP base folder WORKDIR /workspace/examples/digital_fingerprinting/ -# Copy the conda_env file -COPY ./conda_env.yml ./ - # Install DFP dependencies RUN source activate morpheus \ - && mamba env update -n morpheus --file ../../conda/environments/examples_cuda-121_arch-x86_64.yaml + && mamba env update -n morpheus --file /workspace/conda/environments/examples.yaml # Set the tracking URI for mlflow ENV MLFLOW_TRACKING_URI="http://mlflow:5000" diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md b/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md index 3e093266b3..a9c09197d2 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md @@ -47,7 +47,7 @@ Install additonal required dependencies: ```bash mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./morpheus/conda/environments/examples_cuda-121_arch-x86_64.yaml + --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml ``` diff --git a/examples/gnn_fraud_detection_pipeline/README.md b/examples/gnn_fraud_detection_pipeline/README.md index 94b919069d..715aaf4b80 100644 --- a/examples/gnn_fraud_detection_pipeline/README.md +++ b/examples/gnn_fraud_detection_pipeline/README.md @@ -24,7 +24,7 @@ Prior to running the GNN fraud detection pipeline, additional requirements must export CUDA_VER=11.8 mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./morpheus/conda/environments/examples_cuda-121_arch-x86_64.yaml + --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml ``` ## Running diff --git a/examples/llm/agents/README.md b/examples/llm/agents/README.md index cd437f7711..00c84baee9 100644 --- a/examples/llm/agents/README.md +++ b/examples/llm/agents/README.md @@ -97,7 +97,7 @@ Install the required dependencies. ```bash mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./morpheus/conda/environments/examples_cuda-121_arch-x86_64.yaml + --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml ``` diff --git a/examples/llm/completion/README.md b/examples/llm/completion/README.md index 729537286e..e2d2d7091f 100644 --- a/examples/llm/completion/README.md +++ b/examples/llm/completion/README.md @@ -68,7 +68,7 @@ Install the required dependencies. ```bash mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./morpheus/conda/environments/examples_cuda-121_arch-x86_64.yaml + --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml ``` diff --git a/models/training-tuning-scripts/fraud-detection-models/README.md b/models/training-tuning-scripts/fraud-detection-models/README.md index 10ee520d09..14e4b32084 100644 --- a/models/training-tuning-scripts/fraud-detection-models/README.md +++ b/models/training-tuning-scripts/fraud-detection-models/README.md @@ -26,7 +26,7 @@ Install packages for training GNN model. ```bash mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./morpheus/conda/environments/model-utils-121_arch-x86_64.yaml + --file ./conda/environments/model-utils-121_arch-x86_64.yaml ``` ### Options for training and tuning models. diff --git a/morpheus/llm/services/nemo_llm_service.py b/morpheus/llm/services/nemo_llm_service.py index 0173354df9..97324bb9b4 100644 --- a/morpheus/llm/services/nemo_llm_service.py +++ b/morpheus/llm/services/nemo_llm_service.py @@ -26,7 +26,7 @@ IMPORT_ERROR_MESSAGE = ( "NemoLLM not found. Install it and other additional dependencies by running the following command:\n" "`conda env update --solver=libmamba -n morpheus " - "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") + "--file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") try: import nemollm diff --git a/morpheus/llm/services/openai_chat_service.py b/morpheus/llm/services/openai_chat_service.py index 446b9a0ee9..34e6462222 100644 --- a/morpheus/llm/services/openai_chat_service.py +++ b/morpheus/llm/services/openai_chat_service.py @@ -31,7 +31,7 @@ IMPORT_ERROR_MESSAGE = ("OpenAIChatService & OpenAIChatClient require the openai package to be installed. " "Install it by running the following command:\n" "`conda env update --solver=libmamba -n morpheus " - "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") + "--file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") try: import openai diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md index b33b2f995e..905e6e30b5 100644 --- a/tests/benchmarks/README.md +++ b/tests/benchmarks/README.md @@ -200,5 +200,5 @@ You can use the same Dev container created here to run the Production DFP benchm ```bash mamba env update \ -n ${CONDA_DEFAULT_ENV} \ - --file ./morpheus/conda/environments/examples_cuda-121_arch-x86_64.yaml + --file ./conda/environments/examples_cuda-121_arch-x86_64.yaml ``` diff --git a/tests/conftest.py b/tests/conftest.py index c422c18ea9..1c4fd74c94 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1033,7 +1033,7 @@ def nemollm_fixture(fail_missing: bool): """ skip_reason = ("Tests for the NeMoLLMService require the nemollm package to be installed, to install this run:\n" "`conda env update --solver=libmamba -n morpheus " - "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") + "--file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") yield import_or_skip("nemollm", reason=skip_reason, fail_missing=fail_missing) @@ -1044,7 +1044,7 @@ def openai_fixture(fail_missing: bool): """ skip_reason = ("Tests for the OpenAIChatService require the openai package to be installed, to install this run:\n" "`conda env update --solver=libmamba -n morpheus " - "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") + "--file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") yield import_or_skip("openai", reason=skip_reason, fail_missing=fail_missing) diff --git a/tests/examples/llm/common/conftest.py b/tests/examples/llm/common/conftest.py index c0bd0ad9ba..259b535304 100644 --- a/tests/examples/llm/common/conftest.py +++ b/tests/examples/llm/common/conftest.py @@ -60,5 +60,5 @@ def langchain_fixture(fail_missing: bool): skip_reason = ("Tests for the WebScraperStage require the langchain package to be installed, to install this run:\n" "`conda env update --solver=libmamba -n morpheus " - "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") + "--file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") yield import_or_skip("langchain", reason=skip_reason, fail_missing=fail_missing) diff --git a/tests/stages/arxiv/conftest.py b/tests/stages/arxiv/conftest.py index fb1fdad7be..0865faada0 100644 --- a/tests/stages/arxiv/conftest.py +++ b/tests/stages/arxiv/conftest.py @@ -23,7 +23,7 @@ SKIP_REASON = ("Tests for the arxiv_source require a number of packages not installed in the Morpheus development " "environment. To install these run:\n" "`conda env update --solver=libmamba -n morpheus " - "--file morpheus/conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") + "--file conda/environments/dev_cuda-121_arch-x86_64.yaml --prune`") @pytest.fixture(name="arxiv", autouse=True, scope='session') From c9edbfe9e3cd04dc5d4fa6653ea78d3b472f87e4 Mon Sep 17 00:00:00 2001 From: Eli Fajardo Date: Thu, 14 Mar 2024 13:21:20 -0400 Subject: [PATCH 2/2] dockerfile now copies all env yamls to runtime image --- docker/Dockerfile | 22 +++++++++---------- .../production/Dockerfile | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e70732770d..19c0cc00e7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -232,8 +232,15 @@ ARG MORPHEUS_ROOT_HOST ARG CUDA_MAJOR_VER ARG CUDA_MINOR_VER -COPY ${MORPHEUS_ROOT_HOST}/conda/environments/runtime_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml ./conda/environments/runtime.yaml -COPY ${MORPHEUS_ROOT_HOST}/conda/environments/examples_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml ./conda/environments/examples.yaml +# Only copy specific files/folders over that are necessary for runtime +COPY "${MORPHEUS_ROOT_HOST}/conda/environments/*.yaml" "./conda/environments/" +COPY "${MORPHEUS_ROOT_HOST}/docker" "./docker" +COPY "${MORPHEUS_ROOT_HOST}/docs" "./docs" +COPY "${MORPHEUS_ROOT_HOST}/examples" "./examples" +COPY "${MORPHEUS_ROOT_HOST}/models" "./models" +COPY "${MORPHEUS_ROOT_HOST}/scripts" "./scripts" +COPY "${MORPHEUS_ROOT_HOST}/*.md" "./" +COPY "${MORPHEUS_ROOT_HOST}/LICENSE" "./" # Mount Morpheus conda package build in `conda_bld_morpheus` RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target=/opt/conda/conda-bld \ @@ -255,18 +262,9 @@ RUN --mount=type=bind,from=conda_bld_morpheus,source=/opt/conda/conda-bld,target -c defaults \ morpheus &&\ - /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file conda/environments/runtime.yaml && \ + /opt/conda/bin/conda env update --solver=libmamba -n morpheus --file conda/environments/runtime_cuda-${CUDA_MAJOR_VER}${CUDA_MINOR_VER}_arch-x86_64.yaml && \ conda clean -afy -# Only copy specific files/folders over that are necessary for runtime -COPY "${MORPHEUS_ROOT_HOST}/docker" "./docker" -COPY "${MORPHEUS_ROOT_HOST}/docs" "./docs" -COPY "${MORPHEUS_ROOT_HOST}/examples" "./examples" -COPY "${MORPHEUS_ROOT_HOST}/models" "./models" -COPY "${MORPHEUS_ROOT_HOST}/scripts" "./scripts" -COPY "${MORPHEUS_ROOT_HOST}/*.md" "./" -COPY "${MORPHEUS_ROOT_HOST}/LICENSE" "./" - # Use morpheus by default CMD [ "morpheus" ] diff --git a/examples/digital_fingerprinting/production/Dockerfile b/examples/digital_fingerprinting/production/Dockerfile index 962154ad53..4472bce5d5 100644 --- a/examples/digital_fingerprinting/production/Dockerfile +++ b/examples/digital_fingerprinting/production/Dockerfile @@ -28,7 +28,7 @@ WORKDIR /workspace/examples/digital_fingerprinting/ # Install DFP dependencies RUN source activate morpheus \ - && mamba env update -n morpheus --file /workspace/conda/environments/examples.yaml + && mamba env update -n morpheus --file /workspace/conda/environments/examples_cuda-121_arch-x86_64.yaml # Set the tracking URI for mlflow ENV MLFLOW_TRACKING_URI="http://mlflow:5000"