Move data dir to models (#1099)

* Move `morpheus/data` to `models/data` * Replace `morpheus/data` with a symlink to `models/data` * Newly built CI images to fix #1100 Authors: - David Gardner (https://github.com/dagardner-nv) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) URL: #1099
nv-morpheus · Jul 25, 2023 · 96f8df0 · 96f8df0
1 parent f6ca8e1
commit 96f8df0
Show file tree

Hide file tree

Showing 24 changed files with 12 additions and 11 deletions.
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -46,7 +46,7 @@ jobs:
     uses: ./.github/workflows/ci_pipe.yml
     with:
       run_check: ${{ startsWith(github.ref_name, 'pull-request/') }}
-      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-230710
-      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-230710
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-build-230725
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:morpheus-ci-test-230725
     secrets:
       NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
diff --git a/ci/scripts/run_ci_local.sh b/ci/scripts/run_ci_local.sh
@@ -49,7 +49,7 @@ GIT_BRANCH=$(git branch --show-current)
 GIT_COMMIT=$(git log -n 1 --pretty=format:%H)
 
 LOCAL_CI_TMP=${LOCAL_CI_TMP:-${MORPHEUS_ROOT}/.tmp/local_ci_tmp}
-CONTAINER_VER=${CONTAINER_VER:-230710}
+CONTAINER_VER=${CONTAINER_VER:-230725}
 CUDA_VER=${CUDA_VER:-11.8}
 DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
 

diff --git a/morpheus/data/bert-base-cased-hash.txt → models/data/bert-base-cased-hash.txt b/morpheus/data/bert-base-cased-hash.txt → models/data/bert-base-cased-hash.txt
diff --git a/morpheus/data/bert-base-cased-vocab.txt → models/data/bert-base-cased-vocab.txt b/morpheus/data/bert-base-cased-vocab.txt → models/data/bert-base-cased-vocab.txt
diff --git a/morpheus/data/bert-base-uncased-hash.txt → models/data/bert-base-uncased-hash.txt b/morpheus/data/bert-base-uncased-hash.txt → models/data/bert-base-uncased-hash.txt
diff --git a/morpheus/data/bert-base-uncased-vocab.txt → models/data/bert-base-uncased-vocab.txt b/morpheus/data/bert-base-uncased-vocab.txt → models/data/bert-base-uncased-vocab.txt
diff --git a/morpheus/data/columns_ae.txt → models/data/columns_ae.txt b/morpheus/data/columns_ae.txt → models/data/columns_ae.txt
diff --git a/morpheus/data/columns_ae_azure.txt → models/data/columns_ae_azure.txt b/morpheus/data/columns_ae_azure.txt → models/data/columns_ae_azure.txt
diff --git a/morpheus/data/columns_ae_cloudtrail.txt → models/data/columns_ae_cloudtrail.txt b/morpheus/data/columns_ae_cloudtrail.txt → models/data/columns_ae_cloudtrail.txt
diff --git a/morpheus/data/columns_ae_duo.txt → models/data/columns_ae_duo.txt b/morpheus/data/columns_ae_duo.txt → models/data/columns_ae_duo.txt
diff --git a/morpheus/data/columns_fil.txt → models/data/columns_fil.txt b/morpheus/data/columns_fil.txt → models/data/columns_fil.txt
diff --git a/morpheus/data/labels_ae.txt → models/data/labels_ae.txt b/morpheus/data/labels_ae.txt → models/data/labels_ae.txt
diff --git a/morpheus/data/labels_nlp.txt → models/data/labels_nlp.txt b/morpheus/data/labels_nlp.txt → models/data/labels_nlp.txt
diff --git a/morpheus/data/labels_phishing.txt → models/data/labels_phishing.txt b/morpheus/data/labels_phishing.txt → models/data/labels_phishing.txt
diff --git a/morpheus/data/public_suffix_list.dat → models/data/public_suffix_list.dat b/morpheus/data/public_suffix_list.dat → models/data/public_suffix_list.dat
diff --git a/morpheus/data/splunk_notable_regex.yaml → models/data/splunk_notable_regex.yaml b/morpheus/data/splunk_notable_regex.yaml → models/data/splunk_notable_regex.yaml
diff --git a/morpheus/data/windows_event_regex.yaml → models/data/windows_event_regex.yaml b/morpheus/data/windows_event_regex.yaml → models/data/windows_event_regex.yaml
diff --git a/models/training-tuning-scripts/log-parsing-models/resources/bert-base-cased-hash.txt b/models/training-tuning-scripts/log-parsing-models/resources/bert-base-cased-hash.txt
@@ -1 +1 @@
-../../../../morpheus/data/bert-base-cased-hash.txt
+../../../data/bert-base-cased-hash.txt
diff --git a/models/training-tuning-scripts/sid-models/resources/bert-base-cased-hash.txt b/models/training-tuning-scripts/sid-models/resources/bert-base-cased-hash.txt
@@ -1 +1 @@
-../../../../morpheus/data/bert-base-cased-hash.txt
+../../../data/bert-base-cased-hash.txt
diff --git a/models/training-tuning-scripts/sid-models/resources/bert-base-cased-vocab.txt b/models/training-tuning-scripts/sid-models/resources/bert-base-cased-vocab.txt
@@ -1 +1 @@
-../../../../morpheus/data/bert-base-cased-vocab.txt
+../../../data/bert-base-cased-vocab.txt
diff --git a/models/training-tuning-scripts/sid-models/resources/bert-base-uncased-hash.txt b/models/training-tuning-scripts/sid-models/resources/bert-base-uncased-hash.txt
@@ -1 +1 @@
-../../../../morpheus/data/bert-base-uncased-hash.txt
+../../../data/bert-base-uncased-hash.txt
diff --git a/models/training-tuning-scripts/sid-models/resources/bert-base-uncased-vocab.txt b/models/training-tuning-scripts/sid-models/resources/bert-base-uncased-vocab.txt
@@ -1 +1 @@
-../../../../morpheus/data/bert-base-uncased-vocab.txt
+../../../data/bert-base-uncased-vocab.txt
diff --git a/morpheus/data b/morpheus/data
@@ -0,0 +1 @@
+../models/data
diff --git a/tests/test_cli.py b/tests/test_cli.py
@@ -715,7 +715,7 @@ def test_pipeline_nlp(self, config, callback_values):
         assert isinstance(deserialize, DeserializeStage)
 
         assert isinstance(process_nlp, PreprocessNLPStage)
-        assert process_nlp._vocab_hash_file == vocab_file_name
+        assert process_nlp._vocab_hash_file == os.path.realpath(vocab_file_name)
         assert process_nlp._truncation
         assert process_nlp._do_lower_case
         assert not process_nlp._add_special_tokens
@@ -838,7 +838,7 @@ def test_pipeline_nlp_all(self, config, callback_values, tmp_path, mlflow_uri):
         assert dropna._column == 'xyz'
 
         assert isinstance(process_nlp, PreprocessNLPStage)
-        assert process_nlp._vocab_hash_file == vocab_file_name
+        assert process_nlp._vocab_hash_file == os.path.realpath(vocab_file_name)
         assert process_nlp._truncation
         assert process_nlp._do_lower_case
         assert not process_nlp._add_special_tokens
@@ -927,7 +927,7 @@ def test_pipeline_nlp_relative_paths(self, config, callback_values):
         # pylint: disable=unused-variable
         [file_source, deserialize, process_nlp, triton_inf, monitor, add_class, validation, serialize, to_file] = stages
 
-        assert process_nlp._vocab_hash_file == vocab_file_name
+        assert process_nlp._vocab_hash_file == os.path.realpath(vocab_file_name)
 
     @pytest.mark.replace_callback('pipeline_nlp')
     def test_pipeline_nlp_relative_path_precedence(self, config, callback_values, tmp_path):