diff --git a/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md b/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md index 52e1846b92..9b11b12200 100644 --- a/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md +++ b/docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md @@ -84,7 +84,7 @@ inference tasks against a specific set of data, and the capacity for real-time l that can be injected back into the training pipeline. The following content will track the pipeline declared in -`examples/digital_fingerprinting/production/morpheus/dfp_integrated_training_streaming_pipeline.py` +`examples/digital_fingerprinting/production/morpheus/dfp_modules_streaming_pipeline.py` ```python # Setup and command line argument parsing @@ -514,3 +514,134 @@ def write_to_file(builder: mrc.Builder): ``` For a complete reference, refer to: [Write to File](../../modules/core/write_to_file.md) + +## Running Example Modular DFP Pipelines + +The following are steps to run modular DFP pipelines with example Azure and Duo datasets. + +### System requirements +* [Docker](https://docs.docker.com/get-docker/) and [docker-compose](https://docs.docker.com/compose/) installed on the host machine​ +* Supported GPU with [nvidia-docker runtime​](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker) + +> **Note:** For GPU Requirements refer to [getting_started](../../getting_started.md#requirements) + +### Building the services +From the root of the Morpheus repo, run: +```bash +cd examples/digital_fingerprinting/production +docker compose build +``` + +> **Note:** This requires version 1.28.0 or higher of Docker Compose, and preferably v2. If you encounter an error similar to: +> +> ``` +> ERROR: The Compose file './docker-compose.yml' is invalid because: +> services.jupyter.deploy.resources.reservations value Additional properties are not allowed ('devices' was +> unexpected) +> ``` +> +> This is most likely due to using an older version of the `docker-compose` command, instead re-run the build with `docker compose`. Refer to [Migrate to Compose V2](https://docs.docker.com/compose/migrate/) for more information. + +### Downloading the example datasets +First, we will need to install `s3fs` and then run the `examples/digital_fingerprinting/fetch_example_data.py` script. This will download the example data into the `examples/data/dfp` dir. + +From the Morpheus repo, run: +```bash +pip install s3fs +python examples/digital_fingerprinting/fetch_example_data.py all +``` + +### Morpheus Pipeline +From the `examples/digital_fingerprinting/production` dir, run: +```bash +docker compose run morpheus_pipeline bash +``` +To run the DFP pipelines with the example datasets within the container, run: + +* Duo Training Pipeline + ```bash + python dfp_integrated_training_batch_pipeline.py \ + --log_level DEBUG \ + --use_cpp=true \ + --source duo \ + --start_time "2022-08-01" \ + --duration "60d" \ + --train_users generic \ + --input_file "./control_messages/duo_payload_training.json" + ``` + +* Duo Inference Pipeline + ```bash + python dfp_integrated_training_batch_pipeline.py \ + --log_level DEBUG \ + --use_cpp=true \ + --source duo \ + --start_time "2022-08-30" \ + --input_file "./control_messages/duo_payload_inference.json" + ``` + +* Duo Training + Inference Pipeline + ```bash + python dfp_integrated_training_batch_pipeline.py \ + --log_level DEBUG \ + --use_cpp=true \ + --source duo \ + --start_time "2022-08-01" \ + --duration "60d" \ + --train_users generic \ + --input_file "./control_messages/duo_payload_load_training_inference.json" + ``` + +* Azure Training Pipeline + ```bash + python dfp_integrated_training_batch_pipeline.py \ + --log_level DEBUG \ + --use_cpp=true \ + --source azure \ + --start_time "2022-08-01" \ + --duration "60d" \ + --train_users generic \ + --input_file "./control_messages/azure_payload_training.json" + ``` + +* Azure Inference Pipeline + ```bash + python dfp_integrated_training_batch_pipeline.py \ + --log_level DEBUG \ + --use_cpp=true \ + --source azure \ + --start_time "2022-08-30" \ + --input_file "./control_messages/azure_payload_inference.json" + ``` + +* Azure Training + Inference Pipeline + ```bash + python dfp_integrated_training_batch_pipeline.py \ + --log_level DEBUG \ + --use_cpp=true \ + --source azure \ + --start_time "2022-08-01" \ + --duration "60d" \ + --train_users generic \ + --input_file "./control_messages/azure_payload_load_train_inference.json" + ``` + +### Output Fields +The output files, `dfp_detectiions_duo.csv` and `dfp_detections_azure.csv`, will contain those logs from the input dataset for which an anomaly was detected; this is determined by the z-score in the `mean_abs_z` field. By default, any logs with a z-score of 2.0 or higher are considered anomalous. Refer to [`DFPPostprocessingStage`](6_digital_fingerprinting_reference.md#post-processing-stage-dfppostprocessingstage). + +Most of the fields in the output files generated by running the above examples are input fields or derived from input fields. The additional output fields are: +| Field | Type | Description | +| ----- | ---- | ----------- | +| event_time | TEXT | ISO 8601 formatted date string, the time the anomaly was detected by Morpheus | +| model_version | TEXT | Name and version of the model used to performed the inference, in the form of `:` | +| max_abs_z | FLOAT | Max z-score across all features | +| mean_abs_z | FLOAT | Average z-score across all features | + +In addition to this, for each input feature the following output fields will exist: +| Field | Type | Description | +| ----- | ---- | ----------- | +| `_loss` | FLOAT | The loss | +| `_z_loss` | FLOAT | The loss z-score | +| `_pred` | FLOAT | The predicted value | + +Refer to [DFPInferenceStage](6_digital_fingerprinting_reference.md#inference-stage-dfpinferencestage) for more on these fields. \ No newline at end of file diff --git a/examples/digital_fingerprinting/production/README.md b/examples/digital_fingerprinting/production/README.md index 4a3605fa5d..ab06fd598d 100644 --- a/examples/digital_fingerprinting/production/README.md +++ b/examples/digital_fingerprinting/production/README.md @@ -157,6 +157,12 @@ Run Azure Inference Pipeline: python dfp_azure_pipeline.py --train_users none --start_time "2022-08-30" --input_file="../../../data/dfp/azure-inference-data/*.json" ``` +##### Module-based DFP pipelines + +The commands in the previous section run stage-based example DFP pipelines. The Morpheus 23.03 release introduced a new, more flexible module-based approach to build pipelines through the use of control messages. More information about modular DFP pipelines can be found [here](../../../docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md). + +Commands to run equivalent module-based DFP pipelines can be found [here](../../../docs/source/developer_guide/guides/10_modular_pipeline_digital_fingerprinting.md#running-example-modular-dfp-pipelines). + #### Optional MLflow Service Starting either the `morpheus_pipeline` or the `jupyter` service, will start the `mlflow` service in the background. For debugging purposes it can be helpful to view the logs of the running MLflow service. diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md b/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md index 78e0dacd76..7e80c973c8 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/README.md @@ -77,7 +77,7 @@ To provide your own calibration or use other `pytest-benchmark` features with th Morpheus pipeline configurations for each workflow are managed using [pipelines_conf.json](./resource/pipelines_conf.json). For example, this is the Morpheus configuration for `dfp_modules_duo_payload_inference`: ``` "test_dfp_modules_duo_payload_inference_e2e": { - "message_path": "./resource/control_messages/duo_payload_inference.json", + "message_path": "../control_messages/duo_payload_inference.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/conftest.py b/examples/digital_fingerprinting/production/morpheus/benchmarks/conftest.py index 649c4666be..eaaadf9236 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/conftest.py +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/conftest.py @@ -28,7 +28,7 @@ def get_json_lines_count(filename): return len(lines) -def pytest_benchmark_update_json(_, __, output_json): +def pytest_benchmark_update_json(config, benchmarks, output_json): # pylint:disable=unused-argument curr_dir = path.dirname(path.abspath(__file__)) diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/pipelines_conf.json b/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/pipelines_conf.json index ae9f88d399..a15edde34f 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/pipelines_conf.json +++ b/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/pipelines_conf.json @@ -1,7 +1,7 @@ { "tracking_uri": "http://localhost:5000", "test_dfp_modules_azure_payload_inference_e2e": { - "message_path": "./resource/control_messages/azure_payload_inference.json", + "message_path": "../control_messages/azure_payload_inference.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -13,7 +13,7 @@ "use_cpp": true }, "test_dfp_modules_azure_payload_lti_e2e": { - "message_path": "./resource/control_messages/azure_payload_lti.json", + "message_path": "../control_messages/azure_payload_lti.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -25,7 +25,7 @@ "use_cpp": true }, "test_dfp_modules_azure_payload_lti_s3_e2e": { - "message_path": "./resource/control_messages/azure_payload_lti_s3.json", + "message_path": "../control_messages/azure_payload_lti_s3.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -37,7 +37,7 @@ "use_cpp": true }, "test_dfp_modules_azure_payload_training_e2e": { - "message_path": "./resource/control_messages/azure_payload_training.json", + "message_path": "../control_messages/azure_payload_training.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -49,7 +49,7 @@ "use_cpp": true }, "test_dfp_modules_azure_streaming_inference_e2e": { - "message_path": "./resource/control_messages/azure_streaming_inference.json", + "message_path": "../control_messages/azure_streaming_inference.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -61,7 +61,7 @@ "use_cpp": true }, "test_dfp_modules_azure_streaming_lti_e2e": { - "message_path": "./resource/control_messages/azure_streaming_lti.json", + "message_path": "../control_messages/azure_streaming_lti.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -73,7 +73,7 @@ "use_cpp": true }, "test_dfp_modules_azure_streaming_training_e2e": { - "message_path": "./resource/control_messages/azure_streaming_training.json", + "message_path": "../control_messages/azure_streaming_training.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -85,7 +85,7 @@ "use_cpp": true }, "test_dfp_modules_duo_payload_inference_e2e": { - "message_path": "./resource/control_messages/duo_payload_inference.json", + "message_path": "../control_messages/duo_payload_inference.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -97,7 +97,7 @@ "use_cpp": true }, "test_dfp_modules_duo_payload_lti_e2e": { - "message_path": "./resource/control_messages/duo_payload_lti.json", + "message_path": "../control_messages/duo_payload_lti.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -109,7 +109,7 @@ "use_cpp": true }, "test_dfp_modules_duo_payload_only_load_e2e": { - "message_path": "./resource/control_messages/duo_payload_only_load.json", + "message_path": "../control_messages/duo_payload_only_load.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -121,7 +121,7 @@ "use_cpp": true }, "test_dfp_modules_duo_payload_training_e2e": { - "message_path": "./resource/control_messages/duo_payload_training.json", + "message_path": "../control_messages/duo_payload_training.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -133,7 +133,7 @@ "use_cpp": true }, "test_dfp_modules_duo_streaming_inference_e2e": { - "message_path": "./resource/control_messages/duo_streaming_inference.json", + "message_path": "../control_messages/duo_streaming_inference.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -145,7 +145,7 @@ "use_cpp": true }, "test_dfp_modules_duo_streaming_lti_e2e": { - "message_path": "./resource/control_messages/duo_streaming_lti.json", + "message_path": "../control_messages/duo_streaming_lti.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -157,7 +157,7 @@ "use_cpp": true }, "test_dfp_modules_duo_streaming_only_load_e2e": { - "message_path": "./resource/control_messages/duo_streaming_only_load.json", + "message_path": "../control_messages/duo_streaming_only_load.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -169,7 +169,7 @@ "use_cpp": true }, "test_dfp_modules_duo_streaming_payload_e2e": { - "message_path": "./resource/control_messages/duo_streaming_payload.json", + "message_path": "../control_messages/duo_streaming_payload.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, @@ -181,7 +181,7 @@ "use_cpp": true }, "test_dfp_modules_duo_streaming_training_e2e": { - "message_path": "./resource/control_messages/duo_streaming_training.json", + "message_path": "../control_messages/duo_streaming_training.json", "num_threads": 12, "pipeline_batch_size": 256, "edge_buffer_size": 128, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_inference.json b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_inference.json similarity index 79% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_inference.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_inference.json index 1a2048e37d..a258cf3fd5 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_inference.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_inference.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/azure-inference-data/*.json" + "../../../../examples/data/dfp/azure-inference-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_load_train_inference.json b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_load_train_inference.json similarity index 79% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_load_train_inference.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_load_train_inference.json index 4c6da153c1..0286129151 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_load_train_inference.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_load_train_inference.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/azure-training-data/*.json" + "../../../../examples/data/dfp/azure-training-data/*.json" ] } }, @@ -28,7 +28,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/azure-inference-data/*.json" + "../../../../examples/data/dfp/azure-inference-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_load_training.json b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_load_training.json similarity index 79% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_load_training.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_load_training.json index ad682ffbea..d6e028d4eb 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_load_training.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_load_training.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/azure-training-data/*.json" + "../../../../examples/data/dfp/azure-training-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_lti.json b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_lti.json similarity index 87% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_lti.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_lti.json index 363ce70963..97053f82a4 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_lti.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_lti.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/azure-training-data/*.json" + "../../../../examples/data/dfp/azure-training-data/*.json" ] } }, @@ -34,7 +34,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/azure-inference-data/*.json" + "../../../../examples/data/dfp/azure-inference-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_lti_s3.json b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_lti_s3.json similarity index 100% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_lti_s3.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_lti_s3.json diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_training.json b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_training.json similarity index 79% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_training.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_training.json index ad682ffbea..d6e028d4eb 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_payload_training.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_payload_training.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/azure-training-data/*.json" + "../../../../examples/data/dfp/azure-training-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_streaming_inference.json b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_streaming_inference.json similarity index 79% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_streaming_inference.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/azure_streaming_inference.json index f78740c918..d122241e8c 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_streaming_inference.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_streaming_inference.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/azure-inference-data/*.json" + "../../../../examples/data/dfp/azure-inference-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_streaming_lti.json b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_streaming_lti.json similarity index 87% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_streaming_lti.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/azure_streaming_lti.json index 03451ab996..f798a1a475 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_streaming_lti.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_streaming_lti.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/azure-training-data/*.json" + "../../../../examples/data/dfp/azure-training-data/*.json" ] } }, @@ -34,7 +34,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/azure-inference-data/*.json" + "../../../../examples/data/dfp/azure-inference-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_streaming_training.json b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_streaming_training.json similarity index 80% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_streaming_training.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/azure_streaming_training.json index 935f950cfb..8397489cb2 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/azure_streaming_training.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/azure_streaming_training.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/azure-training-data/*.json" + "../../../../examples/data/dfp/azure-training-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_inference.json b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_inference.json similarity index 80% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_inference.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_inference.json index 4e2d5a3893..7b4bb9672a 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_inference.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_inference.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/duo-inference-data/*.json" + "../../../../examples/data/dfp/duo-inference-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_load_train_inference.json b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_load_train_inference.json new file mode 100644 index 0000000000..e378d82760 --- /dev/null +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_load_train_inference.json @@ -0,0 +1,46 @@ +{ + "inputs": [ + { + "tasks": [ + { + "type": "load", + "properties": { + "loader_id": "fsspec", + "files": [ + "../../../../examples/data/dfp/duo-training-data/*.json" + ] + } + }, + { + "type": "training", + "properties": { + } + } + ], + "metadata": { + "data_type": "payload" + } + }, + { + "tasks": [ + { + "type": "load", + "properties": { + "loader_id": "fsspec", + "files": [ + "../../../../examples/data/dfp/duo-inference-data/*.json" + ] + } + }, + { + "type": "inference", + "properties": { + } + } + ], + "metadata": { + "data_type": "payload" + } + } + ] + } diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_lti.json b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_lti.json similarity index 87% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_lti.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_lti.json index 2a851c3e9d..07c69233ba 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_lti.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_lti.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/duo-training-data/*.json" + "../../../../examples/data/dfp/duo-training-data/*.json" ] } }, @@ -34,7 +34,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/duo-inference-data/*.json" + "../../../../examples/data/dfp/duo-inference-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_only_load.json b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_only_load.json similarity index 75% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_only_load.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_only_load.json index 6233282d61..3a214ea810 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_only_load.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_only_load.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/duo-inference-data/*.json" + "../../../../examples/data/dfp/duo-inference-data/*.json" ] } } diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_training.json b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_training.json similarity index 83% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_training.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_training.json index e331b33f98..df21751d36 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_payload_training.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_payload_training.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/duo-training-data/*.json" + "../../../../examples/data/dfp/duo-training-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_inference.json b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_inference.json similarity index 80% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_inference.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_inference.json index d0e82f31c0..fc1db57669 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_inference.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_inference.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/duo-inference-data/*.json" + "../../../../examples/data/dfp/duo-inference-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_lti.json b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_lti.json similarity index 100% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_lti.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_lti.json diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_only_load.json b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_only_load.json similarity index 75% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_only_load.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_only_load.json index 2d0ff5b186..28b2a2f7f1 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_only_load.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_only_load.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/duo-inference-data/*.json" + "../../../../examples/data/dfp/duo-inference-data/*.json" ] } } diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_payload.json b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_payload.json similarity index 79% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_payload.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_payload.json index 38ca26d73f..ab5dadf0e5 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_payload.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_payload.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/duo-training-data/*.json" + "../../../../examples/data/dfp/duo-training-data/*.json" ] } }, @@ -28,7 +28,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/duo-inference-data/*.json" + "../../../../examples/data/dfp/duo-inference-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_training.json b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_training.json similarity index 83% rename from examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_training.json rename to examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_training.json index 844b3c9d86..73cc9046d9 100644 --- a/examples/digital_fingerprinting/production/morpheus/benchmarks/resource/control_messages/duo_streaming_training.json +++ b/examples/digital_fingerprinting/production/morpheus/control_messages/duo_streaming_training.json @@ -7,7 +7,7 @@ "properties": { "loader_id": "fsspec", "files": [ - "../../../../../examples/data/dfp/duo-training-data/*.json" + "../../../../examples/data/dfp/duo-training-data/*.json" ] } }, diff --git a/examples/digital_fingerprinting/production/morpheus/dfp/utils/dfp_arg_parser.py b/examples/digital_fingerprinting/production/morpheus/dfp/utils/dfp_arg_parser.py index dffe16e33c..6bf71a0a3d 100644 --- a/examples/digital_fingerprinting/production/morpheus/dfp/utils/dfp_arg_parser.py +++ b/examples/digital_fingerprinting/production/morpheus/dfp/utils/dfp_arg_parser.py @@ -68,7 +68,11 @@ def __init__(self, self._model_name_formatter = f"DFP-{source}-" + "{user_id}" self._experiment_name_formatter = f"dfp/{source}/training/" + "{reg_model_name}" - def verify_init(self, func): + @staticmethod + def verify_init(func): + """ + Decorator function that verifies DFPArgParser instance has been initialized. + """ def wrapper(self, *args, **kwargs): if not self._initialized: @@ -91,7 +95,6 @@ def time_fields(self): def silence_monitors(self): return self._silence_monitors - @property @verify_init def include_generic(self): return self._include_generic diff --git a/tests/test_cli.py b/tests/test_cli.py index 6bc50ff35d..a2379fe074 100755 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -995,7 +995,7 @@ def test_pipeline_fil_relative_path_precedence(self, config: Config, tmp_path: s f.writelines("\n".join(test_labels)) # Use different labels - test_columns = [f"column{i}" for i in range(29)] + test_columns = [f"column{i}" for i in range(18)] # Overwrite the copied labels with open(columns_file_local, mode="w", encoding='UTF-8') as f: