Replace usage of PreprocessLogParsingStage with PreprocessNLPStage (#842

)

* Code in `PreprocessLogParsingStage` was about 99% the same as `PreprocessNLPStage`
* The only thing `PreprocessLogParsingStage` provided was different default constructor values, along with some special handling of punctuation. However the cudf subword_tokenizer removes all punctuation.

fixes #801

Authors:
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #842

examples/log_parsing/README.md

@@ -108,14 +108,13 @@ From the root of the Morpheus repo run:
 ```bash
 PYTHONPATH="examples/log_parsing" \
 morpheus --log_level INFO \
-	--plugin "preprocessing" \
 	--plugin "inference" \
 	--plugin "postprocessing" \
 	run --num_threads 1 --use_cpp False --pipeline_batch_size 1024 --model_max_batch_size 32  \
 	pipeline-nlp \
 	from-file --filename ./models/datasets/validation-data/log-parsing-validation-data-input.csv  \
 	deserialize \
-	log-preprocess --vocab_hash_file ./models/training-tuning-scripts/sid-models/resources/bert-base-cased-hash.txt --stride 64 \
+	preprocess --vocab_hash_file ./models/training-tuning-scripts/sid-models/resources/bert-base-cased-hash.txt --stride 64 --column=raw \
 	monitor --description "Preprocessing rate" \
 	inf-logparsing --model_name log-parsing-onnx --server_url localhost:8001 --force_convert_inputs=True \
 	monitor --description "Inference rate" --unit inf \

examples/log_parsing/run.py

@@ -17,7 +17,6 @@
 import click
 from inference import LogParsingInferenceStage
 from postprocessing import LogParsingPostProcessingStage
-from preprocessing import PreprocessLogParsingStage
 
 from morpheus.config import Config
 from morpheus.config import CppConfig
@@ -27,6 +26,7 @@
 from morpheus.stages.input.file_source_stage import FileSourceStage
 from morpheus.stages.output.write_to_file_stage import WriteToFileStage
 from morpheus.stages.preprocess.deserialize_stage import DeserializeStage
+from morpheus.stages.preprocess.preprocess_nlp_stage import PreprocessNLPStage
 
 
 @click.command()
@@ -115,12 +115,13 @@ def run_pipeline(
     # Add a preprocessing stage.
     # This stage preprocess the rows in the Dataframe.
     pipeline.add_stage(
-        PreprocessLogParsingStage(config,
-                                  vocab_hash_file=model_vocab_hash_file,
-                                  truncation=False,
-                                  do_lower_case=False,
-                                  stride=64,
-                                  add_special_tokens=False))
+        PreprocessNLPStage(config,
+                           vocab_hash_file=model_vocab_hash_file,
+                           truncation=False,
+                           do_lower_case=False,
+                           stride=64,
+                           add_special_tokens=False,
+                           column="raw"))
 
     # Add a monitor stage.
     # This stage logs the metrics (msg/sec) from the above stage.

morpheus/stages/preprocess/preprocess_nlp_stage.py

@@ -18,7 +18,6 @@
 import mrc
 
 import cudf
-from cudf.core.subword_tokenizer import SubwordTokenizer
 
 import morpheus._lib.stages as _stages
 from morpheus.cli.register_stage import register_stage
@@ -94,8 +93,6 @@ def __init__(self,
         self._do_lower_case = do_lower_case
         self._add_special_tokens = add_special_tokens
 
-        self._tokenizer: SubwordTokenizer = None
-
     @property
     def name(self) -> str:
         return "preprocess-nlp"