Updating tests to force .pyi files to be committed into the repo

.gitignore

@@ -1,7 +1,4 @@
-./build
-*.engine
-.Dockerfile
-
+# Put new items at the bottom of this list!!!
 
 # Created by https://www.gitignore.io/api/vim,c++,cmake,python,synology
 
@@ -181,26 +178,27 @@ tags
 
 
 # End of https://www.gitignore.io/api/vim,c++,cmake,python,synology
+
+./build
+.conda-bld
+.Dockerfile
 .tmp
-examples/data/dfp
-examples/digital_fingerprinting/**/dfp_detections_*.csv
-viz_frames*
-notebooks/output
-dask-worker-space
 *.dirlock
+*.engine
 *.lock
-triton_models/**/*.onnx
-triton_models/**/*.engine
+/build*/
+/conda-build/
+dask-worker-space
+data/*
 docs/source/_lib
 docs/source/_modules
-data/*
+examples/data/dfp
+examples/digital_fingerprinting/**/dfp_detections_*.csv
 mlruns/*
-/build*/
-/conda-build/
-.conda-bld
-
-# Ignore generated pyi files for pybind and cython modules
-morpheus/_lib/**/*.pyi
+notebooks/output
+triton_models/**/*.engine
+triton_models/**/*.onnx
+viz_frames*
 
 # Explicitly ignore .vscode/. Shared settings should go in morpheus.code-workspace
 # and user settings will go in .vscode/

ci/release/update-version.sh

@@ -67,7 +67,11 @@ function sed_runner() {
 # .gitmodules
 git submodule set-branch -b branch-${NEXT_SHORT_TAG} external/morpheus-visualizations
 git submodule set-branch -b branch-${NEXT_SHORT_TAG} morpheus_utils
-git submodule update --remote --recursive
+
+if [[ "$(git diff --name-only | grep .gitmodules)" != "" ]]; then
+   # Only update the submodules if setting the branch changed .gitmodules
+   git submodule update --remote --recursive
+fi
 
 # Root CMakeLists.txt
 sed_runner 's/'"VERSION ${CURRENT_FULL_VERSION}.*"'/'"VERSION ${NEXT_FULL_VERSION}"'/g' CMakeLists.txt

ci/scripts/common.sh

@@ -13,11 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# set -x
-
 export SCRIPT_DIR=${SCRIPT_DIR:-"$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"}
-export REPO_DIR=$(realpath ${REPO_DIR:-"${SCRIPT_DIR}/../.."})
-export PY_ROOT="${REPO_DIR}"
+
+# The root to the Morpheus repo
+export MORPHEUS_ROOT=${MORPHEUS_ROOT:-"$(realpath ${SCRIPT_DIR}/../..)"}
+
+export PY_ROOT="${MORPHEUS_ROOT}"
 export PY_CFG="${PY_ROOT}/setup.cfg"
 export PY_DIRS="${PY_ROOT} ci/scripts"
 
@@ -40,7 +41,7 @@ export SKIP_ISORT=${SKIP_ISORT:-""}
 export SKIP_YAPF=${SKIP_YAPF:-""}
 
 # Set BUILD_DIR to use a different build folder
-export BUILD_DIR=${BUILD_DIR:-"${REPO_DIR}/build"}
+export BUILD_DIR=${BUILD_DIR:-"${MORPHEUS_ROOT}/build"}
 
 # Speficy the clang-tools version to use. Default 14
 export CLANG_TOOLS_VERSION=${CLANG_TOOLS_VERSION:-14}
@@ -194,4 +195,4 @@ function cleanup {
 trap cleanup EXIT
 
 # Change directory to the repo root
-pushd "${REPO_DIR}" &> /dev/null
+pushd "${MORPHEUS_ROOT}" &> /dev/null

ci/scripts/github/test.sh

@@ -21,17 +21,46 @@ source ${WORKSPACE}/ci/scripts/github/common.sh
 
 update_conda_env
 
-download_artifact "wheel.tar.bz"
-download_artifact "cpp_tests.tar.bz"
-download_artifact "morhpeus_libs.tar.bz"
-
-tar xf "${WORKSPACE_TMP}/wheel.tar.bz"
-tar xf "${WORKSPACE_TMP}/morhpeus_libs.tar.bz"
-tar xf "${WORKSPACE_TMP}/cpp_tests.tar.bz"
-
-# Install the built Morpheus python package
-cd ${MORPHEUS_ROOT}
-pip install ${MORPHEUS_ROOT}/build/dist/*.whl
+rapids-logger "Check versions"
+python3 --version
+x86_64-conda-linux-gnu-cc --version
+x86_64-conda-linux-gnu-c++ --version
+cmake --version
+ninja --version
+sccache --version
+
+git submodule update --init --recursive
+
+CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES}"
+CMAKE_FLAGS="${CMAKE_FLAGS} -DCMAKE_BUILD_RPATH_USE_ORIGIN=ON"
+CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_STUBS=ON"
+CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_BUILD_WHEEL=OFF"
+CMAKE_FLAGS="${CMAKE_FLAGS} -DMORPHEUS_PYTHON_PERFORM_INSTALL=ON"
+if [[ "${LOCAL_CI}" == "" ]]; then
+    CMAKE_FLAGS="${CMAKE_FLAGS} -DCCACHE_PROGRAM_PATH=$(which sccache)"
+fi
+
+rapids-logger "Configuring cmake for Morpheus with ${CMAKE_FLAGS}"
+cmake -B build -G Ninja ${CMAKE_FLAGS} .
+
+rapids-logger "Building Morpheus"
+cmake --build build --parallel ${PARALLEL_LEVEL}
+
+if [[ "${LOCAL_CI}" == "" ]]; then
+    rapids-logger "sccache usage for morpheus build:"
+    sccache --show-stats
+fi
+
+rapids-logger "Checking Python stub files"
+
+# Check for git diffs which would mean the build is out of sync with the repo
+if [[ $(git status --short --untracked | grep .pyi) != "" ]]; then
+
+    echo "The Python stubs (*.pyi) are out of sync with repo. Please rerun the build locally with "
+    echo "'-DMORPHEUS_PYTHON_BUILD_STUBS=ON' and commit the stub files into the repo"
+    git status
+    exit 1
+fi
 
 CPP_TESTS=($(find ${MORPHEUS_ROOT}/build -name "*.x"))
 

ci/scripts/run_iwyu_for_ci.sh

@@ -23,7 +23,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
 # Call iwyu_tool.py and append IWYU arguments onto the end
 ${IWYU_TOOL_PY:-iwyu_tool.py} "$@" -- \
-   -Xiwyu --mapping_file=${REPO_DIR:-${SCRIPT_DIR}/../..}/ci/iwyu/mappings.imp \
+   -Xiwyu --mapping_file=${MORPHEUS_ROOT:-${SCRIPT_DIR}/../..}/ci/iwyu/mappings.imp \
    -Xiwyu --verbose=${IWYU_VERBOSITY:-1} \
    -Xiwyu --no_fwd_decls \
    -Xiwyu --quoted_includes_first \