GPUNetIO Integration

.clang-tidy

@@ -91,7 +91,7 @@ CheckOptions:
   - key:             readability-identifier-naming.GetConfigPerFile
     value:           'true'
   - key:             readability-identifier-naming.GlobalConstantCase
-    value:           CamelCase
+    value:           UPPER_CASE
   - key:             readability-identifier-naming.GlobalConstantIgnoredRegexp
     value:           ''
   - key:             readability-identifier-naming.GlobalConstantPrefix

.devcontainer/docker-compose.yml

@@ -21,14 +21,14 @@ services:
     container_name: morpheus-triton
     runtime: nvidia
     image: nvcr.io/nvidia/tritonserver:22.10-py3
-    command: tritonserver --model-repository=/models --exit-on-error=false ${TRITON_MODEL_ARGS}
+    command: tritonserver --model-repository=/models/triton-model-repo --exit-on-error=false ${TRITON_MODEL_ARGS}
     ports:
       - 8000:8000
       - 8001:8001
       - 8002:8002
     volumes:
-      - ${HOST_MORPHEUS_ROOT}/models/triton-model-repo:/models
-      - ${HOST_MORPHEUS_ROOT}/examples/abp_pcap_detection/abp-pcap-xgb:/models/abp-pcap-xgb
+      - ${HOST_MORPHEUS_ROOT}/models:/models
+      - ${HOST_MORPHEUS_ROOT}/examples/abp_pcap_detection/abp-pcap-xgb:/models/triton-model-repo/abp-pcap-xgb
 
   zookeeper:
     image: bitnami/zookeeper:latest

.devcontainer/opt/morpheus/bin/post-attach-command.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.devcontainer/opt/morpheus/bin/update-content-command.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

CMakeLists.txt

@@ -30,6 +30,7 @@ option(MORPHEUS_PYTHON_BUILD_STUBS "Whether or not to generated .pyi stub files
 option(MORPHEUS_PYTHON_BUILD_WHEEL "Whether or not to build the morpheus .whl file" OFF)
 option(MORPHEUS_PYTHON_INPLACE_BUILD "Whether or not to copy built python modules back to the source tree for debug purposes." OFF)
 option(MORPHEUS_PYTHON_PERFORM_INSTALL "Whether or not to automatically `pip install` any built python library. WARNING: This may overwrite any existing installation of the same name." OFF)
+option(MORPHEUS_SUPPORT_DOCA "Whether or not to build doca-related elements of morpheus" OFF)
 option(MORPHEUS_USE_CCACHE "Enable caching compilation results with ccache" OFF)
 option(MORPHEUS_USE_CLANG_TIDY "Enable running clang-tidy as part of the build process" OFF)
 option(MORPHEUS_USE_CONDA "Enables finding dependencies via conda instead of vcpkg. Note: This will disable vcpkg. All dependencies must be installed first in the conda environment" OFF)
@@ -44,11 +45,13 @@ mark_as_advanced(MORPHEUS_CACHE_DIR)
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS ON)
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 set(CMAKE_INSTALL_RPATH "$ORIGIN")
 
+# Disable compile commands until after dependencies
+set(CMAKE_EXPORT_COMPILE_COMMANDS OFF)
+
 enable_testing()
 
 if (MORPHEUS_USE_IWYU AND MORPHEUS_USE_CCACHE)
@@ -57,14 +60,15 @@ endif()
 
 # MRC CMake path and module extensions
 set(MORPHEUS_CMAKE_MODULE_PATH_EXTENSIONS
-    "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
-    "${CMAKE_CURRENT_SOURCE_DIR}/external/utilities/cmake"
-    "${CMAKE_CURRENT_SOURCE_DIR}/external/utilities/cmake/morpheus_utils/package_search"
-    )
+  "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
+  "${CMAKE_CURRENT_SOURCE_DIR}/cmake/package_search"
+  "${CMAKE_CURRENT_SOURCE_DIR}/external/utilities/cmake"
+  "${CMAKE_CURRENT_SOURCE_DIR}/external/utilities/cmake/morpheus_utils/package_search"
+)
 
 set(MORPHEUS_CMAKE_PREFIX_PATH_EXTENSIONS
-    "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
-    )
+  "${CMAKE_CURRENT_SOURCE_DIR}/cmake"
+)
 
 # Prepend path and prefix updates so they take priority in this scope.
 list(PREPEND CMAKE_MODULE_PATH "${MORPHEUS_CMAKE_MODULE_PATH_EXTENSIONS}")
@@ -127,6 +131,9 @@ add_custom_target(${PROJECT_NAME}_style_checks
 ##### Morpheus Python Setup ######
 ##################################
 
+# Re-enable compile commands
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
 morpheus_utils_python_configure()
 
 # Include the main morpheus code
@@ -167,14 +174,14 @@ if(MORPHEUS_ENABLE_DEBUG_INFO)
   morpheus_utils_print_all_targets()
 
   morpheus_utils_print_target_properties(
-      TARGETS
-        morpheus
-        morpheus_utils
-      WRITE_TO_FILE
+    TARGETS
+      morpheus
+      morpheus_utils
+    WRITE_TO_FILE
   )
 
   morpheus_utils_print_global_properties(
-      WRITE_TO_FILE
+    WRITE_TO_FILE
   )
 endif()
 

ci/scripts/cpp_checks.sh

@@ -80,9 +80,20 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then
 
    # Include What You Use
    if [[ "${SKIP_IWYU}" == "" ]]; then
-      IWYU_DIRS="morpheus"
+      # Remove .h, .hpp, and .cu files from the modified list
+      shopt -s extglob
+      IWYU_MODIFIED_FILES=( "${MORPHEUS_MODIFIED_FILES[@]/*.@(h|hpp|cu)/}" )
+
+      # Get the list of compiled files relative to this directory
+      WORKING_PREFIX="${PWD}/"
+      COMPILED_FILES=( $(jq -r .[].file ${BUILD_DIR}/compile_commands.json | sort -u ) )
+      COMPILED_FILES=( "${COMPILED_FILES[@]/#$WORKING_PREFIX/}" )
+
+      # Find the intersection between compiled files and modified files
+      IWYU_MODIFIED_FILES=( $(printf '%s\0' "${COMPILED_FILES[@]} ${IWYU_MODIFIED_FILES[@]}" | sort -z | uniq -d -z | xargs -0n1) )
+
       NUM_PROC=$(get_num_proc)
-      IWYU_OUTPUT=`${IWYU_TOOL} -p ${BUILD_DIR} -j ${NUM_PROC} ${IWYU_DIRS} 2>&1`
+      IWYU_OUTPUT=`${IWYU_TOOL} -p ${BUILD_DIR} -j ${NUM_PROC} ${IWYU_MODIFIED_FILES[@]} 2>&1`
       IWYU_RETVAL=$?
    fi
 else

cmake/dependencies.cmake

@@ -21,6 +21,9 @@ if (VERBOSE)
   morpheus_utils_print_config()
 endif()
 
+# First, load the package_config functions
+include(${CMAKE_CURRENT_LIST_DIR}/package_config/register_api.cmake)
+
 # Load direct physical package dependencies first, so we fail early. Add all dependencies to our export set
 rapids_find_package(Protobuf
   REQUIRED

cmake/package_config/bsd/Configure_bsd.cmake

@@ -0,0 +1,117 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include_guard()
+
+function(morpheus_configure_libbsd)
+  list(APPEND CMAKE_MESSAGE_CONTEXT "libbsd")
+
+  set(MORPHEUS_LIBBSD_VERSION "0.11.7" CACHE STRING "Version of libbsd to build")
+
+  # Check if bsd is available -- download if not
+  rapids_cpm_find(bsd ${MORPHEUS_LIBBSD_VERSION}
+    GLOBAL_TARGETS bsd
+    CPM_ARGS
+      GIT_REPOSITORY          https://gitlab.freedesktop.org/libbsd/libbsd
+      GIT_TAG                 ${MORPHEUS_LIBBSD_VERSION}
+      DOWNLOAD_ONLY           TRUE
+  )
+
+  if (bsd_ADDED)
+    message(STATUS "libbsd was not installed and will be built from source")
+
+    find_package(bsd REQUIRED)
+
+    set(bsd_INSTALL_DIR ${bsd_BINARY_DIR}/install)
+    file(MAKE_DIRECTORY ${bsd_INSTALL_DIR}/include)
+    include(ExternalProject)
+
+    string(TOUPPER ${CMAKE_BUILD_TYPE} BUILD_TYPE_UC)
+
+    # Get libmd components
+    get_target_property(MD_INCLUDE_DIRS md::md INTERFACE_INCLUDE_DIRECTORIES) # Add '-I' to CPPFLAGS
+    get_target_property(MD_LIBRARY md::md IMPORTED_LOCATION) # Add '-L' to LDFLAGS
+
+    cmake_path(GET MD_LIBRARY PARENT_PATH MD_LINK_DIRECTORY)
+
+    message(STATUS "MD_LIBRARY: ${MD_LINK_DIRECTORY}")
+
+    # Get the Compiler settings to forward onto autoconf
+    set(COMPILER_SETTINGS
+      "CXX=${CMAKE_CXX_COMPILER_LAUNCHER} ${CMAKE_CXX_COMPILER}"
+      "CPP=${CMAKE_CXX_COMPILER_LAUNCHER} ${CMAKE_C_COMPILER} -E"
+      "CC=${CMAKE_C_COMPILER_LAUNCHER} ${CMAKE_C_COMPILER}"
+      "AR=${CMAKE_C_COMPILER_AR}"
+      "RANLIB=${CMAKE_C_COMPILER_RANLIB}"
+      "NM=${CMAKE_NM}"
+      "STRIP=${CMAKE_STRIP}"
+      "CFLAGS=${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE_UC}}"
+      "CPPFLAGS=${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BUILD_TYPE_UC}} -I${MD_INCLUDE_DIRS}"
+      "CXXFLAGS=${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BUILD_TYPE_UC}}"
+      "LDFLAGS=${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS_${BUILD_TYPE_UC}} -L${MD_LINK_DIRECTORY}"
+    )
+
+    ExternalProject_Add(bsd
+      PREFIX              ${bsd_BINARY_DIR} # Root directory for bsd
+      SOURCE_DIR          ${bsd_BINARY_DIR} # Move source over from cpm dir and build in binary dir
+      INSTALL_DIR         ${bsd_INSTALL_DIR}
+
+      DOWNLOAD_COMMAND    ${CMAKE_COMMAND} -E copy_directory ${bsd_SOURCE_DIR} ${bsd_BINARY_DIR}
+      # Note, we set SED and GREP here since they can be hard coded in the conda libtoolize
+      CONFIGURE_COMMAND   ${CMAKE_COMMAND} -E env SED=sed GREP=grep <SOURCE_DIR>/autogen
+      COMMAND   <SOURCE_DIR>/configure ${COMPILER_SETTINGS} --prefix=${CMAKE_INSTALL_PREFIX} --enable-static
+
+      BUILD_COMMAND       make -j
+      BUILD_IN_SOURCE     TRUE
+      BUILD_BYPRODUCTS    <INSTALL_DIR>/lib/libbsd.a
+      INSTALL_COMMAND     make install prefix=<INSTALL_DIR>
+      LOG_CONFIGURE       TRUE
+      LOG_BUILD           TRUE
+      LOG_INSTALL         TRUE
+      # Add a target for configuring to allow for style checks on source code
+      STEP_TARGETS        install
+      DEPENDS             md::md
+    )
+
+    # Install only the headers
+    install(
+      DIRECTORY ${md_INSTALL_DIR}/include
+      TYPE INCLUDE
+    )
+
+    add_library(bsd::bsd STATIC IMPORTED GLOBAL)
+    set_target_properties(bsd::bsd
+      PROPERTIES
+        INTERFACE_INCLUDE_DIRECTORIES
+          "$<BUILD_INTERFACE:${bsd_INSTALL_DIR}/include>;$<INSTALL_INTERFACE:include>"
+        INTERFACE_LINK_LIBRARIES
+          "md::md"
+        INTERFACE_POSITION_INDEPENDENT_CODE
+          "ON"
+        IMPORTED_LOCATION
+          "${bsd_INSTALL_DIR}/lib/libbsd.a"
+        IMPORTED_SONAME
+          "libbsd.a"
+    )
+
+    add_dependencies(bsd::bsd bsd)
+
+    message(STATUS "bsd_INSTALL_DIR: ${bsd_INSTALL_DIR}")
+
+  endif()
+
+  LIST(POP_BACK CMAKE_MESSAGE_CONTEXT)
+
+endfunction()