Update morpheus dev container to support flag for building with debug python build + source files.

CONTRIBUTING.md

@@ -90,6 +90,28 @@ This workflow utilizes a docker container to set up most dependencies ensuring a
       DOCKER_IMAGE_TAG=my_tag ./docker/build_container_dev.sh
       ```
       Would build the container `morpheus:my_tag`.
+   1. To build the container with a debugging version of cpython installed, update the docker target as follows:
+   ```shell
+   DOCKER_TARGET=development_pydbg ./docker/build_container_dev.sh
+   ```
+   1. Note: When debugging python code, you just need to add `ci/conda/recipes/python-dbg/source` to your debugger's 
+   source path.
+   1. Once created, you will be able to introspect python objects from within GDB. For example, if we were to break
+   within a generator setup call and examine it's PyFrame_Object `f`, it might look like this:
+   ```shell
+    #4  0x000056498ce685f4 in gen_send_ex (gen=0x7f3ecc07ad40, arg=<optimized out>, exc=<optimized out>, closing=<optimized out>) at Objects/genobject.c:222
+    (gdb) pyo f
+    object address  : 0x7f3eb3888750
+    object refcount : 1
+    object type     : 0x56498cf99c00
+    object type name: frame
+    object repr     : <frame at 0x7f3eb3888750, file '/workspace/morpheus/pipeline/pipeline.py', line 902, code join
+   ```
+   1. Note: Now when running the container, conda should list your python version as `pyxxx_dbg_morpheus`.
+   ```shell
+    (morpheus) user@host:/workspace# conda list | grep python
+    python                    3.8.13          py3.8.13_dbg_morpheus    local
+   ```
    1. Note: This does not build any Morpheus or Neo code and defers building the code until the entire repo can be mounted into a running container. This allows for faster incremental builds during development.
 2. Set up `ssh-agent` to allow container to pull from private repos
    ```bash

ci/conda/recipes/python-dbg/build.sh

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+./configure --with-pydebug --enable-shared --enable-ipv6 --prefix=$PREFIX
+make -j
+make install
+
+
+cd $PREFIX/bin
+ln -s python${PY_VER} python
+ln -s pydoc${PY_VER} pydoc

ci/conda/recipes/python-dbg/conda_build_config.yaml

@@ -0,0 +1,20 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+c_compiler_version:
+  - 9.4
+
+cxx_compiler_version:
+  - 9.4

ci/conda/recipes/python-dbg/meta.yaml

@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{% set version = environ.get('MORPHEUS_PYTHON_VER') %}
+
+package:
+  name: python
+  version: {{ version }}
+
+source:
+  fn: Python-{{ version }}.tar.xz
+  url: https://www.python.org/ftp/python/{{ version }}/Python-{{ version }}.tar.xz
+  # md5: c4b7100dcaace9d33ab1fda9a3a038d6
+  # If you want to build from the github source. This is quite a bit slower than pulling the tarball.
+  #git_url: https://github.com/python/cpython.git
+  #git_rev: 3.8
+
+build:
+  include_recipe: False
+  no_link: bin/python
+  number: 0
+  string: py{{ version }}_dbg_morpheus
+
+requirements:
+  build:
+    - bzip2       [unix]
+    - libffi      [unix] # Required for _ctypes, pip will complain that setuptools doesn't exist if _ctypes is missing.
+    - openssl >=1.1.1n,<2.0   # Required for TLS/SSL capabilities in pip.
+    - readline    [unix]
+    - sqlite      [unix]
+    - tk          [unix]
+    - xz          [unix]
+    - zlib        [unix]
+  run:
+    - bzip2       [unix]
+    - libffi      [unix]
+    - openssl >=1.1.1n,<2.0   # Required for TLS/SSL capabilities in pip.
+    - readline    [unix]
+    - sqlite      [unix]
+    - tk          [unix]
+    - xz          [unix]
+    - zlib        [unix]
+
+test:
+  commands:
+    - python -V               [unix]
+    - python3 -V              [unix]
+    - 2to3 -h
+    - pydoc -h
+    - python3-config --help   [unix]
+
+about:
+  home: http://www.python.org/
+  license: PSF
+  summary: general purpose programming language

ci/conda/recipes/python_dbg_install.sh

@@ -0,0 +1,118 @@
+# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+function usage {
+  echo "$0 -s [CPYTHON_SOURCE] -p [CONDA_INSTALL_FILE_PATH] -i [SOURCE_INSTALL_PATH] -g [GDBINIT_INSTALL_PATH]"
+
+  exit 0
+}
+
+function help {
+echo <<-DOC
+Usage: $0 [OPTION]...
+Install debug version of cpython, into the current conda environment, which has previously been downloaded/built.
+
+Arguments:
+-s            Path to cpython (s)ource tarball, source files will be extracted and installed to path specified by '-i'.
+-c            Path to cpython (c)onda tarball, this is what will be installed into the conda environment.
+-i            Path where cpython source files will be installed.
+              Default: ci/conda/recipes/python-dbg/source
+-g            Path to install point for cpython 'gdbinit' file. Ignored if empty.
+              Note: Requires cpython source install point '-i' to be specified so that we can locate the gdb macros.
+DOC
+
+exit 0
+}
+
+function set_py_ver {
+    ## Get the version information
+    PYDEBUG_VERSION=$(python --version | cut -d ' ' -f2 )
+    PYDEBUG_VERSION_MAJ_MIN=$(echo "${PYDEBUG_VERSION}" | awk '{split($0,ver,"."); print ver[1] "." ver[2]}')
+}
+
+PYDEBUG_CONDA_INSTALL_FILE=""
+PYDEBUG_INSTALL_GDB_PATH=""
+PYDEBUG_INSTALL_GDB_PATH=""
+PYDEBUG_INSTALL_PATH=${PWD}/ci/conda/recipes/python-dbg/source
+PYDEBUG_VERSION=""
+PYDEBUG_VERSION_MAJ_MIN=""
+
+while getopts "s:c:i:gh" opt; do
+  case ${opt} in
+    s) PYDEBUG_SOURCE=${OPTARG};;
+    c) PYDEBUG_CONDA_INSTALL_FILE=${OPTARG};;
+    i) PYDEBUG_INSTALL_PATH=${OPTARG};;
+    g) PYDEBUG_INSTALL_GDB_PATH=${OPTARG};;
+    h) help;;
+    *) usage;;
+  esac
+done
+
+# Install conda package
+if [ -n "${PYDEBUG_CONDA_INSTALL_FILE}" ]; then
+  if [ ! -f "${PYDEBUG_CONDA_INSTALL_FILE}" ]; then
+    echo "Conda install file does not exist or is inaccessible: ${PYDEBUG_CONDA_INSTALL_FILE}"
+    exit 1
+  else
+    echo "Installing cpython debug build: ${PYDEBUG_CONDA_INSTALL_FILE}"
+    mamba install --use-local "${PYDEBUG_CONDA_INSTALL_FILE}"
+
+    set_py_ver
+    ## Conda package will install python source to python3.xd (for development), which the CMake configure files won't find
+    ## Copy the includes to the python3.x folder so CMake can find them.
+    cp -R /opt/conda/envs/morpheus/include/python${PYDEBUG_VERSION_MAJ_MIN}d/* \
+          /opt/conda/envs/morpheus/include/python${PYDEBUG_VERSION_MAJ_MIN}
+  fi
+else
+    echo "No Conda install file specified, skipping..."
+    set_py_ver
+fi
+
+# Install cpython source files
+if [[ -n ${PYDEBUG_SOURCE} && -n ${PYDEBUG_INSTALL_PATH} ]]; then
+  if [[ ! -f ${PYDEBUG_SOURCE} ]]; then
+    echo "Cpython source file does not exist or is inaccessible: ${PYDEBUG_SOURCE}"
+    exit 1
+  fi
+
+  if [[ ! -f ${PYDEBUG_INSTALL_PATH} ]]; then
+    mkdir -p "${PYDEBUG_INSTALL_PATH}"
+  fi
+
+  # Extract cpython source to /workspace \
+  for src_dir in Include Misc Modules Objects Python; do
+      tar --strip-components=1 --extract --wildcards --file="${PYDEBUG_SOURCE}" Python-${PYDEBUG_VERSION}/${src_dir}/*
+      mv ./${src_dir} "${PYDEBUG_INSTALL_PATH}/${src_dir}"
+  done
+else
+    echo "Missing cpython source tarball or install path, skipping..."
+fi
+
+# Install GDB init macros
+if [[ -f ${PYDEBUG_INSTALL_PATH} ]]; then
+  # Move cpython gdb helper macros to ${HOME}/.gdbinit \
+  # See: https://github.com/python/cpython/blob/main/Misc/gdbinit
+  if [[ "${PYDEBUG_INSTALL_GDB_PATH}" != "" ]]; then
+    GDB_SRC_PATH="${PYDEBUG_INSTALL_PATH}/Misc/gdbinit"
+    if [[ ! -f "${GDB_SRC_PATH}" ]]; then
+      echo "gdbinit path does not exist or is inaccessible: ${GDB_SRC_PATH}"
+      exit 1
+    fi
+
+    cp "${GDB_SRC_PATH}" "${PYDEBUG_INSTALL_GDB_PATH}"
+  fi
+fi

ci/conda/recipes/run_conda_build.sh

@@ -140,3 +140,12 @@ if hasArg morpheus; then
    conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ci/conda/recipes/morpheus
    set +x
 fi
+
+if hasArg pydebug; then
+  export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2)
+
+  echo "Running conda-build for python-dbg..."
+  set -x
+  conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ./ci/conda/recipes/python-dbg
+  set +x
+fi

docker/Dockerfile

@@ -83,7 +83,7 @@ RUN --mount=type=ssh \
     source activate base &&\
     # Run with --no-test for now until we can build with builtkit and default nvidia runtime
     # Temp add CONDA_CHANNEL_ALIAS to get around conda-build 404 errors
-    MORPHEUS_ROOT=/workspace CONDA_BLD_DIR=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh libcudf cudf
+    MORPHEUS_ROOT=/workspace CONDA_BLD_PATH=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh libcudf cudf
 
 # ============ Stage: conda_env ============
 # Create the conda environment and install all dependencies
@@ -146,7 +146,7 @@ RUN --mount=type=ssh \
     --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \
     source activate base &&\
     # Temp add CONDA_CHANNEL_ALIAS to get around conda-build 404 errors
-    MORPHEUS_ROOT=/workspace CONDA_BLD_DIR=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh morpheus
+    MORPHEUS_ROOT=/workspace CONDA_BLD_PATH=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh morpheus
 
 # ============ Stage: runtime ============
 # Setup container for runtime environment
@@ -181,7 +181,6 @@ CMD [ "morpheus" ]
 # ============ Stage: development ============
 # Install and configure development only packages
 FROM conda_env_dev as development
-
 # Copy the source
 # COPY . ./
 
@@ -192,3 +191,26 @@ RUN npm install -g camouflage-server
 # greater. See https://marc.info/?l=git&m=164989570902912&w=2. Only enable for
 # development
 RUN git config --global --add safe.directory "*"
+
+# ============ Stage: python_debug_bld ===========
+# Configure and build cpython with debugging symbols
+FROM development as development_pydbg
+
+COPY ci/conda/recipes/python-dbg/ ./ci/conda/recipes/python-dbg
+COPY ci/conda/recipes/run_conda_build.sh ./ci/conda/recipes/run_conda_build.sh
+COPY ci/conda/recipes/python_dbg_install.sh ./ci/conda/recipes/python_dbg_install.sh
+
+# Temporary until #68 goes in
+ARG MORPHEUS_USER="root"
+
+# Build and install debug cpython
+RUN source activate morpheus \
+    && MORPHEUS_ROOT=/workspace \
+       CONDA_BLD_PATH=/opt/conda/conda-bld \
+       CONDA_ARGS="--no-test" \
+       ./ci/conda/recipes/run_conda_build.sh pydebug \
+    && ./ci/conda/recipes/python_dbg_install.sh \
+       -s $( ls /opt/conda/conda-bld/src_cache/Python-${PYTHON_VER}*.tar.xz ) \
+       -c $( ls /opt/conda/conda-bld/linux-64/python-${PYTHON_VER}*.tar.bz2 ) \
+       -i ./ci/conda/recipes/python-dbg/source \
+       -g $(eval echo "~${MORPHEUS_USER}")/.gdbinit