diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 665c54d161..559b7f0a8a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -90,6 +90,28 @@ This workflow utilizes a docker container to set up most dependencies ensuring a DOCKER_IMAGE_TAG=my_tag ./docker/build_container_dev.sh ``` Would build the container `morpheus:my_tag`. + 1. To build the container with a debugging version of cpython installed, update the docker target as follows: + ```shell + DOCKER_TARGET=development_pydbg ./docker/build_container_dev.sh + ``` + 1. Note: When debugging python code, you just need to add `ci/conda/recipes/python-dbg/source` to your debugger's + source path. + 1. Once created, you will be able to introspect python objects from within GDB. For example, if we were to break + within a generator setup call and examine it's PyFrame_Object `f`, it might look like this: + ```shell + #4 0x000056498ce685f4 in gen_send_ex (gen=0x7f3ecc07ad40, arg=, exc=, closing=) at Objects/genobject.c:222 + (gdb) pyo f + object address : 0x7f3eb3888750 + object refcount : 1 + object type : 0x56498cf99c00 + object type name: frame + object repr : =1.1.1n,<2.0 # Required for TLS/SSL capabilities in pip. + - readline [unix] + - sqlite [unix] + - tk [unix] + - xz [unix] + - zlib [unix] + run: + - bzip2 [unix] + - libffi [unix] + - openssl >=1.1.1n,<2.0 # Required for TLS/SSL capabilities in pip. + - readline [unix] + - sqlite [unix] + - tk [unix] + - xz [unix] + - zlib [unix] + +test: + commands: + - python -V [unix] + - python3 -V [unix] + - 2to3 -h + - pydoc -h + - python3-config --help [unix] + +about: + home: http://www.python.org/ + license: PSF + summary: general purpose programming language diff --git a/ci/conda/recipes/python_dbg_install.sh b/ci/conda/recipes/python_dbg_install.sh new file mode 100755 index 0000000000..46fe26ce3c --- /dev/null +++ b/ci/conda/recipes/python_dbg_install.sh @@ -0,0 +1,118 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +function usage { + echo "$0 -s [CPYTHON_SOURCE] -p [CONDA_INSTALL_FILE_PATH] -i [SOURCE_INSTALL_PATH] -g [GDBINIT_INSTALL_PATH]" + + exit 0 +} + +function help { +echo <<-DOC +Usage: $0 [OPTION]... +Install debug version of cpython, into the current conda environment, which has previously been downloaded/built. + +Arguments: +-s Path to cpython (s)ource tarball, source files will be extracted and installed to path specified by '-i'. +-c Path to cpython (c)onda tarball, this is what will be installed into the conda environment. +-i Path where cpython source files will be installed. + Default: ci/conda/recipes/python-dbg/source +-g Path to install point for cpython 'gdbinit' file. Ignored if empty. + Note: Requires cpython source install point '-i' to be specified so that we can locate the gdb macros. +DOC + +exit 0 +} + +function set_py_ver { + ## Get the version information + PYDEBUG_VERSION=$(python --version | cut -d ' ' -f2 ) + PYDEBUG_VERSION_MAJ_MIN=$(echo "${PYDEBUG_VERSION}" | awk '{split($0,ver,"."); print ver[1] "." ver[2]}') +} + +PYDEBUG_CONDA_INSTALL_FILE="" +PYDEBUG_INSTALL_GDB_PATH="" +PYDEBUG_INSTALL_GDB_PATH="" +PYDEBUG_INSTALL_PATH=${PWD}/ci/conda/recipes/python-dbg/source +PYDEBUG_VERSION="" +PYDEBUG_VERSION_MAJ_MIN="" + +while getopts "s:c:i:gh" opt; do + case ${opt} in + s) PYDEBUG_SOURCE=${OPTARG};; + c) PYDEBUG_CONDA_INSTALL_FILE=${OPTARG};; + i) PYDEBUG_INSTALL_PATH=${OPTARG};; + g) PYDEBUG_INSTALL_GDB_PATH=${OPTARG};; + h) help;; + *) usage;; + esac +done + +# Install conda package +if [ -n "${PYDEBUG_CONDA_INSTALL_FILE}" ]; then + if [ ! -f "${PYDEBUG_CONDA_INSTALL_FILE}" ]; then + echo "Conda install file does not exist or is inaccessible: ${PYDEBUG_CONDA_INSTALL_FILE}" + exit 1 + else + echo "Installing cpython debug build: ${PYDEBUG_CONDA_INSTALL_FILE}" + mamba install --use-local "${PYDEBUG_CONDA_INSTALL_FILE}" + + set_py_ver + ## Conda package will install python source to python3.xd (for development), which the CMake configure files won't find + ## Copy the includes to the python3.x folder so CMake can find them. + cp -R /opt/conda/envs/morpheus/include/python${PYDEBUG_VERSION_MAJ_MIN}d/* \ + /opt/conda/envs/morpheus/include/python${PYDEBUG_VERSION_MAJ_MIN} + fi +else + echo "No Conda install file specified, skipping..." + set_py_ver +fi + +# Install cpython source files +if [[ -n ${PYDEBUG_SOURCE} && -n ${PYDEBUG_INSTALL_PATH} ]]; then + if [[ ! -f ${PYDEBUG_SOURCE} ]]; then + echo "Cpython source file does not exist or is inaccessible: ${PYDEBUG_SOURCE}" + exit 1 + fi + + if [[ ! -f ${PYDEBUG_INSTALL_PATH} ]]; then + mkdir -p "${PYDEBUG_INSTALL_PATH}" + fi + + # Extract cpython source to /workspace \ + for src_dir in Include Misc Modules Objects Python; do + tar --strip-components=1 --extract --wildcards --file="${PYDEBUG_SOURCE}" Python-${PYDEBUG_VERSION}/${src_dir}/* + mv ./${src_dir} "${PYDEBUG_INSTALL_PATH}/${src_dir}" + done +else + echo "Missing cpython source tarball or install path, skipping..." +fi + +# Install GDB init macros +if [[ -f ${PYDEBUG_INSTALL_PATH} ]]; then + # Move cpython gdb helper macros to ${HOME}/.gdbinit \ + # See: https://github.com/python/cpython/blob/main/Misc/gdbinit + if [[ "${PYDEBUG_INSTALL_GDB_PATH}" != "" ]]; then + GDB_SRC_PATH="${PYDEBUG_INSTALL_PATH}/Misc/gdbinit" + if [[ ! -f "${GDB_SRC_PATH}" ]]; then + echo "gdbinit path does not exist or is inaccessible: ${GDB_SRC_PATH}" + exit 1 + fi + + cp "${GDB_SRC_PATH}" "${PYDEBUG_INSTALL_GDB_PATH}" + fi +fi diff --git a/ci/conda/recipes/run_conda_build.sh b/ci/conda/recipes/run_conda_build.sh index 2d6ab7cb99..98b25d62b5 100755 --- a/ci/conda/recipes/run_conda_build.sh +++ b/ci/conda/recipes/run_conda_build.sh @@ -140,3 +140,12 @@ if hasArg morpheus; then conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ci/conda/recipes/morpheus set +x fi + +if hasArg pydebug; then + export MORPHEUS_PYTHON_VER=$(python --version | cut -d ' ' -f 2) + + echo "Running conda-build for python-dbg..." + set -x + conda ${CONDA_COMMAND} "${CONDA_ARGS_ARRAY[@]}" ${CONDA_ARGS} ./ci/conda/recipes/python-dbg + set +x +fi diff --git a/docker/Dockerfile b/docker/Dockerfile index 21c11f6aff..9ed45ec6df 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -83,7 +83,7 @@ RUN --mount=type=ssh \ source activate base &&\ # Run with --no-test for now until we can build with builtkit and default nvidia runtime # Temp add CONDA_CHANNEL_ALIAS to get around conda-build 404 errors - MORPHEUS_ROOT=/workspace CONDA_BLD_DIR=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh libcudf cudf + MORPHEUS_ROOT=/workspace CONDA_BLD_PATH=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh libcudf cudf # ============ Stage: conda_env ============ # Create the conda environment and install all dependencies @@ -146,7 +146,7 @@ RUN --mount=type=ssh \ --mount=type=cache,id=conda_pkgs,target=/opt/conda/pkgs,sharing=locked \ source activate base &&\ # Temp add CONDA_CHANNEL_ALIAS to get around conda-build 404 errors - MORPHEUS_ROOT=/workspace CONDA_BLD_DIR=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh morpheus + MORPHEUS_ROOT=/workspace CONDA_BLD_PATH=/opt/conda/conda-bld CONDA_ARGS="--no-test" ./ci/conda/recipes/run_conda_build.sh morpheus # ============ Stage: runtime ============ # Setup container for runtime environment @@ -181,7 +181,6 @@ CMD [ "morpheus" ] # ============ Stage: development ============ # Install and configure development only packages FROM conda_env_dev as development - # Copy the source # COPY . ./ @@ -192,3 +191,26 @@ RUN npm install -g camouflage-server # greater. See https://marc.info/?l=git&m=164989570902912&w=2. Only enable for # development RUN git config --global --add safe.directory "*" + +# ============ Stage: python_debug_bld =========== +# Configure and build cpython with debugging symbols +FROM development as development_pydbg + +COPY ci/conda/recipes/python-dbg/ ./ci/conda/recipes/python-dbg +COPY ci/conda/recipes/run_conda_build.sh ./ci/conda/recipes/run_conda_build.sh +COPY ci/conda/recipes/python_dbg_install.sh ./ci/conda/recipes/python_dbg_install.sh + +# Temporary until #68 goes in +ARG MORPHEUS_USER="root" + +# Build and install debug cpython +RUN source activate morpheus \ + && MORPHEUS_ROOT=/workspace \ + CONDA_BLD_PATH=/opt/conda/conda-bld \ + CONDA_ARGS="--no-test" \ + ./ci/conda/recipes/run_conda_build.sh pydebug \ + && ./ci/conda/recipes/python_dbg_install.sh \ + -s $( ls /opt/conda/conda-bld/src_cache/Python-${PYTHON_VER}*.tar.xz ) \ + -c $( ls /opt/conda/conda-bld/linux-64/python-${PYTHON_VER}*.tar.bz2 ) \ + -i ./ci/conda/recipes/python-dbg/source \ + -g $(eval echo "~${MORPHEUS_USER}")/.gdbinit