From 865c112d0eb237fc2157bfc9255954724d4edbab Mon Sep 17 00:00:00 2001 From: Jeff Cheng <83052155+jeffreyc-splunk@users.noreply.github.com> Date: Tue, 29 Aug 2023 11:43:01 -0400 Subject: [PATCH] Include splunk-otel-js in auto instrumentation deb/rpm (#3540) * Add systemd sample file to splunk-otel-auto-instrumention * Update tests for new libsplunk.so * Include splunk-otel-js in auto instrumentation deb/rpm * Include default /etc/splunk/zeroconfig_node.conf in deb/rpm packages --- .gitlab-ci.yml | 10 ++++ .gitlab/update-nodejs-agent.sh | 55 +++++++++++++++++++ instrumentation/packaging/fpm/common.sh | 41 +++++++++++--- instrumentation/packaging/fpm/deb/build.sh | 8 +-- .../systemd/00-splunk-otel-javaagent.conf | 11 ++-- .../examples/systemd/00-splunk-otel-js.conf | 43 +++++++++++++++ instrumentation/packaging/fpm/rpm/build.sh | 8 +-- .../packaging/fpm/zeroconfig_node.conf | 1 + .../packaging/nodejs-agent-release.txt | 1 + .../instrumentation/instrumentation_test.py | 26 ++++++--- 10 files changed, 171 insertions(+), 33 deletions(-) create mode 100755 .gitlab/update-nodejs-agent.sh create mode 100644 instrumentation/packaging/fpm/examples/systemd/00-splunk-otel-js.conf create mode 100644 instrumentation/packaging/fpm/zeroconfig_node.conf create mode 100644 instrumentation/packaging/nodejs-agent-release.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 23daac4805..279617d428 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -157,6 +157,16 @@ update-javaagent: - .gitlab/install-gh-cli.sh - .gitlab/update-javaagent.sh +update-nodejs-agent: + only: + - schedules + stage: update-deps + needs: [] + dependencies: [] + script: + - .gitlab/install-gh-cli.sh + - .gitlab/update-nodejs-agent.sh + tidy-dependabot-pr: rules: - if: $CI_COMMIT_BRANCH =~ /^dependabot\/go_modules\/.*/ && $CI_COMMIT_AUTHOR =~ /^dependabot.*/ diff --git a/.gitlab/update-nodejs-agent.sh b/.gitlab/update-nodejs-agent.sh new file mode 100755 index 0000000000..4d44fbbc2e --- /dev/null +++ b/.gitlab/update-nodejs-agent.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +# NOTE: this script is meant to be run on the GitLab CI, it depends on GitLab CI variables +# Based on https://github.com/signalfx/splunk-otel-java/blob/c9134906c84e9a32a974dec4b380453fe1757410/scripts/propagate-version.sh + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# shellcheck source-path=SCRIPTDIR +source "${SCRIPT_DIR}/common.sh" + +ROOT_DIR="${SCRIPT_DIR}/../" +cd "${ROOT_DIR}" + +create_collector_pr() { + local repo="signalfx/splunk-otel-collector" + local repo_url="https://srv-gh-o11y-gdi:${GITHUB_TOKEN}@github.com/${repo}.git" + local branch="create-pull-request/update-nodejs-agent" + local message="Update splunk-otel-js to latest" + + echo ">>> Cloning the $repo repository ..." + git clone "$repo_url" collector-mirror + cd collector-mirror + + setup_branch "$branch" "$repo_url" + + echo ">>> Getting latest splunk-otel-js release ..." + tag="$( gh release view --repo "https://github.com/signalfx/splunk-otel-js" --json tagName --jq 'select(.isDraft|not and .isPrelease|not) | .tagName' )" + if [[ -n "$tag" ]]; then + echo ">>> Updating splunk-otel-js version to $tag ..." + echo "$tag" > instrumentation/packaging/nodejs-agent-release.txt + else + echo "ERROR: Failed to get latest release tag from https://github.com/signalfx/splunk-otel-js !" >&2 + exit 1 + fi + + # Only create the PR if there are changes + if ! git diff --exit-code >/dev/null 2>&1; then + git commit -S -am "$message" + git push -f "$repo_url" "$branch" + echo ">>> Creating the PR ..." + gh pr create \ + --draft \ + --repo "$repo" \ + --title "$message" \ + --body "$message" \ + --base main \ + --head "$branch" + fi +} + +setup_gpg +import_gpg_secret_key "$GITHUB_BOT_GPG_KEY" +setup_git +create_collector_pr diff --git a/instrumentation/packaging/fpm/common.sh b/instrumentation/packaging/fpm/common.sh index 68f984246f..04f864f70f 100644 --- a/instrumentation/packaging/fpm/common.sh +++ b/instrumentation/packaging/fpm/common.sh @@ -29,11 +29,18 @@ LIBSPLUNK_INSTALL_PATH="${INSTALL_DIR}/libsplunk.so" JAVA_AGENT_INSTALL_PATH="${INSTALL_DIR}/splunk-otel-javaagent.jar" JAVA_CONFIG_REPO_PATH="${FPM_DIR}/zeroconfig_java.conf" JAVA_CONFIG_INSTALL_PATH="/etc/splunk/zeroconfig_java.conf" +NODE_CONFIG_REPO_PATH="${FPM_DIR}/zeroconfig_node.conf" +NODE_CONFIG_INSTALL_PATH="/etc/splunk/zeroconfig_node.conf" EXAMPLES_INSTALL_DIR="${INSTALL_DIR}/examples" EXAMPLES_DIR="${FPM_DIR}/examples" JAVA_AGENT_RELEASE_PATH="${FPM_DIR}/../java-agent-release.txt" JAVA_AGENT_RELEASE_URL="https://github.com/signalfx/splunk-otel-java/releases/" +JAVA_AGENT_INSTALL_PATH="${INSTALL_DIR}/splunk-otel-javaagent.jar" + +NODEJS_AGENT_RELEASE_PATH="${FPM_DIR}/../nodejs-agent-release.txt" +NODEJS_AGENT_RELEASE_URL="https://github.com/signalfx/splunk-otel-js/releases/" +NODEJS_AGENT_INSTALL_PATH="${INSTALL_DIR}/splunk-otel-js.tgz" PREUNINSTALL_PATH="$FPM_DIR/preuninstall.sh" @@ -66,28 +73,44 @@ download_java_agent() { curl -sfL "$dl_url" -o "$dest" } +download_nodejs_agent() { + local tag="$1" + local dest="$2" + local dl_url="$NODEJS_AGENT_RELEASE_URL/download/$tag/splunk-otel-${tag#v}.tgz" + + echo "Downloading $dl_url ..." + mkdir -p "$( dirname $dest )" + curl -sfL "$dl_url" -o "$dest" +} + setup_files_and_permissions() { - local libsplunk="$1" - local java_agent="$2" - local buildroot="$3" + local arch="$1" + local buildroot="$2" + local libsplunk="$REPO_DIR/instrumentation/dist/libsplunk_${arch}.so" + local java_agent_release="$(cat "$JAVA_AGENT_RELEASE_PATH")" + local nodejs_agent_release="$(cat "$NODEJS_AGENT_RELEASE_PATH")" mkdir -p "$buildroot/$(dirname $LIBSPLUNK_INSTALL_PATH)" cp -f "$libsplunk" "$buildroot/$LIBSPLUNK_INSTALL_PATH" - sudo chown root:root "$buildroot/$LIBSPLUNK_INSTALL_PATH" sudo chmod 755 "$buildroot/$LIBSPLUNK_INSTALL_PATH" - mkdir -p "$buildroot/$(dirname $JAVA_AGENT_INSTALL_PATH)" - cp -f "$java_agent" "$buildroot/$JAVA_AGENT_INSTALL_PATH" - sudo chown root:root "$buildroot/$JAVA_AGENT_INSTALL_PATH" + download_java_agent "$java_agent_release" "${buildroot}/${JAVA_AGENT_INSTALL_PATH}" sudo chmod 755 "$buildroot/$JAVA_AGENT_INSTALL_PATH" + download_nodejs_agent "$nodejs_agent_release" "${buildroot}/${NODEJS_AGENT_INSTALL_PATH}" + sudo chmod 644 "$buildroot/$NODEJS_AGENT_INSTALL_PATH" + mkdir -p "$buildroot/$(dirname $JAVA_CONFIG_INSTALL_PATH)" cp -f "$JAVA_CONFIG_REPO_PATH" "$buildroot/$JAVA_CONFIG_INSTALL_PATH" - sudo chown root:root "$buildroot/$JAVA_CONFIG_INSTALL_PATH" sudo chmod 644 "$buildroot/$JAVA_CONFIG_INSTALL_PATH" + mkdir -p "$buildroot/$(dirname $NODE_CONFIG_INSTALL_PATH)" + cp -f "$NODE_CONFIG_REPO_PATH" "$buildroot/$NODE_CONFIG_INSTALL_PATH" + sudo chmod 644 "$buildroot/$NODE_CONFIG_INSTALL_PATH" + mkdir -p "$buildroot/$INSTALL_DIR" cp -rf "$EXAMPLES_DIR" "$buildroot/$INSTALL_DIR/" - sudo chown -R root:root "$buildroot/$EXAMPLES_INSTALL_DIR" sudo chmod -R 644 "$buildroot/$EXAMPLES_INSTALL_DIR" + + sudo chown -R root:root "$buildroot" } diff --git a/instrumentation/packaging/fpm/deb/build.sh b/instrumentation/packaging/fpm/deb/build.sh index c994e6074a..53d6f3a399 100755 --- a/instrumentation/packaging/fpm/deb/build.sh +++ b/instrumentation/packaging/fpm/deb/build.sh @@ -22,20 +22,15 @@ SCRIPT_DIR="$( cd "$( dirname ${BASH_SOURCE[0]} )" && pwd )" VERSION="${1:-}" ARCH="${2:-amd64}" OUTPUT_DIR="${3:-$REPO_DIR/instrumentation/dist}" -LIBSPLUNK_PATH="$REPO_DIR/instrumentation/dist/libsplunk_${ARCH}.so" -JAVA_AGENT_PATH="$REPO_DIR/instrumentation/dist/splunk-otel-javaagent.jar" -JAVA_AGENT_RELEASE="$(cat $JAVA_AGENT_RELEASE_PATH)" if [[ -z "$VERSION" ]]; then VERSION="$( get_version )" fi VERSION="${VERSION#v}" -download_java_agent "$JAVA_AGENT_RELEASE" "$JAVA_AGENT_PATH" - buildroot="$(mktemp -d)" -setup_files_and_permissions "$LIBSPLUNK_PATH" "$JAVA_AGENT_PATH" "$buildroot" +setup_files_and_permissions "$ARCH" "$buildroot" mkdir -p "$OUTPUT_DIR" @@ -53,6 +48,7 @@ sudo fpm -s dir -t deb -n "$PKG_NAME" -v "$VERSION" -f -p "$OUTPUT_DIR" \ --depends sed \ --depends grep \ --config-files "$JAVA_CONFIG_INSTALL_PATH" \ + --config-files "$NODE_CONFIG_INSTALL_PATH" \ "$buildroot/"=/ dpkg -c "${OUTPUT_DIR}/${PKG_NAME}_${VERSION}_${ARCH}.deb" diff --git a/instrumentation/packaging/fpm/examples/systemd/00-splunk-otel-javaagent.conf b/instrumentation/packaging/fpm/examples/systemd/00-splunk-otel-javaagent.conf index 316dd783a2..e08382f45f 100644 --- a/instrumentation/packaging/fpm/examples/systemd/00-splunk-otel-javaagent.conf +++ b/instrumentation/packaging/fpm/examples/systemd/00-splunk-otel-javaagent.conf @@ -13,14 +13,17 @@ # limitations under the License. # Sample systemd drop-in file to activate and configure Splunk OpenTelemetry Auto Instrumentation for Java +# https://docs.splunk.com/Observability/gdi/get-data-in/application/java/get-started.html # To use this file to activate/configure auto instrumentation for all supported Java applications running as systemd # services (requires root privileges): -# 1. See https://docs.splunk.com/Observability/gdi/get-data-in/application/java/configuration/advanced-java-otel-configuration.html +# 1. Check https://docs.splunk.com/Observability/gdi/get-data-in/application/java/java-otel-requirements.html for +# minimum requirements +# 2. See https://docs.splunk.com/Observability/gdi/get-data-in/application/java/configuration/advanced-java-otel-configuration.html # for supported environment variables and defaults for Java auto instrumentation -# 2. Add/Update the DefaultEnvironment entries below for the desired environment variables and values -# 3. Copy this file to the systemd configuration directory, e.g. /usr/lib/systemd/system.conf.d -# 4. Reboot the system or run the following commands to restart the applicable service(s) for any changes to take effect: +# 3. Add/Update the DefaultEnvironment entries below for the desired environment variables and values +# 4. Copy this file to the systemd configuration directory, e.g. /usr/lib/systemd/system.conf.d/ +# 5. Reboot the system or run the following commands to restart the applicable service(s) for any changes to take effect: # $ sudo systemctl daemon-reload # $ sudo systemctl restart # replace "" and run for each applicable service diff --git a/instrumentation/packaging/fpm/examples/systemd/00-splunk-otel-js.conf b/instrumentation/packaging/fpm/examples/systemd/00-splunk-otel-js.conf new file mode 100644 index 0000000000..fdbcef9801 --- /dev/null +++ b/instrumentation/packaging/fpm/examples/systemd/00-splunk-otel-js.conf @@ -0,0 +1,43 @@ +# Copyright Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Sample systemd drop-in file to activate and configure Splunk OpenTelemetry Auto Instrumentation for Node.js +# https://docs.splunk.com/Observability/gdi/get-data-in/application/nodejs/get-started.html + +# To use this file to activate/configure auto instrumentation for all supported Node.js applications running as systemd +# services (requires root privileges): +# 1. Check https://docs.splunk.com/Observability/gdi/get-data-in/application/nodejs/nodejs-otel-requirements.html for +# minimum requirements. +# 2. Install the provided Node.js auto instrumentation package with npm. For example: +# $ sudo npm install --global /usr/lib/splunk-instrumentation/splunk-otel-js.tar.gz +# 3. See https://docs.splunk.com/Observability/gdi/get-data-in/application/nodejs/configuration/advanced-nodejs-otel-configuration.html +# for supported environment variables and defaults for Node.js auto instrumentation +# 4. Add/Update the DefaultEnvironment entries below for the desired environment variables and values +# 5. Copy this file to the systemd configuration directory, e.g. /usr/lib/systemd/system.conf.d/ +# 6. Reboot the system or run the following commands to restart the applicable service(s) for any changes to take effect: +# $ sudo systemctl daemon-reload +# $ sudo systemctl restart # replace "" and run for each applicable service + +# See https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html for general information +# about systemd configuration and other supported options. + +[Manager] +# Required to activate Splunk OpenTelemetry Auto Instrumentation for Node.js +DefaultEnvironment="NODE_OPTIONS=-r @splunk/otel/instrument" + +# Examples of common configuration options: +#DefaultEnvironment="OTEL_EXPORTER_OTLP_ENDPOINT=http://1.2.3.4:5678" +#DefaultEnvironment="OTEL_RESOURCE_ATTRIBUTES=deployment.environment=my_deployment_environment" +#DefaultEnvironment="OTEL_SERVICE_NAME=my_service_name" +#DefaultEnvironment="SPLUNK_PROFILER_ENABLED=true" diff --git a/instrumentation/packaging/fpm/rpm/build.sh b/instrumentation/packaging/fpm/rpm/build.sh index 858a7fd1e6..8c10f4c4a7 100755 --- a/instrumentation/packaging/fpm/rpm/build.sh +++ b/instrumentation/packaging/fpm/rpm/build.sh @@ -22,9 +22,6 @@ SCRIPT_DIR="$( cd "$( dirname ${BASH_SOURCE[0]} )" && pwd )" VERSION="${1:-}" ARCH="${2:-amd64}" OUTPUT_DIR="${3:-$REPO_DIR/instrumentation/dist}" -LIBSPLUNK_PATH="$REPO_DIR/instrumentation/dist/libsplunk_${ARCH}.so" -JAVA_AGENT_PATH="$REPO_DIR/instrumentation/dist/splunk-otel-javaagent.jar" -JAVA_AGENT_RELEASE="$(cat $JAVA_AGENT_RELEASE_PATH)" if [[ -z "$VERSION" ]]; then VERSION="$( get_version )" @@ -34,11 +31,9 @@ fi VERSION="${VERSION/'-'/'~'}" VERSION="${VERSION#v}" -download_java_agent "$JAVA_AGENT_RELEASE" "$JAVA_AGENT_PATH" - buildroot="$(mktemp -d)" -setup_files_and_permissions "$LIBSPLUNK_PATH" "$JAVA_AGENT_PATH" "$buildroot" +setup_files_and_permissions "$ARCH" "$buildroot" mkdir -p "$OUTPUT_DIR" @@ -62,6 +57,7 @@ sudo fpm -s dir -t rpm -n "$PKG_NAME" -v "$VERSION" -f -p "$OUTPUT_DIR" \ --depends sed \ --depends grep \ --config-files "$JAVA_CONFIG_INSTALL_PATH" \ + --config-files "$NODE_CONFIG_INSTALL_PATH" \ "$buildroot/"=/ rpm -qpli "${OUTPUT_DIR}/${PKG_NAME}-${VERSION}-1.${ARCH}.rpm" diff --git a/instrumentation/packaging/fpm/zeroconfig_node.conf b/instrumentation/packaging/fpm/zeroconfig_node.conf new file mode 100644 index 0000000000..139418d771 --- /dev/null +++ b/instrumentation/packaging/fpm/zeroconfig_node.conf @@ -0,0 +1 @@ +NODE_OPTIONS=-r @splunk/otel/instrument diff --git a/instrumentation/packaging/nodejs-agent-release.txt b/instrumentation/packaging/nodejs-agent-release.txt new file mode 100644 index 0000000000..8721bbc46a --- /dev/null +++ b/instrumentation/packaging/nodejs-agent-release.txt @@ -0,0 +1 @@ +v2.4.0 diff --git a/internal/buildscripts/packaging/tests/instrumentation/instrumentation_test.py b/internal/buildscripts/packaging/tests/instrumentation/instrumentation_test.py index a53e4c55d3..701f6e7dbf 100644 --- a/internal/buildscripts/packaging/tests/instrumentation/instrumentation_test.py +++ b/internal/buildscripts/packaging/tests/instrumentation/instrumentation_test.py @@ -42,16 +42,28 @@ INSTALLER_PATH = REPO_DIR / "internal" / "buildscripts" / "packaging" / "installer" / "install.sh" COLLECTOR_CONFIG_PATH = TESTS_DIR / "instrumentation" / "config.yaml" JAVA_AGENT_PATH = "/usr/lib/splunk-instrumentation/splunk-otel-javaagent.jar" +NODE_AGENT_PATH = "/usr/lib/splunk-instrumentation/splunk-otel-js.tgz" PKG_NAME = "splunk-otel-auto-instrumentation" LIBSPLUNK_PATH = "/usr/lib/splunk-instrumentation/libsplunk.so" JAVA_CONFIG_PATH = "/etc/splunk/zeroconfig_java.conf" CUSTOM_JAVA_CONFIG_PATH = TESTS_DIR / "instrumentation" / "libsplunk-java-test.conf" -DEFAULT_SYSTEMD_CONF_PATH = "/usr/lib/splunk-instrumentation/examples/systemd/00-splunk-otel-javaagent.conf" +NODE_CONFIG_PATH = "/etc/splunk/zeroconfig_node.conf" +SAMPLE_JAVA_SYSTEMD_CONF_PATH = "/usr/lib/splunk-instrumentation/examples/systemd/00-splunk-otel-javaagent.conf" +SAMPLE_NODE_SYSTEMD_CONF_PATH = "/usr/lib/splunk-instrumentation/examples/systemd/00-splunk-otel-js.conf" CUSTOM_SYSTEMD_CONF_PATH = TESTS_DIR / "instrumentation" / "systemd-test.conf" SYSTEMD_CONF_DIR = "/usr/lib/systemd/system.conf.d" PRELOAD_PATH = "/etc/ld.so.preload" +INSTALLED_FILES = [ + JAVA_AGENT_PATH, + NODE_AGENT_PATH, + LIBSPLUNK_PATH, + JAVA_CONFIG_PATH, + NODE_CONFIG_PATH, + SAMPLE_JAVA_SYSTEMD_CONF_PATH, + SAMPLE_NODE_SYSTEMD_CONF_PATH, +] def get_dockerfile(distro): if distro in DEB_DISTROS: @@ -88,6 +100,9 @@ def install_package(container, distro, path): else: run_container_cmd(container, f"rpm -ivh {path}") + for path in INSTALLED_FILES: + assert container_file_exists(container, path), f"{path} not found" + def verify_preload(container, line, exists=True): code, output = container.exec_run(f"cat {PRELOAD_PATH}") @@ -112,7 +127,7 @@ def verify_tomcat_instrumentation(container, otelcol_path, test_case, source, at stream = container.exec_run("journalctl -u splunk-otel-collector -f", stream=True).output else: run_container_cmd(container, f"mkdir -p {SYSTEMD_CONF_DIR}") - run_container_cmd(container, f"cp {DEFAULT_SYSTEMD_CONF_PATH} {SYSTEMD_CONF_DIR}/") + run_container_cmd(container, f"cp {SAMPLE_JAVA_SYSTEMD_CONF_PATH} {SYSTEMD_CONF_DIR}/") if test_case == "custom": copy_file_into_container(container, CUSTOM_SYSTEMD_CONF_PATH, f"{SYSTEMD_CONF_DIR}/99-systemd-test.conf") @@ -194,8 +209,6 @@ def test_tomcat_instrumentation(distro, arch, test_case, source): run_container_cmd(container, f"chmod a+x /test/{otelcol_bin}") install_package(container, distro, f"/test/{pkg_base}") - for path in [JAVA_AGENT_PATH, LIBSPLUNK_PATH, JAVA_CONFIG_PATH, DEFAULT_SYSTEMD_CONF_PATH]: - assert container_file_exists(container, path), f"{path} not found" if test_case == "default": # service name auto-generated by java agent @@ -240,9 +253,6 @@ def test_package_uninstall(distro, arch): verify_preload(container, "# This line should be preserved") - for path in [JAVA_AGENT_PATH, LIBSPLUNK_PATH, JAVA_CONFIG_PATH, DEFAULT_SYSTEMD_CONF_PATH]: - assert container_file_exists(container, path), f"{path} not found" - # verify libsplunk.so was not automatically added to /etc/ld.so.preload verify_preload(container, LIBSPLUNK_PATH, exists=False) @@ -262,7 +272,7 @@ def test_package_uninstall(distro, arch): assert container.exec_run(f"rpm -q {PKG_NAME}").exit_code != 0 # verify files were uninstalled - for path in [JAVA_AGENT_PATH, LIBSPLUNK_PATH, JAVA_CONFIG_PATH, DEFAULT_SYSTEMD_CONF_PATH]: + for path in INSTALLED_FILES: assert not container_file_exists(container, path) # verify libsplunk.so was removed from /etc/ld.so.preload