Add systemd sample file to splunk-otel-auto-instrumention (#3506)

* Add systemd sample file to splunk-otel-auto-instrumention

* Update manual instructions

* Include default /etc/splunk/zeroconfig_java.conf in deb/rpm packages

* Update tests for new libsplunk.so

* Run auto instrumentation workflow for the zero-config-dev branch

.github/workflows/auto-instrumentation.yml

@@ -4,50 +4,56 @@ on:
   push:
     branches:
       - main
+      - zero-config-dev
   pull_request:
     paths:
       - '.github/workflows/auto-instrumentation.yml'
       - 'instrumentation/**'
+      - 'internal/buildscripts/packaging/tests/helpers/**'
+      - 'internal/buildscripts/packaging/tests/instrumentation/**'
+      - 'internal/buildscripts/packaging/tests/requirements.txt'
+      - '!**.md'
 
 concurrency:
   group: auto-instrumentation-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  PYTHON_VERSION: '3.11'
+  PIP_VERSION: '22.0.4'
+  REQUIREMENTS_PATH: "internal/buildscripts/packaging/tests/requirements.txt"
+  GO_VERSION: 1.20.6
+
 jobs:
-  libsplunk:
+  cross-compile:
     # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        ARCH: [ "amd64", "arm64" ]
+        SYS_BINARIES: [ "binaries-linux_amd64", "binaries-linux_arm64" ]
     steps:
       - name: Check out the codebase.
         uses: actions/checkout@v4
 
-      - name: Set up QEMU
-        if: ${{ matrix.ARCH == 'arm64' }}
-        uses: docker/setup-qemu-action@v3
+      - name: Set up Go
+        uses: actions/setup-go@v4
         with:
-          platforms: arm64
-          image: tonistiigi/binfmt:qemu-v7.0.0
-
-      - name: Build libsplunk.so
-        run: make -C instrumentation dist ARCH=${{ matrix.ARCH }}
+          go-version: ${{ env.GO_VERSION }}
+          cache-dependency-path: '**/go.sum'
 
-      - name: Run tests
-        run: make -C instrumentation tests ARCH=${{ matrix.ARCH }}
+      - name: Build Collector
+        run: |
+          make ${{ matrix.SYS_BINARIES }}
 
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
-          name: libsplunk-${{ matrix.ARCH }}
-          path: ./instrumentation/dist/libsplunk_${{ matrix.ARCH }}.so
+          name: ${{ matrix.SYS_BINARIES }}
+          path: |
+            ./bin/*
 
   build-package:
-    name: build-package
-    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
     runs-on: ubuntu-20.04
-    needs: [libsplunk]
     strategy:
       matrix:
         SYS_PACKAGE: [ "deb", "rpm" ]
@@ -58,86 +64,124 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Downloading libsplunk-${{ matrix.ARCH }}
-        uses: actions/download-artifact@v3
+      - uses: docker/setup-qemu-action@v2
+        if: ${{ matrix.ARCH != 'amd64' }}
         with:
-          name: libsplunk-${{ matrix.ARCH }}
-          path: ./instrumentation/dist
+          platforms: ${{ matrix.ARCH }}
+          image: tonistiigi/binfmt:qemu-v7.0.0
 
-      - name: Build ${{ matrix.SYS_PACKAGE }} ${{ matrix.ARCH }} package
-        run: |
-          gitRef=(${GITHUB_REF//// })
-          if [[ "${gitRef[1]}" = "tags" ]] ;
-          then
-            echo "TAG_NAME=${gitRef[2]}" >> $GITHUB_ENV
-          else
-            echo "TAG_NAME=" >> $GITHUB_ENV
-          fi
-          make -C instrumentation ${{ matrix.SYS_PACKAGE }}-package SKIP_COMPILE=true VERSION="${TAG_NAME:-}" ARCH="${{ matrix.ARCH }}"
+      - name: Run tests
+        run: make -C instrumentation tests ARCH=${{ matrix.ARCH }}
+
+      - name: Build ${{ matrix.ARCH }} ${{ matrix.SYS_PACKAGE }} package
+        run: make -C instrumentation/ ${{ matrix.SYS_PACKAGE }}-package ARCH="${{ matrix.ARCH }}"
 
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         with:
           name: splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-${{ matrix.SYS_PACKAGE }}
-          path: ./instrumentation/dist/splunk-otel-auto-instrumentation*.${{ matrix.SYS_PACKAGE }}
+          path: ./instrumentation/dist/*.${{ matrix.SYS_PACKAGE }}
 
-  test-deb-package:
-    name: test-deb-package
-    # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
+  test-package-matrix:
     runs-on: ubuntu-20.04
-    needs: [build-package]
-    strategy:
-      matrix:
-        DISTRO: [ "debian:8", "debian:9", "debian:10", "ubuntu:16.04", "ubuntu:18.04", "ubuntu:20.04" ]
-        ARCH: [ "amd64", "arm64" ]
-        exclude:
-          - DISTRO: debian:8
-            ARCH: arm64
     steps:
-      - name: Check out the codebase.
+      - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Downloading splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-deb
-        uses: actions/download-artifact@v3
-        with:
-          name: splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-deb
-          path: ./instrumentation/dist
-
-      - name: Set up QEMU
-        if: ${{ matrix.ARCH == 'arm64' }}
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: arm64
-          image: tonistiigi/binfmt:qemu-v7.0.0
-
-      - name: Test
-        run: docker run --platform linux/${{ matrix.ARCH }} --rm -v $(pwd):/repo -w /repo ${{ matrix.DISTRO }} /repo/instrumentation/packaging/fpm/test.sh deb ${{ matrix.ARCH }}
-
-  test-rpm-package:
-    name: test-rpm-package
+      - name: Get matrix
+        id: get-matrix
+        run: |
+          # create test matrix for distro and arch
+          dockerfiles=$(find internal/buildscripts/packaging/tests/instrumentation/images/ -name "Dockerfile.*" | cut -d '.' -f2- | sort -u)
+          if [ -z "$dockerfiles" ]; then
+            echo "Failed to get dockerfiles from internal/buildscripts/packaging/tests/instrumentation/images!" >&2
+            exit 1
+          fi
+          distro=$(for d in $dockerfiles; do echo -n "\"$d\","; done)
+          arch="\"amd64\", \"arm64\""
+          matrix="{\"DISTRO\": [${distro%,}], \"ARCH\": [${arch}]}"
+          echo "$matrix" | jq
+          echo "matrix=${matrix}" >> $GITHUB_OUTPUT
+    outputs:
+      matrix: ${{ steps.get-matrix.outputs.matrix }}
+
+  test-package:
     # Use 20.04.5 until https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16450 is resolved
-    runs-on: ubuntu-20.04
-    needs: [build-package]
+    runs-on: ${{ fromJSON('["ubuntu-20.04", "ubuntu-22.04"]')[matrix.DISTRO == 'amazonlinux-2023'] }}
+    timeout-minutes: 60
+    needs: [cross-compile, build-package, test-package-matrix]
     strategy:
-      matrix:
-        DISTRO: [ "centos:7", "centos:8", "amazonlinux:2", "opensuse/leap:42", "opensuse/leap:15", "oraclelinux:7", "oraclelinux:8" ]
-        ARCH: [ "amd64", "arm64" ]
+      matrix: ${{ fromJSON(needs.test-package-matrix.outputs.matrix) }}
+      fail-fast: false
     steps:
       - name: Check out the codebase.
         uses: actions/checkout@v4
 
-      - name: Downloading splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-rpm
-        uses: actions/download-artifact@v3
+      - name: Get package type for ${{ matrix.DISTRO }}
+        run: |
+          for pkg in "deb" "rpm"; do
+            if [[ -f "internal/buildscripts/packaging/tests/instrumentation/images/${pkg}/Dockerfile.${{ matrix.DISTRO }}" ]]; then
+              echo "SYS_PACKAGE=${pkg}" >> $GITHUB_ENV
+              exit 0
+            fi
+          done
+          echo "Unknown distro '${{ matrix.DISTRO }}'!"
+          exit 1
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: binaries-linux_${{ matrix.ARCH }}
+          path: ./bin
+
+      - uses: actions/download-artifact@v3
         with:
-          name: splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-rpm
+          name: splunk-otel-auto-instrumentation-${{ matrix.ARCH }}-${{ env.SYS_PACKAGE }}
           path: ./instrumentation/dist
 
-      - name: Set up QEMU
-        if: ${{ matrix.ARCH == 'arm64'}}
-        uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-qemu-action@v3
+        if: ${{ matrix.ARCH != 'amd64' }}
         with:
-          platforms: arm64
+          platforms: ${{ matrix.ARCH }}
           image: tonistiigi/binfmt:qemu-v7.0.0
 
-      - name: Test
-        run: docker run --platform linux/${{ matrix.ARCH }} --rm -v $(pwd):/repo -w /repo ${{ matrix.DISTRO }} /repo/instrumentation/packaging/fpm/test.sh rpm ${{ matrix.ARCH }}
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+          cache-dependency-path: ${{ env.REQUIREMENTS_PATH }}
+
+      - name: Install pytest
+        run: |
+          if which pip; then
+              pip install --upgrade 'pip==${{ env.PIP_VERSION }}'
+          else
+              curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+              python get-pip.py 'pip==${{ env.PIP_VERSION }}'
+          fi
+          pip install -r "${{ env.REQUIREMENTS_PATH }}"
+
+      - name: Test ${{ env.SYS_PACKAGE }} on ${{ matrix.DISTRO }} ${{ matrix.ARCH }}
+        id: pytest
+        continue-on-error: true
+        run: |
+          distro="${{ matrix.DISTRO }}"
+          if [[ "$distro" = "amazonlinux-2" ]]; then
+            # workaround for pytest substring matching
+            distro="amazonlinux-2 and not amazonlinux-2023"
+          fi
+          python3 -u -m pytest -s --verbose -k "$distro and ${{ matrix.ARCH }}" \
+            internal/buildscripts/packaging/tests/instrumentation/instrumentation_test.py
+
+      # qemu, networking, running systemd in containers, etc., can be flaky
+      - name: Re-run failed tests
+        if: ${{ steps.pytest.outcome == 'failure' }}
+        run: |
+          distro="${{ matrix.DISTRO }}"
+          if [[ "$distro" = "amazonlinux-2" ]]; then
+            # workaround for pytest substring matching
+            distro="amazonlinux-2 and not amazonlinux-2023"
+          fi
+          python3 -u -m pytest -s --verbose -k "$distro and ${{ matrix.ARCH }}" \
+            --last-failed \
+            internal/buildscripts/packaging/tests/instrumentation/instrumentation_test.py

docs/getting-started/linux-manual.md

@@ -328,24 +328,43 @@ and install it with the following commands (requires `root` privileges).
 
 #### Auto Instrumentation Post-Install Configuration
 
-- The `/etc/ld.so.preload` file will be automatically created/updated with the
-  default path to the installed instrumentation library
-  (`/usr/lib/splunk-instrumentation/libsplunk.so`).  If necessary, custom
-  library paths can be manually added to this file.
-- The `/usr/lib/splunk-instrumentation/instrumentation.conf` configuration file
-  can be manually configured for resource attributes and other parameters.  By
-  default, this file will contain the `java_agent_jar` parameter set to the
-  path of the installed [Java Instrumentation Agent](
-  https://github.com/signalfx/splunk-otel-java)
-  (`/usr/lib/splunk-instrumentation/splunk-otel-javaagent.jar`).
-
-See [Linux Java Auto Instrumentation](https://github.com/signalfx/splunk-otel-collector/tree/main/instrumentation#linux-java-auto-instrumentation)
-for more details.
-
-**Note:** After installation/upgrade or any configuration changes, the Java
-application(s) on the host need to be manually started/restarted for automatic
-instrumentation to take effect and/or to source the updated values in the
-configuration file.
+Choose one of the following methods to activate and configure Splunk
+OpenTelemetry Auto Instrumentation ***globally*** with either the provided
+`libsplunk.so` shared object library or sample `systemd` drop-in files. To
+activate and configure auto instrumentation for individual services or
+applications, see
+[Instrument back-end applications to send spans to Splunk APM](
+https://docs.splunk.com/Observability/gdi/get-data-in/application/application.html).
+
+1. Preload method
+   - The `/usr/lib/splunk-instrumentation/libsplunk.so` shared object library
+     can be added to the [`/etc/ld.so.preload`](
+     https://man7.org/linux/man-pages/man8/ld.so.8.html#FILES) file to activate
+     auto instrumentation for ***all*** supported processes.
+   - The `/usr/lib/splunk-instrumentation/instrumentation.conf` configuration
+     file can be configured for resource attributes and other supported
+     parameters. By default, this file will contain the `java_agent_jar`
+     parameter set to the path of the installed [Java Instrumentation Agent](
+     https://github.com/signalfx/splunk-otel-java)
+     (`/usr/lib/splunk-instrumentation/splunk-otel-javaagent.jar`).
+   - See [Linux Java Auto Instrumentation](../../instrumentation/libsplunk.md)
+     for more details.
+
+2. `Systemd` method
+   - The sample `systemd` drop-in files in the
+     `/usr/lib/splunk-instrumentation/examples/systemd/` directory can be
+     configured and copied to the host's [`systemd` configuration
+     directory](
+     https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html),
+     for example `/usr/lib/systemd/system.conf.d/`, to activate and
+     configure auto instrumentation for ***all*** supported applications
+     running as `systemd` services.
+   - See [Splunk OpenTelemetry Zero Configuration Auto Instrumentation for
+     Systemd](../../instrumentation/systemd.md) for more details.
+
+**Note:** After installation/upgrade or any configuration changes, reboot the
+system or restart the application(s) on the host for automatic instrumentation
+to take effect and/or to source the updated values.
 
 #### Fluentd
 

instrumentation/README.md

@@ -0,0 +1,50 @@
+# Splunk OpenTelemetry Zero Configuration Auto Instrumentation for Linux
+
+**Splunk OpenTelemetry Zero Configuration Auto Instrumentation for Linux** (`splunk-otel-auto-instrumentation`)
+Debian/RPM package installs Splunk OpenTelemetry Auto Instrumentation agent(s), the `libsplunk.so` shared object
+library, and sample configuration files to automatically instrument applications and services to capture and report
+distributed traces and metrics to the [Splunk OpenTelemetry Collector](
+https://docs.splunk.com/Observability/gdi/opentelemetry/opentelemetry.html), and then on to [Splunk APM](
+https://docs.splunk.com/Observability/apm/intro-to-apm.html).
+
+Currently, `splunk-otel-auto-instrumentation` installs and supports configuration of the following Auto Instrumentation
+agent(s):
+
+- [Java](https://docs.splunk.com/Observability/gdi/get-data-in/application/java/get-started.html)
+
+## Prerequisites/Requirements
+
+- Check agent compatibility and requirements:
+  - [Java](https://docs.splunk.com/Observability/gdi/get-data-in/application/java/java-otel-requirements.html)
+- [Install and configure](https://docs.splunk.com/Observability/gdi/opentelemetry/install-linux.html) the Splunk
+  OpenTelemetry Collector.
+- Debian or RPM based Linux distribution (amd64/x86_64 or arm64/aarch64).
+
+## Zero Configuration Options
+
+The following options are supported to enable the installed Auto Instrumentation agent(s):
+
+- **[`libsplunk.so`](./libsplunk.md)**: The provided `/usr/lib/splunk-instrumentation/libsplunk.so` shared object
+  library can be added to the `/etc/ld.so.preload` file to enable and configure Auto Instrumentation for ***all***
+  supported processes on the system, or it can be assigned to the `LD_PRELOAD` environment variable for specific
+  applications/services. Configuration of the installed agent(s) is supported by the
+  [`/usr/lib/splunk-instrumentation/instrumentation.conf`](./libsplunk.md#configuration-file) file.
+
+- **[`systemd`](./systemd.md)**: The provided `systemd` drop-in file(s) in the
+  `/usr/lib/splunk-instrumentation/examples/systemd/` directory can be copied to the host's `systemd` configuration
+  directory, e.g. `/usr/lib/systemd/system.conf.d/`, to enable and configure Auto Instrumentation for ***all***
+  supported applications running as `systemd` services. Configuration of the installed agent(s) is supported by
+  modifying these files or adding custom drop-in files with the desired environment variables.
+
+Alternatively, [manually install and configure](
+https://docs.splunk.com/Observability/gdi/get-data-in/application/application.html)
+Auto Instrumentation if neither option is appropriate for the target host or application.
+
+> ### Notes
+> 
+> 1. To prevent conflicts and duplicate traces/metrics, only one option should be enabled on the target system.
+> 2. The configuration files and the options defined within are only applicable for the respective option that is
+>    configured. For example, `/usr/lib/splunk-instrumentation/instrumentation.conf` is only applicable with
+>    `libsplunk.so`, and the systemd drop-in file is not applicable for `libsplunk.so`.
+> 3. The [`splunk.linux-autoinstr.executions`](./libsplunk.md#disable_telemetry-optional) telemetry
+>    metric is currently only provided with `libsplunk.so`.