Merge pull request #49 from SpecterOps/helm

Add Helm for Kubernetes Deployment
SpecterOps · Apr 2, 2024 · 93d12bb · 93d12bb
2 parents e8a75a5 + 283487f
commit 93d12bb
Show file tree

Hide file tree

Showing 178 changed files with 20,342 additions and 18,473 deletions.
diff --git a/.github/workflows/helm-release.yml b/.github/workflows/helm-release.yml
@@ -0,0 +1,190 @@
+name: Release Helm Charts
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Release version (eg: 2.1.1)"
+        required: true
+        type: string
+
+      charts_dir:
+        description: "Charts directory"
+        required: true
+        default: 'helm'
+        type: string
+
+env:
+  PACKAGE_DIR: dist
+
+
+# Only allow one instance of this workflow to run at a time
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Make sure charts directory exists and contains Helm charts
+        env:
+          CHARTS_DIR: ${{ inputs.charts_dir }}
+        run: find $CHARTS_DIR -maxdepth 2 -mindepth 2 -type f -name "Chart.yaml"
+
+      - name: Verify release version is a valid SemVer version string
+        env:
+          VERSION: ${{ inputs.version }}
+        # regex is from the semver.org list of suggested regex strings https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+        run:  echo $VERSION | grep -qP '^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$'
+
+      - name: Ensure a git tag for this version does not already exist
+        env:
+          VERSION: v${{ inputs.version }}
+        run: |
+          if [ $(git tag -l "$VERSION") ]; then
+            echo "Git tag matching release version '$VERSION' already exists"
+            false
+          else
+            true
+          fi
+
+      - name: Ensure a Github release for this version does not already exist
+        env:
+          VERSION: v${{ inputs.version }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          export API_RESULT=$(gh api --silent \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-Github-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/releases/tags/${VERSION} 2>&1)
+
+          if [[ "$API_RESULT" == *"Not Found"* ]]; then
+            true
+          else
+            echo "Release for version '$VERSION' already exists on Github"
+            false
+          fi
+
+  release:
+    needs: verify
+
+    # Provision a Github token with repository and pages write permissions
+    permissions:
+      contents: write
+      pages: write
+      id-token: write
+
+    # Use the github-pages environment. The actions/deploy-pages workflow fails with a
+    # "Invalid environment node id" error if an environment is not specified.
+    # https://github.com/actions/deploy-pages/issues/271
+    environment:
+      name: github-pages
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Configure git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - name: Create a git tag for the release
+        uses: EndBug/add-and-commit@v9
+        with:
+          message: "Nemesis v${{ inputs.version }}"
+          push: true
+          tag: "v${{ inputs.version }}"
+
+      - name: Install Helm
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        uses: azure/setup-helm@v3
+
+      - name: Add chart dependency repositories
+        env:
+          CHARTS_DIR: ${{ inputs.charts_dir }}
+        run: |
+          charts=($(find $CHARTS_DIR -maxdepth 2 -mindepth 2 -type f -name "Chart.yaml" -printf '%h\n'))
+          for chart in "${charts[@]}"; do
+            repos=($(helm dependency list $chart | head -n '-1' | tail -n '+2' | cut -f3))
+            for repo in "${repos[@]}"; do
+              name=$(echo $repo | grep -o '[^/]*$')
+              helm repo add $name $repo
+            done
+          done
+
+      - name: Package Helm charts
+        env:
+          PACKAGE_DIR: ${{ env.PACKAGE_DIR }}
+          CHARTS_DIR: ${{ inputs.charts_dir }}
+        run: |
+          mkdir -p $PACKAGE_DIR
+          find $CHARTS_DIR -maxdepth 2 -mindepth 2 -type f -name "Chart.yaml" -printf '%h\n' | xargs -I % bash -c "helm package -d $PACKAGE_DIR %"
+
+      - name: Pull in previous index.yaml file if it exists
+        env:
+          PACKAGE_DIR: ${{ env.PACKAGE_DIR }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          PAGES_URL=$(gh api \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/pages \
+            | jq -r '.html_url')
+
+          if [[ "$PAGES_URL" != "null" ]]; then
+            HTTP_STATUS=$(curl -sL -w '%{http_code}' "${PAGES_URL%/}/index.yaml" -o ${PACKAGE_DIR}/index.yaml)
+            if [[ "$HTTP_STATUS" != "200" ]]; then
+              rm ${PACKAGE_DIR}/index.yaml
+            fi
+          fi
+
+      - name: Update Helm repository index.yaml file
+        env:
+          PACKAGE_DIR: ${{ env.PACKAGE_DIR }}
+          CHART_BASE_URL: ${{ github.server_url }}/${{ github.repository }}/releases/download/v${{ inputs.version }}
+        run: |
+          if [ -f ${PACKAGE_DIR}/index.yaml ]; then
+            helm repo index $PACKAGE_DIR --merge ${PACKAGE_DIR}/index.yaml --url $CHART_BASE_URL
+          else
+            helm repo index $PACKAGE_DIR --url $CHART_BASE_URL
+          fi
+
+      - name: Create Github release with the Helm charts
+        env:
+          PACKAGE_DIR: ${{ env.PACKAGE_DIR }}
+          VERSION: v${{ inputs.version }}
+          GH_TOKEN: ${{ github.token }}
+        run: gh release create ${VERSION} -R ${{ github.repository }} -t "Nemesis $VERSION" -n "Nemesis $VERSION release" $PACKAGE_DIR/*.tgz
+
+      - name: Remove packaged Helm charts
+        env:
+          PACKAGE_DIR: ${{ env.PACKAGE_DIR }}
+        run: rm -f ${PACKAGE_DIR}/*.tgz
+
+      - name: Setup Github pages
+        uses: actions/configure-pages@v4
+
+      - name: Create Github pages artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ${{ env.PACKAGE_DIR }}
+
+      - name: Deploy Helm chart repository to Github pages
+        uses: actions/deploy-pages@v4
+
+      - name: Remove Github release and tag on failure
+        continue-on-error: true
+        if: ${{ failure() }}
+        env:
+          VERSION: v${{ inputs.version }}
+          GH_TOKEN: ${{ github.token }}
+        run: gh release delete -R ${{ github.repository }} $VERSION -y --cleanup-tag
diff --git a/.gitignore b/.gitignore
@@ -7,4 +7,5 @@ __pycache__
 *.asc
 nemesis.config
 config.yml
-
+submit_to_nemesis.yaml
+submit/
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -20,32 +20,15 @@
                         "${workspaceFolder}/cmd/enrichment": "/app/cmd/enrichment"
                     },
                     "justMyCode": false
-                }
-            ]
-        },
-        {
-            "name": "Kubernetes: Run/Debug Dashboard",
-            "type": "cloudcode.kubernetes",
-            "request": "launch",
-            "skaffoldConfig": "${workspaceFolder}/skaffold.yaml",
-            "skaffoldFlags": {
-                "modules": [
-                    "dashboard"
-                ]
-            },
-            "watch": true,
-            "cleanUp": false,
-            "portForward": true,
-            "debug": [
+                },
                 {
-                    "image": "dashboard",
+                    "image": "enrichment-dev",
+                    "containerName": "enrichment",
                     "sourceFileMap": {
-                        "${workspaceFolder}/cmd/dashboard": "/app/cmd/dashboard"
-                    },
-                    "justMyCode": false
+                        "${workspaceFolder}/cmd/enrichment": "/app/cmd/enrichment"
+                    }
                 }
-            ],
-            "imageRegistry": "docker.io"
+            ]
         }
     ]
 }
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -17,6 +17,9 @@
         },
         // "editor.defaultFormatter": "ms-python.black-formatter"
     },
+    "flake8.args": [
+        "--max-line=length=120"
+    ],
     "files.watcherExclude": {
         "**/__pycache__/**": true,
         "**/.git/objects/**": true,
@@ -31,4 +34,4 @@
         "**/dist/**": true,
         "**/node_modules/*/**": true,
     },
-}
+}
diff --git a/README.md b/README.md
@@ -46,6 +46,7 @@ See [development.md](./docs/development.md)
 
 | Post Name                                   | Publication Date | Link                                                                               |
 |---------------------------------------------|------------------|------------------------------------------------------------------------------------|
+| *Shadow Wizard Registry Gang: Structured Registry Querying* | Sep 5, 2023 | https://posts.specterops.io/shadow-wizard-registry-gang-structured-registry-querying-9a2fab62a26f |
 | *Hacking With Your Nemesis*                 | Aug 9, 2023      | https://posts.specterops.io/hacking-with-your-nemesis-7861f75fcab4                 |
 | *Challenges In Post-Exploitation Workflows* | Aug 2, 2023      | https://posts.specterops.io/challenges-in-post-exploitation-workflows-2b3469810fe9 |
 | *On (Structured) Data*                      | Jul 26, 2023     | https://posts.specterops.io/on-structured-data-707b7d9876c6                        |

diff --git a/cmd/connectors/cobaltstrike-nemesis-connector/README.md b/cmd/connectors/cobaltstrike-nemesis-connector/README.md
@@ -1,53 +1,53 @@
-Aggressor project that ingests Cobalt Strike data into Nemesis.
-
-The project can be run on a teamserver to ingest data tasks by ALL clients, or can be run on an individual client instance to ingest data just from that client.
-
-Also contains the `bof_reg_collect` , a Beacon Object File to collect registry data in a serialized manner.
-
-# Usage
-Make sure the following environment variables are set before running `nemesis-connector.cna`:
-
-| Configuration Variable             | Required | Example                     | Default (if applicable) | Description                                                                                                   |
-|------------------------------------|----------|-----------------------------|-------------------------|---------------------------------------------------------------------------------------------------------------|
-| NEMESIS_COBALTSTRIKE_DOWNLOADS_DIR | Yes      | /tmp/                       |                         | Temporary directory used when syncing downloads from a Cobalt Strike teamserver                               |
-| NEMESIS_BASE_URL                   | Yes      | http://192.168.230.42:8080/ |                         | Base URL used when constructing the URL to web api, elastic, kibana, etc (e.g., NEMESIS_BASE_URL + "/kibana") |
-| NEMESIS_CREDS                      | Yes      | nemesis:Qwerty12345         |                         | Basic auth credentials used when accessing the Nemesis frontend web endpoints                                 |
-| NEMESIS_DEBUG_JSON                 | No       | 1                           | 0                       | Print JSON responses from web API requests                                                                    |
-| NEMESIS_PROJECT                    | Yes      | ASSESS-123                  |                         | Assessmend project ID the teamserver is associated with                                                       |
-| NEMESIS_DATA_EXPIRATION_DAYS       | Yes      | 100                         | 100                     | Number of days after which Nemesis should expire the data                                                     |
-
-Example:
-```
-export NEMESIS_COBALTSTRIKE_DOWNLOADS_DIR=/tmp/
-export NEMESIS_BASE_URL=http://192.168.230.100:8080/
-export NEMESIS_CREDS="nemesis:Qwerty12345"
-# export NEMESIS_DEBUG_JSON="1"
-export NEMESIS_PROJECT=ASSESS-X
-export NEMESIS_DATA_EXPIRATION_DAYS=100
-```
-
-When launching the Cobalt Strike GUI or `agscript`, ensure that `SSLUtils.jar` is added to the classpath, e.g. for `agscript`:
-
-```
-...
-
-java -XX:ParallelGCThreads=4 -XX:+AggressiveHeap -XX:+UseParallelGC -classpath "${CSJAR}:/home/user/Toolkit/cobaltstrike-nemesis-connector/SSLUtils.jar" aggressor.headless.Start $*
-
-```
-
-Then, load `nemesis-connector.cna` using agscript or into the Cobalt Stike client GUI:
-
-```
-./agscript <IP> 50050 nemesis-bot Password123! /home/user/Toolkit/cobaltstrike-nemesis-connector/nemesis-connector.cna
-```
-
-**Note:** the `agscript` command can be run from the teamserver itself, or any other server with Cobalt Strike installed.
-
-
-## Interacting with nemesis-bot
-
-Communication from the client to the `nemesis-bot` on the server occurs through the event log. In order to interact with `nemesis-bot`, go to `Cobalt Strike` -> `Script Console` and use the privmsg() function. All current interaction commands are listed below.
-
-To trigger reprocessing of all downloads:
-
-    aggressor> x privmsg("nemesis-bot", "ReprocessDownloads")
+Aggressor project that ingests Cobalt Strike data into Nemesis.
+
+The project can be run on a teamserver to ingest data tasks by ALL clients, or can be run on an individual client instance to ingest data just from that client.
+
+Also contains the `bof_reg_collect` , a Beacon Object File to collect registry data in a serialized manner.
+
+# Usage
+Make sure the following environment variables are set before running `nemesis-connector.cna`:
+
+| Configuration Variable             | Required | Example                     | Default (if applicable) | Description                                                                                                   |
+|------------------------------------|----------|-----------------------------|-------------------------|---------------------------------------------------------------------------------------------------------------|
+| NEMESIS_COBALTSTRIKE_DOWNLOADS_DIR | Yes      | /tmp/                       |                         | Temporary directory used when syncing downloads from a Cobalt Strike teamserver                               |
+| NEMESIS_BASE_URL                   | Yes      | https://192.168.230.42:8080/ |                         | Base URL used when constructing the URL to web api, elastic, kibana, etc (e.g., NEMESIS_BASE_URL + "/kibana") |
+| NEMESIS_CREDS                      | Yes      | nemesis:Qwerty12345         |                         | Basic auth credentials used when accessing the Nemesis frontend web endpoints                                 |
+| NEMESIS_DEBUG_JSON                 | No       | 1                           | 0                       | Print JSON responses from web API requests                                                                    |
+| NEMESIS_PROJECT                    | Yes      | ASSESS-123                  |                         | Assessmend project ID the teamserver is associated with                                                       |
+| NEMESIS_DATA_EXPIRATION_DAYS       | Yes      | 100                         | 100                     | Number of days after which Nemesis should expire the data                                                     |
+
+Example:
+```
+export NEMESIS_COBALTSTRIKE_DOWNLOADS_DIR=/tmp/
+export NEMESIS_BASE_URL=https://192.168.230.100:8080/
+export NEMESIS_CREDS="nemesis:Qwerty12345"
+# export NEMESIS_DEBUG_JSON="1"
+export NEMESIS_PROJECT=ASSESS-X
+export NEMESIS_DATA_EXPIRATION_DAYS=100
+```
+
+When launching the Cobalt Strike GUI or `agscript`, ensure that `SSLUtils.jar` is added to the classpath, e.g. for `agscript`:
+
+```
+...
+
+java -XX:ParallelGCThreads=4 -XX:+AggressiveHeap -XX:+UseParallelGC -classpath "${CSJAR}:/home/user/Toolkit/cobaltstrike-nemesis-connector/SSLUtils.jar" aggressor.headless.Start $*
+
+```
+
+Then, load `nemesis-connector.cna` using agscript or into the Cobalt Stike client GUI:
+
+```
+./agscript <IP> 50050 nemesis-bot Password123! /home/user/Toolkit/cobaltstrike-nemesis-connector/nemesis-connector.cna
+```
+
+**Note:** the `agscript` command can be run from the teamserver itself, or any other server with Cobalt Strike installed.
+
+
+## Interacting with nemesis-bot
+
+Communication from the client to the `nemesis-bot` on the server occurs through the event log. In order to interact with `nemesis-bot`, go to `Cobalt Strike` -> `Script Console` and use the privmsg() function. All current interaction commands are listed below.
+
+To trigger reprocessing of all downloads:
+
+    aggressor> x privmsg("nemesis-bot", "ReprocessDownloads")