Merge pull request vectordotdev#379 from answerbook/feature/LOG-18882

feat: Vector upstream 0.32.2
aholmberg · Jan 3, 2024 · 8bd9860 · 8bd9860
2 parents 9728bcd + 4742c2f
commit 8bd9860
Show file tree

Hide file tree

Showing 516 changed files with 14,051 additions and 7,055 deletions.
diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md
@@ -41,8 +41,8 @@ On the day of release:
 - [ ] Merge release preparation branch into the release branch
     - `git co v0.<new version number> && git merge --ff-only prepare-v0.<new version number>`
 - [ ] Tag new release
-  - [ ] `git tag v0.<minor>.0 -a -m v0.<minor>.0``
-  - [ ] `git push origin v0.<minor>.0
+  - [ ] `git tag v0.<minor>.0 -a -m v0.<minor>.0`
+  - [ ] `git push origin v0.<minor>.0`
 - [ ] Wait for release workflow to complete
   - Discoverable via [https://github.com/timberio/vector/actions/workflows/release.yml](https://github.com/timberio/vector/actions/workflows/release.yml)
 - [ ] Release updated Helm chart. See [releasing Helm chart](https://github.com/vectordotdev/helm-charts#releasing).

diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt
@@ -36,6 +36,7 @@ Celkon
 Ceph
 Chromecast
 Citrix
+Cloudflare
 Cloudfone
 Cmx
 Coby
@@ -52,6 +53,7 @@ Enot
 Evercoss
 Explay
 FAQs
+FDO
 FQDNs
 Fabro
 Figma
@@ -165,6 +167,7 @@ Samsung
 Sega
 Segoe
 Shopify
+SIGINTs
 Simvalley
 Skype
 Skytex
@@ -244,6 +247,7 @@ compiletime
 coredns
 corejs
 coreutils
+curta
 daemonset
 databend
 datacenter
@@ -295,6 +299,9 @@ gpg
 gql
 grafana
 graphiql
+greptime
+greptimecloud
+greptimedb
 gvisor
 gws
 hadoop
@@ -313,6 +320,7 @@ https
 humungus
 icecream
 ifeq
+ifneq
 imobile
 influxd
 ionik
@@ -418,6 +426,8 @@ userguide
 webhdfs
 winapi
 workarounds
+XCHACHA
+XSALSA
 yandex
 zeek
 zookeeper

diff --git a/.github/actions/spelling/excludes.txt b/.github/actions/spelling/excludes.txt
@@ -48,6 +48,7 @@
 \.otf$
 \.p12$
 \.pattern$
+\.pb$
 \.pdf$
 \.pem$
 \.png$
@@ -68,11 +69,13 @@
 \.zst$
 ^\.github/actions/spelling/
 ^\Q.cargo/config.toml\E$
-^\Q.github/workflows/spelling.yml\E$
 ^\Q.github/workflows/gardener_issue_comment.yml\E$
+^\Q.github/workflows/spelling.yml\E$
 ^\Qbenches/codecs/moby_dick.txt\E$
 ^\Qbenches/dnstap/mod.rs\E$
 ^\Qbenches/transform/route.rs\E$
+^\Qlib/codecs/tests/data/decoding/protobuf/test_protobuf.desc\E$
+^\Qlib/codecs/tests/data/decoding/protobuf/test_protobuf3.desc\E$
 ^\Qlib/dnsmsg-parser/benches/benches.rs\E$
 ^\Qlib/dnsmsg-parser/src/dns_message_parser.rs\E$
 ^\Qlib/lookup/tests/fixtures/lookup/quoted\E$

diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt
@@ -1,9 +1,7 @@
 abcd
 abcdefghijklm
 abcdefghijklmnopqrstuvwxyzand
-abced
 abortable
-acb
 ack'ing
 acking
 Acq
@@ -37,10 +35,8 @@ ARNOTAREALIDD
 arshiyasolei
 asdf
 asdfasdf
-ASMS
 assertverify
 Asterix
-asynk
 atag
 atx
 aty
@@ -140,7 +136,6 @@ casttype
 castvalue
 cbe
 CBOR
-cbs
 cddl
 cdeab
 cdylib
@@ -224,6 +219,7 @@ cwl
 Dailywarehousing
 daschl
 dashmap
+datadir
 dataflows
 datafuselabs
 datasources
@@ -234,6 +230,7 @@ datname
 dbkind
 dbreader
 DBserver
+DCMAKE
 ddagent
 ddev
 ddmetric
@@ -304,7 +301,6 @@ ebfcee
 edenhill
 edns
 eeyun
-efg
 efgh
 Elhage
 emca
@@ -325,7 +321,6 @@ ENVARS
 envsubst
 EOIG
 EOL'ed
-Erfxl
 Err'ing
 errorf
 Errorsfor
@@ -464,12 +459,12 @@ gty
 Guangzhou
 guenter
 gzip'ed
+hadolint
 halfsies
 hannes
 Hashbang
 hashbrown
 hashindex
-hashlink
 hashring
 hashset
 hashsum
@@ -574,7 +569,6 @@ kernelmode
 keybase
 keyclock
 keyid
-keypair
 keyxxxxx
 khvzak
 kib
@@ -778,6 +772,7 @@ NQTP
 nresamples
 nullishness
 numbackends
+oahd
 oap
 OKD
 omfwd
@@ -821,6 +816,7 @@ pathgen
 peekable
 PEMS
 pgmajfault
+pgrep
 PII
 Pitbull
 pkc
@@ -873,6 +869,7 @@ qwe
 raboof
 rande
 RANDFILE
+ratatui
 rawconfig
 rawstring
 rdkafka
@@ -1053,7 +1050,6 @@ supertrait
 suser
 sustainability
 svalue
-Sya
 sysfs
 sysinit
 syslogng
@@ -1114,7 +1110,6 @@ Tomola
 tonydanza
 toolbars
 toolchains
-TOOLSDIRECTORY
 toolset
 toor
 topdir
@@ -1201,6 +1196,7 @@ watchexec
 watchlogs
 wayfor
 webgraphviz
+webpki
 webservers
 websites
 weee

diff --git a/.github/workflows/changes.yml.upstream b/.github/workflows/changes.yml.upstream
@@ -70,6 +70,8 @@ on:
         value: ${{ jobs.int_tests.outputs.fluent }}
       gcp:
         value: ${{ jobs.int_tests.outputs.gcp }}
+      greptimedb:
+        value: ${{ jobs.int_tests.outputs.greptimedb }}
       humio:
         value: ${{ jobs.int_tests.outputs.humio }}
       http-client:
@@ -194,6 +196,7 @@ jobs:
       eventstoredb: ${{ steps.filter.outputs.eventstoredb }}
       fluent: ${{ steps.filter.outputs.fluent }}
       gcp: ${{ steps.filter.outputs.gcp }}
+      greptimedb: ${{ steps.filter.outputs.greptimedb }}
       humio: ${{ steps.filter.outputs.humio }}
       http-client: ${{ steps.filter.outputs.http-client }}
       influxdb: ${{ steps.filter.outputs.influxdb }}

diff --git a/.github/workflows/comment-trigger.yml.upstream b/.github/workflows/comment-trigger.yml.upstream
@@ -15,6 +15,7 @@
 # /ci-run-unit-windows        : runs Unit - Windows
 # /ci-run-environment         : runs Environment Suite
 # /ci-run-regression          : runs Regression Detection Suite
+# /ci-run-k8s                 : runs K8s E2E Suite
 
 name: Comment Trigger
 
@@ -55,15 +56,22 @@ jobs:
           || contains(github.event.comment.body, '/ci-run-unit-windows')
           || contains(github.event.comment.body, '/ci-run-environment')
           || contains(github.event.comment.body, '/ci-run-regression')
+          || contains(github.event.comment.body, '/ci-run-k8s')
       )
     steps:
+      - name: Generate authentication token
+        id: generate_token
+        uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92
+        with:
+          app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }}
+          private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }}
       - name: Get PR comment author
         id: comment
         uses: tspascoal/get-user-teams-membership@v2
         with:
           username: ${{ github.actor }}
           team: 'Vector'
-          GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }}
+          GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }}
 
       - name: Validate author membership
         if: steps.comment.outputs.isTeamMember == 'false'
@@ -116,3 +124,9 @@ jobs:
     if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-regression')
     uses: ./.github/workflows/regression.yml
     secrets: inherit
+
+  k8s:
+    needs: validate
+    if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-k8s')
+    uses: ./.github/workflows/k8s_e2e.yml
+    secrets: inherit
diff --git a/.github/workflows/environment.yml.upstream b/.github/workflows/environment.yml.upstream
@@ -42,7 +42,7 @@ jobs:
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2.2.0
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2.8.0
+        uses: docker/setup-buildx-action@v2.9.1
       - name: Login to DockerHub
         uses: docker/login-action@v2.1.0
         if: github.ref == 'refs/heads/master'

diff --git a/.github/workflows/gardener_issue_comment.yml.upstream b/.github/workflows/gardener_issue_comment.yml.upstream
@@ -20,50 +20,74 @@ jobs:
           GH_TOKEN: ${{ secrets.GH_PROJECT_PAT }}
         run: |
           issue_id=${{ github.event.issue.node_id }}
+          echo "issue_id: $issue_id"
 
           # IDs fetched from https://docs.github.com/en/graphql/overview/explorer
           project_id="PVT_kwDOAQFeYs4AAsTr" # Gardener
           status_field_id="PVTF_lADOAQFeYs4AAsTrzgAXRuU" # Status
           triage_option_id="2a08fafa"
 
-          # ensures that the issue is already on board but also seems to be the only way to fetch
-          # the item id
-          item_id="$(gh api graphql -f query='
-            mutation($project_id: ID!, $content_id: ID!) {
-                addProjectV2ItemById(input: {projectId: $project_id, contentId: $content_id}) {
-                  item {
-                    id
-                  }
-                }
-            }' -f project_id="$project_id" -f content_id="$issue_id" -q '.data.addProjectV2ItemById.item.id'
-          )"
-
-          echo "item_id: $item_id"
-
-          if [ -z "$item_id" ] ; then
-            echo "Issue not found in Gardener board"
-            exit 0
-          else
-            echo "Found issue on Gardener board"
-          fi
-
-          current_status="$(gh api graphql -f query='
+          # Query for project items for the given issue
+          project_items="$(gh api graphql -f query='
             query($item_id: ID!) {
               node(id: $item_id) {
-                ... on ProjectV2Item {
-                  fieldValueByName(name: "Status") {
-                    ... on ProjectV2ItemFieldSingleSelectValue {
-                      name
+                ... on Issue {
+                  projectItems(first: 50) {
+                    ... on ProjectV2ItemConnection {
+                      nodes {
+                        fieldValueByName(name: "Status") {
+                          ... on ProjectV2ItemFieldSingleSelectValue {
+                            name
+                          }
+                        }
+                        ... on ProjectV2Item {
+                          id
+                          project {
+                            ... on ProjectV2 {
+                              id
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+                ... on PullRequest {
+                  projectItems(first: 50) {
+                    ... on ProjectV2ItemConnection {
+                      nodes {
+                        fieldValueByName(name: "Status") {
+                          ... on ProjectV2ItemFieldSingleSelectValue {
+                            name
+                          }
+                        }
+                        ... on ProjectV2Item {
+                          id
+                          project {
+                            ... on ProjectV2 {
+                              id
+                            }
+                          }
+                        }
+                      }
                     }
                   }
                 }
               }
-            }' -f item_id="$item_id"
+            }' -f item_id="$issue_id"
           )"
 
-          current_status=$(echo $current_status | jq -c -r '.["data"]["node"]["fieldValueByName"]["name"]')
+          # Extract the item in the Gardener project
+          project=$(echo $project_items | jq -c -r --arg project_id $project_id '.data.node.projectItems.nodes[] | select(.project.id == $project_id)')
+          current_status=$(echo $project | jq -c -r '.fieldValueByName.name')
+          item_id=$(echo $project | jq -c '.id')
 
-          echo "Current issue status is: '${current_status}'"
+          if [ -z "$current_status" ] ; then
+            echo "Issue not found in Gardener board"
+            exit 0
+          else
+            echo "Found issue on Gardener board. Current issue status is: '${current_status}'"
+          fi
 
           if [ "$current_status" = "Blocked / Waiting" ] ; then
             echo "Moving issue from 'Blocked / Waiting' to 'Triage'"