Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into feature/check-on-gh…
Browse files Browse the repository at this point in the history
…-actions

* upstream/main: (313 commits)
  github-action: delete opentelemetry workflow (elastic#39559)
  updatecli: move to the .github folder and support for signed commits (elastic#39472)
  Osquerybeat: Add action responses data stream (elastic#39143)
  [winlogbeat] performance improvment; avoid rendering event message twice (elastic#39544)
  Fix the AWS SDK dependencies issue causing the "not found, ResolveEndpointV2" error (elastic#39454)
  x-pack/filebeat/input/cel: add http metrics collection (elastic#39503)
  build(deps): bump github.com/elastic/elastic-agent-libs from 0.9.4 to 0.9.7 (elastic#39424)
  Remove unused env vars from pipelines (elastic#39534)
  [BK] - Remove osx steps from branch execution (elastic#39552)
  [BK] - Remove certain steps from running for Branches (elastic#39533)
  Allow dependabot report BK status checks (elastic#39540)
  Remove hardcoded module definitions in CI (elastic#39506)
  Explicitly set DOCKER_PULL, RACE_DETECTOR and TEST_COVERAGE for pipelines (elastic#39510)
  Fixed pipelines formatting (elastic#39513)
  Update filebeat pipeline to match Jenkins steps (elastic#39261)
  Add error check to groupToEvents so we don't blindly add error values (elastic#39404)
  Remove fields not needed for session view in add_session_view processor (elastic#39500)
  `aws-s3` input: Split S3 poller and SQS reader into explicit input objects (elastic#39353)
  ci(jenkins): remove post-build notifications (elastic#39483)
  [DOCS] Add the `read_pipeline` cluster privilege for winlogbeat and the `auto_configure` index privilege to beats documentation (elastic#38534)
  ...
  • Loading branch information
v1v committed May 15, 2024
2 parents 5df2fe0 + 8b58916 commit 4c5e0d6
Show file tree
Hide file tree
Showing 1,261 changed files with 73,270 additions and 595,578 deletions.
360 changes: 289 additions & 71 deletions .buildkite/auditbeat/auditbeat-pipeline.yml

Large diffs are not rendered by default.

8 changes: 0 additions & 8 deletions .buildkite/auditbeat/scripts/crosscompile.sh

This file was deleted.

51 changes: 0 additions & 51 deletions .buildkite/auditbeat/scripts/package-step.sh

This file was deleted.

13 changes: 0 additions & 13 deletions .buildkite/auditbeat/scripts/package.sh

This file was deleted.

51 changes: 0 additions & 51 deletions .buildkite/auditbeat/scripts/unit-tests-win.ps1

This file was deleted.

12 changes: 0 additions & 12 deletions .buildkite/auditbeat/scripts/unit-tests.sh

This file was deleted.

114 changes: 112 additions & 2 deletions .buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,115 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json

env:
IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
MODULE: "kubernetes"

# Other deps
ASDF_KIND_VERSION: "0.20.0"

steps:
- label: "Example test"
command: echo "Hello!"
- group: "Deploy/K8S"
if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/

steps:
- label: "Checks"
command: |
set -euo pipefail
make -C deploy/kubernetes all
make check-no-changes
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
notify:
- github_commit_status:
context: "deploy/k8s checks"

- label: "K8S Test/K8S version: v1.29.0"
key: "k8s-test-129"
env:
K8S_VERSION: "v1.29.0"
MODULE: "${MODULE}"
commands: |
set -euo pipefail
source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
echo "--- Executing Tests"
make -C metricbeat integration-tests
make -C deploy/kubernetes test
retry:
automatic:
- limit: 3
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
notify:
- github_commit_status:
context: "deploy/k8s test v1.29.0"

- label: "K8S Test/K8S version: v1.28.0"
key: "k8s-test-128"
env:
K8S_VERSION: "v1.28.0"
MODULE: "${MODULE}"
commands: |
set -euo pipefail
source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
echo "--- Executing Tests"
make -C metricbeat integration-tests
make -C deploy/kubernetes test
retry:
automatic:
- limit: 3
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
notify:
- github_commit_status:
context: "deploy/k8s test v1.28.0"

- label: "K8S Test/K8S version: v1.27.3"
key: "k8s-test-1273"
env:
K8S_VERSION: "v1.27.3"
MODULE: "${MODULE}"
commands: |
set -euo pipefail
source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
echo "--- Executing Tests"
make -C metricbeat integration-tests
make -C deploy/kubernetes test
retry:
automatic:
- limit: 3
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
notify:
- github_commit_status:
context: "deploy/k8s test v1.27.3"

- label: "K8S Test/K8S version: v1.26.6"
key: "k8s-test-1266"
env:
K8S_VERSION: "v1.26.6"
MODULE: "${MODULE}"
commands: |
set -euo pipefail
source .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
echo "--- Executing Tests"
make -C metricbeat integration-tests
make -C deploy/kubernetes test
retry:
automatic:
- limit: 3
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
notify:
- github_commit_status:
context: "deploy/k8s test v1.26.6"
7 changes: 7 additions & 0 deletions .buildkite/deploy/kubernetes/scripts/kind-setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail

kind create cluster --image kindest/node:${K8S_VERSION}

echo "Cluster info: "
kubectl cluster-info
44 changes: 44 additions & 0 deletions .buildkite/deploy/kubernetes/scripts/setup-k8s-env.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
#!/usr/bin/env bash

set -euo pipefail

source .buildkite/env-scripts/util.sh

export KUBECONFIG="${WORKSPACE}/kubecfg"
export BIN="${WORKSPACE}/bin"
add_bin_path

echo "--- Installing kind & kubectl"
asdf plugin add kind
asdf install kind $ASDF_KIND_VERSION

echo "~~~ Setting up kind"
max_retries=3
timeout=5
retries=0

while true; do
echo "Creating cluster"
script_output=$(.buildkite/deploy/kubernetes/scripts/kind-setup.sh 2>&1)
exit_code=$?

echo "Script Output: $script_output"

if [ $exit_code -eq 0 ]; then
break
else
retries=$((retries + 1))

if [ $retries -gt $max_retries ]; then
kind delete cluster
echo "Kind setup FAILED: $script_output"
exit 1
fi

kind delete cluster

sleep_time=$((timeout * retries))
echo "Retry #$retries failed. Retrying after ${sleep_time}s..."
sleep $sleep_time
fi
done
52 changes: 46 additions & 6 deletions .buildkite/env-scripts/env.sh
Original file line number Diff line number Diff line change
@@ -1,19 +1,59 @@
#!/usr/bin/env bash

SETUP_GVM_VERSION="v0.5.1"
source .buildkite/env-scripts/util.sh

DOCS_CHANGESET="^.*\.(asciidoc|md)$
deploy/kubernetes/.*-kubernetes.yaml"
PACKAGING_CHANGESET="^dev-tools/packaging/
^.go-version"

REPO="beats"
WORKSPACE="$(pwd)"
BIN="${WORKSPACE}/bin"
HW_TYPE="$(uname -m)"
PLATFORM_TYPE="$(uname)"
REPO="beats"
TMP_FOLDER="tmp.${REPO}"
SNAPSHOT="true"
PYTEST_ADDOPTS=""
OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"

SETUP_GVM_VERSION="v0.5.1"
ASDF_MAGE_VERSION="1.14.0"
SETUP_WIN_PYTHON_VERSION="3.11.0"

# Docker & DockerHub
DOCKER_COMPOSE_VERSION="1.21.0"
DOCKER_REGISTRY="docker.elastic.co"

export SETUP_GVM_VERSION
ONLY_DOCS=$(changeset_applies "$DOCS_CHANGESET")
PACKAGING_CHANGES=$(changeset_applies "$PACKAGING_CHANGESET")
GO_MOD_CHANGES=$(changeset_applies "^go.mod")

KIND_VERSION="v0.20.0"
KUBECONFIG="${WORKSPACE}/kubecfg"

export REPO
export WORKSPACE
export BIN
export HW_TYPE
export PLATFORM_TYPE
export REPO
export TMP_FOLDER
export SNAPSHOT
export PYTEST_ADDOPTS
export OSS_MODULE_PATTERN
export XPACK_MODULE_PATTERN

export SETUP_GVM_VERSION
export ASDF_MAGE_VERSION
export SETUP_WIN_PYTHON_VERSION

export DOCKER_COMPOSE_VERSION
export DOCKER_REGISTRY

export ONLY_DOCS
export PACKAGING_CHANGES
export GO_MOD_CHANGES

export KIND_VERSION
export KUBECONFIG

add_bin_path
Loading

0 comments on commit 4c5e0d6

Please sign in to comment.