Skip to content

Commit

Permalink
Merge branch 'main' into rule_templates_auto
Browse files Browse the repository at this point in the history
  • Loading branch information
@orouz
orouz authored Mar 26, 2024
2 parents 030059e + 6b9714e commit 9573600
Show file tree
Hide file tree
Showing 42 changed files with 3,299 additions and 203 deletions.
6 changes: 3 additions & 3 deletions cloudbeat.reference.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1195,9 +1195,9 @@ setup.template.settings:
# to load your own lifecycle policy.
#setup.ilm.policy_file:

# Disable the check for an existing lifecycle policy. The default is true. If
# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy
# can be installed.
# Disable the check for an existing lifecycle policy. The default is true.
# If you set this option to false, lifecycle policy will not be installed,
# even if setup.ilm.overwrite is set to true.
#setup.ilm.check_exists: true

# Overwrite the lifecycle policy at startup. The default is false.
Expand Down
6 changes: 4 additions & 2 deletions deploy/test-environments/delete_env.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,9 @@ else
GCP_FILTER="name:'$ENV_PREFIX*'"
fi

ALL_GCP_DEPLOYMENTS=$(gcloud deployment-manager deployments list --filter="$GCP_FILTER" --format="value(name)")
while IFS= read -r line; do
ALL_GCP_DEPLOYMENTS+=("$line")
done < <(gcloud deployment-manager deployments list --filter="$GCP_FILTER" --format="value(name)")

# Divide environments into those to be deleted and those to be skipped
TO_DELETE_ENVS=()
Expand Down Expand Up @@ -163,7 +165,7 @@ printf "%s\n" "${FAILED_STACKS[@]}"
# Delete GCP deployments
PROJECT_NAME=$(gcloud config get-value core/project)
PROJECT_NUMBER=$(gcloud projects list --filter="${PROJECT_NAME}" --format="value(PROJECT_NUMBER)")
./delete_gcp_env.sh "$PROJECT_NAME" "$PROJECT_NUMBER" "$ALL_GCP_DEPLOYMENTS"
./delete_gcp_env.sh "$PROJECT_NAME" "$PROJECT_NUMBER" "${ALL_GCP_DEPLOYMENTS[@]}"

# Delete Azure groups
FAILED_AZURE_GROUPS=()
Expand Down
30 changes: 16 additions & 14 deletions deploy/test-environments/delete_gcp_env.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@ PROJECT_NUMBER=$2
shift 2
GCP_DEPLOYMENTS=("$@")

echo "Project Name: $PROJECT_NAME"
echo "Project Number: $PROJECT_NUMBER"
echo "GCP Deployments: ${GCP_DEPLOYMENTS[*]}"
# Add the needed roles to delete the templates to the project using the deployment manager
gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled

for DEPLOYMENT in "${GCP_DEPLOYMENTS[@]}"; do
# Add the needed roles to delete the templates to the project using the deployment manager
gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled
DELETED_DEPLOYMENTS=()
FAILED_DEPLOYMENTS=()

for DEPLOYMENT in "${GCP_DEPLOYMENTS[@]}"; do
echo "Deleting GCP deployment: $DEPLOYMENT"
if gcloud deployment-manager deployments delete "$DEPLOYMENT" -q; then
echo "Successfully deleted GCP deployment: $DEPLOYMENT"
DELETED_DEPLOYMENTS+=("$DEPLOYMENT")
Expand All @@ -30,18 +30,20 @@ for DEPLOYMENT in "${GCP_DEPLOYMENTS[@]}"; do
FAILED_DEPLOYMENTS+=("$DEPLOYMENT")
fi

# Remove the roles required to deploy the DM templates
gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled

done

# Print summary of gcp deployments deletions
# Remove the roles required to deploy the DM templates
gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled

echo "Successfully deleted GCP deployments (${#DELETED_DEPLOYMENTS[@]}):"
printf "%s\n" "${DELETED_DEPLOYMENTS[@]}"
# Print summary of gcp deployments deletions
if [ ${#DELETED_DEPLOYMENTS[@]} -gt 0 ]; then
printf "%s\n" "${DELETED_DEPLOYMENTS[@]}"
fi

echo "Failed to delete GCP deployments (${#FAILED_DEPLOYMENTS[@]}):"
if [ ${#FAILED_DEPLOYMENTS[@]} -gt 0 ]; then
echo "Failed to delete GCP deployments (${#FAILED_DEPLOYMENTS[@]}):"
printf "%s\n" "${FAILED_DEPLOYMENTS[@]}"
exit 1
fi
12 changes: 12 additions & 0 deletions dev-docs/Cloud-Env-Upgrade.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,18 @@ The [`Test Upgrade Environment`](https://github.com/elastic/cloudbeat/actions/wo
It also facilitates the upgrade of the environment to a new version of the ELK stack and all installed agents, while also performing checks for findings retrieval. For example, if the target ELK version is 8.12.0 and the base version was not selected, the workflow will automatically calculate the previously released version (e.g., 8.11.3), install that version, and then proceed to upgrade to the specified target version (8.12.0). Essentially, this workflow is designed to test the upgrade feature on upcoming versions that are currently in development or will be release candidates (BC).


## Overview of the Upgrade Process

The upgrade process comprises the following main steps:

1. Install the released version, including all integrations (CSPM/KSPM), and deploy their agents.
2. Upgrade the ELK stack version.
3. Upgrade CSPM/KSPM integration versions:
- If the integration has a `preview` version, the workflow will execute a script to update the integration to the latest `preview` version.
- If the latest version is released (no `preview` suffix), the integration upgrade will be automatically performed after the stack upgrade.
4. Upgrade KSPM agents by reapplying Kubernetes manifests with the latest image versions.
5. Upgrade Linux-type agents (CSPM/CNVM) by using the Fleet upgrade API.

## How to Run the Workflow

Follow these steps to run the workflow:
Expand Down
107 changes: 54 additions & 53 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ module github.com/elastic/cloudbeat
go 1.21

require (
cloud.google.com/go/asset v1.17.2
cloud.google.com/go/iam v1.1.6
cloud.google.com/go/asset v1.18.1
cloud.google.com/go/iam v1.1.7
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.3.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/keyvault/armkeyvault v1.4.0
Expand All @@ -20,40 +20,40 @@ require (
github.com/aquasecurity/go-dep-parser v0.0.0-20231120074854-8322cc2242bf
github.com/aquasecurity/trivy v0.48.3
github.com/aquasecurity/trivy-db v0.0.0-20240220070059-88dc6466aa40
github.com/aws/aws-sdk-go v1.50.35
github.com/aws/aws-sdk-go-v2 v1.25.3
github.com/aws/aws-sdk-go-v2/config v1.27.7
github.com/aws/aws-sdk-go-v2/credentials v1.17.7
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3
github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.28.3
github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.3
github.com/aws/aws-sdk-go-v2/service/cloudformation v1.47.1
github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.39.0
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.36.2
github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.34.3
github.com/aws/aws-sdk-go-v2/service/configservice v1.46.2
github.com/aws/aws-sdk-go-v2/service/ec2 v1.150.0
github.com/aws/aws-sdk-go-v2/service/ecr v1.27.2
github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.24.2
github.com/aws/aws-sdk-go-v2/service/iam v1.31.2
github.com/aws/aws-sdk-go-v2/service/kms v1.29.2
github.com/aws/aws-sdk-go-v2/service/organizations v1.27.1
github.com/aws/aws-sdk-go-v2/service/rds v1.75.1
github.com/aws/aws-sdk-go-v2/service/s3 v1.51.4
github.com/aws/aws-sdk-go-v2/service/s3control v1.44.2
github.com/aws/aws-sdk-go-v2/service/securityhub v1.46.2
github.com/aws/aws-sdk-go-v2/service/sns v1.29.2
github.com/aws/aws-sdk-go-v2/service/sts v1.28.4
github.com/aws/aws-sdk-go v1.51.6
github.com/aws/aws-sdk-go-v2 v1.26.0
github.com/aws/aws-sdk-go-v2/config v1.27.9
github.com/aws/aws-sdk-go-v2/credentials v1.17.9
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0
github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.29.0
github.com/aws/aws-sdk-go-v2/service/autoscaling v1.40.4
github.com/aws/aws-sdk-go-v2/service/cloudformation v1.48.0
github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.39.1
github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.36.3
github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.35.0
github.com/aws/aws-sdk-go-v2/service/configservice v1.46.3
github.com/aws/aws-sdk-go-v2/service/ec2 v1.152.0
github.com/aws/aws-sdk-go-v2/service/ecr v1.27.3
github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing v1.24.3
github.com/aws/aws-sdk-go-v2/service/iam v1.31.3
github.com/aws/aws-sdk-go-v2/service/kms v1.30.0
github.com/aws/aws-sdk-go-v2/service/organizations v1.27.2
github.com/aws/aws-sdk-go-v2/service/rds v1.76.0
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0
github.com/aws/aws-sdk-go-v2/service/s3control v1.44.3
github.com/aws/aws-sdk-go-v2/service/securityhub v1.47.0
github.com/aws/aws-sdk-go-v2/service/sns v1.29.3
github.com/aws/aws-sdk-go-v2/service/sts v1.28.5
github.com/aws/smithy-go v1.20.1
github.com/dgraph-io/ristretto v0.1.1
github.com/djherbis/times v1.6.0
github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240122153931-afbba842e18b
github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240322152048-06a11e7cec0f
github.com/elastic/e2e-testing v1.99.2-0.20231005090720-556e60d449dc
github.com/elastic/elastic-agent-autodiscover v0.6.8
github.com/elastic/elastic-agent-client/v7 v7.6.0
github.com/elastic/elastic-agent-client/v7 v7.8.1
github.com/elastic/elastic-agent-libs v0.7.5
github.com/elastic/go-licenser v0.4.1
github.com/elastic/go-ucfg v0.8.6
github.com/elastic/go-ucfg v0.8.7
github.com/gocarina/gocsv v0.0.0-20231116093920-b87c2d0e983a
github.com/gofrs/uuid v4.4.0+incompatible
github.com/googleapis/gax-go/v2 v2.12.2
Expand All @@ -76,7 +76,7 @@ require (
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616
golang.org/x/oauth2 v0.17.0
google.golang.org/api v0.167.0
google.golang.org/api v0.169.0
gopkg.in/yaml.v2 v2.4.0
gotest.tools/gotestsum v1.11.0
k8s.io/api v0.28.4
Expand All @@ -88,6 +88,7 @@ require (

require (
dario.cat/mergo v1.0.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.6.0 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/aquasecurity/trivy-iac v0.7.1 // indirect
github.com/aquasecurity/trivy-policies v0.6.1-0.20231120231532-f6f2330bf842 // indirect
Expand All @@ -109,7 +110,7 @@ require (
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.48.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect
Expand All @@ -119,14 +120,14 @@ require (
)

require (
cloud.google.com/go v0.112.0 // indirect
cloud.google.com/go v0.112.1 // indirect
cloud.google.com/go/accesscontextmanager v1.8.5 // indirect
cloud.google.com/go/compute v1.24.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/longrunning v0.5.5 // indirect
cloud.google.com/go/orgpolicy v1.12.1 // indirect
cloud.google.com/go/osconfig v1.12.5 // indirect
cloud.google.com/go/storage v1.36.0 // indirect
cloud.google.com/go/storage v1.38.0 // indirect
code.cloudfoundry.org/go-diodes v0.0.0-20190809170250-f77fb823c7ee // indirect
code.cloudfoundry.org/go-loggregator v7.4.0+incompatible // indirect
code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f // indirect
Expand Down Expand Up @@ -180,18 +181,18 @@ require (
github.com/armon/go-radix v1.0.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.4 // indirect
github.com/aws/aws-sdk-go-v2/service/ebs v1.21.7 // indirect
github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.21.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.6 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.20.3 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.3 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
github.com/bmatcuk/doublestar/v4 v4.6.0 // indirect
Expand Down Expand Up @@ -224,7 +225,7 @@ require (
github.com/dnephin/pflag v1.0.7 // indirect
github.com/docker/cli v24.0.5+incompatible // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v24.0.7+incompatible // indirect
github.com/docker/docker v24.0.9+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
Expand All @@ -237,14 +238,14 @@ require (
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 // indirect
github.com/elastic/elastic-agent-system-metrics v0.9.1 // indirect
github.com/elastic/elastic-agent-system-metrics v0.9.2 // indirect
github.com/elastic/go-concert v0.2.0 // indirect
github.com/elastic/go-lumber v0.1.2-0.20220819171948-335fde24ea0f // indirect
github.com/elastic/go-seccomp-bpf v1.4.0 // indirect
github.com/elastic/go-structform v0.0.10 // indirect
github.com/elastic/go-sysinfo v1.11.2 // indirect
github.com/elastic/go-sysinfo v1.13.1 // indirect
github.com/elastic/go-windows v1.0.1 // indirect
github.com/elastic/gosigar v0.14.2 // indirect
github.com/elastic/gosigar v0.14.3 // indirect
github.com/elazarl/goproxy/ext v0.0.0-20230731152917-f99041a5c027 // indirect
github.com/elliotchance/orderedmap v1.5.1 // indirect
github.com/emicklei/go-restful/v3 v3.10.1 // indirect
Expand Down Expand Up @@ -282,7 +283,7 @@ require (
github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
github.com/golang/glog v1.2.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gomodule/redigo v1.8.3 // indirect
github.com/google/btree v1.1.2 // indirect
Expand Down Expand Up @@ -467,21 +468,21 @@ require (
go.opentelemetry.io/otel/trace v1.24.0 // indirect
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.20.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/mod v0.15.0 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sync v0.6.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/term v0.17.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20240228224816-df926f6c8641 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240228224816-df926f6c8641 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240228224816-df926f6c8641 // indirect
google.golang.org/grpc v1.62.0 // indirect
google.golang.org/protobuf v1.32.0
google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect
google.golang.org/grpc v1.62.1 // indirect
google.golang.org/protobuf v1.33.0
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
Expand Down
Loading

0 comments on commit 9573600

Please sign in to comment.