Merge branch 'master' into canvas/fix-32937

.buildkite/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "kibana-buildkite",
+  "version": "1.0.0",
+  "private": true,
+  "dependencies": {
+    "kibana-buildkite-library": "elastic/kibana-buildkite-library"
+  }
+}

.buildkite/pipelines/pull_request.yml

@@ -0,0 +1,17 @@
+env:
+  GITHUB_COMMIT_STATUS_ENABLED: 'true'
+  GITHUB_COMMIT_STATUS_CONTEXT: 'buildkite/kibana-pull-request'
+steps:
+  - command: .buildkite/scripts/lifecycle/pre_build.sh
+    label: Pre-Build
+
+  - wait
+
+  - command: echo 'Hello World'
+    label: Test
+
+  - wait: ~
+    continue_on_failure: true
+
+  - command: .buildkite/scripts/lifecycle/post_build.sh
+    label: Post-Build

.buildkite/pipelines/update_demo_env.yml

@@ -0,0 +1,8 @@
+steps:
+  - command: .buildkite/scripts/steps/demo_env/es_and_init.sh
+    label: Initialize Environment and Deploy ES
+
+  - command: .buildkite/scripts/steps/demo_env/kibana.sh
+    label: Build and Deploy Kibana
+    agents:
+      queue: c2-8

.buildkite/scripts/lifecycle/build_status.js

@@ -0,0 +1,17 @@
+const { BuildkiteClient } = require('kibana-buildkite-library');
+
+(async () => {
+  try {
+    const client = new BuildkiteClient();
+    const status = await client.getCurrentBuildStatus();
+    console.log(status.success ? 'true' : 'false');
+    process.exit(0);
+  } catch (ex) {
+    if (ex.response) {
+      console.error('HTTP Error Response Body', ex.response.data);
+      console.error('HTTP Error Response Status', ex.response.status);
+    }
+    console.error(ex);
+    process.exit(1);
+  }
+})();

.buildkite/scripts/lifecycle/ci_stats_complete.js

@@ -1,15 +1,8 @@
-const ciStats = require('./ci_stats');
-
-// TODO - this is okay for now but should really be replaced with an API call, especially once retries are enabled
-const BUILD_STATUS = process.env.BUILD_FAILED === 'true' ? 'FAILURE' : 'SUCCESS';
+const { CiStats } = require('kibana-buildkite-library');
 
 (async () => {
   try {
-    if (process.env.CI_STATS_BUILD_ID) {
-      await ciStats.post(`/v1/build/_complete?id=${process.env.CI_STATS_BUILD_ID}`, {
-        result: BUILD_STATUS,
-      });
-    }
+    await CiStats.onComplete();
   } catch (ex) {
     console.error(ex);
     process.exit(1);

.buildkite/scripts/lifecycle/ci_stats_start.js

@@ -1,28 +1,8 @@
-const { execSync } = require('child_process');
-const ciStats = require('./ci_stats');
+const { CiStats } = require('kibana-buildkite-library');
 
 (async () => {
   try {
-    const build = await ciStats.post('/v1/build', {
-      jenkinsJobName: process.env.BUILDKITE_PIPELINE_NAME,
-      jenkinsJobId: process.env.BUILDKITE_BUILD_ID,
-      jenkinsUrl: process.env.BUILDKITE_BUILD_URL,
-      prId: process.env.GITHUB_PR_NUMBER || null,
-    });
-
-    execSync(`buildkite-agent meta-data set ci_stats_build_id "${build.id}"`);
-
-    // TODO Will need to set MERGE_BASE for PRs
-
-    await ciStats.post(`/v1/git_info?buildId=${build.id}`, {
-      branch: process.env.BUILDKITE_BRANCH.replace(/^(refs\/heads\/|origin\/)/, ''),
-      commit: process.env.BUILDKITE_COMMIT,
-      targetBranch:
-        process.env.GITHUB_PR_TARGET_BRANCH ||
-        process.env.BUILDKITE_PULL_REQUEST_BASE_BRANCH ||
-        null,
-      mergeBase: process.env.GITHUB_PR_MERGE_BASE || null, // TODO confirm GITHUB_PR_MERGE_BASE or switch to final var
-    });
+    await CiStats.onStart();
   } catch (ex) {
     console.error(ex);
     process.exit(1);

.buildkite/scripts/lifecycle/commit_status_complete.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 
 if [[ "${GITHUB_COMMIT_STATUS_ENABLED:-}" == "true" ]]; then
   COMMIT_STATUS=success
-  if [[ "${BUILD_FAILED:-}"  == "true" ]]; then
+  if [[ "${BUILD_SUCCESSFUL:-}"  != "true" ]]; then
     COMMIT_STATUS=failure
   fi
 

.buildkite/scripts/lifecycle/post_build.sh

@@ -2,8 +2,8 @@
 
 set -euo pipefail
 
-BUILD_FAILED=$(buildkite-agent meta-data get build_failed --default "false")
-export BUILD_FAILED
+BUILD_SUCCESSFUL=$(node "$(dirname "${0}")/build_status.js")
+export BUILD_SUCCESSFUL
 
 "$(dirname "${0}")/commit_status_complete.sh"
 

.buildkite/scripts/lifecycle/pre_command.sh

@@ -2,6 +2,13 @@
 
 set -euo pipefail
 
+cd '.buildkite'
+yarn install
+cd -
+
+BUILDKITE_TOKEN="$(vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
+export BUILDKITE_TOKEN
+
 # Set up a custom ES Snapshot Manifest if one has been specified for this build
 {
   ES_SNAPSHOT_MANIFEST=${ES_SNAPSHOT_MANIFEST:-$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST --default '')}

.buildkite/scripts/steps/demo_env/Dockerfile

@@ -0,0 +1,4 @@
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+COPY ./* /var/lib/example_plugins
+RUN find /var/lib/example_plugins/ -type f -name '*.zip' | xargs -I % /usr/share/kibana/bin/kibana-plugin install 'file://%'

.buildkite/scripts/steps/demo_env/auth.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo '--- Auth and set up kubectl'
+
+gcloud container clusters get-credentials demo-env --region us-central1 --project elastic-kibana-184716
+kubectl config use-context gke_elastic-kibana-184716_us-central1_demo-env

.buildkite/scripts/steps/demo_env/config.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+DEPLOYMENT_VERSION=$(jq -r .version package.json)
+export DEPLOYMENT_VERSION
+
+export DEPLOYMENT_MINOR_VERSION="${DEPLOYMENT_VERSION%.*}"
+export DEPLOYMENT_NAME="kb-${DEPLOYMENT_MINOR_VERSION/./-}"

.buildkite/scripts/steps/demo_env/es_and_init.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "${0}")/config.sh"
+
+"$(dirname "${0}")/auth.sh"
+
+echo '--- Import and publish Elasticsearch image'
+
+mkdir -p target
+
+export ES_IMAGE="gcr.io/elastic-kibana-184716/demo/elasticsearch:$DEPLOYMENT_NAME-$(git rev-parse HEAD)"
+
+DOCKER_EXPORT_URL=$(curl https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$DEPLOYMENT_VERSION/manifest-latest-verified.json | jq -r '.archives | .[] | select(.platform=="docker") | .url')
+curl "$DOCKER_EXPORT_URL" > target/elasticsearch-docker.tar.gz
+docker load < target/elasticsearch-docker.tar.gz
+docker tag "docker.elastic.co/elasticsearch/elasticsearch:$DEPLOYMENT_VERSION-SNAPSHOT" "$ES_IMAGE"
+docker push "$ES_IMAGE"
+
+echo '--- Prepare yaml'
+
+TEMPLATE=$(envsubst < "$(dirname "${0}")/es_and_init.yml")
+
+echo "$TEMPLATE"
+
+cat << EOF | buildkite-agent annotate --style "info" --context demo-env-info
+The demo environment can be accessed here, once Kibana and ES are running:
+
+https://demo.kibana.dev/$DEPLOYMENT_MINOR_VERSION
+
+Logs, etc can be found here:
+
+https://console.cloud.google.com/kubernetes/workload?project=elastic-kibana-184716&pageState=(%22savedViews%22:(%22n%22:%5B%22${DEPLOYMENT_NAME}%22%5D,%22c%22:%5B%22gke%2Fus-central1%2Fdemo-env%22%5D))
+
+EOF
+
+echo '--- Deploy yaml'
+echo "$TEMPLATE" | kubectl apply -f -

.buildkite/scripts/steps/demo_env/es_and_init.yml

@@ -0,0 +1,63 @@
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: $DEPLOYMENT_NAME
+  labels:
+    name: $DEPLOYMENT_NAME
+---
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: $DEPLOYMENT_NAME
+  namespace: $DEPLOYMENT_NAME
+spec:
+  version: $DEPLOYMENT_VERSION
+  image: $ES_IMAGE
+  nodeSets:
+    - name: default
+      count: 1
+      config:
+        node.store.allow_mmap: false
+        xpack.security.authc:
+          anonymous:
+            roles: viewer
+            authz_exception: true
+      podTemplate:
+        spec:
+          containers:
+            - name: elasticsearch
+              env:
+                - name: ES_JAVA_OPTS
+                  value: -Xms2g -Xmx2g
+              resources:
+                requests:
+                  memory: 4Gi
+                  cpu: 2
+                limits:
+                  memory: 4Gi
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: $DEPLOYMENT_NAME
+  namespace: $DEPLOYMENT_NAME
+  annotations:
+    kubernetes.io/ingress.class: 'nginx'
+    cert-manager.io/cluster-issuer: 'letsencrypt-prod'
+    nginx.ingress.kubernetes.io/rewrite-target: /$2
+spec:
+  tls:
+    - hosts:
+        - demo.kibana.dev
+      secretName: tls-certificate
+  rules:
+    - host: demo.kibana.dev
+      http:
+        paths:
+          - path: /$DEPLOYMENT_MINOR_VERSION(/|$)(.*)
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: $DEPLOYMENT_NAME-kb-http
+                port:
+                  number: 5601

.buildkite/scripts/steps/demo_env/kibana.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export DISABLE_BOOTSTRAP_VALIDATION=true
+export BUILD_TS_REFS_DISABLE=true
+
+.buildkite/scripts/bootstrap.sh
+
+source "$(dirname "${0}")/config.sh"
+
+export KIBANA_IMAGE="gcr.io/elastic-kibana-184716/demo/kibana:$DEPLOYMENT_NAME-$(git rev-parse HEAD)"
+
+echo '--- Build Kibana'
+node scripts/build --debug --docker-images --example-plugins --skip-os-packages --skip-docker-ubi
+
+echo '--- Build Docker image with example plugins'
+cd target/example_plugins
+BUILT_IMAGE="docker.elastic.co/kibana/kibana:$DEPLOYMENT_VERSION-SNAPSHOT"
+docker build --build-arg BASE_IMAGE="$BUILT_IMAGE" -t "$KIBANA_IMAGE" -f "$KIBANA_DIR/.buildkite/scripts/steps/demo_env/Dockerfile" .
+docker push "$KIBANA_IMAGE"
+cd -
+
+"$(dirname "${0}")/auth.sh"
+
+echo '--- Prepare yaml'
+
+TEMPLATE=$(envsubst < "$(dirname "${0}")/kibana.yml")
+
+echo "$TEMPLATE"
+
+echo '--- Deploy yaml'
+echo "$TEMPLATE" | kubectl apply -f -