Merge branch 'main' into 122318-index-pattern-deprecated

elastic · Mar 2, 2022 · 438cb7b · 438cb7b
2 parents 66e6924 + 774fd63
commit 438cb7b
Show file tree

Hide file tree

Showing 1,596 changed files with 58,137 additions and 26,808 deletions.
diff --git a/.buildkite/scripts/steps/cloud/build_and_deploy.sh b/.buildkite/scripts/steps/cloud/build_and_deploy.sh
@@ -42,7 +42,9 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
     .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" |
     .name = "'$CLOUD_DEPLOYMENT_NAME'" |
     .resources.kibana[0].plan.kibana.version = "'$VERSION'" |
-    .resources.elasticsearch[0].plan.elasticsearch.version = "'$VERSION'"
+    .resources.elasticsearch[0].plan.elasticsearch.version = "'$VERSION'" |
+    .resources.enterprise_search[0].plan.enterprise_search.version = "'$VERSION'" |
+    .resources.integrations_server[0].plan.integrations_server.version = "'$VERSION'"
     ' .buildkite/scripts/steps/cloud/deploy.json > /tmp/deploy.json
 
   ecctl deployment create --track --output json --file /tmp/deploy.json &> "$JSON_FILE"
@@ -51,6 +53,13 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
   CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$JSON_FILE")
   CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$JSON_FILE")
 
+  # Enable stack monitoring
+  jq '
+    .settings.observability.metrics.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'" |
+    .settings.observability.logging.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'"
+    ' .buildkite/scripts/steps/cloud/stack_monitoring.json > /tmp/stack_monitoring.json
+  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/stack_monitoring.json &> "$JSON_FILE"
+
   # Refresh vault token
   VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
   VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
@@ -59,11 +68,10 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
 
   retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
 else
-  ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
-    .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" |
-    .resources.kibana[0].plan.kibana.version = "'$VERSION'" |
-    .resources.elasticsearch[0].plan.elasticsearch.version = "'$VERSION'"
-    ' > /tmp/deploy.json
+ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
+  .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" |
+  (.. | select(.version? != null).version) = "'$VERSION'"
+  ' > /tmp/deploy.json
   ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json &> "$JSON_FILE"
 fi
 

diff --git a/.buildkite/scripts/steps/cloud/deploy.json b/.buildkite/scripts/steps/cloud/deploy.json
@@ -1,5 +1,27 @@
 {
   "resources": {
+    "integrations_server": [
+      {
+        "elasticsearch_cluster_ref_id": "main-elasticsearch",
+        "region": "gcp-us-west2",
+        "plan": {
+          "cluster_topology": [
+            {
+              "instance_configuration_id": "gcp.integrationsserver.1",
+              "zone_count": 1,
+              "size": {
+                "value": 512,
+                "resource": "memory"
+              }
+            }
+          ],
+          "integrations_server": {
+            "version": null
+          }
+        },
+        "ref_id": "main-integrations_server"
+      }
+    ],
     "elasticsearch": [
       {
         "region": "gcp-us-west2",
@@ -12,7 +34,10 @@
             {
               "zone_count": 2,
               "instance_configuration_id": "gcp.coordinating.1",
-              "node_roles": ["ingest", "remote_cluster_client"],
+              "node_roles": [
+                "ingest",
+                "remote_cluster_client"
+              ],
               "id": "coordinating",
               "size": {
                 "resource": "memory",
@@ -54,7 +79,10 @@
                 "enabled_built_in_plugins": []
               },
               "instance_configuration_id": "gcp.data.highstorage.1",
-              "node_roles": ["data_warm", "remote_cluster_client"],
+              "node_roles": [
+                "data_warm",
+                "remote_cluster_client"
+              ],
               "id": "warm",
               "size": {
                 "resource": "memory",
@@ -70,7 +98,10 @@
                 "enabled_built_in_plugins": []
               },
               "instance_configuration_id": "gcp.data.highstorage.1",
-              "node_roles": ["data_cold", "remote_cluster_client"],
+              "node_roles": [
+                "data_cold",
+                "remote_cluster_client"
+              ],
               "id": "cold",
               "size": {
                 "resource": "memory",
@@ -86,7 +117,9 @@
                 "enabled_built_in_plugins": []
               },
               "instance_configuration_id": "gcp.es.datafrozen.n1.64x10x95",
-              "node_roles": ["data_frozen"],
+              "node_roles": [
+                "data_frozen"
+              ],
               "id": "frozen",
               "size": {
                 "resource": "memory",
@@ -96,7 +129,10 @@
             {
               "zone_count": 3,
               "instance_configuration_id": "gcp.master.1",
-              "node_roles": ["master", "remote_cluster_client"],
+              "node_roles": [
+                "master",
+                "remote_cluster_client"
+              ],
               "id": "master",
               "size": {
                 "resource": "memory",
@@ -109,11 +145,14 @@
             {
               "zone_count": 1,
               "instance_configuration_id": "gcp.ml.1",
-              "node_roles": ["ml", "remote_cluster_client"],
+              "node_roles": [
+                "ml",
+                "remote_cluster_client"
+              ],
               "id": "ml",
               "size": {
-                "resource": "memory",
-                "value": 0
+                "value": 1024,
+                "resource": "memory"
               },
               "elasticsearch": {
                 "enabled_built_in_plugins": []
@@ -130,7 +169,33 @@
         "ref_id": "main-elasticsearch"
       }
     ],
-    "enterprise_search": [],
+    "enterprise_search": [
+      {
+        "elasticsearch_cluster_ref_id": "main-elasticsearch",
+        "region": "gcp-us-west2",
+        "plan": {
+          "cluster_topology": [
+            {
+              "node_type": {
+                "connector": true,
+                "appserver": true,
+                "worker": true
+              },
+              "instance_configuration_id": "gcp.enterprisesearch.1",
+              "zone_count": 1,
+              "size": {
+                "resource": "memory",
+                "value": 2048
+              }
+            }
+          ],
+          "enterprise_search": {
+            "version": null
+          }
+        },
+        "ref_id": "main-enterprise_search"
+      }
+    ],
     "kibana": [
       {
         "elasticsearch_cluster_ref_id": "main-elasticsearch",
@@ -153,8 +218,7 @@
         },
         "ref_id": "main-kibana"
       }
-    ],
-    "apm": []
+    ]
   },
   "name": null,
   "metadata": {

diff --git a/.buildkite/scripts/steps/cloud/stack_monitoring.json b/.buildkite/scripts/steps/cloud/stack_monitoring.json
@@ -0,0 +1,19 @@
+{
+  "prune_orphans": false,
+  "settings": {
+    "observability": {
+      "metrics": {
+        "destination": {
+          "deployment_id": null,
+          "ref_id": "main-elasticsearch"
+        }
+      },
+      "logging": {
+        "destination": {
+          "deployment_id": null,
+          "ref_id": "main-elasticsearch"
+        }
+      }
+    }
+  }
+}
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -422,6 +422,9 @@ x-pack/test/security_solution_cypress @elastic/security-engineering-productivity
 # Security Asset Management
 /x-pack/plugins/osquery @elastic/security-asset-management
 
+# Cloud Posture Security
+/x-pack/plugins/cloud_security_posture/ @elastic/cloud-posture-security
+
 # Design (at the bottom for specificity of SASS files)
 **/*.scss  @elastic/kibana-design
 #CC# /packages/kbn-ui-framework/ @elastic/kibana-design

diff --git a/.github/workflows/add-to-apm-project.yml b/.github/workflows/add-to-apm-project.yml
@@ -14,7 +14,7 @@ jobs:
         with:
           headers: '{"GraphQL-Features": "projects_next_graphql"}'
           query: |
-            mutation add_to_project($projectid:String!,$contentid:String!) {
+            mutation add_to_project($projectid:ID!,$contentid:ID!) {
               addProjectNextItem(input:{projectId:$projectid contentId:$contentid}) {
                 projectNextItem {
                   id
@@ -31,7 +31,7 @@ jobs:
         with:
           headers: '{"GraphQL-Features": "projects_next_graphql"}'
           query: |
-            mutation label_team($projectid:String!,$itemid:String!,$fieldid:String!,$value:String!) {
+            mutation label_team($projectid:ID!,$itemid:ID!,$fieldid:ID!,$value:String!) {
               updateProjectNextItemField(input: { projectId:$projectid itemId:$itemid fieldId:$fieldid value:$value }) {
                 projectNextItem {
                   id

diff --git a/.github/workflows/add-to-fleet-project.yml b/.github/workflows/add-to-fleet-project.yml
@@ -20,7 +20,7 @@ jobs:
         with:
           headers: '{"GraphQL-Features": "projects_next_graphql"}'
           query: |
-            mutation add_to_project($projectid:String!,$contentid:String!) {
+            mutation add_to_project($projectid: ID!, $contentid: ID!) {
               addProjectNextItem(input:{projectId:$projectid contentId:$contentid}) {
                 projectNextItem {
                   id

diff --git a/.github/workflows/backport-next.yml b/.github/workflows/backport-next.yml
@@ -0,0 +1,27 @@
+on:
+  pull_request_target:
+    branches:
+      - main
+    types:
+      - labeled
+      - closed
+
+jobs:
+  backport:
+    name: Backport PR
+    runs-on: ubuntu-latest
+    if: |
+      github.event.pull_request.merged == true
+      && contains(github.event.pull_request.labels.*.name, 'auto-backport-next')
+      && (
+        (github.event.action == 'labeled' && github.event.label.name == 'auto-backport-next')
+        || (github.event.action == 'closed')
+      )
+    steps:
+      - name: Backport Action
+        uses: sqren/backport-github-action@v7.3.1
+        with:
+          github_token: ${{secrets.KIBANAMACHINE_TOKEN}}
+
+      - name: Backport log
+        run: cat /home/runner/.backport/backport.log
diff --git a/.github/workflows/label-qa-fixed-in.yml b/.github/workflows/label-qa-fixed-in.yml
@@ -1,6 +1,13 @@
 name: Add QA labels to Fleet issues
 on:
-  pull_request:
+  # pull_request_target allows running actions on PRs from forks with a read/write GITHUB_TOKEN, but it will not allow
+  # running workflows defined in the PRs itself, only workflows already merged into the target branch. This avoids
+  # potential vulnerabilities that could allow someone to open a PR and retrieve secrets.
+  # It's important that this workflow never runs any checkout actions which could be used to circumvent this protection.
+  # See these links for more information:
+  # - https://github.blog/2020-08-03-github-actions-improvements-for-fork-and-pull-request-workflows/ 
+  # - https://nathandavison.com/blog/github-actions-and-the-threat-of-malicious-pull-requests
+  pull_request_target:
     types:
       - closed
 
@@ -38,7 +45,7 @@ jobs:
             }
           prnumber: ${{ github.event.number }}
         env:
-          GITHUB_TOKEN: ${{ secrets.FLEET_TECH_KIBANA_USER_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - uses: sergeysova/jq-action@v2
         id: issues_to_label
         with:
@@ -69,12 +76,12 @@ jobs:
         id: add_labels_to_closed_issue
         with:
           query: |
-            mutation add_label($issueid:String!, $labelids:[String!]!) {
+            mutation add_label($issueid: ID!, $labelids:[ID!]!) {
               addLabelsToLabelable(input: {labelableId: $issueid, labelIds: $labelids}) {
                 clientMutationId
               }
             }
           issueid: ${{ matrix.issueNodeId }}
-          labelids: ${{ needs.fetch_issues_to_label.outputs.label_ids }}
+          labelids: ${{ fromJSON(needs.fetch_issues_to_label.outputs.label_ids) }}
         env:
-          GITHUB_TOKEN: ${{ secrets.FLEET_TECH_KIBANA_USER_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/project-assigner.yml b/.github/workflows/project-assigner.yml
@@ -20,7 +20,6 @@ jobs:
               {"label": "Feature:Drilldowns", "projectNumber": 68, "columnName": "Inbox"},
               {"label": "Feature:Input Controls", "projectNumber": 72, "columnName": "Inbox"},
               {"label": "Team:Security", "projectNumber": 320, "columnName": "Awaiting triage", "projectScope": "org"},
-              {"label": "Team:Operations", "projectNumber": 314, "columnName": "Triage", "projectScope": "org"},
-              {"label": "Team:Fleet", "projectNumber": 490, "columnName": "Inbox", "projectScope": "org"}
+              {"label": "Team:Operations", "projectNumber": 314, "columnName": "Triage", "projectScope": "org"}
             ]
           ghToken: ${{ secrets.PROJECT_ASSIGNER_TOKEN }}
diff --git a/api_docs/actions.devdocs.json b/api_docs/actions.devdocs.json
@@ -695,7 +695,7 @@
         "label": "ActionParamsType",
         "description": [],
         "signature": [
-          "{ readonly source?: string | undefined; readonly group?: string | undefined; readonly summary?: string | undefined; readonly timestamp?: string | undefined; readonly eventAction?: \"resolve\" | \"trigger\" | \"acknowledge\" | undefined; readonly dedupKey?: string | undefined; readonly severity?: \"error\" | \"info\" | \"warning\" | \"critical\" | undefined; readonly component?: string | undefined; readonly class?: string | undefined; }"
+          "{ readonly source?: string | undefined; readonly summary?: string | undefined; readonly group?: string | undefined; readonly timestamp?: string | undefined; readonly eventAction?: \"resolve\" | \"trigger\" | \"acknowledge\" | undefined; readonly dedupKey?: string | undefined; readonly severity?: \"error\" | \"info\" | \"warning\" | \"critical\" | undefined; readonly component?: string | undefined; readonly class?: string | undefined; }"
         ],
         "path": "x-pack/plugins/actions/server/builtin_action_types/pagerduty.ts",
         "deprecated": false,

diff --git a/api_docs/actions.mdx b/api_docs/actions.mdx
@@ -4,15 +4,15 @@ slug: /kibana-dev-docs/api/actions
 title: "actions"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the actions plugin
-date: 2022-02-11
+date: 2022-02-28
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'actions']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---
 import actionsObj from './actions.devdocs.json';
 
 
 
-Contact [Kibana Alerting](https://github.com/orgs/elastic/teams/kibana-alerting-services) for questions regarding this plugin.
+Contact [Response Ops](https://github.com/orgs/elastic/teams/response-ops) for questions regarding this plugin.
 
 **Code health stats**
 

diff --git a/api_docs/advanced_settings.mdx b/api_docs/advanced_settings.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/advancedSettings
 title: "advancedSettings"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the advancedSettings plugin
-date: 2022-02-11
+date: 2022-02-28
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'advancedSettings']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---