Merge branch 'master' into saved-objects/scoped-client-with-extra-types

* master:
  [apm] Annotation API documentation (elastic#65963)
  [Uptime] Improve responsiveness details page (elastic#67034)
  skip flaky suite (elastic#66669)
  Revert "Integration of a static filesystem for the node_modules (elastic#47998)" (elastic#67124)
  Support api_integration/kibana/stats against remote hosts (elastic#53000)
  chore(NA): add module name mapper for src plugins on x-pack (elastic#67103)
  Change the error message on TSVB in order to be more user friendly (elastic#67090)
  [kbn/optimizer] poll parent process to avoid zombie processes (elastic#67059)
  [Visualize] Lazy load default editor, fix duplicated styles (elastic#66732)
  Bump styled-component dependencies (elastic#66611)
  Bump react-markdown dependencies (elastic#66615)
  Fix Core docs links (elastic#66977)
  Timelion graph is not refreshing content after searching or filtering (elastic#67023)
  Remove `--xpack.endpoint.enabled=true` from README.md file (elastic#67053)
  Move apm tutorial from apm plugin into apm_oss plugin (elastic#66432)
  [Logs UI] Restore call to `UsageCollector.countLogs` (elastic#67051)
  Remove unused license check result from LP Security plugin (elastic#66966)

.i18nrc.json

@@ -56,7 +56,8 @@
     "visTypeVislib": "src/plugins/vis_type_vislib",
     "visTypeXy": "src/plugins/vis_type_xy",
     "visualizations": "src/plugins/visualizations",
-    "visualize": "src/plugins/visualize"
+    "visualize": "src/plugins/visualize",
+    "apmOss": "src/plugins/apm_oss"
   },
   "exclude": [
     "src/legacy/ui/ui_render/ui_render_mixin.js"

docs/apm/api.asciidoc

@@ -9,14 +9,45 @@
 Some APM app features are provided via a REST API:
 
 * <<agent-config-api>>
-
-TIP: Kibana provides additional <<api,REST APIs>>,
-and general information on <<using-api,how to use APIs>>.
+* <<apm-annotation-api>>
+
+Here's an example CURL request that adds an annotation to the APM app:
+
+[source,curl]
+----
+curl -X POST \
+  http://localhost:5601/api/apm/services/opbeans-java/annotation \
+-H 'Content-Type: application/json' \
+-H 'kbn-xsrf: true' \
+-H 'Authorization: Basic YhUlubWZhM0FDbnlQeE6WRtaW49FQmSGZ4RUWXdX' \
+-d '{
+      "@timestamp": "2020-05-11T10:31:30.452Z",
+      "service": {
+        "version": "1.2"
+      },
+      "message": "Revert upgrade",
+      "tags": [
+        "elastic.co", "customer"
+      ]
+    }'
+----
+
+For more information, the Kibana <<api,REST API reference>> provides information on how to use Kibana APIs,
+like required request headers and authentication options.
+
+// AGENT CONFIG API
+// GET --> Feature (APM) Read
+// CREATE/EDIT/DELETE --> Feature (APM) All
+
+// ANNOTATION API
+// Feature (APM) All
+// Index: `observability-annotations`. Privileges: `create_index`, `create_doc`, `manage`, and `read`.
 
 ////
 *******************************************************
 ////
 
+[role="xpack"]
 [[agent-config-api]]
 === Agent Configuration API
 
@@ -274,6 +305,118 @@ POST /api/apm/settings/agent-configuration/search
 }
 --------------------------------------------------
 
+////
+*******************************************************
+*******************************************************
+////
+
+[role="xpack"]
+[[apm-annotation-api]]
+=== Annotation API
+
+The Annotation API allows you to annotate visualizations in the APM app with significant events, like deployments,
+allowing you to easily see how these events are impacting the performance of your existing applications.
+
+The following APIs are available:
+
+* <<apm-annotation-create>> to create an annotation for APM.
+// * <<obs-annotation-create>> POST /api/observability/annotation
+// * <<obs-annotation-get>> GET /api/observability/annotation/:id
+// * <<obs-annotation-delete>> DELETE /api/observability/annotation/:id
+
+By default, annotations are stored in a newly created `observability-annotations` index.
+The name of this index can be changed in your `config.yml` by editing `xpack.observability.annotations.index`.
+
 ////
 *******************************************************
 ////
+
+[[apm-annotation-create]]
+==== Create or update annotation
+
+[[apm-annotation-config-req]]
+===== Request
+
+`POST /api/apm/services/:serviceName/annotation`
+
+[role="child_attributes"]
+[[apm-annotation-config-req-body]]
+===== Request body
+
+`service`::
+(required, object) Service identifying the configuration to create or update.
++
+.Properties of `service`
+[%collapsible%open]
+======
+`version` :::
+  (required, string) Name of service.
+
+`environment` :::
+  (optional, string) Environment of service.
+======
+
+`@timestamp`::
+(required, string) The date and time of the annotation. Must be in https://www.w3.org/TR/NOTE-datetime[ISO 8601] format.
+
+`message`::
+(optional, string) The message displayed in the annotation. Defaults to `service.version`.
+
+`tags`::
+(optional, array) Tags are used by the APM app to distinguish APM annotations from other annotations.
+Tags may have additional functionality in future releases. Defaults to `[apm]`.
+While you can add additional tags, you cannot remove the `apm` tag.
+
+[[apm-annotation-config-example]]
+===== Example
+
+The following example creates an annotation for a service named `opbeans-java`.
+
+[source,console]
+--------------------------------------------------
+POST /api/apm/services/opbeans-java/annotation
+{
+	"@timestamp": "2020-05-08T10:31:30.452Z",
+	"service": {
+		"version": "1.2"
+	},
+	"message": "Deployment 1.2",
+	"tags": [
+		"elastic.co", "customer"
+	]
+}
+--------------------------------------------------
+
+[[apm-annotation-config-body]]
+===== Response body
+
+[source,js]
+--------------------------------------------------
+{
+  "_index": "observability-annotations",
+  "_id": "Lc9I93EBh6DbmkeV7nFX",
+  "_version": 1,
+  "_seq_no": 12,
+  "_primary_term": 1,
+  "found": true,
+  "_source": {
+    "message": "Deployment 1.2",
+    "@timestamp": "2020-05-08T10:31:30.452Z",
+    "service": {
+      "version": "1.2",
+      "name": "opbeans-java"
+    },
+    "tags": [
+      "apm",
+      "elastic.co",
+      "customer"
+    ],
+    "annotation": {
+      "type": "deployment"
+    },
+    "event": {
+      "created": "2020-05-09T02:34:43.937Z"
+    }
+  }
+}
+--------------------------------------------------

docs/apm/deployment-annotations.asciidoc

@@ -3,15 +3,19 @@
 === Track deployments with annotations
 
 ++++
-<titleabbrev>Track deployments</titleabbrev>
+<titleabbrev>Track deployments with annotations</titleabbrev>
 ++++
 
 For enhanced visibility into your deployments, we offer deployment annotations on all transaction charts.
 This feature automatically tags new deployments, so you can easily see if your deploy has increased response times
 for an end-user, or if the memory/CPU footprint of your application has changed.
 Being able to identify bad deployments quickly enables you to rollback and fix issues without causing costly outages.
 
-Deployment annotations are automatically enabled, and appear when the `service.version` of your app changes.
+Deployment annotations are enabled by default, and can be created with the <<apm-annotation-api,annotation API>>.
+If there are no created annotations for the selected time period,
+the APM app will automatically annotate your data if the `service.version` of your application changes.
+
+NOTE: If custom annotations have been created for the selected time period, any derived annotations, i.e., those created automatically when `service.version` changes, will not be shown.
 
 [role="screenshot"]
 image::apm/images/apm-transaction-annotation.png[Example view of transactions annotation in the APM app in Kibana]

docs/settings/apm-settings.asciidoc

@@ -52,6 +52,9 @@ Changing these settings may disable features of the APM App.
 | `xpack.apm.ui.maxTraceItems` {ess-icon}
   | Maximum number of child items displayed when viewing trace details. Defaults to `1000`.
 
+| `xpack.observability.annotations.index`
+  | Index name where Observability annotations are stored. Defaults to `observability-annotations`.
+
 | `apm_oss.indexPattern` {ess-icon}
   | The index pattern used for integrations with Machine Learning and Query Bar.
   It must match all apm indices. Defaults to `apm-*`.

package.json

@@ -240,7 +240,7 @@
     "react-dom": "^16.12.0",
     "react-grid-layout": "^0.16.2",
     "react-input-range": "^1.3.0",
-    "react-markdown": "^3.4.1",
+    "react-markdown": "^4.3.1",
     "react-monaco-editor": "~0.27.0",
     "react-redux": "^7.1.3",
     "react-resize-detector": "^4.2.0",
@@ -298,7 +298,6 @@
     "@elastic/eslint-plugin-eui": "0.0.2",
     "@elastic/github-checks-reporter": "0.0.20b3",
     "@elastic/makelogs": "^5.0.1",
-    "@elastic/static-fs": "1.0.2",
     "@kbn/dev-utils": "1.0.0",
     "@kbn/es": "1.0.0",
     "@kbn/eslint-import-resolver-kibana": "2.0.0",
@@ -387,7 +386,7 @@
     "@types/semver": "^5.5.0",
     "@types/sinon": "^7.0.13",
     "@types/strip-ansi": "^3.0.0",
-    "@types/styled-components": "^4.4.2",
+    "@types/styled-components": "^5.1.0",
     "@types/supertest": "^2.0.5",
     "@types/supertest-as-promised": "^2.0.38",
     "@types/testing-library__react": "^9.1.2",

packages/kbn-babel-preset/package.json

@@ -14,7 +14,7 @@
     "@babel/preset-typescript": "^7.9.0",
     "babel-plugin-add-module-exports": "^1.0.2",
     "babel-plugin-filter-imports": "^3.0.0",
-    "babel-plugin-styled-components": "^1.10.6",
+    "babel-plugin-styled-components": "^1.10.7",
     "babel-plugin-transform-define": "^1.3.1",
     "babel-plugin-transform-imports": "^2.0.0"
   }

packages/kbn-optimizer/src/common/index.ts

@@ -21,6 +21,7 @@ export * from './bundle';
 export * from './bundle_cache';
 export * from './worker_config';
 export * from './worker_messages';
+export * from './parent_messages';
 export * from './compiler_messages';
 export * from './ts_helpers';
 export * from './rxjs_helpers';

packages/kbn-optimizer/src/common/parent_messages.ts

@@ -0,0 +1,33 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export interface ParentPongMsg {
+  type: 'pong';
+}
+
+export const isParentPong = (value: any): value is ParentPongMsg =>
+  typeof value === 'object' && value && value.type === 'pong';
+
+export class ParentMsgs {
+  pong(): ParentPongMsg {
+    return {
+      type: 'pong',
+    };
+  }
+}

packages/kbn-optimizer/src/common/worker_messages.ts

@@ -24,13 +24,17 @@ import {
   CompilerErrorMsg,
 } from './compiler_messages';
 
-export type WorkerMsg =
+export type InternalWorkerMsg =
+  | WorkerPingMsg
   | CompilerRunningMsg
   | CompilerIssueMsg
   | CompilerSuccessMsg
   | CompilerErrorMsg
   | WorkerErrorMsg;
 
+// ping messages are internal, they don't apper in public message streams
+export type WorkerMsg = Exclude<InternalWorkerMsg, WorkerPingMsg>;
+
 /**
  * Message sent when the worker encounters an error that it can't
  * recover from, no more messages will be sent and the worker
@@ -42,6 +46,10 @@ export interface WorkerErrorMsg {
   errorStack?: string;
 }
 
+export interface WorkerPingMsg {
+  type: 'ping';
+}
+
 const WORKER_STATE_TYPES: ReadonlyArray<WorkerMsg['type']> = [
   'running',
   'compiler issue',
@@ -50,10 +58,19 @@ const WORKER_STATE_TYPES: ReadonlyArray<WorkerMsg['type']> = [
   'worker error',
 ];
 
+export const isWorkerPing = (value: any): value is WorkerPingMsg =>
+  typeof value === 'object' && value && value.type === 'ping';
+
 export const isWorkerMsg = (value: any): value is WorkerMsg =>
   typeof value === 'object' && value && WORKER_STATE_TYPES.includes(value.type);
 
 export class WorkerMsgs {
+  ping(): WorkerPingMsg {
+    return {
+      type: 'ping',
+    };
+  }
+
   error(error: Error): WorkerErrorMsg {
     return {
       type: 'worker error',

packages/kbn-optimizer/src/optimizer/observe_worker.ts

@@ -22,12 +22,14 @@ import { Readable } from 'stream';
 import { inspect } from 'util';
 
 import * as Rx from 'rxjs';
-import { map, takeUntil } from 'rxjs/operators';
+import { map, filter, takeUntil } from 'rxjs/operators';
 
-import { isWorkerMsg, WorkerConfig, WorkerMsg, Bundle } from '../common';
+import { isWorkerMsg, isWorkerPing, WorkerConfig, WorkerMsg, Bundle, ParentMsgs } from '../common';
 
 import { OptimizerConfig } from './optimizer_config';
 
+const parentMsgs = new ParentMsgs();
+
 export interface WorkerStdio {
   type: 'worker stdio';
   stream: 'stdout' | 'stderr';
@@ -146,6 +148,16 @@ export function observeWorker(
       observeStdio$(proc.stderr, 'stderr'),
       Rx.fromEvent<[unknown]>(proc, 'message')
         .pipe(
+          // filter out ping messages so they don't end up in the general message stream
+          filter(([msg]) => {
+            if (!isWorkerPing(msg)) {
+              return true;
+            }
+
+            proc.send(parentMsgs.pong());
+            return false;
+          }),
+
           // validate the messages from the process
           map(([msg]) => {
             if (!isWorkerMsg(msg)) {