Add GCP package (elastic#459)

* Audit data stream

* Firewall dataset

* Add vpcflow dataset

* Manifests, dashboards and system tests

* Remove log input

* Add gcp emulator for system testing

* Rename system config files

* Add changelog

* Move all processors to ingest

* Add fake credentials

* Use go client to publish test messages

* Rename system tests service

* Change custom docker container for akroh/stream

* Use stream v0.3.0 for system tests

* Hide advanced settings

* Remove unnecessary config from deploy

packages/gcp/_dev/build/docs/README.md

@@ -0,0 +1,30 @@
+# Google Cloud Integration
+
+The Google Cloud integration collects and parses Google Cloud audit, VPC flow,
+and firewall logs that have been exported from Stackdriver to a Google Pub/Sub topic sink.
+
+## Logs
+
+### Audit
+
+This is the `audit` dataset.
+
+{{event "audit"}}
+
+{{fields "audit"}}
+
+### Firewall
+
+This is the `firewall` dataset.
+
+{{event "firewall"}}
+
+{{fields "firewall"}}
+
+### VPC Flow
+
+This is the `VPC Flow` dataset.
+
+{{event "vpcflow"}}
+
+{{fields "vpcflow"}}

packages/gcp/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,49 @@
+version: '2.3'
+services:
+  gcppubsub-emulator:
+    image: google/cloud-sdk:emulators
+    command: gcloud beta emulators pubsub start --host-port=0.0.0.0:8681
+    ports:
+      - "8681/tcp"
+  gcppubsub-audit:
+    image: akroh/stream:v0.3.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command:
+      - log
+      - --retry=30
+      - --addr=gcppubsub-emulator:8681
+      - -p=gcppubsub
+      - --gcppubsub-clear=true
+      - --gcppubsub-project=audit
+      - /sample_logs/audit.log
+    depends_on:
+      - gcppubsub-emulator
+  gcppubsub-firewall:
+    image: akroh/stream:v0.3.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command:
+      - log
+      - --retry=30
+      - --addr=gcppubsub-emulator:8681
+      - -p=gcppubsub
+      - --gcppubsub-clear=true
+      - --gcppubsub-project=firewall
+      - /sample_logs/firewall.log
+    depends_on:
+      - gcppubsub-emulator
+  gcppubsub-vpcflow:
+    image: akroh/stream:v0.3.0
+    volumes:
+      - ./sample_logs:/sample_logs:ro
+    command:
+      - log
+      - --retry=30
+      - --addr=gcppubsub-emulator:8681
+      - -p=gcppubsub
+      - --gcppubsub-clear=true
+      - --gcppubsub-project=vpcflow
+      - /sample_logs/vpcflow.log
+    depends_on:
+      - gcppubsub-emulator

packages/gcp/_dev/deploy/docker/sample_logs/audit.log

@@ -0,0 +1,7 @@
+{"insertId":"-uihnmjctwo","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"resourcemanager.projects.get","resource":"projects/elastic-beats","resourceAttributes":{}}],"methodName":"GetResourceBillingInfo","request":{"@type":"type.googleapis.com/google.internal.cloudbilling.billingaccount.v1.GetResourceBillingInfoRequest","resourceName":"projects/189716325846"},"requestMetadata":{"callerIp":"192.168.1.1","destinationAttributes":{},"requestAttributes":{}},"resourceName":"projects/elastic-beats","serviceName":"cloudbilling.googleapis.com","status":{}},"receiveTimestamp":"2019-12-19T00:49:36.313482371Z","resource":{"labels":{"project_id":"elastic-beats"},"type":"project"},"severity":"INFO","timestamp":"2019-12-19T00:49:36.086Z"}
+{"insertId":"-h6onuze1h7dg","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":false,"permission":"compute.machineTypes.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.machineTypes.aggregatedList","numResponseItems":"71","request":{"@type":"type.googleapis.com/compute.machineTypes.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:45:51.711Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/machineTypes","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:45:52.367887078Z","resource":{"labels":{"location":"global","method":"compute.machineTypes.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:45:51.228Z"}
+{"insertId":"yonau2dg2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"response":{"@type":"core.k8s.io/v1.Status","apiVersion":"v1","details":{"group":"batch","kind":"jobs","name":"gsuite-exporter-1589294700","uid":"2beff34a-945f-11ea-bacf-42010a80007f"},"kind":"Status","metadata":{},"status":"Success"},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"}
+{"insertId":"yonau3dc2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com","status":{"code":7,"message":"PERMISSION_DENIED"}},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"}
+{"insertId":"87efd529-6349-45d2-b905-fc607e6c5d3b","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cert-manager-webhook:auth-delegator\" of ClusterRole \"system:auth-delegator\" to ServiceAccount \"cert-manager-webhook/cert-manager\""},"logName":"projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"5555555-6349-45d2-b905-fc607e6c5d3b","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:serviceaccount:cert-manager:cert-manager-webhook"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","resource":"authorization.k8s.io/v1beta1/subjectaccessreviews"}],"methodName":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","request":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":false}},"requestMetadata":{"callerIp":"10.11.12.13","callerSuppliedUserAgent":"webhook/v0.0.0 (linux/amd64) kubernetes/$Format"},"resourceName":"authorization.k8s.io/v1beta1/subjectaccessreviews","response":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":true,"reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""}},"serviceName":"k8s.io","status":{"code":0}},"receiveTimestamp":"2020-08-05T21:07:32.157698684Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2020-08-05T21:07:30.974750Z"}
+{"insertId":"v2spcwdzmc2","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"first":true,"id":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"user@mycompany.com"},"authorizationInfo":[{"granted":true,"permission":"compute.images.create","resourceAttributes":{"name":"projects/foo/global/images/windows-server-2016-v20200805","service":"compute","type":"compute.images"}}],"methodName":"v1.compute.images.insert","request":{"@type":"type.googleapis.com/compute.images.insert","family":"windows-server-2016","guestOsFeatures":[{"type":"VIRTIO_SCSI_MULTIQUEUE"},{"type":"WINDOWS"}],"name":"windows-server-2016-v20200805","rawDisk":{"source":"https://storage.googleapis.com/storage/v1/b/foo/o/windows-server-2016-v20200805.tar.gz"},"sourceType":"RAW"},"requestMetadata":{"callerIp":"1.2.3.4","callerSuppliedUserAgent":"google-cloud-sdk gcloud/290.0.1 command/gcloud.compute.images.create invocation-id/032752ad0fa44b4ea951951d2deef6a3 environment/None environment-version/None interactive/True from-script/False python/2.7.17 term/xterm-256color (Macintosh; Intel Mac OS X 19.6.0),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2020-08-05T21:59:27.515Z"}},"resourceLocation":{"currentLocations":["eu"]},"resourceName":"projects/foo/global/images/windows-server-2016-v20200805","response":{"@type":"type.googleapis.com/operation","id":"44919313","insertTime":"2020-08-05T14:59:27.259-07:00","name":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","operationType":"insert","progress":"0","selfLink":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/operation-1596664766354-5ac287c395484-fa3923bd-543e018e","selfLinkWithId":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/4491931805423146320","startTime":"2020-08-05T14:59:27.274-07:00","status":"RUNNING","targetId":"12345","targetLink":"https://www.googleapis.com/compute/v1/projects/foo/global/images/windows-server-2016-v20200805","user":"user@mycompany.com"},"serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T21:59:27.822546978Z","resource":{"labels":{"image_id":"771879043","project_id":"foo"},"type":"gce_image"},"severity":"NOTICE","timestamp":"2020-08-05T21:59:26.456Z"}
+{"insertId":"-c7ctxmd2zab","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"id":"operation-1596646123456-5ac2438b775f6-f8ca1382-e70b6831","last":true,"producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"user@mycompany.com"},"methodName":"beta.compute.instances.stop","request":{"@type":"type.googleapis.com/compute.instances.stop"},"requestMetadata":{"callerIp":"2.3.4.5","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)"},"resourceName":"projects/foo/zones/us-central1-a/instances/win10-test","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T16:56:41.315135528Z","resource":{"labels":{"instance_id":"590261181","project_id":"foo","zone":"us-central1-a"},"type":"gce_instance"},"severity":"NOTICE","timestamp":"2020-08-05T16:56:40.428Z"}