Stackdriver Export to Datadog v2

examples/datadog-sink/README.md

@@ -0,0 +1,53 @@
+# Datadog sink example
+
+The solution helps you set up a log-streaming pipeline from Stackdriver Logging to Datadog.
+
+## Instructions
+
+1. Fill the required variables in the `terraform.tfvars.sample` file located in this directory.
+
+2. Verify the IAM roles for your Terraform service account:
+    - `roles/logging.configWriter` on the project (to create the logsink)
+    - `roles/iam.admin` on the project (to grant write permissions for logsink service account)
+    - `roles/serviceusage.admin` on the destination project (to enable destination API)
+    - `roles/pubsub.admin` on the destination project (to create a pub/sub topic)
+    - `roles/serviceAccount.admin` on the destination project (to create a service account for the logsink subscriber)
+
+2. Run the Terraform automation:
+    ```
+    terraform init
+    terraform apply
+    ```
+
+    You should see similar outputs as the following:
+
+    ![output screenshot](https://github.com/smbreslow/terraform-google-log-export/raw/master/examples/datadog-sink/screenshots/Screen%20Shot%202019-12-09%20at%204.44.11%20PM.png)
+
+3. Navigate to the [Datadog Google Cloud Integration Tile](http://app.datadoghq.com/account/settings#integrations/google_cloud_platform).
+
+4. On the **Configuration** tab, select *Upload Key File* and upload the JSON file located at the specified `output_key_path`.
+    ![datadog screenshot](https://docs.datadoghq.com/images/integrations/google_cloud_platform/ServiceAccountAdded.png?fit=max&auto=format)
+
+5. Press *Install/Update*.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|:----:|:-----:|:-----:|
+| key\_output\_path | The path to a directory where the JSON private key of the new Datadog service account will be created. | string | `"../datadog-sink/datadog-sa-key.json"` | no |
+| parent\_resource\_id | The ID of the project in which pubsub topic destination will be created. | string | n/a | yes |
+| project\_id | The ID of the project in which the log export will be created. | string | n/a | yes |
+| push\_endpoint | The URL locating the endpoint to which messages should be pushed. | string | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| datadog\_service\_account | Datadog service account email |
+| log\_writer |  |
+| pubsub\_subscription\_name | Pub/Sub topic subscription name |
+| pubsub\_topic\_name | Pub/Sub topic name |
+| pubsub\_topic\_project | Pub/Sub topic project id |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/datadog-sink/main.tf

@@ -0,0 +1,76 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+provider "google" {
+  version = "~> 2.0"
+}
+
+locals {
+  datadog_svc = element(google_service_account.datadog-viewer.*.email, 0)
+  log_writ    = module.log_export.writer_identity
+}
+
+resource "google_service_account" "datadog-viewer" {
+  account_id  = "${var.project_id}-datadog-viewer"
+  description = "Service account for Datadog monitoring"
+  project     = var.project_id
+}
+
+resource "google_service_account_key" "datadog-viewer-key" {
+  service_account_id = google_service_account.datadog-viewer.name
+}
+
+resource "local_file" "key_export" {
+  content_base64 = google_service_account_key.datadog-viewer-key.private_key
+  filename       = var.key_output_path
+}
+
+resource "google_project_iam_member" "compute-viewer" {
+  project = var.project_id
+  role    = "roles/compute.viewer"
+  member  = "serviceAccount:${google_service_account.datadog-viewer.email}"
+}
+
+resource "google_project_iam_member" "cloudasset-viewer" {
+  project = var.project_id
+  role    = "roles/cloudasset.viewer"
+  member  = "serviceAccount:${google_service_account.datadog-viewer.email}"
+}
+
+resource "google_project_iam_member" "monitoring-viewer" {
+  project = var.project_id
+  role    = "roles/monitoring.viewer"
+  member  = "serviceAccount:${google_service_account.datadog-viewer.email}"
+}
+
+module "log_export" {
+  source                 = "../../"
+  destination_uri        = module.destination.destination_uri
+  log_sink_name          = "test-datadog-sink"
+  parent_resource_id     = var.parent_resource_id
+  parent_resource_type   = "project"
+  unique_writer_identity = true
+}
+
+module "destination" {
+  source                   = "../../modules/pubsub"
+  project_id               = var.project_id
+  topic_name               = "datadog-sink"
+  log_sink_writer_identity = module.log_export.writer_identity
+  create_subscriber        = false
+  create_push_subscriber   = true
+  push_endpoint            = var.push_endpoint
+}

examples/datadog-sink/outputs.tf

@@ -0,0 +1,39 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "pubsub_topic_name" {
+  description = "Pub/Sub topic name"
+  value       = module.destination.resource_id
+}
+
+output "pubsub_topic_project" {
+  description = "Pub/Sub topic project id"
+  value       = module.destination.project
+}
+
+output "pubsub_subscription_name" {
+  description = "Pub/Sub topic subscription name"
+  value       = module.destination.pubsub_push_subscription
+}
+
+output "datadog_service_account" {
+  description = "Datadog service account email"
+  value       = local.datadog_svc
+}
+
+output "log_writer" {
+  value = local.log_writ
+}

examples/datadog-sink/terraform.tfvars.sample

@@ -0,0 +1,7 @@
+project_id = "smb-dd-lab"
+
+parent_resource_id = "smb-dd-lab"
+
+push_endpoint = "https://gcp-intake.logs.datadoghq.eu/v1/input/<DATADOG_API_KEY>/"
+
+key_output_path = "/home/sbreslow/terraform-google-log-export/examples/datadog-sink/sa-key.json"

examples/datadog-sink/variables.tf

@@ -0,0 +1,36 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The ID of the project in which the log export will be created."
+  type        = string
+}
+
+variable "parent_resource_id" {
+  description = "The ID of the project in which pubsub topic destination will be created."
+  type        = string
+}
+
+variable "push_endpoint" {
+  description = "The URL locating the endpoint to which messages should be pushed."
+  type        = string
+}
+
+variable "key_output_path" {
+  description = "The path to a directory where the JSON private key of the new Datadog service account will be created."
+  type        = string
+  default     = "../datadog-sink/datadog-sa-key.json"
+}

examples/datadog-sink/versions.tf

@@ -0,0 +1,20 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+terraform {
+  required_version = ">= 0.12"
+}

modules/pubsub/README.md

@@ -37,9 +37,11 @@ so that all dependencies are met.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|:----:|:-----:|:-----:|
-| create\_subscriber | Whether to create a subscription to the topic that was created and used for log entries matching the filter. If 'true', a subscription is created along with a service account that is granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic. | bool | `"false"` | no |
+| create\_push\_subscriber | Whether to add a push configuration to the subcription. If 'true', a push subscription is created along with a service account that is granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic. | bool | `"false"` | no |
+| create\_subscriber | Whether to create a subscription to the topic that was created and used for log entries matching the filter. If 'true', a pull subscription is created along with a service account that is granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic. | bool | `"false"` | no |
 | log\_sink\_writer\_identity | The service account that logging uses to write log entries to the destination. (This is available as an output coming from the root module). | string | n/a | yes |
 | project\_id | The ID of the project in which the pubsub topic will be created. | string | n/a | yes |
+| push\_endpoint | The URL locating the endpoint to which messages should be pushed. | string | `""` | no |
 | topic\_labels | A set of key/value label pairs to assign to the pubsub topic. | map(string) | `<map>` | no |
 | topic\_name | The name of the pubsub topic to be created and used for log entries matching the filter. | string | n/a | yes |
 
@@ -50,6 +52,7 @@ so that all dependencies are met.
 | console\_link | The console link to the destination storage bucket |
 | destination\_uri | The destination URI for the topic. |
 | project | The project in which the topic was created. |
+| pubsub\_push\_subscription | Pub/Sub push subscription id (if any) |
 | pubsub\_subscriber | Pub/Sub subscriber email (if any) |
 | pubsub\_subscription | Pub/Sub subscription id (if any) |
 | resource\_id | The resource id for the destination topic |

modules/pubsub/main.tf

@@ -28,6 +28,10 @@ locals {
     concat(google_pubsub_subscription.pubsub_subscription.*.id, [""]),
     0,
   )
+  pubsub_push_subscription = element(
+    concat(google_pubsub_subscription.pubsub_push_subscription.*.id, [""]),
+    0,
+  )
 }
 
 #----------------#
@@ -91,3 +95,13 @@ resource "google_pubsub_subscription" "pubsub_subscription" {
   topic   = local.topic_name
 }
 
+resource "google_pubsub_subscription" "pubsub_push_subscription" {
+  count   = var.create_push_subscriber ? 1 : 0
+  name    = "${local.topic_name}-push-subscription"
+  project = var.project_id
+  topic   = local.topic_name
+
+  push_config {
+    push_endpoint = var.push_endpoint
+  }
+}

modules/pubsub/outputs.tf

@@ -49,3 +49,7 @@ output "pubsub_subscription" {
   value       = local.pubsub_subscription
 }
 
+output "pubsub_push_subscription" {
+  description = "Pub/Sub push subscription id (if any)"
+  value       = local.pubsub_push_subscription
+}

modules/pubsub/variables.tf

@@ -15,11 +15,23 @@
  */
 
 variable "create_subscriber" {
-  description = "Whether to create a subscription to the topic that was created and used for log entries matching the filter. If 'true', a subscription is created along with a service account that is granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic."
+  description = "Whether to create a subscription to the topic that was created and used for log entries matching the filter. If 'true', a pull subscription is created along with a service account that is granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic."
   type        = bool
   default     = false
 }
 
+variable "create_push_subscriber" {
+  description = "Whether to add a push configuration to the subcription. If 'true', a push subscription is created along with a service account that is granted roles/pubsub.subscriber and roles/pubsub.viewer to the topic."
+  type        = bool
+  default     = false
+}
+
+variable "push_endpoint" {
+  description = "The URL locating the endpoint to which messages should be pushed."
+  type        = string
+  default     = ""
+}
+
 variable "log_sink_writer_identity" {
   description = "The service account that logging uses to write log entries to the destination. (This is available as an output coming from the root module)."
   type        = string