Create re-usable Lambda Terraform module

terraform/modules/tf_lambda/README.md

@@ -0,0 +1,57 @@
+# Lambda Module
+This Terraform module creates a single AWS Lambda function and its related components:
+
+* IAM execution role with basic permissions
+* Lambda function
+* Production alias
+* CloudWatch log group
+* CloudWatch metric alarms related to Lambda
+
+All StreamAlert Lambda functions will eventually leverage this module.
+
+The created IAM role has permission to publish CloudWatch logs and metrics. To add function-specific
+permissions, attach/inline them to the created IAM role.
+
+## Example
+```hcl
+module "alert_processor" {
+  function_name     = "alert_processor"
+  handler           = "stream_alert.alert_processor.main.handler"
+  source_bucket     = "SOURCE_BUCKET"
+  source_object_key = "SOURCE_OBJECT_KEY"
+
+  environment_variables = {
+    LOGGER_LEVEL = "info"
+  }
+
+  // Commonly used optional variables
+  enabled                   = true
+  description               = "Function Description"
+  memory_size_mb            = 128
+  timeout_sec               = 60
+  vpc_subnet_ids            = ["abc"]
+  vpc_security_group_ids    = ["id0"]
+  aliased_version           = 1
+  log_retention_days        = 14
+  alarm_actions             = ["SNS_ARN"]
+  errors_alarm_threshold    = 1
+  enable_iterator_age_alarm = true
+}
+
+// Add additional permissions
+resource "aws_iam_role_policy" "policy" {
+  name   = "CustomPolicy"
+  role   = "${module.alert_processor.role_id}"
+  policy = "${data.aws_iam_policy_document.policy.json}"
+}
+
+data "aws_iam_policy_document" "policy" {
+  statement {
+    effect    = "Allow"
+    actions   = ["s3:PutObject"]
+    resources = ["arn:aws:s3:::..."]
+  }
+}
+```
+
+For a complete list of available options and their descriptions, see [`variables.tf`](variables.tf).

terraform/modules/tf_lambda/iam.tf

@@ -0,0 +1,43 @@
+data "aws_iam_policy_document" "lambda_execution_policy" {
+  count = "${var.enabled}"
+
+  statement {
+    effect  = "Allow"
+    actions = ["sts:AssumeRole"]
+
+    principals {
+      type        = "Service"
+      identifiers = ["lambda.amazonaws.com"]
+    }
+  }
+}
+
+// Create the execution role for the Lambda function.
+resource "aws_iam_role" "role" {
+  count              = "${var.enabled}"
+  name               = "${var.function_name}_role"
+  assume_role_policy = "${data.aws_iam_policy_document.lambda_execution_policy.json}"
+}
+
+// Base permissions - Allow creating logs and publishing metrics
+data "aws_iam_policy_document" "logs_metrics_policy" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "cloudwatch:PutMetricData",
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents",
+    ]
+
+    resources = ["*"]
+  }
+}
+
+resource "aws_iam_role_policy" "logs_metrics_policy" {
+  count  = "${var.enabled}"
+  name   = "LogsAndMetrics"
+  role   = "${aws_iam_role.role.id}"
+  policy = "${data.aws_iam_policy_document.logs_metrics_policy.json}"
+}

terraform/modules/tf_lambda/main.tf

@@ -0,0 +1,47 @@
+// Generic module for any StreamAlert Lambda function.
+// TODO - migrate all Lambda functions and Lambda metric alarms to use this module
+
+resource "aws_lambda_function" "function" {
+  count         = "${var.enabled}"
+  function_name = "${var.function_name}"
+  description   = "${var.description}"
+  runtime       = "${var.runtime}"
+  role          = "${aws_iam_role.role.arn}"
+  handler       = "${var.handler}"
+  memory_size   = "${var.memory_size_mb}"
+  timeout       = "${var.timeout_sec}"
+  s3_bucket     = "${var.source_bucket}"
+  s3_key        = "${var.source_object_key}"
+
+  environment {
+    variables = "${var.environment_variables}"
+  }
+
+  // Note: If both of these lists are empty, VPC will not be enabled
+  vpc_config {
+    security_group_ids = "${var.vpc_subnet_ids}"
+    subnet_ids         = "${var.vpc_security_group_ids}"
+  }
+
+  tags {
+    Name = "${var.name_tag}"
+  }
+}
+
+resource "aws_lambda_alias" "production_alias" {
+  count            = "${var.enabled}"
+  name             = "production"
+  description      = "Production alias for ${aws_lambda_function.function.function_name}"
+  function_name    = "${aws_lambda_function.function.function_name}"
+  function_version = "${var.aliased_version == "" ? aws_lambda_function.function.version : var.aliased_version}"
+}
+
+resource "aws_cloudwatch_log_group" "lambda_log_group" {
+  count             = "${var.enabled}"
+  name              = "/aws/lambda/${var.function_name}"
+  retention_in_days = "${var.log_retention_days}"
+
+  tags {
+    Name = "${var.name_tag}"
+  }
+}

terraform/modules/tf_lambda/metrics.tf

@@ -0,0 +1,57 @@
+resource "aws_cloudwatch_metric_alarm" "lambda_invocation_errors" {
+  count               = "${var.enabled}"
+  alarm_name          = "${var.function_name}_invocation_errors"
+  namespace           = "AWS/Lambda"
+  metric_name         = "Errors"
+  statistic           = "Sum"
+  comparison_operator = "GreaterThanThreshold"
+  threshold           = "${var.errors_alarm_threshold}"
+  evaluation_periods  = "${var.errors_alarm_evaluation_periods}"
+  period              = "${var.errors_alarm_period_secs}"
+  alarm_description   = "StreamAlert Lambda Invocation Errors: ${var.function_name}"
+  alarm_actions       = "${var.alarm_actions}"
+
+  dimensions {
+    FunctionName = "${var.function_name}"
+    Resource     = "${var.function_name}:production"
+  }
+}
+
+resource "aws_cloudwatch_metric_alarm" "lambda_throttles" {
+  count               = "${var.enabled}"
+  alarm_name          = "${var.function_name}_throttles"
+  namespace           = "AWS/Lambda"
+  metric_name         = "Throttles"
+  statistic           = "Sum"
+  comparison_operator = "GreaterThanThreshold"
+  threshold           = "${var.throttles_alarm_threshold}"
+  evaluation_periods  = "${var.throttles_alarm_evaluation_periods}"
+  period              = "${var.throttles_alarm_period_secs}"
+  alarm_description   = "StreamAlert Lambda Throttles: ${var.function_name}"
+  alarm_actions       = "${var.alarm_actions}"
+
+  dimensions {
+    FunctionName = "${var.function_name}"
+    Resource     = "${var.function_name}:production"
+  }
+}
+
+// Lambda: IteratorAge
+resource "aws_cloudwatch_metric_alarm" "streamalert_lambda_iterator_age" {
+  count               = "${min(var.enabled, var.enable_iterator_age_alarm)}"
+  alarm_name          = "${var.function_name}_iterator_age"
+  namespace           = "AWS/Lambda"
+  metric_name         = "IteratorAge"
+  statistic           = "Maximum"
+  comparison_operator = "GreaterThanThreshold"
+  threshold           = "${var.iterator_age_alarm_threshold}"
+  evaluation_periods  = "${var.iterator_age_alarm_evaluation_periods}"
+  period              = "${var.iterator_age_alarm_period_secs}"
+  alarm_description   = "StreamAlert Lambda High Iterator Age: ${var.function_name}"
+  alarm_actions       = "${var.alarm_actions}"
+
+  dimensions {
+    FunctionName = "${var.function_name}"
+    Resource     = "${var.function_name}:production"
+  }
+}

terraform/modules/tf_lambda/output.tf

@@ -0,0 +1,11 @@
+output "function_arn" {
+  value = "${aws_lambda_function.function.arn}"
+}
+
+output "role_arn" {
+  value = "${aws_iam_role.role.arn}"
+}
+
+output "role_id" {
+  value = "${aws_iam_role.role.id}"
+}

terraform/modules/tf_lambda/variables.tf

@@ -0,0 +1,132 @@
+// Note: We use this variable because terraform does not support "count" for module resources
+// https://github.com/hashicorp/terraform/issues/953
+variable "enabled" {
+  default     = true
+  description = "If true, the Lambda function and all associated components will be created"
+}
+
+variable "function_name" {
+  description = "Name of the Lambda function"
+}
+
+variable "description" {
+  default     = ""
+  description = "Description of the Lambda function"
+}
+
+variable "runtime" {
+  default     = "python2.7"
+  description = "Function runtime environment"
+}
+
+variable "handler" {
+  description = "Entry point for the function"
+}
+
+variable "memory_size_mb" {
+  default     = 128
+  description = "Memory allocated to the function. CPU and network are allocated proportionally."
+}
+
+variable "timeout_sec" {
+  default     = 30
+  description = "Maximum duration before execution is terminated"
+}
+
+variable "source_bucket" {
+  description = "S3 bucket containing function source code"
+}
+
+variable "source_object_key" {
+  description = "S3 object key pointing to the function source code"
+}
+
+variable "environment_variables" {
+  type        = "map"
+  description = "Map of environment variables available to the running Lambda function"
+}
+
+variable "vpc_subnet_ids" {
+  type        = "list"
+  default     = []
+  description = "Optional list of VPC subnet IDs"
+}
+
+variable "vpc_security_group_ids" {
+  type        = "list"
+  default     = []
+  description = "Optional list of security group IDs (for VPC)"
+}
+
+variable "name_tag" {
+  default     = "StreamAlert"
+  description = "The value for the Name cost tag associated with all applicable components"
+}
+
+variable "aliased_version" {
+  default     = ""
+  description = "Alias points to this version (or the latest published version if not specified)"
+}
+
+variable "log_retention_days" {
+  default     = 14
+  description = "CloudWatch logs for the Lambda function will be retained for this many days"
+}
+
+// CloudWatch metric alarms
+
+variable "alarm_actions" {
+  type        = "list"
+  default     = []
+  description = "Optional list of CloudWatch alarm actions (e.g. SNS topic ARNs)"
+}
+
+variable "errors_alarm_threshold" {
+  default     = 0
+  description = "Alarm if Lambda invocation errors exceed this value in the specified period(s)"
+}
+
+variable "errors_alarm_evaluation_periods" {
+  default     = 1
+  description = "Number of periods over which to evaluate the number invocation errors"
+}
+
+variable "errors_alarm_period_secs" {
+  default     = 60
+  description = "Period over which to count the number of invocation errors"
+}
+
+variable "throttles_alarm_threshold" {
+  default     = 0
+  description = "Alarm if Lambda throttles exceed this value in the specified period(s)"
+}
+
+variable "throttles_alarm_evaluation_periods" {
+  default     = 1
+  description = "Number of periods over which to evaluate the number of throttles"
+}
+
+variable "throttles_alarm_period_secs" {
+  default     = 60
+  description = "Period over which to count the number of throttles"
+}
+
+variable "enable_iterator_age_alarm" {
+  default     = false
+  description = "Enable IteratorAge alarm (applicable only for stream-based invocations like Kinesis)"
+}
+
+variable "iterator_age_alarm_threshold" {
+  default     = 0
+  description = "Alarm if the Lambda IteratorAge exceeds this value in the specified period(s)"
+}
+
+variable "iterator_age_alarm_evaluation_periods" {
+  default     = 1
+  description = "Number of periods over which to evaluate the IteratorAge"
+}
+
+variable "iterator_age_alarm_period_secs" {
+  default     = 60
+  description = "Period over which to evaluate the maximum IteratorAge"
+}