diff --git a/.gitignore b/.gitignore index 9cc6b3f5745a..a68b1396d1db 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ dist .aws-sam .idea pkg/loki/wal +tools/lambda-promtail/main # Workspaces *.work diff --git a/tools/lambda-promtail/Dockerfile b/tools/lambda-promtail/Dockerfile index b0852baac0f6..a54740225ceb 100644 --- a/tools/lambda-promtail/Dockerfile +++ b/tools/lambda-promtail/Dockerfile @@ -15,4 +15,4 @@ RUN go build -o /main -tags lambda.norpc -ldflags="-s -w" lambda-promtail/*.go # copy artifacts to a clean image FROM public.ecr.aws/lambda/provided:al2 COPY --from=build-image /main /main -ENTRYPOINT [ "/main" ] \ No newline at end of file +ENTRYPOINT [ "/main" ] diff --git a/tools/lambda-promtail/README.md b/tools/lambda-promtail/README.md index 32b52db38c6b..27492a44b40f 100644 --- a/tools/lambda-promtail/README.md +++ b/tools/lambda-promtail/README.md @@ -16,7 +16,7 @@ This is a sample deployment for lambda-promtail - Below is a brief explanation o * AWS CLI already configured with Administrator permission * [Terraform](https://www.terraform.io/downloads.html) -If you want to modify the lambda-promtail code you will also need: +If you want to modify the lambda-promtail code you will also need: * [Docker installed](https://www.docker.com/community-edition) * [Golang](https://golang.org) @@ -26,17 +26,11 @@ If you want to modify the lambda-promtail code you will also need: The provided Makefile has targets `build`, and `clean`. -`build` builds the lambda-promtail as a Go static binary. To build the container image properly you should run `docker build . -f tools/lambda-promtail/Dockerfile` from the root of the Loki repository,you can upload this image to your AWS ECR and use via Lambda. `clean` will remove the built Go binary. +`build` builds the lambda-promtail as a Go static binary. To build the container image properly you should run `docker build . -f tools/lambda-promtail/Dockerfile` from the root of the Loki repository, you can upload this image to your AWS ECR and use via Lambda or if you don't pass a `lambda_promtail_image` value, the terraform will build it from the Loki reposiroty, zip it and use it via Lambda. `clean` will remove the built Go binary. ### Packaging and deployment -The easiest way to deploy to AWS Lambda using the Golang runtime is to use the `lambda-promtail` image by uploading it to your ECR. - -Alternatively you can build the Go binary and upload it to Lambda as a zip: -```bash -GOOS=linux CGO_ENABLED=0 go build main.go -zip function.zip main -``` +The easiest way to deploy to AWS Lambda using the Golang runtime is to build the `lambda-promtail` go file, zip it and upload it to the lambda function with terraform. To deploy your application for the first time, first make sure you've set the following value in the Terraform file: - `WRITE_ADDRESS` @@ -47,19 +41,24 @@ The `lambda-promtail` code picks this value up via an environment variable. Also, if your deployment requires a [VPC configuration](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function#vpc_config), make sure to edit the `vpc_config` field in `main.tf` manually. Additonal documentation for the Lambda specific Terraform configuration is [here](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function#vpc_config). If you want to link kinesis data stream to Lambda as event source, see [here](https://docs.aws.amazon.com/ko_kr/lambda/latest/dg/with-kinesis.html). -`lambda-promtail` supports authentication either using HTTP Basic Auth or using Bearer Token. -For development purposes you can set the environment variable SKIP_TLS_VERIFY to `true`, so you can use self signed certificates, but this is not recommended in production. Default is `false`. +`lambda-promtail` supports authentication either using HTTP Basic Auth or using Bearer Token. +For development purposes, you can set the environment variable SKIP_TLS_VERIFY to `true`, so you can use self-signed certificates, but this is not recommended in production. Default is `false`. Then use Terraform to deploy: ```bash ## use cloudwatch log group -terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'bearer_token=' -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'drop_labels="name1,name2"' -var "tenant_id=" -var 'skip_tls_verify="false"' +terraform apply -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'bearer_token=' -var 'log_group_names=["log-group-01", "log-group-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'drop_labels="name1,name2"' -var "tenant_id=" -var 'skip_tls_verify="false"' +``` + +```bash +## use docker image uploaded to ECR +terraform apply -var "lambda_promtail_image=:" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'bearer_token=' -var 'extra_labels="name1,value1,name2,value2"' -var 'drop_labels="name1,name2"' -var "tenant_id=" -var 'skip_tls_verify="false"' ``` ```bash ## use kinesis data stream -terraform apply -var ":" -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'drop_labels="name1,name2"' -var "tenant_id=" -var 'skip_tls_verify="false"' +terraform apply -var "write_address=https://your-loki-url/loki/api/v1/push" -var "password=" -var "username=" -var 'kinesis_stream_name=["kinesis-stream-01", "kinesis-stream-02"]' -var 'extra_labels="name1,value1,name2,value2"' -var 'drop_labels="name1,name2"' -var "tenant_id=" -var 'skip_tls_verify="false"' ``` or CloudFormation: @@ -68,6 +67,8 @@ or CloudFormation: aws cloudformation create-stack --stack-name lambda-promtail-stack --template-body file://template.yaml --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM --region us-east-2 --parameters ParameterKey=WriteAddress,ParameterValue=https://your-loki-url/loki/api/v1/push ParameterKey=Username,ParameterValue= ParameterKey=Password,ParameterValue= ParameterKey=BearerToken,ParameterValue= ParameterKey=LambdaPromtailImage,ParameterValue=: ParameterKey=ExtraLabels,ParameterValue="name1,value1,name2,value2" ParameterKey=TenantID,ParameterValue= ParameterKey=SkipTlsVerify,ParameterValue="false" ``` +**NOTE:** To use CloudFormation, you must build the docker image with `docker build . -f tools/lambda-promtail/Dockerfile` from the root of the Loki repository, upload it to an ECR, and pass it as the `LambdaPromtailImage` parameter to cloudformation. + # Appendix ## Golang installation diff --git a/tools/lambda-promtail/main.tf b/tools/lambda-promtail/main.tf index 3c7e74fdc7e5..11c97513798d 100644 --- a/tools/lambda-promtail/main.tf +++ b/tools/lambda-promtail/main.tf @@ -148,15 +148,46 @@ resource "aws_cloudwatch_log_group" "this" { retention_in_days = 14 } +locals { + binary_path = "${path.module}/lambda-promtail/bootstrap" + archive_path = "${path.module}/lambda-promtail/${var.name}.zip" +} + +resource "null_resource" "function_binary" { + count = var.lambda_promtail_image == "" ? 1 : 0 + triggers = { + always_run = timestamp() + } + + provisioner "local-exec" { + command = "GOOS=linux GOARCH=amd64 CGO_ENABLED=0 GOFLAGS=-trimpath go build -mod=readonly -ldflags='-s -w' -o bootstrap" + working_dir = format("%s/%s", path.module, "lambda-promtail") + } +} + +data "archive_file" "lambda" { + count = var.lambda_promtail_image == "" ? 1 : 0 + depends_on = [null_resource.function_binary[0]] + + type = "zip" + source_file = local.binary_path + output_path = local.archive_path +} + resource "aws_lambda_function" "this" { - image_uri = var.lambda_promtail_image function_name = var.name role = aws_iam_role.this.arn kms_key_arn = var.kms_key_arn + image_uri = var.lambda_promtail_image == "" ? null : var.lambda_promtail_image + filename = var.lambda_promtail_image == "" ? local.archive_path : null + source_code_hash = var.lambda_promtail_image == "" ? data.archive_file.lambda[0].output_base64sha256 : null + runtime = var.lambda_promtail_image == "" ? "provided.al2023" : null + handler = var.lambda_promtail_image == "" ? local.binary_path : null + timeout = 60 memory_size = 128 - package_type = "Image" + package_type = var.lambda_promtail_image == "" ? "Zip" : "Image" # From the Terraform AWS Lambda docs: If both subnet_ids and security_group_ids are empty then vpc_config is considered to be empty or unset. vpc_config { diff --git a/tools/lambda-promtail/versions.tf b/tools/lambda-promtail/versions.tf index c77dc04fa988..c03bf2d9c080 100644 --- a/tools/lambda-promtail/versions.tf +++ b/tools/lambda-promtail/versions.tf @@ -4,5 +4,11 @@ terraform { aws = { source = "hashicorp/aws" } + archive = { + source = "hashicorp/archive" + } + null = { + source = "hashicorp/null" + } } }