Merge pull request buildkite#4 from aurora-tech/kevingu/buildkite-dep…

…loyment-for-aurora

Setup deployment of autoscaler lambda for aurora

.buildkite/pipeline.release.yml

@@ -5,22 +5,6 @@ steps:
     command: ".buildkite/steps/release-version.sh"
     branches: master
     agents:
-      queue: "deploy"
+      queue: "ci-admin"
     concurrency: 1
     concurrency_group: 'release'
-
-  - wait
-  - label: ":github:"
-    command: ".buildkite/steps/github-release.sh"
-    branches: master
-    agents:
-      queue: "deploy"
-    concurrency: 1
-    concurrency_group: 'release'
-    plugins:
-      - ecr#v2.0.0:
-          login: true
-          account-ids: "032379705303"
-      - docker#v3.5.0:
-          image: "032379705303.dkr.ecr.us-east-1.amazonaws.com/deploytools:2020.03"
-          propagate-environment: true

.buildkite/pipeline.yml

@@ -1,22 +1,30 @@
+defaults: &defaults
+  timeout_in_minutes: 60
+  agents:
+    queue: "ci-non-gpu-basic"
+
 steps:
-  - name: ":golang:"
+  - <<: *defaults
+    name: ":golang:"
     command: ".buildkite/steps/tests.sh"
     plugins:
       - docker#v3.1.0:
           image: "golang:1.15"
-          workdir: /go/src/github.com/buildkite/buildkite-agent-scaler
+          propagate-uid-gid: true
 
   - wait
-  - name: ":lambda:"
-    command: .buildkite/steps/build-lambda.sh
+  - <<: *defaults
+    name: ":lambda:"
+    command: ".buildkite/steps/build-lambda.sh"
     artifact_paths:
       - handler.zip
 
   - wait
-  - label: ":s3:"
+  - <<: *defaults
+    label: ":s3:"
     command: ".buildkite/steps/upload-to-s3.sh"
     agents:
-      queue: "deploy"
+      queue: "ci-admin"
 
   - wait
   - name: ":pipeline:"

.buildkite/steps/tests.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euo pipefail
 
-GO111MODULE=off go get gotest.tools/gotestsum
+GO111MODULE=off GOCACHE=/tmp go get gotest.tools/gotestsum
 
 echo '+++ Running tests'
-gotestsum --junitfile "junit-${OSTYPE}.xml" -- -count=1 -failfast ./...
+GOCACHE=/tmp gotestsum --junitfile "junit-${OSTYPE}.xml" -- -count=1 -failfast ./...

.buildkite/steps/upload-to-s3.sh

@@ -28,7 +28,7 @@ EXTRA_REGIONS=(
 )
 
 VERSION=$(buildkite-agent meta-data get "version")
-BASE_BUCKET=buildkite-lambdas
+BASE_BUCKET=aurora-buildkite-lambda
 BUCKET_PATH="buildkite-agent-scaler"
 
 if [[ "${1:-}" == "release" ]] ; then
@@ -41,10 +41,4 @@ echo "~~~ :buildkite: Downloading artifacts"
 buildkite-agent artifact download handler.zip .
 
 echo "--- :s3: Uploading lambda to ${BASE_BUCKET}/${BUCKET_PATH}/ in ${AWS_DEFAULT_REGION}"
-aws s3 cp --acl public-read handler.zip "s3://${BASE_BUCKET}/${BUCKET_PATH}/handler.zip"
-
-for region in "${EXTRA_REGIONS[@]}" ; do
-	bucket="${BASE_BUCKET}-${region}"
-	echo "--- :s3: Copying files to ${bucket}"
-	aws --region "${region}" s3 cp --acl public-read "s3://${BASE_BUCKET}/${BUCKET_PATH}/handler.zip" "s3://${bucket}/${BUCKET_PATH}/handler.zip"
-done
+aws s3 cp handler.zip "s3://${BASE_BUCKET}/${BUCKET_PATH}/handler.zip"

Makefile

@@ -10,6 +10,7 @@ clean:
 
 LAMBDA_S3_BUCKET := aurora-buildkite-autoscaler-lambda
 LAMBDA_S3_BUCKET_PATH := /
+USER := --user $(shell id -u):$(shell id -g)
 
 ifdef BUILDKITE_BUILD_NUMBER
 	LD_FLAGS := -s -w -X version.Build=$(BUILDKITE_BUILD_NUMBER)
@@ -26,12 +27,12 @@ handler.zip: lambda/handler
 
 lambda/handler: lambda/main.go
 	docker run \
-		--volume go-module-cache:/go/pkg/mod \
+		${USER} \
 		--volume $(CURDIR):/go/src/github.com/buildkite/buildkite-agent-scaler \
 		--workdir /go/src/github.com/buildkite/buildkite-agent-scaler \
+		--env GOCACHE=/tmp \
 		--rm golang:1.15 \
 		go build -ldflags="$(LD_FLAGS)" -o ./lambda/handler ./lambda
-	chmod +x lambda/handler
 
 lambda-sync: handler.zip
 	/opt/aurora/bin/aws s3 sync \