Add config for staging cluster (#360)

Add config to create staging image Add config to deploy to staging custer
hikaya-io · Apr 6, 2020 · 21eb241 · 21eb241
1 parent 4461aab
commit 21eb241
Show file tree

Hide file tree

Showing 9 changed files with 216 additions and 39 deletions.
diff --git a/.env.dev b/.env.dev
@@ -6,4 +6,6 @@ DB_NAME=activity_dev
 DB_USER=activity
 DB_PASSWORD=activity
 DB_HOST=db
-DB_PORT=5432
+DB_PORT=5432
+SOCIAL_AUTH_GOOGLE_OAUTH2_KEY='your google auth key'
+SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET="your google auth secret"
diff --git a/.github/workflows/activity_ci_cd.yml b/.github/workflows/activity_ci_cd.yml
@@ -5,10 +5,12 @@ on:
     branches: 
       - dev
       - master
+      - staging
   pull_request:
     branches: 
       - dev
       - master
+      - staging
 
 jobs:
   Build_and_test:
@@ -72,8 +74,14 @@ jobs:
         SERVICE_ACCESS_TOKEN: ${{ secrets.SERVICE_ACCESS_TOKEN }}
         DB_SIZE: ${{ secrets.DB_SIZE }}
         PG_VERSION: ${{ secrets.PG_VERSION }}
-        DB_NAME_DEV_ENV: ${{ secrets.DB_NAME_DEV_ENV}}
-
+        DB_NAME_DEV_ENV: ${{ secrets.DB_NAME_DEV_ENV }}
+        # staging vars, some are reused from dev_env
+        BACKEND_BUCKET_STAGING: ${{ secrets.BACKEND_BUCKET_STAGING }}
+        STATE_FILE_STAGING: ${{ secrets.STATE_FILE_STAGING }}
+        CLUSTER_NAME_STAGING: ${{ secrets.CLUSTER_NAME_STAGING }}
+        K8S_VERSION_STAGING: ${{ secrets.K8S_VERSION_STAGING }}
+        DB_NAME_STAGING: ${{ secrets.DB_NAME_STAGING }}
+
       run: |
         export TRAVIS_EVENT_TYPE="push"
         chmod +x ci-scripts/create_infrastructure.sh
@@ -100,19 +108,24 @@ jobs:
         ACTIVITY_CE_DB_USER_DEV: ${{ secrets.ACTIVITY_CE_DB_USER_DEV }}
         ACTIVITY_CE_DB_PASSWORD_DEV: ${{ secrets.ACTIVITY_CE_DB_PASSWORD_DEV }}
         ACTIVITY_CE_DB_HOST_DEV: ${{ secrets.ACTIVITY_CE_DB_HOST_DEV }}
-        SECRET_KEY_DEV_ENV: ${{ secrets.SECRET_KEY_DEV_ENV }}
+        SECRET_KEY: ${{ secrets.SECRET_KEY }}
         DEBUG: ${{ secrets.DEBUG }}
         DJANGO_ALLOWED_HOSTS: ${{ secrets.DJANGO_ALLOWED_HOSTS }}
         DB_ENGINE: ${{ secrets.DB_ENGINE }}
         ACTIVITY_CE_DB_PORT_DEV: ${{ secrets.ACTIVITY_CE_DB_PORT_DEV }}
         APPLICATION_ENV: ${{ secrets.APPLICATION_ENV }}
         EMAIL_PORT: ${{ secrets.EMAIL_PORT }}
-        EMAIL_HOST_USER_DEV: ${{ secrets.EMAIL_HOST_USER_DEV }}
-        EMAIL_HOST_PASSWORD_DEV: ${{ secrets.EMAIL_HOST_PASSWORD_DEV }}
+        EMAIL_HOST_USER: ${{ secrets.EMAIL_HOST_USER }}
+        EMAIL_HOST_PASSWORD: ${{ secrets.EMAIL_HOST_PASSWORD }}
         EMAIL_HOST: ${{ secrets.EMAIL_HOST }}
-        GOOGLE_MAP_API_KEY_DEV: ${{ secrets.GOOGLE_MAP_API_KEY_DEV }}
+        GOOGLE_MAP_API_KEY: ${{ secrets.GOOGLE_MAP_API_KEY }}
         APPLICATION_NAME: ${{ secrets.APPLICATION_NAME }}
         REGISTRY_OWNER: ${{ secrets.REGISTRY_OWNER }}
+        SOCIAL_AUTH_GOOGLE_OAUTH2_KEY: ${{ secrets.SOCIAL_AUTH_GOOGLE_OAUTH2_KEY }}
+        SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET: ${{ secrets.SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET }}
+        # staging variables. others are reused from dev
+        ACTIVITY_CE_DB_NAME_STAGING: ${{ secrets.ACTIVITY_CE_DB_NAME_STAGING }}
+        APPLICATION_ENV_STAGING: ${{ secrets.APPLICATION_ENV_STAGING }}
 
       run: |
         export TRAVIS_COMMIT=$GITHUB_SHA
@@ -141,21 +154,23 @@ jobs:
         HOST_DOMAIN: ${{ secrets.HOST_DOMAIN }}
         APPLICATION_NAME: ${{ secrets.APPLICATION_NAME }}
         REGISTRY_OWNER: ${{ secrets.REGISTRY_OWNER }}
+        # staging variables
+        CLUSTER_NAME_STAGING: ${{ secrets.CLUSTER_NAME_STAGING }}
+        HOST_DOMAIN_STAGING: ${{ secrets.HOST_DOMAIN_STAGING }}
+        APPLICATION_ENV_STAGING: ${{ secrets.APPLICATION_ENV_STAGING }}
+        SLACK_CHANNEL_HOOK: ${{ secrets.SLACK_CHANNEL_HOOK }}
       run: |
         export TRAVIS_COMMIT=$GITHUB_SHA
         export FILE_PATH="/github/home/.docker/config.json"
         export TRAVIS_EVENT_TYPE="push"
         chmod +x ci-scripts/deployment.sh
         bash ./ci-scripts/deployment.sh
-    
-  Notify_slack:
-    needs: Deploy 
-    runs-on: ubuntu-latest
-    if: github.event.pull_request.merged
-    steps:
-    - uses: 8398a7/action-slack@v2
-      with:
-        status: ${{ job.status }}
-        author_name: Github Actions Deploy
-      env:
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CHANNEL_HOOK }}
+        if [[ $? -eq 0 ]]; then
+            export DEPLOY="success"
+        else
+            export DEPLOY="fail"
+        fi
+        export EMOJI=":github:"
+        export TRAVIS_COMMIT_MESSAGE="No Commit message"
+        chmod +x ci-scripts/notify_slack.sh
+        bash ./ci-scripts/notify_slack.sh
diff --git a/.travis.yml b/.travis.yml
@@ -5,7 +5,7 @@ language: minimal
 
 notifications:
   slack:
-    if: branch = dev
+    if: branch = dev OR branch = staging
     on_pull_requests: false
     secure: OzJ+zoqdrsgO0Fx2cfat1EyfcqwkBi5WJ9zRr5cEZxb6Y3mlkiURuzE1jgLyvOwzJln0H+eWNlUZ6sy9ijpoEdKDJtAftkOkbQuk1WWG+kzbZ2zhMxKP32B0Cemy44VP3aHljPmzQQN58Pfl+IO1GbxGSaGN/DUPW8NXdKC7+I3RlGHoh0pVoqsnJgNKzf2G9uAfVMujs7Mhc9n6qoUNFp8kRHgphOAEP0+B5ioG6/lei2WKhEPzn582ElHODFSfyg/TSzH4lFHAudgZQ6roA5b4CQRfUUg1hLC4m9NBBTFBMXk1IACWt1rBp9ufgKRrRCVs1n1w9JSITXB5c6cmONQuv6IgjIuJuqd+bH9sqxl5Hd/FBemkNXS5jcQ+IgGiO8sMGgvZ+ZCb1rgQdUH6vUykOgHpiIgfDbOXQf5uE/o7yiSVKh/QncbRZ96pStV2HR5KOyYK7d27UBmtRnA9GtNK0EAw+GpQfu4/jn6gaBpSoArpbLpKt8NksYfDcpyo//xLCrxN0u8VdLdayCXcJ2OaFw9imd5NDB55GV4BfxG3JOQnUNA1pAdSZWPmvb8COTfHPeAxyU24qGgURU0xLppCkz3QybSjIjtR55bLAuoqhfKEBsaJhWoYQP6XEG8/eZm18swqZQG4QEVEOEWkPLnxwRPjk9KkJiPYphlCkdI=
 jobs:
@@ -37,7 +37,7 @@ jobs:
       - chmod +x ci-scripts/build_docker_image.sh
       - ./ci-scripts/build_docker_image.sh
 
-    # Stage to deploy application
+    #@--- Stage to deploy application ---@#
     - stage: Deploy
       services:
         - docker

diff --git a/ci-scripts/build_docker_image.sh b/ci-scripts/build_docker_image.sh
@@ -9,29 +9,61 @@ docker_hub_auth() {
 
 }
 
+#@--- Function to export env variables ---@#
+export_variables() {
+    touch .env.deploy
+    echo export SECRET_KEY=${SECRET_KEY} >> .env.deploy
+    echo export DEBUG=${DEBUG} >> .env.deploy
+    echo export DJANGO_ALLOWED_HOSTS=${DJANGO_ALLOWED_HOSTS} >> .env.deploy
+    echo export DB_ENGINE=${DB_ENGINE} >> .env.deploy
+    echo export EMAIL_PORT=${EMAIL_PORT} >> .env.deploy
+    echo export EMAIL_HOST_USER=${EMAIL_HOST_USER} >> .env.deploy
+    echo export EMAIL_HOST_PASSWORD=${EMAIL_HOST_PASSWORD} >> .env.deploy
+    echo export EMAIL_HOST=${EMAIL_HOST} >> .env.deploy
+    echo export GOOGLE_MAP_API_KEY=${GOOGLE_MAP_API_KEY} >> .env.deploy
+    echo export SOCIAL_AUTH_GOOGLE_OAUTH2_KEY=${SOCIAL_AUTH_GOOGLE_OAUTH2_KEY} >> .env.deploy
+    echo export SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET=${SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET} >> .env.deploy
+}
+
 #@--- Build docker image  and push---@#
 build_and_push_image() {
 
     #@--- Build image for deployment ---@#
     echo "++++++++ Start building image +++++++++"
     if [[ $TRAVIS_BRANCH == "dev" ]] || [[ $GITHUB_REF == "refs/heads/dev" ]]; then
-        #@--- Set all the required variables ---@#
-        touch .env.deploy
+
+        #@--- Run export function ---@#
+        export_variables
 
         echo export ACTIVITY_CE_DB_NAME=${ACTIVITY_CE_DB_NAME_DEV} >> .env.deploy
         echo export ACTIVITY_CE_DB_USER=${ACTIVITY_CE_DB_USER_DEV} >> .env.deploy
         echo export ACTIVITY_CE_DB_PASSWORD=${ACTIVITY_CE_DB_PASSWORD_DEV} >> .env.deploy
         echo export ACTIVITY_CE_DB_HOST=${ACTIVITY_CE_DB_HOST_DEV} >> .env.deploy
-        echo export SECRET_KEY=${SECRET_KEY_DEV_ENV} >> .env.deploy
-        echo export DEBUG=${DEBUG} >> .env.deploy
-        echo export DJANGO_ALLOWED_HOSTS=${DJANGO_ALLOWED_HOSTS} >> .env.deploy
-        echo export DB_ENGINE=${DB_ENGINE} >> .env.deploy
         echo export ACTIVITY_CE_DB_PORT=${ACTIVITY_CE_DB_PORT_DEV} >> .env.deploy
-        echo export EMAIL_PORT=${EMAIL_PORT} >> .env.deploy
-        echo export EMAIL_HOST_USER=${EMAIL_HOST_USER_DEV} >> .env.deploy
-        echo export EMAIL_HOST_PASSWORD=${EMAIL_HOST_PASSWORD_DEV} >> .env.deploy
-        echo export EMAIL_HOST=${EMAIL_HOST} >> .env.deploy
-        echo export GOOGLE_MAP_API_KEY=${GOOGLE_MAP_API_KEY_DEV} >> .env.deploy
+
+        docker build -t $REGISTRY_OWNER/activity:$APPLICATION_NAME-$APPLICATION_ENV-$TRAVIS_COMMIT -f docker-deploy/Dockerfile .  
+        echo "-------- Building Image Done! ----------"
+
+        echo "++++++++++++ Push Image built -------"
+        docker push $REGISTRY_OWNER/activity:$APPLICATION_NAME-$APPLICATION_ENV-$TRAVIS_COMMIT
+
+    fi
+
+    #@--- Build staging image ---@#
+
+    if [[ $TRAVIS_BRANCH == "staging" ]] || \
+        [[ $GITHUB_REF == "refs/heads/staging" ]]; then
+        ECHO "++++++ Build Staging Image +++++++++++"
+
+        #@--- Run export function ---@#
+        export_variables
+
+        echo export ACTIVITY_CE_DB_NAME=${ACTIVITY_CE_DB_NAME_STAGING} >> .env.deploy
+        echo export ACTIVITY_CE_DB_USER=${ACTIVITY_CE_DB_USER_DEV} >> .env.deploy
+        echo export ACTIVITY_CE_DB_PASSWORD=${ACTIVITY_CE_DB_PASSWORD_DEV} >> .env.deploy
+        echo export ACTIVITY_CE_DB_HOST=${ACTIVITY_CE_DB_HOST_DEV} >> .env.deploy
+        echo export ACTIVITY_CE_DB_PORT=${ACTIVITY_CE_DB_PORT_DEV} >> .env.deploy
+        export APPLICATION_ENV=${APPLICATION_ENV_STAGING}
 
         docker build -t $REGISTRY_OWNER/activity:$APPLICATION_NAME-$APPLICATION_ENV-$TRAVIS_COMMIT -f docker-deploy/Dockerfile .  
         echo "-------- Building Image Done! ----------"

diff --git a/ci-scripts/create_infrastructure.sh b/ci-scripts/create_infrastructure.sh
@@ -43,6 +43,50 @@ set_up_cluster_dev_env() {
     fi
 }
 
+#@--- Function to setup staging cluster ---@#
+set_up_cluster_staging() {
+
+    if [[ $TRAVIS_BRANCH == "staging" ]] || [[ $GITHUB_REF == "refs/heads/staging" ]]; then
+
+        #@--- Initialize terraform ---@#
+        echo " +++++++ Initialize the backend ++++++++++ "
+        terraform init -backend-config "bucket=$BACKEND_BUCKET_STAGING" \
+            -backend-config "key=$STATE_FILE_STAGING" \
+            -backend-config "access_key=$SPACES_ACCESS_KEY" \
+            -backend-config "secret_key=$SPACES_SECRET_KEY"
+
+        #@--- Run terraform command to plan infrastructure ---@#
+        echo "----- show plan -------------------"
+        terraform plan -lock=false -target=digitalocean_kubernetes_cluster.cluster \
+            -var "cluster_name=$CLUSTER_NAME_STAGING" \
+            -var "cluster_region=$CLUSTER_REGION" \
+            -var "kubernetes_version=$K8S_VERSION_STAGING" \
+            -var "node_type=$NODE_TYPE" \
+            -var "max_node_number=$MAX_NODE_NUM" \
+            -var "min_node_number=$MIN_NODE_NUM" \
+            -var "digital_ocean_token=$SERVICE_ACCESS_TOKEN" \
+            -var "db_size=$DB_SIZE" \
+            -var "postgres_version=$PG_VERSION" \
+            -var "db_name=$DB_NAME_STAGING"
+
+        #@--- Apply the changes ---@#
+        echo "+++++ Apply infrastructure ++++++++++"
+        terraform apply -lock=false -auto-approve -target=digitalocean_kubernetes_cluster.cluster \
+            -var "cluster_name=$CLUSTER_NAME_STAGING" \
+            -var "cluster_region=$CLUSTER_REGION" \
+            -var "kubernetes_version=$K8S_VERSION_STAGING" \
+            -var "node_type=$NODE_TYPE" \
+            -var "max_node_number=$MAX_NODE_NUM" \
+            -var "min_node_number=$MIN_NODE_NUM" \
+            -var "digital_ocean_token=$SERVICE_ACCESS_TOKEN" \
+            -var "db_size=$DB_SIZE" \
+            -var "postgres_version=$PG_VERSION" \
+            -var "db_name=$DB_NAME_STAGING" \
+            || echo "Resources exist"
+    fi
+
+}
+
 #@--- Main function ---@#
 main() {
     cd infrastructure
@@ -51,6 +95,9 @@ main() {
 
             #@--- Run the setup dev-env cluster function ---@#
             set_up_cluster_dev_env
+
+            #@--- Run function for staging cluster ---@#
+            set_up_cluster_staging
     fi
 
 }

diff --git a/ci-scripts/deployment.sh b/ci-scripts/deployment.sh
@@ -4,7 +4,8 @@ set +ex
 
 #@--- install kubectl and doctl ---@#
 install_kubectl_doctl() {
-    if [[ $TRAVIS_BRANCH == "dev" ]]; then
+    if [[ $TRAVIS_BRANCH == "dev" ]] || \
+        [[ $TRAVIS_BRANCH == "staging" ]]; then
         echo "++++++++++++ install kubectl ++++++++++++"
         curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl
 
@@ -30,9 +31,12 @@ install_kubectl_doctl() {
 #@--- Authorize kubectl to cluster ---@#
 auth_kubectl_cluster() {
     # Authenticate kubectl to the cluster
-    if [[ $TRAVIS_BRANCH == "dev" ]] || [[ $GITHUB_REF == "refs/heads/dev" ]]; then
-        doctl -t $SERVICE_ACCESS_TOKEN kubernetes cluster kubeconfig save $CLUSTER_NAME_DEV_ENV
+    if [[ $TRAVIS_BRANCH == "dev" ]] || \
+        [[ $GITHUB_REF == "refs/heads/dev" ]] || \
+        [[ $TRAVIS_BRANCH == "staging" ]] || \
+        [[ $GITHUB_REF == "refs/heads/staging" ]]; then
         doctl auth init -t $SERVICE_ACCESS_TOKEN
+        doctl -t $SERVICE_ACCESS_TOKEN kubernetes cluster kubeconfig save $CLUSTER_NAME
         kubectl create namespace $APPLICATION_ENV || echo "++++++ Namespace Exists ++++++"
         kubectl create namespace ingress-nginx || echo "++++++ Namespace ingress-nginx Exists ++++++"
     fi
@@ -53,7 +57,11 @@ deploy_app() {
         --from-file=.dockerconfigjson=$FILE_PATH \
         --type=kubernetes.io/dockerconfigjson -n $APPLICATION_ENV
 
-    if [[ $TRAVIS_BRANCH == "dev" ]]  || [[ $GITHUB_REF == "refs/heads/dev" ]]; then
+    if [[ $TRAVIS_BRANCH == "dev" ]]  || \
+        [[ $GITHUB_REF == "refs/heads/dev" ]] || \
+        [[ $TRAVIS_BRANCH == "staging" ]] || \
+        [[ $GITHUB_REF == "refs/heads/staging" ]]; then
+
         echo "------- generate deployfiles --------------"
         envsubst < ./deployment_files/deployment > deployment.yaml
         envsubst < ./deployment_files/service > service.yaml
@@ -82,14 +90,32 @@ deploy_app() {
     fi
 }
 
+#@--- Function to replace some key variables ---@#
+replace_variables() {
+    if [[ $TRAVIS_BRANCH == "staging" ]] || \
+        [[ $GITHUB_REF == "refs/heads/staging" ]]; then
+        export APPLICATION_ENV=${APPLICATION_ENV_STAGING}
+        export HOST_DOMAIN=${HOST_DOMAIN_STAGING}
+        export CLUSTER_NAME=${CLUSTER_NAME_STAGING}
+
+    fi
+
+    if [[ $TRAVIS_BRANCH == "dev" ]] || \
+        [[ $GITHUB_REF == "refs/heads/dev" ]]; then
+        export CLUSTER_NAME=${CLUSTER_NAME_DEV_ENV}
+    fi
+}
 
 #@--- Main Function ---@#
 main() {
-
+    
     if [[ $TRAVIS_EVENT_TYPE != "pull_request" ]]; then
         #@--- Run install and setup function ---@#
         install_kubectl_doctl
 
+        #@--- run the replace function ---@#
+        replace_variables
+
         #@--- Run the setup function ---@#
         auth_kubectl_cluster
 

diff --git a/ci-scripts/notify_slack.sh b/ci-scripts/notify_slack.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -ex
+set -o pipefail
+
+#@--- Fuction to replace varibels for branch staging ---@#
+replace_env_varibles() {
+    if [[ $GITHUB_REF == "refs/heads/staging" ]]; then
+        export APPLICATION_ENV=${APPLICATION_ENV_STAGING}
+    fi
+}
+
+
+#@--- Function to setup the message to be send ---@#
+setup_message() {
+    if  [[ $GITHUB_REF == "refs/heads/dev" ]] || \
+        [[ $GITHUB_REF == "refs/heads/staging" ]]; then
+        if [[ $DEPLOY == "success" ]]; then
+            echo "++++++++++++ generate deployment message +++++++++++++"
+            COMMIT_URL="https://github.com/hikaya-io/activity/commit/${TRAVIS_COMMIT}"
+            DEPLOYMENT_MESSAGE="*Success* :white_check_mark: The following commit was deployed to *_activity ${APPLICATION_ENV}_* by ${EMOJI} \n [Message]: $TRAVIS_COMMIT_MESSAGE \n [Link]: ${COMMIT_URL}"
+        else
+            COMMIT_URL="https://github.com/hikaya-io/activity/commit/${TRAVIS_COMMIT}"
+            DEPLOYMENT_MESSAGE="*Failed* :no_entry: The following commit was unable to deploy to *_activity ${APPLICATION_ENV}_* by ${EMOJI} \n [Message]: $TRAVIS_COMMIT_MESSAGE \n [Link]: ${COMMIT_URL}"
+        fi
+    fi
+
+}
+
+#@--- Function to send slack notification ---@#
+send_slack_notification() {
+
+    echo "++++++++++++ sending slack message +++++++++++++"
+    echo $DEPLOYMENT_MESSAGE
+    curl -X POST --data-urlencode \
+    "payload={\"channel\": \"${DEPLOYMENT_CHANNEL}\", \"username\": \"DeployNotification\", \"text\": \"${DEPLOYMENT_MESSAGE}\", \"icon_emoji\": \":rocket:\"}" \
+    "${SLACK_CHANNEL_HOOK}"
+
+}
+
+#@--- Main function ---@#
+main() {
+    #@--- run the replace function ---@#
+    replace_env_varibles
+
+    #@-- Run the message function ---@#
+    setup_message
+
+    #@--- Run slack notify function ---@#
+    send_slack_notification
+}
+
+#@--- Run main function ---@#
+main
diff --git a/ci-scripts/run_tests.sh b/ci-scripts/run_tests.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set +ex
+set -ex
 
 #@--- install and setup python ---@#
 setup_python() {