Add github actions

.github/workflows/cicd.yml

@@ -0,0 +1,142 @@
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions
+name: CI/CD
+
+on:
+  push:
+    branches:
+      - master
+
+# https://docs.github.com/en/actions/learn-github-actions/contexts
+env:
+  CI_REGISTRY_IMAGE: "${{ vars.DOCKER_REGISTRY }}/${{ vars.DOCKER_REPOSITORY }}/${{ github.event.repository.name }}"
+  CI_COMMIT_SHA: "${{ github.sha }}"
+  CI_COMMIT_BRANCH: "${{ github.base_ref || github.head_ref || github.ref_name}}"
+
+jobs:
+  cicd:
+    name: CI/CD
+    runs-on: self-hosted
+    defaults:
+      run:
+        shell: bash
+    steps:
+
+    # Setup workflow
+    - name: Set environment for branch
+      run: |
+        if [[ $GITHUB_REF_NAME == 'master' ]]; then
+            echo "CI_PIPELINE=dev" >> "$GITHUB_ENV"
+        fi
+        echo "JOB_CONTAINER_NAME=$HOSTNAME" >> "$GITHUB_ENV"
+
+    # Copy codebase into runner
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    # Build base image
+    - name: Build base image
+      run: |
+        # Compute $VERSION
+        VERSION=$(sha1sum docker.base.Dockerfile requirements.txt | sha1sum | head -c 40)
+        DESTINATION="$CI_REGISTRY_IMAGE"
+        DESTINATION_VERSION="$DESTINATION:$VERSION"
+
+        echo "CI_IMAGE_VERSION=$VERSION" >> "$GITHUB_ENV"
+        echo "CI_IMAGE_PATH=$DESTINATION_VERSION" >> "$GITHUB_ENV"
+
+        # Check if the image already exists in the registry
+        echo '${{ secrets.DOCKER_CREDENTIALS }}' | docker login -u _json_key --password-stdin https://${{ vars.DOCKER_REGISTRY }}
+
+        TO_CREATE=true
+        docker manifest inspect "$DESTINATION_VERSION" > /dev/null && TO_CREATE=false
+
+        # It doesn't: build and push it
+        if $TO_CREATE; then
+          docker build \
+          --file docker.base.Dockerfile \
+          --build-arg CI_COMMIT_SHA="$CI_COMMIT_SHA" \
+          --build-arg CI_COMMIT_BRANCH="$CI_COMMIT_BRANCH" \
+          -t "$DESTINATION_VERSION" \
+          -t "$DESTINATION:latest" . && \
+          docker push "$DESTINATION_VERSION" && \
+          docker push "$DESTINATION:latest" && \
+          echo "Image $DESTINATION_VERSION build"
+        else
+          echo "Image $DESTINATION_VERSION already exists"
+        fi
+
+    # Lint
+    - name: Pre-commit
+      uses: addnab/docker-run-action@v3
+      with:
+        image: ${{ env.CI_IMAGE_PATH }}
+        options: |
+          --volumes-from=${{ env.JOB_CONTAINER_NAME }}
+          --workdir=${{ github.workspace }}
+        run: |
+          apt-get update -qq && apt-get install -qq -y git
+          pip install --quiet --upgrade pre-commit
+          git config --global --add safe.directory `pwd`
+
+          echo "Running tests for pipeline $CI_PIPELINE"
+          pre-commit run -v --all-files --show-diff-on-failure
+
+    # Test
+    - name: Unit tests
+      uses: addnab/docker-run-action@v3
+      with:
+        image: ${{ env.CI_IMAGE_PATH }}
+        options: |
+          --volumes-from=${{ env.JOB_CONTAINER_NAME }}
+          --workdir=${{ github.workspace }}
+          -e CI_PIPELINE=1
+        run: |
+          cp -R www /_run
+          cd /_run
+          python manage.py test
+
+    # Build app (on top of base)
+    - name: Build full image
+      run: |
+        BASE_VERSION="$CI_IMAGE_PATH"
+        DESTINATION="$CI_REGISTRY_IMAGE/$CI_PIPELINE"
+        DESTINATION_VERSION="$DESTINATION:$CI_COMMIT_SHA"
+
+        echo "Building $DESTINATION_VERSION"
+        echo "CI_IMAGE_FULL_VERSION=$DESTINATION_VERSION" >> "$GITHUB_ENV"
+
+        # Build image
+        sed -i "s%./docker.base.Dockerfile%$BASE_VERSION%" docker.full.Dockerfile
+
+        docker build \
+        --file docker.full.Dockerfile \
+        --build-arg CI_COMMIT_SHA=$CI_COMMIT_SHA \
+        --build-arg CI_COMMIT_BRANCH=$CI_COMMIT_BRANCH \
+        --label "cicd.pipeline=$CI_PIPELINE" \
+        --label "cicd.image=$CI_IMAGE_VERSION" \
+        --label "cicd.ci_commit_sha=$CI_COMMIT_SHA" \
+        -t "$DESTINATION_VERSION" \
+        -t "$DESTINATION:latest" . && \
+        docker push $DESTINATION_VERSION && \
+        docker push $DESTINATION:latest
+
+    # Deploy the new image
+    - name: Deploy
+      uses: addnab/docker-run-action@v3
+      with:
+        image: bitnami/kubectl:latest
+        entrypoint: ""
+        run: |
+          # Set credentials
+          KUBE_CA=/tmp/kube_ca; echo '${{ secrets.KUBE_CA_DATA }}' > "$KUBE_CA"
+
+          kubectl config set-cluster gke --server='${{ vars.KUBE_CLUSTER }}' --embed-certs --certificate-authority "$KUBE_CA"
+          kubectl config set-credentials pipeline --token='${{ secrets.KUBE_TOKEN }}'
+          kubectl config set-context primary --user=pipeline --cluster=gke
+          kubectl config use-context primary
+
+          # Update image
+          kubectl set image deploy/webapp api="${{ env.CI_IMAGE_FULL_VERSION }}"
+
+          message="version change to ${{ env.CI_PIPELINE }}:${{ env.CI_COMMIT_SHA }}"
+          kubectl annotate deploy webapp kubernetes.io/change-cause="$message" --overwrite=true

.gitignore

@@ -38,7 +38,6 @@ Icon?
 *.bak
 *.bak/*
 kubeconfig
-.github
 runner-*
 
 # GitKeep

.gitlab-ci.yml

@@ -100,6 +100,7 @@ test:
     - name: postgres:15
       alias: postgres-test
   variables:
+    CI_PIPELINE: 1
     # Services.postgres variables
     POSTGRES_USER: test
     POSTGRES_PASSWORD: test
@@ -128,7 +129,7 @@ build:
     FROM_BASE: "$CI_REGISTRY_IMAGE:$CI_IMAGE_BASE_VERSION"
     DESTINATION: "$DOCKER_REGISTRY/$DOCKER_REPOSITORY/$CI_PIPELINE/django"
   script:
-    - echo "Building $TAG_VERSION"
+    - echo "Building $DESTINATION"
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
 
     # Build image

DEPLOY_0_PROVISION.md

@@ -141,6 +141,7 @@
 
   # Copy and paste the output from the previous command
   $ DOCKER_REGISTRY='us-west1-docker.pkg.dev';
+  $ DOCKER_REPOSITORY='test-gke-419405/test-gke-repo'
   $ DOCKER_CREDENTIALS='{  "type": "service_account", ...';
 
   $ echo "$DOCKER_CREDENTIALS" | docker login -u _json_key --password-stdin https://$DOCKER_REGISTRY
@@ -154,10 +155,6 @@
 * Tag & push your image
 
   ``` bash
-  $ DOCKER_REPOSITORY=$(terraform output -raw docker_registry_repository)
-  $ echo $DOCKER_REPOSITORY
-  test-gke-419405/test-gke-repo-2
-
   $ IMAGE_NAME=django
   ```
   ``` bash
@@ -250,6 +247,7 @@
   gateway   gke-l7-global-external-managed   34.36.76.231   True         80s
 
   # Give it 2-3 min for the Load Balancer to get up and running
+  # To see what's going on: kubectl events
   $ curl 34.36.76.23
   curl: (52) Empty reply from server
 

DEPLOY_1_GITHUB.md

@@ -0,0 +1,132 @@
+Configuration file: .github/workflows/cicd.yml
+
+## Setup Github
+
+- Create secrets  
+  Settings > Security: Secrets and variables > Actions > 
+
+  - Secrets: New repository secret
+
+    ```
+    DOCKER_CREDENTIALS={  "type": "service_account...
+
+    KUBE_CA_DATA=-----BEGIN ...
+    KUBE_TOKEN=ya29.c.c0A...
+    ```
+
+  - Variables: New repository variable
+
+    ```
+    DOCKER_REGISTRY=us-west1-docker.pkg.dev
+    DOCKER_REPOSITORY=test-gke-419405/test-gke-repo
+
+    KUBE_CLUSTER=https://35.203.177.212
+    ```
+
+---
+
+## Using a local Github runner
+
+* Settings > actions > runners > new self-hosted runner
+
+  ![](https://i.imgur.com/aG51mlB.png)
+
+* Create the Dockerfile  
+  Note we're setting up the container to use docker-in-docker with the host's Docker  
+  Alternative: [install docker in the image](https://github.com/myoung34/docker-github-actions-runner)
+
+  ``` bash
+  $ mkdir runner-github
+  $ cd runner-github
+
+  # Create docker-compose.yml with the appropriate version
+  # Copy the command in Github's "download" section
+  $ curl="curl -o actions-runner-linux-x64-2.315.0.tar.gz -L https://github.com/actions/runner/releases/download/v2.315.0/actions-runner-linux-x64-2.315.0.tar.gz"
+
+  $ version=$([[ "${curl##*-}" =~ (\.?[0-9])* ]] && echo $BASH_REMATCH)
+  $ echo $version
+  2.315.0
+
+  $ docker_group=$(getent group docker | cut -d: -f3)
+  $ echo $docker_group
+  998
+
+  cat <<EOT > docker-compose.yml
+  version: '3'
+
+  services:
+    runner:
+      build:
+        context: .
+        dockerfile: Dockerfile
+        args:
+          RUNNER_VERSION: ${version}
+          DOCKER_GROUP: ${docker_group}
+      restart: "no"
+      volumes:
+        - volume_runner:/home/runner/actions-runner
+        - /usr/bin/docker:/usr/bin/docker
+        - /var/run/docker.sock:/var/run/docker.sock
+      privileged: true
+
+  volumes:
+    volume_runner:
+  EOT
+
+  # Create Dockerfile as-is
+  cat <<'EOT' > Dockerfile
+  # Source: https://dev.to/pwd9000/create-a-docker-based-self-hosted-github-runner-linux-container-48dh
+  FROM ubuntu:20.04
+
+  #input GitHub runner version argument
+  ARG RUNNER_VERSION
+  ARG DOCKER_GROUP
+
+  LABEL RunnerVersion=${RUNNER_VERSION}
+  LABEL DockerGroup=${DOCKER_GROUP}
+
+  ENV DEBIAN_FRONTEND=noninteractive
+
+  # Install dependencies
+  RUN apt-get update -y \
+      && apt-get upgrade -y \
+      && apt-get install -y --no-install-recommends \
+      curl nodejs wget unzip vim git  build-essential libssl-dev libffi-dev python3 python3-venv python3-dev python3-pip
+
+  # Add non-sudo user
+  RUN groupadd docker -g ${DOCKER_GROUP} \
+      && useradd -m runner -G docker -s /bin/bash
+
+  WORKDIR /home/runner/actions-runner
+
+  # Download & install runner
+  RUN curl -O -L https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \
+      && tar xzf ./actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \
+      && rm ./actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz
+  RUN /home/runner/actions-runner/bin/installdependencies.sh
+
+  # Set up runtime
+  RUN chown -R runner: /home/runner
+  USER runner
+
+  CMD ["./run.sh"]
+  EOT
+  ```
+
+* Build and launch
+
+  ``` bash
+  # Build the image
+  docker-compose build
+
+  # Copy the command in Github's "configure" section
+  docker-compose run runner ./config.sh --url https://github.com/a-mt/gcp-gke-django --token AAIK6BREEMCZLYP23CIDL7LGD6H24
+
+  # Run
+  docker-compose up
+  ```
+
+  If your config.sh gives you "Http response code: NotFound from 'POST https://api.github.com/actions/runner-registration'": the token has expired, refresh the "new self-hosted runner" page to get a new token
+
+* Check that your runner is listed in the runners list and is "idle":  
+  Settings > runners

DEPLOY_1_GITLAB.md

@@ -1,7 +1,8 @@
+Configuration file: .gitlab-ci.yml
 
-## Setup Gitlb
+## Setup Gitlab
 
-- Activate the CI/CD menu  
+- Enable the CI/CD menu  
   Settings > General > Visibility, project features, permissions
 
   - Check CI/CD

README.md

@@ -81,4 +81,7 @@ After fixing the files, add them to the staged diffs and commit.
 ## Deployment
 
 - [Provision the infra](DEPLOY_0_PROVISION.md)
-- [Set up Gitlab CI/CD](DEPLOY_1_GITLAB.md)
+
+- Set up a CI/CD pipeline
+  - [On Gitlab](DEPLOY_1_GITLAB.md)
+  - [On Github](DEPLOY_1_GITHUB.md)

docker-compose.cicd.yml

@@ -0,0 +1,17 @@
+version: '3'
+
+services:
+  tests:
+    image: django-base
+    build:
+      context: .
+      dockerfile: docker.base.Dockerfile
+    restart: "no"
+    volumes:
+      - ./www:/srv/www:delegated
+    environment:
+      CI_PIPELINE: 1
+    command:
+      - ./manage.py
+      - test
+    platform: linux/amd64

infra/outputs.tf

@@ -12,6 +12,7 @@ output "postgres_env_vars" {
 output "cicd_docker_credentials" {
   value = <<-EOT
   DOCKER_REGISTRY='${module.docker_registry.docker_registry_hostname}';
+  DOCKER_REPOSITORY='${module.docker_registry.docker_registry_repository}';
   DOCKER_CREDENTIALS='${replace(base64decode(module.docker_registry.docker_registry_write_json_key), "\n", "")}';
   echo "$DOCKER_CREDENTIALS" | docker login -u _json_key --password-stdin https://$DOCKER_REGISTRY
   EOT

www/core/settings/__init__.py

@@ -2,7 +2,12 @@
 import os
 import sys
 
-if 'PRODUCTION' in os.environ:
+if 'CI_PIPELINE' in os.environ:
+    try:
+        from .cicd import *  # noqa
+    except ImportError:
+        raise Exception(f'The CI/CD settings could not be found in {os.path.dirname(__file__)}/cicd')
+elif 'PRODUCTION' in os.environ:
     try:
         from .prod import *  # noqa
     except ImportError:

www/core/settings/cicd.py

@@ -0,0 +1,3 @@
+# flake8: noqa
+import sys
+from .defaults import *