.gitlab-ci.yml

variables:
  CI: "true"
  DOCKER_DRIVER: overlay2
  # Make sure sbt cache are captured completely
  COURSIER_CACHE: $CI_PROJECT_DIR/sbt-cache/coursier
  SBT_OPTS: >
    -Dsbt.global.base=$CI_PROJECT_DIR/sbt-cache/sbtboot
    -Dsbt.boot.directory=$CI_PROJECT_DIR/sbt-cache/boot
    -Dsbt.ivy.home=$CI_PROJECT_DIR/sbt-cache/ivy
  # If you changed dependencies (especially for scala), you might want to prevent your branch from using old cache by increase the version blew
  # gitlab actually use the same technique when you click the `Clear Cache Button`
  CACHE_VERSION: ts-6-scala-5

stages:
  - builders
  - sbt-prebuild
  - prebuild
  - buildtest
  - dockerize
  - preview
  - deploy-dev
  - release
  - deploy-staging
  - deploy-prod

builders-and-yarn:
  stage: builders
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:master
  retry: 1
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - .yarn
  before_script:
    - yarn install --no-progress --child-concurrency 1 --cache-folder .yarn
  services:
    - docker:dind
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.builder=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master
  artifacts:
    paths:
      - "node_modules"
      - "*/node_modules"
      - "yarn.lock"
    expire_in: 7 days

# Make sure sbt depenencies, plugins are in place, cached (only for this job) and pass to following stage as artifacts
sbt-prebuild:
  stage: sbt-prebuild
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - "$CI_PROJECT_DIR/sbt-cache"
  script:
    - "ls sbt-cache || cp -R /sbt-cache sbt-cache"
    - sbt update
  artifacts:
    paths:
      - "sbt-cache"
      - "target"
      - "project/target"
      - "project/project/target"
      - "*/target"
      - "*/project/target"
      - "*/project/project/target"
    expire_in: 7 days

check-scala-formatting:
  stage: prebuild
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG
  script:
    - sbt scalafmtCheckAll

registry-typescript-api:
  stage: prebuild
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG
  script:
    - lerna run generate --scope=@magda/typescript-common --stream
  artifacts:
    paths:
      - "magda-typescript-common/src/generated"
    expire_in: 7 days

buildtest:search-with-index-cache:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG
  retry: 2
  dependencies:
    - builders-and-yarn
    - sbt-prebuild
  before_script:
    - |
      if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
        export DOCKER_HOST='tcp://localhost:2375'
      fi
    - docker info
    - apk update
    - apk upgrade
    - apk add python3 python3-dev py3-pip build-base openssl-dev libffi-dev
    - pip3 install --upgrade pip
    - pip3 install docker-compose
    - docker-compose --version
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - "$CI_PROJECT_DIR/pip-cache"
  services:
    - docker:dind
  variables:
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
    # 2.5gb heap
    # unload classes we don't need
    # aim for < 1000ms gc pauses (largely ignored)
    # use the next-gen collector that every blog on the internet says not to use because we know better than them
    # use < 25% of time for GC
    # put the ivy cache in the workspace where gitlab can cache it
    SBT_OPTS: >
      -Xms2500M -Xmx2500M
      -XX:+CMSClassUnloadingEnabled
      -XX:MaxGCPauseMillis=1000
      -XX:+UseG1GC
      -XX:GCTimeRatio=3
      -Dsbt.global.base=$CI_PROJECT_DIR/sbt-cache/sbtboot
      -Dsbt.boot.directory=$CI_PROJECT_DIR/sbt-cache/boot
      -Dsbt.ivy.home=$CI_PROJECT_DIR/sbt-cache/ivy
  tags:
    - heavy
  script:
    - cd magda-elastic-search
    - docker-compose up -d
    - cd ..
    - sbt -DelasticSearch.serverUrl=elasticsearch://docker:9200 "intTest/testOnly au.csiro.data61.magda.api.*Spec"
    - cd magda-elastic-search
    - docker-compose down
  artifacts:
    paths:
      - "target"
      - "project/target"
      - "project/project/target"
      - "*/target"
      - "*/project/target"
      - "*/project/project/target"
    expire_in: 7 days

buildtest:search-no-index-cache:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG
  retry: 2
  dependencies:
    - builders-and-yarn
    - sbt-prebuild
  before_script:
    - |
      if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
        export DOCKER_HOST='tcp://localhost:2375'
      fi
    - docker info
    - apk update
    - apk upgrade
    - apk add python3 python3-dev py3-pip build-base openssl-dev libffi-dev
    - pip3 install --upgrade pip
    - pip3 install docker-compose
    - docker-compose --version
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - "$CI_PROJECT_DIR/pip-cache"
  services:
    - postgres:9.6
    - docker:dind
  variables:
    POSTGRES_URL: "jdbc:postgresql://postgres/postgres"
    POSTGRES_DB: postgres
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: ""
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
    # 2.5gb heap
    # unload classes we don't need
    # aim for < 1000ms gc pauses (largely ignored)
    # use the next-gen collector that every blog on the internet says not to use because we know better than them
    # use < 25% of time for GC
    # put the ivy cache in the workspace where gitlab can cache it
    SBT_OPTS: >
      -Xms2500M -Xmx2500M
      -XX:+CMSClassUnloadingEnabled
      -XX:MaxGCPauseMillis=1000
      -XX:+UseG1GC
      -XX:GCTimeRatio=3
      -Dsbt.global.base=$CI_PROJECT_DIR/sbt-cache/sbtboot
      -Dsbt.boot.directory=$CI_PROJECT_DIR/sbt-cache/boot
      -Dsbt.ivy.home=$CI_PROJECT_DIR/sbt-cache/ivy
  tags:
    - heavy
  script:
    - cd magda-elastic-search
    - docker-compose up -d
    - cd ..
    - sbt -DelasticSearch.serverUrl=elasticsearch://docker:9200 "intTest/testOnly au.csiro.data61.magda.crawler.*Spec au.csiro.data61.magda.indexer.*Spec au.csiro.data61.magda.spatial.*Spec"
    - cd magda-elastic-search
    - docker-compose down
  artifacts:
    paths:
      - "target"
      - "project/target"
      - "project/project/target"
      - "*/target"
      - "*/project/target"
      - "*/project/project/target"
    expire_in: 7 days

buildtest:ui:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-nodejs:$CI_COMMIT_REF_SLUG
  retry: 1
  dependencies:
    - builders-and-yarn
  cache:
    paths: []
  variables:
    REACT_APP_SHA1: "$CI_COMMIT_SHA"
  script:
    - yarn install
    - yarn run eslint
    - yarn run in-submodules -- -f categories.ui=true -- run build --include-filtered-dependencies
    - yarn run in-submodules -- -f categories.ui=true -- run test
  artifacts:
    paths:
      - "*/build"
      - "*/dist"
    expire_in: 7 days

buildtest:registry:
  stage: buildtest
  retry: 1
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG
  dependencies:
    - builders-and-yarn
    - sbt-prebuild
  services:
    - postgres:9.6
  variables:
    POSTGRES_URL: "jdbc:postgresql://postgres/postgres"
    POSTGRES_DB: postgres
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: ""
  script:
    # The tests in opa/RecordOpaSpec are not fully automated yet therefore skipped.
    # Please see opa/README.md for details.
    - sbt "registryApi/testOnly au.csiro.data61.magda.registry.*"
  artifacts:
    paths:
      - "target"
      - "project/target"
      - "project/project/target"
      - "*/target"
      - "*/project/target"
      - "*/project/project/target"
    expire_in: 7 days

buildtest:map:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-nodejs:$CI_COMMIT_REF_SLUG
  retry: 1
  dependencies:
    - builders-and-yarn
  cache:
    paths: []
  script:
    - yarn install
    - yarn run in-submodules -- -f categories.map=true -- run build --include-filtered-dependencies
    - yarn run in-submodules -- -f categories.map=true -- run test
  artifacts:
    paths:
      - "*/build"
      - "*/dist"
      - "magda-preview-map/wwwroot/build"
    expire_in: 7 days

buildtest:typescript-apis:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 1
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
  before_script:
    - |
      if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
        export DOCKER_HOST='tcp://localhost:2375'
      fi
    - docker info
    - apk update
    - apk upgrade
    - apk add python3 python3-dev py3-pip build-base openssl-dev libffi-dev
    - pip3 install --upgrade pip
    - pip3 install docker-compose
    - docker-compose --version
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - "$CI_PROJECT_DIR/pip-cache"
  services:
    - postgres:9.6
    - docker:dind
  variables:
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
    POSTGRES_HOST: postgres
    POSTGRES_DB: postgres
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: ""
    OPA_URL: "http://docker:8181/"
  script:
    - cd deploy/helm/magda/charts/opa
    - docker-compose up -d
    - cd ../../../../..
    - yarn run in-submodules -- -f categories.api=true -f categories.testStandAlone=false -f language=typescript -- run build --include-filtered-dependencies
    - yarn run in-submodules -- -f categories.api=true -f categories.testStandAlone=false -f language=typescript -- run test --include-filtered-dependencies
    - cd magda-typescript-common && yarn test && cd ..
    - cd deploy/helm/magda/charts/opa
    - docker-compose down
  artifacts:
    paths:
      - "*/dist"
    expire_in: 7 days

buildtest:storage-api:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 1
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
  before_script:
    - |
      if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
        export DOCKER_HOST='tcp://localhost:2375'
      fi
    - docker info
    - apk update
    - apk upgrade
    - apk add python3 python3-dev py3-pip build-base openssl-dev libffi-dev
    - pip3 install --upgrade pip
    - pip3 install docker-compose
    - docker-compose --version
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - "$CI_PROJECT_DIR/pip-cache"
  services:
    - docker:dind
  variables:
    PIP_CACHE_DIR: "$CI_PROJECT_DIR/pip-cache"
    MINIO_HOST: "docker"
    MINIO_PORT: "9000"
  script:
    - cd magda-storage-api
    - docker-compose up -d
    - yarn run build
    - yarn run test
    - docker-compose down
  artifacts:
    paths:
      - "*/dist"
    expire_in: 7 days

buildtest:minions:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-nodejs:$CI_COMMIT_REF_SLUG
  retry: 1
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
  cache:
    paths: []
  script:
    - yarn run in-submodules -- -f categories.minion=true -- run build --include-filtered-dependencies
    - yarn run in-submodules -- -f categories.minion=true -- run test
  artifacts:
    paths:
      - "*/dist"
    expire_in: 7 days

buildtest:connectors:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-nodejs:$CI_COMMIT_REF_SLUG
  retry: 1
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
  cache:
    paths: []
  script:
    - yarn run in-submodules -- -f categories.connector=true -- run build --include-filtered-dependencies
    - yarn run in-submodules -- -f categories.connector=true -- run test
  artifacts:
    paths:
      - "*/dist"
    expire_in: 7 days

buildtest:opa-policies:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 1
  dependencies:
    - builders-and-yarn
  cache:
    paths: []
  services:
    - docker:dind
  script:
    - docker run -v $PWD/deploy/helm/magda/charts/opa/policies:/policies openpolicyagent/opa:0.10.7 test -v ./policies

dockerize:scala:
  stage: dockerize
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG
  retry: 2
  dependencies:
    - registry-typescript-api
    - sbt-prebuild
    - buildtest:registry
    - buildtest:search-no-index-cache
    - buildtest:search-with-index-cache
  services:
    - docker:dind
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - cat /dev/null | sbt -Drepository=$CI_REGISTRY/magda-data/magda -Dversion=$CI_COMMIT_REF_SLUG dockerBuildAndPush

dockerize:ui:
  stage: dockerize
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  cache:
    paths: []
  services:
    - docker:dind
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
    - buildtest:ui
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.ui=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master

dockerize:map:
  stage: dockerize
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  cache:
    paths: []
  services:
    - docker:dind
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
    - buildtest:map
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.map=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master

dockerize:typescript-apis:
  stage: dockerize
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  cache:
    paths: []
  services:
    - docker:dind
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
    - buildtest:typescript-apis
    - buildtest:storage-api
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.api=true -f language=typescript -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master

dockerize:minions:
  stage: dockerize
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  cache:
    paths: []
  services:
    - docker:dind
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
    - buildtest:minions
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.minion=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master

dockerize:connectors:
  stage: dockerize
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  cache:
    paths: []
  services:
    - docker:dind
  dependencies:
    - builders-and-yarn
    - registry-typescript-api
    - buildtest:connectors
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.connector=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master

dockerize:migrators:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  cache:
    paths: []
  services:
    - docker:dind
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.migrator=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master

dockerize:dockerExtensions:
  stage: buildtest
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 2
  cache:
    paths: []
  services:
    - docker:dind
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - yarn run in-submodules -- -f categories.dockerExtension=true -- run docker-build-prod --include-filtered-dependencies -- -- --repository=$CI_REGISTRY/magda-data/magda --version=$CI_COMMIT_REF_SLUG --cacheFromVersion=master



inttest:registryAuth:
  # https://github.com/kind-ci/examples/blob/master/.gitlab-ci.yml
  stage: preview
  retry: 1
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-scala:$CI_COMMIT_REF_SLUG
  allow_failure: true
  dependencies:
    - sbt-prebuild
    - builders-and-yarn
    - buildtest:registry
  services:
    - docker:dind
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - /usr/local/bin/kind
      - /usr/local/bin/kubectl
      - /usr/local/bin/helm
  variables:
    KUBECTL: v1.15.0
    KIND: v0.7.0
  script:
    - apk add --update postgresql-client sed curl
    - curl -Lo /usr/local/bin/kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND}/kind-$(uname)-amd64 -C -
    - chmod +x /usr/local/bin/kind
    - curl -Lo /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL}/bin/linux/amd64/kubectl -C -
    - chmod +x /usr/local/bin/kubectl
    - curl -L https://storage.googleapis.com/kubernetes-helm/helm-v2.16.1-linux-amd64.tar.gz -C - | tar -xzO linux-amd64/helm > /usr/local/bin/helm
    - chmod +x /usr/local/bin/helm
    - kind create cluster --config=./kind-config.yaml --wait 20m --image kindest/node:v1.15.7@sha256:e2df133f80ef633c53c0200114fce2ed5e1f6947477dbc83261a6a921169488d
    # Set up KUBECONFIG that links to kind - we use sed to edit it so it's pointing to gitlab's docker host
    - kind get kubeconfig | sed -E -e 's/localhost|0\.0\.0\.0/docker/g' > ./kubeconfig
    - export KUBECONFIG="$(pwd)/kubeconfig"
    - kubectl apply -f deploy/kubernetes/rbac-config.yaml
    - helm init --service-account tiller --wait
    - export JWT_SECRET=udIsbcYaKs1G4n6AdiMSIvPx5KpxQAy8FA2aIcD46iCipNAZvds4jeXFLZKhVvSJZvhYb5Pvgvmtonk7UFfhGnYcd3DXM7KzHG7gBmGO8PCsOZ4t7icqZoJbpdDqYWMmd9XnrVXtJhR6HVFBmEmbk9AmFJ1Gz9ipYPGYLoFcavPs9iZ63KPXgdt4aBdWQcmICkGPYiY8CQOvqOoiU7hUhKDTkJgRRTSaax6UQDOveTQvQnd5uyXuV4os0tlahzRX
    - kubectl create ns test2
    - 'echo "{ \"apiVersion\": \"v1\", \"kind\": \"Secret\", \"metadata\": {\"name\": \"auth-secrets\"}, \"type\": \"Opaque\", \"data\": {\"jwt-secret\": \"dWRJc2JjWWFLczFHNG42QWRpTVNJdlB4NUtweFFBeThGQTJhSWNENDZpQ2lwTkFadmRzNGplWEZMWktoVnZTSlp2aFliNVB2Z3ZtdG9uazdVRmZoR25ZY2QzRFhNN0t6SEc3Z0JtR084UENzT1o0dDdpY3Fab0picGREcVlXTW1kOVhuclZYdEpoUjZIVkZCbUVtYms5QW1GSjFHejlpcFlQR1lMb0ZjYXZQczlpWjYzS1BYZ2R0NGFCZFdRY21JQ2tHUFlpWThDUU92cU9vaVU3aFVoS0RUa0pnUlJUU2FheDZVUURPdmVUUXZRbmQ1dXlYdVY0b3MwdGxhaHpSWA==\"}}" | kubectl apply --namespace test2 -f -'
    - 'echo "{ \"apiVersion\": \"v1\", \"kind\": \"Secret\", \"metadata\": {\"name\": \"regcred\"}, \"type\": \"kubernetes.io/dockerconfigjson\", \"data\": { \".dockerconfigjson\": \"$DOCKERCONFIGJSON\" }}" | kubectl apply --namespace test2 -f -'
    - helm upgrade test2 deploy/helm/magda --namespace test2 --install -f deploy/helm/local-auth-test.yml --set global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=$CI_COMMIT_REF_SLUG
    # Forward local ports to kind
    - kubectl port-forward combined-db-0 5432 --namespace test2 >/dev/null 2>&1 < /dev/null &
    - kubectl port-forward deployment/authorization-api 6104:80 --namespace test2 >/dev/null 2>&1 < /dev/null &
    # Wait for port forwarding to start
    - timeout -t 22 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 5432
    - timeout -t 22 sh -c 'until nc -z $0 $1; do sleep 1; done' localhost 6104
    # Set up some values
    - psql -h localhost -p 5432 -U postgres -d auth -f magda-registry-api/src/test/resources/data/organizations.sql
    - psql -h localhost -p 5432 -U postgres -d auth -f magda-registry-api/src/test/resources/data/users.sql
    # Test!
    - sbt "registryApi/testOnly au.csiro.data61.magda.opa.*"
    
(Full) Run As Preview: &runAsPreview
  stage: preview
  when: manual
  only:
    - branches
  except:
    - master
    - /^v.*RC.*$/
  dependencies: []
  cache:
    paths: []
  image:
    name: dtzar/helm-kubectl:2.16.1
  retry: 1
  environment:
    name: preview/$CI_COMMIT_REF_NAME
    url: https://$CI_COMMIT_REF_SLUG.dev.magda.io
    on_stop: Stop Preview
  before_script:
    # Add PWGEN for generating passwords
    - apk add --update pwgen

    # Env vars
    - JWT_SECRET="${JWT_SECRET:-$(pwgen 16 1)}"
    - SESSION_SECRET="${SESSION_SECRET:-$(pwgen 16 1)}"
    - DB_PASSWORD="${DB_PASSWORD:-p4ssw0rd}"
    - MINIO_HOST="${MINIO_HOST:-localhost}"
    - MINIO_PORT="${MINIO_PORT:-9000}"
    - MINIO_ACCESS_KEY="${MINIO_ACCESS_KEY:-}"
    - MINIO_SECRET_KEY="${MINIO_SECRET_KEY:-}"

    # Kube Config
    - echo "$KUBECTL_CONFIG" > kubectlconfig.yaml
    - export KUBECONFIG=kubectlconfig.yaml

    # Create kube namespace
    - kubectl get namespace $CI_COMMIT_REF_SLUG || kubectl create namespace $CI_COMMIT_REF_SLUG

    # Create kube secrets
    - kubectl create secret generic auth-secrets --from-literal=jwt-secret=$JWT_SECRET --from-literal=session-secret=$SESSION_SECRET --namespace $CI_COMMIT_REF_SLUG --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
    - kubectl create secret docker-registry regcred --namespace $CI_COMMIT_REF_SLUG --docker-server=registry.gitlab.com --docker-username=gitlab-ci-token --docker-password=$CI_JOB_TOKEN --docker-email=alex.gilleran@data61.csiro.au --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
    - kubectl create secret generic db-passwords --from-literal=combined-db=$DB_PASSWORD --from-literal=authorization-db=$DB_PASSWORD --from-literal=content-db=$DB_PASSWORD --from-literal=session-db=$DB_PASSWORD  --from-literal=registry-db=$DB_PASSWORD --from-literal=combined-db-client=$DB_PASSWORD --from-literal=authorization-db-client=$DB_PASSWORD --from-literal=content-db-client=$DB_PASSWORD --from-literal=session-db-client=$DB_PASSWORD --from-literal=registry-db-client=$DB_PASSWORD --from-literal=tenant-db=$DB_PASSWORD --from-literal=tenant-db-client=$DB_PASSWORD --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
    - kubectl create secret generic oauth-secrets --from-literal=facebook-client-secret=$FACEBOOK_CLIENT_SECRET --from-literal=google-client-secret=$GOOGLE_CLIENT_SECRET --from-literal arcgis-client-secret=$ARCGIS_CLIENT_SECRET --from-literal vanguard-certificate="$VANGUARD_CERT" --namespace $CI_COMMIT_REF_SLUG --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
    - kubectl create secret generic smtp-secret --from-literal=username=$SMTP_USERNAME --from-literal=password=$SMTP_PASSWORD --namespace $CI_COMMIT_REF_SLUG --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
    - echo "$DB_SERVICE_ACCOUNT_PRIVATE_KEY" > db-service-account-private-key.json
    - kubectl create secret generic storage-account-credentials --from-file db-service-account-private-key.json --namespace $CI_COMMIT_REF_SLUG --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -
    - kubectl create secret generic storage-secrets --from-literal=accesskey=$MINIO_ACCESS_KEY --from-literal=secretkey=$MINIO_SECRET_KEY --namespace $CI_COMMIT_REF_SLUG --dry-run -o json | kubectl apply --namespace $CI_COMMIT_REF_SLUG -f -

    # Helm upgrade
    - helm init --upgrade
  script:
    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/magda --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview.yml --set global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=$CI_COMMIT_REF_SLUG,ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,global.externalUrl=https://$CI_COMMIT_REF_SLUG.dev.magda.io,global.namespace=$CI_COMMIT_REF_SLUG --timeout 3600 --wait
    - echo "Successfully deployed to https://${CI_COMMIT_REF_SLUG}.dev.magda.io"

(UI) Run As Preview:
  <<: *runAsPreview
  script:
    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/magda --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview.yml --set global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=$CI_COMMIT_REF_SLUG,ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,ingress.targetService=web,tags.all=false,tags.web-server=true,web-server.baseUrl=https://dev.magda.io,web-server.useLocalStyleSheet=true --timeout 3600 --wait
    - echo "Successfully deployed to https://${CI_COMMIT_REF_SLUG}.dev.magda.io"

(No Data) Run As Preview:
  <<: *runAsPreview
  script:
    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/magda --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview.yml --set global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=$CI_COMMIT_REF_SLUG,ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,combined-db.waleBackup.method=NONE,elasticsearch.useGcsSnapshots=false,global.externalUrl=https://$CI_COMMIT_REF_SLUG.dev.magda.io,global.namespace=$CI_COMMIT_REF_SLUG --timeout 3600 --wait
    - echo "Successfully deployed to https://${CI_COMMIT_REF_SLUG}.dev.magda.io"
(No Data) Run As Multi-tenant Preview:
  <<: *runAsPreview
  script:
    - helm upgrade $CI_COMMIT_REF_SLUG deploy/helm/magda --install --recreate-pods --namespace $CI_COMMIT_REF_SLUG -f deploy/helm/preview-multi-tenant.yml --set global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=$CI_COMMIT_REF_SLUG,ingress.hostname=$CI_COMMIT_REF_SLUG.dev.magda.io,combined-db.waleBackup.method=NONE,elasticsearch.useGcsSnapshots=false,global.externalUrl=https://$CI_COMMIT_REF_SLUG.dev.magda.io,global.magdaAdminPortalName=admin-$CI_COMMIT_REF_SLUG.dev.magda.io,global.namespace=$CI_COMMIT_REF_SLUG --timeout 3600 --wait
    - echo "Successfully deployed to https://${CI_COMMIT_REF_SLUG}.dev.magda.io"

Stop Preview: &stopPreview
  stage: preview
  when: manual
  only:
    - branches
  except:
    - master
    - /^v.*RC.*$/
  dependencies: []
  cache:
    paths: []
  image:
    name: dtzar/helm-kubectl:2.16.1
  retry: 1
  before_script: []
  environment:
    name: preview/$CI_COMMIT_REF_NAME
    action: stop
  script:
    - echo "$KUBECTL_CONFIG" > kubectlconfig.yaml
    - export KUBECONFIG=kubectlconfig.yaml
    - helm del --purge $CI_COMMIT_REF_SLUG
    - kubectl delete namespace $CI_COMMIT_REF_SLUG

Deploy Master To Dev:
  stage: deploy-dev
  only:
    - master
  cache: {}
  image:
    name: dtzar/helm-kubectl:2.16.1
  retry: 1
  before_script: []
  environment:
    name: dev
    url: https://dev.magda.io
  script:
    - echo "$KUBECTL_CONFIG" > kubectlconfig.yaml
    - export KUBECONFIG=kubectlconfig.yaml
    - kubectl create secret docker-registry regcred --namespace default --docker-server=registry.gitlab.com --docker-username=gitlab-ci-token --docker-password=$CI_JOB_TOKEN --docker-email=alex.gilleran@data61.csiro.au --dry-run -o json | kubectl apply --namespace default -f -
    - helm upgrade magda deploy/helm/magda --install --recreate-pods -f deploy/helm/magda-dev.yml --set global.image.repository=registry.gitlab.com/magda-data/magda/data61,global.image.tag=master --timeout 3600 --wait

Release Tags To Docker Hub:
  stage: release
  only:
    - /^v.*\..*\..*$/
  except:
    - branches
    - triggers
  dependencies:
    - builders-and-yarn
  cache: {}
  services:
    - docker:dind
  image: registry.gitlab.com/magda-data/magda/data61/magda-builder-docker:$CI_COMMIT_REF_SLUG
  retry: 1
  script:
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
    - docker login -u magdabot -p $DOCKER_HUB_PASSWORD
    - yarn install
    - yarn run retag-and-push -- -- --fromPrefix=registry.gitlab.com/magda-data/magda/ --fromVersion=$CI_COMMIT_REF_SLUG

Publish Helm Chart:
  stage: release
  only:
    - /^v.*\..*\..*$/
  except:
    - branches
    - triggers
  dependencies: []
  image:
    name: dtzar/helm-kubectl:2.16.1
  cache:
    key: $CI_JOB_NAME-$CACHE_VERSION
    paths:
      - deploy/helm/chart-repo
  retry: 1
  script:
    # Get AWS CLI
    - apk -v --update add python py-pip groff less mailcap
    - pip install --upgrade awscli==1.14.5 s3cmd==2.0.1 python-magic

    # Init helm
    - helm init --client-only

    # Run deploy
    - cd deploy/helm
    - mkdir -p chart-repo
    - aws s3 sync s3://magda-charts/ chart-repo
    - helm package magda -d chart-repo
    - helm repo index chart-repo
    - aws s3 sync chart-repo s3://magda-charts/ --acl public-read