test.yml

# This is a GitHub workflow defining a set of jobs with a set of steps.
# ref: https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
#
name: Tests

on:
  pull_request:
    paths-ignore:
      - "**.md"
      - "**.rst"
      - "docs/**"
      - "examples/**"
      - ".github/workflows/**"
      - "!.github/workflows/test.yml"
  push:
    paths-ignore:
      - "**.md"
      - "**.rst"
      - "docs/**"
      - "examples/**"
      - ".github/workflows/**"
      - "!.github/workflows/test.yml"
    branches-ignore:
      - "dependabot/**"
      - "pre-commit-ci-update-config"
      - "update-*"
  workflow_dispatch:

jobs:
  lint:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v3

      - name: install requirements
        run: pip install ruamel.yaml

      - name: check embedded chart code
        run: ./ci/check_embedded_chart_code.py

  # Most of the "main", "auth" and "helm" jobs are the same and only differ
  # in small things. Unfortunately there is no easy way to share steps between
  # jobs or have "template" jobs, so we use `if` conditions on steps
  tests:
    runs-on: ubuntu-20.04
    timeout-minutes: 10
    strategy:
      # keep running so we can see if tests with other k3s/k8s/helm versions pass
      fail-fast: false
      matrix:
        k3s-channel:
          # Available channels: https://github.com/k3s-io/k3s/blob/HEAD/channel.yaml
          - latest
        test:
          - main
          - auth
          - helm
        include:
          # Chart.yaml contains the chart's oldest supported k8s version, we
          # test against that and the oldest known supported helm cli version
          # which isn't documented anywhere, but is currently at 3.5.0. We also
          # test with the latest k8s and helm versions.
          #
          - k3s-channel: v1.20
            helm-version: v3.5.0
            test: helm
          - k3s-channel: latest
            test: helm
            test-variation: upgrade
            # upgrade-from represents a release channel, see: https://jupyterhub.github.io/helm-chart/info.json
            upgrade-from: dev
            upgrade-from-extra-args: ""
    steps:
      - uses: actions/checkout@v3
        with:
          # chartpress requires the full history
          fetch-depth: 0

      - uses: jupyterhub/action-k3s-helm@v3
        with:
          k3s-channel: ${{ matrix.k3s-channel }}
          helm-version: ${{ matrix.helm-version }}
          metrics-enabled: false
          traefik-enabled: false
          docker-enabled: true

      - name: Setup OS level dependencies
        run: |
          sudo apt-get update
          sudo apt-get install --yes \
            build-essential \
            curl \
            libcurl4-openssl-dev \
            libssl-dev

      - uses: actions/setup-node@v3
        id: setup-node
        with:
          node-version: "16"

      - name: Cache npm
        uses: actions/cache@v3
        with:
          path: ~/.npm
          key: node-${{ steps.setup-node.outputs.node-version }}-${{ hashFiles('**/package.json') }}-${{ github.job }}

      - name: Run webpack to build static assets
        run: |
          npm install
          npm run webpack

      - uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: "3.9"

      - name: Cache pip
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: python-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/*requirements.txt') }}-${{ github.job }}

      - name: Update pip
        run: |
          pip install --upgrade pip
          pip install --upgrade setuptools wheel

      - name: Setup Python package dependencies
        run: |
          pip install -r dev-requirements.txt -r helm-chart/images/binderhub/requirements.txt
          pip install .

      - name: Install JupyterHub chart for main tests
        if: matrix.test == 'main'
        run: |
          ./testing/local-binder-k8s-hub/install-jupyterhub-chart

      - name: Install JupyterHub chart for auth tests
        if: matrix.test == 'auth'
        run: |
          ./testing/local-binder-k8s-hub/install-jupyterhub-chart --auth

      - name: Use chartpress to create the helm chart
        if: matrix.test == 'helm'
        run: |
          export DOCKER_BUILDKIT=1

          # Use chartpress to create the helm chart and build its images
          helm dependency update ./helm-chart/binderhub
          (cd helm-chart && chartpress)
          git --no-pager diff --color=always

      - name: Generate values.schema.json from schema.yaml
        if: matrix.test == 'helm'
        run: |
          tools/generate-json-schema.py

      - name: "Helm template --validate (with lint-and-validate-values.yaml)"
        if: matrix.test == 'helm'
        run: |
          helm template --validate binderhub-test helm-chart/binderhub \
              --values tools/templates/lint-and-validate-values.yaml

      - name: "Helm template --validate for dind (with lint-and-validate-values.yaml)"
        if: matrix.test == 'helm'
        run: |
          helm template --validate binderhub-test helm-chart/binderhub \
              --values tools/templates/lint-and-validate-values.yaml \
              --set imageBuilderType=dind

      - name: "Helm template --validate for pink (with lint-and-validate-values.yaml)"
        if: matrix.test == 'helm'
        run: |
          helm template --validate binderhub-test helm-chart/binderhub \
              --values tools/templates/lint-and-validate-values.yaml \
              --set imageBuilderType=pink

      - name: Validate the chart against the k8s API
        if: matrix.test == 'helm'
        run: |
          helm template --validate binderhub-test helm-chart/binderhub \
              --values testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml \
              --set config.BinderHub.hub_url=http://localhost:30902 \
              --set config.GitHubRepoProvider.access_token=$GITHUB_ACCESS_TOKEN

      - name: "(Upgrade) Install ${{ matrix.upgrade-from }} chart"
        if: matrix.test-variation == 'upgrade'
        run: |
          . ./ci/common
          UPGRADE_FROM_VERSION=$(curl -sS https://jupyterhub.github.io/helm-chart/info.json | jq -er '.binderhub.${{ matrix.upgrade-from }}')
          echo "UPGRADE_FROM_VERSION=$UPGRADE_FROM_VERSION" >> $GITHUB_ENV

          echo ""
          echo "Installing already released binderhub version $UPGRADE_FROM_VERSION"

          # FIXME: We change the directory so binderhub the chart name won't be
          #        misunderstood as the local folder name.
          #
          #        https://github.com/helm/helm/issues/9244
          cd ci

          old_config="../testing/k8s-binder-k8s-hub/binderhub-chart-config-old.yaml"
          if [ -f "$old_config" ]; then
            echo "using old config"
          else
            old_config="../testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml"
          fi

          helm install binderhub-test \
              --repo https://jupyterhub.github.io/helm-chart/ binderhub \
              --disable-openapi-validation \
              --version=$UPGRADE_FROM_VERSION \
              --values "$old_config" \
              --set config.BinderHub.hub_url=http://localhost:30902 \
              --set config.BinderHub.hub_url_local=http://proxy-public \
              --set config.GitHubRepoProvider.access_token=$GITHUB_ACCESS_TOKEN \
              ${{ matrix.upgrade-from-extra-args }}

      - name: "(Upgrade) Install helm diff"
        if: matrix.test-variation == 'upgrade'
        run: |
          helm plugin install https://github.com/databus23/helm-diff

      - name: "(Upgrade) Helm diff ${{ matrix.upgrade-from }} chart with local chart"
        if: matrix.test-variation == 'upgrade'
        run: |
          helm diff upgrade binderhub-test ./helm-chart/binderhub \
              --values testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml \
              --set config.BinderHub.hub_url=http://localhost:30902 \
              --set config.BinderHub.hub_url_local=http://proxy-public \
              --set config.GitHubRepoProvider.access_token=$GITHUB_ACCESS_TOKEN \
              ${{ matrix.local-chart-extra-args }}

      - name: "(Upgrade) Await ${{ matrix.upgrade-from }} chart"
        if: matrix.test-variation == 'upgrade'
        uses: jupyterhub/action-k8s-await-workloads@v1
        with:
          timeout: 150
          max-restarts: 1

      - name: Install the chart
        if: matrix.test == 'helm'
        run: |
          helm upgrade --install binderhub-test helm-chart/binderhub \
              --values testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml \
              --set config.BinderHub.hub_url=http://localhost:30902 \
              --set config.BinderHub.hub_url_local=http://proxy-public \
              --set config.GitHubRepoProvider.access_token=$GITHUB_ACCESS_TOKEN \
              ${{ matrix.local-chart-extra-args }}

      - name: Await and curl JupyterHub
        run: |
          . ci/common
          await_jupyterhub

          echo curl http://localhost:30902/hub/api/ should print the JupyterHub version
          curl http://localhost:30902/hub/api/ --max-time 5 --retry 5 --retry-delay 1 --retry-connrefused

      - name: Await and curl BinderHub
        if: matrix.test == 'helm'
        run: |
          . ci/common
          await_binderhub binderhub-test

          echo curl http://localhost:30901/health to check BinderHub\'s health
          curl http://localhost:30901/health --max-time 5 --retry 5 --retry-delay 1 --retry-connrefused

      - name: Run main tests
        if: matrix.test == 'main'
        # running the "main" tests means "all tests that aren't auth"
        run: pytest -m "not auth" -v --maxfail=10 --cov binderhub --durations=10 --color=yes

      - name: Run auth tests
        if: matrix.test == 'auth'
        # running the "auth" tests means "all tests that are marked as auth"
        run: pytest -m "auth" -v --maxfail=10 --cov binderhub --durations=10 --color=yes

      - name: Run helm tests
        if: matrix.test == 'helm'
        run: |
          export BINDER_URL=http://localhost:30901
          pytest --helm -m "remote" -v --maxfail=10 --cov binderhub --durations=10 --color=yes

      # GitHub Action reference: https://github.com/jupyterhub/action-k8s-namespace-report
      - name: Kubernetes namespace report
        uses: jupyterhub/action-k8s-namespace-report@v1
        if: always()
        with:
          important-workloads: deploy/binder deploy/hub deploy/proxy

      # GitHub action reference: https://github.com/codecov/codecov-action
      - name: Upload coverage stats
        uses: codecov/codecov-action@v3
        if: ${{ always() }}

  test-local:
    runs-on: ubuntu-20.04
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@v3

      - name: Setup OS level dependencies
        run: |
          sudo apt-get update
          sudo apt-get install --yes \
            build-essential \
            curl \
            libcurl4-openssl-dev \
            libssl-dev

      - uses: actions/setup-node@v3
        id: setup-node
        with:
          node-version: "16"

      - name: Cache npm
        uses: actions/cache@v3
        with:
          path: ~/.npm
          key: node-${{ steps.setup-node.outputs.node-version }}-${{ hashFiles('**/package.json') }}-${{ github.job }}

      - uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: "3.10"

      - name: Cache pip
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: python-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/*requirements.txt') }}-${{ github.job }}

      - name: Update pip
        run: |
          pip install --upgrade pip
          pip install --upgrade setuptools wheel

      - name: Setup Python package dependencies
        run: |
          pip install -r dev-requirements.txt -r testing/local-binder-local-hub/requirements.txt
          pip install .

      - name: Setup JupyterHub NPM dependencies
        run: npm install -g configurable-http-proxy

      - name: Await and curl JupyterHub
        run: |
          cd testing/local-binder-local-hub
          jupyterhub --config=jupyterhub_config.py > jupyterhub.log 2>&1 &
          sleep 5

          echo curl http://localhost:8000/hub/api/ should print the JupyterHub version
          curl http://localhost:8000/hub/api/ --max-time 5 --retry 5 --retry-delay 1 --retry-connrefused

      - name: Run remote tests
        run: |
          export BINDER_URL=http://localhost:8000/services/binder/
          pytest -m remote -v --color=yes

      - name: Show hub logs
        if: always()
        run: cat testing/local-binder-local-hub/jupyterhub.log