build.yml

name: Build
on: [push, pull_request]
env:
  GCS_BUCKET: autobuilds.grr-response.com
  GCS_BUCKET_OPENAPI: autobuilds-grr-openapi
  GCS_LATEST_PATH: _latest_server_deb
  DOCKER_REPOSITORY: ghcr.io/google/grr
jobs:
  test-devenv:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v3
      - name: check_deps
        run: |
          sudo apt-get update && sudo apt-get install -y podman curl jq
          sudo usermod --add-subuids 500000-565535 --add-subgids 500000-565535 "${USER}"
          devenv/devenv.sh check_deps
      - name: start
        run: |
          devenv/devenv.sh start
          OK=false
          for attempt in $(seq 10); do
            curl -su admin:admin http://localhost:4280/api/clients |
              sed 1d |
              jq -r ".items[].value.client_id.value" |
              egrep -e "^C[.][0-9a-f]{16}\$" && { OK=true; break; }
            echo "attempt ${attempt}: devenv not ready"
            sleep 5
          done
          [[ $OK = true ]]
  test-ubuntu:
    runs-on: ubuntu-22.04
    env:
      CHROME_DEB: google-chrome-stable_current_amd64.deb
    steps:
      - uses: actions/checkout@v3
      - name: Install
        run: |
          free -hmw
          lscpu
          sudo apt install -y python3-dev python3-pip python3-venv python3-mysqldb
          if [[ -z "$(type google-chrome 2>/dev/null)" ]]; then
            wget "https://dl.google.com/linux/direct/${CHROME_DEB}" && sudo apt install -y "./${CHROME_DEB}";
          fi
          python3 -m venv --system-site-packages "${HOME}/INSTALL"
          travis/install.sh
      - name: Test
        run: |
          source "${HOME}/INSTALL/bin/activate"
          pip install pytest-xdist==2.2.1 pytest==6.2.5
          # We have 4 vCPUs available, but only use 3 here to avoid timeouts like
          # https://ci.appveyor.com/project/grr/grr-ia94e/builds/20483467/messages ,
          # which happen when tests stall.
          pytest --verbose -n 3 grr/ --ignore grr/server/grr_response_server/gui/selenium_tests/ --ignore grr/client/grr_response_client/client_actions/windows/
          # jsTree tests seem to fail on Chrome 71 headless due to https://github.com/GoogleChrome/puppeteer/issues/3463
          if [ $(google-chrome --version | grep -Eo " [0-9]{1,3}") != "71" ]; then (cd grr/server/grr_response_server/gui/static/ && npm run gulp test); fi
  build-openapi:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v3
      - name: Install
        run: |
          sudo apt install -y python3-dev python3-pip python3-venv python3-mysqldb
          python3 -m venv --system-site-packages "${HOME}/INSTALL"
          travis/install.sh
      - name: Build
        run: |
          source "${HOME}/INSTALL/bin/activate"
          mkdir -p _openapi_artifacts/openapi_description
          mkdir -p _openapi_artifacts/documentation
          travis/build_api_documentation.sh "_openapi_artifacts/openapi_description/openapi_description.json" "_openapi_artifacts/documentation/openapi_documentation.html"
          ls -la _openapi_artifacts/*
      - name: Upload OpenAPI to GitHub artifacts
        uses: actions/upload-artifact@v3
        with:
          name: openapi
          path: _openapi_artifacts/
          retention-days: 1

  build-ubuntu:
    runs-on: ubuntu-22.04
    env:
      GCS_TAG: ubuntu_64bit
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Set up
        run: |
          sudo apt install fakeroot debhelper libffi-dev libssl-dev
          pip install virtualenv
          virtualenv "${HOME}/INSTALL"
      - name: Build
        run: |
          travis/install_client_builder.sh
          travis/build_templates.sh
          ls -la gcs_upload_dir
      - name: Upload installers to GitHub artifacts
        uses: actions/upload-artifact@v3
        with:
          name: ubuntu-installers
          path: gcs_upload_dir/
          retention-days: 1

  build-osx:
    runs-on: macos-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Set up
        run: |
          pip install --upgrade setuptools virtualenv
          virtualenv "${HOME}/INSTALL"
      - name: Build installers
        run: |
          travis/install_client_builder.sh
          travis/build_templates.sh
          ls -la gcs_upload_dir
      - name: Upload installers to GitHub artifacts
        uses: actions/upload-artifact@v3
        with:
          name: osx-installers
          path: gcs_upload_dir/
          retention-days: 1

  build-centos:
    runs-on: ubuntu-22.04
    env:
      GCS_TAG: centos_64bit
      DOCKER_IMG: grrdocker/centos7-python39
      DOCKER_CONTAINER: centos_64bit_container
      DOCKER_USER: grrbot
    steps:
      - uses: actions/checkout@v3
      - name: Build installers
        run: |
          docker run -dit \
            --volume "${PWD}:/mnt/grr" \
            --workdir /mnt/grr \
            --env DOCKER_USER="${DOCKER_USER}" \
            --env TRAVIS_OS_NAME="linux" \
            --name "${DOCKER_CONTAINER}" \
            "${DOCKER_IMG}"
          # Using `bash -l` here and below to make sure devtools
          # (including the C++ 14 compatible compiler) are properly
          # registered in the environment variables.
          docker exec "${DOCKER_CONTAINER}" bash -l travis/set_up_test_user.sh
          docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l -c '/usr/local/bin/python3.9 -m venv "/home/${DOCKER_USER}/INSTALL"'
          docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l travis/install_client_builder.sh
          docker exec --user "${DOCKER_USER}" "${DOCKER_CONTAINER}" bash -l travis/build_templates.sh
          docker exec "${DOCKER_CONTAINER}" rpm -vih gcs_upload_dir/*.rpm
          ls -la gcs_upload_dir
      - name: Upload installers to GitHub artifacts
        uses: actions/upload-artifact@v3
        with:
          name: centos-installers
          path: gcs_upload_dir/
          retention-days: 1

  build-windows:
    runs-on: windows-2022
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Build installers
        shell: bash
        run: |
          set -ex
          pip install virtualenv wheel
          python -u appveyor/windows_templates/build_windows_templates.py --grr_src=$GITHUB_WORKSPACE --output_dir=$GITHUB_WORKSPACE/output --test_repack_install
          mkdir -p gcs_upload_dir
          mv -v output*/* gcs_upload_dir
          ls -la gcs_upload_dir
      - name: Upload installers to GitHub artifacts
        uses: actions/upload-artifact@v3
        with:
          name: windows-installers
          path: gcs_upload_dir/
          retention-days: 1

  build-server-deb:
    runs-on: ubuntu-22.04
    env:
      GCS_TAG: server_deb
    needs:
      - build-centos
      - build-ubuntu
      - build-osx
      - build-windows
    steps:
      - uses: actions/checkout@v3
      - name: Download installers from GitHub artifacts
        id: download
        uses: actions/download-artifact@v3
        with:
          path: ~/_artifacts
      - name: Set up
        run: |
          sudo apt-get update
          sudo apt-get install -y fakeroot debhelper libffi-dev libssl-dev python3-dev python3-pip python3-venv python3-mysqldb wget openjdk-8-jdk zip git devscripts libmysqlclient-dev dh-virtualenv dh-make libc6-i386 lib32z1
          python3 -m venv --system-site-packages "${HOME}/INSTALL"
          "${HOME}/INSTALL/bin/python3" -m pip install --upgrade pip 'setuptools<58.3.1' wheel
      - name: Build
        run: |
          travis/install.sh
          mkdir -p grr/config/grr_response_templates/templates
          mv -v ~/_artifacts/windows-installers/GRR_*_amd64.msi.zip grr/config/grr_response_templates/templates
          mv -v ~/_artifacts/ubuntu-installers/grr_*_amd64.deb.zip grr/config/grr_response_templates/templates
          mv -v ~/_artifacts/centos-installers/grr_*_amd64.rpm.zip grr/config/grr_response_templates/templates
          mv -v ~/_artifacts/osx-installers/grr_*_amd64.xar.zip grr/config/grr_response_templates/templates
          travis/build_local_pyindex.sh
          travis/build_server_deb.sh
          ls -la gcs_upload_dir
      - name: Upload installers to GitHub artifacts
        uses: actions/upload-artifact@v3
        with:
          name: server-deb
          path: gcs_upload_dir/
          retention-days: 1

  test-ubuntu-e2e:
    continue-on-error: true  # Debug follow up step.
    runs-on: ubuntu-22.04
    env:
      GRR_ADMIN_PASS: 'e2e_tests'
      APPVEYOR_MYSQL_PASS: 'root'
    needs:
      - build-server-deb
    steps:
      - uses: actions/checkout@v3
      - name: Set up MySQL
        run: |
          printf "\n[mysqld]\nmax_allowed_packet=42M\nlog_bin_trust_function_creators=1\n" | sudo tee -a /etc/mysql/my.cnf
          sudo /etc/init.d/mysql start
      - name: Download installers from GitHub artifacts
        id: download
        uses: actions/download-artifact@v3
        with:
          name: server-deb
          path: _artifacts
      - name: Install
        run: |
          free -hmw
          lscpu
          sudo -EH ./appveyor/e2e_tests/install_mem_usage_cron.sh
          sudo -EH ./appveyor/e2e_tests/install_latest_server_deb.sh
      - name: Test
        run: |
          sudo -EH ./appveyor/e2e_tests/run_e2e_tests.sh
          sudo -EH ./appveyor/e2e_tests/test_repack.sh
      - name: Upload logs and configs to GitHub artifacts
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: e2e-test
          path: /var/log/grr
          retention-days: 1

  build-push-docker:
    runs-on: ubuntu-22.04
    needs:
      - build-server-deb
      # - test-ubuntu-e2e  # TODO: Comment back in after debugging is finished.
      - test-ubuntu
      - build-openapi
    steps:
      - uses: actions/checkout@v3
      - name: Download installers from GitHub artifacts
        id: download
        uses: actions/download-artifact@v3
        with:
          name: server-deb
          path: _artifacts
      - name: Build Docker image
        run: |
          export BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
          ./appveyor/docker_build/build_docker_image.sh
      - if: ${{ github.event_name == 'push' }}
        name: Login to GitHub Container registry
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - if: ${{ github.event_name == 'push' }}
        name: Push to GitHub Container registry
        run: |
          docker push -a ${{ env.DOCKER_REPOSITORY }}
  upload:
    if: ${{ github.event_name == 'push' }}
    permissions:
      contents: 'read'
      id-token: 'write'
    runs-on: ubuntu-22.04
    needs:
      - build-push-docker
    steps:
      - uses: actions/checkout@v3
      - name: Download installers from GitHub artifacts
        id: download
        uses: actions/download-artifact@v3
        with:
          path: _artifacts
      - run: |
          ls -la _artifacts/*/
          COMMIT_TIME=$(git show -s --date='format-local:%Y-%m-%dT%H:%M:%SZ' --format="%cd" $GITHUB_SHA)
          OUTPUT_DIR=gcs_upload_dir/${COMMIT_TIME}_${GITHUB_SHA}/
          echo "OUTPUT_DIR=$OUTPUT_DIR" >> $GITHUB_ENV
          mkdir -p $OUTPUT_DIR/centos/
          mv -v _artifacts/centos-installers/* $OUTPUT_DIR/centos
          mkdir -p $OUTPUT_DIR/ubuntu/
          mv -v _artifacts/ubuntu-installers/* $OUTPUT_DIR/ubuntu
          mkdir -p $OUTPUT_DIR/osx/
          mv -v _artifacts/osx-installers/* $OUTPUT_DIR/osx
          mkdir -p $OUTPUT_DIR/windows/
          mv -v _artifacts/windows-installers/* $OUTPUT_DIR/windows
          mkdir -p $OUTPUT_DIR/server_deb/
          mv -v _artifacts/server-deb/* $OUTPUT_DIR/server_deb
      - name: Authenticate
        uses: 'google-github-actions/auth@v1'
        with:
          credentials_json: ${{ secrets.GCP_SA_KEY }}
          export_environment_variables: true
      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@v1.1.0
      - name: Upload installers to GCS
        uses: google-github-actions/upload-cloud-storage@v1.0.0
        with:
          path: gcs_upload_dir/
          destination: ${{ env.GCS_BUCKET }}
          # Omit `path` (e.g. /home/runner/deploy/) in final GCS path.
          parent: false
      - name: Replace ${{ env.GCS_LATEST_PATH }} folder in GCS
        run: |
          gsutil rm gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/** || true
          gsutil cp -r $OUTPUT_DIR/server_deb/* gs://${{ env.GCS_BUCKET }}/${{ env.GCS_LATEST_PATH }}/
      - name: Upload OpenAPI to GCS
        uses: google-github-actions/upload-cloud-storage@v1.0.0
        with:
          path: _artifacts/openapi/
          destination: ${{ env.GCS_BUCKET_OPENAPI }}
          # Omit `path` (e.g. /home/runner/deploy/) in final GCS path.
          parent: false