build_and_publish.yml

# GitHub Action workflow to build and run filecrawler's tests
#

name: Filecrawler release build

on:
  release:
    types: [published]

env:
  LATEST_TAG: filecrawler:latest

jobs:
  lint:
    name: Check syntaxs errors and warnings
    runs-on: ubuntu-latest
    if:
      github.event_name == 'push' || github.event.pull_request.head.repo.full_name !=
      github.repository

    steps:
      - name: Checkout filecrawler
        uses: actions/checkout@v3

      - name: Setup Python 3.9
        uses: actions/setup-python@v3
        with:
          python-version: 3.9

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install flake8

      - name: Check syntax errors
        run: |
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics

      - name: Check PEP8 warnings
        run: |
          flake8 . --count --ignore=E1,E2,E3,E501,W291,W293 --exit-zero --max-complexity=65 --max-line-length=127 --statistics

  binutils_dependencies:
    name: Build external dependencies
    needs: lint
    runs-on: ubuntu-latest
    if:
      github.event_name == 'push' || github.event.pull_request.head.repo.full_name !=
      github.repository

    strategy:
      fail-fast: false

    steps:
      - name: Install dependencies
        uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: wget curl jq zip
          version: 1.0

      - name: Create temp path
        run: |
          mkdir -p /tmp/filecrawler/bin

      - name: Get tika server .jar
        run: |
          VER=$(curl -s "https://raw.githubusercontent.com/chrismattmann/tika-python/master/tika/tika.py" | grep 'TIKA_VERSION' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}' || echo '2.6.0')
          SERVER_HASH=$(curl -s "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar.sha1")
          wget -nv -O "/tmp/filecrawler/bin/tika-server.jar" "http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server-standard/$VER/tika-server-standard-$VER.jar"
          echo "${SERVER_HASH} /tmp/filecrawler/bin/tika-server.jar" | sha1sum -c - || exit 1

      - name: Get JADX
        run: |
          VER=$(curl -s  "https://api.github.com/repos/skylot/jadx/tags" | jq -r '.[0].name' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')
          wget -nv -O "/tmp/jadx.zip" "https://github.com/skylot/jadx/releases/download/v$VER/jadx-$VER.zip"
          mkdir -p /tmp/jadx/
          unzip -o /tmp/jadx.zip -d /tmp/jadx/
          FILE=$(find /tmp/jadx/ -name "jadx*.jar")
          mv "$FILE" "/tmp/filecrawler/bin/jadx.jar"

      - name: Get APKTool
        run: |
          VER=$(curl -s  "https://api.github.com/repos/iBotPeaches/Apktool/tags" | jq -r '.[0].name' | grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')
          wget -nv -O "/tmp/apktool.jar" "https://github.com/iBotPeaches/Apktool/releases/download/v$VER/apktool_$VER.jar"
          mv "/tmp/apktool.jar" "/tmp/filecrawler/bin/"

      - name: Create bin zip
        run: |
          zip -j -5 /tmp/filecrawler_extra_libs.zip /tmp/filecrawler/bin/*

      - name: Upload binutils release asset
        id: filecrawler_extra_libs
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ github.event.release.upload_url }}
          asset_path: /tmp/filecrawler_extra_libs.zip
          asset_name: filecrawler_extra_libs.zip
          asset_content_type: application/zip

  build:
    name: Build
    needs: binutils_dependencies
    runs-on: ubuntu-latest
    if:
      github.event_name == 'push' || github.event.pull_request.head.repo.full_name !=
      github.repository

    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
        experimental: [false]
        include:
          - python-version: "3.11"
            experimental: true
    continue-on-error: ${{ matrix.experimental }}

    steps:
      - name: Checkout filecrawler
        uses: actions/checkout@v3

      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip wheel
          python -m pip install -r requirements.txt -r tests/requirements-test.txt

      - name: Install filecrawler
        run: |
          pip install .

      - name: Filecrawler basic test
        run: |
          filecrawler -h

      - name: Build artifact
        run: |
          python setup.py sdist

  docker_build:
    name: Check docker build and run
    needs: build
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      #- name: Set up Docker Buildx
      #  uses: docker/setup-buildx-action@v2

      #- name: Build and export to Docker
      #  uses: docker/build-push-action@v4
      #  with:
      #    context: .
      #    push: true
      #    tags: ${{ env.LATEST_TAG }}

      #- name: Test
      #  run: |
      #    docker run --rm ${{ env.LATEST_TAG }} -h

  publish:
    name: Publish on Live Environment
    needs: docker_build
    runs-on: ubuntu-latest
    if:
      github.event_name == 'push' || github.event.pull_request.head.repo.full_name !=
      github.repository

    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9"]
        experimental: [false]
    continue-on-error: ${{ matrix.experimental }}

    steps:
      - name: Checkout filecrawler
        uses: actions/checkout@v3

      - name: Install dependencies
        uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: wget curl jq zip
          version: 1.0

      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Python dependencies
        run: |
          python -m pip install --upgrade pip wheel
          python -m pip install -r requirements.txt -r tests/requirements-test.txt

      - name: Build artifact
        run: |
          python setup.py sdist

      - name: Publish package in Live Environment
        #if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.prod_pypi_password }}