main.yml

name: Python Build

on:
  push:
  pull_request:
    branches: [ master ]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:
    inputs:
      tmate_enabled:
        type: boolean
        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
        required: false
        default: false

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # https://github.com/actions/setup-python/issues/544#issuecomment-1333893110
        # Python 3.6 is removed from workflow matrices now.
        # One alternative is to support 3.6 but duplicate all the workflow steps in
        # another ubuntu-20.04. But that would result in a full copy of all the
        # workflow steps, causing many problems in future upgrade.
        # Afaik, there is no good way to avoid copying right now: https://stackoverflow.com/a/71570847
        python-version: [ 3.7 ]
        torch-version: [ 1.5.0, 1.6.0 ]
        tensorflow-version: [ 1.15.0 ]
        include:
          - python-version: 3.8
            torch-version: 1.7.1
            tensorflow-version: 2.2.0
          - python-version: 3.8
            torch-version: 1.8.1
            tensorflow-version: 2.2.0
          - python-version: 3.9
            torch-version: 1.7.1
            tensorflow-version: 2.5.0
          - python-version: 3.9
            torch-version: 1.8.1
            tensorflow-version: 2.5.0
    env:
      notebook-details-dep: "huggingface nltk"
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-pip-
            ${{ runner.os }}-
      - name: Install dependencies
        run: |
          sudo apt-get install -y libsndfile1-dev
          python -m pip install --progress-bar off --upgrade pip
          pip install --progress-bar off Django django-guardian
          pip install --progress-bar off black==22.12.0
          python -m pip install ipykernel
          python -m ipykernel install --user
      - name: Format check with Black
        run: |
          black --line-length 80 --check forte/
      - name: Obtain Stave Database Examples
        run: |
          git clone https://github.com/asyml/stave.git
          cd stave/simple-backend
          python manage.py migrate
          cat sample_sql/*.sql | sqlite3 db.sqlite3
          cd ../..
          # Simply keep the database file but remove the repo.
          cp stave/simple-backend/db.sqlite3 .
          rm -rf stave
      - name: Install Forte
        run: |
          pip install --progress-bar off .[data_aug,ir,remote,audio_ext,stave,models,test,wikipedia,nlp,extractor,payload]
      - name: Install deep learning frameworks
        run: |
          pip install --progress-bar off torch==${{ matrix.torch-version }}
          pip install --progress-bar off tensorflow==${{ matrix.tensorflow-version }}
      - name: Install linting tools after regular installation
        # Remember to update pre commit hook (pre-commit-config.yaml) if the following is updated.
        # Better run after other installations (e.g. tensorflow), since those packages may ask for different versions.
        run: |
          pip install --progress-bar off pylint==2.14.5 flake8==5.0.4 mypy==0.991 pytest==7.2.0 black==22.12.0
          pip install --progress-bar off types-PyYAML==5.4.8 types-typed-ast==1.4.4 types-requests==2.25.6 types-dataclasses==0.1.7
          pip install --progress-bar off --upgrade typing-extensions
          pip install --progress-bar off coverage codecov
      - name: Build ontology
        run: |
          ./scripts/build_ontology_specs.sh
      - name: Setup tmate session
        # Add a tmate step after most of the installation.
        uses: mxschmitt/action-tmate@v3
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.tmate_enabled }}
      - name: Lint with flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 forte/ examples/ ft/ scripts/ tests/ --count --select=E9,F63,F7,F82 --show-source --statistics
          # Test the rest but ignore the tests/ folder The GitHub editor is 127 chars wide
          flake8 forte/ examples/ ft/ scripts/ --ignore E203,W503 --count --max-complexity=10 --max-line-length=127 --statistics
      - name: Lint with pylint
        run: |
          pip show typing-extensions
          pylint forte/
      - name: Lint main code with mypy when torch version is not 1.5.0 and python is 3.9
        run: |
          if [[ ${{ matrix.torch-version }} != "1.5.0" && ${{ matrix.python-version }} == "3.9" ]]; then mypy forte --check-untyped-defs; fi
      - name: Test with pytest and run coverage
        run: |
          coverage run -m pytest tests  --ignore=tests/forte/notebooks --ignore=tests/forte/forte_backbone_test.py
          coverage run --append -m pytest --doctest-modules forte
      - name: Upload coverage
        run: |
          codecov
      - name: Test notebook tutorials with installing Fortex
        if: ${{ contains(fromJson('["1.5.0", "1.7.1"]'), matrix.torch-version) }}
        run: |
          git clone https://github.com/asyml/forte-wrappers.git
          cd forte-wrappers
          for d in ${{ env.notebook-details-dep }}; do pip install "src/"$d; done
          cd ..
          if [ ${{ matrix.torch-version }} == "1.5.0" ]
          then
            coverage run -m pytest tests/forte/notebooks  --ignore=tests/forte/notebooks/ocr_test.py --ignore=tests/forte/notebooks/tutorial_MT_with_forte_test.py
          fi
          if [ ${{ matrix.torch-version }} == "1.7.1" ]
          then
            coverage run -m pytest tests/forte/notebooks/tutorial_MT_with_forte_test.py
          fi

  test_ocr_example:
    runs-on: ubuntu-latest
    env:
      python-version: 3.7
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ env.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.python-version }}
      - name: Cache pip
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-pip-
            ${{ runner.os }}-
      - name: ocr test
        run: |
          sudo apt-get install -y automake ca-certificates g++ git libtool libleptonica-dev make pkg-config
          sudo apt install tesseract-ocr
          pip install --progress-bar off .[ocr_tutorial,test]
          python tests/forte/notebooks/ocr_test.py


  test_backbone:
    runs-on: ubuntu-latest
    env:
      python-version: 3.7
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ env.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.python-version }}
      - name: Cache pip
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-pip-
            ${{ runner.os }}-
      - name: Test backbone Forte import test
        run: |
          # Try to install Forte backbone only and test basic imports.
          pip install --progress-bar off .
          python tests/forte/forte_backbone_test.py

  test_modules:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ 3.7 ]
        torch-version: [ 1.5.0 ]
        tensorflow-version: [ 1.15.0 ]
        modules:
          - { module: "data_aug", test_file: "tests/forte/processors/data_augment"}
          - { module: "ir",test_file: "tests/forte/processors/ir/"}
          - { module: "remote", test_file: "tests/forte/remote_processor_test.py"}
          - { module: "audio_ext",test_file: "tests/forte/data/readers/audio_reader_test.py"}
          - { module: "stave",test_file: "tests/forte/data/readers/stave_reader_test.py tests/forte/processors/stave_processor_test.py"}
          - { module: "models", test_file: "tests/forte/models"}
          - { module: "wikipedia", test_file: "tests/forte/datasets/wikipedia"}
          - { module: "nlp",test_file: "tests/forte/processors/subword_tokenizer_test.py tests/forte/processors/pretrained_encoder_processors_test.py"}
          - { module: "extractor",test_file: "tests/forte/train_preprocessor_test.py forte/data/extractors tests/forte/data/data_pack_dataset_test.py tests/forte/data/converter/converter_test.py"}
          - { module: "payload", test_file: "tests/forte/utils/payload_decorator_test.py"}
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-pip-
            ${{ runner.os }}-
      - name: install forte
        run: |
          pip install --progress-bar off .
      - name: install dependencies
        run: |
          sudo apt-get install -y libsndfile1-dev
          python -m pip install --progress-bar off --upgrade pip
          pip install --progress-bar off Django django-guardian
      - name: Obtain Stave Database Examples
        run: |
          git clone https://github.com/asyml/stave.git
          cd stave/simple-backend
          python manage.py migrate
          cat sample_sql/*.sql | sqlite3 db.sqlite3
          cd ../..
          # Simply keep the database file but remove the repo.
          cp stave/simple-backend/db.sqlite3 .
          rm -rf stave
      - name: Test modules need extra packages
        run: |
          pip install --progress-bar off .[test]
          pip install --progress-bar off .[${{ matrix.modules.module }}]
          pip install --progress-bar off torch==${{ matrix.torch-version }}
          pip install --progress-bar off pytest==5.1.3
      - name: Setup tmate session
        # Add a tmate step before the actual testing.
        uses: mxschmitt/action-tmate@v3
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.tmate_enabled }}
      - name: Run the test
        run: |
          pytest ${{ matrix.modules.test_file }}

  docs:
    needs: build
    runs-on: ubuntu-latest
    env:
      python-version: 3.7
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ env.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.python-version }}
      - name: Cache pip
        uses: actions/cache@v2
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-pip-
            ${{ runner.os }}-
      - name: Install dependencies
        run: |
          python -m pip install --progress-bar off --upgrade pip
          sudo apt-get install pandoc
          pip install --progress-bar off -r requirements.txt
          pip install --progress-bar off -r docs/requirements.txt
          git clone https://github.com/asyml/texar-pytorch.git
          cd texar-pytorch
          pip install --progress-bar off .
          cd ..
          rm -rf texar-pytorch
      - name: Build Docs
        run: |
          cd docs
          sphinx-build -W -b html -d _build/doctrees . _build/html
          sphinx-build -W -b spelling -d _build/doctrees . _build/spelling
          cd ..
  dispatch:
    needs: build
    runs-on: ubuntu-latest
    if: github.repository == 'asyml/forte' && github.ref == 'refs/heads/master'
    steps:
      - name: Repository Dispatch
        uses: peter-evans/repository-dispatch@v1
        with:
          token: ${{ secrets.REPO_DISPATCH_PAT_HECTOR }}
          repository: asyml/forte-wrappers
          event-type: trigger-forte-wrappers

  readme:
    needs: build
    runs-on: ubuntu-latest
    env:
      python-version: 3.9
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ env.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ env.python-version }}

      - name: Test README.md when python version is 3.9
        run: |
          pip install mkcodes
          pip install --progress-bar off .
          pip install --progress-bar off forte.spacy nltk
          mkcodes --github --output tests/temp_readme_test.py README.md
          python tests/temp_readme_test.py
          rm tests/temp_readme_test.py