tests.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Tests

on:
  push:
  pull_request:
    branches: [ master ]

jobs:

  lint:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python 3.8
      uses: actions/setup-python@v4.0.0
      with:
        python-version: 3.8
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install flake8
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 ./shitspotter --count --select=E9,F63,F7,F82 --show-source --statistics
    #    # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
    #    flake8 . --count --exit-zero --max-complexity=20 --max-line-length=127 --statistics

  build_and_test_sdist:
    name: Test sdist Python 3.8
    runs-on: ubuntu-latest
    #needs: [lint]
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python 3.8
      uses: actions/setup-python@v4.0.0
      with:
        python-version: 3.8
    - name: Upgrade pip 
      run: |
        python -m pip install --upgrade pip
        python -m pip install -r requirements/tests.txt
        python -m pip install -r requirements/runtime.txt
    - name: Build sdist
      run: |
        python setup.py sdist
    - name: Install sdist
      run: |
        ls -al ./dist
        pip install dist/shitspotter*.tar.gz -v
        mv shitspotter ignore_src_shitspotter
    - name: Test minimal sdist
      run: |
        pwd
        ls -al
        # Run the tests
        # Get path to installed package
        MOD_DPATH=$(python -c "import shitspotter, os; print(os.path.dirname(shitspotter.__file__))")
        echo "MOD_DPATH = $MOD_DPATH"
        python -m pytest -p pytester -p no:doctest --xdoctest --cov=shitspotter $MOD_DPATH ./tests

    - name: Test full sdist
      run: |
        pwd
        ls -al
        python -m pip install -r requirements/optional.txt
        # Run the tests
        # Get path to installed package
        MOD_DPATH=$(python -c "import shitspotter, os; print(os.path.dirname(shitspotter.__file__))")
        echo "MOD_DPATH = $MOD_DPATH"
        python -m pytest -p pytester -p no:doctest --xdoctest $MOD_DPATH ./tests


    - name: Upload sdist artifact
      uses: actions/upload-artifact@v3
      with:
        name: wheels
        path: ./dist/*.tar.gz

  build_and_test_wheels:
    # TODO: handle different python versions: https://github.com/actions/setup-python
    name: ${{ matrix.python-version }} on ${{ matrix.os }}, arch=${{ matrix.arch }}
    runs-on: ${{ matrix.os }}
    #needs: [lint]
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macOS-latest]
        python-version: 
          - '3.5'
          - '3.6'
          - '3.7'
          - '3.8'
        arch: [auto]
        #cibw_build: [cp3*-*]
        #cibw_skip: ["*-win32"]
        # Add additional workers to reduce overall build time
        #include:
        #  - os: windows-latest
        #    cibw_build: cp3*-win32
        #    arch: auto
        #    cibw_skip: ""
        #  - os: ubuntu-latest
        #    arch: aarch64
        #    cibw_build: cp35-*
        #  - os: ubuntu-latest
        #    arch: aarch64
        #    cibw_build: cp36-*
        #  - os: ubuntu-latest
        #    arch: aarch64
        #    cibw_build: cp37-*
        #  - os: ubuntu-latest
        #    arch: aarch64
        #    cibw_build: cp38-*
        #  - os: ubuntu-latest
        #    arch: aarch64
        #    cibw_build: cp39-*


    steps:
    - name: Checkout source
      uses: actions/checkout@v3

    # Configure compilers for Windows 64bit.
    - name: Enable MSVC 64bit
      if: matrix.os == 'windows-latest' && matrix.cibw_build != 'cp3*-win32'
      uses: ilammy/msvc-dev-cmd@v1
    
    # Configure compilers for Windows 32bit.
    - name: Enable MSVC 32bit
      if: matrix.os == 'windows-latest' && matrix.cibw_build == 'cp3*-win32'
      uses: ilammy/msvc-dev-cmd@v1
      with:
        arch: x86

    # Emulate aarch64 ppc64le s390x under linux
    - name: Set up QEMU
      if: runner.os == 'Linux' && matrix.arch != 'auto'
      uses: docker/setup-qemu-action@v2
      with:
        platforms: all


    # See: https://github.com/pypa/cibuildwheel/blob/main/action.yml
    #- name: Build wheels
    #  uses: pypa/cibuildwheel@v1.11.0
    #  with:
    #    output-dir: wheelhouse
    #  # to supply options, put them in 'env', like:
    #  env:
    #    CIBW_SKIP: ${{ matrix.cibw_skip }}
    #    CIBW_BUILD: ${{ matrix.cibw_build }}
    #    CIBW_TEST_REQUIRES: -r requirements/tests.txt
    #    CIBW_TEST_COMMAND: python {project}/run_tests.py
    #    # configure cibuildwheel to build native archs ('auto'), or emulated ones
    #    CIBW_ARCHS_LINUX: ${{ matrix.arch }}
    
    # Setup Python environemtn for a pure wheel
    - uses: actions/setup-python@v4.0.0
      with:
        python-version: ${{ matrix.python-version }}
    
    - name: Build pure wheel
      shell: bash
      run: |
        python -m pip install setuptools>=0.8 wheel
        python -m pip wheel --wheel-dir wheelhouse .

    - name: Test pure wheel
      shell: bash
      env:
        CI_PYTHON_VERSION: py${{ matrix.python-version }}
        #OS_NAME: ${{ matrix.os }}
      run: | 
        # Remove source directory (ensure it doesn't conflict)
        mv shitspotter ignore_src_shitspotter
        rm pytest.ini
        # Install the wheel
        python -m pip install wheelhouse/shitspotter*.whl
        python -m pip install -r requirements/tests.txt
        # Run in a sandboxed directory
        WORKSPACE_DNAME="testdir_${CI_PYTHON_VERSION}_${GITHUB_RUN_ID}_${RUNNER_OS}"
        mkdir -p $WORKSPACE_DNAME
        cd $WORKSPACE_DNAME
        # Run the tests
        # Get path to installed package
        MOD_DPATH=$(python -c "import shitspotter, os; print(os.path.dirname(shitspotter.__file__))")
        echo "MOD_DPATH = $MOD_DPATH"
        python -m pytest -p pytester -p no:doctest --xdoctest --cov-config ../.coveragerc --cov-report term --cov=shitspotter $MOD_DPATH ../tests
        mv .coverage "../.coverage.$WORKSPACE_DNAME"
        cd ..
        # Move coverage file to a new name

    - name: Show built files
      shell: bash
      run: ls -la wheelhouse

    - name: Set up Python 3.8 to combine coverage Linux
      if: runner.os == 'Linux'
      uses: actions/setup-python@v4.0.0
      with:
        python-version: 3.8

    - name: Combine coverage Linux
      if: runner.os == 'Linux'
      run: |
        echo '############ PWD'
        pwd
        ls -al
        python -m pip install coverage[toml]
        echo '############ combine'
        coverage combine .
        echo '############ XML'
        coverage xml -o ./tests/coverage.xml
        echo '############ FIND'
        find . -name .coverage.*
        find . -name coverage.xml

    - name: Codecov Upload
      uses: codecov/codecov-action@v3
      with:
        file: ./tests/coverage.xml

    - name: Upload wheels artifact
      uses: actions/upload-artifact@v3
      with:
        name: wheels
        path: ./wheelhouse/shitspotter*.whl

  deploy:
    # Publish on the real PyPI
    name: Uploading to PyPi
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
    needs: [build_and_test_wheels, build_and_test_sdist]
    steps:
      - name: Checkout source
        uses: actions/checkout@v3

      - name: Download wheels and sdist
        uses: actions/download-artifact@v3
        with:
          name: wheels
          path: dist

      - name: Show files to upload
        shell: bash
        run: ls -la dist

      # Note:
      # See ../../dev/setup_secrets.sh for details on how secrets are deployed securely
      - name: Sign and Publish
        env:
          TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/
          TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
          EROTEMIC_CI_SECRET: ${{ secrets.EROTEMIC_CI_SECRET }}
        run: |
          ls -al
          GPG_EXECUTABLE=gpg
          $GPG_EXECUTABLE --version
          openssl version
          $GPG_EXECUTABLE --list-keys
          echo "Decrypting Keys"
          GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import 
          GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust 
          GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import
          echo "Finish Decrypt Keys"
          $GPG_EXECUTABLE --list-keys || true
          $GPG_EXECUTABLE --list-keys  || echo "first invocation of gpg creates directories and returns 1"
          $GPG_EXECUTABLE --list-keys  
          MB_PYTHON_TAG=$(python -c "import setup; print(setup.MB_PYTHON_TAG)")
          VERSION=$(python -c "import setup; print(setup.VERSION)") 
          pip install twine
          pip install six pyopenssl ndg-httpsclient pyasn1 -U --user
          pip install requests[security] twine --user
          GPG_KEYID=$(cat dev/public_gpg_key)
          echo "GPG_KEYID = '$GPG_KEYID'"
          MB_PYTHON_TAG=$MB_PYTHON_TAG \
              DO_GPG=True GPG_KEYID=$GPG_KEYID \
              TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} \
              TWINE_PASSWORD=$TWINE_PASSWORD \
              TWINE_USERNAME=$TWINE_USERNAME \
              GPG_EXECUTABLE=$GPG_EXECUTABLE \
              DO_UPLOAD=True \
              DO_TAG=False ./publish.sh 

  test_deploy:
    # Publish on the test PyPI
    name: Uploading to Test PyPi
    runs-on: ubuntu-latest
    #if: github.event_name == 'push' && (startsWith(github.event.ref, 'refs/heads/main') || startsWith(github.event.ref, 'refs/heads/master'))
    if: github.event_name == 'push' && !startsWith(github.event.ref, 'refs/tags')
    needs: [build_and_test_wheels, build_and_test_sdist]
    steps:
      - name: Checkout source
        uses: actions/checkout@v3

      - name: Download wheels and sdist
        uses: actions/download-artifact@v3
        with:
          name: wheels
          path: dist

      - name: Show files to upload
        shell: bash
        run: ls -la dist
      - name: Sign and Publish
        env:
          TEST_TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
          #TEST_TWINE_USERNAME: ${{ secrets.TEST_TWINE_USERNAME }}
          #TEST_TWINE_PASSWORD: ${{ secrets.TEST_TWINE_PASSWORD }}
          #export TEST_TWINE_USERNAME=${{ secrets.EROTEMIC_TEST_TWINE_USERNAME }}
          #export TEST_TWINE_PASSWORD=${{ secrets.EROTEMIC_TEST_TWINE_PASSWORD }}
          #PYUTILS_CI_SECRET: ${{ secrets.PYUTILS_CI_SECRET }}
          TEST_TWINE_USERNAME: ${{ secrets.TEST_TWINE_USERNAME }}
          TEST_TWINE_PASSWORD: ${{ secrets.TEST_TWINE_PASSWORD }}
          EROTEMIC_CI_SECRET: ${{ secrets.EROTEMIC_CI_SECRET }}
        run: |
          ls -al
          GPG_EXECUTABLE=gpg
          $GPG_EXECUTABLE --version
          openssl version
          $GPG_EXECUTABLE --list-keys
          echo "Decrypting Keys"
          GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import 
          GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust 
          GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import
          echo "Finish Decrypt Keys"
          $GPG_EXECUTABLE --list-keys  || echo "first invocation of gpg creates directories and returns 1"
          $GPG_EXECUTABLE --list-keys  
          MB_PYTHON_TAG=$(python -c "import setup; print(setup.MB_PYTHON_TAG)")
          VERSION=$(python -c "import setup; print(setup.VERSION)") 
          pip install twine
          pip install six pyopenssl ndg-httpsclient pyasn1 -U --user
          pip install requests[security] twine --user
          GPG_KEYID=$(cat dev/public_gpg_key)
          echo "GPG_KEYID = '$GPG_KEYID'"
          MB_PYTHON_TAG=$MB_PYTHON_TAG \
              DO_GPG=True GPG_KEYID=$GPG_KEYID \
              TWINE_REPOSITORY_URL=${TEST_TWINE_REPOSITORY_URL} \
              TWINE_USERNAME=${TEST_TWINE_USERNAME} \
              TWINE_PASSWORD=${TEST_TWINE_PASSWORD} \
              GPG_EXECUTABLE=$GPG_EXECUTABLE \
              DO_UPLOAD=True \
              DO_TAG=False ./publish.sh

       
###
# Unfortunately we cant (yet) use the yaml docstring trick here
# https://gh.neting.ccmunity/t/allow-unused-keys-in-workflow-yaml-files/172120
#__doc__: |
#    # How to run locally
#    # https://packaging.python.org/guides/using-testpypi/
#    cd $HOME/code
#    git clone https://github.com/nektos/act.git $HOME/code/act
#    cd $HOME/code/act
#    chmod +x install.sh
#    ./install.sh -b $HOME/.local/opt/act
#    cd $HOME/code/shitspotter

#    load_secrets
#    unset GITHUB_TOKEN
#    $HOME/.local/opt/act/act \
#        --secret=EROTEMIC_TWINE_PASSWORD=$EROTEMIC_TWINE_PASSWORD \
#        --secret=EROTEMIC_TWINE_USERNAME=$EROTEMIC_TWINE_USERNAME \
#        --secret=EROTEMIC_CI_SECRET=$EROTEMIC_CI_SECRET \
#        --secret=EROTEMIC_TEST_TWINE_USERNAME=$EROTEMIC_TEST_TWINE_USERNAME \
#        --secret=EROTEMIC_TEST_TWINE_PASSWORD=$EROTEMIC_TEST_TWINE_PASSWORD