wip #236
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a variety of Python versions | |
| # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | |
| name: Tests | |
| on: | |
| push: | |
| pull_request: | |
| branches: [ master ] | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v4.0.0 | |
| with: | |
| python-version: 3.8 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install flake8 | |
| - name: Lint with flake8 | |
| run: | | |
| # stop the build if there are Python syntax errors or undefined names | |
| flake8 ./shitspotter --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
| # flake8 . --count --exit-zero --max-complexity=20 --max-line-length=127 --statistics | |
| build_and_test_sdist: | |
| name: Test sdist Python 3.8 | |
| runs-on: ubuntu-latest | |
| #needs: [lint] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v4.0.0 | |
| with: | |
| python-version: 3.8 | |
| - name: Upgrade pip | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install -r requirements/tests.txt | |
| python -m pip install -r requirements/runtime.txt | |
| - name: Build sdist | |
| run: | | |
| python setup.py sdist | |
| - name: Install sdist | |
| run: | | |
| ls -al ./dist | |
| pip install dist/shitspotter*.tar.gz -v | |
| mv shitspotter ignore_src_shitspotter | |
| - name: Test minimal sdist | |
| run: | | |
| pwd | |
| ls -al | |
| # Run the tests | |
| # Get path to installed package | |
| MOD_DPATH=$(python -c "import shitspotter, os; print(os.path.dirname(shitspotter.__file__))") | |
| echo "MOD_DPATH = $MOD_DPATH" | |
| python -m pytest -p pytester -p no:doctest --xdoctest --cov=shitspotter $MOD_DPATH ./tests | |
| - name: Test full sdist | |
| run: | | |
| pwd | |
| ls -al | |
| python -m pip install -r requirements/optional.txt | |
| # Run the tests | |
| # Get path to installed package | |
| MOD_DPATH=$(python -c "import shitspotter, os; print(os.path.dirname(shitspotter.__file__))") | |
| echo "MOD_DPATH = $MOD_DPATH" | |
| python -m pytest -p pytester -p no:doctest --xdoctest $MOD_DPATH ./tests | |
| - name: Upload sdist artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: ./dist/*.tar.gz | |
| build_and_test_wheels: | |
| # TODO: handle different python versions: https://github.com/actions/setup-python | |
| name: ${{ matrix.python-version }} on ${{ matrix.os }}, arch=${{ matrix.arch }} | |
| runs-on: ${{ matrix.os }} | |
| #needs: [lint] | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macOS-latest] | |
| python-version: | |
| - '3.5' | |
| - '3.6' | |
| - '3.7' | |
| - '3.8' | |
| arch: [auto] | |
| #cibw_build: [cp3*-*] | |
| #cibw_skip: ["*-win32"] | |
| # Add additional workers to reduce overall build time | |
| #include: | |
| # - os: windows-latest | |
| # cibw_build: cp3*-win32 | |
| # arch: auto | |
| # cibw_skip: "" | |
| # - os: ubuntu-latest | |
| # arch: aarch64 | |
| # cibw_build: cp35-* | |
| # - os: ubuntu-latest | |
| # arch: aarch64 | |
| # cibw_build: cp36-* | |
| # - os: ubuntu-latest | |
| # arch: aarch64 | |
| # cibw_build: cp37-* | |
| # - os: ubuntu-latest | |
| # arch: aarch64 | |
| # cibw_build: cp38-* | |
| # - os: ubuntu-latest | |
| # arch: aarch64 | |
| # cibw_build: cp39-* | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v3 | |
| # Configure compilers for Windows 64bit. | |
| - name: Enable MSVC 64bit | |
| if: matrix.os == 'windows-latest' && matrix.cibw_build != 'cp3*-win32' | |
| uses: ilammy/msvc-dev-cmd@v1 | |
| # Configure compilers for Windows 32bit. | |
| - name: Enable MSVC 32bit | |
| if: matrix.os == 'windows-latest' && matrix.cibw_build == 'cp3*-win32' | |
| uses: ilammy/msvc-dev-cmd@v1 | |
| with: | |
| arch: x86 | |
| # Emulate aarch64 ppc64le s390x under linux | |
| - name: Set up QEMU | |
| if: runner.os == 'Linux' && matrix.arch != 'auto' | |
| uses: docker/setup-qemu-action@v2 | |
| with: | |
| platforms: all | |
| # See: https://github.com/pypa/cibuildwheel/blob/main/action.yml | |
| #- name: Build wheels | |
| # uses: pypa/cibuildwheel@v1.11.0 | |
| # with: | |
| # output-dir: wheelhouse | |
| # # to supply options, put them in 'env', like: | |
| # env: | |
| # CIBW_SKIP: ${{ matrix.cibw_skip }} | |
| # CIBW_BUILD: ${{ matrix.cibw_build }} | |
| # CIBW_TEST_REQUIRES: -r requirements/tests.txt | |
| # CIBW_TEST_COMMAND: python {project}/run_tests.py | |
| # # configure cibuildwheel to build native archs ('auto'), or emulated ones | |
| # CIBW_ARCHS_LINUX: ${{ matrix.arch }} | |
| # Setup Python environemtn for a pure wheel | |
| - uses: actions/setup-python@v4.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Build pure wheel | |
| shell: bash | |
| run: | | |
| python -m pip install setuptools>=0.8 wheel | |
| python -m pip wheel --wheel-dir wheelhouse . | |
| - name: Test pure wheel | |
| shell: bash | |
| env: | |
| CI_PYTHON_VERSION: py${{ matrix.python-version }} | |
| #OS_NAME: ${{ matrix.os }} | |
| run: | | |
| # Remove source directory (ensure it doesn't conflict) | |
| mv shitspotter ignore_src_shitspotter | |
| rm pytest.ini | |
| # Install the wheel | |
| python -m pip install wheelhouse/shitspotter*.whl | |
| python -m pip install -r requirements/tests.txt | |
| # Run in a sandboxed directory | |
| WORKSPACE_DNAME="testdir_${CI_PYTHON_VERSION}_${GITHUB_RUN_ID}_${RUNNER_OS}" | |
| mkdir -p $WORKSPACE_DNAME | |
| cd $WORKSPACE_DNAME | |
| # Run the tests | |
| # Get path to installed package | |
| MOD_DPATH=$(python -c "import shitspotter, os; print(os.path.dirname(shitspotter.__file__))") | |
| echo "MOD_DPATH = $MOD_DPATH" | |
| python -m pytest -p pytester -p no:doctest --xdoctest --cov-config ../.coveragerc --cov-report term --cov=shitspotter $MOD_DPATH ../tests | |
| mv .coverage "../.coverage.$WORKSPACE_DNAME" | |
| cd .. | |
| # Move coverage file to a new name | |
| - name: Show built files | |
| shell: bash | |
| run: ls -la wheelhouse | |
| - name: Set up Python 3.8 to combine coverage Linux | |
| if: runner.os == 'Linux' | |
| uses: actions/setup-python@v4.0.0 | |
| with: | |
| python-version: 3.8 | |
| - name: Combine coverage Linux | |
| if: runner.os == 'Linux' | |
| run: | | |
| echo '############ PWD' | |
| pwd | |
| ls -al | |
| python -m pip install coverage[toml] | |
| echo '############ combine' | |
| coverage combine . | |
| echo '############ XML' | |
| coverage xml -o ./tests/coverage.xml | |
| echo '############ FIND' | |
| find . -name .coverage.* | |
| find . -name coverage.xml | |
| - name: Codecov Upload | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./tests/coverage.xml | |
| - name: Upload wheels artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: wheels | |
| path: ./wheelhouse/shitspotter*.whl | |
| deploy: | |
| # Publish on the real PyPI | |
| name: Uploading to PyPi | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') | |
| needs: [build_and_test_wheels, build_and_test_sdist] | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v3 | |
| - name: Download wheels and sdist | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| path: dist | |
| - name: Show files to upload | |
| shell: bash | |
| run: ls -la dist | |
| # Note: | |
| # See ../../dev/setup_secrets.sh for details on how secrets are deployed securely | |
| - name: Sign and Publish | |
| env: | |
| TWINE_REPOSITORY_URL: https://upload.pypi.org/legacy/ | |
| TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} | |
| EROTEMIC_CI_SECRET: ${{ secrets.EROTEMIC_CI_SECRET }} | |
| run: | | |
| ls -al | |
| GPG_EXECUTABLE=gpg | |
| $GPG_EXECUTABLE --version | |
| openssl version | |
| $GPG_EXECUTABLE --list-keys | |
| echo "Decrypting Keys" | |
| GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import | |
| GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust | |
| GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import | |
| echo "Finish Decrypt Keys" | |
| $GPG_EXECUTABLE --list-keys || true | |
| $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" | |
| $GPG_EXECUTABLE --list-keys | |
| MB_PYTHON_TAG=$(python -c "import setup; print(setup.MB_PYTHON_TAG)") | |
| VERSION=$(python -c "import setup; print(setup.VERSION)") | |
| pip install twine | |
| pip install six pyopenssl ndg-httpsclient pyasn1 -U --user | |
| pip install requests[security] twine --user | |
| GPG_KEYID=$(cat dev/public_gpg_key) | |
| echo "GPG_KEYID = '$GPG_KEYID'" | |
| MB_PYTHON_TAG=$MB_PYTHON_TAG \ | |
| DO_GPG=True GPG_KEYID=$GPG_KEYID \ | |
| TWINE_REPOSITORY_URL=${TWINE_REPOSITORY_URL} \ | |
| TWINE_PASSWORD=$TWINE_PASSWORD \ | |
| TWINE_USERNAME=$TWINE_USERNAME \ | |
| GPG_EXECUTABLE=$GPG_EXECUTABLE \ | |
| DO_UPLOAD=True \ | |
| DO_TAG=False ./publish.sh | |
| test_deploy: | |
| # Publish on the test PyPI | |
| name: Uploading to Test PyPi | |
| runs-on: ubuntu-latest | |
| #if: github.event_name == 'push' && (startsWith(github.event.ref, 'refs/heads/main') || startsWith(github.event.ref, 'refs/heads/master')) | |
| if: github.event_name == 'push' && !startsWith(github.event.ref, 'refs/tags') | |
| needs: [build_and_test_wheels, build_and_test_sdist] | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v3 | |
| - name: Download wheels and sdist | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: wheels | |
| path: dist | |
| - name: Show files to upload | |
| shell: bash | |
| run: ls -la dist | |
| - name: Sign and Publish | |
| env: | |
| TEST_TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/ | |
| #TEST_TWINE_USERNAME: ${{ secrets.TEST_TWINE_USERNAME }} | |
| #TEST_TWINE_PASSWORD: ${{ secrets.TEST_TWINE_PASSWORD }} | |
| #export TEST_TWINE_USERNAME=${{ secrets.EROTEMIC_TEST_TWINE_USERNAME }} | |
| #export TEST_TWINE_PASSWORD=${{ secrets.EROTEMIC_TEST_TWINE_PASSWORD }} | |
| #PYUTILS_CI_SECRET: ${{ secrets.PYUTILS_CI_SECRET }} | |
| TEST_TWINE_USERNAME: ${{ secrets.TEST_TWINE_USERNAME }} | |
| TEST_TWINE_PASSWORD: ${{ secrets.TEST_TWINE_PASSWORD }} | |
| EROTEMIC_CI_SECRET: ${{ secrets.EROTEMIC_CI_SECRET }} | |
| run: | | |
| ls -al | |
| GPG_EXECUTABLE=gpg | |
| $GPG_EXECUTABLE --version | |
| openssl version | |
| $GPG_EXECUTABLE --list-keys | |
| echo "Decrypting Keys" | |
| GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_public_gpg_key.pgp.enc | $GPG_EXECUTABLE --import | |
| GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/gpg_owner_trust.enc | $GPG_EXECUTABLE --import-ownertrust | |
| GLKWS=$EROTEMIC_CI_SECRET openssl enc -aes-256-cbc -pbkdf2 -md SHA512 -pass env:GLKWS -d -a -in dev/ci_secret_gpg_subkeys.pgp.enc | $GPG_EXECUTABLE --import | |
| echo "Finish Decrypt Keys" | |
| $GPG_EXECUTABLE --list-keys || echo "first invocation of gpg creates directories and returns 1" | |
| $GPG_EXECUTABLE --list-keys | |
| MB_PYTHON_TAG=$(python -c "import setup; print(setup.MB_PYTHON_TAG)") | |
| VERSION=$(python -c "import setup; print(setup.VERSION)") | |
| pip install twine | |
| pip install six pyopenssl ndg-httpsclient pyasn1 -U --user | |
| pip install requests[security] twine --user | |
| GPG_KEYID=$(cat dev/public_gpg_key) | |
| echo "GPG_KEYID = '$GPG_KEYID'" | |
| MB_PYTHON_TAG=$MB_PYTHON_TAG \ | |
| DO_GPG=True GPG_KEYID=$GPG_KEYID \ | |
| TWINE_REPOSITORY_URL=${TEST_TWINE_REPOSITORY_URL} \ | |
| TWINE_USERNAME=${TEST_TWINE_USERNAME} \ | |
| TWINE_PASSWORD=${TEST_TWINE_PASSWORD} \ | |
| GPG_EXECUTABLE=$GPG_EXECUTABLE \ | |
| DO_UPLOAD=True \ | |
| DO_TAG=False ./publish.sh | |
| ### | |
| # Unfortunately we cant (yet) use the yaml docstring trick here | |
| # https://gh.neting.ccmunity/t/allow-unused-keys-in-workflow-yaml-files/172120 | |
| #__doc__: | | |
| # # How to run locally | |
| # # https://packaging.python.org/guides/using-testpypi/ | |
| # cd $HOME/code | |
| # git clone https://github.com/nektos/act.git $HOME/code/act | |
| # cd $HOME/code/act | |
| # chmod +x install.sh | |
| # ./install.sh -b $HOME/.local/opt/act | |
| # cd $HOME/code/shitspotter | |
| # load_secrets | |
| # unset GITHUB_TOKEN | |
| # $HOME/.local/opt/act/act \ | |
| # --secret=EROTEMIC_TWINE_PASSWORD=$EROTEMIC_TWINE_PASSWORD \ | |
| # --secret=EROTEMIC_TWINE_USERNAME=$EROTEMIC_TWINE_USERNAME \ | |
| # --secret=EROTEMIC_CI_SECRET=$EROTEMIC_CI_SECRET \ | |
| # --secret=EROTEMIC_TEST_TWINE_USERNAME=$EROTEMIC_TEST_TWINE_USERNAME \ | |
| # --secret=EROTEMIC_TEST_TWINE_PASSWORD=$EROTEMIC_TEST_TWINE_PASSWORD |