Refactor query execution to avoid toJSON call in SparkExpectationsAct… #238
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build | |
| on: | |
| pull_request: | |
| types: [ opened, synchronize ] | |
| push: | |
| branches: [ main ] | |
| release: | |
| types: [ created ] | |
| jobs: | |
| unittesting: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| max-parallel: 1 | |
| matrix: | |
| python-version: [ '3.9' ] | |
| os: [ ubuntu-latest ] | |
| env: | |
| HADOOP_USER_HOME: 'hadoop' | |
| UNIT_TESTING_ENV: 'spark_expectations_unit_testing_on_github_actions' | |
| steps: | |
| - uses: actions/checkout@v3 # use latest version of the checkout action | |
| - name: Set up JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| - name: Setup Kafka | |
| run: | | |
| curl -SL "https://archive.apache.org/dist/kafka/3.0.0/kafka_2.13-3.0.0.tgz" | tar -xzC /tmp | |
| export PATH=$PATH:/tmp/kafka_2.13-3.0.0/bin | |
| echo "KAFKA_HOME=/tmp/kafka_2.13-3.0.0" >> $GITHUB_ENV | |
| - name: Install pip | |
| run: python -m pip install --upgrade pip | |
| - name: Install and configure Poetry | |
| uses: snok/install-poetry@v1 | |
| - name: Install poetry and build tools | |
| run: | | |
| export PATH=$PATH:$HOME/.local/bin | |
| poetry self add "poetry-dynamic-versioning[plugin]" | |
| - name: Install dependencies | |
| run: | | |
| export PATH=$PATH:$HOME/.local/bin | |
| make poetry | |
| - name: Install, lint and test | |
| run: | | |
| export PATH=$PATH:$HOME/.local/bin | |
| export GITHUB_ACTIONS=true | |
| make cov | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v3 | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| files: coverage.xml | |
| fail_ci_if_error: true | |
| deploy: | |
| name: Deploy to PyPi | |
| runs-on: ubuntu-latest | |
| needs: | |
| - unittesting | |
| if: github.event_name == 'release' | |
| steps: | |
| - uses: actions/checkout@v3 # use latest version of the checkout action | |
| - name: Set up JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| - name: Install pip | |
| run: python -m pip install --upgrade pip | |
| - name: Install and configure Poetry | |
| uses: snok/install-poetry@v1 | |
| - name: Install build tools | |
| run: | | |
| export PATH=$PATH:$HOME/.local/bin | |
| poetry self add "poetry-dynamic-versioning[plugin]" | |
| - name: Install dependencies | |
| run: | | |
| export PATH=$PATH:$HOME/.local/bin | |
| make poetry | |
| - name: Install wheel and twine | |
| run: python -m pip install wheel twine | |
| - name: Build and publish | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| export PATH=$PATH:$HOME/.local/bin | |
| make build | |
| twine upload dist/* |