Merge pull request #359 from smart-on-fhir/dependabot/pip/pyarrow-lt-19 #1047
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'docs/**' | |
| - '*.md' | |
| # The goal here is to cancel older workflows when a PR is updated (because it's pointless work) | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| unittest: | |
| name: unit tests | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| # Don't go crazy with the Python versions as the test suite is a beast. | |
| # Just test the minimum version we support. | |
| python-version: ["3.10"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install .[tests] | |
| - name: Check out MS tool | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: microsoft/Tools-for-Health-Data-Anonymization | |
| path: mstool | |
| - name: Build MS tool | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install dotnet6 | |
| dotnet publish \ | |
| --runtime=linux-x64 \ | |
| --configuration=Release \ | |
| -p:PublishSingleFile=true \ | |
| --output=$HOME/.local/bin \ | |
| mstool/FHIR/src/Microsoft.Health.Fhir.Anonymizer.R4.CommandLineTool | |
| - name: Test with pytest | |
| run: | | |
| python -m pytest --cov=cumulus_etl --cov-report=xml | |
| - name: Log missing coverage | |
| run: | | |
| coverage report -m --skip-covered | |
| - name: Check coverage report | |
| if: github.ref != 'refs/heads/main' | |
| uses: orgoro/coverage@v3.2 | |
| with: | |
| coverageFile: coverage.xml | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| thresholdAll: .98 | |
| thresholdNew: 1 | |
| thresholdModified: 1 | |
| nlp-regression: | |
| runs-on: ubuntu-latest | |
| env: | |
| UMLS_API_KEY: ${{ secrets.UMLS_API_KEY }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Docker | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build ETL image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| load: true # just build, no push | |
| tags: smartonfhir/cumulus-etl:latest | |
| build-args: | | |
| LOCAL_VERSION=${{ github.sha }} | |
| - name: Download NLP images | |
| run: | | |
| # Just start the specific services we need, to conserve CI disk space | |
| docker compose up -d --quiet-pull cnlpt-negation ctakes-covid | |
| - name: Run NLP | |
| run: | | |
| export DATADIR=$(realpath tests/data/nlp-regression) | |
| # Run the NLP task | |
| docker compose run --rm \ | |
| --volume $DATADIR:/in \ | |
| cumulus-etl \ | |
| /in/input \ | |
| /in/run-output \ | |
| /in/phi \ | |
| --export-group nlp-test \ | |
| --export-timestamp 2024-08-29 \ | |
| --output-format=ndjson \ | |
| --task covid_symptom__nlp_results | |
| # Compare results | |
| export OUTDIR=$DATADIR/run-output/covid_symptom__nlp_results | |
| sudo chown -R $(id -u) $OUTDIR | |
| sed -i 's/"generated_on": "[^"]*", //g' $OUTDIR/*.ndjson | |
| diff -upr $DATADIR/expected-output $OUTDIR | |
| echo "All Good!" | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install linters | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install .[dev] | |
| - name: Run ruff | |
| run: ruff check --output-format=github . |