Peskas Kenya Data Pipeline #31
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Peskas Kenya Data Pipeline | |
| on: | |
| push: | |
| schedule: | |
| - cron: '0 0 */2 * *' | |
| env: | |
| KOBO_ASSET_ID: ${{ secrets.KOBO_ASSET_ID }} | |
| KOBO_USERNAME: ${{ secrets.KOBO_USERNAME }} | |
| KOBO_PASSWORD: ${{ secrets.KOBO_PASSWORD }} | |
| MONGODB_CONNECTION_STRING: ${{ secrets.MONGODB_CONNECTION_STRING }} | |
| GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }} | |
| GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| jobs: | |
| build-container: | |
| name: Build R container | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Build image with cache | |
| id: build-docker | |
| uses: whoan/docker-build-with-cache-action@v5 | |
| with: | |
| username: $GITHUB_ACTOR | |
| password: "${{ secrets.GITHUB_TOKEN }}" | |
| registry: docker.pkg.github.com | |
| image_name: r-runner-peskas-kenya | |
| dockerfile: Dockerfile.prod | |
| ingest-landings: | |
| name: Ingest ongoing landings | |
| needs: build-container | |
| runs-on: ubuntu-latest | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.kenya.data.pipeline/r-runner-peskas-kenya | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call ingest_landings() | |
| run: Rscript -e 'peskas.kenya.data.pipeline::ingest_landings()' | |
| preprocess-legacy-landings: | |
| name: Preprocess legacy landings | |
| needs: ingest-landings | |
| runs-on: ubuntu-latest | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.kenya.data.pipeline/r-runner-peskas-kenya | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call preprocess_legacy_landings() | |
| run: Rscript -e 'peskas.kenya.data.pipeline::preprocess_legacy_landings()' | |
| preprocess-landings: | |
| name: Preprocess landings | |
| needs: ingest-landings | |
| runs-on: ubuntu-latest | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.kenya.data.pipeline/r-runner-peskas-kenya | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call preprocess_landings() | |
| run: Rscript -e 'peskas.kenya.data.pipeline::preprocess_landings()' | |
| merge-landings: | |
| name: Merge legacy and ongoing landings | |
| needs: [preprocess-legacy-landings, preprocess-landings] | |
| runs-on: ubuntu-latest | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.kenya.data.pipeline/r-runner-peskas-kenya | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call merge_landings() | |
| run: Rscript -e 'peskas.kenya.data.pipeline::merge_landings()' | |
| validate-landings: | |
| name: Validate landings | |
| needs: merge-landings | |
| runs-on: ubuntu-latest | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.kenya.data.pipeline/r-runner-peskas-kenya | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call validate_landings() | |
| run: Rscript -e 'peskas.kenya.data.pipeline::validate_landings()' | |
| export-landings: | |
| name: Export summarized landings for web application | |
| needs: validate-landings | |
| runs-on: ubuntu-latest | |
| container: | |
| image: docker.pkg.github.com/worldfishcenter/peskas.kenya.data.pipeline/r-runner-peskas-kenya | |
| credentials: | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| steps: | |
| - name: Set env to production | |
| if: endsWith(github.ref, '/main') | |
| run: echo "R_CONFIG_ACTIVE=production" >> $GITHUB_ENV | |
| - name: Call export_summaries() | |
| run: Rscript -e 'peskas.kenya.data.pipeline::export_summaries()' |