run_false-bay-forecast.yml

name: Run False Bay Forecast model

on:
  workflow_call:
    inputs:
      # the branch on which the model is run e.g. "stable"
      BRANCH_REF:
        required: true
        type: string
      # model run date in format "yyyymmdd"
      MODEL_RUN_DATE:
        required: true
        type: string
      # toolkit image in format ghcr.io/saeon/somisana_toolkit_$BRANCH_REF:sha-<commit>
      TOOLKIT_IMAGE:
        required: true
        type: string
      # directory where forcing input data over the EEZ is downloaded to
      EEZ_DATA_DIR:
        required: true
        type: string

env:
  REGISTRY: ghcr.io

jobs:

  # Cleanup old runs temp files
  cleanup-old-run-temp-files:
    runs-on: somisana
    continue-on-error: true
    steps:
      - name: Clean /home/runner/somisana/false-bay-forecast/${{ inputs.BRANCH_REF }}
        run: >-
          find \
            /home/runner/somisana/false-bay-forecast/${{ inputs.BRANCH_REF }}/* \
            -maxdepth 0 \
            -type d \
            -ctime +5 \
            -exec \
              rm \
                -rf {} \;

  envs:
    runs-on: ubuntu-latest
    outputs:
      CROCO_IMAGE: ${{ steps.ENVS.outputs.CROCO_IMAGE }}
      MODEL_RUN_YEAR_MONTH: ${{ steps.ENVS.outputs.MODEL_RUN_YEAR_MONTH }}
      RESTART_FILE_DATE: ${{ steps.ENVS.outputs.RESTART_FILE_DATE }}
    steps:
      - name: Configure restart date
        id: restart_date
        run: |
          echo "value=$(date -d '${{ inputs.MODEL_RUN_DATE }} -1 days' +'%Y%m%d')" >> $GITHUB_OUTPUT
      - name: Configure run date as yyyymm format
        id: run_date_yyyymm
        run: |
          echo "value=$(date -d '${{ inputs.MODEL_RUN_DATE }}' +'%Y%m')" >> $GITHUB_OUTPUT
      - name: Set envs
        id: ENVS
        run: |
          echo "CROCO_IMAGE=${{ github.repository }}_false_bay_forecast_croco_${{ inputs.BRANCH_REF }}" >> $GITHUB_OUTPUT
          echo "MODEL_RUN_YEAR_MONTH=${{ steps.run_date_yyyymm.outputs.value }}" >> $GITHUB_OUTPUT
          echo "RESTART_FILE_DATE=${{ steps.restart_date.outputs.value }}" >> $GITHUB_OUTPUT

  # Compile CROCO model for the False Bay forecast
  compile-croco:
    needs: [envs]
    runs-on: ubuntu-latest
    env:
      CROCO_IMAGE: ${{ needs.envs.outputs.CROCO_IMAGE }}
    outputs:
      image: ${{ steps.meta.outputs.tags }}
    steps:
      - name: Checkout source code
        uses: actions/checkout@main
        with:
          ref: ${{ inputs.BRANCH_REF }}
      - name: Log in to the Container registry
        uses: docker/login-action@master
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@master
        with:
          images: ${{ env.REGISTRY }}/${{ env.CROCO_IMAGE }}
          tags: |
            type=sha
      - name: Build and push
        uses: docker/build-push-action@master
        with:
          context: models/false-bay-forecast
          file: models/false-bay-forecast/croco.Dockerfile
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}

  # Create a tmp folder structure for mode run assets. The format is:
  ## YYYYMMDD/
  ##  - croco/
  ##     - forcing/
  ##     - forecast/
  ##     - scratch/
  workdir:
    needs: [envs]
    runs-on: somisana
    env:
      RESTART_FILE_DATE: ${{ needs.envs.outputs.RESTART_FILE_DATE }}
    outputs:
      WORKDIR: ${{ steps.WORKDIR.outputs.WORKDIR}}
      WORKDIR_YESTERDAY: ${{ steps.WORKDIR.outputs.WORKDIR_YESTERDAY }}
    steps:
      - name: Create /home/runner/somisana/false-bay-forecast/${{ inputs.BRANCH_REF }}/${{ inputs.MODEL_RUN_DATE }} directory structure
        run: |
          rm -rf /home/runner/somisana/false-bay-forecast/${{ inputs.BRANCH_REF }}/${{ inputs.MODEL_RUN_DATE }}
          mkdir -p /home/runner/somisana/false-bay-forecast/${{ inputs.BRANCH_REF }}/${{ inputs.MODEL_RUN_DATE }}/croco/{forcing,forecast,scratch}
          chown -R :runners /home/runner/somisana/false-bay-forecast/${{ inputs.BRANCH_REF }}/${{ inputs.MODEL_RUN_DATE }}
          chmod -R 774 /home/runner/somisana/false-bay-forecast/${{ inputs.BRANCH_REF }}/${{ inputs.MODEL_RUN_DATE }}
      - name: Set WORKDIRs
        id: WORKDIR
        run: |
          echo "WORKDIR=/home/runner/somisana/false-bay-forecast/${{ inputs.BRANCH_REF }}/${{ inputs.MODEL_RUN_DATE }}" >> $GITHUB_OUTPUT
          echo "WORKDIR_YESTERDAY=/home/runner/somisana/false-bay-forecast/${{ inputs.BRANCH_REF }}/${{ env.RESTART_FILE_DATE }}" >> $GITHUB_OUTPUT

  # CROCOTOOLS is a collection of MatLab scripts for converting environmental data (i.e. the boundary data downloaded previously)
  # into NetCDF files that can be used as input to the CROCO model. https://www.croco-ocean.org/documentation/crocotools-documentation/
  crocotools:
    needs: [workdir, envs]
    runs-on: croco
    env:
      WORKDIR: ${{ needs.workdir.outputs.WORKDIR }}
      RESTART_FILE_DATE: ${{ needs.envs.outputs.RESTART_FILE_DATE }}
      RESTART_FILE_1_PATH: ${{ needs.workdir.outputs.WORKDIR_YESTERDAY }}/croco/scratch/rst.nc
      RESTART_FILE_2_PATH: ${{ needs.workdir.outputs.WORKDIR_YESTERDAY }}/croco/scratch/rst.nc.1
    steps:
      - name: Check out source code
        uses: actions/checkout@main
        with:
          ref: ${{ inputs.BRANCH_REF }}
      - name: Copy yesterday's parent restart file (if it exists)
        run: cp ${{ env.RESTART_FILE_1_PATH }} ${{ env.WORKDIR }}/croco/forcing/rst_${{ env.RESTART_FILE_DATE }}.nc
        continue-on-error: true # Restart file doesn't have to exist
      - name: Copy yesterday's child restart file (if it exists)
        run: cp ${{ env.RESTART_FILE_2_PATH }} ${{ env.WORKDIR }}/croco/forcing/rst_${{ env.RESTART_FILE_DATE }}.nc.1
        continue-on-error: true # Restart file doesn't have to exist
      - name: copy the MatLab .env file generated during the download global GFS data step to the working directory
        run: cp ${{ inputs.EEZ_DATA_DIR }}/.env ${{ env.WORKDIR }}
        continue-on-error: true # Allow for the case where the restart file doesn't exist
      - name: Configure MatLab env restart file path
        run: |
          echo "RESTART_FILE_1_PATH=/tmp/somisana/current/croco/forcing/rst_${{ env.RESTART_FILE_DATE }}.nc" >> ${{ env.WORKDIR }}/.env
          echo "RESTART_FILE_2_PATH=/tmp/somisana/current/croco/forcing/rst_${{ env.RESTART_FILE_DATE }}.nc.1" >> ${{ env.WORKDIR }}/.env
      - name: Make forcing files
        run: >-
          docker run \
            --rm \
            --mac-address 02:42:ff:ff:ff:ff \
            -v /opt/licenses/matlab-r2022a/license.lic:/licenses/license.lic \
            -v $(pwd)/models/false-bay-forecast/crocotools:/crocotools/ \
            -v $(pwd)/models/false-bay-forecast/lib/grd.nc:/crocotools/croco/forcing/grd.nc \
            -v $(pwd)/models/false-bay-forecast/lib/grd.nc.1:/crocotools/croco/forcing/grd.nc.1 \
            -v ${{ env.WORKDIR }}:/tmp/somisana/current \
            -v ${{ inputs.EEZ_DATA_DIR }}:/tmp/somisana/global-data \
            -e MLM_LICENSE_FILE=/licenses/license.lic \
            ${{ env.REGISTRY }}/saeon/somisana_matlab:r2022a \
              -batch "run('/crocotools/run.m')"

  # Execute the CROCO model using the forcing files created previously
  # The CROCO model executable is compiled a part of a Docker build, and is baked into a docker image.
  # As such the CROCO model run must be in the context of a container instantiated from that Docker image
  croco:
    needs: [compile-croco, crocotools, envs, workdir]
    runs-on: croco
    env:
      WORKDIR: ${{ needs.workdir.outputs.WORKDIR }}
      RESTART_FILE_DATE: ${{ needs.envs.outputs.RESTART_FILE_DATE }}
    steps:
      - name: Check out source code
        uses: actions/checkout@main
        with:
          ref: ${{ inputs.BRANCH_REF }}
      - name: Execute CROCO binary
        run: >-
          docker run \
            --rm \
            -v $WORKDIR:/false-bay-forecast/current \
            -v $(pwd)/models/false-bay-forecast/lib/grd.nc:/false-bay-forecast/current/croco/forcing/grd.nc \
            -v $(pwd)/models/false-bay-forecast/lib/grd.nc.1:/false-bay-forecast/current/croco/forcing/grd.nc.1 \
            -e NP_XI=2 \
            -e NP_ETA=4 \
            --cpus 8 \
            ${{ needs.compile-croco.outputs.image }} \
              ./run_croco.bash \
                /false-bay-forecast/current \
                ${{ inputs.MODEL_RUN_DATE }} \
                ${{ env.RESTART_FILE_DATE }}
      - name: Move CROCO output
        run: |
          mv ${{ env.WORKDIR }}/croco/scratch/avg.nc ${{ env.WORKDIR }}/croco/forecast/hourly-avg-${{ inputs.MODEL_RUN_DATE }}.nc
          mv ${{ env.WORKDIR }}/croco/scratch/avg.nc.1 ${{ env.WORKDIR }}/croco/forecast/hourly-avg-${{ inputs.MODEL_RUN_DATE }}.nc.1

  # Regrid CROCO u,v to rho grid,
  # rotate u,v components from grid aligned to east/north aligned and
  # work out depth levels of sigma grid in meters (tier 1),
  # interpolate data to constant vertical levels (tier 2),
  # and interpolate data to a constant horizontal grid (tier 3)
  post-processing:
    needs: [croco, workdir]
    runs-on: somisana
    env:
      WORKDIR: ${{ needs.workdir.outputs.WORKDIR }}
    steps:
      - name: Check out source code
        uses: actions/checkout@main
        with:
          ref: ${{ inputs.BRANCH_REF }}
      - name: subset the raw CROCO NetCDF file for archiving - parent
        continue-on-error: true
        run: >-
          docker run \
            --rm \
            --entrypoint /bin/bash \
            -v $WORKDIR:/tmp/somisana/current \
            ghcr.io/saeon/somisana_geopython:3.10.6 \
              -c "ncks -v temp,salt,u,v,w,zeta,sustr,svstr -d time,120, -O /tmp/somisana/current/croco/forecast/hourly-avg-${{ inputs.MODEL_RUN_DATE }}.nc /tmp/somisana/current/croco/forecast/hourly-avg-${{ inputs.MODEL_RUN_DATE }}-archive.nc"
      - name: subset the raw CROCO NetCDF file for archiving - child
        continue-on-error: true
        run: >-
          docker run \
            --rm \
            --entrypoint /bin/bash \
            -v $WORKDIR:/tmp/somisana/current \
            ghcr.io/saeon/somisana_geopython:3.10.6 \
              -c "ncks -v temp,salt,u,v,w,zeta,sustr,svstr -d time,120, -O /tmp/somisana/current/croco/forecast/hourly-avg-${{ inputs.MODEL_RUN_DATE }}.nc.1 /tmp/somisana/current/croco/forecast/hourly-avg-${{ inputs.MODEL_RUN_DATE }}-archive.nc.1"
      - name: Tier 1 regridding
        run: >-
          docker run \
            --rm \
            -v $WORKDIR:/tmp/somisana/current \
            -v $(pwd)/models/false-bay-forecast/lib/grd.nc:/tmp/somisana/current/croco/forcing/grd.nc \
            -v $(pwd)/models/false-bay-forecast/lib/grd.nc.1:/tmp/somisana/current/croco/forcing/grd.nc.1 \
            ${{ inputs.TOOLKIT_IMAGE }} \
              croco \
                regrid-tier1 \
                  --id false-bay-forecast \
                  --grid /tmp/somisana/current/croco/forcing/grd.nc.1 \
                  --input /tmp/somisana/current/croco/forecast/hourly-avg-${{ inputs.MODEL_RUN_DATE }}.nc.1 \
                  --output /tmp/somisana/current/croco/forecast/false-bay-forecast-${{ inputs.MODEL_RUN_DATE }}-tier1.nc
      - name: Tier 2 regridding
        continue-on-error: true
        run: >-
          docker run \
            --rm \
            -v $WORKDIR:/tmp/somisana/current \
            ${{ inputs.TOOLKIT_IMAGE }} \
              croco \
                regrid-tier2 \
                  --input /tmp/somisana/current/croco/forecast/false-bay-forecast-${{ inputs.MODEL_RUN_DATE }}-tier1.nc \
                  --depths 0,1,2,5,10,15,20,30,40,50,60,70,100,150,200,500,1000,1500,2000,99999 \
                  --output /tmp/somisana/current/croco/forecast/false-bay-forecast-${{ inputs.MODEL_RUN_DATE }}-tier2.nc
      - name: Tier 3 regridding
        continue-on-error: true
        run: >-
          docker run \
            --rm \
            -v $WORKDIR:/tmp/somisana/current \
            ${{ inputs.TOOLKIT_IMAGE }} \
              croco \
                regrid-tier3 \
                  --input /tmp/somisana/current/croco/forecast/false-bay-forecast-${{ inputs.MODEL_RUN_DATE }}-tier2.nc \
                  --output /tmp/somisana/current/croco/forecast/false-bay-forecast-${{ inputs.MODEL_RUN_DATE }}-tier3.nc \
                  --spacing 0.02

  # Archive output
  archive-data:
    needs: [post-processing, workdir, envs]
    runs-on: somisana
    env:
      WORKDIR: ${{ needs.workdir.outputs.WORKDIR }}
      MODEL_RUN_YEAR_MONTH: ${{ needs.envs.outputs.MODEL_RUN_YEAR_MONTH }}      
    steps:
      - name: Load $PATH
        run: |
          source ~/.bashrc
      - name: Archive raw CROCO output (hindcast / forecast) - child grid only
        continue-on-error: false
        run: |
          curl \
            --silent \
            --keepalive-time 2400 \
            -X PUT \
            -H "Authorization: ${{ secrets.MNEMOSYNE_TOKEN }}" \
            -H "Content-Type: application/octet-stream" \
            -T ${{ env.WORKDIR }}/croco/forecast/hourly-avg-${{ inputs.MODEL_RUN_DATE }}.nc.1 \
            https://mnemosyne.somisana.ac.za/somisana/sw-cape/5-day-forecast/${{ env.MODEL_RUN_YEAR_MONTH }}/${{ inputs.MODEL_RUN_DATE }}_hourly_avg.nc;
      - name: Archive truncated CROCO output (forecast) - child grid only
        continue-on-error: false
        run: |
          curl \
            --silent \
            --keepalive-time 2400 \
            -X PUT \
            -H "Content-Type: application/octet-stream" \
            -H "Authorization: ${{ secrets.MNEMOSYNE_TOKEN }}" \
            -T ${{ env.WORKDIR }}/croco/forecast/hourly-avg-${{ inputs.MODEL_RUN_DATE }}-archive.nc.1 \
            https://mnemosyne.somisana.ac.za/somisana/sw-cape/5-day-forecast/${{ env.MODEL_RUN_YEAR_MONTH }}/${{ inputs.MODEL_RUN_DATE }}_hourly_avg_fcst.nc;
      - name: Archive tier 1 output
        continue-on-error: false
        run: |
          curl \
            --silent \
            --keepalive-time 2400 \
            -X PUT \
            -H "Content-Type: application/octet-stream" \
            -H "Authorization: ${{ secrets.MNEMOSYNE_TOKEN }}" \
            -T ${{ env.WORKDIR }}/croco/forecast/false-bay-forecast-${{ inputs.MODEL_RUN_DATE }}-tier1.nc \
            https://mnemosyne.somisana.ac.za/somisana/sw-cape/5-day-forecast/${{ env.MODEL_RUN_YEAR_MONTH }}/${{ inputs.MODEL_RUN_DATE }}_hourly_avg_t1.nc;
      - name: Archive tier 2 output
        continue-on-error: false
        run: |
          curl \
            --silent \
            --keepalive-time 2400 \
            -X PUT \
            -H "Content-Type: application/octet-stream" \
            -H "Authorization: ${{ secrets.MNEMOSYNE_TOKEN }}" \
            -T ${{ env.WORKDIR }}/croco/forecast/false-bay-forecast-${{ inputs.MODEL_RUN_DATE }}-tier2.nc \
            https://mnemosyne.somisana.ac.za/somisana/sw-cape/5-day-forecast/${{ env.MODEL_RUN_YEAR_MONTH }}/${{ inputs.MODEL_RUN_DATE }}_hourly_avg_t2.nc;
      - name: Archive tier 3 output
        continue-on-error: false
        run: |
          curl \
            --silent \
            --keepalive-time 2400 \
            -X PUT \
            -H "Content-Type: application/octet-stream" \
            -H "Authorization: ${{ secrets.MNEMOSYNE_TOKEN }}" \
            -T ${{ env.WORKDIR }}/croco/forecast/false-bay-forecast-${{ inputs.MODEL_RUN_DATE }}-tier3.nc \
            https://mnemosyne.somisana.ac.za/somisana/sw-cape/5-day-forecast/${{ env.MODEL_RUN_YEAR_MONTH }}/${{ inputs.MODEL_RUN_DATE }}_hourly_avg_t3.nc;
      - name: Kerchunk NetCDF tier 3 output
        continue-on-error: true
        run: |
          docker run \
            --rm \
            -v $WORKDIR:/tmp/somisana/current \
            ${{ inputs.TOOLKIT_IMAGE }} \
              kerchunk \
                run \
                  --inputs https://mnemosyne.somisana.ac.za/somisana/sw-cape/5-day-forecast/${{ env.MODEL_RUN_YEAR_MONTH }}/${{ inputs.MODEL_RUN_DATE }}_hourly_avg_t3.nc \
                  --output /tmp/somisana/current/${{ inputs.MODEL_RUN_DATE }}-hourly-avg-t3.kerchunk.json
      - name: Upload Kerchunked tier 3 output to Mnemosyne
        continue-on-error: true
        run: |
          curl \
            --silent \
            --keepalive-time 1200 \
            -X PUT \
            -H "Authorization: ${{ secrets.MNEMOSYNE_TOKEN }}" \
            -H "Content-Type: application/octet-stream" \
            -T ${{ env.WORKDIR}}/${{ inputs.MODEL_RUN_DATE }}-hourly-avg-t3.kerchunk.json \
            https://mnemosyne.somisana.ac.za/somisana/sw-cape/5-day-forecast/${{ env.MODEL_RUN_YEAR_MONTH }}/${{ inputs.MODEL_RUN_DATE }}_hourly_avg_t3.kerchunk.json;

  # Load raster data into postgis using raster2pgsql
  load-postgis:
    needs: [post-processing, envs, workdir]
    runs-on: postgres
    env:
      WORKDIR: ${{ needs.workdir.outputs.WORKDIR }}
    steps:
      - name: Get PG_HOST secret name
        id: _PG_HOST_
        run: |
          STRING=PG_HOST_${{ inputs.BRANCH_REF }}
          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
      - name: Get PG_PORT secret name
        id: _PG_PORT_
        run: |
          STRING=PG_PORT_${{ inputs.BRANCH_REF }}
          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
      - name: Get PG_DB secret name
        id: _PG_DB_
        run: |
          STRING=PG_DB_${{ inputs.BRANCH_REF }}
          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
      - name: Get PG_USERNAME secret name
        id: _PG_USERNAME_
        run: |
          STRING=PG_USERNAME_${{ inputs.BRANCH_REF }}
          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
      - name: Get PG_PASSWORD secret name
        id: _PG_PASSWORD_
        run: |
          STRING=PG_PASSWORD_${{ inputs.BRANCH_REF }}
          STRING_UPPERCASE=$(echo $STRING | tr '[:lower:]' '[:upper:]')
          echo "uppercase=$STRING_UPPERCASE" >> $GITHUB_OUTPUT
      - name: Load normalized NetCDF output to PostGIS
        env:
          PG_HOST: ${{ secrets[steps._PG_HOST_.outputs.uppercase] }}
          PG_PORT: ${{ secrets[steps._PG_PORT_.outputs.uppercase] }}
          PG_DB: ${{ secrets[steps._PG_DB_.outputs.uppercase] }}
          PG_USERNAME: ${{ secrets[steps._PG_USERNAME_.outputs.uppercase] }}
          PG_PASSWORD: ${{ secrets[steps._PG_PASSWORD_.outputs.uppercase] }}
        run: >-
          docker run \
            --rm \
            -v $WORKDIR:/tmp/somisana/current \
            -e PG_HOST=$PG_HOST \
            -e PG_PORT=$PG_PORT \
            -e PG_USERNAME=$PG_USERNAME \
            -e PG_PASSWORD=$PG_PASSWORD \
            -e PG_DB=$PG_DB \
            ${{ inputs.TOOLKIT_IMAGE }} \
              pg \
                load-croco-tier1-output-to-pg \
                  --input /tmp/somisana/current/croco/forecast/false-bay-forecast-${{ inputs.MODEL_RUN_DATE }}-tier1.nc \
                  --parallelization 32
      - name: Delete old PostgreSQL data
        continue-on-error: true
        env:
          PG_HOST: ${{ secrets[steps._PG_HOST_.outputs.uppercase] }}
          PG_PORT: ${{ secrets[steps._PG_PORT_.outputs.uppercase] }}
          PG_DB: ${{ secrets[steps._PG_DB_.outputs.uppercase] }}
          PG_USERNAME: ${{ secrets[steps._PG_USERNAME_.outputs.uppercase] }}
          PG_PASSWORD: ${{ secrets[steps._PG_PASSWORD_.outputs.uppercase] }}
        run: >-
          docker run \
            --rm \
            -e PG_HOST=$PG_HOST \
            -e PG_PORT=$PG_PORT \
            -e PG_USERNAME=$PG_USERNAME \
            -e PG_PASSWORD=$PG_PASSWORD \
            -e PG_DB=$PG_DB \
            ${{ inputs.TOOLKIT_IMAGE }} \
              pg \
                prune-values