streamline plotting workflow #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: postprocess and plot | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| MODEL_TYPE: | ||
| description: 'what kind of model are we running e.g. oil - a user defined input in run_ops.yml' | ||
| required: true | ||
| type: string | ||
| BRANCH_REF: | ||
| description: 'what branch are we on - defined dynamically in run_ops.yml' | ||
| required: true | ||
| type: string | ||
| CONFIG_NAME: | ||
| description: 'configuration name - defined dynamically in run_ops.yml' | ||
| required: true | ||
| type: string | ||
| RUN_DATE: | ||
| description: 'time of T0 of the croco workflow in format YYYYMMDD_HH - defined dynamically in run_ops.yml' | ||
| required: true | ||
| type: string | ||
| RUNNER_NAME: | ||
| description: 'specify the runner name to determine what server we are running on' | ||
| required: true | ||
| type: string | ||
| WIND: | ||
| description: 'name of wind forcing e.g. GFS' | ||
| required: true | ||
| type: string | ||
| OGCM: | ||
| description: 'name of boundary forcing e.g. MERCATOR' | ||
| required: true | ||
| type: string | ||
| EXTENTS: | ||
| description: 'spatial extent of the gridded output and plot, in format lon0,lon1,lat0,lat1. If None, then this is dynamically determined from the geographic extent of the particles' | ||
| required: true | ||
| type: string | ||
| DX_M: | ||
| description: 'grid size (m) used to grid particle output. If None, If None, then a 50 x 50 regular grid is generated' | ||
| required: true | ||
| type: string | ||
| env: | ||
| # the directory to get mounted when running the docker image | ||
| DATE_DIR: /home/somisana/ops/${{ inputs.BRANCH_REF }}/${{ inputs.RUN_DATE }} | ||
| # path to the run directory, as seen inside the running docker container | ||
| RUN_DIR_DOCKER: /mnt/tmp/opendrift_${{ inputs.MODEL_TYPE }}/${{ inputs.CONFIG_NAME }}/${{ inputs.OGCM }}_${{ inputs.WIND }} | ||
| # path to the run directory, as seen inside the local server | ||
| RUN_DIR: /home/somisana/ops/${{ inputs.BRANCH_REF }}/${{ inputs.RUN_DATE }}/opendrift_${{ inputs.MODEL_TYPE }}/${{ inputs.CONFIG_NAME }}/${{ inputs.OGCM }}_${{ inputs.WIND }} | ||
| jobs: | ||
| gridding: | ||
| runs-on: ${{ inputs.RUNNER_NAME }} | ||
| steps: | ||
| - name: go from particle trajectories to a Eulerian grid of particle density | ||
| run: | | ||
| # we're specifying --user root due to permission issues I wasn't clever enough to solve | ||
| docker run \ | ||
| --rm \ | ||
| --user root \ | ||
| -v ${{ env.DATE_DIR }}:/mnt/tmp \ | ||
| ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \ | ||
| grid_particles \ | ||
| --config_dir ${{ env.RUN_DIR_DOCKER }} \ | ||
| --extents ${{ inputs.EXTENTS }} \ | ||
| --dx_m ${{ inputs.DX_M }} | ||
| plot_particles: | ||
| runs-on: ${{ inputs.RUNNER_NAME }} | ||
| steps: | ||
| - name: make animation of the particles | ||
| run: | | ||
| # we're specifying --user root due to permission issues I wasn't clever enough to solve | ||
| # animation of the particles | ||
| docker run \ | ||
| --rm \ | ||
| --user root \ | ||
| -v ${{ env.DATE_DIR }}:/mnt/tmp \ | ||
| ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \ | ||
| animate \ | ||
| --type particles \ | ||
| --config_dir ${{ env.RUN_DIR_DOCKER }} \ | ||
| --extents ${{ inputs.EXTENTS }} \ | ||
| --gif_out trajectories.gif | ||
| plot_gridded: | ||
| needs: [gridding] | ||
| runs-on: ${{ inputs.RUNNER_NAME }} | ||
| steps: | ||
| # animation of the gridded output | ||
| docker run \ | ||
| --rm \ | ||
| --user root \ | ||
| -v ${{ env.DATE_DIR }}:/mnt/tmp \ | ||
| ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \ | ||
| animate \ | ||
| --type gridded \ | ||
| --config_dir ${{ env.RUN_DIR_DOCKER }} \ | ||
| --extents ${{ inputs.EXTENTS }} \ | ||
| --gif_out gridded.gif | ||
| permissions: | ||
| needs: [gridding,plot_gridded,plot_particles] | ||
| runs-on: ${{ inputs.RUNNER_NAME }} | ||
| steps: | ||
| - name: set permissions of output | ||
| run: | | ||
| # specifying --user root means the outputs have root permissions so hard to delete them, hence this hack | ||
| docker run \ | ||
| --rm \ | ||
| --user root \ | ||
| --entrypoint /bin/bash \ | ||
| -v ${{ env.DATE_DIR }}:/mnt/tmp \ | ||
| ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \ | ||
| -c "chown -R $(id -u):$(id -g) ${{ env.RUN_DIR_DOCKER }}" | ||
| # copy data over to where it will be archived | ||
| archive: | ||
| needs: [gridding,gridding,permissions,plotting] | ||
| runs-on: ${{ inputs.RUNNER_NAME }} | ||
| env: | ||
| ARCHIVE_DIR: /mnt/saeon-somisana/data/opendrift/${{ inputs.RUN_DATE }}/${{ inputs.CONFIG_NAME }}/${{ inputs.OGCM }}-${{ inputs.WIND }} | ||
| steps: | ||
| - name: copy files to the archive directory if needed | ||
| run: | | ||
| # only copy if we're on the main branch | ||
| # we need to use sudo since the archive directory is owned by root | ||
| # we have set up the somisana user to be able to do this without the need for a password | ||
| if [ ${{ inputs.BRANCH_REF }} = "main" ]; then | ||
| if [ ! -d ${{ env.ARCHIVE_DIR }} ]; then | ||
| sudo mkdir -p ${{ env.ARCHIVE_DIR }} | ||
| sudo chmod -R 775 ${{ env.ARCHIVE_DIR }} | ||
| fi | ||
| sudo cp -f ${{ env.RUN_DIR }}/config.py ${{ env.ARCHIVE_DIR }} | ||
| sudo cp -f ${{ env.RUN_DIR }}/*.nc ${{ env.ARCHIVE_DIR }} | ||
| sudo cp -f ${{ env.RUN_DIR }}/*.gif ${{ env.ARCHIVE_DIR }} | ||
| fi | ||
