add functions for combining multiple outputs, and add to operational …

…workflow

.github/workflows/combine_output.yml

@@ -0,0 +1,209 @@
+name: combine output
+# combine output from all simulations (combinations of forcing) into single files 
+# so the combined footprint can be viewed
+
+on:
+  workflow_call:
+    inputs:
+      MODEL_TYPE:
+        description: 'what kind of model are we running e.g. oil - a user defined input in run_ops.yml'
+        required: true
+        type: string
+      BRANCH_REF:
+        description: 'what branch are we on - defined dynamically in run_ops.yml'
+        required: true
+        type: string
+      CONFIG_NAME:
+        description: 'configuration name - defined dynamically in run_ops.yml'
+        required: true
+        type: string
+      RUN_DATE:
+        description: 'time of T0 of the croco workflow in format YYYYMMDD_HH - defined dynamically in run_ops.yml'
+        required: true
+        type: string
+      RUNNER_NAME:
+        description: 'specify the runner name to determine what server we are running on'
+        required: true
+        type: string
+      EXTENTS:
+        description: 'spatial extent of the gridded output and plot, in format lon0,lon1,lat0,lat1. If None, then this is dynamically determined from the geographic extent of the particles'
+        required: true
+        type: string
+
+env:
+  # path to the directory containing multiple run directories, as seen inside the local server 
+  DIR_WITH_DIRS: /home/somisana/ops/${{ inputs.BRANCH_REF }}/${{ inputs.RUN_DATE }}/opendrift_${{ inputs.MODEL_TYPE }}/${{ inputs.CONFIG_NAME }}
+
+jobs:
+  combine_particle_density:
+    runs-on: ${{ inputs.RUNNER_NAME }}
+    steps:
+      - name: combine gridded particle density files
+        run: |
+          docker run \
+            --rm \
+            --user root \
+            -v ${{ env.DIR_WITH_DIRS }}:/mnt/tmp \
+            ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \
+                    combine_gridded  \
+                      --dir_with_dirs /mnt/tmp \
+                      --fname_gridded gridded_particle_density.nc \
+                      --var_name particle_density
+
+      - name: animate the combined gridded particle density
+        run: |
+          docker run \
+            --rm \
+            --user root \
+            -v ${{ env.DIR_WITH_DIRS }}:/mnt/tmp \
+            ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \
+              animate  \
+                --type gridded  \
+                --config_dir /mnt/tmp/combined \
+                --fname_gridded gridded_particle_density.nc \
+                --extents ${{ inputs.EXTENTS }} \
+                --gif_out gridded_particle_density.gif
+
+      - name: plot maximum combined gridded particle density
+        run: |
+          docker run \
+            --rm \
+            --user root \
+            -v ${{ env.DIR_WITH_DIRS }}:/mnt/tmp \
+            ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \
+              animate  \
+                --type gridded_stats  \
+                --config_dir /mnt/tmp/combined \
+                --fname_gridded gridded_particle_density.nc \
+                --var_str maximum \
+                --cbar_label 'max. particle density (%)' \
+                --extents ${{ inputs.EXTENTS }} \
+                --jpg_out gridded_particle_density_max.jpg
+      
+      - name: plot minimum time to arrival
+        # this doesn't relate to particle density, but just sneaking it into this job
+        run: |
+          docker run \
+            --rm \
+            --user root \
+            -v ${{ env.DIR_WITH_DIRS }}:/mnt/tmp \
+            ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \
+              animate  \
+                --type gridded_stats  \
+                --config_dir /mnt/tmp/combined \
+                --fname_gridded gridded_particle_density.nc \
+                --var_str minimum_time \
+                --ticks 0,0.5,1,1.5,2,3,5,7 \
+                --cbar_label 'minimum time (days)' \
+                --extents ${{ inputs.EXTENTS }} \
+                --jpg_out gridded_min_time.jpg
+
+  combine_surface_oil:
+    runs-on: ${{ inputs.RUNNER_NAME }}
+    if: ${{ inputs.MODEL_TYPE == 'oil' }}
+    steps:
+      - name: combined surface oil thickness from particles (average over all runs)
+        run: |
+          docker run \
+            --rm \
+            --user root \
+            -v ${{ env.DIR_WITH_DIRS }}:/mnt/tmp \
+            ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \
+              combine_gridded  \
+                --dir_with_dirs /mnt/tmp \
+                --fname_gridded gridded_surface_oil.nc \
+                --var_name surface_thickness
+
+      - name: animate the combined surface oil thickness
+        run: |
+          docker run \
+            --rm \
+            --user root \
+            -v ${{ env.DIR_WITH_DIRS }}:/mnt/tmp \
+            ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \
+              animate  \
+                --type gridded  \
+                --config_dir /mnt/tmp/combined \
+                --fname_gridded gridded_surface_oil.nc \
+                --extents ${{ inputs.EXTENTS }} \
+                --var_str surface_thickness \
+                --ticks 0,0.01,0.1,1,10 \
+                --cbar_label 'surface oil thickness ($\mu$m)' \
+                --cmap Greys \
+                --gif_out gridded_surface_oil.gif
+
+      - name: plot maximum surface oil thickness
+        run: |
+          docker run \
+            --rm \
+            --user root \
+            -v ${{ env.DIR_WITH_DIRS }}:/mnt/tmp \
+            ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \
+              animate  \
+                --type gridded_stats  \
+                --config_dir /mnt/tmp/combined \
+                --fname_gridded gridded_surface_oil.nc \
+                --extents ${{ inputs.EXTENTS }} \
+                --var_str maximum \
+                --ticks 0,0.01,0.1,1,10 \
+                --cbar_label 'max. surface oil thickness ($\mu$m)' \
+                --cmap Greys \
+                --jpg_out gridded_surface_oil_max.jpg
+
+  combine_stranded_oil:
+    runs-on: ${{ inputs.RUNNER_NAME }}
+    if: ${{ inputs.MODEL_TYPE == 'oil' }}
+    steps:
+      - name: combined gridded stranded oil concentration (average over all runs)
+        run: |
+          docker run \
+            --rm \
+            --user root \
+            -v ${{ env.DIR_WITH_DIRS }}:/mnt/tmp \
+            ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \
+              combine_gridded  \
+                --dir_with_dirs /mnt/tmp \
+                --fname_gridded gridded_stranded_oil.nc \
+                --var_name stranded_oil
+
+      - name: plot maximum stranded oil concentration
+        run: |
+          docker run \
+            --rm \
+            --user root \
+            -v ${{ env.DIR_WITH_DIRS }}:/mnt/tmp \
+            ghcr.io/saeon/somisana-opendrift_${{ inputs.BRANCH_REF }}:latest \
+              animate  \
+                --type gridded_stats  \
+                --config_dir /mnt/tmp/combined \
+                --fname_gridded gridded_stranded_oil.nc \
+                --extents ${{ inputs.EXTENTS }} \
+                --var_str maximum \
+                --ticks 0,10,20,50,100,200,500,1000 \
+                --cbar_label 'max. stranded oil (g m$^{-2}$)' \
+                --cmap plasma \
+                --jpg_out gridded_stranded_oil_max.jpg
+  
+  # copy data over to where it will be archived
+  archive:
+    needs: [combine_particle_density,combine_surface_oil,combine_stranded_oil]
+    runs-on: ${{ inputs.RUNNER_NAME }}
+    env:
+      ARCHIVE_DIR: /mnt/saeon-somisana/data/opendrift/${{ inputs.RUN_DATE }}/${{ inputs.CONFIG_NAME }}/combined
+    steps:
+      - name: copy files to the archive directory if needed
+        run: |
+          # start by changing ownership of all files to the somisana user
+          sudo chown -R somisana:somisana ${{ env.DIR_WITH_DIRS }}/combined
+          # only copy if we're on the main branch
+          # we need to use sudo since the archive directory is owned by root
+          # we have set up the somisana user to be able to do this without the need for a password
+          if [ ${{ inputs.BRANCH_REF }} = "main" ]; then
+            if [ ! -d ${{ env.ARCHIVE_DIR }} ]; then
+              sudo mkdir -p ${{ env.ARCHIVE_DIR }}
+              sudo chmod -R 775 ${{ env.ARCHIVE_DIR }}
+            fi
+            sudo cp -f ${{ env.DIR_WITH_DIRS }}/combined/*.nc ${{ env.ARCHIVE_DIR }}
+            sudo cp -f ${{ env.DIR_WITH_DIRS }}/combined/*.gif ${{ env.ARCHIVE_DIR }}
+            sudo cp -f ${{ env.DIR_WITH_DIRS }}/combined/*.jpg ${{ env.ARCHIVE_DIR }}
+          fi

.github/workflows/postprocess.yml

@@ -244,8 +244,6 @@ jobs:
   # copy data over to where it will be archived
   archive:
     needs: [animate_particles,particle_density,surface_oil,stranded_oil,oil_mass_balance]
-    #plot_grid_oil_surface]
-    if: ${{ always() && github.event.inputs.do_post == 'true' }}
     runs-on: ${{ inputs.RUNNER_NAME }}
     env:
       ARCHIVE_DIR: /mnt/saeon-somisana/data/opendrift/${{ inputs.RUN_DATE }}/${{ inputs.CONFIG_NAME }}/${{ inputs.OGCM }}-${{ inputs.WIND }}

.github/workflows/run_ops.yml

@@ -116,4 +116,16 @@ jobs:
       EXTENTS: ${{ github.event.inputs.extents }}
       DX_M: ${{ github.event.inputs.dx_m }}
 
-  # get some combined output from runs with different forcings?
+  # get some combined output from runs with different forcings
+  # maybe we need a check to see if more than 1 run was actually successful?
+  combine_output:
+    needs: [envs,run_SAWS_MERCATOR,run_GFS_MERCATOR]
+    if: ${{ always() && ${{ github.event.inputs.do_post }} == 'true' }}
+    uses: ./.github/workflows/combine_output.yml  # Path to your reusable workflow
+    with:
+      MODEL_TYPE: ${{ github.event.inputs.model_type }}
+      BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }}
+      CONFIG_NAME: ${{ needs.envs.outputs.CONFIG_NAME }}
+      RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }}
+      RUNNER_NAME: mims1
+      EXTENTS: ${{ github.event.inputs.extents }}

cli.py

@@ -13,7 +13,7 @@
 from opendrift_tools.run import oil as run_oil
 from opendrift_tools.run import leeway as run_leeway
 from opendrift_tools.run import oceandrift as run_oceandrift
-from opendrift_tools.postprocess import grid_particles, oil_massbal
+from opendrift_tools.postprocess import grid_particles, oil_massbal, combine_gridded, combine_trajectories
 from opendrift_tools.plotting import plot_particles, plot_gridded, plot_gridded_stats, plot_budget
 from opendrift_tools.stochastic import run_stochastic, grid_stochastic, gridded_stats, gridded_stats_polygon, stochasitic_massbal
 
@@ -30,6 +30,12 @@ def parse_list(value):
     else:
         return [float(x.strip()) for x in value.split(',')]
 
+def parse_list_str(value):
+    if value is None or value == 'None':
+        return None
+    else:
+        return [x.strip() for x in value.split(',')]
+
 def parse_float(value):
     if value is None or value == 'None':
         return None
@@ -99,6 +105,36 @@ def grid_particles_handler(args):
                        max_only=args.max_only)
     parser_grid_particles.set_defaults(func=grid_particles_handler)
 
+    # ----------------------------------------------------
+    # combine multiple gridded outputs into a single file
+    # ----------------------------------------------------
+    parser_combine_gridded = subparsers.add_parser('combine_gridded', 
+            help='combine gridded output from multiple simulations into a single file')
+    parser_combine_gridded.add_argument('--dir_with_dirs', required=True, type=str, help='path to a directory containing multiple directories, where each directory contains opendrift output')
+    parser_combine_gridded.add_argument('--dirs', required=False, type=parse_list_str, default=None, help='a comma separated list of dir names inside dir_with_dirs which contain opendrift ouput. If None, then dirs is found dynamically from all dirs inside dir_with_dirs (except for the dirname \'combined\')')
+    parser_combine_gridded.add_argument('--fname_gridded', required=False, type=str, default='gridded.nc', help='the gridded filename to be found in every dir in dirs')
+    parser_combine_gridded.add_argument('--var_name', required=False, type=parse_str, default=None, help='the time-dependent variable in fname_gridded. If None, then only the time-independent \'maximum\' and \'minimum_time\' variables are used')
+    def combine_gridded_handler(args):
+        combine_gridded(args.dir_with_dirs,
+                       dirs=args.dirs,
+                       fname_gridded=args.fname_gridded,
+                       var_name=args.var_name)
+    parser_combine_gridded.set_defaults(func=combine_gridded_handler)
+
+    # ---------------------------------------------------------
+    # combine multiple trajectories outputs into a single file
+    # ---------------------------------------------------------
+    parser_combine_trajectories = subparsers.add_parser('combine_trajectories', 
+            help='combine raw opendrift output from multiple simulations into a single file')
+    parser_combine_trajectories.add_argument('--dir_with_dirs', required=True, type=str, help='path to a directory containing multiple directories, where each directory contains opendrift output')
+    parser_combine_trajectories.add_argument('--dirs', required=False, type=parse_list_str, default=None, help='a comma separated list of dir names inside dir_with_dirs which contain opendrift ouput. If None, then dirs is found dynamically from all dirs inside dir_with_dirs (except for the dirname \'combined\')')
+    parser_combine_trajectories.add_argument('--fname_traj', required=False, type=str, default='trajectories.nc', help='the output opendrift filename to be found in every dir in dirs')
+    def combine_trajectories_handler(args):
+        combine_trajectories(args.dir_with_dirs,
+                       dirs=args.dirs,
+                       fname_traj=args.fname_traj)
+    parser_combine_trajectories.set_defaults(func=combine_trajectories_handler)
+
     # -------------------------------------------
     # get the oil mass balance of an OpenOil run
     # -------------------------------------------