Prepare tools for icenet

tools/icenet/.shed.yml

@@ -0,0 +1,14 @@
+name: icenet
+owner: climate
+suite: 
+  name: "suite_icenet"
+  description: "A suite of icenet tools"
+  type: repository_suite_definition
+description: A collection of tools to create a forecast of the sea ice concentration with IceNet and visualize it
+homepage_url: https://zenodo.org/record/5176573
+long_description: |
+    IceNet is a probabilistic, deep learning sea ice forecasting system. It consists of an ensemble of U-Net networks, which learns how sea ice changes from climate simulations and observational data to forecast up to 6 months of monthly-averaged sea ice concentration maps at 25 km resolution. The forecast tool takes pretrained IceNet models downloaded from the Polar Data Centre and Observational Data generated with the "Preprocess Icenet Data Tool" as an input and creates one netcdf forecast file as an output. This forecast file can be used as an input for the "Visualize Icenet Forecast Tool" to plot an interactive figure.
+remote_repository_url: https://github.com/NordicESMhub/galaxy-tools/tree/master/tools/icenet
+type: unrestricted
+categories:
+- Climate Analysis

tools/icenet/2021_09_03_1300_icenet_config.json

@@ -0,0 +1,207 @@
+{
+  "dataloader_name": "icenet_demo",
+  "dataset_name": "dataset1",
+  "input_data": {
+    "siconca": {
+      "abs": {
+        "include": true,
+        "max_lag": 12
+      },
+      "anom": {
+        "include": false,
+        "max_lag": 3
+      },
+      "linear_trend": {
+        "include": true
+      }
+    },
+    "tas": {
+      "abs": {
+        "include": false,
+        "max_lag": 3
+      },
+      "anom": {
+        "include": true,
+        "max_lag": 3
+      }
+    },
+    "ta500": {
+      "abs": {
+        "include": false,
+        "max_lag": 3
+      },
+      "anom": {
+        "include": true,
+        "max_lag": 3
+      }
+    },
+    "tos": {
+      "abs": {
+        "include": false,
+        "max_lag": 3
+      },
+      "anom": {
+        "include": true,
+        "max_lag": 3
+      }
+    },
+    "rsds": {
+      "abs": {
+        "include": false,
+        "max_lag": 3
+      },
+      "anom": {
+        "include": true,
+        "max_lag": 3
+      }
+    },
+    "rsus": {
+      "abs": {
+        "include": false,
+        "max_lag": 3
+      },
+      "anom": {
+        "include": true,
+        "max_lag": 3
+      }
+    },
+    "psl": {
+      "abs": {
+        "include": false,
+        "max_lag": 3
+      },
+      "anom": {
+        "include": true,
+        "max_lag": 3
+      }
+    },
+    "zg500": {
+      "abs": {
+        "include": false,
+        "max_lag": 3
+      },
+      "anom": {
+        "include": true,
+        "max_lag": 3
+      }
+    },
+    "zg250": {
+      "abs": {
+        "include": false,
+        "max_lag": 3
+      },
+      "anom": {
+        "include": true,
+        "max_lag": 3
+      }
+    },
+    "ua10": {
+      "abs": {
+        "include": true,
+        "max_lag": 3
+      },
+      "anom": {
+        "include": false,
+        "max_lag": 3
+      }
+    },
+    "uas": {
+      "abs": {
+        "include": true,
+        "max_lag": 1
+      },
+      "anom": {
+        "include": false,
+        "max_lag": 1
+      }
+    },
+    "vas": {
+      "abs": {
+        "include": true,
+        "max_lag": 1
+      },
+      "anom": {
+        "include": false,
+        "max_lag": 1
+      }
+    },
+    "land": {
+      "metadata": true,
+      "include": true
+    },
+    "circmonth": {
+      "metadata": true,
+      "include": true
+    }
+  },
+  "batch_size": 2,
+  "shuffle": true,
+  "n_forecast_months": 6,
+  "sample_IDs": {
+    "obs_train_dates": [
+      "1980-1-1",
+      "2011-6-1"
+    ],
+    "obs_val_dates": [
+      "2012-1-1",
+      "2017-6-1"
+    ],
+    "obs_test_dates": [
+      "2018-1-1",
+      "2019-6-1"
+    ]
+  },
+  "cmip6_run_dict": {
+    "EC-Earth3": {
+      "r2i1p1f1": [
+        "1851-1-1",
+        "2099-6-1"
+      ],
+      "r7i1p1f1": [
+        "1851-1-1",
+        "2099-6-1"
+      ],
+      "r10i1p1f1": [
+        "1851-1-1",
+        "2099-6-1"
+      ],
+      "r12i1p1f1": [
+        "1851-1-1",
+        "2099-6-1"
+      ],
+      "r14i1p1f1": [
+        "1851-1-1",
+        "2099-6-1"
+      ]
+    },
+    "MRI-ESM2-0": {
+      "r1i1p1f1": [
+        "1851-1-1",
+        "2099-6-1"
+      ],
+      "r2i1p1f1": [
+        "1851-1-1",
+        "2029-6-1"
+      ],
+      "r3i1p1f1": [
+        "1851-1-1",
+        "2029-6-1"
+      ],
+      "r4i1p1f1": [
+        "1851-1-1",
+        "2029-6-1"
+      ],
+      "r5i1p1f1": [
+        "1851-1-1",
+        "2029-6-1"
+      ]
+    }
+  },
+  "raw_data_shape": [
+    432,
+    432
+  ],
+  "default_seed": 42,
+  "loss_weight_months": true,
+  "verbose_level": 0
+}

tools/icenet/concatenate.py

@@ -0,0 +1,17 @@
+import xarray as xr
+import argparse
+'''
+Concatenates monthly averaged OSI-SAF SIC data from 1979-present. The data was downloaded from the following sites:
+    - OSI-450 (1979-2016): https://thredds.met.no/thredds/dodsC/osisaf/met.no/reprocessed/ice/conc_v2p0_nh_agg.html
+    - OSI-430-b (2016-present): https://thredds.met.no/thredds/dodsC/osisaf/met.no/reprocessed/ice/conc_crb_nh_agg.html
+'''
+parser = argparse.ArgumentParser()
+parser.add_argument("-i", "--input", type=str, nargs="+", help="input files")
+args = parser.parse_args()
+
+pathlist = args.input
+
+concat_data = xr.open_mfdataset(pathlist, preprocess=lambda f: f["ice_conc"], data_vars=["ice_conc"])
+concat_data = concat_data.resample(time="1MS").mean(dim="time")
+
+concat_data.to_netcdf('siconca.nc')

tools/icenet/config.py

@@ -0,0 +1,89 @@
+"""
+Code taken from https://github.com/tom-andersson/icenet-paper and slightly adjusted
+to fit the galaxy interface.
+"""
+
+import os
+import pandas as pd
+'''
+Defines globals used throughout the codebase.
+'''
+
+###############################################################################
+# Folder structure naming system
+###############################################################################
+
+data_folder = 'data'
+obs_data_folder = os.path.join(data_folder, 'obs')
+cmip6_data_folder = os.path.join(data_folder, 'cmip6')
+mask_data_folder = os.path.join(data_folder, 'masks')
+forecast_data_folder = os.path.join(data_folder, 'forecasts')
+network_dataset_folder = os.path.join(data_folder, 'network_datasets')
+
+dataloader_config_folder = 'dataloader_configs'
+
+networks_folder = 'trained_networks'
+
+results_folder = 'results'
+forecast_results_folder = os.path.join(results_folder, 'forecast_results')
+permute_and_predict_results_folder = os.path.join(results_folder, 'permute_and_predict_results')
+uncertainty_results_folder = os.path.join(results_folder, 'uncertainty_results')
+
+figure_folder = 'figures'
+
+video_folder = 'videos'
+
+active_grid_cell_file_format = 'active_grid_cell_mask_{}.npy'
+land_mask_filename = 'land_mask.npy'
+region_mask_filename = 'region_mask.npy'
+
+###############################################################################
+# Polar hole/missing months
+###############################################################################
+
+# Pre-defined polar hole radii (in number of 25km x 25km grid cells)
+#   The polar hole radii were determined from Sections 2.1, 2.2, and 2.3 of
+#   http://osisaf.met.no/docs/osisaf_cdop3_ss2_pum_sea-ice-conc-climate-data-record_v2p0.pdf
+polarhole1_radius = 28
+polarhole2_radius = 11
+polarhole3_radius = 3
+
+# Whether or not to mask out the 3rd polar hole mask from
+# Nov 2005 to Dec 2015 with a radius of only 3 grid cells. Including it creates
+# some complications when analysing performance on a validation set that
+# overlaps with the 3rd polar hole period.
+use_polarhole3 = False
+
+polarhole1_fname = 'polarhole1_mask.npy'
+polarhole2_fname = 'polarhole2_mask.npy'
+polarhole3_fname = 'polarhole3_mask.npy'
+
+# Final month that each of the polar holes apply
+# NOTE: 1st of the month chosen arbitrarily throughout as always working wit
+#   monthly averages
+polarhole1_final_date = pd.Timestamp('1987-06-01')  # 1987 June
+polarhole2_final_date = pd.Timestamp('2005-10-01')  # 2005 Oct
+polarhole3_final_date = pd.Timestamp('2015-12-01')  # 2015 Dec
+
+missing_dates = [pd.Timestamp('1986-4-1'), pd.Timestamp('1986-5-1'),
+                 pd.Timestamp('1986-6-1'), pd.Timestamp('1987-12-1')]
+
+###############################################################################
+# Weights and biases config (https://docs.wandb.ai/guides/track/advanced/environment-variables)
+###############################################################################
+
+# Get API key from https://wandb.ai/authorize
+WANDB_API_KEY = 'YOUR-KEY-HERE'
+# Absolute path to store wandb generated files (folder must exist)
+#   Note: user must have write access
+WANDB_DIR = '/path/to/wandb/dir'
+# Absolute path to wandb config dir (
+WANDB_CONFIG_DIR = '/path/to/wandb/config/dir'
+WANDB_CACHE_DIR = '/path/to/wandb/cache/dir'
+
+###############################################################################
+# ECMWF details
+###############################################################################
+
+ECMWF_API_KEY = 'YOUR-KEY-HERE'
+ECMWF_API_EMAIL = 'YOUR-KEY-HERE'