Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prepare tools for icenet #60

Open
wants to merge 12 commits into
base: master
Choose a base branch
from
Open

Prepare tools for icenet #60

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
567453a
adding tool for preprocessing icenet data
vanessa-tamara Feb 10, 2023
189b627
added delta to size in bytes
vanessa-tamara Feb 10, 2023
1bac89d
added header for python files
vanessa-tamara Feb 14, 2023
b603ff4
changed preprocessing tool
vanessa-tamara Mar 4, 2023
40de7a7
preprocessing has no test data for now
vanessa-tamara Apr 5, 2023
82d17a1
updated preprocess tool (flake8 + xml + config file parameter)
vanessa-tamara Apr 10, 2023
a9255b5
icenet folder with icenet tools
vanessa-tamara Jun 16, 2023
9868ed6
created all of the tools needed for the icenet workflow
vanessa-tamara Sep 13, 2023
1385860
added createRequest with correct output filename
vanessa-tamara Sep 22, 2023
c779997
specify numpy format to binary
vanessa-tamara Sep 26, 2023
bde063a
mistake in output dir
vanessa-tamara Sep 28, 2023
5560540
removed hardcoded ensemble members
vanessa-tamara Oct 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
207 changes: 207 additions & 0 deletions tools/preprocess/2021_09_03_1300_icenet_demo.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,207 @@
{
"dataloader_name": "icenet_demo",
"dataset_name": "dataset1",
"input_data": {
"siconca": {
"abs": {
"include": true,
"max_lag": 12
},
"anom": {
"include": false,
"max_lag": 3
},
"linear_trend": {
"include": true
}
},
"tas": {
"abs": {
"include": false,
"max_lag": 3
},
"anom": {
"include": true,
"max_lag": 3
}
},
"ta500": {
"abs": {
"include": false,
"max_lag": 3
},
"anom": {
"include": true,
"max_lag": 3
}
},
"tos": {
"abs": {
"include": false,
"max_lag": 3
},
"anom": {
"include": true,
"max_lag": 3
}
},
"rsds": {
"abs": {
"include": false,
"max_lag": 3
},
"anom": {
"include": true,
"max_lag": 3
}
},
"rsus": {
"abs": {
"include": false,
"max_lag": 3
},
"anom": {
"include": true,
"max_lag": 3
}
},
"psl": {
"abs": {
"include": false,
"max_lag": 3
},
"anom": {
"include": true,
"max_lag": 3
}
},
"zg500": {
"abs": {
"include": false,
"max_lag": 3
},
"anom": {
"include": true,
"max_lag": 3
}
},
"zg250": {
"abs": {
"include": false,
"max_lag": 3
},
"anom": {
"include": true,
"max_lag": 3
}
},
"ua10": {
"abs": {
"include": true,
"max_lag": 3
},
"anom": {
"include": false,
"max_lag": 3
}
},
"uas": {
"abs": {
"include": true,
"max_lag": 1
},
"anom": {
"include": false,
"max_lag": 1
}
},
"vas": {
"abs": {
"include": true,
"max_lag": 1
},
"anom": {
"include": false,
"max_lag": 1
}
},
"land": {
"metadata": true,
"include": true
},
"circmonth": {
"metadata": true,
"include": true
}
},
"batch_size": 2,
"shuffle": true,
"n_forecast_months": 6,
"sample_IDs": {
"obs_train_dates": [
"1980-1-1",
"1980-2-1"
],
"obs_val_dates": [
"2012-1-1",
"2017-6-1"
],
"obs_test_dates": [
"2018-1-1",
"2019-6-1"
]
},
"cmip6_run_dict": {
"EC-Earth3": {
"r2i1p1f1": [
"1851-1-1",
"2099-6-1"
],
"r7i1p1f1": [
"1851-1-1",
"2099-6-1"
],
"r10i1p1f1": [
"1851-1-1",
"2099-6-1"
],
"r12i1p1f1": [
"1851-1-1",
"2099-6-1"
],
"r14i1p1f1": [
"1851-1-1",
"2099-6-1"
]
},
"MRI-ESM2-0": {
"r1i1p1f1": [
"1851-1-1",
"2099-6-1"
],
"r2i1p1f1": [
"1851-1-1",
"2029-6-1"
],
"r3i1p1f1": [
"1851-1-1",
"2029-6-1"
],
"r4i1p1f1": [
"1851-1-1",
"2029-6-1"
],
"r5i1p1f1": [
"1851-1-1",
"2029-6-1"
]
}
},
"raw_data_shape": [
432,
432
],
"default_seed": 42,
"loss_weight_months": true,
"verbose_level": 0
}
16 changes: 7 additions & 9 deletions tools/preprocess/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""
Code taken from https://github.com/tom-andersson/icenet-paper and slightly adjusted
to fit the galaxy interface.
Code taken from https://github.com/tom-andersson/icenet-paper and slightly adjusted
to fit the galaxy interface.
"""

import os
Expand All @@ -11,7 +11,7 @@
'''

###############################################################################
### Folder structure naming system
# Folder structure naming system
###############################################################################

data_folder = 'data'
Expand Down Expand Up @@ -39,7 +39,7 @@
region_mask_filename = 'region_mask.npy'

###############################################################################
### Polar hole/missing months
# Polar hole/missing months
###############################################################################

# Pre-defined polar hole radii (in number of 25km x 25km grid cells)
Expand All @@ -65,12 +65,10 @@
polarhole1_final_date = pd.Timestamp('1987-06-01') # 1987 June
polarhole2_final_date = pd.Timestamp('2005-10-01') # 2005 Oct
polarhole3_final_date = pd.Timestamp('2015-12-01') # 2015 Dec

missing_dates = [pd.Timestamp('1986-4-1'), pd.Timestamp('1986-5-1'),
pd.Timestamp('1986-6-1'), pd.Timestamp('1987-12-1')]
missing_dates = [pd.Timestamp('1986-4-1'), pd.Timestamp('1986-5-1'), pd.Timestamp('1986-6-1'), pd.Timestamp('1987-12-1')]

###############################################################################
### Weights and biases config (https://docs.wandb.ai/guides/track/advanced/environment-variables)
# Weights and biases config (https://docs.wandb.ai/guides/track/advanced/environment-variables)
###############################################################################

# Get API key from https://wandb.ai/authorize
Expand All @@ -83,7 +81,7 @@
WANDB_CACHE_DIR = '/path/to/wandb/cache/dir'

###############################################################################
### ECMWF details
# ECMWF details
###############################################################################

ECMWF_API_KEY = 'YOUR-KEY-HERE'
Expand Down
37 changes: 18 additions & 19 deletions tools/preprocess/gen_masks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
"""
Code taken from https://github.com/tom-andersson/icenet-paper and slightly adjusted
to fit the galaxy interface.
Code taken from https://github.com/tom-andersson/icenet-paper and slightly adjusted
to fit the galaxy interface.
"""

import sys
import os
sys.path.insert(0, os.path.join(os.getcwd(), 'icenet')) # if using jupyter kernel
import numpy as np
import xarray as xr
import os
import shutil
import config
import iris
import warnings
import matplotlib.pyplot as plt
import matplotlib as mpl
from mpl_toolkits.axes_grid1 import make_axes_locatable
sys.path.insert(0, os.path.join(os.getcwd(), 'icenet')) # if using jupyter kernel

'''
Obtains masks for land, the polar holes, OSI-SAF monthly maximum ice extent (the 'active
Expand All @@ -34,7 +33,7 @@
save_land_mask = True # Save the land mask (constant across months)
save_arctic_region_mask = True # Save Arctic region mask from NSIDC, with coastline cells
save_polarhole_masks = True # Save the polarhole masks
save_figures = False # Figures of the max extent/region masks
save_figures = False # Figures of the max extent/region masks

temp_ice_data_folder = os.path.join(config.mask_data_folder, 'temp')

Expand All @@ -45,8 +44,8 @@
' ftp://osisaf.met.no/reprocessed/ice/conc/v2p0/{:04d}/{:02d}/' + '{}'
filename_template_osi450 = 'ice_conc_nh_ease2-250_cdr-v2p0_{:04d}{:02d}021200.nc'

#### Generate the land-lake-sea mask using the second day from each month of
#### the year 2000 (chosen arbitrarily as the mask is fixed within a calendar month)
# Generate the land-lake-sea mask using the second day from each month of
# the year 2000 (chosen arbitrarily as the mask is fixed within a calendar month)
###############################################################################

print("Generating active grid cell region & and masks\n")
Expand All @@ -58,7 +57,7 @@
if not os.path.exists(fig_folder):
os.makedirs(fig_folder)

#### Active grid cell masks and land mask
# Active grid cell masks and land mask
###############################################################################

for month in range(1, 13):
Expand Down Expand Up @@ -111,11 +110,11 @@
# Delete the downloaded daily data
shutil.rmtree(temp_ice_data_folder)

#### Arctic region mask
# Arctic region mask
###############################################################################

if save_arctic_region_mask:
### Authors: Tony Phillips (BAS), Tom Andersson (BAS)
# Authors: Tony Phillips (BAS), Tom Andersson (BAS)

print("Generating NSIDC Arctic sea region array\n")

Expand Down Expand Up @@ -146,7 +145,7 @@
# ellipsoid -- see http://nsidc.org/data/polar-stereo/ps_grids.html
a = 6378273.0
e = 0.081816153
b = a * ((1.0 - e*e) ** 0.5)
b = a * ((1.0 - e * e) ** 0.5)
ellipsoid = iris.coord_systems.GeogCS(semi_major_axis=a, semi_minor_axis=b)

# coordinate system -- see ftp://sidads.colorado.edu/pub/tools/mapx/nsidc_maps/Nps.mpp
Expand All @@ -163,8 +162,8 @@
cy = 233.5

# derive X and Y coordinates of pixel centres -- Y reversed so it starts at the bottom-left
x = np.linspace(-cx, (nx-1)-cx, num=nx) * grid_length
y = np.linspace(cy-(ny-1), cy, num=ny) * grid_length
x = np.linspace(-cx, (nx - 1) - cx, num=nx) * grid_length
y = np.linspace(cy - (ny - 1), cy, num=ny) * grid_length

# read region data
region_file_path = os.path.join(config.mask_data_folder, 'region_n.msk')
Expand All @@ -187,7 +186,7 @@
# Save the mask as a Numpy array and remove the temporary files
arctic_region_mask = regions_ease.data.data

###### Extract the coastline properly using numpy
# Extract the coastline properly using numpy
land_mask_path = os.path.join(config.mask_data_folder, config.land_mask_filename)
land_mask = np.load(land_mask_path)

Expand Down Expand Up @@ -250,12 +249,12 @@
'Greenland Sea', 'Barents and Kara Seas', 'Arctic Ocean',
'Canadian Archipelago', 'Gulf of St. Lawrence', 'Land', 'Coast']
formatter = mpl.ticker.FuncFormatter(lambda val, loc: region_names[loc])
plt.colorbar(im, cax, ticks=[0,1,2,3,4,5,6,7,8,9,10,11,12,13], format=formatter)
plt.colorbar(im, cax, ticks=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13], format=formatter)
axes[0].set_title('Arctic region segmentation', fontsize=20)
getattr(mpl.cm, 'Blues_r').set_bad(color='black')
axes[1].imshow(sic_EASE_cube.data[0, :], cmap='Blues_r')
axes[1].contour(land_map, levels=[0.5], colors='k', linewidths=0.5)
axes[1].contour(regions_ease.data.data, levels=np.arange(0,13,1), colors='r', linewidths=2)
axes[1].contour(regions_ease.data.data, levels=np.arange(0, 13, 1), colors='r', linewidths=2)
axes[1].set_title('SIC day map with Arctic region segmentation', fontsize=20)
plt.savefig(os.path.join(fig_folder, 'all_regions.pdf'))
plt.savefig(os.path.join(fig_folder, 'all_regions.png'))
Expand All @@ -266,7 +265,7 @@

ax.contour(land_map, levels=[0.5], colors='k', linewidths=0.5)
ax.imshow(sic_EASE_cube.data[0, :], cmap='Blues_r')
ax.contour(regions_ease.data.data == i, levels=np.arange(0,13,1), colors='r', linewidths=2)
ax.contour(regions_ease.data.data == i, levels=np.arange(0, 13, 1), colors='r', linewidths=2)

plt.savefig(os.path.join(fig_folder, region_str + '.pdf'))
plt.savefig(os.path.join(fig_folder, region_str + '.png'))
Expand All @@ -275,14 +274,14 @@
os.remove(sic_day_fpath)
os.remove(region_file_path)

#### Polar hole masks
# Polar hole masks
###############################################################################

if save_polarhole_masks:

print("Generating polar hole masks\n")

#### Generate the polar hole masks
# Generate the polar hole masks
x = np.tile(np.arange(0, 432).reshape(432, 1), (1, 432)).astype(np.float32) - 215.5
y = np.tile(np.arange(0, 432).reshape(1, 432), (432, 1)).astype(np.float32) - 215.5
squaresum = np.square(x) + np.square(y)
Expand Down
Loading