New recipe and diagnostic for Arctic-midlatitude research

config-user_EG_levante.yml

@@ -0,0 +1,82 @@
+###############################################################################
+# User's configuration file for the ESMValTool
+###############################################################################
+---
+
+# Rootpaths to the data from different projects (lists are also possible)
+rootpath:
+#  CMIP3: /work/bd0854/DATA/ESMValTool2/CMIP3/
+  CMIP5: [/work/kd0956/CMIP5/data/cmip5/output1]
+  CMIP6: [/work/ik1017/CMIP6/data/CMIP6/]
+  DAMIP: [/work/ik1017/CMIP6/data/DAMIP/]
+  OBS: [/work/bd0854/DATA/ESMValTool2/OBS, /work/bd0854/b380971/output/revision/data_formatting_20230112_110337]
+#  OBS6: [/work/bd0854/b309137/data_ext/, /mnt/lustre02/work/bd0854/DATA/ESMValTool2/OBS]
+#  obs4MIPs: [/work/bd0854/b309137/data_ext/, /mnt/lustre02/work/bd0854/DATA/ESMValTool2/OBS]
+#  ana4mips: ~/ana4mips_inputpath
+#  native6:  ~/native6_inputpath
+  RAWOBS: /work/bd0854/b380971/data_temp/raw_obs/Tier2/HadISST #/work/bd0854/DATA/ESMValTool2/OBS
+  default: /work/bd0854/DATA/ESMValTool2/OBS
+  native6: [/work/bd0854/DATA/ESMValTool2/RAWOBS, /work/bd0854/b380971/data_temp/CDS/]
+
+# Directory structure for input data: [default]/BADC/DKRZ/ETHZ/etc
+# See config-developer.yml for definitions.
+drs:
+  CMIP3: DKRZ
+  CMIP5: DKRZ
+  CMIP6: DKRZ
+
+# Directory tree created by automatically downloading from ESGF
+#rootpath:
+#  CMIP3: ~/climate_data
+#  CMIP5: ~/climate_data
+#  CMIP6: /work/bd0854/b380971/data_temp/automatic_download
+#  CORDEX: ~/climate_data
+#  obs4MIPs: ~/climate_data
+#drs:
+#  CMIP3: ESGF
+#  CMIP5: ESGF
+#  CMIP6: ESGF
+#  CORDEX: ESGF
+#  obs4MIPs: ESGF
+
+# Run at most this many tasks in parallel [null]/1/2/3/4/..
+# Set to null to use the number of available CPUs.
+# If you run out of memory, try setting max_parallel_tasks to 1 and check the
+# amount of memory you need for that by inspecting the file
+# run/resource_usage.txt in the output directory. Using the number there you
+# can increase the number of parallel tasks again to a reasonable number for
+# the amount of memory available in your system.
+max_parallel_tasks: null
+
+# Destination directory
+output_dir: /work/bd0854/b380971/output/revision/
+
+# Directory for storing downloaded climate data
+download_dir: /work/bd0854/b380971/data_temp/CMIP6
+
+# Auxiliary data directory (used for some additional datasets)
+auxiliary_data_dir: ./auxiliary_data
+
+# Set the console log level debug, [info], warning, error
+log_level: info
+
+# Disable the automatic download of missing CMIP3, CMIP5, CMIP6, CORDEX,
+# and obs4MIPs data from ESGF by default. This is useful if you are working
+# on a computer without an internet connection.
+offline: true
+# Exit on warning (only for NCL diagnostic scripts)? true/[false]
+exit_on_warning: false
+# Plot file format? [png]/pdf/ps/eps/epsi
+output_file_type: png
+# Use netCDF compression true/[false]
+compress_netcdf: false
+# Save intermediary cubes in the preprocessor true/[false]
+save_intermediary_cubes: false
+# Remove the preproc dir if all fine
+remove_preproc_dir: false
+# Path to custom config-developer file, to customise project configurations.
+# See config-developer.yml for an example. Set to None to use the default
+# config_developer_file: null
+# Get profiling information for diagnostics
+# Only available for Python diagnostics
+profile_diagnostic: true

esmvaltool/diag_scripts/cvdp/cvdp/driver.ncl

@@ -2,68 +2,68 @@
 ; CVDP driver script. To run the CVDP at the command line type: ncl driver.ncl
 ; To run the CVDP at the command line, put it in background mode, and write the terminal output
 ; to a file named file.out, type: ncl driver.ncl >&! file.out &
-; 
+;
 ;============================================================================================
   outdir            = "/project/CVDP/"   ; location of output files   (must end in a "/")
-                                                           ; It is recommended that a new or empty directory be pointed to here 
+                                                           ; It is recommended that a new or empty directory be pointed to here
                                                            ; as existing files in outdir can get removed.
 
   namelists_only       = "False"             ; Set to True to only create the variable namelists. Useful
                                              ; upon running the package for the first time to verify that the correct
                                              ; files are being selected by the package. (See files in namelist_byvar/ directory)
-                                             ; Set to False to run the entire package. 
-                                             
+                                             ; Set to False to run the entire package.
+
   obs                  = "True"              ; True = analyze and plot observations (specified in namelist_obs), False = do not
   scale_timeseries     = "False"              ; True = scale timeseries so that x-axis length is comparable across timeseries, False = do not
   output_data          = "True"              ; True = output selected calculated data to a netCDF file. Make sure .nc files from previous CVDP
-                                             ;        runs are not in outdir or they will get added to or modified.  
-  compute_modes_mon    = "True"              ; True = compute DJF, MAM, JJA, SON, Annual and Monthly Atmospheric Modes of Variability 
-                                             ; False = do not compute the Monthly Atmospheric Modes of Variability  (saves computation time)                               
-;- - - - - - - - - - - - - - - - - -   
+                                             ;        runs are not in outdir or they will get added to or modified.
+  compute_modes_mon    = "True"              ; True = compute DJF, MAM, JJA, SON, Annual and Monthly Atmospheric Modes of Variability
+                                             ; False = do not compute the Monthly Atmospheric Modes of Variability  (saves computation time)
+;- - - - - - - - - - - - - - - - - -
   opt_climo         = "Full"                 ; Full  = remove climatology based on full record of each simulation,
-                                             ; Custom = set climatological period using climo_syear (climatological start year) and climo_eyear (climatological end year) 
+                                             ; Custom = set climatological period using climo_syear (climatological start year) and climo_eyear (climatological end year)
 
   if (opt_climo.eq."Custom") then            ; When climo_syear and climo_eyear are positive, remove the climatology/annual cycle based on these years.
-     climo_syear    = -30                    ;   Both settings should be within the range of years of all specified model runs and observational datasets. 
+     climo_syear    = -30                    ;   Both settings should be within the range of years of all specified model runs and observational datasets.
      climo_eyear    = 0                    ; When climo_syear is negative, remove the climatology/annual cycle relative to the end of each model run
   end if                                     ;   or observational dataset. Example: climo_syear = -25, climo_eyear = 0 will result in the climatology
-                                             ;   being removed from the last 26 years of each model run and observations. 
-;- - - - - - - - - - - - - - - - - -   
+                                             ;   being removed from the last 26 years of each model run and observations.
+;- - - - - - - - - - - - - - - - - -
   colormap          = 0               ; 0 = default colormaps, 1 = colormaps better for color blindness
 
-  output_type       = "png"           ; png = create png files, ps = create postscript files as well as png files (for web viewing).  
-  
-  png_scale         = 1.5             ; Set the output .png size. Value between .1->5.  Any value > 1 (< 1) increases (decreases) png size. 
+  output_type       = "png"           ; png = create png files, ps = create postscript files as well as png files (for web viewing).
+
+  png_scale         = 1.5             ; Set the output .png size. Value between .1->5.  Any value > 1 (< 1) increases (decreases) png size.
                                       ; When output_type = "png" a value of 1 will result in a png sized 1500 (H) x 1500 (W) before automatic cropping of white space
                                       ; When output_type = "ps"  a value of 1 will result in a png density setting of 144 before automatic cropping of white space
-  webpage_title     = "Title goes here"          ; Set webpage title   
-
-  tar_output        = "False"               ; True = tar up all output in outdir and remove individual files, False = do not                                      
-                                            ; Note: ALL files in outdir will be tarred up and then removed from the outdir directory. 
+  webpage_title     = "Title goes here"          ; Set webpage title
 
-;---Advanced Options----------------------------------------------------------------------                                             
+  tar_output        = "False"               ; True = tar up all output in outdir and remove individual files, False = do not
+                                            ; Note: ALL files in outdir will be tarred up and then removed from the outdir directory.
+
+;---Advanced Options----------------------------------------------------------------------
   zp = "ncl_scripts/"    ; directory path of CVDP NCL scripts. (must end in a "/")
                          ; Examples: "ncl_scripts/" if all code is local, or on CGD or CISL systems: "~asphilli/CESM-diagnostics/CVDP/Release/v4.1.0/ncl_scripts/"
-                         ; Regardless of this setting the following files should be in one directory: namelist, driver.ncl, and namelist_obs. 
+                         ; Regardless of this setting the following files should be in one directory: namelist, driver.ncl, and namelist_obs.
                          ; If pointing to code in ~asphilli make sure the driver script version #s match between this script and the script in ~asphilli.
-  
-  ncl_exec = "ncl"       ; This can be changed to a different path if a different version of NCL needs to be used, such as "/different/path/to/bin/ncl"                           
-                         
-  run_style = "parallel"   ; parallel = allow simple python-based parallelization to occur. X number of CVDP NCL scripts will be called at once. 
+
+  ncl_exec = "ncl"       ; This can be changed to a different path if a different version of NCL needs to be used, such as "/different/path/to/bin/ncl"
+
+  run_style = "parallel"   ; parallel = allow simple python-based parallelization to occur. X number of CVDP NCL scripts will be called at once.
                          ;            X is set via max_num_tasks. Terminal output will be harder to follow.
-                         ; serial = call CVDP NCL scripts serially. (Default)    
-                         
-  max_num_tasks = 4      ; if run_parallel = "parallel", how many CVDP NCL scripts can be called at once? (If unsure set to 3)     
-  
-  modular = "False"      ; True = Run only those CVDP scripts specified in modular_list.
+                         ; serial = call CVDP NCL scripts serially. (Default)
+
+  max_num_tasks = 4      ; if run_parallel = "parallel", how many CVDP NCL scripts can be called at once? (If unsure set to 3)
+
+  modular = "True"      ; True = Run only those CVDP scripts specified in modular_list.
                          ; False = Run all CVDP scripts (Default)
 
-  modular_list = "pdo,aice.trends_timeseries,sst.indices"  ; When modular = "True" list the CVDP scripts that will be run. 
+  modular_list = "sst.indices,psl.nam_nao"  ; When modular = "True" list the CVDP scripts that will be run.
                                            ; Example: modular_list = "amoc,amo,pr.trends_timeseries"
-                                           ; For a list of available scripts see complete_list at line 72. 
+                                           ; For a list of available scripts see complete_list at line 72.
 
   machine_casesen = "True"  ; True = Your filesystem is case sensitive  (Default)
-                             ; False = Your filesystem is case insensitive                                                          
+                             ; False = Your filesystem is case insensitive
 ;========END USER MODIFICATIONS===========================================================
   version = "5.0.0"
 
@@ -78,7 +78,7 @@
   do gg = 0,dimsizes(outfiles)-1
      if (isfilepresent2("obs_"+outfiles(gg))) then
         system("rm obs_"+outfiles(gg))
-     end if 
+     end if
   end do
 
   if (isfilepresent2(outdir+"metrics_orig.txt")) then    ; remove metrics_orig.txt file if present
@@ -88,7 +88,7 @@
   if (opt_climo.eq."Custom") then
      if (climo_syear.ge.climo_eyear) then
         print("Specified custom climatology start year (climo_syear) cannot be greater than or equal to the specified end year (climo_eyear), exiting CVDP.")
-        exit 
+        exit
      end if
   else
      climo_syear = -999
@@ -102,8 +102,8 @@
          "export OUTPUT_DATA="+output_data+";   export VERSION="+version+";   export PNG_SCALE="+png_scale+";   "+\
          "export OPT_CLIMO="+opt_climo+";   export CLIMO_SYEAR="+climo_syear+";   export CLIMO_EYEAR="+climo_eyear+";   "+\
          "export COMPUTE_MODES_MON="+compute_modes_mon+";   export OUTPUT_TYPE="+output_type+";   export MACHINE="+machine_casesen+";   "+\
-         "export COLORMAP="+colormap+";   export CVDP_SCRIPTS="+zp+";   export MAX_TASKS="+max_num_tasks+";"  
-  ncl_exec = ncl_exec+" -n -Q" 
+         "export COLORMAP="+colormap+";   export CVDP_SCRIPTS="+zp+";   export MAX_TASKS="+max_num_tasks+";"
+  ncl_exec = ncl_exec+" -n -Q"
 
   system(envvar_str + " "+str_sub_str(ncl_exec," -Q","")+" "+zp+"namelist.ncl")     ; create variable namelists
   if (namelists_only.eq."True") then
@@ -112,10 +112,10 @@
      do gg = 0,dimsizes(outfiles)-1
         if (isfilepresent2("obs_"+outfiles(gg))) then
            system("rm obs_"+outfiles(gg))
-        end if 
+        end if
      end do
      exit
-  end if   
+  end if
 
   if (modular.eq."True") then
      modular_list = str_sub_str(modular_list," ","")   ; remove spaces if present
@@ -127,25 +127,25 @@
      else
         modular_list = str_sub_str(modular_list,","," "+ncl_exec+" "+zp)
         modular_list = str_sub_str(modular_list,".ncl",".ncl;")
-        system(envvar_str+" "+ncl_exec+" "+zp+modular_list) 
+        system(envvar_str+" "+ncl_exec+" "+zp+modular_list)
      end if
   else
      complete_list = str_sub_str(complete_list,",",".ncl,")   ; add .ncl in to end of each script name
      complete_list = complete_list+".ncl"                     ; add .ncl in to last script name
      if (run_style.eq."parallel") then
         complete_list = str_sub_str(complete_list,","," "+zp)
-        system(envvar_str+" python "+zp+"runTasks.py "+zp+complete_list)		   
+        system(envvar_str+" python "+zp+"runTasks.py "+zp+complete_list)
      else
         complete_list = str_sub_str(complete_list,","," "+ncl_exec+" "+zp)
         complete_list = str_sub_str(complete_list,".ncl",".ncl;")
-        system(envvar_str+" "+ncl_exec+" "+zp+complete_list)                 
+        system(envvar_str+" "+ncl_exec+" "+zp+complete_list)
      end if
   end if
 
-  if (output_data.eq."True") then        
+  if (output_data.eq."True") then
      system(envvar_str + " "+ncl_exec+" "+zp+"ncfiles.append.ncl")
-  end if 
-      
+  end if
+
   if (output_type.eq."png") then
      ofiles = systemfunc("ls "+outdir+"*.png")
      do gg = 0,dimsizes(ofiles)-1
@@ -162,7 +162,7 @@
         end if
      end do
      delete([/filesize,ofilesS,ofilesT/])
-     
+
      ofiles = systemfunc("ls "+outdir+"*."+output_type)
      ofiles_png = str_sub_str(ofiles,"."+output_type,".png")
      d_opt = 144*png_scale
@@ -172,7 +172,7 @@
      end do
      print("Done with "+output_type+"->png conversion")
   end if
-  
+
   system("cp "+zp+"cas-cvdp.png "+outdir)
   system("cp namelist_byvar/* "+outdir)
   system("cp namelist "+outdir)
@@ -204,11 +204,11 @@
   do gg = 0,dimsizes(outfiles)-1
      if (isfilepresent2("obs_"+outfiles(gg))) then
         system("rm obs_"+outfiles(gg))
-     end if 
+     end if
   end do
 
   delete([/ofiles,outfiles,outdir,obs,scale_timeseries,output_data,opt_climo,climo_syear,climo_eyear,\
            png_scale,webpage_title,compute_modes_mon,met_files/])
 
   print("Finished: Climate Variability Diagnostics Package ("+systemfunc("date")+")")
-  
+