From e24264f046a5bb23d124e5ca47f7bf79032e399d Mon Sep 17 00:00:00 2001 From: Balwinder Date: Thu, 1 Jun 2017 20:49:47 -0400 Subject: [PATCH] Merge branch 'steveghan/atm/iediagnostics' (PR #1400) Add aerosol and cloud history for diagnosing aerosol effects on cloud... radiative forcing Add aerosol and cloud history for diagnosing relationships that drive aerosol effects on cloud radiative forcing. The additional fields are calculated if namelist option do_aerocom_ind3 =.true. and written every 3 hours if atm_in contains: mfilt = 1,8 nhtfrq = 0,-3 fincl2 = 'AODVIS:A', 'angstrm:A','cod:A', 'cdr:A', 'cdnc:A', 'cdnum:A', 'icnum:A', 'clt:A', 'lcc:A', 'lwp:A', 'iwp:A', 'icc:A', 'icnc:A', 'icr:A', 'LHFLX:A', 'SHFLX:A', 'OMEGA500:A', 'rh700:A', 'colrv:A', 'ccn:A', 'ccn.1bl:A', 'ccn.3bl:A', 'ptop:A', 'ttop:A', 'rwp:A','lwp2:A', 'iwp2:A', 'autoconv:A','accretn:A', 'FSUTOA_d1:A', 'FSUTOAC_d1:A', 'FSUTOA:A', 'FSUTOAC:A', 'FLUT:A', 'FLUTC:A','TH7001000', 'PRECL:A' rad_diag_1 = 'A:Q:H2O', 'N:O2:O2', 'N:CO2:CO2', 'A:O3:O3', 'N:N2O:N2O', 'N:CH4:CH4', 'N:CFC11:CFC11', 'N:CFC12:CFC12' [BFB] [NML] * steveghan/atm/iediagnostics: Modified "use abortutils" to "use cam_abortutils" mods to set do_aerocom_ind3 with namelist - Steve Ghan 4-11-2017 aerosol indirect effects diagnostics mods - Steve Ghan 4-11-2017 aerosol indirect effects diagnostics - Steve Ghan 4-11-2017 Conflicts: components/cam/src/physics/cam/phys_control.F90 --- CMakeLists.txt | 48 +- ChangeLog | 3829 +++++++++++++ LICENSE.TXT | 8 +- README | 31 - README.md | 29 +- README.unit_testing | 16 - .../acme/allactive/config_compsets.xml.cime2 | 2355 -------- cime_config/acme/allactive/config_pes.xml | 2223 -------- .../acme/allactive/config_pes.xml.cime2 | 1439 ----- .../testmods_dirs/cam/outfrq9s/shell_commands | 5 - cime_config/acme/machines/syslog.titan | 46 - .../allactive/cam/outfrq9s/xmlchange_cmnds | 5 - .../acme/allactive/config_compsets.xml | 48 + .../acme/allactive/config_pesall.xml | 462 +- .../acme/allactive/testlist_allactive.xml | 0 .../usermods_dirs/b1850/shell_commands | 0 .../b1850_cism/include_user_mods | 0 .../usermods_dirs/b1850_cism/user_nl_cism | 0 .../acme/config_archive.xml | 68 +- {cime_config => config}/acme/config_files.xml | 174 +- {cime_config => config}/acme/config_grids.xml | 25 + .../acme/machines/Depends.cetus | 0 .../acme/machines/Depends.corip1 | 0 .../acme/machines/Depends.cray | 0 .../acme/machines/Depends.gnu | 0 .../acme/machines/Depends.intel | 0 .../acme/machines/Depends.intel14 | 0 .../acme/machines/Depends.intelmic | 0 .../acme/machines/Depends.intelmic14 | 0 .../acme/machines/Depends.mira | 0 .../acme/machines/Depends.nag | 0 .../acme/machines/Depends.titan.cray | 0 .../acme/machines/Depends.titan.intel | 0 .../acme/machines/Depends.titan.pgi | 0 .../acme/machines/Depends.titan.pgiacc | 0 .../acme/machines/Makefile | 53 +- {cime_config => config}/acme/machines/README | 0 .../acme/machines/config_batch.xml | 90 +- .../acme/machines/config_batch.xsd | 0 .../acme/machines/config_compilers.xml | 178 +- .../acme/machines/config_lt_archive.xml | 0 .../acme/machines/config_machines.xml | 688 ++- .../acme/machines/config_pio.xml | 6 + .../acme/machines/mkDepends | 0 .../acme/machines/mkSrcfiles | 0 .../acme/machines/syslog.anvil | 5 +- .../acme/machines/syslog.cetus | 28 +- .../acme/machines/syslog.cori-haswell | 42 +- .../acme/machines/syslog.cori-knl | 42 +- .../acme/machines/syslog.edison | 43 +- .../acme/machines/syslog.mira | 28 +- .../acme/machines/syslog.noop | 0 config/acme/machines/syslog.titan | 58 + .../acme/machines/template.case.run | 4 +- .../acme/machines/template.case.test | 4 +- .../acme}/machines/template.lt_archive | 6 +- .../acme/machines/template.ocn_diagnostics | 0 .../acme}/machines/template.st_archive | 33 +- .../acme/machines/template.timeseries | 0 .../userdefined_laptop_template/README.md | 0 .../config_compilers.xml | 4 +- .../config_machines.xml | 0 .../config_pes.xml | 0 .../allactive/cam/cosplite/user_nl_cam | 1 + .../allactive/cam/cosplite/xmlchange_cmnds | 3 + .../cam/cosplite_nhtfrq5/user_nl_cam | 3 + .../cam/cosplite_nhtfrq5/xmlchange_cmnds | 3 + .../allactive/cam/outfrq9s/user_nl_cam | 0 .../allactive/cam/outfrq9s/user_nl_clm | 0 .../allactive/cam/outfrq9s/xmlchange_cmnds | 1 + .../allactive/cam/rtm_null/xmlchange_cmnds | 1 + .../allactive/clm/betr/xmlchange_cmnds | 2 + .../allactive/clm/eca/user_nl_clm | 2 + .../allactive/clm/fates/user_nl_clm | 11 + .../allactive/force_netcdf_pio/shell_commands | 0 .../force_netcdf_pio/xmlchange_cmnds | 0 .../cesm/config_archive.xml | 0 {cime_config => config}/cesm/config_files.xml | 164 +- {cime_config => config}/cesm/config_grids.xml | 507 +- .../cesm/machines/Depends.babbageKnc | 0 .../cesm/machines/Depends.bluewaters | 0 .../cesm/machines/Depends.corip1 | 0 .../cesm/machines/Depends.cray | 0 .../cesm/machines/Depends.gnu | 0 .../cesm/machines/Depends.goldbach | 0 .../cesm/machines/Depends.intel | 0 .../cesm/machines/Depends.intel14 | 0 .../cesm/machines/Depends.intelmic | 0 .../cesm/machines/Depends.intelmic14 | 0 .../cesm/machines/Depends.mira | 0 .../cesm/machines/Depends.nag | 0 .../cesm/machines/Makefile | 80 +- {cime_config => config}/cesm/machines/README | 0 .../cesm/machines/config_batch.xml | 2 +- .../cesm/machines/config_compilers.xml | 42 +- .../cesm/machines/config_lt_archive.xml | 0 .../cesm/machines/config_machines.xml | 51 +- .../cesm/machines/config_pio.xml | 10 +- .../cesm/machines/mkDepends | 0 .../cesm/machines/mkSrcfiles | 0 .../cesm/machines/nag_mpi_argument.txt | 0 .../cesm/machines/template.case.run | 4 +- .../cesm/machines/template.case.test | 4 +- .../cesm}/machines/template.lt_archive | 6 +- .../cesm}/machines/template.st_archive | 13 +- .../userdefined_laptop_template/README.md | 0 .../config_compilers.xml | 4 +- .../config_machines.xml | 0 .../config_pes.xml | 0 {cime_config => config}/config_headers.xml | 13 +- {cime_config => config}/config_tests.xml | 71 + .../xml_schemas/cimeteststatus.xsd | 0 .../xml_schemas/config_archive.xsd | 0 .../xml_schemas/config_batch.xsd | 3 +- .../xml_schemas/config_compilers_v2.xsd | 0 .../xml_schemas/config_compsets.xsd | 2 + .../xml_schemas/config_grids_v2.xsd | 8 +- .../xml_schemas/config_machines.xsd | 3 + .../xml_schemas/config_machines_template.xml | 0 .../xml_schemas/config_pes.xsd | 0 .../xml_schemas/entry_id.xsd | 3 + .../xml_schemas/entry_id_base.xsd | 3 +- .../xml_schemas/entry_id_namelist.xsd | 2 + config/xml_schemas/env_archive.xsd | 19 + config/xml_schemas/env_entry_id.xsd | 47 + .../xml_schemas/testlist.xsd | 0 externals/mct/mpi-serial/NOTES | 46 - externals/mct/mpi-serial/mpif.h | 334 -- externals/pio1/timing/COPYING | 674 --- externals/pio2/CMakeLists.txt | 128 - externals/pio2/ctest/runcdash-nersc-cray.sh | 38 - externals/pio2/ctest/runcdash-nersc-intel.sh | 38 - externals/pio2/doc/CMakeLists.txt | 21 - externals/pio2/examples/c/CMakeLists.txt | 51 - externals/pio2/src/clib/CMakeLists.txt | 211 - externals/pio2/src/clib/bget.c | 1672 ------ externals/pio2/src/clib/bget.h | 36 - externals/pio2/src/clib/pio.h | 609 -- externals/pio2/src/clib/pio_darray.c | 1787 ------ externals/pio2/src/clib/pio_file.c | 461 -- externals/pio2/src/clib/pio_get_nc.c | 4988 ---------------- externals/pio2/src/clib/pio_internal.h | 371 -- externals/pio2/src/clib/pio_lists.c | 221 - externals/pio2/src/clib/pio_nc.c | 4404 -------------- externals/pio2/src/clib/pio_nc4.c | 1078 ---- externals/pio2/src/clib/pio_put_nc.c | 5073 ----------------- externals/pio2/src/clib/pio_rearrange.c | 1692 ------ externals/pio2/src/clib/pio_spmd.c | 650 --- externals/pio2/src/clib/pioc.c | 599 -- externals/pio2/src/clib/pioc_sc.c | 444 -- externals/pio2/src/clib/pioc_support.c | 493 -- externals/pio2/src/clib/topology.c | 363 -- externals/pio2/src/flib/CMakeLists.txt | 269 - externals/pio2/src/gptl/CMakeLists.txt | 156 - externals/pio2/src/gptl/COPYING | 674 --- externals/pio2/tests/general/CMakeLists.txt | 327 -- .../pio2/tests/general/ncdf_get_put.F90.in | 120 - .../tests/general/pio_decomp_tests.F90.in | 251 - externals/pio2/tests/unit/CMakeLists.txt | 91 - scripts/Tools/acme_check_env | 4 +- scripts/Tools/bless_test_results | 4 +- scripts/Tools/case.build | 20 +- scripts/Tools/case.cmpgen_namelists | 4 +- scripts/Tools/case.setup | 7 +- scripts/Tools/case.submit | 4 +- scripts/Tools/case_diff | 4 +- scripts/Tools/check_case | 5 +- scripts/Tools/check_input_data | 11 +- scripts/Tools/check_lockedfiles | 4 +- scripts/Tools/cime_bisect | 6 +- scripts/Tools/code_checker | 4 +- scripts/Tools/compare_namelists | 4 +- scripts/Tools/compare_test_results | 4 +- scripts/Tools/component_compare_baseline | 4 +- scripts/Tools/component_compare_copy | 4 +- scripts/Tools/component_compare_test | 4 +- scripts/Tools/component_generate_baseline | 4 +- scripts/Tools/cron_script | 14 - scripts/Tools/cs.status | 83 + scripts/Tools/cs_status | 60 - scripts/Tools/getTiming | 4 +- scripts/Tools/jenkins_generic_job | 52 +- scripts/Tools/jenkins_script | 15 +- scripts/Tools/list_acme_tests | 4 +- scripts/Tools/normalize_cases | 4 +- scripts/Tools/pelayout | 4 +- scripts/Tools/preview_namelists | 40 +- scripts/Tools/preview_run | 69 + scripts/Tools/save_provenance | 4 +- scripts/Tools/simple_compare | 4 +- scripts/Tools/standard_script_setup.py | 2 +- scripts/Tools/testreporter.py | 228 + scripts/Tools/update_acme_tests | 10 +- scripts/Tools/wait_for_tests | 4 +- scripts/Tools/xmlchange | 41 +- .../xmlconvertors/config_pes_converter.py | 10 +- .../Tools/xmlconvertors/grid_xml_converter.py | 10 +- scripts/Tools/xmlquery | 38 +- scripts/create_clone | 6 +- scripts/create_newcase | 45 +- scripts/create_test | 28 +- .../data_assimilation/da_no_data_mod.sh | 0 .../Examples/circle_area/src/CMakeLists.txt | 0 .../Examples/circle_area/src/circle.F90 | 0 .../Examples/circle_area/tests/CMakeLists.txt | 4 +- .../circle_area/tests/CTest/CMakeLists.txt | 0 .../circle_area/tests/CTest/test_driver.F90 | 0 .../circle_area/tests/pFUnit/CMakeLists.txt | 0 .../circle_area/tests/pFUnit/test_circle.pf | 0 .../interpolate_1d/src/CMakeLists.txt | 0 .../interpolate_1d/src/interpolate_1d.F90 | 0 .../interpolate_1d/tests/CMakeLists.txt | 5 +- .../interpolate_1d/tests/CTest/CMakeLists.txt | 0 .../tests/CTest/test_driver.F90 | 0 .../tests/pFUnit/CMakeLists.txt | 0 .../tests/pFUnit/test_interpolate_point.pf | 0 .../Examples/test_list.xml | 0 .../fortran_unit_testing}/README | 64 +- .../fortran_unit_testing}/python/.gitignore | 0 .../fortran_unit_testing}/python/printer.py | 0 .../python/test_xml_test_list.py | 5 +- .../python/xml_test_list.py | 5 +- .../fortran_unit_testing}/run_tests.py | 150 +- .../lib}/CIME/BuildTools/__init__.py | 0 .../lib}/CIME/BuildTools/cmakemacroswriter.py | 22 +- .../lib}/CIME/BuildTools/configure.py | 12 +- .../CIME/BuildTools/macroconditiontree.py | 0 .../lib}/CIME/BuildTools/macrowriterbase.py | 0 .../lib}/CIME/BuildTools/makemacroswriter.py | 0 .../lib}/CIME/BuildTools/possiblevalues.py | 0 .../lib}/CIME/BuildTools/valuesetting.py | 0 .../lib}/CIME/SystemTests/README | 0 .../lib}/CIME/SystemTests/__init__.py | 0 .../lib}/CIME/SystemTests/dae.py | 11 +- .../lib}/CIME/SystemTests/eri.py | 0 .../lib}/CIME/SystemTests/erio.py | 0 .../lib}/CIME/SystemTests/erp.py | 4 +- .../lib}/CIME/SystemTests/err.py | 0 scripts/lib/CIME/SystemTests/erri.py | 34 + .../lib}/CIME/SystemTests/ers.py | 0 .../lib}/CIME/SystemTests/ert.py | 0 .../lib}/CIME/SystemTests/homme.py | 29 +- .../lib}/CIME/SystemTests/icp.py | 0 .../lib}/CIME/SystemTests/lii.py | 0 .../lib}/CIME/SystemTests/nck.py | 6 +- .../lib}/CIME/SystemTests/ncr.py | 0 scripts/lib/CIME/SystemTests/nodefail.py | 76 + .../lib}/CIME/SystemTests/pea.py | 0 .../lib}/CIME/SystemTests/pem.py | 4 +- .../lib}/CIME/SystemTests/pet.py | 6 +- .../lib}/CIME/SystemTests/pfs.py | 0 .../lib}/CIME/SystemTests/pre.py | 71 +- .../lib}/CIME/SystemTests/rep.py | 0 .../lib}/CIME/SystemTests/seq.py | 22 +- .../lib}/CIME/SystemTests/sms.py | 0 .../lib}/CIME/SystemTests/ssp.py | 2 + .../CIME/SystemTests/system_tests_common.py | 93 +- .../SystemTests/system_tests_compare_two.py | 6 + .../CIME/SystemTests/test_utils/__init__.py | 0 .../SystemTests/test_utils/user_nl_utils.py | 0 .../lib}/CIME/XML/__init__.py | 0 .../lib}/CIME/XML/archive.py | 13 +- .../python => scripts/lib}/CIME/XML/batch.py | 0 .../lib}/CIME/XML/compilerblock.py | 0 .../lib}/CIME/XML/compilers.py | 0 .../lib}/CIME/XML/component.py | 0 .../lib}/CIME/XML/compsets.py | 13 +- .../lib}/CIME/XML/entry_id.py | 44 +- .../lib}/CIME/XML/env_archive.py | 5 +- .../lib}/CIME/XML/env_base.py | 20 +- .../lib}/CIME/XML/env_batch.py | 256 +- .../lib}/CIME/XML/env_build.py | 3 +- .../lib}/CIME/XML/env_case.py | 3 +- .../lib}/CIME/XML/env_mach_pes.py | 10 +- .../lib}/CIME/XML/env_mach_specific.py | 64 +- .../lib}/CIME/XML/env_run.py | 4 +- .../lib}/CIME/XML/env_test.py | 19 + .../python => scripts/lib}/CIME/XML/files.py | 7 +- .../lib}/CIME/XML/generic_xml.py | 6 +- .../python => scripts/lib}/CIME/XML/grids.py | 80 +- .../lib}/CIME/XML/headers.py | 2 +- .../lib}/CIME/XML/lt_archive.py | 0 .../lib}/CIME/XML/machines.py | 35 +- .../lib}/CIME/XML/namelist_definition.py | 21 +- {utils/python => scripts/lib}/CIME/XML/pes.py | 0 {utils/python => scripts/lib}/CIME/XML/pio.py | 0 .../lib}/CIME/XML/standard_module_setup.py | 2 +- scripts/lib/CIME/XML/test_reporter.py | 113 + .../lib}/CIME/XML/testlist.py | 0 .../python => scripts/lib}/CIME/XML/tests.py | 0 .../lib}/CIME/XML/testspec.py | 0 .../python => scripts/lib}/CIME/__init__.py | 0 {utils/python => scripts/lib}/CIME/aprun.py | 29 +- .../lib}/CIME/bless_test_results.py | 2 +- {utils/python => scripts/lib}/CIME/build.py | 453 +- .../python => scripts/lib}/CIME/buildlib.py | 16 +- .../python => scripts/lib}/CIME/buildnml.py | 7 +- {utils/python => scripts/lib}/CIME/case.py | 286 +- .../lib}/CIME/case_cmpgen_namelists.py | 5 +- .../lib}/CIME/case_lt_archive.py | 18 +- .../python => scripts/lib}/CIME/case_run.py | 141 +- .../python => scripts/lib}/CIME/case_setup.py | 76 +- .../lib}/CIME/case_st_archive.py | 138 +- .../lib}/CIME/case_submit.py | 33 +- .../python => scripts/lib}/CIME/case_test.py | 8 +- .../lib}/CIME/check_input_data.py | 7 + .../lib}/CIME/check_lockedfiles.py | 9 +- .../lib}/CIME/code_checker.py | 9 +- .../lib}/CIME/compare_namelists.py | 0 .../lib}/CIME/compare_test_results.py | 40 +- .../python => scripts/lib}/CIME/get_timing.py | 12 +- .../python => scripts/lib}/CIME/hist_utils.py | 21 +- .../python => scripts/lib}/CIME/namelist.py | 460 +- {utils/python => scripts/lib}/CIME/nmlgen.py | 34 +- .../lib}/CIME/preview_namelists.py | 73 +- .../python => scripts/lib}/CIME/provenance.py | 190 +- .../lib}/CIME/simple_compare.py | 0 .../lib}/CIME/test_scheduler.py | 134 +- .../lib}/CIME/test_status.py | 26 +- .../python => scripts/lib}/CIME/test_utils.py | 0 .../python => scripts/lib}/CIME/tests/README | 0 .../lib}/CIME/tests/SystemTests/__init__.py | 0 .../test_system_tests_compare_two.py | 0 ..._tests_compare_two_link_to_case2_output.py | 0 .../tests/SystemTests/test_utils/__init__.py | 0 .../test_utils/test_user_nl_utils.py | 0 .../lib}/CIME/tests/__init__.py | 0 .../lib}/CIME/tests/case_fake.py | 0 .../lib}/CIME/tests/test_case_fake.py | 0 .../lib/CIME/tests/test_user_mod_support.py | 171 + .../lib}/CIME/user_mod_support.py | 47 +- {utils/python => scripts/lib}/CIME/utils.py | 214 +- .../lib}/CIME/wait_for_tests.py | 2 +- scripts/lib/cs.status.template | 3 + .../python => scripts/lib}/cs.submit.template | 0 .../lib}/jenkins_generic_job.py | 49 +- scripts/lib/testreporter.template | 3 + .../lib}/update_acme_tests.py | 18 +- scripts/manage_case | 7 +- scripts/manage_pes | 4 +- scripts/manage_testlists | 10 +- scripts/query_testlists | 8 +- .../python => scripts}/tests/CMakeLists.txt | 0 .../tests/CTestConfig.cmake | 0 .../python => scripts}/tests/cpl.hi1.nc.test | Bin .../python => scripts}/tests/cpl.hi2.nc.test | Bin .../tests/cpl.log.failmemleak.gz | Bin .../tests/cpl.log.passmemleak.gz | Bin {utils/python => scripts}/tests/list_tests | 0 .../tests/scripts_regression_tests.py | 283 +- .../tests/user_mods_test1/include_user_mods | 0 .../tests/user_mods_test1/user_nl_comp | 0 .../tests/user_mods_test1/user_nl_cpl | 0 .../SourceMods/src.drv/somefile.F90 | 0 .../tests/user_mods_test2/user_nl_cpl | 0 share/csm_share/include/shr_assert.h | 10 - .../test/unit/shr_assert_test/test_macro.pf | 52 - .../test/unit/shr_assert_test/test_ndebug.pf | 45 - share/timing/COPYING | 674 --- .../build_scripts}/buildlib.csm_share | 18 +- .../build_scripts}/buildlib.gptl | 6 +- .../build_scripts}/buildlib.mct | 2 +- .../build_scripts}/buildlib.mpi-serial | 2 +- .../build_scripts}/buildlib.pio | 21 +- .../data_comps/datm/atm_comp_mct.F90 | 0 .../data_comps/datm/cime_config/buildlib | 2 +- .../data_comps/datm/cime_config/buildnml | 9 +- .../datm/cime_config/config_archive.xml | 0 .../datm/cime_config/config_component.xml | 29 +- .../cime_config/namelist_definition_datm.xml | 206 +- .../data_comps/datm/cime_config/user_nl_datm | 0 .../data_comps/datm/datm_comp_mod.F90 | 26 - .../data_comps/datm/datm_shr_mod.F90 | 0 .../components}/data_comps/datm/doc/SOM.pdf | Bin .../data_comps/datm/doc/bookinfo.xml | 0 .../data_comps/datm/doc/datacomps.xml | 0 .../components}/data_comps/datm/doc/intro.xml | 0 .../data_comps/datm/doc/rundocbook.csh | 0 .../data_comps/datm/doc/streams.xml | 2 +- .../components}/data_comps/datm/doc/ug.xml | 0 .../data_comps/desp/cime_config/buildlib | 2 +- .../data_comps/desp/cime_config/buildnml | 11 +- .../desp/cime_config/config_component.xml | 0 .../cime_config/namelist_definition_desp.xml | 0 .../data_comps/desp/cime_config/user_nl_desp | 0 .../data_comps/desp/desp_comp_mod.F90 | 109 +- .../data_comps/desp/esp_comp_mct.F90 | 46 +- .../components}/data_comps/desp/esp_utils.F90 | 0 .../data_comps/dice/cime_config/buildlib | 2 +- .../data_comps/dice/cime_config/buildnml | 3 +- .../dice/cime_config/config_archive.xml | 0 .../dice/cime_config/config_component.xml | 2 - .../cime_config/namelist_definition_dice.xml | 33 +- .../data_comps/dice/cime_config/user_nl_dice | 0 .../data_comps/dice/dice_comp_mod.F90 | 1 - .../data_comps/dice/ice_comp_mct.F90 | 0 .../data_comps/dlnd/cime_config/buildlib | 2 +- .../data_comps/dlnd/cime_config/buildnml | 8 +- .../dlnd/cime_config/config_archive.xml | 0 .../dlnd/cime_config/config_component.xml | 50 +- .../cime_config/namelist_definition_dlnd.xml | 0 .../data_comps/dlnd/cime_config/user_nl_dlnd | 0 .../data_comps/dlnd/dlnd_comp_mod.F90 | 0 .../data_comps/dlnd/lnd_comp_mct.F90 | 0 .../data_comps/docn/cime_config/buildlib | 2 +- .../data_comps/docn/cime_config/buildnml | 12 +- .../docn/cime_config/config_archive.xml | 0 .../docn/cime_config/config_component.xml | 100 +- .../cime_config/namelist_definition_docn.xml | 58 +- .../data_comps/docn/cime_config/user_nl_docn | 0 .../data_comps/docn/docn_comp_mod.F90 | 347 +- .../data_comps/docn/ocn_comp_mct.F90 | 0 .../data_comps/docn/tools/pop_som_frc/README | 0 .../data_comps/docn/tools/pop_som_frc/SOM.doc | Bin .../data_comps/docn/tools/pop_som_frc/SOM.pdf | Bin .../docn/tools/pop_som_frc/pop_frc.csh | 0 .../docn/tools/pop_som_frc/pop_frc_mlann.ncl | 0 .../docn/tools/pop_som_frc/pop_frc_mlt.ncl | 0 .../docn/tools/pop_som_frc/pop_interp.ncl | 0 .../docn/tools/pop_som_frc/read_from_mss.csh | 0 .../tools/pop_som_frc/read_from_mss_month.csh | 0 .../data_comps/drof/cime_config/buildlib | 2 +- .../data_comps/drof/cime_config/buildnml | 3 +- .../drof/cime_config/config_archive.xml | 0 .../drof/cime_config/config_component.xml | 0 .../cime_config/namelist_definition_drof.xml | 17 +- .../data_comps/drof/cime_config/user_nl_drof | 0 .../data_comps/drof/drof_comp_mod.F90 | 5 +- .../data_comps/drof/rof_comp_mct.F90 | 0 .../components}/data_comps/dwav/README | 0 .../components}/data_comps/dwav/bld/README | 0 .../data_comps/dwav/cime_config/buildlib | 2 +- .../data_comps/dwav/cime_config/buildnml | 26 +- .../dwav/cime_config/config_archive.xml | 0 .../dwav/cime_config/config_component.xml | 10 +- .../cime_config/namelist_definition_dwav.xml | 47 +- .../data_comps/dwav/cime_config/user_nl_dwav | 0 .../data_comps/dwav/dwav_comp_mod.F90 | 97 +- .../data_comps/dwav/wav_comp_mct.F90 | 0 .../stub_comps/satm/cime_config/buildlib | 2 +- .../stub_comps/satm/cime_config/buildnml | 0 .../satm/cime_config/config_component.xml | 0 .../stub_comps/satm/cpl/atm_comp_mct.F90 | 0 .../stub_comps/sesp/cime_config/buildlib | 2 +- .../stub_comps/sesp/cime_config/buildnml | 0 .../sesp/cime_config/config_component.xml | 0 .../stub_comps/sesp/cpl/esp_comp_mct.F90 | 0 .../stub_comps/sglc/cime_config/buildlib | 2 +- .../stub_comps/sglc/cime_config/buildnml | 0 .../sglc/cime_config/config_component.xml | 0 .../stub_comps/sglc/cpl/glc_comp_mct.F90 | 0 .../stub_comps/sice/cime_config/buildlib | 2 +- .../stub_comps/sice/cime_config/buildnml | 0 .../sice/cime_config/config_component.xml | 0 .../stub_comps/sice/cpl/ice_comp_mct.F90 | 0 .../stub_comps/slnd/cime_config/buildlib | 2 +- .../stub_comps/slnd/cime_config/buildnml | 0 .../slnd/cime_config/config_component.xml | 0 .../stub_comps/slnd/cpl/lnd_comp_mct.F90 | 0 .../stub_comps/socn/cime_config/buildlib | 2 +- .../stub_comps/socn/cime_config/buildnml | 0 .../socn/cime_config/config_component.xml | 0 .../stub_comps/socn/cpl/ocn_comp_mct.F90 | 0 .../stub_comps/srof/cime_config/buildlib | 2 +- .../stub_comps/srof/cime_config/buildnml | 0 .../srof/cime_config/config_component.xml | 0 .../stub_comps/srof/cpl/rof_comp_mct.F90 | 0 .../stub_comps/swav/cime_config/buildlib | 2 +- .../stub_comps/swav/cime_config/buildnml | 0 .../swav/cime_config/config_component.xml | 0 .../stub_comps/swav/cpl/wav_comp_mct.F90 | 0 .../xcpl_comps/xatm/cime_config/buildlib | 2 +- .../xcpl_comps/xatm/cime_config/buildnml | 2 +- .../xatm/cime_config/config_component.xml | 0 .../xcpl_comps/xatm/cpl/atm_comp_mct.F90 | 0 .../xcpl_comps/xglc/cime_config/buildlib | 2 +- .../xcpl_comps/xglc/cime_config/buildnml | 2 +- .../xglc/cime_config/config_component.xml | 0 .../xcpl_comps/xglc/cpl/glc_comp_mct.F90 | 0 .../xcpl_comps/xice/cime_config/buildlib | 2 +- .../xcpl_comps/xice/cime_config/buildnml | 2 +- .../xice/cime_config/config_component.xml | 0 .../xcpl_comps/xice/cpl/ice_comp_mct.F90 | 0 .../xcpl_comps/xlnd/cime_config/buildlib | 2 +- .../xcpl_comps/xlnd/cime_config/buildnml | 2 +- .../xlnd/cime_config/config_component.xml | 0 .../xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 | 0 .../xcpl_comps/xocn/cime_config/buildlib | 2 +- .../xcpl_comps/xocn/cime_config/buildnml | 2 +- .../xocn/cime_config/config_component.xml | 0 .../xcpl_comps/xocn/cpl/ocn_comp_mct.F90 | 0 .../xcpl_comps/xrof/cime_config/buildlib | 2 +- .../xcpl_comps/xrof/cime_config/buildnml | 2 +- .../xrof/cime_config/config_component.xml | 0 .../xcpl_comps/xrof/cpl/rof_comp_mct.F90 | 0 .../xcpl_comps/xshare/dead_data_mod.F90 | 0 .../xcpl_comps/xshare/dead_mct_mod.F90 | 0 .../xcpl_comps/xshare/dead_mod.F90 | 0 .../xcpl_comps/xwav/cime_config/buildlib | 2 +- .../xcpl_comps/xwav/cime_config/buildnml | 2 +- .../xwav/cime_config/config_component.xml | 0 .../xcpl_comps/xwav/cpl/wav_comp_mct.F90 | 0 .../namelist_files/namelist_defaults_drv.xml | 0 .../namelist_definition_drv.xml | 0 .../namelist_definition_drv_flds.xml | 0 .../namelist_definition_modio.xml | 0 .../bld/testdir/Buildconf/camconf/drv_flds_in | 0 .../bld/testdir/Buildconf/clmconf/drv_flds_in | 0 .../drivers/mct}/bld/testdir/env_build.xml | 0 .../drivers/mct}/bld/testdir/env_case.xml | 0 .../drivers/mct}/bld/testdir/env_mach_pes.xml | 0 .../drivers/mct}/bld/testdir/env_run.xml | 2 +- .../drivers/mct}/bld/testdir/runtest | 0 .../drivers/mct}/bld/testdir/user_nl_cpl | 0 .../drivers/mct}/cime_config/buildexe | 8 +- .../drivers/mct}/cime_config/buildnml | 53 +- .../mct}/cime_config/config_archive.xml | 0 .../mct}/cime_config/config_component.xml | 102 +- .../cime_config/config_component_acme.xml | 60 +- .../cime_config/config_component_cesm.xml | 30 + .../mct}/cime_config/config_compsets.xml | 40 +- .../drivers/mct}/cime_config/config_pes.xml | 40 + .../cime_config/namelist_definition_drv.xml | 115 +- .../namelist_definition_drv_flds.xml | 148 + .../namelist_definition_modelio.xml | 0 .../cime_config/testdefs/testlist_drv.xml | 0 .../testmods_dirs/drv/5steps/shell_commands | 0 .../drivers/mct}/cime_config/user_nl_cpl | 0 .../drivers/mct}/doc/bookinfo.xml | 0 {driver_cpl => src/drivers/mct}/doc/chap1.xml | 4 +- {driver_cpl => src/drivers/mct}/doc/chap2.xml | 0 {driver_cpl => src/drivers/mct}/doc/figx1.jpg | Bin .../drivers/mct}/doc/namelist.xml | 0 .../drivers/mct}/doc/rundocbook.csh | 0 .../drivers/mct}/doc/stylesheet.dsl | 0 {driver_cpl => src/drivers/mct}/doc/ug.xml | 0 .../drivers/mct/main}/CMakeLists.txt | 0 .../drivers/mct/main}/cesm_comp_mod.F90 | 149 +- .../drivers/mct/main}/cesm_driver.F90 | 7 +- .../drivers/mct/main}/component_mod.F90 | 0 .../drivers/mct/main}/component_type_mod.F90 | 0 .../mct/main}/cplcomp_exchange_mod.F90 | 0 .../drivers/mct/main}/map_glc2lnd_mod.F90 | 10 +- .../drivers/mct/main}/map_lnd2glc_mod.F90 | 8 +- .../mct/main}/map_lnd2rof_irrig_mod.F90 | 4 +- .../drivers/mct/main}/mrg_mod.F90 | 0 .../drivers/mct/main}/prep_aoflux_mod.F90 | 0 .../drivers/mct/main}/prep_atm_mod.F90 | 0 .../drivers/mct/main}/prep_glc_mod.F90 | 27 + .../drivers/mct/main}/prep_ice_mod.F90 | 0 .../drivers/mct/main}/prep_lnd_mod.F90 | 0 .../drivers/mct/main}/prep_ocn_mod.F90 | 0 .../drivers/mct/main}/prep_rof_mod.F90 | 0 .../drivers/mct/main}/prep_wav_mod.F90 | 0 .../drivers/mct/main}/seq_diag_mct.F90 | 0 .../drivers/mct/main}/seq_domain_mct.F90 | 0 .../drivers/mct/main}/seq_flux_mct.F90 | 0 .../drivers/mct/main}/seq_frac_mct.F90 | 0 .../drivers/mct/main}/seq_hist_mod.F90 | 0 .../drivers/mct/main}/seq_io_mod.F90 | 15 +- .../drivers/mct/main}/seq_map_mod.F90 | 0 .../drivers/mct/main}/seq_map_type_mod.F90 | 0 .../drivers/mct/main}/seq_rest_mod.F90 | 8 +- .../drivers/mct/main}/t_driver_timers_mod.F90 | 0 ...vertical_gradient_calculator_2nd_order.F90 | 14 +- .../vertical_gradient_calculator_base.F90 | 0 .../vertical_gradient_calculator_factory.F90 | 2 +- .../drivers/mct}/shr/CMakeLists.txt | 0 .../drivers/mct}/shr/glc_elevclass_mod.F90 | 2 +- .../drivers/mct}/shr/seq_cdata_mod.F90 | 0 .../drivers/mct}/shr/seq_comm_mct.F90 | 132 +- .../drivers/mct}/shr/seq_drydep_mod.F90 | 0 .../drivers/mct}/shr/seq_flds_mod.F90 | 40 +- .../drivers/mct}/shr/seq_infodata_mod.F90 | 592 +- .../drivers/mct}/shr/seq_io_read_mod.F90 | 0 .../drivers/mct}/shr/seq_timemgr_mod.F90 | 514 +- .../drivers/mct}/shr/shr_carma_mod.F90 | 0 .../drivers/mct}/shr/shr_expr_parser_mod.F90 | 0 .../drivers/mct}/shr/shr_fire_emis_mod.F90 | 0 .../drivers/mct}/shr/shr_megan_mod.F90 | 0 src/drivers/mct/shr/shr_ndep_mod.F90 | 117 + .../drivers/mct}/unit_test/CMakeLists.txt | 16 +- .../avect_wrapper_test/CMakeLists.txt | 0 .../avect_wrapper_test/test_avect_wrapper.pf | 0 .../glc_elevclass_test/CMakeLists.txt | 0 .../glc_elevclass_test/test_glc_elevclass.pf | 0 .../unit_test/map_glc2lnd_test/CMakeLists.txt | 0 .../map_glc2lnd_test/test_map_glc2lnd.pf | 0 .../unit_test/map_lnd2glc_test/CMakeLists.txt | 0 .../map_lnd2glc_test/test_map_lnd2glc.pf | 0 .../map_lnd2rof_irrig_test/CMakeLists.txt | 0 .../test_map_lnd2rof_irrig.pf | 8 +- .../unit_test/seq_map_test/CMakeLists.txt | 0 .../unit_test/seq_map_test/test_seq_map.pf | 0 .../mct}/unit_test/stubs/CMakeLists.txt | 1 + .../mct/unit_test/stubs/seq_timemgr_mod.F90 | 19 + .../vertical_gradient_calculator_constant.F90 | 0 .../mct}/unit_test/utils/CMakeLists.txt | 0 .../unit_test/utils/avect_wrapper_mod.F90 | 0 .../unit_test/utils/create_mapper_mod.F90 | 0 .../mct}/unit_test/utils/mct_wrapper_mod.F90 | 0 .../mct}/unit_test/utils/simple_map_mod.F90 | 4 +- .../CMakeLists.txt | 0 .../vertical_gradient_calculator_test/README | 0 .../gradient_example.txt | 0 .../plot_gradient | 0 ..._vertical_gradient_calculator_2nd_order.pf | 0 ...st_vertical_gradient_calculator_factory.pf | 0 {externals => src/externals}/CMake/.gitignore | 0 .../externals}/CMake/CESM_utils.cmake | 0 src/externals/CMake/CIME_initial_setup.cmake | 16 + .../externals}/CMake/CIME_utils.cmake | 12 +- {externals => src/externals}/CMake/ChangeLog | 0 .../externals}/CMake/Compilers.cmake | 0 .../externals}/CMake/FindpFUnit.cmake | 0 {externals => src/externals}/CMake/LICENSE | 0 {externals => src/externals}/CMake/README.md | 8 +- .../externals}/CMake/Sourcelist_utils.cmake | 0 .../externals}/CMake/TryCSizeOf.f90 | 0 .../externals}/CMake/TryMPIIO.f90 | 0 .../externals}/CMake/TryMPIMod.f90 | 0 .../externals}/CMake/TryMPISERIAL.f90 | 0 .../externals}/CMake/TryPnetcdf_inc.f90 | 0 .../externals}/CMake/TryPnetcdf_mod.f90 | 0 .../externals}/CMake/genf90_utils.cmake | 0 .../externals}/CMake/mpiexec.cmake | 0 .../externals}/CMake/pFUnit_utils.cmake | 3 +- {externals => src/externals}/genf90/ChangeLog | 0 {externals => src/externals}/genf90/genf90.pl | 0 {externals => src/externals}/mct/.gitignore | 0 {externals => src/externals}/mct/COPYRIGHT | 0 {externals => src/externals}/mct/Makefile | 0 .../externals}/mct/Makefile.conf.in | 0 {externals => src/externals}/mct/README | 0 {externals => src/externals}/mct/aclocal.m4 | 0 .../externals}/mct/benchmarks/.gitignore | 0 .../externals}/mct/benchmarks/Makefile | 0 .../mct/benchmarks/RouterTestDis.F90 | 0 .../mct/benchmarks/RouterTestOvr.F90 | 0 .../externals}/mct/benchmarks/T42.8pC | 0 .../externals}/mct/benchmarks/T42.8pR | 0 .../externals}/mct/benchmarks/gx1.8pC | 0 .../externals}/mct/benchmarks/gx1.8pR | 0 .../externals}/mct/benchmarks/importBench.F90 | 0 {externals => src/externals}/mct/config.h.in | 0 {externals => src/externals}/mct/configure | 0 {externals => src/externals}/mct/configure.ac | 0 .../externals}/mct/doc/.gitignore | 0 {externals => src/externals}/mct/doc/Makefile | 0 {externals => src/externals}/mct/doc/README | 0 .../externals}/mct/doc/coupler.bib | 0 .../externals}/mct/doc/mct_APIs.tex | 0 .../externals}/mct/doc/texsrc/.gitignore | 0 .../externals}/mct/doc/texsrc/Makefile | 0 .../externals}/mct/doc/texsrc/SRCS_tex.mk | 0 .../externals}/mct/examples/Makefile | 0 .../externals}/mct/examples/README | 0 .../mct/examples/climate_concur1/.gitignore | 0 .../mct/examples/climate_concur1/Makefile | 0 .../mct/examples/climate_concur1/README | 0 .../mct/examples/climate_concur1/coupler.F90 | 0 .../mct/examples/climate_concur1/master.F90 | 0 .../mct/examples/climate_concur1/model.F90 | 0 .../mct/examples/climate_sequen1/.gitignore | 0 .../mct/examples/climate_sequen1/Makefile | 0 .../mct/examples/climate_sequen1/README | 0 .../mct/examples/climate_sequen1/TS1.dat | 0 .../mct/examples/climate_sequen1/coupler.F90 | 0 .../mct/examples/climate_sequen1/dst.rc | 0 .../mct/examples/climate_sequen1/dstmodel.F90 | 0 .../mct/examples/climate_sequen1/master.F90 | 0 .../mct/examples/climate_sequen1/mutils.F90 | 0 .../mct/examples/climate_sequen1/src.rc | 0 .../mct/examples/climate_sequen1/srcmodel.F90 | 0 .../externals}/mct/examples/simple/.gitignore | 0 .../externals}/mct/examples/simple/Makefile | 0 .../externals}/mct/examples/simple/README | 0 .../mct/examples/simple/script.babyblue | 0 .../mct/examples/simple/twocmp.con.F90 | 0 .../mct/examples/simple/twocmp.seq.F90 | 0 .../mct/examples/simple/twocmp.seqNB.F90 | 0 .../mct/examples/simple/twocmp.seqUnvn.F90 | 0 {externals => src/externals}/mct/install-sh | 0 {externals => src/externals}/mct/m4/README | 0 .../externals}/mct/m4/acx_mpi.m4 | 0 .../externals}/mct/m4/ax_fc_version.m4 | 0 .../externals}/mct/m4/fortran.m4 | 0 {externals => src/externals}/mct/mct/Makefile | 4 +- {externals => src/externals}/mct/mct/README | 0 .../externals}/mct/mct/m_Accumulator.F90 | 0 .../externals}/mct/mct/m_AccumulatorComms.F90 | 0 .../externals}/mct/mct/m_AttrVect.F90 | 211 +- .../externals}/mct/mct/m_AttrVectComms.F90 | 0 .../externals}/mct/mct/m_AttrVectReduce.F90 | 0 .../externals}/mct/mct/m_ConvertMaps.F90 | 0 .../externals}/mct/mct/m_ExchangeMaps.F90 | 0 .../externals}/mct/mct/m_GeneralGrid.F90 | 0 .../externals}/mct/mct/m_GeneralGridComms.F90 | 0 .../externals}/mct/mct/m_GlobalMap.F90 | 0 .../externals}/mct/mct/m_GlobalSegMap.F90 | 254 +- .../mct/mct/m_GlobalSegMapComms.F90 | 0 .../externals}/mct/mct/m_GlobalToLocal.F90 | 0 .../externals}/mct/mct/m_MCTWorld.F90 | 0 .../externals}/mct/mct/m_MatAttrVectMul.F90 | 48 +- .../externals}/mct/mct/m_Merge.F90 | 0 .../externals}/mct/mct/m_Navigator.F90 | 0 .../externals}/mct/mct/m_Rearranger.F90 | 125 +- .../externals}/mct/mct/m_Router.F90 | 329 +- src/externals/mct/mct/m_SPMDutils.F90 | 1148 ++++ .../externals}/mct/mct/m_SparseMatrix.F90 | 0 .../mct/mct/m_SparseMatrixComms.F90 | 0 .../mct/mct/m_SparseMatrixDecomp.F90 | 0 .../externals}/mct/mct/m_SparseMatrixPlus.F90 | 0 .../mct/mct/m_SparseMatrixToMaps.F90 | 0 .../externals}/mct/mct/m_SpatialIntegral.F90 | 0 .../externals}/mct/mct/m_SpatialIntegralV.F90 | 0 .../externals}/mct/mct/m_Transfer.F90 | 0 .../externals}/mct/mkinstalldirs | 0 .../externals}/mct/mpeu/Makefile | 0 {externals => src/externals}/mct/mpeu/README | 0 .../externals}/mct/mpeu/assertmpeu.H | 0 .../externals}/mct/mpeu/get_zeits.c | 0 .../externals}/mct/mpeu/m_FcComms.F90 | 9 +- .../externals}/mct/mpeu/m_FileResolv.F90 | 0 .../externals}/mct/mpeu/m_Filename.F90 | 0 .../externals}/mct/mpeu/m_IndexBin_char.F90 | 0 .../mct/mpeu/m_IndexBin_integer.F90 | 0 .../mct/mpeu/m_IndexBin_logical.F90 | 0 .../externals}/mct/mpeu/m_List.F90 | 0 .../externals}/mct/mpeu/m_MergeSorts.F90 | 0 .../externals}/mct/mpeu/m_Permuter.F90 | 0 .../externals}/mct/mpeu/m_SortingTools.F90 | 0 .../externals}/mct/mpeu/m_StrTemplate.F90 | 0 .../externals}/mct/mpeu/m_String.F90 | 0 .../mct/mpeu/m_StringLinkedList.F90 | 0 .../externals}/mct/mpeu/m_TraceBack.F90 | 0 .../externals}/mct/mpeu/m_chars.F90 | 0 .../externals}/mct/mpeu/m_die.F90 | 0 .../externals}/mct/mpeu/m_dropdead.F90 | 0 .../externals}/mct/mpeu/m_flow.F90 | 0 .../externals}/mct/mpeu/m_inpak90.F90 | 0 .../externals}/mct/mpeu/m_ioutil.F90 | 0 .../externals}/mct/mpeu/m_mall.F90 | 0 .../externals}/mct/mpeu/m_mpif.F90 | 0 .../externals}/mct/mpeu/m_mpif90.F90 | 0 .../externals}/mct/mpeu/m_mpout.F90 | 0 .../externals}/mct/mpeu/m_rankMerge.F90 | 0 .../externals}/mct/mpeu/m_realkinds.F90 | 0 .../externals}/mct/mpeu/m_stdio.F90 | 0 .../externals}/mct/mpeu/m_zeit.F90 | 0 .../externals}/mct/mpi-serial/.gitignore | 0 .../externals}/mct/mpi-serial/Makefile | 2 + .../mct/mpi-serial/Makefile.conf.in | 0 .../externals}/mct/mpi-serial/README | 42 +- .../externals}/mct/mpi-serial/aclocal.m4 | 0 .../externals}/mct/mpi-serial/cart.c | 0 .../externals}/mct/mpi-serial/collective.c | 21 - .../externals}/mct/mpi-serial/comm.c | 0 .../externals}/mct/mpi-serial/config.h.in | 0 .../externals}/mct/mpi-serial/configure | 0 .../externals}/mct/mpi-serial/configure.in | 0 .../externals}/mct/mpi-serial/copy.c | 0 src/externals/mct/mpi-serial/error.c | 13 + .../externals}/mct/mpi-serial/fort.F90 | 0 .../externals}/mct/mpi-serial/getcount.c | 0 .../externals}/mct/mpi-serial/group.c | 0 .../externals}/mct/mpi-serial/handles.c | 0 src/externals/mct/mpi-serial/ic_merge.c | 15 + .../externals}/mct/mpi-serial/info.c | 0 .../externals}/mct/mpi-serial/list.c | 0 .../externals}/mct/mpi-serial/list.h | 0 .../externals}/mct/mpi-serial/listP.h | 0 .../externals}/mct/mpi-serial/listops.h | 0 .../externals}/mct/mpi-serial/m4/README | 0 .../mct/mpi-serial/m4/ax_fc_version.m4 | 0 .../externals}/mct/mpi-serial/mpi.c | 0 .../externals}/mct/mpi-serial/mpi.h | 19 +- .../externals}/mct/mpi-serial/mpiP.h | 0 .../externals}/mct/mpi-serial/mpif.F90 | 0 src/externals/mct/mpi-serial/mpif.h | 327 ++ .../externals}/mct/mpi-serial/op.c | 0 .../externals}/mct/mpi-serial/pack.c | 0 .../externals}/mct/mpi-serial/probe.c | 0 .../externals}/mct/mpi-serial/protify.awk | 0 .../externals}/mct/mpi-serial/recv.c | 0 .../externals}/mct/mpi-serial/req.c | 0 .../externals}/mct/mpi-serial/send.c | 0 .../mct/mpi-serial/tests/.gitignore | 0 .../externals}/mct/mpi-serial/tests/Makefile | 0 .../externals}/mct/mpi-serial/tests/ctest.c | 0 .../mct/mpi-serial/tests/ctest_old.c | 0 .../externals}/mct/mpi-serial/tests/ftest.F90 | 0 .../mct/mpi-serial/tests/ftest_internal.F90 | 0 .../mct/mpi-serial/tests/ftest_old.F90 | 8 +- .../externals}/mct/mpi-serial/time.c | 0 .../externals}/mct/mpi-serial/type.c | 17 + .../externals}/mct/mpi-serial/type.h | 0 .../externals}/mct/mpi-serial/type_const.c | 0 .../externals}/mct/protex/protex | 0 .../externals}/mct/testsystem/Makefile | 0 .../mct/testsystem/testall/.gitignore | 0 .../mct/testsystem/testall/Makefile | 0 .../testall/ReadSparseMatrixAsc.F90 | 2 + .../mct/testsystem/testall/UNTESTED | 0 .../externals}/mct/testsystem/testall/ccm.F90 | 0 .../mct/testsystem/testall/convertPOPT.F90 | 0 .../mct/testsystem/testall/convertgauss.F90 | 0 .../externals}/mct/testsystem/testall/cpl.F90 | 2 +- .../mct/testsystem/testall/job.ut-all.jaguar | 0 .../mct/testsystem/testall/m_ACTEST.F90 | 0 .../mct/testsystem/testall/m_AVTEST.F90 | 0 .../mct/testsystem/testall/m_GGRIDTEST.F90 | 0 .../mct/testsystem/testall/m_GMAPTEST.F90 | 0 .../mct/testsystem/testall/m_GSMAPTEST.F90 | 0 .../mct/testsystem/testall/m_MCTWORLDTEST.F90 | 0 .../mct/testsystem/testall/m_ROUTERTEST.F90 | 0 .../mct/testsystem/testall/m_SMATTEST.F90 | 0 .../mct/testsystem/testall/master.F90 | 0 .../externals}/mct/testsystem/testall/mph.F90 | 0 .../externals}/mct/testsystem/testall/pop.F90 | 0 .../mct/testsystem/testall/processors_map.in | 0 .../mct/testsystem/testall/script.jag | 0 .../mct/testsystem/testall/ut_SparseMatrix.rc | 0 .../externals}/mct/testunit/.gitignore | 0 .../externals}/mct/testunit/AttrVect_Test.F90 | 0 .../externals}/mct/testunit/Makefile | 0 .../externals}/mct/testunit/master.F90 | 0 .../externals}/pio1/CMakeLists.txt | 0 {externals => src/externals}/pio1/ChangeLog | 0 .../externals}/pio1/ChangeLog_template | 0 {externals => src/externals}/pio1/Doxyfile | 0 .../externals}/pio1/DoxygenLayout.xml | 0 .../externals}/pio1/SVN_EXTERNAL_DIRECTORIES | 0 .../externals}/pio1/customdoxygen.css | 0 .../externals}/pio1/doc/CAMexample.txt | 0 .../externals}/pio1/doc/Decomp.txt | 0 .../externals}/pio1/doc/DoxygenLayout.xml | 0 .../externals}/pio1/doc/Error.txt | 0 .../externals}/pio1/doc/Examples.txt | 0 .../externals}/pio1/doc/GettingStarted.txt | 0 .../externals}/pio1/doc/Installing.txt | 0 {externals => src/externals}/pio1/doc/api.txt | 0 .../externals}/pio1/doc/base.txt | 0 .../externals}/pio1/doc/example/errorhandle | 0 .../externals}/pio1/doc/example/simple-bc | 0 .../pio1/doc/example/simple-bc-rearr | 0 .../pio1/doc/example/simple-bc-rearr-pe1 | 0 .../pio1/doc/example/simple-bc-rearr-pe2 | 0 .../externals}/pio1/doc/example/simple-dof | 0 .../pio1/doc/example/simple-dof-rearr | 0 {externals => src/externals}/pio1/doc/faq.txt | 0 .../externals}/pio1/doc/footer.html | 0 .../externals}/pio1/doc/header.html | 0 .../pio1/doc/images/baseimage.graffle | 0 .../pio1/doc/images/block-cyclic-rearr.eps | 0 .../doc/images/block-cyclic-rearr.graffle | 0 .../pio1/doc/images/block-cyclic-rearr.png | Bin .../pio1/doc/images/block-cyclic.eps | 0 .../pio1/doc/images/block-cyclic.graffle | 0 .../pio1/doc/images/block-cyclic.png | Bin .../externals}/pio1/doc/images/dof-rearr.eps | 0 .../pio1/doc/images/dof-rearr.graffle | 0 .../externals}/pio1/doc/images/dof-rearr.png | Bin .../externals}/pio1/doc/images/dof.eps | 0 .../externals}/pio1/doc/images/dof.graffle | 0 .../externals}/pio1/doc/images/dof.png | Bin .../externals}/pio1/doc/testpio_example.txt | 0 {externals => src/externals}/pio1/doxygen.sty | 0 .../externals}/pio1/pio/CMakeLists.txt | 13 +- .../externals}/pio1/pio/C_interface_mod.F90 | 0 .../externals}/pio1/pio/Makefile.conf.in | 0 .../externals}/pio1/pio/README.config | 0 .../externals}/pio1/pio/alloc_mod.F90.in | 0 .../externals}/pio1/pio/box_rearrange.F90.in | 0 .../externals}/pio1/pio/calcdecomp.F90 | 0 .../externals}/pio1/pio/calcdisplace_mod.F90 | 0 .../externals}/pio1/pio/config.h.in | 0 .../externals}/pio1/pio/dtypes.h | 0 .../externals}/pio1/pio/fdepends.awk | 0 .../externals}/pio1/pio/iompi_mod.F90.in | 0 .../externals}/pio1/pio/ionf_mod.F90 | 0 .../externals}/pio1/pio/nf_mod.F90 | 0 {externals => src/externals}/pio1/pio/pio.F90 | 2 +- .../externals}/pio1/pio/pio_kinds.F90 | 0 .../externals}/pio1/pio/pio_mpi_utils.F90 | 0 .../externals}/pio1/pio/pio_msg_callbacks.F90 | 0 .../pio1/pio/pio_msg_getput_callbacks.F90.in | 0 .../externals}/pio1/pio/pio_msg_mod.F90 | 0 .../externals}/pio1/pio/pio_nf_utils.F90 | 0 .../externals}/pio1/pio/pio_spmd_utils.F90.in | 0 .../externals}/pio1/pio/pio_support.F90 | 0 .../externals}/pio1/pio/pio_types.F90 | 2 +- .../externals}/pio1/pio/pio_utils.F90 | 0 .../externals}/pio1/pio/piodarray.F90.in | 0 .../externals}/pio1/pio/piolib_mod.F90 | 75 +- .../externals}/pio1/pio/pionfatt_mod.F90.in | 0 .../externals}/pio1/pio/pionfget_mod.F90.in | 0 .../externals}/pio1/pio/pionfput_mod.F90.in | 3 +- .../externals}/pio1/pio/pionfread_mod.F90.in | 0 .../externals}/pio1/pio/pionfwrite_mod.F90.in | 0 .../externals}/pio1/pio/piovdc.F90 | 0 .../externals}/pio1/pio/rearr_options.h | 0 .../externals}/pio1/pio/rearrange.F90.in | 0 .../externals}/pio1/pio/topology.c | 0 .../externals}/pio1/scripts/Utils.pm | 0 .../externals}/pio1/scripts/config.pl | 0 .../pio1/scripts/testpio_yellowstone.pl | 0 .../externals}/pio1/tests/testpio/CAM05.csh | 0 .../pio1/tests/testpio/CMakeLists.txt | 0 .../pio1/tests/testpio/MPASA30km.csh | 0 .../pio1/tests/testpio/MPASA60km.csh | 0 .../externals}/pio1/tests/testpio/POPB.csh | 0 .../externals}/pio1/tests/testpio/POPC.csh | 0 .../externals}/pio1/tests/testpio/POPD.csh | 0 .../externals}/pio1/tests/testpio/POPDv0.csh | 0 .../externals}/pio1/tests/testpio/POPDv1.csh | 0 .../externals}/pio1/tests/testpio/POPDv2.csh | 0 .../externals}/pio1/tests/testpio/POPDv3.csh | 0 .../externals}/pio1/tests/testpio/POPDv4.csh | 0 .../externals}/pio1/tests/testpio/POPDv5.csh | 0 .../pio1/tests/testpio/README.testpio | 0 .../externals}/pio1/tests/testpio/WRFB.csh | 0 .../pio1/tests/testpio/build_defaults.xml | 0 .../pio1/tests/testpio/check_mod.F90 | 0 .../pio1/tests/testpio/config_bench.xml | 0 .../pio1/tests/testpio/fdepends.awk | 0 .../pio1/tests/testpio/gdecomp_mod.F90 | 0 .../pio1/tests/testpio/kinds_mod.F90 | 0 .../pio1/tests/testpio/kraken.128.csh | 0 .../pio1/tests/testpio/kraken.1K.csh | 0 .../pio1/tests/testpio/kraken.256.csh | 0 .../pio1/tests/testpio/kraken.512.csh | 0 .../pio1/tests/testpio/kraken.64.csh | 0 .../pio1/tests/testpio/namelist_mod.F90 | 0 .../tests/testpio/namelists/testpio_in.apb05 | 0 .../tests/testpio/namelists/testpio_in.asb01 | 0 .../tests/testpio/namelists/testpio_in.asb04 | 0 .../tests/testpio/namelists/testpio_in.b01 | 0 .../tests/testpio/namelists/testpio_in.b02 | 0 .../tests/testpio/namelists/testpio_in.b03 | 0 .../tests/testpio/namelists/testpio_in.b04 | 0 .../tests/testpio/namelists/testpio_in.b05 | 0 .../tests/testpio/namelists/testpio_in.b06 | 0 .../tests/testpio/namelists/testpio_in.b07 | 0 .../tests/testpio/namelists/testpio_in.b08 | 0 .../tests/testpio/namelists/testpio_in.b09 | 0 .../tests/testpio/namelists/testpio_in.b10 | 0 .../tests/testpio/namelists/testpio_in.b11 | 0 .../tests/testpio/namelists/testpio_in.b12 | 0 .../tests/testpio/namelists/testpio_in.b13 | 0 .../tests/testpio/namelists/testpio_in.bb01 | 0 .../tests/testpio/namelists/testpio_in.bb02 | 0 .../tests/testpio/namelists/testpio_in.bb03 | 0 .../tests/testpio/namelists/testpio_in.bb04 | 0 .../tests/testpio/namelists/testpio_in.bb05 | 0 .../tests/testpio/namelists/testpio_in.bb06 | 0 .../tests/testpio/namelists/testpio_in.bb07 | 0 .../tests/testpio/namelists/testpio_in.bb08 | 0 .../tests/testpio/namelists/testpio_in.bn01 | 0 .../tests/testpio/namelists/testpio_in.bn02 | 0 .../tests/testpio/namelists/testpio_in.bn03 | 0 .../tests/testpio/namelists/testpio_in.n4b01 | 0 .../tests/testpio/namelists/testpio_in.n4b02 | 0 .../tests/testpio/namelists/testpio_in.n4b03 | 0 .../tests/testpio/namelists/testpio_in.n4b04 | 0 .../tests/testpio/namelists/testpio_in.n4b05 | 0 .../tests/testpio/namelists/testpio_in.n4b06 | 0 .../tests/testpio/namelists/testpio_in.n4b07 | 0 .../tests/testpio/namelists/testpio_in.n4b08 | 0 .../tests/testpio/namelists/testpio_in.n4n01 | 0 .../tests/testpio/namelists/testpio_in.n4n02 | 0 .../tests/testpio/namelists/testpio_in.n4n03 | 0 .../tests/testpio/namelists/testpio_in.pb01 | 0 .../tests/testpio/namelists/testpio_in.pb02 | 0 .../tests/testpio/namelists/testpio_in.pb03 | 0 .../tests/testpio/namelists/testpio_in.pb04 | 0 .../tests/testpio/namelists/testpio_in.pb05 | 0 .../tests/testpio/namelists/testpio_in.pb06 | 0 .../tests/testpio/namelists/testpio_in.pb07 | 0 .../tests/testpio/namelists/testpio_in.pb08 | 0 .../tests/testpio/namelists/testpio_in.pn01 | 0 .../tests/testpio/namelists/testpio_in.pn02 | 0 .../tests/testpio/namelists/testpio_in.pn03 | 0 .../tests/testpio/namelists/testpio_in.sb01 | 0 .../tests/testpio/namelists/testpio_in.sb02 | 0 .../tests/testpio/namelists/testpio_in.sb03 | 0 .../tests/testpio/namelists/testpio_in.sb04 | 0 .../tests/testpio/namelists/testpio_in.sb05 | 0 .../tests/testpio/namelists/testpio_in.sb06 | 0 .../tests/testpio/namelists/testpio_in.sb07 | 0 .../tests/testpio/namelists/testpio_in.sb08 | 0 .../tests/testpio/namelists/testpio_in.sn01 | 0 .../tests/testpio/namelists/testpio_in.sn02 | 0 .../tests/testpio/namelists/testpio_in.sn03 | 0 .../tests/testpio/namelists/testpio_in.wr01 | 0 .../tests/testpio/namelists/testpio_in.wr02 | 0 .../pio1/tests/testpio/perl5lib/ChangeLog | 0 .../pio1/tests/testpio/perl5lib/README | 0 .../pio1/tests/testpio/perl5lib/XML/Changes | 0 .../pio1/tests/testpio/perl5lib/XML/Lite.pm | 0 .../testpio/perl5lib/XML/Lite/Element.pm | 0 .../pio1/tests/testpio/perl5lib/XML/README | 0 .../testpio/perl5lib/XML/man3/XML::Lite.3 | 0 .../perl5lib/XML/man3/XML::Lite::Element.3 | 0 .../externals}/pio1/tests/testpio/test.csh | 0 .../pio1/tests/testpio/test_lib.F90 | 0 .../pio1/tests/testpio/testdecomp.F90 | 0 .../tests/testpio/testdecomp.bluefire.run | 0 .../pio1/tests/testpio/testdecomp_in | 0 .../externals}/pio1/tests/testpio/testpio.F90 | 0 .../pio1/tests/testpio/testpio_bench.pl | 0 .../pio1/tests/testpio/testpio_build.pl | 0 .../pio1/tests/testpio/testpio_run.pl | 0 .../pio1/tests/testpio/utils_mod.F90 | 0 .../externals}/pio1/tests/testpio/ystest.sh | 0 .../pio1/tests/unittests/CMakeLists.txt | 0 .../pio1/tests/unittests/Levy_Notes | 0 .../externals}/pio1/tests/unittests/README | 0 .../pio1/tests/unittests/basic_tests.F90 | 0 .../pio1/tests/unittests/driver.F90 | 0 .../pio1/tests/unittests/global_vars.F90 | 0 .../externals}/pio1/tests/unittests/input.nl | 0 .../pio1/tests/unittests/nc_set_log_level2.c | 0 .../pio1/tests/unittests/ncdf_tests.F90 | 0 .../pio1/tests/unittests/not_netcdf.ieee | Bin .../externals}/pio1/timing/CMakeLists.txt | 0 src/externals/pio1/timing/COPYING | 17 + .../externals}/pio1/timing/ChangeLog | 0 .../externals}/pio1/timing/GPTLget_memusage.c | 0 .../pio1/timing/GPTLprint_memusage.c | 0 .../externals}/pio1/timing/GPTLutil.c | 0 .../externals}/pio1/timing/Makefile | 0 .../externals}/pio1/timing/README | 0 .../externals}/pio1/timing/XXXdotF/perf_mod.F | 0 .../pio1/timing/XXXdotF/perf_utils.F | 0 .../externals}/pio1/timing/f_wrappers.c | 0 .../externals}/pio1/timing/gptl.c | 0 .../externals}/pio1/timing/gptl.h | 0 .../externals}/pio1/timing/gptl.inc | 0 .../externals}/pio1/timing/gptl_papi.c | 0 .../externals}/pio1/timing/perf_mod.F90 | 0 .../externals}/pio1/timing/perf_utils.F90 | 0 .../externals}/pio1/timing/private.h | 0 .../externals}/pio1/timing/threadutil.c | 0 {externals => src/externals}/pio2/.gitignore | 0 src/externals/pio2/CMakeLists.txt | 198 + .../externals}/pio2/CTestConfig.cmake | 12 +- .../externals}/pio2/CTestScript.cmake | 42 +- {externals => src/externals}/pio2/README.md | 0 .../externals}/pio2/cmake/FindGPTL.cmake | 0 .../externals}/pio2/cmake/FindHDF5.cmake | 0 .../externals}/pio2/cmake/FindLIBRT.cmake | 0 .../externals}/pio2/cmake/FindLIBZ.cmake | 0 .../externals}/pio2/cmake/FindMPE.cmake | 0 .../externals}/pio2/cmake/FindMPISERIAL.cmake | 0 .../externals}/pio2/cmake/FindNetCDF.cmake | 5 + .../externals}/pio2/cmake/FindPAPI.cmake | 0 .../externals}/pio2/cmake/FindPnetCDF.cmake | 0 .../externals}/pio2/cmake/FindSZIP.cmake | 0 .../externals}/pio2/cmake/LibCheck.cmake | 28 +- .../externals}/pio2/cmake/LibFind.cmake | 0 .../externals}/pio2/cmake/LibMPI.cmake | 28 +- .../externals}/pio2/cmake/TryHDF5_HAS_SZIP.c | 0 .../externals}/pio2/cmake/TryNetCDF_DAP.c | 0 .../pio2/cmake/TryNetCDF_PARALLEL.c | 0 .../externals}/pio2/cmake/TryNetCDF_PNETCDF.c | 0 .../externals}/pio2/cmake/mpiexec.alcf | 0 .../externals}/pio2/cmake/mpiexec.ncsa | 0 .../externals}/pio2/cmake/mpiexec.nersc | 0 .../externals}/pio2/cmake/mpiexec.nwsc | 0 src/externals/pio2/cmake/mpiexec.nwscla | 12 + src/externals/pio2/cmake/mpiexec.olcf | 12 + .../pio2/ctest/CTestEnvironment-alcf.cmake | 0 .../CTestEnvironment-anlworkstation.cmake | 30 + .../pio2/ctest/CTestEnvironment-cgd.cmake | 0 .../pio2/ctest/CTestEnvironment-ncsa.cmake | 0 .../pio2/ctest/CTestEnvironment-nersc.cmake | 0 .../pio2/ctest/CTestEnvironment-nwsc.cmake | 6 +- .../pio2/ctest/CTestEnvironment-nwscla.cmake | 18 + .../pio2/ctest/CTestEnvironment-unknown.cmake | 0 .../pio2/ctest/CTestScript-Test.cmake | 0 .../pio2/ctest/runcdash-alcf-ibm.sh | 4 +- .../pio2/ctest/runcdash-anlworkstation.sh | 45 + .../pio2/ctest/runcdash-cgd-gnu-openmpi.sh | 38 + .../externals}/pio2/ctest/runcdash-cgd-nag.sh | 14 +- .../pio2/ctest/runcdash-nersc-cray.sh | 74 + .../pio2/ctest/runcdash-nersc-intel.sh | 73 + .../pio2/ctest/runcdash-nwsc-gnu.sh | 10 +- .../ctest/runcdash-nwsc-intel-mpiserial.sh | 6 +- .../pio2/ctest/runcdash-nwsc-intel.sh | 10 +- .../pio2/ctest/runcdash-nwsc-pgi.sh | 10 +- .../pio2/ctest/runcdash-nwscla-intel.sh | 40 + .../externals}/pio2/ctest/runctest-alcf.sh | 0 .../pio2/ctest/runctest-anlworkstation.sh | 18 + .../externals}/pio2/ctest/runctest-cgd.sh | 6 +- .../externals}/pio2/ctest/runctest-ncsa.sh | 0 .../externals}/pio2/ctest/runctest-nersc.sh | 26 +- .../externals}/pio2/ctest/runctest-nwsc.sh | 0 src/externals/pio2/ctest/runctest-nwscla.sh | 44 + .../externals}/pio2/ctest/runctest-unknown.sh | 0 .../doc/CMakeFiles/3.2.3/CMakeSystem.cmake | 0 .../pio2/doc/CMakeFiles/CMakeOutput.log | 0 .../pio2/doc/CMakeFiles/cmake.check_cache | 0 src/externals/pio2/doc/CMakeLists.txt | 43 + .../externals}/pio2/doc/Doxyfile.in | 7 +- .../externals}/pio2/doc/DoxygenLayout.xml | 0 .../externals}/pio2/doc/customdoxygen.css | 0 .../externals}/pio2/doc/doxygen.sty | 0 .../pio2/doc/images/baseimage.graffle | 0 .../pio2/doc/images/block-cyclic-rearr.eps | 0 .../doc/images/block-cyclic-rearr.graffle | 0 .../pio2/doc/images/block-cyclic-rearr.png | Bin .../pio2/doc/images/block-cyclic.eps | 0 .../pio2/doc/images/block-cyclic.graffle | 0 .../pio2/doc/images/block-cyclic.png | Bin .../externals}/pio2/doc/images/dof-rearr.eps | 0 .../pio2/doc/images/dof-rearr.graffle | 0 .../externals}/pio2/doc/images/dof-rearr.png | Bin .../externals}/pio2/doc/images/dof.eps | 0 .../externals}/pio2/doc/images/dof.graffle | 0 .../externals}/pio2/doc/images/dof.png | Bin .../externals}/pio2/doc/source/CAMexample.txt | 0 .../externals}/pio2/doc/source/Decomp.txt | 0 .../externals}/pio2/doc/source/Error.txt | 3 - .../externals}/pio2/doc/source/Examples.txt | 2 + .../pio2/doc/source/GettingStarted.txt | 0 .../externals}/pio2/doc/source/Installing.txt | 2 +- src/externals/pio2/doc/source/Testing.txt | 181 + .../externals}/pio2/doc/source/api.txt | 0 .../externals}/pio2/doc/source/base.txt | 3 + .../pio2/doc/source/contributing_code.txt | 268 + .../pio2/doc/source/example/errorhandle | 0 .../pio2/doc/source/example/simple-bc | 0 .../pio2/doc/source/example/simple-bc-rearr | 0 .../doc/source/example/simple-bc-rearr-pe1 | 0 .../doc/source/example/simple-bc-rearr-pe2 | 0 .../pio2/doc/source/example/simple-dof | 0 .../pio2/doc/source/example/simple-dof-rearr | 0 .../externals}/pio2/doc/source/faq.txt | 0 .../pio2/doc/source/mach_walkthrough.txt | 61 +- .../pio2/doc/source/testpio_example.txt | 0 .../externals}/pio2/examples/CMakeLists.txt | 12 +- .../externals}/pio2/examples/basic/CAM05.csh | 0 .../pio2/examples/basic/CMakeLists.txt | 0 .../pio2/examples/basic/MPASA30km.csh | 0 .../pio2/examples/basic/MPASA60km.csh | 0 .../externals}/pio2/examples/basic/POPB.csh | 0 .../externals}/pio2/examples/basic/POPC.csh | 0 .../externals}/pio2/examples/basic/POPD.csh | 0 .../externals}/pio2/examples/basic/POPDv0.csh | 0 .../externals}/pio2/examples/basic/POPDv1.csh | 0 .../externals}/pio2/examples/basic/POPDv2.csh | 0 .../externals}/pio2/examples/basic/POPDv3.csh | 0 .../externals}/pio2/examples/basic/POPDv4.csh | 0 .../externals}/pio2/examples/basic/POPDv5.csh | 0 .../pio2/examples/basic/README.testpio | 0 .../externals}/pio2/examples/basic/WRFB.csh | 0 .../pio2/examples/basic/alloc_mod.F90.in | 0 .../pio2/examples/basic/build_defaults.xml | 0 .../pio2/examples/basic/check_mod.F90 | 0 .../pio2/examples/basic/config_bench.xml | 0 .../pio2/examples/basic/fdepends.awk | 0 .../pio2/examples/basic/gdecomp_mod.F90 | 0 .../pio2/examples/basic/kinds_mod.F90 | 0 .../pio2/examples/basic/kraken.128.csh | 0 .../pio2/examples/basic/kraken.1K.csh | 0 .../pio2/examples/basic/kraken.256.csh | 0 .../pio2/examples/basic/kraken.512.csh | 0 .../pio2/examples/basic/kraken.64.csh | 0 .../pio2/examples/basic/namelist_mod.F90 | 0 .../examples/basic/namelists/testpio_in.apb05 | 0 .../examples/basic/namelists/testpio_in.asb01 | 0 .../examples/basic/namelists/testpio_in.asb04 | 0 .../examples/basic/namelists/testpio_in.n4b01 | 0 .../examples/basic/namelists/testpio_in.n4b02 | 0 .../examples/basic/namelists/testpio_in.n4b03 | 0 .../examples/basic/namelists/testpio_in.n4b04 | 0 .../examples/basic/namelists/testpio_in.n4b05 | 0 .../examples/basic/namelists/testpio_in.n4b06 | 0 .../examples/basic/namelists/testpio_in.n4b07 | 0 .../examples/basic/namelists/testpio_in.n4b08 | 0 .../examples/basic/namelists/testpio_in.n4n01 | 0 .../examples/basic/namelists/testpio_in.n4n02 | 0 .../examples/basic/namelists/testpio_in.n4n03 | 0 .../examples/basic/namelists/testpio_in.pb01 | 0 .../examples/basic/namelists/testpio_in.pb02 | 0 .../examples/basic/namelists/testpio_in.pb03 | 0 .../examples/basic/namelists/testpio_in.pb04 | 0 .../examples/basic/namelists/testpio_in.pb05 | 0 .../examples/basic/namelists/testpio_in.pb06 | 0 .../examples/basic/namelists/testpio_in.pb07 | 0 .../examples/basic/namelists/testpio_in.pb08 | 0 .../examples/basic/namelists/testpio_in.pn01 | 0 .../examples/basic/namelists/testpio_in.pn02 | 0 .../examples/basic/namelists/testpio_in.pn03 | 0 .../examples/basic/namelists/testpio_in.ps01 | 0 .../examples/basic/namelists/testpio_in.ps02 | 0 .../examples/basic/namelists/testpio_in.ps03 | 0 .../examples/basic/namelists/testpio_in.ps04 | 0 .../examples/basic/namelists/testpio_in.ps05 | 0 .../examples/basic/namelists/testpio_in.ps06 | 0 .../examples/basic/namelists/testpio_in.ps07 | 0 .../examples/basic/namelists/testpio_in.ps08 | 0 .../examples/basic/namelists/testpio_in.sb01 | 0 .../examples/basic/namelists/testpio_in.sb02 | 0 .../examples/basic/namelists/testpio_in.sb03 | 0 .../examples/basic/namelists/testpio_in.sb04 | 0 .../examples/basic/namelists/testpio_in.sb05 | 0 .../examples/basic/namelists/testpio_in.sb06 | 0 .../examples/basic/namelists/testpio_in.sb07 | 0 .../examples/basic/namelists/testpio_in.sb08 | 0 .../examples/basic/namelists/testpio_in.sn01 | 0 .../examples/basic/namelists/testpio_in.sn02 | 0 .../examples/basic/namelists/testpio_in.sn03 | 0 .../examples/basic/namelists/testpio_in.wr01 | 0 .../examples/basic/namelists/testpio_in.wr02 | 0 .../pio2/examples/basic/perl5lib/ChangeLog | 0 .../pio2/examples/basic/perl5lib/README | 0 .../pio2/examples/basic/perl5lib/XML/Changes | 0 .../pio2/examples/basic/perl5lib/XML/Lite.pm | 0 .../basic/perl5lib/XML/Lite/Element.pm | 0 .../pio2/examples/basic/perl5lib/XML/README | 0 .../basic/perl5lib/XML/man3/XML::Lite.3 | 0 .../perl5lib/XML/man3/XML::Lite::Element.3 | 0 .../externals}/pio2/examples/basic/test.csh | 0 .../pio2/examples/basic/testdecomp.F90 | 0 .../examples/basic/testdecomp.bluefire.run | 0 .../pio2/examples/basic/testdecomp_in | 0 .../pio2/examples/basic/testpio.F90 | 0 .../pio2/examples/basic/testpio_bench.pl | 0 .../pio2/examples/basic/testpio_build.pl | 0 .../pio2/examples/basic/testpio_run.pl | 0 .../pio2/examples/basic/utils_mod.F90 | 0 .../externals}/pio2/examples/basic/wstest.c | 0 .../externals}/pio2/examples/basic/ystest.sh | 0 src/externals/pio2/examples/c/CMakeLists.txt | 68 + src/externals/pio2/examples/c/darray_async.c | 387 ++ .../pio2/examples/c/darray_no_async.c | 358 ++ .../externals}/pio2/examples/c/example1.c | 63 +- .../externals}/pio2/examples/c/example2.c | 7 +- .../externals}/pio2/examples/c/examplePio.c | 24 +- .../pio2/examples/c/valsupp_example1.supp | 15 + .../pio2/examples/cxx/CMakeLists.txt | 0 .../pio2/examples/cxx/examplePio.cxx | 0 .../pio2/examples/f03/CMakeLists.txt | 3 +- .../pio2/examples/f03/examplePio.f90 | 2 +- .../externals}/pio2/scripts/prune_decomps.pl | 0 .../externals}/pio2/src/CMakeLists.txt | 20 +- src/externals/pio2/src/clib/CMakeLists.txt | 162 + src/externals/pio2/src/clib/bget.c | 1677 ++++++ src/externals/pio2/src/clib/bget.h | 41 + .../externals}/pio2/src/clib/config.h.in | 6 + .../externals}/pio2/src/clib/dtypes.h | 0 .../externals}/pio2/src/clib/ncparser.pl | 0 .../pio2/src/clib/ncputgetparser.pl | 0 src/externals/pio2/src/clib/pio.h | 1189 ++++ .../pio2/src/clib/pio_c_get_template.c | 0 .../pio2/src/clib/pio_c_put_template.c | 0 .../externals}/pio2/src/clib/pio_c_template.c | 0 src/externals/pio2/src/clib/pio_darray.c | 690 +++ src/externals/pio2/src/clib/pio_darray_int.c | 1522 +++++ src/externals/pio2/src/clib/pio_file.c | 422 ++ src/externals/pio2/src/clib/pio_get_nc.c | 1119 ++++ src/externals/pio2/src/clib/pio_getput_int.c | 1329 +++++ src/externals/pio2/src/clib/pio_internal.h | 564 ++ src/externals/pio2/src/clib/pio_lists.c | 305 + src/externals/pio2/src/clib/pio_msg.c | 2530 ++++++++ src/externals/pio2/src/clib/pio_nc.c | 2756 +++++++++ src/externals/pio2/src/clib/pio_nc4.c | 1031 ++++ src/externals/pio2/src/clib/pio_put_nc.c | 1190 ++++ src/externals/pio2/src/clib/pio_rearrange.c | 2103 +++++++ src/externals/pio2/src/clib/pio_spmd.c | 494 ++ src/externals/pio2/src/clib/pio_varm.c | 1811 ++++++ src/externals/pio2/src/clib/pioc.c | 1541 +++++ src/externals/pio2/src/clib/pioc_sc.c | 440 ++ src/externals/pio2/src/clib/pioc_support.c | 2280 ++++++++ src/externals/pio2/src/clib/topology.c | 363 ++ src/externals/pio2/src/flib/CMakeLists.txt | 268 + .../externals}/pio2/src/flib/pio.F90 | 25 +- .../externals}/pio2/src/flib/pio_kinds.F90 | 0 .../externals}/pio2/src/flib/pio_nf.F90 | 332 +- .../externals}/pio2/src/flib/pio_support.F90 | 0 .../externals}/pio2/src/flib/pio_types.F90 | 78 +- .../externals}/pio2/src/flib/piodarray.F90.in | 42 +- .../externals}/pio2/src/flib/piolib_mod.F90 | 123 +- .../pio2/src/flib/pionfatt_mod.F90.in | 57 +- .../pio2/src/flib/pionfget_mod.F90.in | 16 +- .../pio2/src/flib/pionfput_mod.F90.in | 44 +- src/externals/pio2/src/gptl/CMakeLists.txt | 156 + src/externals/pio2/src/gptl/COPYING | 17 + .../externals}/pio2/src/gptl/ChangeLog | 0 .../pio2/src/gptl/GPTLget_memusage.c | 0 .../pio2/src/gptl/GPTLprint_memusage.c | 0 .../externals}/pio2/src/gptl/GPTLutil.c | 0 .../externals}/pio2/src/gptl/README | 0 .../externals}/pio2/src/gptl/f_wrappers.c | 0 .../externals}/pio2/src/gptl/gptl.c | 18 +- .../externals}/pio2/src/gptl/gptl.h | 0 .../externals}/pio2/src/gptl/gptl.inc | 0 .../externals}/pio2/src/gptl/gptl_papi.c | 0 .../externals}/pio2/src/gptl/perf_mod.F90 | 2 +- .../externals}/pio2/src/gptl/perf_utils.F90 | 0 .../externals}/pio2/src/gptl/private.h | 0 .../externals}/pio2/src/gptl/threadutil.c | 0 .../externals}/pio2/tests/CMakeLists.txt | 23 +- src/externals/pio2/tests/cunit/CMakeLists.txt | 216 + src/externals/pio2/tests/cunit/pio_tests.h | 103 + .../pio2/tests/cunit/test_async_3proc.c | 128 + .../pio2/tests/cunit/test_async_4proc.c | 121 + .../pio2/tests/cunit/test_async_simple.c | 130 + src/externals/pio2/tests/cunit/test_common.c | 926 +++ src/externals/pio2/tests/cunit/test_darray.c | 375 ++ .../pio2/tests/cunit/test_darray_1d.c | 891 +++ .../pio2/tests/cunit/test_darray_3d.c | 422 ++ .../pio2/tests/cunit/test_darray_multi.c | 473 ++ .../pio2/tests/cunit/test_darray_multivar.c | 604 ++ .../pio2/tests/cunit/test_darray_multivar2.c | 287 + .../pio2/tests/cunit/test_decomp_uneven.c | 380 ++ src/externals/pio2/tests/cunit/test_decomps.c | 420 ++ .../pio2/tests/cunit/test_intercomm2.c | 558 ++ .../pio2/tests/cunit/test_iosystem2.c | 226 + .../pio2/tests/cunit/test_iosystem2_simple.c | 235 + .../pio2/tests/cunit/test_iosystem2_simple2.c | 142 + .../pio2/tests/cunit/test_iosystem3.c | 375 ++ .../pio2/tests/cunit/test_iosystem3_simple.c | 123 + .../pio2/tests/cunit/test_iosystem3_simple2.c | 101 + src/externals/pio2/tests/cunit/test_pioc.c | 2138 +++++++ .../pio2/tests/cunit/test_pioc_fill.c | 820 +++ .../pio2/tests/cunit/test_pioc_putget.c | 2082 +++++++ .../pio2/tests/cunit/test_pioc_unlim.c | 188 + src/externals/pio2/tests/cunit/test_rearr.c | 1542 +++++ src/externals/pio2/tests/cunit/test_shared.c | 198 + src/externals/pio2/tests/cunit/test_spmd.c | 532 ++ .../pio2/tests/general/CMakeLists.txt | 832 +++ src/externals/pio2/tests/general/README.md | 142 + .../pio2/tests/general/ncdf_fail.F90.in | 14 +- .../pio2/tests/general/ncdf_get_put.F90.in | 569 ++ .../pio2/tests/general/ncdf_inq.F90.in | 192 + .../tests/general/ncdf_simple_tests.F90.in | 0 .../tests/general/pio_decomp_fillval.F90.in | 0 .../tests/general/pio_decomp_fillval2.F90 | 0 .../general/pio_decomp_frame_tests.F90.in | 420 ++ .../tests/general/pio_decomp_tests.F90.in | 305 + .../tests/general/pio_decomp_tests_1d.F90.in | 306 + .../tests/general/pio_decomp_tests_2d.F90.in | 367 ++ .../tests/general/pio_decomp_tests_3d.F90.in | 209 + .../pio2/tests/general/pio_fail.F90.in | 0 .../pio2/tests/general/pio_file_fail.F90.in | 0 .../general/pio_file_simple_tests.F90.in | 3 +- .../tests/general/pio_init_finalize.F90.in | 0 .../tests/general/pio_iosystem_tests.F90.in | 309 + .../tests/general/pio_iosystem_tests2.F90.in | 238 + .../tests/general/pio_iosystem_tests3.F90.in | 284 + .../pio2/tests/general/pio_rearr.F90.in | 252 + .../pio2/tests/general/pio_rearr_opts.F90.in | 483 ++ .../pio2/tests/general/pio_rearr_opts2.F90.in | 420 ++ .../pio2/tests/general/test_memleak.c | 68 +- .../pio2/tests/general/util/pio_tf_f90gen.pl | 86 +- .../pio2/tests/general/util/pio_tutil.F90 | 290 +- .../pio2/tests/performance/CMakeLists.txt | 18 +- .../pio2/tests/performance/gensimple.pl | 0 .../tests/performance/kt.PIO1.perfmakefile | 0 .../pio2/tests/performance/pioperformance.F90 | 0 src/externals/pio2/tests/unit/CMakeLists.txt | 63 + .../externals}/pio2/tests/unit/Levy_Notes | 0 .../externals}/pio2/tests/unit/README | 0 .../pio2/tests/unit/basic_tests.F90 | 53 +- .../externals}/pio2/tests/unit/driver.F90 | 29 +- .../pio2/tests/unit/global_vars.F90 | 2 +- .../externals}/pio2/tests/unit/input.nl | 0 .../pio2/tests/unit/nc_set_log_level2.c | 6 +- .../externals}/pio2/tests/unit/ncdf_tests.F90 | 6 +- .../pio2/tests/unit/not_netcdf.ieee | Bin .../externals}/pio2/tests/unit/test_names.c | 0 .../externals}/pio2/tests/unit/test_nc4.c | 0 {share/csm_share => src/share}/README | 10 +- .../share/RandNum}/include/dSFMT-common.h | 0 .../share/RandNum}/include/dSFMT-params.h | 0 .../RandNum}/include/dSFMT-params19937.h | 0 .../share/RandNum}/include/dSFMT.h | 0 .../share/RandNum}/src/dsfmt_f03/dSFMT.c | 0 .../src/dsfmt_f03/dSFMT_interface.F90 | 0 .../RandNum}/src/dsfmt_f03/dSFMT_utils.c | 0 .../share/RandNum}/src/kissvec/kissvec.c | 0 .../RandNum}/src/kissvec/kissvec_mod.F90 | 0 .../src/mt19937/mersennetwister_mod.F90 | 0 .../share/RandNum}/src/shr_RandNum_mod.F90 | 0 .../share/RandNum}/test/bench/Makefile | 0 .../RandNum}/test/bench/test_shr_RandNum.F90 | 0 .../share}/esmf_wrf_timemgr/CMakeLists.txt | 0 .../share}/esmf_wrf_timemgr/ESMF.F90 | 0 .../esmf_wrf_timemgr/ESMF_AlarmClockMod.F90 | 0 .../share}/esmf_wrf_timemgr/ESMF_AlarmMod.F90 | 0 .../share}/esmf_wrf_timemgr/ESMF_BaseMod.F90 | 0 .../esmf_wrf_timemgr/ESMF_BaseTimeMod.F90 | 0 .../esmf_wrf_timemgr/ESMF_CalendarMod.F90 | 0 .../share}/esmf_wrf_timemgr/ESMF_ClockMod.F90 | 0 .../esmf_wrf_timemgr/ESMF_FractionMod.F90 | 0 .../share}/esmf_wrf_timemgr/ESMF_Macros.inc | 0 .../esmf_wrf_timemgr/ESMF_ShrTimeMod.F90 | 0 .../share}/esmf_wrf_timemgr/ESMF_Stubs.F90 | 39 +- .../esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90 | 0 .../share}/esmf_wrf_timemgr/ESMF_TimeMgr.inc | 0 .../share}/esmf_wrf_timemgr/ESMF_TimeMod.F90 | 0 .../share}/esmf_wrf_timemgr/Makefile | 0 .../share}/esmf_wrf_timemgr/MeatMod.F90 | 0 {share => src/share}/esmf_wrf_timemgr/README | 0 .../esmf_wrf_timemgr/unittests/Makefile | 0 .../share}/esmf_wrf_timemgr/unittests/go.csh | 0 .../esmf_wrf_timemgr/unittests/test.F90 | 0 .../esmf_wrf_timemgr/unittests/wrf_stuff.F90 | 0 .../esmf_wrf_timemgr/wrf_error_fatal.F90 | 0 .../share}/esmf_wrf_timemgr/wrf_message.F90 | 0 .../share}/include/dynamic_vector_procdef.inc | 0 .../share}/include/dynamic_vector_typedef.inc | 0 src/share/include/shr_assert.h | 22 + .../share}/test/old_unit_testers/Makefile | 0 .../share}/test/old_unit_testers/Mkdepends | 0 .../share}/test/old_unit_testers/Mksrcfiles | 0 .../test/old_unit_testers/bundle_expected.F90 | 0 .../share}/test/old_unit_testers/config.h | 0 .../share}/test/old_unit_testers/make.Macros | 0 .../share}/test/old_unit_testers/namelist | 0 .../share}/test/old_unit_testers/nl/atm.stdin | 0 .../share}/test/old_unit_testers/nl/cpl.stdin | 0 .../share}/test/old_unit_testers/nl/ice.stdin | 0 .../share}/test/old_unit_testers/nl/lnd.stdin | 0 .../share}/test/old_unit_testers/nl/ocn.stdin | 0 .../old_unit_testers/run_dshr_bundle_test | 0 .../test/old_unit_testers/run_file_test | 0 .../share}/test/old_unit_testers/test_mod.F90 | 0 .../test/old_unit_testers/test_shr_file.F90 | 0 .../test/old_unit_testers/test_shr_log.F90 | 0 .../test/old_unit_testers/test_shr_mpi.F90 | 0 .../test/old_unit_testers/test_shr_orb.F90 | 0 .../test/old_unit_testers/test_shr_scam.F90 | 0 .../old_unit_testers/test_shr_streams.F90 | 0 .../test/old_unit_testers/test_shr_sys.F90 | 0 .../old_unit_testers/test_shr_tInterp.F90 | 0 .../share}/test/unit/CMakeLists.txt | 0 .../test/unit/dynamic_vector/CMakeLists.txt | 0 .../character16_vector_tests.pf.in | 0 .../dynamic_vector_base_tests.inc | 0 .../dynamic_vector_character16.F90 | 0 .../dynamic_vector/dynamic_vector_int_ptr.F90 | 0 .../dynamic_vector/dynamic_vector_integer.F90 | 0 .../unit/dynamic_vector/dynamic_vector_r8.F90 | 0 .../dynamic_vector/int_ptr_vector_tests.pf.in | 0 .../dynamic_vector/integer_vector_tests.pf.in | 0 .../test/unit/dynamic_vector/ptr_wrapper.F90 | 0 .../unit/dynamic_vector/r8_vector_tests.pf.in | 0 .../share}/test/unit/mock/CMakeLists.txt | 0 .../mock/shr_sys_mod.nompi_abortthrows.F90 | 0 .../test/unit/shr_assert_test/CMakeLists.txt | 0 .../test/unit/shr_assert_test/test_assert.pf | 12 +- .../unit/shr_assert_test/test_assert_array.pf | 8 +- .../test/unit/shr_assert_test/test_macro.pf | 87 + .../test/unit/shr_assert_test/test_ndebug.pf | 63 + .../test/unit/shr_infnan_test/CMakeLists.txt | 0 .../test/unit/shr_infnan_test/test_infnan.F90 | 0 .../test/unit/shr_log_test/CMakeLists.txt | 0 .../unit/shr_log_test/test_error_printers.pf | 0 .../test/unit/shr_precip_test/CMakeLists.txt | 0 .../unit/shr_precip_test/test_shr_precip.pf | 0 .../test/unit/shr_spfn_test/CMakeLists.txt | 0 .../test/unit/shr_spfn_test/test_erf_r4.pf | 0 .../test/unit/shr_spfn_test/test_erf_r8.pf | 0 .../shr_spfn_test/test_gamma_factorial.pf | 0 .../test/unit/shr_spfn_test/test_igamma.pf | 0 .../unit/shr_strconvert_test/CMakeLists.txt | 0 .../unit/shr_strconvert_test/test_toString.pf | 0 .../test/unit/shr_string_test/CMakeLists.txt | 0 .../unit/shr_string_test/test_shr_string.pf | 0 .../test/unit/shr_vmath_test/CMakeLists.txt | 0 .../test/unit/shr_vmath_test/test_vmath.F90 | 0 .../test/unit/shr_wv_sat_test/CMakeLists.txt | 0 .../test/unit/shr_wv_sat_test/test_wv_sat.pf | 0 .../test_wv_sat_each_method.pf | 0 {share => src/share}/timing/CMakeLists.txt | 0 src/share/timing/COPYING | 17 + {share => src/share}/timing/ChangeLog | 0 .../share}/timing/GPTLget_memusage.c | 0 .../share}/timing/GPTLprint_memusage.c | 0 {share => src/share}/timing/GPTLutil.c | 0 {share => src/share}/timing/Makefile | 0 {share => src/share}/timing/README | 0 {share => src/share}/timing/f_wrappers.c | 0 {share => src/share}/timing/gptl.c | 0 {share => src/share}/timing/gptl.h | 0 {share => src/share}/timing/gptl.inc | 0 {share => src/share}/timing/gptl_papi.c | 0 {share => src/share}/timing/perf_mod.F90 | 0 {share => src/share}/timing/perf_utils.F90 | 0 {share => src/share}/timing/private.h | 0 .../share}/unit_test_stubs/README | 0 .../share}/unit_test_stubs/pio/CMakeLists.txt | 0 .../share}/unit_test_stubs/pio/README | 0 .../share}/unit_test_stubs/pio/pio.F90.in | 44 +- .../shr => src/share/util}/CMakeLists.txt | 0 .../shr => src/share/util}/mct_mod.F90 | 1 - .../share/util}/shr_assert_mod.F90.in | 60 +- .../shr => src/share/util}/shr_cal_mod.F90 | 0 .../shr => src/share/util}/shr_const_mod.F90 | 0 .../shr => src/share/util}/shr_dmodel_mod.F90 | 0 .../shr => src/share/util}/shr_file_mod.F90 | 0 .../shr => src/share/util}/shr_flux_mod.F90 | 0 .../shr => src/share/util}/shr_frz_mod.F90.in | 0 .../share/util}/shr_infnan_mod.F90.in | 0 .../shr => src/share/util}/shr_kind_mod.F90 | 0 .../shr => src/share/util}/shr_log_mod.F90 | 7 + .../shr => src/share/util}/shr_map_mod.F90 | 0 .../shr => src/share/util}/shr_mct_mod.F90 | 0 .../shr => src/share/util}/shr_mem_mod.F90 | 0 .../shr => src/share/util}/shr_mpi_mod.F90 | 0 .../shr => src/share/util}/shr_msg_mod.F90 | 0 .../shr => src/share/util}/shr_ncread_mod.F90 | 0 .../shr => src/share/util}/shr_nl_mod.F90 | 0 .../shr => src/share/util}/shr_orb_mod.F90 | 0 .../shr => src/share/util}/shr_pcdf_mod.F90 | 0 .../shr => src/share/util}/shr_pio_mod.F90 | 87 +- .../shr => src/share/util}/shr_precip_mod.F90 | 0 .../share/util}/shr_reprosum_mod.F90 | 0 .../shr => src/share/util}/shr_reprosumx86.c | 0 .../shr => src/share/util}/shr_scam_mod.F90 | 0 .../shr => src/share/util}/shr_spfn_mod.F90 | 0 .../share/util}/shr_strconvert_mod.F90 | 0 .../share/util}/shr_strdata_mod.F90 | 18 +- .../shr => src/share/util}/shr_stream_mod.F90 | 0 .../shr => src/share/util}/shr_string_mod.F90 | 2 +- .../shr => src/share/util}/shr_sys_mod.F90 | 0 .../share/util}/shr_tInterp_mod.F90 | 0 .../shr => src/share/util}/shr_timer_mod.F90 | 0 .../shr => src/share/util}/shr_vmath_mod.F90 | 0 .../shr => src/share/util}/shr_wv_sat_mod.F90 | 0 .../shr => src/share/util}/water_isotopes.F90 | 0 .../shr => src/share/util}/water_types.F90 | 0 tools/Readme.configure | 74 +- tools/configure | 43 +- tools/cprnc/CMakeLists.txt | 7 +- tools/cprnc/Makefile | 2 +- tools/cprnc/README | 4 +- tools/cprnc/summarize_cprnc_diffs | 137 +- tools/mapping/check_maps/.gitignore | 2 + tools/mapping/check_maps/README | 32 +- .../src/ESMF_RegridWeightGenCheck.F90 | 2 + tools/mapping/check_maps/src/Makefile | 13 +- tools/mapping/gen_domain_files/INSTALL | 21 +- tools/mapping/gen_domain_files/src/Makefile | 18 +- tools/mapping/gen_mapping_files/README | 4 +- .../gen_ESMF_mapping_file/.gitignore | 2 + .../gen_ESMF_mapping_file/README | 4 +- .../gen_ESMF_mapping_file/create_ESMF_map.sh | 24 +- .../gen_mapping_files/runoff_to_ocn/INSTALL | 21 +- .../gen_mapping_files/runoff_to_ocn/README | 211 +- .../map_r05_to_gx1v7_nnsm_e1000r300.nml | 16 + .../map_r05_to_gx3v7_nnsm_e1000r500.nml | 16 + .../map_rx1_to_gx1v7_nnsm_e1000r300.nml | 16 + .../map_rx1_to_gx3v7_nnsm_e1000r500.nml | 16 + .../map_wr50a_to_ar9v4_nnsm_e1000r300.nml | 14 + .../map_wr50a_to_gx3v7_nnsm_e1000r300.nml | 14 + .../runoff_to_ocn/maps/.gitignore | 1 + .../runoff_to_ocn/ncl/merge_mapping_files.ncl | 210 + .../runoff_to_ocn/run_merge_mapping_files.sh | 130 + .../runoff_to_ocn/src/Makefile | 18 +- .../runoff_to_ocn/src/fixroff_mod.F90 | 5 - .../runoff_to_ocn/src/kind_mod.F90 | 5 - .../runoff_to_ocn/src/main.F90 | 127 +- .../runoff_to_ocn/src/map_mod.F90 | 206 +- .../runoff_to_ocn/src/mapsort_mod.F90 | 5 - .../runoff_to_ocn/src/shr_kind_mod.F90 | 5 - .../runoff_to_ocn/src/shr_sys_mod.F90 | 5 - .../runoff_to_ocn/src/shr_timer_mod.F90 | 5 - .../runoff_to_ocn/src/smooth_mod.F90 | 97 +- tools/mapping/map_field/INSTALL | 21 +- tools/mapping/map_field/src/Makefile | 26 +- tools/mapping/map_field/src/map_field.F90 | 3 +- tools/unit_testing/python/comparable.py | 29 - tools/unit_testing/python/environment.py | 263 - tools/unit_testing/python/machine_setup.py | 296 - tools/unit_testing/python/test_environment.py | 237 - tools/unit_testing/python/test_xml_utils.py | 294 - tools/unit_testing/python/xml_utils.py | 202 - utils/perl5lib/Build/Namelist.pm | 7 +- utils/perl5lib/compilers_translation_tool.pl | 2 +- utils/python/cs.status.template | 3 - 1585 files changed, 65285 insertions(+), 41954 deletions(-) delete mode 100644 README delete mode 100644 README.unit_testing delete mode 100644 cime_config/acme/allactive/config_compsets.xml.cime2 delete mode 100644 cime_config/acme/allactive/config_pes.xml delete mode 100644 cime_config/acme/allactive/config_pes.xml.cime2 delete mode 100644 cime_config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands delete mode 100755 cime_config/acme/machines/syslog.titan delete mode 100644 cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/xmlchange_cmnds rename {cime_config => config}/acme/allactive/config_compsets.xml (78%) rename {cime_config => config}/acme/allactive/config_pesall.xml (94%) rename {cime_config => config}/acme/allactive/testlist_allactive.xml (100%) rename {cime_config => config}/acme/allactive/usermods_dirs/b1850/shell_commands (100%) rename {cime_config => config}/acme/allactive/usermods_dirs/b1850_cism/include_user_mods (100%) rename {cime_config => config}/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism (100%) rename {cime_config => config}/acme/config_archive.xml (64%) rename {cime_config => config}/acme/config_files.xml (61%) rename {cime_config => config}/acme/config_grids.xml (98%) rename {cime_config => config}/acme/machines/Depends.cetus (100%) rename {cime_config => config}/acme/machines/Depends.corip1 (100%) rename {cime_config => config}/acme/machines/Depends.cray (100%) rename {cime_config => config}/acme/machines/Depends.gnu (100%) rename {cime_config => config}/acme/machines/Depends.intel (100%) rename {cime_config => config}/acme/machines/Depends.intel14 (100%) rename {cime_config => config}/acme/machines/Depends.intelmic (100%) rename {cime_config => config}/acme/machines/Depends.intelmic14 (100%) rename {cime_config => config}/acme/machines/Depends.mira (100%) rename {cime_config => config}/acme/machines/Depends.nag (100%) rename {cime_config => config}/acme/machines/Depends.titan.cray (100%) rename {cime_config => config}/acme/machines/Depends.titan.intel (100%) rename {cime_config => config}/acme/machines/Depends.titan.pgi (100%) rename {cime_config => config}/acme/machines/Depends.titan.pgiacc (100%) rename {cime_config => config}/acme/machines/Makefile (93%) rename {cime_config => config}/acme/machines/README (100%) rename {cime_config => config}/acme/machines/config_batch.xml (82%) rename {cime_config => config}/acme/machines/config_batch.xsd (100%) rename {cime_config => config}/acme/machines/config_compilers.xml (90%) rename {cime_config => config}/acme/machines/config_lt_archive.xml (100%) rename {cime_config => config}/acme/machines/config_machines.xml (80%) rename {cime_config => config}/acme/machines/config_pio.xml (96%) rename {cime_config => config}/acme/machines/mkDepends (100%) rename {cime_config => config}/acme/machines/mkSrcfiles (100%) rename {cime_config => config}/acme/machines/syslog.anvil (93%) rename {cime_config => config}/acme/machines/syslog.cetus (56%) rename {cime_config => config}/acme/machines/syslog.cori-haswell (52%) rename {cime_config => config}/acme/machines/syslog.cori-knl (52%) rename {cime_config => config}/acme/machines/syslog.edison (51%) rename {cime_config => config}/acme/machines/syslog.mira (56%) rename {cime_config => config}/acme/machines/syslog.noop (100%) create mode 100755 config/acme/machines/syslog.titan rename {cime_config => config}/acme/machines/template.case.run (95%) rename {cime_config => config}/acme/machines/template.case.test (95%) rename {cime_config/cesm => config/acme}/machines/template.lt_archive (96%) rename {cime_config => config}/acme/machines/template.ocn_diagnostics (100%) rename {cime_config/cesm => config/acme}/machines/template.st_archive (53%) rename {cime_config => config}/acme/machines/template.timeseries (100%) rename {cime_config => config}/acme/machines/userdefined_laptop_template/README.md (100%) rename {cime_config => config}/acme/machines/userdefined_laptop_template/config_compilers.xml (92%) rename {cime_config => config}/acme/machines/userdefined_laptop_template/config_machines.xml (100%) rename {cime_config => config}/acme/machines/userdefined_laptop_template/config_pes.xml (100%) create mode 100644 config/acme/testmods_dirs/allactive/cam/cosplite/user_nl_cam create mode 100644 config/acme/testmods_dirs/allactive/cam/cosplite/xmlchange_cmnds create mode 100644 config/acme/testmods_dirs/allactive/cam/cosplite_nhtfrq5/user_nl_cam create mode 100644 config/acme/testmods_dirs/allactive/cam/cosplite_nhtfrq5/xmlchange_cmnds rename {cime_config => config}/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_cam (100%) rename {cime_config => config}/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_clm (100%) create mode 100644 config/acme/testmods_dirs/allactive/cam/outfrq9s/xmlchange_cmnds create mode 100644 config/acme/testmods_dirs/allactive/cam/rtm_null/xmlchange_cmnds create mode 100644 config/acme/testmods_dirs/allactive/clm/betr/xmlchange_cmnds create mode 100644 config/acme/testmods_dirs/allactive/clm/eca/user_nl_clm create mode 100644 config/acme/testmods_dirs/allactive/clm/fates/user_nl_clm rename {cime_config => config}/acme/testmods_dirs/allactive/force_netcdf_pio/shell_commands (100%) rename {cime_config => config}/acme/testmods_dirs/allactive/force_netcdf_pio/xmlchange_cmnds (100%) rename {cime_config => config}/cesm/config_archive.xml (100%) rename {cime_config => config}/cesm/config_files.xml (65%) rename {cime_config => config}/cesm/config_grids.xml (81%) rename {cime_config => config}/cesm/machines/Depends.babbageKnc (100%) rename {cime_config => config}/cesm/machines/Depends.bluewaters (100%) rename {cime_config => config}/cesm/machines/Depends.corip1 (100%) rename {cime_config => config}/cesm/machines/Depends.cray (100%) rename {cime_config => config}/cesm/machines/Depends.gnu (100%) rename {cime_config => config}/cesm/machines/Depends.goldbach (100%) rename {cime_config => config}/cesm/machines/Depends.intel (100%) rename {cime_config => config}/cesm/machines/Depends.intel14 (100%) rename {cime_config => config}/cesm/machines/Depends.intelmic (100%) rename {cime_config => config}/cesm/machines/Depends.intelmic14 (100%) rename {cime_config => config}/cesm/machines/Depends.mira (100%) rename {cime_config => config}/cesm/machines/Depends.nag (100%) rename {cime_config => config}/cesm/machines/Makefile (96%) rename {cime_config => config}/cesm/machines/README (100%) rename {cime_config => config}/cesm/machines/config_batch.xml (99%) rename {cime_config => config}/cesm/machines/config_compilers.xml (95%) rename {cime_config => config}/cesm/machines/config_lt_archive.xml (100%) rename {cime_config => config}/cesm/machines/config_machines.xml (97%) rename {cime_config => config}/cesm/machines/config_pio.xml (94%) rename {cime_config => config}/cesm/machines/mkDepends (100%) rename {cime_config => config}/cesm/machines/mkSrcfiles (100%) rename {cime_config => config}/cesm/machines/nag_mpi_argument.txt (100%) rename {cime_config => config}/cesm/machines/template.case.run (95%) rename {cime_config => config}/cesm/machines/template.case.test (95%) rename {cime_config/acme => config/cesm}/machines/template.lt_archive (95%) rename {cime_config/acme => config/cesm}/machines/template.st_archive (77%) rename {cime_config => config}/cesm/machines/userdefined_laptop_template/README.md (100%) rename {cime_config => config}/cesm/machines/userdefined_laptop_template/config_compilers.xml (92%) rename {cime_config => config}/cesm/machines/userdefined_laptop_template/config_machines.xml (100%) rename {cime_config => config}/cesm/machines/userdefined_laptop_template/config_pes.xml (100%) rename {cime_config => config}/config_headers.xml (89%) rename {cime_config => config}/config_tests.xml (85%) rename {cime_config => config}/xml_schemas/cimeteststatus.xsd (100%) rename {cime_config => config}/xml_schemas/config_archive.xsd (100%) rename {cime_config => config}/xml_schemas/config_batch.xsd (98%) rename {cime_config => config}/xml_schemas/config_compilers_v2.xsd (100%) rename {cime_config => config}/xml_schemas/config_compsets.xsd (95%) rename {cime_config => config}/xml_schemas/config_grids_v2.xsd (96%) rename {cime_config => config}/xml_schemas/config_machines.xsd (98%) rename {cime_config => config}/xml_schemas/config_machines_template.xml (100%) rename {cime_config => config}/xml_schemas/config_pes.xsd (100%) rename {cime_config => config}/xml_schemas/entry_id.xsd (88%) rename {cime_config => config}/xml_schemas/entry_id_base.xsd (95%) rename {cime_config => config}/xml_schemas/entry_id_namelist.xsd (93%) create mode 100644 config/xml_schemas/env_archive.xsd create mode 100644 config/xml_schemas/env_entry_id.xsd rename {cime_config => config}/xml_schemas/testlist.xsd (100%) delete mode 100644 externals/mct/mpi-serial/NOTES delete mode 100644 externals/mct/mpi-serial/mpif.h delete mode 100644 externals/pio1/timing/COPYING delete mode 100644 externals/pio2/CMakeLists.txt delete mode 100755 externals/pio2/ctest/runcdash-nersc-cray.sh delete mode 100755 externals/pio2/ctest/runcdash-nersc-intel.sh delete mode 100644 externals/pio2/doc/CMakeLists.txt delete mode 100644 externals/pio2/examples/c/CMakeLists.txt delete mode 100644 externals/pio2/src/clib/CMakeLists.txt delete mode 100644 externals/pio2/src/clib/bget.c delete mode 100644 externals/pio2/src/clib/bget.h delete mode 100644 externals/pio2/src/clib/pio.h delete mode 100644 externals/pio2/src/clib/pio_darray.c delete mode 100644 externals/pio2/src/clib/pio_file.c delete mode 100644 externals/pio2/src/clib/pio_get_nc.c delete mode 100644 externals/pio2/src/clib/pio_internal.h delete mode 100644 externals/pio2/src/clib/pio_lists.c delete mode 100644 externals/pio2/src/clib/pio_nc.c delete mode 100644 externals/pio2/src/clib/pio_nc4.c delete mode 100644 externals/pio2/src/clib/pio_put_nc.c delete mode 100644 externals/pio2/src/clib/pio_rearrange.c delete mode 100644 externals/pio2/src/clib/pio_spmd.c delete mode 100644 externals/pio2/src/clib/pioc.c delete mode 100644 externals/pio2/src/clib/pioc_sc.c delete mode 100644 externals/pio2/src/clib/pioc_support.c delete mode 100644 externals/pio2/src/clib/topology.c delete mode 100644 externals/pio2/src/flib/CMakeLists.txt delete mode 100644 externals/pio2/src/gptl/CMakeLists.txt delete mode 100644 externals/pio2/src/gptl/COPYING delete mode 100644 externals/pio2/tests/general/CMakeLists.txt delete mode 100644 externals/pio2/tests/general/ncdf_get_put.F90.in delete mode 100644 externals/pio2/tests/general/pio_decomp_tests.F90.in delete mode 100644 externals/pio2/tests/unit/CMakeLists.txt delete mode 100755 scripts/Tools/cron_script create mode 100755 scripts/Tools/cs.status delete mode 100755 scripts/Tools/cs_status create mode 100755 scripts/Tools/preview_run create mode 100755 scripts/Tools/testreporter.py rename {utils => scripts}/data_assimilation/da_no_data_mod.sh (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/circle_area/src/CMakeLists.txt (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/circle_area/src/circle.F90 (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/circle_area/tests/CMakeLists.txt (89%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/circle_area/tests/CTest/CMakeLists.txt (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/circle_area/tests/CTest/test_driver.F90 (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/circle_area/tests/pFUnit/CMakeLists.txt (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/circle_area/tests/pFUnit/test_circle.pf (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/interpolate_1d/src/CMakeLists.txt (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/interpolate_1d/src/interpolate_1d.F90 (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/interpolate_1d/tests/CMakeLists.txt (89%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/interpolate_1d/tests/CTest/CMakeLists.txt (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/interpolate_1d/tests/CTest/test_driver.F90 (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/interpolate_1d/tests/pFUnit/CMakeLists.txt (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/interpolate_1d/tests/pFUnit/test_interpolate_point.pf (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/Examples/test_list.xml (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/README (84%) rename {tools/unit_testing => scripts/fortran_unit_testing}/python/.gitignore (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/python/printer.py (100%) rename {tools/unit_testing => scripts/fortran_unit_testing}/python/test_xml_test_list.py (97%) rename {tools/unit_testing => scripts/fortran_unit_testing}/python/xml_test_list.py (96%) rename {tools/unit_testing => scripts/fortran_unit_testing}/run_tests.py (67%) rename {utils/python => scripts/lib}/CIME/BuildTools/__init__.py (100%) rename {utils/python => scripts/lib}/CIME/BuildTools/cmakemacroswriter.py (82%) rename {utils/python => scripts/lib}/CIME/BuildTools/configure.py (91%) rename {utils/python => scripts/lib}/CIME/BuildTools/macroconditiontree.py (100%) rename {utils/python => scripts/lib}/CIME/BuildTools/macrowriterbase.py (100%) rename {utils/python => scripts/lib}/CIME/BuildTools/makemacroswriter.py (100%) rename {utils/python => scripts/lib}/CIME/BuildTools/possiblevalues.py (100%) rename {utils/python => scripts/lib}/CIME/BuildTools/valuesetting.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/README (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/__init__.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/dae.py (94%) rename {utils/python => scripts/lib}/CIME/SystemTests/eri.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/erio.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/erp.py (98%) rename {utils/python => scripts/lib}/CIME/SystemTests/err.py (100%) create mode 100644 scripts/lib/CIME/SystemTests/erri.py rename {utils/python => scripts/lib}/CIME/SystemTests/ers.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/ert.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/homme.py (73%) rename {utils/python => scripts/lib}/CIME/SystemTests/icp.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/lii.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/nck.py (93%) rename {utils/python => scripts/lib}/CIME/SystemTests/ncr.py (100%) create mode 100644 scripts/lib/CIME/SystemTests/nodefail.py rename {utils/python => scripts/lib}/CIME/SystemTests/pea.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/pem.py (87%) rename {utils/python => scripts/lib}/CIME/SystemTests/pet.py (93%) rename {utils/python => scripts/lib}/CIME/SystemTests/pfs.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/pre.py (61%) rename {utils/python => scripts/lib}/CIME/SystemTests/rep.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/seq.py (86%) rename {utils/python => scripts/lib}/CIME/SystemTests/sms.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/ssp.py (97%) rename {utils/python => scripts/lib}/CIME/SystemTests/system_tests_common.py (88%) rename {utils/python => scripts/lib}/CIME/SystemTests/system_tests_compare_two.py (97%) rename {utils/python => scripts/lib}/CIME/SystemTests/test_utils/__init__.py (100%) rename {utils/python => scripts/lib}/CIME/SystemTests/test_utils/user_nl_utils.py (100%) rename {utils/python => scripts/lib}/CIME/XML/__init__.py (100%) rename {utils/python => scripts/lib}/CIME/XML/archive.py (81%) rename {utils/python => scripts/lib}/CIME/XML/batch.py (100%) rename {utils/python => scripts/lib}/CIME/XML/compilerblock.py (100%) rename {utils/python => scripts/lib}/CIME/XML/compilers.py (100%) rename {utils/python => scripts/lib}/CIME/XML/component.py (100%) rename {utils/python => scripts/lib}/CIME/XML/compsets.py (90%) rename {utils/python => scripts/lib}/CIME/XML/entry_id.py (90%) rename {utils/python => scripts/lib}/CIME/XML/env_archive.py (93%) rename {utils/python => scripts/lib}/CIME/XML/env_base.py (88%) rename {utils/python => scripts/lib}/CIME/XML/env_batch.py (66%) rename {utils/python => scripts/lib}/CIME/XML/env_build.py (74%) rename {utils/python => scripts/lib}/CIME/XML/env_case.py (74%) rename {utils/python => scripts/lib}/CIME/XML/env_mach_pes.py (91%) rename {utils/python => scripts/lib}/CIME/XML/env_mach_specific.py (89%) rename {utils/python => scripts/lib}/CIME/XML/env_run.py (79%) rename {utils/python => scripts/lib}/CIME/XML/env_test.py (82%) rename {utils/python => scripts/lib}/CIME/XML/files.py (80%) rename {utils/python => scripts/lib}/CIME/XML/generic_xml.py (99%) rename {utils/python => scripts/lib}/CIME/XML/grids.py (84%) rename {utils/python => scripts/lib}/CIME/XML/headers.py (94%) rename {utils/python => scripts/lib}/CIME/XML/lt_archive.py (100%) rename {utils/python => scripts/lib}/CIME/XML/machines.py (91%) rename {utils/python => scripts/lib}/CIME/XML/namelist_definition.py (95%) rename {utils/python => scripts/lib}/CIME/XML/pes.py (100%) rename {utils/python => scripts/lib}/CIME/XML/pio.py (100%) rename {utils/python => scripts/lib}/CIME/XML/standard_module_setup.py (76%) create mode 100644 scripts/lib/CIME/XML/test_reporter.py rename {utils/python => scripts/lib}/CIME/XML/testlist.py (100%) rename {utils/python => scripts/lib}/CIME/XML/tests.py (100%) rename {utils/python => scripts/lib}/CIME/XML/testspec.py (100%) rename {utils/python => scripts/lib}/CIME/__init__.py (100%) rename {utils/python => scripts/lib}/CIME/aprun.py (81%) rename {utils/python => scripts/lib}/CIME/bless_test_results.py (99%) rename {utils/python => scripts/lib}/CIME/build.py (89%) rename {utils/python => scripts/lib}/CIME/buildlib.py (85%) rename {utils/python => scripts/lib}/CIME/buildnml.py (95%) rename {utils/python => scripts/lib}/CIME/case.py (83%) rename {utils/python => scripts/lib}/CIME/case_cmpgen_namelists.py (97%) rename {utils/python => scripts/lib}/CIME/case_lt_archive.py (79%) rename {utils/python => scripts/lib}/CIME/case_run.py (64%) rename {utils/python => scripts/lib}/CIME/case_setup.py (78%) rename {utils/python => scripts/lib}/CIME/case_st_archive.py (80%) mode change 100755 => 100644 rename {utils/python => scripts/lib}/CIME/case_submit.py (74%) rename {utils/python => scripts/lib}/CIME/case_test.py (91%) rename {utils/python => scripts/lib}/CIME/check_input_data.py (96%) rename {utils/python => scripts/lib}/CIME/check_lockedfiles.py (96%) rename {utils/python => scripts/lib}/CIME/code_checker.py (92%) rename {utils/python => scripts/lib}/CIME/compare_namelists.py (100%) rename {utils/python => scripts/lib}/CIME/compare_test_results.py (84%) rename {utils/python => scripts/lib}/CIME/get_timing.py (98%) rename {utils/python => scripts/lib}/CIME/hist_utils.py (96%) rename {utils/python => scripts/lib}/CIME/namelist.py (83%) rename {utils/python => scripts/lib}/CIME/nmlgen.py (96%) rename {utils/python => scripts/lib}/CIME/preview_namelists.py (67%) rename {utils/python => scripts/lib}/CIME/provenance.py (65%) rename {utils/python => scripts/lib}/CIME/simple_compare.py (100%) rename {utils/python => scripts/lib}/CIME/test_scheduler.py (90%) rename {utils/python => scripts/lib}/CIME/test_status.py (96%) rename {utils/python => scripts/lib}/CIME/test_utils.py (100%) rename {utils/python => scripts/lib}/CIME/tests/README (100%) rename {utils/python => scripts/lib}/CIME/tests/SystemTests/__init__.py (100%) rename {utils/python => scripts/lib}/CIME/tests/SystemTests/test_system_tests_compare_two.py (100%) rename {utils/python => scripts/lib}/CIME/tests/SystemTests/test_system_tests_compare_two_link_to_case2_output.py (100%) rename {utils/python => scripts/lib}/CIME/tests/SystemTests/test_utils/__init__.py (100%) rename {utils/python => scripts/lib}/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py (100%) rename {utils/python => scripts/lib}/CIME/tests/__init__.py (100%) rename {utils/python => scripts/lib}/CIME/tests/case_fake.py (100%) rename {utils/python => scripts/lib}/CIME/tests/test_case_fake.py (100%) create mode 100644 scripts/lib/CIME/tests/test_user_mod_support.py rename {utils/python => scripts/lib}/CIME/user_mod_support.py (75%) rename {utils/python => scripts/lib}/CIME/utils.py (87%) rename {utils/python => scripts/lib}/CIME/wait_for_tests.py (99%) create mode 100644 scripts/lib/cs.status.template rename {utils/python => scripts/lib}/cs.submit.template (100%) rename {utils/python => scripts/lib}/jenkins_generic_job.py (82%) create mode 100644 scripts/lib/testreporter.template rename {utils/python => scripts/lib}/update_acme_tests.py (94%) rename {utils/python => scripts}/tests/CMakeLists.txt (100%) rename {utils/python => scripts}/tests/CTestConfig.cmake (100%) rename {utils/python => scripts}/tests/cpl.hi1.nc.test (100%) rename {utils/python => scripts}/tests/cpl.hi2.nc.test (100%) rename {utils/python => scripts}/tests/cpl.log.failmemleak.gz (100%) rename {utils/python => scripts}/tests/cpl.log.passmemleak.gz (100%) rename {utils/python => scripts}/tests/list_tests (100%) rename {utils/python => scripts}/tests/scripts_regression_tests.py (87%) rename {utils/python => scripts}/tests/user_mods_test1/include_user_mods (100%) rename {utils/python => scripts}/tests/user_mods_test1/user_nl_comp (100%) rename {utils/python => scripts}/tests/user_mods_test1/user_nl_cpl (100%) rename {utils/python => scripts}/tests/user_mods_test2/SourceMods/src.drv/somefile.F90 (100%) rename {utils/python => scripts}/tests/user_mods_test2/user_nl_cpl (100%) delete mode 100644 share/csm_share/include/shr_assert.h delete mode 100644 share/csm_share/test/unit/shr_assert_test/test_macro.pf delete mode 100644 share/csm_share/test/unit/shr_assert_test/test_ndebug.pf delete mode 100644 share/timing/COPYING rename {cime_config => src/build_scripts}/buildlib.csm_share (89%) rename {cime_config => src/build_scripts}/buildlib.gptl (78%) rename {cime_config => src/build_scripts}/buildlib.mct (95%) rename {cime_config => src/build_scripts}/buildlib.mpi-serial (94%) rename {cime_config => src/build_scripts}/buildlib.pio (90%) rename {components => src/components}/data_comps/datm/atm_comp_mct.F90 (100%) rename {components => src/components}/data_comps/datm/cime_config/buildlib (90%) rename {components => src/components}/data_comps/datm/cime_config/buildnml (98%) rename {components => src/components}/data_comps/datm/cime_config/config_archive.xml (100%) rename {components => src/components}/data_comps/datm/cime_config/config_component.xml (90%) rename {components => src/components}/data_comps/datm/cime_config/namelist_definition_datm.xml (91%) rename {components => src/components}/data_comps/datm/cime_config/user_nl_datm (100%) rename {components => src/components}/data_comps/datm/datm_comp_mod.F90 (98%) rename {components => src/components}/data_comps/datm/datm_shr_mod.F90 (100%) rename {components => src/components}/data_comps/datm/doc/SOM.pdf (100%) rename {components => src/components}/data_comps/datm/doc/bookinfo.xml (100%) rename {components => src/components}/data_comps/datm/doc/datacomps.xml (100%) rename {components => src/components}/data_comps/datm/doc/intro.xml (100%) rename {components => src/components}/data_comps/datm/doc/rundocbook.csh (100%) rename {components => src/components}/data_comps/datm/doc/streams.xml (99%) rename {components => src/components}/data_comps/datm/doc/ug.xml (100%) rename {components => src/components}/data_comps/desp/cime_config/buildlib (90%) rename {components => src/components}/data_comps/desp/cime_config/buildnml (97%) rename {components => src/components}/data_comps/desp/cime_config/config_component.xml (100%) rename {components => src/components}/data_comps/desp/cime_config/namelist_definition_desp.xml (100%) rename {components => src/components}/data_comps/desp/cime_config/user_nl_desp (100%) rename {components => src/components}/data_comps/desp/desp_comp_mod.F90 (90%) rename {components => src/components}/data_comps/desp/esp_comp_mct.F90 (83%) rename {components => src/components}/data_comps/desp/esp_utils.F90 (100%) rename {components => src/components}/data_comps/dice/cime_config/buildlib (90%) rename {components => src/components}/data_comps/dice/cime_config/buildnml (99%) rename {components => src/components}/data_comps/dice/cime_config/config_archive.xml (100%) rename {components => src/components}/data_comps/dice/cime_config/config_component.xml (95%) rename {components => src/components}/data_comps/dice/cime_config/namelist_definition_dice.xml (95%) rename {components => src/components}/data_comps/dice/cime_config/user_nl_dice (100%) rename {components => src/components}/data_comps/dice/dice_comp_mod.F90 (99%) rename {components => src/components}/data_comps/dice/ice_comp_mct.F90 (100%) rename {components => src/components}/data_comps/dlnd/cime_config/buildlib (90%) rename {components => src/components}/data_comps/dlnd/cime_config/buildnml (99%) rename {components => src/components}/data_comps/dlnd/cime_config/config_archive.xml (100%) rename {components => src/components}/data_comps/dlnd/cime_config/config_component.xml (54%) rename {components => src/components}/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml (100%) rename {components => src/components}/data_comps/dlnd/cime_config/user_nl_dlnd (100%) rename {components => src/components}/data_comps/dlnd/dlnd_comp_mod.F90 (100%) rename {components => src/components}/data_comps/dlnd/lnd_comp_mct.F90 (100%) rename {components => src/components}/data_comps/docn/cime_config/buildlib (90%) rename {components => src/components}/data_comps/docn/cime_config/buildnml (96%) rename {components => src/components}/data_comps/docn/cime_config/config_archive.xml (100%) rename {components => src/components}/data_comps/docn/cime_config/config_component.xml (65%) rename {components => src/components}/data_comps/docn/cime_config/namelist_definition_docn.xml (92%) rename {components => src/components}/data_comps/docn/cime_config/user_nl_docn (100%) rename {components => src/components}/data_comps/docn/docn_comp_mod.F90 (74%) rename {components => src/components}/data_comps/docn/ocn_comp_mct.F90 (100%) rename {components => src/components}/data_comps/docn/tools/pop_som_frc/README (100%) rename {components => src/components}/data_comps/docn/tools/pop_som_frc/SOM.doc (100%) rename {components => src/components}/data_comps/docn/tools/pop_som_frc/SOM.pdf (100%) rename {components => src/components}/data_comps/docn/tools/pop_som_frc/pop_frc.csh (100%) rename {components => src/components}/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl (100%) rename {components => src/components}/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl (100%) rename {components => src/components}/data_comps/docn/tools/pop_som_frc/pop_interp.ncl (100%) rename {components => src/components}/data_comps/docn/tools/pop_som_frc/read_from_mss.csh (100%) rename {components => src/components}/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh (100%) rename {components => src/components}/data_comps/drof/cime_config/buildlib (90%) rename {components => src/components}/data_comps/drof/cime_config/buildnml (99%) rename {components => src/components}/data_comps/drof/cime_config/config_archive.xml (100%) rename {components => src/components}/data_comps/drof/cime_config/config_component.xml (100%) rename {components => src/components}/data_comps/drof/cime_config/namelist_definition_drof.xml (95%) rename {components => src/components}/data_comps/drof/cime_config/user_nl_drof (100%) rename {components => src/components}/data_comps/drof/drof_comp_mod.F90 (99%) rename {components => src/components}/data_comps/drof/rof_comp_mct.F90 (100%) rename {components => src/components}/data_comps/dwav/README (100%) rename {components => src/components}/data_comps/dwav/bld/README (100%) rename {components => src/components}/data_comps/dwav/cime_config/buildlib (90%) rename {components => src/components}/data_comps/dwav/cime_config/buildnml (93%) rename {components => src/components}/data_comps/dwav/cime_config/config_archive.xml (100%) rename {components => src/components}/data_comps/dwav/cime_config/config_component.xml (76%) rename {components => src/components}/data_comps/dwav/cime_config/namelist_definition_dwav.xml (94%) rename {components => src/components}/data_comps/dwav/cime_config/user_nl_dwav (100%) rename {components => src/components}/data_comps/dwav/dwav_comp_mod.F90 (89%) rename {components => src/components}/data_comps/dwav/wav_comp_mct.F90 (100%) rename {components => src/components}/stub_comps/satm/cime_config/buildlib (90%) rename {components => src/components}/stub_comps/satm/cime_config/buildnml (100%) rename {components => src/components}/stub_comps/satm/cime_config/config_component.xml (100%) rename {components => src/components}/stub_comps/satm/cpl/atm_comp_mct.F90 (100%) rename {components => src/components}/stub_comps/sesp/cime_config/buildlib (90%) rename {components => src/components}/stub_comps/sesp/cime_config/buildnml (100%) rename {components => src/components}/stub_comps/sesp/cime_config/config_component.xml (100%) rename {components => src/components}/stub_comps/sesp/cpl/esp_comp_mct.F90 (100%) rename {components => src/components}/stub_comps/sglc/cime_config/buildlib (90%) rename {components => src/components}/stub_comps/sglc/cime_config/buildnml (100%) rename {components => src/components}/stub_comps/sglc/cime_config/config_component.xml (100%) rename {components => src/components}/stub_comps/sglc/cpl/glc_comp_mct.F90 (100%) rename {components => src/components}/stub_comps/sice/cime_config/buildlib (90%) rename {components => src/components}/stub_comps/sice/cime_config/buildnml (100%) rename {components => src/components}/stub_comps/sice/cime_config/config_component.xml (100%) rename {components => src/components}/stub_comps/sice/cpl/ice_comp_mct.F90 (100%) rename {components => src/components}/stub_comps/slnd/cime_config/buildlib (90%) rename {components => src/components}/stub_comps/slnd/cime_config/buildnml (100%) rename {components => src/components}/stub_comps/slnd/cime_config/config_component.xml (100%) rename {components => src/components}/stub_comps/slnd/cpl/lnd_comp_mct.F90 (100%) rename {components => src/components}/stub_comps/socn/cime_config/buildlib (90%) rename {components => src/components}/stub_comps/socn/cime_config/buildnml (100%) rename {components => src/components}/stub_comps/socn/cime_config/config_component.xml (100%) rename {components => src/components}/stub_comps/socn/cpl/ocn_comp_mct.F90 (100%) rename {components => src/components}/stub_comps/srof/cime_config/buildlib (90%) rename {components => src/components}/stub_comps/srof/cime_config/buildnml (100%) rename {components => src/components}/stub_comps/srof/cime_config/config_component.xml (100%) rename {components => src/components}/stub_comps/srof/cpl/rof_comp_mct.F90 (100%) rename {components => src/components}/stub_comps/swav/cime_config/buildlib (90%) rename {components => src/components}/stub_comps/swav/cime_config/buildnml (100%) rename {components => src/components}/stub_comps/swav/cime_config/config_component.xml (100%) rename {components => src/components}/stub_comps/swav/cpl/wav_comp_mct.F90 (100%) rename {components => src/components}/xcpl_comps/xatm/cime_config/buildlib (90%) rename {components => src/components}/xcpl_comps/xatm/cime_config/buildnml (95%) rename {components => src/components}/xcpl_comps/xatm/cime_config/config_component.xml (100%) rename {components => src/components}/xcpl_comps/xatm/cpl/atm_comp_mct.F90 (100%) rename {components => src/components}/xcpl_comps/xglc/cime_config/buildlib (90%) rename {components => src/components}/xcpl_comps/xglc/cime_config/buildnml (95%) rename {components => src/components}/xcpl_comps/xglc/cime_config/config_component.xml (100%) rename {components => src/components}/xcpl_comps/xglc/cpl/glc_comp_mct.F90 (100%) rename {components => src/components}/xcpl_comps/xice/cime_config/buildlib (90%) rename {components => src/components}/xcpl_comps/xice/cime_config/buildnml (95%) rename {components => src/components}/xcpl_comps/xice/cime_config/config_component.xml (100%) rename {components => src/components}/xcpl_comps/xice/cpl/ice_comp_mct.F90 (100%) rename {components => src/components}/xcpl_comps/xlnd/cime_config/buildlib (90%) rename {components => src/components}/xcpl_comps/xlnd/cime_config/buildnml (95%) rename {components => src/components}/xcpl_comps/xlnd/cime_config/config_component.xml (100%) rename {components => src/components}/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 (100%) rename {components => src/components}/xcpl_comps/xocn/cime_config/buildlib (90%) rename {components => src/components}/xcpl_comps/xocn/cime_config/buildnml (95%) rename {components => src/components}/xcpl_comps/xocn/cime_config/config_component.xml (100%) rename {components => src/components}/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 (100%) rename {components => src/components}/xcpl_comps/xrof/cime_config/buildlib (90%) rename {components => src/components}/xcpl_comps/xrof/cime_config/buildnml (95%) rename {components => src/components}/xcpl_comps/xrof/cime_config/config_component.xml (100%) rename {components => src/components}/xcpl_comps/xrof/cpl/rof_comp_mct.F90 (100%) rename {components => src/components}/xcpl_comps/xshare/dead_data_mod.F90 (100%) rename {components => src/components}/xcpl_comps/xshare/dead_mct_mod.F90 (100%) rename {components => src/components}/xcpl_comps/xshare/dead_mod.F90 (100%) rename {components => src/components}/xcpl_comps/xwav/cime_config/buildlib (90%) rename {components => src/components}/xcpl_comps/xwav/cime_config/buildnml (95%) rename {components => src/components}/xcpl_comps/xwav/cime_config/config_component.xml (100%) rename {components => src/components}/xcpl_comps/xwav/cpl/wav_comp_mct.F90 (100%) rename {driver_cpl => src/drivers/mct}/bld/namelist_files/namelist_defaults_drv.xml (100%) rename {driver_cpl => src/drivers/mct}/bld/namelist_files/namelist_definition_drv.xml (100%) rename {driver_cpl => src/drivers/mct}/bld/namelist_files/namelist_definition_drv_flds.xml (100%) rename {driver_cpl => src/drivers/mct}/bld/namelist_files/namelist_definition_modio.xml (100%) rename {driver_cpl => src/drivers/mct}/bld/testdir/Buildconf/camconf/drv_flds_in (100%) rename {driver_cpl => src/drivers/mct}/bld/testdir/Buildconf/clmconf/drv_flds_in (100%) rename {driver_cpl => src/drivers/mct}/bld/testdir/env_build.xml (100%) rename {driver_cpl => src/drivers/mct}/bld/testdir/env_case.xml (100%) rename {driver_cpl => src/drivers/mct}/bld/testdir/env_mach_pes.xml (100%) rename {driver_cpl => src/drivers/mct}/bld/testdir/env_run.xml (99%) rename {driver_cpl => src/drivers/mct}/bld/testdir/runtest (100%) rename {driver_cpl => src/drivers/mct}/bld/testdir/user_nl_cpl (100%) rename {driver_cpl => src/drivers/mct}/cime_config/buildexe (86%) rename {driver_cpl => src/drivers/mct}/cime_config/buildnml (87%) rename {driver_cpl => src/drivers/mct}/cime_config/config_archive.xml (100%) rename {driver_cpl => src/drivers/mct}/cime_config/config_component.xml (96%) rename {driver_cpl => src/drivers/mct}/cime_config/config_component_acme.xml (92%) rename {driver_cpl => src/drivers/mct}/cime_config/config_component_cesm.xml (94%) rename {driver_cpl => src/drivers/mct}/cime_config/config_compsets.xml (72%) rename {driver_cpl => src/drivers/mct}/cime_config/config_pes.xml (76%) rename {driver_cpl => src/drivers/mct}/cime_config/namelist_definition_drv.xml (96%) create mode 100644 src/drivers/mct/cime_config/namelist_definition_drv_flds.xml rename {driver_cpl => src/drivers/mct}/cime_config/namelist_definition_modelio.xml (100%) rename {driver_cpl => src/drivers/mct}/cime_config/testdefs/testlist_drv.xml (100%) rename {driver_cpl => src/drivers/mct}/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands (100%) rename {driver_cpl => src/drivers/mct}/cime_config/user_nl_cpl (100%) rename {driver_cpl => src/drivers/mct}/doc/bookinfo.xml (100%) rename {driver_cpl => src/drivers/mct}/doc/chap1.xml (99%) rename {driver_cpl => src/drivers/mct}/doc/chap2.xml (100%) rename {driver_cpl => src/drivers/mct}/doc/figx1.jpg (100%) rename {driver_cpl => src/drivers/mct}/doc/namelist.xml (100%) rename {driver_cpl => src/drivers/mct}/doc/rundocbook.csh (100%) rename {driver_cpl => src/drivers/mct}/doc/stylesheet.dsl (100%) rename {driver_cpl => src/drivers/mct}/doc/ug.xml (100%) rename {driver_cpl/driver => src/drivers/mct/main}/CMakeLists.txt (100%) rename {driver_cpl/driver => src/drivers/mct/main}/cesm_comp_mod.F90 (97%) rename {driver_cpl/driver => src/drivers/mct/main}/cesm_driver.F90 (92%) rename {driver_cpl/driver => src/drivers/mct/main}/component_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/component_type_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/cplcomp_exchange_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/map_glc2lnd_mod.F90 (97%) rename {driver_cpl/driver => src/drivers/mct/main}/map_lnd2glc_mod.F90 (98%) rename {driver_cpl/driver => src/drivers/mct/main}/map_lnd2rof_irrig_mod.F90 (99%) rename {driver_cpl/driver => src/drivers/mct/main}/mrg_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/prep_aoflux_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/prep_atm_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/prep_glc_mod.F90 (94%) rename {driver_cpl/driver => src/drivers/mct/main}/prep_ice_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/prep_lnd_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/prep_ocn_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/prep_rof_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/prep_wav_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/seq_diag_mct.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/seq_domain_mct.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/seq_flux_mct.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/seq_frac_mct.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/seq_hist_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/seq_io_mod.F90 (99%) rename {driver_cpl/driver => src/drivers/mct/main}/seq_map_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/seq_map_type_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/seq_rest_mod.F90 (98%) rename {driver_cpl/driver => src/drivers/mct/main}/t_driver_timers_mod.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/vertical_gradient_calculator_2nd_order.F90 (96%) rename {driver_cpl/driver => src/drivers/mct/main}/vertical_gradient_calculator_base.F90 (100%) rename {driver_cpl/driver => src/drivers/mct/main}/vertical_gradient_calculator_factory.F90 (98%) rename {driver_cpl => src/drivers/mct}/shr/CMakeLists.txt (100%) rename {driver_cpl => src/drivers/mct}/shr/glc_elevclass_mod.F90 (99%) rename {driver_cpl => src/drivers/mct}/shr/seq_cdata_mod.F90 (100%) rename {driver_cpl => src/drivers/mct}/shr/seq_comm_mct.F90 (96%) rename {driver_cpl => src/drivers/mct}/shr/seq_drydep_mod.F90 (100%) rename {driver_cpl => src/drivers/mct}/shr/seq_flds_mod.F90 (98%) rename {driver_cpl => src/drivers/mct}/shr/seq_infodata_mod.F90 (89%) rename {driver_cpl => src/drivers/mct}/shr/seq_io_read_mod.F90 (100%) rename {driver_cpl => src/drivers/mct}/shr/seq_timemgr_mod.F90 (86%) rename {driver_cpl => src/drivers/mct}/shr/shr_carma_mod.F90 (100%) rename {driver_cpl => src/drivers/mct}/shr/shr_expr_parser_mod.F90 (100%) rename {driver_cpl => src/drivers/mct}/shr/shr_fire_emis_mod.F90 (100%) rename {driver_cpl => src/drivers/mct}/shr/shr_megan_mod.F90 (100%) create mode 100644 src/drivers/mct/shr/shr_ndep_mod.F90 rename {driver_cpl => src/drivers/mct}/unit_test/CMakeLists.txt (81%) rename {driver_cpl => src/drivers/mct}/unit_test/avect_wrapper_test/CMakeLists.txt (100%) rename {driver_cpl => src/drivers/mct}/unit_test/avect_wrapper_test/test_avect_wrapper.pf (100%) rename {driver_cpl => src/drivers/mct}/unit_test/glc_elevclass_test/CMakeLists.txt (100%) rename {driver_cpl => src/drivers/mct}/unit_test/glc_elevclass_test/test_glc_elevclass.pf (100%) rename {driver_cpl => src/drivers/mct}/unit_test/map_glc2lnd_test/CMakeLists.txt (100%) rename {driver_cpl => src/drivers/mct}/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf (100%) rename {driver_cpl => src/drivers/mct}/unit_test/map_lnd2glc_test/CMakeLists.txt (100%) rename {driver_cpl => src/drivers/mct}/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf (100%) rename {driver_cpl => src/drivers/mct}/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt (100%) rename {driver_cpl => src/drivers/mct}/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf (96%) rename {driver_cpl => src/drivers/mct}/unit_test/seq_map_test/CMakeLists.txt (100%) rename {driver_cpl => src/drivers/mct}/unit_test/seq_map_test/test_seq_map.pf (100%) rename {driver_cpl => src/drivers/mct}/unit_test/stubs/CMakeLists.txt (82%) create mode 100644 src/drivers/mct/unit_test/stubs/seq_timemgr_mod.F90 rename {driver_cpl => src/drivers/mct}/unit_test/stubs/vertical_gradient_calculator_constant.F90 (100%) rename {driver_cpl => src/drivers/mct}/unit_test/utils/CMakeLists.txt (100%) rename {driver_cpl => src/drivers/mct}/unit_test/utils/avect_wrapper_mod.F90 (100%) rename {driver_cpl => src/drivers/mct}/unit_test/utils/create_mapper_mod.F90 (100%) rename {driver_cpl => src/drivers/mct}/unit_test/utils/mct_wrapper_mod.F90 (100%) rename {driver_cpl => src/drivers/mct}/unit_test/utils/simple_map_mod.F90 (98%) rename {driver_cpl => src/drivers/mct}/unit_test/vertical_gradient_calculator_test/CMakeLists.txt (100%) rename {driver_cpl => src/drivers/mct}/unit_test/vertical_gradient_calculator_test/README (100%) rename {driver_cpl => src/drivers/mct}/unit_test/vertical_gradient_calculator_test/gradient_example.txt (100%) rename {driver_cpl => src/drivers/mct}/unit_test/vertical_gradient_calculator_test/plot_gradient (100%) rename {driver_cpl => src/drivers/mct}/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf (100%) rename {driver_cpl => src/drivers/mct}/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf (100%) rename {externals => src/externals}/CMake/.gitignore (100%) rename {externals => src/externals}/CMake/CESM_utils.cmake (100%) create mode 100644 src/externals/CMake/CIME_initial_setup.cmake rename {externals => src/externals}/CMake/CIME_utils.cmake (87%) rename {externals => src/externals}/CMake/ChangeLog (100%) rename {externals => src/externals}/CMake/Compilers.cmake (100%) rename {externals => src/externals}/CMake/FindpFUnit.cmake (100%) rename {externals => src/externals}/CMake/LICENSE (100%) rename {externals => src/externals}/CMake/README.md (74%) rename {externals => src/externals}/CMake/Sourcelist_utils.cmake (100%) rename {externals => src/externals}/CMake/TryCSizeOf.f90 (100%) rename {externals => src/externals}/CMake/TryMPIIO.f90 (100%) rename {externals => src/externals}/CMake/TryMPIMod.f90 (100%) rename {externals => src/externals}/CMake/TryMPISERIAL.f90 (100%) rename {externals => src/externals}/CMake/TryPnetcdf_inc.f90 (100%) rename {externals => src/externals}/CMake/TryPnetcdf_mod.f90 (100%) rename {externals => src/externals}/CMake/genf90_utils.cmake (100%) rename {externals => src/externals}/CMake/mpiexec.cmake (100%) rename {externals => src/externals}/CMake/pFUnit_utils.cmake (98%) rename {externals => src/externals}/genf90/ChangeLog (100%) rename {externals => src/externals}/genf90/genf90.pl (100%) rename {externals => src/externals}/mct/.gitignore (100%) rename {externals => src/externals}/mct/COPYRIGHT (100%) rename {externals => src/externals}/mct/Makefile (100%) rename {externals => src/externals}/mct/Makefile.conf.in (100%) rename {externals => src/externals}/mct/README (100%) rename {externals => src/externals}/mct/aclocal.m4 (100%) rename {externals => src/externals}/mct/benchmarks/.gitignore (100%) rename {externals => src/externals}/mct/benchmarks/Makefile (100%) rename {externals => src/externals}/mct/benchmarks/RouterTestDis.F90 (100%) rename {externals => src/externals}/mct/benchmarks/RouterTestOvr.F90 (100%) rename {externals => src/externals}/mct/benchmarks/T42.8pC (100%) rename {externals => src/externals}/mct/benchmarks/T42.8pR (100%) rename {externals => src/externals}/mct/benchmarks/gx1.8pC (100%) rename {externals => src/externals}/mct/benchmarks/gx1.8pR (100%) rename {externals => src/externals}/mct/benchmarks/importBench.F90 (100%) rename {externals => src/externals}/mct/config.h.in (100%) rename {externals => src/externals}/mct/configure (100%) rename {externals => src/externals}/mct/configure.ac (100%) rename {externals => src/externals}/mct/doc/.gitignore (100%) rename {externals => src/externals}/mct/doc/Makefile (100%) rename {externals => src/externals}/mct/doc/README (100%) rename {externals => src/externals}/mct/doc/coupler.bib (100%) rename {externals => src/externals}/mct/doc/mct_APIs.tex (100%) rename {externals => src/externals}/mct/doc/texsrc/.gitignore (100%) rename {externals => src/externals}/mct/doc/texsrc/Makefile (100%) rename {externals => src/externals}/mct/doc/texsrc/SRCS_tex.mk (100%) rename {externals => src/externals}/mct/examples/Makefile (100%) rename {externals => src/externals}/mct/examples/README (100%) rename {externals => src/externals}/mct/examples/climate_concur1/.gitignore (100%) rename {externals => src/externals}/mct/examples/climate_concur1/Makefile (100%) rename {externals => src/externals}/mct/examples/climate_concur1/README (100%) rename {externals => src/externals}/mct/examples/climate_concur1/coupler.F90 (100%) rename {externals => src/externals}/mct/examples/climate_concur1/master.F90 (100%) rename {externals => src/externals}/mct/examples/climate_concur1/model.F90 (100%) rename {externals => src/externals}/mct/examples/climate_sequen1/.gitignore (100%) rename {externals => src/externals}/mct/examples/climate_sequen1/Makefile (100%) rename {externals => src/externals}/mct/examples/climate_sequen1/README (100%) rename {externals => src/externals}/mct/examples/climate_sequen1/TS1.dat (100%) rename {externals => src/externals}/mct/examples/climate_sequen1/coupler.F90 (100%) rename {externals => src/externals}/mct/examples/climate_sequen1/dst.rc (100%) rename {externals => src/externals}/mct/examples/climate_sequen1/dstmodel.F90 (100%) rename {externals => src/externals}/mct/examples/climate_sequen1/master.F90 (100%) rename {externals => src/externals}/mct/examples/climate_sequen1/mutils.F90 (100%) rename {externals => src/externals}/mct/examples/climate_sequen1/src.rc (100%) rename {externals => src/externals}/mct/examples/climate_sequen1/srcmodel.F90 (100%) rename {externals => src/externals}/mct/examples/simple/.gitignore (100%) rename {externals => src/externals}/mct/examples/simple/Makefile (100%) rename {externals => src/externals}/mct/examples/simple/README (100%) rename {externals => src/externals}/mct/examples/simple/script.babyblue (100%) rename {externals => src/externals}/mct/examples/simple/twocmp.con.F90 (100%) rename {externals => src/externals}/mct/examples/simple/twocmp.seq.F90 (100%) rename {externals => src/externals}/mct/examples/simple/twocmp.seqNB.F90 (100%) rename {externals => src/externals}/mct/examples/simple/twocmp.seqUnvn.F90 (100%) rename {externals => src/externals}/mct/install-sh (100%) rename {externals => src/externals}/mct/m4/README (100%) rename {externals => src/externals}/mct/m4/acx_mpi.m4 (100%) rename {externals => src/externals}/mct/m4/ax_fc_version.m4 (100%) rename {externals => src/externals}/mct/m4/fortran.m4 (100%) rename {externals => src/externals}/mct/mct/Makefile (97%) rename {externals => src/externals}/mct/mct/README (100%) rename {externals => src/externals}/mct/mct/m_Accumulator.F90 (100%) rename {externals => src/externals}/mct/mct/m_AccumulatorComms.F90 (100%) rename {externals => src/externals}/mct/mct/m_AttrVect.F90 (96%) rename {externals => src/externals}/mct/mct/m_AttrVectComms.F90 (100%) rename {externals => src/externals}/mct/mct/m_AttrVectReduce.F90 (100%) rename {externals => src/externals}/mct/mct/m_ConvertMaps.F90 (100%) rename {externals => src/externals}/mct/mct/m_ExchangeMaps.F90 (100%) rename {externals => src/externals}/mct/mct/m_GeneralGrid.F90 (100%) rename {externals => src/externals}/mct/mct/m_GeneralGridComms.F90 (100%) rename {externals => src/externals}/mct/mct/m_GlobalMap.F90 (100%) rename {externals => src/externals}/mct/mct/m_GlobalSegMap.F90 (92%) rename {externals => src/externals}/mct/mct/m_GlobalSegMapComms.F90 (100%) rename {externals => src/externals}/mct/mct/m_GlobalToLocal.F90 (100%) rename {externals => src/externals}/mct/mct/m_MCTWorld.F90 (100%) rename {externals => src/externals}/mct/mct/m_MatAttrVectMul.F90 (93%) rename {externals => src/externals}/mct/mct/m_Merge.F90 (100%) rename {externals => src/externals}/mct/mct/m_Navigator.F90 (100%) rename {externals => src/externals}/mct/mct/m_Rearranger.F90 (93%) rename {externals => src/externals}/mct/mct/m_Router.F90 (74%) create mode 100644 src/externals/mct/mct/m_SPMDutils.F90 rename {externals => src/externals}/mct/mct/m_SparseMatrix.F90 (100%) rename {externals => src/externals}/mct/mct/m_SparseMatrixComms.F90 (100%) rename {externals => src/externals}/mct/mct/m_SparseMatrixDecomp.F90 (100%) rename {externals => src/externals}/mct/mct/m_SparseMatrixPlus.F90 (100%) rename {externals => src/externals}/mct/mct/m_SparseMatrixToMaps.F90 (100%) rename {externals => src/externals}/mct/mct/m_SpatialIntegral.F90 (100%) rename {externals => src/externals}/mct/mct/m_SpatialIntegralV.F90 (100%) rename {externals => src/externals}/mct/mct/m_Transfer.F90 (100%) rename {externals => src/externals}/mct/mkinstalldirs (100%) rename {externals => src/externals}/mct/mpeu/Makefile (100%) rename {externals => src/externals}/mct/mpeu/README (100%) rename {externals => src/externals}/mct/mpeu/assertmpeu.H (100%) rename {externals => src/externals}/mct/mpeu/get_zeits.c (100%) rename {externals => src/externals}/mct/mpeu/m_FcComms.F90 (98%) rename {externals => src/externals}/mct/mpeu/m_FileResolv.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_Filename.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_IndexBin_char.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_IndexBin_integer.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_IndexBin_logical.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_List.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_MergeSorts.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_Permuter.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_SortingTools.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_StrTemplate.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_String.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_StringLinkedList.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_TraceBack.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_chars.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_die.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_dropdead.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_flow.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_inpak90.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_ioutil.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_mall.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_mpif.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_mpif90.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_mpout.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_rankMerge.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_realkinds.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_stdio.F90 (100%) rename {externals => src/externals}/mct/mpeu/m_zeit.F90 (100%) rename {externals => src/externals}/mct/mpi-serial/.gitignore (100%) rename {externals => src/externals}/mct/mpi-serial/Makefile (97%) rename {externals => src/externals}/mct/mpi-serial/Makefile.conf.in (100%) rename {externals => src/externals}/mct/mpi-serial/README (71%) rename {externals => src/externals}/mct/mpi-serial/aclocal.m4 (100%) rename {externals => src/externals}/mct/mpi-serial/cart.c (100%) rename {externals => src/externals}/mct/mpi-serial/collective.c (97%) rename {externals => src/externals}/mct/mpi-serial/comm.c (100%) rename {externals => src/externals}/mct/mpi-serial/config.h.in (100%) rename {externals => src/externals}/mct/mpi-serial/configure (100%) rename {externals => src/externals}/mct/mpi-serial/configure.in (100%) rename {externals => src/externals}/mct/mpi-serial/copy.c (100%) create mode 100644 src/externals/mct/mpi-serial/error.c rename {externals => src/externals}/mct/mpi-serial/fort.F90 (100%) rename {externals => src/externals}/mct/mpi-serial/getcount.c (100%) rename {externals => src/externals}/mct/mpi-serial/group.c (100%) rename {externals => src/externals}/mct/mpi-serial/handles.c (100%) create mode 100644 src/externals/mct/mpi-serial/ic_merge.c rename {externals => src/externals}/mct/mpi-serial/info.c (100%) rename {externals => src/externals}/mct/mpi-serial/list.c (100%) rename {externals => src/externals}/mct/mpi-serial/list.h (100%) rename {externals => src/externals}/mct/mpi-serial/listP.h (100%) rename {externals => src/externals}/mct/mpi-serial/listops.h (100%) rename {externals => src/externals}/mct/mpi-serial/m4/README (100%) rename {externals => src/externals}/mct/mpi-serial/m4/ax_fc_version.m4 (100%) rename {externals => src/externals}/mct/mpi-serial/mpi.c (100%) rename {externals => src/externals}/mct/mpi-serial/mpi.h (96%) rename {externals => src/externals}/mct/mpi-serial/mpiP.h (100%) rename {externals => src/externals}/mct/mpi-serial/mpif.F90 (100%) create mode 100644 src/externals/mct/mpi-serial/mpif.h rename {externals => src/externals}/mct/mpi-serial/op.c (100%) rename {externals => src/externals}/mct/mpi-serial/pack.c (100%) rename {externals => src/externals}/mct/mpi-serial/probe.c (100%) rename {externals => src/externals}/mct/mpi-serial/protify.awk (100%) rename {externals => src/externals}/mct/mpi-serial/recv.c (100%) rename {externals => src/externals}/mct/mpi-serial/req.c (100%) rename {externals => src/externals}/mct/mpi-serial/send.c (100%) rename {externals => src/externals}/mct/mpi-serial/tests/.gitignore (100%) rename {externals => src/externals}/mct/mpi-serial/tests/Makefile (100%) rename {externals => src/externals}/mct/mpi-serial/tests/ctest.c (100%) rename {externals => src/externals}/mct/mpi-serial/tests/ctest_old.c (100%) rename {externals => src/externals}/mct/mpi-serial/tests/ftest.F90 (100%) rename {externals => src/externals}/mct/mpi-serial/tests/ftest_internal.F90 (100%) rename {externals => src/externals}/mct/mpi-serial/tests/ftest_old.F90 (98%) rename {externals => src/externals}/mct/mpi-serial/time.c (100%) rename {externals => src/externals}/mct/mpi-serial/type.c (97%) rename {externals => src/externals}/mct/mpi-serial/type.h (100%) rename {externals => src/externals}/mct/mpi-serial/type_const.c (100%) rename {externals => src/externals}/mct/protex/protex (100%) rename {externals => src/externals}/mct/testsystem/Makefile (100%) rename {externals => src/externals}/mct/testsystem/testall/.gitignore (100%) rename {externals => src/externals}/mct/testsystem/testall/Makefile (100%) rename {externals => src/externals}/mct/testsystem/testall/ReadSparseMatrixAsc.F90 (98%) rename {externals => src/externals}/mct/testsystem/testall/UNTESTED (100%) rename {externals => src/externals}/mct/testsystem/testall/ccm.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/convertPOPT.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/convertgauss.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/cpl.F90 (99%) rename {externals => src/externals}/mct/testsystem/testall/job.ut-all.jaguar (100%) rename {externals => src/externals}/mct/testsystem/testall/m_ACTEST.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/m_AVTEST.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/m_GGRIDTEST.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/m_GMAPTEST.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/m_GSMAPTEST.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/m_MCTWORLDTEST.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/m_ROUTERTEST.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/m_SMATTEST.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/master.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/mph.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/pop.F90 (100%) rename {externals => src/externals}/mct/testsystem/testall/processors_map.in (100%) rename {externals => src/externals}/mct/testsystem/testall/script.jag (100%) rename {externals => src/externals}/mct/testsystem/testall/ut_SparseMatrix.rc (100%) rename {externals => src/externals}/mct/testunit/.gitignore (100%) rename {externals => src/externals}/mct/testunit/AttrVect_Test.F90 (100%) rename {externals => src/externals}/mct/testunit/Makefile (100%) rename {externals => src/externals}/mct/testunit/master.F90 (100%) rename {externals => src/externals}/pio1/CMakeLists.txt (100%) rename {externals => src/externals}/pio1/ChangeLog (100%) rename {externals => src/externals}/pio1/ChangeLog_template (100%) rename {externals => src/externals}/pio1/Doxyfile (100%) rename {externals => src/externals}/pio1/DoxygenLayout.xml (100%) rename {externals => src/externals}/pio1/SVN_EXTERNAL_DIRECTORIES (100%) rename {externals => src/externals}/pio1/customdoxygen.css (100%) rename {externals => src/externals}/pio1/doc/CAMexample.txt (100%) rename {externals => src/externals}/pio1/doc/Decomp.txt (100%) rename {externals => src/externals}/pio1/doc/DoxygenLayout.xml (100%) rename {externals => src/externals}/pio1/doc/Error.txt (100%) rename {externals => src/externals}/pio1/doc/Examples.txt (100%) rename {externals => src/externals}/pio1/doc/GettingStarted.txt (100%) rename {externals => src/externals}/pio1/doc/Installing.txt (100%) rename {externals => src/externals}/pio1/doc/api.txt (100%) rename {externals => src/externals}/pio1/doc/base.txt (100%) rename {externals => src/externals}/pio1/doc/example/errorhandle (100%) rename {externals => src/externals}/pio1/doc/example/simple-bc (100%) rename {externals => src/externals}/pio1/doc/example/simple-bc-rearr (100%) rename {externals => src/externals}/pio1/doc/example/simple-bc-rearr-pe1 (100%) rename {externals => src/externals}/pio1/doc/example/simple-bc-rearr-pe2 (100%) rename {externals => src/externals}/pio1/doc/example/simple-dof (100%) rename {externals => src/externals}/pio1/doc/example/simple-dof-rearr (100%) rename {externals => src/externals}/pio1/doc/faq.txt (100%) rename {externals => src/externals}/pio1/doc/footer.html (100%) rename {externals => src/externals}/pio1/doc/header.html (100%) rename {externals => src/externals}/pio1/doc/images/baseimage.graffle (100%) rename {externals => src/externals}/pio1/doc/images/block-cyclic-rearr.eps (100%) rename {externals => src/externals}/pio1/doc/images/block-cyclic-rearr.graffle (100%) rename {externals => src/externals}/pio1/doc/images/block-cyclic-rearr.png (100%) rename {externals => src/externals}/pio1/doc/images/block-cyclic.eps (100%) rename {externals => src/externals}/pio1/doc/images/block-cyclic.graffle (100%) rename {externals => src/externals}/pio1/doc/images/block-cyclic.png (100%) rename {externals => src/externals}/pio1/doc/images/dof-rearr.eps (100%) rename {externals => src/externals}/pio1/doc/images/dof-rearr.graffle (100%) rename {externals => src/externals}/pio1/doc/images/dof-rearr.png (100%) rename {externals => src/externals}/pio1/doc/images/dof.eps (100%) rename {externals => src/externals}/pio1/doc/images/dof.graffle (100%) rename {externals => src/externals}/pio1/doc/images/dof.png (100%) rename {externals => src/externals}/pio1/doc/testpio_example.txt (100%) rename {externals => src/externals}/pio1/doxygen.sty (100%) rename {externals => src/externals}/pio1/pio/CMakeLists.txt (96%) rename {externals => src/externals}/pio1/pio/C_interface_mod.F90 (100%) rename {externals => src/externals}/pio1/pio/Makefile.conf.in (100%) rename {externals => src/externals}/pio1/pio/README.config (100%) rename {externals => src/externals}/pio1/pio/alloc_mod.F90.in (100%) rename {externals => src/externals}/pio1/pio/box_rearrange.F90.in (100%) rename {externals => src/externals}/pio1/pio/calcdecomp.F90 (100%) rename {externals => src/externals}/pio1/pio/calcdisplace_mod.F90 (100%) rename {externals => src/externals}/pio1/pio/config.h.in (100%) rename {externals => src/externals}/pio1/pio/dtypes.h (100%) rename {externals => src/externals}/pio1/pio/fdepends.awk (100%) rename {externals => src/externals}/pio1/pio/iompi_mod.F90.in (100%) rename {externals => src/externals}/pio1/pio/ionf_mod.F90 (100%) rename {externals => src/externals}/pio1/pio/nf_mod.F90 (100%) rename {externals => src/externals}/pio1/pio/pio.F90 (98%) rename {externals => src/externals}/pio1/pio/pio_kinds.F90 (100%) rename {externals => src/externals}/pio1/pio/pio_mpi_utils.F90 (100%) rename {externals => src/externals}/pio1/pio/pio_msg_callbacks.F90 (100%) rename {externals => src/externals}/pio1/pio/pio_msg_getput_callbacks.F90.in (100%) rename {externals => src/externals}/pio1/pio/pio_msg_mod.F90 (100%) rename {externals => src/externals}/pio1/pio/pio_nf_utils.F90 (100%) rename {externals => src/externals}/pio1/pio/pio_spmd_utils.F90.in (100%) rename {externals => src/externals}/pio1/pio/pio_support.F90 (100%) rename {externals => src/externals}/pio1/pio/pio_types.F90 (99%) rename {externals => src/externals}/pio1/pio/pio_utils.F90 (100%) rename {externals => src/externals}/pio1/pio/piodarray.F90.in (100%) rename {externals => src/externals}/pio1/pio/piolib_mod.F90 (96%) rename {externals => src/externals}/pio1/pio/pionfatt_mod.F90.in (100%) rename {externals => src/externals}/pio1/pio/pionfget_mod.F90.in (100%) rename {externals => src/externals}/pio1/pio/pionfput_mod.F90.in (99%) rename {externals => src/externals}/pio1/pio/pionfread_mod.F90.in (100%) rename {externals => src/externals}/pio1/pio/pionfwrite_mod.F90.in (100%) rename {externals => src/externals}/pio1/pio/piovdc.F90 (100%) rename {externals => src/externals}/pio1/pio/rearr_options.h (100%) rename {externals => src/externals}/pio1/pio/rearrange.F90.in (100%) rename {externals => src/externals}/pio1/pio/topology.c (100%) rename {externals => src/externals}/pio1/scripts/Utils.pm (100%) rename {externals => src/externals}/pio1/scripts/config.pl (100%) rename {externals => src/externals}/pio1/scripts/testpio_yellowstone.pl (100%) rename {externals => src/externals}/pio1/tests/testpio/CAM05.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/CMakeLists.txt (100%) rename {externals => src/externals}/pio1/tests/testpio/MPASA30km.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/MPASA60km.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/POPB.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/POPC.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/POPD.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/POPDv0.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/POPDv1.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/POPDv2.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/POPDv3.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/POPDv4.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/POPDv5.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/README.testpio (100%) rename {externals => src/externals}/pio1/tests/testpio/WRFB.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/build_defaults.xml (100%) rename {externals => src/externals}/pio1/tests/testpio/check_mod.F90 (100%) rename {externals => src/externals}/pio1/tests/testpio/config_bench.xml (100%) rename {externals => src/externals}/pio1/tests/testpio/fdepends.awk (100%) rename {externals => src/externals}/pio1/tests/testpio/gdecomp_mod.F90 (100%) rename {externals => src/externals}/pio1/tests/testpio/kinds_mod.F90 (100%) rename {externals => src/externals}/pio1/tests/testpio/kraken.128.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/kraken.1K.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/kraken.256.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/kraken.512.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/kraken.64.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/namelist_mod.F90 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.apb05 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.asb01 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.asb04 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b01 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b02 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b03 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b04 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b05 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b06 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b07 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b08 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b09 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b10 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b11 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b12 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.b13 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.bb01 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.bb02 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.bb03 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.bb04 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.bb05 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.bb06 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.bb07 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.bb08 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.bn01 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.bn02 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.bn03 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.n4b01 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.n4b02 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.n4b03 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.n4b04 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.n4b05 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.n4b06 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.n4b07 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.n4b08 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.n4n01 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.n4n02 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.n4n03 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.pb01 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.pb02 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.pb03 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.pb04 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.pb05 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.pb06 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.pb07 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.pb08 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.pn01 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.pn02 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.pn03 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.sb01 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.sb02 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.sb03 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.sb04 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.sb05 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.sb06 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.sb07 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.sb08 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.sn01 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.sn02 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.sn03 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.wr01 (100%) rename {externals => src/externals}/pio1/tests/testpio/namelists/testpio_in.wr02 (100%) rename {externals => src/externals}/pio1/tests/testpio/perl5lib/ChangeLog (100%) rename {externals => src/externals}/pio1/tests/testpio/perl5lib/README (100%) rename {externals => src/externals}/pio1/tests/testpio/perl5lib/XML/Changes (100%) rename {externals => src/externals}/pio1/tests/testpio/perl5lib/XML/Lite.pm (100%) rename {externals => src/externals}/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm (100%) rename {externals => src/externals}/pio1/tests/testpio/perl5lib/XML/README (100%) rename {externals => src/externals}/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 (100%) rename {externals => src/externals}/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 (100%) rename {externals => src/externals}/pio1/tests/testpio/test.csh (100%) rename {externals => src/externals}/pio1/tests/testpio/test_lib.F90 (100%) rename {externals => src/externals}/pio1/tests/testpio/testdecomp.F90 (100%) rename {externals => src/externals}/pio1/tests/testpio/testdecomp.bluefire.run (100%) rename {externals => src/externals}/pio1/tests/testpio/testdecomp_in (100%) rename {externals => src/externals}/pio1/tests/testpio/testpio.F90 (100%) rename {externals => src/externals}/pio1/tests/testpio/testpio_bench.pl (100%) rename {externals => src/externals}/pio1/tests/testpio/testpio_build.pl (100%) rename {externals => src/externals}/pio1/tests/testpio/testpio_run.pl (100%) rename {externals => src/externals}/pio1/tests/testpio/utils_mod.F90 (100%) rename {externals => src/externals}/pio1/tests/testpio/ystest.sh (100%) rename {externals => src/externals}/pio1/tests/unittests/CMakeLists.txt (100%) rename {externals => src/externals}/pio1/tests/unittests/Levy_Notes (100%) rename {externals => src/externals}/pio1/tests/unittests/README (100%) rename {externals => src/externals}/pio1/tests/unittests/basic_tests.F90 (100%) rename {externals => src/externals}/pio1/tests/unittests/driver.F90 (100%) rename {externals => src/externals}/pio1/tests/unittests/global_vars.F90 (100%) rename {externals => src/externals}/pio1/tests/unittests/input.nl (100%) rename {externals => src/externals}/pio1/tests/unittests/nc_set_log_level2.c (100%) rename {externals => src/externals}/pio1/tests/unittests/ncdf_tests.F90 (100%) rename {externals => src/externals}/pio1/tests/unittests/not_netcdf.ieee (100%) rename {externals => src/externals}/pio1/timing/CMakeLists.txt (100%) create mode 100644 src/externals/pio1/timing/COPYING rename {externals => src/externals}/pio1/timing/ChangeLog (100%) rename {externals => src/externals}/pio1/timing/GPTLget_memusage.c (100%) rename {externals => src/externals}/pio1/timing/GPTLprint_memusage.c (100%) rename {externals => src/externals}/pio1/timing/GPTLutil.c (100%) rename {externals => src/externals}/pio1/timing/Makefile (100%) rename {externals => src/externals}/pio1/timing/README (100%) rename {externals => src/externals}/pio1/timing/XXXdotF/perf_mod.F (100%) rename {externals => src/externals}/pio1/timing/XXXdotF/perf_utils.F (100%) rename {externals => src/externals}/pio1/timing/f_wrappers.c (100%) rename {externals => src/externals}/pio1/timing/gptl.c (100%) rename {externals => src/externals}/pio1/timing/gptl.h (100%) rename {externals => src/externals}/pio1/timing/gptl.inc (100%) rename {externals => src/externals}/pio1/timing/gptl_papi.c (100%) rename {externals => src/externals}/pio1/timing/perf_mod.F90 (100%) rename {externals => src/externals}/pio1/timing/perf_utils.F90 (100%) rename {externals => src/externals}/pio1/timing/private.h (100%) rename {externals => src/externals}/pio1/timing/threadutil.c (100%) rename {externals => src/externals}/pio2/.gitignore (100%) create mode 100644 src/externals/pio2/CMakeLists.txt rename {externals => src/externals}/pio2/CTestConfig.cmake (54%) rename {externals => src/externals}/pio2/CTestScript.cmake (79%) rename {externals => src/externals}/pio2/README.md (100%) rename {externals => src/externals}/pio2/cmake/FindGPTL.cmake (100%) rename {externals => src/externals}/pio2/cmake/FindHDF5.cmake (100%) rename {externals => src/externals}/pio2/cmake/FindLIBRT.cmake (100%) rename {externals => src/externals}/pio2/cmake/FindLIBZ.cmake (100%) rename {externals => src/externals}/pio2/cmake/FindMPE.cmake (100%) rename {externals => src/externals}/pio2/cmake/FindMPISERIAL.cmake (100%) rename {externals => src/externals}/pio2/cmake/FindNetCDF.cmake (96%) rename {externals => src/externals}/pio2/cmake/FindPAPI.cmake (100%) rename {externals => src/externals}/pio2/cmake/FindPnetCDF.cmake (100%) rename {externals => src/externals}/pio2/cmake/FindSZIP.cmake (100%) rename {externals => src/externals}/pio2/cmake/LibCheck.cmake (95%) rename {externals => src/externals}/pio2/cmake/LibFind.cmake (100%) rename {externals => src/externals}/pio2/cmake/LibMPI.cmake (75%) rename {externals => src/externals}/pio2/cmake/TryHDF5_HAS_SZIP.c (100%) rename {externals => src/externals}/pio2/cmake/TryNetCDF_DAP.c (100%) rename {externals => src/externals}/pio2/cmake/TryNetCDF_PARALLEL.c (100%) rename {externals => src/externals}/pio2/cmake/TryNetCDF_PNETCDF.c (100%) rename {externals => src/externals}/pio2/cmake/mpiexec.alcf (100%) rename {externals => src/externals}/pio2/cmake/mpiexec.ncsa (100%) rename {externals => src/externals}/pio2/cmake/mpiexec.nersc (100%) rename {externals => src/externals}/pio2/cmake/mpiexec.nwsc (100%) create mode 100755 src/externals/pio2/cmake/mpiexec.nwscla create mode 100755 src/externals/pio2/cmake/mpiexec.olcf rename {externals => src/externals}/pio2/ctest/CTestEnvironment-alcf.cmake (100%) create mode 100644 src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake rename {externals => src/externals}/pio2/ctest/CTestEnvironment-cgd.cmake (100%) rename {externals => src/externals}/pio2/ctest/CTestEnvironment-ncsa.cmake (100%) rename {externals => src/externals}/pio2/ctest/CTestEnvironment-nersc.cmake (100%) rename {externals => src/externals}/pio2/ctest/CTestEnvironment-nwsc.cmake (77%) create mode 100644 src/externals/pio2/ctest/CTestEnvironment-nwscla.cmake rename {externals => src/externals}/pio2/ctest/CTestEnvironment-unknown.cmake (100%) rename {externals => src/externals}/pio2/ctest/CTestScript-Test.cmake (100%) rename {externals => src/externals}/pio2/ctest/runcdash-alcf-ibm.sh (89%) create mode 100755 src/externals/pio2/ctest/runcdash-anlworkstation.sh create mode 100755 src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh rename {externals => src/externals}/pio2/ctest/runcdash-cgd-nag.sh (65%) create mode 100755 src/externals/pio2/ctest/runcdash-nersc-cray.sh create mode 100755 src/externals/pio2/ctest/runcdash-nersc-intel.sh rename {externals => src/externals}/pio2/ctest/runcdash-nwsc-gnu.sh (72%) rename {externals => src/externals}/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh (75%) rename {externals => src/externals}/pio2/ctest/runcdash-nwsc-intel.sh (71%) rename {externals => src/externals}/pio2/ctest/runcdash-nwsc-pgi.sh (72%) create mode 100755 src/externals/pio2/ctest/runcdash-nwscla-intel.sh rename {externals => src/externals}/pio2/ctest/runctest-alcf.sh (100%) create mode 100755 src/externals/pio2/ctest/runctest-anlworkstation.sh rename {externals => src/externals}/pio2/ctest/runctest-cgd.sh (86%) rename {externals => src/externals}/pio2/ctest/runctest-ncsa.sh (100%) rename {externals => src/externals}/pio2/ctest/runctest-nersc.sh (68%) rename {externals => src/externals}/pio2/ctest/runctest-nwsc.sh (100%) create mode 100755 src/externals/pio2/ctest/runctest-nwscla.sh rename {externals => src/externals}/pio2/ctest/runctest-unknown.sh (100%) rename {externals => src/externals}/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake (100%) rename {externals => src/externals}/pio2/doc/CMakeFiles/CMakeOutput.log (100%) rename {externals => src/externals}/pio2/doc/CMakeFiles/cmake.check_cache (100%) create mode 100644 src/externals/pio2/doc/CMakeLists.txt rename {externals => src/externals}/pio2/doc/Doxyfile.in (99%) rename {externals => src/externals}/pio2/doc/DoxygenLayout.xml (100%) rename {externals => src/externals}/pio2/doc/customdoxygen.css (100%) rename {externals => src/externals}/pio2/doc/doxygen.sty (100%) rename {externals => src/externals}/pio2/doc/images/baseimage.graffle (100%) rename {externals => src/externals}/pio2/doc/images/block-cyclic-rearr.eps (100%) rename {externals => src/externals}/pio2/doc/images/block-cyclic-rearr.graffle (100%) rename {externals => src/externals}/pio2/doc/images/block-cyclic-rearr.png (100%) rename {externals => src/externals}/pio2/doc/images/block-cyclic.eps (100%) rename {externals => src/externals}/pio2/doc/images/block-cyclic.graffle (100%) rename {externals => src/externals}/pio2/doc/images/block-cyclic.png (100%) rename {externals => src/externals}/pio2/doc/images/dof-rearr.eps (100%) rename {externals => src/externals}/pio2/doc/images/dof-rearr.graffle (100%) rename {externals => src/externals}/pio2/doc/images/dof-rearr.png (100%) rename {externals => src/externals}/pio2/doc/images/dof.eps (100%) rename {externals => src/externals}/pio2/doc/images/dof.graffle (100%) rename {externals => src/externals}/pio2/doc/images/dof.png (100%) rename {externals => src/externals}/pio2/doc/source/CAMexample.txt (100%) rename {externals => src/externals}/pio2/doc/source/Decomp.txt (100%) rename {externals => src/externals}/pio2/doc/source/Error.txt (88%) rename {externals => src/externals}/pio2/doc/source/Examples.txt (99%) rename {externals => src/externals}/pio2/doc/source/GettingStarted.txt (100%) rename {externals => src/externals}/pio2/doc/source/Installing.txt (99%) create mode 100644 src/externals/pio2/doc/source/Testing.txt rename {externals => src/externals}/pio2/doc/source/api.txt (100%) rename {externals => src/externals}/pio2/doc/source/base.txt (96%) create mode 100644 src/externals/pio2/doc/source/contributing_code.txt rename {externals => src/externals}/pio2/doc/source/example/errorhandle (100%) rename {externals => src/externals}/pio2/doc/source/example/simple-bc (100%) rename {externals => src/externals}/pio2/doc/source/example/simple-bc-rearr (100%) rename {externals => src/externals}/pio2/doc/source/example/simple-bc-rearr-pe1 (100%) rename {externals => src/externals}/pio2/doc/source/example/simple-bc-rearr-pe2 (100%) rename {externals => src/externals}/pio2/doc/source/example/simple-dof (100%) rename {externals => src/externals}/pio2/doc/source/example/simple-dof-rearr (100%) rename {externals => src/externals}/pio2/doc/source/faq.txt (100%) rename {externals => src/externals}/pio2/doc/source/mach_walkthrough.txt (87%) rename {externals => src/externals}/pio2/doc/source/testpio_example.txt (100%) rename {externals => src/externals}/pio2/examples/CMakeLists.txt (64%) rename {externals => src/externals}/pio2/examples/basic/CAM05.csh (100%) rename {externals => src/externals}/pio2/examples/basic/CMakeLists.txt (100%) rename {externals => src/externals}/pio2/examples/basic/MPASA30km.csh (100%) rename {externals => src/externals}/pio2/examples/basic/MPASA60km.csh (100%) rename {externals => src/externals}/pio2/examples/basic/POPB.csh (100%) rename {externals => src/externals}/pio2/examples/basic/POPC.csh (100%) rename {externals => src/externals}/pio2/examples/basic/POPD.csh (100%) rename {externals => src/externals}/pio2/examples/basic/POPDv0.csh (100%) rename {externals => src/externals}/pio2/examples/basic/POPDv1.csh (100%) rename {externals => src/externals}/pio2/examples/basic/POPDv2.csh (100%) rename {externals => src/externals}/pio2/examples/basic/POPDv3.csh (100%) rename {externals => src/externals}/pio2/examples/basic/POPDv4.csh (100%) rename {externals => src/externals}/pio2/examples/basic/POPDv5.csh (100%) rename {externals => src/externals}/pio2/examples/basic/README.testpio (100%) rename {externals => src/externals}/pio2/examples/basic/WRFB.csh (100%) rename {externals => src/externals}/pio2/examples/basic/alloc_mod.F90.in (100%) rename {externals => src/externals}/pio2/examples/basic/build_defaults.xml (100%) rename {externals => src/externals}/pio2/examples/basic/check_mod.F90 (100%) rename {externals => src/externals}/pio2/examples/basic/config_bench.xml (100%) rename {externals => src/externals}/pio2/examples/basic/fdepends.awk (100%) rename {externals => src/externals}/pio2/examples/basic/gdecomp_mod.F90 (100%) rename {externals => src/externals}/pio2/examples/basic/kinds_mod.F90 (100%) rename {externals => src/externals}/pio2/examples/basic/kraken.128.csh (100%) rename {externals => src/externals}/pio2/examples/basic/kraken.1K.csh (100%) rename {externals => src/externals}/pio2/examples/basic/kraken.256.csh (100%) rename {externals => src/externals}/pio2/examples/basic/kraken.512.csh (100%) rename {externals => src/externals}/pio2/examples/basic/kraken.64.csh (100%) rename {externals => src/externals}/pio2/examples/basic/namelist_mod.F90 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.apb05 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.asb01 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.asb04 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.n4b01 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.n4b02 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.n4b03 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.n4b04 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.n4b05 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.n4b06 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.n4b07 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.n4b08 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.n4n01 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.n4n02 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.n4n03 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.pb01 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.pb02 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.pb03 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.pb04 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.pb05 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.pb06 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.pb07 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.pb08 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.pn01 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.pn02 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.pn03 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.ps01 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.ps02 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.ps03 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.ps04 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.ps05 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.ps06 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.ps07 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.ps08 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.sb01 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.sb02 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.sb03 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.sb04 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.sb05 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.sb06 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.sb07 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.sb08 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.sn01 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.sn02 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.sn03 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.wr01 (100%) rename {externals => src/externals}/pio2/examples/basic/namelists/testpio_in.wr02 (100%) rename {externals => src/externals}/pio2/examples/basic/perl5lib/ChangeLog (100%) rename {externals => src/externals}/pio2/examples/basic/perl5lib/README (100%) rename {externals => src/externals}/pio2/examples/basic/perl5lib/XML/Changes (100%) rename {externals => src/externals}/pio2/examples/basic/perl5lib/XML/Lite.pm (100%) rename {externals => src/externals}/pio2/examples/basic/perl5lib/XML/Lite/Element.pm (100%) rename {externals => src/externals}/pio2/examples/basic/perl5lib/XML/README (100%) rename {externals => src/externals}/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3 (100%) rename {externals => src/externals}/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3 (100%) rename {externals => src/externals}/pio2/examples/basic/test.csh (100%) rename {externals => src/externals}/pio2/examples/basic/testdecomp.F90 (100%) rename {externals => src/externals}/pio2/examples/basic/testdecomp.bluefire.run (100%) rename {externals => src/externals}/pio2/examples/basic/testdecomp_in (100%) rename {externals => src/externals}/pio2/examples/basic/testpio.F90 (100%) rename {externals => src/externals}/pio2/examples/basic/testpio_bench.pl (100%) rename {externals => src/externals}/pio2/examples/basic/testpio_build.pl (100%) rename {externals => src/externals}/pio2/examples/basic/testpio_run.pl (100%) rename {externals => src/externals}/pio2/examples/basic/utils_mod.F90 (100%) rename {externals => src/externals}/pio2/examples/basic/wstest.c (100%) rename {externals => src/externals}/pio2/examples/basic/ystest.sh (100%) create mode 100644 src/externals/pio2/examples/c/CMakeLists.txt create mode 100644 src/externals/pio2/examples/c/darray_async.c create mode 100644 src/externals/pio2/examples/c/darray_no_async.c rename {externals => src/externals}/pio2/examples/c/example1.c (90%) rename {externals => src/externals}/pio2/examples/c/example2.c (99%) rename {externals => src/externals}/pio2/examples/c/examplePio.c (97%) create mode 100644 src/externals/pio2/examples/c/valsupp_example1.supp rename {externals => src/externals}/pio2/examples/cxx/CMakeLists.txt (100%) rename {externals => src/externals}/pio2/examples/cxx/examplePio.cxx (100%) rename {externals => src/externals}/pio2/examples/f03/CMakeLists.txt (96%) rename {externals => src/externals}/pio2/examples/f03/examplePio.f90 (99%) rename {externals => src/externals}/pio2/scripts/prune_decomps.pl (100%) rename {externals => src/externals}/pio2/src/CMakeLists.txt (72%) create mode 100644 src/externals/pio2/src/clib/CMakeLists.txt create mode 100644 src/externals/pio2/src/clib/bget.c create mode 100644 src/externals/pio2/src/clib/bget.h rename {externals => src/externals}/pio2/src/clib/config.h.in (76%) rename {externals => src/externals}/pio2/src/clib/dtypes.h (100%) rename {externals => src/externals}/pio2/src/clib/ncparser.pl (100%) rename {externals => src/externals}/pio2/src/clib/ncputgetparser.pl (100%) create mode 100644 src/externals/pio2/src/clib/pio.h rename {externals => src/externals}/pio2/src/clib/pio_c_get_template.c (100%) rename {externals => src/externals}/pio2/src/clib/pio_c_put_template.c (100%) rename {externals => src/externals}/pio2/src/clib/pio_c_template.c (100%) create mode 100644 src/externals/pio2/src/clib/pio_darray.c create mode 100644 src/externals/pio2/src/clib/pio_darray_int.c create mode 100644 src/externals/pio2/src/clib/pio_file.c create mode 100644 src/externals/pio2/src/clib/pio_get_nc.c create mode 100644 src/externals/pio2/src/clib/pio_getput_int.c create mode 100644 src/externals/pio2/src/clib/pio_internal.h create mode 100644 src/externals/pio2/src/clib/pio_lists.c create mode 100644 src/externals/pio2/src/clib/pio_msg.c create mode 100644 src/externals/pio2/src/clib/pio_nc.c create mode 100644 src/externals/pio2/src/clib/pio_nc4.c create mode 100644 src/externals/pio2/src/clib/pio_put_nc.c create mode 100644 src/externals/pio2/src/clib/pio_rearrange.c create mode 100644 src/externals/pio2/src/clib/pio_spmd.c create mode 100644 src/externals/pio2/src/clib/pio_varm.c create mode 100644 src/externals/pio2/src/clib/pioc.c create mode 100644 src/externals/pio2/src/clib/pioc_sc.c create mode 100644 src/externals/pio2/src/clib/pioc_support.c create mode 100644 src/externals/pio2/src/clib/topology.c create mode 100644 src/externals/pio2/src/flib/CMakeLists.txt rename {externals => src/externals}/pio2/src/flib/pio.F90 (90%) rename {externals => src/externals}/pio2/src/flib/pio_kinds.F90 (100%) rename {externals => src/externals}/pio2/src/flib/pio_nf.F90 (94%) rename {externals => src/externals}/pio2/src/flib/pio_support.F90 (100%) rename {externals => src/externals}/pio2/src/flib/pio_types.F90 (76%) rename {externals => src/externals}/pio2/src/flib/piodarray.F90.in (97%) rename {externals => src/externals}/pio2/src/flib/piolib_mod.F90 (91%) rename {externals => src/externals}/pio2/src/flib/pionfatt_mod.F90.in (95%) rename {externals => src/externals}/pio2/src/flib/pionfget_mod.F90.in (98%) rename {externals => src/externals}/pio2/src/flib/pionfput_mod.F90.in (98%) create mode 100644 src/externals/pio2/src/gptl/CMakeLists.txt create mode 100644 src/externals/pio2/src/gptl/COPYING rename {externals => src/externals}/pio2/src/gptl/ChangeLog (100%) rename {externals => src/externals}/pio2/src/gptl/GPTLget_memusage.c (100%) rename {externals => src/externals}/pio2/src/gptl/GPTLprint_memusage.c (100%) rename {externals => src/externals}/pio2/src/gptl/GPTLutil.c (100%) rename {externals => src/externals}/pio2/src/gptl/README (100%) rename {externals => src/externals}/pio2/src/gptl/f_wrappers.c (100%) rename {externals => src/externals}/pio2/src/gptl/gptl.c (99%) rename {externals => src/externals}/pio2/src/gptl/gptl.h (100%) rename {externals => src/externals}/pio2/src/gptl/gptl.inc (100%) rename {externals => src/externals}/pio2/src/gptl/gptl_papi.c (100%) rename {externals => src/externals}/pio2/src/gptl/perf_mod.F90 (99%) rename {externals => src/externals}/pio2/src/gptl/perf_utils.F90 (100%) rename {externals => src/externals}/pio2/src/gptl/private.h (100%) rename {externals => src/externals}/pio2/src/gptl/threadutil.c (100%) rename {externals => src/externals}/pio2/tests/CMakeLists.txt (58%) create mode 100644 src/externals/pio2/tests/cunit/CMakeLists.txt create mode 100644 src/externals/pio2/tests/cunit/pio_tests.h create mode 100644 src/externals/pio2/tests/cunit/test_async_3proc.c create mode 100644 src/externals/pio2/tests/cunit/test_async_4proc.c create mode 100644 src/externals/pio2/tests/cunit/test_async_simple.c create mode 100644 src/externals/pio2/tests/cunit/test_common.c create mode 100644 src/externals/pio2/tests/cunit/test_darray.c create mode 100644 src/externals/pio2/tests/cunit/test_darray_1d.c create mode 100644 src/externals/pio2/tests/cunit/test_darray_3d.c create mode 100644 src/externals/pio2/tests/cunit/test_darray_multi.c create mode 100644 src/externals/pio2/tests/cunit/test_darray_multivar.c create mode 100644 src/externals/pio2/tests/cunit/test_darray_multivar2.c create mode 100644 src/externals/pio2/tests/cunit/test_decomp_uneven.c create mode 100644 src/externals/pio2/tests/cunit/test_decomps.c create mode 100644 src/externals/pio2/tests/cunit/test_intercomm2.c create mode 100644 src/externals/pio2/tests/cunit/test_iosystem2.c create mode 100644 src/externals/pio2/tests/cunit/test_iosystem2_simple.c create mode 100644 src/externals/pio2/tests/cunit/test_iosystem2_simple2.c create mode 100644 src/externals/pio2/tests/cunit/test_iosystem3.c create mode 100644 src/externals/pio2/tests/cunit/test_iosystem3_simple.c create mode 100644 src/externals/pio2/tests/cunit/test_iosystem3_simple2.c create mode 100644 src/externals/pio2/tests/cunit/test_pioc.c create mode 100644 src/externals/pio2/tests/cunit/test_pioc_fill.c create mode 100644 src/externals/pio2/tests/cunit/test_pioc_putget.c create mode 100644 src/externals/pio2/tests/cunit/test_pioc_unlim.c create mode 100644 src/externals/pio2/tests/cunit/test_rearr.c create mode 100644 src/externals/pio2/tests/cunit/test_shared.c create mode 100644 src/externals/pio2/tests/cunit/test_spmd.c create mode 100644 src/externals/pio2/tests/general/CMakeLists.txt create mode 100644 src/externals/pio2/tests/general/README.md rename {externals => src/externals}/pio2/tests/general/ncdf_fail.F90.in (98%) create mode 100644 src/externals/pio2/tests/general/ncdf_get_put.F90.in create mode 100644 src/externals/pio2/tests/general/ncdf_inq.F90.in rename {externals => src/externals}/pio2/tests/general/ncdf_simple_tests.F90.in (100%) rename {externals => src/externals}/pio2/tests/general/pio_decomp_fillval.F90.in (100%) rename {externals => src/externals}/pio2/tests/general/pio_decomp_fillval2.F90 (100%) create mode 100644 src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in create mode 100644 src/externals/pio2/tests/general/pio_decomp_tests.F90.in create mode 100644 src/externals/pio2/tests/general/pio_decomp_tests_1d.F90.in create mode 100644 src/externals/pio2/tests/general/pio_decomp_tests_2d.F90.in create mode 100644 src/externals/pio2/tests/general/pio_decomp_tests_3d.F90.in rename {externals => src/externals}/pio2/tests/general/pio_fail.F90.in (100%) rename {externals => src/externals}/pio2/tests/general/pio_file_fail.F90.in (100%) rename {externals => src/externals}/pio2/tests/general/pio_file_simple_tests.F90.in (97%) rename {externals => src/externals}/pio2/tests/general/pio_init_finalize.F90.in (100%) create mode 100644 src/externals/pio2/tests/general/pio_iosystem_tests.F90.in create mode 100644 src/externals/pio2/tests/general/pio_iosystem_tests2.F90.in create mode 100644 src/externals/pio2/tests/general/pio_iosystem_tests3.F90.in create mode 100644 src/externals/pio2/tests/general/pio_rearr.F90.in create mode 100644 src/externals/pio2/tests/general/pio_rearr_opts.F90.in create mode 100644 src/externals/pio2/tests/general/pio_rearr_opts2.F90.in rename {externals => src/externals}/pio2/tests/general/test_memleak.c (93%) rename {externals => src/externals}/pio2/tests/general/util/pio_tf_f90gen.pl (90%) rename {externals => src/externals}/pio2/tests/general/util/pio_tutil.F90 (72%) rename {externals => src/externals}/pio2/tests/performance/CMakeLists.txt (56%) rename {externals => src/externals}/pio2/tests/performance/gensimple.pl (100%) rename {externals => src/externals}/pio2/tests/performance/kt.PIO1.perfmakefile (100%) rename {externals => src/externals}/pio2/tests/performance/pioperformance.F90 (100%) create mode 100644 src/externals/pio2/tests/unit/CMakeLists.txt rename {externals => src/externals}/pio2/tests/unit/Levy_Notes (100%) rename {externals => src/externals}/pio2/tests/unit/README (100%) rename {externals => src/externals}/pio2/tests/unit/basic_tests.F90 (87%) rename {externals => src/externals}/pio2/tests/unit/driver.F90 (90%) rename {externals => src/externals}/pio2/tests/unit/global_vars.F90 (96%) rename {externals => src/externals}/pio2/tests/unit/input.nl (100%) rename {externals => src/externals}/pio2/tests/unit/nc_set_log_level2.c (64%) rename {externals => src/externals}/pio2/tests/unit/ncdf_tests.F90 (99%) rename {externals => src/externals}/pio2/tests/unit/not_netcdf.ieee (100%) rename {externals => src/externals}/pio2/tests/unit/test_names.c (100%) rename {externals => src/externals}/pio2/tests/unit/test_nc4.c (100%) rename {share/csm_share => src/share}/README (79%) rename {share/shr_RandNum => src/share/RandNum}/include/dSFMT-common.h (100%) rename {share/shr_RandNum => src/share/RandNum}/include/dSFMT-params.h (100%) rename {share/shr_RandNum => src/share/RandNum}/include/dSFMT-params19937.h (100%) rename {share/shr_RandNum => src/share/RandNum}/include/dSFMT.h (100%) rename {share/shr_RandNum => src/share/RandNum}/src/dsfmt_f03/dSFMT.c (100%) rename {share/shr_RandNum => src/share/RandNum}/src/dsfmt_f03/dSFMT_interface.F90 (100%) rename {share/shr_RandNum => src/share/RandNum}/src/dsfmt_f03/dSFMT_utils.c (100%) rename {share/shr_RandNum => src/share/RandNum}/src/kissvec/kissvec.c (100%) rename {share/shr_RandNum => src/share/RandNum}/src/kissvec/kissvec_mod.F90 (100%) rename {share/shr_RandNum => src/share/RandNum}/src/mt19937/mersennetwister_mod.F90 (100%) rename {share/shr_RandNum => src/share/RandNum}/src/shr_RandNum_mod.F90 (100%) rename {share/shr_RandNum => src/share/RandNum}/test/bench/Makefile (100%) rename {share/shr_RandNum => src/share/RandNum}/test/bench/test_shr_RandNum.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/CMakeLists.txt (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_AlarmMod.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_BaseMod.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_CalendarMod.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_ClockMod.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_FractionMod.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_Macros.inc (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_Stubs.F90 (79%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_TimeMgr.inc (100%) rename {share => src/share}/esmf_wrf_timemgr/ESMF_TimeMod.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/Makefile (100%) rename {share => src/share}/esmf_wrf_timemgr/MeatMod.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/README (100%) rename {share => src/share}/esmf_wrf_timemgr/unittests/Makefile (100%) rename {share => src/share}/esmf_wrf_timemgr/unittests/go.csh (100%) rename {share => src/share}/esmf_wrf_timemgr/unittests/test.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/unittests/wrf_stuff.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/wrf_error_fatal.F90 (100%) rename {share => src/share}/esmf_wrf_timemgr/wrf_message.F90 (100%) rename {share/csm_share => src/share}/include/dynamic_vector_procdef.inc (100%) rename {share/csm_share => src/share}/include/dynamic_vector_typedef.inc (100%) create mode 100644 src/share/include/shr_assert.h rename {share/csm_share => src/share}/test/old_unit_testers/Makefile (100%) rename {share/csm_share => src/share}/test/old_unit_testers/Mkdepends (100%) rename {share/csm_share => src/share}/test/old_unit_testers/Mksrcfiles (100%) rename {share/csm_share => src/share}/test/old_unit_testers/bundle_expected.F90 (100%) rename {share/csm_share => src/share}/test/old_unit_testers/config.h (100%) rename {share/csm_share => src/share}/test/old_unit_testers/make.Macros (100%) rename {share/csm_share => src/share}/test/old_unit_testers/namelist (100%) rename {share/csm_share => src/share}/test/old_unit_testers/nl/atm.stdin (100%) rename {share/csm_share => src/share}/test/old_unit_testers/nl/cpl.stdin (100%) rename {share/csm_share => src/share}/test/old_unit_testers/nl/ice.stdin (100%) rename {share/csm_share => src/share}/test/old_unit_testers/nl/lnd.stdin (100%) rename {share/csm_share => src/share}/test/old_unit_testers/nl/ocn.stdin (100%) rename {share/csm_share => src/share}/test/old_unit_testers/run_dshr_bundle_test (100%) rename {share/csm_share => src/share}/test/old_unit_testers/run_file_test (100%) rename {share/csm_share => src/share}/test/old_unit_testers/test_mod.F90 (100%) rename {share/csm_share => src/share}/test/old_unit_testers/test_shr_file.F90 (100%) rename {share/csm_share => src/share}/test/old_unit_testers/test_shr_log.F90 (100%) rename {share/csm_share => src/share}/test/old_unit_testers/test_shr_mpi.F90 (100%) rename {share/csm_share => src/share}/test/old_unit_testers/test_shr_orb.F90 (100%) rename {share/csm_share => src/share}/test/old_unit_testers/test_shr_scam.F90 (100%) rename {share/csm_share => src/share}/test/old_unit_testers/test_shr_streams.F90 (100%) rename {share/csm_share => src/share}/test/old_unit_testers/test_shr_sys.F90 (100%) rename {share/csm_share => src/share}/test/old_unit_testers/test_shr_tInterp.F90 (100%) rename {share/csm_share => src/share}/test/unit/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/dynamic_vector/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/dynamic_vector/character16_vector_tests.pf.in (100%) rename {share/csm_share => src/share}/test/unit/dynamic_vector/dynamic_vector_base_tests.inc (100%) rename {share/csm_share => src/share}/test/unit/dynamic_vector/dynamic_vector_character16.F90 (100%) rename {share/csm_share => src/share}/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90 (100%) rename {share/csm_share => src/share}/test/unit/dynamic_vector/dynamic_vector_integer.F90 (100%) rename {share/csm_share => src/share}/test/unit/dynamic_vector/dynamic_vector_r8.F90 (100%) rename {share/csm_share => src/share}/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in (100%) rename {share/csm_share => src/share}/test/unit/dynamic_vector/integer_vector_tests.pf.in (100%) rename {share/csm_share => src/share}/test/unit/dynamic_vector/ptr_wrapper.F90 (100%) rename {share/csm_share => src/share}/test/unit/dynamic_vector/r8_vector_tests.pf.in (100%) rename {share/csm_share => src/share}/test/unit/mock/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90 (100%) rename {share/csm_share => src/share}/test/unit/shr_assert_test/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/shr_assert_test/test_assert.pf (70%) rename {share/csm_share => src/share}/test/unit/shr_assert_test/test_assert_array.pf (95%) create mode 100644 src/share/test/unit/shr_assert_test/test_macro.pf create mode 100644 src/share/test/unit/shr_assert_test/test_ndebug.pf rename {share/csm_share => src/share}/test/unit/shr_infnan_test/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/shr_infnan_test/test_infnan.F90 (100%) rename {share/csm_share => src/share}/test/unit/shr_log_test/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/shr_log_test/test_error_printers.pf (100%) rename {share/csm_share => src/share}/test/unit/shr_precip_test/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/shr_precip_test/test_shr_precip.pf (100%) rename {share/csm_share => src/share}/test/unit/shr_spfn_test/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/shr_spfn_test/test_erf_r4.pf (100%) rename {share/csm_share => src/share}/test/unit/shr_spfn_test/test_erf_r8.pf (100%) rename {share/csm_share => src/share}/test/unit/shr_spfn_test/test_gamma_factorial.pf (100%) rename {share/csm_share => src/share}/test/unit/shr_spfn_test/test_igamma.pf (100%) rename {share/csm_share => src/share}/test/unit/shr_strconvert_test/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/shr_strconvert_test/test_toString.pf (100%) rename {share/csm_share => src/share}/test/unit/shr_string_test/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/shr_string_test/test_shr_string.pf (100%) rename {share/csm_share => src/share}/test/unit/shr_vmath_test/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/shr_vmath_test/test_vmath.F90 (100%) rename {share/csm_share => src/share}/test/unit/shr_wv_sat_test/CMakeLists.txt (100%) rename {share/csm_share => src/share}/test/unit/shr_wv_sat_test/test_wv_sat.pf (100%) rename {share/csm_share => src/share}/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf (100%) rename {share => src/share}/timing/CMakeLists.txt (100%) create mode 100644 src/share/timing/COPYING rename {share => src/share}/timing/ChangeLog (100%) rename {share => src/share}/timing/GPTLget_memusage.c (100%) rename {share => src/share}/timing/GPTLprint_memusage.c (100%) rename {share => src/share}/timing/GPTLutil.c (100%) rename {share => src/share}/timing/Makefile (100%) rename {share => src/share}/timing/README (100%) rename {share => src/share}/timing/f_wrappers.c (100%) rename {share => src/share}/timing/gptl.c (100%) rename {share => src/share}/timing/gptl.h (100%) rename {share => src/share}/timing/gptl.inc (100%) rename {share => src/share}/timing/gptl_papi.c (100%) rename {share => src/share}/timing/perf_mod.F90 (100%) rename {share => src/share}/timing/perf_utils.F90 (100%) rename {share => src/share}/timing/private.h (100%) rename {share/csm_share => src/share}/unit_test_stubs/README (100%) rename {share/csm_share => src/share}/unit_test_stubs/pio/CMakeLists.txt (100%) rename {share/csm_share => src/share}/unit_test_stubs/pio/README (100%) rename {share/csm_share => src/share}/unit_test_stubs/pio/pio.F90.in (93%) rename {share/csm_share/shr => src/share/util}/CMakeLists.txt (100%) rename {share/csm_share/shr => src/share/util}/mct_mod.F90 (99%) rename {share/csm_share/shr => src/share/util}/shr_assert_mod.F90.in (85%) rename {share/csm_share/shr => src/share/util}/shr_cal_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_const_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_dmodel_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_file_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_flux_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_frz_mod.F90.in (100%) rename {share/csm_share/shr => src/share/util}/shr_infnan_mod.F90.in (100%) rename {share/csm_share/shr => src/share/util}/shr_kind_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_log_mod.F90 (88%) rename {share/csm_share/shr => src/share/util}/shr_map_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_mct_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_mem_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_mpi_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_msg_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_ncread_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_nl_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_orb_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_pcdf_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_pio_mod.F90 (92%) rename {share/csm_share/shr => src/share/util}/shr_precip_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_reprosum_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_reprosumx86.c (100%) rename {share/csm_share/shr => src/share/util}/shr_scam_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_spfn_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_strconvert_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_strdata_mod.F90 (99%) rename {share/csm_share/shr => src/share/util}/shr_stream_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_string_mod.F90 (99%) rename {share/csm_share/shr => src/share/util}/shr_sys_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_tInterp_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_timer_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_vmath_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/shr_wv_sat_mod.F90 (100%) rename {share/csm_share/shr => src/share/util}/water_isotopes.F90 (100%) rename {share/csm_share/shr => src/share/util}/water_types.F90 (100%) create mode 100644 tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/.gitignore create mode 100644 tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx1v7_nnsm_e1000r300.nml create mode 100644 tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx3v7_nnsm_e1000r500.nml create mode 100644 tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx1v7_nnsm_e1000r300.nml create mode 100644 tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx3v7_nnsm_e1000r500.nml create mode 100644 tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_ar9v4_nnsm_e1000r300.nml create mode 100644 tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_gx3v7_nnsm_e1000r300.nml create mode 100644 tools/mapping/gen_mapping_files/runoff_to_ocn/maps/.gitignore create mode 100644 tools/mapping/gen_mapping_files/runoff_to_ocn/ncl/merge_mapping_files.ncl create mode 100755 tools/mapping/gen_mapping_files/runoff_to_ocn/run_merge_mapping_files.sh delete mode 100644 tools/unit_testing/python/comparable.py delete mode 100644 tools/unit_testing/python/environment.py delete mode 100644 tools/unit_testing/python/machine_setup.py delete mode 100755 tools/unit_testing/python/test_environment.py delete mode 100755 tools/unit_testing/python/test_xml_utils.py delete mode 100644 tools/unit_testing/python/xml_utils.py delete mode 100644 utils/python/cs.status.template diff --git a/CMakeLists.txt b/CMakeLists.txt index ae90c8bd74c..6329c529576 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,13 +2,15 @@ cmake_minimum_required(VERSION 2.8) include(ExternalProject) set(CIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") +list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY}) +include(CIME_initial_setup) + project(cime_tests Fortran C) # We rely on pio for cmake utilities like findnetcdf.cmake, so that we don't # need to duplicate this cmake code -list(APPEND CMAKE_MODULE_PATH "${CIME_ROOT}/externals/pio2/cmake") +list(APPEND CMAKE_MODULE_PATH "${CIME_ROOT}/src/externals/pio2/cmake") -list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY}) include(CIME_utils) find_package(NetCDF COMPONENTS C Fortran) include_directories(${NetCDF_C_INCLUDE_DIRS} ${NetCDF_Fortran_INCLUDE_DIRS}) @@ -19,16 +21,23 @@ include_directories(${NetCDF_C_INCLUDE_DIRS} ${NetCDF_Fortran_INCLUDE_DIRS}) # ------------------------------------------------------------------------ # Build mct # ------------------------------------------------------------------------ -set(MCT_ROOT "${CIME_ROOT}/externals/mct") +set(MCT_ROOT "${CIME_ROOT}/src/externals/mct") + +if (USE_MPI_SERIAL) + set(ENABLE_MPI_SERIAL "--enable-mpiserial") +else() + set(ENABLE_MPI_SERIAL "") +endif() ExternalProject_add(mct_project PREFIX ${CMAKE_CURRENT_BINARY_DIR} SOURCE_DIR ${MCT_ROOT} BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/mct - CONFIGURE_COMMAND ${MCT_ROOT}/configure --enable-debugging --prefix=${CMAKE_CURRENT_BINARY_DIR} CFLAGS=${CFLAGS} FCFLAGS=${FFLAGS} SRCDIR=${MCT_ROOT} DEBUG="-g" - BUILD_COMMAND $(MAKE) + CONFIGURE_COMMAND ${MCT_ROOT}/configure ${ENABLE_MPI_SERIAL} --enable-debugging --prefix=${CMAKE_CURRENT_BINARY_DIR} CC=${CMAKE_C_COMPILER} FC=${CMAKE_Fortran_COMPILER} CFLAGS=${CFLAGS} FCFLAGS=${FFLAGS} SRCDIR=${MCT_ROOT} DEBUG="-g" + BUILD_COMMAND $(MAKE) SRCDIR=${MCT_ROOT} # Leave things in rather than "installing", because we have - # no need to move things around inside of the CMake binary directory. + # no need to move things around inside of the CMake binary directory. Also, + # mpi-serial doesn't install properly in the out-of-source build INSTALL_COMMAND : ) # This copy_makefiles step is needed because mct currently doesn't support an @@ -43,12 +52,27 @@ ExternalProject_add_step(mct_project copy_makefiles COMMAND mkdir -p mpeu COMMAND cp -p /mpeu/Makefile mpeu/ ) +if (USE_MPI_SERIAL) + ExternalProject_add_step(mct_project copy_mpi_serial_files + DEPENDEES configure + DEPENDERS build + WORKING_DIRECTORY + COMMAND mkdir -p mpi-serial + COMMAND cp -p /mpi-serial/Makefile mpi-serial/ + COMMAND cp /mpi-serial/mpif.h mpi-serial/ + COMMAND cp /mpi-serial/mpi.h mpi-serial/ + ) +endif() # Tell cmake to look for libraries & mod files here, because this is where we built libraries include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mct) include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu) link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mct) link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu) +if (USE_MPI_SERIAL) + include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpi-serial) + link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpi-serial) +endif() # ------------------------------------------------------------------------ # Done MCT build @@ -58,14 +82,14 @@ link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu) # csm_share (we don't build it here because it seems to be built differently # by different tests?) -set(SHARE_ROOT "${CIME_ROOT}/share/csm_share") -add_subdirectory(${SHARE_ROOT}/shr csm_share) +set(SHARE_ROOT "${CIME_ROOT}/src/share") +add_subdirectory(${SHARE_ROOT}/util csm_share) include_directories(${SHARE_ROOT}/include) # esmf_wrf_timemgr not built here because it depends on csm_share. -add_subdirectory(${CIME_ROOT}/share/esmf_wrf_timemgr esmf_wrf_timemgr) -include_directories(${CIME_ROOT}/share/esmf_wrf_timemgr) +add_subdirectory(${SHARE_ROOT}/esmf_wrf_timemgr esmf_wrf_timemgr) +include_directories(${SHARE_ROOT}/esmf_wrf_timemgr) # Now the actual test directories. -add_subdirectory(driver_cpl/unit_test) -add_subdirectory(share/csm_share/test/unit) +add_subdirectory(${CIME_ROOT}/src/drivers/mct/unit_test) +add_subdirectory(${SHARE_ROOT}/test/unit) diff --git a/ChangeLog b/ChangeLog index f769eb097e5..aa0351fc0c9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,3834 @@ ====================================================================== +Originator: Chris Fischer +Date: 5-2-207 +Tag: cime5.3.0-alpha.10 +Answer Changes: [None, Round Off, Climate Changing] +Tests: scripts_regression_tests.py with PIO_VERSION set to 2 in driver config_components.xml + scripts_regression test + ERP_Ln9.f09_f09.F1850_DONOTUSE.yellowstone_intel.cam-outfrq9s_clm5 + summarize_cprnc_diffs by hand + code-checker +Dependencies: + +Brief Summary: + - Update to version 2.2.1 of the pio external. + - Update the top-level READMEs. + - New optional ndep stream from atm. + - Creation of new aquaplanet capability in DOCN. + - Fix typo in parse_args. + - All command line arguments should conform. Allowed for one exception, the -value option to xmlquery. + - ERP fix again. + - Pick up multi-instance cprnc.out files in summarize_cprnc_diffs. + - Fix an issue with pio1 determining if netcdf4 is available and removes a debug print statement. + - Improve error message when config_files.xml is not found. + - Fix pylint error. + - Put explicit dot in paths in some instructions. + - Remove scripts adjustment of pio settings. + + +User interface changes: + - Now accepting only conforming command line arguments. + +PR summary: git log --oneline --first-parent [previous_tag]..master +0dcd36c Merge pull request #1447 from jedwards4b/pio2_external_update +357ad45 Merge pull request #1432 from ESMCI/rljacob/README-cleanup +c76d946 Merge pull request #1448 from ESMCI/mvertens/ndep +e5f144d Merge pull request #1451 from ESMCI/mvertens/aquap +d4dfbf1 Merge pull request #1452 from jedwards4b/acme_run_template_fix +505ce67 Merge pull request #1393 from jedwards4b/test_cli +c1e5a38 Merge pull request #1445 from jedwards4b/erp_fix_again +f6c05d1 Merge pull request #1442 from billsacks/summarize_cprnc_multiinst +71cc74e Merge pull request #1425 from jedwards4b/pio1_fixes +aa9018a Merge pull request #1440 from jedwards4b/better_msg_for_config_files_not_found +1089e53 Merge pull request #1446 from ESMCI/jgfouca/fix_pylint_err_nml +3a498d2 Merge pull request #1371 from billsacks/source_file_from_path +ae2a5ca Add Sandia Corporation to CIME LICENSE.TXT +7692745 Merge pull request #1441 from jedwards4b/pio_default_settings + + +Modified files: git diff --name-status [previous_tag] + +M LICENSE.TXT +D README +M README.md +M config/acme/machines/template.case.run +M config/acme/machines/template.case.test +M config/acme/machines/template.lt_archive +M config/acme/machines/template.st_archive +M config/cesm/machines/config_compilers.xml +M config/cesm/machines/template.case.run +M config/cesm/machines/template.case.test +M config/cesm/machines/template.lt_archive +M config/cesm/machines/template.st_archive +M scripts/Tools/acme_check_env +M scripts/Tools/bless_test_results +M scripts/Tools/case.build +M scripts/Tools/case.cmpgen_namelists +M scripts/Tools/case.setup +M scripts/Tools/case.submit +M scripts/Tools/case_diff +M scripts/Tools/check_case +M scripts/Tools/check_input_data +M scripts/Tools/check_lockedfiles +M scripts/Tools/cime_bisect +M scripts/Tools/code_checker +M scripts/Tools/compare_namelists +M scripts/Tools/compare_test_results +M scripts/Tools/component_compare_baseline +M scripts/Tools/component_compare_copy +M scripts/Tools/component_compare_test +M scripts/Tools/component_generate_baseline +M scripts/Tools/getTiming +M scripts/Tools/jenkins_generic_job +M scripts/Tools/list_acme_tests +M scripts/Tools/normalize_cases +M scripts/Tools/pelayout +M scripts/Tools/preview_namelists +M scripts/Tools/save_provenance +M scripts/Tools/simple_compare +M scripts/Tools/testreporter.py +M scripts/Tools/update_acme_tests +M scripts/Tools/wait_for_tests +M scripts/Tools/xmlchange +M scripts/Tools/xmlconvertors/config_pes_converter.py +M scripts/Tools/xmlconvertors/grid_xml_converter.py +M scripts/Tools/xmlquery +M scripts/create_clone +M scripts/create_newcase +M scripts/create_test +M scripts/fortran_unit_testing/run_tests.py +M scripts/lib/CIME/SystemTests/erp.py +M scripts/lib/CIME/SystemTests/seq.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/XML/env_mach_specific.py +M scripts/lib/CIME/XML/files.py +M scripts/lib/CIME/buildlib.py +M scripts/lib/CIME/buildnml.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/case_setup.py +M scripts/lib/CIME/namelist.py +M scripts/lib/CIME/utils.py +M scripts/manage_case +M scripts/manage_pes +M scripts/query_testlists +M scripts/tests/scripts_regression_tests.py +M src/build_scripts/buildlib.gptl +M src/build_scripts/buildlib.pio +M src/components/data_comps/docn/cime_config/buildnml +M src/components/data_comps/docn/cime_config/config_component.xml +M src/components/data_comps/docn/cime_config/namelist_definition_docn.xml +M src/components/data_comps/docn/docn_comp_mod.F90 +M src/drivers/mct/cime_config/namelist_definition_drv_flds.xml +M src/drivers/mct/shr/seq_flds_mod.F90 +A src/drivers/mct/shr/shr_ndep_mod.F90 +M src/externals/pio1/pio/CMakeLists.txt +M src/externals/pio1/pio/pionfput_mod.F90.in +M src/externals/pio2/CMakeLists.txt +M src/externals/pio2/CTestScript.cmake +M src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake +M src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake +A src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh +M src/externals/pio2/ctest/runcdash-cgd-nag.sh +M src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +M src/externals/pio2/ctest/runcdash-nwscla-intel.sh +M src/externals/pio2/ctest/runctest-cgd.sh +M src/externals/pio2/ctest/runctest-nwscla.sh +M src/externals/pio2/doc/source/contributing_code.txt +M src/externals/pio2/examples/c/CMakeLists.txt +A src/externals/pio2/examples/c/darray_async.c +A src/externals/pio2/examples/c/darray_no_async.c +M src/externals/pio2/examples/c/example1.c +M src/externals/pio2/src/clib/CMakeLists.txt +M src/externals/pio2/src/clib/bget.c +M src/externals/pio2/src/clib/config.h.in +M src/externals/pio2/src/clib/pio.h +M src/externals/pio2/src/clib/pio_darray.c +M src/externals/pio2/src/clib/pio_darray_int.c +M src/externals/pio2/src/clib/pio_file.c +M src/externals/pio2/src/clib/pio_get_nc.c +M src/externals/pio2/src/clib/pio_getput_int.c +M src/externals/pio2/src/clib/pio_internal.h +M src/externals/pio2/src/clib/pio_lists.c +M src/externals/pio2/src/clib/pio_msg.c +M src/externals/pio2/src/clib/pio_nc.c +M src/externals/pio2/src/clib/pio_nc4.c +M src/externals/pio2/src/clib/pio_put_nc.c +M src/externals/pio2/src/clib/pio_rearrange.c +M src/externals/pio2/src/clib/pio_spmd.c +M src/externals/pio2/src/clib/pio_varm.c +M src/externals/pio2/src/clib/pioc.c +M src/externals/pio2/src/clib/pioc_sc.c +M src/externals/pio2/src/clib/pioc_support.c +M src/externals/pio2/tests/cunit/CMakeLists.txt +M src/externals/pio2/tests/cunit/pio_tests.h +D src/externals/pio2/tests/cunit/test_async_2comp.c +M src/externals/pio2/tests/cunit/test_async_3proc.c +M src/externals/pio2/tests/cunit/test_async_4proc.c +M src/externals/pio2/tests/cunit/test_async_simple.c +M src/externals/pio2/tests/cunit/test_darray_1d.c +M src/externals/pio2/tests/cunit/test_darray_3d.c +A src/externals/pio2/tests/cunit/test_darray_multi.c +M src/externals/pio2/tests/cunit/test_darray_multivar.c +A src/externals/pio2/tests/cunit/test_darray_multivar2.c +A src/externals/pio2/tests/cunit/test_decomp_uneven.c +M src/externals/pio2/tests/cunit/test_decomps.c +M src/externals/pio2/tests/cunit/test_intercomm2.c +M src/externals/pio2/tests/cunit/test_iosystem2_simple.c +M src/externals/pio2/tests/cunit/test_pioc.c +M src/externals/pio2/tests/cunit/test_pioc_fill.c +A src/externals/pio2/tests/cunit/test_rearr.c +M src/externals/pio2/tests/cunit/test_shared.c +M src/externals/pio2/tests/cunit/test_spmd.c +M src/externals/pio2/tests/general/ncdf_get_put.F90.in +M src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in +M src/externals/pio2/tests/general/util/pio_tutil.F90 +M src/share/util/shr_pio_mod.F90 +M src/share/util/shr_strdata_mod.F90 +M tools/configure +M tools/cprnc/README +M tools/cprnc/summarize_cprnc_diffs +M tools/mapping/check_maps/.gitignore +M tools/mapping/check_maps/README +M tools/mapping/check_maps/src/ESMF_RegridWeightGenCheck.F90 +M tools/mapping/check_maps/src/Makefile +M tools/mapping/gen_domain_files/INSTALL +M tools/mapping/gen_domain_files/src/Makefile +M tools/mapping/gen_mapping_files/README +A tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/.gitignore +M tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/README +M tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/create_ESMF_map.sh +M tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL +M tools/mapping/map_field/INSTALL +M tools/mapping/map_field/src/Makefile + +====================================================================== + +====================================================================== + +Originator: Chris Fischer +Date: 4-30-2017 +Tag: cime5.3.0-alpha.09 +Answer Changes: None +Tests: ERP_Ln9.f09_f09.F1850_DONOTUSE.yellowstone_intel.cam-outfrq9s_clm5 + scripts_regression_tests.py --fast, hand test xmlchange and xmlquery +Dependencies: + +Brief Summary: + - Fix erp test build. + - Add Support for '+=' in Namelists. + - Add --component Option to preview_namelist. + - Add --file option to xmlquery and enable option in xmlchange. + +User interface changes: + - Added support for '+=' to namelist parser. + - Added --component option to preview_namelist. + - New --file option in xmlquery. + +PR summary: git log --oneline --first-parent [previous_tag]..master +6d8b24b Merge pull request #1444 from jedwards4b/erp_test_fix +84b7bde Merge pull request #1389 from ESMCI/erichlf/namelist/plusEqual +45af658 Merge pull request #1436 from ESMCI/erichlf/preview_namelists/component +c4d4437 Merge pull request #1419 from jedwards4b/xmlcq_fileoption + + +Modified files: git diff --name-status [previous_tag] + +M scripts/Tools/preview_namelists +M scripts/Tools/xmlchange +M scripts/Tools/xmlquery +M scripts/lib/CIME/SystemTests/erp.py +M scripts/lib/CIME/XML/env_mach_specific.py +M scripts/lib/CIME/XML/env_test.py +M scripts/lib/CIME/XML/generic_xml.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/namelist.py +M scripts/lib/CIME/preview_namelists.py + +====================================================================== + +====================================================================== + +Originator: Chris Fischer +Date: 4-28-2017 +Tag: cime5.3.0-alpha.08 +Answer Changes: [None, Round Off, Climate Changing] +Tests: scripts_regression_tests, SMS_D_Ld3.f45_g37_rx1.A.hobart_nag, + ERS_Ln9.T62_g37.DTEST.yellowstone_intel, CESM prealpha on hobart_nag +Dependencies: cime_config0.0.1.alpha.07, cism2_1_30, clm4_5_15_r235 + +Brief Summary: + - Use ESMF7 on yellowstone and cheyenne. + - Pass annual-average fields to GLC. + - Give users more info and control over key run commands. + - Moves unit testing infrastructure from tools/unit_testing to + scripts/fortran_unit_testing. + - There were tabs in create_newcase. None prevented the script from + running, but should be removed for consistency. + - nag compliler needs an additional library flag when built with debug. + - Update case members in case_setup. + - Handle the case when executable is None. + - Fix big mistake in scripts_regression_tests. + - Minor changes to xmlchange. + - Fix minor bug in test_scheduler. + - Rewrite testreporter into python. + - Print time built per model. + - Add input-dir to create_newcase. + - Get unit test build and run working with serial or parallel pFUnit. + - Add Support of true and false in Namelist. + - Remove the cprnc.out files prior to running tests. + - Fix bug where LockedFiles weren't checked if there was a dot in + filepath. + - Addition of new dwav and dlnd compsets and functionality. + - Errput may be empty string or None. + - Return aprun args but use executable string from xml. + - Add illegal char : to check_name test. + - Simplify netcdf lib args on hobart. + - Bug fix for DART in case_st_archive.py. + - Get Fortran unit tests working on cheyenne. + - Two changes to fix the Fortran unit tests that had gotten broken + recently. + - Improve create_test output by dumping case errors directly to screen. + - Fix NODEFAIL test on cheyenne. + - Fix indentation problem in case_setup. + - Allow user_mods directory in compset definition. + - Fix typo in cori-knl bind command. + - Big cs.status upgrade. + - Force user to always go through case.submit. + - Promote to warning if machine doesn't match probed machine. + - Make Machines.get_value more likely to return values of the correct + type. + - Remove conditional pio1 code. + - Fixes for PRE test. + - Set rearr opts using a pio interface. + - Bring in latest version of PIO1 to CIME. + +User interface changes: + - Moves run_tests.py to scripts/fortran_unit_testing. + - Adds PET test to cime_developer, PET behavior change. + - Output time per built component. + - config_compilers should now give MPILIB and compile_threaded + attributes for PFUNIT. + - The default mode of operation for run_tests.py is now serial. + - Other components that have unit tests (CLM and CAM) will need to + rework their top-level CMakeLists.txt file slightly. + - Do not allow ':' in testid or case name. + - Output changes for create_test. + - Significant to cs.status. + - Minor output changes to create_newcase. + - New PIO1 interface to set rearranger options. + - Signficant: new preview_run tool and new XML variables. + + +PR summary: git log --oneline --first-parent [previous_tag]..master +6d8ccb3 Merge pull request #1403 from fvitt/esmf +fe82bd6 Merge pull request #1413 from billsacks/billsacks/glc_tavg_v2 +dc29942 Merge pull request #1359 from ESMCI/jgfouca/new_query_tools +9c5b1aa Merge pull request #1428 from billsacks/move_unit_tests +c726ded Merge pull request #1435 from ESMCI/erichlf/create_newcase/tabs2spaces +8924bb3 Merge pull request #1421 from jedwards4b/hobart_nag_fix +f59dd4d Merge pull request #1414 from ESMCI/jgfouca/pet_fix_plus_coverage +67b1be7 Merge pull request #1410 from jedwards4b/executable_none_fix +0fe9e3d Merge pull request #1412 from ESMCI/jgfouca/fix_scripts_regr_test_rc +7c28d90 Merge pull request #1405 from ESMCI/jgfouca/xmlchange_append_add_space +68ab80b Merge pull request #1406 from ESMCI/jgfouca/test_sched_bugfix +da9d3b8 Merge pull request #1299 from ESMCI/fischer/testreporter_rewrite_python +f8f0fec Merge pull request #1402 from ESMCI/jgfouca/show_build_time_per_component +dc1a7ce Merge pull request #1401 from ESMCI/jgfouca/add_input_dir_to_create_newcase +962b20b Merge pull request #1396 from billsacks/compiler_from_macros +af32abd Merge pull request #1395 from ESMCI/erichlf/Build/Namelist.pm +9f5d489 Merge pull request #1394 from jedwards4b/system_test_rerun_improvement +1db632a Merge pull request #1391 from ESMCI/sarich/fix_bug_in_check_lockedfiles +5cda778 Merge pull request #1365 from ESMCI/mvertens/data_wave +2742c17 Merge pull request #1390 from ESMCI/jgfouca/fix_bug_in_errput_run_cmd +6dd85dd Merge pull request #1385 from jedwards4b/aprun_fix +4fb16a1 Merge pull request #1383 from jedwards4b/new_illegal_char +40a6017 Merge pull request #1386 from jedwards4b/hobart_netcdf_slib_fix +5a141f8 Merge pull request #1375 from ESMCI/bertinia/st_archive +06c5645 Merge pull request #1381 from billsacks/cheyenne_unit_tests +9629886 Merge pull request #1378 from jedwards4b/funit_test_fix +d79922e Merge pull request #1377 from ESMCI/jgfouca/improve_create_test_output +9d9f09c Merge pull request #1370 from ESMCI/jgfouca/minor_fix_for_cheyenne +a99eab3 Merge pull request #1373 from jedwards4b/pio_adjust_indentation_fix +ca289d9 Merge pull request #1366 from ESMCI/mvertens/usermods_in_compset +aaee38a Merge pull request #1369 from jedwards4b/cori-knlfix +d0e277d Merge pull request #1363 from ESMCI/jgfouca/big_cs_status_upgrade +dd5a30b Merge pull request #1362 from ESMCI/jgfouca/force_resubmit_to_use_submit +8282a8f Merge pull request #1361 from ESMCI/jgfouca/inform_user_of_machine +2530c66 Merge pull request #1360 from ESMCI/jgfouca/machine_get_value_type +1335a62 Merge pull request #1357 from ESMCI/jayeshkrishna/rem_cond_pio1_code +9903472 Merge pull request #1350 from gold2718/pausePOP +9f9da67 Merge pull request #1354 from ESMCI/jayeshkrishna/pio_use_set_rearr_opts +be0f9a4 Merge pull request #1352 from ESMCI/jayeshkrishna/bring_latest_pio1 + + +Modified files: git diff --name-status [previous_tag] + +M CMakeLists.txt +D README.unit_testing +M config/acme/machines/config_compilers.xml +M config/acme/machines/userdefined_laptop_template/config_compilers.xml +M config/cesm/config_grids.xml +M config/cesm/machines/config_compilers.xml +M config/cesm/machines/config_machines.xml +M config/cesm/machines/userdefined_laptop_template/config_compilers.xml +M config/config_tests.xml +M config/xml_schemas/config_compsets.xsd +M config/xml_schemas/config_grids_v2.xsd +M config/xml_schemas/config_machines.xsd +A scripts/Tools/cs.status +D scripts/Tools/cs_status +A scripts/Tools/preview_run +A scripts/Tools/testreporter.py +M scripts/Tools/xmlchange +M scripts/create_newcase +M scripts/create_test +A scripts/fortran_unit_testing/Examples/circle_area/src/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/circle_area/src/circle.F90 +A scripts/fortran_unit_testing/Examples/circle_area/tests/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/circle_area/tests/CTest/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/circle_area/tests/CTest/test_driver.F90 +A scripts/fortran_unit_testing/Examples/circle_area/tests/pFUnit/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/circle_area/tests/pFUnit/test_circle.pf +A scripts/fortran_unit_testing/Examples/interpolate_1d/src/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/interpolate_1d/src/interpolate_1d.F90 +A scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CTest/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/interpolate_1d/tests/CTest/test_driver.F90 +A scripts/fortran_unit_testing/Examples/interpolate_1d/tests/pFUnit/CMakeLists.txt +A scripts/fortran_unit_testing/Examples/interpolate_1d/tests/pFUnit/test_interpolate_point.pf +A scripts/fortran_unit_testing/Examples/test_list.xml +A scripts/fortran_unit_testing/README +A scripts/fortran_unit_testing/python/.gitignore +A scripts/fortran_unit_testing/python/printer.py +A scripts/fortran_unit_testing/python/test_xml_test_list.py +A scripts/fortran_unit_testing/python/xml_test_list.py +A scripts/fortran_unit_testing/run_tests.py +M scripts/lib/CIME/BuildTools/cmakemacroswriter.py +M scripts/lib/CIME/BuildTools/configure.py +M scripts/lib/CIME/SystemTests/nck.py +M scripts/lib/CIME/SystemTests/nodefail.py +M scripts/lib/CIME/SystemTests/pre.py +M scripts/lib/CIME/SystemTests/system_tests_common.py +M scripts/lib/CIME/XML/compsets.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/XML/env_mach_specific.py +M scripts/lib/CIME/XML/machines.py +A scripts/lib/CIME/XML/test_reporter.py +M scripts/lib/CIME/aprun.py +M scripts/lib/CIME/build.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/case_run.py +M scripts/lib/CIME/case_setup.py +M scripts/lib/CIME/case_st_archive.py +M scripts/lib/CIME/case_test.py +M scripts/lib/CIME/check_lockedfiles.py +M scripts/lib/CIME/code_checker.py +M scripts/lib/CIME/nmlgen.py +M scripts/lib/CIME/test_scheduler.py +M scripts/lib/CIME/test_status.py +A scripts/lib/CIME/tests/test_user_mod_support.py +M scripts/lib/CIME/user_mod_support.py +M scripts/lib/CIME/utils.py +M scripts/lib/cs.status.template +A scripts/lib/testreporter.template +M scripts/lib/update_acme_tests.py +M scripts/tests/scripts_regression_tests.py +M src/components/data_comps/datm/cime_config/config_component.xml +M src/components/data_comps/datm/cime_config/namelist_definition_datm.xml +M src/components/data_comps/datm/datm_comp_mod.F90 +M src/components/data_comps/desp/desp_comp_mod.F90 +M src/components/data_comps/desp/esp_comp_mct.F90 +M src/components/data_comps/dice/cime_config/config_component.xml +M src/components/data_comps/dice/cime_config/namelist_definition_dice.xml +M src/components/data_comps/dice/dice_comp_mod.F90 +M src/components/data_comps/dlnd/cime_config/buildnml +M src/components/data_comps/dlnd/cime_config/config_component.xml +M src/components/data_comps/docn/cime_config/config_component.xml +M src/components/data_comps/docn/cime_config/namelist_definition_docn.xml +M src/components/data_comps/docn/docn_comp_mod.F90 +M src/components/data_comps/drof/cime_config/namelist_definition_drof.xml +M src/components/data_comps/drof/drof_comp_mod.F90 +M src/components/data_comps/dwav/cime_config/buildnml +M src/components/data_comps/dwav/cime_config/config_component.xml +M src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml +M src/components/data_comps/dwav/dwav_comp_mod.F90 +M src/drivers/mct/cime_config/buildexe +M src/drivers/mct/cime_config/buildnml +M src/drivers/mct/cime_config/config_component.xml +M src/drivers/mct/cime_config/config_component_acme.xml +M src/drivers/mct/cime_config/config_component_cesm.xml +M src/drivers/mct/cime_config/config_compsets.xml +M src/drivers/mct/cime_config/namelist_definition_drv.xml +M src/drivers/mct/main/cesm_comp_mod.F90 +M src/drivers/mct/main/prep_glc_mod.F90 +M src/drivers/mct/main/seq_io_mod.F90 +M src/drivers/mct/main/seq_rest_mod.F90 +M src/drivers/mct/shr/seq_comm_mct.F90 +M src/drivers/mct/shr/seq_infodata_mod.F90 +M src/drivers/mct/shr/seq_timemgr_mod.F90 +M src/drivers/mct/unit_test/CMakeLists.txt +M src/drivers/mct/unit_test/stubs/CMakeLists.txt +A src/drivers/mct/unit_test/stubs/seq_timemgr_mod.F90 +A src/externals/CMake/CIME_initial_setup.cmake +M src/externals/CMake/CIME_utils.cmake +M src/externals/CMake/README.md +M src/externals/CMake/pFUnit_utils.cmake +M src/externals/pio1/pio/CMakeLists.txt +M src/externals/pio1/pio/pio.F90 +M src/externals/pio1/pio/piolib_mod.F90 +M src/share/unit_test_stubs/pio/pio.F90.in +M src/share/util/shr_pio_mod.F90 +D tools/unit_testing/Examples/circle_area/src/CMakeLists.txt +D tools/unit_testing/Examples/circle_area/src/circle.F90 +D tools/unit_testing/Examples/circle_area/tests/CMakeLists.txt +D tools/unit_testing/Examples/circle_area/tests/CTest/CMakeLists.txt +D tools/unit_testing/Examples/circle_area/tests/CTest/test_driver.F90 +D tools/unit_testing/Examples/circle_area/tests/pFUnit/CMakeLists.txt +D tools/unit_testing/Examples/circle_area/tests/pFUnit/test_circle.pf +D tools/unit_testing/Examples/interpolate_1d/src/CMakeLists.txt +D tools/unit_testing/Examples/interpolate_1d/src/interpolate_1d.F90 +D tools/unit_testing/Examples/interpolate_1d/tests/CMakeLists.txt +D tools/unit_testing/Examples/interpolate_1d/tests/CTest/CMakeLists.txt +D tools/unit_testing/Examples/interpolate_1d/tests/CTest/test_driver.F90 +D tools/unit_testing/Examples/interpolate_1d/tests/pFUnit/CMakeLists.txt +D tools/unit_testing/Examples/interpolate_1d/tests/pFUnit/test_interpolate_point.pf +D tools/unit_testing/Examples/test_list.xml +D tools/unit_testing/README +D tools/unit_testing/python/.gitignore +D tools/unit_testing/python/comparable.py +D tools/unit_testing/python/environment.py +D tools/unit_testing/python/machine_setup.py +D tools/unit_testing/python/printer.py +D tools/unit_testing/python/test_environment.py +D tools/unit_testing/python/test_xml_test_list.py +D tools/unit_testing/python/test_xml_utils.py +D tools/unit_testing/python/xml_test_list.py +D tools/unit_testing/python/xml_utils.py +D tools/unit_testing/run_tests.py +M utils/perl5lib/Build/Namelist.pm +====================================================================== + +====================================================================== + +Originator: Chris Fischer +Date: 4-14-2017 +Tag: cime5.3.0-alpha.07 +Answer Changes: None +Tests on various PRs: by-hand, code_checker, pylint, scripts_regression_tests, + cime_developer with gnu compiler on cheyenne, All gnu debug tests in CESM + test suites, create B compset with ww3 at T31 resolution, + J_TestCreateNewcase, some cesm tests on cori-knl +Dependencies: + +Brief Summary: + -Updates to get going on cori-knl + -Update domain_b in nnsm map + -Fix bug where create_test was not handling user-selected projects correctly + in the case where the test_root depended on PROJECT. + -HOMME Improvement + -Unit testing: Determine machine-specific settings automatically + -Mapping/update runoff to ocn + -Fixes an issue with --user-compset flag + -Fixes for cheyenne issue + -Validate the case and test_id arguments + -Add more checks for gnu in debug mode. + -Avoid a problem with pio_typename in clone with keepexe. + -Fix blues NETCDFROOT unresolved error. + -Fix TOTAL_CORES. + -Refactor BUILD_THREADED to match documentation. + -Refactor run_cmd to handle file output streams more easily. + -More-robust way of getting path to create_test. + -Better content in TestStatus.log file. + -Improved formatting/content of TestStatus.log + -Better handling of poorly implemented batch systems. + -Fix atm grid match for T31 atm2wav map. + -Change flag for answer to create_newcase + -Fix bug in xmlquery, add regr test to catch in future + -Improve error message when user forgets --user-compset. + -Cleanup TEST_ROOT if everything works. + -Reduce newlines in CaseStatus. + +User interface changes: + -BUILD_THREADED behaves differently + -for create_newcase, --answer becomes --handle-preexisting-dirs + -Formatting changes to CaseStatus + +PR summary: git log --oneline --first-parent [previous_tag]..master +30243ac Merge pull request #1351 from jedwards4b/cori_update +1f4c4e9 Merge pull request #1349 from mnlevy1981/mapping/runoff_metadata_bugfix +7664249 Merge pull request #1348 from ESMCI/jgfouca/fix_custom_proj_test_root +6156e0a Merge pull request #1346 from ESMCI/jgfouca/homme_improvement +3779ec1 Merge pull request #1343 from billsacks/run_tests_defaults +5074c4a Merge pull request #1280 from mnlevy1981/mapping/update_runoff_to_ocn +60bfc01 Merge pull request #1341 from jedwards4b/user-compset +5e79b04 Merge pull request #1340 from jedwards4b/cheyenne_fixes +37aac0f Merge pull request #1337 from jedwards4b/validate_case_and_test_id_args +5edf3fa Merge pull request #1339 from billsacks/gnu_debug_checks +82b6181 Merge pull request #1305 from jedwards4b/clone_keepexe_and_pio_typename +4d62d80 Merge pull request #1333 from ESMCI/sarich/fix-blues-netcdfroot +12bc9b3 Merge pull request #1336 from ESMCI/jgfouca/fix_total_cores +1adf8d1 Merge pull request #1335 from ESMCI/jgfouca/cleanup_build_threaded +315e849 Merge pull request #1326 from ESMCI/jgfouca/refactor_run_cmd +997ec60 Merge pull request #1321 from ESMCI/jgfouca/cime_bisect_robustness +cb7bd10 Merge pull request #1317 from ESMCI/jgfouca/better_testlog_content +7fb258b Merge pull request #1316 from ESMCI/jgfouca/handle_titan_batch_errs +2b4f7fe Merge pull request #1320 from billsacks/t31_wav_map +7370b02 Merge pull request #1311 from ESMCI/jgfouca/better_name_for_answer +b14ebe0 Merge pull request #1312 from ESMCI/jgfouca/fix_xmlquery_listall +f741180 Merge pull request #1314 from billsacks/user_compset_errmsg +243fafa Merge pull request #1310 from ESMCI/jgfouca/better_regr_cleanup +a4c7ab7 Merge pull request #1307 from ESMCI/jgfouca/remove_useless_newlines_from_casestatus + + +Modified files: git diff --name-status [previous_tag] +M README.unit_testing +M config/acme/machines/config_machines.xml +M config/cesm/config_grids.xml +M config/cesm/machines/config_batch.xml +M config/cesm/machines/config_compilers.xml +M config/cesm/machines/config_machines.xml +M config/cesm/machines/config_pio.xml +M config/xml_schemas/config_machines.xsd +M scripts/Tools/cime_bisect +M scripts/Tools/xmlquery +M scripts/create_clone +M scripts/create_newcase +M scripts/create_test +M scripts/lib/CIME/SystemTests/erp.py +M scripts/lib/CIME/SystemTests/homme.py +M scripts/lib/CIME/SystemTests/system_tests_common.py +M scripts/lib/CIME/SystemTests/system_tests_compare_two.py +M scripts/lib/CIME/XML/compsets.py +M scripts/lib/CIME/XML/entry_id.py +M scripts/lib/CIME/XML/env_base.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/XML/env_mach_specific.py +M scripts/lib/CIME/XML/machines.py +M scripts/lib/CIME/aprun.py +M scripts/lib/CIME/build.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/case_run.py +M scripts/lib/CIME/case_setup.py +M scripts/lib/CIME/case_st_archive.py +M scripts/lib/CIME/hist_utils.py +M scripts/lib/CIME/preview_namelists.py +M scripts/lib/CIME/provenance.py +M scripts/lib/CIME/test_scheduler.py +M scripts/lib/CIME/utils.py +M scripts/lib/CIME/wait_for_tests.py +M scripts/tests/scripts_regression_tests.py +M src/drivers/mct/cime_config/config_component.xml +M src/externals/pio1/timing/COPYING +M src/externals/pio2/src/gptl/COPYING +M src/share/timing/COPYING +M src/share/util/mct_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL +M tools/mapping/gen_mapping_files/runoff_to_ocn/README +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx1v7_nnsm_e1000r300.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx3v7_nnsm_e1000r500.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx1v7_nnsm_e1000r300.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx3v7_nnsm_e1000r500.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_ar9v4_nnsm_e1000r300.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_gx3v7_nnsm_e1000r300.nml +A tools/mapping/gen_mapping_files/runoff_to_ocn/maps/.gitignore +A tools/mapping/gen_mapping_files/runoff_to_ocn/ncl/merge_mapping_files.ncl +A tools/mapping/gen_mapping_files/runoff_to_ocn/run_merge_mapping_files.sh +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map.nml +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map_r05_gx3v7.nml +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map_rx1_gx1v6.nml +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map_tx01.nml +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map_wr50a_ar9v4.nml +D tools/mapping/gen_mapping_files/runoff_to_ocn/runoff_map_wr50a_gx3v7.nml +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/Makefile +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/map_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90 +M tools/mapping/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90 +M tools/unit_testing/run_tests.py + +====================================================================== + +Originator:Chris Fischer +Date: 4-4-2017 +Tag: cime5.3.0-alpha.06 +Answer Changes: None +Tests: code_checker, NODEFAIL +Dependencies: + +Brief Summary: + - Fix handling of spare nodes + +User interface changes: + +PR summary: git log --oneline --first-parent [previous_tag]..master +95019a3 Merge pull request #1302 from ESMCI/jgfouca/fixes_to_recent_prs + +Modified files: git diff --name-status [previous_tag] +M scripts/lib/CIME/XML/env_mach_pes.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/case_run.py + +====================================================================== + +Originator: Chris Fischer +Date: 4-4-2017 +Tag: cime5.3.0-alpha.05 +Answer Changes: None +Tests: scripts_regression_test, code checker +Dependencies: + +Brief Summary: + - First try to support titan restart. + - Pull back ACME repo changes of CIME since last subtree split (late + January?) up to March 29 back into CIME master. + - Allow a custom input root through create_test. + - Update pio stubs to include new derived field needed in pio_init. + - Set a default value for esmf_logging so it does not have to appear in + drv_in. + - Add logging options to ESMF_Initialize + - Fix pylint issues. + - Fix manage_case --query-grids for cesm. + - Adds support for namelist entries of the form foo(3) = 'a' + - Add mpilib and compiler command line options to configure. + - summarize_cprnc_diffs: Print unnormalized RMS diffs. + - Make sure pio_typename is valid. + - New hooks for runnning external scripts for pre-run and post-run. + - Enable PIO rearranger options for PIO2. + - code_checker needs to work outside of ESMCI/CIME. + - Fix pylint error in buildnml. + - Add flags to make create_newcase non-interactive. + - Added mct/cime_config/namelist_definition_drv_flds.xml with updated + schema. + - Some driver code no longer needed to do this for cam. + - Supress a future warning on the use of condition in expect subroutine. + - Couple fixes to recent PR that added SUBMIT phase. + +User interface changes: + - Add --input-dir argument to create_test + - esmf_logging (and therefore drv_in) are now optional + - New run time XML variable: ESMF_LOGFILE_KIND + - New --answer flag to create_newcase + - Removes a distracting warning + +PR summary: git log --oneline --first-parent [previous_tag]..master +eeb1e2c Merge pull request #1275 from ESMCI/jgfouca/titan_restart +fbf1b0c Merge pull request #1287 from ESMCI/agsalin/merge-from-acme-03292017 +dda631e Merge pull request #1298 from ESMCI/jgfouca/allow_custom_input_root +2dc9a8a Merge pull request #1301 from jedwards4b/unit_test_stubs_update +3d76b10 Merge pull request #1300 from gold2718/esmfLog +0d320ec Merge pull request #1289 from gold2718/esmfLog +8890873 Merge pull request #1290 from ESMCI/jgfouca/grids_pylint_fixes +55a084b Merge pull request #1285 from ESMCI/mvertens/manage_case +9d8396a Merge pull request #1259 from jedwards4b/index_in_namelist +4e125ee Merge pull request #1268 from jedwards4b/configure_update +9759e15 Merge pull request #1279 from billsacks/summarize_rms +840ae3c Merge pull request #1273 from jedwards4b/pio_typename_fix +07e61ed Merge pull request #1256 from ESMCI/mvertens/phys_cycle +85f831d Merge pull request #1260 from ESMCI/jayeshkrishna/enable_rearr_opts_pio2 +fbfa996 Merge pull request #1269 from ESMCI/jgfouca/code_checker_work_outside_esmci +300cbf3 Merge pull request #1253 from ESMCI/jgfouca/pylint_fix_to_buildnml +d44970c Merge pull request #1249 from ESMCI/jgfouca/create_newcase_non_interactive +8270602 Merge pull request #1231 from ESMCI/mvertens/drv_flds_in +5f3072c Merge pull request #1237 from jedwards4b/jpe_fix_atm_simple_settings +ffa61ea Merge pull request #1242 from jedwards4b/jpe/futurewarning +fecaaad Merge pull request #1240 from ESMCI/jgfouca/fixes_to_submit_phase_pr + + +Modified files: git diff --name-status [previous_tag] +M config/acme/allactive/config_pes.xml +M config/acme/allactive/config_pesall.xml +M config/acme/config_archive.xml +M config/acme/config_files.xml +M config/acme/config_grids.xml +M config/acme/machines/Makefile +M config/acme/machines/config_batch.xml +M config/acme/machines/config_compilers.xml +M config/acme/machines/config_machines.xml +M config/acme/machines/config_pio.xml +A config/acme/machines/syslog.anvil +A config/acme/testmods_dirs/allactive/cam/outfrq9s/xmlchange_cmnds +A config/acme/testmods_dirs/allactive/force_netcdf_pio/shell_commands +A config/acme/testmods_dirs/allactive/force_netcdf_pio/xmlchange_cmnds +M config/cesm/config_files.xml +M config/cesm/config_grids.xml +M config/cesm/machines/config_pio.xml +M config/config_tests.xml +M scripts/Tools/check_input_data +M scripts/create_newcase +M scripts/create_test +M scripts/lib/CIME/BuildTools/configure.py +M scripts/lib/CIME/BuildTools/macrowriterbase.py +M scripts/lib/CIME/SystemTests/dae.py +M scripts/lib/CIME/SystemTests/homme.py +A scripts/lib/CIME/SystemTests/nodefail.py +M scripts/lib/CIME/XML/compilers.py +M scripts/lib/CIME/XML/entry_id.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/XML/env_mach_pes.py +M scripts/lib/CIME/XML/env_mach_specific.py +M scripts/lib/CIME/XML/grids.py +M scripts/lib/CIME/XML/namelist_definition.py +A scripts/lib/CIME/aprun.py +M scripts/lib/CIME/bless_test_results.py +M scripts/lib/CIME/build.py +M scripts/lib/CIME/buildnml.py +M scripts/lib/CIME/case.py +M scripts/lib/CIME/case_run.py +M scripts/lib/CIME/case_setup.py +M scripts/lib/CIME/case_st_archive.py +M scripts/lib/CIME/case_submit.py +M scripts/lib/CIME/code_checker.py +M scripts/lib/CIME/get_timing.py +M scripts/lib/CIME/namelist.py +M scripts/lib/CIME/nmlgen.py +M scripts/lib/CIME/preview_namelists.py +M scripts/lib/CIME/test_scheduler.py +M scripts/lib/CIME/utils.py +M scripts/lib/update_acme_tests.py +M scripts/manage_case +M scripts/tests/scripts_regression_tests.py +M src/build_scripts/buildlib.pio +M src/components/data_comps/datm/cime_config/buildnml +M src/components/data_comps/desp/cime_config/buildnml +M src/components/data_comps/dice/cime_config/buildnml +M src/components/data_comps/dlnd/cime_config/buildnml +M src/components/data_comps/docn/cime_config/buildnml +M src/components/data_comps/drof/cime_config/buildnml +M src/components/data_comps/dwav/cime_config/buildnml +D src/drivers/mct/bld/namelist_files/namelist_defaults_drv.xml +D src/drivers/mct/bld/namelist_files/namelist_definition_drv.xml +D src/drivers/mct/bld/namelist_files/namelist_definition_drv_flds.xml +D src/drivers/mct/bld/namelist_files/namelist_definition_modio.xml +D src/drivers/mct/bld/testdir/Buildconf/camconf/drv_flds_in +D src/drivers/mct/bld/testdir/Buildconf/clmconf/drv_flds_in +D src/drivers/mct/bld/testdir/env_build.xml +D src/drivers/mct/bld/testdir/env_case.xml +D src/drivers/mct/bld/testdir/env_mach_pes.xml +D src/drivers/mct/bld/testdir/env_run.xml +D src/drivers/mct/bld/testdir/runtest +D src/drivers/mct/bld/testdir/user_nl_cpl +M src/drivers/mct/cime_config/buildnml +M src/drivers/mct/cime_config/config_component.xml +M src/drivers/mct/cime_config/config_component_acme.xml +M src/drivers/mct/cime_config/namelist_definition_drv.xml +A src/drivers/mct/cime_config/namelist_definition_drv_flds.xml +M src/drivers/mct/main/cesm_driver.F90 +M src/drivers/mct/shr/seq_comm_mct.F90 +M src/externals/pio1/pio/CMakeLists.txt +M src/share/esmf_wrf_timemgr/ESMF_Stubs.F90 +M src/share/unit_test_stubs/pio/pio.F90.in +M src/share/util/shr_pio_mod.F90 +M tools/Readme.configure +M tools/configure +M tools/cprnc/summarize_cprnc_diffs +M tools/mapping/map_field/src/map_field.F90 + +====================================================================== + +Originator: Chris Fischer +Date: 3-14-2017 +Tag: cime5.3.0-alpha.04 +Answer Changes: Answer changes for gx3v7 +Tests: scripts_regression_test, SMS.f09_g16.B1850 +Dependencies: + +Brief Summary: + -Need to set RTM to NULL in both cases in order to have this work with fix + needed for upcoming pythonization of rtm and mosart buildnml + -Improve create_test output + -Remove a misleading print statement from tests. + -Better way to handle batch errors. + -Change in location of $SRCROOT/config/allactive to $SRCROOT/cime_config. + -Update so that all relevent grids contain a mask variable. + +User interface changes: New SUBMIT phase! + +PR summary: git log --oneline --first-parent [previous_tag]..master +34a244a Merge pull request #1239 from ESMCI/mvertens/fix_ssptest +867711d Merge pull request #1238 from ESMCI/jgfouca/create_test_better_output +d050c08 Merge pull request #1235 from jedwards4b/mask_updates +5058c33 Merge pull request #1234 from ESMCI/fischer/config_cesm_allactive_mv +64d6bd2 Merge pull request #1230 from ESMCI/jgfouca/add_submit_phase_2 +dec7437 Merge pull request #1232 from jedwards4b/remove_incorrect_statement + + + +Modified files: git diff --name-status [previous_tag] +M config/cesm/config_files.xml +M config/cesm/config_grids.xml +M config/xml_schemas/config_grids_v2.xsd +M scripts/lib/CIME/SystemTests/ssp.py +M scripts/lib/CIME/SystemTests/system_tests_common.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/case_submit.py +M scripts/lib/CIME/case_test.py +M scripts/lib/CIME/test_scheduler.py +M scripts/lib/CIME/test_status.py +M scripts/tests/scripts_regression_tests.p + +====================================================================== +Originator: Chris Fischer +Date: 3-11-2017 +Tag: cime5.3.0-alpha.03 +Answer Changes: None +Tests: scripts_regression_tests, various CESM tests. +Dependencies: + +Brief Summary: + Added new aliases for resolutions supporting a gx1v7 mask. + Move location of config/cesm/allactive. + Get rid of hard coded comp_class settings and use get_values instead. + Fix build model message. + Ignore prereq if forced. + Additional include path needed for cmake check_function_exists. + PIO 2 import. + +User interface changes: + +PR summary: git log --oneline --first-parent [previous_tag]..master + 3627e16 Merge pull request #1228 from jtruesdal/add_gx1v7 + 7b10920 Merge pull request #1227 from ESMCI/fischer/config_cesm_allactive_mv + ab579b3 Merge pull request #1226 from jedwards4b/compclasses_cleanup + dc0a87c Merge pull request #1225 from ESMCI/jgfouca/fix_build_msg + 5f4f989 Merge pull request #1218 from jedwards4b/ignore_prereq_if_forced + ff3d97f Merge pull request #1219 from jedwards4b/cmake_module_fix + 37fe05b Merge pull request #1202 from jedwards4b/pio2_import + +Modified files: git diff --name-status [previous_tag] +M config/acme/machines/Makefile +M config/cesm/config_files.xml +M config/cesm/config_grids.xml +M config/cesm/machines/Makefile +M scripts/lib/CIME/SystemTests/pem.py +M scripts/lib/CIME/SystemTests/pet.py +M scripts/lib/CIME/XML/env_batch.py +M scripts/lib/CIME/build.py +M scripts/lib/CIME/get_timing.py +M scripts/lib/CIME/test_scheduler.py +M src/build_scripts/buildlib.pio +M src/components/data_comps/docn/cime_config/config_component.xml +M src/externals/pio2/CMakeLists.txt +M src/externals/pio2/CTestConfig.cmake +M src/externals/pio2/CTestScript.cmake +M src/externals/pio2/cmake/FindNetCDF.cmake +M src/externals/pio2/cmake/LibCheck.cmake +M src/externals/pio2/cmake/LibMPI.cmake +A src/externals/pio2/cmake/mpiexec.nwscla +A src/externals/pio2/cmake/mpiexec.olcf +A src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake +M src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake +A src/externals/pio2/ctest/CTestEnvironment-nwscla.cmake +M src/externals/pio2/ctest/runcdash-alcf-ibm.sh +A src/externals/pio2/ctest/runcdash-anlworkstation.sh +M src/externals/pio2/ctest/runcdash-cgd-nag.sh +M src/externals/pio2/ctest/runcdash-nersc-cray.sh +M src/externals/pio2/ctest/runcdash-nersc-intel.sh +M src/externals/pio2/ctest/runcdash-nwsc-gnu.sh +M src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +M src/externals/pio2/ctest/runcdash-nwsc-intel.sh +M src/externals/pio2/ctest/runcdash-nwsc-pgi.sh +A src/externals/pio2/ctest/runcdash-nwscla-intel.sh +A src/externals/pio2/ctest/runctest-anlworkstation.sh +M src/externals/pio2/ctest/runctest-cgd.sh +M src/externals/pio2/ctest/runctest-nersc.sh +A src/externals/pio2/ctest/runctest-nwscla.sh +M src/externals/pio2/doc/CMakeLists.txt +M src/externals/pio2/doc/Doxyfile.in +M src/externals/pio2/doc/source/Error.txt +M src/externals/pio2/doc/source/Examples.txt +M src/externals/pio2/doc/source/Installing.txt +A src/externals/pio2/doc/source/Testing.txt +M src/externals/pio2/doc/source/base.txt +A src/externals/pio2/doc/source/contributing_code.txt +M src/externals/pio2/doc/source/mach_walkthrough.txt +M src/externals/pio2/examples/CMakeLists.txt +M src/externals/pio2/examples/c/CMakeLists.txt +M src/externals/pio2/examples/c/example1.c +M src/externals/pio2/examples/c/example2.c +M src/externals/pio2/examples/c/examplePio.c +A src/externals/pio2/examples/c/valsupp_example1.supp +M src/externals/pio2/examples/f03/CMakeLists.txt +M src/externals/pio2/examples/f03/examplePio.f90 +M src/externals/pio2/src/CMakeLists.txt +M src/externals/pio2/src/clib/CMakeLists.txt +M src/externals/pio2/src/clib/bget.c +M src/externals/pio2/src/clib/bget.h +M src/externals/pio2/src/clib/config.h.in +D src/externals/pio2/src/clib/ncparser.pl +D src/externals/pio2/src/clib/ncputgetparser.pl +M src/externals/pio2/src/clib/pio.h +D src/externals/pio2/src/clib/pio_c_get_template.c +D src/externals/pio2/src/clib/pio_c_put_template.c +D src/externals/pio2/src/clib/pio_c_template.c +M src/externals/pio2/src/clib/pio_darray.c +A src/externals/pio2/src/clib/pio_darray_int.c +M src/externals/pio2/src/clib/pio_file.c +M src/externals/pio2/src/clib/pio_get_nc.c +A src/externals/pio2/src/clib/pio_getput_int.c +M src/externals/pio2/src/clib/pio_internal.h +M src/externals/pio2/src/clib/pio_lists.c +A src/externals/pio2/src/clib/pio_msg.c +M src/externals/pio2/src/clib/pio_nc.c +M src/externals/pio2/src/clib/pio_nc4.c +M src/externals/pio2/src/clib/pio_put_nc.c +M src/externals/pio2/src/clib/pio_rearrange.c +M src/externals/pio2/src/clib/pio_spmd.c +A src/externals/pio2/src/clib/pio_varm.c +M src/externals/pio2/src/clib/pioc.c +M src/externals/pio2/src/clib/pioc_sc.c +M src/externals/pio2/src/clib/pioc_support.c +M src/externals/pio2/src/clib/topology.c +M src/externals/pio2/src/flib/CMakeLists.txt +M src/externals/pio2/src/flib/pio.F90 +M src/externals/pio2/src/flib/pio_nf.F90 +M src/externals/pio2/src/flib/pio_types.F90 +M src/externals/pio2/src/flib/piodarray.F90.in +M src/externals/pio2/src/flib/piolib_mod.F90 +M src/externals/pio2/src/flib/pionfatt_mod.F90.in +M src/externals/pio2/src/flib/pionfget_mod.F90.in +M src/externals/pio2/src/flib/pionfput_mod.F90.in +M src/externals/pio2/src/gptl/CMakeLists.txt +M src/externals/pio2/src/gptl/gptl.c +M src/externals/pio2/src/gptl/perf_mod.F90 +M src/externals/pio2/tests/CMakeLists.txt +A src/externals/pio2/tests/cunit/CMakeLists.txt +A src/externals/pio2/tests/cunit/pio_tests.h +A src/externals/pio2/tests/cunit/test_async_2comp.c +A src/externals/pio2/tests/cunit/test_async_3proc.c +A src/externals/pio2/tests/cunit/test_async_4proc.c +A src/externals/pio2/tests/cunit/test_async_simple.c +A src/externals/pio2/tests/cunit/test_common.c +A src/externals/pio2/tests/cunit/test_darray.c +A src/externals/pio2/tests/cunit/test_darray_1d.c +A src/externals/pio2/tests/cunit/test_darray_3d.c +A src/externals/pio2/tests/cunit/test_darray_multivar.c +A src/externals/pio2/tests/cunit/test_decomps.c +A src/externals/pio2/tests/cunit/test_intercomm2.c +A src/externals/pio2/tests/cunit/test_iosystem2.c +A src/externals/pio2/tests/cunit/test_iosystem2_simple.c +A src/externals/pio2/tests/cunit/test_iosystem2_simple2.c +A src/externals/pio2/tests/cunit/test_iosystem3.c +A src/externals/pio2/tests/cunit/test_iosystem3_simple.c +A src/externals/pio2/tests/cunit/test_iosystem3_simple2.c +A src/externals/pio2/tests/cunit/test_pioc.c +A src/externals/pio2/tests/cunit/test_pioc_fill.c +A src/externals/pio2/tests/cunit/test_pioc_putget.c +A src/externals/pio2/tests/cunit/test_pioc_unlim.c +A src/externals/pio2/tests/cunit/test_shared.c +A src/externals/pio2/tests/cunit/test_spmd.c +M src/externals/pio2/tests/general/CMakeLists.txt +A src/externals/pio2/tests/general/README.md +M src/externals/pio2/tests/general/ncdf_fail.F90.in +M src/externals/pio2/tests/general/ncdf_get_put.F90.in +A src/externals/pio2/tests/general/ncdf_inq.F90.in +D src/externals/pio2/tests/general/pio_decomp_fillval2.F90 +A src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in +M src/externals/pio2/tests/general/pio_decomp_tests.F90.in +A src/externals/pio2/tests/general/pio_decomp_tests_1d.F90.in +A src/externals/pio2/tests/general/pio_decomp_tests_2d.F90.in +A src/externals/pio2/tests/general/pio_decomp_tests_3d.F90.in +M src/externals/pio2/tests/general/pio_file_simple_tests.F90.in +A src/externals/pio2/tests/general/pio_iosystem_tests.F90.in +A src/externals/pio2/tests/general/pio_iosystem_tests2.F90.in +A src/externals/pio2/tests/general/pio_iosystem_tests3.F90.in +A src/externals/pio2/tests/general/pio_rearr.F90.in +A src/externals/pio2/tests/general/pio_rearr_opts.F90.in +A src/externals/pio2/tests/general/pio_rearr_opts2.F90.in +M src/externals/pio2/tests/general/test_memleak.c +M src/externals/pio2/tests/general/util/pio_tf_f90gen.pl +M src/externals/pio2/tests/general/util/pio_tutil.F90 +M src/externals/pio2/tests/performance/CMakeLists.txt +M src/externals/pio2/tests/unit/CMakeLists.txt +D src/externals/pio2/tests/unit/README +M src/externals/pio2/tests/unit/basic_tests.F90 +M src/externals/pio2/tests/unit/driver.F90 +M src/externals/pio2/tests/unit/global_vars.F90 +M src/externals/pio2/tests/unit/nc_set_log_level2.c +M src/externals/pio2/tests/unit/ncdf_tests.F90 +D src/externals/pio2/tests/unit/test_names.c +D src/externals/pio2/tests/unit/test_nc4.c + +====================================================================== +Originator: Chris Fischer +Date: 3-7-2017 +Tag: cime5.3.0-alpha.02 +Answer Changes: None +Tests: +Dependencies: + +Brief Summary: Revert PR# 1210, remove hardcoded compclasses + +User interface changes: + +PR summary: git log --oneline --first-parent [previous_tag]..master + 4cc7e22 Merge pull request #1213 from ESMCI/revert-1210-compclasses_cleanup + +Modified files: git diff --name-status [previous_tag] +M scripts/lib/CIME/SystemTests/ncr.py +M scripts/lib/CIME/SystemTests/pem.py +M scripts/lib/CIME/SystemTests/pet.py +M scripts/lib/CIME/get_timing.py + +====================================================================== + +Originator: Chris Fischer +Date: 3-6-2017 +Tag: cime5.3.0-alpha.01 +Answer Changes: None +Tests: code_checker,hist_utils.py,scripts_regression_tests.py, on + different PRs.. +Dependencies: + +Brief Summary: + - Adjust pio layout if pe layout is changed. + - Make baseline perms setting more robust. + - Allows user to rm -rf their build area and rebuild without having + to re-run case.setup. + - Big refactor of CaseStatus management. + - cdf64 only valid for serial netcdf, fix seq test + - Fix bug in ERP namelist comparison. + - Refactor env_batch + - fix issue with compset not printing to timing file + - copy rather than follow links in create_clone + - update compare_xml to find more differences + - remove hardcoded compclasses + +User interface changes: + - case.setup has a new option --no-adjust-pio + - case.build behavior slightly different case.build behavior slightly + different + - CaseStatus will look slightly different + - env_batch.xml now looks different + +PR summary: git log --oneline --first-parent [previous_tag]..master +3fd75db Merge pull request #1210 from jedwards4b/compclasses_cleanup +78cab9a Merge pull request #1207 from jedwards4b/fix_compare_xml +bd0f19d Merge pull request #1209 from jedwards4b/symlinks_in_clone +d308873 Merge pull request #1208 from jedwards4b/fix_compset_not_in_timing +77dc910 Merge pull request #1204 from ESMCI/jgfouca/refactor_env_batch +c10fe46 Merge pull request #1203 from ESMCI/jgfouca/fix_erp_namelist_issue +30e8e79 Merge pull request #1200 from jedwards4b/seq_test_fix +5eee02a Merge pull request #1199 from ESMCI/jgfouca/refactor_case_status +4219bfd Merge pull request #1196 from ESMCI/jgfouca/build_needs_to_make_dirs +33ac01a Merge pull request #1197 from ESMCI/jgfouca/minor_hist_chg +7e7f805 Merge pull request #1195 from jedwards4b/pio_layout_adjust +0131a12 resolve merge conflicts +6c93fc3 Merge branch 'agsalin/mkdir_src' +215e459 Merge pull request #1182 from jedwards4b/config_archive_split + + +Modified files: git diff --name-status [previous_tag] +M CMakeLists.txt +M ChangeLog +D cime_config/acme/allactive/config_compsets.xml +D cime_config/acme/allactive/config_compsets.xml.cime2 +D cime_config/acme/allactive/config_pes.xml +D cime_config/acme/allactive/config_pes.xml.cime2 +D cime_config/acme/allactive/config_pesall.xml +D cime_config/acme/allactive/testlist_allactive.xml +D cime_config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands +D cime_config/acme/allactive/testmods_dirs/force_netcdf_pio/shell_commands +D cime_config/acme/allactive/usermods_dirs/b1850/shell_commands +D cime_config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods +D cime_config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism +D cime_config/acme/config_archive.xml +D cime_config/acme/config_files.xml +D cime_config/acme/config_grids.xml +D cime_config/acme/machines/Depends.cetus +D cime_config/acme/machines/Depends.corip1 +D cime_config/acme/machines/Depends.cray +D cime_config/acme/machines/Depends.gnu +D cime_config/acme/machines/Depends.intel +D cime_config/acme/machines/Depends.intel14 +D cime_config/acme/machines/Depends.intelmic +D cime_config/acme/machines/Depends.intelmic14 +D cime_config/acme/machines/Depends.mira +D cime_config/acme/machines/Depends.nag +D cime_config/acme/machines/Depends.titan.cray +D cime_config/acme/machines/Depends.titan.intel +D cime_config/acme/machines/Depends.titan.pgi +D cime_config/acme/machines/Depends.titan.pgiacc +D cime_config/acme/machines/Makefile +D cime_config/acme/machines/README +D cime_config/acme/machines/config_batch.xml +D cime_config/acme/machines/config_batch.xsd +D cime_config/acme/machines/config_compilers.xml +D cime_config/acme/machines/config_lt_archive.xml +D cime_config/acme/machines/config_machines.xml +D cime_config/acme/machines/config_pio.xml +D cime_config/acme/machines/mkDepends +D cime_config/acme/machines/mkSrcfiles +D cime_config/acme/machines/syslog.cetus +D cime_config/acme/machines/syslog.cori-haswell +D cime_config/acme/machines/syslog.cori-knl +D cime_config/acme/machines/syslog.edison +D cime_config/acme/machines/syslog.mira +D cime_config/acme/machines/syslog.noop +D cime_config/acme/machines/syslog.titan +D cime_config/acme/machines/template.case.run +D cime_config/acme/machines/template.case.test +D cime_config/acme/machines/template.lt_archive +D cime_config/acme/machines/template.ocn_diagnostics +D cime_config/acme/machines/template.st_archive +D cime_config/acme/machines/template.timeseries +D cime_config/acme/machines/userdefined_laptop_template/README.md +D cime_config/acme/machines/userdefined_laptop_template/config_compilers.xml +D cime_config/acme/machines/userdefined_laptop_template/config_machines.xml +D cime_config/acme/machines/userdefined_laptop_template/config_pes.xml +D cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_cam +D cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_clm +D cime_config/buildlib.csm_share +D cime_config/buildlib.gptl +D cime_config/buildlib.mct +D cime_config/buildlib.mpi-serial +D cime_config/buildlib.pio +D cime_config/cesm/config_archive.xml +D cime_config/cesm/config_files.xml +D cime_config/cesm/config_grids.xml +D cime_config/cesm/machines/Depends.babbageKnc +D cime_config/cesm/machines/Depends.bluewaters +D cime_config/cesm/machines/Depends.corip1 +D cime_config/cesm/machines/Depends.cray +D cime_config/cesm/machines/Depends.gnu +D cime_config/cesm/machines/Depends.goldbach +D cime_config/cesm/machines/Depends.intel +D cime_config/cesm/machines/Depends.intel14 +D cime_config/cesm/machines/Depends.intelmic +D cime_config/cesm/machines/Depends.intelmic14 +D cime_config/cesm/machines/Depends.mira +D cime_config/cesm/machines/Depends.nag +D cime_config/cesm/machines/Makefile +D cime_config/cesm/machines/README +D cime_config/cesm/machines/config_batch.xml +D cime_config/cesm/machines/config_compilers.xml +D cime_config/cesm/machines/config_lt_archive.xml +D cime_config/cesm/machines/config_machines.xml +D cime_config/cesm/machines/config_pio.xml +D cime_config/cesm/machines/mkDepends +D cime_config/cesm/machines/mkSrcfiles +D cime_config/cesm/machines/nag_mpi_argument.txt +D cime_config/cesm/machines/template.case.run +D cime_config/cesm/machines/template.case.test +D cime_config/cesm/machines/template.lt_archive +D cime_config/cesm/machines/template.st_archive +D cime_config/cesm/machines/userdefined_laptop_template/README.md +D cime_config/cesm/machines/userdefined_laptop_template/config_compilers.xml +D cime_config/cesm/machines/userdefined_laptop_template/config_machines.xml +D cime_config/cesm/machines/userdefined_laptop_template/config_pes.xml +D cime_config/config_headers.xml +D cime_config/config_tests.xml +D cime_config/xml_schemas/archive.xsd +D cime_config/xml_schemas/cimeteststatus.xsd +D cime_config/xml_schemas/config_batch.xsd +D cime_config/xml_schemas/config_compilers_v2.xsd +D cime_config/xml_schemas/config_compsets.xsd +D cime_config/xml_schemas/config_grids_v2.xsd +D cime_config/xml_schemas/config_machines.xsd +D cime_config/xml_schemas/config_machines_template.xml +D cime_config/xml_schemas/config_pes.xsd +D cime_config/xml_schemas/entry_id.xsd +D cime_config/xml_schemas/entry_id_base.xsd +D cime_config/xml_schemas/entry_id_namelist.xsd +D cime_config/xml_schemas/testlist.xsd +D components/data_comps/datm/atm_comp_mct.F90 +D components/data_comps/datm/cime_config/buildlib +D components/data_comps/datm/cime_config/buildnml +D components/data_comps/datm/cime_config/config_component.xml +D components/data_comps/datm/cime_config/namelist_definition_datm.xml +D components/data_comps/datm/cime_config/user_nl_datm +D components/data_comps/datm/datm_comp_mod.F90 +D components/data_comps/datm/datm_shr_mod.F90 +D components/data_comps/datm/doc/SOM.pdf +D components/data_comps/datm/doc/bookinfo.xml +D components/data_comps/datm/doc/datacomps.xml +D components/data_comps/datm/doc/intro.xml +D components/data_comps/datm/doc/rundocbook.csh +D components/data_comps/datm/doc/streams.xml +D components/data_comps/datm/doc/ug.xml +D components/data_comps/desp/cime_config/buildlib +D components/data_comps/desp/cime_config/buildnml +D components/data_comps/desp/cime_config/config_component.xml +D components/data_comps/desp/cime_config/namelist_definition_desp.xml +D components/data_comps/desp/cime_config/user_nl_desp +D components/data_comps/desp/desp_comp_mod.F90 +D components/data_comps/desp/esp_comp_mct.F90 +D components/data_comps/desp/esp_utils.F90 +D components/data_comps/dice/cime_config/buildlib +D components/data_comps/dice/cime_config/buildnml +D components/data_comps/dice/cime_config/config_component.xml +D components/data_comps/dice/cime_config/namelist_definition_dice.xml +D components/data_comps/dice/cime_config/user_nl_dice +D components/data_comps/dice/dice_comp_mod.F90 +D components/data_comps/dice/ice_comp_mct.F90 +D components/data_comps/dlnd/cime_config/buildlib +D components/data_comps/dlnd/cime_config/buildnml +D components/data_comps/dlnd/cime_config/config_component.xml +D components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml +D components/data_comps/dlnd/cime_config/user_nl_dlnd +D components/data_comps/dlnd/dlnd_comp_mod.F90 +D components/data_comps/dlnd/lnd_comp_mct.F90 +D components/data_comps/docn/cime_config/buildlib +D components/data_comps/docn/cime_config/buildnml +D components/data_comps/docn/cime_config/config_component.xml +D components/data_comps/docn/cime_config/namelist_definition_docn.xml +D components/data_comps/docn/cime_config/user_nl_docn +D components/data_comps/docn/docn_comp_mod.F90 +D components/data_comps/docn/ocn_comp_mct.F90 +D components/data_comps/docn/tools/pop_som_frc/README +D components/data_comps/docn/tools/pop_som_frc/SOM.doc +D components/data_comps/docn/tools/pop_som_frc/SOM.pdf +D components/data_comps/docn/tools/pop_som_frc/pop_frc.csh +D components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl +D components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl +D components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl +D components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh +D components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh +D components/data_comps/drof/cime_config/buildlib +D components/data_comps/drof/cime_config/buildnml +D components/data_comps/drof/cime_config/config_component.xml +D components/data_comps/drof/cime_config/namelist_definition_drof.xml +D components/data_comps/drof/cime_config/user_nl_drof +D components/data_comps/drof/drof_comp_mod.F90 +D components/data_comps/drof/rof_comp_mct.F90 +D components/data_comps/dwav/README +D components/data_comps/dwav/bld/README +D components/data_comps/dwav/cime_config/buildlib +D components/data_comps/dwav/cime_config/buildnml +D components/data_comps/dwav/cime_config/config_component.xml +D components/data_comps/dwav/cime_config/namelist_definition_dwav.xml +D components/data_comps/dwav/cime_config/user_nl_dwav +D components/data_comps/dwav/dwav_comp_mod.F90 +D components/data_comps/dwav/wav_comp_mct.F90 +D components/stub_comps/satm/cime_config/buildlib +D components/stub_comps/satm/cime_config/buildnml +D components/stub_comps/satm/cime_config/config_component.xml +D components/stub_comps/satm/cpl/atm_comp_mct.F90 +D components/stub_comps/sesp/cime_config/buildlib +D components/stub_comps/sesp/cime_config/buildnml +D components/stub_comps/sesp/cime_config/config_component.xml +D components/stub_comps/sesp/cpl/esp_comp_mct.F90 +D components/stub_comps/sglc/cime_config/buildlib +D components/stub_comps/sglc/cime_config/buildnml +D components/stub_comps/sglc/cime_config/config_component.xml +D components/stub_comps/sglc/cpl/glc_comp_mct.F90 +D components/stub_comps/sice/cime_config/buildlib +D components/stub_comps/sice/cime_config/buildnml +D components/stub_comps/sice/cime_config/config_component.xml +D components/stub_comps/sice/cpl/ice_comp_mct.F90 +D components/stub_comps/slnd/cime_config/buildlib +D components/stub_comps/slnd/cime_config/buildnml +D components/stub_comps/slnd/cime_config/config_component.xml +D components/stub_comps/slnd/cpl/lnd_comp_mct.F90 +D components/stub_comps/socn/cime_config/buildlib +D components/stub_comps/socn/cime_config/buildnml +D components/stub_comps/socn/cime_config/config_component.xml +D components/stub_comps/socn/cpl/ocn_comp_mct.F90 +D components/stub_comps/srof/cime_config/buildlib +D components/stub_comps/srof/cime_config/buildnml +D components/stub_comps/srof/cime_config/config_component.xml +D components/stub_comps/srof/cpl/rof_comp_mct.F90 +D components/stub_comps/swav/cime_config/buildlib +D components/stub_comps/swav/cime_config/buildnml +D components/stub_comps/swav/cime_config/config_component.xml +D components/stub_comps/swav/cpl/wav_comp_mct.F90 +D components/xcpl_comps/xatm/cime_config/buildlib +D components/xcpl_comps/xatm/cime_config/buildnml +D components/xcpl_comps/xatm/cime_config/config_component.xml +D components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 +D components/xcpl_comps/xglc/cime_config/buildlib +D components/xcpl_comps/xglc/cime_config/buildnml +D components/xcpl_comps/xglc/cime_config/config_component.xml +D components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 +D components/xcpl_comps/xice/cime_config/buildlib +D components/xcpl_comps/xice/cime_config/buildnml +D components/xcpl_comps/xice/cime_config/config_component.xml +D components/xcpl_comps/xice/cpl/ice_comp_mct.F90 +D components/xcpl_comps/xlnd/cime_config/buildlib +D components/xcpl_comps/xlnd/cime_config/buildnml +D components/xcpl_comps/xlnd/cime_config/config_component.xml +D components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 +D components/xcpl_comps/xocn/cime_config/buildlib +D components/xcpl_comps/xocn/cime_config/buildnml +D components/xcpl_comps/xocn/cime_config/config_component.xml +D components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 +D components/xcpl_comps/xrof/cime_config/buildlib +D components/xcpl_comps/xrof/cime_config/buildnml +D components/xcpl_comps/xrof/cime_config/config_component.xml +D components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 +D components/xcpl_comps/xshare/dead_data_mod.F90 +D components/xcpl_comps/xshare/dead_mct_mod.F90 +D components/xcpl_comps/xshare/dead_mod.F90 +D components/xcpl_comps/xwav/cime_config/buildlib +D components/xcpl_comps/xwav/cime_config/buildnml +D components/xcpl_comps/xwav/cime_config/config_component.xml +D components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 +A config/acme/allactive/config_compsets.xml +A config/acme/allactive/config_compsets.xml.cime2 +A config/acme/allactive/config_pes.xml +A config/acme/allactive/config_pes.xml.cime2 +A config/acme/allactive/config_pesall.xml +A config/acme/allactive/testlist_allactive.xml +A config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands +A config/acme/allactive/testmods_dirs/force_netcdf_pio/shell_commands +A config/acme/allactive/usermods_dirs/b1850/shell_commands +A config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods +A config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism +A config/acme/config_archive.xml +A config/acme/config_files.xml +A config/acme/config_grids.xml +A config/acme/machines/Depends.cetus +A config/acme/machines/Depends.corip1 +A config/acme/machines/Depends.cray +A config/acme/machines/Depends.gnu +A config/acme/machines/Depends.intel +A config/acme/machines/Depends.intel14 +A config/acme/machines/Depends.intelmic +A config/acme/machines/Depends.intelmic14 +A config/acme/machines/Depends.mira +A config/acme/machines/Depends.nag +A config/acme/machines/Depends.titan.cray +A config/acme/machines/Depends.titan.intel +A config/acme/machines/Depends.titan.pgi +A config/acme/machines/Depends.titan.pgiacc +A config/acme/machines/Makefile +A config/acme/machines/README +A config/acme/machines/config_batch.xml +A config/acme/machines/config_batch.xsd +A config/acme/machines/config_compilers.xml +A config/acme/machines/config_lt_archive.xml +A config/acme/machines/config_machines.xml +A config/acme/machines/config_pio.xml +A config/acme/machines/mkDepends +A config/acme/machines/mkSrcfiles +A config/acme/machines/syslog.cetus +A config/acme/machines/syslog.cori-haswell +A config/acme/machines/syslog.cori-knl +A config/acme/machines/syslog.edison +A config/acme/machines/syslog.mira +A config/acme/machines/syslog.noop +A config/acme/machines/syslog.titan +A config/acme/machines/template.case.run +A config/acme/machines/template.case.test +A config/acme/machines/template.lt_archive +A config/acme/machines/template.ocn_diagnostics +A config/acme/machines/template.st_archive +A config/acme/machines/template.timeseries +A config/acme/machines/userdefined_laptop_template/README.md +A config/acme/machines/userdefined_laptop_template/config_compilers.xml +A config/acme/machines/userdefined_laptop_template/config_machines.xml +A config/acme/machines/userdefined_laptop_template/config_pes.xml +A config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_cam +A config/acme/testmods_dirs/allactive/cam/outfrq9s/user_nl_clm +A config/cesm/config_archive.xml +A config/cesm/config_files.xml +A config/cesm/config_grids.xml +A config/cesm/machines/Depends.babbageKnc +A config/cesm/machines/Depends.bluewaters +A config/cesm/machines/Depends.corip1 +A config/cesm/machines/Depends.cray +A config/cesm/machines/Depends.gnu +A config/cesm/machines/Depends.goldbach +A config/cesm/machines/Depends.intel +A config/cesm/machines/Depends.intel14 +A config/cesm/machines/Depends.intelmic +A config/cesm/machines/Depends.intelmic14 +A config/cesm/machines/Depends.mira +A config/cesm/machines/Depends.nag +A config/cesm/machines/Makefile +A config/cesm/machines/README +A config/cesm/machines/config_batch.xml +A config/cesm/machines/config_compilers.xml +A config/cesm/machines/config_lt_archive.xml +A config/cesm/machines/config_machines.xml +A config/cesm/machines/config_pio.xml +A config/cesm/machines/mkDepends +A config/cesm/machines/mkSrcfiles +A config/cesm/machines/nag_mpi_argument.txt +A config/cesm/machines/template.case.run +A config/cesm/machines/template.case.test +A config/cesm/machines/template.lt_archive +A config/cesm/machines/template.st_archive +A config/cesm/machines/userdefined_laptop_template/README.md +A config/cesm/machines/userdefined_laptop_template/config_compilers.xml +A config/cesm/machines/userdefined_laptop_template/config_machines.xml +A config/cesm/machines/userdefined_laptop_template/config_pes.xml +A config/config_headers.xml +A config/config_tests.xml +A config/xml_schemas/cimeteststatus.xsd +A config/xml_schemas/config_archive.xsd +A config/xml_schemas/config_batch.xsd +A config/xml_schemas/config_compilers_v2.xsd +A config/xml_schemas/config_compsets.xsd +A config/xml_schemas/config_grids_v2.xsd +A config/xml_schemas/config_machines.xsd +A config/xml_schemas/config_machines_template.xml +A config/xml_schemas/config_pes.xsd +A config/xml_schemas/entry_id.xsd +A config/xml_schemas/entry_id_base.xsd +A config/xml_schemas/entry_id_namelist.xsd +A config/xml_schemas/env_archive.xsd +A config/xml_schemas/env_entry_id.xsd +A config/xml_schemas/testlist.xsd +D driver_cpl/bld/namelist_files/namelist_defaults_drv.xml +D driver_cpl/bld/namelist_files/namelist_definition_drv.xml +D driver_cpl/bld/namelist_files/namelist_definition_drv_flds.xml +D driver_cpl/bld/namelist_files/namelist_definition_modio.xml +D driver_cpl/bld/testdir/Buildconf/camconf/drv_flds_in +D driver_cpl/bld/testdir/Buildconf/clmconf/drv_flds_in +D driver_cpl/bld/testdir/env_build.xml +D driver_cpl/bld/testdir/env_case.xml +D driver_cpl/bld/testdir/env_mach_pes.xml +D driver_cpl/bld/testdir/env_run.xml +D driver_cpl/bld/testdir/runtest +D driver_cpl/bld/testdir/user_nl_cpl +D driver_cpl/cime_config/buildexe +D driver_cpl/cime_config/buildnml +D driver_cpl/cime_config/config_component.xml +D driver_cpl/cime_config/config_component_acme.xml +D driver_cpl/cime_config/config_component_cesm.xml +D driver_cpl/cime_config/config_compsets.xml +D driver_cpl/cime_config/config_pes.xml +D driver_cpl/cime_config/namelist_definition_drv.xml +D driver_cpl/cime_config/namelist_definition_modelio.xml +D driver_cpl/cime_config/testdefs/testlist_drv.xml +D driver_cpl/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands +D driver_cpl/cime_config/user_nl_cpl +D driver_cpl/doc/bookinfo.xml +D driver_cpl/doc/chap1.xml +D driver_cpl/doc/chap2.xml +D driver_cpl/doc/figx1.jpg +D driver_cpl/doc/namelist.xml +D driver_cpl/doc/rundocbook.csh +D driver_cpl/doc/stylesheet.dsl +D driver_cpl/doc/ug.xml +D driver_cpl/driver/CMakeLists.txt +D driver_cpl/driver/cesm_comp_mod.F90 +D driver_cpl/driver/cesm_driver.F90 +D driver_cpl/driver/component_mod.F90 +D driver_cpl/driver/component_type_mod.F90 +D driver_cpl/driver/cplcomp_exchange_mod.F90 +D driver_cpl/driver/map_glc2lnd_mod.F90 +D driver_cpl/driver/map_lnd2glc_mod.F90 +D driver_cpl/driver/map_lnd2rof_irrig_mod.F90 +D driver_cpl/driver/mrg_mod.F90 +D driver_cpl/driver/prep_aoflux_mod.F90 +D driver_cpl/driver/prep_atm_mod.F90 +D driver_cpl/driver/prep_glc_mod.F90 +D driver_cpl/driver/prep_ice_mod.F90 +D driver_cpl/driver/prep_lnd_mod.F90 +D driver_cpl/driver/prep_ocn_mod.F90 +D driver_cpl/driver/prep_rof_mod.F90 +D driver_cpl/driver/prep_wav_mod.F90 +D driver_cpl/driver/seq_diag_mct.F90 +D driver_cpl/driver/seq_domain_mct.F90 +D driver_cpl/driver/seq_flux_mct.F90 +D driver_cpl/driver/seq_frac_mct.F90 +D driver_cpl/driver/seq_hist_mod.F90 +D driver_cpl/driver/seq_io_mod.F90 +D driver_cpl/driver/seq_map_mod.F90 +D driver_cpl/driver/seq_map_type_mod.F90 +D driver_cpl/driver/seq_rest_mod.F90 +D driver_cpl/driver/t_driver_timers_mod.F90 +D driver_cpl/driver/vertical_gradient_calculator_2nd_order.F90 +D driver_cpl/driver/vertical_gradient_calculator_base.F90 +D driver_cpl/driver/vertical_gradient_calculator_factory.F90 +D driver_cpl/shr/CMakeLists.txt +D driver_cpl/shr/glc_elevclass_mod.F90 +D driver_cpl/shr/seq_cdata_mod.F90 +D driver_cpl/shr/seq_comm_mct.F90 +D driver_cpl/shr/seq_drydep_mod.F90 +D driver_cpl/shr/seq_flds_mod.F90 +D driver_cpl/shr/seq_infodata_mod.F90 +D driver_cpl/shr/seq_io_read_mod.F90 +D driver_cpl/shr/seq_timemgr_mod.F90 +D driver_cpl/shr/shr_carma_mod.F90 +D driver_cpl/shr/shr_expr_parser_mod.F90 +D driver_cpl/shr/shr_fire_emis_mod.F90 +D driver_cpl/shr/shr_megan_mod.F90 +D driver_cpl/unit_test/CMakeLists.txt +D driver_cpl/unit_test/avect_wrapper_test/CMakeLists.txt +D driver_cpl/unit_test/avect_wrapper_test/test_avect_wrapper.pf +D driver_cpl/unit_test/glc_elevclass_test/CMakeLists.txt +D driver_cpl/unit_test/glc_elevclass_test/test_glc_elevclass.pf +D driver_cpl/unit_test/map_glc2lnd_test/CMakeLists.txt +D driver_cpl/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf +D driver_cpl/unit_test/map_lnd2glc_test/CMakeLists.txt +D driver_cpl/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf +D driver_cpl/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt +D driver_cpl/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf +D driver_cpl/unit_test/seq_map_test/CMakeLists.txt +D driver_cpl/unit_test/seq_map_test/test_seq_map.pf +D driver_cpl/unit_test/stubs/CMakeLists.txt +D driver_cpl/unit_test/stubs/vertical_gradient_calculator_constant.F90 +D driver_cpl/unit_test/utils/CMakeLists.txt +D driver_cpl/unit_test/utils/avect_wrapper_mod.F90 +D driver_cpl/unit_test/utils/create_mapper_mod.F90 +D driver_cpl/unit_test/utils/mct_wrapper_mod.F90 +D driver_cpl/unit_test/utils/simple_map_mod.F90 +D driver_cpl/unit_test/vertical_gradient_calculator_test/CMakeLists.txt +D driver_cpl/unit_test/vertical_gradient_calculator_test/README +D driver_cpl/unit_test/vertical_gradient_calculator_test/gradient_example.txt +D driver_cpl/unit_test/vertical_gradient_calculator_test/plot_gradient +D driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf +D driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf +D externals/CMake/.gitignore +D externals/CMake/CESM_utils.cmake +D externals/CMake/CIME_utils.cmake +D externals/CMake/ChangeLog +D externals/CMake/Compilers.cmake +D externals/CMake/FindpFUnit.cmake +D externals/CMake/LICENSE +D externals/CMake/README.md +D externals/CMake/Sourcelist_utils.cmake +D externals/CMake/TryCSizeOf.f90 +D externals/CMake/TryMPIIO.f90 +D externals/CMake/TryMPIMod.f90 +D externals/CMake/TryMPISERIAL.f90 +D externals/CMake/TryPnetcdf_inc.f90 +D externals/CMake/TryPnetcdf_mod.f90 +D externals/CMake/genf90_utils.cmake +D externals/CMake/mpiexec.cmake +D externals/CMake/pFUnit_utils.cmake +D externals/genf90/ChangeLog +D externals/genf90/genf90.pl +D externals/mct/.gitignore +D externals/mct/COPYRIGHT +D externals/mct/Makefile +D externals/mct/Makefile.conf.in +D externals/mct/README +D externals/mct/aclocal.m4 +D externals/mct/benchmarks/.gitignore +D externals/mct/benchmarks/Makefile +D externals/mct/benchmarks/RouterTestDis.F90 +D externals/mct/benchmarks/RouterTestOvr.F90 +D externals/mct/benchmarks/T42.8pC +D externals/mct/benchmarks/T42.8pR +D externals/mct/benchmarks/gx1.8pC +D externals/mct/benchmarks/gx1.8pR +D externals/mct/benchmarks/importBench.F90 +D externals/mct/config.h.in +D externals/mct/configure +D externals/mct/configure.ac +D externals/mct/doc/.gitignore +D externals/mct/doc/Makefile +D externals/mct/doc/README +D externals/mct/doc/coupler.bib +D externals/mct/doc/mct_APIs.tex +D externals/mct/doc/texsrc/.gitignore +D externals/mct/doc/texsrc/Makefile +D externals/mct/doc/texsrc/SRCS_tex.mk +D externals/mct/examples/Makefile +D externals/mct/examples/README +D externals/mct/examples/climate_concur1/.gitignore +D externals/mct/examples/climate_concur1/Makefile +D externals/mct/examples/climate_concur1/README +D externals/mct/examples/climate_concur1/coupler.F90 +D externals/mct/examples/climate_concur1/master.F90 +D externals/mct/examples/climate_concur1/model.F90 +D externals/mct/examples/climate_sequen1/.gitignore +D externals/mct/examples/climate_sequen1/Makefile +D externals/mct/examples/climate_sequen1/README +D externals/mct/examples/climate_sequen1/TS1.dat +D externals/mct/examples/climate_sequen1/coupler.F90 +D externals/mct/examples/climate_sequen1/dst.rc +D externals/mct/examples/climate_sequen1/dstmodel.F90 +D externals/mct/examples/climate_sequen1/master.F90 +D externals/mct/examples/climate_sequen1/mutils.F90 +D externals/mct/examples/climate_sequen1/src.rc +D externals/mct/examples/climate_sequen1/srcmodel.F90 +D externals/mct/examples/simple/.gitignore +D externals/mct/examples/simple/Makefile +D externals/mct/examples/simple/README +D externals/mct/examples/simple/script.babyblue +D externals/mct/examples/simple/twocmp.con.F90 +D externals/mct/examples/simple/twocmp.seq.F90 +D externals/mct/examples/simple/twocmp.seqNB.F90 +D externals/mct/examples/simple/twocmp.seqUnvn.F90 +D externals/mct/install-sh +D externals/mct/m4/README +D externals/mct/m4/acx_mpi.m4 +D externals/mct/m4/ax_fc_version.m4 +D externals/mct/m4/fortran.m4 +D externals/mct/mct/Makefile +D externals/mct/mct/README +D externals/mct/mct/m_Accumulator.F90 +D externals/mct/mct/m_AccumulatorComms.F90 +D externals/mct/mct/m_AttrVect.F90 +D externals/mct/mct/m_AttrVectComms.F90 +D externals/mct/mct/m_AttrVectReduce.F90 +D externals/mct/mct/m_ConvertMaps.F90 +D externals/mct/mct/m_ExchangeMaps.F90 +D externals/mct/mct/m_GeneralGrid.F90 +D externals/mct/mct/m_GeneralGridComms.F90 +D externals/mct/mct/m_GlobalMap.F90 +D externals/mct/mct/m_GlobalSegMap.F90 +D externals/mct/mct/m_GlobalSegMapComms.F90 +D externals/mct/mct/m_GlobalToLocal.F90 +D externals/mct/mct/m_MCTWorld.F90 +D externals/mct/mct/m_MatAttrVectMul.F90 +D externals/mct/mct/m_Merge.F90 +D externals/mct/mct/m_Navigator.F90 +D externals/mct/mct/m_Rearranger.F90 +D externals/mct/mct/m_Router.F90 +D externals/mct/mct/m_SparseMatrix.F90 +D externals/mct/mct/m_SparseMatrixComms.F90 +D externals/mct/mct/m_SparseMatrixDecomp.F90 +D externals/mct/mct/m_SparseMatrixPlus.F90 +D externals/mct/mct/m_SparseMatrixToMaps.F90 +D externals/mct/mct/m_SpatialIntegral.F90 +D externals/mct/mct/m_SpatialIntegralV.F90 +D externals/mct/mct/m_Transfer.F90 +D externals/mct/mkinstalldirs +D externals/mct/mpeu/Makefile +D externals/mct/mpeu/README +D externals/mct/mpeu/assertmpeu.H +D externals/mct/mpeu/get_zeits.c +D externals/mct/mpeu/m_FcComms.F90 +D externals/mct/mpeu/m_FileResolv.F90 +D externals/mct/mpeu/m_Filename.F90 +D externals/mct/mpeu/m_IndexBin_char.F90 +D externals/mct/mpeu/m_IndexBin_integer.F90 +D externals/mct/mpeu/m_IndexBin_logical.F90 +D externals/mct/mpeu/m_List.F90 +D externals/mct/mpeu/m_MergeSorts.F90 +D externals/mct/mpeu/m_Permuter.F90 +D externals/mct/mpeu/m_SortingTools.F90 +D externals/mct/mpeu/m_StrTemplate.F90 +D externals/mct/mpeu/m_String.F90 +D externals/mct/mpeu/m_StringLinkedList.F90 +D externals/mct/mpeu/m_TraceBack.F90 +D externals/mct/mpeu/m_chars.F90 +D externals/mct/mpeu/m_die.F90 +D externals/mct/mpeu/m_dropdead.F90 +D externals/mct/mpeu/m_flow.F90 +D externals/mct/mpeu/m_inpak90.F90 +D externals/mct/mpeu/m_ioutil.F90 +D externals/mct/mpeu/m_mall.F90 +D externals/mct/mpeu/m_mpif.F90 +D externals/mct/mpeu/m_mpif90.F90 +D externals/mct/mpeu/m_mpout.F90 +D externals/mct/mpeu/m_rankMerge.F90 +D externals/mct/mpeu/m_realkinds.F90 +D externals/mct/mpeu/m_stdio.F90 +D externals/mct/mpeu/m_zeit.F90 +D externals/mct/mpi-serial/.gitignore +D externals/mct/mpi-serial/Makefile +D externals/mct/mpi-serial/Makefile.conf.in +D externals/mct/mpi-serial/NOTES +D externals/mct/mpi-serial/README +D externals/mct/mpi-serial/aclocal.m4 +D externals/mct/mpi-serial/cart.c +D externals/mct/mpi-serial/collective.c +D externals/mct/mpi-serial/comm.c +D externals/mct/mpi-serial/config.h.in +D externals/mct/mpi-serial/configure +D externals/mct/mpi-serial/configure.in +D externals/mct/mpi-serial/copy.c +D externals/mct/mpi-serial/fort.F90 +D externals/mct/mpi-serial/getcount.c +D externals/mct/mpi-serial/group.c +D externals/mct/mpi-serial/handles.c +D externals/mct/mpi-serial/info.c +D externals/mct/mpi-serial/list.c +D externals/mct/mpi-serial/list.h +D externals/mct/mpi-serial/listP.h +D externals/mct/mpi-serial/listops.h +D externals/mct/mpi-serial/m4/README +D externals/mct/mpi-serial/m4/ax_fc_version.m4 +D externals/mct/mpi-serial/mpi.c +D externals/mct/mpi-serial/mpi.h +D externals/mct/mpi-serial/mpiP.h +D externals/mct/mpi-serial/mpif.F90 +D externals/mct/mpi-serial/mpif.h +D externals/mct/mpi-serial/op.c +D externals/mct/mpi-serial/pack.c +D externals/mct/mpi-serial/probe.c +D externals/mct/mpi-serial/protify.awk +D externals/mct/mpi-serial/recv.c +D externals/mct/mpi-serial/req.c +D externals/mct/mpi-serial/send.c +D externals/mct/mpi-serial/tests/.gitignore +D externals/mct/mpi-serial/tests/Makefile +D externals/mct/mpi-serial/tests/ctest.c +D externals/mct/mpi-serial/tests/ctest_old.c +D externals/mct/mpi-serial/tests/ftest.F90 +D externals/mct/mpi-serial/tests/ftest_internal.F90 +D externals/mct/mpi-serial/tests/ftest_old.F90 +D externals/mct/mpi-serial/time.c +D externals/mct/mpi-serial/type.c +D externals/mct/mpi-serial/type.h +D externals/mct/mpi-serial/type_const.c +D externals/mct/protex/protex +D externals/mct/testsystem/Makefile +D externals/mct/testsystem/testall/.gitignore +D externals/mct/testsystem/testall/Makefile +D externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 +D externals/mct/testsystem/testall/UNTESTED +D externals/mct/testsystem/testall/ccm.F90 +D externals/mct/testsystem/testall/convertPOPT.F90 +D externals/mct/testsystem/testall/convertgauss.F90 +D externals/mct/testsystem/testall/cpl.F90 +D externals/mct/testsystem/testall/job.ut-all.jaguar +D externals/mct/testsystem/testall/m_ACTEST.F90 +D externals/mct/testsystem/testall/m_AVTEST.F90 +D externals/mct/testsystem/testall/m_GGRIDTEST.F90 +D externals/mct/testsystem/testall/m_GMAPTEST.F90 +D externals/mct/testsystem/testall/m_GSMAPTEST.F90 +D externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 +D externals/mct/testsystem/testall/m_ROUTERTEST.F90 +D externals/mct/testsystem/testall/m_SMATTEST.F90 +D externals/mct/testsystem/testall/master.F90 +D externals/mct/testsystem/testall/mph.F90 +D externals/mct/testsystem/testall/pop.F90 +D externals/mct/testsystem/testall/processors_map.in +D externals/mct/testsystem/testall/script.jag +D externals/mct/testsystem/testall/ut_SparseMatrix.rc +D externals/mct/testunit/.gitignore +D externals/mct/testunit/AttrVect_Test.F90 +D externals/mct/testunit/Makefile +D externals/mct/testunit/master.F90 +D externals/pio1/CMakeLists.txt +D externals/pio1/ChangeLog +D externals/pio1/ChangeLog_template +D externals/pio1/Doxyfile +D externals/pio1/DoxygenLayout.xml +D externals/pio1/SVN_EXTERNAL_DIRECTORIES +D externals/pio1/customdoxygen.css +D externals/pio1/doc/CAMexample.txt +D externals/pio1/doc/Decomp.txt +D externals/pio1/doc/DoxygenLayout.xml +D externals/pio1/doc/Error.txt +D externals/pio1/doc/Examples.txt +D externals/pio1/doc/GettingStarted.txt +D externals/pio1/doc/Installing.txt +D externals/pio1/doc/api.txt +D externals/pio1/doc/base.txt +D externals/pio1/doc/example/errorhandle +D externals/pio1/doc/example/simple-bc +D externals/pio1/doc/example/simple-bc-rearr +D externals/pio1/doc/example/simple-bc-rearr-pe1 +D externals/pio1/doc/example/simple-bc-rearr-pe2 +D externals/pio1/doc/example/simple-dof +D externals/pio1/doc/example/simple-dof-rearr +D externals/pio1/doc/faq.txt +D externals/pio1/doc/footer.html +D externals/pio1/doc/header.html +D externals/pio1/doc/images/baseimage.graffle +D externals/pio1/doc/images/block-cyclic-rearr.eps +D externals/pio1/doc/images/block-cyclic-rearr.graffle +D externals/pio1/doc/images/block-cyclic-rearr.png +D externals/pio1/doc/images/block-cyclic.eps +D externals/pio1/doc/images/block-cyclic.graffle +D externals/pio1/doc/images/block-cyclic.png +D externals/pio1/doc/images/dof-rearr.eps +D externals/pio1/doc/images/dof-rearr.graffle +D externals/pio1/doc/images/dof-rearr.png +D externals/pio1/doc/images/dof.eps +D externals/pio1/doc/images/dof.graffle +D externals/pio1/doc/images/dof.png +D externals/pio1/doc/testpio_example.txt +D externals/pio1/doxygen.sty +D externals/pio1/pio/CMakeLists.txt +D externals/pio1/pio/C_interface_mod.F90 +D externals/pio1/pio/Makefile.conf.in +D externals/pio1/pio/README.config +D externals/pio1/pio/alloc_mod.F90.in +D externals/pio1/pio/box_rearrange.F90.in +D externals/pio1/pio/calcdecomp.F90 +D externals/pio1/pio/calcdisplace_mod.F90 +D externals/pio1/pio/config.h.in +D externals/pio1/pio/dtypes.h +D externals/pio1/pio/fdepends.awk +D externals/pio1/pio/iompi_mod.F90.in +D externals/pio1/pio/ionf_mod.F90 +D externals/pio1/pio/nf_mod.F90 +D externals/pio1/pio/pio.F90 +D externals/pio1/pio/pio_kinds.F90 +D externals/pio1/pio/pio_mpi_utils.F90 +D externals/pio1/pio/pio_msg_callbacks.F90 +D externals/pio1/pio/pio_msg_getput_callbacks.F90.in +D externals/pio1/pio/pio_msg_mod.F90 +D externals/pio1/pio/pio_nf_utils.F90 +D externals/pio1/pio/pio_spmd_utils.F90.in +D externals/pio1/pio/pio_support.F90 +D externals/pio1/pio/pio_types.F90 +D externals/pio1/pio/pio_utils.F90 +D externals/pio1/pio/piodarray.F90.in +D externals/pio1/pio/piolib_mod.F90 +D externals/pio1/pio/pionfatt_mod.F90.in +D externals/pio1/pio/pionfget_mod.F90.in +D externals/pio1/pio/pionfput_mod.F90.in +D externals/pio1/pio/pionfread_mod.F90.in +D externals/pio1/pio/pionfwrite_mod.F90.in +D externals/pio1/pio/piovdc.F90 +D externals/pio1/pio/rearr_options.h +D externals/pio1/pio/rearrange.F90.in +D externals/pio1/pio/topology.c +D externals/pio1/scripts/Utils.pm +D externals/pio1/scripts/config.pl +D externals/pio1/scripts/testpio_yellowstone.pl +D externals/pio1/tests/testpio/CAM05.csh +D externals/pio1/tests/testpio/CMakeLists.txt +D externals/pio1/tests/testpio/MPASA30km.csh +D externals/pio1/tests/testpio/MPASA60km.csh +D externals/pio1/tests/testpio/POPB.csh +D externals/pio1/tests/testpio/POPC.csh +D externals/pio1/tests/testpio/POPD.csh +D externals/pio1/tests/testpio/POPDv0.csh +D externals/pio1/tests/testpio/POPDv1.csh +D externals/pio1/tests/testpio/POPDv2.csh +D externals/pio1/tests/testpio/POPDv3.csh +D externals/pio1/tests/testpio/POPDv4.csh +D externals/pio1/tests/testpio/POPDv5.csh +D externals/pio1/tests/testpio/README.testpio +D externals/pio1/tests/testpio/WRFB.csh +D externals/pio1/tests/testpio/build_defaults.xml +D externals/pio1/tests/testpio/check_mod.F90 +D externals/pio1/tests/testpio/config_bench.xml +D externals/pio1/tests/testpio/fdepends.awk +D externals/pio1/tests/testpio/gdecomp_mod.F90 +D externals/pio1/tests/testpio/kinds_mod.F90 +D externals/pio1/tests/testpio/kraken.128.csh +D externals/pio1/tests/testpio/kraken.1K.csh +D externals/pio1/tests/testpio/kraken.256.csh +D externals/pio1/tests/testpio/kraken.512.csh +D externals/pio1/tests/testpio/kraken.64.csh +D externals/pio1/tests/testpio/namelist_mod.F90 +D externals/pio1/tests/testpio/namelists/testpio_in.apb05 +D externals/pio1/tests/testpio/namelists/testpio_in.asb01 +D externals/pio1/tests/testpio/namelists/testpio_in.asb04 +D externals/pio1/tests/testpio/namelists/testpio_in.b01 +D externals/pio1/tests/testpio/namelists/testpio_in.b02 +D externals/pio1/tests/testpio/namelists/testpio_in.b03 +D externals/pio1/tests/testpio/namelists/testpio_in.b04 +D externals/pio1/tests/testpio/namelists/testpio_in.b05 +D externals/pio1/tests/testpio/namelists/testpio_in.b06 +D externals/pio1/tests/testpio/namelists/testpio_in.b07 +D externals/pio1/tests/testpio/namelists/testpio_in.b08 +D externals/pio1/tests/testpio/namelists/testpio_in.b09 +D externals/pio1/tests/testpio/namelists/testpio_in.b10 +D externals/pio1/tests/testpio/namelists/testpio_in.b11 +D externals/pio1/tests/testpio/namelists/testpio_in.b12 +D externals/pio1/tests/testpio/namelists/testpio_in.b13 +D externals/pio1/tests/testpio/namelists/testpio_in.bb01 +D externals/pio1/tests/testpio/namelists/testpio_in.bb02 +D externals/pio1/tests/testpio/namelists/testpio_in.bb03 +D externals/pio1/tests/testpio/namelists/testpio_in.bb04 +D externals/pio1/tests/testpio/namelists/testpio_in.bb05 +D externals/pio1/tests/testpio/namelists/testpio_in.bb06 +D externals/pio1/tests/testpio/namelists/testpio_in.bb07 +D externals/pio1/tests/testpio/namelists/testpio_in.bb08 +D externals/pio1/tests/testpio/namelists/testpio_in.bn01 +D externals/pio1/tests/testpio/namelists/testpio_in.bn02 +D externals/pio1/tests/testpio/namelists/testpio_in.bn03 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b01 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b02 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b03 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b04 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b05 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b06 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b07 +D externals/pio1/tests/testpio/namelists/testpio_in.n4b08 +D externals/pio1/tests/testpio/namelists/testpio_in.n4n01 +D externals/pio1/tests/testpio/namelists/testpio_in.n4n02 +D externals/pio1/tests/testpio/namelists/testpio_in.n4n03 +D externals/pio1/tests/testpio/namelists/testpio_in.pb01 +D externals/pio1/tests/testpio/namelists/testpio_in.pb02 +D externals/pio1/tests/testpio/namelists/testpio_in.pb03 +D externals/pio1/tests/testpio/namelists/testpio_in.pb04 +D externals/pio1/tests/testpio/namelists/testpio_in.pb05 +D externals/pio1/tests/testpio/namelists/testpio_in.pb06 +D externals/pio1/tests/testpio/namelists/testpio_in.pb07 +D externals/pio1/tests/testpio/namelists/testpio_in.pb08 +D externals/pio1/tests/testpio/namelists/testpio_in.pn01 +D externals/pio1/tests/testpio/namelists/testpio_in.pn02 +D externals/pio1/tests/testpio/namelists/testpio_in.pn03 +D externals/pio1/tests/testpio/namelists/testpio_in.sb01 +D externals/pio1/tests/testpio/namelists/testpio_in.sb02 +D externals/pio1/tests/testpio/namelists/testpio_in.sb03 +D externals/pio1/tests/testpio/namelists/testpio_in.sb04 +D externals/pio1/tests/testpio/namelists/testpio_in.sb05 +D externals/pio1/tests/testpio/namelists/testpio_in.sb06 +D externals/pio1/tests/testpio/namelists/testpio_in.sb07 +D externals/pio1/tests/testpio/namelists/testpio_in.sb08 +D externals/pio1/tests/testpio/namelists/testpio_in.sn01 +D externals/pio1/tests/testpio/namelists/testpio_in.sn02 +D externals/pio1/tests/testpio/namelists/testpio_in.sn03 +D externals/pio1/tests/testpio/namelists/testpio_in.wr01 +D externals/pio1/tests/testpio/namelists/testpio_in.wr02 +D externals/pio1/tests/testpio/perl5lib/ChangeLog +D externals/pio1/tests/testpio/perl5lib/README +D externals/pio1/tests/testpio/perl5lib/XML/Changes +D externals/pio1/tests/testpio/perl5lib/XML/Lite.pm +D externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm +D externals/pio1/tests/testpio/perl5lib/XML/README +D externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 +D externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 +D externals/pio1/tests/testpio/test.csh +D externals/pio1/tests/testpio/test_lib.F90 +D externals/pio1/tests/testpio/testdecomp.F90 +D externals/pio1/tests/testpio/testdecomp.bluefire.run +D externals/pio1/tests/testpio/testdecomp_in +D externals/pio1/tests/testpio/testpio.F90 +D externals/pio1/tests/testpio/testpio_bench.pl +D externals/pio1/tests/testpio/testpio_build.pl +D externals/pio1/tests/testpio/testpio_run.pl +D externals/pio1/tests/testpio/utils_mod.F90 +D externals/pio1/tests/testpio/ystest.sh +D externals/pio1/tests/unittests/CMakeLists.txt +D externals/pio1/tests/unittests/Levy_Notes +D externals/pio1/tests/unittests/README +D externals/pio1/tests/unittests/basic_tests.F90 +D externals/pio1/tests/unittests/driver.F90 +D externals/pio1/tests/unittests/global_vars.F90 +D externals/pio1/tests/unittests/input.nl +D externals/pio1/tests/unittests/nc_set_log_level2.c +D externals/pio1/tests/unittests/ncdf_tests.F90 +D externals/pio1/tests/unittests/not_netcdf.ieee +D externals/pio1/timing/CMakeLists.txt +D externals/pio1/timing/COPYING +D externals/pio1/timing/ChangeLog +D externals/pio1/timing/GPTLget_memusage.c +D externals/pio1/timing/GPTLprint_memusage.c +D externals/pio1/timing/GPTLutil.c +D externals/pio1/timing/Makefile +D externals/pio1/timing/README +D externals/pio1/timing/XXXdotF/perf_mod.F +D externals/pio1/timing/XXXdotF/perf_utils.F +D externals/pio1/timing/f_wrappers.c +D externals/pio1/timing/gptl.c +D externals/pio1/timing/gptl.h +D externals/pio1/timing/gptl.inc +D externals/pio1/timing/gptl_papi.c +D externals/pio1/timing/perf_mod.F90 +D externals/pio1/timing/perf_utils.F90 +D externals/pio1/timing/private.h +D externals/pio1/timing/threadutil.c +D externals/pio2/.gitignore +D externals/pio2/CMakeLists.txt +D externals/pio2/CTestConfig.cmake +D externals/pio2/CTestScript.cmake +D externals/pio2/README.md +D externals/pio2/cmake/FindGPTL.cmake +D externals/pio2/cmake/FindHDF5.cmake +D externals/pio2/cmake/FindLIBRT.cmake +D externals/pio2/cmake/FindLIBZ.cmake +D externals/pio2/cmake/FindMPE.cmake +D externals/pio2/cmake/FindMPISERIAL.cmake +D externals/pio2/cmake/FindNetCDF.cmake +D externals/pio2/cmake/FindPAPI.cmake +D externals/pio2/cmake/FindPnetCDF.cmake +D externals/pio2/cmake/FindSZIP.cmake +D externals/pio2/cmake/LibCheck.cmake +D externals/pio2/cmake/LibFind.cmake +D externals/pio2/cmake/LibMPI.cmake +D externals/pio2/cmake/TryHDF5_HAS_SZIP.c +D externals/pio2/cmake/TryNetCDF_DAP.c +D externals/pio2/cmake/TryNetCDF_PARALLEL.c +D externals/pio2/cmake/TryNetCDF_PNETCDF.c +D externals/pio2/cmake/mpiexec.alcf +D externals/pio2/cmake/mpiexec.ncsa +D externals/pio2/cmake/mpiexec.nersc +D externals/pio2/cmake/mpiexec.nwsc +D externals/pio2/ctest/CTestEnvironment-alcf.cmake +D externals/pio2/ctest/CTestEnvironment-cgd.cmake +D externals/pio2/ctest/CTestEnvironment-ncsa.cmake +D externals/pio2/ctest/CTestEnvironment-nersc.cmake +D externals/pio2/ctest/CTestEnvironment-nwsc.cmake +D externals/pio2/ctest/CTestEnvironment-unknown.cmake +D externals/pio2/ctest/CTestScript-Test.cmake +D externals/pio2/ctest/runcdash-alcf-ibm.sh +D externals/pio2/ctest/runcdash-cgd-nag.sh +D externals/pio2/ctest/runcdash-nersc-cray.sh +D externals/pio2/ctest/runcdash-nersc-intel.sh +D externals/pio2/ctest/runcdash-nwsc-gnu.sh +D externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +D externals/pio2/ctest/runcdash-nwsc-intel.sh +D externals/pio2/ctest/runcdash-nwsc-pgi.sh +D externals/pio2/ctest/runctest-alcf.sh +D externals/pio2/ctest/runctest-cgd.sh +D externals/pio2/ctest/runctest-ncsa.sh +D externals/pio2/ctest/runctest-nersc.sh +D externals/pio2/ctest/runctest-nwsc.sh +D externals/pio2/ctest/runctest-unknown.sh +D externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake +D externals/pio2/doc/CMakeFiles/CMakeOutput.log +D externals/pio2/doc/CMakeFiles/cmake.check_cache +D externals/pio2/doc/CMakeLists.txt +D externals/pio2/doc/Doxyfile.in +D externals/pio2/doc/DoxygenLayout.xml +D externals/pio2/doc/customdoxygen.css +D externals/pio2/doc/doxygen.sty +D externals/pio2/doc/images/baseimage.graffle +D externals/pio2/doc/images/block-cyclic-rearr.eps +D externals/pio2/doc/images/block-cyclic-rearr.graffle +D externals/pio2/doc/images/block-cyclic-rearr.png +D externals/pio2/doc/images/block-cyclic.eps +D externals/pio2/doc/images/block-cyclic.graffle +D externals/pio2/doc/images/block-cyclic.png +D externals/pio2/doc/images/dof-rearr.eps +D externals/pio2/doc/images/dof-rearr.graffle +D externals/pio2/doc/images/dof-rearr.png +D externals/pio2/doc/images/dof.eps +D externals/pio2/doc/images/dof.graffle +D externals/pio2/doc/images/dof.png +D externals/pio2/doc/source/CAMexample.txt +D externals/pio2/doc/source/Decomp.txt +D externals/pio2/doc/source/Error.txt +D externals/pio2/doc/source/Examples.txt +D externals/pio2/doc/source/GettingStarted.txt +D externals/pio2/doc/source/Installing.txt +D externals/pio2/doc/source/api.txt +D externals/pio2/doc/source/base.txt +D externals/pio2/doc/source/example/errorhandle +D externals/pio2/doc/source/example/simple-bc +D externals/pio2/doc/source/example/simple-bc-rearr +D externals/pio2/doc/source/example/simple-bc-rearr-pe1 +D externals/pio2/doc/source/example/simple-bc-rearr-pe2 +D externals/pio2/doc/source/example/simple-dof +D externals/pio2/doc/source/example/simple-dof-rearr +D externals/pio2/doc/source/faq.txt +D externals/pio2/doc/source/mach_walkthrough.txt +D externals/pio2/doc/source/testpio_example.txt +D externals/pio2/examples/CMakeLists.txt +D externals/pio2/examples/basic/CAM05.csh +D externals/pio2/examples/basic/CMakeLists.txt +D externals/pio2/examples/basic/MPASA30km.csh +D externals/pio2/examples/basic/MPASA60km.csh +D externals/pio2/examples/basic/POPB.csh +D externals/pio2/examples/basic/POPC.csh +D externals/pio2/examples/basic/POPD.csh +D externals/pio2/examples/basic/POPDv0.csh +D externals/pio2/examples/basic/POPDv1.csh +D externals/pio2/examples/basic/POPDv2.csh +D externals/pio2/examples/basic/POPDv3.csh +D externals/pio2/examples/basic/POPDv4.csh +D externals/pio2/examples/basic/POPDv5.csh +D externals/pio2/examples/basic/README.testpio +D externals/pio2/examples/basic/WRFB.csh +D externals/pio2/examples/basic/alloc_mod.F90.in +D externals/pio2/examples/basic/build_defaults.xml +D externals/pio2/examples/basic/check_mod.F90 +D externals/pio2/examples/basic/config_bench.xml +D externals/pio2/examples/basic/fdepends.awk +D externals/pio2/examples/basic/gdecomp_mod.F90 +D externals/pio2/examples/basic/kinds_mod.F90 +D externals/pio2/examples/basic/kraken.128.csh +D externals/pio2/examples/basic/kraken.1K.csh +D externals/pio2/examples/basic/kraken.256.csh +D externals/pio2/examples/basic/kraken.512.csh +D externals/pio2/examples/basic/kraken.64.csh +D externals/pio2/examples/basic/namelist_mod.F90 +D externals/pio2/examples/basic/namelists/testpio_in.apb05 +D externals/pio2/examples/basic/namelists/testpio_in.asb01 +D externals/pio2/examples/basic/namelists/testpio_in.asb04 +D externals/pio2/examples/basic/namelists/testpio_in.n4b01 +D externals/pio2/examples/basic/namelists/testpio_in.n4b02 +D externals/pio2/examples/basic/namelists/testpio_in.n4b03 +D externals/pio2/examples/basic/namelists/testpio_in.n4b04 +D externals/pio2/examples/basic/namelists/testpio_in.n4b05 +D externals/pio2/examples/basic/namelists/testpio_in.n4b06 +D externals/pio2/examples/basic/namelists/testpio_in.n4b07 +D externals/pio2/examples/basic/namelists/testpio_in.n4b08 +D externals/pio2/examples/basic/namelists/testpio_in.n4n01 +D externals/pio2/examples/basic/namelists/testpio_in.n4n02 +D externals/pio2/examples/basic/namelists/testpio_in.n4n03 +D externals/pio2/examples/basic/namelists/testpio_in.pb01 +D externals/pio2/examples/basic/namelists/testpio_in.pb02 +D externals/pio2/examples/basic/namelists/testpio_in.pb03 +D externals/pio2/examples/basic/namelists/testpio_in.pb04 +D externals/pio2/examples/basic/namelists/testpio_in.pb05 +D externals/pio2/examples/basic/namelists/testpio_in.pb06 +D externals/pio2/examples/basic/namelists/testpio_in.pb07 +D externals/pio2/examples/basic/namelists/testpio_in.pb08 +D externals/pio2/examples/basic/namelists/testpio_in.pn01 +D externals/pio2/examples/basic/namelists/testpio_in.pn02 +D externals/pio2/examples/basic/namelists/testpio_in.pn03 +D externals/pio2/examples/basic/namelists/testpio_in.ps01 +D externals/pio2/examples/basic/namelists/testpio_in.ps02 +D externals/pio2/examples/basic/namelists/testpio_in.ps03 +D externals/pio2/examples/basic/namelists/testpio_in.ps04 +D externals/pio2/examples/basic/namelists/testpio_in.ps05 +D externals/pio2/examples/basic/namelists/testpio_in.ps06 +D externals/pio2/examples/basic/namelists/testpio_in.ps07 +D externals/pio2/examples/basic/namelists/testpio_in.ps08 +D externals/pio2/examples/basic/namelists/testpio_in.sb01 +D externals/pio2/examples/basic/namelists/testpio_in.sb02 +D externals/pio2/examples/basic/namelists/testpio_in.sb03 +D externals/pio2/examples/basic/namelists/testpio_in.sb04 +D externals/pio2/examples/basic/namelists/testpio_in.sb05 +D externals/pio2/examples/basic/namelists/testpio_in.sb06 +D externals/pio2/examples/basic/namelists/testpio_in.sb07 +D externals/pio2/examples/basic/namelists/testpio_in.sb08 +D externals/pio2/examples/basic/namelists/testpio_in.sn01 +D externals/pio2/examples/basic/namelists/testpio_in.sn02 +D externals/pio2/examples/basic/namelists/testpio_in.sn03 +D externals/pio2/examples/basic/namelists/testpio_in.wr01 +D externals/pio2/examples/basic/namelists/testpio_in.wr02 +D externals/pio2/examples/basic/perl5lib/ChangeLog +D externals/pio2/examples/basic/perl5lib/README +D externals/pio2/examples/basic/perl5lib/XML/Changes +D externals/pio2/examples/basic/perl5lib/XML/Lite.pm +D externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm +D externals/pio2/examples/basic/perl5lib/XML/README +D externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3 +D externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3 +D externals/pio2/examples/basic/test.csh +D externals/pio2/examples/basic/testdecomp.F90 +D externals/pio2/examples/basic/testdecomp.bluefire.run +D externals/pio2/examples/basic/testdecomp_in +D externals/pio2/examples/basic/testpio.F90 +D externals/pio2/examples/basic/testpio_bench.pl +D externals/pio2/examples/basic/testpio_build.pl +D externals/pio2/examples/basic/testpio_run.pl +D externals/pio2/examples/basic/utils_mod.F90 +D externals/pio2/examples/basic/wstest.c +D externals/pio2/examples/basic/ystest.sh +D externals/pio2/examples/c/CMakeLists.txt +D externals/pio2/examples/c/example1.c +D externals/pio2/examples/c/example2.c +D externals/pio2/examples/c/examplePio.c +D externals/pio2/examples/cxx/CMakeLists.txt +D externals/pio2/examples/cxx/examplePio.cxx +D externals/pio2/examples/f03/CMakeLists.txt +D externals/pio2/examples/f03/examplePio.f90 +D externals/pio2/scripts/prune_decomps.pl +D externals/pio2/src/CMakeLists.txt +D externals/pio2/src/clib/CMakeLists.txt +D externals/pio2/src/clib/bget.c +D externals/pio2/src/clib/bget.h +D externals/pio2/src/clib/config.h.in +D externals/pio2/src/clib/dtypes.h +D externals/pio2/src/clib/ncparser.pl +D externals/pio2/src/clib/ncputgetparser.pl +D externals/pio2/src/clib/pio.h +D externals/pio2/src/clib/pio_c_get_template.c +D externals/pio2/src/clib/pio_c_put_template.c +D externals/pio2/src/clib/pio_c_template.c +D externals/pio2/src/clib/pio_darray.c +D externals/pio2/src/clib/pio_file.c +D externals/pio2/src/clib/pio_get_nc.c +D externals/pio2/src/clib/pio_internal.h +D externals/pio2/src/clib/pio_lists.c +D externals/pio2/src/clib/pio_nc.c +D externals/pio2/src/clib/pio_nc4.c +D externals/pio2/src/clib/pio_put_nc.c +D externals/pio2/src/clib/pio_rearrange.c +D externals/pio2/src/clib/pio_spmd.c +D externals/pio2/src/clib/pioc.c +D externals/pio2/src/clib/pioc_sc.c +D externals/pio2/src/clib/pioc_support.c +D externals/pio2/src/clib/topology.c +D externals/pio2/src/flib/CMakeLists.txt +D externals/pio2/src/flib/pio.F90 +D externals/pio2/src/flib/pio_kinds.F90 +D externals/pio2/src/flib/pio_nf.F90 +D externals/pio2/src/flib/pio_support.F90 +D externals/pio2/src/flib/pio_types.F90 +D externals/pio2/src/flib/piodarray.F90.in +D externals/pio2/src/flib/piolib_mod.F90 +D externals/pio2/src/flib/pionfatt_mod.F90.in +D externals/pio2/src/flib/pionfget_mod.F90.in +D externals/pio2/src/flib/pionfput_mod.F90.in +D externals/pio2/src/gptl/CMakeLists.txt +D externals/pio2/src/gptl/COPYING +D externals/pio2/src/gptl/ChangeLog +D externals/pio2/src/gptl/GPTLget_memusage.c +D externals/pio2/src/gptl/GPTLprint_memusage.c +D externals/pio2/src/gptl/GPTLutil.c +D externals/pio2/src/gptl/README +D externals/pio2/src/gptl/f_wrappers.c +D externals/pio2/src/gptl/gptl.c +D externals/pio2/src/gptl/gptl.h +D externals/pio2/src/gptl/gptl.inc +D externals/pio2/src/gptl/gptl_papi.c +D externals/pio2/src/gptl/perf_mod.F90 +D externals/pio2/src/gptl/perf_utils.F90 +D externals/pio2/src/gptl/private.h +D externals/pio2/src/gptl/threadutil.c +D externals/pio2/tests/CMakeLists.txt +D externals/pio2/tests/general/CMakeLists.txt +D externals/pio2/tests/general/ncdf_fail.F90.in +D externals/pio2/tests/general/ncdf_get_put.F90.in +D externals/pio2/tests/general/ncdf_simple_tests.F90.in +D externals/pio2/tests/general/pio_decomp_fillval.F90.in +D externals/pio2/tests/general/pio_decomp_fillval2.F90 +D externals/pio2/tests/general/pio_decomp_tests.F90.in +D externals/pio2/tests/general/pio_fail.F90.in +D externals/pio2/tests/general/pio_file_fail.F90.in +D externals/pio2/tests/general/pio_file_simple_tests.F90.in +D externals/pio2/tests/general/pio_init_finalize.F90.in +D externals/pio2/tests/general/test_memleak.c +D externals/pio2/tests/general/util/pio_tf_f90gen.pl +D externals/pio2/tests/general/util/pio_tutil.F90 +D externals/pio2/tests/performance/CMakeLists.txt +D externals/pio2/tests/performance/gensimple.pl +D externals/pio2/tests/performance/kt.PIO1.perfmakefile +D externals/pio2/tests/performance/pioperformance.F90 +D externals/pio2/tests/unit/CMakeLists.txt +D externals/pio2/tests/unit/Levy_Notes +D externals/pio2/tests/unit/README +D externals/pio2/tests/unit/basic_tests.F90 +D externals/pio2/tests/unit/driver.F90 +D externals/pio2/tests/unit/global_vars.F90 +D externals/pio2/tests/unit/input.nl +D externals/pio2/tests/unit/nc_set_log_level2.c +D externals/pio2/tests/unit/ncdf_tests.F90 +D externals/pio2/tests/unit/not_netcdf.ieee +D externals/pio2/tests/unit/test_names.c +D externals/pio2/tests/unit/test_nc4.c +D scripts/Testing/Testcases/ICP_script +D scripts/Testing/Testcases/LAR_script +D scripts/Testing/Testcases/OCP_script +D scripts/Testing/Testcases/README +D scripts/Testing/testreporter.pl +M scripts/Tools/case.build +M scripts/Tools/case.setup +M scripts/Tools/standard_script_setup.py +M scripts/Tools/update_acme_tests +M scripts/Tools/xmlchange +M scripts/Tools/xmlconvertors/config_pes_converter.py +M scripts/Tools/xmlconvertors/grid_xml_converter.py +M scripts/Tools/xmlquery +M scripts/create_newcase +A scripts/data_assimilation/da_no_data_mod.sh +A scripts/lib/CIME/BuildTools/__init__.py +A scripts/lib/CIME/BuildTools/cmakemacroswriter.py +A scripts/lib/CIME/BuildTools/configure.py +A scripts/lib/CIME/BuildTools/macroconditiontree.py +A scripts/lib/CIME/BuildTools/macrowriterbase.py +A scripts/lib/CIME/BuildTools/makemacroswriter.py +A scripts/lib/CIME/BuildTools/possiblevalues.py +A scripts/lib/CIME/BuildTools/valuesetting.py +A scripts/lib/CIME/SystemTests/README +A scripts/lib/CIME/SystemTests/__init__.py +A scripts/lib/CIME/SystemTests/dae.py +A scripts/lib/CIME/SystemTests/eri.py +A scripts/lib/CIME/SystemTests/erio.py +A scripts/lib/CIME/SystemTests/erp.py +A scripts/lib/CIME/SystemTests/err.py +A scripts/lib/CIME/SystemTests/ers.py +A scripts/lib/CIME/SystemTests/ert.py +A scripts/lib/CIME/SystemTests/homme.py +A scripts/lib/CIME/SystemTests/icp.py +A scripts/lib/CIME/SystemTests/lii.py +A scripts/lib/CIME/SystemTests/nck.py +A scripts/lib/CIME/SystemTests/ncr.py +A scripts/lib/CIME/SystemTests/pea.py +A scripts/lib/CIME/SystemTests/pem.py +A scripts/lib/CIME/SystemTests/pet.py +A scripts/lib/CIME/SystemTests/pfs.py +A scripts/lib/CIME/SystemTests/pre.py +A scripts/lib/CIME/SystemTests/rep.py +A scripts/lib/CIME/SystemTests/seq.py +A scripts/lib/CIME/SystemTests/sms.py +A scripts/lib/CIME/SystemTests/ssp.py +A scripts/lib/CIME/SystemTests/system_tests_common.py +A scripts/lib/CIME/SystemTests/system_tests_compare_two.py +A scripts/lib/CIME/SystemTests/test_utils/__init__.py +A scripts/lib/CIME/SystemTests/test_utils/user_nl_utils.py +A scripts/lib/CIME/XML/__init__.py +A scripts/lib/CIME/XML/archive.py +A scripts/lib/CIME/XML/batch.py +A scripts/lib/CIME/XML/compilerblock.py +A scripts/lib/CIME/XML/compilers.py +A scripts/lib/CIME/XML/component.py +A scripts/lib/CIME/XML/compsets.py +A scripts/lib/CIME/XML/entry_id.py +A scripts/lib/CIME/XML/env_archive.py +A scripts/lib/CIME/XML/env_base.py +A scripts/lib/CIME/XML/env_batch.py +A scripts/lib/CIME/XML/env_build.py +A scripts/lib/CIME/XML/env_case.py +A scripts/lib/CIME/XML/env_mach_pes.py +A scripts/lib/CIME/XML/env_mach_specific.py +A scripts/lib/CIME/XML/env_run.py +A scripts/lib/CIME/XML/env_test.py +A scripts/lib/CIME/XML/files.py +A scripts/lib/CIME/XML/generic_xml.py +A scripts/lib/CIME/XML/grids.py +A scripts/lib/CIME/XML/headers.py +A scripts/lib/CIME/XML/lt_archive.py +A scripts/lib/CIME/XML/machines.py +A scripts/lib/CIME/XML/namelist_definition.py +A scripts/lib/CIME/XML/pes.py +A scripts/lib/CIME/XML/pio.py +A scripts/lib/CIME/XML/standard_module_setup.py +A scripts/lib/CIME/XML/testlist.py +A scripts/lib/CIME/XML/tests.py +A scripts/lib/CIME/XML/testspec.py +A scripts/lib/CIME/__init__.py +A scripts/lib/CIME/bless_test_results.py +A scripts/lib/CIME/build.py +A scripts/lib/CIME/buildlib.py +A scripts/lib/CIME/buildnml.py +A scripts/lib/CIME/case.py +A scripts/lib/CIME/case_cmpgen_namelists.py +A scripts/lib/CIME/case_lt_archive.py +A scripts/lib/CIME/case_run.py +A scripts/lib/CIME/case_setup.py +A scripts/lib/CIME/case_st_archive.py +A scripts/lib/CIME/case_submit.py +A scripts/lib/CIME/case_test.py +A scripts/lib/CIME/check_input_data.py +A scripts/lib/CIME/check_lockedfiles.py +A scripts/lib/CIME/code_checker.py +A scripts/lib/CIME/compare_namelists.py +A scripts/lib/CIME/compare_test_results.py +A scripts/lib/CIME/get_timing.py +A scripts/lib/CIME/hist_utils.py +A scripts/lib/CIME/namelist.py +A scripts/lib/CIME/nmlgen.py +A scripts/lib/CIME/preview_namelists.py +A scripts/lib/CIME/provenance.py +A scripts/lib/CIME/simple_compare.py +A scripts/lib/CIME/test_scheduler.py +A scripts/lib/CIME/test_status.py +A scripts/lib/CIME/test_utils.py +A scripts/lib/CIME/tests/README +A scripts/lib/CIME/tests/SystemTests/__init__.py +A scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py +A scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two_link_to_case2_output.py +A scripts/lib/CIME/tests/SystemTests/test_utils/__init__.py +A scripts/lib/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py +A scripts/lib/CIME/tests/__init__.py +A scripts/lib/CIME/tests/case_fake.py +A scripts/lib/CIME/tests/test_case_fake.py +A scripts/lib/CIME/user_mod_support.py +A scripts/lib/CIME/utils.py +A scripts/lib/CIME/wait_for_tests.py +A scripts/lib/cs.status.template +A scripts/lib/cs.submit.template +A scripts/lib/jenkins_generic_job.py +A scripts/lib/update_acme_tests.py +M scripts/manage_testlists +A scripts/tests/CMakeLists.txt +A scripts/tests/CTestConfig.cmake +A scripts/tests/cpl.hi1.nc.test +A scripts/tests/cpl.hi2.nc.test +A scripts/tests/cpl.log.failmemleak.gz +A scripts/tests/cpl.log.passmemleak.gz +A scripts/tests/list_tests +A scripts/tests/scripts_regression_tests.py +A scripts/tests/user_mods_test1/include_user_mods +A scripts/tests/user_mods_test1/user_nl_comp +A scripts/tests/user_mods_test1/user_nl_cpl +A scripts/tests/user_mods_test2/SourceMods/src.drv/somefile.F90 +A scripts/tests/user_mods_test2/user_nl_cpl +D share/csm_share/README +D share/csm_share/include/dynamic_vector_procdef.inc +D share/csm_share/include/dynamic_vector_typedef.inc +D share/csm_share/include/shr_assert.h +D share/csm_share/shr/CMakeLists.txt +D share/csm_share/shr/mct_mod.F90 +D share/csm_share/shr/shr_assert_mod.F90.in +D share/csm_share/shr/shr_cal_mod.F90 +D share/csm_share/shr/shr_const_mod.F90 +D share/csm_share/shr/shr_dmodel_mod.F90 +D share/csm_share/shr/shr_file_mod.F90 +D share/csm_share/shr/shr_flux_mod.F90 +D share/csm_share/shr/shr_frz_mod.F90.in +D share/csm_share/shr/shr_infnan_mod.F90.in +D share/csm_share/shr/shr_kind_mod.F90 +D share/csm_share/shr/shr_log_mod.F90 +D share/csm_share/shr/shr_map_mod.F90 +D share/csm_share/shr/shr_mct_mod.F90 +D share/csm_share/shr/shr_mem_mod.F90 +D share/csm_share/shr/shr_mpi_mod.F90 +D share/csm_share/shr/shr_msg_mod.F90 +D share/csm_share/shr/shr_ncread_mod.F90 +D share/csm_share/shr/shr_nl_mod.F90 +D share/csm_share/shr/shr_orb_mod.F90 +D share/csm_share/shr/shr_pcdf_mod.F90 +D share/csm_share/shr/shr_pio_mod.F90 +D share/csm_share/shr/shr_precip_mod.F90 +D share/csm_share/shr/shr_reprosum_mod.F90 +D share/csm_share/shr/shr_reprosumx86.c +D share/csm_share/shr/shr_scam_mod.F90 +D share/csm_share/shr/shr_spfn_mod.F90 +D share/csm_share/shr/shr_strconvert_mod.F90 +D share/csm_share/shr/shr_strdata_mod.F90 +D share/csm_share/shr/shr_stream_mod.F90 +D share/csm_share/shr/shr_string_mod.F90 +D share/csm_share/shr/shr_sys_mod.F90 +D share/csm_share/shr/shr_tInterp_mod.F90 +D share/csm_share/shr/shr_timer_mod.F90 +D share/csm_share/shr/shr_vmath_mod.F90 +D share/csm_share/shr/shr_wv_sat_mod.F90 +D share/csm_share/shr/water_isotopes.F90 +D share/csm_share/shr/water_types.F90 +D share/csm_share/test/old_unit_testers/Makefile +D share/csm_share/test/old_unit_testers/Mkdepends +D share/csm_share/test/old_unit_testers/Mksrcfiles +D share/csm_share/test/old_unit_testers/bundle_expected.F90 +D share/csm_share/test/old_unit_testers/config.h +D share/csm_share/test/old_unit_testers/make.Macros +D share/csm_share/test/old_unit_testers/namelist +D share/csm_share/test/old_unit_testers/nl/atm.stdin +D share/csm_share/test/old_unit_testers/nl/cpl.stdin +D share/csm_share/test/old_unit_testers/nl/ice.stdin +D share/csm_share/test/old_unit_testers/nl/lnd.stdin +D share/csm_share/test/old_unit_testers/nl/ocn.stdin +D share/csm_share/test/old_unit_testers/run_dshr_bundle_test +D share/csm_share/test/old_unit_testers/run_file_test +D share/csm_share/test/old_unit_testers/test_mod.F90 +D share/csm_share/test/old_unit_testers/test_shr_file.F90 +D share/csm_share/test/old_unit_testers/test_shr_log.F90 +D share/csm_share/test/old_unit_testers/test_shr_mpi.F90 +D share/csm_share/test/old_unit_testers/test_shr_orb.F90 +D share/csm_share/test/old_unit_testers/test_shr_scam.F90 +D share/csm_share/test/old_unit_testers/test_shr_streams.F90 +D share/csm_share/test/old_unit_testers/test_shr_sys.F90 +D share/csm_share/test/old_unit_testers/test_shr_tInterp.F90 +D share/csm_share/test/unit/CMakeLists.txt +D share/csm_share/test/unit/dynamic_vector/CMakeLists.txt +D share/csm_share/test/unit/dynamic_vector/character16_vector_tests.pf.in +D share/csm_share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc +D share/csm_share/test/unit/dynamic_vector/dynamic_vector_character16.F90 +D share/csm_share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90 +D share/csm_share/test/unit/dynamic_vector/dynamic_vector_integer.F90 +D share/csm_share/test/unit/dynamic_vector/dynamic_vector_r8.F90 +D share/csm_share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in +D share/csm_share/test/unit/dynamic_vector/integer_vector_tests.pf.in +D share/csm_share/test/unit/dynamic_vector/ptr_wrapper.F90 +D share/csm_share/test/unit/dynamic_vector/r8_vector_tests.pf.in +D share/csm_share/test/unit/mock/CMakeLists.txt +D share/csm_share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90 +D share/csm_share/test/unit/shr_assert_test/CMakeLists.txt +D share/csm_share/test/unit/shr_assert_test/test_assert.pf +D share/csm_share/test/unit/shr_assert_test/test_assert_array.pf +D share/csm_share/test/unit/shr_assert_test/test_macro.pf +D share/csm_share/test/unit/shr_assert_test/test_ndebug.pf +D share/csm_share/test/unit/shr_infnan_test/CMakeLists.txt +D share/csm_share/test/unit/shr_infnan_test/test_infnan.F90 +D share/csm_share/test/unit/shr_log_test/CMakeLists.txt +D share/csm_share/test/unit/shr_log_test/test_error_printers.pf +D share/csm_share/test/unit/shr_precip_test/CMakeLists.txt +D share/csm_share/test/unit/shr_precip_test/test_shr_precip.pf +D share/csm_share/test/unit/shr_spfn_test/CMakeLists.txt +D share/csm_share/test/unit/shr_spfn_test/test_erf_r4.pf +D share/csm_share/test/unit/shr_spfn_test/test_erf_r8.pf +D share/csm_share/test/unit/shr_spfn_test/test_gamma_factorial.pf +D share/csm_share/test/unit/shr_spfn_test/test_igamma.pf +D share/csm_share/test/unit/shr_strconvert_test/CMakeLists.txt +D share/csm_share/test/unit/shr_strconvert_test/test_toString.pf +D share/csm_share/test/unit/shr_string_test/CMakeLists.txt +D share/csm_share/test/unit/shr_string_test/test_shr_string.pf +D share/csm_share/test/unit/shr_vmath_test/CMakeLists.txt +D share/csm_share/test/unit/shr_vmath_test/test_vmath.F90 +D share/csm_share/test/unit/shr_wv_sat_test/CMakeLists.txt +D share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat.pf +D share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf +D share/csm_share/unit_test_stubs/README +D share/csm_share/unit_test_stubs/pio/CMakeLists.txt +D share/csm_share/unit_test_stubs/pio/README +D share/csm_share/unit_test_stubs/pio/pio.F90.in +D share/esmf_wrf_timemgr/CMakeLists.txt +D share/esmf_wrf_timemgr/ESMF.F90 +D share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90 +D share/esmf_wrf_timemgr/ESMF_AlarmMod.F90 +D share/esmf_wrf_timemgr/ESMF_BaseMod.F90 +D share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90 +D share/esmf_wrf_timemgr/ESMF_CalendarMod.F90 +D share/esmf_wrf_timemgr/ESMF_ClockMod.F90 +D share/esmf_wrf_timemgr/ESMF_FractionMod.F90 +D share/esmf_wrf_timemgr/ESMF_Macros.inc +D share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90 +D share/esmf_wrf_timemgr/ESMF_Stubs.F90 +D share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90 +D share/esmf_wrf_timemgr/ESMF_TimeMgr.inc +D share/esmf_wrf_timemgr/ESMF_TimeMod.F90 +D share/esmf_wrf_timemgr/Makefile +D share/esmf_wrf_timemgr/MeatMod.F90 +D share/esmf_wrf_timemgr/README +D share/esmf_wrf_timemgr/unittests/Makefile +D share/esmf_wrf_timemgr/unittests/go.csh +D share/esmf_wrf_timemgr/unittests/test.F90 +D share/esmf_wrf_timemgr/unittests/wrf_stuff.F90 +D share/esmf_wrf_timemgr/wrf_error_fatal.F90 +D share/esmf_wrf_timemgr/wrf_message.F90 +D share/shr_RandNum/include/dSFMT-common.h +D share/shr_RandNum/include/dSFMT-params.h +D share/shr_RandNum/include/dSFMT-params19937.h +D share/shr_RandNum/include/dSFMT.h +D share/shr_RandNum/src/dsfmt_f03/dSFMT.c +D share/shr_RandNum/src/dsfmt_f03/dSFMT_interface.F90 +D share/shr_RandNum/src/dsfmt_f03/dSFMT_utils.c +D share/shr_RandNum/src/kissvec/kissvec.c +D share/shr_RandNum/src/kissvec/kissvec_mod.F90 +D share/shr_RandNum/src/mt19937/mersennetwister_mod.F90 +D share/shr_RandNum/src/shr_RandNum_mod.F90 +D share/shr_RandNum/test/bench/Makefile +D share/shr_RandNum/test/bench/test_shr_RandNum.F90 +D share/timing/CMakeLists.txt +D share/timing/COPYING +D share/timing/ChangeLog +D share/timing/GPTLget_memusage.c +D share/timing/GPTLprint_memusage.c +D share/timing/GPTLutil.c +D share/timing/Makefile +D share/timing/README +D share/timing/f_wrappers.c +D share/timing/gptl.c +D share/timing/gptl.h +D share/timing/gptl.inc +D share/timing/gptl_papi.c +D share/timing/perf_mod.F90 +D share/timing/perf_utils.F90 +D share/timing/private.h +A src/build_scripts/buildlib.csm_share +A src/build_scripts/buildlib.gptl +A src/build_scripts/buildlib.mct +A src/build_scripts/buildlib.mpi-serial +A src/build_scripts/buildlib.pio +A src/components/data_comps/datm/atm_comp_mct.F90 +A src/components/data_comps/datm/cime_config/buildlib +A src/components/data_comps/datm/cime_config/buildnml +A src/components/data_comps/datm/cime_config/config_archive.xml +A src/components/data_comps/datm/cime_config/config_component.xml +A src/components/data_comps/datm/cime_config/namelist_definition_datm.xml +A src/components/data_comps/datm/cime_config/user_nl_datm +A src/components/data_comps/datm/datm_comp_mod.F90 +A src/components/data_comps/datm/datm_shr_mod.F90 +A src/components/data_comps/datm/doc/SOM.pdf +A src/components/data_comps/datm/doc/bookinfo.xml +A src/components/data_comps/datm/doc/datacomps.xml +A src/components/data_comps/datm/doc/intro.xml +A src/components/data_comps/datm/doc/rundocbook.csh +A src/components/data_comps/datm/doc/streams.xml +A src/components/data_comps/datm/doc/ug.xml +A src/components/data_comps/desp/cime_config/buildlib +A src/components/data_comps/desp/cime_config/buildnml +A src/components/data_comps/desp/cime_config/config_component.xml +A src/components/data_comps/desp/cime_config/namelist_definition_desp.xml +A src/components/data_comps/desp/cime_config/user_nl_desp +A src/components/data_comps/desp/desp_comp_mod.F90 +A src/components/data_comps/desp/esp_comp_mct.F90 +A src/components/data_comps/desp/esp_utils.F90 +A src/components/data_comps/dice/cime_config/buildlib +A src/components/data_comps/dice/cime_config/buildnml +A src/components/data_comps/dice/cime_config/config_archive.xml +A src/components/data_comps/dice/cime_config/config_component.xml +A src/components/data_comps/dice/cime_config/namelist_definition_dice.xml +A src/components/data_comps/dice/cime_config/user_nl_dice +A src/components/data_comps/dice/dice_comp_mod.F90 +A src/components/data_comps/dice/ice_comp_mct.F90 +A src/components/data_comps/dlnd/cime_config/buildlib +A src/components/data_comps/dlnd/cime_config/buildnml +A src/components/data_comps/dlnd/cime_config/config_archive.xml +A src/components/data_comps/dlnd/cime_config/config_component.xml +A src/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml +A src/components/data_comps/dlnd/cime_config/user_nl_dlnd +A src/components/data_comps/dlnd/dlnd_comp_mod.F90 +A src/components/data_comps/dlnd/lnd_comp_mct.F90 +A src/components/data_comps/docn/cime_config/buildlib +A src/components/data_comps/docn/cime_config/buildnml +A src/components/data_comps/docn/cime_config/config_archive.xml +A src/components/data_comps/docn/cime_config/config_component.xml +A src/components/data_comps/docn/cime_config/namelist_definition_docn.xml +A src/components/data_comps/docn/cime_config/user_nl_docn +A src/components/data_comps/docn/docn_comp_mod.F90 +A src/components/data_comps/docn/ocn_comp_mct.F90 +A src/components/data_comps/docn/tools/pop_som_frc/README +A src/components/data_comps/docn/tools/pop_som_frc/SOM.doc +A src/components/data_comps/docn/tools/pop_som_frc/SOM.pdf +A src/components/data_comps/docn/tools/pop_som_frc/pop_frc.csh +A src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl +A src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl +A src/components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl +A src/components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh +A src/components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh +A src/components/data_comps/drof/cime_config/buildlib +A src/components/data_comps/drof/cime_config/buildnml +A src/components/data_comps/drof/cime_config/config_archive.xml +A src/components/data_comps/drof/cime_config/config_component.xml +A src/components/data_comps/drof/cime_config/namelist_definition_drof.xml +A src/components/data_comps/drof/cime_config/user_nl_drof +A src/components/data_comps/drof/drof_comp_mod.F90 +A src/components/data_comps/drof/rof_comp_mct.F90 +A src/components/data_comps/dwav/README +A src/components/data_comps/dwav/bld/README +A src/components/data_comps/dwav/cime_config/buildlib +A src/components/data_comps/dwav/cime_config/buildnml +A src/components/data_comps/dwav/cime_config/config_archive.xml +A src/components/data_comps/dwav/cime_config/config_component.xml +A src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml +A src/components/data_comps/dwav/cime_config/user_nl_dwav +A src/components/data_comps/dwav/dwav_comp_mod.F90 +A src/components/data_comps/dwav/wav_comp_mct.F90 +A src/components/stub_comps/satm/cime_config/buildlib +A src/components/stub_comps/satm/cime_config/buildnml +A src/components/stub_comps/satm/cime_config/config_component.xml +A src/components/stub_comps/satm/cpl/atm_comp_mct.F90 +A src/components/stub_comps/sesp/cime_config/buildlib +A src/components/stub_comps/sesp/cime_config/buildnml +A src/components/stub_comps/sesp/cime_config/config_component.xml +A src/components/stub_comps/sesp/cpl/esp_comp_mct.F90 +A src/components/stub_comps/sglc/cime_config/buildlib +A src/components/stub_comps/sglc/cime_config/buildnml +A src/components/stub_comps/sglc/cime_config/config_component.xml +A src/components/stub_comps/sglc/cpl/glc_comp_mct.F90 +A src/components/stub_comps/sice/cime_config/buildlib +A src/components/stub_comps/sice/cime_config/buildnml +A src/components/stub_comps/sice/cime_config/config_component.xml +A src/components/stub_comps/sice/cpl/ice_comp_mct.F90 +A src/components/stub_comps/slnd/cime_config/buildlib +A src/components/stub_comps/slnd/cime_config/buildnml +A src/components/stub_comps/slnd/cime_config/config_component.xml +A src/components/stub_comps/slnd/cpl/lnd_comp_mct.F90 +A src/components/stub_comps/socn/cime_config/buildlib +A src/components/stub_comps/socn/cime_config/buildnml +A src/components/stub_comps/socn/cime_config/config_component.xml +A src/components/stub_comps/socn/cpl/ocn_comp_mct.F90 +A src/components/stub_comps/srof/cime_config/buildlib +A src/components/stub_comps/srof/cime_config/buildnml +A src/components/stub_comps/srof/cime_config/config_component.xml +A src/components/stub_comps/srof/cpl/rof_comp_mct.F90 +A src/components/stub_comps/swav/cime_config/buildlib +A src/components/stub_comps/swav/cime_config/buildnml +A src/components/stub_comps/swav/cime_config/config_component.xml +A src/components/stub_comps/swav/cpl/wav_comp_mct.F90 +A src/components/xcpl_comps/xatm/cime_config/buildlib +A src/components/xcpl_comps/xatm/cime_config/buildnml +A src/components/xcpl_comps/xatm/cime_config/config_component.xml +A src/components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 +A src/components/xcpl_comps/xglc/cime_config/buildlib +A src/components/xcpl_comps/xglc/cime_config/buildnml +A src/components/xcpl_comps/xglc/cime_config/config_component.xml +A src/components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 +A src/components/xcpl_comps/xice/cime_config/buildlib +A src/components/xcpl_comps/xice/cime_config/buildnml +A src/components/xcpl_comps/xice/cime_config/config_component.xml +A src/components/xcpl_comps/xice/cpl/ice_comp_mct.F90 +A src/components/xcpl_comps/xlnd/cime_config/buildlib +A src/components/xcpl_comps/xlnd/cime_config/buildnml +A src/components/xcpl_comps/xlnd/cime_config/config_component.xml +A src/components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 +A src/components/xcpl_comps/xocn/cime_config/buildlib +A src/components/xcpl_comps/xocn/cime_config/buildnml +A src/components/xcpl_comps/xocn/cime_config/config_component.xml +A src/components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 +A src/components/xcpl_comps/xrof/cime_config/buildlib +A src/components/xcpl_comps/xrof/cime_config/buildnml +A src/components/xcpl_comps/xrof/cime_config/config_component.xml +A src/components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 +A src/components/xcpl_comps/xshare/dead_data_mod.F90 +A src/components/xcpl_comps/xshare/dead_mct_mod.F90 +A src/components/xcpl_comps/xshare/dead_mod.F90 +A src/components/xcpl_comps/xwav/cime_config/buildlib +A src/components/xcpl_comps/xwav/cime_config/buildnml +A src/components/xcpl_comps/xwav/cime_config/config_component.xml +A src/components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 +A src/drivers/mct/bld/namelist_files/namelist_defaults_drv.xml +A src/drivers/mct/bld/namelist_files/namelist_definition_drv.xml +A src/drivers/mct/bld/namelist_files/namelist_definition_drv_flds.xml +A src/drivers/mct/bld/namelist_files/namelist_definition_modio.xml +A src/drivers/mct/bld/testdir/Buildconf/camconf/drv_flds_in +A src/drivers/mct/bld/testdir/Buildconf/clmconf/drv_flds_in +A src/drivers/mct/bld/testdir/env_build.xml +A src/drivers/mct/bld/testdir/env_case.xml +A src/drivers/mct/bld/testdir/env_mach_pes.xml +A src/drivers/mct/bld/testdir/env_run.xml +A src/drivers/mct/bld/testdir/runtest +A src/drivers/mct/bld/testdir/user_nl_cpl +A src/drivers/mct/cime_config/buildexe +A src/drivers/mct/cime_config/buildnml +A src/drivers/mct/cime_config/config_archive.xml +A src/drivers/mct/cime_config/config_component.xml +A src/drivers/mct/cime_config/config_component_acme.xml +A src/drivers/mct/cime_config/config_component_cesm.xml +A src/drivers/mct/cime_config/config_compsets.xml +A src/drivers/mct/cime_config/config_pes.xml +A src/drivers/mct/cime_config/namelist_definition_drv.xml +A src/drivers/mct/cime_config/namelist_definition_modelio.xml +A src/drivers/mct/cime_config/testdefs/testlist_drv.xml +A src/drivers/mct/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands +A src/drivers/mct/cime_config/user_nl_cpl +A src/drivers/mct/doc/bookinfo.xml +A src/drivers/mct/doc/chap1.xml +A src/drivers/mct/doc/chap2.xml +A src/drivers/mct/doc/figx1.jpg +A src/drivers/mct/doc/namelist.xml +A src/drivers/mct/doc/rundocbook.csh +A src/drivers/mct/doc/stylesheet.dsl +A src/drivers/mct/doc/ug.xml +A src/drivers/mct/main/CMakeLists.txt +A src/drivers/mct/main/cesm_comp_mod.F90 +A src/drivers/mct/main/cesm_driver.F90 +A src/drivers/mct/main/component_mod.F90 +A src/drivers/mct/main/component_type_mod.F90 +A src/drivers/mct/main/cplcomp_exchange_mod.F90 +A src/drivers/mct/main/map_glc2lnd_mod.F90 +A src/drivers/mct/main/map_lnd2glc_mod.F90 +A src/drivers/mct/main/map_lnd2rof_irrig_mod.F90 +A src/drivers/mct/main/mrg_mod.F90 +A src/drivers/mct/main/prep_aoflux_mod.F90 +A src/drivers/mct/main/prep_atm_mod.F90 +A src/drivers/mct/main/prep_glc_mod.F90 +A src/drivers/mct/main/prep_ice_mod.F90 +A src/drivers/mct/main/prep_lnd_mod.F90 +A src/drivers/mct/main/prep_ocn_mod.F90 +A src/drivers/mct/main/prep_rof_mod.F90 +A src/drivers/mct/main/prep_wav_mod.F90 +A src/drivers/mct/main/seq_diag_mct.F90 +A src/drivers/mct/main/seq_domain_mct.F90 +A src/drivers/mct/main/seq_flux_mct.F90 +A src/drivers/mct/main/seq_frac_mct.F90 +A src/drivers/mct/main/seq_hist_mod.F90 +A src/drivers/mct/main/seq_io_mod.F90 +A src/drivers/mct/main/seq_map_mod.F90 +A src/drivers/mct/main/seq_map_type_mod.F90 +A src/drivers/mct/main/seq_rest_mod.F90 +A src/drivers/mct/main/t_driver_timers_mod.F90 +A src/drivers/mct/main/vertical_gradient_calculator_2nd_order.F90 +A src/drivers/mct/main/vertical_gradient_calculator_base.F90 +A src/drivers/mct/main/vertical_gradient_calculator_factory.F90 +A src/drivers/mct/shr/CMakeLists.txt +A src/drivers/mct/shr/glc_elevclass_mod.F90 +A src/drivers/mct/shr/seq_cdata_mod.F90 +A src/drivers/mct/shr/seq_comm_mct.F90 +A src/drivers/mct/shr/seq_drydep_mod.F90 +A src/drivers/mct/shr/seq_flds_mod.F90 +A src/drivers/mct/shr/seq_infodata_mod.F90 +A src/drivers/mct/shr/seq_io_read_mod.F90 +A src/drivers/mct/shr/seq_timemgr_mod.F90 +A src/drivers/mct/shr/shr_carma_mod.F90 +A src/drivers/mct/shr/shr_expr_parser_mod.F90 +A src/drivers/mct/shr/shr_fire_emis_mod.F90 +A src/drivers/mct/shr/shr_megan_mod.F90 +A src/drivers/mct/unit_test/CMakeLists.txt +A src/drivers/mct/unit_test/avect_wrapper_test/CMakeLists.txt +A src/drivers/mct/unit_test/avect_wrapper_test/test_avect_wrapper.pf +A src/drivers/mct/unit_test/glc_elevclass_test/CMakeLists.txt +A src/drivers/mct/unit_test/glc_elevclass_test/test_glc_elevclass.pf +A src/drivers/mct/unit_test/map_glc2lnd_test/CMakeLists.txt +A src/drivers/mct/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf +A src/drivers/mct/unit_test/map_lnd2glc_test/CMakeLists.txt +A src/drivers/mct/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf +A src/drivers/mct/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt +A src/drivers/mct/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf +A src/drivers/mct/unit_test/seq_map_test/CMakeLists.txt +A src/drivers/mct/unit_test/seq_map_test/test_seq_map.pf +A src/drivers/mct/unit_test/stubs/CMakeLists.txt +A src/drivers/mct/unit_test/stubs/vertical_gradient_calculator_constant.F90 +A src/drivers/mct/unit_test/utils/CMakeLists.txt +A src/drivers/mct/unit_test/utils/avect_wrapper_mod.F90 +A src/drivers/mct/unit_test/utils/create_mapper_mod.F90 +A src/drivers/mct/unit_test/utils/mct_wrapper_mod.F90 +A src/drivers/mct/unit_test/utils/simple_map_mod.F90 +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/CMakeLists.txt +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/README +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/gradient_example.txt +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/plot_gradient +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf +A src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf +A src/externals/CMake/.gitignore +A src/externals/CMake/CESM_utils.cmake +A src/externals/CMake/CIME_utils.cmake +A src/externals/CMake/ChangeLog +A src/externals/CMake/Compilers.cmake +A src/externals/CMake/FindpFUnit.cmake +A src/externals/CMake/LICENSE +A src/externals/CMake/README.md +A src/externals/CMake/Sourcelist_utils.cmake +A src/externals/CMake/TryCSizeOf.f90 +A src/externals/CMake/TryMPIIO.f90 +A src/externals/CMake/TryMPIMod.f90 +A src/externals/CMake/TryMPISERIAL.f90 +A src/externals/CMake/TryPnetcdf_inc.f90 +A src/externals/CMake/TryPnetcdf_mod.f90 +A src/externals/CMake/genf90_utils.cmake +A src/externals/CMake/mpiexec.cmake +A src/externals/CMake/pFUnit_utils.cmake +A src/externals/genf90/ChangeLog +A src/externals/genf90/genf90.pl +A src/externals/mct/.gitignore +A src/externals/mct/COPYRIGHT +A src/externals/mct/Makefile +A src/externals/mct/Makefile.conf.in +A src/externals/mct/README +A src/externals/mct/aclocal.m4 +A src/externals/mct/benchmarks/.gitignore +A src/externals/mct/benchmarks/Makefile +A src/externals/mct/benchmarks/RouterTestDis.F90 +A src/externals/mct/benchmarks/RouterTestOvr.F90 +A src/externals/mct/benchmarks/T42.8pC +A src/externals/mct/benchmarks/T42.8pR +A src/externals/mct/benchmarks/gx1.8pC +A src/externals/mct/benchmarks/gx1.8pR +A src/externals/mct/benchmarks/importBench.F90 +A src/externals/mct/config.h.in +A src/externals/mct/configure +A src/externals/mct/configure.ac +A src/externals/mct/doc/.gitignore +A src/externals/mct/doc/Makefile +A src/externals/mct/doc/README +A src/externals/mct/doc/coupler.bib +A src/externals/mct/doc/mct_APIs.tex +A src/externals/mct/doc/texsrc/.gitignore +A src/externals/mct/doc/texsrc/Makefile +A src/externals/mct/doc/texsrc/SRCS_tex.mk +A src/externals/mct/examples/Makefile +A src/externals/mct/examples/README +A src/externals/mct/examples/climate_concur1/.gitignore +A src/externals/mct/examples/climate_concur1/Makefile +A src/externals/mct/examples/climate_concur1/README +A src/externals/mct/examples/climate_concur1/coupler.F90 +A src/externals/mct/examples/climate_concur1/master.F90 +A src/externals/mct/examples/climate_concur1/model.F90 +A src/externals/mct/examples/climate_sequen1/.gitignore +A src/externals/mct/examples/climate_sequen1/Makefile +A src/externals/mct/examples/climate_sequen1/README +A src/externals/mct/examples/climate_sequen1/TS1.dat +A src/externals/mct/examples/climate_sequen1/coupler.F90 +A src/externals/mct/examples/climate_sequen1/dst.rc +A src/externals/mct/examples/climate_sequen1/dstmodel.F90 +A src/externals/mct/examples/climate_sequen1/master.F90 +A src/externals/mct/examples/climate_sequen1/mutils.F90 +A src/externals/mct/examples/climate_sequen1/src.rc +A src/externals/mct/examples/climate_sequen1/srcmodel.F90 +A src/externals/mct/examples/simple/.gitignore +A src/externals/mct/examples/simple/Makefile +A src/externals/mct/examples/simple/README +A src/externals/mct/examples/simple/script.babyblue +A src/externals/mct/examples/simple/twocmp.con.F90 +A src/externals/mct/examples/simple/twocmp.seq.F90 +A src/externals/mct/examples/simple/twocmp.seqNB.F90 +A src/externals/mct/examples/simple/twocmp.seqUnvn.F90 +A src/externals/mct/install-sh +A src/externals/mct/m4/README +A src/externals/mct/m4/acx_mpi.m4 +A src/externals/mct/m4/ax_fc_version.m4 +A src/externals/mct/m4/fortran.m4 +A src/externals/mct/mct/Makefile +A src/externals/mct/mct/README +A src/externals/mct/mct/m_Accumulator.F90 +A src/externals/mct/mct/m_AccumulatorComms.F90 +A src/externals/mct/mct/m_AttrVect.F90 +A src/externals/mct/mct/m_AttrVectComms.F90 +A src/externals/mct/mct/m_AttrVectReduce.F90 +A src/externals/mct/mct/m_ConvertMaps.F90 +A src/externals/mct/mct/m_ExchangeMaps.F90 +A src/externals/mct/mct/m_GeneralGrid.F90 +A src/externals/mct/mct/m_GeneralGridComms.F90 +A src/externals/mct/mct/m_GlobalMap.F90 +A src/externals/mct/mct/m_GlobalSegMap.F90 +A src/externals/mct/mct/m_GlobalSegMapComms.F90 +A src/externals/mct/mct/m_GlobalToLocal.F90 +A src/externals/mct/mct/m_MCTWorld.F90 +A src/externals/mct/mct/m_MatAttrVectMul.F90 +A src/externals/mct/mct/m_Merge.F90 +A src/externals/mct/mct/m_Navigator.F90 +A src/externals/mct/mct/m_Rearranger.F90 +A src/externals/mct/mct/m_Router.F90 +A src/externals/mct/mct/m_SparseMatrix.F90 +A src/externals/mct/mct/m_SparseMatrixComms.F90 +A src/externals/mct/mct/m_SparseMatrixDecomp.F90 +A src/externals/mct/mct/m_SparseMatrixPlus.F90 +A src/externals/mct/mct/m_SparseMatrixToMaps.F90 +A src/externals/mct/mct/m_SpatialIntegral.F90 +A src/externals/mct/mct/m_SpatialIntegralV.F90 +A src/externals/mct/mct/m_Transfer.F90 +A src/externals/mct/mkinstalldirs +A src/externals/mct/mpeu/Makefile +A src/externals/mct/mpeu/README +A src/externals/mct/mpeu/assertmpeu.H +A src/externals/mct/mpeu/get_zeits.c +A src/externals/mct/mpeu/m_FcComms.F90 +A src/externals/mct/mpeu/m_FileResolv.F90 +A src/externals/mct/mpeu/m_Filename.F90 +A src/externals/mct/mpeu/m_IndexBin_char.F90 +A src/externals/mct/mpeu/m_IndexBin_integer.F90 +A src/externals/mct/mpeu/m_IndexBin_logical.F90 +A src/externals/mct/mpeu/m_List.F90 +A src/externals/mct/mpeu/m_MergeSorts.F90 +A src/externals/mct/mpeu/m_Permuter.F90 +A src/externals/mct/mpeu/m_SortingTools.F90 +A src/externals/mct/mpeu/m_StrTemplate.F90 +A src/externals/mct/mpeu/m_String.F90 +A src/externals/mct/mpeu/m_StringLinkedList.F90 +A src/externals/mct/mpeu/m_TraceBack.F90 +A src/externals/mct/mpeu/m_chars.F90 +A src/externals/mct/mpeu/m_die.F90 +A src/externals/mct/mpeu/m_dropdead.F90 +A src/externals/mct/mpeu/m_flow.F90 +A src/externals/mct/mpeu/m_inpak90.F90 +A src/externals/mct/mpeu/m_ioutil.F90 +A src/externals/mct/mpeu/m_mall.F90 +A src/externals/mct/mpeu/m_mpif.F90 +A src/externals/mct/mpeu/m_mpif90.F90 +A src/externals/mct/mpeu/m_mpout.F90 +A src/externals/mct/mpeu/m_rankMerge.F90 +A src/externals/mct/mpeu/m_realkinds.F90 +A src/externals/mct/mpeu/m_stdio.F90 +A src/externals/mct/mpeu/m_zeit.F90 +A src/externals/mct/mpi-serial/.gitignore +A src/externals/mct/mpi-serial/Makefile +A src/externals/mct/mpi-serial/Makefile.conf.in +A src/externals/mct/mpi-serial/NOTES +A src/externals/mct/mpi-serial/README +A src/externals/mct/mpi-serial/aclocal.m4 +A src/externals/mct/mpi-serial/cart.c +A src/externals/mct/mpi-serial/collective.c +A src/externals/mct/mpi-serial/comm.c +A src/externals/mct/mpi-serial/config.h.in +A src/externals/mct/mpi-serial/configure +A src/externals/mct/mpi-serial/configure.in +A src/externals/mct/mpi-serial/copy.c +A src/externals/mct/mpi-serial/fort.F90 +A src/externals/mct/mpi-serial/getcount.c +A src/externals/mct/mpi-serial/group.c +A src/externals/mct/mpi-serial/handles.c +A src/externals/mct/mpi-serial/info.c +A src/externals/mct/mpi-serial/list.c +A src/externals/mct/mpi-serial/list.h +A src/externals/mct/mpi-serial/listP.h +A src/externals/mct/mpi-serial/listops.h +A src/externals/mct/mpi-serial/m4/README +A src/externals/mct/mpi-serial/m4/ax_fc_version.m4 +A src/externals/mct/mpi-serial/mpi.c +A src/externals/mct/mpi-serial/mpi.h +A src/externals/mct/mpi-serial/mpiP.h +A src/externals/mct/mpi-serial/mpif.F90 +A src/externals/mct/mpi-serial/mpif.h +A src/externals/mct/mpi-serial/op.c +A src/externals/mct/mpi-serial/pack.c +A src/externals/mct/mpi-serial/probe.c +A src/externals/mct/mpi-serial/protify.awk +A src/externals/mct/mpi-serial/recv.c +A src/externals/mct/mpi-serial/req.c +A src/externals/mct/mpi-serial/send.c +A src/externals/mct/mpi-serial/tests/.gitignore +A src/externals/mct/mpi-serial/tests/Makefile +A src/externals/mct/mpi-serial/tests/ctest.c +A src/externals/mct/mpi-serial/tests/ctest_old.c +A src/externals/mct/mpi-serial/tests/ftest.F90 +A src/externals/mct/mpi-serial/tests/ftest_internal.F90 +A src/externals/mct/mpi-serial/tests/ftest_old.F90 +A src/externals/mct/mpi-serial/time.c +A src/externals/mct/mpi-serial/type.c +A src/externals/mct/mpi-serial/type.h +A src/externals/mct/mpi-serial/type_const.c +A src/externals/mct/protex/protex +A src/externals/mct/testsystem/Makefile +A src/externals/mct/testsystem/testall/.gitignore +A src/externals/mct/testsystem/testall/Makefile +A src/externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 +A src/externals/mct/testsystem/testall/UNTESTED +A src/externals/mct/testsystem/testall/ccm.F90 +A src/externals/mct/testsystem/testall/convertPOPT.F90 +A src/externals/mct/testsystem/testall/convertgauss.F90 +A src/externals/mct/testsystem/testall/cpl.F90 +A src/externals/mct/testsystem/testall/job.ut-all.jaguar +A src/externals/mct/testsystem/testall/m_ACTEST.F90 +A src/externals/mct/testsystem/testall/m_AVTEST.F90 +A src/externals/mct/testsystem/testall/m_GGRIDTEST.F90 +A src/externals/mct/testsystem/testall/m_GMAPTEST.F90 +A src/externals/mct/testsystem/testall/m_GSMAPTEST.F90 +A src/externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 +A src/externals/mct/testsystem/testall/m_ROUTERTEST.F90 +A src/externals/mct/testsystem/testall/m_SMATTEST.F90 +A src/externals/mct/testsystem/testall/master.F90 +A src/externals/mct/testsystem/testall/mph.F90 +A src/externals/mct/testsystem/testall/pop.F90 +A src/externals/mct/testsystem/testall/processors_map.in +A src/externals/mct/testsystem/testall/script.jag +A src/externals/mct/testsystem/testall/ut_SparseMatrix.rc +A src/externals/mct/testunit/.gitignore +A src/externals/mct/testunit/AttrVect_Test.F90 +A src/externals/mct/testunit/Makefile +A src/externals/mct/testunit/master.F90 +A src/externals/pio1/CMakeLists.txt +A src/externals/pio1/ChangeLog +A src/externals/pio1/ChangeLog_template +A src/externals/pio1/Doxyfile +A src/externals/pio1/DoxygenLayout.xml +A src/externals/pio1/SVN_EXTERNAL_DIRECTORIES +A src/externals/pio1/customdoxygen.css +A src/externals/pio1/doc/CAMexample.txt +A src/externals/pio1/doc/Decomp.txt +A src/externals/pio1/doc/DoxygenLayout.xml +A src/externals/pio1/doc/Error.txt +A src/externals/pio1/doc/Examples.txt +A src/externals/pio1/doc/GettingStarted.txt +A src/externals/pio1/doc/Installing.txt +A src/externals/pio1/doc/api.txt +A src/externals/pio1/doc/base.txt +A src/externals/pio1/doc/example/errorhandle +A src/externals/pio1/doc/example/simple-bc +A src/externals/pio1/doc/example/simple-bc-rearr +A src/externals/pio1/doc/example/simple-bc-rearr-pe1 +A src/externals/pio1/doc/example/simple-bc-rearr-pe2 +A src/externals/pio1/doc/example/simple-dof +A src/externals/pio1/doc/example/simple-dof-rearr +A src/externals/pio1/doc/faq.txt +A src/externals/pio1/doc/footer.html +A src/externals/pio1/doc/header.html +A src/externals/pio1/doc/images/baseimage.graffle +A src/externals/pio1/doc/images/block-cyclic-rearr.eps +A src/externals/pio1/doc/images/block-cyclic-rearr.graffle +A src/externals/pio1/doc/images/block-cyclic-rearr.png +A src/externals/pio1/doc/images/block-cyclic.eps +A src/externals/pio1/doc/images/block-cyclic.graffle +A src/externals/pio1/doc/images/block-cyclic.png +A src/externals/pio1/doc/images/dof-rearr.eps +A src/externals/pio1/doc/images/dof-rearr.graffle +A src/externals/pio1/doc/images/dof-rearr.png +A src/externals/pio1/doc/images/dof.eps +A src/externals/pio1/doc/images/dof.graffle +A src/externals/pio1/doc/images/dof.png +A src/externals/pio1/doc/testpio_example.txt +A src/externals/pio1/doxygen.sty +A src/externals/pio1/pio/CMakeLists.txt +A src/externals/pio1/pio/C_interface_mod.F90 +A src/externals/pio1/pio/Makefile.conf.in +A src/externals/pio1/pio/README.config +A src/externals/pio1/pio/alloc_mod.F90.in +A src/externals/pio1/pio/box_rearrange.F90.in +A src/externals/pio1/pio/calcdecomp.F90 +A src/externals/pio1/pio/calcdisplace_mod.F90 +A src/externals/pio1/pio/config.h.in +A src/externals/pio1/pio/dtypes.h +A src/externals/pio1/pio/fdepends.awk +A src/externals/pio1/pio/iompi_mod.F90.in +A src/externals/pio1/pio/ionf_mod.F90 +A src/externals/pio1/pio/nf_mod.F90 +A src/externals/pio1/pio/pio.F90 +A src/externals/pio1/pio/pio_kinds.F90 +A src/externals/pio1/pio/pio_mpi_utils.F90 +A src/externals/pio1/pio/pio_msg_callbacks.F90 +A src/externals/pio1/pio/pio_msg_getput_callbacks.F90.in +A src/externals/pio1/pio/pio_msg_mod.F90 +A src/externals/pio1/pio/pio_nf_utils.F90 +A src/externals/pio1/pio/pio_spmd_utils.F90.in +A src/externals/pio1/pio/pio_support.F90 +A src/externals/pio1/pio/pio_types.F90 +A src/externals/pio1/pio/pio_utils.F90 +A src/externals/pio1/pio/piodarray.F90.in +A src/externals/pio1/pio/piolib_mod.F90 +A src/externals/pio1/pio/pionfatt_mod.F90.in +A src/externals/pio1/pio/pionfget_mod.F90.in +A src/externals/pio1/pio/pionfput_mod.F90.in +A src/externals/pio1/pio/pionfread_mod.F90.in +A src/externals/pio1/pio/pionfwrite_mod.F90.in +A src/externals/pio1/pio/piovdc.F90 +A src/externals/pio1/pio/rearr_options.h +A src/externals/pio1/pio/rearrange.F90.in +A src/externals/pio1/pio/topology.c +A src/externals/pio1/scripts/Utils.pm +A src/externals/pio1/scripts/config.pl +A src/externals/pio1/scripts/testpio_yellowstone.pl +A src/externals/pio1/tests/testpio/CAM05.csh +A src/externals/pio1/tests/testpio/CMakeLists.txt +A src/externals/pio1/tests/testpio/MPASA30km.csh +A src/externals/pio1/tests/testpio/MPASA60km.csh +A src/externals/pio1/tests/testpio/POPB.csh +A src/externals/pio1/tests/testpio/POPC.csh +A src/externals/pio1/tests/testpio/POPD.csh +A src/externals/pio1/tests/testpio/POPDv0.csh +A src/externals/pio1/tests/testpio/POPDv1.csh +A src/externals/pio1/tests/testpio/POPDv2.csh +A src/externals/pio1/tests/testpio/POPDv3.csh +A src/externals/pio1/tests/testpio/POPDv4.csh +A src/externals/pio1/tests/testpio/POPDv5.csh +A src/externals/pio1/tests/testpio/README.testpio +A src/externals/pio1/tests/testpio/WRFB.csh +A src/externals/pio1/tests/testpio/build_defaults.xml +A src/externals/pio1/tests/testpio/check_mod.F90 +A src/externals/pio1/tests/testpio/config_bench.xml +A src/externals/pio1/tests/testpio/fdepends.awk +A src/externals/pio1/tests/testpio/gdecomp_mod.F90 +A src/externals/pio1/tests/testpio/kinds_mod.F90 +A src/externals/pio1/tests/testpio/kraken.128.csh +A src/externals/pio1/tests/testpio/kraken.1K.csh +A src/externals/pio1/tests/testpio/kraken.256.csh +A src/externals/pio1/tests/testpio/kraken.512.csh +A src/externals/pio1/tests/testpio/kraken.64.csh +A src/externals/pio1/tests/testpio/namelist_mod.F90 +A src/externals/pio1/tests/testpio/namelists/testpio_in.apb05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.asb01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.asb04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b06 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b07 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b08 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b09 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b10 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b11 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b12 +A src/externals/pio1/tests/testpio/namelists/testpio_in.b13 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb06 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb07 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bb08 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bn01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bn02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.bn03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b06 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b07 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4b08 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4n01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4n02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.n4n03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb06 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb07 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pb08 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pn01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pn02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.pn03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb04 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb05 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb06 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb07 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sb08 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sn01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sn02 +A src/externals/pio1/tests/testpio/namelists/testpio_in.sn03 +A src/externals/pio1/tests/testpio/namelists/testpio_in.wr01 +A src/externals/pio1/tests/testpio/namelists/testpio_in.wr02 +A src/externals/pio1/tests/testpio/perl5lib/ChangeLog +A src/externals/pio1/tests/testpio/perl5lib/README +A src/externals/pio1/tests/testpio/perl5lib/XML/Changes +A src/externals/pio1/tests/testpio/perl5lib/XML/Lite.pm +A src/externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm +A src/externals/pio1/tests/testpio/perl5lib/XML/README +A src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 +A src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 +A src/externals/pio1/tests/testpio/test.csh +A src/externals/pio1/tests/testpio/test_lib.F90 +A src/externals/pio1/tests/testpio/testdecomp.F90 +A src/externals/pio1/tests/testpio/testdecomp.bluefire.run +A src/externals/pio1/tests/testpio/testdecomp_in +A src/externals/pio1/tests/testpio/testpio.F90 +A src/externals/pio1/tests/testpio/testpio_bench.pl +A src/externals/pio1/tests/testpio/testpio_build.pl +A src/externals/pio1/tests/testpio/testpio_run.pl +A src/externals/pio1/tests/testpio/utils_mod.F90 +A src/externals/pio1/tests/testpio/ystest.sh +A src/externals/pio1/tests/unittests/CMakeLists.txt +A src/externals/pio1/tests/unittests/Levy_Notes +A src/externals/pio1/tests/unittests/README +A src/externals/pio1/tests/unittests/basic_tests.F90 +A src/externals/pio1/tests/unittests/driver.F90 +A src/externals/pio1/tests/unittests/global_vars.F90 +A src/externals/pio1/tests/unittests/input.nl +A src/externals/pio1/tests/unittests/nc_set_log_level2.c +A src/externals/pio1/tests/unittests/ncdf_tests.F90 +A src/externals/pio1/tests/unittests/not_netcdf.ieee +A src/externals/pio1/timing/CMakeLists.txt +A src/externals/pio1/timing/COPYING +A src/externals/pio1/timing/ChangeLog +A src/externals/pio1/timing/GPTLget_memusage.c +A src/externals/pio1/timing/GPTLprint_memusage.c +A src/externals/pio1/timing/GPTLutil.c +A src/externals/pio1/timing/Makefile +A src/externals/pio1/timing/README +A src/externals/pio1/timing/XXXdotF/perf_mod.F +A src/externals/pio1/timing/XXXdotF/perf_utils.F +A src/externals/pio1/timing/f_wrappers.c +A src/externals/pio1/timing/gptl.c +A src/externals/pio1/timing/gptl.h +A src/externals/pio1/timing/gptl.inc +A src/externals/pio1/timing/gptl_papi.c +A src/externals/pio1/timing/perf_mod.F90 +A src/externals/pio1/timing/perf_utils.F90 +A src/externals/pio1/timing/private.h +A src/externals/pio1/timing/threadutil.c +A src/externals/pio2/.gitignore +A src/externals/pio2/CMakeLists.txt +A src/externals/pio2/CTestConfig.cmake +A src/externals/pio2/CTestScript.cmake +A src/externals/pio2/README.md +A src/externals/pio2/cmake/FindGPTL.cmake +A src/externals/pio2/cmake/FindHDF5.cmake +A src/externals/pio2/cmake/FindLIBRT.cmake +A src/externals/pio2/cmake/FindLIBZ.cmake +A src/externals/pio2/cmake/FindMPE.cmake +A src/externals/pio2/cmake/FindMPISERIAL.cmake +A src/externals/pio2/cmake/FindNetCDF.cmake +A src/externals/pio2/cmake/FindPAPI.cmake +A src/externals/pio2/cmake/FindPnetCDF.cmake +A src/externals/pio2/cmake/FindSZIP.cmake +A src/externals/pio2/cmake/LibCheck.cmake +A src/externals/pio2/cmake/LibFind.cmake +A src/externals/pio2/cmake/LibMPI.cmake +A src/externals/pio2/cmake/TryHDF5_HAS_SZIP.c +A src/externals/pio2/cmake/TryNetCDF_DAP.c +A src/externals/pio2/cmake/TryNetCDF_PARALLEL.c +A src/externals/pio2/cmake/TryNetCDF_PNETCDF.c +A src/externals/pio2/cmake/mpiexec.alcf +A src/externals/pio2/cmake/mpiexec.ncsa +A src/externals/pio2/cmake/mpiexec.nersc +A src/externals/pio2/cmake/mpiexec.nwsc +A src/externals/pio2/ctest/CTestEnvironment-alcf.cmake +A src/externals/pio2/ctest/CTestEnvironment-cgd.cmake +A src/externals/pio2/ctest/CTestEnvironment-ncsa.cmake +A src/externals/pio2/ctest/CTestEnvironment-nersc.cmake +A src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake +A src/externals/pio2/ctest/CTestEnvironment-unknown.cmake +A src/externals/pio2/ctest/CTestScript-Test.cmake +A src/externals/pio2/ctest/runcdash-alcf-ibm.sh +A src/externals/pio2/ctest/runcdash-cgd-nag.sh +A src/externals/pio2/ctest/runcdash-nersc-cray.sh +A src/externals/pio2/ctest/runcdash-nersc-intel.sh +A src/externals/pio2/ctest/runcdash-nwsc-gnu.sh +A src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +A src/externals/pio2/ctest/runcdash-nwsc-intel.sh +A src/externals/pio2/ctest/runcdash-nwsc-pgi.sh +A src/externals/pio2/ctest/runctest-alcf.sh +A src/externals/pio2/ctest/runctest-cgd.sh +A src/externals/pio2/ctest/runctest-ncsa.sh +A src/externals/pio2/ctest/runctest-nersc.sh +A src/externals/pio2/ctest/runctest-nwsc.sh +A src/externals/pio2/ctest/runctest-unknown.sh +A src/externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake +A src/externals/pio2/doc/CMakeFiles/CMakeOutput.log +A src/externals/pio2/doc/CMakeFiles/cmake.check_cache +A src/externals/pio2/doc/CMakeLists.txt +A src/externals/pio2/doc/Doxyfile.in +A src/externals/pio2/doc/DoxygenLayout.xml +A src/externals/pio2/doc/customdoxygen.css +A src/externals/pio2/doc/doxygen.sty +A src/externals/pio2/doc/images/baseimage.graffle +A src/externals/pio2/doc/images/block-cyclic-rearr.eps +A src/externals/pio2/doc/images/block-cyclic-rearr.graffle +A src/externals/pio2/doc/images/block-cyclic-rearr.png +A src/externals/pio2/doc/images/block-cyclic.eps +A src/externals/pio2/doc/images/block-cyclic.graffle +A src/externals/pio2/doc/images/block-cyclic.png +A src/externals/pio2/doc/images/dof-rearr.eps +A src/externals/pio2/doc/images/dof-rearr.graffle +A src/externals/pio2/doc/images/dof-rearr.png +A src/externals/pio2/doc/images/dof.eps +A src/externals/pio2/doc/images/dof.graffle +A src/externals/pio2/doc/images/dof.png +A src/externals/pio2/doc/source/CAMexample.txt +A src/externals/pio2/doc/source/Decomp.txt +A src/externals/pio2/doc/source/Error.txt +A src/externals/pio2/doc/source/Examples.txt +A src/externals/pio2/doc/source/GettingStarted.txt +A src/externals/pio2/doc/source/Installing.txt +A src/externals/pio2/doc/source/api.txt +A src/externals/pio2/doc/source/base.txt +A src/externals/pio2/doc/source/example/errorhandle +A src/externals/pio2/doc/source/example/simple-bc +A src/externals/pio2/doc/source/example/simple-bc-rearr +A src/externals/pio2/doc/source/example/simple-bc-rearr-pe1 +A src/externals/pio2/doc/source/example/simple-bc-rearr-pe2 +A src/externals/pio2/doc/source/example/simple-dof +A src/externals/pio2/doc/source/example/simple-dof-rearr +A src/externals/pio2/doc/source/faq.txt +A src/externals/pio2/doc/source/mach_walkthrough.txt +A src/externals/pio2/doc/source/testpio_example.txt +A src/externals/pio2/examples/CMakeLists.txt +A src/externals/pio2/examples/basic/CAM05.csh +A src/externals/pio2/examples/basic/CMakeLists.txt +A src/externals/pio2/examples/basic/MPASA30km.csh +A src/externals/pio2/examples/basic/MPASA60km.csh +A src/externals/pio2/examples/basic/POPB.csh +A src/externals/pio2/examples/basic/POPC.csh +A src/externals/pio2/examples/basic/POPD.csh +A src/externals/pio2/examples/basic/POPDv0.csh +A src/externals/pio2/examples/basic/POPDv1.csh +A src/externals/pio2/examples/basic/POPDv2.csh +A src/externals/pio2/examples/basic/POPDv3.csh +A src/externals/pio2/examples/basic/POPDv4.csh +A src/externals/pio2/examples/basic/POPDv5.csh +A src/externals/pio2/examples/basic/README.testpio +A src/externals/pio2/examples/basic/WRFB.csh +A src/externals/pio2/examples/basic/alloc_mod.F90.in +A src/externals/pio2/examples/basic/build_defaults.xml +A src/externals/pio2/examples/basic/check_mod.F90 +A src/externals/pio2/examples/basic/config_bench.xml +A src/externals/pio2/examples/basic/fdepends.awk +A src/externals/pio2/examples/basic/gdecomp_mod.F90 +A src/externals/pio2/examples/basic/kinds_mod.F90 +A src/externals/pio2/examples/basic/kraken.128.csh +A src/externals/pio2/examples/basic/kraken.1K.csh +A src/externals/pio2/examples/basic/kraken.256.csh +A src/externals/pio2/examples/basic/kraken.512.csh +A src/externals/pio2/examples/basic/kraken.64.csh +A src/externals/pio2/examples/basic/namelist_mod.F90 +A src/externals/pio2/examples/basic/namelists/testpio_in.apb05 +A src/externals/pio2/examples/basic/namelists/testpio_in.asb01 +A src/externals/pio2/examples/basic/namelists/testpio_in.asb04 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b01 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b02 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b03 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b04 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b05 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b06 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b07 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4b08 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4n01 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4n02 +A src/externals/pio2/examples/basic/namelists/testpio_in.n4n03 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb01 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb02 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb03 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb04 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb05 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb06 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb07 +A src/externals/pio2/examples/basic/namelists/testpio_in.pb08 +A src/externals/pio2/examples/basic/namelists/testpio_in.pn01 +A src/externals/pio2/examples/basic/namelists/testpio_in.pn02 +A src/externals/pio2/examples/basic/namelists/testpio_in.pn03 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps01 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps02 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps03 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps04 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps05 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps06 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps07 +A src/externals/pio2/examples/basic/namelists/testpio_in.ps08 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb01 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb02 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb03 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb04 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb05 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb06 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb07 +A src/externals/pio2/examples/basic/namelists/testpio_in.sb08 +A src/externals/pio2/examples/basic/namelists/testpio_in.sn01 +A src/externals/pio2/examples/basic/namelists/testpio_in.sn02 +A src/externals/pio2/examples/basic/namelists/testpio_in.sn03 +A src/externals/pio2/examples/basic/namelists/testpio_in.wr01 +A src/externals/pio2/examples/basic/namelists/testpio_in.wr02 +A src/externals/pio2/examples/basic/perl5lib/ChangeLog +A src/externals/pio2/examples/basic/perl5lib/README +A src/externals/pio2/examples/basic/perl5lib/XML/Changes +A src/externals/pio2/examples/basic/perl5lib/XML/Lite.pm +A src/externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm +A src/externals/pio2/examples/basic/perl5lib/XML/README +A src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3 +A src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3 +A src/externals/pio2/examples/basic/test.csh +A src/externals/pio2/examples/basic/testdecomp.F90 +A src/externals/pio2/examples/basic/testdecomp.bluefire.run +A src/externals/pio2/examples/basic/testdecomp_in +A src/externals/pio2/examples/basic/testpio.F90 +A src/externals/pio2/examples/basic/testpio_bench.pl +A src/externals/pio2/examples/basic/testpio_build.pl +A src/externals/pio2/examples/basic/testpio_run.pl +A src/externals/pio2/examples/basic/utils_mod.F90 +A src/externals/pio2/examples/basic/wstest.c +A src/externals/pio2/examples/basic/ystest.sh +A src/externals/pio2/examples/c/CMakeLists.txt +A src/externals/pio2/examples/c/example1.c +A src/externals/pio2/examples/c/example2.c +A src/externals/pio2/examples/c/examplePio.c +A src/externals/pio2/examples/cxx/CMakeLists.txt +A src/externals/pio2/examples/cxx/examplePio.cxx +A src/externals/pio2/examples/f03/CMakeLists.txt +A src/externals/pio2/examples/f03/examplePio.f90 +A src/externals/pio2/scripts/prune_decomps.pl +A src/externals/pio2/src/CMakeLists.txt +A src/externals/pio2/src/clib/CMakeLists.txt +A src/externals/pio2/src/clib/bget.c +A src/externals/pio2/src/clib/bget.h +A src/externals/pio2/src/clib/config.h.in +A src/externals/pio2/src/clib/dtypes.h +A src/externals/pio2/src/clib/ncparser.pl +A src/externals/pio2/src/clib/ncputgetparser.pl +A src/externals/pio2/src/clib/pio.h +A src/externals/pio2/src/clib/pio_c_get_template.c +A src/externals/pio2/src/clib/pio_c_put_template.c +A src/externals/pio2/src/clib/pio_c_template.c +A src/externals/pio2/src/clib/pio_darray.c +A src/externals/pio2/src/clib/pio_file.c +A src/externals/pio2/src/clib/pio_get_nc.c +A src/externals/pio2/src/clib/pio_internal.h +A src/externals/pio2/src/clib/pio_lists.c +A src/externals/pio2/src/clib/pio_nc.c +A src/externals/pio2/src/clib/pio_nc4.c +A src/externals/pio2/src/clib/pio_put_nc.c +A src/externals/pio2/src/clib/pio_rearrange.c +A src/externals/pio2/src/clib/pio_spmd.c +A src/externals/pio2/src/clib/pioc.c +A src/externals/pio2/src/clib/pioc_sc.c +A src/externals/pio2/src/clib/pioc_support.c +A src/externals/pio2/src/clib/topology.c +A src/externals/pio2/src/flib/CMakeLists.txt +A src/externals/pio2/src/flib/pio.F90 +A src/externals/pio2/src/flib/pio_kinds.F90 +A src/externals/pio2/src/flib/pio_nf.F90 +A src/externals/pio2/src/flib/pio_support.F90 +A src/externals/pio2/src/flib/pio_types.F90 +A src/externals/pio2/src/flib/piodarray.F90.in +A src/externals/pio2/src/flib/piolib_mod.F90 +A src/externals/pio2/src/flib/pionfatt_mod.F90.in +A src/externals/pio2/src/flib/pionfget_mod.F90.in +A src/externals/pio2/src/flib/pionfput_mod.F90.in +A src/externals/pio2/src/gptl/CMakeLists.txt +A src/externals/pio2/src/gptl/COPYING +A src/externals/pio2/src/gptl/ChangeLog +A src/externals/pio2/src/gptl/GPTLget_memusage.c +A src/externals/pio2/src/gptl/GPTLprint_memusage.c +A src/externals/pio2/src/gptl/GPTLutil.c +A src/externals/pio2/src/gptl/README +A src/externals/pio2/src/gptl/f_wrappers.c +A src/externals/pio2/src/gptl/gptl.c +A src/externals/pio2/src/gptl/gptl.h +A src/externals/pio2/src/gptl/gptl.inc +A src/externals/pio2/src/gptl/gptl_papi.c +A src/externals/pio2/src/gptl/perf_mod.F90 +A src/externals/pio2/src/gptl/perf_utils.F90 +A src/externals/pio2/src/gptl/private.h +A src/externals/pio2/src/gptl/threadutil.c +A src/externals/pio2/tests/CMakeLists.txt +A src/externals/pio2/tests/general/CMakeLists.txt +A src/externals/pio2/tests/general/ncdf_fail.F90.in +A src/externals/pio2/tests/general/ncdf_get_put.F90.in +A src/externals/pio2/tests/general/ncdf_simple_tests.F90.in +A src/externals/pio2/tests/general/pio_decomp_fillval.F90.in +A src/externals/pio2/tests/general/pio_decomp_fillval2.F90 +A src/externals/pio2/tests/general/pio_decomp_tests.F90.in +A src/externals/pio2/tests/general/pio_fail.F90.in +A src/externals/pio2/tests/general/pio_file_fail.F90.in +A src/externals/pio2/tests/general/pio_file_simple_tests.F90.in +A src/externals/pio2/tests/general/pio_init_finalize.F90.in +A src/externals/pio2/tests/general/test_memleak.c +A src/externals/pio2/tests/general/util/pio_tf_f90gen.pl +A src/externals/pio2/tests/general/util/pio_tutil.F90 +A src/externals/pio2/tests/performance/CMakeLists.txt +A src/externals/pio2/tests/performance/gensimple.pl +A src/externals/pio2/tests/performance/kt.PIO1.perfmakefile +A src/externals/pio2/tests/performance/pioperformance.F90 +A src/externals/pio2/tests/unit/CMakeLists.txt +A src/externals/pio2/tests/unit/Levy_Notes +A src/externals/pio2/tests/unit/README +A src/externals/pio2/tests/unit/basic_tests.F90 +A src/externals/pio2/tests/unit/driver.F90 +A src/externals/pio2/tests/unit/global_vars.F90 +A src/externals/pio2/tests/unit/input.nl +A src/externals/pio2/tests/unit/nc_set_log_level2.c +A src/externals/pio2/tests/unit/ncdf_tests.F90 +A src/externals/pio2/tests/unit/not_netcdf.ieee +A src/externals/pio2/tests/unit/test_names.c +A src/externals/pio2/tests/unit/test_nc4.c +A src/share/README +A src/share/RandNum/include/dSFMT-common.h +A src/share/RandNum/include/dSFMT-params.h +A src/share/RandNum/include/dSFMT-params19937.h +A src/share/RandNum/include/dSFMT.h +A src/share/RandNum/src/dsfmt_f03/dSFMT.c +A src/share/RandNum/src/dsfmt_f03/dSFMT_interface.F90 +A src/share/RandNum/src/dsfmt_f03/dSFMT_utils.c +A src/share/RandNum/src/kissvec/kissvec.c +A src/share/RandNum/src/kissvec/kissvec_mod.F90 +A src/share/RandNum/src/mt19937/mersennetwister_mod.F90 +A src/share/RandNum/src/shr_RandNum_mod.F90 +A src/share/RandNum/test/bench/Makefile +A src/share/RandNum/test/bench/test_shr_RandNum.F90 +A src/share/esmf_wrf_timemgr/CMakeLists.txt +A src/share/esmf_wrf_timemgr/ESMF.F90 +A src/share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_AlarmMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_BaseMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_CalendarMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_ClockMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_FractionMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_Macros.inc +A src/share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_Stubs.F90 +A src/share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90 +A src/share/esmf_wrf_timemgr/ESMF_TimeMgr.inc +A src/share/esmf_wrf_timemgr/ESMF_TimeMod.F90 +A src/share/esmf_wrf_timemgr/Makefile +A src/share/esmf_wrf_timemgr/MeatMod.F90 +A src/share/esmf_wrf_timemgr/README +A src/share/esmf_wrf_timemgr/unittests/Makefile +A src/share/esmf_wrf_timemgr/unittests/go.csh +A src/share/esmf_wrf_timemgr/unittests/test.F90 +A src/share/esmf_wrf_timemgr/unittests/wrf_stuff.F90 +A src/share/esmf_wrf_timemgr/wrf_error_fatal.F90 +A src/share/esmf_wrf_timemgr/wrf_message.F90 +A src/share/include/dynamic_vector_procdef.inc +A src/share/include/dynamic_vector_typedef.inc +A src/share/include/shr_assert.h +A src/share/test/old_unit_testers/Makefile +A src/share/test/old_unit_testers/Mkdepends +A src/share/test/old_unit_testers/Mksrcfiles +A src/share/test/old_unit_testers/bundle_expected.F90 +A src/share/test/old_unit_testers/config.h +A src/share/test/old_unit_testers/make.Macros +A src/share/test/old_unit_testers/namelist +A src/share/test/old_unit_testers/nl/atm.stdin +A src/share/test/old_unit_testers/nl/cpl.stdin +A src/share/test/old_unit_testers/nl/ice.stdin +A src/share/test/old_unit_testers/nl/lnd.stdin +A src/share/test/old_unit_testers/nl/ocn.stdin +A src/share/test/old_unit_testers/run_dshr_bundle_test +A src/share/test/old_unit_testers/run_file_test +A src/share/test/old_unit_testers/test_mod.F90 +A src/share/test/old_unit_testers/test_shr_file.F90 +A src/share/test/old_unit_testers/test_shr_log.F90 +A src/share/test/old_unit_testers/test_shr_mpi.F90 +A src/share/test/old_unit_testers/test_shr_orb.F90 +A src/share/test/old_unit_testers/test_shr_scam.F90 +A src/share/test/old_unit_testers/test_shr_streams.F90 +A src/share/test/old_unit_testers/test_shr_sys.F90 +A src/share/test/old_unit_testers/test_shr_tInterp.F90 +A src/share/test/unit/CMakeLists.txt +A src/share/test/unit/dynamic_vector/CMakeLists.txt +A src/share/test/unit/dynamic_vector/character16_vector_tests.pf.in +A src/share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc +A src/share/test/unit/dynamic_vector/dynamic_vector_character16.F90 +A src/share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90 +A src/share/test/unit/dynamic_vector/dynamic_vector_integer.F90 +A src/share/test/unit/dynamic_vector/dynamic_vector_r8.F90 +A src/share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in +A src/share/test/unit/dynamic_vector/integer_vector_tests.pf.in +A src/share/test/unit/dynamic_vector/ptr_wrapper.F90 +A src/share/test/unit/dynamic_vector/r8_vector_tests.pf.in +A src/share/test/unit/mock/CMakeLists.txt +A src/share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90 +A src/share/test/unit/shr_assert_test/CMakeLists.txt +A src/share/test/unit/shr_assert_test/test_assert.pf +A src/share/test/unit/shr_assert_test/test_assert_array.pf +A src/share/test/unit/shr_assert_test/test_macro.pf +A src/share/test/unit/shr_assert_test/test_ndebug.pf +A src/share/test/unit/shr_infnan_test/CMakeLists.txt +A src/share/test/unit/shr_infnan_test/test_infnan.F90 +A src/share/test/unit/shr_log_test/CMakeLists.txt +A src/share/test/unit/shr_log_test/test_error_printers.pf +A src/share/test/unit/shr_precip_test/CMakeLists.txt +A src/share/test/unit/shr_precip_test/test_shr_precip.pf +A src/share/test/unit/shr_spfn_test/CMakeLists.txt +A src/share/test/unit/shr_spfn_test/test_erf_r4.pf +A src/share/test/unit/shr_spfn_test/test_erf_r8.pf +A src/share/test/unit/shr_spfn_test/test_gamma_factorial.pf +A src/share/test/unit/shr_spfn_test/test_igamma.pf +A src/share/test/unit/shr_strconvert_test/CMakeLists.txt +A src/share/test/unit/shr_strconvert_test/test_toString.pf +A src/share/test/unit/shr_string_test/CMakeLists.txt +A src/share/test/unit/shr_string_test/test_shr_string.pf +A src/share/test/unit/shr_vmath_test/CMakeLists.txt +A src/share/test/unit/shr_vmath_test/test_vmath.F90 +A src/share/test/unit/shr_wv_sat_test/CMakeLists.txt +A src/share/test/unit/shr_wv_sat_test/test_wv_sat.pf +A src/share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf +A src/share/timing/CMakeLists.txt +A src/share/timing/COPYING +A src/share/timing/ChangeLog +A src/share/timing/GPTLget_memusage.c +A src/share/timing/GPTLprint_memusage.c +A src/share/timing/GPTLutil.c +A src/share/timing/Makefile +A src/share/timing/README +A src/share/timing/f_wrappers.c +A src/share/timing/gptl.c +A src/share/timing/gptl.h +A src/share/timing/gptl.inc +A src/share/timing/gptl_papi.c +A src/share/timing/perf_mod.F90 +A src/share/timing/perf_utils.F90 +A src/share/timing/private.h +A src/share/unit_test_stubs/README +A src/share/unit_test_stubs/pio/CMakeLists.txt +A src/share/unit_test_stubs/pio/README +A src/share/unit_test_stubs/pio/pio.F90.in +A src/share/util/CMakeLists.txt +A src/share/util/mct_mod.F90 +A src/share/util/shr_assert_mod.F90.in +A src/share/util/shr_cal_mod.F90 +A src/share/util/shr_const_mod.F90 +A src/share/util/shr_dmodel_mod.F90 +A src/share/util/shr_file_mod.F90 +A src/share/util/shr_flux_mod.F90 +A src/share/util/shr_frz_mod.F90.in +A src/share/util/shr_infnan_mod.F90.in +A src/share/util/shr_kind_mod.F90 +A src/share/util/shr_log_mod.F90 +A src/share/util/shr_map_mod.F90 +A src/share/util/shr_mct_mod.F90 +A src/share/util/shr_mem_mod.F90 +A src/share/util/shr_mpi_mod.F90 +A src/share/util/shr_msg_mod.F90 +A src/share/util/shr_ncread_mod.F90 +A src/share/util/shr_nl_mod.F90 +A src/share/util/shr_orb_mod.F90 +A src/share/util/shr_pcdf_mod.F90 +A src/share/util/shr_pio_mod.F90 +A src/share/util/shr_precip_mod.F90 +A src/share/util/shr_reprosum_mod.F90 +A src/share/util/shr_reprosumx86.c +A src/share/util/shr_scam_mod.F90 +A src/share/util/shr_spfn_mod.F90 +A src/share/util/shr_strconvert_mod.F90 +A src/share/util/shr_strdata_mod.F90 +A src/share/util/shr_stream_mod.F90 +A src/share/util/shr_string_mod.F90 +A src/share/util/shr_sys_mod.F90 +A src/share/util/shr_tInterp_mod.F90 +A src/share/util/shr_timer_mod.F90 +A src/share/util/shr_vmath_mod.F90 +A src/share/util/shr_wv_sat_mod.F90 +A src/share/util/water_isotopes.F90 +A src/share/util/water_types.F90 +D tests/unit/test_names.c +M tools/configure +M tools/cprnc/CMakeLists.txt +M tools/cprnc/Makefile +M tools/unit_testing/run_tests.py +D utils/data_assimilation/da_no_data_mod.sh +M utils/perl5lib/compilers_translation_tool.pl +D utils/python/CIME/BuildTools/__init__.py +D utils/python/CIME/BuildTools/cmakemacroswriter.py +D utils/python/CIME/BuildTools/configure.py +D utils/python/CIME/BuildTools/macroconditiontree.py +D utils/python/CIME/BuildTools/macrowriterbase.py +D utils/python/CIME/BuildTools/makemacroswriter.py +D utils/python/CIME/BuildTools/possiblevalues.py +D utils/python/CIME/BuildTools/valuesetting.py +D utils/python/CIME/SystemTests/README +D utils/python/CIME/SystemTests/__init__.py +D utils/python/CIME/SystemTests/dae.py +D utils/python/CIME/SystemTests/eri.py +D utils/python/CIME/SystemTests/erio.py +D utils/python/CIME/SystemTests/erp.py +D utils/python/CIME/SystemTests/err.py +D utils/python/CIME/SystemTests/ers.py +D utils/python/CIME/SystemTests/ert.py +D utils/python/CIME/SystemTests/homme.py +D utils/python/CIME/SystemTests/icp.py +D utils/python/CIME/SystemTests/lii.py +D utils/python/CIME/SystemTests/nck.py +D utils/python/CIME/SystemTests/ncr.py +D utils/python/CIME/SystemTests/pea.py +D utils/python/CIME/SystemTests/pem.py +D utils/python/CIME/SystemTests/pet.py +D utils/python/CIME/SystemTests/pfs.py +D utils/python/CIME/SystemTests/pre.py +D utils/python/CIME/SystemTests/rep.py +D utils/python/CIME/SystemTests/seq.py +D utils/python/CIME/SystemTests/sms.py +D utils/python/CIME/SystemTests/ssp.py +D utils/python/CIME/SystemTests/system_tests_common.py +D utils/python/CIME/SystemTests/system_tests_compare_two.py +D utils/python/CIME/SystemTests/test_utils/__init__.py +D utils/python/CIME/SystemTests/test_utils/user_nl_utils.py +D utils/python/CIME/XML/__init__.py +D utils/python/CIME/XML/archive.py +D utils/python/CIME/XML/batch.py +D utils/python/CIME/XML/compilerblock.py +D utils/python/CIME/XML/compilers.py +D utils/python/CIME/XML/component.py +D utils/python/CIME/XML/compsets.py +D utils/python/CIME/XML/entry_id.py +D utils/python/CIME/XML/env_archive.py +D utils/python/CIME/XML/env_base.py +D utils/python/CIME/XML/env_batch.py +D utils/python/CIME/XML/env_build.py +D utils/python/CIME/XML/env_case.py +D utils/python/CIME/XML/env_mach_pes.py +D utils/python/CIME/XML/env_mach_specific.py +D utils/python/CIME/XML/env_run.py +D utils/python/CIME/XML/env_test.py +D utils/python/CIME/XML/files.py +D utils/python/CIME/XML/generic_xml.py +D utils/python/CIME/XML/grids.py +D utils/python/CIME/XML/headers.py +D utils/python/CIME/XML/lt_archive.py +D utils/python/CIME/XML/machines.py +D utils/python/CIME/XML/namelist_definition.py +D utils/python/CIME/XML/pes.py +D utils/python/CIME/XML/pio.py +D utils/python/CIME/XML/standard_module_setup.py +D utils/python/CIME/XML/testlist.py +D utils/python/CIME/XML/tests.py +D utils/python/CIME/XML/testspec.py +D utils/python/CIME/__init__.py +D utils/python/CIME/bless_test_results.py +D utils/python/CIME/build.py +D utils/python/CIME/buildlib.py +D utils/python/CIME/buildnml.py +D utils/python/CIME/case.py +D utils/python/CIME/case_cmpgen_namelists.py +D utils/python/CIME/case_lt_archive.py +D utils/python/CIME/case_run.py +D utils/python/CIME/case_setup.py +D utils/python/CIME/case_st_archive.py +D utils/python/CIME/case_submit.py +D utils/python/CIME/case_test.py +D utils/python/CIME/check_input_data.py +D utils/python/CIME/check_lockedfiles.py +D utils/python/CIME/code_checker.py +D utils/python/CIME/compare_namelists.py +D utils/python/CIME/compare_test_results.py +D utils/python/CIME/get_timing.py +D utils/python/CIME/hist_utils.py +D utils/python/CIME/namelist.py +D utils/python/CIME/nmlgen.py +D utils/python/CIME/preview_namelists.py +D utils/python/CIME/provenance.py +D utils/python/CIME/simple_compare.py +D utils/python/CIME/test_scheduler.py +D utils/python/CIME/test_status.py +D utils/python/CIME/test_utils.py +D utils/python/CIME/tests/README +D utils/python/CIME/tests/SystemTests/__init__.py +D utils/python/CIME/tests/SystemTests/test_system_tests_compare_two.py +D utils/python/CIME/tests/SystemTests/test_system_tests_compare_two_link_to_case2_output.py +D utils/python/CIME/tests/SystemTests/test_utils/__init__.py +D utils/python/CIME/tests/SystemTests/test_utils/test_user_nl_utils.py +D utils/python/CIME/tests/__init__.py +D utils/python/CIME/tests/case_fake.py +D utils/python/CIME/tests/test_case_fake.py +D utils/python/CIME/user_mod_support.py +D utils/python/CIME/utils.py +D utils/python/CIME/wait_for_tests.py +D utils/python/cs.status.template +D utils/python/cs.submit.template +D utils/python/jenkins_generic_job.py +D utils/python/tests/CMakeLists.txt +D utils/python/tests/CTestConfig.cmake +D utils/python/tests/cpl.hi1.nc.test +D utils/python/tests/cpl.hi2.nc.test +D utils/python/tests/cpl.log.failmemleak.gz +D utils/python/tests/cpl.log.passmemleak.gz +D utils/python/tests/list_tests +D utils/python/tests/scripts_regression_tests.py +D utils/python/tests/user_mods_test1/include_user_mods +D utils/python/tests/user_mods_test1/user_nl_comp +D utils/python/tests/user_mods_test1/user_nl_cpl +D utils/python/tests/user_mods_test2/SourceMods/src.drv/somefile.F90 +D utils/python/tests/user_mods_test2/user_nl_cpl +D utils/python/update_acme_tests.py + +====================================================================== + +====================================================================== + Originator: Chris Fischer Date: 2-23-2017 Tag: cime5.2.0-alpha.26 diff --git a/LICENSE.TXT b/LICENSE.TXT index d832be2be93..6d0ae1a9a2e 100644 --- a/LICENSE.TXT +++ b/LICENSE.TXT @@ -1,9 +1,15 @@ Copyright (c) 2015, University Corporation for Atmospheric Research (UCAR) All rights reserved. + and +Copyright (c) 2017, Sandia Corporation. +Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, +the U.S. Government retains certain rights in this software. Developed by: University Corporation for Atmospheric Research - National Center for Atmospheric Research https://www2.cesm.ucar.edu/working-groups/sewg + and + DOE BER ACME project team members, including those at SNL and ANL Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), @@ -17,7 +23,7 @@ the Software is furnished to do so, subject to the following conditions: - Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimers in the documentation and/or other materials provided with the distribution. - - Neither the names of [Name of Development Group, UCAR], + - Neither the names of UCAR or Sandia Corporation, nor the names of its contributors may be used to endorse or promote products derived from this Software without specific prior written permission. diff --git a/README b/README deleted file mode 100644 index ec161702495..00000000000 --- a/README +++ /dev/null @@ -1,31 +0,0 @@ -The cime directory structure (other than the externals/ directory) -was generated from the following trunk tags: - -cime/driver_cpl https://svn-ccsm-models.cgd.ucar.edu/drv/seq_mct/trunk_tags/drvseq5_1_15 - -cime/components/data_comps/datm https://svn-ccsm-models.cgd.ucar.edu/datm7/trunk_tags/datm8_150310 -cime/components/data_comps/dice https://svn-ccsm-models.cgd.ucar.edu/dice7/trunk_tags/dice8_150310 -cime/components/data_comps/dlnd https://svn-ccsm-models.cgd.ucar.edu/dlnd7/trunk_tags/dlnd8_150310 -cime/components/data_comps/docn https://svn-ccsm-models.cgd.ucar.edu/docn7/trunk_tags/docn8_150310 -cime/components/data_comps/drof https://svn-ccsm-models.cgd.ucar.edu/drof/trunk_tags/drof_150310 -cime/components/stub_comps https://svn-ccsm-models.cgd.ucar.edu/stubs/trunk_tags/stubs1_4_08 -cime/components/xcpl_comps https://svn-ccsm-models.cgd.ucar.edu/dead7/trunk_tags/dead7_8_04 - -cime/machines https://svn-ccsm-models.cgd.ucar.edu/Machines/trunk_tags/Machines_150309 - -cime/scripts https://svn-ccsm-models.cgd.ucar.edu/scripts/trunk_tags/scripts_150309 - -cime/share/csm_share https://svn-ccsm-models.cgd.ucar.edu/csm_share/trunk_tags/share3_150116 -cime/share/esmf_wrf_timemgr https://svn-ccsm-models.cgd.ucar.edu/esmf_wrf_timemgr/trunk_tags/esmf_wrf_timemgr_141217 -cime/share/timing https://svn-ccsm-models.cgd.ucar.edu/timing/trunk_tags/timing_150302 - -cime/utils/pythonlib https://svn-ccsm-models.cgd.ucar.edu/scripts/trunk_tags/scripts4_150204/scripts/ccsm_utils/Tools/pythonlib -cime/utils/perl5lib https://svn-ccsm-models.cgd.ucar.edu/perl5lib/trunk_tags/perl5lib_150302 - -cime/tools/load_balancing_tool https://svn-ccsm-models.cgd.ucar.edu/tools/load_balancing_tool/trunk_tags/load_balancing_tool_141008 -cime/tools/unit_testing https://svn-ccsm-models.cgd.ucar.edu/unit_testing/trunk_tags/unit_testing_0_16 -cime/tools/statistical_ensemble_test https://svn-ccsm-models.cgd.ucar.edu/validation_testing/trunk_tags/validation_20140708/run_CESM/ -cime/tools/mapping https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106 -cime/tools/cprnc https://svn-ccsm-models.cgd.ucar.edu/tools/cprnc/trunk_tags/cprnc_150301 - - diff --git a/README.md b/README.md index 73e19ea18d8..05a643e2f47 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,16 @@ Common Infrastructure for Modeling the Earth **cime** (pronounced: seem) represents the infrastructure code for the Community Earth System Model - (CESM) and the Accelerated Climate Model for Energy (ACME). -*cime* includes scripts for configuration, build, and testing of -models, as well as data and stub models for climate components, -and a code for coupling different climate components together. - + (CESM) and the +Accelerated Climate Model for Energy (ACME). +*cime* providess scripts for configuration, build, and testing of +models, as well as code for data and stub models for climate components, +and a driver code for bringing all the climate components together in a single executable. + +# Documentation + +See esmci.github.io/cime + # Developers *cime* was initially extracted from CESM as a stand-alone capability in 2015 @@ -28,18 +33,12 @@ Mariana Vertenstein | NCAR | 1 - D | NSF, DOE Jim Edwards | NCAR | 1 - D | NSF (1-D), DOE(1-2) Jim Foucar | SNL | 5 - D | DOE Rob Jacob | ANL | 5 - D | DOE +Bill Sacks | NCAR | 1 - D | NSF Andreas Wilke | ANL | 5 - D | DOE Jason Sarich | ANL | 5 - D | DOE -??Sean Santos | NCAR | 1 - 4 | NSF +Michael Deakin | SNL | 5 - D | DOE +Erich Foster | SNL | 5 - D | DOE +Sean Santos | NCAR | 1 - 4 | NSF _Key: D = Current development version (i.e. still active on project)_ -# Documentation - -Currently, this is a work in progress. - - - - - - diff --git a/README.unit_testing b/README.unit_testing deleted file mode 100644 index 7b2a3d1983e..00000000000 --- a/README.unit_testing +++ /dev/null @@ -1,16 +0,0 @@ -# To run all CIME unit tests on caldera, run the following command: -# (Note that this must be done from an interactive caldera session, not from yellowstone) -# Note also that this requires module load all-python-libs -# -# The creation of a temporary directory ensures that you are doing a completely -# clean build of the unit tests. (The use of the --clean flag to run_tests.py -# cleans most, but not all of the files created by the unit test build.) For -# rerunning the tests after an incremental change, you can instead use an -# existing build directory. -# -# We would encourage you to port these tests to other platforms. -# The test requires an install of pFunit available from -# https://sourceforge.net/projects/pfunit/ -# - -tools/unit_testing/run_tests.py --test-spec-dir=. --compiler=intel --mpilib=mpich2 --use-openmp --mpirun-command=mpirun.lsf --build-dir `mktemp -d --tmpdir=. unit_tests.XXXXXXXX` diff --git a/cime_config/acme/allactive/config_compsets.xml.cime2 b/cime_config/acme/allactive/config_compsets.xml.cime2 deleted file mode 100644 index 1a902e21f71..00000000000 --- a/cime_config/acme/allactive/config_compsets.xml.cime2 +++ /dev/null @@ -1,2355 +0,0 @@ - - - - - - - -========================================= -COMPSET naming convention -========================================= -The compset longname has the specified order - atm, lnd, ice, ocn, river, glc wave cesm-options -The notation for the compset longname is - TIME_ATM[%phys]_LND[%phys]_ICE[%phys]_OCN[%phys]_ROF[%phys]_GLC[%phys]_WAV[%phys][_BGC%phys] -Where - TIME = Time period (e.g. 2000, 20TR, RCP8...) - ATM = [CAM4, CAM5, DATM, SATM, XATM] - LND = [CLM40, CLM45, CLM50, DLND, SLND, XLND] - ICE = [CICE, DICE, SICE, SICE, MPASCICE] - OCN = [POP, DOCN, SOCN, XOCN,AQUAP,MPASO] - ROF = [RTM, MOSART, DROF, SROF, XROF] - GLC = [CISM1, CISM2S, CISM2P, MPASLISIA, MPASLIALB, SGLC, XGLC] - WAV = [WW3, DWAV, SWAV, XWAV] - BGC = optional BGC scenario -The OPTIONAL %phys attributes specify submodes of the given system - For example DOCN%DOM is the data ocean model for DOCN - ALL the possible %phys choices for each component are listed - with the -list command for create_newcase - ALL data models must have a %phys option that corresponds to the data - model mode -Each compset is associated with five names - - longname (see above) - - shortname (for backwards compatibility) - - alias (even shorter notation than shortname) (optional) - - grid (requirements for grid to work with the compset) (regular expression match) (optional) - - support_level (description of the support level for this compset) (optional) - -========================================= -GRID naming convention -========================================= -The long grid name has the order atm,lnd,ocn/ice,river,mask,glc -The following shortname is used - a% => atm, l% => lnd, oi% => ocn/ice, r% => river, m% => mask, g% => glc, w% => wave -The notation is - a%name_l%name_oi%name_r%name_m%mask_g%name_w%name -Each grid is associated with three names - - longname (a%name_l%name_oi%name_r%name_m%mask_g%name_w%name) - - shortname (for backwards compatibility) - - alias (even shorter notation than shortname) - - - - - - - - - -2000_CAM5%AV1C-L_CLM45%SPBC_MPASCICE_MPASO_MOSART_SGLC_SWAV -2000_CAM5%AV1C-L_CLM45%SPBC_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAV -1850_CAM5%AV1C-L_CLM45%SPBC_MPASCICE_MPASO_MOSART_SGLC_SWAV -1850_CAM5%AV1C-L_CLM45%SPBC_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAV -20TR_CAM5%AV1C-L_CLM45%SPBC_MPASCICE_MPASO_MOSART_SGLC_SWAV -20TR_CAM5%AV1C-L_CLM45%SPBC_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAV - - -1850_CAM5_CLM45%SP_MPASCICE_MPASO_MOSART_SGLC_SWAV -2000_CAM5_CLM45%SP_MPASCICE_MPASO_MOSART_SGLC_SWAV - -2000_CAM5%AV1C-L_CLM45%SPBC_MPASCICE_MPASO_RTM_MPASLIALB_SWAV - - -2000_DATM%NYF_DLND%NULL_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV -2000_DATM%IAF_SLND_DICE%SIAF_DOCN%IAF_DROF%IAF_SGLC_SWAV -2000_DATM%WW3_DLND%NULL_DICE%COPY_DOCN%COPY_DROF%NULL_SGLC_WW3 - - - -2000_CAM4_CLM40%SP_CICE_POP2_RTM_SGLC_SWAV -2000_CAM4_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -2000_CAM5_CLM40%SP_CICE_POP2_RTM_SGLC_SWAV -2000_CAM5_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -2000_CAM5_CLM45%BGC_CICE_POP2_RTM_SGLC_SWAV -2000_CAM4%FCHM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -2000_CAM4%TMOZ_CLM40%SP_CICE_POP2_RTM_SGLC_SWAV -2000_CAM4%MOZS_CLM40%SP_CICE_POP2_RTM_SGLC_SWAV -2000_CAM5%MOZM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -2000_CAM5%SMA3_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -2000_CAM5%SMA7_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -2000_CAM4%WCCM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV - -1850_CAM4_CLM40%SP_CICE_POP2_RTM_SGLC_SWAV -1850_CAM5_CLM40%SP_CICE_POP2_RTM_SGLC_SWAV -1850_CAM5%UNI_CLM40%SP_CICE_POP2_RTM_SGLC_SWAV -1850_CAM5_CLM40%SP_CICE_POP2_MOSART_SGLC_SWAV -1850_CAM4_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -1850_CAM4_CLM40%CN_CICE_POP2_MOSART_SGLC_SWAV -1850_CAM5_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -1850_CAM5_CLM45%BGC_CICE_POP2_RTM_SGLC_SWAV -1850_CAM4%WCCM_CLM40%SP_CICE_POP2_RTM_SGLC_SWAV -1850_CAM4%FCHM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -1850_CAM4%WCCM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -1850_CAM4%WCSC_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -1850_CAM4%RCO2_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -1850_CAM5%WCCM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV - -1850_CAM4_CLM45%CN_CICE_POP2_RTM_SGLC_SWAV - -1850_CAM4_CLM40%CN_CICE_POP2%ECO_RTM_SGLC_SWAV_BGC%BPRP -1850_CAM4_CLM40%CN_CICE_POP2%ECO_RTM_SGLC_SWAV_BGC%BDRD - -20TR_CAM4_CLM40%SP_CICE_POP2_RTM_SGLC_SWAV -20TR_CAM5_CLM40%SP_CICE_POP2_RTM_SGLC_SWAV -20TR_CAM4_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -20TR_CAM5_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -20TR_CAM5_CLM45%BGC_CICE_POP2_RTM_SGLC_SWAV -20TR_CAM4%FCHM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -20TR_CAM4%WCCM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -5505_CAM4%WCCM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -5505_CAM4%WCSC_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV - -20TR_CAM4_CLM40%CN_CICE_POP2%ECO_RTM_SGLC_SWAV_BGC%BPRP - -1850_CAM4_CLM45%BGC_CICE_POP2%ECO_RTM_SGLC_SWAV_BGC%BPRP -1850_CAM5_CLM45%BGC_CICE_POP2%ECO_RTM_SGLC_SWAV_BGC%BPRP - -20TR_CAM4_CLM40%CN_CICE_POP2%ECO_RTM_SGLC_SWAV_BGC%BDRD - -RCP2_CAM4_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP4_CAM4_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP4_CAM4_CLM40%CN_CICE_POP2%ECO_RTM_SGLC_SWAV_BGC%BDRD -RCP6_CAM4_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP8_CAM4_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP8_CAM4_CLM40%CN_CICE_POP2%ECO_RTM_SGLC_SWAV_BGC%BDRD -RCP8_CAM4_CLM40%CN_CICE_POP2%ECO_RTM_SGLC_SWAV_BGC%BPRP -RCP2_CAM5_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP4_CAM5_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP6_CAM5_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP8_CAM5_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP8_CAM5_CLM45%BGC_CICE_POP2_RTM_SGLC_SWAV -RCP2_CAM4%WCCM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP4_CAM4%WCCM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP8_CAM4%WCCM_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP2_CAM4%WCSC_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP4_CAM4%WCSC_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV -RCP8_CAM4%WCSC_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV - -2013_CAM4%WCBC_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV - - -1850_CAM5_CLM45%SP_MPASCICE_MPASO_RTM_SGLC_SWAV -1850_CAM5_CLM45%CN_MPASCICE_MPASO_RTM_SGLC_SWAV -1850_CAM5_CLM45%BGC_MPASCICE_MPASO_RTM_SGLC_SWAV - -2000_CAM5_CLM45%SP_MPASCICE_MPASO_RTM_SGLC_SWAV -2000_CAM5_CLM45%CN_MPASCICE_MPASO_RTM_SGLC_SWAV -2000_CAM5_CLM45%BGC_MPASCICE_MPASO_RTM_SGLC_SWAV - -2000_CAM5%ATMMOD_CLM45%SPBC_MPASCICE_MPASO_RTM_SGLC_SWAV - - - -2000_CAM4_CLM40%CN_CICE_POP2_RTM_CISM1_SWAV -2000_CAM4_CLM40%CN_CICE_POP2_RTM_CISM2P_SWAV - -1850_CAM4_CLM40%CN_CICE_POP2_RTM_CISM1_SWAV -20TR_CAM4_CLM40%CN_CICE_POP2_RTM_CISM1_SWAV -RCP2_CAM4_CLM40%CN_CICE_POP2_RTM_CISM1_SWAV -RCP4_CAM4_CLM40%CN_CICE_POP2_RTM_CISM1_SWAV -RCP6_CAM4_CLM40%CN_CICE_POP2_RTM_CISM1_SWAV -RCP8_CAM4_CLM40%CN_CICE_POP2_RTM_CISM1_SWAV - - -1850_CAM5_CLM45%CN_MPASCICE_MPASO_RTM_MPASLI_SWAV - - - -2000_DATM%NYF_SLND_DICE%SSMI_POP2_DROF%NYF_SGLC_SWAV -2000_DATM%IAF_SLND_DICE%SIAF_POP2_DROF%IAF_SGLC_SWAV -1850_DATM%NYF_SLND_DICE%SSMI_POP2%ECO_DROF%NYF_SGLC_SWAV - - -2000_DATM%NYF_SLND_DICE%SSMI_MPASO_DROF%NYF_SGLC_SWAV -2000_DATM%IAF_SLND_DICE%SIAF_MPASO_DROF%IAF_SGLC_SWAV - - -2000_DATM%NYF_SLND_CICE_DOCN%SOM_DROF%NYF_SGLC_SWAV -2000_DATM%IAF_SLND_CICE_DOCN%SOM_DROF%IAF_SGLC_SWAV -2000_DATM%NYF_SLND_CICE_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST - - -2000_DATM%NYF_SLND_MPASCICE_DOCN%SOM_DROF%NYF_SGLC_SWAV_TEST - - - -2000_CAM4_CLM40%SP_CICE_DOCN%SOM_RTM_SGLC_SWAV_TEST -1850_CAM5_CLM40%SP_CICE_DOCN%SOM_RTM_SGLC_SWAV -1850_CAM5_CLM40%SP_CICE_DOCN%SOM_RTM_SGLC_SWAV_TEST -1850_CAM4_CLM40%CN_CICE_DOCN%SOM_RTM_SGLC_SWAV -1850_CAM5_CLM40%CN_CICE_DOCN%SOM_RTM_SGLC_SWAV -1850_CAM5_CLM40%CN_CICE_DOCN%SOM_RTM_SGLC_SWAV_TEST -1850_CAM5_CLM45%SP_CICE_DOCN%SOM_RTM_SGLC_SWAV -1850_CAM5_CLM45%CN_CICE_DOCN%SOM_RTM_SGLC_SWAV -1850_CAM5_CLM45%BGC_CICE_DOCN%SOM_RTM_SGLC_SWAV - - - -1850_CAM4_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM5_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM5_CLM45%BGC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM4_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%MAM4_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%MAM4%PLMOD_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%MAM4%RESUS_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%ATMMOD_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1C_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1C-00_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM5%AV1C-01_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM5%AV1C-02_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM5%AV1C-03_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM5%AV1C-04_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM5%AV1C-04P_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM5%AV1C-04P2_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM5%AV1C-L_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1C-01_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1C-02_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1C-03_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1C-04_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1C-04P_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1C-04P2_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1C-L_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1F_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1F-00_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%AV1F-01_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -20TR_CAM5%AV1C-03_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -20TR_CAM5%AV1C-04_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -20TR_CAM5%AV1C-04P_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -20TR_CAM5%AV1C-04P2_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -20TR_CAM5%AV1C-L_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%ATMMODCOSP_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%COSP_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%UNI_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -AMIP_CAM4_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -AMIP_CAM5_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -AMIP_CAM5%UNI_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -AMIP_CAM5%L60_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -PIPD_CAM5_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -20TR_CAM5_CLM45%BGC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM4_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5_CLM45%BGC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -AMIP_CAM4_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -AMIP_CAM5_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -20TR_CAM4_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLB_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%MG2_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2L72_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2L72_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2LINMAM4RESUS_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2LINMAM4RESUSMOM_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2MAM4MOM_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2MAM4_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2MAM4RESUS_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2MAM4RESUSMOM_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2MAM4RESUSCOSP_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2MAM4RESUSMOMCOSP_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2MAM4RESUS_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2MAM4RESUSMOM_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2MAM4RESUSCOSP_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2MAM4RESUSMOMCOSP_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%CLBMG2MAM4RESUSMOMCOSP_CLM45%SPBC_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM5%PM_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%PM_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM4%WCCM_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM4%TMOZ_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM4%WCCM_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM4%WCSC_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM4%WCMX_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1996_CAM4%WCMX_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%WCCM_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM4%MOZS_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM4%SSOA_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -1850_CAM4%FCHM_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%MOZM_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%SMA3_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -2000_CAM5%SMA7_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -5505_CAM4%WCCM_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -GEOS_CAM4%WCCM_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -GEOS_CAM4%WCSF_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -GEOS_CAM4%TMOZ_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -GEOS_CAM4%TBAM_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -GEOS_CAM4%SSOA_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -GEOS_CAM5%SMA3_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -AR95_CAM4%SCAM_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV -AR97_CAM5%SCAM_CLM40%SP_CICE%PRES_DOCN%DOM_RTM_SGLC_SWAV - -2000_CAM4_SLND_SICE_AQUAP_SROF_SGLC_SWAV -2000_CAM5_SLND_SICE_AQUAP_SROF_SGLC_SWAV -2000_CAM%IDEAL_SLND_SICE_SOCN_SROF_SGLC_SWAV -2000_CAM%ADIAB_SLND_SICE_SOCN_SROF_SGLC_SWAV - - - - -2000_CAM4_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_CISM1_SWAV -1850_CAM4_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_CISM1_SWAV -2000_CAM4_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_CISM2P_SWAV -20TR_CAM4_CLM40%CN_CICE%PRES_DOCN%DOM_RTM_CISM1_SWAV - - - -2000_DATM%NYF_SLND_CICE_POP2_DROF%NYF_SGLC_SWAV -2000_DATM%NYF_SLND_MPASCICE_MPASO_DROF%NYF_SGLC_SWAV -2000_DATM%IAF_SLND_MPASCICE_MPASO_DROF%IAF_SGLC_SWAV -2000_DATM%IAF_SLND_CICE_POP2_DROF%IAF_SGLC_SWAV -1850_DATM%NYF_SLND_CICE_POP2%ECO_DROF%NYF_SGLC_SWAV - - - -2000_CAM4_CLM40.*_DICE%PRES_DOCN%DOM_SROF_SGLC_SWAV - - - - - -2000_DATM%1PT_CLM40%SP_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%1PT_CLM45%SP_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%1PT_CLM50%SP_SICE_SOCN_RTM_SGLC_SWAV - - - -2003_DATM%QIA_CLM40%SP_SICE_SOCN_RTM_SGLC_SWAV_TEST -2003_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_SGLC_SWAV_TEST -2003_DATM%QIA_CLM45%CN_SICE_SOCN_RTM_SGLC_SWAV_TEST -1850_DATM%QIA_CLM40%SP_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%SP_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%SP_SICE_SOCN_MOSART_SGLC_SWAV -1850_DATM%QIA_CLM45%SPBC_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%SPBC_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%QIA_CLM40%SP_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%SP_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%SP_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%QIA_CLM45%SPBC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%SPBC_SICE_SOCN_MOSART_SGLC_SWAV -4804_DATM%QIA_CLM40%SP_SICE_SOCN_RTM_SGLC_SWAV -4804_DATM%QIA_CLM45%SP_SICE_SOCN_RTM_SGLC_SWAV -20TR_DATM%QIA_CLM40%SP_SICE_SOCN_RTM_SGLC_SWAV -20TR_DATM%QIA_CLM45%SP_SICE_SOCN_RTM_SGLC_SWAV -20TR_DATM%QIA_CLM45%CN_SICE_SOCN_RTM_SGLC_SWAV -20TR_SATM%QIA_CLM45%CN_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM40%CN_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%NLDAS_CLM40%CN_SICE_SOCN_MOSART_SGLC_SWAV -1850_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%CN_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%CN_SICE_SOCN_MOSART_SGLC_SWAV -1850_SATM%QIA_CLM45%CN_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%CN-ED_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%CN_SICE_SOCN_RTM%FLOOD_SGLC_SWAV -20TR_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_SGLC_SWAV -RCP8_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_SGLC_SWAV -RCP8_DATM%QIA_CLM45%CN_SICE_SOCN_RTM_SGLC_SWAV -RCP6_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_SGLC_SWAV -RCP6_DATM%QIA_CLM45%CN_SICE_SOCN_RTM_SGLC_SWAV -RCP4_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_SGLC_SWAV -RCP2_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_SGLC_SWAV -RCP2_DATM%QIA_CLM45%CN_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM40%CN-CROP_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%CN-CROP_SICE_SOCN_RTM_SGLC_SWAV -RCP4_DATM%QIA_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV -RCP8_DATM%QIA_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%CRDCTCBC_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%CNRDCTCBC_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%CNPRDCTCBC_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%CECACTCBC_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%CNPECACTCBC_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%CRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV -1850_DATM%QIA_CLM45%CNRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV -1850_DATM%QIA_CLM45%CNPRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV -1850_DATM%QIA_CLM45%CNECACTCBC_SICE_SOCN_MOSART_SGLC_SWAV -1850_DATM%QIA_CLM45%CNPECACTCBC_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%QIA_CLM45%CRDCTCBC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%CNRDCTCBC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%CNPRDCTCBC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%CNECACTCBC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%CNPECACTCBC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%CRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%QIA_CLM45%CNRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%QIA_CLM45%CNPRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%QIA_CLM45%CNECACTCBC_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%QIA_CLM45%CNPECACTCBC_SICE_SOCN_MOSART_SGLC_SWAV - - - -2000_DATM%CRU_CLM45_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%CRU_CLM40%CN_SICE_SOCN_RTM_SGLC_SWAV - -1850_DATM%CRU_CLM45%CN_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%CRU_CLM45%BGCDV_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%CRU_CLM45%BGC-CROP_SICE_SOCN_RTM_SGLC_SWAV -2003_DATM%CRU_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV_TEST -2000_DATM%CRU_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%CRU_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV -20TR_DATM%CRU_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%CRU_CLM45%CRDCTCBC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%CRU_CLM45%CNRDCTCBC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%CRU_CLM45%CNPRDCTCBC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%CRU_CLM45%CNECACTCBC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%CRU_CLM45%CNPECACTCBC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%CRU_CLM45%CRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%CRU_CLM45%CNRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%CRU_CLM45%CNPRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%CRU_CLM45%CNECACTCBC_SICE_SOCN_MOSART_SGLC_SWAV -2000_DATM%CRU_CLM45%CNPECACTCBC_SICE_SOCN_MOSART_SGLC_SWAV - -2000_DATM%CRU_CLM50%BGC_SICE_SOCN_RTM_SGLC_SWAV - - - -2000_DATM%QIA_CLM40%CNDV-CROP_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%CNDV_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%BGCDV-CROP_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%SP-VIC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%CN-ED_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV - -1850_DATM%QIA_CLM45%CRDCTCBC_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%CRDCTCBC_SICE_SOCN_MOSART_SGLC_SWAV - -2000_DATM%QIA_CLM45%BGC-ED_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%BGC-ED_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%QIA_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV -1850_SATM%QIA_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%QIA_CLM45%BGC-CROP_SICE_SOCN_RTM_SGLC_SWAV -2000_DATM%CRU_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV -20TR_DATM%QIA_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV -20TR_SATM%QIA_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV - - - -1850_DATM%S1850_CLM40%CN_SICE_SOCN_RTM_SGLC_SWAV -1850_DATM%S1850_CLM45%BGC_SICE_SOCN_RTM_SGLC_SWAV - - - -1850_DATM%QIA_CLM40%SP_SICE_SOCN_RTM_CISM1_SWAV -1850_DATM%QIA_CLM45%SP_SICE_SOCN_RTM_CISM1_SWAV -2000_DATM%QIA_CLM40%SP_SICE_SOCN_RTM_CISM1_SWAV -2000_DATM%QIA_CLM40%SP_SICE_SOCN_RTM_CISM2P_SWAV -2000_DATM%QIA_CLM45%SP_SICE_SOCN_RTM_CISM1_SWAV -2000_DATM%QIA_CLM45%SP_SICE_SOCN_RTM_CISM2P_SWAV -20TR_DATM%QIA_CLM40%SP_SICE_SOCN_RTM_CISM1_SWAV -20TR_DATM%QIA_CLM45%SP_SICE_SOCN_RTM_CISM1_SWAV -1850_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_CISM1_SWAV -2000_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_CISM1_SWAV -20TR_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_CISM1_SWAV -RCP8_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_CISM1_SWAV -RCP6_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_CISM1_SWAV -RCP4_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_CISM1_SWAV -RCP2_DATM%QIA_CLM40%CN_SICE_SOCN_RTM_CISM1_SWAV -20TR_DATM%QIA_CLM45%CN_SICE_SOCN_RTM_CISM1_SWAV -RCP8_DATM%QIA_CLM45%CN_SICE_SOCN_RTM_CISM1_SWAV -RCP4_DATM%QIA_CLM45%CN_SICE_SOCN_RTM_CISM1_SWAV - -1850_DATM%QIA_CLM50%SP_SICE_SOCN_RTM_CISM1_SWAV - - - -2000_DATM%QIA_CLM45%CN_SICE_SOCN_RTM_CISM1_SWAV_TEST - - - -2000_DATM%QIA_CLM45%SP_SICE_SOCN_RTM_MPASLI_SWAV - - - -2000_SATM_DLND%SCPL_SICE_SOCN_SROF_CISM1_SWAV -1850_SATM_DLND%SCPL_SICE_SOCN_SROF_CISM1_SWAV -20TR_SATM_DLND%SCPL_SICE_SOCN_SROF_CISM1_SWAV -RCP8_SATM_DLND%SCPL_SICE_SOCN_SROF_CISM1_SWAV - -2000_SATM_DLND%SCPL_SICE_SOCN_SROF_CISM2S_SWAV -2000_SATM_DLND%SCPL_SICE_SOCN_SROF_CISM2P_SWAV -1850_SATM_DLND%SCPL_SICE_SOCN_SROF_CISM2P_SWAV -20TR_SATM_DLND%SCPL_SICE_SOCN_SROF_CISM2P_SWAV -RCP8_SATM_DLND%SCPL_SICE_SOCN_SROF_CISM2P_SWAV - - -2000_SATM_DLND%SCPL_SICE_SOCN_SROF_MPASLI_SWAV - - -RCP8_SATM_SLND_SICE_SOCN_SROF_MPASLISIA_SWAV -RCP8_SATM_SLND_SICE_SOCN_SROF_MPASLIALB_SWAV - - -2000_DATM%NYF_SLND_DICE%SSMI_MPASO_DROF%NYF_MPASLI_SWAV - - -2000_DATM%NYF_SLND_MPASCICE_MPASO_DROF%NYF_MPASLI_SWAV - - - -2000_SATM_SLND_SICE_SOCN_SROF_SGLC_SWAV -2000_XATM_XLND_XICE_XOCN_XROF_XGLC_XWAV - - - -1850_CAM4_CLM40%CN_CICE_POP2%DAR_BGC%BDRD_RTM_SGLC_SWAV -1850_DATM%NYF_SLND_CICE_POP2%DAR_DROF_SGLC_SWAV -2000_DATM%NYF_SLND_CICE_POP2%DAR_DROF_SGLC_SWAV -1850_DATM%NYF_SLND_DICE%NYF_POP2%DAR_DROF_SGLC_SWAV - - - -2000_CAM4_CLM40%SP_CICE_POP2%1D_RTM_SGLC_SWAV -2000_DATM%NYF_SLND_DICE%SSMI_POP2%1D_DROF%NYF_SGLC_SWAV - ---DO NOT USE FOR LONG SIMULATIONS: - - - - - - - - - - - - -MG1.0 w/ modified activation: -UNICON (modified mg1.0): - - - -cam -datm -satm -xatm - -clm -dlnd -slnd -xlnd - -rtm -mosart -drof -srof -xrof - -cice -dice -sice -xice -mpas-cice - -pop -docn -socn -xocn -aquap -mpas-o - -cism -sglc -xglc -mpasli - -ww3 -swav -dwav -xwav - -CAM: -DATM: -SATM: -XATM: - -CLM: -DLND: -SLND: -XLND: - -RTM: -MOSART: -DROF: -SROF: -Xrof: - -CICE: -DICE: -SICE: -XICE: -MPAS-CICE: (Experimental) - -POP2: -DOCN: -SOCN: -XOCN: -AQUAP: -MPAS-O: (Experimental) - -CISM: -SGLC: -XGLC: -MPASLI: (Experimental) - -WW3: -DWAV: -SWAV: -XWAV: - - - - - -1850 to present day: -pre-industrial: -present day: -present day: -1850 to 2000 transient: -1955 to 2005 transient: -RCP8.5 future scenario: -RCP6.0 future scenario: -RCP4.5 future scenario: -RCP2.6 future scenario: -RCP4.5 based scenario from 2013 (control for WACCM/CARMA nuclear winter study): -present day with conditions for solar minimum in 1996: -1992 to 2005 transient: -GEOS5 meteorology: -CLM transient land use: - ------------------------------WARNING ------------------------------------------------ -The compset F_1850-PDAY_CAM5 uses complete atmospheric forcing data from observed sources -up to the year 2005. Following this period it is a combination of observed sources -(SST, sea ice, CO2, CH4, N2O) to present day and IPCC RCP4.5 scenario data. -------------------------------------------------------------------------------------- - - - ------------------------------WARNING ------------------------------------------------ -This compset is not spun-up! In later versions of the model, spun-up initial -conditions will be provided and this warning will be removed. -------------------------------------------------------------------------------------- - - - -========================================= -Time period (first four characters) -========================================= -1850 => pre-industrial -2000 => present day -PDAY => present day -20TR => transient 1850 to 2000 -PIPD => transient pre-industrial to present day -5505 => transient 1955 to 2005 -9205 => transient 1992 to 2005 -RCP8 => transient RCP8.5 future scenario -RCP6 => transient RCP6.0 future scenario -RCP4 => transient RCP4.5 future scenario -RCP2 => transient RCP2.6 future scenario -2013 => RCP4.5 based scenario from 2013 (control for WACCM/CARMA nuclear winter study) -1996 => present day with conditions for solar minimum in 1996 -AMIP => transient for "stand-alone" CAM (1979 startdate) -GEOS => GEOS5 meteorology for "stand-alone" CAM - - - - - - -fv -eul -se - --phys cam4 --phys cam5 --chem trop_mam4 --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom -rain_evap_to_coarse_aero --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom_soag -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom_soag -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom_soag -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom_soag -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom_soag -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom_soag -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom_soag -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem trop_mam4_resus -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem trop_mam4_resus -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem trop_mam4_resus_soag -rain_evap_to_coarse_aero -nlev 72 --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom -rain_evap_to_coarse_aero -cosp --cosp --clubb_sgs --microphys mg2 --clubb_sgs -microphys mg2 --clubb_sgs -microphys mg2 -nlev 72 --clubb_sgs -microphys mg2 -nlev 72 --clubb_sgs -microphys mg2 -chem linoz_mam4_resus -rain_evap_to_coarse_aero --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom -rain_evap_to_coarse_aero --clubb_sgs -microphys mg2 -chem trop_mam4 --clubb_sgs -microphys mg2 -chem trop_mam4_mom --clubb_sgs -microphys mg2 -chem trop_mam4 -rain_evap_to_coarse_aero --clubb_sgs -microphys mg2 -chem trop_mam4_resus_mom -rain_evap_to_coarse_aero --clubb_sgs -microphys mg2 -chem trop_mam4 -rain_evap_to_coarse_aero -cosp --clubb_sgs -microphys mg2 -chem trop_mam4_resus_mom -rain_evap_to_coarse_aero -cosp --clubb_sgs -microphys mg2 -chem linoz_mam4_resus_mom -rain_evap_to_coarse_aero -cosp --unicon -cppdefs -DMODIFY_ACTIVATE --chem trop_bam --chem trop_mozart --chem trop_mozart_soa -age_of_air_trcs --chem trop_mozart_mam3 -age_of_air_trcs --chem none --chem trop_strat_mam3 -age_of_air_trcs --chem trop_strat_mam7 -age_of_air_trcs --chem trop_strat_soa -age_of_air_trcs - - - --chem waccm_mozart --chem waccm_mozart_mam3 --chem waccm_ghg --carma bc_strat --chem waccm_mozart_sulfur -carma sulfate --waccmx - --offline_dyn --nlev 56 --nlev 56 --nlev 88 - --chem super_fast_llnl --chem super_fast_llnl -age_of_air_trcs --co2_cycle --dyn eul -scam - --nlev 60 - --phys ideal --phys adiabatic --ocn aquaplanet - --bc_dep_to_snow_updates - - - - - - - - -2000_cam4_trop_chem -1850_cam4 -1850_cam5 -1850-2005_cam4 -1850-2005_cam5 -1850-2005_cam5 -1850-2005_cam4 -1850-2005_cam4_bgc -2005-2100_cam4_rcp26 -2005-2100_cam4_rcp45 -2005-2100_cam4_rcp45_bgc -2005-2100_cam4_rcp60 -2005-2100_cam4_rcp85 -2005-2100_cam4_rcp85_bgc -2006-2100_cam5_rcp26 -2006-2100_cam5_rcp45 -2006-2100_cam5_rcp60 -2006-2100_cam5_rcp85 -1850_cam4_super_fast_llnl -2000_cam4_super_fast_llnl -1850-2005_cam4_super_fast_llnl -2000_cam4_trop_chem -2000_cam5_trop_moz_mam3 -1850_cam5_pm -2000_cam5_plmod -2000_cam5_plmod -2000_cam5_plmod_plus_resus -2000_cam5_clubb_mg2_l72 -2000_cam5_clubb_mg2_bc_l72 -2000_cam5_linoz_plmod_plus_resus -2000_cam5_atmmod -2000_cam5_av1c -2000_cam5_av1c-00 -1850_cam5_av1c-01 -1850_cam5_av1c-02 -1850_cam5_av1c-03 -1850_cam5_av1c-04 -1850_cam5_av1c-04p -1850_cam5_av1c-04p2 -1850_cam5_av1c-04p2 -2000_cam5_av1c-01 -2000_cam5_av1c-02 -2000_cam5_av1c-03 -2000_cam5_av1c-04 -2000_cam5_av1c-04p -2000_cam5_av1c-04p2 -2000_cam5_av1c-04p2 -2000_cam5_av1f -2000_cam5_av1f-00 -2000_cam5_av1f-01 -20TR_cam5_av1c-03 -20TR_cam5_av1c-04 -20TR_cam5_av1c-04p -20TR_cam5_av1c-04p2 -20TR_cam5_av1c-04p2 -2000_cam5_cosp -waccmx_2000_cam4 -waccmx_1996_cam4 -waccm_1955-2005_cam4 -waccm_sc_1850_cam4 -waccm_sc_1955-2005_cam4 -waccm_sc_2000_cam4 -waccm_sc_2005-2100_cam4_rcp26 -waccm_sc_2005-2100_cam4_rcp45 -waccm_sc_2005-2100_cam4_rcp85 -waccm_1850_cam4 -waccm_2000_cam4 -waccm_1850-2005_cam4 -waccm_2005-2100_cam4_rcp26 -waccm_2005-2100_cam4_rcp45 -waccm_2005-2100_cam4_rcp85 -waccm_1850_cam5 -waccm_2000_cam5 -waccm_carma_bc_2013_cam4 -cam4_chem_radpsv_geos5 -cam4_bam_radpsv_geos5 -sd_waccm_geos5 -sd_waccm_sulfur -soa_chem_megan_emis -2000_cam5_trop_strat_mam3 -2000_cam5_trop_strat_mam7 -2000_cam4_trop_strat_soa -sd_cam5_trop_strat_mam3 -sd_cam4_trop_strat_soa -scam_arm95 -scam_arm97 -1850-PD_cam5 - - - scenario_ghg='RAMP_CO2_ONLY'ramp_co2_annual_rate=1 - - - co2_cycle_rad_passive=.true. - - - - - - - - - - -TRUE -36.6 -262.5 - -cam4 physics: -cam5 physics: -CAM trop_mam4 chemistry: -CAM with COSP turned on: -CAM with all polar mods: -CAM with polar mods, MAM4 and resuspension: -CAM with all ACME atmosphere mods: -CAM with complete set of ACME atmospheric mods for V1 (72 layers model): -CAM with complete set of ACME atmospheric mods for V1 (72 layers model)- v00: -CAM with complete set of ACME atmospheric mods for V1 (72 layers model) and ACES4BGC SOAG emissions- v01: -CAM with complete set of ACME atmospheric mods for V1 (72 layers model) and ACES4BGC SOAG emissions- v02: -CAM with complete set of ACME atmospheric mods for V1 (72 layers model) and ACES4BGC SOAG emissions- v03: -CAM with complete set of ACME atmospheric mods for V1 (72 layers model) and ACES4BGC SOAG emissions- v04: -CAM with complete set of ACME atmospheric mods for V1 (72 layers model) and ACES4BGC SOAG emissions- v04p: -CAM with complete set of ACME atmospheric mods for V1 (72 layers model) and ACES4BGC SOAG emissions- v04p2: -CAM with complete set of ACME atmospheric mods for V1 (72 layers model) and ACES4BGC SOAG emissions- v04p: -CAM with familiar set of ACME atmospheric mods for V1 (72 Layers model): -CAM with familiar set of ACME atmospheric mods for V1 (72 Layers model)- v00: -CAM with familiar set of ACME atmospheric mods for V1 (72 Layers model) and ACES4BGC SOAG emissions- v01: -CAM with all ACME atmosphere mods and COSP: -CAM CLUBB: -CAM MG2: -CAM CLUBB MG2: -CAM CLUBB MG2 with 72 layers: -CAM CLUBB MG2 BC dep on snow with 72 layers: -CAM CLUBB MG2 Linoz2 MAM4 resuspension: -CAM CLUBB MG2 Linoz2 MAM4 resuspension MOM: -CAM CLUBB MG2 MAM4: -CAM CLUBB MG2 MAM4 MOM: -CAM CLUBB MG2 MAM4 resuspension: -CAM CLUBB MG2 MAM4 resuspension MOM: -CAM CLUBB MG2 MAM4 resuspension COSP: -CAM CLUBB MG2 MAM4 resuspension MOM COSP: -CAM CLUBB MG2 Linoz2 MAM4 resuspension MOM COSP: -CAM prescribed modal aerosols: -CAM WACCM with daily solar data and SPEs: -CAM WACCM-X: -CAM WACCM specified chemistry: -CAM WACCM with the stratospheric black carbon CARMA model: -CAM WACCM with sulfur chemistry and the sulfate CARMA model: -CAM super_fast_llnl chemistry: -CAM trop_mozart chemistry: -CAM trop_bam chemistry: -CAM trop_mozart_mam3 chemistry: -CAM trop_mozart_soa chemistry: -CAM trop_strat_mam3 chemistry: -CAM trop_strat_mam7 chemistry: -CAM trop_strat_soa chemistry: -CAM CO2 ramp: -stand-alone single column CAM ARM95 IOP test case: -stand-alone single column CAM ARM97 IOP test case: -CAM winds and temperature driven by GEOS5 meteorology: -CAM WACCM winds and temperature nudged toward GEOS5 meteorology each time step: -CAM 60 layers and full gravity wave spectrum: -CAM adiabatic physics: -CAM ideal physics: - -Atmospheric Model Intercomparison Project protocol: -CAM atmosphere in aquaplanet mode: - - -========================================= -CAM naming conventions in compset name -========================================= -CAM4% => cam4 physics -CAM5% => cam5 physics -CAM[45]%WCCM => CAM WACCM with daily solar data and SPEs: -CAM[45]%WCMX => CAM WACCM-X: -CAM[45]%WCSC => CAM WACCM specified chemistry: -CAM[45]%WCBC => CAM WACCM with the stratospheric black carbon CARMA model: -CAM[45]%WCSF => CAM WACCM with sulfur chemistry and the sulfate CARMA model: -CAM[45]%FCHM => CAM super_fast_llnl chemistry: -CAM[45]%TMOZ => CAM trop_mozart chemistry: -CAM[45]%MOZM => CAM trop_mozart_mam3 chemistry: -CAM[45]%MOZS => CAM trop_mozart_soa chemistry: -CAM[45]%SMA3 => CAM trop_strat_mam3 chemistry: -CAM[45]%SMA7 => CAM trop_strat_mam7 chemistry: -CAM[45]%SSOA => CAM trop_strat_soa chemistry: -CAM[45]%RCO2 => CAM CO2 ramp: -CAM[45]%SCAM => CAM single column mode: -CAM[45]%L60 => CAM with 60 layers and full gravity wave spectrum: - - - - - - --phys clm4_0 --bgc cn --bgc cndv --crop on - --phys clm4_5 --phys clm5_0 - --phys clm4_5 -cppdefs -DMODAL_AER - - - - - - - - -2000_control -2000_control -2000_control -1850-2100_rcp4.5_transient -2000_control -1850_control -20thC_transient -1850-2100_rcp6_transient -1850-2100_rcp4.5_transient -1850-2100_rcp2.6_transient -1850-2100_rcp8.5_transient -1850_glacierMEC_control -2000_glacierMEC_control -2000_glacierMEC_control -2000_glacierMEC_control -2000_glacierMEC_control -20thC_glacierMEC_transient -1850-2100_rcp8.5_glacierMEC_transient -1850-2100_rcp6_glacierMEC_transient -1850-2100_rcp4.5_glacierMEC_transient -1850-2100_rcp2.6_glacierMEC_transient -1850-2100_rcp4.5_glacierMEC_transient -20thC_transient -20thC_transient -glacierMEC_pd - -1850-2100_rcp4.5_transient -1850-2100_rcp4.5_transient -1850-2100_rcp4.5_transient - - --ignore_ic_year --bgc sp --bgc sp -fsnowoptics lnd/clm2/snicardata/snicar_optics_5bnd_mam_c140303.nc --bgc cn --bgc bgc --bgc cn -crop --bgc bgc -crop --bgc cn -dynamic_vegetation --bgc bgc -dynamic_vegetation --bgc cn -dynamic_vegetation -crop --bgc bgc -dynamic_vegetation -crop --bgc cn -ed_mode --bgc bgc -ed_mode --bgc cn -nutrient c -nutrient_comp_pathway rd -soil_decomp ctc -methane -nitrif_denitrif -fsnowoptics lnd/clm2/snicardata/snicar_optics_5bnd_mam_c140303.nc --bgc cn -nutrient cn -nutrient_comp_pathway rd -soil_decomp ctc -methane -nitrif_denitrif -fsnowoptics lnd/clm2/snicardata/snicar_optics_5bnd_mam_c140303.nc --bgc cn -nutrient cnp -nutrient_comp_pathway rd -soil_decomp ctc -methane -nitrif_denitrif -fsnowoptics lnd/clm2/snicardata/snicar_optics_5bnd_mam_c140303.nc --bgc cn -nutrient cn -nutrient_comp_pathway eca -soil_decomp ctc -methane -nitrif_denitrif -fsnowoptics lnd/clm2/snicardata/snicar_optics_5bnd_mam_c140303.nc --bgc cn -nutrient cnp -nutrient_comp_pathway eca -soil_decomp ctc -methane -nitrif_denitrif -fsnowoptics lnd/clm2/snicardata/snicar_optics_5bnd_mam_c140303.nc - --bgc sp --bgc bgc - -constant -diagnostic -diagnostic -prognostic -diagnostic -diagnostic - -on -off -on - - - - - - - - - - - - -clm4.0 physics: -clm4.0 Satellite phenology: -clm4.0 cn: -clm4.0 cndv: -clm4.0 prognostic crop: - -clm4.5 physics: -clm4.5 Satellite phenology: -clm4.5 Satellite phenology with black carbon deposition: -clm4.5 cn: -clm4.5 cn with dynamic vegetation: -clm4.5 bgc (cn and methane): -clm4.5 prognostic crop: -clm4.5 vic hydrology: -clm4.5 ED (Ecosystem Demography): (experimental) -clm4.5 bgc (cn and methane) with dynamic vegetation: -alm1.0 C only, nutirent competition via relative demand, ctc soil cascade with black carbon deposition: -alm1.0 C-N, nutirent competition via relative demand, ctc soil cascade with black carbon deposition: -alm1.0 C-N-P, nutirent competition via relative demand, ctc soil cascade with black carbon deposition: -alm1.0 C-N, nutirent competition via equilibrium chemistry approximation, ctc soil cascade with black carbon deposition: -alm1.0 C-N-P, nutirent competition via equilibrium chemistry approximation, ctc soil cascade with black carbon deposition: - -clm5.0 physics: -clm5.0 Satellite phenology: -clm5.0 bgc (cn and methane): - -========================================= -CLM naming conventions in compset name -========================================= -note: [^_]* means match zero or more of any character BUT an underbar. - (in other words make sure there is NOT a underbar before the string afterwards) - -CLM40 => clm4.0 Physics -CLM40%[^_]*SP => clm4.0 Satellite phenology -CLM40%[^_]*CN => clm4.0 Carbon Nitrogen -CLM40%[^_]*CNDV => clm4.0 Carbon Nitrogen Dynamic Vegetation -CLM40%[^_]*CROP => clm4.0 Prognostic crop - -CLM45 => clm4.5 Physics -CLM45%[^_]*SP => clm4.5 Satellite phenology -CLM45%[^_]*BC => clm4.5 Black carbon deposition scheme updates turned on -CLM45%[^_]*CN => clm4.5 Carbon Nitrogen Biogeochemistry (BGC) (as in CLM4.0) -CLM45%[^_]*CNDV => clm4.5 Carbon Nitrogen BGC with Dynamic Vegetation -CLM45%[^_]*BGC => clm4.5 BGC (CN with vertically resolved soil BGC, based on Century with Methane) -CLM45%[^_]*CROP => clm4.5 Prognostic crop -CLM45%[^_]*VIC => clm4.5 VIC hydrology -CLM45%[^_]*ED => clm4.5 Ecosystem Demography (experimental) -CLM45%[^_]*BGCDV => clm4.5 BGC (CN with vertically resolved soil BGC, based on Century with Methane) with dynamic veg - -CLM50 => clm5.0 Physics -CLM50%[^_]*SP => clm5.0 Satellite phenology -CLM50%[^_]*BGC => clm5.0 BGC (CN with vertically resolved soil BGC, based on Century with Methane) - - - - - - -cold_start -spunup - - - - - - - - -spunup mpas-cice: -cold_start mpas-cice: - - -========================================= -MPASCICE naming conventions in compset name -========================================= -MPASCICE => use cold start initial conditions for MPASCICE -MPASCICE%SPUNUP => use spunup initial conditions for MPASCICE - - - - - - -prognostic -prescribed - --bc_dep_to_snow_updates - -cam5=.true. - - - - - - - - - -prescribed cice: -prognostic cice: - - -========================================= -CICE naming conventions in compset name -========================================= -CICE => prognostic cice -CICE%PRES => prescribed cice - - - - - - - -TRUE - - -iage -cfc -cfc -ecosys -$OCN_SUBMODEL -iage $OCN_SUBMODEL - - - - - - - - -darwin -darwin - -moby -moby - -full -partial - -1850-2000 -1850-2000 -rcp4.5 -rcp8.5 - -inactive -active - -prognostic -diagnostic -diagnostic - - -constant -prognostic -diagnostic -constant - -TRUE - - - - - - - - - - - - - - - -POP2 default: -Single column POP2: -POP2/Ecosystem: -Darwin marine ecosystem: (requires additional MOBY code) - - -========================================= -POP2 naming conventions in compset name -========================================= -POP2 => POP2 default -POP2%1D => POP2/Single column -POP2%ECO => POP2/Ecosystem -POP2%DAR => Darwin marine ecosystem (not supported in community releases) - - - - - - -ACTIVE -NULL -ACTIVE - --simyr 2000 --simyr 1850 --simyr 1850 --simyr 1850 --simyr 2000 - -rtm_effvel='ACTIVE' -rtm_effvel='ACTIVE' - - - - - - - - - -RTM: -RTM model with flood: - - -========================================= -RTM naming conventions in compset name -========================================= -RTM => default RTM model -RTM%FLOOD => RTM model with flood - - - - - - -ACTIVE -NULL -ACTIVE - - - - - - - - - -MOSART: -MOSART model with flood: - - -========================================= -MOSART naming conventions in compset name -========================================= -MOSART => default MOSART model -MOSART%FLOOD => MOSART model with flood - - - - - - -cism1 -cism2 - -FALSE -TRUE -FALSE - -10 -10 - -TRUE -TRUE - -TRUE - -cism1: -cism2 (parallel): -cism2 (serial): - - -========================================= -CISM naming conventions in compset name -========================================= -CISM1 => cism1 (default, serial only) -CISM2P => cism2 (parallel) -CISM2S => cism2 (serial) - - - - - - - -10 - -TRUE -TRUE - -TRUE - - -FALSE -TRUE - -MPAS Land Ice (shallow ice solver): -MPAS Land Ice (Albany-FO solver): - - -========================================= -MPASLI naming conventions in compset name -========================================= -MPASLISIA => native shallow ice solver -MPASLIALBFO => external Albany First-order solver - - - - - - - -CORE2_NYF -CORE2_IAF -CLM_QIAN -NLDAS -CLMCRUNCEP -CLM1PT -CPLHIST3HrWx -COPYALL_NPS_v1 -COPYALL_NPS_CORE2_v1 -WW3 - -clim_2000 -clim_2000 -clim_2000 -clim_2000 -clim_1850 -rcp8.5 -rcp6.0 -rcp4.5 -rcp2.6 -trans_1850-2000 -clim_1850 -clim_2000 -none -none - -b40.1850.track1.1deg.006a -1 -960 -1030 - -1 -1972 -2004 - -1 -1948 -1972 - -1 -1980 -1980 - -1 -1901 -1920 - -1 -1972 -2004 - -1895 -1948 -1972 - -1901 -1901 -1920 - -1 -1948 -2004 - -2004 -1972 -2004 - -2005 -1991 -2010 - -1 -2002 -2003 - -1 -1901 -1920 - -1 -1991 -2010 - -1 -2002 -2003 - -1 -2003 -2003 - -none -rcp8.5 -rcp6.0 -rcp4.5 -rcp2.6 -20tr - - - - - - - - - - - - - - - - -QIAN atm input data for 1948-1972: -QIAN atm input data for 1972-2004: -NLDAS atm input data for 1980: -QIAN atm input data for 2002-2003: -QIAN atm input data for 1948-1972: -QIAN atm input data for 1948-2004: -QIAN atm input data for 1972-2004: -CRUNCEP atm input data for 1901-1920: -CRUNCEP atm input data for 1991-2010: -CRUNCEP atm input data for 2002-2003: -CRUNCEP atm input data for 1901-1920: -CRUNCEP atm input data for 1991-2010: -CPL history input data: -single point tower site atm input data: - -COREv2 datm normal year forcing: (requires additional user-supplied data) -COREv2 datm interannual year forcing: (requires additional user-supplied data) - -DATM NPS forcing: - - -========================================= -DATM naming conventions in compset name -========================================= -DATM%QIA => QIAN atm input data (1948-1972) -DATM%NLDAS => NLDAS atm input data (1980) -DATM%CRU => CRUNCEP atm input data for (1901-2010) -DATM%S1850 => CPL history atm input data -DATM%1PT => single point tower site atm input data -DATM%NYF => COREv2 datm normal year forcing (requires extra user data) -DATM%IAF => COREv2 datm interannual year forcing (requires extra user data) -DATM%NPS => DATM NPS forcing - - - - - - -NULL - -CPLHIST - -GLC_CPLHIST - -$DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 -b.e10.BG20TRCN.f09_g16.002 - 1 -1976 -2005 - -$DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG1850CN.f09_g16.002_c121001 -b.e10.BG1850CN.f09_g16.002 - 1 - 26 -100 - -$DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 -b.e10.BG20TRCN.f09_g16.002 -1850 -1850 -2005 - -$DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BGRCP85CN.f09_g16.002_c121001 -b.e10.BGRCP85CN.f09_g16.002 -2006 -2006 -2100 - - -$DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 -b.e10.BG20TRCN.f09_g16.002 - 1 -1976 -2005 - - - - - - - - - - -dlnd modes are DLND_MODE=NULL: -dlnd modes are DLND_MODE=GLC_CPLHIST: -dlnd modes are DLND_MODE=CPLHIST: - - -========================================= -DLND naming conventions -========================================= -DLND%NULL => dlnd_mode is NULL -DLND%SCPL => dlnd_mode is GLC_CPLHIST (used for TG) -DLND%LCPL => dlnd_mode is CPLHIST - - - - - - -NULL -DIATREN_ANN_RX1 -DIATREN_IAF_RX1 -NULL - - - - - - -NULL drof mode: -COREv2 drof normal year forcing: -COREv2 drof interannual year forcing: - - -========================================= -DROF naming conventions -========================================= -DROF%NYF => COREv2 drof normal year forcing -DROF%IAF => COREv2 drof interannual year forcing -DROF%NULL => null mode - - - - - - -ssmi -ssmi_iaf -prescribed -copyall -null - - - - - - -dice mode is ssmi: -dice mode is ssmi_iaf: -dice mode is prescribed: -dice mode is copy: -dice mode is null: - - -========================================= -DICE naming conventions -========================================= -DICE%SSMI => dice mode is ssmi -DICE%SIAF => dice mode is ssmi_iaf -DICE%PRES => dice mode is prescribed -DICE%COPY => dice mode is copy -DICE%NULL => dice mode is null - - - - - - -null -prescribed -som -us20 -copyall -interannual - -pop_frc.1x1d.090130.nc - - - - - - - - - -docn null mode: -docn slab ocean mode: -docn data mode: -docn us20 mode: -docn copy mode: -docn interannual mode: - - -========================================= -DOCN naming conventions -========================================= -DOCN%NULL => docn null mode -DOCN%SOM => docn slab ocean mode -DOCN%DOM => docn data mode -DOCN%US20 => docn us20 mode -DOCN%COPY => docn copy mode - - - - - - - - - -CAMDATA -1 -0 -0 -$DIN_LOC_ROOT/ocn/docn7/domain.ocn.1x1.111007.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1x1_clim_c101029.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.48x96_gx3v7_100114.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_48x96_clim_c050526.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.1.9x2.5_gx1v6_090403.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1.9x2.5_clim_c061031.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_c040926.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.47x0.63_gx1v6_090408.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.47x0.63_clim_c061106.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.23x0.31_gx1v6_101108.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.23x0.31_clim_c110526.nc - -CAMDATA -1850 -1850 -2012 -$DIN_LOC_ROOT/ocn/docn7/domain.ocn.1x1.111007.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1x1_1850_2012_c130411.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.48x96_gx3v7_100114.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_48x96_1850_2012_c130411.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.1.9x2.5_gx1v6_090403.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1.9x2.5_1850_2012_c130411.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2012_c130411.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.47x0.63_gx1v6_090408.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.47x0.63_1850_2012_c130411.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.23x0.31_gx1v6_101108.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.23x0.31_1850_2012_c130411.nc - -CAMDATA -1 -0 -0 -$DIN_LOC_ROOT/ocn/docn7/domain.ocn.1x1.111007.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1x1_clim_pi_c101029.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.48x96_gx3v7_100114.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_48x96_clim_pi_c101028.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.1.9x2.5_gx1v6_090403.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1.9x2.5_clim_pi_c101028.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_pi_c101028.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.47x0.63_gx1v6_090408.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.47x0.63_clim_pi_c101028.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.23x0.31_gx1v6_101108.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.23x0.31_clim_pi_c101028.nc -$DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc -$DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_c040926.nc - - -WW3 -1 -2000 -2000 -$DIN_LOC_ROOT/wav/ww3/core2_G4_ice_30min_20000601_to_05.nc -$DIN_LOC_ROOT/wav/ww3/core2_G4_ice_30min_20000601_to_05.nc - - - - - - - - - - - - - - - - - -1979-01-01 -1850-01-01 -1955-01-01 -2000-01-01 -1996-01-01 -2005-01-01 -2010-01-01 -2008-01-01 -2008-01-01 -2005-01-01 -2005-01-01 -2013-01-01 -0001-01-01 -1850-01-01 -2005-01-01 - -1980-01-15 -1997-12-31 - -0001-01-01 -0001-01-01 -1850-01-01 -2006-01-01 - - -0001-01-01 - -1995-07-18 -19800 - -1997-06-18 -84585 - -1993-12-01 -nsteps -158 - -1992-08-12 -nsteps -331 - -0001-08-12 -nsteps -22772 - -ndays -2 - - -nyears -10 - - -ndays -5 -nyears -10 - -GREGORIAN - -nyears -1 - -nmonths -1 - -FALSE -TRUE -TRUE -TRUE - -false - true - true - true - -off -ocn - - - - - - - - - - - - - - - - - - - 48 - 48 - 48 - 48 -144 -288 - 48 - 72 - 4 - 24 - 24 - 24 - 48 - 48 - 1 - -96 -12 -12 -96 -96 -144 -144 -96 -144 -144 -144 -288 -48 -48 -24 -24 -48 - -$ATM_NCPL -1 -4 -1 -1 -1 -1 -1 - - CESM1_MOD_TIGHT -CESM1_MOD -CESM1_MOD -CESM1_MOD -CESM1_MOD -CESM1_MOD -CESM1_MOD -RASM_OPTION1 - -8 -6 -6 -$ATM_NCPL -$ATM_NCPL -$ATM_NCPL -$ATM_NCPL -$ATM_NCPL - -1 -1 -1 -1 -1 -1 - -year - -1 -1 -1 -1 -1 -1 -year - -hour - - -day -1 -1 -1 -1 -1 -1 - - -day -24 -24 -24 -24 -1 -24 - - -day -48 -48 -48 -48 -1 -8 - - -day -12 -12 -12 -12 -1 -4 - - -day -96 -96 -96 -48 -96 -1 -8 - - - - - - - - - - - - - - -379.000 -284.7 -284.7 -0.000001 -368.9 -367.0 -379.000 -284.7 -379.000 -379.000 -367.0 -367.0 -367.0 -367.0 -367.0 -367.0 -379.000 -0.000001 -0.000001 -0.000001 -0.000001 -0.000001 -0.000001 -0.000001 -0.000001 -0.000001 -368.9 -368.9 -367.0 -284.725 -284.725 -368.865 -368.865 -368.865 -368.865 -368.865 - - -0.000001 -0.000001 -0.000001 -368.9 -0.000001 -0.000001 -0.000001 -0.000001 -0.000001 -0.000001 - - - - - - - - - - - - -CO2A -none -CO2C -CO2C -CO2A -CO2A - - - - - - - - -BGC CO2=prog, rad CO2=prog: -BGC CO2=diag, rad CO2=diag: -ECO in POP: - - - - - -b40.1850.track1.2deg.wcm.007 -0156-01-01 - -b40.1850.track1.2deg.wcm.007 -0156-01-01 - -b40.1850.track1.2deg.003 -0501-01-01 - -b40_1850_c02c_76jpf -0221-01-01 - -b40_1850_2d_r07c5cn_160jp -0070-01-01 - -b40.1850.track1.2deg.003 -0501-01-01 - -b40.1850.track1.2deg.003 -0501-01-01 - -b40_1850_c02c_76jpf -0221-01-01 - -b40_1850_2d_r07c5cn_160jp -0070-01-01 - -b40.1850.track1.2deg.wcm.007 -0156-01-01 - -b40.20th.track1.2deg.wcm.007 -1955-01-01 - -b40.20th.track1.2deg.wcm.007 -1955-01-01 - -I1850CN_f19_g16_c100503 -0001-01-01 - -I2000CN_f19_g16_c100503 -0001-01-01 - -I2000CN_f19_g16_c100503 -0001-01-01 - -I1850CN_f19_g16_c100503 -0001-01-01 - -I1850CN_f19_g16_c100503 -0001-01-01 - -I2000CN_f19_g16_c100503 -0001-01-01 - -I1850CN_f19_g16_c100503 -0001-01-01 - -I2000CN_f19_g16_c100503 -0001-01-01 - -I2000CN_f19_g16_c100503 -0001-01-01 - -I2000CN_f19_g16_c100503 -0001-01-01 - -I2000CN_f19_g16_c100503 -0001-01-01 - -b40.1955-2005.2deg.wcm.002 -2005-01-01 - -b40.1955-2005.2deg.wcm.002 -2005-01-01 - -b40.1955-2005.2deg.wcm.002 -2005-01-01 - -b40.1955-2005.2deg.wcm.002 -2005-01-01 - -b40.1955-2005.2deg.wcm.002 -2005-01-01 - -b40.1955-2005.2deg.wcm.002 -2005-01-01 - -b40_1850_1d_b08c5cn_138j -0320-01-01 - -b40.1850.track1.1deg.006 -0863-01-01 - -bg40.1850.track1.1deg.006b -0863-01-01 - -b40.1850.track1.1deg.006 -0863-01-01 - -trk1_1deg_chm_1850_b55.01 -0086-01-01 - -b40.1850.track1.1deg.006 -0863-01-01 - -trk1_1deg_chm_1850_b55.01 -0086-01-01 - -b40_1850_1d_b08c5cn_138j -0320-01-01 - -b40.20th.track1.1deg.008 -2005-01-01 - -b.e10.B20TRC5CN.f09_g16.001 -2006-01-01 - -b40.20th.track1.1deg.008 -2005-01-01 - -b.e10.B20TRC5CN.f09_g16.001 -2006-01-01 - -b40.20th.track1.1deg.008 -2005-01-01 - -b.e10.B20TRC5CN.f09_g16.001 -2006-01-01 - -b40.20th.track1.1deg.008 -2005-01-01 - -b.e10.B20TRC5CN.f09_g16.001 -2006-01-01 - -I2000CN_f09_g16_c100503 -0001-01-01 - -I2000CN_f09_g16_c100503 -0001-01-01 - -I1850CN_f09_g16_c100503 -0001-01-01 - -I1850CN_f09_g16_c100503 -0001-01-01 - -I1850CLM40CRUCN_f09_g16_clm4500_c130514 -1122-01-01 - -I1850CLM40CRUCN_f09_g16_clm4500_c130514 -1122-01-01 - -I2000CN_f09_g16_c100503 -0001-01-01 - -I1850CN_f09_g16_c100503 -0001-01-01 - -I2000CN_f09_g16_c100503 -0001-01-01 - -I2000CN_f09_g16_c100503 -0001-01-01 - -I2000CN_f09_g16_c100503 -0001-01-01 - -I2000CN_f09_g16_c100503 -0001-01-01 - -b40_20th_1d_b08c5cn_139jp -1979-01-01 - -b40_20th_1d_b08c5cn_139jp -1979-01-01 - -f.e12.FAMIPC5.ne30_ne30.amip_L30.001 -1979-01-01 - -f.e12.FAMIPC5.ne30_ne30.amip_L60.001 -1979-01-01 - -f.e12.FAMIPC5.ne30_ne30.amip_L30.001 -1979-01-01 - - -b40.20th.track1.1deg.004 -1979-01-01 - -b40_20th_2d_r07c5cn_161jp -1979-01-01 - -b40.20th.track1.2deg.001 -1979-01-01 - -b40.t31x3.037c -0507-01-01 - -b.e11.B1850C5CN.ne30_g16.tuning.004 -0026-01-01 - -b40.rcp4_5.2deg.wcm.carma.bc0tg.002 -2013-01-01 - - - - - - - - - - - - - --3 --3 --1 - 1 - 1 --1 - 1 --2 --2 --5 --5 --6 - - 0 - 0 - 0 - 0 - - 1 - 3 - -1 -3 -1 -3 -3 -3 -3 -1 -2 -2 - - 2 - 1 - 1 - - - - - - - - - - - - -NO - - - diff --git a/cime_config/acme/allactive/config_pes.xml b/cime_config/acme/allactive/config_pes.xml deleted file mode 100644 index 820aa89ae0a..00000000000 --- a/cime_config/acme/allactive/config_pes.xml +++ /dev/null @@ -1,2223 +0,0 @@ - - - - - - none - - -4 - -4 - -4 - -4 - -4 - -4 - -4 - -4 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 960 - 960 - 960 - 960 - 960 - 960 - 960 - 960 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 1024 - 1024 - 1024 - 1024 - 1024 - 1024 - 1024 - 1024 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 1024 - 1024 - 1024 - 1024 - 1024 - 1024 - 1024 - 1024 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 64 - 64 - 64 - 64 - 64 - 64 - 64 - 64 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 128 - 128 - 128 - 128 - 128 - 128 - 128 - 128 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 1024 - 1024 - 1024 - 1024 - 1024 - 1024 - 1024 - 1024 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 675 - 168 - 168 - 512 - 512 - 1 - 512 - 512 - - - 2 - 2 - 2 - 2 - 1 - 2 - 2 - 2 - - - 0 - 512 - 512 - 0 - 680 - 512 - 0 - 0 - - - - - - - - none - - 256 - 256 - 256 - 256 - 256 - 256 - 256 - 256 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 9600 - 9600 - 9600 - 9600 - 9600 - 9600 - 1 - 9600 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 9600 - 9600 - 9600 - 9600 - 9600 - 9600 - 1 - 9600 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 7200 - 256 - 256 - 6688 - 992 - 1 - 1 - 7200 - - - 16 - 16 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 256 - 512 - 7200 - 0 - 0 - 0 - - - - - - - - none - - 7200 - 256 - 256 - 6688 - 992 - 1 - 1 - 7200 - - - 16 - 16 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 256 - 512 - 7200 - 0 - 0 - 0 - - - - - - - - none - - 7200 - 992 - 992 - 992 - 992 - 1 - 1 - 992 - - - 16 - 16 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 7200 - 7200 - 7200 - 7200 - 7200 - 7200 - 7200 - - - - - - - - none - - 3200 - 1600 - 3200 - 1600 - 32 - 3200 - 3200 - 3200 - - - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - - - 0 - 0 - 0 - 1600 - 3200 - 0 - 0 - 0 - - - - - - - - none - - 2000 - 960 - 2000 - 1040 - 32 - 1 - 2000 - 960 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 960 - 2000 - 0 - 0 - 0 - - - - - - - - none - - 960 - 48 - 960 - 912 - 48 - 1 - 960 - 960 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 48 - 960 - 0 - 0 - 0 - - - - - - - - none - - 16 - 16 - 16 - 16 - 16 - 16 - 16 - 16 - - - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 8 - 8 - 8 - 5 - 4 - 8 - 8 - 8 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 1 - 1 - 1 - 32 - 96 - 1 - 1 - 32 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 16 - 0 - 0 - 32 - 0 - 0 - 0 - - - - - - - - none - - 1 - 1 - 1 - 80 - 192 - 1 - 1 - 80 - - - 1 - 1 - 1 - 4 - 4 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 80 - 0 - 0 - 0 - - - - - - - - none - - 8 - 8 - 8 - 8 - 480 - 8 - 8 - 32 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 8 - 0 - 16 - 32 - 24 - 0 - 0 - - - - - - - - none - - 16 - 8 - 16 - 480 - 480 - 8 - 16 - 32 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 16 - 0 - 32 - 32 - 24 - 0 - 0 - - - - - - - - none - - 48 - 48 - 48 - 48 - 1024 - 48 - 48 - 48 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 48 - 0 - 192 - 0 - 96 - 0 - 144 - - - - - - - - none - - 48 - 48 - 48 - 1024 - 1024 - 48 - 48 - 48 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 48 - 0 - 0 - 0 - 96 - 0 - 144 - - - - - - - - none - - 48 - 48 - 48 - 960 - 960 - 48 - 48 - 48 - - - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - - - 0 - 48 - 0 - 0 - 0 - 96 - 0 - 144 - - - - - - - - none - - 8 - 8 - 8 - 5 - 4 - 8 - 8 - 8 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 8 - 8 - 8 - 5 - 4 - 8 - 8 - 8 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 160 - 160 - 160 - 160 - 240 - 160 - 160 - 160 - - - 2 - 1 - 2 - 2 - 1 - 2 - 2 - 1 - - - 0 - 0 - 0 - 0 - 160 - 0 - 0 - 0 - - - - - - - - none - - 32 - 32 - 32 - 32 - 32 - 32 - 32 - 32 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 64 - 32 - 64 - 32 - 64 - 64 - 64 - 64 - - - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - - - 0 - 0 - 0 - 32 - 0 - 0 - 0 - 0 - - - - - - - - none - - 32 - 32 - 32 - 32 - 32 - 32 - 32 - 32 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 384 - 64 - 384 - 320 - 32 - 384 - 384 - 64 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 1 - - - 0 - 0 - 0 - 64 - 384 - 0 - 0 - 0 - - - - - - - - none - - 768 - 128 - 768 - 640 - 60 - 768 - 768 - 128 - - - 3 - 3 - 3 - 3 - 3 - 3 - 3 - 1 - - - 0 - 0 - 0 - 128 - 768 - 0 - 0 - 0 - - - - - - - - none - - 128 - 128 - 128 - 128 - 128 - 128 - 128 - 128 - - - 2 - 2 - 2 - 2 - 2 - 2 - 2 - 2 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 1525 - 71 - 71 - 1454 - 256 - 1 - 1525 - 1525 - - - 16 - 16 - 16 - 16 - 1 - 1 - 16 - 4 - - - 0 - 0 - 0 - 71 - 1525 - 0 - 0 - 0 - - - - - - - - none - - 1525 - 71 - 71 - 1454 - 256 - 1 - 1525 - 1525 - - - 16 - 16 - 16 - 16 - 1 - 1 - 16 - 4 - - - 0 - 0 - 0 - 71 - 1525 - 0 - 0 - 0 - - - - - - - - none - - 512 - 512 - 512 - 512 - 512 - 512 - 512 - 512 - - - 16 - 16 - 16 - 16 - 16 - 16 - 16 - 16 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 1525 - 71 - 71 - 1454 - 256 - 1 - 1525 - 1525 - - - 16 - 16 - 16 - 16 - 1 - 1 - 16 - 4 - - - 0 - 0 - 0 - 71 - 1525 - 0 - 0 - 0 - - - - - - - - none - - 1525 - 71 - 71 - 1454 - 256 - 1 - 1525 - 1525 - - - 16 - 16 - 16 - 16 - 1 - 1 - 16 - 4 - - - 0 - 0 - 0 - 71 - 1525 - 0 - 0 - 0 - - - - - - - - none - - 496 - 176 - 496 - 320 - 16 - 496 - 496 - 160 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 320 - 0 - 0 - 496 - 0 - 0 - 0 - - - - - - - - none - - 512 - 192 - 512 - 320 - 64 - 512 - 512 - 512 - - - 4 - 4 - 4 - 1 - 1 - 4 - 4 - 1 - - - 0 - 320 - 0 - 0 - 512 - 0 - 0 - 0 - - - - - - - - none - - 480 - 416 - 480 - 480 - 480 - 480 - 480 - 480 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 1024 - 416 - 1024 - 1024 - 1024 - 1024 - 1024 - 1024 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 480 - 416 - 480 - 480 - 1232 - 480 - 480 - 432 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 480 - 0 - 0 - 0 - - - - - - - - none - - 1024 - 416 - 1024 - 1024 - 2356 - 1024 - 1024 - 432 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 1024 - 0 - 0 - 0 - - - - - - - - none - - 1664 - 416 - 1664 - 1800 - 3476 - 1664 - 1664 - 432 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 1800 - 0 - 0 - 0 - - - - - - - - none - - 496 - 336 - 496 - 160 - 16 - 496 - 496 - 160 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 160 - 0 - 0 - 496 - 0 - 0 - 0 - - - - - - - - none - - 1916 - 120 - 120 - 1796 - 132 - 1 - 1916 - 1916 - - - 16 - 16 - 16 - 16 - 1 - 1 - 16 - 16 - - - 0 - 0 - 0 - 120 - 1916 - 0 - 0 - 0 - - - - - - - - none - - 2560 - 512 - 2560 - 512 - 512 - 2560 - 2560 - 512 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 1536 - 512 - 1536 - 0 - 0 - 1536 - 1536 - 1023 - - - - - - - - none - - 9600 - 960 - 960 - 960 - 960 - 960 - 9600 - 960 - - - 4 - 4 - 4 - 4 - 4 - 4 - 4 - 4 - - - 0 - 0 - 5800 - 960 - 1920 - 4840 - 0 - 3880 - - - - - - - - none - - 2048 - 112 - 2048 - 1800 - 4028 - 2048 - 2048 - 2048 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 2048 - 0 - 2160 - 3960 - 0 - 0 - 0 - - - - - - - - none - - 512 - 512 - 512 - 512 - 512 - 512 - 512 - 512 - - - 6 - 6 - 6 - 1 - 1 - 6 - 6 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 2048 - 112 - 2048 - 1800 - 4028 - 2048 - 2048 - 1800 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 2048 - 0 - 2160 - 3960 - 0 - 0 - 0 - - - - - - - - none - - 1824 - 112 - 1824 - 1600 - 3600 - 1824 - 1824 - 1600 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 1824 - 0 - 1936 - 3536 - 0 - 0 - 0 - - - - - - - - none - - 512 - 512 - 512 - 1800 - 2048 - 512 - 512 - 512 - - - 6 - 6 - 6 - 1 - 1 - 6 - 6 - 1 - - - 0 - 512 - 0 - 512 - 2312 - 0 - 0 - 0 - - - - - - - - none - - 512 - 512 - 512 - 1800 - 2048 - 512 - 512 - 512 - - - 6 - 6 - 6 - 1 - 1 - 6 - 6 - 1 - - - 0 - 512 - 0 - 512 - 2312 - 0 - 0 - 0 - - - - - - - - none - - 1440 - 512 - 1440 - 512 - 2048 - 1440 - 1440 - 512 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 1440 - 0 - 0 - 0 - - - - - - - - none - - 3600 - 512 - 3600 - 512 - 512 - 3600 - 3600 - 512 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - - - - none - - 14400 - 512 - 14400 - 512 - 512 - 14400 - 14400 - 512 - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - 0 - 0 - 0 - 0 - 0 - 0 - 0 - 0 - - - - - diff --git a/cime_config/acme/allactive/config_pes.xml.cime2 b/cime_config/acme/allactive/config_pes.xml.cime2 deleted file mode 100644 index ecb6dfd95fe..00000000000 --- a/cime_config/acme/allactive/config_pes.xml.cime2 +++ /dev/null @@ -1,1439 +0,0 @@ - - - - - - - - - - 16 1 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - - -1-1 pnetcdf 1 - 0 - - - - -1-1 netcdf 1 - 0 - - - - - $MAX_TASKS_PER_NODE 1 0 - 2cm - - - - - 12 4 01 - - - - - 8 4 01 - - - - - - - - 64 1 01 - - - - 2048 1 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - - -1-1 pnetcdf 1 - 0 - - - - - - 1024 1 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - - -1-1 pnetcdf 1 - 0 - - - - 4800 4 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - - -124 pnetcdf 1 - 0 - - - - 3200 2 0 1 - 1600 $NTHRDS_ATM $ROOTPE_ATM 1 - 1600 $NTHRDS_ATM 1600 1 - 32 $NTHRDS_ATM $NTASKS_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - - -1-1 pnetcdf 1 - 0 - - - - 128 1 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - -1-1 pnetcdf 1 - 0 - - - - 64 1 0 1 - - - - 960 4 0 1 - -124 pnetcdf 1 - 0 - - - - 960 4 0 1 - -132 pnetcdf 1 - 0 - - - - 32 1 0 1 - 2cm - - - - 1280 1 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - -1-1 pnetcdf 1 - 0 - - - - 1200 2 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - -1-1 pnetcdf 1 - 0 - - - - 1200201 - 600 $NTHRDS_ATM $ROOTPE_ATM 1 - 600 $NTHRDS_ATM 600 1 - 120 1 1200 1 - 1320 $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - -1-1 pnetcdf 1 - 0 - - - - 2000 1 0 1 - 960 $NTHRDS_ATM $ROOTPE_ATM 1 - 1040 $NTHRDS_ATM 960 1 - 32 1 2000 1 - 960 $NTHRDS_ATM $ROOTPE_ATM - 1 $NTHRDS_ATM $ROOTPE_ATM 1 - -132 pnetcdf 1 - 0 - - - - 256 1 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - -1-1 pnetcdf 1 - 0 - - - - 4 4 0 - 0p - - - - 128 4 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM 1 $ROOTPE_ATM 1 - $NTASKS_ATM 1 $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - -1-1 pnetcdf 1 - - - - netcdf - - - - - - 16 1 0 - 1r - - - - 12 4 0 - - - - 16 4 0 - - - - 4 4 0 - 1rp - - - - 8 1 0 - 1r - - - - 2 4 0 - 1rp - - - - 64 1 0 - 1r - - - - 48 1 0 - 1r - - - - 32 4 0 - 1r - - - - 16 4 0 - 1rp - - - - 128 4 0 - 1r - - - - 32 4 0 - 1rp - - - - 24 4 0 - 1rp - - - - 64 1 0 - 1r - - - - 96 4 0 - - - - 16 4 0 - 1rp - - - - 24 4 0 - 1rp - - - - 64 1 0 - 1r - - - - 192 4 0 - 1r - - - - 32 4 0 - 1rp - - - - 48 4 0 - 1rp - - - - 64 1 0 1 - 1r - - - - 48 1 0 - 48 1 0 - 48 1 0 - 48 1 0 - 48 1 0 - 48 1 0 1 - 48 1 0 1 - 48 1 0 - 1m - - - - 128 1 0 - 128 1 0 - 128 1 0 - 128 1 0 - 128 1 0 - 128 1 0 1 - 128 1 0 1 - 128 1 0 - - - - 48 1 0 - 48 1 0 - 48 1 0 - 48 1 0 - 48 1 0 - 48 1 0 1 - 48 1 0 1 - 48 1 0 - - - - 960 4 0 - 48 4 0 - 912 4 48 - 48 4 960 - 960 4 0 - 1 4 0 - 4rcmp - - - - 256 1 0 - 1r - - - - 64 4 0 - 1rp - - - - 512 1 0 - 1r - - - - 128 4 0 - 1rp - - - - - 32 1 0 - 1rc - - - - 25 1 0 - 1r - - - - 64 1 0 - 1r - - - - 40 1 0 - 1r - - - - 64 1 0 - 64 1 0 - 64 1 0 - 64 1 0 - 64 1 0 - 64 1 0 - 1r - - - - pnetcdf - -8 - - - - -1-1 pnetcdf 1 - - - - - - - -2048 16 0 -2048 16 0 -2048 16 0 -2048 16 0 -2048 16 0 -2048 16 0 1 -2048 16 0 1 -2048 16 0 --164 pnetcdf 1 -1m - - - - -512 16 0 -512 16 0 -512 16 0 -512 16 0 -512 16 0 -512 16 0 1 -512 16 0 1 -512 16 0 -1m - - - -8 16 0 -8 16 0 -8 16 0 -8 16 0 -8 16 0 -8 16 0 1 -8 16 0 1 -8 16 0 -1m - - - - - - - 16 2 0 - 1r - - - - 8 4 0 - 8 4 0 - 5 4 0 - 8 4 0 - 8 4 0 - 4 4 0 - 2rp - - - - 1 1 0 - 1 1 16 - 32 1 0 - 96 1 32 - 32 1 0 - 1 1 0 - 2rc - - - - 1 1 0 - 1 1 0 - 80 4 0 - 192 4 80 - 80 1 0 - 1 1 0 - 3rcp - - - - 8 1 0 - 8 1 8 - 8 1 16 - 8 1 24 - 32 1 0 - 480 1 32 - 3rcp - - - - 8 1 0 - 8 1 8 - 8 1 16 - 8 1 24 - 32 1 0 - 480 1 32 - 3rcp - - - - 16 1 0 - 8 1 16 - 8 1 24 - 32 1 0 - 480 1 32 - 480 1 32 - 3rcp - - - - 48 1 0 - 48 1 48 - 48 1 96 - 48 1 144 - 48 1 192 - 1024 1 0 - 3rcp - - - - 48 1 0 - 48 1 48 - 48 1 96 - 48 1 144 - 1024 1 0 - 48 1 192 - 3rcp - - - - 48 1 0 - 48 1 48 - 48 1 96 - 48 1 144 - 1024 1 0 - 1024 1 0 - 3rcp - - - - 48 2 0 - 48 2 48 - 48 2 96 - 48 2 144 - 960 2 0 - 960 2 0 - 3rcp - - - - 96 4 0 - 1r - - - - - - 8 4 0 - 8 4 0 - 6 4 0 - 8 4 0 - 8 4 0 - 8 4 0 - 2rp - - - - 8 4 0 - 8 4 0 - 5 4 0 - 8 4 0 - 8 4 0 - 4 4 0 - 2rp - - - - 8 4 0 - 8 4 0 - 5 4 0 - 8 4 0 - 8 4 0 - 4 4 0 - 2rp - - - - - - 160 2 0 - 160 1 0 - 160 2 0 - 160 1 0 - 160 2 0 - 240 1 160 - 3rcm - - - - 32 4 0 - 32 4 0 - 32 4 0 - 32 4 0 - 32 4 0 - 32 4 0 - 2rp - - - - 64 2 0 - 32 2 0 - 32 2 32 - 64 2 0 - 64 2 0 - 64 2 0 - 2rp - - - - 32 4 0 - 32 4 0 - 32 4 0 - 32 4 0 - 32 4 0 - 32 4 0 - 2rp - - - - 384 4 0 - 64 4 0 - 320 4 64 - 32 4 384 - 64 1 0 - 384 4 0 - 3rcm - - - - 768 3 0 - 128 3 0 - 640 3 128 - 60 3 768 - 128 1 0 - 768 3 0 - 3rcm - - - - 128 2 0 - 128 2 0 - 128 2 0 - 128 2 0 - 128 2 0 - 128 2 0 - 3rcm - - - - 120 1 0 - 96 1 0 - 12 1 96 - 12 1 108 - 96 1 0 - 1 1 0 - 1c - - - - 25 1 0 - 25 1 0 - 32 1 0 - 32 1 0 - 64 1 0 - 1 1 0 - 2rc - - - - 256 1 0 - 1 1 0 - 256 1 0 - 256 1 0 - 256 1 0 - 1 1 0 - 2rc - - - - 256 1 0 - 1 1 0 - 16 1 0 - 16 1 0 - 256 1 0 - 1 1 0 - 2rc - - - - - -1525 16 0 -71 16 0 -1454 16 71 -71 16 0 -256 1 1525 -1 1 $ROOTPE_ATM 1 -$NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 -$NTASKS_ATM 4 $ROOTPE_ATM -1m - - - -512 16 0 -512 16 0 -512 16 0 -512 16 0 -512 16 0 -512 16 0 1 -512 16 0 1 -512 16 0 -1m - - - -1525 16 0 -71 16 0 -1454 16 71 -71 16 0 -256 1 1525 -1 1 $ROOTPE_ATM 1 -$NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 -$NTASKS_ATM 4 $ROOTPE_ATM -1m - - - - 496 1 0 - 176 1 320 - 320 1 0 - 160 1 0 - 496 1 0 - 16 1 496 - 3rcm - - - - 512 4 0 - 192 4 320 - 320 1 0 - 512 1 0 - 512 4 0 - 64 1 512 - 3rcm - - - - 480 1 0 - 416 1 0 - 480 1 0 - 480 1 0 - 480 1 0 - 480 1 0 - 3rcp - - - - 1024 1 0 - 416 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 3rcp - - - - 480 1 0 - 416 1 0 - 480 1 0 - 432 1 0 - 480 1 0 - 1232 1 480 - 3rcp - - - - 1024 1 0 - 416 1 0 - 1024 1 0 - 432 1 0 - 1024 1 0 - 2356 1 1024 - 3rcp - - - - 1664 1 0 - 416 1 0 - 1800 1 0 - 432 1 0 - 1664 1 0 - 3476 1 1800 - 3rcp - - - - - - 496 1 0 - 336 1 160 - 160 1 0 - 160 1 0 - 496 1 0 - 16 1 496 - 3rcm - - - - - -2048 16 0 -2048 16 0 -2048 16 0 -2048 16 0 -2048 16 0 -2048 16 0 -2048 16 0 -2048 16 0 -1m - - - -1916 16 0 -120 16 0 -1796 16 120 -120 16 0 -132 1 1916 -1 1 $ROOTPE_ATM 1 -1916 $NTHRDS_ATM $ROOTPE_ATM 1 -1916 16 $ROOTPE_ATM -1m - - - - 2560 1 1536 - 512 1 512 - 512 1 0 - 512 1 0 - 512 1 1023 - 2560 1 1536 - 1m - - - - 9600 4 0 - 960 4 0 - 960 4 960 - 960 4 1920 - 960 4 3880 - 960 4 4840 - 960 4 5800 - 1m - - - - 2048 1 0 - 112 1 2048 - 1800 1 2160 - 2048 1 0 - 2048 1 0 - 4028 1 3960 - 1m - - - - - - - 512 6 0 - 512 6 0 - 512 1 0 - 512 1 0 - 512 6 0 - 512 1 0 - 2rm - - - - 2048 1 0 - 112 1 2048 - 1800 1 2160 - 1800 1 0 - 2048 1 0 - 4028 1 3960 - 4rcmp - - - - - 1824 1 0 - 112 1 1824 - 1600 1 1936 - 1600 1 0 - 1824 1 0 - 3600 1 3536 - 4rcmp - - - - - 512 6 0 - 512 6 512 - 1800 1 512 - 512 1 0 - 512 6 0 - 2048 1 2312 - 4rcmp - - - - - 512 6 0 - 512 6 512 - 1800 1 512 - 512 1 0 - 512 6 0 - 2048 1 2312 - 4rcmp - - - - 1440 1 0 - 512 1 0 - 512 1 0 - 512 1 0 - 1440 1 0 - 2048 1 1440 - 3rcm - - - - 3600 1 0 - 512 1 0 - 512 1 0 - 512 1 0 - 3600 1 0 - 512 1 0 - 3rcm - - - - 14400 1 0 - 512 1 0 - 512 1 0 - 512 1 0 - 14400 1 0 - 512 1 0 - 3rcm - - - - $NTASKS_LND $NTHRDS_LND $ROOTPE_LND - - - - - - - - - 192 1 0 - 192 1 0 - 192 1 0 - 192 1 0 - 192 1 0 - 192 1 0 - - - - 96 1 0 - 96 1 0 - 96 1 0 - 96 1 0 - 96 1 0 - 96 1 0 - - - - 32 1 0 - 32 1 0 - 32 1 0 - 32 1 0 - 32 1 0 - 32 1 0 - - - - - - - - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 1c - - - - 1 $ROOTPE_CPL - - - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - - - 1 - - - - - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - - - - 8 1 8 - 2 1 0 - 2 1 2 - 4 1 4 - 8 1 24 - 4 1 20 - 4 1 16 - 1p - - - - 8 1 8 - 4 1 0 - 4 1 12 - 8 1 16 - 8 1 24 - 4 1 32 - 8 1 36 - 4 1 44 - 1p - - - - 4 1 4 - 4 1 0 - 4 1 8 - 4 1 12 - 4 1 16 - 4 1 20 - 4 1 24 - 4 1 28 - 1p - - - - - - 16 1 0 - 1c - - - - 4 4 0 - 1c - - - - 8 1 0 - 1c - - - - 2 4 0 - 1c - - - - 2 4 0 - 2cp - - - - - 6 4 0 - 2cmp - - - - 8 4 0 - 2cmp - - - - $MAX_TASKS_PER_NODE 1 0 - 2cmp - - - - $MAX_TASKS_PER_NODE 4 0 - 2cmp - - - - -64 16 0 -64 16 0 -64 1 0 -64 16 0 -64 1 0 -64 16 0 1 -64 16 0 1 -64 16 0 -1m - - - - - - 1 1 0 - 1r - - - - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 1r - - - - 5 1 0 - 1r - - - - 64 1 0 - 64 1 0 - 64 1 0 - 64 1 0 - 64 1 0 - 64 1 0 - 8 1 0 - 64 1 0 - 1r - - - - - - 960 1 0 - 960 1 0 - 960 1 0 - 960 1 0 - 960 1 0 - 960 1 0 - 960 1 0 - 960 1 0 - 1r - - - - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1r - - - - 675 2 0 - 168 2 512 - 512 2 0 - 512 1 680 - 168 2 512 - 1 2 512 - 512 2 0 - 512 2 0 - 1r - - - - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1r - - - - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1024 1 0 - 1r - - - - - 9600 1 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - 1 1 0 1 - -1-1 pnetcdf 1 - - - 7200160 1 - 256 160 1 - 66881 512 1 - 992 1 72001 - 72001 0 - 1 1 0 1 - 256 1 256 1 - 1 1 0 1 - -164 pnetcdf 1 - - - - - - 9600 1 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - 1 1 0 1 - -1-1 pnetcdf 1 - - - 7200160 1 - 256 160 1 - 66881 512 1 - 992 1 72001 - 72001 0 - 1 1 0 1 - 256 1 256 1 - 1 1 0 1 - -164 pnetcdf 1 - - - - - 7200 160 1 - 992 167200 1 - $NTASKS_LND1 $ROOTPE_LND1 - $NTASKS_LND1 $ROOTPE_LND1 - $NTASKS_LND1 $ROOTPE_LND1 - $NTASKS_LND1 $ROOTPE_LND - 1 1 $ROOTPE_LND1 - 1 1 $ROOTPE_LND1 - -1 64pnetcdf1 - - - - - - 240 4 0 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM 1 $ROOTPE_ATM 1 - $NTASKS_ATM 1 $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - $NTASKS_ATM $NTHRDS_ATM $ROOTPE_ATM 1 - -1-1 pnetcdf 1 - - - - - - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 64160 - 64160 - 1 1 0 - 1r - - - - - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 1 1 0 - 6410 - 6410 - 1 1 0 - - - - diff --git a/cime_config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands b/cime_config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands deleted file mode 100644 index ac78d092c67..00000000000 --- a/cime_config/acme/allactive/testmods_dirs/cam/outfrq9s/shell_commands +++ /dev/null @@ -1,5 +0,0 @@ -if [ `./xmlquery MACH -value` = "melvin" ]; then - ./xmlchange PIO_TYPENAME=netcdf -fi - -./xmlchange ROF_NCPL=48 diff --git a/cime_config/acme/machines/syslog.titan b/cime_config/acme/machines/syslog.titan deleted file mode 100755 index a2ff159dbb7..00000000000 --- a/cime_config/acme/machines/syslog.titan +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/csh -f -# titan syslog script: -# mach_syslog - -set sec = 0 -set sample_interval = $1 -set jid = $2 -set lid = $3 -set run = $4 -set timing = $5 -set dir = $6 - -# wait until job mapping information is output before saving output file -# (note that calling script 'touch'es the acme log file before spawning this script, so that 'wc' does not fail) -set nnodes = `qstat -f $jid | grep Resource_List.nodes | sed 's/ *Resource_List.nodes = *\([0-9]*\):ppn=*\([0-9]*\) */\1/' ` -set outlth = 0 -while ($outlth < $nnodes) - sleep 10 - set outlth = `wc \-l $run/acme.log.$lid | sed 's/ *\([0-9]*\) *.*/\1/' ` -end -set remaining = `qstat -f $jid | grep Walltime.Remaining | sed 's/ *Walltime.Remaining = *\([0-9]*\) */\1/' ` -cat > $run/Walltime.Remaining < 0) - grep -a -i -e "nstep" -e "model date" $run/*atm.log.$lid | tail > $dir/atm.log.$lid.nstep.$remaining -# grep -a -i "nstep" $run/acme.log.$lid | tail > $dir/acme.log.$lid.nstep.$remaining - grep -a -i -e "timestep" -e "model date" $run/*lnd.log.$lid | tail > $dir/lnd.log.$lid.timestep.$remaining - grep -a -i -e "timestep" -e "Step number" -e "model date" $run/*ocn.log.$lid | tail > $dir/ocn.log.$lid.stepnum.$remaining - grep -a -i -e "timestep" -e "istep" -e "model date" $run/*ice.log.$lid | tail > $dir/ice.log.$lid.istep.$remaining - grep -a -i "model date" $run/*cpl.log.$lid | tail > $dir/cpl.log.$lid.modeldata.$remaining - /bin/cp --preserve=timestamps -u $timing/* $dir - xtnodestat > $dir/xtnodestat.$lid.$remaining - showq > $dir/showq.$lid.$remaining - chmod a+r $dir/* - sleep $sample_interval - set remaining = `qstat -f $jid | grep Walltime.Remaining | sed 's/ *Walltime.Remaining = *\([0-9]*\) */\1/' ` - if ("X$remaining" == "X") set remaining = 0 - cat > $run/Walltime.Remaining << EOF2 -$remaining $sample_interval -EOF2 - -end - diff --git a/cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/xmlchange_cmnds b/cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/xmlchange_cmnds deleted file mode 100644 index ac78d092c67..00000000000 --- a/cime_config/acme/testmods_dirs/allactive/cam/outfrq9s/xmlchange_cmnds +++ /dev/null @@ -1,5 +0,0 @@ -if [ `./xmlquery MACH -value` = "melvin" ]; then - ./xmlchange PIO_TYPENAME=netcdf -fi - -./xmlchange ROF_NCPL=48 diff --git a/cime_config/acme/allactive/config_compsets.xml b/config/acme/allactive/config_compsets.xml similarity index 78% rename from cime_config/acme/allactive/config_compsets.xml rename to config/acme/allactive/config_compsets.xml index 1ad8c923df4..305b9227647 100644 --- a/cime_config/acme/allactive/config_compsets.xml +++ b/config/acme/allactive/config_compsets.xml @@ -88,6 +88,26 @@ 2000_CAM5%AV1C-H01A_CLM45%SPBC_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAV + + A_WCYCL1850_H01A + 1850_CAM5%AV1C-H01A_CLM45%SPBC_MPASCICE_MPASO_MOSART_SGLC_SWAV + + + + A_WCYCL1850_H01AS + 1850_CAM5%AV1C-H01A_CLM45%SPBC_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAV + + + + + A_WCYCL20TR_H01A + 20TR_CAM5%AV1C-H01A_CLM45%SPBC_MPASCICE_MPASO_MOSART_SGLC_SWAV + + + + A_WCYCL20TR_H01AS + 20TR_CAM5%AV1C-H01A_CLM45%SPBC_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAV + @@ -100,6 +120,17 @@ 2000_CAM5_CLM45%SP_MPASCICE_MPASO_MOSART_SGLC_SWAVi + + A_WCYCL1850S_v0atm + 1850_CAM5_CLM45%SP_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAV + + + + A_WCYCL2000S_v0atm + 2000_CAM5_CLM45%SP_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAVi + + + @@ -158,4 +189,21 @@ 1850_CAM5_CLM45%BGC_CICE_DOCN%SOM_RTM_SGLC_SWAV + + E1850C5CNECACTCBC + 1850_CAM5_CLM45%CNECACTCBC_CICE_DOCN%SOM_RTM_SGLC_SWAV + + + + E20TRC5CNECACTCBC + 20TR_CAM5_CLM45%CNECACTCBC_CICE_DOCN%SOM_RTM_SGLC_SWAV + + + + + + 1850-01-01 + + + diff --git a/cime_config/acme/allactive/config_pesall.xml b/config/acme/allactive/config_pesall.xml similarity index 94% rename from cime_config/acme/allactive/config_pesall.xml rename to config/acme/allactive/config_pesall.xml index 4f00631f832..eabee41dbf5 100644 --- a/cime_config/acme/allactive/config_pesall.xml +++ b/config/acme/allactive/config_pesall.xml @@ -554,7 +554,7 @@ - + none @@ -590,6 +590,43 @@ + + + + none + + 256 + 256 + 256 + 256 + 256 + 256 + 256 + 256 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + @@ -961,7 +998,7 @@ - + none @@ -1367,6 +1404,43 @@ + + + + none + + 128 + 128 + 128 + 128 + 128 + 128 + 128 + 128 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + @@ -1590,7 +1664,44 @@ - + + + none + + 96 + 96 + 96 + 96 + 96 + 96 + 96 + 96 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + + + none @@ -1775,7 +1886,44 @@ - + + + none + + 64 + 64 + 64 + 64 + 64 + 64 + 64 + 64 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + + + none @@ -1997,7 +2145,7 @@ - + none @@ -2034,7 +2182,7 @@ - + none @@ -2070,6 +2218,43 @@ + + + + none + + 128 + 128 + 128 + 128 + 128 + 128 + 128 + 128 + + + 4 + 4 + 4 + 4 + 4 + 4 + 4 + 4 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + @@ -3958,7 +4143,7 @@ - + none @@ -5466,19 +5651,19 @@ - + - - none + + "133 node version gets 6 SYPD. This will be the default and M size" - 960 - 960 - 960 - 960 - 960 - 960 - 960 - 960 + 2700 + 312 + 312 + 2400 + 480 + 312 + 2400 + 2400 1 @@ -5492,11 +5677,11 @@ 0 - 0 - 0 + 2400 + 2400 0 - 0 - 0 + 2712 + 2400 0 0 @@ -5505,17 +5690,17 @@ - - "PMC - 173 node version gets 6 SYPD. This will be the default and M size" + + "39 node version gets 2.1 SYPD." - 2700 - 312 - 312 - 2400 - 1440 - 312 - 2400 - 2400 + 675 + 56 + 56 + 640 + 240 + 56 + 56 + 640 1 @@ -5529,11 +5714,11 @@ 0 - 2400 - 2400 + 640 + 640 0 - 2712 - 2400 + 696 + 640 0 0 @@ -5543,13 +5728,13 @@ - "PMC - 375 node version gets 10.7 SYPD" + "265 node version gets 10.4 SYPD" 5400 600 600 - 4800 - 3600 + 3200 + 960 600 4800 4800 @@ -5577,6 +5762,43 @@ + + + + "first attempt for testing" + + 5400 + 640 + 640 + 4800 + 3600 + 640 + 4800 + 4800 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 4800 + 4800 + 0 + 5440 + 4800 + 0 + 0 + + + + @@ -5984,10 +6206,10 @@ - + - - T62_oQU120.CMPASO-NYF on 16 nodes pure-MPI + + T62_oQU120 grid for MPAS tests on 16 nodes pure-MPI 64 64 @@ -6023,7 +6245,7 @@ - + -compset A_WCYCL* -res ne4_oQU240 on 7 nodes pure-MPI 96 @@ -6201,8 +6423,111 @@ - - + + + none + + 32 + 32 + 32 + 32 + 32 + 32 + 32 + 32 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + none + + 48 + 48 + 48 + 48 + 48 + 48 + 48 + 48 + + + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + none + + 96 + 96 + 96 + 96 + 96 + 96 + 96 + 96 + + + 4 + 4 + 4 + 4 + 4 + 4 + 4 + 4 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + + + + none 32 @@ -6235,10 +6560,6 @@ 0 - - - - none @@ -6272,10 +6593,6 @@ 0 - - - - none @@ -6313,7 +6630,7 @@ - + none 32 @@ -6348,4 +6665,41 @@ + + + + any compset on ne4 grid + + 6 + 6 + 6 + 6 + 6 + 6 + 6 + 6 + + + 16 + 16 + 16 + 16 + 16 + 16 + 16 + 16 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/cime_config/acme/allactive/testlist_allactive.xml b/config/acme/allactive/testlist_allactive.xml similarity index 100% rename from cime_config/acme/allactive/testlist_allactive.xml rename to config/acme/allactive/testlist_allactive.xml diff --git a/cime_config/acme/allactive/usermods_dirs/b1850/shell_commands b/config/acme/allactive/usermods_dirs/b1850/shell_commands similarity index 100% rename from cime_config/acme/allactive/usermods_dirs/b1850/shell_commands rename to config/acme/allactive/usermods_dirs/b1850/shell_commands diff --git a/cime_config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods b/config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods similarity index 100% rename from cime_config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods rename to config/acme/allactive/usermods_dirs/b1850_cism/include_user_mods diff --git a/cime_config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism b/config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism similarity index 100% rename from cime_config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism rename to config/acme/allactive/usermods_dirs/b1850_cism/user_nl_cism diff --git a/cime_config/acme/config_archive.xml b/config/acme/config_archive.xml similarity index 64% rename from cime_config/acme/config_archive.xml rename to config/acme/config_archive.xml index a4843a2b572..fc2d71239b1 100644 --- a/cime_config/acme/config_archive.xml +++ b/config/acme/config_archive.xml @@ -11,6 +11,15 @@ + + \.r.* + unset + + rpointer$NINST_STRING.atm + $CASE.datm$NINST_STRING.r.$DATENAME.nc,$CASE.datm$NINST_STRING.rs1.$DATENAME.bin + + + \.[ri]\..* \.rh.* @@ -22,6 +31,15 @@ + + \.r.* + unset + + rpointer$NINST_STRING.lnd + $CASE.dlnd$NINST_STRING.r.$DATENAME.nc,$CASE.dlnd$NINST_STRING.rs1.$DATENAME.bin + + + \.r.* \.h.*.nc$ @@ -42,6 +60,15 @@ + + \.r.* + unset + + rpointer$NINST_STRING.rof + $CASE.drof$NINST_STRING.r.$DATENAME.nc,$CASE.drof$NINST_STRING.rs1.$DATENAME.bin + + + \.[ri].* \.h.*.nc$ @@ -54,13 +81,22 @@ \.rst.* - \.hist.* + \.hist.*\.*\.*\.nc unset rpointer.ice + + \.r.* + unset + + rpointer$NINST_STRING.ice + $CASE.dice$NINST_STRING.r.$DATENAME.nc,$CASE.dice$NINST_STRING.rs1.$DATENAME.bin + + + \.r.* \.h.*.nc$|\.d[dovt]\. @@ -77,13 +113,22 @@ \.rst.* - \.hist.* + \.hist.*\.*\.*\.nc unset rpointer.ocn + + \.r.* + unset + + rpointer$NINST_STRING.ocn + $CASE.docn$NINST_STRING.r.$DATENAME.nc,$CASE.docn$NINST_STRING.rs1.$DATENAME.bin + + + \.[ri]\..* \.h\..*\.nc$ @@ -114,6 +159,25 @@ + + \.r.* + unset + + rpointer$NINST_STRING.wav + $CASE.dwav$NINST_STRING.r.$DATENAME.nc,$CASE.dwav$NINST_STRING.rs1.$DATENAME.bin + + + + + \.r\..* + \.h.*.nc$ + unset + + rpointer.cpl + $CASE.cpl.r.$DATENAME.nc + + + inflate_restart.* \.True_State.* diff --git a/cime_config/acme/config_files.xml b/config/acme/config_files.xml similarity index 61% rename from cime_config/acme/config_files.xml rename to config/acme/config_files.xml index dcd98320309..97867caa304 100644 --- a/cime_config/acme/config_files.xml +++ b/config/acme/config_files.xml @@ -18,7 +18,7 @@ char - $CIMEROOT/cime_config/config_headers.xml + $CIMEROOT/config/config_headers.xml case_der env_case.xml contains both header and group information for all the case env_*.xml files @@ -26,16 +26,16 @@ char - $CIMEROOT/cime_config/$MODEL/machines/config_batch.xml + $CIMEROOT/config/$MODEL/machines/config_batch.xml case_last env_case.xml file containing batch system details for target system (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_batch.xsd + $CIMEROOT/config/xml_schemas/config_batch.xsd char - $CIMEROOT/cime_config/$MODEL/config_grids.xml + $CIMEROOT/config/$MODEL/config_grids.xml case_last env_case.xml file containing specification of all supported model grids, domains and mapping files (for documentation only - DO NOT EDIT) @@ -43,16 +43,16 @@ char - $CIMEROOT/cime_config/$MODEL/machines/config_compilers.xml + $CIMEROOT/config/$MODEL/machines/config_compilers.xml case_last env_case.xml file containing compiler specifications for target model primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_compilers_v2.xsd + $CIMEROOT/config/xml_schemas/config_compilers_v2.xsd char - $CIMEROOT/cime_config/$MODEL/machines/config_machines.xml + $CIMEROOT/config/$MODEL/machines/config_machines.xml case_last env_case.xml file containing machine specifications for target model primary component (for documentation only - DO NOT EDIT) @@ -60,7 +60,7 @@ char - $CIMEROOT/cime_config/$MODEL/machines/config_pio.xml + $CIMEROOT/config/$MODEL/machines/config_pio.xml case_last env_case.xml file containing specification of pio settings for target model possible machine, compiler, mpilib, compset and/or grid attributes (for documentation only - DO NOT EDIT) @@ -68,7 +68,7 @@ char - $CIMEROOT/cime_config/config_tests.xml + $CIMEROOT/config/config_tests.xml test env_test.xml file containing system test descriptions @@ -83,8 +83,8 @@ char unset - $CIMEROOT/cime_config/$MODEL/allactive/config_compsets.xml - $CIMEROOT/driver_cpl/cime_config/config_compsets.xml + $CIMEROOT/config/$MODEL/allactive/config_compsets.xml + $CIMEROOT/src/drivers/mct/cime_config/config_compsets.xml $SRCROOT/components/cam/cime_config/config_compsets.xml $SRCROOT/components/cism/cime_config/config_compsets.xml $SRCROOT/components/clm/cime_config/config_compsets.xml @@ -97,62 +97,58 @@ case_last env_case.xml file containing specification of all compsets for primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_compsets.xsd + $CIMEROOT/config/xml_schemas/config_compsets.xsd char unset - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/driver_cpl/cime_config/config_pes.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml - $CIMEROOT/cime_config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/src/drivers/mct/cime_config/config_pes.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml + $CIMEROOT/config/$MODEL/allactive/config_pesall.xml case_last env_case.xml file containing specification of all pe-layouts for primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_pes.xsd + $CIMEROOT/config/xml_schemas/config_pes.xsd - char - $CIMEROOT/cime_config/acme/config_archive.xml - $CIMEROOT/driver_cpl/cime_config/config_archive.xml + $CIMEROOT/config/acme/config_archive.xml + $CIMEROOT/src/drivers/mct/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/drof/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/datm/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/dice/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/dlnd/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/docn/cime_config/config_archive.xml - $CIMEROOT/components/data_comps/dwav/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/drof/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/datm/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/dice/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/dlnd/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/docn/cime_config/config_archive.xml + $CIMEROOT/src/components/data_comps/dwav/cime_config/config_archive.xml $SRCROOT/components/cam/cime_config/config_archive.xml $SRCROOT/components/cism/cime_config/config_archive.xml $SRCROOT/components/clm/cime_config/config_archive.xml $SRCROOT/components/cice/cime_config/config_archive.xml $SRCROOT/components/pop/cime_config/config_archive.xml - $SRCROOT/components/mpas-o/cime_config/config_archive.xml - $SRCROOT/components/mpasli/cime_config/config_archive.xml - $SRCROOT/components/mpas-cice/cime_config/config_archive.xml case_last env_case.xml file containing specification of archive files for each component (for documentation only - DO NOT EDIT) - $CIMEROOT/cime_config/xml_schemas/config_archive.xsd + $CIMEROOT/config/xml_schemas/config_archive.xsd char - $CIMEROOT/utils/python/CIME/SystemTests + $CIMEROOT/scripts/lib/CIME/SystemTests $SRCROOT/components/clm/cime_config/SystemTests $SRCROOT/components/cam/cime_config/SystemTests $SRCROOT/components/pop/cime_config/SystemTests @@ -168,8 +164,8 @@ char unset - $CIMEROOT/cime_config/$MODEL/allactive/testlist_allactive.xml - $CIMEROOT/driver_cpl/cime_config/testdefs/testlist_drv.xml + $CIMEROOT/config/$MODEL/allactive/testlist_allactive.xml + $CIMEROOT/src/drivers/mct/cime_config/testdefs/testlist_drv.xml $SRCROOT/components/cam/cime_config/testdefs/testlist_cam.xml $SRCROOT/components/cism/cime_config/testdefs/testlist_cism.xml $SRCROOT/components/clm/cime_config/testdefs/testlist_clm.xml @@ -187,15 +183,15 @@ char unset - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs - $CIMEROOT/cime_config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs + $CIMEROOT/config/$MODEL/testmods_dirs case_last env_case.xml @@ -206,8 +202,8 @@ char unset - $CIMEROOT/cime_config/$MODEL/allactive/usermods_dirs - $CIMEROOT/driver_cpl/cime_config/usermods_dirs + $CIMEROOT/config/$MODEL/allactive/usermods_dirs + $CIMEROOT/src/drivers/mct/cime_config/usermods_dirs $SRCROOT/components/cam/cime_config/usermods_dirs $SRCROOT/components/cism/cime_config/usermods_dirs $SRCROOT/components/clm/cime_config/usermods_dirs @@ -225,16 +221,16 @@ char unset - $CIMEROOT/driver_cpl/cime_config/namelist_definition_modelio.xml - $CIMEROOT/driver_cpl/bld/namelist_files/namelist_definition_drv_flds.xml - $CIMEROOT/driver_cpl/cime_config/namelist_definition_drv.xml + $CIMEROOT/src/drivers/mct/cime_config/namelist_definition_modelio.xml + $CIMEROOT/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml + $CIMEROOT/src/drivers/mct/cime_config/namelist_definition_drv.xml - $CIMEROOT/components/data_comps/drof/cime_config/namelist_definition_drof.xml - $CIMEROOT/components/data_comps/datm/cime_config/namelist_definition_datm.xml - $CIMEROOT/components/data_comps/dice/cime_config/namelist_definition_dice.xml - $CIMEROOT/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml - $CIMEROOT/components/data_comps/docn/cime_config/namelist_definition_docn.xml - $CIMEROOT/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml + $CIMEROOT/src/components/data_comps/drof/cime_config/namelist_definition_drof.xml + $CIMEROOT/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml + $CIMEROOT/src/components/data_comps/dice/cime_config/namelist_definition_dice.xml + $CIMEROOT/src/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml + $CIMEROOT/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml + $CIMEROOT/src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml @@ -1707,6 +1720,13 @@ cpl/gridmaps/fv0.23x0.31/map_fv0.23x0.31_to_ne120np4_aave_110331.nc cpl/gridmaps/fv0.23x0.31/map_fv0.23x0.31_to_ne120np4_aave_110331.nc + + cpl/gridmaps/ne120np4/map_ne120np4_to_oRRS18to6v3_aave.170111.nc + cpl/gridmaps/ne120np4/map_ne120np4_to_oRRS18to6v3_conserve.170111.nc + cpl/gridmaps/ne120np4/map_ne120np4_to_oRRS18to6v3_conserve.170111.nc + cpl/gridmaps/oRRS18to6v3/map_oRRS18to6v3_to_ne120np4_aave.170111.nc + cpl/gridmaps/oRRS18to6v3/map_oRRS18to6v3_to_ne120np4_aave.170111.nc + cpl/gridmaps/ne240np4/map_ne240np4_to_gx1v6_aave_110428.nc @@ -2244,6 +2264,11 @@ cpl/cpl6/map_r0125_to_oRRS18to6v3_nn.170111.nc + + cpl/cpl6/map_r0125_to_oRRS18to6v3_nn.170111.nc + cpl/cpl6/map_r0125_to_oRRS18to6v3_nn.170111.nc + + cpl/cpl6/map_r0125_to_oRRS15to5_nn.160720.nc cpl/cpl6/map_r0125_to_oRRS15to5_nn.160720.nc diff --git a/cime_config/acme/machines/Depends.cetus b/config/acme/machines/Depends.cetus similarity index 100% rename from cime_config/acme/machines/Depends.cetus rename to config/acme/machines/Depends.cetus diff --git a/cime_config/acme/machines/Depends.corip1 b/config/acme/machines/Depends.corip1 similarity index 100% rename from cime_config/acme/machines/Depends.corip1 rename to config/acme/machines/Depends.corip1 diff --git a/cime_config/acme/machines/Depends.cray b/config/acme/machines/Depends.cray similarity index 100% rename from cime_config/acme/machines/Depends.cray rename to config/acme/machines/Depends.cray diff --git a/cime_config/acme/machines/Depends.gnu b/config/acme/machines/Depends.gnu similarity index 100% rename from cime_config/acme/machines/Depends.gnu rename to config/acme/machines/Depends.gnu diff --git a/cime_config/acme/machines/Depends.intel b/config/acme/machines/Depends.intel similarity index 100% rename from cime_config/acme/machines/Depends.intel rename to config/acme/machines/Depends.intel diff --git a/cime_config/acme/machines/Depends.intel14 b/config/acme/machines/Depends.intel14 similarity index 100% rename from cime_config/acme/machines/Depends.intel14 rename to config/acme/machines/Depends.intel14 diff --git a/cime_config/acme/machines/Depends.intelmic b/config/acme/machines/Depends.intelmic similarity index 100% rename from cime_config/acme/machines/Depends.intelmic rename to config/acme/machines/Depends.intelmic diff --git a/cime_config/acme/machines/Depends.intelmic14 b/config/acme/machines/Depends.intelmic14 similarity index 100% rename from cime_config/acme/machines/Depends.intelmic14 rename to config/acme/machines/Depends.intelmic14 diff --git a/cime_config/acme/machines/Depends.mira b/config/acme/machines/Depends.mira similarity index 100% rename from cime_config/acme/machines/Depends.mira rename to config/acme/machines/Depends.mira diff --git a/cime_config/acme/machines/Depends.nag b/config/acme/machines/Depends.nag similarity index 100% rename from cime_config/acme/machines/Depends.nag rename to config/acme/machines/Depends.nag diff --git a/cime_config/acme/machines/Depends.titan.cray b/config/acme/machines/Depends.titan.cray similarity index 100% rename from cime_config/acme/machines/Depends.titan.cray rename to config/acme/machines/Depends.titan.cray diff --git a/cime_config/acme/machines/Depends.titan.intel b/config/acme/machines/Depends.titan.intel similarity index 100% rename from cime_config/acme/machines/Depends.titan.intel rename to config/acme/machines/Depends.titan.intel diff --git a/cime_config/acme/machines/Depends.titan.pgi b/config/acme/machines/Depends.titan.pgi similarity index 100% rename from cime_config/acme/machines/Depends.titan.pgi rename to config/acme/machines/Depends.titan.pgi diff --git a/cime_config/acme/machines/Depends.titan.pgiacc b/config/acme/machines/Depends.titan.pgiacc similarity index 100% rename from cime_config/acme/machines/Depends.titan.pgiacc rename to config/acme/machines/Depends.titan.pgiacc diff --git a/cime_config/acme/machines/Makefile b/config/acme/machines/Makefile similarity index 93% rename from cime_config/acme/machines/Makefile rename to config/acme/machines/Makefile index 1ecad3188a5..75b11dd70f4 100644 --- a/cime_config/acme/machines/Makefile +++ b/config/acme/machines/Makefile @@ -300,11 +300,22 @@ ifeq ($(findstring -DLINUX,$(CPPDEFS)),-DLINUX) endif endif +# Atleast on Titan+cray mpi, MPI_Irsends() are buggy, causing hangs during I/O +# Force PIO to use MPI_Isends instead of the default, MPI_Irsends +ifeq ($(PIO_VERSION),2) + EXTRA_PIO_CPPDEFS = -DUSE_MPI_ISEND_FOR_FC +else + EXTRA_PIO_CPPDEFS = -D_NO_MPI_RSEND +endif + ifdef CPRE FPPDEFS := $(subst $(comma),\\$(comma),$(CPPDEFS)) FPPDEFS := $(patsubst -D%,$(CPRE)%,$(FPPDEFS)) + EXTRA_PIO_FPPDEFS := $(subst $(comma),\\$(comma),$(EXTRA_PIO_CPPDEFS)) + EXTRA_PIO_FPPDEFS := $(patsubst -D%,$(CPRE)%,$(EXTRA_PIO_FPPDEFS)) else FPPDEFS := $(CPPDEFS) + EXTRA_PIO_FPPDEFS := $(EXTRA_PIO_CPPDEFS) endif #=============================================================================== @@ -427,8 +438,8 @@ ifndef CISM_LIBDIR CISM_LIBDIR=$(GLC_DIR)/lib endif -INCLDIR += -I$(INSTALL_SHAREDPATH)/include -I$(CIMEROOT)/share/csm_share/shr \ - -I$(CIMEROOT)/share/csm_share/include -I$(CIMEROOT)/share/shr_RandNum/include +INCLDIR += -I$(INSTALL_SHAREDPATH)/include -I$(CIMEROOT)/src/share/util \ + -I$(CIMEROOT)/src/share/include -I$(CIMEROOT)/src/share/RandNum/include # # Use the MCT dir for the cache for all configure calls because it is the first one # @@ -569,25 +580,25 @@ endif $(SHAREDLIBROOT)/$(SHAREDPATH)/mct/Makefile.conf: @echo "SHAREDLIBROOT |$(SHAREDLIBROOT)| SHAREDPATH |$(SHAREDPATH)|"; \ - $(CONFIG_SHELL) $(CIMEROOT)/externals/mct/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/externals/mct + $(CONFIG_SHELL) $(CIMEROOT)/src/externals/mct/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/src/externals/mct $(SHAREDLIBROOT)/$(SHAREDPATH)/mct/mpi-serial/Makefile.conf: @echo "SHAREDLIBROOT |$(SHAREDLIBROOT)| SHAREDPATH |$(SHAREDPATH)|"; \ - $(CONFIG_SHELL) $(CIMEROOT)/externals/mct/mpi-serial/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/externals/mct/mpi-serial + $(CONFIG_SHELL) $(CIMEROOT)/src/externals/mct/mpi-serial/configure $(CONFIG_ARGS) --srcdir $(CIMEROOT)/src/externals/mct/mpi-serial ifeq ($(PIO_VERSION),2) # This is a pio2 library PIOLIB = $(PIO_LIBDIR)/libpiof.a $(PIO_LIBDIR)/libpioc.a PIOLIBNAME = -lpiof -lpioc - PIO_SRC_DIR = $(CIMEROOT)/externals/pio2 + PIO_SRC_DIR = $(CIMEROOT)/src/externals/pio2 else # This is a pio1 library PIOLIB = $(PIO_LIBDIR)/libpio.a PIOLIBNAME = -lpio - ifneq ("$(wildcard $(CIMEROOT)/externals/pio1/pio)", "") - PIO_SRC_DIR = $(CIMEROOT)/externals/pio1 + ifneq ("$(wildcard $(CIMEROOT)/src/externals/pio1/pio)", "") + PIO_SRC_DIR = $(CIMEROOT)/src/externals/pio1 else - PIO_SRC_DIR = $(CIMEROOT)/externals/pio1/pio + PIO_SRC_DIR = $(CIMEROOT)/src/externals/pio1/pio endif endif #endif @@ -609,13 +620,13 @@ endif # FIXEDFLAGS, so that both free & fixed code can be built (cmake # doesn't seem to be able to differentiate between free & fixed # fortran flags) -CMAKE_OPTS += -D CMAKE_Fortran_FLAGS:STRING="$(FFLAGS) $(INCLDIR)" \ - -D CMAKE_C_FLAGS:STRING="$(CFLAGS) $(INCLDIR)" \ - -D CMAKE_CXX_FLAGS:STRING="$(CXXFLAGS) $(INCLDIR)" \ +CMAKE_OPTS += -D CMAKE_Fortran_FLAGS:STRING="$(FFLAGS) $(EXTRA_PIO_FPPDEFS) $(INCLDIR)" \ + -D CMAKE_C_FLAGS:STRING="$(CFLAGS) $(EXTRA_PIO_CPPDEFS) $(INCLDIR)" \ + -D CMAKE_CXX_FLAGS:STRING="$(CXXFLAGS) $(EXTRA_PIO_CPPDEFS) $(INCLDIR)" \ -D CMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -D GPTL_PATH:STRING=$(INSTALL_SHAREDPATH) \ -D PIO_ENABLE_TESTS:BOOL=OFF \ - -D USER_CMAKE_MODULE_PATH:STRING=$(CIMEROOT)/externals/CMake + -D USER_CMAKE_MODULE_PATH:LIST="$(CIMEROOT)/src/externals/CMake;$(CIMEROOT)/src/externals/pio2/cmake" \ # Allow for separate installations of the NetCDF C and Fortran libraries ifeq ($(NETCDF_SEPARATE), false) @@ -832,26 +843,26 @@ $(COMPLIB): $(OBJS) $(CXX) -c $(INCLDIR) $(INCS) $(CXXFLAGS) $< %.F90: %.F90.in - $(CIMEROOT)/externals/genf90/genf90.pl $< > $@ + $(CIMEROOT)/src/externals/genf90/genf90.pl $< > $@ cleanatm: $(RM) -f $(LIBROOT)/libatm.a - cd $(EXEROOT)/atm/obj; $(RM) -f *.o *.mod + $(RM) -fr $(EXEROOT)/atm/obj cleancpl: - cd $(EXEROOT)/cpl/obj; $(RM) -f *.o *.mod + $(RM) -fr $(EXEROOT)/cpl/obj cleanocn: $(RM) -f $(LIBROOT)/libocn.a - cd $(EXEROOT)/ocn/obj ; $(RM) -f *.o *.mod + $(RM) -fr $(EXEROOT)/ocn/obj cleanwav: $(RM) -f $(LIBROOT)/libwav.a - cd $(EXEROOT)/wav/obj ; $(RM) -f *.o *.mod + $(RM) -fr $(EXEROOT)/wav/obj cleanesp: $(RM) -f $(LIBROOT)/libesp.a - cd $(EXEROOT)/esp/obj ; $(RM) -f *.o *.mod + $(RM) -fr $(EXEROOT)/esp/obj cleanglc: $(RM) -f $(LIBROOT)/libglc.a @@ -859,15 +870,15 @@ cleanglc: cleanice: $(RM) -f $(LIBROOT)/libice.a - cd $(EXEROOT)/ice/obj ; $(RM) -f *.o *.mod + $(RM) -fr $(EXEROOT)/ice/obj cleanrof: $(RM) -f $(LIBROOT)/librof.a - cd $(EXEROOT)/rof/obj ; $(RM) -f *.o *.mod + $(RM) -fr $(EXEROOT)/rof/obj cleanlnd: $(RM) -f $(LIBROOT)/liblnd.a - cd $(EXEROOT)/lnd/obj ; $(RM) -f *.o *.mod + $(RM) -fr $(EXEROOT)/lnd/obj cleancsmshare: $(RM) -f $(CSMSHARELIB) diff --git a/cime_config/acme/machines/README b/config/acme/machines/README similarity index 100% rename from cime_config/acme/machines/README rename to config/acme/machines/README diff --git a/cime_config/acme/machines/config_batch.xml b/config/acme/machines/config_batch.xml similarity index 82% rename from cime_config/acme/machines/config_batch.xml rename to config/acme/machines/config_batch.xml index 48893de566c..39420fc7f29 100644 --- a/cime_config/acme/machines/config_batch.xml +++ b/config/acme/machines/config_batch.xml @@ -130,6 +130,29 @@ -S {{ shell }} + + + + msub + #MSUB + (\d+)$ + -l depend=jobid + %H:%M:%S + + -V + -q {{ job_queue }} + -N {{ job_id }} + -l nodes={{ num_nodes }}:ppn={{ tasks_per_node }} + -l walltime={{ job_wallclock_time }} + -o {{ output_error_path }}.out + -e {{ output_error_path }}.err + -m be + -A {{ project }} + + + pbatch + + squeue @@ -158,8 +181,8 @@ -l nodes={{ num_nodes }}:ppn={{ tasks_per_node }} - shared - batch + shared + batch @@ -177,8 +200,8 @@ + debug regular - debug @@ -189,7 +212,7 @@ -l nodes={{ num_nodes }} - batch + batch @@ -198,8 +221,8 @@ --constraint=haswell + debug regular - debug @@ -208,8 +231,8 @@ --constraint=knl,quad,cache + debug regular - debug @@ -225,6 +248,22 @@ + + + default + + + + + + --mail-type=END + --mail-user=email@pnnl.gov + + + small + + + --mail-type=END @@ -244,7 +283,7 @@ --mail-user=email@pnnl.gov --output=slurm.out --error=slurm.err - + slurm @@ -259,6 +298,15 @@ + + + --ntasks-per-node={{ tasks_per_node }} + + + ec + + + --ntasks-per-node={{ tasks_per_node }} @@ -275,18 +323,22 @@ - - -l nodes={{ num_nodes }}:ppn={{ tasks_per_node }} - - - moab - + + -l nodes={{ num_nodes }}:ppn={{ tasks_per_node }} + -l qos=standard + + + moab + -l nodes={{ num_nodes }}:ppn={{ tasks_per_node }} + + moab + @@ -341,24 +393,28 @@ batch - debug + debug + + + --ntasks-per-node={{ tasks_per_node }} - --exclusive - lr2 + lr3 + + + --ntasks-per-node={{ tasks_per_node }} - --exclusive lr3 diff --git a/cime_config/acme/machines/config_batch.xsd b/config/acme/machines/config_batch.xsd similarity index 100% rename from cime_config/acme/machines/config_batch.xsd rename to config/acme/machines/config_batch.xsd diff --git a/cime_config/acme/machines/config_compilers.xml b/config/acme/machines/config_compilers.xml similarity index 90% rename from cime_config/acme/machines/config_compilers.xml rename to config/acme/machines/config_compilers.xml index 237cd880b31..f1b40b26510 100644 --- a/cime_config/acme/machines/config_compilers.xml +++ b/config/acme/machines/config_compilers.xml @@ -29,7 +29,7 @@ if $MPILIB is set to mpi-serial then FC = $SFC MPICC = $SCC MPIFC = $SFC - INC_MPI = $(CIMEROOT)/externals/mct/mpi-serial + INC_MPI = $(CIMEROOT)/src/externals/mct/mpi-serial ======================================================================== Options for including C++ code in the build @@ -521,7 +521,7 @@ for mct, etc. USERDEFINED_MUST_EDIT_THIS - # USERDEFINED $(shell $(NETCDF_PATH)/bin/nc-config --flibs) + # USERDEFINED $(shell $(NETCDF_PATH)/bin/nf-config --flibs) @@ -561,9 +561,46 @@ for mct, etc. /global/project/projectdirs/acme/software/AlbanyTrilinos_09232015/Albany/build/install + + -DFORTRANUNDERSCORE -DNO_R16 -DCPRINTEL + --host=Linux + + -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -fp-model source + -O2 -fp-model precise -std=gnu99 + -O0 + -free + -fixed -132 + + -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -qno-opt-dynamic-align + -O2 -debug minimal -qno-opt-dynamic-align + + -O0 -g + -O2 -debug minimal + -r8 + + -qopenmp + -qopenmp + -qopenmp + -qopenmp + + -L$(NETCDF_DIR) -lnetcdff -Wl,--as-needed,-L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf + -mkl -lpthread -lm + -DHAVE_PAPI + -DHAVE_SLASHPROC + ftn + cc + CC + ifort + icc + icpc + FORTRAN + -cxxlib + TRUE + TRUE + $(PETSC_DIR) + + - -xCORE-AVX2 - -xCORE-AVX2 -O2 -qno-opt-dynamic-align -O2 -qopenmp @@ -572,8 +609,7 @@ for mct, etc. -qopenmp --host=Linux -L$(NETCDF_DIR) -lnetcdff -Wl,--as-needed,-L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf - ${MKLROOT}/lib/intel64/libmkl_scalapack_lp64.a -Wl,--start-group ${MKLROOT}/lib/intel64/libmkl_intel_lp64.a ${MKLROOT}/lib/intel64/libmkl_core.a ${MKLROOT}/lib/intel64/libmkl_sequential.a -Wl,--end-group ${MKLROOT}/lib/intel64/libmkl_blacs_intelmpi_lp64.a -lpthread -lm - + -mkl -lpthread -lm -DHAVE_PAPI -DHAVE_SLASHPROC ftn @@ -595,10 +631,8 @@ for mct, etc. --host=Linux -L$(NETCDF_DIR) -lnetcdff -Wl,--as-needed,-L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf -mkl -lpthread -lm - -DHAVE_PAPI - -DHAVE_SLASHPROC - + -DARCH_MIC_KNL -DHAVE_SLASHPROC ftn cc CC @@ -630,7 +664,7 @@ for mct, etc. $(NETCDF_PATH) $(PNETCDF_PATH) - $(shell $(NETCDF_PATH)/bin/nc-config --flibs) + $(shell $(NETCDF_PATH)/bin/nf-config --flibs) @@ -682,8 +716,6 @@ for mct, etc. -O2 -O2 --host=Linux - $(NETCDFROOT) - $(PNETCDFROOT) $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -lblas -llapack -lstdc++ -lmpi_cxx /projects/install/rhel6-x86_64/ACME/AlbanyTrilinos/Albany/build/install @@ -704,6 +736,21 @@ for mct, etc. /projects/ccsm/AlbanyTrilinos/Albany/build/install + + -O2 + -O2 + $(NETCDFROOT) + $(PNETCDFROOT) + /opt/openmpi-1.8-intel + /projects/ccsm/esmf-6.3.0rp1/lib/libO/Linux.intel.64.openmpi.default + --host=Linux + $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -L/projects/ccsm/BLAS-intel -lblas_LINUX + lustre + -mkl=cluster + -mkl + /projects/ccsm/AlbanyTrilinos/Albany/build/install + + -O2 -O2 @@ -730,7 +777,7 @@ for mct, etc. /usr/local/tools/mvapich2-pgi-1.7/ mpich -DNO_SHR_VMATH -DCNL - $(shell /usr/local/tools/netcdf-pgi-4.1.3/bin/nc-config --flibs) + $(shell /usr/local/tools/netcdf-pgi-4.1.3/bin/nf-config --flibs) -Wl,-rpath /usr/local/tools/netcdf-pgi-4.1.3/lib -llapack -lblas @@ -744,13 +791,57 @@ for mct, etc. mpich -DNO_SHR_VMATH -DCNL -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv - $(shell /usr/local/tools/netcdf-intel-4.1.3/bin/nc-config --flibs) + $(shell /usr/local/tools/netcdf-intel-4.1.3/bin/nf-config --flibs) -llapack -lblas -D_USE_FLOW_CONTROL + + /usr/local/tools/netcdf-pgi-4.1.3 + /usr/local/tools/mvapich2-pgi-1.7/ + mpich + -DNO_SHR_VMATH -DCNL + $(shell /usr/local/tools/netcdf-pgi-4.1.3/bin/nc-config --flibs) + -Wl,-rpath /usr/local/tools/netcdf-pgi-4.1.3/lib -llapack -lblas + + + + -O2 + -O2 + /usr/local/tools/netcdf-intel-4.1.3 + --host=Linux + /usr/local/tools/mvapich2-intel-2.1/ + mpich + -DNO_SHR_VMATH -DCNL + -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv + $(shell /usr/local/tools/netcdf-intel-4.1.3/bin/nc-config --flibs) + -llapack -lblas + + + + /usr/local/tools/netcdf-pgi-4.1.3 + /usr/local/tools/mvapich2-pgi-1.7/ + mpich + -DNO_SHR_VMATH -DCNL + $(shell /usr/local/tools/netcdf-pgi-4.1.3/bin/nc-config --flibs) + -Wl,-rpath /usr/local/tools/netcdf-pgi-4.1.3/lib -llapack -lblas + + + + -O2 + -O2 + /usr/local/tools/netcdf-intel-4.1.3 + --host=Linux + /usr/local/tools/mvapich2-intel-2.1/ + mpich + -DNO_SHR_VMATH -DCNL + -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv + $(shell /usr/local/tools/netcdf-intel-4.1.3/bin/nc-config --flibs) + -llapack -lblas + + /projects/cesm/devtools/netcdf-4.1.3-gcc4.8.1-mpich3.0.4/ -L/user/lib64 -llapack -lblas -lnetcdff @@ -814,7 +905,6 @@ for mct, etc. -lmkl_core -lmkl_gnu_thread -lmkl_blacs_openmpi_lp64 \ -Wl,--end-group -lgomp -lrt - @@ -923,7 +1013,6 @@ for mct, etc. mpixlf77_r /soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/current/ /soft/libraries/pnetcdf/1.6.0/cnk-xl/current/ - /home/santos/pFUnit/pFUnit_IBM /soft/libraries/hdf5/1.8.14/cnk-xl/current/ -L$(NETCDF_PATH)/lib -lnetcdff -lnetcdf -L$(HDF5_PATH)/lib -lhdf5_hl -lhdf5 -L/soft/libraries/alcf/current/xl/ZLIB/lib -lz -L/soft/libraries/alcf/current/xl/LAPACK/lib -llapack -L/soft/libraries/alcf/current/xl/BLAS/lib -lblas -L/bgsys/drivers/ppcfloor/comm/sys/lib -L$(IBM_MAIN_DIR)/xlf/bg/14.1/bglib64 -lxlfmath -lxlf90_r -lxlopt -lxl -L$(IBM_MAIN_DIR)/xlsmp/bg/3.1/bglib64 -lxlsmp @@ -946,7 +1035,6 @@ for mct, etc. mpixlf77_r /soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/current/ /soft/libraries/pnetcdf/1.6.0/cnk-xl/current/ - /home/santos/pFUnit/pFUnit_IBM /soft/libraries/hdf5/1.8.14/cnk-xl/current/ -L$(NETCDF_PATH)/lib -lnetcdff -lnetcdf -L$(HDF5_PATH)/lib -lhdf5_hl -lhdf5 -L/soft/libraries/alcf/current/xl/ZLIB/lib -lz -L/soft/libraries/alcf/current/xl/LAPACK/lib -llapack -L/soft/libraries/alcf/current/xl/BLAS/lib -lblas -L/bgsys/drivers/ppcfloor/comm/sys/lib -L$(IBM_MAIN_DIR)/xlf/bg/14.1/bglib64 -lxlfmath -lxlf90_r -lxlopt -lxl -L$(IBM_MAIN_DIR)/xlsmp/bg/3.1/bglib64 -lxlsmp @@ -960,6 +1048,24 @@ for mct, etc. /projects/ccsm/libs/AlbanyTrilinos/Albany/build/install + + ftn + cc + CC + -O2 -xMIC-AVX512 + -O2 -xMIC-AVX512 + -O0 -g -xMIC-AVX512 + -O0 -g -xMIC-AVX512 + -qopenmp + -qopenmp + -qopenmp + -qopenmp + --host=Linux + -L$(NETCDF_DIR)/lib -lnetcdff -L$(NETCDF_DIR)/lib -lnetcdf -Wl,-rpath -Wl,$(NETCDF_DIR)/lib + -mkl -lpthread -lm + -DHAVE_COMM_F2C + + $(PNETCDFROOT) $(NETCDFROOT) @@ -968,7 +1074,7 @@ for mct, etc. mpi mpi mpich - $(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas + $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas -rpath $(NETCDFROOT)/lib gpfs @@ -980,7 +1086,7 @@ for mct, etc. /soft/mpich2/1.4.1-intel-13.1 mpi mpich - $(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas + $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas gpfs -mkl=cluster -mkl=cluster @@ -996,7 +1102,7 @@ for mct, etc. $(NETCDFROOT) /soft/mvapich2/2.2b_psm/intel-15.0 mpi - $(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas + $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas -Wl,-rpath -Wl,$(NETCDFROOT)/lib -mkl=cluster -mkl=cluster @@ -1013,7 +1119,7 @@ for mct, etc. $(NETCDFROOT) /blues/gpfs/home/software/spack/opt/spack/linux-x86_64/gcc-5.3.0/mvapich2-2.2b-sdh7nhddicl4sh5mgxjyzxtxox3ajqey mpi - $(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas + $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY gpfs @@ -1023,24 +1129,24 @@ for mct, etc. $(NETCDFROOT) /home/robl/soft/mpich-3.1.4-nag-6.0 mpi - $(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas + $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas gpfs - $(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas -mkl + $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas -mkl -Wl,-rpath -Wl,$(NETCDF_PATH)/lib gpfs - $(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas + $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY gpfs - $(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas + $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -llapack -lblas -rpath $(NETCDF_PATH)/lib gpfs @@ -1098,22 +1204,15 @@ for mct, etc. -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi -L$(MKLROOT) -lmkl_rt - - - $(NETCDF_LIB)/.. + $(NETCDF_HOME) $(PNETCDFROOT) lustre --host=Linux --enable-filesystem-hints=lustre -DLINUX -g -traceback -O0 -fpe0 -check all -check noarg_temp_created -ftrapuv - -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi -L$(MKL_PATH) -lmkl_rt - -mkl=cluster + -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi -L$(MKL_PATH)/lib/intel64 -lmkl_rt -mkl=cluster - -mkl=cluster - -mkl=cluster - -mkl=cluster - -mkl=cluster -mkl @@ -1160,6 +1259,7 @@ for mct, etc. + lustre mpicc mpif90 mpic++ @@ -1171,6 +1271,7 @@ for mct, etc. + lustre mpicc mpif90 mpic++ @@ -1232,11 +1333,13 @@ for mct, etc. + $(NETCDF_DIR) -lnetcdff -lnetcdf -mkl -DHAVE_VPRINTF -DHAVE_GETTIMEOFDAY + $(NETCDF_DIR) -lnetcdff -lnetcdf -mkl -DHAVE_VPRINTF -DHAVE_GETTIMEOFDAY @@ -1296,6 +1399,13 @@ lude -L/soft/netcdf/fortran-4.4-intel-sp1-update3-parallel/lib -lnetcdff -L/soft/hdf5/hdf5-1.8.13-intel-2013-sp1-update3-impi-5.0.0.028/lib -openmp -fPIC -lnetcdf -lnetcdf -L/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -lpthread -lm + + -DHAVE_VPRINTF -DHAVE_GETTIMEOFDAY -DHAVE_BACKTRACE + -L$(NETCDF_HOME)/lib/ -lnetcdff -lnetcdf -lcurl -llapack -lblas + $(NETCDF_HOME) + + + -std=c99 diff --git a/cime_config/acme/machines/config_lt_archive.xml b/config/acme/machines/config_lt_archive.xml similarity index 100% rename from cime_config/acme/machines/config_lt_archive.xml rename to config/acme/machines/config_lt_archive.xml diff --git a/cime_config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml similarity index 80% rename from cime_config/acme/machines/config_machines.xml rename to config/acme/machines/config_machines.xml index 507b4cb25a9..206b343bcfb 100644 --- a/cime_config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -51,6 +51,8 @@ the 'executable' tag must have arguments required for the chosen mpirun, as well as the executable name. + unit_testing: can be 'true' or 'false'. + This allows using a different mpirun command to launch unit tests --> @@ -58,16 +60,16 @@ NERSC XC30, os is CNL, 24 pes/node, batch system is SLURM edison acme_developer - intel,gnu,cray + intel,intel17,gnu,cray mpt,mpi-serial - $ENV{SCRATCH}/acme_scratch/edison + $ENV{CSCRATCH}/acme_scratch/edison $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld $CIME_OUTPUT_ROOT/archive/$CASE csm/$CASE CNL slurm - cseg + acme 8 24 24 @@ -124,20 +126,27 @@ PrgEnv-intel intel intel/15.0.1.133 - - cray-libsci + cray-mpich/7.2.5 + + + PrgEnv-intel + intel + intel/17.0.1.132 + cray-libsci + cray-mpich/7.5.1 PrgEnv-cray - cce cce/8.4.3 + cce cce/8.5.1 + cray-libsci/16.07.1 + cray-mpich/7.5.1 PrgEnv-gnu - gcc gcc/5.1.0 - - + gcc gcc/6.2.0 cray-libsci/16.07.1 + cray-mpich/7.5.1 @@ -146,10 +155,6 @@ craype-ivybridge pmi pmi/5.0.10-1.0000.11050.0.0.ari - - - - cray-mpich/7.2.5 @@ -162,6 +167,7 @@ cray-parallel-netcdf/1.6.1 + git/2.4.6 papi/5.4.3.2 perl/5.20.0 cmake/3.3.2 @@ -173,8 +179,9 @@ 1 1 1 - 64M + yes + yes @@ -185,7 +192,7 @@ acme_developer intel,gnu,cray mpt,mpi-serial - $ENV{SCRATCH}/acme_scratch + $ENV{SCRATCH}/acme_scratch/cori-haswell $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /project/projectdirs/acme/inputdata @@ -209,7 +216,7 @@ --label -n $TOTALPES - + -c 2 @@ -254,24 +261,23 @@ craype/2.5.7 craype-haswell - cray-mpich/7.4.4 + + cray-mpich/7.5.3 PrgEnv-intel intel - - - intel/17.0.1.132 + intel/17.0.2.174 PrgEnv-cray - cce cce/8.5.0 + cce cce/8.5.4 PrgEnv-gnu - gcc gcc/6.2.0 + gcc gcc/6.3.0 @@ -290,12 +296,11 @@ - git + git/2.9.1 cmake/3.3.2 - pmi/5.0.10-1.0000.11050.0.0.ari + pmi/5.0.10-1.0000.11069.183.8.ari papi/5.4.3.2 - zlib - + zlib/1.2.8 @@ -308,7 +313,7 @@ 128M spread threads - + yes @@ -319,7 +324,7 @@ acme_developer intel,gnu,cray mpt,mpi-serial - $ENV{SCRATCH}/acme_scratch + $ENV{SCRATCH}/acme_scratch/cori-knl $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /project/projectdirs/acme/inputdata @@ -343,7 +348,7 @@ --label -n $TOTALPES - + -c 4 --cpu_bind=cores @@ -358,9 +363,6 @@ module module - - PrgEnv-cray - PrgEnv-gnu intel cce gcc @@ -386,21 +388,23 @@ PrgEnv-intel intel - - - intel/17.0.1.132 + intel/17.0.2.174 + PrgEnv-intel PrgEnv-cray - cce cce/8.5.0 + cce + cce/8.5.4 + PrgEnv-intel PrgEnv-gnu - gcc gcc/6.2.0 + gcc + gcc/6.3.0 - cray-libsci/16.09.1 + cray-libsci/16.09.1 @@ -425,17 +429,16 @@ cray-parallel-netcdf/1.7.0 - git + git/2.9.1 cmake/3.3.2 - pmi/5.0.10-1.0000.11050.0.0.ari + pmi/5.0.10-1.0000.11069.183.8.ari papi/5.4.3.2 - zlib + zlib/1.2.8 - 1 1 @@ -444,7 +447,8 @@ spread threads - + yes + @@ -561,7 +565,7 @@ Linux workstation at Sandia on SRN with SEMS TPL modules - (s999964|climate) + (s999964|climate|penn) wwwproxy.sandia.gov:80 acme_developer LINUX @@ -605,7 +609,7 @@ sems-python/2.7.9 - sems-gcc/5.1.0 + sems-gcc/5.3.0 sems-intel/15.0.2 @@ -662,16 +666,27 @@ +gcc-6.2.0 +szip-2.1-gcc-6.2.0 - +mpich-3.2-gcc-6.2.0 - +hdf5-1.8.16-gcc-6.2.0-mpich-3.2-parallel - +netcdf-4.3.3.1c-4.2cxx-4.4.2f-parallel-gcc6.2.0-mpich3.2 - +pnetcdf-1.6.1-gcc-6.2.0-mpich-3.2 +cmake-2.8.12 + + +netcdf-4.4.1c-4.2cxx-4.4.4f-serial-gcc6.2.0 + + + +mpich-3.2-gcc-6.2.0 + +hdf5-1.8.16-gcc-6.2.0-mpich-3.2-parallel + +netcdf-4.3.3.1c-4.2cxx-4.4.2f-parallel-gcc6.2.0-mpich3.2 + +pnetcdf-1.6.1-gcc-6.2.0-mpich-3.2 + - $SHELL{dirname $(dirname $(which ncdump))} - $SHELL{dirname $(dirname $(which pnetcdf_version))} + $SHELL{dirname $(dirname $(which ncdump))} + + + + /soft/apps/packages/climate/hdf5/1.8.16-serial/gcc-6.2.0/lib:$ENV{LD_LIBRARY_PATH} + + + $SHELL{dirname $(dirname $(which pnetcdf_version))} @@ -755,6 +770,86 @@ + + SNL clust + chama-login + wwwproxy.sandia.gov:80 + acme_integration + intel + openmpi,mpi-serial + LINUX + /gscratch/$USER/acme_scratch/chama + $CIME_OUTPUT_ROOT/$CASE/run + $CIME_OUTPUT_ROOT/$CASE/bld + /projects/ccsm/inputdata + /projects/ccsm/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + USERDEFINED_optional_run + /projects/ccsm/ccsm_baselines + /projects/ccsm/cprnc/build/cprnc_wrap + /projects/ccsm/timings + slurm + jgfouca at sandia dot gov + 8 + 16 + 16 + 1 + TRUE + fy150001 + + + mpiexec + + --bind-to-core + --n $TOTALPES + --npernode $PES_PER_NODE + + + + + + + /usr/share/Modules/init/python.py + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/sh + /usr/share/Modules/init/csh + /usr/bin/modulecmd python + /usr/bin/modulecmd perl + module + module + + + sems-env + sems-git + sems-python/2.7.9 + sems-cmake + gnu/4.9.2 + intel/intel-15.0.3.187 + openmpi-intel/1.6 + libraries/intel-mkl-15.0.2.164 + + + + + + + + + + /projects/ccsm/tpl/netcdf/4.3.2/intel/13.0.1/openmpi/1.6.5/bin:$ENV{PATH} + /projects/ccsm/tpl/netcdf/4.3.2/intel/13.0.1/openmpi/1.6.5/lib:$ENV{LD_LIBRARY_PATH} + /projects/ccsm/tpl/netcdf/4.3.2/intel/13.0.1/openmpi/1.6.5/include + /projects/ccsm/tpl/netcdf/4.3.2/intel/13.0.1/openmpi/1.6.5/lib + /projects/ccsm/tpl/netcdf/4.3.2/intel/13.0.1/openmpi/1.6.5 + /projects/ccsm/tpl/netcdf/4.3.2/intel/13.0.1/openmpi/1.6.5 + + + + + 64M + + + SNL clust redsky-login @@ -900,8 +995,8 @@ /soft/climate/netcdf/4.3.3.1c-4.2cxx-4.4.2f-serial/intel-15.0.1 /soft/climate/netcdf/4.3.3.1c-4.2cxx-4.4.2f-serial/intel-15.0.1/include /soft/climate/netcdf/4.3.3.1c-4.2cxx-4.4.2f-serial/intel-15.0.1/lib - $ENV{NETCDFROOT}/bin:$ENV{PATH} - $ENV{NETCDFROOT}/lib:/soft/intel/15.0.1/mkl/lib/intel64:$ENV{LD_LIBRARY_PATH} + /soft/climate/netcdf/4.3.3.1c-4.2cxx-4.4.2f-serial/intel-15.0.1/bin:$ENV{PATH} + /soft/climate/netcdf/4.3.3.1c-4.2cxx-4.4.2f-serial/intel-15.0.1/lib:/soft/intel/15.0.1/mkl/lib/intel64:$ENV{LD_LIBRARY_PATH} /soft/climate/pnetcdf/1.6.1/intel-15.0.1/mvapich2-2.2a-intel-15.0 @@ -998,8 +1093,8 @@ - `which nc-config | xargs dirname | xargs dirname` - `which pnetcdf_version | xargs dirname | xargs dirname` + $SHELL{which nf-config | xargs dirname | xargs dirname} + $SHELL{which pnetcdf_version | xargs dirname | xargs dirname} 256M @@ -1070,67 +1165,110 @@ - - Linux workstation: Andy's at SNL - penn - acme_developer - LINUX - gnu - openmpi,mpi-serial - $ENV{HOME}/acme/scratch - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - $ENV{HOME}/acme/inputdata - $ENV{HOME}/acme/ptclmdata - $CIME_OUTPUT_ROOT/archive/$CASE - csm/$CASE - - /home/agsalin/acme/baselines - $CCSMROOT/tools/cprnc/build/cprnc - agsalin at sandia dot gov - 20 - 20 - 2 + + LLNL Linux Cluster, Linux (pgi), 16 pes/node, batch system is Slurm + intel, pgi + mpich,mpi-serial + /p/lscratche/$CCSMUSER/ACME/$CASE/run + /p/lscratche/$CCSMUSER/$CASE/bld + /p/lscratche/$USER + /usr/gdata/climdat/ccsm3data/inputdata + /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 + /p/lscratche/$CCSMUSER/archive/$CASE + FALSE + UNSET + /p/lscratchd/$CCSMUSER/ccsm_baselines + /p/lscratchd/ma21/ccsm3data/tools/cprnc/cprnc + LINUX + bogenschutz1 -at- llnl.gov + 8 + 16 + 16 + lc_slurm + /usr/gdata/climdat/timing_performance + + + + + srun + + + use + use + /usr/global/tools/dotkit/init.csh + /usr/global/tools/dotkit/init.sh + + pgi-14.3 + mvapich2-pgi-1.7 + netcdf-pgi-4.1.3 + + + ic-14.0.211 + mvapich2-intel-2.1 + netcdf-intel-4.1.3 + + + + /usr/local/tools/netcdf-pgi-4.1.3/ + /usr/local/tools/parallel-netcdf-pgi-1.6.1/ + + + /usr/local/tools/netcdf-intel-4.1.3/ + /usr/local/tools/parallel-netcdf-intel-1.6.1/ + - - LLNL Linux Cluster, Linux (pgi), 12 pes/node, batch system is Moab + + LLNL Linux Cluster, Linux (pgi), 16 pes/node, batch system is Slurm intel, pgi mpich,mpi-serial /p/lscratche/$CCSMUSER/ACME/$CASE/run /p/lscratche/$CCSMUSER/$CASE/bld /p/lscratche/$USER - /p/lscratchd/ma21/ccsm3data/inputdata - /p/lscratchd/ma21/ccsm3data/inputdata/atm/datm7 + /usr/gdata/climdat/ccsm3data/inputdata + /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 /p/lscratche/$CCSMUSER/archive/$CASE FALSE UNSET /p/lscratchd/$CCSMUSER/ccsm_baselines /p/lscratchd/ma21/ccsm3data/tools/cprnc/cprnc LINUX - mshow - msub - - wlin -at- bnl.gov + bogenschutz1 -at- llnl.gov 8 - 12 - - - pbatch - - - walltime=01:00:00 - - - - srun - - -N {{ num_nodes }} - - + 16 + 16 + lc_slurm + /usr/gdata/climdat/timing_performance - + + + srun + + + use + use + /usr/global/tools/dotkit/init.csh + /usr/global/tools/dotkit/init.sh + + pgi-14.3 + mvapich2-pgi-1.7 + netcdf-pgi-4.1.3 + + + ic-14.0.211 + mvapich2-intel-2.1 + netcdf-intel-4.1.3 + + + + /usr/local/tools/netcdf-pgi-4.1.3/ + /usr/local/tools/parallel-netcdf-pgi-1.6.1/ + + + /usr/local/tools/netcdf-intel-4.1.3/ + /usr/local/tools/parallel-netcdf-intel-1.6.1/ + @@ -1192,6 +1330,101 @@ + + ALCF Cray XC* KNL, os is CNL, 64 pes/node, batch system is cobalt + theta.* + acme_developer + intel,gnu,cray + mpt + /projects/EarlyPerf_theta/acme/scratch/$USER + /projects/EarlyPerf_theta/acme/inputdata + /projects/EarlyPerf_theta/acme/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + /projects/EarlyPerf_theta/acme/csm/$CASE + /projects/EarlyPerf_theta/acme/baselines + /projects/EarlyPerf_theta/acme/tools/cprnc + /projects/EarlyPerf_theta/acme + CNL + cobalt_theta + acme + 8 + 64 + 64 + Theta_ESP + TRUE + -D PIO_BUILD_TIMING:BOOL=ON + + aprun + + -n $TOTALPES + -N $PES_PER_NODE + --cc depth -d $OMP_NUM_THREADS + -e OMP_STACKSIZE=128M + -e OMP_NUM_THREADS=$OMP_NUM_THREADS + + + + /opt/modules/default/init/perl.pm + /opt/modules/default/init/python.py + /opt/modules/default/init/sh + /opt/modules/default/init/csh + /opt/modules/default/bin/modulecmd perl + /opt/modules/default/bin/modulecmd python + module + module + + PrgEnv-intel + PrgEnv-cray + PrgEnv-gnu + intel + cce + cray-parallel-netcdf + cray-hdf5-parallel + pmi + cray-libsci + cray-mpich + cray-netcdf + cray-hdf5 + cray-netcdf-hdf5parallel + craype + + + PrgEnv-intel/6.0.3 + intel intel/17.0.2.174 + cray-libsci + + + PrgEnv-cray/6.0.3 + cce cce/8.5.4 + + + PrgEnv-gnu/6.0.3 + gcc gcc/6.2.0 + + + craype craype/2.5.9 + + + cray-libsci/16.09.1 + + + cray-mpich/7.5.3 + + + cray-netcdf-hdf5parallel/4.4.1.1 + cray-hdf5-parallel/1.10.0.1 + cray-parallel-netcdf/1.8.0 + + + + 1 + 1 + 128M + spread + threads + + + PNL cluster, OS is Linux, batch system is SLURM sooty @@ -1266,28 +1499,75 @@ - - PNL cluster, os is Linux (pgi), batch system is SLURM - cascade - acme_developer - LINUX - intel,nag - mpich - /dtemp/$USER/csmruns/$CASE/run - /dtemp/$USER/csmruns/$CASE/bld - /dtemp/sing201/inputdata/CAM/CSMDATA_CAM/aerocom/csmdata - /dtemp/sing201/inputdata/CAM/CSMDATA_CAM/aerocom/csmdata/atm/datm7 - /dtemp/$USER/archive/$CASE - csm/$CASE - /dtemp/sing201/acme_testing/acme_baselines/ - /dtemp/$USER/csmruns - /home/sing201/CAM/cprnc/cprnc - slurm - balwinder.singh at pnnl dot gov - 8 - 16 - - + + PNL Intel KNC cluster, OS is Linux, batch system is SLURM + LINUX + intel + mvapich2 + glogin + /dtemp/$USER + /dtemp/$USER/csmruns/$CASE/run + /dtemp/$USER/csmruns/$CASE/bld + /dtemp/$USER + /dtemp/sing201/acme/inputdata/ + /dtemp/sing201/acme/inputdata/atm/datm7 + /dtemp/sing201/$USER/archive/$CASE + UNSET + /dtemp/sing201/acme/acme_baselines + /dtemp/sing201/acme/acme_baselines/cprnc/cprnc + /dtemp/sing201/acme/timing_acme + slurm + balwinder.singh -at- pnnl.gov + 8 + 16 + 16 + TRUE + + + + + srun + + --mpi=none + --ntasks=$TOTALPES + --cpu_bind=sockets --cpu_bind=verbose + --kill-on-bad-exit + + + + /opt/lmod/5.0.1/init/env_modules_python.py + /etc/profile.d/modules.csh + /etc/profile.d/modules.sh + /opt/lmod/lmod/libexec/lmod python + module + module + + + + + python/2.7.9 + + + intel/15.0.090 + mkl/14.0 + + + mvapich2/1.9 + + + netcdf/4.3.0 + + + + 64M + $ENV{NETCDF_ROOT} + + + $ENV{MLIBHOME} + intel + + + PNL Haswell cluster, OS is Linux, batch system is SLURM LINUX @@ -1487,12 +1767,13 @@ csm/$CASE /lustre/or-hydra/cades-ccsi/scratch /lustre/or-hydra/cades-ccsi/proj-shared/project_acme/baselines - $CIMEROOT/tools/cprnc/build/cprnc + /lustre/or-hydra/cades-ccsi/proj-shared/tools/cprnc.orcondo yinj -at- ornl.gov 4 32 16 pbs + FALSE /software/dev_tools/swtree/cs400/xalt/0.7.5/centos7.3/bin/mpirun @@ -1532,6 +1813,7 @@ ORNL XK6, os is CNL, 16 pes/node, batch system is PBS titan + Received node event ec_node acme_developer pgi,pgiacc,intel,cray mpich,mpi-serial @@ -1548,9 +1830,11 @@ CNL pbs 16 + acme 8 16 + 10 TRUE cli115 -D PIO_BUILD_TIMING:BOOL=ON @@ -1773,7 +2057,7 @@ cray-parallel-netcdf/1.6.1 - cmake/2.8.11.2 + cmake3/3.2.3 python/2.7.9 @@ -1792,7 +2076,7 @@ gr-fe.*.lanl.gov acme_developer gnu,intel - openmpi + mvapich,openmpi LINUX /lustre/scratch3/turquoise/$ENV{USER}/ACME/cases/$CASE/run /lustre/scratch3/turquoise/$ENV{USER}/ACME/cases/$CASE/bld @@ -1816,7 +2100,6 @@ /usr/projects/climate/SHARED_CLIMATE/modulefiles/all - friendly-testing python/anaconda-2.7-climate @@ -1829,7 +2112,9 @@ openmpi/1.10.5 - + + mvapich2/2.2 + netcdf/4.4.1 @@ -1838,21 +2123,26 @@ - $LD_LIBRARY_PATH:$NETCDF_ROOT/lib - romio_ds_write=disable;romio_ds_read=disable + romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable moab mpirun - -n {{ num_tasks }} + -n $TOTALPES + + + + srun + + -n $TOTALPES mpirun - -n {{ num_tasks }} + -n $TOTALPES @@ -1872,57 +2162,24 @@ acme_developer intel,gnu openmpi,mvapich,mpi-serial - LINUX + /lustre/scratch3/turquoise/$ENV{USER}/ACME/scratch /lustre/scratch3/turquoise/$ENV{USER}/ACME/cases/$CASE/run /lustre/scratch3/turquoise/$ENV{USER}/ACME/cases/$CASE/bld - /lustre/scratch3/turquoise/$ENV{USER}/ACME/input_data - /lustre/scratch3/turquoise/$ENV{USER}/ACME/input_data/atm/datm7 /lustre/scratch3/turquoise/$ENV{USER}/ACME/archive/$CASE UNSET + LINUX + moab + jonbob -at- lanl.gov + 4 + 16 + 16 + climateacme + TRUE + /lustre/scratch3/turquoise/$ENV{USER}/ACME/input_data + /lustre/scratch3/turquoise/$ENV{USER}/ACME/input_data/atm/datm7 /lustre/scratch3/turquoise/$ENV{USER}/ACME/input_data/ccsm_baselines - /lustre/scratch3/turquoise/$ENV{USER}/ACME/scratch - $CASEROOT/timings /turquoise/usr/projects/climate/SHARED_CLIMATE/software/wolf/cprnc/v0.40/cprnc - - /usr/share/Modules/init/perl.pm - /usr/share/Modules/init/python.py - /usr/share/Modules/init/sh - /usr/share/Modules/init/csh - /usr/bin/modulecmd perl - /usr/bin/modulecmd python - module - module - - - /usr/projects/climate/SHARED_CLIMATE/modulefiles/all - friendly-testing - python/anaconda-2.7-climate - - - intel/15.0.5 - mkl/11.3.3 - - - gcc/4.8.2 - - - openmpi/1.6.5 - - - mvapich2/1.8 - - - netcdf/4.4.0 - - - parallel-netcdf/1.5.0 - - - - $LD_LIBRARY_PATH:$NETCDF_ROOT/lib - - - moab + $CASEROOT/timings mpirun @@ -1944,12 +2201,44 @@ - 4 - 16 - 16 - climateacme - TRUE - jonbob -at- lanl.gov + + /usr/share/Modules/init/perl.pm + /usr/share/Modules/init/python.py + /usr/share/Modules/init/sh + /usr/share/Modules/init/csh + /usr/bin/modulecmd perl + /usr/bin/modulecmd python + module + module + + + /usr/projects/climate/SHARED_CLIMATE/modulefiles/all + friendly-testing + python/anaconda-2.7-climate + + + intel/15.0.5 + mkl/11.3.3 + + + gcc/4.8.2 + + + openmpi/1.6.5 + + + mvapich2/1.8 + + + netcdf/4.4.0 + + + parallel-netcdf/1.5.0 + + + + $ENV{LD_LIBRARY_PATH}:$ENV{NETCDF_ROOT}/lib + @@ -2131,6 +2420,7 @@ Lawrencium LR3 cluster at LBL, OS is Linux (intel), batch system is SLURM LINUX + n000* intel openmpi,mpi-serial /global/scratch/$ENV{USER} @@ -2194,6 +2484,7 @@ Lawrencium LR2 cluster at LBL, OS is Linux (intel), batch system is SLURM LINUX + n000* intel openmpi,mpi-serial /global/scratch/$ENV{USER} @@ -2254,6 +2545,39 @@ + + small developer workhorse at lbl climate sciences + LINUX + gnu + openmpi,mpi-serial + ngeet + /home/lbleco/acme/ + /home/lbleco/cesm/cesm_input_datasets/ + /home/lbleco/cesm/cesm_input_datasets/atm/datm7/ + /home/lbleco/acme/cesm_archive/$CASE + csm/$CASE + /home/lbleco/acme/cesm_baselines + /home/lbleco/cesm/cesm_tools/cprnc/cprnc + 1 + none + rgknox at lbl gov + 4 + 4 + FALSE + + + + + mpirun + + -np $TOTALPES + -npernode $PES_PER_NODE + + + + + + ${EXEROOT}/acme.exe >> acme.log.$LID 2>&1 diff --git a/cime_config/acme/machines/config_pio.xml b/config/acme/machines/config_pio.xml similarity index 96% rename from cime_config/acme/machines/config_pio.xml rename to config/acme/machines/config_pio.xml index e1716e68c5b..f76c751661a 100644 --- a/cime_config/acme/machines/config_pio.xml +++ b/config/acme/machines/config_pio.xml @@ -23,6 +23,7 @@ $PES_PER_NODE 60 128 + 64 @@ -46,8 +47,10 @@ pnetcdf netcdf + netcdf netcdf netcdf + netcdf netcdf netcdf netcdf @@ -56,6 +59,8 @@ netcdf netcdf netcdf + netcdf + netcdf netcdf @@ -86,6 +91,7 @@ 1 + 1 1 diff --git a/cime_config/acme/machines/mkDepends b/config/acme/machines/mkDepends similarity index 100% rename from cime_config/acme/machines/mkDepends rename to config/acme/machines/mkDepends diff --git a/cime_config/acme/machines/mkSrcfiles b/config/acme/machines/mkSrcfiles similarity index 100% rename from cime_config/acme/machines/mkSrcfiles rename to config/acme/machines/mkSrcfiles diff --git a/cime_config/acme/machines/syslog.anvil b/config/acme/machines/syslog.anvil similarity index 93% rename from cime_config/acme/machines/syslog.anvil rename to config/acme/machines/syslog.anvil index c95edbac999..7790e2f770b 100755 --- a/cime_config/acme/machines/syslog.anvil +++ b/config/acme/machines/syslog.anvil @@ -25,6 +25,8 @@ cat > $run/Walltime.Remaining < $dir/qstatr.$lid.$remaining +qstat -1 -n acme > $dir/qstatn.$lid.$remaining while ($remaining > 0) echo "Wallclock time remaining: $remaining" >> $dir/atm.log.$lid.step @@ -41,7 +43,8 @@ while ($remaining > 0) echo "Wallclock time remaining: $remaining" >> $dir/cpl.log.$lid.step tail -n 4 $dir/cpl.log.$lid.step-all >> $dir/cpl.log.$lid.step /bin/cp --preserve=timestamps -u $timing/* $dir - qstat -f acme > $dir/qstatf.$lid.$remaining + qstat -r acme > $dir/qstatr.$lid.$remaining + qstat -1 -n acme > $dir/qstatn.$lid.$remaining chmod a+r $dir/* sleep $sample_interval set remaining = `qstat -f $jid | grep -F Walltime.Remaining | sed 's/ *Walltime.Remaining = *\([0-9]*\) */\1/' ` diff --git a/cime_config/acme/machines/syslog.cetus b/config/acme/machines/syslog.cetus similarity index 56% rename from cime_config/acme/machines/syslog.cetus rename to config/acme/machines/syslog.cetus index 17d775b5d09..77828e53473 100755 --- a/cime_config/acme/machines/syslog.cetus +++ b/config/acme/machines/syslog.cetus @@ -2,7 +2,6 @@ # cetus syslog script: # mach_syslog -set sec = 0 set sample_interval = $1 set jid = $2 set lid = $3 @@ -12,13 +11,14 @@ set dir = $6 # wait until output file is nonempty before checking remaining time # (note that calling script 'touch'es the acme log file before spawning this script, so that 'wc' does not fail) +set ncpus = `qstat -lf $jid | grep -F Procs | sed 's/^ *Procs *: *\([0-9]*\).*/\1/' ` set outlth = 0 -while ($outlth < 1) +while ($outlth < $ncpus) sleep 10 set outlth = `wc \-l $run/acme.log.$lid | sed 's/ *\([0-9]*\) *.*/\1/' ` end -set TimeRemaining = `qstat -lf $jid | grep TimeRemaining | sed 's/^ *TimeRemaining *: *\([0-9]*:[0-9]*:[0-9]*\) */\1/' ` +set TimeRemaining = `qstat -lf $jid | grep -F TimeRemaining | sed 's/^ *TimeRemaining *: *\([0-9]*:[0-9]*:[0-9]*\) */\1/' ` set rem_hours = `echo $TimeRemaining | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` set rem_mins = `echo $TimeRemaining | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` set rem_secs = `echo $TimeRemaining | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` @@ -32,16 +32,24 @@ EOF1 /bin/cp --preserve=timestamps $run/acme.log.$lid $dir/acme.log.$lid.$remaining while ($remaining > 0) - grep -a -i -e "nstep" -e "model date" $run/*atm.log.$lid | tail > $dir/atm.log.$lid.nstep.$remaining -# grep -a -i "nstep" $run/acme.log.$lid | tail > $dir/acme.log.$lid.nstep.$remaining - grep -a -i -e "timestep" -e "model date" $run/*lnd.log.$lid | tail > $dir/lnd.log.$lid.timestep.$remaining - grep -a -i -e "timestep" -e "Step number" -e "model date" $run/*ocn.log.$lid | tail > $dir/ocn.log.$lid.stepnum.$remaining - grep -a -i -e "timestep" -e "istep" -e "model date" $run/*ice.log.$lid | tail > $dir/ice.log.$lid.istep.$remaining - grep -a -i "model date" $run/*cpl.log.$lid | tail > $dir/cpl.log.$lid.modeldata.$remaining + echo "Wallclock time remaining: $remaining" >> $dir/atm.log.$lid.step + grep -Fa -e "nstep" -e "model date" $run/*atm.log.$lid | tail -n 4 >> $dir/atm.log.$lid.step + echo "Wallclock time remaining: $remaining" >> $dir/lnd.log.$lid.step + grep -Fa -e "timestep" -e "model date" $run/*lnd.log.$lid | tail -n 4 >> $dir/lnd.log.$lid.step + echo "Wallclock time remaining: $remaining" >> $dir/ocn.log.$lid.step + grep -Fa -e "timestep" -e "Step number" -e "model date" $run/*ocn.log.$lid | tail -n 4 >> $dir/ocn.log.$lid.step + echo "Wallclock time remaining: $remaining" >> $dir/ice.log.$lid.step + grep -Fa -e "timestep" -e "istep" -e "model date" $run/*ice.log.$lid | tail -n 4 >> $dir/ice.log.$lid.step + echo "Wallclock time remaining: $remaining" >> $dir/rof.log.$lid.step + grep -Fa "model date" $run/*rof.log.$lid | tail -n 4 >> $dir/rof.log.$lid.step + grep -Fa "model date" $run/*cpl.log.$lid > $dir/cpl.log.$lid.step-all + echo "Wallclock time remaining: $remaining" >> $dir/cpl.log.$lid.step + tail -n 4 $dir/cpl.log.$lid.step-all >> $dir/cpl.log.$lid.step /bin/cp --preserve=timestamps -u $timing/* $dir + qstat -f > $dir/qstatf.$lid.$remaining chmod a+r $dir/* sleep $sample_interval - set TimeRemaining = `qstat -lf $jid | grep TimeRemaining | sed 's/^ *TimeRemaining *: *\([0-9]*:[0-9]*:[0-9]*\) */\1/' ` + set TimeRemaining = `qstat -lf $jid | grep -F TimeRemaining | sed 's/^ *TimeRemaining *: *\([0-9]*:[0-9]*:[0-9]*\) */\1/' ` set rem_hours = `echo $TimeRemaining | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` set rem_mins = `echo $TimeRemaining | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` set rem_secs = `echo $TimeRemaining | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` diff --git a/cime_config/acme/machines/syslog.cori-haswell b/config/acme/machines/syslog.cori-haswell similarity index 52% rename from cime_config/acme/machines/syslog.cori-haswell rename to config/acme/machines/syslog.cori-haswell index 548d0446b3d..2fc5e2e3be5 100755 --- a/cime_config/acme/machines/syslog.cori-haswell +++ b/config/acme/machines/syslog.cori-haswell @@ -1,8 +1,7 @@ #!/bin/csh -f # cori-haswell syslog script: -# mach_syslog @@ -1991,10 +1891,10 @@ 1.5 1.e30 - 2.0 - 2.0 - 2.0 - 2.0 + 2.0 + 2.0 + 2.0 + 2.0 2.0 @@ -2014,12 +1914,8 @@ null - null - u:v u:v u:v - u:v - null @@ -2147,7 +2043,7 @@ 1 - -1 + -1 diff --git a/components/data_comps/datm/cime_config/user_nl_datm b/src/components/data_comps/datm/cime_config/user_nl_datm similarity index 100% rename from components/data_comps/datm/cime_config/user_nl_datm rename to src/components/data_comps/datm/cime_config/user_nl_datm diff --git a/components/data_comps/datm/datm_comp_mod.F90 b/src/components/data_comps/datm/datm_comp_mod.F90 similarity index 98% rename from components/data_comps/datm/datm_comp_mod.F90 rename to src/components/data_comps/datm/datm_comp_mod.F90 index 48a7d63e417..e6099e42b63 100644 --- a/components/data_comps/datm/datm_comp_mod.F90 +++ b/src/components/data_comps/datm/datm_comp_mod.F90 @@ -475,7 +475,6 @@ subroutine datm_comp_init( EClock, cdata, x2a, a2x, NLFilename ) if (trim(atm_mode) == 'NULL' .or. & trim(atm_mode) == 'CORE2_NYF' .or. & trim(atm_mode) == 'CORE2_IAF' .or. & - trim(atm_mode) == 'WRF' .or. & trim(atm_mode) == 'CLMNCEP' .or. & trim(atm_mode) == 'CPLHIST' .or. & trim(atm_mode) == 'COPYALL' ) then @@ -975,31 +974,6 @@ subroutine datm_comp_run( EClock, cdata, x2a, a2x) case('CPLHIST') ! do nothing extra - case ('WRF') - lsize = mct_avect_lsize(a2x) - do n = 1,lsize - - !--- fabricate required swdn components from total swdn --- - a2x%rAttr(kswvdr,n) = avstrm%rAttr(sswdn,n)*(0.28_R8) - a2x%rAttr(kswndr,n) = avstrm%rAttr(sswdn,n)*(0.31_R8) - a2x%rAttr(kswvdf,n) = avstrm%rAttr(sswdn,n)*(0.24_R8) - a2x%rAttr(kswndf,n) = avstrm%rAttr(sswdn,n)*(0.17_R8) - - !--- just a diagnostic, not really needed - a2x%rAttr(kswnet,n) = avstrm%rAttr(sswdn,n)-avstrm%rAttr(sswup,n) - - !--- convert from hPa - a2x%rAttr(kpslv,n) = a2x%rAttr(kpslv,n)*100._R8 - a2x%rAttr(kpbot,n) = a2x%rAttr(kpbot,n)*100._R8 - - !--- tcraig, file has terrain height on it, set to 10m - a2x%rAttr(kz,n) = 10.0_R8 - - !--- convert to degK from degC - a2x%rAttr(ktbot,n) = a2x%rAttr(ktbot,n) + tKFrz - - enddo - case('CORE2_NYF','CORE2_IAF') if (firstcall) then if (sprec < 1 .or. sswdn < 1) then diff --git a/components/data_comps/datm/datm_shr_mod.F90 b/src/components/data_comps/datm/datm_shr_mod.F90 similarity index 100% rename from components/data_comps/datm/datm_shr_mod.F90 rename to src/components/data_comps/datm/datm_shr_mod.F90 diff --git a/components/data_comps/datm/doc/SOM.pdf b/src/components/data_comps/datm/doc/SOM.pdf similarity index 100% rename from components/data_comps/datm/doc/SOM.pdf rename to src/components/data_comps/datm/doc/SOM.pdf diff --git a/components/data_comps/datm/doc/bookinfo.xml b/src/components/data_comps/datm/doc/bookinfo.xml similarity index 100% rename from components/data_comps/datm/doc/bookinfo.xml rename to src/components/data_comps/datm/doc/bookinfo.xml diff --git a/components/data_comps/datm/doc/datacomps.xml b/src/components/data_comps/datm/doc/datacomps.xml similarity index 100% rename from components/data_comps/datm/doc/datacomps.xml rename to src/components/data_comps/datm/doc/datacomps.xml diff --git a/components/data_comps/datm/doc/intro.xml b/src/components/data_comps/datm/doc/intro.xml similarity index 100% rename from components/data_comps/datm/doc/intro.xml rename to src/components/data_comps/datm/doc/intro.xml diff --git a/components/data_comps/datm/doc/rundocbook.csh b/src/components/data_comps/datm/doc/rundocbook.csh similarity index 100% rename from components/data_comps/datm/doc/rundocbook.csh rename to src/components/data_comps/datm/doc/rundocbook.csh diff --git a/components/data_comps/datm/doc/streams.xml b/src/components/data_comps/datm/doc/streams.xml similarity index 99% rename from components/data_comps/datm/doc/streams.xml rename to src/components/data_comps/datm/doc/streams.xml index e3175863120..6d3ad3066bf 100644 --- a/components/data_comps/datm/doc/streams.xml +++ b/src/components/data_comps/datm/doc/streams.xml @@ -77,7 +77,7 @@ The strdata (short for "stream data") input is set via a fortran namelist called shr_strdata_nml. That namelist, the strdata datatype, and the methods are contained in the share source - code file, models/csm_share/shr/shr_strdata_mod.F90. In general, + code file, share/util/shr_strdata_mod.F90. In general, strdata input defines an array of input streams and operations to perform on those streams. Therefore, many namelist inputs are arrays of character strings. Different variable of the same index are diff --git a/components/data_comps/datm/doc/ug.xml b/src/components/data_comps/datm/doc/ug.xml similarity index 100% rename from components/data_comps/datm/doc/ug.xml rename to src/components/data_comps/datm/doc/ug.xml diff --git a/components/data_comps/desp/cime_config/buildlib b/src/components/data_comps/desp/cime_config/buildlib similarity index 90% rename from components/data_comps/desp/cime_config/buildlib rename to src/components/data_comps/desp/cime_config/buildlib index 5f9588e3652..682cbf335f5 100755 --- a/components/data_comps/desp/cime_config/buildlib +++ b/src/components/data_comps/desp/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/desp/cime_config/buildnml b/src/components/data_comps/desp/cime_config/buildnml similarity index 97% rename from components/data_comps/desp/cime_config/buildnml rename to src/components/data_comps/desp/cime_config/buildnml index 45ac5321569..6c938536441 100755 --- a/components/data_comps/desp/cime_config/buildnml +++ b/src/components/data_comps/desp/cime_config/buildnml @@ -11,7 +11,7 @@ import os, shutil, sys, glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -65,7 +65,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): config['desp_mode'] = desp_mode #---------------------------------------------------- - # Initialize namelist defaults + # Initialize namelist defaults #---------------------------------------------------- nmlgen.init_defaults(infile, config) @@ -99,7 +99,7 @@ def buildnml(case, caseroot, compname): os.makedirs(confdir) #---------------------------------------------------- - # Construct the namelist generator + # Construct the namelist generator #---------------------------------------------------- # Determine directory for user modified namelist_definitions.xml and namelist_defaults.xml user_xml_dir = os.path.join(caseroot, "SourceMods", "src." + compname) @@ -107,7 +107,7 @@ def buildnml(case, caseroot, compname): "user_xml_dir %s does not exist " %user_xml_dir) # NOTE: User definition *replaces* existing definition. - namelist_xml_dir = os.path.join(cimeroot, "components", "data_comps", compname, "cime_config") + namelist_xml_dir = os.path.join(cimeroot, "src", "components", "data_comps", compname, "cime_config") definition_file = [os.path.join(namelist_xml_dir, "namelist_definition_desp.xml")] user_definition = os.path.join(user_xml_dir, "namelist_definition_desp.xml") if os.path.isfile(user_definition): @@ -117,12 +117,11 @@ def buildnml(case, caseroot, compname): # Create the namelist generator object - independent of instance nmlgen = NamelistGenerator(case, definition_file) - + #---------------------------------------------------- # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/desp/cime_config/config_component.xml b/src/components/data_comps/desp/cime_config/config_component.xml similarity index 100% rename from components/data_comps/desp/cime_config/config_component.xml rename to src/components/data_comps/desp/cime_config/config_component.xml diff --git a/components/data_comps/desp/cime_config/namelist_definition_desp.xml b/src/components/data_comps/desp/cime_config/namelist_definition_desp.xml similarity index 100% rename from components/data_comps/desp/cime_config/namelist_definition_desp.xml rename to src/components/data_comps/desp/cime_config/namelist_definition_desp.xml diff --git a/components/data_comps/desp/cime_config/user_nl_desp b/src/components/data_comps/desp/cime_config/user_nl_desp similarity index 100% rename from components/data_comps/desp/cime_config/user_nl_desp rename to src/components/data_comps/desp/cime_config/user_nl_desp diff --git a/components/data_comps/desp/desp_comp_mod.F90 b/src/components/data_comps/desp/desp_comp_mod.F90 similarity index 90% rename from components/data_comps/desp/desp_comp_mod.F90 rename to src/components/data_comps/desp/desp_comp_mod.F90 index ab86eb3c241..d48b5e06a5d 100644 --- a/components/data_comps/desp/desp_comp_mod.F90 +++ b/src/components/data_comps/desp/desp_comp_mod.F90 @@ -15,7 +15,8 @@ module desp_comp_mod use shr_strdata_mod, only: shr_strdata_type, shr_strdata_advance use shr_strdata_mod, only: shr_strdata_pioinit - use seq_timemgr_mod, only: seq_timemgr_EClockGetData, seq_timemgr_RestartAlarmIsOn + use seq_timemgr_mod, only: seq_timemgr_EClockGetData + use seq_timemgr_mod, only: seq_timemgr_RestartAlarmIsOn use seq_comm_mct, only: seq_comm_inst, seq_comm_name, seq_comm_suffix implicit none @@ -30,39 +31,29 @@ module desp_comp_mod public :: desp_comp_final !-------------------------------------------------------------------------- -! Public data +! Public module data !-------------------------------------------------------------------------- - - integer, parameter, public :: atm_ind = 1 - integer, parameter, public :: lnd_ind = 2 - integer, parameter, public :: ice_ind = 3 - integer, parameter, public :: ocn_ind = 4 - integer, parameter, public :: glc_ind = 5 - integer, parameter, public :: rof_ind = 6 - integer, parameter, public :: wav_ind = 7 - integer, parameter, public :: cpl_ind = 8 - integer, parameter, public :: max_ind = cpl_ind + integer, public, parameter :: desp_num_comps = 8 + character(len=3), public, parameter :: comp_names(desp_num_comps) = & + (/ 'atm', 'lnd', 'ice', 'ocn', 'glc', 'rof', 'wav', 'drv' /) !-------------------------------------------------------------------------- ! Private data !-------------------------------------------------------------------------- - character(len=CS) :: myModelName = 'esp' ! user defined model name + character(len=CS) :: myModelName = 'esp' ! user defined model name integer(IN) :: mpicom - integer(IN) :: COMPID ! mct comp id - integer(IN) :: my_task ! my task in mpi communicator mpicom - integer(IN) :: npes ! total number of tasks - integer(IN), parameter :: master_task=0 ! task number of master task - integer(IN) :: logunit ! logging unit number + integer(IN) :: COMPID ! mct comp id + integer(IN) :: my_task ! my task in mpi communicator mpicom + integer(IN) :: npes ! total number of tasks + integer(IN), parameter :: master_task=0 ! task number of master task + integer(IN) :: logunit ! logging unit number integer(IN) :: loglevel - integer :: inst_index ! number of current instance (ie. 1) - character(len=16) :: inst_name ! fullname of current instance (ie. "lnd_0001") - character(len=16) :: inst_suffix ! char string associated with instance - ! (ie. "_0001" or "") - character(len=CL) :: desp_mode ! mode of operation - - character(len=3), parameter :: comp_names(8) = (/ 'atm', 'lnd', 'ice', & - 'ocn', 'glc', 'rof', 'wav', 'drv' /) + integer :: inst_index ! number of current instance (ie. 1) + character(len=16) :: inst_name ! fullname of current instance (ie. "lnd_0001") + character(len=16) :: inst_suffix ! char string associated with instance + ! (ie. "_0001" or "") + character(len=CL) :: desp_mode ! mode of operation character(len=*), parameter :: rpprefix = 'rpointer.' character(len=*), parameter :: rpfile = rpprefix//'esp' character(len=*), parameter :: nullstr = 'undefined' @@ -148,16 +139,10 @@ subroutine desp_comp_init(EClock, espid, mpicom_in, phase, read_restart, & desp_mode, info_debug, restfilm !--- formats --- - character(*), parameter :: F00 = "('(desp_comp_init) ',8a)" - character(*), parameter :: F0L = "('(desp_comp_init) ',a, l2)" - character(*), parameter :: F01 = "('(desp_comp_init) ',a,5i8)" - character(*), parameter :: F02 = "('(desp_comp_init) ',a,4es13.6)" - character(*), parameter :: F03 = "('(desp_comp_init) ',a,i8,a)" - character(*), parameter :: F04 = "('(desp_comp_init) ',2a,2i8,'s')" - character(*), parameter :: F05 = "('(desp_comp_init) ',a,2f10.4)" - character(*), parameter :: F90 = "('(desp_comp_init) ',73('='))" - character(*), parameter :: F91 = "('(desp_comp_init) ',73('-'))" character(*), parameter :: subName = "(desp_comp_init) " + character(*), parameter :: F00 = "('"//subName//"',8a)" + character(*), parameter :: F01 = "('"//subName//"',a,5i8)" + character(*), parameter :: F04 = "('"//subName//"',2a,2i8,'s')" !------------------------------------------------------------------------------- call t_startf('DESP_INIT') @@ -219,8 +204,8 @@ subroutine desp_comp_init(EClock, espid, mpicom_in, phase, read_restart, & write(logunit,F01) 'ERROR: reading input namelist, '//trim(filename)//' iostat=',ierr call shr_sys_abort(subName//': namelist read error '//trim(filename)) end if - write(logunit,F01)' info_debug = ',info_debug - write(logunit,F00)' restfilm = ',trim(restfilm) + write(logunit,F01) 'info_debug = ',info_debug + write(logunit,F00) 'restfilm = ',trim(restfilm) write(logunit,F01) 'inst_index = ',inst_index write(logunit,F00) 'inst_name = ',trim(inst_name) write(logunit,F00) 'inst_suffix = ',trim(inst_suffix) @@ -360,7 +345,7 @@ subroutine desp_comp_run(EClock, case_name, pause_sig, atm_resume, & type(ESMF_Clock), intent(in) :: EClock character(len=*), intent(in) :: case_name - logical, intent(in) :: pause_sig(max_ind) + logical, intent(in) :: pause_sig(desp_num_comps) character(len=CL), intent(inout) :: atm_resume(num_inst_atm) character(len=CL), intent(inout) :: lnd_resume(num_inst_lnd) character(len=CL), intent(inout) :: rof_resume(num_inst_rof) @@ -445,45 +430,45 @@ subroutine desp_comp_run(EClock, case_name, pause_sig, atm_resume, & ! Find the active components and their restart files ! Note hard-coded variable names are just for testing. This could be ! changed if this feature comes to regular use - do ind = 1, max_ind + do ind = 1, desp_num_comps if (pause_sig(ind)) then - select case (ind) - case(atm_ind) + select case (comp_names(ind)) + case('atm') call get_restart_filenames(ind, atm_resume, errcode) allocate(rfilenames(size(atm_resume))) rfilenames = atm_resume varname = 'T' - case(lnd_ind) + case('lnd') call get_restart_filenames(ind, lnd_resume, errcode) allocate(rfilenames(size(lnd_resume))) rfilenames = lnd_resume varname = 'T' - case(ice_ind) + case('ice') call get_restart_filenames(ind, ice_resume, errcode) allocate(rfilenames(size(ice_resume))) rfilenames = ice_resume varname = 'T' - case(ocn_ind) + case('ocn') call get_restart_filenames(ind, ocn_resume, errcode) allocate(rfilenames(size(ocn_resume))) rfilenames = ocn_resume - varname = 'T' - case(glc_ind) + varname = 'PSURF_CUR' + case('glc') call get_restart_filenames(ind, glc_resume, errcode) allocate(rfilenames(size(glc_resume))) rfilenames = glc_resume varname = 'T' - case(rof_ind) + case('rof') call get_restart_filenames(ind, rof_resume, errcode) allocate(rfilenames(size(rof_resume))) rfilenames = rof_resume varname = 'T' - case(wav_ind) + case('wav') call get_restart_filenames(ind, wav_resume, errcode) allocate(rfilenames(size(wav_resume))) rfilenames = wav_resume varname = 'T' - case(cpl_ind) + case('drv') call get_restart_filenames(ind, cpl_resume, errcode) allocate(rfilenames(1)) rfilenames(1) = cpl_resume @@ -520,6 +505,9 @@ subroutine desp_comp_run(EClock, case_name, pause_sig, atm_resume, & write(logunit, *) subname, 'Found restart file ',trim(rfilenames(inst)) end if call esp_pio_modify_variable(COMPID, mpicom, rfilenames(inst), varname, var_found) + if (.not. var_found) then + call shr_sys_abort(subname//'Variable, '//trim(varname)//', not found on '//rfilenames(inst)) + end if end do case (null_mode) ! Since DESP is not 'present' for this mode, we should not get here. @@ -642,22 +630,22 @@ subroutine get_restart_filenames_a(comp_ind, filenames, retcode) filenames = ' ' num_inst = size(filenames) allocate(ids(num_inst)) - select case (comp_ind) - case(atm_ind) + select case (comp_names(comp_ind)) + case('atm') ids = ATMID - case(lnd_ind) + case('lnd') ids = LNDID - case(ice_ind) + case('ice') ids = ICEID - case(ocn_ind) + case('ocn') ids = OCNID - case(glc_ind) + case('glc') ids = GLCID - case(rof_ind) + case('rof') ids = ROFID - case(wav_ind) + case('wav') ids = WAVID - case(cpl_ind) + case('drv') ids = CPLID case default call shr_sys_abort(subname//'Unrecognized comp_ind') @@ -668,6 +656,11 @@ subroutine get_restart_filenames_a(comp_ind, filenames, retcode) rpointer_name = rpprefix//comp_names(comp_ind)//trim(seq_comm_suffix(ids(ind))) if (my_task == master_task) then inquire(file=rpointer_name, EXIST=file_exists) + ! POP decided to not follow the convention + if ((.not. file_exists) .and. (trim(comp_names(comp_ind)) == 'ocn')) then + rpointer_name = rpprefix//comp_names(comp_ind)//".restart"//trim(seq_comm_suffix(ids(ind))) + inquire(file=rpointer_name, EXIST=file_exists) + end if if (.not. file_exists) then retcode = NO_RPOINTER else diff --git a/components/data_comps/desp/esp_comp_mct.F90 b/src/components/data_comps/desp/esp_comp_mct.F90 similarity index 83% rename from components/data_comps/desp/esp_comp_mct.F90 rename to src/components/data_comps/desp/esp_comp_mct.F90 index 549a1ce1027..3f3c07a9887 100644 --- a/components/data_comps/desp/esp_comp_mct.F90 +++ b/src/components/data_comps/desp/esp_comp_mct.F90 @@ -2,10 +2,11 @@ module esp_comp_mct ! !USES: - use esmf, only: ESMF_Clock - use mct_mod, only: mct_aVect - use seq_cdata_mod, only: seq_cdata - use seq_infodata_mod, only: seq_infodata_type + use esmf, only: ESMF_Clock + use mct_mod, only: mct_aVect + use seq_cdata_mod, only: seq_cdata + use seq_infodata_mod, only: seq_infodata_type + use desp_comp_mod, only: desp_num_comps ! !PUBLIC TYPES: implicit none @@ -19,6 +20,11 @@ module esp_comp_mct public :: esp_run_mct public :: esp_final_mct + !-------------------------------------------------------------------------- + ! Private module data + !-------------------------------------------------------------------------- + integer :: comp_index(desp_num_comps) + !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONTAINS !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -36,9 +42,11 @@ module esp_comp_mct ! !INTERFACE: --------------------------------------------------------------- subroutine esp_init_mct(EClock, cdata, x2a, a2x, NLFilename) - use desp_comp_mod, only: desp_comp_init + use desp_comp_mod, only: desp_comp_init, comp_names use seq_cdata_mod, only: seq_cdata_setptrs use seq_infodata_mod, only: seq_infodata_putData, seq_infodata_GetData + use seq_timemgr_mod, only: seq_timemgr_pause_component_index + ! !INPUT/OUTPUT PARAMETERS: @@ -49,6 +57,7 @@ subroutine esp_init_mct(EClock, cdata, x2a, a2x, NLFilename) !EOP + integer :: ind integer :: ESPID integer :: mpicom_esp integer :: esp_phase @@ -73,6 +82,11 @@ subroutine esp_init_mct(EClock, cdata, x2a, a2x, NLFilename) call seq_infodata_PutData(infodata, & esp_present=esp_present, esp_prognostic=esp_prognostic) + ! Retrieve component indices from the time manager + do ind = 1, desp_num_comps + comp_index(ind) = seq_timemgr_pause_component_index(comp_names(ind)) + end do + end subroutine esp_init_mct !============================================================================ @@ -88,14 +102,14 @@ end subroutine esp_init_mct ! !INTERFACE: ------------------------------------------------------------------ subroutine esp_run_mct( EClock, cdata, x2a, a2x) + use shr_kind_mod, only: CL=>SHR_KIND_CL use seq_comm_mct, only: num_inst_atm, num_inst_lnd, num_inst_rof use seq_comm_mct, only: num_inst_ocn, num_inst_ice, num_inst_glc use seq_comm_mct, only: num_inst_wav - use desp_comp_mod, only: desp_comp_run, atm_ind, lnd_ind, ocn_ind - use desp_comp_mod, only: ice_ind, glc_ind, rof_ind, wav_ind, cpl_ind, max_ind + use desp_comp_mod, only: desp_comp_run use seq_cdata_mod, only: seq_cdata_setptrs use seq_infodata_mod, only: seq_infodata_putData, seq_infodata_GetData - use shr_kind_mod, only: CL=>SHR_KIND_CL + use seq_timemgr_mod, only: seq_timemgr_pause_component_active ! !INPUT/OUTPUT PARAMETERS: @@ -106,8 +120,9 @@ subroutine esp_run_mct( EClock, cdata, x2a, a2x) !EOP + integer :: ind type(seq_infodata_type), pointer :: infodata - logical :: pause_sig(max_ind) + logical :: pause_sig(desp_num_comps) character(len=CL) :: atm_resume(num_inst_atm) character(len=CL) :: lnd_resume(num_inst_lnd) character(len=CL) :: rof_resume(num_inst_rof) @@ -120,16 +135,13 @@ subroutine esp_run_mct( EClock, cdata, x2a, a2x) character(len=*), parameter :: subName = "(esp_run_mct) " !-------------------------------------------------------------------------- - ! Grab infodata + ! Grab infodata and case name call seq_cdata_setptrs(cdata, infodata=infodata) + call seq_infodata_GetData(infodata, case_name=case_name) ! Find out if we should be running - ! The data ESP component only runs during a pause alarm - call seq_infodata_GetData(infodata, atm_pause=pause_sig(atm_ind), & - lnd_pause=pause_sig(lnd_ind), ocn_pause=pause_sig(ocn_ind), & - ice_pause=pause_sig(ice_ind), glc_pause=pause_sig(glc_ind), & - rof_pause=pause_sig(rof_ind), wav_pause=pause_sig(wav_ind), & - cpl_pause=pause_sig(cpl_ind), case_name=case_name) - + do ind = 1, desp_num_comps + pause_sig(ind) = seq_timemgr_pause_component_active(comp_index(ind)) + end do call desp_comp_run(EClock, case_name, pause_sig, atm_resume, lnd_resume, & rof_resume, ocn_resume, ice_resume, glc_resume, wav_resume, & cpl_resume) diff --git a/components/data_comps/desp/esp_utils.F90 b/src/components/data_comps/desp/esp_utils.F90 similarity index 100% rename from components/data_comps/desp/esp_utils.F90 rename to src/components/data_comps/desp/esp_utils.F90 diff --git a/components/data_comps/dice/cime_config/buildlib b/src/components/data_comps/dice/cime_config/buildlib similarity index 90% rename from components/data_comps/dice/cime_config/buildlib rename to src/components/data_comps/dice/cime_config/buildlib index 4730f0ea102..02b3e57703a 100755 --- a/components/data_comps/dice/cime_config/buildlib +++ b/src/components/data_comps/dice/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/dice/cime_config/buildnml b/src/components/data_comps/dice/cime_config/buildnml similarity index 99% rename from components/data_comps/dice/cime_config/buildnml rename to src/components/data_comps/dice/cime_config/buildnml index e4d2be36b20..eef42ff3925 100755 --- a/components/data_comps/dice/cime_config/buildnml +++ b/src/components/data_comps/dice/cime_config/buildnml @@ -11,7 +11,7 @@ import os, shutil, sys, glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -164,7 +164,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/dice/cime_config/config_archive.xml b/src/components/data_comps/dice/cime_config/config_archive.xml similarity index 100% rename from components/data_comps/dice/cime_config/config_archive.xml rename to src/components/data_comps/dice/cime_config/config_archive.xml diff --git a/components/data_comps/dice/cime_config/config_component.xml b/src/components/data_comps/dice/cime_config/config_component.xml similarity index 95% rename from components/data_comps/dice/cime_config/config_component.xml rename to src/components/data_comps/dice/cime_config/config_component.xml index 26591c92f2b..92bc2a30261 100644 --- a/components/data_comps/dice/cime_config/config_component.xml +++ b/src/components/data_comps/dice/cime_config/config_component.xml @@ -21,7 +21,6 @@ ssmi ssmi_iaf prescribed - copyall null run_component_dice @@ -48,7 +47,6 @@ dice mode is ssmi: dice mode is ssmi_iaf: dice mode is prescribed: - dice mode is copy: dice mode is null: diff --git a/components/data_comps/dice/cime_config/namelist_definition_dice.xml b/src/components/data_comps/dice/cime_config/namelist_definition_dice.xml similarity index 95% rename from components/data_comps/dice/cime_config/namelist_definition_dice.xml rename to src/components/data_comps/dice/cime_config/namelist_definition_dice.xml index 57b32f6bce5..e5b2cdb7db0 100644 --- a/components/data_comps/dice/cime_config/namelist_definition_dice.xml +++ b/src/components/data_comps/dice/cime_config/namelist_definition_dice.xml @@ -25,9 +25,9 @@ 'streamN.txt year_align year_first year_last ' Currently the following datamode values are supported - SSMI_IAF - SSMI NULL + COPYALL + SSTDATA **********IMPORTANT NOTE: ************* In the value matches below, regular expressions are used **** @@ -55,11 +55,10 @@ streams_file List of streams used for the given dice_mode. - NULL - SSMI - SSMI_IAF + NULL + SSMI + SSMI_IAF prescribed - copyall @@ -69,7 +68,6 @@ derived does not appear in namelist - only used to set domain and data information - $SSTICE_GRID_FILENAME $SSTICE_GRID_FILENAME @@ -80,7 +78,6 @@ derived does not appear in namelist - only used to set domain and data information - $SSTICE_GRID_FILENAME $SSTICE_GRID_FILENAME @@ -122,11 +119,6 @@ area area mask mask - - time time - gridLon lon - gridLat lat - @@ -226,9 +218,6 @@ ifrac ifrac - - iceCon ifrac - @@ -252,7 +241,6 @@ 1 1 $SSTICE_YEAR_ALIGN - $SSTICE_YEAR_ALIGN @@ -266,7 +254,6 @@ 1 1948 $SSTICE_YEAR_START - $SSTICE_YEAR_START @@ -280,7 +267,6 @@ 1 2009 $SSTICE_YEAR_END - $SSTICE_YEAR_END @@ -326,11 +312,10 @@ If DICE_MODE is set to null, datamodel will be set to NULL - NULL - SSTDATA - SSTDATA + NULL + SSTDATA + SSTDATA SSTDATA - COPYALL @@ -347,7 +332,7 @@ $ICE_DOMAIN_PATH/$ICE_DOMAIN_FILE $SSTICE_GRID_FILENAME - $SSTICE_GRID_FILENAME + $SSTICE_GRID_FILENAME diff --git a/components/data_comps/dice/cime_config/user_nl_dice b/src/components/data_comps/dice/cime_config/user_nl_dice similarity index 100% rename from components/data_comps/dice/cime_config/user_nl_dice rename to src/components/data_comps/dice/cime_config/user_nl_dice diff --git a/components/data_comps/dice/dice_comp_mod.F90 b/src/components/data_comps/dice/dice_comp_mod.F90 similarity index 99% rename from components/data_comps/dice/dice_comp_mod.F90 rename to src/components/data_comps/dice/dice_comp_mod.F90 index ec31b04cdb7..692b07c2c64 100644 --- a/components/data_comps/dice/dice_comp_mod.F90 +++ b/src/components/data_comps/dice/dice_comp_mod.F90 @@ -621,7 +621,6 @@ subroutine dice_comp_run( EClock, cdata, x2i, i2x) integer(IN) :: nl ! ice frac index integer(IN) :: lsize ! size of attr vect integer(IN) :: shrlogunit, shrloglev ! original log unit and level - logical :: glcrun_alarm ! is glc going to run now logical :: newdata ! has newdata been read logical :: mssrmlf ! remove old data integer(IN) :: idt ! integer timestep diff --git a/components/data_comps/dice/ice_comp_mct.F90 b/src/components/data_comps/dice/ice_comp_mct.F90 similarity index 100% rename from components/data_comps/dice/ice_comp_mct.F90 rename to src/components/data_comps/dice/ice_comp_mct.F90 diff --git a/components/data_comps/dlnd/cime_config/buildlib b/src/components/data_comps/dlnd/cime_config/buildlib similarity index 90% rename from components/data_comps/dlnd/cime_config/buildlib rename to src/components/data_comps/dlnd/cime_config/buildlib index 81bad4d7411..9f624fae95c 100755 --- a/components/data_comps/dlnd/cime_config/buildlib +++ b/src/components/data_comps/dlnd/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/dlnd/cime_config/buildnml b/src/components/data_comps/dlnd/cime_config/buildnml similarity index 99% rename from components/data_comps/dlnd/cime_config/buildnml rename to src/components/data_comps/dlnd/cime_config/buildnml index 04c6de1f9d6..46b0d82ad1a 100755 --- a/components/data_comps/dlnd/cime_config/buildnml +++ b/src/components/data_comps/dlnd/cime_config/buildnml @@ -1,6 +1,6 @@ #!/usr/bin/env python -"""Namelist creator for CIME's data ocn model. +"""Namelist creator for CIME's data land model. """ # Typically ignore this. @@ -11,7 +11,7 @@ import os, shutil, sys, glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -84,13 +84,11 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): # Construct the list of streams. #---------------------------------------------------- streams = nmlgen.get_streams() - #---------------------------------------------------- # For each stream, create stream text file and update # shr_strdata_nml group and input data list. #---------------------------------------------------- for stream in streams: - # Ignore null values. if stream is None or stream in ("NULL", ""): continue @@ -162,12 +160,10 @@ def buildnml(case, caseroot, compname): # Create the namelist generator object - independent of instance nmlgen = NamelistGenerator(case, definition_file, files=files) - #---------------------------------------------------- # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/dlnd/cime_config/config_archive.xml b/src/components/data_comps/dlnd/cime_config/config_archive.xml similarity index 100% rename from components/data_comps/dlnd/cime_config/config_archive.xml rename to src/components/data_comps/dlnd/cime_config/config_archive.xml diff --git a/components/data_comps/dlnd/cime_config/config_component.xml b/src/components/data_comps/dlnd/cime_config/config_component.xml similarity index 54% rename from components/data_comps/dlnd/cime_config/config_component.xml rename to src/components/data_comps/dlnd/cime_config/config_component.xml index f74f63c47d7..3e8d2a02d61 100644 --- a/components/data_comps/dlnd/cime_config/config_component.xml +++ b/src/components/data_comps/dlnd/cime_config/config_component.xml @@ -34,11 +34,11 @@ char UNSET - $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 - $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG1850CN.f09_g16.002_c121001 - $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 - $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BGRCP85CN.f09_g16.002_c121001 - $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 + $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 + $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG1850CN.f09_g16.002_c121001 + $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 + $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BGRCP85CN.f09_g16.002_c121001 + $DIN_LOC_ROOT/lnd/dlnd7/CPLHIST_SNO/b.e10.BG20TRCN.f09_g16.002_c121001 run_component_dlnd env_run.xml @@ -49,11 +49,11 @@ char UNSET - b.e10.BG20TRCN.f09_g16.002 - b.e10.BG1850CN.f09_g16.002 - b.e10.BG20TRCN.f09_g16.002 - b.e10.BG20TRCN.f09_g16.002 - b.e10.BGRCP85CN.f09_g16.002 + b.e10.BG20TRCN.f09_g16.002 + b.e10.BG1850CN.f09_g16.002 + b.e10.BG20TRCN.f09_g16.002 + b.e10.BG20TRCN.f09_g16.002 + b.e10.BGRCP85CN.f09_g16.002 run_component_dlnd env_run.xml @@ -64,11 +64,11 @@ integer 1 - 1 - 1 - 1850 - 1850 - 2006 + 1 + 1 + 1850 + 1850 + 2006 run_component_dlnd env_run.xml @@ -79,11 +79,11 @@ integer 1 - 1976 - 26 - 1850 - 1850 - 2006 + 1976 + 26 + 1850 + 1850 + 2006 run_component_dlnd env_run.xml @@ -94,11 +94,11 @@ integer 1 - 2005 - 100 - 2005 - 2005 - 2100 + 2005 + 100 + 2005 + 2005 + 2100 run_component_dlnd env_run.xml diff --git a/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml b/src/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml similarity index 100% rename from components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml rename to src/components/data_comps/dlnd/cime_config/namelist_definition_dlnd.xml diff --git a/components/data_comps/dlnd/cime_config/user_nl_dlnd b/src/components/data_comps/dlnd/cime_config/user_nl_dlnd similarity index 100% rename from components/data_comps/dlnd/cime_config/user_nl_dlnd rename to src/components/data_comps/dlnd/cime_config/user_nl_dlnd diff --git a/components/data_comps/dlnd/dlnd_comp_mod.F90 b/src/components/data_comps/dlnd/dlnd_comp_mod.F90 similarity index 100% rename from components/data_comps/dlnd/dlnd_comp_mod.F90 rename to src/components/data_comps/dlnd/dlnd_comp_mod.F90 diff --git a/components/data_comps/dlnd/lnd_comp_mct.F90 b/src/components/data_comps/dlnd/lnd_comp_mct.F90 similarity index 100% rename from components/data_comps/dlnd/lnd_comp_mct.F90 rename to src/components/data_comps/dlnd/lnd_comp_mct.F90 diff --git a/components/data_comps/docn/cime_config/buildlib b/src/components/data_comps/docn/cime_config/buildlib similarity index 90% rename from components/data_comps/docn/cime_config/buildlib rename to src/components/data_comps/docn/cime_config/buildlib index 75ed5840ce4..1084a766a54 100755 --- a/components/data_comps/docn/cime_config/buildlib +++ b/src/components/data_comps/docn/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/docn/cime_config/buildnml b/src/components/data_comps/docn/cime_config/buildnml similarity index 96% rename from components/data_comps/docn/cime_config/buildnml rename to src/components/data_comps/docn/cime_config/buildnml index 42ec4c42417..360c84e2db2 100755 --- a/components/data_comps/docn/cime_config/buildnml +++ b/src/components/data_comps/docn/cime_config/buildnml @@ -9,9 +9,9 @@ # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import,wrong-import-position -import os, shutil, sys, glob +import os, shutil, sys, glob, re -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -108,6 +108,13 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): else: nmlgen.create_stream_file_and_update_shr_strdata_nml(config, stream, stream_path, data_list_path) + # For aquaplanet prescribed have no streams + match = re.match(r'^sst_aquap\d+',docn_mode) + if match is not None: + if match.group(0): + value = ['null'] + nmlgen.set_value("streams",value) + #---------------------------------------------------- # Create `shr_strdata_nml` namelist group. #---------------------------------------------------- @@ -166,7 +173,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/docn/cime_config/config_archive.xml b/src/components/data_comps/docn/cime_config/config_archive.xml similarity index 100% rename from components/data_comps/docn/cime_config/config_archive.xml rename to src/components/data_comps/docn/cime_config/config_archive.xml diff --git a/components/data_comps/docn/cime_config/config_component.xml b/src/components/data_comps/docn/cime_config/config_component.xml similarity index 65% rename from components/data_comps/docn/cime_config/config_component.xml rename to src/components/data_comps/docn/cime_config/config_component.xml index 335a75481aa..ff71aa17f1b 100644 --- a/components/data_comps/docn/cime_config/config_component.xml +++ b/src/components/data_comps/docn/cime_config/config_component.xml @@ -15,14 +15,24 @@ char - prescribed,som,copyall,interannual,null + prescribed,sst_aquap1,sst_aquap2,sst_aquap3,sst_aquap4,sst_aquap5,sst_aquap6,sst_aquap7,sst_aquap8,sst_aquap9,sst_aquap10,som,som_aquap,copyall,interannual,null prescribed null prescribed som - us20 + som_aquap interannual + sst_aquap1 + sst_aquap2 + sst_aquap3 + sst_aquap4 + sst_aquap5 + sst_aquap6 + sst_aquap7 + sst_aquap8 + sst_aquap9 + sst_aquap10 copyall run_component_docn @@ -31,7 +41,7 @@ driver. The atmosphere/ocean fluxes are computed in the coupler. Therefore, the data ocean model does not compute fluxes like the data ice model. DOCN has two distinct modes of operation. It can - arun as a pure data model, reading in ocean SSTs (normally + run as a pure data model, reading in ocean SSTs (normally climatological) from input datasets, performing time/spatial interpolations, and passing these to the coupler. Alternatively, DOCN can compute updated SSTs by running as a slab ocean model where @@ -40,18 +50,14 @@ from the driver. --- A setting of prescribed assumes the only field in the input stream is SST. It also assumes the SST is in Celsius and must be converted to Kelvin. - All other fields are set to zero except for ocean salinity, which - is set to a constant reference salinity value. - Normally the ice fraction data is found in the same data files that - provide SST data to the data ocean model. They are normally found in - the same file because the SST and ice fraction data are derived from - the same observational data sets and are consistent with each other. - to the data ocean model. They are normally found in the same file - because the SST and ice fraction data are derived from the same - observational data sets and are consistent with each other. - --- A setting of som (slab ocean model) mode is a prognostic mode. This mode - computes a prognostic sea surface temperature and a freeze/melt - potential (surface Q-flux) used by the sea ice model. This + All other fields are set to zero except for ocean salinity, which is set to a + constant reference salinity value. Normally the ice fraction data is found in + the same data files that provide SST data to the data ocean model. They are + normally found in the same file because the SST and ice fraction data are derived + from the same observational data sets and are consistent with each other. + --- Settings of som (slab ocean model) or som_aquap (aquaplanet slab ocean) are + prognostic modes which compute a prognostic sea surface temperature and a + freeze/melt potential (surface Q-flux) used by the sea ice model. This calculation requires an external SOM forcing data file that includes ocean mixed layer depths and bottom-of-the-slab Q-fluxes. Scientifically appropriate bottom-of-the-slab Q-fluxes are normally @@ -61,7 +67,14 @@ appropriate and is provided for testing and development purposes only. Users must create scientifically appropriate data for their particular application. A tool is available to derive valid SOM forcing. - Default is prescribed. + --- A setting of sst_aquapN (where “N” is an integer index value) is a + type of prescribed SST mode used specifically for an aquaplanet setup in + which global SSTs correspond to an analytic form set by the index value. + Currently, indices for 10 SST profiles are supported [e.g., index 3 corresponds + to the “QOBS” profile of Neale and Hoskins (2001, Atmos. Sci. Lett.)]. + With source code modifications, it is possible for users to create their own + analytic SST distributions and match them to indices 11 or greater. + @@ -69,7 +82,9 @@ UNSET - pop_frc.1x1d.090130.nc + pop_frc.1x1d.090130.nc + default.som.forcing.aquaplanet.Qflux0_h30_sstQOBS.2degFV_c20170421.nc + default.som.forcing.aquaplanet.Qflux0_h30_sstQOBS.1degFV_c20170421.nc run_component_docn env_run.xml @@ -80,9 +95,6 @@ char CAMDATA - - WW3 - run_component_docn env_run.xml Prescribed SST and ice coverage stream name. @@ -100,9 +112,9 @@ $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.47x0.63_clim_c061106.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.23x0.31_clim_c110526.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1x1_1850_2012_c130411.nc - $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_48x96_1850_2012_c130411.nc - $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1.9x2.5_1850_2012_c130411.nc - $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2012_c130411.nc + $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_48x96_1850_2012_c130411.nc + $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1.9x2.5_1850_2012_c130411.nc + $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_1850_2012_c130411.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.47x0.63_1850_2012_c130411.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.23x0.31_1850_2012_c130411.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_1x1_clim_pi_c101029.nc @@ -113,7 +125,8 @@ $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.23x0.31_clim_pi_c101028.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_c040926.nc $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_0.9x1.25_clim_c040926.nc - $DIN_LOC_ROOT/wav/ww3/core2_G4_ice_30min_20000601_to_05.nc + $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_4x5_clim_c110526.nc + $DIN_LOC_ROOT/atm/cam/sst/sst_HadOIBl_bc_4x5_clim_c110526.nc run_component_docn env_run.xml @@ -128,24 +141,28 @@ $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.48x96_gx3v7_100114.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.1.9x2.5_gx1v6_090403.nc - $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.fv0.9x1.25_gx1v7.151020.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.47x0.63_gx1v6_090408.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.23x0.31_gx1v6_101108.nc $DIN_LOC_ROOT/ocn/docn7/domain.ocn.1x1.111007.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.48x96_gx3v7_100114.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.1.9x2.5_gx1v6_090403.nc - $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.fv0.9x1.25_gx1v7.151020.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.47x0.63_gx1v6_090408.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.23x0.31_gx1v6_101108.nc $DIN_LOC_ROOT/ocn/docn7/domain.ocn.1x1.111007.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.48x96_gx3v7_100114.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.1.9x2.5_gx1v6_090403.nc - $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/share/domains/domain.ocn.fv0.9x1.25_gx1v7.151020.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.47x0.63_gx1v6_090408.nc $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.23x0.31_gx1v6_101108.nc - $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc - $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc - $DIN_LOC_ROOT/wav/ww3/core2_G4_ice_30min_20000601_to_05.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.0.9x1.25_gx1v6_090403.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.4x5_gx3v7_100120.nc + $DIN_LOC_ROOT/atm/cam/ocnfrac/domain.camocn.4x5_gx3v7_100120.nc run_component_cam_sstice env_run.xml @@ -195,7 +212,6 @@ 0 2012 - 2000 run_component_cam_sstice env_run.xml @@ -206,12 +222,21 @@ - docn null mode: - docn slab ocean mode: - docn data mode: - docn us20 mode: - docn copy mode: - docn interannual mode: + docn null mode + docn slab ocean mode + docn aquaplanet slab ocean mode + docn interannual mode + docn aquaplanet mode: + docn prescribed aquaplanet sst - option 1 + docn prescribed aquaplanet sst - option 2 + docn prescribed aquaplanet sst - option 3 + docn prescribed aquaplanet sst - option 4 + docn prescribed aquaplanet sst - option 5 + docn prescribed aquaplanet sst - option 6 + docn prescribed aquaplanet sst - option 7 + docn prescribed aquaplanet sst - option 8 + docn prescribed aquaplanet sst - option 9 + docn prescribed aquaplanet sst - option 10 @@ -221,4 +246,3 @@ - diff --git a/components/data_comps/docn/cime_config/namelist_definition_docn.xml b/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml similarity index 92% rename from components/data_comps/docn/cime_config/namelist_definition_docn.xml rename to src/components/data_comps/docn/cime_config/namelist_definition_docn.xml index 32d2d7f5ba5..9f75abca493 100644 --- a/components/data_comps/docn/cime_config/namelist_definition_docn.xml +++ b/src/components/data_comps/docn/cime_config/namelist_definition_docn.xml @@ -33,7 +33,6 @@ prescribed SSTDATA (Run with prescribed SST, ICE_COV) som SOM (Slab ocean model) null NULL (NULL mode) - copyall COPYALL (Copy fields only) --> @@ -46,10 +45,20 @@ streams_file List of streams used for the given docn_mode. - prescribed - som + prescribed + '' + '' + '' + '' + '' + '' + '' + '' + '' + '' + som + som interannual - copyall @@ -59,7 +68,6 @@ derived does not appear in namelist - only used to set domain and data information - $SSTICE_GRID_FILENAME $SSTICE_GRID_FILENAME @@ -70,7 +78,6 @@ derived does not appear in namelist - only used to set domain and data information - $SSTICE_DATA_FILENAME $SSTICE_DATA_FILENAME @@ -112,11 +119,6 @@ area area mask mask - - time time - gridLon lon - gridLat lat - time time lon lon @@ -172,9 +174,6 @@ sst t - - iceCon s - T t S s @@ -212,7 +211,6 @@ -999 $SSTICE_YEAR_ALIGN - $SSTICE_YEAR_ALIGN 1 1 @@ -226,7 +224,6 @@ -999 $SSTICE_YEAR_START - $SSTICE_YEAR_START 1 1850 @@ -239,10 +236,9 @@ Last year of stream. -999 - $SSTICE_YEAR_END - $SSTICE_YEAR_END - 1 - 2014 + $SSTICE_YEAR_END + 1 + 2014 @@ -258,7 +254,7 @@ char streams shr_strdata_nml - SSTDATA,SOM,IAF,NULL,COPYALL + SSTDATA,SST_AQUAP1,SST_AQUAP2,SST_AQUAP3,SST_AQUAP4,SST_AQUAP5,SST_AQUAP6,SST_AQUAP7,SST_AQUAP8,SST_AQUAP9,SST_AQUAP10,SOM,SOM_AQUAP,IAF,NULL,COPYALL General method that operates on the data. This is generally implemented in the data models but is set in the strdata method for @@ -305,16 +301,28 @@ If DOCN_MODE is prescribed, datamode will be set to SSTDATA If DOCN_MODE is interannual, datamode will be set to IAF If DOCN_MODE is som , datamode will be set to SOM + If DOCN_MODE is sst_aqup[n], datamode will be set to SST_AQUAP + If DOCN_MODE is som_aqup[n], datamode will be set to SOM_AQUAP If DOCN_MODE is null, datamode will be set to NULL default: SSTDATA (prescribed setting for DOCN_MODE)' - NULL - SSTDATA - SOM + NULL + SSTDATA + SST_AQUAP1 + SST_AQUAP2 + SST_AQUAP3 + SST_AQUAP4 + SST_AQUAP5 + SST_AQUAP6 + SST_AQUAP7 + SST_AQUAP8 + SST_AQUAP9 + SST_AQUAP10 + SOM + SOM_AQUAP IAF - COPYALL diff --git a/components/data_comps/docn/cime_config/user_nl_docn b/src/components/data_comps/docn/cime_config/user_nl_docn similarity index 100% rename from components/data_comps/docn/cime_config/user_nl_docn rename to src/components/data_comps/docn/cime_config/user_nl_docn diff --git a/components/data_comps/docn/docn_comp_mod.F90 b/src/components/data_comps/docn/docn_comp_mod.F90 similarity index 74% rename from components/data_comps/docn/docn_comp_mod.F90 rename to src/components/data_comps/docn/docn_comp_mod.F90 index 88cd3bc70a4..f557311f7f2 100644 --- a/components/data_comps/docn/docn_comp_mod.F90 +++ b/src/components/data_comps/docn/docn_comp_mod.F90 @@ -79,13 +79,17 @@ module docn_comp_mod integer(IN) :: kt,ks,ku,kv,kdhdx,kdhdy,kq,kswp ! field indices integer(IN) :: kswnet,klwup,klwdn,ksen,klat,kmelth,ksnow,krofi integer(IN) :: kh,kqbot + integer(IN) :: index_lat, index_lon type(shr_strdata_type) :: SDOCN - type(mct_rearr) :: rearr - type(mct_avect) :: avstrm ! av of data from stream - real(R8), pointer :: somtp(:) - real(R8), pointer :: tfreeze(:) - integer , pointer :: imask(:) + type(mct_rearr) :: rearr + type(mct_avect) :: avstrm ! av of data from stream + real(R8), pointer :: somtp(:) + real(R8), pointer :: tfreeze(:) + integer , pointer :: imask(:) + real(R8), pointer :: xc(:), yc(:) ! arryas of model latitudes and longitudes + integer(IN) :: aquap_option + character(len=*),parameter :: flds_strm = 'strm_h:strm_qbot' integer(IN),parameter :: ktrans = 29 @@ -144,6 +148,7 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) integer(IN) :: shrlogunit, shrloglev ! original log unit and level integer(IN) :: nunit ! unit number integer(IN) :: kmask ! field reference + integer(IN) :: kfrac ! field reference logical :: ocn_present ! flag logical :: ocn_prognostic ! flag logical :: ocnrof_prognostic ! flag @@ -292,15 +297,31 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) ocn_mode = trim(SDOCN%dataMode) + ! Special logic for prescribed aquaplanet + if (ocn_mode(1:9) == 'SST_AQUAP') then + ! First determine the prescribed aquaplanet option + if (len_trim(ocn_mode) == 10) then + read(ocn_mode(10:10),'(i1)') aquap_option + else if (len_trim(ocn_mode) == 11) then + read(ocn_mode(10:11),'(i2)') aquap_option + end if + ! Now remove the index from the ocn_mode value, to have a generic setting + ! for use below + ocn_mode = "SST_AQUAP" + end if + ! check that we know how to handle the mode - if (trim(ocn_mode) == 'NULL' .or. & - trim(ocn_mode) == 'SSTDATA' .or. & - trim(ocn_mode) == 'COPYALL' .or. & - trim(ocn_mode) == 'IAF' .or. & - trim(ocn_mode) == 'SOM') then - if (my_task == master_task) & + if (trim(ocn_mode) == 'NULL' .or. & + trim(ocn_mode) == 'SSTDATA' .or. & + trim(ocn_mode) == 'SST_AQUAP' .or. & + trim(ocn_mode) == 'COPYALL' .or. & + trim(ocn_mode) == 'IAF' .or. & + trim(ocn_mode) == 'SOM' .or. & + trim(ocn_mode) == 'SOM_AQUAP') then + if (my_task == master_task) then write(logunit,F00) ' ocn mode = ',trim(ocn_mode) + end if else write(logunit,F00) ' ERROR illegal ocn mode = ',trim(ocn_mode) call shr_sys_abort() @@ -338,7 +359,7 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) ocnrof_prognostic = .true. endif - if (trim(ocn_mode) == 'SOM') then + if (trim(ocn_mode) == 'SOM' .or. trim(ocn_mode) == 'SOM_AQUAP') then ocn_prognostic = .true. endif @@ -353,11 +374,11 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) !---------------------------------------------------------------------------- call seq_infodata_PutData(infodata, ocnrof_prognostic=ocnrof_prognostic, & - ocn_present=ocn_present, ocn_prognostic=ocn_prognostic, & - ocn_nx=SDOCN%nxg, ocn_ny=SDOCN%nyg ) + ocn_present=ocn_present, ocn_prognostic=ocn_prognostic, & + ocn_nx=SDOCN%nxg, ocn_ny=SDOCN%nyg ) !---------------------------------------------------------------------------- - ! Initialize MCT global seg map, 1d decomp + ! Initialize data model MCT global seg map, 1d decomp !---------------------------------------------------------------------------- call t_startf('docn_initgsmaps') @@ -374,14 +395,26 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) call t_stopf('docn_initgsmaps') !---------------------------------------------------------------------------- - ! Initialize MCT domain + ! Initialize data model MCT domain !---------------------------------------------------------------------------- call t_startf('docn_initmctdom') if (my_task == master_task) write(logunit,F00) 'copy domains' call shr_sys_flush(logunit) - if (ocn_present) call shr_dmodel_rearrGGrid(SDOCN%grid, ggrid, gsmap, rearr, mpicom) + if (ocn_present) then + call shr_dmodel_rearrGGrid(SDOCN%grid, ggrid, gsmap, rearr, mpicom) + end if + + ! Special logic for either prescribed or som aquaplanet - overwrite and + ! set mask/frac to 1 + if (ocn_mode == 'SST_AQUAP' .or. ocn_mode == 'SOM_AQUAP') then + kmask = mct_aVect_indexRA(ggrid%data,'mask') + ggrid%data%rattr(kmask,:) = 1 + kfrac = mct_aVect_indexRA(ggrid%data,'frac') + ggrid%data%rattr(kfrac,:) = 1.0_r8 + write(logunit,F00) ' Resetting the data ocean mask and frac to 1 for aquaplanet' + end if call t_stopf('docn_initmctdom') @@ -426,10 +459,18 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) allocate(somtp(lsize)) allocate(tfreeze(lsize)) allocate(imask(lsize)) + allocate(xc(lsize)) + allocate(yc(lsize)) kmask = mct_aVect_indexRA(ggrid%data,'mask') imask(:) = nint(ggrid%data%rAttr(kmask,:)) + index_lon = mct_aVect_indexRA(ggrid%data,'lon') + xc(:) = ggrid%data%rAttr(index_lon,:) + + index_lat = mct_aVect_indexRA(ggrid%data,'lat') + yc(:) = ggrid%data%rAttr(index_lat,:) + call t_stopf('docn_initmctavs') !---------------------------------------------------------------------------- @@ -466,7 +507,7 @@ subroutine docn_comp_init( EClock, cdata, x2o, o2x, NLFilename ) endif endif call shr_mpi_bcast(exists,mpicom,'exists') - if (trim(ocn_mode) == 'SOM') then + if (trim(ocn_mode) == 'SOM' .or. trim(ocn_mode) == 'SOM_AQUAP') then if (my_task == master_task) write(logunit,F00) ' reading ',trim(rest_file) call shr_pcdf_readwrite('read',iosystem,SDOCN%io_type,trim(rest_file),mpicom,gsmap,rf1=somtp,rf1n='somtp') endif @@ -539,7 +580,6 @@ subroutine docn_comp_run( EClock, cdata, x2o, o2x) integer(IN) :: nl ! ocn frac index integer(IN) :: lsize ! size of attr vect integer(IN) :: shrlogunit, shrloglev ! original log unit and level - logical :: glcrun_alarm ! is glc going to run now logical :: newdata ! has newdata been read logical :: mssrmlf ! remove old data integer(IN) :: idt ! integer timestep @@ -583,23 +623,6 @@ subroutine docn_comp_run( EClock, cdata, x2o, o2x) call t_stopf('docn_run1') - !-------------------- - ! UNPACK - !-------------------- - - call t_startf('docn_unpack') - -! lsize = mct_avect_lsize(x2o) -! nflds_x2o = mct_avect_nRattr(x2o) - -! do nf=1,nflds_x2o -! do n=1,lsize -! ?? = x2o%rAttr(nf,n) -! enddo -! enddo - - call t_stopf('docn_unpack') - !-------------------- ! ADVANCE OCN !-------------------- @@ -658,6 +681,16 @@ subroutine docn_comp_run( EClock, cdata, x2o, o2x) o2x%rAttr(kswp ,n) = swp enddo + case('SST_AQUAP') + lsize = mct_avect_lsize(o2x) + do n = 1,lsize + o2x%rAttr(:,n) = 0.0_r8 + end do + call prescribed_sst(xc, yc, lsize, aquap_option, o2x%rAttr(kt,:)) + do n = 1,lsize + o2x%rAttr(kt,n) = o2x%rAttr(kt,n) + TkFrz + enddo + case('IAF') lsize = mct_avect_lsize(o2x) do n = 1,lsize @@ -709,6 +742,41 @@ subroutine docn_comp_run( EClock, cdata, x2o, o2x) enddo endif ! firstcall + case('SOM_AQUAP') + lsize = mct_avect_lsize(o2x) + do n = 1,SDOCN%nstreams + call shr_dmodel_translateAV(SDOCN%avs(n),avstrm,avifld,avofld,rearr) + enddo + if (firstcall) then + do n = 1,lsize + if (.not. read_restart) then + somtp(n) = o2x%rAttr(kt,n) + TkFrz + endif + o2x%rAttr(kt,n) = somtp(n) + o2x%rAttr(kq,n) = 0.0_R8 + enddo + else ! firstcall + tfreeze = shr_frz_freezetemp(o2x%rAttr(ks,:)) + TkFrz + do n = 1,lsize + !--- pull out h from av for resuse below --- + hn = avstrm%rAttr(kh,n) + !--- compute new temp --- + o2x%rAttr(kt,n) = somtp(n) + & + (x2o%rAttr(kswnet,n) + & ! shortwave + x2o%rAttr(klwup ,n) + & ! longwave + x2o%rAttr(klwdn ,n) + & ! longwave + x2o%rAttr(ksen ,n) + & ! sensible + x2o%rAttr(klat ,n) + & ! latent + x2o%rAttr(kmelth,n) - & ! ice melt + avstrm%rAttr(kqbot ,n) - & ! flux at bottom + (x2o%rAttr(ksnow,n)+x2o%rAttr(krofi,n))*latice) * & ! latent by prec and roff + dt/(cpsw*rhosw*hn) + !--- compute ice formed or melt potential --- + o2x%rAttr(kq,n) = (tfreeze(n) - o2x%rAttr(kt,n))*(cpsw*rhosw*hn)/dt ! ice formed q>0 + somtp(n) = o2x%rAttr(kt,n) ! save temp + enddo + endif ! firstcall + end select call t_stopf('docn_mode') @@ -730,7 +798,7 @@ subroutine docn_comp_run( EClock, cdata, x2o, o2x) close(nu) call shr_file_freeUnit(nu) endif - if (trim(ocn_mode) == 'SOM') then + if (trim(ocn_mode) == 'SOM' .or. trim(ocn_mode) == 'SOM_AQUAP') then if (my_task == master_task) write(logunit,F04) ' writing ',trim(rest_file),currentYMD,currentTOD call shr_pcdf_readwrite('write',iosystem,SDOCN%io_type,trim(rest_file),mpicom,gsmap,clobber=.true., & rf1=somtp,rf1n='somtp') @@ -801,8 +869,213 @@ subroutine docn_comp_final() call t_stopf('DOCN_FINAL') end subroutine docn_comp_final + !=============================================================================== !=============================================================================== +subroutine prescribed_sst(xc, yc, lsize, sst_option, sst) + + real(R8) , intent(in) :: xc(:) !degrees + real(R8) , intent(in) :: yc(:) !degrees + integer(IN) , intent(in) :: lsize + integer(IN) , intent(in) :: sst_option + real(R8) , intent(inout) :: sst(:) + + ! local + integer :: i + real(r8) :: tmp, tmp1, pi + real(r8) :: rlon(lsize), rlat(lsize) + + real(r8), parameter :: pio180 = SHR_CONST_PI/180._r8 + + ! Parameters for zonally symmetric experiments + real(r8), parameter :: t0_max = 27._r8 + real(r8), parameter :: t0_min = 0._r8 + real(r8), parameter :: maxlat = 60._r8*pio180 + real(r8), parameter :: shift = 5._r8*pio180 + real(r8), parameter :: shift9 = 10._r8*pio180 + real(r8), parameter :: shift10 = 15._r8*pio180 + + ! Parameters for zonally asymmetric experiments + real(r8), parameter :: t0_max6 = 1._r8 + real(r8), parameter :: t0_max7 = 3._r8 + real(r8), parameter :: latcen = 0._r8*pio180 + real(r8), parameter :: loncen = 0._r8*pio180 + real(r8), parameter :: latrad6 = 15._r8*pio180 + real(r8), parameter :: latrad8 = 30._r8*pio180 + real(r8), parameter :: lonrad = 30._r8*pio180 + !------------------------------------------------------------------------------- + + pi = SHR_CONST_PI + + ! convert xc and yc from degrees to radians + + rlon(:) = xc(:) * pio180 + rlat(:) = yc(:) * pio180 + + ! Control + + if (sst_option < 1 .or. sst_option > 10) then + call shr_sys_abort ('prescribed_sst: ERROR: sst_option must be between 1 and 10') + end if + + if (sst_option == 1 .or. sst_option == 6 .or. sst_option == 7 .or. sst_option == 8) then + do i = 1,lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else + tmp = sin(rlat(i)*pi*0.5_r8/maxlat) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + + ! Flat + + if (sst_option == 2) then + do i = 1,lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else + tmp = sin(rlat(i)*pi*0.5_r8/maxlat) + tmp = 1._r8 - tmp*tmp*tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + + ! Qobs + + if (sst_option == 3) then + do i = 1,lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else + tmp = sin(rlat(i)*pi*0.5_r8/maxlat) + tmp = (2._r8 - tmp*tmp*tmp*tmp - tmp*tmp)*0.5_r8 + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + + ! Peaked + + if (sst_option == 4) then + do i = 1,lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else + tmp = (maxlat - abs(rlat(i)))/maxlat + tmp1 = 1._r8 - tmp + sst(i) = t0_max*tmp + t0_min*tmp1 + end if + end do + end if + + ! Control-5N + + if (sst_option == 5) then + do i = 1,lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else if (rlat(i) > shift) then + tmp = sin((rlat(i)-shift)*pi*0.5_r8/(maxlat-shift)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + else + tmp = sin((rlat(i)-shift)*pi*0.5_r8/(maxlat+shift)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + + ! 1KEQ + + if (sst_option == 6) then + do i = 1,lsize + if (abs(rlat(i)-latcen) <= latrad6) then + tmp1 = cos((rlat(i)-latcen)*pi*0.5_r8/latrad6) + tmp1 = tmp1*tmp1 + tmp = abs(rlon(i)-loncen) + tmp = min(tmp , 2._r8*pi-tmp) + if(tmp <= lonrad) then + tmp = cos(tmp*pi*0.5_r8/lonrad) + tmp = tmp*tmp + sst(i) = sst(i) + t0_max6*tmp*tmp1 + end if + end if + end do + end if + + ! 3KEQ + + if (sst_option == 7) then + do i = 1, lsize + if (abs(rlat(i)-latcen) <= latrad6) then + tmp1 = cos((rlat(i)-latcen)*pi*0.5_r8/latrad6) + tmp1 = tmp1*tmp1 + tmp = abs(rlon(i)-loncen) + tmp = min(tmp , 2._r8*pi-tmp) + if (tmp <= lonrad) then + tmp = cos(tmp*pi*0.5_r8/lonrad) + tmp = tmp*tmp + sst(i) = sst(i) + t0_max7*tmp*tmp1 + end if + end if + end do + end if + + ! 3KW1 + + if (sst_option == 8) then + do i = 1, lsize + if (abs(rlat(i)-latcen) <= latrad8) then + tmp1 = cos((rlat(i)-latcen)*pi*0.5_r8/latrad8) + tmp1 = tmp1*tmp1 + tmp = cos(rlon(i)-loncen) + sst(i) = sst(i) + t0_max7*tmp*tmp1 + end if + end do + end if + + ! Control-10N + + if (sst_option == 9) then + do i = 1, lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else if (rlat(i) > shift9) then + tmp = sin((rlat(i)-shift9)*pi*0.5_r8/(maxlat-shift9)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + else + tmp = sin((rlat(i)-shift9)*pi*0.5_r8/(maxlat+shift9)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + + ! Control-15N + + if (sst_option == 10) then + do i = 1, lsize + if (abs(rlat(i)) > maxlat) then + sst(i) = t0_min + else if(rlat(i) > shift10) then + tmp = sin((rlat(i)-shift10)*pi*0.5_r8/(maxlat-shift10)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + else + tmp = sin((rlat(i)-shift10)*pi*0.5_r8/(maxlat+shift10)) + tmp = 1._r8 - tmp*tmp + sst(i) = tmp*(t0_max - t0_min) + t0_min + end if + end do + end if + +end subroutine prescribed_sst end module docn_comp_mod diff --git a/components/data_comps/docn/ocn_comp_mct.F90 b/src/components/data_comps/docn/ocn_comp_mct.F90 similarity index 100% rename from components/data_comps/docn/ocn_comp_mct.F90 rename to src/components/data_comps/docn/ocn_comp_mct.F90 diff --git a/components/data_comps/docn/tools/pop_som_frc/README b/src/components/data_comps/docn/tools/pop_som_frc/README similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/README rename to src/components/data_comps/docn/tools/pop_som_frc/README diff --git a/components/data_comps/docn/tools/pop_som_frc/SOM.doc b/src/components/data_comps/docn/tools/pop_som_frc/SOM.doc similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/SOM.doc rename to src/components/data_comps/docn/tools/pop_som_frc/SOM.doc diff --git a/components/data_comps/docn/tools/pop_som_frc/SOM.pdf b/src/components/data_comps/docn/tools/pop_som_frc/SOM.pdf similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/SOM.pdf rename to src/components/data_comps/docn/tools/pop_som_frc/SOM.pdf diff --git a/components/data_comps/docn/tools/pop_som_frc/pop_frc.csh b/src/components/data_comps/docn/tools/pop_som_frc/pop_frc.csh similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/pop_frc.csh rename to src/components/data_comps/docn/tools/pop_som_frc/pop_frc.csh diff --git a/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl b/src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl rename to src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlann.ncl diff --git a/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl b/src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl rename to src/components/data_comps/docn/tools/pop_som_frc/pop_frc_mlt.ncl diff --git a/components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl b/src/components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl rename to src/components/data_comps/docn/tools/pop_som_frc/pop_interp.ncl diff --git a/components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh b/src/components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh rename to src/components/data_comps/docn/tools/pop_som_frc/read_from_mss.csh diff --git a/components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh b/src/components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh similarity index 100% rename from components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh rename to src/components/data_comps/docn/tools/pop_som_frc/read_from_mss_month.csh diff --git a/components/data_comps/drof/cime_config/buildlib b/src/components/data_comps/drof/cime_config/buildlib similarity index 90% rename from components/data_comps/drof/cime_config/buildlib rename to src/components/data_comps/drof/cime_config/buildlib index b53123f5dac..66bc96b84ae 100755 --- a/components/data_comps/drof/cime_config/buildlib +++ b/src/components/data_comps/drof/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/drof/cime_config/buildnml b/src/components/data_comps/drof/cime_config/buildnml similarity index 99% rename from components/data_comps/drof/cime_config/buildnml rename to src/components/data_comps/drof/cime_config/buildnml index e76eea5a8c3..37b0a3a7a3f 100755 --- a/components/data_comps/drof/cime_config/buildnml +++ b/src/components/data_comps/drof/cime_config/buildnml @@ -11,7 +11,7 @@ import os, shutil, sys, glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -164,7 +164,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/drof/cime_config/config_archive.xml b/src/components/data_comps/drof/cime_config/config_archive.xml similarity index 100% rename from components/data_comps/drof/cime_config/config_archive.xml rename to src/components/data_comps/drof/cime_config/config_archive.xml diff --git a/components/data_comps/drof/cime_config/config_component.xml b/src/components/data_comps/drof/cime_config/config_component.xml similarity index 100% rename from components/data_comps/drof/cime_config/config_component.xml rename to src/components/data_comps/drof/cime_config/config_component.xml diff --git a/components/data_comps/drof/cime_config/namelist_definition_drof.xml b/src/components/data_comps/drof/cime_config/namelist_definition_drof.xml similarity index 95% rename from components/data_comps/drof/cime_config/namelist_definition_drof.xml rename to src/components/data_comps/drof/cime_config/namelist_definition_drof.xml index a5a953ba8ed..03a819b67f1 100644 --- a/components/data_comps/drof/cime_config/namelist_definition_drof.xml +++ b/src/components/data_comps/drof/cime_config/namelist_definition_drof.xml @@ -212,7 +212,7 @@ char streams shr_strdata_nml - NULL,CPLHIST,DIATREN_ANN_RX1,DIATREN_IAF_RX1 + NULL,COPYALL The runoff data is associated with the river model. datamode = "NULL" @@ -223,21 +223,10 @@ dataMode = "COPYALL" Copies all fields directly from the input data streams Any required fields not found on an input stream will be set to zero. - datamode = "CPLHIST" - Reads in data from coupler history files generated by a previous run - datamode = DIATREN_ANN_RX1 - Reads in annual forcing river data used for CORE2 forcing runs - datamode = DIATREN_ANN_RX1,DIATREN_IAF_RX1 - Reads in intra-annual forcing river data used for CORE2 forcing runs - Set by the the DROF_MODE xml variables in env_run.xml - specifies values for rof mode: CPLHIST,DIATREN_ANN_RX1,DIATREN_IAF_RX1,NULL - default value: DIATREN_ANN_RX1 - NULL - CPLHIST - DIATREN_ANN_RX1 - DIATREN_IAF_RX1 + COPYALL + NULL diff --git a/components/data_comps/drof/cime_config/user_nl_drof b/src/components/data_comps/drof/cime_config/user_nl_drof similarity index 100% rename from components/data_comps/drof/cime_config/user_nl_drof rename to src/components/data_comps/drof/cime_config/user_nl_drof diff --git a/components/data_comps/drof/drof_comp_mod.F90 b/src/components/data_comps/drof/drof_comp_mod.F90 similarity index 99% rename from components/data_comps/drof/drof_comp_mod.F90 rename to src/components/data_comps/drof/drof_comp_mod.F90 index 060b5fad678..a2c9974bd3a 100644 --- a/components/data_comps/drof/drof_comp_mod.F90 +++ b/src/components/data_comps/drof/drof_comp_mod.F90 @@ -255,10 +255,7 @@ subroutine drof_comp_init( EClock, cdata, x2r, r2x, NLFilename ) ! check that we know how to handle the mode - if (trim(rof_mode) == 'NULL' .or. & - trim(rof_mode) == 'CPLHIST' .or. & - trim(rof_mode) == 'DIATREN_ANN_RX1' .or. & - trim(rof_mode) == 'DIATREN_IAF_RX1') then + if (trim(rof_mode) == 'NULL' .or. trim(rof_mode) == 'COPYALL') then if (my_task == master_task) write(logunit,F00) 'rof mode = ',trim(rof_mode) else write(logunit,F00) ' ERROR illegal rof mode = ',trim(rof_mode) diff --git a/components/data_comps/drof/rof_comp_mct.F90 b/src/components/data_comps/drof/rof_comp_mct.F90 similarity index 100% rename from components/data_comps/drof/rof_comp_mct.F90 rename to src/components/data_comps/drof/rof_comp_mct.F90 diff --git a/components/data_comps/dwav/README b/src/components/data_comps/dwav/README similarity index 100% rename from components/data_comps/dwav/README rename to src/components/data_comps/dwav/README diff --git a/components/data_comps/dwav/bld/README b/src/components/data_comps/dwav/bld/README similarity index 100% rename from components/data_comps/dwav/bld/README rename to src/components/data_comps/dwav/bld/README diff --git a/components/data_comps/dwav/cime_config/buildlib b/src/components/data_comps/dwav/cime_config/buildlib similarity index 90% rename from components/data_comps/dwav/cime_config/buildlib rename to src/components/data_comps/dwav/cime_config/buildlib index 1c9cb28ac5f..5d3b8e78b3f 100755 --- a/components/data_comps/dwav/cime_config/buildlib +++ b/src/components/data_comps/dwav/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/data_comps/dwav/cime_config/buildnml b/src/components/data_comps/dwav/cime_config/buildnml similarity index 93% rename from components/data_comps/dwav/cime_config/buildnml rename to src/components/data_comps/dwav/cime_config/buildnml index 3a412278f2d..3aeffe5afbb 100755 --- a/components/data_comps/dwav/cime_config/buildnml +++ b/src/components/data_comps/dwav/cime_config/buildnml @@ -1,7 +1,6 @@ #!/usr/bin/env python -"""Namelist creator for CIME's data ocn model. -`build_namelist` function. +"""Namelist creator for CIME's data wave model. """ # Typically ignore this. # pylint: disable=invalid-name @@ -9,12 +8,9 @@ # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import,wrong-import-position -import os -import shutil -import sys -import glob +import os, shutil, sys, glob -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -63,8 +59,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): #---------------------------------------------------- # Clear out old data. #---------------------------------------------------- - data_list_path = os.path.join(case.get_case_root(), "Buildconf", - "dwav.input_data_list") + data_list_path = os.path.join(case.get_case_root(), "Buildconf", "dwav.input_data_list") if os.path.exists(data_list_path): os.remove(data_list_path) @@ -97,8 +92,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): inst_stream = stream + inst_string logger.debug("DWAV stream is %s", inst_stream) - stream_path = os.path.join(confdir, - "dwav.streams.txt." + inst_stream) + stream_path = os.path.join(confdir, "dwav.streams.txt." + inst_stream) user_stream_path = os.path.join(case.get_case_root(), "user_dwav.streams.txt." + inst_stream) @@ -113,11 +107,10 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): #---------------------------------------------------- # Create dwav_nml namelists group #---------------------------------------------------- + # set per-stream variables nmlgen.create_shr_strdata_nml() - nmlgen.add_default("decomp", "1d") - nmlgen.add_default("force_prognostic_true", value=".false.") - nmlgen.add_default("restfilm", value="undefined") - nmlgen.add_default("restfils", value="undefined") + + # set variables that are not per-stream if wav_domain_file != "UNSET": full_domain_path = os.path.join(wav_domain_path, wav_domain_file) nmlgen.add_default("domainfile", value=full_domain_path) @@ -126,7 +119,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen): # Finally, write out all the namelists. #---------------------------------------------------- namelist_file = os.path.join(confdir, "dwav_in") - nmlgen.write_output_file(namelist_file, data_list_path) + nmlgen.write_output_file(namelist_file, data_list_path, groups=['dwav_nml','shr_strdata_nml']) ############################################################################### def buildnml(case, caseroot, compname): @@ -170,7 +163,6 @@ def buildnml(case, caseroot, compname): # Loop over instances #---------------------------------------------------- for inst_counter in range(1, ninst+1): - # determine instance string inst_string = "" if ninst > 1: diff --git a/components/data_comps/dwav/cime_config/config_archive.xml b/src/components/data_comps/dwav/cime_config/config_archive.xml similarity index 100% rename from components/data_comps/dwav/cime_config/config_archive.xml rename to src/components/data_comps/dwav/cime_config/config_archive.xml diff --git a/components/data_comps/dwav/cime_config/config_component.xml b/src/components/data_comps/dwav/cime_config/config_component.xml similarity index 76% rename from components/data_comps/dwav/cime_config/config_component.xml rename to src/components/data_comps/dwav/cime_config/config_component.xml index a7fb6c1c055..3b73827e7fd 100644 --- a/components/data_comps/dwav/cime_config/config_component.xml +++ b/src/components/data_comps/dwav/cime_config/config_component.xml @@ -13,11 +13,11 @@ char - null,copyall - copyall + NULL,CLIMO + NULL - null - copyall + NULL + CLIMO run_component_dwav env_run.xml @@ -28,7 +28,7 @@ dwav null mode: - dwav copy mode: + dwav climatological mode: diff --git a/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml b/src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml similarity index 94% rename from components/data_comps/dwav/cime_config/namelist_definition_dwav.xml rename to src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml index c2f12ca9b4e..81398a316f7 100644 --- a/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml +++ b/src/components/data_comps/dwav/cime_config/namelist_definition_dwav.xml @@ -47,10 +47,10 @@ char(100) streams streams_file - List of streams used for the given datm_mode. + List of streams used for the given dwav_mode. - NULL - wav.copyall + null + climo @@ -60,7 +60,7 @@ streams_file Stream domain file directory. - $DIN_LOC_ROOT/wav/dwav + $DIN_LOC_ROOT/wav/dwav @@ -70,7 +70,7 @@ streams_file Stream domain file path(s). - waveclim.mon.ww3a.150612.nc + waveclim.mon.ww3a.150612.nc @@ -80,7 +80,7 @@ streams_file Stream domain variable name(s). - + time time xc lon yc lat @@ -96,7 +96,7 @@ streams_file Stream data file directory. - $DIN_LOC_ROOT/wav/dwav + $DIN_LOC_ROOT/wav/dwav @@ -106,7 +106,7 @@ streams_file Stream data file path(s). - waveclim.mon.ww3a.150612.nc + waveclim.mon.ww3a.150612.nc @@ -116,7 +116,7 @@ streams_file Stream data variable name(s). - + lamult lamult ustokes ustokes vstokes vstokes @@ -140,7 +140,7 @@ streams_file Simulation year to align stream to. - 1 + 1 @@ -150,7 +150,7 @@ streams_file First year of stream. - 1 + 1 @@ -160,7 +160,7 @@ streams_file Last year of stream. - 1 + 1 @@ -188,15 +188,14 @@ fields not found on an input stream will be set to zero. Set by the following xml variables in env_run.xml DWAV_MODE specifies values for wav mode: copyall,null - default value: copyall - null - copyall + copyall + null - + char streams abs @@ -207,7 +206,7 @@ for all input data for this strdata input. - + null @@ -253,7 +252,7 @@ and fillmask are ignored. - + NOT_SET @@ -268,7 +267,7 @@ save and reuse a set of weights later. - + NOT_SET @@ -317,7 +316,7 @@ is set, mapalgo and mapmask are ignored. - + NOT_SET @@ -332,7 +331,7 @@ allows a user to save and reuse a set of weights later. - + NOT_SET @@ -401,7 +400,7 @@ - + char streams shr_strdata_nml @@ -483,8 +482,8 @@ logical - drof - drof_nml + dwav + dwav_nml If TRUE, prognostic is forced to true. .false. diff --git a/components/data_comps/dwav/cime_config/user_nl_dwav b/src/components/data_comps/dwav/cime_config/user_nl_dwav similarity index 100% rename from components/data_comps/dwav/cime_config/user_nl_dwav rename to src/components/data_comps/dwav/cime_config/user_nl_dwav diff --git a/components/data_comps/dwav/dwav_comp_mod.F90 b/src/components/data_comps/dwav/dwav_comp_mod.F90 similarity index 89% rename from components/data_comps/dwav/dwav_comp_mod.F90 rename to src/components/data_comps/dwav/dwav_comp_mod.F90 index 4c45d364d4a..f3301988797 100644 --- a/components/data_comps/dwav/dwav_comp_mod.F90 +++ b/src/components/data_comps/dwav/dwav_comp_mod.F90 @@ -1,3 +1,6 @@ +#ifdef AIX +@PROCESS ALIAS_SIZE(805306368) +#endif module dwav_comp_mod ! !USES: @@ -51,9 +54,8 @@ module dwav_comp_mod integer(IN),parameter :: master_task=0 ! task number of master task integer(IN) :: logunit ! logging unit number integer :: inst_index ! number of current instance (ie. 1) - character(len=16) :: inst_name ! fullname of current instance (ie. "lnd_0001") - character(len=16) :: inst_suffix ! char string associated with instance - ! (ie. "_0001" or "") + character(len=16) :: inst_name ! fullname of current instance (ie. "wav_0001") + character(len=16) :: inst_suffix ! char string associated with instance (ie. "_0001" or "") character(CL) :: wav_mode ! mode integer(IN) :: dbug = 0 ! debug level (higher is more) @@ -66,10 +68,8 @@ module dwav_comp_mod type(mct_rearr) :: rearr integer(IN),parameter :: ktrans = 3 - character(12),parameter :: avifld(1:ktrans) = & - (/"lamult ","ustokes ","vstokes "/) - character(12),parameter :: avofld(1:ktrans) = & - (/"Sw_lamult ","Sw_ustokes ","Sw_vstokes "/) + character(12),parameter :: avifld(1:ktrans) = (/"lamult ","ustokes ","vstokes "/) + character(12),parameter :: avofld(1:ktrans) = (/"Sw_lamult ","Sw_ustokes ","Sw_vstokes "/) !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CONTAINS @@ -128,21 +128,25 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) integer(IN) :: yearLast ! last year to use in data stream integer(IN) :: yearAlign ! data year that aligns with yearFirst - character(CL) :: wav_in ! dshr wav namelist - character(CL) :: decomp ! decomp strategy - character(CL) :: rest_file ! restart filename - character(CL) :: rest_file_strm ! restart filename for stream - character(CL) :: restfilm ! restart filename for namelist - character(CL) :: restfils ! restart filename for stream for namelist + character(CL) :: wav_in ! dshr wav namelist + character(CL) :: decomp ! decomp strategy + character(CL) :: rest_file ! restart filename + character(CL) :: rest_file_strm ! restart filename for stream + character(CL) :: restfilm ! restart filename for namelist + character(CL) :: restfils ! restart filename for stream for namelist + logical :: force_prognostic_true ! if true set prognostic true + logical :: exists ! file existance integer(IN) :: nu ! unit number !----- define namelist ----- namelist / dwav_nml / & - decomp, restfilm, restfils + decomp, restfilm, restfils, & + force_prognostic_true !--- formats --- character(*), parameter :: F00 = "('(dwav_comp_init) ',8a)" + character(*), parameter :: F0L = "('(docn_comp_init) ',a, l2)" character(*), parameter :: F01 = "('(dwav_comp_init) ',a,5i8)" character(*), parameter :: F02 = "('(dwav_comp_init) ',a,4es13.6)" character(*), parameter :: F03 = "('(dwav_comp_init) ',a,i8,a)" @@ -151,8 +155,7 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) character(*), parameter :: F90 = "('(dwav_comp_init) ',73('='))" character(*), parameter :: F91 = "('(dwav_comp_init) ',73('-'))" character(*), parameter :: subName = "(dwav_comp_init) " - -!------------------------------------------------------------------------------- + !------------------------------------------------------------------------------- call t_startf('DWAV_INIT') @@ -202,7 +205,6 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) !---------------------------------------------------------------------------- call t_startf('dwav_readnml') - !write(logunit,F00)' dwav_readnml...' filename = "dwav_in"//trim(inst_suffix) decomp = "1d" @@ -218,35 +220,37 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) write(logunit,F01) 'ERROR: reading input namelist, '//trim(filename)//' iostat=',ierr call shr_sys_abort(subName//': namelist read error '//trim(filename)) end if - write(logunit,F00)' wav_in = ',trim(wav_in) write(logunit,F00)' decomp = ',trim(decomp) write(logunit,F00)' restfilm = ',trim(restfilm) write(logunit,F00)' restfils = ',trim(restfils) + write(logunit,F0L)' force_prognostic_true = ',force_prognostic_true endif call shr_mpi_bcast(decomp,mpicom,'decomp') call shr_mpi_bcast(restfilm,mpicom,'restfilm') call shr_mpi_bcast(restfils,mpicom,'restfils') + call shr_mpi_bcast(force_prognostic_true,mpicom,'force_prognostic_true') rest_file = trim(restfilm) rest_file_strm = trim(restfils) + if (force_prognostic_true) then + wav_present = .true. + wav_prognostic = .true. + endif !---------------------------------------------------------------------------- ! Read dshr namelist !---------------------------------------------------------------------------- - !write(logunit,F00)' read dshr nml...' call shr_strdata_readnml(SDWAV,trim(filename),mpicom=mpicom) !---------------------------------------------------------------------------- ! Validate mode !---------------------------------------------------------------------------- - !write(logunit,F00)' validate mode...' - wav_mode = trim(SDWAV%dataMode) ! check that we know how to handle the mode - if (trim(wav_mode) == 'null' .or. & + if (trim(wav_mode) == 'null' .or. & trim(wav_mode) == 'copyall') then if (my_task == master_task) & write(logunit,F00) ' wav mode = ',trim(wav_mode) @@ -262,7 +266,6 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) !---------------------------------------------------------------------------- call t_startf('dwav_strdata_init') - !write(logunit,F00)' dwav_strdata_init...' if (trim(wav_mode) /= 'null') then wav_present = .true. @@ -271,8 +274,7 @@ subroutine dwav_comp_init( EClock, cdata, x2w, w2x, NLFilename ) call shr_strdata_pioinit(SDWAV, iosystem, shr_pio_getiotype(trim(inst_name))) - call shr_strdata_init(SDWAV,mpicom,compid,name='wav', & - calendar=calendar) + call shr_strdata_init(SDWAV,mpicom,compid,name='wav', calendar=calendar) endif if (my_task == master_task) then @@ -440,25 +442,20 @@ subroutine dwav_comp_run( EClock, cdata, x2w, w2x) type(mct_gsMap) , pointer :: gsmap type(mct_gGrid) , pointer :: ggrid - integer(IN) :: CurrentYMD ! model date - integer(IN) :: CurrentTOD ! model sec into model date - integer(IN) :: yy,mm,dd ! year month day - integer(IN) :: n ! indices - integer(IN) :: nf ! fields loop index - integer(IN) :: nl ! ocn frac index - integer(IN) :: lsize ! size of attr vect + integer(IN) :: CurrentYMD ! model date + integer(IN) :: CurrentTOD ! model sec into model date + integer(IN) :: yy,mm,dd ! year month day + integer(IN) :: n ! indices + integer(IN) :: nf ! fields loop index + integer(IN) :: lsize ! size of attr vect integer(IN) :: shrlogunit, shrloglev ! original log unit and level -! logical :: glcrun_alarm ! is glc going to run now - logical :: newdata ! has newdata been read - logical :: mssrmlf ! remove old data - integer(IN) :: idt ! integer timestep - real(R8) :: dt ! timestep -! real(R8) :: hn ! h field - logical :: write_restart ! restart now - character(CL) :: case_name ! case name - character(CL) :: rest_file ! restart_file - character(CL) :: rest_file_strm ! restart_file for stream - integer(IN) :: nu ! unit number + integer(IN) :: idt ! integer timestep + real(R8) :: dt ! timestep + logical :: write_restart ! restart now + character(CL) :: case_name ! case name + character(CL) :: rest_file ! restart_file + character(CL) :: rest_file_strm ! restart_file for stream + integer(IN) :: nu ! unit number integer(IN) :: nflds_x2w type(seq_infodata_type), pointer :: infodata @@ -494,14 +491,14 @@ subroutine dwav_comp_run( EClock, cdata, x2w, w2x) call t_startf('dwav_unpack') -! lsize = mct_avect_lsize(x2o) -! nflds_x2o = mct_avect_nRattr(x2o) + ! lsize = mct_avect_lsize(x2o) + ! nflds_x2o = mct_avect_nRattr(x2o) -! do nf=1,nflds_x2o -! do n=1,lsize -! ?? = x2o%rAttr(nf,n) -! enddo -! enddo + ! do nf=1,nflds_x2o + ! do n=1,lsize + ! ?? = x2o%rAttr(nf,n) + ! enddo + ! enddo call t_stopf('dwav_unpack') diff --git a/components/data_comps/dwav/wav_comp_mct.F90 b/src/components/data_comps/dwav/wav_comp_mct.F90 similarity index 100% rename from components/data_comps/dwav/wav_comp_mct.F90 rename to src/components/data_comps/dwav/wav_comp_mct.F90 diff --git a/components/stub_comps/satm/cime_config/buildlib b/src/components/stub_comps/satm/cime_config/buildlib similarity index 90% rename from components/stub_comps/satm/cime_config/buildlib rename to src/components/stub_comps/satm/cime_config/buildlib index 83e8d9a5b58..0da3c680e27 100755 --- a/components/stub_comps/satm/cime_config/buildlib +++ b/src/components/stub_comps/satm/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/satm/cime_config/buildnml b/src/components/stub_comps/satm/cime_config/buildnml similarity index 100% rename from components/stub_comps/satm/cime_config/buildnml rename to src/components/stub_comps/satm/cime_config/buildnml diff --git a/components/stub_comps/satm/cime_config/config_component.xml b/src/components/stub_comps/satm/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/satm/cime_config/config_component.xml rename to src/components/stub_comps/satm/cime_config/config_component.xml diff --git a/components/stub_comps/satm/cpl/atm_comp_mct.F90 b/src/components/stub_comps/satm/cpl/atm_comp_mct.F90 similarity index 100% rename from components/stub_comps/satm/cpl/atm_comp_mct.F90 rename to src/components/stub_comps/satm/cpl/atm_comp_mct.F90 diff --git a/components/stub_comps/sesp/cime_config/buildlib b/src/components/stub_comps/sesp/cime_config/buildlib similarity index 90% rename from components/stub_comps/sesp/cime_config/buildlib rename to src/components/stub_comps/sesp/cime_config/buildlib index c54e5d4cfc8..47b48f32b3b 100755 --- a/components/stub_comps/sesp/cime_config/buildlib +++ b/src/components/stub_comps/sesp/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/sesp/cime_config/buildnml b/src/components/stub_comps/sesp/cime_config/buildnml similarity index 100% rename from components/stub_comps/sesp/cime_config/buildnml rename to src/components/stub_comps/sesp/cime_config/buildnml diff --git a/components/stub_comps/sesp/cime_config/config_component.xml b/src/components/stub_comps/sesp/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/sesp/cime_config/config_component.xml rename to src/components/stub_comps/sesp/cime_config/config_component.xml diff --git a/components/stub_comps/sesp/cpl/esp_comp_mct.F90 b/src/components/stub_comps/sesp/cpl/esp_comp_mct.F90 similarity index 100% rename from components/stub_comps/sesp/cpl/esp_comp_mct.F90 rename to src/components/stub_comps/sesp/cpl/esp_comp_mct.F90 diff --git a/components/stub_comps/sglc/cime_config/buildlib b/src/components/stub_comps/sglc/cime_config/buildlib similarity index 90% rename from components/stub_comps/sglc/cime_config/buildlib rename to src/components/stub_comps/sglc/cime_config/buildlib index dfaa11d6d12..7615fefee80 100755 --- a/components/stub_comps/sglc/cime_config/buildlib +++ b/src/components/stub_comps/sglc/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/sglc/cime_config/buildnml b/src/components/stub_comps/sglc/cime_config/buildnml similarity index 100% rename from components/stub_comps/sglc/cime_config/buildnml rename to src/components/stub_comps/sglc/cime_config/buildnml diff --git a/components/stub_comps/sglc/cime_config/config_component.xml b/src/components/stub_comps/sglc/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/sglc/cime_config/config_component.xml rename to src/components/stub_comps/sglc/cime_config/config_component.xml diff --git a/components/stub_comps/sglc/cpl/glc_comp_mct.F90 b/src/components/stub_comps/sglc/cpl/glc_comp_mct.F90 similarity index 100% rename from components/stub_comps/sglc/cpl/glc_comp_mct.F90 rename to src/components/stub_comps/sglc/cpl/glc_comp_mct.F90 diff --git a/components/stub_comps/sice/cime_config/buildlib b/src/components/stub_comps/sice/cime_config/buildlib similarity index 90% rename from components/stub_comps/sice/cime_config/buildlib rename to src/components/stub_comps/sice/cime_config/buildlib index a179856cc9a..6058bc7f6f7 100755 --- a/components/stub_comps/sice/cime_config/buildlib +++ b/src/components/stub_comps/sice/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/sice/cime_config/buildnml b/src/components/stub_comps/sice/cime_config/buildnml similarity index 100% rename from components/stub_comps/sice/cime_config/buildnml rename to src/components/stub_comps/sice/cime_config/buildnml diff --git a/components/stub_comps/sice/cime_config/config_component.xml b/src/components/stub_comps/sice/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/sice/cime_config/config_component.xml rename to src/components/stub_comps/sice/cime_config/config_component.xml diff --git a/components/stub_comps/sice/cpl/ice_comp_mct.F90 b/src/components/stub_comps/sice/cpl/ice_comp_mct.F90 similarity index 100% rename from components/stub_comps/sice/cpl/ice_comp_mct.F90 rename to src/components/stub_comps/sice/cpl/ice_comp_mct.F90 diff --git a/components/stub_comps/slnd/cime_config/buildlib b/src/components/stub_comps/slnd/cime_config/buildlib similarity index 90% rename from components/stub_comps/slnd/cime_config/buildlib rename to src/components/stub_comps/slnd/cime_config/buildlib index d464e7a1087..9bf3a0e5f8d 100755 --- a/components/stub_comps/slnd/cime_config/buildlib +++ b/src/components/stub_comps/slnd/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/slnd/cime_config/buildnml b/src/components/stub_comps/slnd/cime_config/buildnml similarity index 100% rename from components/stub_comps/slnd/cime_config/buildnml rename to src/components/stub_comps/slnd/cime_config/buildnml diff --git a/components/stub_comps/slnd/cime_config/config_component.xml b/src/components/stub_comps/slnd/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/slnd/cime_config/config_component.xml rename to src/components/stub_comps/slnd/cime_config/config_component.xml diff --git a/components/stub_comps/slnd/cpl/lnd_comp_mct.F90 b/src/components/stub_comps/slnd/cpl/lnd_comp_mct.F90 similarity index 100% rename from components/stub_comps/slnd/cpl/lnd_comp_mct.F90 rename to src/components/stub_comps/slnd/cpl/lnd_comp_mct.F90 diff --git a/components/stub_comps/socn/cime_config/buildlib b/src/components/stub_comps/socn/cime_config/buildlib similarity index 90% rename from components/stub_comps/socn/cime_config/buildlib rename to src/components/stub_comps/socn/cime_config/buildlib index 559ea1c36ea..18ecb7480a4 100755 --- a/components/stub_comps/socn/cime_config/buildlib +++ b/src/components/stub_comps/socn/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/socn/cime_config/buildnml b/src/components/stub_comps/socn/cime_config/buildnml similarity index 100% rename from components/stub_comps/socn/cime_config/buildnml rename to src/components/stub_comps/socn/cime_config/buildnml diff --git a/components/stub_comps/socn/cime_config/config_component.xml b/src/components/stub_comps/socn/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/socn/cime_config/config_component.xml rename to src/components/stub_comps/socn/cime_config/config_component.xml diff --git a/components/stub_comps/socn/cpl/ocn_comp_mct.F90 b/src/components/stub_comps/socn/cpl/ocn_comp_mct.F90 similarity index 100% rename from components/stub_comps/socn/cpl/ocn_comp_mct.F90 rename to src/components/stub_comps/socn/cpl/ocn_comp_mct.F90 diff --git a/components/stub_comps/srof/cime_config/buildlib b/src/components/stub_comps/srof/cime_config/buildlib similarity index 90% rename from components/stub_comps/srof/cime_config/buildlib rename to src/components/stub_comps/srof/cime_config/buildlib index d448e16bcf8..7a3ad9192d6 100755 --- a/components/stub_comps/srof/cime_config/buildlib +++ b/src/components/stub_comps/srof/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/srof/cime_config/buildnml b/src/components/stub_comps/srof/cime_config/buildnml similarity index 100% rename from components/stub_comps/srof/cime_config/buildnml rename to src/components/stub_comps/srof/cime_config/buildnml diff --git a/components/stub_comps/srof/cime_config/config_component.xml b/src/components/stub_comps/srof/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/srof/cime_config/config_component.xml rename to src/components/stub_comps/srof/cime_config/config_component.xml diff --git a/components/stub_comps/srof/cpl/rof_comp_mct.F90 b/src/components/stub_comps/srof/cpl/rof_comp_mct.F90 similarity index 100% rename from components/stub_comps/srof/cpl/rof_comp_mct.F90 rename to src/components/stub_comps/srof/cpl/rof_comp_mct.F90 diff --git a/components/stub_comps/swav/cime_config/buildlib b/src/components/stub_comps/swav/cime_config/buildlib similarity index 90% rename from components/stub_comps/swav/cime_config/buildlib rename to src/components/stub_comps/swav/cime_config/buildlib index 0cb5ff5666a..04589b411ac 100755 --- a/components/stub_comps/swav/cime_config/buildlib +++ b/src/components/stub_comps/swav/cime_config/buildlib @@ -6,7 +6,7 @@ build stub model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/stub_comps/swav/cime_config/buildnml b/src/components/stub_comps/swav/cime_config/buildnml similarity index 100% rename from components/stub_comps/swav/cime_config/buildnml rename to src/components/stub_comps/swav/cime_config/buildnml diff --git a/components/stub_comps/swav/cime_config/config_component.xml b/src/components/stub_comps/swav/cime_config/config_component.xml similarity index 100% rename from components/stub_comps/swav/cime_config/config_component.xml rename to src/components/stub_comps/swav/cime_config/config_component.xml diff --git a/components/stub_comps/swav/cpl/wav_comp_mct.F90 b/src/components/stub_comps/swav/cpl/wav_comp_mct.F90 similarity index 100% rename from components/stub_comps/swav/cpl/wav_comp_mct.F90 rename to src/components/stub_comps/swav/cpl/wav_comp_mct.F90 diff --git a/components/xcpl_comps/xatm/cime_config/buildlib b/src/components/xcpl_comps/xatm/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xatm/cime_config/buildlib rename to src/components/xcpl_comps/xatm/cime_config/buildlib index 37fde2cc30d..594aa4a3459 100755 --- a/components/xcpl_comps/xatm/cime_config/buildlib +++ b/src/components/xcpl_comps/xatm/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xatm/cime_config/buildnml b/src/components/xcpl_comps/xatm/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xatm/cime_config/buildnml rename to src/components/xcpl_comps/xatm/cime_config/buildnml index b8104013501..50a814c79d3 100755 --- a/components/xcpl_comps/xatm/cime_config/buildnml +++ b/src/components/xcpl_comps/xatm/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xatm/cime_config/config_component.xml b/src/components/xcpl_comps/xatm/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xatm/cime_config/config_component.xml rename to src/components/xcpl_comps/xatm/cime_config/config_component.xml diff --git a/components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 b/src/components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 rename to src/components/xcpl_comps/xatm/cpl/atm_comp_mct.F90 diff --git a/components/xcpl_comps/xglc/cime_config/buildlib b/src/components/xcpl_comps/xglc/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xglc/cime_config/buildlib rename to src/components/xcpl_comps/xglc/cime_config/buildlib index 33fcc2187de..90e2e0afd43 100755 --- a/components/xcpl_comps/xglc/cime_config/buildlib +++ b/src/components/xcpl_comps/xglc/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xglc/cime_config/buildnml b/src/components/xcpl_comps/xglc/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xglc/cime_config/buildnml rename to src/components/xcpl_comps/xglc/cime_config/buildnml index a02ca3f951f..64077a469c3 100755 --- a/components/xcpl_comps/xglc/cime_config/buildnml +++ b/src/components/xcpl_comps/xglc/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xglc/cime_config/config_component.xml b/src/components/xcpl_comps/xglc/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xglc/cime_config/config_component.xml rename to src/components/xcpl_comps/xglc/cime_config/config_component.xml diff --git a/components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 b/src/components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 rename to src/components/xcpl_comps/xglc/cpl/glc_comp_mct.F90 diff --git a/components/xcpl_comps/xice/cime_config/buildlib b/src/components/xcpl_comps/xice/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xice/cime_config/buildlib rename to src/components/xcpl_comps/xice/cime_config/buildlib index 57137c68a9d..4b4157bd79e 100755 --- a/components/xcpl_comps/xice/cime_config/buildlib +++ b/src/components/xcpl_comps/xice/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xice/cime_config/buildnml b/src/components/xcpl_comps/xice/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xice/cime_config/buildnml rename to src/components/xcpl_comps/xice/cime_config/buildnml index f6fcbcc8a3b..e023dc7c6d2 100755 --- a/components/xcpl_comps/xice/cime_config/buildnml +++ b/src/components/xcpl_comps/xice/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xice/cime_config/config_component.xml b/src/components/xcpl_comps/xice/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xice/cime_config/config_component.xml rename to src/components/xcpl_comps/xice/cime_config/config_component.xml diff --git a/components/xcpl_comps/xice/cpl/ice_comp_mct.F90 b/src/components/xcpl_comps/xice/cpl/ice_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xice/cpl/ice_comp_mct.F90 rename to src/components/xcpl_comps/xice/cpl/ice_comp_mct.F90 diff --git a/components/xcpl_comps/xlnd/cime_config/buildlib b/src/components/xcpl_comps/xlnd/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xlnd/cime_config/buildlib rename to src/components/xcpl_comps/xlnd/cime_config/buildlib index 52d6dde1154..1f61fb8c23d 100755 --- a/components/xcpl_comps/xlnd/cime_config/buildlib +++ b/src/components/xcpl_comps/xlnd/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xlnd/cime_config/buildnml b/src/components/xcpl_comps/xlnd/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xlnd/cime_config/buildnml rename to src/components/xcpl_comps/xlnd/cime_config/buildnml index f2354a124ae..760d75e5c06 100755 --- a/components/xcpl_comps/xlnd/cime_config/buildnml +++ b/src/components/xcpl_comps/xlnd/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xlnd/cime_config/config_component.xml b/src/components/xcpl_comps/xlnd/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xlnd/cime_config/config_component.xml rename to src/components/xcpl_comps/xlnd/cime_config/config_component.xml diff --git a/components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 b/src/components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 rename to src/components/xcpl_comps/xlnd/cpl/lnd_comp_mct.F90 diff --git a/components/xcpl_comps/xocn/cime_config/buildlib b/src/components/xcpl_comps/xocn/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xocn/cime_config/buildlib rename to src/components/xcpl_comps/xocn/cime_config/buildlib index f376d7ae2ee..5a4aa3bfc21 100755 --- a/components/xcpl_comps/xocn/cime_config/buildlib +++ b/src/components/xcpl_comps/xocn/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xocn/cime_config/buildnml b/src/components/xcpl_comps/xocn/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xocn/cime_config/buildnml rename to src/components/xcpl_comps/xocn/cime_config/buildnml index 246cea26602..c6f9453766e 100755 --- a/components/xcpl_comps/xocn/cime_config/buildnml +++ b/src/components/xcpl_comps/xocn/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xocn/cime_config/config_component.xml b/src/components/xcpl_comps/xocn/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xocn/cime_config/config_component.xml rename to src/components/xcpl_comps/xocn/cime_config/config_component.xml diff --git a/components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 b/src/components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 rename to src/components/xcpl_comps/xocn/cpl/ocn_comp_mct.F90 diff --git a/components/xcpl_comps/xrof/cime_config/buildlib b/src/components/xcpl_comps/xrof/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xrof/cime_config/buildlib rename to src/components/xcpl_comps/xrof/cime_config/buildlib index 3976c96932d..2245cae8312 100755 --- a/components/xcpl_comps/xrof/cime_config/buildlib +++ b/src/components/xcpl_comps/xrof/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xrof/cime_config/buildnml b/src/components/xcpl_comps/xrof/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xrof/cime_config/buildnml rename to src/components/xcpl_comps/xrof/cime_config/buildnml index 06e9f1b86bd..182cf39efa5 100755 --- a/components/xcpl_comps/xrof/cime_config/buildnml +++ b/src/components/xcpl_comps/xrof/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xrof/cime_config/config_component.xml b/src/components/xcpl_comps/xrof/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xrof/cime_config/config_component.xml rename to src/components/xcpl_comps/xrof/cime_config/config_component.xml diff --git a/components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 b/src/components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 rename to src/components/xcpl_comps/xrof/cpl/rof_comp_mct.F90 diff --git a/components/xcpl_comps/xshare/dead_data_mod.F90 b/src/components/xcpl_comps/xshare/dead_data_mod.F90 similarity index 100% rename from components/xcpl_comps/xshare/dead_data_mod.F90 rename to src/components/xcpl_comps/xshare/dead_data_mod.F90 diff --git a/components/xcpl_comps/xshare/dead_mct_mod.F90 b/src/components/xcpl_comps/xshare/dead_mct_mod.F90 similarity index 100% rename from components/xcpl_comps/xshare/dead_mct_mod.F90 rename to src/components/xcpl_comps/xshare/dead_mct_mod.F90 diff --git a/components/xcpl_comps/xshare/dead_mod.F90 b/src/components/xcpl_comps/xshare/dead_mod.F90 similarity index 100% rename from components/xcpl_comps/xshare/dead_mod.F90 rename to src/components/xcpl_comps/xshare/dead_mod.F90 diff --git a/components/xcpl_comps/xwav/cime_config/buildlib b/src/components/xcpl_comps/xwav/cime_config/buildlib similarity index 90% rename from components/xcpl_comps/xwav/cime_config/buildlib rename to src/components/xcpl_comps/xwav/cime_config/buildlib index df2c40dd11b..8ee3b413b33 100755 --- a/components/xcpl_comps/xwav/cime_config/buildlib +++ b/src/components/xcpl_comps/xwav/cime_config/buildlib @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xwav/cime_config/buildnml b/src/components/xcpl_comps/xwav/cime_config/buildnml similarity index 95% rename from components/xcpl_comps/xwav/cime_config/buildnml rename to src/components/xcpl_comps/xwav/cime_config/buildnml index ba0ae95a8a4..b7b8576babd 100755 --- a/components/xcpl_comps/xwav/cime_config/buildnml +++ b/src/components/xcpl_comps/xwav/cime_config/buildnml @@ -6,7 +6,7 @@ build data model library import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * diff --git a/components/xcpl_comps/xwav/cime_config/config_component.xml b/src/components/xcpl_comps/xwav/cime_config/config_component.xml similarity index 100% rename from components/xcpl_comps/xwav/cime_config/config_component.xml rename to src/components/xcpl_comps/xwav/cime_config/config_component.xml diff --git a/components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 b/src/components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 similarity index 100% rename from components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 rename to src/components/xcpl_comps/xwav/cpl/wav_comp_mct.F90 diff --git a/driver_cpl/bld/namelist_files/namelist_defaults_drv.xml b/src/drivers/mct/bld/namelist_files/namelist_defaults_drv.xml similarity index 100% rename from driver_cpl/bld/namelist_files/namelist_defaults_drv.xml rename to src/drivers/mct/bld/namelist_files/namelist_defaults_drv.xml diff --git a/driver_cpl/bld/namelist_files/namelist_definition_drv.xml b/src/drivers/mct/bld/namelist_files/namelist_definition_drv.xml similarity index 100% rename from driver_cpl/bld/namelist_files/namelist_definition_drv.xml rename to src/drivers/mct/bld/namelist_files/namelist_definition_drv.xml diff --git a/driver_cpl/bld/namelist_files/namelist_definition_drv_flds.xml b/src/drivers/mct/bld/namelist_files/namelist_definition_drv_flds.xml similarity index 100% rename from driver_cpl/bld/namelist_files/namelist_definition_drv_flds.xml rename to src/drivers/mct/bld/namelist_files/namelist_definition_drv_flds.xml diff --git a/driver_cpl/bld/namelist_files/namelist_definition_modio.xml b/src/drivers/mct/bld/namelist_files/namelist_definition_modio.xml similarity index 100% rename from driver_cpl/bld/namelist_files/namelist_definition_modio.xml rename to src/drivers/mct/bld/namelist_files/namelist_definition_modio.xml diff --git a/driver_cpl/bld/testdir/Buildconf/camconf/drv_flds_in b/src/drivers/mct/bld/testdir/Buildconf/camconf/drv_flds_in similarity index 100% rename from driver_cpl/bld/testdir/Buildconf/camconf/drv_flds_in rename to src/drivers/mct/bld/testdir/Buildconf/camconf/drv_flds_in diff --git a/driver_cpl/bld/testdir/Buildconf/clmconf/drv_flds_in b/src/drivers/mct/bld/testdir/Buildconf/clmconf/drv_flds_in similarity index 100% rename from driver_cpl/bld/testdir/Buildconf/clmconf/drv_flds_in rename to src/drivers/mct/bld/testdir/Buildconf/clmconf/drv_flds_in diff --git a/driver_cpl/bld/testdir/env_build.xml b/src/drivers/mct/bld/testdir/env_build.xml similarity index 100% rename from driver_cpl/bld/testdir/env_build.xml rename to src/drivers/mct/bld/testdir/env_build.xml diff --git a/driver_cpl/bld/testdir/env_case.xml b/src/drivers/mct/bld/testdir/env_case.xml similarity index 100% rename from driver_cpl/bld/testdir/env_case.xml rename to src/drivers/mct/bld/testdir/env_case.xml diff --git a/driver_cpl/bld/testdir/env_mach_pes.xml b/src/drivers/mct/bld/testdir/env_mach_pes.xml similarity index 100% rename from driver_cpl/bld/testdir/env_mach_pes.xml rename to src/drivers/mct/bld/testdir/env_mach_pes.xml diff --git a/driver_cpl/bld/testdir/env_run.xml b/src/drivers/mct/bld/testdir/env_run.xml similarity index 99% rename from driver_cpl/bld/testdir/env_run.xml rename to src/drivers/mct/bld/testdir/env_run.xml index 1a1092f6f52..5711dcda702 100644 --- a/driver_cpl/bld/testdir/env_run.xml +++ b/src/drivers/mct/bld/testdir/env_run.xml @@ -1215,7 +1215,7 @@ char constant,diagnostic,prognostic - 1 + Determines how CLM will determine where CO2 is set. If value is constant, it will be set to CCSM_CO2_PPMV, if value is either diagnostic or prognostic, the atmosphere model diff --git a/driver_cpl/bld/testdir/runtest b/src/drivers/mct/bld/testdir/runtest similarity index 100% rename from driver_cpl/bld/testdir/runtest rename to src/drivers/mct/bld/testdir/runtest diff --git a/driver_cpl/bld/testdir/user_nl_cpl b/src/drivers/mct/bld/testdir/user_nl_cpl similarity index 100% rename from driver_cpl/bld/testdir/user_nl_cpl rename to src/drivers/mct/bld/testdir/user_nl_cpl diff --git a/driver_cpl/cime_config/buildexe b/src/drivers/mct/cime_config/buildexe similarity index 86% rename from driver_cpl/cime_config/buildexe rename to src/drivers/mct/cime_config/buildexe index 95c3d098e97..627330611a9 100755 --- a/driver_cpl/cime_config/buildexe +++ b/src/drivers/mct/cime_config/buildexe @@ -6,7 +6,7 @@ build model executable import sys, os -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * @@ -31,11 +31,15 @@ def _main_func(): gmake = case.get_value("GMAKE") gmake_j = case.get_value("GMAKE_J") model = case.get_value("MODEL") + num_esp = case.get_value("NUM_COMP_INST_ESP") os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION")) + expect((num_esp is None) or (int(num_esp) == 1), "ESP component restricted to one instance") + + with open('Filepath', 'w') as out: out.write(os.path.join(caseroot, "SourceMods", "src.drv") + "\n") - out.write(os.path.join(cimeroot, "driver_cpl", "driver") + "\n") + out.write(os.path.join(cimeroot, "src", "drivers", "mct", "main") + "\n") # build model executable diff --git a/driver_cpl/cime_config/buildnml b/src/drivers/mct/cime_config/buildnml similarity index 87% rename from driver_cpl/cime_config/buildnml rename to src/drivers/mct/cime_config/buildnml index 4e29c3ff83c..c2076dcad0b 100755 --- a/driver_cpl/cime_config/buildnml +++ b/src/drivers/mct/cime_config/buildnml @@ -7,16 +7,16 @@ # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import,wrong-import-position -import os, shutil, sys, glob +import os, shutil, sys, glob, itertools -_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..") +_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..") sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools")) from standard_script_setup import * from CIME.case import Case from CIME.nmlgen import NamelistGenerator from CIME.utils import expect -from CIME.utils import run_cmd_no_fail, get_model +from CIME.utils import run_cmd_no_fail, get_model, get_time_in_seconds from CIME.buildnml import create_namelist_infile, parse_input from CIME.XML.files import Files @@ -93,6 +93,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): expect(False, "basedt invalid overflow for NCPL_BASE_PERIOD %s " %ncpl_base_period) comps = case.get_values("COMP_CLASSES") + mindt = basedt for comp in comps: ncpl = case.get_value(comp.upper() + '_NCPL') if ncpl is not None: @@ -100,7 +101,8 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): totaldt = cpl_dt * int(ncpl) if totaldt != basedt: expect(False, " %s ncpl doesn't divide base dt evenly" %comp) - nmlgen.set_value(comp.lower() + '_cpl_dt', value=cpl_dt) + nmlgen.add_default(comp.lower() + '_cpl_dt', value=cpl_dt) + mindt = min(mindt, cpl_dt) # elif comp.lower() is not 'cpl': #-------------------------------- @@ -142,6 +144,7 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): # Set up the pause_component_list if pause is active pauseo = case.get_value('PAUSE_OPTION') if pauseo != 'never' and pauseo != 'none': + pausen = case.get_value('PAUSE_N') pcl = nmlgen.get_default('pause_component_list') nmlgen.add_default('pause_component_list', pcl) # Check to make sure pause_component_list is valid @@ -154,6 +157,13 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): "Invalid PAUSE_COMPONENT_LIST, %s is not a valid component type"%comp) # End for # End if + # Set esp interval + if 'nstep' in pauseo: + esp_time = mindt + else: + esp_time = get_time_in_seconds(pausen, pauseo) + + nmlgen.set_value('esp_cpl_dt', value=esp_time) # End if pause is active #-------------------------------- @@ -190,15 +200,43 @@ def _create_drv_namelists(case, infile, confdir, nmlgen, files): infiles.append(infile) if len(infiles) != 0: + + # First read the drv_flds_in files and make sure that + # for any key there are not two conflicting values + dicts = {} + for infile in infiles: + dict_ = {} + with open(infile) as myfile: + for line in myfile: + if "=" in line and '!' not in line: + name, var = line.partition("=")[::2] + name = name.strip() + var = var.strip() + dict_[name] = var + dicts[infile] = dict_ + + for first,second in itertools.combinations(dicts.keys(),2): + compare_drv_flds_in(dicts[first], dicts[second], first, second) + + # Now create drv_flds_in config = {} definition_file = [files.get_value("NAMELIST_DEFINITION_FILE", attribute={"component":"drv_flds"})] - nmlgen = NamelistGenerator(case, definition_file, files=files) skip_entry_loop = True nmlgen.init_defaults(infiles, config, skip_entry_loop=skip_entry_loop) drv_flds_in = os.path.join(caseroot, "CaseDocs", "drv_flds_in") nmlgen.write_output_file(drv_flds_in) +############################################################################### +def compare_drv_flds_in(first, second, infile1, infile2): +############################################################################### + sharedKeys = set(first.keys()).intersection(second.keys()) + for key in sharedKeys: + if first[key] != second[key]: + print('Key: {}, \n Value 1: {}, \n Value 2: {}'.format(key, first[key], second[key])) + expect(False, "incompatible settings in drv_flds_in from \n %s \n and \n %s" + % (infile1, infile2)) + ############################################################################### def _create_component_modelio_namelists(case, files): ############################################################################### @@ -290,12 +328,9 @@ def buildnml(case, caseroot, component): if case.get_value('COMP_ATM') == 'cam': # cam is actually changing the driver namelist settings cam_config_opts = case.get_value("CAM_CONFIG_OPTS") - if "adiabatic" in cam_config_opts: - infile_text = "atm_adiabatic = .true." - if "ideal" in cam_config_opts: - infile_text = "atm_ideal_phys = .true." if "aquaplanet" in cam_config_opts: infile_text = "aqua_planet = .true. \n aqua_planet_sst = 1" + user_nl_file = os.path.join(caseroot, "user_nl_cpl") namelist_infile = os.path.join(confdir, "namelist_infile") create_namelist_infile(case, user_nl_file, namelist_infile, infile_text) diff --git a/driver_cpl/cime_config/config_archive.xml b/src/drivers/mct/cime_config/config_archive.xml similarity index 100% rename from driver_cpl/cime_config/config_archive.xml rename to src/drivers/mct/cime_config/config_archive.xml diff --git a/driver_cpl/cime_config/config_component.xml b/src/drivers/mct/cime_config/config_component.xml similarity index 96% rename from driver_cpl/cime_config/config_component.xml rename to src/drivers/mct/cime_config/config_component.xml index eeda67bca8a..b6a872f7d7c 100644 --- a/driver_cpl/cime_config/config_component.xml +++ b/src/drivers/mct/cime_config/config_component.xml @@ -303,7 +303,7 @@ run_begin_stop_restart env_run.xml - Sets the run length along with STOP_N and STOP_DATE (must be nyear(s) for _GLC compsets for restarts to work properly). + Sets the run length along with STOP_N and STOP_DATE @@ -335,8 +335,7 @@ run_begin_stop_restart env_run.xml - sets frequency of model restart writes (same options as STOP_OPTION) (must be nyear(s) for _GLC compsets) - (must be nyear(s) for _GLC compsets) + sets frequency of model restart writes (same options as STOP_OPTION) @@ -368,7 +367,7 @@ run_begin_stop_restart env_run.xml - Sets the pause frequency along with PAUSE_N (must be nyear(s) for _GLC compsets for restarts to resume properly). + Sets the pause frequency along with PAUSE_N @@ -418,6 +417,23 @@ + + logical + TRUE,FALSE + FALSE + run_begin_stop_restart + env_run.xml + + ESP component runs after driver 'pause cycle' + If any component 'pauses' (see PAUSE_OPTION, PAUSE_N and + PAUSE_COMPONENT_LIST XML variables), the ESP component (if + present) will be run to process the component 'pause' (restart) + files and set any required 'resume' signals. + If true, esp_cpl_dt and esp_cpl_offset settings are ignored. + default: false + + + logical TRUE,FALSE @@ -490,12 +506,12 @@ logical TRUE,FALSE - FALSE + TRUE run_data_archive env_run.xml Logical to archive all interim restart files, not just those at eor If TRUE, perform short term archiving on all interim restart files, - not just those at the end of the run. By default, this value is FALSE. + not just those at the end of the run. By default, this value is TRUE. The restart files are saved under the specific component directory ($DOUT_S_ROOT/$CASE/$COMPONENT/rest rather than the top-level $DOUT_S_ROOT/$CASE/rest directory). Interim restart files are created using the REST_N and REST_OPTION variables. @@ -544,7 +560,7 @@ char none - moab,pbs,lsf,slurm,cobalt,cobalt_theta,none + lc_slurm,moab,pbs,lsf,slurm,cobalt,cobalt_theta,none config_batch env_batch.xml The batch system type to use for this machine. @@ -915,15 +931,6 @@ timer output depth - - char - - timing - run_flags - env_run.xml - Where to auto archive timing data - - logical TRUE,FALSE @@ -935,6 +942,23 @@ this to work. + + char + ESMF_LOGKIND_SINGLE,ESMF_LOGKIND_MULTI,ESMF_LOGKIND_NONE + ESMF_LOGKIND_NONE + run_cesm + env_run.xml + + Determines what ESMF log files (if any) are generated when + USE_ESMF_LIB is TRUE. + ESMF_LOGKIND_SINGLE: Use a single log file, combining messages from + all of the PETs. Not supported on some platforms. + ESMF_LOGKIND_MULTI: Use multiple log files -- one per PET. + ESMF_LOGKIND_NONE: Do not issue messages to a log file. + By default, no ESMF log files are generated. + + + logical TRUE,FALSE @@ -1804,6 +1828,16 @@ + + char + run_din + env_run.xml + + A regular expression to search for an indication that a run failure was caused by a node failure + and should therefore be re-attempted. + + + char UNSET @@ -1872,13 +1906,21 @@ UNSET run_mpi env_run.xml - mpi run command + override the mpi run command, do not include model executable + + integer + 0 + mach_pes + env_mach_pes.xml + Percent of extra spare nodes to allocate + + integer @@ -2477,6 +2519,23 @@ machines/compilers. + + + + char + + external_tools + env_run.xml + External script to be run before model completion + + + char + + external_tools + env_run.xml + External script to be run after model completion + + @@ -2523,6 +2582,15 @@ The machine wallclock setting. Default determined in config_machines.xml can be overwritten by testing + + char + + + job_submission + env_batch.xml + Override the batch submit command this job. Do not include executable or dependencies + + char diff --git a/driver_cpl/cime_config/config_component_acme.xml b/src/drivers/mct/cime_config/config_component_acme.xml similarity index 92% rename from driver_cpl/cime_config/config_component_acme.xml rename to src/drivers/mct/cime_config/config_component_acme.xml index 73d9fcc336d..0198b7eecb1 100644 --- a/driver_cpl/cime_config/config_component_acme.xml +++ b/src/drivers/mct/cime_config/config_component_acme.xml @@ -23,6 +23,15 @@ logical to save timing files in rundir + + char + + UNSET + run_flags + env_run.xml + Where to auto archive timing data + + integer 20 @@ -213,7 +222,7 @@ year year day - hour + day day day day @@ -247,6 +256,16 @@ 1 1 24 + 12 + 12 + 24 + 48 + 48 + 48 + 96 + 96 + 96 + 96 48 12 96 @@ -306,10 +325,7 @@ 1 1 24 - 48 - $ATM_NCPL - 12 - 96 + $ATM_NCPL run_coupling env_run.xml @@ -335,11 +351,17 @@ 1 1 24 - 48 - $ATM_NCPL - 12 - 48 - 96 + 12 + 12 + 24 + 48 + 48 + 48 + 48 + 48 + 48 + 96 + run_coupling env_run.xml @@ -365,6 +387,24 @@ Number of glc coupling intervals per NCPL_BASE_PERIOD. + + char + glc_coupling_period,yearly + glc_coupling_period + run_coupling + env_run.xml + Period at which coupler averages fields sent to GLC. + This supports doing the averaging to GLC less frequently than GLC is called + (i.e., separating the averaging frequency from the calling frequency). + This is useful because there are benefits to only averaging the GLC inputs + as frequently as they are really needed (yearly for CISM), but GLC needs to + still be called more frequently than that in order to support mid-year restarts. + + Setting GLC_AVG_PERIOD to 'glc_coupling_period' means that the averaging is + done exactly when the GLC is called (governed by GLC_NCPL). + + + integer 8 diff --git a/driver_cpl/cime_config/config_component_cesm.xml b/src/drivers/mct/cime_config/config_component_cesm.xml similarity index 94% rename from driver_cpl/cime_config/config_component_cesm.xml rename to src/drivers/mct/cime_config/config_component_cesm.xml index fbad2fde32f..a14438fdc1c 100644 --- a/driver_cpl/cime_config/config_component_cesm.xml +++ b/src/drivers/mct/cime_config/config_component_cesm.xml @@ -23,6 +23,15 @@ logical to save timing files in rundir + + char + + timing + run_flags + env_run.xml + Where to auto archive timing data + + integer 0 @@ -200,6 +209,27 @@ Number of glc coupling intervals per NCPL_BASE_PERIOD. + + char + glc_coupling_period,yearly + yearly + run_coupling + env_run.xml + Period at which coupler averages fields sent to GLC. + This supports doing the averaging to GLC less frequently than GLC is called + (i.e., separating the averaging frequency from the calling frequency). + This is useful because there are benefits to only averaging the GLC inputs + as frequently as they are really needed (yearly for CISM), but GLC needs to + still be called more frequently than that in order to support mid-year restarts. + + Setting GLC_AVG_PERIOD to 'glc_coupling_period' means that the averaging is + done exactly when the GLC is called (governed by GLC_NCPL). + + IMPORTANT: In order to restart mid-year when running with CISM, you MUST specify GLC_AVG_PERIOD = 'yearly'. + If using GLC_AVG_PERIOD = 'glc_coupling_period' with CISM, you can only restart on year boundaries. + + + integer 8 diff --git a/driver_cpl/cime_config/config_compsets.xml b/src/drivers/mct/cime_config/config_compsets.xml similarity index 72% rename from driver_cpl/cime_config/config_compsets.xml rename to src/drivers/mct/cime_config/config_compsets.xml index c11354e1450..fb129089b51 100644 --- a/driver_cpl/cime_config/config_compsets.xml +++ b/src/drivers/mct/cime_config/config_compsets.xml @@ -9,19 +9,22 @@ The compset longname below has the specified order atm, lnd, ice, ocn, river, glc wave esp cesm-options - The notation for the compset longname is + The notation for the compset longname below is TIME_ATM[%phys]_LND[%phys]_ICE[%phys]_OCN[%phys]_ROF[%phys]_GLC[%phys]_WAV[%phys][_ESP][_BGC%phys] - Where for the CAM specific compsets below the following is supported + + The following compsets are those that can be tested in CIME stand-alone configurations + without any prognostic components. + For the compsets below the following are the only allowable values of the components. + TIME = Time period (e.g. 2000, HIST, RCP8...) - ATM = [CAM4, CAM5] - LND = [CLM40, CLM45, CLM50, SLND] - ICE = [CICE, DICE, SICE] - OCN = [DOCN, ,AQUAP, SOCN] - ROF = [RTM, SROF] - GLC = [CISM1, CISM2, SGLC] - WAV = [SWAV WW3] - ESP = [SESP DESP] - BGC = optional BGC scenario + ATM = [DATM, SATM, XATM] + LND = [DLND, SLND, XLND] + ICE = [DICE, SICE, XICE] + OCN = [DOCN, SOCN, XOCN] + ROF = [DROF, SROF, XROF] + GLC = [ SGLC ] + WAV = [DWAV, SWAV ] + ESP = [DESP, SESP ] The OPTIONAL %phys attributes specify submodes of the given system For example DOCN%DOM is the data ocean model for DOCN @@ -42,6 +45,16 @@ 2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV + + ADLND + 2000_SATM_DLND%SCPL_SICE_SOCN_SROF_SGLC_SWAV + + + + ADWAV + 2000_SATM_SLND_SICE_SOCN_SROF_SGLC_DWAV%CLIMO + + ADESP 2000_DATM%NYF_SLND_DICE%SSMI_DOCN%DOM_DROF%NYF_SGLC_SWAV_DESP @@ -57,11 +70,6 @@ 2000_DATM%IAF_SLND_DICE%IAF_DOCN%IAF_DROF%IAF_SGLC_SWAV - - AWAV - 2000_DATM%WW3_SLND_DICE%COPY_DOCN%COPY_SROF_SGLC_WW3 - - S 2000_SATM_SLND_SICE_SOCN_SROF_SGLC_SWAV_SESP diff --git a/driver_cpl/cime_config/config_pes.xml b/src/drivers/mct/cime_config/config_pes.xml similarity index 76% rename from driver_cpl/cime_config/config_pes.xml rename to src/drivers/mct/cime_config/config_pes.xml index f1fa86ae5fd..ad332b3ff03 100644 --- a/driver_cpl/cime_config/config_pes.xml +++ b/src/drivers/mct/cime_config/config_pes.xml @@ -166,5 +166,45 @@ + + + + PE layout for tests + + 64 + 64 + 64 + 64 + 64 + 64 + 64 + 64 + 64 + + + 16 + 16 + 16 + 16 + 16 + 16 + 16 + 16 + 16 + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/driver_cpl/cime_config/namelist_definition_drv.xml b/src/drivers/mct/cime_config/namelist_definition_drv.xml similarity index 96% rename from driver_cpl/cime_config/namelist_definition_drv.xml rename to src/drivers/mct/cime_config/namelist_definition_drv.xml index 844bcc65512..82a7d634e78 100644 --- a/driver_cpl/cime_config/namelist_definition_drv.xml +++ b/src/drivers/mct/cime_config/namelist_definition_drv.xml @@ -1,6 +1,6 @@ - + @@ -352,7 +352,7 @@ - + logical expdef seq_infodata_inparm @@ -1387,7 +1387,7 @@ - + integer time seq_timemgr_inparm @@ -1398,12 +1398,9 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, and has valid values: hour,day,year,decade - - -999 - - + integer time seq_timemgr_inparm @@ -1414,12 +1411,9 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade - - -999 - - + integer time seq_timemgr_inparm @@ -1428,12 +1422,9 @@ currently set by default to 10800 seconds. default: 10800 - - 10800 - - + integer time seq_timemgr_inparm @@ -1444,12 +1435,9 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade - - -999 - - + integer time seq_timemgr_inparm @@ -1460,12 +1448,9 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade - - -999 - - + integer time seq_timemgr_inparm @@ -1476,12 +1461,30 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade + + + + char + time + seq_timemgr_inparm + glc_coupling_period,yearly - -999 + $GLC_AVG_PERIOD + + Period at which coupler averages fields sent to GLC. + This supports doing the averaging to GLC less frequently than GLC is called + (i.e., separating the averaging frequency from the calling frequency). + This is useful because there are benefits to only averaging the GLC inputs + as frequently as they are really needed (yearly for CISM), but GLC needs to + still be called more frequently than that in order to support mid-year restarts. + + Setting glc_avg_period to 'glc_coupling_period' means that the averaging is + done exactly when the GLC is called (governed by GLC_NCPL). + - + integer time seq_timemgr_inparm @@ -1492,6 +1495,20 @@ NCPL_BASE_PERIOD is also set in env_run.xml and is the base period associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade + + + + integer + time + seq_timemgr_inparm + + esp run interval in seconds + esp_cpl_dt is the number of times the esp is run per NCPL_BASE_PERIOD + NCPL_BASE_PERIOD is set in env_run.xml and is the base period + associated with NCPL coupling frequency, nad has valid values: hour,day,year,decade + default value set by buildnml to be the pause interval if pause is active + otherwise, it is set to the shortest component coupling time + -999 @@ -1569,6 +1586,36 @@ + + integer + time + seq_timemgr_inparm + + esp coupling interval offset in seconds default: 0 + + + 0 + + + + + logical + time + seq_timemgr_inparm + + true => ESP component runs after driver 'pause cycle' + If any component 'pauses' (see PAUSE_OPTION, PAUSE_N and + PAUSE_COMPONENT_LIST XML variables), the ESP component (if + present) will be run to process the component 'pause' (restart) + files and set any required 'resume' signals. + If true, esp_cpl_dt and esp_cpl_offset settings are ignored. + default: true + + + .true. + + + char time @@ -2602,6 +2649,24 @@ + + char + ccsm_pes + ccsm_pes + + Determines what ESMF log files (if any) are generated when + USE_ESMF_LIB is TRUE. + ESMF_LOGKIND_SINGLE: Use a single log file, combining messages from + all of the PETs. Not supported on some platforms. + ESMF_LOGKIND_MULTI: Use multiple log files — one per PET. + ESMF_LOGKIND_NONE: Do not issue messages to a log file. + By default, no ESMF log files are generated. + + + $ESMF_LOGFILE_KIND + + + diff --git a/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml b/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml new file mode 100644 index 00000000000..08847103afd --- /dev/null +++ b/src/drivers/mct/cime_config/namelist_definition_drv_flds.xml @@ -0,0 +1,148 @@ + + + + + + + + + + + + + + char + abs + drv_flds_in + megan_emis_nl + + File containing MEGAN emissions factors. Includes the list of MEGAN compounds that can be + used in the Comp_Name variable on the file. + + + + + char(100) + drv_flds_in + megan_emis_nl + + MEGAN specifier. This is in the form of: Chem-compound = megan_compound(s) + where megan_compound(s) can be the sum of megan compounds with a "+" between them. + In each equation, the item to the left of the equal sign is a CAM chemistry compound, the + items to the right are compounds known to the MEGAN model (single or combinations). + For example: megan_specifier = 'ISOP = isoprene', 'C10H16 = pinene_a + carene_3 + thujene_a' + + + + + logical + drv_flds_in + megan_emis_nl + + MEGAN mapped isoprene emissions factors switch + If TRUE then use mapped MEGAN emissions factors for isoprene. + + + + + char(150) + drv_flds_in + drv_physics + + List of possible MEGAN compounds to use + (the list used by the simulation is on the megan_factors_file as the Comp_Name) + + + + + + + + + char + dry-deposition + drydep_inparm + xactive_lnd,xactive_atm,table + + Where dry deposition is calculated (from land, atmosphere, or from a table) + This specifies the method used to calculate dry + deposition velocities of gas-phase chemical species. The available methods are: + 'table' - prescribed method in CAM + 'xactive_atm' - interactive method in CAM + 'xactive_lnd' - interactive method in CLM + + + + + char(300) + dry-deposition + drydep_inparm + + List of species that undergo dry deposition. + + + + + + + + + char(2) + nitrogen deposition + ndep_inparm + + List of nitrogen deposition fluxes to be sent from CAM to surfae models. + + + + + + + + + char + abs + Fire_emissions + fire_emis_nl + + File containing fire emissions factors. + + + + + char(100) + Fire_emissions + fire_emis_nl + + Fire emissions specifier. + + + + + logical + Fire_emissions + fire_emis_nl + + If ture fire emissions are input into atmosphere as elevated forcings. + Otherwise they are treated as surface emissions. + + + + + + + + + char + carma + carma_inparm + + List of fluxes needed by the CARMA model, from CLM to CAM. + + + + diff --git a/driver_cpl/cime_config/namelist_definition_modelio.xml b/src/drivers/mct/cime_config/namelist_definition_modelio.xml similarity index 100% rename from driver_cpl/cime_config/namelist_definition_modelio.xml rename to src/drivers/mct/cime_config/namelist_definition_modelio.xml diff --git a/driver_cpl/cime_config/testdefs/testlist_drv.xml b/src/drivers/mct/cime_config/testdefs/testlist_drv.xml similarity index 100% rename from driver_cpl/cime_config/testdefs/testlist_drv.xml rename to src/drivers/mct/cime_config/testdefs/testlist_drv.xml diff --git a/driver_cpl/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands b/src/drivers/mct/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands similarity index 100% rename from driver_cpl/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands rename to src/drivers/mct/cime_config/testdefs/testmods_dirs/drv/5steps/shell_commands diff --git a/driver_cpl/cime_config/user_nl_cpl b/src/drivers/mct/cime_config/user_nl_cpl similarity index 100% rename from driver_cpl/cime_config/user_nl_cpl rename to src/drivers/mct/cime_config/user_nl_cpl diff --git a/driver_cpl/doc/bookinfo.xml b/src/drivers/mct/doc/bookinfo.xml similarity index 100% rename from driver_cpl/doc/bookinfo.xml rename to src/drivers/mct/doc/bookinfo.xml diff --git a/driver_cpl/doc/chap1.xml b/src/drivers/mct/doc/chap1.xml similarity index 99% rename from driver_cpl/doc/chap1.xml rename to src/drivers/mct/doc/chap1.xml index d302fa67b65..74356103095 100644 --- a/driver_cpl/doc/chap1.xml +++ b/src/drivers/mct/doc/chap1.xml @@ -20,8 +20,8 @@ and carries out mapping (interpolation), merging, diagnostics, and other calculations. The name cpl7 refers to the source code associated with both the driver and the coupler parts of the model. cpl7 code is located in the CIME source -tree under driver_cpl/ and the main program of CESM2 is -driver_cpl/driver/cesm_driver.F90. +tree under drivers/mct/ and the main program of CESM2 is +drivers/mct/main/cesm_driver.F90. This document provides a general overview of the cpl7 design. Then diff --git a/driver_cpl/doc/chap2.xml b/src/drivers/mct/doc/chap2.xml similarity index 100% rename from driver_cpl/doc/chap2.xml rename to src/drivers/mct/doc/chap2.xml diff --git a/driver_cpl/doc/figx1.jpg b/src/drivers/mct/doc/figx1.jpg similarity index 100% rename from driver_cpl/doc/figx1.jpg rename to src/drivers/mct/doc/figx1.jpg diff --git a/driver_cpl/doc/namelist.xml b/src/drivers/mct/doc/namelist.xml similarity index 100% rename from driver_cpl/doc/namelist.xml rename to src/drivers/mct/doc/namelist.xml diff --git a/driver_cpl/doc/rundocbook.csh b/src/drivers/mct/doc/rundocbook.csh similarity index 100% rename from driver_cpl/doc/rundocbook.csh rename to src/drivers/mct/doc/rundocbook.csh diff --git a/driver_cpl/doc/stylesheet.dsl b/src/drivers/mct/doc/stylesheet.dsl similarity index 100% rename from driver_cpl/doc/stylesheet.dsl rename to src/drivers/mct/doc/stylesheet.dsl diff --git a/driver_cpl/doc/ug.xml b/src/drivers/mct/doc/ug.xml similarity index 100% rename from driver_cpl/doc/ug.xml rename to src/drivers/mct/doc/ug.xml diff --git a/driver_cpl/driver/CMakeLists.txt b/src/drivers/mct/main/CMakeLists.txt similarity index 100% rename from driver_cpl/driver/CMakeLists.txt rename to src/drivers/mct/main/CMakeLists.txt diff --git a/driver_cpl/driver/cesm_comp_mod.F90 b/src/drivers/mct/main/cesm_comp_mod.F90 similarity index 97% rename from driver_cpl/driver/cesm_comp_mod.F90 rename to src/drivers/mct/main/cesm_comp_mod.F90 index afc6b863ff4..f9a8637fa1a 100644 --- a/driver_cpl/driver/cesm_comp_mod.F90 +++ b/src/drivers/mct/main/cesm_comp_mod.F90 @@ -91,6 +91,7 @@ module cesm_comp_mod use seq_timemgr_mod, only: seq_timemgr_alarm_ocnrun use seq_timemgr_mod, only: seq_timemgr_alarm_icerun use seq_timemgr_mod, only: seq_timemgr_alarm_glcrun + use seq_timemgr_mod, only: seq_timemgr_alarm_glcrun_avg use seq_timemgr_mod, only: seq_timemgr_alarm_ocnnext use seq_timemgr_mod, only: seq_timemgr_alarm_tprof use seq_timemgr_mod, only: seq_timemgr_alarm_histavg @@ -99,13 +100,15 @@ module cesm_comp_mod use seq_timemgr_mod, only: seq_timemgr_alarm_esprun use seq_timemgr_mod, only: seq_timemgr_alarm_barrier use seq_timemgr_mod, only: seq_timemgr_alarm_pause - use seq_timemgr_mod, only: pause_component_list=>seq_timemgr_pause_component_list + use seq_timemgr_mod, only: seq_timemgr_pause_active + use seq_timemgr_mod, only: seq_timemgr_pause_component_active + use seq_timemgr_mod, only: seq_timemgr_pause_component_index ! "infodata" gathers various control flags into one datatype use seq_infodata_mod, only: seq_infodata_putData, seq_infodata_GetData use seq_infodata_mod, only: seq_infodata_init, seq_infodata_exchange use seq_infodata_mod, only: seq_infodata_type, seq_infodata_orb_variable_year - use seq_infodata_mod, only: seq_infodata_print + use seq_infodata_mod, only: seq_infodata_print, seq_infodata_init2 ! domain related routines use seq_domain_mct, only : seq_domain_check @@ -252,12 +255,12 @@ module cesm_comp_mod logical :: ocnrun_alarm ! ocn run alarm logical :: ocnnext_alarm ! ocn run alarm on next timestep logical :: glcrun_alarm ! glc run alarm + logical :: glcrun_avg_alarm ! glc run averaging alarm logical :: rofrun_alarm ! rof run alarm logical :: wavrun_alarm ! wav run alarm logical :: esprun_alarm ! esp run alarm logical :: tprof_alarm ! timing profile alarm logical :: barrier_alarm ! barrier alarm - logical :: pause_alarm ! pause alarm logical :: t1hr_alarm ! alarm every hour logical :: t2hr_alarm ! alarm every two hours logical :: t3hr_alarm ! alarm every three hours @@ -265,6 +268,8 @@ module cesm_comp_mod logical :: t12hr_alarm ! alarm every twelve hours logical :: t24hr_alarm ! alarm every twentyfour hours logical :: t1yr_alarm ! alarm every year, at start of year + logical :: pause_alarm ! pause alarm + integer :: drv_index ! seq_timemgr index for driver real(r8) :: days_per_year = 365.0 ! days per year @@ -1002,6 +1007,11 @@ subroutine cesm_pre_init2() call seq_timemgr_clockPrint(seq_SyncClock) endif + !---------------------------------------------------------- + !| Initialize infodata items which need the clocks + !---------------------------------------------------------- + call seq_infodata_init2(infodata, GLOID) + call seq_infodata_getData(infodata, & orb_iyear=orb_iyear, & orb_iyear_align=orb_iyear_align, & @@ -1966,6 +1976,11 @@ subroutine cesm_init() call t_stopf('comp_init_cc_atm2') endif ! atm present + !---------------------------------------------------------- + !| Get time manager's index for driver + !---------------------------------------------------------- + drv_index = seq_timemgr_pause_component_index('drv') + !---------------------------------------------------------- !| Read driver restart file, overwrite anything previously sent or computed !---------------------------------------------------------- @@ -2052,8 +2067,8 @@ end subroutine cesm_init !=============================================================================== subroutine cesm_run() - use seq_comm_mct, only: atm_layout, lnd_layout, ice_layout, glc_layout, rof_layout, & - ocn_layout, wav_layout, esp_layout + use seq_comm_mct, only: atm_layout, lnd_layout, ice_layout, glc_layout, & + rof_layout, ocn_layout, wav_layout, esp_layout use shr_string_mod, only: shr_string_listGetIndexF ! gptl timer lookup variables @@ -2062,6 +2077,7 @@ subroutine cesm_run() ! Driver pause/resume logical :: drv_pause ! Driver writes pause restart file character(len=CL) :: drv_resume ! Driver resets state from restart file + integer :: iamroot_ESPID 101 format( A, 2i8, 12A, A, F8.2, A, F8.2 ) 102 format( A, 2i8, A, 8L3 ) @@ -2117,6 +2133,11 @@ subroutine cesm_run() ! (this is time that models should have before they return ! to the driver). Write timestamp and run alarm status !---------------------------------------------------------- + ! Note that the glcrun_avg_alarm just controls what is passed to glc in terms + ! of averaged fields - it does NOT control when glc is called currently - + ! glc will be called on the glcrun_alarm setting - but it might not be passed relevant + ! info if the time averaging period to accumulate information passed to glc is greater + ! than the glcrun interval call seq_timemgr_clockAdvance( seq_SyncClock, force_stop, force_stop_ymd, force_stop_tod) call seq_timemgr_EClockGetData( EClock_d, curr_ymd=ymd, curr_tod=tod ) @@ -2127,6 +2148,7 @@ subroutine cesm_run() rofrun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_rofrun) icerun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_icerun) glcrun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_glcrun) + glcrun_avg_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_glcrun_avg) wavrun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_wavrun) esprun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_esprun) ocnrun_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_ocnrun) @@ -2138,49 +2160,19 @@ subroutine cesm_run() barrier_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_barrier) pause_alarm = seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_pause) - ! Determine wich components need to write pause (restart) files - if (pause_alarm) then - if (trim(pause_component_list) == 'all') then - drv_pause = .true. - call seq_infodata_putData(infodata, atm_pause=.true., & - lnd_pause=.true., ocn_pause=.true., ice_pause=.true., & - glc_pause=.true., rof_pause=.true., wav_pause=.true., & - cpl_pause=.true.) - else if (trim(pause_component_list) /= 'none') then - if (shr_string_listGetIndexF(pause_component_list, 'atm') > 0) then - call seq_infodata_putData(infodata, atm_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'lnd') > 0) then - call seq_infodata_putData(infodata, lnd_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'ocn') > 0) then - call seq_infodata_putData(infodata, ocn_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'ice') > 0) then - call seq_infodata_putData(infodata, ice_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'glc') > 0) then - call seq_infodata_putData(infodata, glc_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'rof') > 0) then - call seq_infodata_putData(infodata, rof_pause=.true.) - end if - if (shr_string_listGetIndexF(pause_component_list, 'wav') > 0) then - call seq_infodata_putData(infodata, wav_pause=.true.) - end if - if ( (shr_string_listGetIndexF(pause_component_list, 'cpl') > 0) .or.& - (shr_string_listGetIndexF(pause_component_list, 'drv') > 0)) then - drv_pause = .true. - call seq_infodata_putData(infodata, cpl_pause=.true.) - end if - end if - else - drv_pause = .false. - call seq_infodata_putData(infodata, atm_pause=.false., & - lnd_pause=.false., ocn_pause=.false., ice_pause=.false., & - glc_pause=.false., rof_pause=.false., wav_pause=.false., & - cpl_pause=.false.) - end if ! pause alarm + ! Does the driver need to pause? + drv_pause = pause_alarm .and. seq_timemgr_pause_component_active(drv_index) + + ! Check alarm consistency + if (glcrun_avg_alarm .and. .not. glcrun_alarm) then + write(logunit,*) 'ERROR: glcrun_avg_alarm is true, but glcrun_alarm is false' + write(logunit,*) 'Make sure that NCPL_BASE_PERIOD, GLC_NCPL and GLC_AVG_PERIOD' + write(logunit,*) 'are set so that glc averaging only happens at glc coupling times.' + write(logunit,*) '(It is allowable for glc coupling to be more frequent than glc averaging,' + write(logunit,*) 'but not for glc averaging to be more frequent than glc coupling.)' + call shr_sys_abort(subname//' glcrun_avg_alarm is true, but glcrun_alarm is false') + end if + ! this probably belongs in seq_timemgr somewhere using proper clocks t1hr_alarm = .false. @@ -2198,8 +2190,6 @@ subroutine cesm_run() if (tod == 0) t24hr_alarm = .true. if (month==1 .and. day==1 .and. tod==0) t1yr_alarm = .true. - call seq_infodata_putData(infodata, glcrun_alarm=glcrun_alarm) - if (seq_timemgr_alarmIsOn(EClock_d,seq_timemgr_alarm_datestop)) then if (iamroot_CPLID) then write(logunit,*) ' ' @@ -2927,20 +2917,36 @@ subroutine cesm_run() if (drv_threading) call seq_comm_setnthreads(nthreads_CPLID) if (lnd_c2_glc) then - call prep_glc_accum_avg(timer='CPL:glcprep_avg') + ! NOTE - only create appropriate input to glc if the avg_alarm is on + if (glcrun_avg_alarm) then + call prep_glc_accum_avg(timer='CPL:glcprep_avg') - ! Note that l2x_gx is obtained from mapping the module variable l2gacc_lx - call prep_glc_calc_l2x_gx(fractions_lx, timer='CPL:glcprep_lnd2glc') + ! Note that l2x_gx is obtained from mapping the module variable l2gacc_lx + call prep_glc_calc_l2x_gx(fractions_lx, timer='CPL:glcprep_lnd2glc') - call prep_glc_mrg(infodata, fractions_gx, timer_mrg='CPL:glcprep_mrgx2g') + call prep_glc_mrg(infodata, fractions_gx, timer_mrg='CPL:glcprep_mrgx2g') - call component_diag(infodata, glc, flow='x2c', comment='send glc', & - info_debug=info_debug, timer_diag='CPL:glcprep_diagav') - endif + call component_diag(infodata, glc, flow='x2c', comment='send glc', & + info_debug=info_debug, timer_diag='CPL:glcprep_diagav') + + else + call prep_glc_zero_fields() + end if ! glcrun_avg_alarm + end if ! lnd_c2_glc if (drv_threading) call seq_comm_setnthreads(nthreads_GLOID) call t_drvstopf ('CPL:GLCPREP',cplrun=.true.) - endif + + end if ! iamin_CPLID .and. glc_prognostic + + ! Set the infodata field on all tasks (not just those with iamin_CPLID). + if (glc_prognostic) then + if (glcrun_avg_alarm) then + call seq_infodata_PutData(infodata, glc_valid_input=.true.) + else + call seq_infodata_PutData(infodata, glc_valid_input=.false.) + end if + end if !---------------------------------------------------- !| cpl -> glc @@ -3726,6 +3732,10 @@ subroutine cesm_run() comp_prognostic=esp_prognostic, comp_num=comp_num_esp, & timer_barrier= 'CPL:ESP_RUN_BARRIER', timer_comp_run='CPL:ESP_RUN', & run_barriers=run_barriers, ymd=ymd, tod=tod,comp_layout=esp_layout) + !--------------------------------------------------------------------- + !| ESP computes resume options for other components -- update everyone + !--------------------------------------------------------------------- + call seq_infodata_exchange(infodata, CPLALLESPID, 'esp2cpl_run') endif !---------------------------------------------------------- @@ -3733,16 +3743,19 @@ subroutine cesm_run() !---------------------------------------------------------- call seq_infodata_GetData(infodata, cpl_resume=drv_resume) if (len_trim(drv_resume) > 0) then - if (iamroot_CPLID) then - write(logunit,103) subname,' Reading restart (resume) file ',trim(drv_resume) - call shr_sys_flush(logunit) - end if - if (iamin_CPLID) then - call seq_rest_read(drv_resume, infodata, & - atm, lnd, ice, ocn, rof, glc, wav, esp, & - fractions_ax, fractions_lx, fractions_ix, fractions_ox, & - fractions_rx, fractions_gx, fractions_wx) - end if + if (iamroot_CPLID) then + write(logunit,103) subname,' Reading restart (resume) file ',trim(drv_resume) + call shr_sys_flush(logunit) + end if + if (iamin_CPLID) then + call seq_rest_read(drv_resume, infodata, & + atm, lnd, ice, ocn, rof, glc, wav, esp, & + fractions_ax, fractions_lx, fractions_ix, fractions_ox, & + fractions_rx, fractions_gx, fractions_wx) + end if + ! Clear the resume file so we don't try to read it again + drv_resume = ' ' + call seq_infodata_PutData(infodata, cpl_resume=drv_resume) end if !---------------------------------------------------------- diff --git a/driver_cpl/driver/cesm_driver.F90 b/src/drivers/mct/main/cesm_driver.F90 similarity index 92% rename from driver_cpl/driver/cesm_driver.F90 rename to src/drivers/mct/main/cesm_driver.F90 index c846dea2887..7c22d4aa53c 100644 --- a/driver_cpl/driver/cesm_driver.F90 +++ b/src/drivers/mct/main/cesm_driver.F90 @@ -20,8 +20,9 @@ program cesm_driver !---------------------------------------------------------------------------- ! share code & libs !---------------------------------------------------------------------------- - use perf_mod - use ESMF + use perf_mod, only : t_startf, t_adj_detailf, t_stopf + use ESMF, only : ESMF_Initialize, ESMF_Finalize + use seq_comm_mct, only : esmf_logfile_kind use cesm_comp_mod, only : cesm_pre_init1 use cesm_comp_mod, only : cesm_pre_init2 use cesm_comp_mod, only : cesm_init @@ -40,7 +41,7 @@ program cesm_driver ! because it is needed for the time manager, even if the ESMF_INTERFACE ! is not used. !-------------------------------------------------------------------------- - call ESMF_Initialize() + call ESMF_Initialize(logkindflag=esmf_logfile_kind) !-------------------------------------------------------------------------- ! Read in the configuration information and initialize the time manager. diff --git a/driver_cpl/driver/component_mod.F90 b/src/drivers/mct/main/component_mod.F90 similarity index 100% rename from driver_cpl/driver/component_mod.F90 rename to src/drivers/mct/main/component_mod.F90 diff --git a/driver_cpl/driver/component_type_mod.F90 b/src/drivers/mct/main/component_type_mod.F90 similarity index 100% rename from driver_cpl/driver/component_type_mod.F90 rename to src/drivers/mct/main/component_type_mod.F90 diff --git a/driver_cpl/driver/cplcomp_exchange_mod.F90 b/src/drivers/mct/main/cplcomp_exchange_mod.F90 similarity index 100% rename from driver_cpl/driver/cplcomp_exchange_mod.F90 rename to src/drivers/mct/main/cplcomp_exchange_mod.F90 diff --git a/driver_cpl/driver/map_glc2lnd_mod.F90 b/src/drivers/mct/main/map_glc2lnd_mod.F90 similarity index 97% rename from driver_cpl/driver/map_glc2lnd_mod.F90 rename to src/drivers/mct/main/map_glc2lnd_mod.F90 index 4a8ab0d0389..9bb67b1de57 100644 --- a/driver_cpl/driver/map_glc2lnd_mod.F90 +++ b/src/drivers/mct/main/map_glc2lnd_mod.F90 @@ -248,7 +248,7 @@ subroutine get_glc_elevation_classes(glc_topo, glc_elevclass) !----------------------------------------------------------------------- npts = size(glc_elevclass) - SHR_ASSERT((size(glc_topo) == npts), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL((size(glc_topo) == npts), __FILE__, __LINE__) do glc_pt = 1, npts call glc_get_elevation_class(glc_topo(glc_pt), glc_elevclass(glc_pt), err_code) @@ -291,8 +291,8 @@ subroutine get_frac_this_ec(glc_frac, glc_elevclass, this_elevclass, glc_frac_th !----------------------------------------------------------------------- npts = size(glc_frac_this_ec) - SHR_ASSERT((size(glc_frac) == npts), errMsg(__FILE__, __LINE__)) - SHR_ASSERT((size(glc_elevclass) == npts), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL((size(glc_frac) == npts), __FILE__, __LINE__) + SHR_ASSERT_FL((size(glc_elevclass) == npts), __FILE__, __LINE__) if (this_elevclass == 0) then glc_frac_this_ec(:) = 1._r8 - glc_frac(:) @@ -340,7 +340,7 @@ subroutine set_topo_in_virtual_columns(elev_class, glc_frac_this_ec_l, & ! Extract fields from attribute vectors lsize = mct_aVect_lsize(glc_frac_this_ec_l) - SHR_ASSERT(mct_aVect_lsize(glc_topo_this_ec_l) == lsize, errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL(mct_aVect_lsize(glc_topo_this_ec_l) == lsize, __FILE__, __LINE__) allocate(frac_l(lsize)) allocate(topo_l(lsize)) call mct_aVect_exportRattr(glc_frac_this_ec_l, frac_field, frac_l) @@ -388,7 +388,7 @@ subroutine make_aVect_frac_times_icemask(frac_av, mask_av, frac_field, icemask_f !----------------------------------------------------------------------- lsize = mct_aVect_lsize(frac_av) - SHR_ASSERT(mct_aVect_lsize(mask_av) == lsize, errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL(mct_aVect_lsize(mask_av) == lsize, __FILE__, __LINE__) call mct_aVect_init(frac_times_icemask_av, rList = frac_times_icemask_field, lsize = lsize) call mct_aVect_copy(aVin = frac_av, aVout = frac_times_icemask_av, & diff --git a/driver_cpl/driver/map_lnd2glc_mod.F90 b/src/drivers/mct/main/map_lnd2glc_mod.F90 similarity index 98% rename from driver_cpl/driver/map_lnd2glc_mod.F90 rename to src/drivers/mct/main/map_lnd2glc_mod.F90 index 3cc273c5976..da2ea515880 100644 --- a/driver_cpl/driver/map_lnd2glc_mod.F90 +++ b/src/drivers/mct/main/map_lnd2glc_mod.F90 @@ -224,8 +224,8 @@ subroutine get_glc_elevation_classes(glc_ice_covered, glc_topo, glc_elevclass) !----------------------------------------------------------------------- npts = size(glc_elevclass) - SHR_ASSERT((size(glc_ice_covered) == npts), errMsg(__FILE__, __LINE__)) - SHR_ASSERT((size(glc_topo) == npts), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL((size(glc_ice_covered) == npts), __FILE__, __LINE__) + SHR_ASSERT_FL((size(glc_topo) == npts), __FILE__, __LINE__) do glc_pt = 1, npts if (abs(glc_ice_covered(glc_pt) - 1._r8) < ice_covered_tol) then @@ -287,7 +287,7 @@ subroutine map_bare_land(l2x_l, landfrac_l, fieldname, mapper, data_g_bare_land) character(len=*), parameter :: subname = 'map_bare_land' !----------------------------------------------------------------------- - SHR_ASSERT(associated(data_g_bare_land), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL(associated(data_g_bare_land), __FILE__, __LINE__) lsize_g = size(data_g_bare_land) elevclass_as_string = glc_elevclass_as_string(0) @@ -379,7 +379,7 @@ subroutine map_one_elevation_class(l2x_l, landfrac_l, fieldname, elevclass, & !----------------------------------------------------------------------- lsize_g = size(data_g_thisEC) - SHR_ASSERT((size(topo_g) == lsize_g), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL((size(topo_g) == lsize_g), __FILE__, __LINE__) ! ------------------------------------------------------------------------ ! Create temporary attribute vectors diff --git a/driver_cpl/driver/map_lnd2rof_irrig_mod.F90 b/src/drivers/mct/main/map_lnd2rof_irrig_mod.F90 similarity index 99% rename from driver_cpl/driver/map_lnd2rof_irrig_mod.F90 rename to src/drivers/mct/main/map_lnd2rof_irrig_mod.F90 index eee6049c879..b7be99281fa 100644 --- a/driver_cpl/driver/map_lnd2rof_irrig_mod.F90 +++ b/src/drivers/mct/main/map_lnd2rof_irrig_mod.F90 @@ -258,8 +258,8 @@ subroutine map_rof2lnd_volr(volr_r, mapper_Fr2l, volr_l) character(len=*), parameter :: volr_field = 'volr' !--------------------------------------------------------------- - SHR_ASSERT(associated(volr_r), errMsg(sourcefile, __LINE__)) - SHR_ASSERT(associated(volr_l), errMsg(sourcefile, __LINE__)) + SHR_ASSERT_FL(associated(volr_r), sourcefile, __LINE__) + SHR_ASSERT_FL(associated(volr_l), sourcefile, __LINE__) lsize_r = size(volr_r) lsize_l = size(volr_l) diff --git a/driver_cpl/driver/mrg_mod.F90 b/src/drivers/mct/main/mrg_mod.F90 similarity index 100% rename from driver_cpl/driver/mrg_mod.F90 rename to src/drivers/mct/main/mrg_mod.F90 diff --git a/driver_cpl/driver/prep_aoflux_mod.F90 b/src/drivers/mct/main/prep_aoflux_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_aoflux_mod.F90 rename to src/drivers/mct/main/prep_aoflux_mod.F90 diff --git a/driver_cpl/driver/prep_atm_mod.F90 b/src/drivers/mct/main/prep_atm_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_atm_mod.F90 rename to src/drivers/mct/main/prep_atm_mod.F90 diff --git a/driver_cpl/driver/prep_glc_mod.F90 b/src/drivers/mct/main/prep_glc_mod.F90 similarity index 94% rename from driver_cpl/driver/prep_glc_mod.F90 rename to src/drivers/mct/main/prep_glc_mod.F90 index 0bf96b83b0a..52eeb02bfa1 100644 --- a/driver_cpl/driver/prep_glc_mod.F90 +++ b/src/drivers/mct/main/prep_glc_mod.F90 @@ -32,6 +32,8 @@ module prep_glc_mod public :: prep_glc_calc_l2x_gx + public :: prep_glc_zero_fields + public :: prep_glc_get_l2x_gx public :: prep_glc_get_l2gacc_lx public :: prep_glc_get_l2gacc_lx_cnt @@ -444,6 +446,31 @@ end subroutine prep_glc_map_one_field_lnd2glc !================================================================================================ + subroutine prep_glc_zero_fields() + + !--------------------------------------------------------------- + ! Description + ! Set glc inputs to zero + ! + ! This is appropriate during time intervals when we're not sending valid data to glc. + ! In principle we shouldn't need to zero the fields at these times (instead, glc + ! should just ignore the fields at these times). However, some tests (like an ERS or + ! ERI test that stops the final run segment mid-year) can fail if we don't explicitly + ! zero the fields, because these x2g fields can then differ upon restart. + + ! Local Variables + integer :: egi + type(mct_avect), pointer :: x2g_gx + !--------------------------------------------------------------- + + do egi = 1,num_inst_glc + x2g_gx => component_get_x2c_cx(glc(egi)) + call mct_aVect_zero(x2g_gx) + end do + end subroutine prep_glc_zero_fields + + !================================================================================================ + function prep_glc_get_l2x_gx() type(mct_aVect), pointer :: prep_glc_get_l2x_gx(:) prep_glc_get_l2x_gx => l2x_gx(:) diff --git a/driver_cpl/driver/prep_ice_mod.F90 b/src/drivers/mct/main/prep_ice_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_ice_mod.F90 rename to src/drivers/mct/main/prep_ice_mod.F90 diff --git a/driver_cpl/driver/prep_lnd_mod.F90 b/src/drivers/mct/main/prep_lnd_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_lnd_mod.F90 rename to src/drivers/mct/main/prep_lnd_mod.F90 diff --git a/driver_cpl/driver/prep_ocn_mod.F90 b/src/drivers/mct/main/prep_ocn_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_ocn_mod.F90 rename to src/drivers/mct/main/prep_ocn_mod.F90 diff --git a/driver_cpl/driver/prep_rof_mod.F90 b/src/drivers/mct/main/prep_rof_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_rof_mod.F90 rename to src/drivers/mct/main/prep_rof_mod.F90 diff --git a/driver_cpl/driver/prep_wav_mod.F90 b/src/drivers/mct/main/prep_wav_mod.F90 similarity index 100% rename from driver_cpl/driver/prep_wav_mod.F90 rename to src/drivers/mct/main/prep_wav_mod.F90 diff --git a/driver_cpl/driver/seq_diag_mct.F90 b/src/drivers/mct/main/seq_diag_mct.F90 similarity index 100% rename from driver_cpl/driver/seq_diag_mct.F90 rename to src/drivers/mct/main/seq_diag_mct.F90 diff --git a/driver_cpl/driver/seq_domain_mct.F90 b/src/drivers/mct/main/seq_domain_mct.F90 similarity index 100% rename from driver_cpl/driver/seq_domain_mct.F90 rename to src/drivers/mct/main/seq_domain_mct.F90 diff --git a/driver_cpl/driver/seq_flux_mct.F90 b/src/drivers/mct/main/seq_flux_mct.F90 similarity index 100% rename from driver_cpl/driver/seq_flux_mct.F90 rename to src/drivers/mct/main/seq_flux_mct.F90 diff --git a/driver_cpl/driver/seq_frac_mct.F90 b/src/drivers/mct/main/seq_frac_mct.F90 similarity index 100% rename from driver_cpl/driver/seq_frac_mct.F90 rename to src/drivers/mct/main/seq_frac_mct.F90 diff --git a/driver_cpl/driver/seq_hist_mod.F90 b/src/drivers/mct/main/seq_hist_mod.F90 similarity index 100% rename from driver_cpl/driver/seq_hist_mod.F90 rename to src/drivers/mct/main/seq_hist_mod.F90 diff --git a/driver_cpl/driver/seq_io_mod.F90 b/src/drivers/mct/main/seq_io_mod.F90 similarity index 99% rename from driver_cpl/driver/seq_io_mod.F90 rename to src/drivers/mct/main/seq_io_mod.F90 index 058e5160979..d27fc581c6e 100644 --- a/driver_cpl/driver/seq_io_mod.F90 +++ b/src/drivers/mct/main/seq_io_mod.F90 @@ -37,7 +37,8 @@ module seq_io_mod use shr_kind_mod, only: r4 => shr_kind_r4, r8 => shr_kind_r8, in => shr_kind_in use shr_kind_mod, only: cl => shr_kind_cl, cs => shr_kind_cs use shr_sys_mod ! system calls - use seq_comm_mct + use seq_comm_mct, only: logunit, CPLID, seq_comm_setptrs + use seq_comm_mct, only: seq_comm_namelen, seq_comm_name use seq_flds_mod, only : seq_flds_lookup use mct_mod ! mct wrappers use pio @@ -184,7 +185,9 @@ subroutine seq_io_wopen(filename,clobber,cdf64,file_ind) if (exists) then if (lclobber) then nmode = pio_clobber - if (lcdf64) nmode = ior(nmode,PIO_64BIT_OFFSET) + !lcdf64 only applies to classic NETCDF files. + if (lcdf64 .and. cpl_pio_iotype == PIO_IOTYPE_NETCDF) & + nmode = ior(nmode,PIO_64BIT_OFFSET) rcode = pio_createfile(cpl_io_subsystem, cpl_io_file(lfile_ind), cpl_pio_iotype, trim(filename), nmode) if(iam==0) write(logunit,*) subname,' create file ',trim(filename) rcode = pio_put_att(cpl_io_file(lfile_ind),pio_global,"file_version",version) @@ -203,7 +206,9 @@ subroutine seq_io_wopen(filename,clobber,cdf64,file_ind) endif else nmode = pio_noclobber - if (lcdf64) nmode = ior(nmode,PIO_64BIT_OFFSET) + !lcdf64 only applies to classic NETCDF files. + if (lcdf64 .and. cpl_pio_iotype == PIO_IOTYPE_NETCDF) & + nmode = ior(nmode,PIO_64BIT_OFFSET) rcode = pio_createfile(cpl_io_subsystem, cpl_io_file(lfile_ind), cpl_pio_iotype, trim(filename), nmode) if(iam==0) write(logunit,*) subname,' create file ',trim(filename) rcode = pio_put_att(cpl_io_file(lfile_ind),pio_global,"file_version",version) @@ -1541,7 +1546,7 @@ subroutine seq_io_read_av(filename,gsmap,AV,dname,pre) implicit none character(len=*),intent(in) :: filename ! file type(mct_gsMap), intent(in) :: gsmap - type(mct_aVect) ,intent(inout):: AV ! data to be written + type(mct_aVect) ,intent(inout):: AV ! data to be read character(len=*),intent(in) :: dname ! name of data character(len=*),intent(in),optional :: pre ! prefix name @@ -1667,7 +1672,7 @@ subroutine seq_io_read_avs(filename,gsmap,AVS,dname,pre) implicit none character(len=*),intent(in) :: filename ! file type(mct_gsMap), intent(in) :: gsmap - type(mct_aVect) ,intent(inout):: AVS(:) ! data to be written + type(mct_aVect) ,intent(inout):: AVS(:) ! data to be read character(len=*),intent(in) :: dname ! name of data character(len=*),intent(in),optional :: pre ! prefix name diff --git a/driver_cpl/driver/seq_map_mod.F90 b/src/drivers/mct/main/seq_map_mod.F90 similarity index 100% rename from driver_cpl/driver/seq_map_mod.F90 rename to src/drivers/mct/main/seq_map_mod.F90 diff --git a/driver_cpl/driver/seq_map_type_mod.F90 b/src/drivers/mct/main/seq_map_type_mod.F90 similarity index 100% rename from driver_cpl/driver/seq_map_type_mod.F90 rename to src/drivers/mct/main/seq_map_type_mod.F90 diff --git a/driver_cpl/driver/seq_rest_mod.F90 b/src/drivers/mct/main/seq_rest_mod.F90 similarity index 98% rename from driver_cpl/driver/seq_rest_mod.F90 rename to src/drivers/mct/main/seq_rest_mod.F90 index fa26e06d028..6994d954b81 100644 --- a/driver_cpl/driver/seq_rest_mod.F90 +++ b/src/drivers/mct/main/seq_rest_mod.F90 @@ -40,16 +40,17 @@ module seq_rest_mod seq_comm_iamin, CPLID, GLOID, logunit, loglevel ! "infodata" gathers various control flags into one datatype - use seq_infodata_mod + use seq_infodata_mod, only: seq_infodata_type, seq_infodata_getData ! clock & alarm routines - use seq_timemgr_mod + use seq_timemgr_mod, only: seq_timemgr_type, seq_timemgr_EClockGetData ! diagnostic routines use seq_diag_mct, only: budg_datag ! lower level io routines - use seq_io_mod + use seq_io_mod, only: seq_io_read, seq_io_write, seq_io_enddef + use seq_io_mod, only: seq_io_wopen, seq_io_close ! prep modules - coupler communication between different components use prep_ocn_mod, only: prep_ocn_get_x2oacc_ox @@ -412,7 +413,6 @@ subroutine seq_rest_write(EClock_d, seq_SyncClock, infodata, & if (fk == 1) then whead = .true. wdata = .false. -!! call seq_io_redef(rest_file) elseif (fk == 2) then whead = .false. wdata = .true. diff --git a/driver_cpl/driver/t_driver_timers_mod.F90 b/src/drivers/mct/main/t_driver_timers_mod.F90 similarity index 100% rename from driver_cpl/driver/t_driver_timers_mod.F90 rename to src/drivers/mct/main/t_driver_timers_mod.F90 diff --git a/driver_cpl/driver/vertical_gradient_calculator_2nd_order.F90 b/src/drivers/mct/main/vertical_gradient_calculator_2nd_order.F90 similarity index 96% rename from driver_cpl/driver/vertical_gradient_calculator_2nd_order.F90 rename to src/drivers/mct/main/vertical_gradient_calculator_2nd_order.F90 index b0a9cdfd5cb..e53a7894d1d 100644 --- a/driver_cpl/driver/vertical_gradient_calculator_2nd_order.F90 +++ b/src/drivers/mct/main/vertical_gradient_calculator_2nd_order.F90 @@ -100,8 +100,8 @@ function constructor(field, topo, elevclass_bounds) result(this) this%num_points = size(field, 1) this%nelev = size(field, 2) - SHR_ASSERT_ALL((ubound(topo) == (/this%num_points, this%nelev/)), errMsg(__FILE__, __LINE__)) - SHR_ASSERT_ALL((ubound(elevclass_bounds) == (/this%nelev/)), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_ALL_FL((ubound(topo) == (/this%num_points, this%nelev/)), __FILE__, __LINE__) + SHR_ASSERT_ALL_FL((ubound(elevclass_bounds) == (/this%nelev/)), __FILE__, __LINE__) allocate(this%elevclass_bounds(0:this%nelev)) this%elevclass_bounds(:) = elevclass_bounds(:) @@ -231,8 +231,8 @@ subroutine get_gradients_one_class(this, elevation_class, gradients) ! Assert pre-conditions - SHR_ASSERT(this%calculated, errMsg(__FILE__, __LINE__)) - SHR_ASSERT((size(gradients) == this%num_points), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL(this%calculated, __FILE__, __LINE__) + SHR_ASSERT_FL((size(gradients) == this%num_points), __FILE__, __LINE__) if (elevation_class < 1 .or. elevation_class > this%nelev) then write(logunit,*) subname, ': ERROR: elevation class out of bounds: ', & @@ -266,9 +266,9 @@ subroutine get_gradients_one_point(this, point, gradients) character(len=*), parameter :: subname = 'get_gradients_one_point' !----------------------------------------------------------------------- - SHR_ASSERT(this%calculated, errMsg(__FILE__, __LINE__)) - SHR_ASSERT(point <= this%num_points, errMsg(__FILE__, __LINE__)) - SHR_ASSERT((size(gradients) == this%nelev), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_FL(this%calculated, __FILE__, __LINE__) + SHR_ASSERT_FL(point <= this%num_points, __FILE__, __LINE__) + SHR_ASSERT_FL((size(gradients) == this%nelev), __FILE__, __LINE__) gradients(:) = this%vertical_gradient(point, :) diff --git a/driver_cpl/driver/vertical_gradient_calculator_base.F90 b/src/drivers/mct/main/vertical_gradient_calculator_base.F90 similarity index 100% rename from driver_cpl/driver/vertical_gradient_calculator_base.F90 rename to src/drivers/mct/main/vertical_gradient_calculator_base.F90 diff --git a/driver_cpl/driver/vertical_gradient_calculator_factory.F90 b/src/drivers/mct/main/vertical_gradient_calculator_factory.F90 similarity index 98% rename from driver_cpl/driver/vertical_gradient_calculator_factory.F90 rename to src/drivers/mct/main/vertical_gradient_calculator_factory.F90 index 22455ce66cc..c6501a87e53 100644 --- a/driver_cpl/driver/vertical_gradient_calculator_factory.F90 +++ b/src/drivers/mct/main/vertical_gradient_calculator_factory.F90 @@ -55,7 +55,7 @@ function create_vertical_gradient_calculator_2nd_order( & !----------------------------------------------------------------------- nelev = size(elevclass_names) - SHR_ASSERT_ALL((ubound(elevclass_bounds) == (/nelev/)), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_ALL_FL((ubound(elevclass_bounds) == (/nelev/)), __FILE__, __LINE__) call extract_data_from_attr_vect(attr_vect, fieldname, toponame, elevclass_names, & field, topo) diff --git a/driver_cpl/shr/CMakeLists.txt b/src/drivers/mct/shr/CMakeLists.txt similarity index 100% rename from driver_cpl/shr/CMakeLists.txt rename to src/drivers/mct/shr/CMakeLists.txt diff --git a/driver_cpl/shr/glc_elevclass_mod.F90 b/src/drivers/mct/shr/glc_elevclass_mod.F90 similarity index 99% rename from driver_cpl/shr/glc_elevclass_mod.F90 rename to src/drivers/mct/shr/glc_elevclass_mod.F90 index def51ebecf3..8ae9ac652a5 100644 --- a/driver_cpl/shr/glc_elevclass_mod.F90 +++ b/src/drivers/mct/shr/glc_elevclass_mod.F90 @@ -126,7 +126,7 @@ subroutine glc_elevclass_init_override(my_glc_nec, my_topomax) character(len=*), parameter :: subname = 'glc_elevclass_init_override' !----------------------------------------------------------------------- - SHR_ASSERT_ALL((ubound(my_topomax) == (/my_glc_nec/)), errMsg(__FILE__, __LINE__)) + SHR_ASSERT_ALL_FL((ubound(my_topomax) == (/my_glc_nec/)), __FILE__, __LINE__) glc_nec = my_glc_nec allocate(topomax(0:glc_nec)) diff --git a/driver_cpl/shr/seq_cdata_mod.F90 b/src/drivers/mct/shr/seq_cdata_mod.F90 similarity index 100% rename from driver_cpl/shr/seq_cdata_mod.F90 rename to src/drivers/mct/shr/seq_cdata_mod.F90 diff --git a/driver_cpl/shr/seq_comm_mct.F90 b/src/drivers/mct/shr/seq_comm_mct.F90 similarity index 96% rename from driver_cpl/shr/seq_comm_mct.F90 rename to src/drivers/mct/shr/seq_comm_mct.F90 index 44dbd98f2ed..6dc12259f16 100644 --- a/driver_cpl/shr/seq_comm_mct.F90 +++ b/src/drivers/mct/shr/seq_comm_mct.F90 @@ -20,11 +20,13 @@ module seq_comm_mct use shr_sys_mod , only : shr_sys_abort, shr_sys_flush use shr_mpi_mod , only : shr_mpi_chkerr, shr_mpi_bcast, shr_mpi_max use shr_file_mod, only : shr_file_getUnit, shr_file_freeUnit + use esmf , only : ESMF_LogKind_Flag, ESMF_LOGKIND_NONE + use esmf , only : ESMF_LOGKIND_SINGLE, ESMF_LOGKIND_MULTI implicit none private -#include +#include save !-------------------------------------------------------------------------- @@ -94,7 +96,6 @@ module seq_comm_mct !!! internal communications, and one is needed for the global space. !!! All instances of a component type also share a separate communicator !!! All instances of a component type share a communicator with the coupler - !!! Note that ESP models do not need coupler communicators integer, parameter, public :: num_inst_phys = num_inst_atm + num_inst_lnd + & num_inst_ocn + num_inst_ice + & @@ -103,7 +104,7 @@ module seq_comm_mct integer, parameter, public :: num_cpl_phys = num_inst_atm + num_inst_lnd + & num_inst_ocn + num_inst_ice + & num_inst_glc + num_inst_rof + & - num_inst_wav + num_inst_wav + num_inst_esp integer, parameter :: ncomps = (1 + ncouplers + ncomptypes + nphysmod + num_inst_phys + num_cpl_phys) integer, public :: GLOID @@ -125,7 +126,7 @@ module seq_comm_mct integer, public :: CPLALLGLCID integer, public :: CPLALLROFID integer, public :: CPLALLWAVID - integer, public, parameter :: CPLALLESPID = -1 + integer, public :: CPLALLESPID integer, public :: ATMID(num_inst_atm) integer, public :: LNDID(num_inst_lnd) @@ -145,7 +146,10 @@ module seq_comm_mct integer, public :: CPLWAVID(num_inst_wav) integer, public :: CPLESPID(num_inst_esp) + type(ESMF_LogKind_Flag), public :: esmf_logfile_kind + integer, parameter, public :: seq_comm_namelen=16 + type seq_comm_type character(len=seq_comm_namelen) :: name ! my name character(len=seq_comm_namelen) :: suffix ! recommended suffix @@ -161,7 +165,9 @@ module seq_comm_mct integer :: gloroot ! the global task number of each comps root on all pes integer :: pethreads ! max number of threads on my task integer :: cplpe ! a common task in mpicom from the cpl group for join mpicoms + ! cplpe is used to broadcast information from the coupler to the component integer :: cmppe ! a common task in mpicom from the component group for join mpicoms + ! cmppe is used to broadcast information from the component to the coupler logical :: set ! has this datatype been set integer, pointer :: petlist(:) ! esmf pet list logical :: petlist_allocated ! whether the petlist pointer variable was allocated @@ -182,19 +188,19 @@ module seq_comm_mct character(len=32), public :: & atm_layout, lnd_layout, ice_layout, glc_layout, rof_layout, & ocn_layout, wav_layout, esp_layout - + logical :: seq_comm_mct_initialized = .false. ! whether this module has been initialized !======================================================================= contains !====================================================================== - integer function seq_comm_get_ncomps() + integer function seq_comm_get_ncomps() seq_comm_get_ncomps = ncomps end function seq_comm_get_ncomps - + subroutine seq_comm_init(Comm_in, nmlfile) - + !---------------------------------------------------------- ! ! Arguments @@ -236,6 +242,8 @@ subroutine seq_comm_init(Comm_in, nmlfile) ocn_ntasks, ocn_rootpe, ocn_pestride, ocn_nthreads, & esp_ntasks, esp_rootpe, esp_pestride, esp_nthreads, & cpl_ntasks, cpl_rootpe, cpl_pestride, cpl_nthreads + character(len=24) :: esmf_logging + namelist /ccsm_pes/ & atm_ntasks, atm_rootpe, atm_pestride, atm_nthreads, atm_layout, & lnd_ntasks, lnd_rootpe, lnd_pestride, lnd_nthreads, lnd_layout, & @@ -245,7 +253,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) rof_ntasks, rof_rootpe, rof_pestride, rof_nthreads, rof_layout, & ocn_ntasks, ocn_rootpe, ocn_pestride, ocn_nthreads, ocn_layout, & esp_ntasks, esp_rootpe, esp_pestride, esp_nthreads, esp_layout, & - cpl_ntasks, cpl_rootpe, cpl_pestride, cpl_nthreads + cpl_ntasks, cpl_rootpe, cpl_pestride, cpl_nthreads, esmf_logging !---------------------------------------------------------- ! make sure this is first pass and set comms unset @@ -264,7 +272,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) seq_comms(n)%inst = 0 seq_comms(n)%set = .false. seq_comms(n)%petlist_allocated = .false. - seq_comms(n)%mpicom = MPI_COMM_NULL ! do some initialization here + seq_comms(n)%mpicom = MPI_COMM_NULL ! do some initialization here seq_comms(n)%iam = -1 seq_comms(n)%iamroot = .false. seq_comms(n)%npes = -1 @@ -288,7 +296,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) ! Initialize gloiam on all IDs global_mype = mype - + do n = 1,ncomps seq_comms(n)%gloiam = mype enddo @@ -352,6 +360,8 @@ subroutine seq_comm_init(Comm_in, nmlfile) cpl_pestride = 1 cpl_nthreads = 1 + esmf_logging = "ESMF_LOGKIND_NONE" + ! Read namelist if it exists nu = shr_file_getUnit() @@ -369,7 +379,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end if !--- compute some other num_inst values - + num_inst_xao = max(num_inst_atm,num_inst_ocn) num_inst_frc = num_inst_ice @@ -385,7 +395,8 @@ subroutine seq_comm_init(Comm_in, nmlfile) num_inst_min = min(num_inst_min, num_inst_glc) num_inst_min = min(num_inst_min, num_inst_wav) num_inst_min = min(num_inst_min, num_inst_rof) - num_inst_min = min(num_inst_min, num_inst_esp) +! ESP is currently limited to one instance, should not affect other comps +! num_inst_min = min(num_inst_min, num_inst_esp) num_inst_max = num_inst_atm num_inst_max = max(num_inst_max, num_inst_lnd) num_inst_max = max(num_inst_max, num_inst_ocn) @@ -403,7 +414,10 @@ subroutine seq_comm_init(Comm_in, nmlfile) if (num_inst_glc /= num_inst_min .and. num_inst_glc /= num_inst_max) error_state = .true. if (num_inst_wav /= num_inst_min .and. num_inst_wav /= num_inst_max) error_state = .true. if (num_inst_rof /= num_inst_min .and. num_inst_rof /= num_inst_max) error_state = .true. - if (num_inst_esp /= num_inst_min .and. num_inst_esp /= num_inst_max) error_state = .true. + if (num_inst_esp /= 1) then + write(logunit,*) trim(subname),' ERROR: ESP restricted to one instance' + error_state = .true. + end if if (error_state) then write(logunit,*) trim(subname),' ERROR: num_inst inconsistent' @@ -450,6 +464,8 @@ subroutine seq_comm_init(Comm_in, nmlfile) CPLALLROFID = count count = count + 1 CPLALLWAVID = count + count = count + 1 + CPLALLESPID = count do n = 1, num_inst_atm count = count + 1 @@ -457,21 +473,21 @@ subroutine seq_comm_init(Comm_in, nmlfile) count = count + 1 CPLATMID(n) = count end do - + do n = 1, num_inst_lnd count = count + 1 LNDID(n) = count count = count + 1 CPLLNDID(n) = count end do - + do n = 1, num_inst_ocn count = count + 1 OCNID(n) = count count = count + 1 CPLOCNID(n) = count end do - + do n = 1, num_inst_ice count = count + 1 ICEID(n) = count @@ -526,7 +542,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) if (rof_rootpe < 0) error_state = .true. if (esp_rootpe < 0) error_state = .true. if (cpl_rootpe < 0) error_state = .true. - + if (error_state) then write(logunit,*) trim(subname),' ERROR: rootpes must be >= 0' call shr_sys_abort(trim(subname)//' ERROR: rootpes >= 0') @@ -577,7 +593,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! Ocean instance tasks - + if (trim(ocn_layout) == trim(layout_concurrent)) then ocn_inst_tasks = ocn_ntasks / num_inst_ocn droot = (ocn_inst_tasks * ocn_pestride) @@ -597,7 +613,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! Sea ice instance tasks - + if (trim(ice_layout) == trim(layout_concurrent)) then ice_inst_tasks = ice_ntasks / num_inst_ice droot = (ice_inst_tasks * ice_pestride) @@ -617,7 +633,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! Glacier instance tasks - + if (trim(glc_layout) == trim(layout_concurrent)) then glc_inst_tasks = glc_ntasks / num_inst_glc droot = (glc_inst_tasks * glc_pestride) @@ -637,7 +653,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! Runoff instance tasks - + if (trim(rof_layout) == trim(layout_concurrent)) then rof_inst_tasks = rof_ntasks / num_inst_rof droot = (rof_inst_tasks * rof_pestride) @@ -657,7 +673,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! Wave instance tasks - + if (trim(wav_layout) == trim(layout_concurrent)) then wav_inst_tasks = wav_ntasks / num_inst_wav droot = (wav_inst_tasks * wav_pestride) @@ -677,7 +693,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) end do !! External System Processing instance tasks - + if (trim(esp_layout) == trim(layout_concurrent)) then esp_inst_tasks = esp_ntasks / num_inst_esp droot = (esp_inst_tasks * esp_pestride) @@ -832,6 +848,7 @@ subroutine seq_comm_init(Comm_in, nmlfile) call seq_comm_setcomm(ESPID(n), pelist, esp_nthreads, 'ESP', n, num_inst_esp) end do call seq_comm_jcommarr(ESPID,ALLESPID,'ALLESPID',1,1) + call seq_comm_joincomm(CPLID,ALLESPID,CPLALLESPID,'CPLALLESPID',1,1) !! Count the total number of threads @@ -890,6 +907,23 @@ subroutine seq_comm_init(Comm_in, nmlfile) deallocate(comps,comms) + ! ESMF logging (only has effect if ESMF libraries are used) + call mpi_bcast(esmf_logging, len(esmf_logging), MPI_CHARACTER, 0, GLOBAL_COMM, ierr) + + select case(esmf_logging) + case ("ESMF_LOGKIND_SINGLE") + esmf_logfile_kind = ESMF_LOGKIND_SINGLE + case ("ESMF_LOGKIND_MULTI") + esmf_logfile_kind = ESMF_LOGKIND_MULTI + case ("ESMF_LOGKIND_NONE") + esmf_logfile_kind = ESMF_LOGKIND_NONE + case default + if (mype == 0) then + write(logunit,*) trim(subname),' ERROR: Invalid value for esmf_logging, ',esmf_logging + endif + call shr_sys_abort(trim(subname)//' ERROR: Invalid value for esmf_logging '//esmf_logging) + end select + call seq_comm_printcomms() end subroutine seq_comm_init @@ -904,10 +938,10 @@ subroutine seq_comm_clean() ! seq_comm_init. integer :: id - + character(*), parameter :: subName = '(seq_comm_clean) ' !---------------------------------------------------------- - + if (.not. seq_comm_mct_initialized) then write(logunit,*) trim(subname),' ERROR seq_comm_init has not been called ' call shr_sys_abort() @@ -919,11 +953,11 @@ subroutine seq_comm_clean() deallocate(seq_comms(id)%petlist) end if end do - + call mct_world_clean() end subroutine seq_comm_clean - + !--------------------------------------------------------- subroutine seq_comm_setcomm(ID,pelist,nthreads,iname,inst,tinst) @@ -947,7 +981,7 @@ subroutine seq_comm_setcomm(ID,pelist,nthreads,iname,inst,tinst) if (ID < 1 .or. ID > ncomps) then write(logunit,*) subname,' ID out of range, abort ',ID call shr_sys_abort() - endif + endif call mpi_comm_group(GLOBAL_COMM, mpigrp_world, ierr) call shr_mpi_chkerr(ierr,subname//' mpi_comm_group mpigrp_world') @@ -1057,11 +1091,11 @@ subroutine seq_comm_joincomm(ID1,ID2,ID,iname,inst,tinst) if (ID1 < 1 .or. ID1 > ncomps) then write(logunit,*) subname,' ID1 out of range, abort ',ID1 call shr_sys_abort() - endif + endif if (ID2 < 1 .or. ID2 > ncomps) then write(logunit,*) subname,' ID2 out of range, abort ',ID2 call shr_sys_abort() - endif + endif if (ID < 1 .or. ID > ncomps) then write(logunit,*) subname,' ID out of range, abort ',ID call shr_sys_abort() @@ -1130,36 +1164,6 @@ subroutine seq_comm_joincomm(ID1,ID2,ID,iname,inst,tinst) seq_comms(ID)%iamroot = .false. endif -! needs to be excluded until mpi_group_size is added to serial mpi in mct -#if (1 == 0) - if (loglevel > 3) then - ! some debug code to prove the join is working ok - ! when joining mpicomms, the local rank may be quite different - ! from either the global or local ranks of the joining comms - call mpi_group_size(seq_comms(ID1)%mpigrp,nsize,ierr) - allocate(pe_t1(nsize),pe_t2(nsize)) - do n = 1,nsize - pe_t1(n) = n-1 - pe_t2(n) = -1 - enddo - call mpi_group_translate_ranks(seq_comms(ID1)%mpigrp, nsize, pe_t1, mpigrp, pe_t2, ierr) - write(logunit,*) 'ID1 ranks ',pe_t1 - write(logunit,*) 'ID1-JOIN ranks ',pe_t2 - deallocate(pe_t1,pe_t2) - - call mpi_group_size(seq_comms(ID2)%mpigrp,nsize,ierr) - allocate(pe_t1(nsize),pe_t2(nsize)) - do n = 1,nsize - pe_t1(n) = n-1 - pe_t2(n) = -1 - enddo - call mpi_group_translate_ranks(seq_comms(ID2)%mpigrp, nsize, pe_t1, mpigrp, pe_t2, ierr) - write(logunit,*) 'ID2 ranks ',pe_t1 - write(logunit,*) 'ID2-JOIN ranks ',pe_t2 - deallocate(pe_t1,pe_t2) - endif -#endif - allocate(pe_t1(1),pe_t2(1)) pe_t1(1) = 0 call mpi_group_translate_ranks(seq_comms(ID1)%mpigrp, 1, pe_t1, mpigrp, pe_t2, ierr) @@ -1211,7 +1215,7 @@ subroutine seq_comm_jcommarr(IDs,ID,iname,inst,tinst) if (IDs(n) < 1 .or. IDs(n) > ncomps) then write(logunit,*) subname,' IDs out of range, abort ',n,IDs(n) call shr_sys_abort() - endif + endif if (.not. seq_comms(IDs(n))%set) then write(logunit,*) subname,' IDs not set ',n,IDs(n) call shr_sys_abort() @@ -1267,7 +1271,7 @@ subroutine seq_comm_jcommarr(IDs,ID,iname,inst,tinst) seq_comms(ID)%mpicom = mpicom seq_comms(ID)%mpigrp = mpigrp - + seq_comms(ID)%nthreads = 1 do n = 1,nids seq_comms(ID)%nthreads = max(seq_comms(ID)%nthreads,seq_comms(IDs(n))%nthreads) @@ -1355,7 +1359,7 @@ subroutine seq_comm_setptrs(ID,mpicom,mpigrp,npes,nthreads,iam,iamroot,gloiam,gl if ((ID == 0) .or. (ID > ncomps)) then write(logunit,*) subname,' ID out of range, return ',ID return - endif + endif if (present(mpicom)) then if (ID > 0) then diff --git a/driver_cpl/shr/seq_drydep_mod.F90 b/src/drivers/mct/shr/seq_drydep_mod.F90 similarity index 100% rename from driver_cpl/shr/seq_drydep_mod.F90 rename to src/drivers/mct/shr/seq_drydep_mod.F90 diff --git a/driver_cpl/shr/seq_flds_mod.F90 b/src/drivers/mct/shr/seq_flds_mod.F90 similarity index 98% rename from driver_cpl/shr/seq_flds_mod.F90 rename to src/drivers/mct/shr/seq_flds_mod.F90 index d00f84f87b1..4c9de8fd468 100644 --- a/driver_cpl/shr/seq_flds_mod.F90 +++ b/src/drivers/mct/shr/seq_flds_mod.F90 @@ -121,13 +121,14 @@ module seq_flds_mod ! variables CCSM_VOC, CCSM_BGC and GLC_NEC. !==================================================================== - use shr_kind_mod, only : CX => shr_kind_CX, CXX => shr_kind_CXX - use shr_sys_mod, only : shr_sys_abort - use seq_drydep_mod, only : seq_drydep_init, seq_drydep_readnl, lnd_drydep - use seq_comm_mct, only : seq_comm_iamroot, seq_comm_setptrs, logunit - use shr_megan_mod, only : shr_megan_readnl, shr_megan_mechcomps_n - use shr_fire_emis_mod, only : shr_fire_emis_readnl, shr_fire_emis_mechcomps_n, shr_fire_emis_ztop_token - use shr_carma_mod, only : shr_carma_readnl + use shr_kind_mod , only : CX => shr_kind_CX, CXX => shr_kind_CXX + use shr_sys_mod , only : shr_sys_abort + use seq_comm_mct , only : seq_comm_iamroot, seq_comm_setptrs, logunit + use seq_drydep_mod , only : seq_drydep_init, seq_drydep_readnl, lnd_drydep + use shr_megan_mod , only : shr_megan_readnl, shr_megan_mechcomps_n + use shr_fire_emis_mod , only : shr_fire_emis_readnl, shr_fire_emis_mechcomps_n, shr_fire_emis_ztop_token + use shr_carma_mod , only : shr_carma_readnl + use shr_ndep_mod , only : shr_ndep_readnl implicit none public @@ -143,8 +144,10 @@ module seq_flds_mod character(len=CXX) :: megan_voc_fields ! List of MEGAN VOC emission fields character(len=CXX) :: fire_emis_fields ! List of fire emission fields character(len=CX) :: carma_fields ! List of CARMA fields from lnd->atm + character(len=CX) :: ndep_fields ! List of nitrogen deposition fields from atm->lnd/ocn integer :: ice_ncat ! number of sea ice thickness categories logical :: seq_flds_i2o_per_cat! .true. if select per ice thickness category fields are passed from ice to ocean + logical :: add_ndep_fields ! .true. => add ndep fields !---------------------------------------------------------------------------- ! metadata @@ -3126,7 +3129,6 @@ subroutine seq_flds_set(nmlfile, ID, infodata) units = 'm' call metadata_set(shr_fire_emis_ztop_token, longname, stdname, units) - endif !----------------------------------------------------------------------------- @@ -3146,11 +3148,31 @@ subroutine seq_flds_set(nmlfile, ID, infodata) longname = 'dry deposition velocity' stdname = 'drydep_vel' units = 'cm/sec' - call metadata_set(seq_drydep_fields, longname, stdname, units) + call metadata_set(seq_drydep_fields, longname, stdname, units) endif call seq_drydep_init( ) + !----------------------------------------------------------------------------- + ! Nitrogen Deposition fields + ! First read namelist and figure out the ndepdep field list to pass + ! Then check if file exists and if not, n_drydep will be zero + ! Then add nitrogen deposition fields to atm export, lnd import and ocn import + !----------------------------------------------------------------------------- + + call shr_ndep_readnl(nlfilename="drv_flds_in", ID=ID, ndep_fields=ndep_fields, add_ndep_fields=add_ndep_fields) + if (add_ndep_fields) then + call seq_flds_add(a2x_fluxes, ndep_fields) + call seq_flds_add(x2l_fluxes, ndep_fields) + call seq_flds_add(x2o_fluxes, ndep_fields) + + longname = 'nitrogen deposition flux' + stdname = 'nitrogen_deposition' + units = 'kg(N)/m2/sec' + + call metadata_set(ndep_fields, longname, stdname, units) + end if + !---------------------------------------------------------------------------- ! state + flux fields !---------------------------------------------------------------------------- diff --git a/driver_cpl/shr/seq_infodata_mod.F90 b/src/drivers/mct/shr/seq_infodata_mod.F90 similarity index 89% rename from driver_cpl/shr/seq_infodata_mod.F90 rename to src/drivers/mct/shr/seq_infodata_mod.F90 index 7341e4c2af2..d62014fa87f 100644 --- a/driver_cpl/shr/seq_infodata_mod.F90 +++ b/src/drivers/mct/shr/seq_infodata_mod.F90 @@ -46,6 +46,7 @@ MODULE seq_infodata_mod ! !PUBLIC MEMBER FUNCTIONS public :: seq_infodata_Init ! Initialize + public :: seq_infodata_Init2 ! Init after clocks initialized public :: seq_infodata_GetData ! Get values from object public :: seq_infodata_PutData ! Change values public :: seq_infodata_Print ! print current info @@ -66,14 +67,6 @@ MODULE seq_infodata_mod ! Type to hold pause/resume signaling information type seq_pause_resume_type private - logical :: atm_pause = .false. ! atm write pause restart file - logical :: lnd_pause = .false. ! lnd write pause restart file - logical :: ice_pause = .false. ! ice write pause restart file - logical :: ocn_pause = .false. ! ocn write pause restart file - logical :: glc_pause = .false. ! glc write pause restart file - logical :: rof_pause = .false. ! rof write pause restart file - logical :: wav_pause = .false. ! wav write pause restart file - logical :: cpl_pause = .false. ! cpl write pause restart file character(SHR_KIND_CL) :: atm_resume(num_inst_atm) = ' ' ! atm resume file character(SHR_KIND_CL) :: lnd_resume(num_inst_lnd) = ' ' ! lnd resume file character(SHR_KIND_CL) :: ice_resume(num_inst_ice) = ' ' ! ice resume file @@ -244,12 +237,13 @@ MODULE seq_infodata_mod integer(SHR_KIND_IN) :: wav_phase ! wav phase integer(SHR_KIND_IN) :: esp_phase ! esp phase logical :: atm_aero ! atmosphere aerosols - logical :: glcrun_alarm ! glc run alarm logical :: glc_g2lupdate ! update glc2lnd fields in lnd model type(seq_pause_resume_type), pointer :: pause_resume => NULL() real(shr_kind_r8) :: max_cplstep_time ! abort if cplstep time exceeds this value !--- set from restart file --- character(SHR_KIND_CL) :: rest_case_name ! Short case identification + !--- set by driver and may be time varying + logical :: glc_valid_input ! is valid accumulated data being sent to prognostic glc end type seq_infodata_type ! --- public interfaces -------------------------------------------------------- @@ -293,12 +287,12 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid) ! !USES: - use shr_file_mod, only : shr_file_getUnit, shr_file_freeUnit - use shr_string_mod, only : shr_string_toUpper, shr_string_listAppend - use shr_mpi_mod, only : shr_mpi_bcast - use seq_io_read_mod - use pio, only : file_desc_t - implicit none + use shr_file_mod, only : shr_file_getUnit, shr_file_freeUnit + use shr_string_mod, only : shr_string_toUpper, shr_string_listAppend + use shr_mpi_mod, only : shr_mpi_bcast + use seq_timemgr_mod, only : seq_timemgr_pause_active + use seq_io_read_mod, only : seq_io_read + use pio, only : file_desc_t ! !INPUT/OUTPUT PARAMETERS: @@ -555,6 +549,7 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid) mct_usealltoall = .false. mct_usevector = .false. max_cplstep_time = 0.0 + !--------------------------------------------------------------------------- ! Read in namelist !--------------------------------------------------------------------------- @@ -715,15 +710,14 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid) infodata%rof_phase = 1 infodata%wav_phase = 1 infodata%atm_aero = .false. - infodata%glcrun_alarm = .false. infodata%glc_g2lupdate = .false. + infodata%glc_valid_input = .true. if (associated(infodata%pause_resume)) then deallocate(infodata%pause_resume) end if nullify(infodata%pause_resume) infodata%max_cplstep_time = max_cplstep_time - !--------------------------------------------------------------- ! check orbital mode, reset unused parameters, validate settings !--------------------------------------------------------------- @@ -858,6 +852,43 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid) END SUBROUTINE seq_infodata_Init +!=============================================================================== +!BOP =========================================================================== +! +! !IROUTINE: seq_infodata_Init2 -- initialize infodata structures +! +! !DESCRIPTION: +! +! Initialize infodata items that depend on the time manager setup +! +! !INTERFACE: ------------------------------------------------------------------ + +SUBROUTINE seq_infodata_Init2(infodata, ID) + +! !USES: + + use seq_timemgr_mod, only : seq_timemgr_pause_active + +! !INPUT/OUTPUT PARAMETERS: + + type(seq_infodata_type), intent(INOUT) :: infodata ! infodata object + integer(SHR_KIND_IN), intent(IN) :: ID ! seq_comm ID +!EOP + + !----- local ----- + integer :: mpicom ! MPI communicator + + call seq_comm_setptrs(ID, mpicom=mpicom) + !---------------------------------------------------------- + !| If pause/resume is active, initialize the resume data + !---------------------------------------------------------- + if (seq_timemgr_pause_active() .and. (.not. associated(infodata%pause_resume))) then + allocate(infodata%pause_resume) + end if + call seq_infodata_bcast(infodata, mpicom) + +END SUBROUTINE seq_infodata_Init2 + !=============================================================================== !=============================================================================== ! !IROUTINE: seq_infodata_GetData_explicit -- Get values from infodata object @@ -883,7 +914,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ ice_gnam, rof_gnam, glc_gnam, wav_gnam, & atm_gnam, ocn_gnam, info_debug, dead_comps, read_restart, & shr_map_dopole, vect_map, aoflux_grid, flux_epbalfact, & - nextsw_cday, precip_fact, flux_epbal, flux_albav, glcrun_alarm, & + nextsw_cday, precip_fact, flux_epbal, flux_albav, & glc_g2lupdate, atm_aero, run_barriers, esmf_map_flag, & do_budgets, do_histinit, drv_threading, flux_diurnal, gust_fac, & budget_inst, budget_daily, budget_month, wall_time_limit, & @@ -896,17 +927,16 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ cpl_cdf64, orb_iyear, orb_iyear_align, orb_mode, orb_mvelp, & orb_eccen, orb_obliqr, orb_lambm0, orb_mvelpp, wv_sat_scheme, & wv_sat_transition_start, wv_sat_use_tables, wv_sat_table_spacing, & - tfreeze_option, & + tfreeze_option, & glc_phase, rof_phase, atm_phase, lnd_phase, ocn_phase, ice_phase, & wav_phase, esp_phase, wav_nx, wav_ny, atm_nx, atm_ny, & lnd_nx, lnd_ny, rof_nx, rof_ny, ice_nx, ice_ny, ocn_nx, ocn_ny, & glc_nx, glc_ny, eps_frac, eps_amask, & eps_agrid, eps_aarea, eps_omask, eps_ogrid, eps_oarea, & reprosum_use_ddpdd, reprosum_diffmax, reprosum_recompute, & - atm_pause, lnd_pause, ocn_pause, ice_pause, glc_pause, rof_pause, & - wav_pause, cpl_pause, atm_resume, lnd_resume, ocn_resume, & - ice_resume, glc_resume, rof_resume, wav_resume, cpl_resume, & - mct_usealltoall, mct_usevector, max_cplstep_time) + atm_resume, lnd_resume, ocn_resume, ice_resume, & + glc_resume, rof_resume, wav_resume, cpl_resume, & + mct_usealltoall, mct_usevector, max_cplstep_time, glc_valid_input) implicit none @@ -1065,17 +1095,9 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ integer(SHR_KIND_IN), optional, intent(OUT) :: wav_phase ! wav phase integer(SHR_KIND_IN), optional, intent(OUT) :: esp_phase ! wav phase logical, optional, intent(OUT) :: atm_aero ! atmosphere aerosols - logical, optional, intent(OUT) :: glcrun_alarm ! glc run alarm logical, optional, intent(OUT) :: glc_g2lupdate ! update glc2lnd fields in lnd model real(shr_kind_r8), optional, intent(out) :: max_cplstep_time - logical, optional, intent(OUT) :: atm_pause ! atm write pause restart file - logical, optional, intent(OUT) :: lnd_pause ! lnd write pause restart file - logical, optional, intent(OUT) :: ice_pause ! ice write pause restart file - logical, optional, intent(OUT) :: ocn_pause ! ocn write pause restart file - logical, optional, intent(OUT) :: glc_pause ! glc write pause restart file - logical, optional, intent(OUT) :: rof_pause ! rof write pause restart file - logical, optional, intent(OUT) :: wav_pause ! wav write pause restart file - logical, optional, intent(OUT) :: cpl_pause ! cpl write pause restart file + logical, optional, intent(OUT) :: glc_valid_input character(SHR_KIND_CL), optional, intent(OUT) :: atm_resume(:) ! atm read resume state character(SHR_KIND_CL), optional, intent(OUT) :: lnd_resume(:) ! lnd read resume state character(SHR_KIND_CL), optional, intent(OUT) :: ice_resume(:) ! ice read resume state @@ -1255,64 +1277,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ if ( present(wav_phase) ) wav_phase = infodata%wav_phase if ( present(esp_phase) ) esp_phase = infodata%esp_phase if ( present(atm_aero) ) atm_aero = infodata%atm_aero - if ( present(glcrun_alarm) ) glcrun_alarm = infodata%glcrun_alarm if ( present(glc_g2lupdate) ) glc_g2lupdate = infodata%glc_g2lupdate - if ( present(atm_pause) ) then - if (associated(infodata%pause_resume)) then - atm_pause = infodata%pause_resume%atm_pause - else - atm_pause = .false. - end if - end if - if ( present(lnd_pause) ) then - if (associated(infodata%pause_resume)) then - lnd_pause = infodata%pause_resume%lnd_pause - else - lnd_pause = .false. - end if - end if - if ( present(ice_pause) ) then - if (associated(infodata%pause_resume)) then - ice_pause = infodata%pause_resume%ice_pause - else - ice_pause = .false. - end if - end if - if ( present(ocn_pause) ) then - if (associated(infodata%pause_resume)) then - ocn_pause = infodata%pause_resume%ocn_pause - else - ocn_pause = .false. - end if - end if - if ( present(glc_pause) ) then - if (associated(infodata%pause_resume)) then - glc_pause = infodata%pause_resume%glc_pause - else - glc_pause = .false. - end if - end if - if ( present(rof_pause) ) then - if (associated(infodata%pause_resume)) then - rof_pause = infodata%pause_resume%rof_pause - else - rof_pause = .false. - end if - end if - if ( present(wav_pause) ) then - if (associated(infodata%pause_resume)) then - wav_pause = infodata%pause_resume%wav_pause - else - wav_pause = .false. - end if - end if - if ( present(cpl_pause) ) then - if (associated(infodata%pause_resume)) then - cpl_pause = infodata%pause_resume%cpl_pause - else - cpl_pause = .false. - end if - end if if ( present(atm_resume) ) then if (associated(infodata%pause_resume)) then atm_resume(:) = infodata%pause_resume%atm_resume(:) @@ -1370,6 +1335,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_ end if end if if ( present(max_cplstep_time) ) max_cplstep_time = infodata%max_cplstep_time + if ( present(glc_valid_input)) glc_valid_input = infodata%glc_valid_input END SUBROUTINE seq_infodata_GetData_explicit @@ -1385,7 +1351,7 @@ END SUBROUTINE seq_infodata_GetData_explicit SUBROUTINE seq_infodata_GetData_bytype( component_firstletter, infodata, & comp_present, comp_prognostic, comp_gnam, histavg_comp, & - comp_phase, comp_nx, comp_ny, comp_pause, comp_resume) + comp_phase, comp_nx, comp_ny, comp_resume) implicit none @@ -1401,7 +1367,6 @@ SUBROUTINE seq_infodata_GetData_bytype( component_firstletter, infodata, & integer(SHR_KIND_IN), optional, intent(OUT) :: comp_ny ! nx,ny 2d grid size global integer(SHR_KIND_IN), optional, intent(OUT) :: comp_phase logical, optional, intent(OUT) :: histavg_comp - logical, optional, intent(OUT) :: comp_pause character(SHR_KIND_CL), optional, intent(OUT) :: comp_resume(:) !----- local ----- @@ -1413,37 +1378,37 @@ SUBROUTINE seq_infodata_GetData_bytype( component_firstletter, infodata, & call seq_infodata_GetData(infodata, atm_present=comp_present, & atm_prognostic=comp_prognostic, atm_gnam=comp_gnam, & atm_phase=comp_phase, atm_nx=comp_nx, atm_ny=comp_ny, & - histavg_atm=histavg_comp, atm_pause=comp_pause, atm_resume=comp_resume) + histavg_atm=histavg_comp, atm_resume=comp_resume) else if (component_firstletter == 'l') then call seq_infodata_GetData(infodata, lnd_present=comp_present, & lnd_prognostic=comp_prognostic, lnd_gnam=comp_gnam, & lnd_phase=comp_phase, lnd_nx=comp_nx, lnd_ny=comp_ny, & - histavg_lnd=histavg_comp, lnd_pause=comp_pause, lnd_resume=comp_resume) + histavg_lnd=histavg_comp, lnd_resume=comp_resume) else if (component_firstletter == 'i') then call seq_infodata_GetData(infodata, ice_present=comp_present, & ice_prognostic=comp_prognostic, ice_gnam=comp_gnam, & ice_phase=comp_phase, ice_nx=comp_nx, ice_ny=comp_ny, & - histavg_ice=histavg_comp, ice_pause=comp_pause, ice_resume=comp_resume) + histavg_ice=histavg_comp, ice_resume=comp_resume) else if (component_firstletter == 'o') then call seq_infodata_GetData(infodata, ocn_present=comp_present, & ocn_prognostic=comp_prognostic, ocn_gnam=comp_gnam, & ocn_phase=comp_phase, ocn_nx=comp_nx, ocn_ny=comp_ny, & - histavg_ocn=histavg_comp, ocn_pause=comp_pause, ocn_resume=comp_resume) + histavg_ocn=histavg_comp, ocn_resume=comp_resume) else if (component_firstletter == 'r') then call seq_infodata_GetData(infodata, rof_present=comp_present, & rof_prognostic=comp_prognostic, rof_gnam=comp_gnam, & rof_phase=comp_phase, rof_nx=comp_nx, rof_ny=comp_ny, & - histavg_rof=histavg_comp, rof_pause=comp_pause, rof_resume=comp_resume) + histavg_rof=histavg_comp, rof_resume=comp_resume) else if (component_firstletter == 'g') then call seq_infodata_GetData(infodata, glc_present=comp_present, & glc_prognostic=comp_prognostic, glc_gnam=comp_gnam, & glc_phase=comp_phase, glc_nx=comp_nx, glc_ny=comp_ny, & - histavg_glc=histavg_comp, glc_pause=comp_pause, glc_resume=comp_resume) + histavg_glc=histavg_comp, glc_resume=comp_resume) else if (component_firstletter == 'w') then call seq_infodata_GetData(infodata, wav_present=comp_present, & wav_prognostic=comp_prognostic, wav_gnam=comp_gnam, & wav_phase=comp_phase, wav_nx=comp_nx, wav_ny=comp_ny, & - histavg_wav=histavg_comp, wav_pause=comp_pause, wav_resume=comp_resume) + histavg_wav=histavg_comp, wav_resume=comp_resume) else if (component_firstletter == 'e') then if (present(comp_gnam)) then comp_gnam = '' @@ -1469,12 +1434,6 @@ SUBROUTINE seq_infodata_GetData_bytype( component_firstletter, infodata, & write(logunit,*) trim(subname),' Note: ESP type has no histavg property' end if end if - if (present(comp_pause)) then - comp_pause = .false. - if ((loglevel > 1) .and. seq_comm_iamroot(1)) then - write(logunit,*) trim(subname),' Note: ESP type has no pause property' - end if - end if if (present(comp_resume)) then comp_resume = ' ' if ((loglevel > 1) .and. seq_comm_iamroot(1)) then @@ -1515,7 +1474,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ ice_gnam, rof_gnam, glc_gnam, wav_gnam, & atm_gnam, ocn_gnam, info_debug, dead_comps, read_restart, & shr_map_dopole, vect_map, aoflux_grid, run_barriers, & - nextsw_cday, precip_fact, flux_epbal, flux_albav, glcrun_alarm, & + nextsw_cday, precip_fact, flux_epbal, flux_albav, & glc_g2lupdate, atm_aero, esmf_map_flag, wall_time_limit, & do_budgets, do_histinit, drv_threading, flux_diurnal, gust_fac, & budget_inst, budget_daily, budget_month, force_stop_at, & @@ -1535,10 +1494,9 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ glc_nx, glc_ny, eps_frac, eps_amask, & eps_agrid, eps_aarea, eps_omask, eps_ogrid, eps_oarea, & reprosum_use_ddpdd, reprosum_diffmax, reprosum_recompute, & - atm_pause, lnd_pause, ocn_pause, ice_pause, glc_pause, rof_pause, & - wav_pause, cpl_pause, atm_resume, lnd_resume, ocn_resume, & - ice_resume, glc_resume, rof_resume, wav_resume, cpl_resume, & - mct_usealltoall, mct_usevector ) + atm_resume, lnd_resume, ocn_resume, ice_resume, & + glc_resume, rof_resume, wav_resume, cpl_resume, & + mct_usealltoall, mct_usevector, glc_valid_input) implicit none @@ -1696,16 +1654,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ integer(SHR_KIND_IN), optional, intent(IN) :: wav_phase ! wav phase integer(SHR_KIND_IN), optional, intent(IN) :: esp_phase ! esp phase logical, optional, intent(IN) :: atm_aero ! atm aerosols - logical, optional, intent(IN) :: glcrun_alarm ! glc run alarm logical, optional, intent(IN) :: glc_g2lupdate ! update glc2lnd fields in lnd model - logical, optional, intent(IN) :: atm_pause ! atm pause - logical, optional, intent(IN) :: lnd_pause ! lnd pause - logical, optional, intent(IN) :: ice_pause ! ice pause - logical, optional, intent(IN) :: ocn_pause ! ocn pause - logical, optional, intent(IN) :: glc_pause ! glc pause - logical, optional, intent(IN) :: rof_pause ! rof pause - logical, optional, intent(IN) :: wav_pause ! wav pause - logical, optional, intent(IN) :: cpl_pause ! cpl pause + logical, optional, intent(IN) :: glc_valid_input character(SHR_KIND_CL), optional, intent(IN) :: atm_resume(:) ! atm resume character(SHR_KIND_CL), optional, intent(IN) :: lnd_resume(:) ! lnd resume character(SHR_KIND_CL), optional, intent(IN) :: ice_resume(:) ! ice resume @@ -1873,72 +1823,8 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_ if ( present(wav_phase) ) infodata%wav_phase = wav_phase if ( present(esp_phase) ) infodata%esp_phase = esp_phase if ( present(atm_aero) ) infodata%atm_aero = atm_aero - if ( present(glcrun_alarm) ) infodata%glcrun_alarm = glcrun_alarm if ( present(glc_g2lupdate) ) infodata%glc_g2lupdate = glc_g2lupdate - if ( present(atm_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%atm_pause = atm_pause - else if (atm_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%atm_pause = atm_pause - end if - end if - if ( present(lnd_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%lnd_pause = lnd_pause - else if (lnd_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%lnd_pause = lnd_pause - end if - end if - if ( present(ice_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%ice_pause = ice_pause - else if (ice_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%ice_pause = ice_pause - end if - end if - if ( present(ocn_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%ocn_pause = ocn_pause - else if (ocn_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%ocn_pause = ocn_pause - end if - end if - if ( present(glc_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%glc_pause = glc_pause - else if (glc_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%glc_pause = glc_pause - end if - end if - if ( present(rof_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%rof_pause = rof_pause - else if (rof_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%rof_pause = rof_pause - end if - end if - if ( present(wav_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%wav_pause = wav_pause - else if (wav_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%wav_pause = wav_pause - end if - end if - if ( present(cpl_pause) ) then - if (associated(infodata%pause_resume)) then - infodata%pause_resume%cpl_pause = cpl_pause - else if (cpl_pause) then - allocate(infodata%pause_resume) - infodata%pause_resume%cpl_pause = cpl_pause - end if - end if + if ( present(glc_valid_input) ) infodata%glc_valid_input = glc_valid_input if ( present(atm_resume) ) then if (associated(infodata%pause_resume)) then infodata%pause_resume%atm_resume(:) = atm_resume(:) @@ -2018,7 +1904,7 @@ END SUBROUTINE seq_infodata_PutData_explicit SUBROUTINE seq_infodata_PutData_bytype( component_firstletter, infodata, & comp_present, comp_prognostic, comp_gnam, & - histavg_comp, comp_phase, comp_nx, comp_ny, comp_pause, comp_resume) + histavg_comp, comp_phase, comp_nx, comp_ny, comp_resume) implicit none @@ -2033,7 +1919,6 @@ SUBROUTINE seq_infodata_PutData_bytype( component_firstletter, infodata, & integer(SHR_KIND_IN), optional, intent(IN) :: comp_ny ! nx,ny 2d grid size global integer(SHR_KIND_IN), optional, intent(IN) :: comp_phase logical, optional, intent(IN) :: histavg_comp - logical, optional, intent(IN) :: comp_pause character(SHR_KIND_CL), optional, intent(IN) :: comp_resume(:) !EOP @@ -2047,37 +1932,37 @@ SUBROUTINE seq_infodata_PutData_bytype( component_firstletter, infodata, & call seq_infodata_PutData(infodata, atm_present=comp_present, & atm_prognostic=comp_prognostic, atm_gnam=comp_gnam, & atm_phase=comp_phase, atm_nx=comp_nx, atm_ny=comp_ny, & - histavg_atm=histavg_comp, atm_pause=comp_pause, atm_resume=comp_resume) + histavg_atm=histavg_comp, atm_resume=comp_resume) else if (component_firstletter == 'l') then call seq_infodata_PutData(infodata, lnd_present=comp_present, & lnd_prognostic=comp_prognostic, lnd_gnam=comp_gnam, & lnd_phase=comp_phase, lnd_nx=comp_nx, lnd_ny=comp_ny, & - histavg_lnd=histavg_comp, lnd_pause=comp_pause, lnd_resume=comp_resume) + histavg_lnd=histavg_comp, lnd_resume=comp_resume) else if (component_firstletter == 'i') then call seq_infodata_PutData(infodata, ice_present=comp_present, & ice_prognostic=comp_prognostic, ice_gnam=comp_gnam, & ice_phase=comp_phase, ice_nx=comp_nx, ice_ny=comp_ny, & - histavg_ice=histavg_comp, ice_pause=comp_pause, ice_resume=comp_resume) + histavg_ice=histavg_comp, ice_resume=comp_resume) else if (component_firstletter == 'o') then call seq_infodata_PutData(infodata, ocn_present=comp_present, & ocn_prognostic=comp_prognostic, ocn_gnam=comp_gnam, & ocn_phase=comp_phase, ocn_nx=comp_nx, ocn_ny=comp_ny, & - histavg_ocn=histavg_comp, ocn_pause=comp_pause, ocn_resume=comp_resume) + histavg_ocn=histavg_comp, ocn_resume=comp_resume) else if (component_firstletter == 'r') then call seq_infodata_PutData(infodata, rof_present=comp_present, & rof_prognostic=comp_prognostic, rof_gnam=comp_gnam, & rof_phase=comp_phase, rof_nx=comp_nx, rof_ny=comp_ny, & - histavg_rof=histavg_comp, rof_pause=comp_pause, rof_resume=comp_resume) + histavg_rof=histavg_comp, rof_resume=comp_resume) else if (component_firstletter == 'g') then call seq_infodata_PutData(infodata, glc_present=comp_present, & glc_prognostic=comp_prognostic, glc_gnam=comp_gnam, & glc_phase=comp_phase, glc_nx=comp_nx, glc_ny=comp_ny, & - histavg_glc=histavg_comp, glc_pause=comp_pause, glc_resume=comp_resume) + histavg_glc=histavg_comp, glc_resume=comp_resume) else if (component_firstletter == 'w') then call seq_infodata_PutData(infodata, wav_present=comp_present, & wav_prognostic=comp_prognostic, wav_gnam=comp_gnam, & wav_phase=comp_phase, wav_nx=comp_nx, wav_ny=comp_ny, & - histavg_wav=histavg_comp, wav_pause=comp_pause, wav_resume=comp_resume) + histavg_wav=histavg_comp, wav_resume=comp_resume) else if (component_firstletter == 'e') then if ((loglevel > 1) .and. seq_comm_iamroot(1)) then if (present(comp_gnam)) then @@ -2092,9 +1977,6 @@ SUBROUTINE seq_infodata_PutData_bytype( component_firstletter, infodata, & if (present(histavg_comp)) then write(logunit,*) trim(subname),' Note: ESP type has no histavg property' end if - if (present(comp_pause)) then - write(logunit,*) trim(subname),' Note: ESP type has no pause property' - end if if (present(comp_resume)) then write(logunit,*) trim(subname),' Note: ESP type has no resume property' end if @@ -2111,6 +1993,74 @@ END SUBROUTINE seq_infodata_PutData_bytype #endif ! ^ ifndef CPRPGI +!=============================================================================== +!BOP =========================================================================== +! +! !IROUTINE: seq_infodata_pauseresume_bcast -- Broadcast pause/resume data from root pe +! +! !DESCRIPTION: +! +! Broadcast the pause_resume data from an infodata across pes +! +! !INTERFACE: ------------------------------------------------------------------ + +subroutine seq_infodata_pauseresume_bcast(infodata, mpicom, pebcast) + + use shr_mpi_mod, only : shr_mpi_bcast + +! !INPUT/OUTPUT PARAMETERS: + + type(seq_infodata_type), intent(INOUT) :: infodata ! assume valid on root pe + integer(SHR_KIND_IN), intent(IN) :: mpicom ! MPI Communicator + integer(SHR_KIND_IN), optional, intent(IN) :: pebcast ! pe sending + +!EOP + + !----- local ----- + integer :: ind + integer(SHR_KIND_IN) :: pebcast_local + character(len=*), parameter :: subname = '(seq_infodata_pauseresume_bcast) ' + + if (present(pebcast)) then + pebcast_local = pebcast + else + pebcast_local = 0 + end if + + if (associated(infodata%pause_resume)) then + do ind = 1, num_inst_atm + call shr_mpi_bcast(infodata%pause_resume%atm_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_lnd + call shr_mpi_bcast(infodata%pause_resume%lnd_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_ice + call shr_mpi_bcast(infodata%pause_resume%ice_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_ocn + call shr_mpi_bcast(infodata%pause_resume%ocn_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_glc + call shr_mpi_bcast(infodata%pause_resume%glc_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_rof + call shr_mpi_bcast(infodata%pause_resume%rof_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + do ind = 1, num_inst_wav + call shr_mpi_bcast(infodata%pause_resume%wav_resume(ind), mpicom, & + pebcast=pebcast_local) + end do + call shr_mpi_bcast(infodata%pause_resume%cpl_resume, mpicom, & + pebcast=pebcast_local) + end if +end subroutine seq_infodata_pauseresume_bcast + !=============================================================================== !BOP =========================================================================== ! @@ -2291,40 +2241,10 @@ subroutine seq_infodata_bcast(infodata,mpicom) call shr_mpi_bcast(infodata%rof_phase, mpicom) call shr_mpi_bcast(infodata%wav_phase, mpicom) call shr_mpi_bcast(infodata%atm_aero, mpicom) - call shr_mpi_bcast(infodata%glcrun_alarm, mpicom) call shr_mpi_bcast(infodata%glc_g2lupdate, mpicom) - if (associated(infodata%pause_resume)) then - call shr_mpi_bcast(infodata%pause_resume%atm_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%lnd_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%ice_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%ocn_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%glc_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%rof_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%wav_pause, mpicom) - call shr_mpi_bcast(infodata%pause_resume%cpl_pause, mpicom) - do ind = 1, num_inst_atm - call shr_mpi_bcast(infodata%pause_resume%atm_resume(ind), mpicom) - end do - do ind = 1, num_inst_lnd - call shr_mpi_bcast(infodata%pause_resume%lnd_resume(ind), mpicom) - end do - do ind = 1, num_inst_ice - call shr_mpi_bcast(infodata%pause_resume%ice_resume(ind), mpicom) - end do - do ind = 1, num_inst_ocn - call shr_mpi_bcast(infodata%pause_resume%ocn_resume(ind), mpicom) - end do - do ind = 1, num_inst_glc - call shr_mpi_bcast(infodata%pause_resume%glc_resume(ind), mpicom) - end do - do ind = 1, num_inst_rof - call shr_mpi_bcast(infodata%pause_resume%rof_resume(ind), mpicom) - end do - do ind = 1, num_inst_wav - call shr_mpi_bcast(infodata%pause_resume%wav_resume(ind), mpicom) - end do - call shr_mpi_bcast(infodata%pause_resume%cpl_resume, mpicom) - end if + call shr_mpi_bcast(infodata%glc_valid_input, mpicom) + + call seq_infodata_pauseresume_bcast(infodata, mpicom) end subroutine seq_infodata_bcast @@ -2355,7 +2275,8 @@ subroutine seq_infodata_Exchange(infodata,ID,type) !----- local ----- integer(SHR_KIND_IN) :: mpicom ! mpicom - integer(SHR_KIND_IN) :: pebcast ! pe sending + integer(SHR_KIND_IN) :: cmppe ! component 'root' for broadcast + integer(SHR_KIND_IN) :: cplpe ! coupler 'root' for broadcast logical :: atm2cpli,atm2cplr logical :: lnd2cpli,lnd2cplr logical :: rof2cpli,rof2cplr @@ -2363,6 +2284,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) logical :: ice2cpli,ice2cplr logical :: glc2cpli,glc2cplr logical :: wav2cpli,wav2cplr + logical :: esp2cpli,esp2cplr logical :: cpl2i,cpl2r logical :: logset logical :: deads ! local variable to hold info temporarily @@ -2372,8 +2294,7 @@ subroutine seq_infodata_Exchange(infodata,ID,type) ! Notes: !------------------------------------------------------------------------------- - ! assume the comp pe is going to broadcast, change to cplpe below if appropriate - call seq_comm_setptrs(ID,mpicom=mpicom,cmppe=pebcast) + call seq_comm_setptrs(ID, mpicom=mpicom, cmppe=cmppe, cplpe=cplpe) logset = .false. @@ -2391,6 +2312,8 @@ subroutine seq_infodata_Exchange(infodata,ID,type) glc2cplr = .false. wav2cpli = .false. wav2cplr = .false. + esp2cpli = .false. + esp2cplr = .false. cpl2i = .false. cpl2r = .false. @@ -2466,16 +2389,26 @@ subroutine seq_infodata_Exchange(infodata,ID,type) logset = .true. endif + if (trim(type) == 'esp2cpl_init') then + esp2cpli = .true. + esp2cplr = .true. + logset = .true. + endif + if (trim(type) == 'esp2cpl_run') then + esp2cplr = .true. + logset = .true. + endif + if (trim(type) == 'cpl2atm_init' .or. & trim(type) == 'cpl2lnd_init' .or. & trim(type) == 'cpl2rof_init' .or. & trim(type) == 'cpl2ocn_init' .or. & trim(type) == 'cpl2glc_init' .or. & trim(type) == 'cpl2wav_init' .or. & + trim(type) == 'cpl2esp_init' .or. & trim(type) == 'cpl2ice_init') then cpl2i = .true. cpl2r = .true. - call seq_comm_setptrs(ID,cplpe=pebcast) logset = .true. endif @@ -2487,7 +2420,6 @@ subroutine seq_infodata_Exchange(infodata,ID,type) trim(type) == 'cpl2wav_run' .or. & trim(type) == 'cpl2ice_run') then cpl2r = .true. - call seq_comm_setptrs(ID,cplpe=pebcast) logset = .true. endif @@ -2501,129 +2433,143 @@ subroutine seq_infodata_Exchange(infodata,ID,type) ! --- now execute exchange --- if (atm2cpli) then - call shr_mpi_bcast(infodata%atm_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_ny, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_aero, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%atm_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%atm_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%atm_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%atm_ny, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%atm_aero, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (lnd2cpli) then - call shr_mpi_bcast(infodata%lnd_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%lnd_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%lnd_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%lnd_ny, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%lnd_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%lnd_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%lnd_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%lnd_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (rof2cpli) then - call shr_mpi_bcast(infodata%rof_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rofice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rof_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rof_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rof_ny, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%flood_present, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%rof_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rofice_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rof_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rof_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%rof_ny, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%flood_present, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (ocn2cpli) then - call shr_mpi_bcast(infodata%ocn_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocn_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocnrof_prognostic,mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocn_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocn_ny, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%ocn_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocn_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ocn_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (ice2cpli) then - call shr_mpi_bcast(infodata%ice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ice_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%iceberg_prognostic,mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ice_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ice_ny, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%ice_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ice_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ice_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%ice_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (glc2cpli) then - call shr_mpi_bcast(infodata%glc_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glclnd_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glcocn_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glcice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_ny, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%glc_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glclnd_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glcocn_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glcice_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glc_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glc_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%glc_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif if (wav2cpli) then - call shr_mpi_bcast(infodata%wav_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%wav_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%wav_nx, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%wav_ny, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%wav_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%wav_prognostic, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%wav_nx, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%wav_ny, mpicom, pebcast=cmppe) ! dead_comps is true if it's ever set to true deads = infodata%dead_comps - call shr_mpi_bcast(deads, mpicom,pebcast=pebcast) + call shr_mpi_bcast(deads, mpicom, pebcast=cmppe) if (deads .or. infodata%dead_comps) infodata%dead_comps = .true. endif + if (esp2cpli) then + call shr_mpi_bcast(infodata%esp_present, mpicom, pebcast=cmppe) + call shr_mpi_bcast(infodata%esp_prognostic, mpicom, pebcast=cmppe) + call seq_infodata_pauseresume_bcast(infodata, mpicom, pebcast=cmppe) + endif + if (cpl2i) then - call shr_mpi_bcast(infodata%atm_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%lnd_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%lnd_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rof_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rofice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%rof_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%flood_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocn_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocn_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ocnrof_prognostic,mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%ice_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%iceberg_prognostic,mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glclnd_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glcocn_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glcice_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%wav_present, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%wav_prognostic, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%dead_comps, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%atm_aero, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%atm_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%atm_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%lnd_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%lnd_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%rof_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%rofice_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%rof_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%flood_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ocn_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ocn_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ocnrof_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ice_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%ice_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%iceberg_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glc_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glclnd_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glcocn_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glcice_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glc_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%wav_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%wav_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%esp_present, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%esp_prognostic, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%dead_comps, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%atm_aero, mpicom, pebcast=cplpe) endif + ! Run-time data exchanges if (atm2cplr) then - call shr_mpi_bcast(infodata%nextsw_cday, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%nextsw_cday, mpicom, pebcast=cmppe) endif if (ocn2cplr) then - call shr_mpi_bcast(infodata%precip_fact, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cmppe) + endif + + if (esp2cplr) then + call seq_infodata_pauseresume_bcast(infodata, mpicom, pebcast=cmppe) endif if (cpl2r) then - call shr_mpi_bcast(infodata%nextsw_cday, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%precip_fact, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glcrun_alarm, mpicom,pebcast=pebcast) - call shr_mpi_bcast(infodata%glc_g2lupdate, mpicom,pebcast=pebcast) + call shr_mpi_bcast(infodata%nextsw_cday, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%precip_fact, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glc_g2lupdate, mpicom, pebcast=cplpe) + call shr_mpi_bcast(infodata%glc_valid_input, mpicom, pebcast=cplpe) + call seq_infodata_pauseresume_bcast(infodata, mpicom, pebcast=cplpe) endif end subroutine seq_infodata_Exchange @@ -2754,33 +2700,6 @@ subroutine seq_infodata_Check( infodata ) call shr_sys_abort(subname//': vect_map invalid = '//trim(infodata%vect_map)) endif - if (associated(infodata%pause_resume)) then - if (infodata%pause_resume%atm_pause .and. ANY(len_trim(infodata%pause_resume%atm_resume) > 0)) then - call shr_sys_abort(subname//': If atm_pause is .true., then atm_resume should not be set') - end if - if (infodata%pause_resume%lnd_pause .and. ANY(len_trim(infodata%pause_resume%lnd_resume) > 0)) then - call shr_sys_abort(subname//': If lnd_pause is .true., then lnd_resume should not be set') - end if - if (infodata%pause_resume%ocn_pause .and. ANY(len_trim(infodata%pause_resume%ocn_resume) > 0)) then - call shr_sys_abort(subname//': If ocn_pause is .true., then ocn_resume should not be set') - end if - if (infodata%pause_resume%ice_pause .and. ANY(len_trim(infodata%pause_resume%ice_resume) > 0)) then - call shr_sys_abort(subname//': If ice_pause is .true., then ice_resume should not be set') - end if - if (infodata%pause_resume%rof_pause .and. ANY(len_trim(infodata%pause_resume%rof_resume) > 0)) then - call shr_sys_abort(subname//': If rof_pause is .true., then rof_resume should not be set') - end if - if (infodata%pause_resume%glc_pause .and. ANY(len_trim(infodata%pause_resume%glc_resume) > 0)) then - call shr_sys_abort(subname//': If glc_pause is .true., then glc_resume should not be set') - end if - if (infodata%pause_resume%wav_pause .and. ANY(len_trim(infodata%pause_resume%wav_resume) > 0)) then - call shr_sys_abort(subname//': If wav_pause is .true., then wav_resume should not be set') - end if - if (infodata%pause_resume%cpl_pause .and. (len_trim(infodata%pause_resume%cpl_resume) > 0)) then - call shr_sys_abort(subname//': If cpl_pause is .true., then cpl_resume should not be set') - end if - end if - END SUBROUTINE seq_infodata_Check !=============================================================================== @@ -2992,17 +2911,8 @@ SUBROUTINE seq_infodata_print( infodata ) write(logunit,F0S) subname,'rof_phase = ', infodata%rof_phase write(logunit,F0S) subname,'wav_phase = ', infodata%wav_phase - write(logunit,F0L) subname,'glcrun_alarm = ', infodata%glcrun_alarm write(logunit,F0L) subname,'glc_g2lupdate = ', infodata%glc_g2lupdate if (associated(infodata%pause_resume)) then - write(logunit,F0L) subname,'atm_pause = ', infodata%pause_resume%atm_pause - write(logunit,F0L) subname,'lnd_pause = ', infodata%pause_resume%lnd_pause - write(logunit,F0L) subname,'ocn_pause = ', infodata%pause_resume%ocn_pause - write(logunit,F0L) subname,'ice_pause = ', infodata%pause_resume%ice_pause - write(logunit,F0L) subname,'glc_pause = ', infodata%pause_resume%glc_pause - write(logunit,F0S) subname,'rof_pause = ', infodata%pause_resume%rof_pause - write(logunit,F0L) subname,'wav_pause = ', infodata%pause_resume%wav_pause - write(logunit,F0L) subname,'cpl_pause = ', infodata%pause_resume%cpl_pause do ind = 1, num_inst_atm if (len_trim(infodata%pause_resume%atm_resume(ind)) > 0) then write(logunit,FIA) subname,'atm_resume(',ind,') = ', trim(infodata%pause_resume%atm_resume(ind)) diff --git a/driver_cpl/shr/seq_io_read_mod.F90 b/src/drivers/mct/shr/seq_io_read_mod.F90 similarity index 100% rename from driver_cpl/shr/seq_io_read_mod.F90 rename to src/drivers/mct/shr/seq_io_read_mod.F90 diff --git a/driver_cpl/shr/seq_timemgr_mod.F90 b/src/drivers/mct/shr/seq_timemgr_mod.F90 similarity index 86% rename from driver_cpl/shr/seq_timemgr_mod.F90 rename to src/drivers/mct/shr/seq_timemgr_mod.F90 index 8a297ce9e04..ed36dbdfd2a 100644 --- a/driver_cpl/shr/seq_timemgr_mod.F90 +++ b/src/drivers/mct/shr/seq_timemgr_mod.F90 @@ -1,8 +1,4 @@ !=============================================================================== -! SVN $Id: seq_timemgr_mod.F90 68253 2015-02-18 22:24:57Z mvertens $ -! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/drv/seq_mct/trunk_tags/drvseq5_1_15/shr/seq_timemgr_mod.F90 $ -!=============================================================================== -!BOP =========================================================================== ! ! !MODULE: seq_timemgr_mod --- Time-manager module ! @@ -60,9 +56,11 @@ module seq_timemgr_mod public :: seq_timemgr_restartAlarmIsOn ! Is a restart alarm ringing public :: seq_timemgr_stopAlarmIsOn ! Is a stop alarm ringing public :: seq_timemgr_historyAlarmIsOn ! Is a history alarm ringing - - ! --- Data that belongs to the driver (but is here to avoid loops) - character(SHR_KIND_CS),public :: seq_timemgr_pause_component_list ! Pause - resume components + public :: seq_timemgr_pauseAlarmIsOn ! Is a pause alarm ringing + ! --- ESP components need to know about the state of other components + public :: seq_timemgr_pause_active ! Pause/resume is enabled + public :: seq_timemgr_pause_component_index ! Index of named component + public :: seq_timemgr_pause_component_active ! .true. is comp should pause ! ! PUBLIC PARAMETERS: @@ -104,6 +102,7 @@ module seq_timemgr_mod ! seq_timemgr_alarm_ocnrun ! seq_timemgr_alarm_icerun ! seq_timemgr_alarm_glcrun +! seq_timemgr_alarm_glcrun_avg ! seq_timemgr_alarm_wavrun ! seq_timemgr_alarm_esprun ! seq_timemgr_alarm_ocnnext @@ -112,8 +111,6 @@ module seq_timemgr_mod ! seq_timemgr_alarm_pause ! seq_timemgr_alarm_barrier -!EOP - private:: seq_timemgr_alarmGet private:: seq_timemgr_alarmInit private:: seq_timemgr_EClockInit @@ -141,7 +138,8 @@ module seq_timemgr_mod seq_timemgr_optYearly = "yearly" , & seq_timemgr_optDate = "date" , & seq_timemgr_optIfdays0 = "ifdays0" , & - seq_timemgr_optEnd = "end" + seq_timemgr_optEnd = "end" , & + seq_timemgr_optGLCCouplingPeriod = "glc_coupling_period" integer(SHR_KIND_IN),private,parameter :: & seq_timemgr_nclock_drv = 1, & @@ -153,6 +151,7 @@ module seq_timemgr_mod seq_timemgr_nclock_wav = 7, & seq_timemgr_nclock_rof = 8, & seq_timemgr_nclock_esp = 9 + integer(SHR_KIND_IN),private,parameter :: max_clocks = 9 character(len=*),public,parameter :: & seq_timemgr_clock_drv = 'seq_timemgr_clock_drv' , & @@ -168,68 +167,77 @@ module seq_timemgr_mod (/'drv ','atm ','lnd ','ocn ', & 'ice ','glc ','wav ','rof ','esp '/) + ! Alarms on both component clocks and driver clock integer(SHR_KIND_IN),private,parameter :: & - seq_timemgr_nalarm_restart = 1, & - seq_timemgr_nalarm_run = 2, & - seq_timemgr_nalarm_stop = 3, & - seq_timemgr_nalarm_datestop= 4, & - seq_timemgr_nalarm_history = 5, & - seq_timemgr_nalarm_atmrun = 6, & - seq_timemgr_nalarm_lndrun = 7, & - seq_timemgr_nalarm_ocnrun = 8, & - seq_timemgr_nalarm_icerun = 9, & - seq_timemgr_nalarm_glcrun =10, & - seq_timemgr_nalarm_ocnnext =11, & - seq_timemgr_nalarm_tprof =12, & - seq_timemgr_nalarm_histavg =13, & - seq_timemgr_nalarm_rofrun =14, & - seq_timemgr_nalarm_wavrun =15, & - seq_timemgr_nalarm_esprun =16, & - seq_timemgr_nalarm_pause =17, & - seq_timemgr_nalarm_barrier =18, & + seq_timemgr_nalarm_restart = 1 , & ! driver and component clock alarm + seq_timemgr_nalarm_run = 2 , & ! driver and component clock alarm + seq_timemgr_nalarm_stop = 3 , & ! driver and component clock alarm + seq_timemgr_nalarm_datestop = 4 , & ! driver and component clock alarm + seq_timemgr_nalarm_history = 5 , & ! driver and component clock alarm + seq_timemgr_nalarm_atmrun = 6 , & ! driver only clock alarm + seq_timemgr_nalarm_lndrun = 7 , & ! driver only clock alarm + seq_timemgr_nalarm_ocnrun = 8 , & ! driver only clock alarm + seq_timemgr_nalarm_icerun = 9 , & ! driver only clock alarm + seq_timemgr_nalarm_glcrun =10 , & ! driver only clock alarm + seq_timemgr_nalarm_glcrun_avg =11 , & ! driver only clock alarm + seq_timemgr_nalarm_ocnnext =12 , & ! driver only clock alarm + seq_timemgr_nalarm_tprof =13 , & ! driver and component clock alarm + seq_timemgr_nalarm_histavg =14 , & ! driver and component clock alarm + seq_timemgr_nalarm_rofrun =15 , & ! driver only clock alarm + seq_timemgr_nalarm_wavrun =16 , & ! driver only clock alarm + seq_timemgr_nalarm_esprun =17 , & ! driver only clock alarm + seq_timemgr_nalarm_pause =18 , & + seq_timemgr_nalarm_barrier =19 , & ! driver and component clock alarm max_alarms = seq_timemgr_nalarm_barrier + character(len=*),public,parameter :: & - seq_timemgr_alarm_restart = 'seq_timemgr_alarm_restart ', & - seq_timemgr_alarm_run = 'seq_timemgr_alarm_run ', & - seq_timemgr_alarm_stop = 'seq_timemgr_alarm_stop ', & - seq_timemgr_alarm_datestop= 'seq_timemgr_alarm_datestop', & - seq_timemgr_alarm_history = 'seq_timemgr_alarm_history ', & - seq_timemgr_alarm_atmrun = 'seq_timemgr_alarm_atmrun ', & - seq_timemgr_alarm_lndrun = 'seq_timemgr_alarm_lndrun ', & - seq_timemgr_alarm_ocnrun = 'seq_timemgr_alarm_ocnrun ', & - seq_timemgr_alarm_icerun = 'seq_timemgr_alarm_icerun ', & - seq_timemgr_alarm_glcrun = 'seq_timemgr_alarm_glcrun ', & - seq_timemgr_alarm_ocnnext = 'seq_timemgr_alarm_ocnnext ', & - seq_timemgr_alarm_tprof = 'seq_timemgr_alarm_tprof ', & - seq_timemgr_alarm_histavg = 'seq_timemgr_alarm_histavg ', & - seq_timemgr_alarm_rofrun = 'seq_timemgr_alarm_rofrun ', & - seq_timemgr_alarm_wavrun = 'seq_timemgr_alarm_wavrun ', & - seq_timemgr_alarm_esprun = 'seq_timemgr_alarm_esprun ', & - seq_timemgr_alarm_pause = 'seq_timemgr_alarm_pause ', & - seq_timemgr_alarm_barrier = 'seq_timemgr_alarm_barrier ' + seq_timemgr_alarm_restart = 'seq_timemgr_alarm_restart ', & + seq_timemgr_alarm_run = 'seq_timemgr_alarm_run ', & + seq_timemgr_alarm_stop = 'seq_timemgr_alarm_stop ', & + seq_timemgr_alarm_datestop = 'seq_timemgr_alarm_datestop', & + seq_timemgr_alarm_history = 'seq_timemgr_alarm_history ', & + seq_timemgr_alarm_atmrun = 'seq_timemgr_alarm_atmrun ', & + seq_timemgr_alarm_lndrun = 'seq_timemgr_alarm_lndrun ', & + seq_timemgr_alarm_ocnrun = 'seq_timemgr_alarm_ocnrun ', & + seq_timemgr_alarm_icerun = 'seq_timemgr_alarm_icerun ', & + seq_timemgr_alarm_glcrun = 'seq_timemgr_alarm_glcrun ', & + seq_timemgr_alarm_glcrun_avg = 'seq_timemgr_alarm_glcrun_avg' , & + seq_timemgr_alarm_ocnnext = 'seq_timemgr_alarm_ocnnext ', & + seq_timemgr_alarm_tprof = 'seq_timemgr_alarm_tprof ', & + seq_timemgr_alarm_histavg = 'seq_timemgr_alarm_histavg ', & + seq_timemgr_alarm_rofrun = 'seq_timemgr_alarm_rofrun ', & + seq_timemgr_alarm_wavrun = 'seq_timemgr_alarm_wavrun ', & + seq_timemgr_alarm_esprun = 'seq_timemgr_alarm_esprun ', & + seq_timemgr_alarm_pause = 'seq_timemgr_alarm_pause ', & + seq_timemgr_alarm_barrier = 'seq_timemgr_alarm_barrier ' + + ! Active pause - resume components + logical, private :: pause_active(max_clocks) = .false. type EClock_pointer ! needed for array of pointers type(ESMF_Clock),pointer :: EClock => null() end type EClock_pointer + type seq_timemgr_type private type(EClock_pointer) :: ECP(max_clocks) ! ESMF clocks, array of pointers type(ESMF_Alarm) :: EAlarm(max_clocks,max_alarms) ! array of clock alarms end type seq_timemgr_type - ! --- Private local data ------------------------------------------------------- + ! --- Private local data ---------------------------------------------------- + + type(ESMF_Calendar), target, save :: seq_timemgr_cal ! calendar + character(SHR_KIND_CL) ,save :: seq_timemgr_calendar ! calendar string + integer, parameter :: SecPerDay = 86400 ! Seconds per day - type(ESMF_Calendar), target, save :: seq_timemgr_cal ! calendar - character(SHR_KIND_CL) ,save :: seq_timemgr_calendar ! calendar string - logical :: seq_timemgr_end_restart ! write restarts at end of run? - logical, save :: seq_timemgr_setCalendar = .false. ! if calendar has been set - integer, parameter :: SecPerDay = 86400 ! Seconds per day + integer :: seq_timemgr_pause_sig_index ! Index of pause comp with smallest dt + logical :: seq_timemgr_esp_run_on_pause ! Run ESP component on pause cycle + logical :: seq_timemgr_end_restart ! write restarts at end of run? !=============================================================================== contains -!=============================================================================== !=============================================================================== ! !IROUTINE: seq_timemgr_clockInit -- Initializes clocks ! @@ -244,8 +252,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi Eclock_rof, EClock_wav, Eclock_esp) ! !USES: - use pio, only : file_desc_T - use shr_string_mod, only : shr_string_toupper + use pio, only : file_desc_T use shr_file_mod, only : shr_file_getunit, shr_file_freeunit use shr_mpi_mod, only : shr_mpi_bcast use seq_io_read_mod @@ -269,7 +276,6 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi type(ESMF_clock),target, intent(IN) :: EClock_wav ! wav clock type(ESMF_clock),target, intent(IN) :: EClock_esp ! esp clock type(file_desc_t) :: pioid -!EOP !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_clockInit) ' @@ -280,10 +286,11 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi type(ESMF_Time) :: StopTime1 ! Stop time type(ESMF_Time) :: StopTime2 ! Stop time type(ESMF_TimeInterval) :: TimeStep ! Clock time-step - type(ESMF_TimeInterval) :: AlarmInterval ! Alarm interval type(ESMF_CalKind_Flag) :: esmf_caltype ! local esmf calendar integer :: rc ! Return code - integer :: n ! index + integer :: n, i ! index + logical :: found + integer :: min_dt ! smallest time step integer :: dtime(max_clocks) ! time-step to use integer :: offset(max_clocks) ! run offset integer :: unitn ! i/o unit number @@ -323,6 +330,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi integer(SHR_KIND_IN) :: ice_cpl_dt ! Sea-Ice coupling interval integer(SHR_KIND_IN) :: ocn_cpl_dt ! Ocean coupling interval integer(SHR_KIND_IN) :: glc_cpl_dt ! Glc coupling interval + character(SHR_KIND_CS) :: glc_avg_period ! Glc avering coupling period integer(SHR_KIND_IN) :: rof_cpl_dt ! Runoff coupling interval integer(SHR_KIND_IN) :: wav_cpl_dt ! Wav coupling interval integer(SHR_KIND_IN) :: esp_cpl_dt ! Esp coupling interval @@ -334,8 +342,8 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi integer(SHR_KIND_IN) :: wav_cpl_offset ! Wav coupling interval integer(SHR_KIND_IN) :: rof_cpl_offset ! Runoff coupling interval integer(SHR_KIND_IN) :: esp_cpl_offset ! Esp coupling interval + logical :: esp_run_on_pause ! Run ESP on pause cycle logical :: end_restart ! Write restart at end of run - integer(SHR_KIND_IN) :: nlUnit ! Namelist unit number integer(SHR_KIND_IN) :: ierr ! Return code character(len=*), parameter :: F0A = "(2A,A)" @@ -353,22 +361,22 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi start_ymd, start_tod, ref_ymd, ref_tod, & atm_cpl_dt, ocn_cpl_dt, ice_cpl_dt, lnd_cpl_dt, & atm_cpl_offset, lnd_cpl_offset, ocn_cpl_offset, & - ice_cpl_offset, glc_cpl_dt, glc_cpl_offset, & + ice_cpl_offset, glc_cpl_dt, glc_cpl_offset, glc_avg_period, & wav_cpl_dt, wav_cpl_offset, esp_cpl_dt, esp_cpl_offset, & - rof_cpl_dt, rof_cpl_offset, end_restart + rof_cpl_dt, rof_cpl_offset, esp_run_on_pause, end_restart !------------------------------------------------------------------------------- ! Notes: !------------------------------------------------------------------------------- - SyncClock%ECP(seq_timemgr_nclock_drv)%EClock => EClock_drv - SyncClock%ECP(seq_timemgr_nclock_atm)%EClock => EClock_atm - SyncClock%ECP(seq_timemgr_nclock_lnd)%EClock => EClock_lnd - SyncClock%ECP(seq_timemgr_nclock_ocn)%EClock => EClock_ocn - SyncClock%ECP(seq_timemgr_nclock_ice)%EClock => EClock_ice - SyncClock%ECP(seq_timemgr_nclock_glc)%EClock => EClock_glc - SyncClock%ECP(seq_timemgr_nclock_rof)%EClock => EClock_rof - SyncClock%ECP(seq_timemgr_nclock_wav)%EClock => EClock_wav - SyncClock%ECP(seq_timemgr_nclock_esp)%EClock => EClock_esp + SyncClock%ECP(seq_timemgr_nclock_drv)%EClock => EClock_drv + SyncClock%ECP(seq_timemgr_nclock_atm)%EClock => EClock_atm + SyncClock%ECP(seq_timemgr_nclock_lnd)%EClock => EClock_lnd + SyncClock%ECP(seq_timemgr_nclock_ocn)%EClock => EClock_ocn + SyncClock%ECP(seq_timemgr_nclock_ice)%EClock => EClock_ice + SyncClock%ECP(seq_timemgr_nclock_glc)%EClock => EClock_glc + SyncClock%ECP(seq_timemgr_nclock_rof)%EClock => EClock_rof + SyncClock%ECP(seq_timemgr_nclock_wav)%EClock => EClock_wav + SyncClock%ECP(seq_timemgr_nclock_esp)%EClock => EClock_esp call mpi_comm_rank(mpicom,iam,ierr) @@ -415,6 +423,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi ice_cpl_dt = 0 ocn_cpl_dt = 0 glc_cpl_dt = 0 + glc_avg_period = seq_timemgr_optGLCCouplingPeriod rof_cpl_dt = 0 wav_cpl_dt = 0 esp_cpl_dt = 0 @@ -426,6 +435,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi rof_cpl_offset = 0 wav_cpl_offset = 0 esp_cpl_offset = 0 + esp_run_on_pause = .true. end_restart = .true. !--------------------------------------------------------------------------- @@ -534,6 +544,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi write(logunit,F0A) trim(subname),' pause_option = ',trim(pause_option) write(logunit,F0I) trim(subname),' pause_n = ',pause_n write(logunit,F0A) trim(subname),' pause_component_list = ',trim(pause_component_list) + write(logunit,F0L) trim(subname),' esp_run_on_pause = ',esp_run_on_pause write(logunit,F0A) trim(subname),' history_option = ',trim(history_option) write(logunit,F0I) trim(subname),' history_n = ',history_n write(logunit,F0I) trim(subname),' history_ymd = ',history_ymd @@ -555,6 +566,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi write(logunit,F0I) trim(subname),' ice_cpl_dt = ',ice_cpl_dt write(logunit,F0I) trim(subname),' ocn_cpl_dt = ',ocn_cpl_dt write(logunit,F0I) trim(subname),' glc_cpl_dt = ',glc_cpl_dt + write(logunit,F0A) trim(subname),' glc_avg_period = ',glc_avg_period write(logunit,F0I) trim(subname),' rof_cpl_dt = ',rof_cpl_dt write(logunit,F0I) trim(subname),' wav_cpl_dt = ',wav_cpl_dt write(logunit,F0I) trim(subname),' esp_cpl_dt = ',esp_cpl_dt @@ -644,6 +656,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi call shr_mpi_bcast( ice_cpl_dt, mpicom ) call shr_mpi_bcast( ocn_cpl_dt, mpicom ) call shr_mpi_bcast( glc_cpl_dt, mpicom ) + call shr_mpi_bcast( glc_avg_period, mpicom ) call shr_mpi_bcast( rof_cpl_dt, mpicom ) call shr_mpi_bcast( wav_cpl_dt, mpicom ) call shr_mpi_bcast( esp_cpl_dt, mpicom ) @@ -655,6 +668,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi call shr_mpi_bcast( rof_cpl_offset, mpicom ) call shr_mpi_bcast( wav_cpl_offset, mpicom ) call shr_mpi_bcast( esp_cpl_offset, mpicom ) + call shr_mpi_bcast( esp_run_on_pause, mpicom ) call shr_mpi_bcast( end_restart, mpicom ) ! --- derive a couple things --- @@ -682,8 +696,60 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi ! --- Initialize generic stuff --- seq_timemgr_calendar = shr_cal_calendarName(calendar) + seq_timemgr_esp_run_on_pause = esp_run_on_pause seq_timemgr_end_restart = end_restart - seq_timemgr_pause_component_list = pause_component_list + ! --- Figure out which components (if any) are doing pause this run + rc = 1 + i = 1 + if (trim(pause_component_list) == 'all') then + pause_active = .true. + else if (trim(pause_component_list) == 'none') then + pause_active = .false. + else + do + i = scan(trim(pause_component_list(rc:)), ':') - 1 + if ((i < 0) .and. (len_trim(pause_component_list) >= rc)) then + i = len_trim(pause_component_list(rc:)) + end if + if (i > 0) then + found = .false. + do n = 1, max_clocks + if (pause_component_list(rc:rc+i-1) == trim(seq_timemgr_clocks(n))) then + pause_active(n) = .true. + found = .true. + exit + end if + end do + ! Special case for cpl -- synonym for drv + if ((.not. found) .and. (pause_component_list(rc:rc+i-1) == 'cpl')) then + pause_active(seq_timemgr_nclock_drv) = .true. + found = .true. + end if + if (.not. found) then + call shr_sys_abort(subname//': unknown pause component, '//pause_component_list(rc:rc+i-1)) + end if + rc = rc + i + if (pause_component_list(rc:rc) == ':') then + rc = rc + 1 + end if + if (rc >= len_trim(pause_component_list)) then + exit + end if + else + exit + end if + end do + end if + if ( ANY(pause_active) .and. & + (trim(pause_option) /= seq_timemgr_optNONE) .and. & + (trim(pause_option) /= seq_timemgr_optNever)) then + do n = 1, max_clocks + if (pause_active(n)) then + write(logunit, '(4a)') subname, ': Pause active for ', & + trim(seq_timemgr_clocks(n)),' component' + end if + end do + end if ! --- Create the new calendar if not already set ------ if ( trim(seq_timemgr_calendar) == trim(seq_timemgr_noleap)) then @@ -706,30 +772,47 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi call seq_timemgr_ETimeInit( RefTime , ref_ymd , ref_tod , "Reference date" ) call seq_timemgr_ETimeInit( CurrTime , curr_ymd , curr_tod , "Current date") - ! --- Figure out what CCSM time-stepping interval should be. --------------- + ! --- Figure out what time-stepping interval should be. --------------- dtime = 0 - dtime(seq_timemgr_nclock_atm ) = atm_cpl_dt - dtime(seq_timemgr_nclock_lnd ) = lnd_cpl_dt - dtime(seq_timemgr_nclock_ocn ) = ocn_cpl_dt - dtime(seq_timemgr_nclock_ice ) = ice_cpl_dt - dtime(seq_timemgr_nclock_glc ) = glc_cpl_dt - dtime(seq_timemgr_nclock_rof ) = rof_cpl_dt - dtime(seq_timemgr_nclock_wav ) = wav_cpl_dt - dtime(seq_timemgr_nclock_esp ) = esp_cpl_dt + dtime(seq_timemgr_nclock_atm ) = atm_cpl_dt + dtime(seq_timemgr_nclock_lnd ) = lnd_cpl_dt + dtime(seq_timemgr_nclock_ocn ) = ocn_cpl_dt + dtime(seq_timemgr_nclock_ice ) = ice_cpl_dt + dtime(seq_timemgr_nclock_glc ) = glc_cpl_dt + dtime(seq_timemgr_nclock_rof ) = rof_cpl_dt + dtime(seq_timemgr_nclock_wav ) = wav_cpl_dt + dtime(seq_timemgr_nclock_esp ) = esp_cpl_dt ! --- this finds the min of dtime excluding the driver value --- dtime(seq_timemgr_nclock_drv) = maxval(dtime) dtime(seq_timemgr_nclock_drv) = minval(dtime) + ! --- For figuring pause cycle + min_dt = maxval(dtime) + seq_timemgr_pause_sig_index = -1 + do n = 1,max_clocks if ( mod(dtime(n),dtime(seq_timemgr_nclock_drv)) /= 0) then write(logunit,*) trim(subname),' ERROR: dtime inconsistent = ',dtime call shr_sys_abort( subname//' :coupling intervals not compatible' ) endif + if (pause_active(n) .and. (dtime(n) < min_dt)) then + min_dt = dtime(n) + seq_timemgr_pause_sig_index = n + end if enddo + if (ANY(pause_active)) then + if (seq_timemgr_pause_sig_index < 1) then + write(logunit, *) subname,"ERROR: No pause_sig_index even with active pause" + call shr_sys_abort(subname//"ERROR: No pause_sig_index even with active pause") + end if + else + ! Don't try to run ESP on non-existent pauses + seq_timemgr_esp_run_on_pause = .false. + end if - ! --- Initialize clocks and alarms --- + ! --- Initialize component and driver clocks and alarms common to components and driver clocks --- do n = 1,max_clocks call ESMF_TimeIntervalSet( TimeStep, s=dtime(n), rc=rc ) @@ -739,8 +822,9 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & EAlarm = SyncClock%EAlarm(n,seq_timemgr_nalarm_run), & - option = seq_timemgr_optNSeconds, & - opt_n = dtime(n), RefTime = CurrTime, & + option = seq_timemgr_optNSeconds, & + opt_n = dtime(n), & + RefTime = CurrTime, & alarmname = trim(seq_timemgr_alarm_run)) call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & @@ -757,7 +841,7 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi option = seq_timemgr_optDate, & opt_ymd = stop_ymd, & opt_tod = stop_tod, & - RefTime = StartTime, & + RefTime = StartTime, & alarmname = trim(seq_timemgr_alarm_datestop)) call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & @@ -809,12 +893,21 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi endif ! Set the pause option if pause/resume is active - call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & - EAlarm = SyncClock%EAlarm(n,seq_timemgr_nalarm_pause), & - option = pause_option, & - opt_n = pause_n, & - RefTime = StartTime, & - alarmname = trim(seq_timemgr_alarm_pause)) + if (pause_active(n)) then + call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & + EAlarm = SyncClock%EAlarm(n,seq_timemgr_nalarm_pause), & + option = pause_option, & + opt_n = pause_n, & + RefTime = CurrTime, & + alarmname = trim(seq_timemgr_alarm_pause)) + else + call seq_timemgr_alarmInit(SyncClock%ECP(n)%EClock, & + EAlarm = SyncClock%EAlarm(n,seq_timemgr_nalarm_pause), & + option = seq_timemgr_optNever, & + opt_n = -1, & + RefTime = StartTime, & + alarmname = trim(seq_timemgr_alarm_pause)) + endif enddo @@ -829,15 +922,15 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi ! via namelist. tcraig, 10/2007 ! -------------------------------------------------------------------- - offset(seq_timemgr_nclock_drv) = 0 - offset(seq_timemgr_nclock_atm) = atm_cpl_offset - offset(seq_timemgr_nclock_lnd) = lnd_cpl_offset - offset(seq_timemgr_nclock_ocn) = ocn_cpl_offset - offset(seq_timemgr_nclock_ice) = ice_cpl_offset - offset(seq_timemgr_nclock_glc) = glc_cpl_offset - offset(seq_timemgr_nclock_rof) = rof_cpl_offset - offset(seq_timemgr_nclock_wav) = wav_cpl_offset - offset(seq_timemgr_nclock_esp) = esp_cpl_offset + offset(seq_timemgr_nclock_drv) = 0 + offset(seq_timemgr_nclock_atm) = atm_cpl_offset + offset(seq_timemgr_nclock_lnd) = lnd_cpl_offset + offset(seq_timemgr_nclock_ocn) = ocn_cpl_offset + offset(seq_timemgr_nclock_ice) = ice_cpl_offset + offset(seq_timemgr_nclock_glc) = glc_cpl_offset + offset(seq_timemgr_nclock_rof) = rof_cpl_offset + offset(seq_timemgr_nclock_wav) = wav_cpl_offset + offset(seq_timemgr_nclock_esp) = esp_cpl_offset do n = 1,max_clocks if (abs(offset(n)) > dtime(n)) then @@ -901,16 +994,6 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi RefTime = OffsetTime, & alarmname = trim(seq_timemgr_alarm_wavrun)) - call ESMF_TimeIntervalSet( TimeStep, s=offset(seq_timemgr_nclock_esp), rc=rc ) - OffsetTime = CurrTime + TimeStep - call seq_timemgr_alarmInit(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock, & - EAlarm = SyncClock%EAlarm(seq_timemgr_nclock_drv,seq_timemgr_nalarm_esprun), & - option = seq_timemgr_optNSeconds, & - opt_n = dtime(seq_timemgr_nclock_esp), & - RefTime = OffsetTime, & - alarmname = trim(seq_timemgr_alarm_esprun)) - - ! --- this is the glcrun alarm (there ^) offset by a -dtime of the driver call ESMF_TimeIntervalSet( TimeStep, s=offset(seq_timemgr_nclock_glc), rc=rc ) OffsetTime = CurrTime + TimeStep call ESMF_TimeIntervalSet( TimeStep, s=-offset(seq_timemgr_nclock_drv), rc=rc ) @@ -921,6 +1004,23 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi opt_n = dtime(seq_timemgr_nclock_glc), & RefTime = OffsetTime, & alarmname = trim(seq_timemgr_alarm_glcrun)) + if (glc_avg_period == seq_timemgr_optGLCCouplingPeriod) then + ! Create this alarm identically to the glcrun alarm (which is created above) + call seq_timemgr_alarmInit(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock, & + EAlarm = SyncClock%EAlarm(seq_timemgr_nclock_drv,seq_timemgr_nalarm_glcrun_avg), & + option = seq_timemgr_optNSeconds, & + opt_n = dtime(seq_timemgr_nclock_glc), & + RefTime = OffsetTime, & + alarmname = trim(seq_timemgr_alarm_glcrun_avg)) + else if (glc_avg_period == seq_timemgr_optYearly) then + call seq_timemgr_alarmInit(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock, & + EAlarm = SyncClock%EAlarm(seq_timemgr_nclock_drv,seq_timemgr_nalarm_glcrun_avg), & + option = seq_timemgr_optYearly, & + RefTime = OffsetTime, & + alarmname = trim(seq_timemgr_alarm_glcrun_avg)) + else + call shr_sys_abort(subname//':: glc_avg_period can only be glc_coupling_period or yearly') + end if call ESMF_TimeIntervalSet( TimeStep, s=offset(seq_timemgr_nclock_ocn), rc=rc ) OffsetTime = CurrTime + TimeStep @@ -943,6 +1043,15 @@ subroutine seq_timemgr_clockInit(SyncClock, nmlfile, restart, restart_file, pioi RefTime = OffsetTime, & alarmname = trim(seq_timemgr_alarm_ocnnext)) + call ESMF_TimeIntervalSet( TimeStep, s=offset(seq_timemgr_nclock_esp), rc=rc ) + OffsetTime = CurrTime + TimeStep + call seq_timemgr_alarmInit(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock, & + EAlarm = SyncClock%EAlarm(seq_timemgr_nclock_drv,seq_timemgr_nalarm_esprun), & + option = seq_timemgr_optNSeconds, & + opt_n = dtime(seq_timemgr_nclock_esp), & + RefTime = OffsetTime, & + alarmname = trim(seq_timemgr_alarm_esprun)) + end subroutine seq_timemgr_clockInit !=============================================================================== @@ -991,8 +1100,6 @@ subroutine seq_timemgr_EClockGetData( EClock, curr_yr, curr_mon, curr_day, & ! and reference date character(len=*) , intent(OUT), optional :: calendar ! calendar type -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_EClockGetData) ' type(ESMF_Time) :: CurrentTime ! Current time @@ -1008,9 +1115,6 @@ subroutine seq_timemgr_EClockGetData( EClock, curr_yr, curr_mon, curr_day, & integer(SHR_KIND_IN) :: ymd ! Date (YYYYMMDD) integer(SHR_KIND_IN) :: tod ! time of day (sec) integer(SHR_KIND_IN) :: ldtime ! local dtime - integer(SHR_KIND_IN) :: intyrs ! alarm variable - integer(SHR_KIND_IN) :: intmon ! alarm variable - integer(SHR_KIND_IN) :: intsec ! alarm variable integer(SHR_KIND_IN) :: days ! number of whole days in time interval integer(SHR_KIND_IN) :: seconds ! number of seconds in time interval integer(SHR_KIND_IN) :: acount ! number of valid alarms @@ -1124,8 +1228,6 @@ subroutine seq_timemgr_clockAdvance( SyncClock, force_stop, force_stop_ymd, forc integer, optional, intent(in) :: force_stop_ymd ! force stop ymd integer, optional, intent(in) :: force_stop_tod ! force stop tod -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_clockAdvance) ' integer :: n @@ -1205,6 +1307,16 @@ subroutine seq_timemgr_clockAdvance( SyncClock, force_stop, force_stop_ymd, forc call seq_timemgr_ESMFCodeCheck(rc, msg=subname//"Error from esp ESMF_ClockAdvance") endif + ! Special handling of ESP component if linked to pause cycles + if (seq_timemgr_esp_run_on_pause) then + ! We need to figure out if any pause clock is ringing + call seq_timemgr_alarmSetOff(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock,seq_timemgr_alarm_esprun) + if (seq_timemgr_alarmIsOn(SyncClock%ECP(seq_timemgr_pause_sig_index)%EClock,seq_timemgr_alarm_pause)) then + call seq_timemgr_alarmSetOn(SyncClock%ECP(seq_timemgr_nclock_drv)%EClock,seq_timemgr_alarm_esprun) + end if + end if + + ! Special handling of restart alarm if end_restart is .true. if (seq_timemgr_end_restart) then do n = 1,max_clocks if (seq_timemgr_alarmIsOn(SyncClock%ECP(n)%EClock,seq_timemgr_alarm_stop) .or. & @@ -1241,8 +1353,6 @@ subroutine seq_timemgr_alarmInit( EClock, EAlarm, option, opt_n, opt_ymd, opt_to type(ESMF_Time) ,optional, intent(IN) :: RefTime ! ref time character(len=*) ,optional, intent(IN) :: alarmname ! alarm name -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_alarmInit): ' integer :: rc ! Return code @@ -1254,7 +1364,6 @@ subroutine seq_timemgr_alarmInit( EClock, EAlarm, option, opt_n, opt_ymd, opt_to logical :: update_nextalarm ! update next alarm type(ESMF_Time) :: CurrTime ! Current Time type(ESMF_Time) :: NextAlarm ! Next restart alarm time - type(ESMF_Time) :: AltAlarm ! Alternate alarm time type(ESMF_TimeInterval) :: AlarmInterval ! Alarm interval !------------------------------------------------------------------------------- @@ -1485,8 +1594,6 @@ subroutine seq_timemgr_alarmGet( EAlarm, next_ymd, next_tod, prev_ymd, prev_tod, integer(SHR_KIND_IN), intent(OUT), optional :: IntYrs ! alarm int yrs character(len=*) , intent(OUT), optional :: name ! alarm name -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_alarmGet) ' integer :: yy, mm, dd, sec ! Return time values @@ -1552,8 +1659,6 @@ subroutine seq_timemgr_AlarmSetOn( EClock, alarmname) type(ESMF_Clock), intent(INOUT) :: EClock ! clock/alarm character(len=*), intent(IN), optional :: alarmname ! alarmname -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_alarmSetOn) ' character(len=*), parameter :: xalarm = 'unset' @@ -1562,7 +1667,6 @@ subroutine seq_timemgr_AlarmSetOn( EClock, alarmname) logical :: found logical :: set character(len=64) :: name - type(ESMF_Alarm),pointer :: EAlarm type(ESMF_Alarm),pointer :: EAlarm_list(:) integer(SHR_KIND_IN) :: AlarmCount ! Number of valid alarms @@ -1636,8 +1740,6 @@ subroutine seq_timemgr_AlarmSetOff( EClock, alarmname) type(ESMF_Clock), intent(INOUT) :: EClock ! clock/alarm character(len=*), intent(IN), optional :: alarmname ! alarmname -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_alarmSetOff) ' character(len=*), parameter :: xalarm = 'unset' @@ -1646,7 +1748,6 @@ subroutine seq_timemgr_AlarmSetOff( EClock, alarmname) logical :: found logical :: set character(len=64) :: name - type(ESMF_Alarm),pointer :: EAlarm type(ESMF_Alarm),pointer :: EAlarm_list(:) integer(SHR_KIND_IN) :: AlarmCount ! Number of valid alarms @@ -1720,18 +1821,14 @@ logical function seq_timemgr_alarmIsOn( EClock, alarmname) type(ESMF_Clock), intent(IN) :: EClock ! clock/alarm character(len=*), intent(IN) :: alarmname ! which alarm -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_alarmIsOn) ' character(len=*), parameter :: xalarm = 'unset' integer :: n integer :: rc logical :: found - logical :: set character(len=64) :: name type(ESMF_Time) :: ETime1, ETime2 - type(ESMF_Alarm),pointer :: EAlarm type(ESMF_Alarm),pointer :: EAlarm_list(:) integer(SHR_KIND_IN) :: AlarmCount ! Number of valid alarms @@ -1803,10 +1900,7 @@ logical function seq_timemgr_restartAlarmIsOn( EClock) type(ESMF_Clock) , intent(IN) :: EClock ! clock/alarm -!EOP - !----- local ----- - integer :: rc character(len=*), parameter :: subname = '(seq_timemgr_restartAlarmIsOn) ' !------------------------------------------------------------------------------- @@ -1836,8 +1930,6 @@ logical function seq_timemgr_stopAlarmIsOn( EClock) type(ESMF_Clock) , intent(IN) :: EClock ! clock/alarm -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_stopAlarmIsOn) ' @@ -1868,10 +1960,7 @@ logical function seq_timemgr_historyAlarmIsOn( EClock) type(ESMF_Clock) , intent(IN) :: EClock ! clock/alarm -!EOP - !----- local ----- - integer :: rc character(len=*), parameter :: subname = '(seq_timemgr_historyAlarmIsOn) ' !------------------------------------------------------------------------------- @@ -1883,6 +1972,137 @@ logical function seq_timemgr_historyAlarmIsOn( EClock) end function seq_timemgr_historyAlarmIsOn +!=============================================================================== +!=============================================================================== +! !IROUTINE: seq_timemgr_pauseAlarmIsOn -- check if an alarm is ringing +! +! !DESCRIPTION: +! +! check if an alarm is ringing +! +! !INTERFACE: ------------------------------------------------------------------ + +logical function seq_timemgr_pauseAlarmIsOn( EClock) + + implicit none + +! !INPUT/OUTPUT PARAMETERS: + + type(ESMF_Clock) , intent(IN) :: EClock ! clock/alarm + +!EOP + + !----- local ----- + character(len=*), parameter :: subname = '(seq_timemgr_pauseAlarmIsOn) ' + +!------------------------------------------------------------------------------- +! Notes: +!------------------------------------------------------------------------------- + + seq_timemgr_pauseAlarmIsOn = & + seq_timemgr_alarmIsOn(EClock, alarmname=seq_timemgr_alarm_pause) + + end function seq_timemgr_pauseAlarmIsOn + +!=============================================================================== +!=============================================================================== +! !IROUTINE: seq_timemgr_pause_active -- Is pause/resume active this run? +! +! !DESCRIPTION: +! +! Return .true. if any component is configured for pause/resume +! +! !INTERFACE: ------------------------------------------------------------------ + + logical function seq_timemgr_pause_active() + +! !INPUT/OUTPUT PARAMETERS: + +!EOP + +!------------------------------------------------------------------------------- +! Notes: +!------------------------------------------------------------------------------- + + seq_timemgr_pause_active = ANY(pause_active) + + end function seq_timemgr_pause_active + +!=============================================================================== +!=============================================================================== +! !IROUTINE: seq_timemgr_pause_component_index -- return an index for a component +! +! !DESCRIPTION: +! +! Look up a component's internal index for faster processing +! +! !INTERFACE: ------------------------------------------------------------------ + + integer function seq_timemgr_pause_component_index(component_name) + +! !INPUT/OUTPUT PARAMETERS: + + character(len=*), intent(IN) :: component_name + +!EOP + + !----- local ----- + integer :: ind + character(len=*), parameter :: subname = '(seq_timemgr_pause_component_index) ' + +!------------------------------------------------------------------------------- +! Notes: +!------------------------------------------------------------------------------- + seq_timemgr_pause_component_index = 0 + do ind = 1, max_clocks + if (trim(component_name) == trim(seq_timemgr_clocks(ind))) then + seq_timemgr_pause_component_index = ind + exit + end if + end do + if (seq_timemgr_pause_component_index < 1) then + if (trim(component_name) == 'cpl') then + seq_timemgr_pause_component_index = seq_timemgr_nclock_drv + end if + end if + if (seq_timemgr_pause_component_index < 1) then + call shr_sys_abort(subname//': No index for component '//trim(component_name)) + end if + + end function seq_timemgr_pause_component_index + +!=============================================================================== +!=============================================================================== +! !IROUTINE: seq_timemgr_pause_component_active -- Check if component paused +! +! !DESCRIPTION: +! +! Return .true. if component is active in driver pause +! +! !INTERFACE: ------------------------------------------------------------------ + + logical function seq_timemgr_pause_component_active(component_index) + +! !INPUT/OUTPUT PARAMETERS: + + integer, intent(IN) :: component_index + +!EOP + + !----- local ----- + character(len=*), parameter :: subname = '(seq_timemgr_pause_component_active) ' + +!------------------------------------------------------------------------------- +! Notes: +!------------------------------------------------------------------------------- + + if ((component_index < 1) .or. (component_index > max_clocks)) then + call shr_sys_abort(subname//': component_index out of range') + end if + seq_timemgr_pause_component_active = pause_active(component_index) + + end function seq_timemgr_pause_component_active + !=============================================================================== !=============================================================================== ! !IROUTINE: seq_timemgr_ETimeInit -- Create ESMF_Time object based on YMD values @@ -1906,8 +2126,6 @@ subroutine seq_timemgr_ETimeInit( ETime, ymd, tod, desc ) integer , intent(in), optional :: tod ! Time of day in seconds character(len=*), intent(in), optional :: desc ! Description of time to set -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_ETimeInit) ' integer :: yr, mon, day ! Year, month, day as integers @@ -1965,8 +2183,6 @@ subroutine seq_timemgr_ETimeGet( ETime, offset, ymd, tod ) integer, optional, intent(OUT) :: ymd ! date of day integer, optional, intent(OUT) :: tod ! Time of day -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_ETimeGet) ' type(ESMF_Time) :: ETimeAdd ! ESMF time + offset @@ -2035,8 +2251,6 @@ subroutine seq_timemgr_EClockInit( TimeStep, StartTime, RefTime, CurrTime, ECloc type(ESMF_Time) , intent(IN) :: CurrTime ! Current time type(ESMF_Clock) , intent(OUT) :: EClock ! Output ESMF clock -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_EClockInit) ' integer :: rc ! ESMF return code @@ -2098,8 +2312,6 @@ logical function seq_timemgr_EClockDateInSync( EClock, ymd, tod, prev) integer, intent(IN) :: tod ! Time of day (sec) logical, optional,intent(IN) :: prev ! If should get previous time -!EOP - !----- local ----- character(len=*), parameter :: subname = "(seq_timemgr_EClockDateInSync) " type(ESMF_Time) :: ETime @@ -2150,9 +2362,7 @@ subroutine seq_timemgr_clockPrint( SyncClock ) ! !INPUT/OUTPUT PARAMETERS: - type(seq_timemgr_type), intent(in) :: SyncClock ! Input clock to print - -!EOP + type(seq_timemgr_type), intent(in) :: SyncClock ! Input clock to print character(len=*), parameter :: subname = "(seq_timemgr_clockPrint) " integer(SHR_KIND_IN) :: m,n @@ -2258,8 +2468,6 @@ subroutine seq_timemgr_ESMFDebug( EClock, ETime, ETimeInterval, istring ) type(ESMF_TimeInterval), optional, intent(inout) :: ETimeInterval ! ESMF Time Interval character(len=*), optional, intent(in) :: istring -!EOP - !----- local ----- character(len=*), parameter :: subname = '(seq_timemgr_ESMFDebug) ' character(len=128) :: timestring @@ -2332,8 +2540,6 @@ subroutine seq_timemgr_ESMFCodeCheck( rc, msg ) integer, intent(in) :: rc ! return code from ESMF character(len=*),optional,intent(in) :: msg ! error message -!EOP - character(len=*),parameter :: subname = 'seq_timemgr_ESMFCodeCheck' !------------------------------------------------------------------------------- ! Notes: diff --git a/driver_cpl/shr/shr_carma_mod.F90 b/src/drivers/mct/shr/shr_carma_mod.F90 similarity index 100% rename from driver_cpl/shr/shr_carma_mod.F90 rename to src/drivers/mct/shr/shr_carma_mod.F90 diff --git a/driver_cpl/shr/shr_expr_parser_mod.F90 b/src/drivers/mct/shr/shr_expr_parser_mod.F90 similarity index 100% rename from driver_cpl/shr/shr_expr_parser_mod.F90 rename to src/drivers/mct/shr/shr_expr_parser_mod.F90 diff --git a/driver_cpl/shr/shr_fire_emis_mod.F90 b/src/drivers/mct/shr/shr_fire_emis_mod.F90 similarity index 100% rename from driver_cpl/shr/shr_fire_emis_mod.F90 rename to src/drivers/mct/shr/shr_fire_emis_mod.F90 diff --git a/driver_cpl/shr/shr_megan_mod.F90 b/src/drivers/mct/shr/shr_megan_mod.F90 similarity index 100% rename from driver_cpl/shr/shr_megan_mod.F90 rename to src/drivers/mct/shr/shr_megan_mod.F90 diff --git a/src/drivers/mct/shr/shr_ndep_mod.F90 b/src/drivers/mct/shr/shr_ndep_mod.F90 new file mode 100644 index 00000000000..64cadf107f3 --- /dev/null +++ b/src/drivers/mct/shr/shr_ndep_mod.F90 @@ -0,0 +1,117 @@ +module shr_ndep_mod + + !======================================================================== + ! Module for handling nitrogen depostion of tracers. + ! This module is shared by land and atmosphere models for the computations of + ! dry deposition of tracers + !======================================================================== + + !USES: + use shr_sys_mod, only : shr_sys_abort + use shr_log_mod, only : s_loglev => shr_log_Level + use shr_kind_mod, only : r8 => shr_kind_r8, CS => SHR_KIND_CS, CX => SHR_KIND_CX + + implicit none + save + + private + + ! !PUBLIC MEMBER FUNCTIONS + public :: shr_ndep_readnl ! Read namelist + !==================================================================================== + +CONTAINS + + !==================================================================================== + subroutine shr_ndep_readnl(NLFilename, ID, ndep_fields, add_ndep_fields) + + !======================================================================== + ! reads ndep_inparm namelist and sets up driver list of fields for + ! atmosphere -> land and atmosphere -> ocn communications. + !======================================================================== + + use shr_file_mod , only : shr_file_getUnit, shr_file_freeUnit + use shr_log_mod , only : s_logunit => shr_log_Unit + use seq_comm_mct , only : seq_comm_iamroot, seq_comm_setptrs + use shr_mpi_mod , only : shr_mpi_bcast + use shr_nl_mod , only : shr_nl_find_group_name + implicit none + + character(len=*), intent(in) :: NLFilename ! Namelist filename + integer , intent(in) :: ID ! seq_comm ID + character(len=*), intent(out) :: ndep_fields + logical , intent(out) :: add_ndep_fields + + !----- local ----- + integer :: i ! Indices + integer :: unitn ! namelist unit number + integer :: ierr ! error code + logical :: exists ! if file exists or not + character(len=8) :: token ! dry dep field name to add + integer :: mpicom ! MPI communicator + + integer, parameter :: maxspc = 100 ! Maximum number of species + character(len=32) :: ndep_list(maxspc) = '' ! List of ndep species + + !----- formats ----- + character(*),parameter :: subName = '(shr_ndep_read) ' + character(*),parameter :: F00 = "('(shr_ndep_read) ',8a)" + character(*),parameter :: FI1 = "('(shr_ndep_init) ',a,I2)" + + namelist /ndep_inparm/ ndep_list + + !----------------------------------------------------------------------------- + ! Read namelist and figure out the ndep field list to pass + ! First check if file exists and if not, n_ndep will be zero + !----------------------------------------------------------------------------- + + !--- Open and read namelist --- + if ( len_trim(NLFilename) == 0 ) then + call shr_sys_abort( subName//'ERROR: nlfilename not set' ) + end if + call seq_comm_setptrs(ID,mpicom=mpicom) + if (seq_comm_iamroot(ID)) then + inquire( file=trim(NLFileName), exist=exists) + if ( exists ) then + unitn = shr_file_getUnit() + open( unitn, file=trim(NLFilename), status='old' ) + if ( s_loglev > 0 ) then + write(s_logunit,F00) 'Read in ndep_inparm namelist from: ', trim(NLFilename) + end if + call shr_nl_find_group_name(unitn, 'ndep_inparm', ierr) + if (ierr == 0) then + ierr = 1 + do while ( ierr /= 0 ) + read(unitn, ndep_inparm, iostat=ierr) + if (ierr < 0) then + call shr_sys_abort( subName//'ERROR: encountered end-of-file on namelist read' ) + endif + end do + close( unitn ) + else + write(s_logunit,*) 'shr_ndep_readnl: no ndep_inparm namelist found in ',NLFilename + endif + call shr_file_freeUnit( unitn ) + end if + end if + call shr_mpi_bcast( ndep_list, mpicom ) + + ndep_fields = ' ' + if (len_trim(ndep_list(1)) == 0) then + add_ndep_fields = .false. + else + ! Loop over species to fill list of fields to communicate for ndep + add_ndep_fields = .true. + do i=1,maxspc + if ( len_trim(ndep_list(i))==0 ) exit + if ( i == 1 ) then + ndep_fields = 'Faxa_' // trim(ndep_list(i)) + else + ndep_fields = trim(ndep_fields)//':'//'Faxa_' // trim(ndep_list(i)) + endif + enddo + end if + + end subroutine shr_ndep_readnl + +end module shr_ndep_mod diff --git a/driver_cpl/unit_test/CMakeLists.txt b/src/drivers/mct/unit_test/CMakeLists.txt similarity index 81% rename from driver_cpl/unit_test/CMakeLists.txt rename to src/drivers/mct/unit_test/CMakeLists.txt index 490759d1973..a747f9e23a2 100644 --- a/driver_cpl/unit_test/CMakeLists.txt +++ b/src/drivers/mct/unit_test/CMakeLists.txt @@ -1,4 +1,4 @@ -set(DRV_ROOT "${CIME_ROOT}/driver_cpl") +set(DRV_ROOT "${CIME_ROOT}/src/drivers/mct") add_definitions( -DNUM_COMP_INST_ATM=1 @@ -11,20 +11,25 @@ add_definitions( -DNUM_COMP_INST_ESP=1 ) +# The following definitions are needed when building with the mpi-serial library +if (USE_MPI_SERIAL) + add_definitions(-DNO_MPI2 -DNO_MPIMOD) +endif() + # Add source directories from stubs. This should be done first, so that in the # case of name collisions, the drv versions take precedence (when there are two # files with the same name, the one added later wins). -add_subdirectory(${CIME_ROOT}/share/csm_share/unit_test_stubs/pio pio) +add_subdirectory(${CIME_ROOT}/src/share/unit_test_stubs/pio pio) # Add drv source directories add_subdirectory(${DRV_ROOT}/shr drv_shr) -add_subdirectory(${DRV_ROOT}/driver drv_driver) +add_subdirectory(${DRV_ROOT}/main drv_main) # Add general unit test directories (stubbed out files, etc.) add_subdirectory(utils drv_unit_test_utils) add_subdirectory(stubs drv_unit_test_stubs) -# Build libraries containing stuff needed for the unit tests. +# Build libraries containing stuff needed for the unit tests. # Eventually, these add_library calls should probably be distributed into the # correct location, rather than being in this top-level CMakeLists.txt file. # Note that we are including the stub pio in the csm_share library for simplicity. @@ -46,6 +51,9 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) # tests need all of these libraries, but it's easiest just to set the same list # for everyone. set(DRV_UNIT_TEST_LIBS drv;csm_share;esmf_wrf_timemgr;mct;mpeu) +if (USE_MPI_SERIAL) + list(APPEND DRV_UNIT_TEST_LIBS mpi-serial) +endif() list(APPEND DRV_UNIT_TEST_LIBS ${NETCDF_LIBRARIES}) # Add the test directories diff --git a/driver_cpl/unit_test/avect_wrapper_test/CMakeLists.txt b/src/drivers/mct/unit_test/avect_wrapper_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/avect_wrapper_test/CMakeLists.txt rename to src/drivers/mct/unit_test/avect_wrapper_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/avect_wrapper_test/test_avect_wrapper.pf b/src/drivers/mct/unit_test/avect_wrapper_test/test_avect_wrapper.pf similarity index 100% rename from driver_cpl/unit_test/avect_wrapper_test/test_avect_wrapper.pf rename to src/drivers/mct/unit_test/avect_wrapper_test/test_avect_wrapper.pf diff --git a/driver_cpl/unit_test/glc_elevclass_test/CMakeLists.txt b/src/drivers/mct/unit_test/glc_elevclass_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/glc_elevclass_test/CMakeLists.txt rename to src/drivers/mct/unit_test/glc_elevclass_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/glc_elevclass_test/test_glc_elevclass.pf b/src/drivers/mct/unit_test/glc_elevclass_test/test_glc_elevclass.pf similarity index 100% rename from driver_cpl/unit_test/glc_elevclass_test/test_glc_elevclass.pf rename to src/drivers/mct/unit_test/glc_elevclass_test/test_glc_elevclass.pf diff --git a/driver_cpl/unit_test/map_glc2lnd_test/CMakeLists.txt b/src/drivers/mct/unit_test/map_glc2lnd_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/map_glc2lnd_test/CMakeLists.txt rename to src/drivers/mct/unit_test/map_glc2lnd_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf b/src/drivers/mct/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf similarity index 100% rename from driver_cpl/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf rename to src/drivers/mct/unit_test/map_glc2lnd_test/test_map_glc2lnd.pf diff --git a/driver_cpl/unit_test/map_lnd2glc_test/CMakeLists.txt b/src/drivers/mct/unit_test/map_lnd2glc_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/map_lnd2glc_test/CMakeLists.txt rename to src/drivers/mct/unit_test/map_lnd2glc_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf b/src/drivers/mct/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf similarity index 100% rename from driver_cpl/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf rename to src/drivers/mct/unit_test/map_lnd2glc_test/test_map_lnd2glc.pf diff --git a/driver_cpl/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt b/src/drivers/mct/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt rename to src/drivers/mct/unit_test/map_lnd2rof_irrig_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf b/src/drivers/mct/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf similarity index 96% rename from driver_cpl/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf rename to src/drivers/mct/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf index a491d5e227b..f50bf8f3b1a 100644 --- a/driver_cpl/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf +++ b/src/drivers/mct/unit_test/map_lnd2rof_irrig_test/test_map_lnd2rof_irrig.pf @@ -73,10 +73,10 @@ contains nlnd = map_l2r%get_n_source_points() nrof = map_l2r%get_n_dest_points() - call shr_assert(map_r2l%get_n_dest_points() == nlnd, errMsg(sourcefile, __LINE__)) - call shr_assert(map_r2l%get_n_source_points() == nrof, errMsg(sourcefile, __LINE__)) - call shr_assert(size(irrig_l) == nlnd, errMsg(sourcefile, __LINE__)) - call shr_assert(size(volr_r) == nrof, errMsg(sourcefile, __LINE__)) + call shr_assert(map_r2l%get_n_dest_points() == nlnd, file=sourcefile, line=__LINE__) + call shr_assert(map_r2l%get_n_source_points() == nrof, file=sourcefile, line=__LINE__) + call shr_assert(size(irrig_l) == nlnd, file=sourcefile, line=__LINE__) + call shr_assert(size(volr_r) == nrof, file=sourcefile, line=__LINE__) call create_aVect_with_data_rows_are_points(this%l2r_l, & attr_tags = [irrig_flux_field], & diff --git a/driver_cpl/unit_test/seq_map_test/CMakeLists.txt b/src/drivers/mct/unit_test/seq_map_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/seq_map_test/CMakeLists.txt rename to src/drivers/mct/unit_test/seq_map_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/seq_map_test/test_seq_map.pf b/src/drivers/mct/unit_test/seq_map_test/test_seq_map.pf similarity index 100% rename from driver_cpl/unit_test/seq_map_test/test_seq_map.pf rename to src/drivers/mct/unit_test/seq_map_test/test_seq_map.pf diff --git a/driver_cpl/unit_test/stubs/CMakeLists.txt b/src/drivers/mct/unit_test/stubs/CMakeLists.txt similarity index 82% rename from driver_cpl/unit_test/stubs/CMakeLists.txt rename to src/drivers/mct/unit_test/stubs/CMakeLists.txt index 4e0db12da1a..a3097917ed9 100644 --- a/driver_cpl/unit_test/stubs/CMakeLists.txt +++ b/src/drivers/mct/unit_test/stubs/CMakeLists.txt @@ -1,4 +1,5 @@ list(APPEND drv_sources + seq_timemgr_mod.F90 vertical_gradient_calculator_constant.F90 ) diff --git a/src/drivers/mct/unit_test/stubs/seq_timemgr_mod.F90 b/src/drivers/mct/unit_test/stubs/seq_timemgr_mod.F90 new file mode 100644 index 00000000000..f88a96d2da8 --- /dev/null +++ b/src/drivers/mct/unit_test/stubs/seq_timemgr_mod.F90 @@ -0,0 +1,19 @@ +module seq_timemgr_mod + + ! Stub for routines from seq_timemgr_mod that are needed by other modules built by the + ! unit tests. + + implicit none + private + + public :: seq_timemgr_pause_active + +contains + + logical function seq_timemgr_pause_active() + ! Stub for seq_timemgr_pause_active - always returns .false. + + seq_timemgr_pause_active = .false. + end function seq_timemgr_pause_active + +end module seq_timemgr_mod diff --git a/driver_cpl/unit_test/stubs/vertical_gradient_calculator_constant.F90 b/src/drivers/mct/unit_test/stubs/vertical_gradient_calculator_constant.F90 similarity index 100% rename from driver_cpl/unit_test/stubs/vertical_gradient_calculator_constant.F90 rename to src/drivers/mct/unit_test/stubs/vertical_gradient_calculator_constant.F90 diff --git a/driver_cpl/unit_test/utils/CMakeLists.txt b/src/drivers/mct/unit_test/utils/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/utils/CMakeLists.txt rename to src/drivers/mct/unit_test/utils/CMakeLists.txt diff --git a/driver_cpl/unit_test/utils/avect_wrapper_mod.F90 b/src/drivers/mct/unit_test/utils/avect_wrapper_mod.F90 similarity index 100% rename from driver_cpl/unit_test/utils/avect_wrapper_mod.F90 rename to src/drivers/mct/unit_test/utils/avect_wrapper_mod.F90 diff --git a/driver_cpl/unit_test/utils/create_mapper_mod.F90 b/src/drivers/mct/unit_test/utils/create_mapper_mod.F90 similarity index 100% rename from driver_cpl/unit_test/utils/create_mapper_mod.F90 rename to src/drivers/mct/unit_test/utils/create_mapper_mod.F90 diff --git a/driver_cpl/unit_test/utils/mct_wrapper_mod.F90 b/src/drivers/mct/unit_test/utils/mct_wrapper_mod.F90 similarity index 100% rename from driver_cpl/unit_test/utils/mct_wrapper_mod.F90 rename to src/drivers/mct/unit_test/utils/mct_wrapper_mod.F90 diff --git a/driver_cpl/unit_test/utils/simple_map_mod.F90 b/src/drivers/mct/unit_test/utils/simple_map_mod.F90 similarity index 98% rename from driver_cpl/unit_test/utils/simple_map_mod.F90 rename to src/drivers/mct/unit_test/utils/simple_map_mod.F90 index 8738a86ef6f..485dca44481 100644 --- a/driver_cpl/unit_test/utils/simple_map_mod.F90 +++ b/src/drivers/mct/unit_test/utils/simple_map_mod.F90 @@ -66,8 +66,8 @@ function constructor(source_indices, dest_indices, overlap_weights) result(this) !----------------------------------------------------------------------- n_overlaps = size(overlap_weights) - call shr_assert(size(source_indices) == n_overlaps, errMsg(__FILE__, __LINE__)) - call shr_assert(size(dest_indices) == n_overlaps, errMsg(__FILE__, __LINE__)) + call shr_assert(size(source_indices) == n_overlaps, file=__FILE__, line=__LINE__) + call shr_assert(size(dest_indices) == n_overlaps, file=__FILE__, line=__LINE__) this%n_overlaps = n_overlaps this%source_indices = source_indices diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/CMakeLists.txt b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/CMakeLists.txt similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/CMakeLists.txt rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/CMakeLists.txt diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/README b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/README similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/README rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/README diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/gradient_example.txt b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/gradient_example.txt similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/gradient_example.txt rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/gradient_example.txt diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/plot_gradient b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/plot_gradient similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/plot_gradient rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/plot_gradient diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_2nd_order.pf diff --git a/driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf b/src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf similarity index 100% rename from driver_cpl/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf rename to src/drivers/mct/unit_test/vertical_gradient_calculator_test/test_vertical_gradient_calculator_factory.pf diff --git a/externals/CMake/.gitignore b/src/externals/CMake/.gitignore similarity index 100% rename from externals/CMake/.gitignore rename to src/externals/CMake/.gitignore diff --git a/externals/CMake/CESM_utils.cmake b/src/externals/CMake/CESM_utils.cmake similarity index 100% rename from externals/CMake/CESM_utils.cmake rename to src/externals/CMake/CESM_utils.cmake diff --git a/src/externals/CMake/CIME_initial_setup.cmake b/src/externals/CMake/CIME_initial_setup.cmake new file mode 100644 index 00000000000..c3e83317574 --- /dev/null +++ b/src/externals/CMake/CIME_initial_setup.cmake @@ -0,0 +1,16 @@ +# Module used for CIME testing. +# +# This module does some initial setup that must be done BEFORE the 'project' +# line in the main CMakeLists.txt file. + +include(${CMAKE_BINARY_DIR}/Macros.cmake RESULT_VARIABLE FOUND) +if(NOT FOUND) + message(FATAL_ERROR "You must generate a Macros.cmake file using CIME's configure") +endif() +if("$ENV{MPILIB}" STREQUAL "mpi-serial") + set(CMAKE_C_COMPILER ${SCC}) + set(CMAKE_Fortran_COMPILER ${SFC}) +else() + set(CMAKE_C_COMPILER ${MPICC}) + set(CMAKE_Fortran_COMPILER ${MPIFC}) +endif() diff --git a/externals/CMake/CIME_utils.cmake b/src/externals/CMake/CIME_utils.cmake similarity index 87% rename from externals/CMake/CIME_utils.cmake rename to src/externals/CMake/CIME_utils.cmake index eeca4ae464f..bd0f254b1d8 100644 --- a/externals/CMake/CIME_utils.cmake +++ b/src/externals/CMake/CIME_utils.cmake @@ -3,6 +3,10 @@ # This module contains statements that would otherwise be boilerplate in # most CIME tests. It enables CTest testing, handles the USE_COLOR and # ENABLE_GENF90 arguments, and includes several other modules. +# +# Some of the things done here must be done AFTER the 'project' line in the main +# CMakeLists.txt file. This assumes that CIME_initial_setup has already been +# included. #========================================================================== # Copyright (c) 2013-2014, University Corporation for Atmospheric Research @@ -30,16 +34,10 @@ set(CMAKE_COLOR_MAKEFILE "${USE_COLOR}") # Compiler info #================================================= -include(${CMAKE_BINARY_DIR}/Macros.cmake RESULT_VARIABLE FOUND) list(APPEND CMAKE_MODULE_PATH "../pio2/cmake") -set(CMAKE_C_COMPILER ${MPICC}) -set(CMAKE_Fortran_COMPILER ${MPIF90}) set(CMAKE_C_FLAGS "${CPPDEFS} ${CFLAGS}") set(CMAKE_Fortran_FLAGS "${CPPDEFS} ${FFLAGS}") -set (CMAKE_EXE_LINKER_FLAGS ${LDFLAGS}) -if(NOT FOUND) - message(FATAL_ERROR "You must generate a Macros.cmake file using CIME's configure") -endif() +set(CMAKE_EXE_LINKER_FLAGS "${LDFLAGS} ${SLIBS}") include(Compilers) diff --git a/externals/CMake/ChangeLog b/src/externals/CMake/ChangeLog similarity index 100% rename from externals/CMake/ChangeLog rename to src/externals/CMake/ChangeLog diff --git a/externals/CMake/Compilers.cmake b/src/externals/CMake/Compilers.cmake similarity index 100% rename from externals/CMake/Compilers.cmake rename to src/externals/CMake/Compilers.cmake diff --git a/externals/CMake/FindpFUnit.cmake b/src/externals/CMake/FindpFUnit.cmake similarity index 100% rename from externals/CMake/FindpFUnit.cmake rename to src/externals/CMake/FindpFUnit.cmake diff --git a/externals/CMake/LICENSE b/src/externals/CMake/LICENSE similarity index 100% rename from externals/CMake/LICENSE rename to src/externals/CMake/LICENSE diff --git a/externals/CMake/README.md b/src/externals/CMake/README.md similarity index 74% rename from externals/CMake/README.md rename to src/externals/CMake/README.md index 21d36fb3be6..ae583206b0e 100644 --- a/externals/CMake/README.md +++ b/src/externals/CMake/README.md @@ -28,6 +28,12 @@ Sourcelist_utils - Use source file lists defined over multiple directories. Modules that are CESM-specific and/or incomplete: -CIME_utils - Handles a few options, and includes several other modules. +CIME\_initial\_setup - Handles setup that must be done before the 'project' +line. This must be included before the 'project' line in the main CMakeLists.txt +file. + +CIME_utils - Handles a few options, and includes several other modules. This +must be included after the 'project' line in the main CMakeLists.txt file, and +after the inclusion of CIME\_initial\_setup. Compilers - Specify compiler-specific behavior, add build types for CESM. diff --git a/externals/CMake/Sourcelist_utils.cmake b/src/externals/CMake/Sourcelist_utils.cmake similarity index 100% rename from externals/CMake/Sourcelist_utils.cmake rename to src/externals/CMake/Sourcelist_utils.cmake diff --git a/externals/CMake/TryCSizeOf.f90 b/src/externals/CMake/TryCSizeOf.f90 similarity index 100% rename from externals/CMake/TryCSizeOf.f90 rename to src/externals/CMake/TryCSizeOf.f90 diff --git a/externals/CMake/TryMPIIO.f90 b/src/externals/CMake/TryMPIIO.f90 similarity index 100% rename from externals/CMake/TryMPIIO.f90 rename to src/externals/CMake/TryMPIIO.f90 diff --git a/externals/CMake/TryMPIMod.f90 b/src/externals/CMake/TryMPIMod.f90 similarity index 100% rename from externals/CMake/TryMPIMod.f90 rename to src/externals/CMake/TryMPIMod.f90 diff --git a/externals/CMake/TryMPISERIAL.f90 b/src/externals/CMake/TryMPISERIAL.f90 similarity index 100% rename from externals/CMake/TryMPISERIAL.f90 rename to src/externals/CMake/TryMPISERIAL.f90 diff --git a/externals/CMake/TryPnetcdf_inc.f90 b/src/externals/CMake/TryPnetcdf_inc.f90 similarity index 100% rename from externals/CMake/TryPnetcdf_inc.f90 rename to src/externals/CMake/TryPnetcdf_inc.f90 diff --git a/externals/CMake/TryPnetcdf_mod.f90 b/src/externals/CMake/TryPnetcdf_mod.f90 similarity index 100% rename from externals/CMake/TryPnetcdf_mod.f90 rename to src/externals/CMake/TryPnetcdf_mod.f90 diff --git a/externals/CMake/genf90_utils.cmake b/src/externals/CMake/genf90_utils.cmake similarity index 100% rename from externals/CMake/genf90_utils.cmake rename to src/externals/CMake/genf90_utils.cmake diff --git a/externals/CMake/mpiexec.cmake b/src/externals/CMake/mpiexec.cmake similarity index 100% rename from externals/CMake/mpiexec.cmake rename to src/externals/CMake/mpiexec.cmake diff --git a/externals/CMake/pFUnit_utils.cmake b/src/externals/CMake/pFUnit_utils.cmake similarity index 98% rename from externals/CMake/pFUnit_utils.cmake rename to src/externals/CMake/pFUnit_utils.cmake index 0089e4989f4..2accabe20e1 100644 --- a/externals/CMake/pFUnit_utils.cmake +++ b/src/externals/CMake/pFUnit_utils.cmake @@ -215,7 +215,8 @@ function(create_pFUnit_test test_name executable_name pf_file_list fortran_sourc endif() # Prefix command with an mpirun command - set (MY_COMMAND ${PFUNIT_MPIRUN} ${MY_COMMAND}) + separate_arguments(PFUNIT_MPIRUN_LIST UNIX_COMMAND ${PFUNIT_MPIRUN}) + set (MY_COMMAND ${PFUNIT_MPIRUN_LIST} ${MY_COMMAND}) # Do the work add_pFUnit_executable(${executable_name} "${pf_file_list}" diff --git a/externals/genf90/ChangeLog b/src/externals/genf90/ChangeLog similarity index 100% rename from externals/genf90/ChangeLog rename to src/externals/genf90/ChangeLog diff --git a/externals/genf90/genf90.pl b/src/externals/genf90/genf90.pl similarity index 100% rename from externals/genf90/genf90.pl rename to src/externals/genf90/genf90.pl diff --git a/externals/mct/.gitignore b/src/externals/mct/.gitignore similarity index 100% rename from externals/mct/.gitignore rename to src/externals/mct/.gitignore diff --git a/externals/mct/COPYRIGHT b/src/externals/mct/COPYRIGHT similarity index 100% rename from externals/mct/COPYRIGHT rename to src/externals/mct/COPYRIGHT diff --git a/externals/mct/Makefile b/src/externals/mct/Makefile similarity index 100% rename from externals/mct/Makefile rename to src/externals/mct/Makefile diff --git a/externals/mct/Makefile.conf.in b/src/externals/mct/Makefile.conf.in similarity index 100% rename from externals/mct/Makefile.conf.in rename to src/externals/mct/Makefile.conf.in diff --git a/externals/mct/README b/src/externals/mct/README similarity index 100% rename from externals/mct/README rename to src/externals/mct/README diff --git a/externals/mct/aclocal.m4 b/src/externals/mct/aclocal.m4 similarity index 100% rename from externals/mct/aclocal.m4 rename to src/externals/mct/aclocal.m4 diff --git a/externals/mct/benchmarks/.gitignore b/src/externals/mct/benchmarks/.gitignore similarity index 100% rename from externals/mct/benchmarks/.gitignore rename to src/externals/mct/benchmarks/.gitignore diff --git a/externals/mct/benchmarks/Makefile b/src/externals/mct/benchmarks/Makefile similarity index 100% rename from externals/mct/benchmarks/Makefile rename to src/externals/mct/benchmarks/Makefile diff --git a/externals/mct/benchmarks/RouterTestDis.F90 b/src/externals/mct/benchmarks/RouterTestDis.F90 similarity index 100% rename from externals/mct/benchmarks/RouterTestDis.F90 rename to src/externals/mct/benchmarks/RouterTestDis.F90 diff --git a/externals/mct/benchmarks/RouterTestOvr.F90 b/src/externals/mct/benchmarks/RouterTestOvr.F90 similarity index 100% rename from externals/mct/benchmarks/RouterTestOvr.F90 rename to src/externals/mct/benchmarks/RouterTestOvr.F90 diff --git a/externals/mct/benchmarks/T42.8pC b/src/externals/mct/benchmarks/T42.8pC similarity index 100% rename from externals/mct/benchmarks/T42.8pC rename to src/externals/mct/benchmarks/T42.8pC diff --git a/externals/mct/benchmarks/T42.8pR b/src/externals/mct/benchmarks/T42.8pR similarity index 100% rename from externals/mct/benchmarks/T42.8pR rename to src/externals/mct/benchmarks/T42.8pR diff --git a/externals/mct/benchmarks/gx1.8pC b/src/externals/mct/benchmarks/gx1.8pC similarity index 100% rename from externals/mct/benchmarks/gx1.8pC rename to src/externals/mct/benchmarks/gx1.8pC diff --git a/externals/mct/benchmarks/gx1.8pR b/src/externals/mct/benchmarks/gx1.8pR similarity index 100% rename from externals/mct/benchmarks/gx1.8pR rename to src/externals/mct/benchmarks/gx1.8pR diff --git a/externals/mct/benchmarks/importBench.F90 b/src/externals/mct/benchmarks/importBench.F90 similarity index 100% rename from externals/mct/benchmarks/importBench.F90 rename to src/externals/mct/benchmarks/importBench.F90 diff --git a/externals/mct/config.h.in b/src/externals/mct/config.h.in similarity index 100% rename from externals/mct/config.h.in rename to src/externals/mct/config.h.in diff --git a/externals/mct/configure b/src/externals/mct/configure similarity index 100% rename from externals/mct/configure rename to src/externals/mct/configure diff --git a/externals/mct/configure.ac b/src/externals/mct/configure.ac similarity index 100% rename from externals/mct/configure.ac rename to src/externals/mct/configure.ac diff --git a/externals/mct/doc/.gitignore b/src/externals/mct/doc/.gitignore similarity index 100% rename from externals/mct/doc/.gitignore rename to src/externals/mct/doc/.gitignore diff --git a/externals/mct/doc/Makefile b/src/externals/mct/doc/Makefile similarity index 100% rename from externals/mct/doc/Makefile rename to src/externals/mct/doc/Makefile diff --git a/externals/mct/doc/README b/src/externals/mct/doc/README similarity index 100% rename from externals/mct/doc/README rename to src/externals/mct/doc/README diff --git a/externals/mct/doc/coupler.bib b/src/externals/mct/doc/coupler.bib similarity index 100% rename from externals/mct/doc/coupler.bib rename to src/externals/mct/doc/coupler.bib diff --git a/externals/mct/doc/mct_APIs.tex b/src/externals/mct/doc/mct_APIs.tex similarity index 100% rename from externals/mct/doc/mct_APIs.tex rename to src/externals/mct/doc/mct_APIs.tex diff --git a/externals/mct/doc/texsrc/.gitignore b/src/externals/mct/doc/texsrc/.gitignore similarity index 100% rename from externals/mct/doc/texsrc/.gitignore rename to src/externals/mct/doc/texsrc/.gitignore diff --git a/externals/mct/doc/texsrc/Makefile b/src/externals/mct/doc/texsrc/Makefile similarity index 100% rename from externals/mct/doc/texsrc/Makefile rename to src/externals/mct/doc/texsrc/Makefile diff --git a/externals/mct/doc/texsrc/SRCS_tex.mk b/src/externals/mct/doc/texsrc/SRCS_tex.mk similarity index 100% rename from externals/mct/doc/texsrc/SRCS_tex.mk rename to src/externals/mct/doc/texsrc/SRCS_tex.mk diff --git a/externals/mct/examples/Makefile b/src/externals/mct/examples/Makefile similarity index 100% rename from externals/mct/examples/Makefile rename to src/externals/mct/examples/Makefile diff --git a/externals/mct/examples/README b/src/externals/mct/examples/README similarity index 100% rename from externals/mct/examples/README rename to src/externals/mct/examples/README diff --git a/externals/mct/examples/climate_concur1/.gitignore b/src/externals/mct/examples/climate_concur1/.gitignore similarity index 100% rename from externals/mct/examples/climate_concur1/.gitignore rename to src/externals/mct/examples/climate_concur1/.gitignore diff --git a/externals/mct/examples/climate_concur1/Makefile b/src/externals/mct/examples/climate_concur1/Makefile similarity index 100% rename from externals/mct/examples/climate_concur1/Makefile rename to src/externals/mct/examples/climate_concur1/Makefile diff --git a/externals/mct/examples/climate_concur1/README b/src/externals/mct/examples/climate_concur1/README similarity index 100% rename from externals/mct/examples/climate_concur1/README rename to src/externals/mct/examples/climate_concur1/README diff --git a/externals/mct/examples/climate_concur1/coupler.F90 b/src/externals/mct/examples/climate_concur1/coupler.F90 similarity index 100% rename from externals/mct/examples/climate_concur1/coupler.F90 rename to src/externals/mct/examples/climate_concur1/coupler.F90 diff --git a/externals/mct/examples/climate_concur1/master.F90 b/src/externals/mct/examples/climate_concur1/master.F90 similarity index 100% rename from externals/mct/examples/climate_concur1/master.F90 rename to src/externals/mct/examples/climate_concur1/master.F90 diff --git a/externals/mct/examples/climate_concur1/model.F90 b/src/externals/mct/examples/climate_concur1/model.F90 similarity index 100% rename from externals/mct/examples/climate_concur1/model.F90 rename to src/externals/mct/examples/climate_concur1/model.F90 diff --git a/externals/mct/examples/climate_sequen1/.gitignore b/src/externals/mct/examples/climate_sequen1/.gitignore similarity index 100% rename from externals/mct/examples/climate_sequen1/.gitignore rename to src/externals/mct/examples/climate_sequen1/.gitignore diff --git a/externals/mct/examples/climate_sequen1/Makefile b/src/externals/mct/examples/climate_sequen1/Makefile similarity index 100% rename from externals/mct/examples/climate_sequen1/Makefile rename to src/externals/mct/examples/climate_sequen1/Makefile diff --git a/externals/mct/examples/climate_sequen1/README b/src/externals/mct/examples/climate_sequen1/README similarity index 100% rename from externals/mct/examples/climate_sequen1/README rename to src/externals/mct/examples/climate_sequen1/README diff --git a/externals/mct/examples/climate_sequen1/TS1.dat b/src/externals/mct/examples/climate_sequen1/TS1.dat similarity index 100% rename from externals/mct/examples/climate_sequen1/TS1.dat rename to src/externals/mct/examples/climate_sequen1/TS1.dat diff --git a/externals/mct/examples/climate_sequen1/coupler.F90 b/src/externals/mct/examples/climate_sequen1/coupler.F90 similarity index 100% rename from externals/mct/examples/climate_sequen1/coupler.F90 rename to src/externals/mct/examples/climate_sequen1/coupler.F90 diff --git a/externals/mct/examples/climate_sequen1/dst.rc b/src/externals/mct/examples/climate_sequen1/dst.rc similarity index 100% rename from externals/mct/examples/climate_sequen1/dst.rc rename to src/externals/mct/examples/climate_sequen1/dst.rc diff --git a/externals/mct/examples/climate_sequen1/dstmodel.F90 b/src/externals/mct/examples/climate_sequen1/dstmodel.F90 similarity index 100% rename from externals/mct/examples/climate_sequen1/dstmodel.F90 rename to src/externals/mct/examples/climate_sequen1/dstmodel.F90 diff --git a/externals/mct/examples/climate_sequen1/master.F90 b/src/externals/mct/examples/climate_sequen1/master.F90 similarity index 100% rename from externals/mct/examples/climate_sequen1/master.F90 rename to src/externals/mct/examples/climate_sequen1/master.F90 diff --git a/externals/mct/examples/climate_sequen1/mutils.F90 b/src/externals/mct/examples/climate_sequen1/mutils.F90 similarity index 100% rename from externals/mct/examples/climate_sequen1/mutils.F90 rename to src/externals/mct/examples/climate_sequen1/mutils.F90 diff --git a/externals/mct/examples/climate_sequen1/src.rc b/src/externals/mct/examples/climate_sequen1/src.rc similarity index 100% rename from externals/mct/examples/climate_sequen1/src.rc rename to src/externals/mct/examples/climate_sequen1/src.rc diff --git a/externals/mct/examples/climate_sequen1/srcmodel.F90 b/src/externals/mct/examples/climate_sequen1/srcmodel.F90 similarity index 100% rename from externals/mct/examples/climate_sequen1/srcmodel.F90 rename to src/externals/mct/examples/climate_sequen1/srcmodel.F90 diff --git a/externals/mct/examples/simple/.gitignore b/src/externals/mct/examples/simple/.gitignore similarity index 100% rename from externals/mct/examples/simple/.gitignore rename to src/externals/mct/examples/simple/.gitignore diff --git a/externals/mct/examples/simple/Makefile b/src/externals/mct/examples/simple/Makefile similarity index 100% rename from externals/mct/examples/simple/Makefile rename to src/externals/mct/examples/simple/Makefile diff --git a/externals/mct/examples/simple/README b/src/externals/mct/examples/simple/README similarity index 100% rename from externals/mct/examples/simple/README rename to src/externals/mct/examples/simple/README diff --git a/externals/mct/examples/simple/script.babyblue b/src/externals/mct/examples/simple/script.babyblue similarity index 100% rename from externals/mct/examples/simple/script.babyblue rename to src/externals/mct/examples/simple/script.babyblue diff --git a/externals/mct/examples/simple/twocmp.con.F90 b/src/externals/mct/examples/simple/twocmp.con.F90 similarity index 100% rename from externals/mct/examples/simple/twocmp.con.F90 rename to src/externals/mct/examples/simple/twocmp.con.F90 diff --git a/externals/mct/examples/simple/twocmp.seq.F90 b/src/externals/mct/examples/simple/twocmp.seq.F90 similarity index 100% rename from externals/mct/examples/simple/twocmp.seq.F90 rename to src/externals/mct/examples/simple/twocmp.seq.F90 diff --git a/externals/mct/examples/simple/twocmp.seqNB.F90 b/src/externals/mct/examples/simple/twocmp.seqNB.F90 similarity index 100% rename from externals/mct/examples/simple/twocmp.seqNB.F90 rename to src/externals/mct/examples/simple/twocmp.seqNB.F90 diff --git a/externals/mct/examples/simple/twocmp.seqUnvn.F90 b/src/externals/mct/examples/simple/twocmp.seqUnvn.F90 similarity index 100% rename from externals/mct/examples/simple/twocmp.seqUnvn.F90 rename to src/externals/mct/examples/simple/twocmp.seqUnvn.F90 diff --git a/externals/mct/install-sh b/src/externals/mct/install-sh similarity index 100% rename from externals/mct/install-sh rename to src/externals/mct/install-sh diff --git a/externals/mct/m4/README b/src/externals/mct/m4/README similarity index 100% rename from externals/mct/m4/README rename to src/externals/mct/m4/README diff --git a/externals/mct/m4/acx_mpi.m4 b/src/externals/mct/m4/acx_mpi.m4 similarity index 100% rename from externals/mct/m4/acx_mpi.m4 rename to src/externals/mct/m4/acx_mpi.m4 diff --git a/externals/mct/m4/ax_fc_version.m4 b/src/externals/mct/m4/ax_fc_version.m4 similarity index 100% rename from externals/mct/m4/ax_fc_version.m4 rename to src/externals/mct/m4/ax_fc_version.m4 diff --git a/externals/mct/m4/fortran.m4 b/src/externals/mct/m4/fortran.m4 similarity index 100% rename from externals/mct/m4/fortran.m4 rename to src/externals/mct/m4/fortran.m4 diff --git a/externals/mct/mct/Makefile b/src/externals/mct/mct/Makefile similarity index 97% rename from externals/mct/mct/Makefile rename to src/externals/mct/mct/Makefile index edc5d11a8df..97aa186e77b 100644 --- a/externals/mct/mct/Makefile +++ b/src/externals/mct/mct/Makefile @@ -31,6 +31,7 @@ SRCS_F90 = m_MCTWorld.F90 \ m_SparseMatrixPlus.F90 \ m_Router.F90 \ m_Rearranger.F90 \ + m_SPMDutils.F90 \ m_Transfer.F90 OBJS_ALL = $(SRCS_F90:.F90=.o) @@ -82,6 +83,7 @@ m_GlobalMap.o: m_GlobalSegMap.o: m_GlobalSegMapComms.o: m_GlobalSegMap.o m_Navigator.o: +m_SPMDutils.o: m_AttrVectComms.o: m_AttrVect.o m_GlobalMap.o m_AttrVectReduce.o: m_AttrVect.o m_AccumulatorComms.o: m_AttrVect.o m_GlobalMap.o m_AttrVectComms.o @@ -91,7 +93,7 @@ m_GeneralGridComms.o: m_AttrVect.o m_GeneralGrid.o m_AttrVectComms.o m_GlobalMap m_MatAttrVectMul.o: m_AttrVect.o m_SparseMatrix.o m_GlobalMap.o m_GlobalSegMap.o m_SparseMatrixPlus.o m_Rearranger.o m_Merge.o: m_AttrVect.o m_GeneralGrid.o m_Router.o: m_GlobalToLocal.o m_MCTWorld.o m_GlobalSegMap.o m_ExchangeMaps.o -m_Rearranger.o: m_Router.o m_MCTWorld.o m_GlobalSegMap.o m_AttrVect.o +m_Rearranger.o: m_Router.o m_MCTWorld.o m_GlobalSegMap.o m_AttrVect.o m_SPMDutils.o m_GlobalToLocal.o: m_GlobalSegMap.o m_ExchangeMaps.o: m_GlobalMap.o m_GlobalSegMap.o m_MCTWorld.o m_ConvertMaps.o m_ConvertMaps.o: m_GlobalMap.o m_GlobalSegMap.o m_MCTWorld.o diff --git a/externals/mct/mct/README b/src/externals/mct/mct/README similarity index 100% rename from externals/mct/mct/README rename to src/externals/mct/mct/README diff --git a/externals/mct/mct/m_Accumulator.F90 b/src/externals/mct/mct/m_Accumulator.F90 similarity index 100% rename from externals/mct/mct/m_Accumulator.F90 rename to src/externals/mct/mct/m_Accumulator.F90 diff --git a/externals/mct/mct/m_AccumulatorComms.F90 b/src/externals/mct/mct/m_AccumulatorComms.F90 similarity index 100% rename from externals/mct/mct/m_AccumulatorComms.F90 rename to src/externals/mct/mct/m_AccumulatorComms.F90 diff --git a/externals/mct/mct/m_AttrVect.F90 b/src/externals/mct/mct/m_AttrVect.F90 similarity index 96% rename from externals/mct/mct/m_AttrVect.F90 rename to src/externals/mct/mct/m_AttrVect.F90 index 93f0f14d1cd..d186fb84ce5 100644 --- a/externals/mct/mct/m_AttrVect.F90 +++ b/src/externals/mct/mct/m_AttrVect.F90 @@ -844,15 +844,19 @@ subroutine zero_(aV, zeroReals, zeroInts) if(myZeroInts) then ! zero out INTEGER attributes if(List_allocated(aV%iList)) then -!CDIR COLLAPSE - if(associated(aV%iAttr) .and. (nIAttr_(aV)>0)) aV%iAttr=0 + if(associated(aV%iAttr) .and. (nIAttr_(aV)>0)) then +!DIR$ COLLAPSE + aV%iAttr=0 + endif endif endif if(myZeroReals) then ! zero out REAL attributes if(List_allocated(aV%rList)) then -!CDIR COLLAPSE - if(associated(aV%rAttr) .and. (nRAttr_(aV)>0)) aV%rAttr=0._FP + if(associated(aV%rAttr) .and. (nRAttr_(aV)>0)) then +!DIR$ COLLAPSE + aV%rAttr=0._FP + endif endif endif @@ -2502,9 +2506,6 @@ subroutine RCopy_(aVin, aVout, vector, sharedIndices) use m_die , only : die use m_stdio , only : stderr - use m_List, only : GetSharedListIndices - use m_List, only : GetIndices => get_indices - implicit none ! !INPUT PARAMETERS: @@ -2527,8 +2528,7 @@ subroutine RCopy_(aVin, aVout, vector, sharedIndices) integer :: i,j,ier ! dummy variables integer :: aVsize ! The lsize of aVin and aVout - integer :: num_inindices, num_outindices ! Number of matching indices in aV - integer :: inxmin, outxmin, inx, outx ! Index variables + integer :: inxmin, outxmin ! Index variables logical :: usevector ! true if vector flag is present and true. character*7 :: data_flag ! character variable used as data type flag type(AVSharedIndicesOneType) :: mySharedIndices ! copied from sharedIndices, or @@ -2586,23 +2586,20 @@ subroutine RCopy_(aVin, aVout, vector, sharedIndices) if(mySharedIndices%contiguous) then + outxmin=mySharedIndices%aVindices2(1)-1 + inxmin=mySharedIndices%aVindices1(1)-1 if(usevector) then - outxmin=mySharedIndices%aVindices2(1)-1 - inxmin=mySharedIndices%aVindices1(1)-1 !$OMP PARALLEL DO PRIVATE(i,j) do i=1,mySharedIndices%num_indices !CDIR SELECT(VECTOR) -!DIR$ CONCURRENT +!DIR$ IVDEP do j=1,aVsize aVout%rAttr(outxmin+i,j) = aVin%rAttr(inxmin+i,j) enddo enddo else - outxmin=mySharedIndices%aVindices2(1)-1 - inxmin=mySharedIndices%aVindices1(1)-1 -!$OMP PARALLEL DO PRIVAtE(j,i) +!$OMP PARALLEL DO PRIVATE(j,i) COLLAPSE(2) do j=1,aVsize -!DIR$ CONCURRENT do i=1,mySharedIndices%num_indices aVout%rAttr(outxmin+i,j) = aVin%rAttr(inxmin+i,j) enddo @@ -2611,13 +2608,10 @@ subroutine RCopy_(aVin, aVout, vector, sharedIndices) else -!$OMP PARALLEL DO PRIVATE(j,i,outx,inx) +!$OMP PARALLEL DO PRIVATE(j,i) COLLAPSE(2) do j=1,aVsize -!DIR$ CONCURRENT do i=1,mySharedIndices%num_indices - outx=mySharedIndices%aVindices2(i) - inx=mySharedIndices%aVindices1(i) - aVout%rAttr(outx,j) = aVin%rAttr(inx,j) + aVout%rAttr(mySharedIndices%aVindices2(i),j) = aVin%rAttr(mySharedIndices%aVindices1(i),j) enddo enddo @@ -2665,7 +2659,6 @@ subroutine RCopyL_(aVin, aVout, rList, TrList, vector) use m_die , only : die use m_stdio , only : stderr - use m_List, only : GetSharedListIndices use m_List, only : GetIndices => get_indices implicit none @@ -2693,12 +2686,10 @@ subroutine RCopyL_(aVin, aVout, rList, TrList, vector) integer :: i,j,ier ! dummy variables integer :: num_indices ! Overlapping attribute index number integer :: aVsize ! The lsize of aVin and aVout - integer :: num_inindices, num_outindices ! Number of matching indices in aV - integer :: inxmin, outxmin, inx, outx ! Index variables + integer :: inxmin, outxmin ! Index variables logical :: TrListIsPresent ! true if list argument is present logical :: contiguous ! true if index segments are contiguous in memory logical :: usevector ! true if vector flag is present and true. - character*7 :: data_flag ! character variable used as data type flag ! Overlapping attribute index storage arrays: integer, dimension(:), pointer :: aVinindices, aVoutindices @@ -2756,48 +2747,48 @@ subroutine RCopyL_(aVin, aVout, rList, TrList, vector) ! Check if the indices are contiguous in memory for faster copy contiguous=.true. do i=2,num_indices - if(aVinindices(i) /= aVinindices(i-1)+1) contiguous = .false. + if(aVinindices(i) /= aVinindices(i-1)+1) then + contiguous = .false. + exit + endif enddo if(contiguous) then do i=2,num_indices - if(aVoutindices(i) /= aVoutindices(i-1)+1) contiguous=.false. + if(aVoutindices(i) /= aVoutindices(i-1)+1) then + contiguous=.false. + exit + endif enddo endif ! Start copying (arranged loop order optimized for xlf90) if(contiguous) then + outxmin=aVoutindices(1)-1 + inxmin=aVinindices(1)-1 if(usevector) then - outxmin=aVoutindices(1)-1 - inxmin=aVinindices(1)-1 !$OMP PARALLEL DO PRIVATE(i,j) do i=1,num_indices -!DIR$ CONCURRENT - do j=1,aVsize - aVout%rAttr(outxmin+i,j) = aVin%rAttr(inxmin+i,j) - enddo - enddo +!DIR$ IVDEP + do j=1,aVsize + aVout%rAttr(outxmin+i,j) = aVin%rAttr(inxmin+i,j) + enddo + enddo else - outxmin=aVoutindices(1)-1 - inxmin=aVinindices(1)-1 -!$OMP PARALLEL DO PRIVATE(j,i) +!$OMP PARALLEL DO PRIVATE(j,i) COLLAPSE(2) do j=1,aVsize -!DIR$ CONCURRENT - do i=1,num_indices - aVout%rAttr(outxmin+i,j) = aVin%rAttr(inxmin+i,j) - enddo - enddo + do i=1,num_indices + aVout%rAttr(outxmin+i,j) = aVin%rAttr(inxmin+i,j) + enddo + enddo endif else -!$OMP PARALLEL DO PRIVATE(j,i,outx,inx) +!$OMP PARALLEL DO PRIVATE(j,i) COLLAPSE(2) do j=1,aVsize -!DIR$ CONCURRENT do i=1,num_indices - outx=aVoutindices(i) - inx=aVinindices(i) - aVout%rAttr(outx,j) = aVin%rAttr(inx,j) + aVout%rAttr(aVoutindices(i),j) = aVin%rAttr(aVinindices(i),j) enddo enddo @@ -2842,9 +2833,6 @@ subroutine ICopy_(aVin, aVout, vector, sharedIndices) use m_die , only : die use m_stdio , only : stderr - use m_List, only : GetSharedListIndices - use m_List, only : GetIndices => get_indices - implicit none ! !INPUT PARAMETERS: @@ -2868,8 +2856,7 @@ subroutine ICopy_(aVin, aVout, vector, sharedIndices) integer :: i,j,ier ! dummy variables integer :: aVsize ! The lsize of aVin and aVout - integer :: num_inindices, num_outindices ! Number of matching indices in aV - integer :: inxmin, outxmin, inx, outx ! Index variables + integer :: inxmin, outxmin ! Index variables logical :: usevector ! true if vector flag is present and true. character*7 :: data_flag ! character variable used as data type flag type(AVSharedIndicesOneType) :: mySharedIndices ! copied from sharedIndices, or @@ -2926,23 +2913,20 @@ subroutine ICopy_(aVin, aVout, vector, sharedIndices) if(mySharedIndices%contiguous) then + outxmin=mySharedIndices%aVindices2(1)-1 + inxmin=mySharedIndices%aVindices1(1)-1 if(usevector) then - outxmin=mySharedIndices%aVindices2(1)-1 - inxmin=mySharedIndices%aVindices1(1)-1 !$OMP PARALLEL DO PRIVATE(i,j) do i=1,mySharedIndices%num_indices !CDIR SELECT(VECTOR) -!DIR$ CONCURRENT +!DIR$ IVDEP do j=1,aVsize aVout%iAttr(outxmin+i,j) = aVin%iAttr(inxmin+i,j) enddo enddo else - outxmin=mySharedIndices%aVindices2(1)-1 - inxmin=mySharedIndices%aVindices1(1)-1 -!$OMP PARALLEL DO PRIVATE(j,i) +!$OMP PARALLEL DO PRIVATE(j,i) COLLAPSE(2) do j=1,aVsize -!DIR$ CONCURRENT do i=1,mySharedIndices%num_indices aVout%iAttr(outxmin+i,j) = aVin%iAttr(inxmin+i,j) enddo @@ -2951,13 +2935,10 @@ subroutine ICopy_(aVin, aVout, vector, sharedIndices) else -!$OMP PARALLEL DO PRIVATE(j,i,outx,inx) +!$OMP PARALLEL DO PRIVATE(j,i) COLLAPSE(2) do j=1,aVsize -!DIR$ CONCURRENT do i=1,mySharedIndices%num_indices - outx=mySharedIndices%aVindices2(i) - inx=mySharedIndices%aVindices1(i) - aVout%iAttr(outx,j) = aVin%iAttr(inx,j) + aVout%iAttr(mySharedIndices%aVindices2(i),j) = aVin%iAttr(mySharedIndices%aVindices1(i),j) enddo enddo @@ -3031,12 +3012,10 @@ subroutine ICopyL_(aVin, aVout, iList, TiList, vector) integer :: i,j,ier ! dummy variables integer :: num_indices ! Overlapping attribute index number integer :: aVsize ! The lsize of aVin and aVout - integer :: num_inindices, num_outindices ! Number of matching indices in aV - integer :: inxmin, outxmin, inx, outx ! Index variables + integer :: inxmin, outxmin ! Index variables logical :: TiListIsPresent ! true if list argument is present logical :: contiguous ! true if index segments are contiguous in memory logical :: usevector ! true if vector flag is present and true. - character*7 :: data_flag ! character variable used as data type flag ! Overlapping attribute index storage arrays: integer, dimension(:), pointer :: aVinindices, aVoutindices @@ -3094,49 +3073,49 @@ subroutine ICopyL_(aVin, aVout, iList, TiList, vector) ! Check if the indices are contiguous in memory for faster copy contiguous=.true. do i=2,num_indices - if(aVinindices(i) /= aVinindices(i-1)+1) contiguous = .false. + if(aVinindices(i) /= aVinindices(i-1)+1) then + contiguous = .false. + exit + endif enddo if(contiguous) then do i=2,num_indices - if(aVoutindices(i) /= aVoutindices(i-1)+1) contiguous=.false. + if(aVoutindices(i) /= aVoutindices(i-1)+1) then + contiguous=.false. + exit + endif enddo endif ! Start copying (arranged loop order optimized for xlf90) if(contiguous) then + outxmin=aVoutindices(1)-1 + inxmin=aVinindices(1)-1 if(usevector) then - outxmin=aVoutindices(1)-1 - inxmin=aVinindices(1)-1 !$OMP PARALLEL DO PRIVAtE(i,j) do i=1,num_indices !CDIR SELECT(VECTOR) -!DIR$ CONCURRENT +!DIR$ IVDEP do j=1,aVsize - aVout%iAttr(outxmin+i,j) = aVin%iAttr(inxmin+i,j) + aVout%iAttr(outxmin+i,j) = aVin%iAttr(inxmin+i,j) enddo enddo else - outxmin=aVoutindices(1)-1 - inxmin=aVinindices(1)-1 -!$OMP PARALLEL DO PRIVATE(j,i) +!$OMP PARALLEL DO PRIVATE(j,i) COLLAPSE(2) do j=1,aVsize -!DIR$ CONCURRENT do i=1,num_indices - aVout%iAttr(outxmin+i,j) = aVin%iAttr(inxmin+i,j) + aVout%iAttr(outxmin+i,j) = aVin%iAttr(inxmin+i,j) enddo enddo endif else -!$OMP PARALLEL DO PRIVATE(j,i,outx,inx) +!$OMP PARALLEL DO PRIVATE(j,i) COLLAPSE(2) do j=1,aVsize -!DIR$ CONCURRENT do i=1,num_indices - outx=aVoutindices(i) - inx=aVinindices(i) - aVout%iAttr(outx,j) = aVin%iAttr(inx,j) + aVout%iAttr(aVoutindices(i),j) = aVin%iAttr(aVinindices(i),j) enddo enddo @@ -3198,9 +3177,6 @@ subroutine Copy_(aVin, aVout, rList, TrList, iList, TiList, vector, sharedIndice use m_die , only : die, warn use m_stdio , only : stderr - use m_List, only : GetSharedListIndices - use m_List, only : GetIndices => get_indices - implicit none ! !INPUT PARAMETERS: @@ -3244,7 +3220,6 @@ subroutine Copy_(aVin, aVout, rList, TrList, iList, TiList, vector, sharedIndice logical :: TiListIsPresent, TrListIsPresent! true if list argument is present logical :: contiguous ! true if index segments are contiguous in memory logical :: usevector ! true if vector flag is present and true. - character*7 :: data_flag ! character variable used as data type flag ! Overlapping attribute index storage arrays: integer, dimension(:), pointer :: aVinindices, aVoutindices @@ -3267,48 +3242,48 @@ subroutine Copy_(aVin, aVout, rList, TrList, iList, TiList, vector, sharedIndice ! Copy the listed real attributes if(present(rList)) then - ! TrList is present if it is provided and its length>0 - TrListIsPresent = .false. - if(present(TrList)) then - if(len_trim(TrList) > 0) then - TrListIsPresent = .true. - endif - endif + ! TrList is present if it is provided and its length>0 + TrListIsPresent = .false. + if(present(TrList)) then + if(len_trim(TrList) > 0) then + TrListIsPresent = .true. + endif + endif - if(present(sharedIndices)) then - call warn(myname_,'Use of sharedIndices not implemented in RCopyL; & + if(present(sharedIndices)) then + call warn(myname_,'Use of sharedIndices not implemented in RCopyL; & &ignoring sharedIndices',1) - end if + endif - if(TrListIsPresent) then - call RCopyL_(aVin,aVout,rList,TrList,vector=usevector) - else - call RCopyL_(aVin,aVout,rList,vector=usevector) - endif + if(TrListIsPresent) then + call RCopyL_(aVin,aVout,rList,TrList,vector=usevector) + else + call RCopyL_(aVin,aVout,rList,vector=usevector) + endif endif ! if(present(rList) ! Copy the listed integer attributes if(present(iList)) then - ! TiList is present if its provided and its length>0 - TiListIsPresent = .false. - if(present(TiList)) then - if(len_trim(TiList) > 0) then - TiListIsPresent = .true. - endif - endif + ! TiList is present if its provided and its length>0 + TiListIsPresent = .false. + if(present(TiList)) then + if(len_trim(TiList) > 0) then + TiListIsPresent = .true. + endif + endif - if(present(sharedIndices)) then - call warn(myname_,'Use of sharedIndices not implemented in ICopyL; & + if(present(sharedIndices)) then + call warn(myname_,'Use of sharedIndices not implemented in ICopyL; & &ignoring sharedIndices',1) - end if + endif - if(TiListIsPresent) then - call ICopyL_(aVin,aVout,iList,TiList,vector=usevector) - else - call ICopyL_(aVin,aVout,iList,vector=usevector) - endif + if(TiListIsPresent) then + call ICopyL_(aVin,aVout,iList,TiList,vector=usevector) + else + call ICopyL_(aVin,aVout,iList,vector=usevector) + endif endif ! if(present(iList)) @@ -4009,12 +3984,14 @@ subroutine SharedIndicesOneType_(aV1, aV2, attrib, sharedIndices) do i=2,sharedIndices%num_indices if(sharedIndices%aVindices1(i) /= sharedIndices%aVindices1(i-1)+1) then sharedIndices%contiguous = .false. + exit endif enddo if(sharedIndices%contiguous) then do i=2,sharedIndices%num_indices if(sharedIndices%aVindices2(i) /= sharedIndices%aVindices2(i-1)+1) then sharedIndices%contiguous=.false. + exit endif enddo endif diff --git a/externals/mct/mct/m_AttrVectComms.F90 b/src/externals/mct/mct/m_AttrVectComms.F90 similarity index 100% rename from externals/mct/mct/m_AttrVectComms.F90 rename to src/externals/mct/mct/m_AttrVectComms.F90 diff --git a/externals/mct/mct/m_AttrVectReduce.F90 b/src/externals/mct/mct/m_AttrVectReduce.F90 similarity index 100% rename from externals/mct/mct/m_AttrVectReduce.F90 rename to src/externals/mct/mct/m_AttrVectReduce.F90 diff --git a/externals/mct/mct/m_ConvertMaps.F90 b/src/externals/mct/mct/m_ConvertMaps.F90 similarity index 100% rename from externals/mct/mct/m_ConvertMaps.F90 rename to src/externals/mct/mct/m_ConvertMaps.F90 diff --git a/externals/mct/mct/m_ExchangeMaps.F90 b/src/externals/mct/mct/m_ExchangeMaps.F90 similarity index 100% rename from externals/mct/mct/m_ExchangeMaps.F90 rename to src/externals/mct/mct/m_ExchangeMaps.F90 diff --git a/externals/mct/mct/m_GeneralGrid.F90 b/src/externals/mct/mct/m_GeneralGrid.F90 similarity index 100% rename from externals/mct/mct/m_GeneralGrid.F90 rename to src/externals/mct/mct/m_GeneralGrid.F90 diff --git a/externals/mct/mct/m_GeneralGridComms.F90 b/src/externals/mct/mct/m_GeneralGridComms.F90 similarity index 100% rename from externals/mct/mct/m_GeneralGridComms.F90 rename to src/externals/mct/mct/m_GeneralGridComms.F90 diff --git a/externals/mct/mct/m_GlobalMap.F90 b/src/externals/mct/mct/m_GlobalMap.F90 similarity index 100% rename from externals/mct/mct/m_GlobalMap.F90 rename to src/externals/mct/mct/m_GlobalMap.F90 diff --git a/externals/mct/mct/m_GlobalSegMap.F90 b/src/externals/mct/mct/m_GlobalSegMap.F90 similarity index 92% rename from externals/mct/mct/m_GlobalSegMap.F90 rename to src/externals/mct/mct/m_GlobalSegMap.F90 index f59901e929c..a1960885fa8 100644 --- a/externals/mct/mct/m_GlobalSegMap.F90 +++ b/src/externals/mct/mct/m_GlobalSegMap.F90 @@ -1742,9 +1742,6 @@ subroutine active_pes_(GSMap, n_active, pe_list) ! !USES: ! use m_die , only : die - use m_SortingTools , only : IndexSet - use m_SortingTools , only : IndexSort - use m_SortingTools , only : Permute implicit none @@ -1764,22 +1761,25 @@ subroutine active_pes_(GSMap, n_active, pe_list) character(len=*),parameter :: myname_=myname//'::active_pes_' integer :: count, i, n, ngseg, ierr - logical :: new - integer, dimension(:), allocatable :: temp_list - integer, dimension(:), allocatable :: perm + integer :: max_activepe, p + logical, dimension(:), allocatable :: process_list ! retrieve total number of segments in the map: ngseg = ngseg_(GSMap) + ! retrieve maximum active process id in the map: + + max_activepe = maxval(GSMap%pe_loc(:)) + ! allocate workspace to tally process id list: - allocate(temp_list(ngseg), stat=ierr) - if(ierr /= 0) call die(myname_,'allocate(temp_list...',ierr) + allocate(process_list(0:max_activepe), stat=ierr) + if(ierr /= 0) call die(myname_,'allocate(process_list)',ierr) - ! initialize temp_list to -1 (which can never be a process id) + ! initialize process_list to false (i.e. no active pes) - temp_list = -1 + process_list = .false. ! initialize the distinct active process count: @@ -1790,25 +1790,10 @@ subroutine active_pes_(GSMap, n_active, pe_list) do n=1,ngseg if(GSMap%pe_loc(n) >= 0) then ! a legitimate pe_location - ! assume initially that GSMap%pe_loc(n) is a process id previously - ! not encountered - - new = .true. - - ! test this proposition against the growing list of distinct - ! process ids stored in temp_list(:) - - do i=1, count - if(GSMap%pe_loc(n) == temp_list(i)) new = .false. - end do - - ! If GSMap%pe_loc(n) represents a previously unencountered - ! process id, increment the count, and add this id to the list - - if(new) then - count = count + 1 - temp_list(count) = GSMap%pe_loc(n) - endif + if (.not. process_list(GSMap%pe_loc(n))) then + process_list(GSMap%pe_loc(n)) = .true. + count = count + 1 + endif else ! a negative entry in GSMap%pe_loc(n) ierr = 2 @@ -1817,41 +1802,37 @@ subroutine active_pes_(GSMap, n_active, pe_list) end do ! If the argument pe_list is present, we must allocate this - ! array, fill it, and sort it + ! array and fill it if(present(pe_list)) then - ! allocate pe_list and permutation array perm + ! allocate pe_list - allocate(pe_list(count), perm(count), stat=ierr) + allocate(pe_list(count), stat=ierr) if (ierr /= 0) then - call die(myname_,'allocate(pe_list...',ierr) + call die(myname_,'allocate(pe_list)',ierr) endif - do n=1,count - pe_list(n) = temp_list(n) - end do - - ! sorting and permutation... - - call IndexSet(perm) - call IndexSort(count, perm, pe_list, descend=.false.) - call Permute(pe_list, perm, count) - - ! deallocate permutation array... - - deallocate(perm, stat=ierr) - if (ierr /= 0) then - call die(myname_,'deallocate(perm)',ierr) + i = 0 + do p=0,max_activepe + if (process_list(p)) then + i = i+1 + if (i > count) exit + pe_list(i) = p + endif + enddo + + if (i > count) then + call die(myname_,'pe_list fill error',count) endif endif ! if(present(pe_list))... - ! deallocate work array temp_list... + ! deallocate work array process_list... - deallocate(temp_list, stat=ierr) + deallocate(process_list, stat=ierr) if (ierr /= 0) then - call die(myname_,'deallocate(temp_list)',ierr) + call die(myname_,'deallocate(process_list)',ierr) endif ! finally, store the active process count in output variable @@ -1903,15 +1884,37 @@ subroutine peLocs_(pointGSMap, npoints, points, pe_locs) ! !REVISION HISTORY: ! 18Apr01 - J.W. Larson - initial version. +! 18Oct16 - P. Worley - added algorithm options: +! new default changes complexity from O(npoints*ngseg) to +! O(gsize + ngseg) (worst case), and much better in current +! usage. Worst case memory requirements are O(gsize), but +! not seen in current usage. Other new algorithm is a little +! slower in practice, and worst case memory requirement is +! O(ngseg), which is also not seen in current usage. +! Original algorithm is recovered if compiled with +! LOW_MEMORY_PELOCS defined. Otherwise nondefault new +! algorithm is enabled if compiled with MEDIUM_MEMORY_PELOCS +! defined. !EOP ___________________________________________________________________ character(len=*),parameter :: myname_=myname//'::peLocs_' integer :: ierr integer :: iseg, ngseg, ipoint integer :: lower_index, upper_index + integer :: min_points_index, max_points_index +#if defined MEDIUM_MEMORY_PELOCS + integer :: ifseg, nfseg + integer, dimension(:), allocatable :: feasible_seg +#else + integer, dimension(:), allocatable :: pindices_to_pes +#endif ! Input argument checks: + if (npoints < 1) then + return + endif + if(size(points) < npoints) then ierr = size(points) call die(myname_,'input points list array too small',ierr) @@ -1934,21 +1937,158 @@ subroutine peLocs_(pointGSMap, npoints, points, pe_locs) ngseg = ngseg_(pointGSMap) +#if defined LOW_MEMORY_PELOCS + do ipoint=1,npoints ! loop over points do iseg=1,ngseg ! loop over segments - lower_index = pointGSMap%start(iseg) - upper_index = lower_index + pointGSMap%length(iseg) - 1 + lower_index = pointGSMap%start(iseg) + upper_index = lower_index + pointGSMap%length(iseg) - 1 - if((points(ipoint) >= lower_index) .and. & - (points(ipoint) <= upper_index)) then - pe_locs(ipoint) = pointGSMap%pe_loc(iseg) - endif + if((points(ipoint) >= lower_index) .and. & + (points(ipoint) <= upper_index)) then + pe_locs(ipoint) = pointGSMap%pe_loc(iseg) + + exit + + endif end do ! do iseg=1, ngseg + + end do ! do ipoint=1,npoints + +#elif defined MEDIUM_MEMORY_PELOCS + +! Determine index range for points vector + max_points_index = 0 + min_points_index = pointGSMap%gsize + 1 + do ipoint=1,npoints ! loop over points + + max_points_index = max(points(ipoint), max_points_index) + min_points_index = min(points(ipoint), min_points_index) + + end do ! do ipoint=1,npoints + +! Determine number of segments that need to be examined + nfseg = 0 + do iseg=1,ngseg ! loop over segments + + lower_index = pointGSMap%start(iseg) + upper_index = lower_index + pointGSMap%length(iseg) - 1 + + if ((lower_index <= max_points_index) .and. & + (upper_index >= min_points_index) ) then + + nfseg = nfseg + 1 + + endif + + end do ! do iseg=1, ngseg + + if(nfseg < 1) then + ierr = nfseg + call die(myname_,'no feasible segments',ierr) + endif + + ! Allocate temporary array + allocate(feasible_seg(nfseg), stat=ierr) + if (ierr /= 0) then + call die(myname_,'allocate(feasible_seg)',ierr) + endif + + ! Determine segments that need to be examined + feasible_seg(:) = 1 + nfseg = 0 + do iseg=1,ngseg ! loop over segments + + lower_index = pointGSMap%start(iseg) + upper_index = lower_index + pointGSMap%length(iseg) - 1 + + if ((lower_index <= max_points_index) .and. & + (upper_index >= min_points_index) ) then + + nfseg = nfseg + 1 + feasible_seg(nfseg) = iseg + + endif + + end do ! do iseg=1, ngseg + + ! Calculate map from local points to pes + do ipoint=1,npoints ! loop over points + + do ifseg=1,nfseg ! loop over feasible segments + + iseg = feasible_seg(ifseg) + lower_index = pointGSMap%start(iseg) + upper_index = lower_index + pointGSMap%length(iseg) - 1 + + if((points(ipoint) >= lower_index) .and. & + (points(ipoint) <= upper_index) ) then + pe_locs(ipoint) = pointGSMap%pe_loc(iseg) + exit + endif + + end do ! do ifseg=1,nfseg + end do ! do ipoint=1,npoints + + ! Clean up + deallocate(feasible_seg, stat=ierr) + if (ierr /= 0) then + call die(myname_,'deallocate(feasible_seg)',ierr) + endif + +#else + +! Determine index range for points assigned to points vector + max_points_index = 0 + min_points_index = pointGSMap%gsize + 1 + do ipoint=1,npoints ! loop over points + + max_points_index = max(points(ipoint), max_points_index) + min_points_index = min(points(ipoint), min_points_index) + + end do ! do ipoint=1,npoints + +! Allocate temporary array + allocate(pindices_to_pes(min_points_index:max_points_index), stat=ierr) + if (ierr /= 0) then + call die(myname_,'allocate(pindices_to_pes)',ierr) + endif + +! Calculate map from (global) point indices to pes + do iseg=1,ngseg ! loop over segments + + lower_index = pointGSMap%start(iseg) + upper_index = lower_index + pointGSMap%length(iseg) - 1 + + lower_index = max(lower_index, min_points_index) + upper_index = min(upper_index, max_points_index) + + if (lower_index <= upper_index) then + do ipoint=lower_index,upper_index + pindices_to_pes(ipoint) = pointGSMap%pe_loc(iseg) + enddo + endif + + end do ! do iseg=1, ngseg + +! Calculate map from local point indices to pes + do ipoint=1,npoints ! loop over points + + pe_locs(ipoint) = pindices_to_pes(points(ipoint)) + end do ! do ipoint=1,npoints +! Clean up + deallocate(pindices_to_pes, stat=ierr) + if (ierr /= 0) then + call die(myname_,'deallocate(pindices_to_pes)',ierr) + endif + +#endif + end subroutine peLocs_ !~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/externals/mct/mct/m_GlobalSegMapComms.F90 b/src/externals/mct/mct/m_GlobalSegMapComms.F90 similarity index 100% rename from externals/mct/mct/m_GlobalSegMapComms.F90 rename to src/externals/mct/mct/m_GlobalSegMapComms.F90 diff --git a/externals/mct/mct/m_GlobalToLocal.F90 b/src/externals/mct/mct/m_GlobalToLocal.F90 similarity index 100% rename from externals/mct/mct/m_GlobalToLocal.F90 rename to src/externals/mct/mct/m_GlobalToLocal.F90 diff --git a/externals/mct/mct/m_MCTWorld.F90 b/src/externals/mct/mct/m_MCTWorld.F90 similarity index 100% rename from externals/mct/mct/m_MCTWorld.F90 rename to src/externals/mct/mct/m_MCTWorld.F90 diff --git a/externals/mct/mct/m_MatAttrVectMul.F90 b/src/externals/mct/mct/m_MatAttrVectMul.F90 similarity index 93% rename from externals/mct/mct/m_MatAttrVectMul.F90 rename to src/externals/mct/mct/m_MatAttrVectMul.F90 index f6937c17b26..080214c677a 100644 --- a/externals/mct/mct/m_MatAttrVectMul.F90 +++ b/src/externals/mct/mct/m_MatAttrVectMul.F90 @@ -108,7 +108,6 @@ subroutine sMatAvMult_DataLocal_(xAV, sMat, yAV, Vector, rList, TrList) use m_AttrVect, only : AttrVect use m_AttrVect, only : AttrVect_lsize => lsize use m_AttrVect, only : AttrVect_zero => zero - use m_AttrVect, only : AttrVect_nRAttr => nRAttr use m_AttrVect, only : AttrVect_indexRA => indexRA use m_AttrVect, only : SharedAttrIndexList @@ -188,6 +187,7 @@ subroutine sMatAvMult_DataLocal_(xAV, sMat, yAV, Vector, rList, TrList) logical :: usevector,TrListIsPresent,rListIsPresent logical :: contiguous,ycontiguous + usevector = .false. if(present(Vector)) then if(Vector) usevector = .true. @@ -237,11 +237,11 @@ subroutine sMatAvMult_DataLocal_(xAV, sMat, yAV, Vector, rList, TrList) call SparseMatrix_vecinit(sMat) endif -!DIR$ CONCURRENT +!DIR$ IVDEP do m=1,num_indices do l=1,sMat%tbl_end !CDIR NOLOOPCHG -!DIR$ CONCURRENT +!DIR$ IVDEP do i=sMat%row_s(l),sMat%row_e(l) col = sMat%tcol(i,l) wgt = sMat%twgt(i,l) @@ -261,7 +261,7 @@ subroutine sMatAvMult_DataLocal_(xAV, sMat, yAV, Vector, rList, TrList) ! loop over attributes being regridded. -!DIR$ CONCURRENT +!DIR$ IVDEP do m=1,num_indices yAV%rAttr(m,row) = yAV%rAttr(m,row) + wgt * xAV%rAttr(m,col) @@ -318,13 +318,17 @@ subroutine sMatAvMult_DataLocal_(xAV, sMat, yAV, Vector, rList, TrList) contiguous=.true. ycontiguous=.true. do i=2,num_indices - if(xaVindices(i) /= xAVindices(i-1)+1) contiguous = .false. + if(xaVindices(i) /= xAVindices(i-1)+1) then + contiguous = .false. + exit + endif enddo if(contiguous) then do i=2,num_indices if(yAVindices(i) /= yAVindices(i-1)+1) then contiguous=.false. ycontiguous=.false. + exit endif enddo endif @@ -335,6 +339,7 @@ subroutine sMatAvMult_DataLocal_(xAV, sMat, yAV, Vector, rList, TrList) if(ycontiguous) then outxmin=yaVindices(1)-1 +!dir$ collapse do j=1,ysize do i=1,numav yAV%rAttr(outxmin+i,j)=0._FP @@ -360,14 +365,13 @@ subroutine sMatAvMult_DataLocal_(xAV, sMat, yAV, Vector, rList, TrList) wgt = sMat%data%rAttr(iwgt,n) ! loop over attributes being regridded. -!DIR$ CONCURRENT +!DIR$ IVDEP do m=1,num_indices yAV%rAttr(outxmin+m,row) = & yAV%rAttr(outxmin+m,row) + & wgt * xAV%rAttr(inxmin+m,col) end do ! m=1,num_indices end do ! n=1,num_elements - else do n=1,num_elements @@ -376,7 +380,7 @@ subroutine sMatAvMult_DataLocal_(xAV, sMat, yAV, Vector, rList, TrList) wgt = sMat%data%rAttr(iwgt,n) ! loop over attributes being regridded. -!DIR$ CONCURRENT +!DIR$ IVDEP do m=1,num_indices yAV%rAttr(yAVindices(m),row) = & yAV%rAttr(yAVindices(m),row) + & @@ -508,7 +512,8 @@ subroutine sMatAvMult_SMPlus_(xAV, sMatPlus, yAV, Vector, rList, TrList) call AttrVect_zero(xPrimeAV) ! Rearrange data from x to get x' call Rearrange(xAV, xPrimeAV, sMatPlus%XToXPrime, & - sMatPlus%Tag ,vector=usevector) + tag=sMatPlus%Tag, vector=usevector,& + alltoall=.true., handshake=.true. ) ! Perform perfectly data-local multiply y = Mx' if (present(TrList).and.present(rList)) then @@ -554,13 +559,15 @@ subroutine sMatAvMult_SMPlus_(xAV, sMatPlus, yAV, Vector, rList, TrList) ! Rearrange/reduce partial sums in y' to get y if (present(TrList).or.present(rList)) then - call Rearrange(yPrimeAV, yAVre, sMatPlus%YPrimeToY, sMatPlus%Tag, & - .TRUE., Vector=usevector) + call Rearrange(yPrimeAV, yAVre, sMatPlus%YPrimeToY, & + tag=sMatPlus%Tag, sum=.TRUE., Vector=usevector, & + alltoall=.true., handshake=.true. ) call AttrVect_Rcopy(yAVre,yAV,vector=usevector) call AttrVect_clean(yAVre, ierr) else - call Rearrange(yPrimeAV, yAV, sMatPlus%YPrimeToY, sMatPlus%Tag, & - .TRUE., Vector=usevector) + call Rearrange(yPrimeAV, yAV, sMatPlus%YPrimeToY, & + tag=sMatPlus%Tag, sum=.TRUE., Vector=usevector, & + alltoall=.true., handshake=.true. ) endif ! Clean up space occupied by y' call AttrVect_clean(yPrimeAV, ierr) @@ -586,8 +593,9 @@ subroutine sMatAvMult_SMPlus_(xAV, sMatPlus, yAV, Vector, rList, TrList) endif ! Rearrange data from x to get x' - call Rearrange(xAV, xPrimeAV, sMatPlus%XToXPrime, sMatPlus%Tag, & - Vector=usevector) + call Rearrange(xAV, xPrimeAV, sMatPlus%XToXPrime, & + tag=sMatPlus%Tag, Vector=usevector, & + alltoall=.true., handshake=.true. ) ! Perform perfectly data-local multiply y' = Mx' if (present(TrList).and.present(rList)) then @@ -603,13 +611,15 @@ subroutine sMatAvMult_SMPlus_(xAV, sMatPlus, yAV, Vector, rList, TrList) ! Rearrange/reduce partial sums in y' to get y if (present(TrList).or.present(rList)) then - call Rearrange(yPrimeAV, yAVre, sMatPlus%YPrimeToY, sMatPlus%Tag, & - .TRUE., Vector=usevector) + call Rearrange(yPrimeAV, yAVre, sMatPlus%YPrimeToY, & + tag=sMatPlus%Tag, sum=.TRUE., Vector=usevector, & + alltoall=.true., handshake=.true. ) call AttrVect_Rcopy(yAVre,yAV,vector=usevector) call AttrVect_clean(yAVre, ierr) else - call Rearrange(yPrimeAV, yAV, sMatPlus%YPrimeToY, sMatPlus%Tag, & - .TRUE., Vector=usevector) + call Rearrange(yPrimeAV, yAV, sMatPlus%YPrimeToY, & + tag=sMatPlus%Tag, sum=.TRUE., Vector=usevector, & + alltoall=.true., handshake=.true. ) endif ! Clean up space occupied by x' diff --git a/externals/mct/mct/m_Merge.F90 b/src/externals/mct/mct/m_Merge.F90 similarity index 100% rename from externals/mct/mct/m_Merge.F90 rename to src/externals/mct/mct/m_Merge.F90 diff --git a/externals/mct/mct/m_Navigator.F90 b/src/externals/mct/mct/m_Navigator.F90 similarity index 100% rename from externals/mct/mct/m_Navigator.F90 rename to src/externals/mct/mct/m_Navigator.F90 diff --git a/externals/mct/mct/m_Rearranger.F90 b/src/externals/mct/mct/m_Rearranger.F90 similarity index 93% rename from externals/mct/mct/m_Rearranger.F90 rename to src/externals/mct/mct/m_Rearranger.F90 index dc81dde9b5d..d75a4a54a1d 100644 --- a/externals/mct/mct/m_Rearranger.F90 +++ b/src/externals/mct/mct/m_Rearranger.F90 @@ -531,7 +531,8 @@ end subroutine clean_ ! ! !INTERFACE: - subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) + subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,& + Vector,AlltoAll,HandShake,ISend,MaxReq) ! ! !USES: @@ -548,6 +549,7 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) use m_AttrVect, only : nIAttr,nRAttr use m_AttrVect, only : Permute,Unpermute use m_Router, only : Router + use m_SPMDutils, only : m_swapm_int, m_swapm_FP use m_realkinds, only : FP use m_mpif90 use m_die @@ -567,6 +569,9 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) logical, optional, intent(in) :: Sum logical, optional, intent(in) :: Vector logical, optional, intent(in) :: AlltoAll + logical, optional, intent(in) :: HandShake + logical, optional, intent(in) :: ISend + integer, optional, intent(in) :: MaxReq ! !REVISION HISTORY: ! 31Jan02 - E.T. Ong - initial prototype @@ -581,6 +586,8 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) ! 14Oct06 - R. Jacob - check value of Sum argument. ! 25Jan08 - R. Jacob - Permute/unpermute if the internal ! routers permarr is defined. +! 29Sep16 - P. Worley - added swapm variant of +! alltoall option !EOP ___________________________________________________________________ character(len=*),parameter :: myname_=myname//'::Rearrange_' @@ -591,11 +598,11 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) integer :: mp_Type_rp integer :: mytag integer :: ISendSize, RSendSize, IRecvSize, RRecvSize - logical :: usevector, usealltoall + logical :: usevector, usealltoall, useswapm logical :: DoSum logical :: Sendunordered logical :: Recvunordered - real(FP) :: realtyp + real(FP):: realtyp !----------------------------------------------------------------------- ! DECLARE STRUCTURES FOR MPI ARGUMENTS. @@ -633,6 +640,10 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) integer,dimension(:),allocatable :: IRecvBuf real(FP),dimension(:),allocatable :: RRecvBuf + ! declare arrays to hold MPI data types for m_swapm_XXX calls + integer :: ITypes(0:max_nprocs-1) + integer :: RTypes(0:max_nprocs-1) + ! Structure to hold MPI request information for sends integer :: send_ireqs(max_nprocs) integer :: send_rreqs(max_nprocs) @@ -654,6 +665,16 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) type(AttrVect),pointer :: SourceAv type(AttrVect),target :: SourceAvtmp + ! local swapm protocol variables and defaults + logical,parameter :: DEF_SWAPM_HS = .true. + logical swapm_hs + + logical,parameter :: DEF_SWAPM_ISEND = .false. + logical swapm_isend + + integer,parameter :: DEF_SWAPM_MAXREQ = 512 + integer swapm_maxreq + !:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: Sendunordered=associated(InRearranger%SendRouter%permarr) @@ -708,8 +729,44 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) endif usealltoall=.false. - if(present(Alltoall)) then - if(Alltoall) usealltoall=.true. + if(present(AlltoAll)) then + if(AlltoAll) usealltoall=.true. + endif +!pw++ + ! forcing use of alltoall protocol until additional tuning + ! capabilities are added to calling routines +!pw usealltoall=.true. +!pw-- + + useswapm=.false. + if (usealltoall) then + ! if any swapm-related optional parameters are present, + ! enable swapm variant of alltoall + + swapm_hs = DEF_SWAPM_HS + if(present(HandShake)) then + if(HandShake) swapm_hs=.true. + useswapm=.true. + endif + + swapm_isend = DEF_SWAPM_ISEND + if(present(ISend)) then + if(ISend) swapm_isend=.true. + useswapm=.true. + endif + + swapm_maxreq = DEF_SWAPM_MAXREQ + if(present(MaxReq)) then + swapm_maxreq=MaxReq + useswapm=.true. + endif + +!pw++ + ! forcing use of swapm variant of alltoall protocol + ! until additional tuning capabilities are added to + ! calling routines +!pw useswapm=.true. +!pw-- endif DoSum=.false. @@ -735,9 +792,6 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) ! ALLOCATE DATA STRUCTURES ! - ! IF SENDING DATA - if(SendRout%nprocs > 0) then - ! IF SENDING INTEGER DATA if(numi .ge. 1) then @@ -769,10 +823,7 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) endif - endif - ! IF RECEVING DATA - if(RecvRout%nprocs > 0) then ! IF RECEIVING INTEGER DATA if(numi .ge. 1) then @@ -805,8 +856,6 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) endif - endif - !:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: ! INVERT PE LIST ! @@ -855,6 +904,10 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) RRdispls(pe) = RRecvLoc(proc) - 1 endif enddo + + ! SET MPI DATA TYPES + ITypes(:) = MP_INTEGER + RTypes(:) = mp_Type_rp endif !:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: @@ -1090,6 +1143,26 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) if (usealltoall) then + if (useswapm) then + + if (numi .ge. 1) then + call m_swapm_int(max_pe, myPid, & + ISendBuf, ISendSize, ISendCnts, ISdispls, ITypes, & + IRecvBuf, IRecvSize, IRecvCnts, IRdispls, ITypes, & + ThisMCTWorld%MCT_comm, & + swapm_hs, swapm_isend, swapm_maxreq ) + endif + + if (numr .ge. 1) then + call m_swapm_FP (max_pe, myPid, & + RSendBuf, RSendSize, RSendCnts, RSdispls, RTypes, & + RRecvBuf, RRecvSize, RRecvCnts, RRdispls, RTypes, & + ThisMCTWorld%MCT_comm, & + swapm_hs, swapm_isend, swapm_maxreq ) + endif + + else + if (numi .ge. 1) then call MPI_Alltoallv(ISendBuf, ISendCnts, ISdispls, MP_INTEGER, & IRecvBuf, IRecvCnts, IRdispls, MP_INTEGER, & @@ -1102,6 +1175,8 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) ThisMCTWorld%MCT_comm,ier) endif + endif + else ! WAIT FOR THE NONBLOCKING SENDS TO COMPLETE @@ -1250,30 +1325,12 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) ! DEALLOCATE ALL STRUCTURES - if(SendRout%nprocs > 0) then - if(numi .ge. 1) then ! Deallocate the send buffer deallocate(ISendBuf,stat=ier) if(ier/=0) call die(myname_,'deallocate(ISendBuf)',ier) - endif - - if(numr .ge. 1) then - - ! Deallocate the send buffer - deallocate(RSendBuf,stat=ier) - if(ier/=0) call die(myname_,'deallocate(RSendBuf)',ier) - - endif - - endif - - if(RecvRout%nprocs > 0) then - - if(numi .ge. 1) then - ! Deallocate the receive buffer deallocate(IRecvBuf,stat=ier) if(ier/=0) call die(myname_,'deallocate(IRecvBuf)',ier) @@ -1282,14 +1339,16 @@ subroutine rearrange_(SourceAVin,TargetAV,InRearranger,Tag,Sum,Vector,AlltoAll) if(numr .ge. 1) then + ! Deallocate the send buffer + deallocate(RSendBuf,stat=ier) + if(ier/=0) call die(myname_,'deallocate(RSendBuf)',ier) + ! Deallocate the receive buffer deallocate(RRecvBuf,stat=ier) if(ier/=0) call die(myname_,'deallocate(RRecvBuf)',ier) endif - endif - nullify(SendRout,RecvRout) end subroutine rearrange_ diff --git a/externals/mct/mct/m_Router.F90 b/src/externals/mct/mct/m_Router.F90 similarity index 74% rename from externals/mct/mct/m_Router.F90 rename to src/externals/mct/mct/m_Router.F90 index 305d2d7e389..f8788d608f7 100644 --- a/externals/mct/mct/m_Router.F90 +++ b/src/externals/mct/mct/m_Router.F90 @@ -268,6 +268,8 @@ subroutine initp_(inGSMap,inRGSMap,mycomm,Rout,name ) ! increasing. Instead, permute it to increasing and proceed. ! 07Sep12 - T. Craig - Replace a double loop with a single ! to improve speed for large proc and segment counts. +! 12Nov16 - P. Worley - eliminate iterations in nested +! loop that can be determined to be unnecessary !EOP ------------------------------------------------------------------- character(len=*),parameter :: myname_=myname//'::initp_' @@ -281,24 +283,30 @@ subroutine initp_(inGSMap,inRGSMap,mycomm,Rout,name ) integer :: my_left ! Left point in local segment (global memory) integer :: my_right ! Right point in local segment (global memory) + integer :: my_leftmost ! Leftmost point in local segments (global memory) + integer :: my_rightmost ! Rightmost point in local segments (global memory) integer :: r_left ! Left point in remote segment (global memory) integer :: r_right ! Right point in remote segment (global memory) + integer :: r_leftmost ! Leftmost point and rightmost point + integer :: r_rightmost ! in remote segments in given process (global memory) integer :: nsegs_overlap ! Number of segments that overlap between two procs integer :: ngseg, nlseg integer :: myseg, rseg - integer :: prev_right ! Rightmost local point in previous overlapped segment + integer :: rseg_leftbase, rseg_start + integer :: prev_right ! Rightmost local point in previous overlapped segment integer :: local_left, local_right integer,allocatable :: mygs_lb(:),mygs_ub(:),mygs_len(:),mygs_lstart(:) integer :: r_ngseg - integer :: r_max_nlseg ! max number of local segments in RGSMap integer,allocatable :: rgs_count(:),rgs_lb(:,:),rgs_ub(:,:) integer,allocatable :: nsegs_overlap_arr(:) integer :: overlap_left, overlap_right, overlap_diff integer :: proc, nprocs + integer :: feas_proc, feas_nprocs + integer,allocatable :: feas_procs(:), inv_feas_procs(:) integer :: max_rgs_count, max_overlap_segs type(GlobalSegMap) :: GSMap @@ -416,71 +424,109 @@ subroutine initp_(inGSMap,inRGSMap,mycomm,Rout,name ) call zeit_co(trim(tagname)) endif - !! -!! determine the segments in RGSMap that are local to each proc +!! determine the possibly overlapping segments +!! in RGSMap that are local to each proc !! - nprocs=ThisMCTWorld%nprocspid(othercomp) r_ngseg = GlobalSegMap_ngseg(RGSMap) - !! original size of rgs_lb()/ub() was (r_ngseg,nprocs) - !! at the cost of looping to compute it (within GlobalSegMap_max_nlseg), - !! reduced size to (r_max_nlseg,nprocs) - !! further reduction could be made by flattening it to one dimension - !! of size (r_ngseg) and allocating another array to index into it. - !! would not improve overall mem use unless this were also done for - !! tmpsegstart()/count() and possibly seg_starts()/lengths (the - !! latter would be a major change). + if (nlseg > 0) then + my_leftmost = mygs_lb(1) + my_rightmost = mygs_ub(nlseg) - if(present(name)) then - tagname='04'//name//'rloop' - call zeit_ci(trim(tagname)) - endif - r_max_nlseg = GlobalSegMap_max_nlseg(RGSMap) - - allocate( rgs_count(nprocs) , & - rgs_lb(r_max_nlseg,nprocs), rgs_ub(r_max_nlseg,nprocs), & - nsegs_overlap_arr(nprocs), stat=ier ) - if(ier/=0) call die(myname_,'allocate rgs, nsegs',ier) - -! tcraig, updated loop - rgs_count = 0 !! number of segments in RGSMap local to proc - - do i=1,r_ngseg - proc = RGSMap%pe_loc(i) + 1 -! if (proc < 1 .or. proc > nprocs) then -! write(stderr,*) myname_,"proc pe_loc error",i,proc -! call die(myname_,'pe_loc error',0) -! endif - rgs_count(proc) = rgs_count(proc) +1 - rgs_lb( rgs_count(proc) , proc )=RGSMap%start(i) - rgs_ub( rgs_count(proc) , proc )=RGSMap%start(i) + RGSMap%length(i) -1 - enddo +!! +!! count number of potentially overlapping remote segments +!! and which and how many processes hold these +!! + if(present(name)) then + tagname='04'//name//'rloop' + call zeit_ci(trim(tagname)) + endif - if(present(name)) then - call zeit_co(trim(tagname)) - endif + !! number of potentially overlapping segments in RGSMap local to proc + !! and mapping from processes that hold these to actual process id + allocate( rgs_count(nprocs), feas_procs(nprocs), & + inv_feas_procs(nprocs), stat=ier ) + if(ier/=0) call die(myname_,'allocate rgs_count, feas_procs',ier) + + rgs_count = 0 + do i=1,r_ngseg + r_left = RGSMap%start(i) + r_right = RGSMap%start(i) + RGSMap%length(i) - 1 + + if (.not. (my_rightmost < r_left .or. & ! potential overlap + my_leftmost > r_right ) ) then + proc = RGSMap%pe_loc(i) + 1 +! if (proc < 1 .or. proc > nprocs) then +! write(stderr,*) myname_,"proc pe_loc error",i,proc +! call die(myname_,'pe_loc error',0) +! endif + rgs_count(proc) = rgs_count(proc) + 1 + endif -!!! -!!! this is purely for error checking + enddo - if(present(name)) then - tagname='05'//name//'erchck' - call zeit_ci(trim(tagname)) - endif - do proc = 1, nprocs - if (rgs_count(proc) > r_max_nlseg) then - write(stderr,*) myname_,"overflow on rgs array",proc,rgs_count(proc) - call die(myname_,'overflow on rgs',0) + feas_nprocs = 0 + feas_procs = -1 + inv_feas_procs = -1 + do proc=1,nprocs + if (rgs_count(proc) > 0) then + feas_nprocs = feas_nprocs + 1 + feas_procs(feas_nprocs) = proc + inv_feas_procs(proc) = feas_nprocs + endif + enddo + +!! +!! build list of potentially overlapping remote segments +!! + !! original size of rgs_lb()/ub() was (r_ngseg,nprocs) + !! at the cost of looping to compute it (within GlobalSegMap_max_nlseg), + !! reduced size to (r_max_nlseg,nprocs) + !! then further reduced to (max_rgs_count,feas_nprocs) + + max_rgs_count=0 + do proc=1,nprocs + max_rgs_count = max( max_rgs_count, rgs_count(proc) ) + enddo + + allocate( rgs_lb(max_rgs_count,feas_nprocs), & + rgs_ub(max_rgs_count,feas_nprocs), & + nsegs_overlap_arr(feas_nprocs), stat=ier ) + if(ier/=0) call die(myname_,'allocate rgs, nsegs',ier) + + !! (note: redefining rgs_count to be indexed as 1:feas_nprocs + !! instead of as 1:nprocs) + rgs_count = 0 + do i=1,r_ngseg + r_left = RGSMap%start(i) + r_right = RGSMap%start(i) + RGSMap%length(i) -1 + + if (.not. (my_rightmost < r_left .or. & ! potential overlap + my_leftmost > r_right) ) then + proc = RGSMap%pe_loc(i) + 1 + feas_proc = inv_feas_procs(proc) + rgs_count(feas_proc) = rgs_count(feas_proc) + 1 + rgs_lb( rgs_count(feas_proc) , feas_proc ) = RGSMap%start(i) + rgs_ub( rgs_count(feas_proc) , feas_proc ) = RGSMap%start(i) + RGSMap%length(i) -1 + endif + + enddo + + deallocate(inv_feas_procs,stat=ier) + if(ier/=0) call die(myname_,'deallocate inv_feas_procs',ier) + + if(present(name)) then + call zeit_co(trim(tagname)) endif - enddo - if(present(name)) then - call zeit_co(trim(tagname)) - endif -!!! + else + + max_rgs_count = 0 + feas_nprocs = 0 + endif !!!!!!!!!!!!!!!!!! @@ -492,86 +538,97 @@ subroutine initp_(inGSMap,inRGSMap,mycomm,Rout,name ) tagname='06'//name//'loop2' call zeit_ci(trim(tagname)) endif - max_rgs_count=0 - do proc=1,nprocs - max_rgs_count = max( max_rgs_count, rgs_count(proc) ) - enddo max_overlap_segs = max(nlseg,max_rgs_count) - allocate(tmpsegcount(ThisMCTWorld%nprocspid(othercomp), max_overlap_segs),& - tmpsegstart(ThisMCTWorld%nprocspid(othercomp), max_overlap_segs),& - tmppe_list(ThisMCTWorld%nprocspid(othercomp)),stat=ier) + allocate(tmpsegcount(feas_nprocs, max_overlap_segs),& + tmpsegstart(feas_nprocs, max_overlap_segs),& + tmppe_list(feas_nprocs),stat=ier) if(ier/=0) & call die( myname_,'allocate tmpsegcount etc. size ', & - ThisMCTWorld%nprocspid(othercomp), & - ' by ',max_overlap_segs) + feas_nprocs, ' by ',max_overlap_segs) - - tmpsegcount=0 - tmpsegstart=0 + if (feas_nprocs > 0) then + tmpsegcount=0 + tmpsegstart=0 + endif count =0 maxsegcount=0 !!!!!!!!!!!!!!!!!! - - do proc = 1, nprocs + do feas_proc = 1, feas_nprocs nsegs_overlap = 0 - tmppe_list(proc) = .FALSE. ! no overlaps with proc yet + tmppe_list(feas_proc) = .FALSE. ! no overlaps with proc yet + + r_leftmost = rgs_lb(1,feas_proc) + r_rightmost = rgs_ub(rgs_count(feas_proc),feas_proc) + + rseg_leftbase = 0 + do myseg = 1, nlseg ! loop over local segs on 'myPID' + + my_left = mygs_lb(myseg) + my_right= mygs_ub(myseg) - if ( rgs_count(proc) > 0 ) then - do myseg = 1, nlseg ! loop over local segs on 'myPID' + ! determine whether any overlap + if (.not. (my_right < r_leftmost .or. & + my_left > r_rightmost) ) then - my_left = mygs_lb(myseg) - my_right= mygs_ub(myseg) + rseg_start = rseg_leftbase + 1 ! rseg loop index to start searching from - do rseg = 1, rgs_count(proc) ! loop over remote segs on 'proc' + ! loop over candidate overlapping remote segs on 'feas_proc' + do rseg = rseg_start, rgs_count(feas_proc) - r_left = rgs_lb(rseg,proc) - r_right = rgs_ub(rseg,proc) + r_right = rgs_ub(rseg,feas_proc) + if (r_right < my_left ) then ! to the left + rseg_leftbase = rseg ! remember to start to the right of + ! this for next myseg + cycle ! try the next remote segment + endif - if (.not. (my_right < r_left .or. & ! overlap - my_left > r_right) ) then + r_left = rgs_lb(rseg,feas_proc) + if (r_left > my_right) exit ! to the right, so no more segments + ! need to be examined - if (nsegs_overlap == 0) then ! first overlap w/this proc - count = count + 1 - tmppe_list(proc) = .TRUE. - prev_right = -9999 - else - prev_right = local_right - endif + ! otherwise, overlaps + if (nsegs_overlap == 0) then ! first overlap w/this proc + count = count + 1 + tmppe_list(feas_proc) = .TRUE. + prev_right = -9999 + else + prev_right = local_right + endif - overlap_left=max(my_left, r_left) - overlap_right=min(my_right, r_right) - overlap_diff= overlap_right - overlap_left + overlap_left=max(my_left, r_left) + overlap_right=min(my_right, r_right) + overlap_diff= overlap_right - overlap_left - local_left = mygs_lstart(myseg) + (overlap_left - my_left) - local_right = local_left + overlap_diff + local_left = mygs_lstart(myseg) + (overlap_left - my_left) + local_right = local_left + overlap_diff - ! non-contiguous w/prev one - if (local_left /= (prev_right+1) ) then - nsegs_overlap = nsegs_overlap + 1 - tmpsegstart(count, nsegs_overlap) = local_left - endif + ! non-contiguous w/prev one + if (local_left /= (prev_right+1) ) then + nsegs_overlap = nsegs_overlap + 1 + tmpsegstart(count, nsegs_overlap) = local_left + endif - tmpsegcount(count, nsegs_overlap) = & - tmpsegcount(count, nsegs_overlap) + overlap_diff + 1 + tmpsegcount(count, nsegs_overlap) = & + tmpsegcount(count, nsegs_overlap) + overlap_diff + 1 - endif enddo - enddo - endif - nsegs_overlap_arr(proc)=nsegs_overlap + endif + + enddo + + nsegs_overlap_arr(feas_proc)=nsegs_overlap enddo !! pull this out of the loop to vectorize - do proc=1,nprocs - maxsegcount=max(maxsegcount,nsegs_overlap_arr(proc)) + do feas_proc = 1, feas_nprocs + maxsegcount=max(maxsegcount,nsegs_overlap_arr(feas_proc)) enddo - if (maxsegcount > max_overlap_segs) & call die( myname_,'overran max_overlap_segs =', & max_overlap_segs, ' count = ',maxsegcount) @@ -581,11 +638,14 @@ subroutine initp_(inGSMap,inRGSMap,mycomm,Rout,name ) ! 'mysize =',mysize - deallocate( mygs_lb, mygs_ub, mygs_len, mygs_lstart, & - rgs_count, rgs_lb, rgs_ub, & - nsegs_overlap_arr, stat=ier) - if(ier/=0) call die(myname_,'deallocate mygs,rgs,nsegs',ier) + deallocate( mygs_lb, mygs_ub, mygs_len, mygs_lstart, stat=ier) + if(ier/=0) call die(myname_,'deallocate mygs,nsegs',ier) + if (nlseg > 0) then + deallocate( rgs_count, rgs_lb, rgs_ub, & + nsegs_overlap_arr, stat=ier) + if(ier/=0) call die(myname_,'deallocate p_rgs, nsegs',ier) + endif ! call shr_timer_stop(t_loop2) ! rml timers if(present(name)) then @@ -593,7 +653,6 @@ subroutine initp_(inGSMap,inRGSMap,mycomm,Rout,name ) endif - !. . . . . . . . . . . . . . . . . . . . . . . . @@ -631,43 +690,45 @@ subroutine initp_(inGSMap,inRGSMap,mycomm,Rout,name ) allocate(Rout%rp1(count),stat=ier) if(ier/=0) call die(myname_,'allocate(rp1)',ier) - - m=0 - do i=1,ThisMCTWorld%nprocspid(othercomp) - if(tmppe_list(i))then + do i=1,feas_nprocs + if(tmppe_list(i))then m=m+1 ! load processor rank in MCT_comm - Rout%pe_list(m)=ThisMCTWorld%idGprocid(othercomp,i-1) - endif - enddo + proc = feas_procs(i) + Rout%pe_list(m)=ThisMCTWorld%idGprocid(othercomp,proc-1) + endif + enddo - lmaxsize=0 - do i=1,count - totallength=0 - do j=1,maxsegcount - if(tmpsegcount(i,j) /= 0) then - Rout%num_segs(i)=j - Rout%seg_starts(i,j)=tmpsegstart(i,j) - Rout%seg_lengths(i,j)=tmpsegcount(i,j) - totallength=totallength+Rout%seg_lengths(i,j) - endif - enddo - Rout%locsize(i)=totallength - lmaxsize=MAX(lmaxsize,totallength) + lmaxsize=0 + do i=1,count + totallength=0 + do j=1,maxsegcount + if(tmpsegcount(i,j) /= 0) then + Rout%num_segs(i)=j + Rout%seg_starts(i,j)=tmpsegstart(i,j) + Rout%seg_lengths(i,j)=tmpsegcount(i,j) + totallength=totallength+Rout%seg_lengths(i,j) + endif enddo + Rout%locsize(i)=totallength + lmaxsize=MAX(lmaxsize,totallength) + enddo - Rout%maxsize=lmaxsize - Rout%lAvsize=mysize + Rout%maxsize=lmaxsize + Rout%lAvsize=mysize + if (nlseg > 0) then + deallocate(feas_procs,stat=ier) + if(ier/=0) call die(myname_,'deallocate feas_procs',ier) + endif deallocate(tmpsegstart,tmpsegcount,tmppe_list,stat=ier) - if(ier/=0) call die(myname_,'deallocate()',ier) + if(ier/=0) call die(myname_,'deallocate tmp',ier) call GlobalSegMap_clean(RGSMap) call GlobalSegMap_clean(GSMap) - if(present(name)) then call zeit_co(trim(tagname)) endif diff --git a/src/externals/mct/mct/m_SPMDutils.F90 b/src/externals/mct/mct/m_SPMDutils.F90 new file mode 100644 index 00000000000..d2bbd59cfa5 --- /dev/null +++ b/src/externals/mct/mct/m_SPMDutils.F90 @@ -0,0 +1,1148 @@ +!~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +! Math and Computer Science Division, Argonne National Laboratory ! +!----------------------------------------------------------------------- +! CVS $Id$ +! CVS $Name$ +!BOP ------------------------------------------------------------------- +! +! !MODULE: m_SPMDutils -- Communication operators to address performance +! issues for specific communication patterns +! +! !DESCRIPTION: +! This module provides the swapm equivalent to MPI_Alltoallv that +! has proven to be more robust with respect to performance than the +! MPI collective or the native MCT communication algorithms when the +! communication pattern is sparse and when load imbalance or send/receive +! asymmetry leads some processes to be flooded by unexpected messages. +! +! Based on algorithms implemented in CAM, but this version modelled after +! pio_spmd_utils.F90 in PIO1 +! +! !SEE ALSO: +! m_Rearranger +! +! +! !INTERFACE: + +! Disable the use of the MPI ready send protocol by default, to +! address recurrent issues with poor performance or incorrect +! functionality in MPI libraries. When support is known to be robust, +! or for experimentation, can be re-enabled by defining the CPP token +! _USE_MPI_RSEND during the build process. +! +#ifndef _USE_MPI_RSEND +#define MPI_RSEND MPI_SEND +#define mpi_rsend mpi_send +#define MPI_IRSEND MPI_ISEND +#define mpi_irsend mpi_isend +#endif + + module m_SPMDutils + + implicit none + + private ! except + +! !PUBLIC MEMBER FUNCTIONS: + + public :: m_swapm_int ! swapm alternative to MPI_AlltoallV for integer data + public :: m_swapm_FP ! swapm alternative to MPI_AlltoallV for FP data + +! !DEFINED PARAMETERS: + + character(len=*), parameter :: myname='MCT::m_SPMDutils' + +! !REVISION HISTORY: +! 28Sep16 - P. Worley - initial prototype +!EOP ___________________________________________________________________ + + contains + +!======================================================================== +! + + integer function pair(np,p,k) + + integer np,p,k,q + q = ieor(p,k) + if(q.gt.np-1) then + pair = -1 + else + pair = q + endif + return + + end function pair + +! +!======================================================================== +! + + integer function ceil2(n) + integer n,p + p=1 + do while(p.lt.n) + p=p*2 + enddo + ceil2=p + return + end function ceil2 + +! +!======================================================================== +! + subroutine m_swapm_int ( nprocs, mytask, & + sndbuf, sbuf_siz, sndlths, sdispls, stypes, & + rcvbuf, rbuf_siz, rcvlths, rdispls, rtypes, & + comm, comm_hs, comm_isend, comm_maxreq ) + +!----------------------------------------------------------------------- +! +!> Purpose: +!! Reduced version of original swapm (for swap of multiple messages +!! using MPI point-to-point routines), more efficiently implementing a +!! subset of the swap protocols. +!! +!! Method: +!! comm_protocol: +!! comm_isend == .true.: use nonblocking send, else use blocking send +!! comm_hs == .true.: use handshaking protocol +!! comm_maxreq: +!! =-1,0: do not limit number of outstanding send/receive requests +!! >0: do not allow more than min(comm_maxreq, steps) outstanding +!! nonblocking send requests or nonblocking receive requests +!! +!! Author of original version: P. Worley +!! Ported from PIO1: P. Worley, September 2016 +!< +!----------------------------------------------------------------------- +!----------------------------------------------------------------------- + use m_mpif90 + use m_realkinds, only : FP + use m_die, only : MP_perr_die + + implicit none +!---------------------------Input arguments-------------------------- +! + integer, intent(in) :: nprocs ! size of communicator + integer, intent(in) :: mytask ! MPI task id with communicator + integer, intent(in) :: sbuf_siz ! size of send buffer + integer, intent(in) :: rbuf_siz ! size of receive buffer + + integer, intent(in) :: sndlths(0:nprocs-1)! length of outgoing message + integer, intent(in) :: sdispls(0:nprocs-1)! offset from beginning of send + ! buffer where outgoing messages + ! should be sent from + integer, intent(in) :: stypes(0:nprocs-1) ! MPI data types + integer, intent(in) :: rcvlths(0:nprocs-1)! length of incoming messages + integer, intent(in) :: rdispls(0:nprocs-1)! offset from beginning of receive + ! buffer where incoming messages + ! should be placed + integer, intent(in) :: rtypes(0:nprocs-1) ! MPI data types + integer, intent(in) :: sndbuf(sbuf_siz) ! outgoing message buffer + + integer, intent(in) :: comm ! MPI communicator + logical, intent(in) :: comm_hs ! handshaking protocol? + logical, intent(in) :: comm_isend ! nonblocking send protocol? + integer, intent(in) :: comm_maxreq ! maximum number of outstanding + ! nonblocking requests + +!---------------------------Output arguments-------------------------- +! + integer, intent(out) :: rcvbuf(rbuf_siz) ! incoming message buffer + +! +!---------------------------Local workspace------------------------------------------- +! + character(len=*), parameter :: subName=myname//'::m_swapm_int' + + integer :: steps ! number of swaps to initiate + integer :: swapids(nprocs) ! MPI process id of swap partners + integer :: p ! process index + integer :: istep ! loop index + integer :: tag ! MPI message tag + integer :: offset_t ! MPI message tag offset, for addressing + ! message conflict bug (if necessary) + integer :: offset_s ! index of message beginning in + ! send buffer + integer :: offset_r ! index of message beginning in + ! receive buffer + integer :: sndids(nprocs) ! send request ids + integer :: rcvids(nprocs) ! receive request ids + integer :: hs_rcvids(nprocs) ! handshake receive request ids + + integer :: maxreq, maxreqh ! maximum number of outstanding + ! nonblocking requests (and half) + integer :: hs ! handshake variable + integer :: rstep ! "receive" step index + + logical :: handshake, sendd ! protocol option flags + + integer :: ier ! return error status + integer :: status(MP_STATUS_SIZE) ! MPI status +! +!------------------------------------------------------------------------------------- +! +#ifdef _NO_M_SWAPM_TAG_OFFSET + offset_t = 0 +#else + offset_t = nprocs +#endif +! + ! if necessary, send to self + if (sndlths(mytask) > 0) then + tag = mytask + offset_t + + offset_r = rdispls(mytask)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(mytask), rtypes(mytask), & + mytask, tag, comm, rcvids(1), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + + offset_s = sdispls(mytask)+1 + call mpi_send( sndbuf(offset_s), sndlths(mytask), stypes(mytask), & + mytask, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + + call mpi_wait( rcvids(1), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + ! calculate swap partners and communication ordering + steps = 0 + do istep=1,ceil2(nprocs)-1 + p = pair(nprocs,istep,mytask) + if (p >= 0) then + if (sndlths(p) > 0 .or. rcvlths(p) > 0) then + steps = steps + 1 + swapids(steps) = p + end if + end if + end do + + if (steps .eq. 0) return + + ! identify communication protocol + if (comm_isend) then + sendd = .false. + else + sendd = .true. + endif + handshake = comm_hs + + ! identify maximum number of outstanding nonblocking requests to permit + if (steps .eq. 1) then + maxreq = 1 + maxreqh = 1 + else + if (comm_maxreq >= -1) then + maxreq = comm_maxreq + else + maxreq = steps + endif + + if ((maxreq .le. steps) .and. (maxreq > 0)) then + if (maxreq > 1) then + maxreqh = maxreq/2 + else + maxreq = 2 + maxreqh = 1 + endif + else + maxreq = steps + maxreqh = steps + endif + endif + +! Four protocol options: +! (1) handshaking + blocking sends + if ((handshake) .and. (sendd)) then + + ! Initialize hs variable + hs = 1 + + ! Post initial handshake receive requests + do istep=1,maxreq + p = swapids(istep) + if (sndlths(p) > 0) then + tag = mytask + offset_t + call mpi_irecv( hs, 1, MP_INTEGER, p, tag, comm, & + hs_rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + enddo + + ! Post initial receive requests + do istep=1,maxreq + p = swapids(istep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + + call mpi_send ( hs, 1, MP_INTEGER, p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + endif + enddo + rstep = maxreq + + ! Send (and start receiving) data + do istep=1,steps + p = swapids(istep) + + ! Submit new rsend request + if (sndlths(p) > 0) then + tag = mytask + offset_t + + offset_s = sdispls(p)+1 + call mpi_wait ( hs_rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + + call mpi_rsend ( sndbuf(offset_s), sndlths(p), stypes(p), & + p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_RSEND',ier) + endif + + if (istep > maxreqh) then + + ! Wait for oldest irecv request to complete + p = swapids(istep-maxreqh) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + if (rstep < steps) then + rstep = rstep + 1 + p = swapids(rstep) + + ! Submit a new handshake irecv request + if (sndlths(p) > 0) then + tag = mytask + offset_t + call mpi_irecv( hs, 1, MP_INTEGER, p, tag, comm, & + hs_rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + + ! Submit a new irecv request + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + + call mpi_send ( hs, 1, MP_INTEGER, p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + endif + endif + + endif +! + enddo + + ! wait for rest of receive requests to complete + do istep=steps-maxreqh+1,steps + p = swapids(istep) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + enddo + +! (2) handshaking + nonblocking sends + elseif ((handshake) .and. (.not. sendd)) then + + ! Initialize hs variable + hs = 1 + + ! Post initial handshake receive requests + do istep=1,maxreq + p = swapids(istep) + if (sndlths(p) > 0) then + tag = mytask + offset_t + call mpi_irecv( hs, 1, MP_INTEGER, p, tag, comm, & + hs_rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + enddo + + ! Post initial receive requests + do istep=1,maxreq + p = swapids(istep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + + call mpi_send ( hs, 1, MP_INTEGER, p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + endif + enddo + rstep = maxreq + + ! Send (and start receiving) data + do istep=1,steps + p = swapids(istep) + + ! Submit new irsend request + if (sndlths(p) > 0) then + tag = mytask + offset_t + + offset_s = sdispls(p)+1 + call mpi_wait ( hs_rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + + call mpi_irsend( sndbuf(offset_s), sndlths(p), stypes(p), & + p, tag, comm, sndids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRSEND',ier) + endif + + if (istep > maxreqh) then + + ! Wait for oldest irecv request to complete + p = swapids(istep-maxreqh) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + if (rstep < steps) then + rstep = rstep + 1 + p = swapids(rstep) + + ! Submit a new handshake irecv request + if (sndlths(p) > 0) then + tag = mytask + offset_t + call mpi_irecv( hs, 1, MP_INTEGER, p, tag, comm, & + hs_rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + + ! Submit a new irecv request + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + + call mpi_send ( hs, 1, MP_INTEGER, p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + endif + endif + + ! Wait for outstanding i(r)send request to complete + p = swapids(istep-maxreqh) + if (sndlths(p) > 0) then + call mpi_wait( sndids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + endif + + enddo + + ! wait for rest of send and receive requests to complete + do istep=steps-maxreqh+1,steps + p = swapids(istep) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + if (sndlths(p) > 0) then + call mpi_wait( sndids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + enddo + +! (3) no handshaking + blocking sends + elseif ((.not. handshake) .and. (sendd)) then + + ! Post receive requests + do istep=1,maxreq + p = swapids(istep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + enddo + rstep = maxreq + + ! Send (and start receiving) data + do istep=1,steps + p = swapids(istep) + + ! Submit new send request + if (sndlths(p) > 0) then + tag = mytask + offset_t + + offset_s = sdispls(p)+1 + call mpi_send( sndbuf(offset_s), sndlths(p), stypes(p), & + p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + endif + + if (istep > maxreqh) then + + ! Wait for oldest irecv request to complete + p = swapids(istep-maxreqh) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + ! Submit a new irecv request + if (rstep < steps) then + rstep = rstep + 1 + p = swapids(rstep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + endif + + endif + + enddo + + ! wait for rest of send and receive requests to complete + do istep=steps-maxreqh+1,steps + p = swapids(istep) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + enddo + +! (4) no handshaking + nonblocking sends + elseif ((.not. handshake) .and. (.not. sendd)) then + + ! Post receive requests + do istep=1,maxreq + p = swapids(istep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + enddo + rstep = maxreq + + ! Send (and start receiving) data + do istep=1,steps + p = swapids(istep) + + ! Submit new isend request + if (sndlths(p) > 0) then + tag = mytask + offset_t + + offset_s = sdispls(p)+1 + call mpi_isend( sndbuf(offset_s), sndlths(p), stypes(p), & + p, tag, comm, sndids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_ISEND',ier) + endif + + if (istep > maxreqh) then + + ! Wait for oldest irecv request to complete + p = swapids(istep-maxreqh) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + ! Submit a new irecv request + if (rstep < steps) then + rstep = rstep + 1 + p = swapids(rstep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + endif + + ! Wait for outstanding i(r)send request to complete + p = swapids(istep-maxreqh) + if (sndlths(p) > 0) then + call mpi_wait( sndids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + endif + + enddo + + ! wait for rest of send and receive requests to complete + do istep=steps-maxreqh+1,steps + p = swapids(istep) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + if (sndlths(p) > 0) then + call mpi_wait( sndids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + enddo + + endif + + return + + end subroutine m_swapm_int + +! +!======================================================================== +! + subroutine m_swapm_FP ( nprocs, mytask, & + sndbuf, sbuf_siz, sndlths, sdispls, stypes, & + rcvbuf, rbuf_siz, rcvlths, rdispls, rtypes, & + comm, comm_hs, comm_isend, comm_maxreq ) + +!----------------------------------------------------------------------- +! +!> Purpose: +!! Reduced version of original swapm (for swap of multiple messages +!! using MPI point-to-point routines), more efficiently implementing a +!! subset of the swap protocols. +!! +!! Method: +!! comm_protocol: +!! comm_isend == .true.: use nonblocking send, else use blocking send +!! comm_hs == .true.: use handshaking protocol +!! comm_maxreq: +!! =-1,0: do not limit number of outstanding send/receive requests +!! >0: do not allow more than min(comm_maxreq, steps) outstanding +!! nonblocking send requests or nonblocking receive requests +!! +!! Author of original version: P. Worley +!! Ported from PIO1: P. Worley, September 2016 +!< +!----------------------------------------------------------------------- +!----------------------------------------------------------------------- + use m_mpif90 + use m_realkinds, only : FP + use m_die, only : MP_perr_die + + implicit none +!---------------------------Input arguments-------------------------- +! + integer, intent(in) :: nprocs ! size of communicator + integer, intent(in) :: mytask ! MPI task id with communicator + integer, intent(in) :: sbuf_siz ! size of send buffer + integer, intent(in) :: rbuf_siz ! size of receive buffer + + integer, intent(in) :: sndlths(0:nprocs-1)! length of outgoing message + integer, intent(in) :: sdispls(0:nprocs-1)! offset from beginning of send + ! buffer where outgoing messages + ! should be sent from + integer, intent(in) :: stypes(0:nprocs-1) ! MPI data types + integer, intent(in) :: rcvlths(0:nprocs-1)! length of incoming messages + integer, intent(in) :: rdispls(0:nprocs-1)! offset from beginning of receive + ! buffer where incoming messages + ! should be placed + integer, intent(in) :: rtypes(0:nprocs-1) ! MPI data types + real(FP),intent(in) :: sndbuf(sbuf_siz) ! outgoing message buffer + + integer, intent(in) :: comm ! MPI communicator + logical, intent(in) :: comm_hs ! handshaking protocol? + logical, intent(in) :: comm_isend ! nonblocking send protocol? + integer, intent(in) :: comm_maxreq ! maximum number of outstanding + ! nonblocking requests + +!---------------------------Output arguments-------------------------- +! + real(FP), intent(out) :: rcvbuf(rbuf_siz) ! incoming message buffer + +! +!---------------------------Local workspace------------------------------------------- +! + character(len=*), parameter :: subName=myname//'::m_swapm_FP' + + integer :: steps ! number of swaps to initiate + integer :: swapids(nprocs) ! MPI process id of swap partners + integer :: p ! process index + integer :: istep ! loop index + integer :: tag ! MPI message tag + integer :: offset_t ! MPI message tag offset, for addressing + ! message conflict bug (if necessary) + integer :: offset_s ! index of message beginning in + ! send buffer + integer :: offset_r ! index of message beginning in + ! receive buffer + integer :: sndids(nprocs) ! send request ids + integer :: rcvids(nprocs) ! receive request ids + integer :: hs_rcvids(nprocs) ! handshake receive request ids + + integer :: maxreq, maxreqh ! maximum number of outstanding + ! nonblocking requests (and half) + integer :: hs ! handshake variable + integer :: rstep ! "receive" step index + + logical :: handshake, sendd ! protocol option flags + + integer :: ier ! return error status + integer :: status(MP_STATUS_SIZE) ! MPI status +! +!------------------------------------------------------------------------------------- +! +#ifdef _NO_M_SWAPM_TAG_OFFSET + offset_t = 0 +#else + offset_t = nprocs +#endif +! + ! if necessary, send to self + if (sndlths(mytask) > 0) then + tag = mytask + offset_t + + offset_r = rdispls(mytask)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(mytask), rtypes(mytask), & + mytask, tag, comm, rcvids(1), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + + offset_s = sdispls(mytask)+1 + call mpi_send( sndbuf(offset_s), sndlths(mytask), stypes(mytask), & + mytask, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + + call mpi_wait( rcvids(1), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + ! calculate swap partners and communication ordering + steps = 0 + do istep=1,ceil2(nprocs)-1 + p = pair(nprocs,istep,mytask) + if (p >= 0) then + if (sndlths(p) > 0 .or. rcvlths(p) > 0) then + steps = steps + 1 + swapids(steps) = p + end if + end if + end do + + if (steps .eq. 0) return + + ! identify communication protocol + if (comm_isend) then + sendd = .false. + else + sendd = .true. + endif + handshake = comm_hs + + ! identify maximum number of outstanding nonblocking requests to permit + if (steps .eq. 1) then + maxreq = 1 + maxreqh = 1 + else + if (comm_maxreq >= -1) then + maxreq = comm_maxreq + else + maxreq = steps + endif + + if ((maxreq .le. steps) .and. (maxreq > 0)) then + if (maxreq > 1) then + maxreqh = maxreq/2 + else + maxreq = 2 + maxreqh = 1 + endif + else + maxreq = steps + maxreqh = steps + endif + endif + +! Four protocol options: +! (1) handshaking + blocking sends + if ((handshake) .and. (sendd)) then + + ! Initialize hs variable + hs = 1 + + ! Post initial handshake receive requests + do istep=1,maxreq + p = swapids(istep) + if (sndlths(p) > 0) then + tag = mytask + offset_t + call mpi_irecv( hs, 1, MP_INTEGER, p, tag, comm, & + hs_rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + enddo + + ! Post initial receive requests + do istep=1,maxreq + p = swapids(istep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + + call mpi_send ( hs, 1, MP_INTEGER, p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + endif + enddo + rstep = maxreq + + ! Send (and start receiving) data + do istep=1,steps + p = swapids(istep) + + ! Submit new rsend request + if (sndlths(p) > 0) then + tag = mytask + offset_t + + offset_s = sdispls(p)+1 + call mpi_wait ( hs_rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + + call mpi_rsend ( sndbuf(offset_s), sndlths(p), stypes(p), & + p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_RSEND',ier) + endif + + if (istep > maxreqh) then + + ! Wait for oldest irecv request to complete + p = swapids(istep-maxreqh) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + if (rstep < steps) then + rstep = rstep + 1 + p = swapids(rstep) + + ! Submit a new handshake irecv request + if (sndlths(p) > 0) then + tag = mytask + offset_t + call mpi_irecv( hs, 1, MP_INTEGER, p, tag, comm, & + hs_rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + + ! Submit a new irecv request + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + + call mpi_send ( hs, 1, MP_INTEGER, p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + endif + endif + + endif +! + enddo + + ! wait for rest of receive requests to complete + do istep=steps-maxreqh+1,steps + p = swapids(istep) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + enddo + +! (2) handshaking + nonblocking sends + elseif ((handshake) .and. (.not. sendd)) then + + ! Initialize hs variable + hs = 1 + + ! Post initial handshake receive requests + do istep=1,maxreq + p = swapids(istep) + if (sndlths(p) > 0) then + tag = mytask + offset_t + call mpi_irecv( hs, 1, MP_INTEGER, p, tag, comm, & + hs_rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + enddo + + ! Post initial receive requests + do istep=1,maxreq + p = swapids(istep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + + call mpi_send ( hs, 1, MP_INTEGER, p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + endif + enddo + rstep = maxreq + + ! Send (and start receiving) data + do istep=1,steps + p = swapids(istep) + + ! Submit new irsend request + if (sndlths(p) > 0) then + tag = mytask + offset_t + + offset_s = sdispls(p)+1 + call mpi_wait ( hs_rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + + call mpi_irsend( sndbuf(offset_s), sndlths(p), stypes(p), & + p, tag, comm, sndids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRSEND',ier) + endif + + if (istep > maxreqh) then + + ! Wait for oldest irecv request to complete + p = swapids(istep-maxreqh) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + if (rstep < steps) then + rstep = rstep + 1 + p = swapids(rstep) + + ! Submit a new handshake irecv request + if (sndlths(p) > 0) then + tag = mytask + offset_t + call mpi_irecv( hs, 1, MP_INTEGER, p, tag, comm, & + hs_rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + + ! Submit a new irecv request + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + + call mpi_send ( hs, 1, MP_INTEGER, p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + endif + endif + + ! Wait for outstanding i(r)send request to complete + p = swapids(istep-maxreqh) + if (sndlths(p) > 0) then + call mpi_wait( sndids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + endif + + enddo + + ! wait for rest of send and receive requests to complete + do istep=steps-maxreqh+1,steps + p = swapids(istep) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + if (sndlths(p) > 0) then + call mpi_wait( sndids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + enddo + +! (3) no handshaking + blocking sends + elseif ((.not. handshake) .and. (sendd)) then + + ! Post receive requests + do istep=1,maxreq + p = swapids(istep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + enddo + rstep = maxreq + + ! Send (and start receiving) data + do istep=1,steps + p = swapids(istep) + + ! Submit new send request + if (sndlths(p) > 0) then + tag = mytask + offset_t + + offset_s = sdispls(p)+1 + call mpi_send( sndbuf(offset_s), sndlths(p), stypes(p), & + p, tag, comm, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_SEND',ier) + endif + + if (istep > maxreqh) then + + ! Wait for oldest irecv request to complete + p = swapids(istep-maxreqh) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + ! Submit a new irecv request + if (rstep < steps) then + rstep = rstep + 1 + p = swapids(rstep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + endif + + endif + + enddo + + ! wait for rest of send and receive requests to complete + do istep=steps-maxreqh+1,steps + p = swapids(istep) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + enddo + +! (4) no handshaking + nonblocking sends + elseif ((.not. handshake) .and. (.not. sendd)) then + + ! Post receive requests + do istep=1,maxreq + p = swapids(istep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + enddo + rstep = maxreq + + ! Send (and start receiving) data + do istep=1,steps + p = swapids(istep) + + ! Submit new isend request + if (sndlths(p) > 0) then + tag = mytask + offset_t + + offset_s = sdispls(p)+1 + call mpi_isend( sndbuf(offset_s), sndlths(p), stypes(p), & + p, tag, comm, sndids(istep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_ISEND',ier) + endif + + if (istep > maxreqh) then + + ! Wait for oldest irecv request to complete + p = swapids(istep-maxreqh) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + ! Submit a new irecv request + if (rstep < steps) then + rstep = rstep + 1 + p = swapids(rstep) + if (rcvlths(p) > 0) then + tag = p + offset_t + + offset_r = rdispls(p)+1 + call mpi_irecv( rcvbuf(offset_r), rcvlths(p), rtypes(p), & + p, tag, comm, rcvids(rstep), ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_IRECV',ier) + endif + endif + + ! Wait for outstanding i(r)send request to complete + p = swapids(istep-maxreqh) + if (sndlths(p) > 0) then + call mpi_wait( sndids(istep-maxreqh), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + + endif + + enddo + + ! wait for rest of send and receive requests to complete + do istep=steps-maxreqh+1,steps + p = swapids(istep) + if (rcvlths(p) > 0) then + call mpi_wait( rcvids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + if (sndlths(p) > 0) then + call mpi_wait( sndids(istep), status, ier ) + if(ier /= 0) call MP_perr_die(subName,'MPI_WAIT',ier) + endif + enddo + + endif + + return + + end subroutine m_swapm_FP + +end module m_SPMDutils + + + + + diff --git a/externals/mct/mct/m_SparseMatrix.F90 b/src/externals/mct/mct/m_SparseMatrix.F90 similarity index 100% rename from externals/mct/mct/m_SparseMatrix.F90 rename to src/externals/mct/mct/m_SparseMatrix.F90 diff --git a/externals/mct/mct/m_SparseMatrixComms.F90 b/src/externals/mct/mct/m_SparseMatrixComms.F90 similarity index 100% rename from externals/mct/mct/m_SparseMatrixComms.F90 rename to src/externals/mct/mct/m_SparseMatrixComms.F90 diff --git a/externals/mct/mct/m_SparseMatrixDecomp.F90 b/src/externals/mct/mct/m_SparseMatrixDecomp.F90 similarity index 100% rename from externals/mct/mct/m_SparseMatrixDecomp.F90 rename to src/externals/mct/mct/m_SparseMatrixDecomp.F90 diff --git a/externals/mct/mct/m_SparseMatrixPlus.F90 b/src/externals/mct/mct/m_SparseMatrixPlus.F90 similarity index 100% rename from externals/mct/mct/m_SparseMatrixPlus.F90 rename to src/externals/mct/mct/m_SparseMatrixPlus.F90 diff --git a/externals/mct/mct/m_SparseMatrixToMaps.F90 b/src/externals/mct/mct/m_SparseMatrixToMaps.F90 similarity index 100% rename from externals/mct/mct/m_SparseMatrixToMaps.F90 rename to src/externals/mct/mct/m_SparseMatrixToMaps.F90 diff --git a/externals/mct/mct/m_SpatialIntegral.F90 b/src/externals/mct/mct/m_SpatialIntegral.F90 similarity index 100% rename from externals/mct/mct/m_SpatialIntegral.F90 rename to src/externals/mct/mct/m_SpatialIntegral.F90 diff --git a/externals/mct/mct/m_SpatialIntegralV.F90 b/src/externals/mct/mct/m_SpatialIntegralV.F90 similarity index 100% rename from externals/mct/mct/m_SpatialIntegralV.F90 rename to src/externals/mct/mct/m_SpatialIntegralV.F90 diff --git a/externals/mct/mct/m_Transfer.F90 b/src/externals/mct/mct/m_Transfer.F90 similarity index 100% rename from externals/mct/mct/m_Transfer.F90 rename to src/externals/mct/mct/m_Transfer.F90 diff --git a/externals/mct/mkinstalldirs b/src/externals/mct/mkinstalldirs similarity index 100% rename from externals/mct/mkinstalldirs rename to src/externals/mct/mkinstalldirs diff --git a/externals/mct/mpeu/Makefile b/src/externals/mct/mpeu/Makefile similarity index 100% rename from externals/mct/mpeu/Makefile rename to src/externals/mct/mpeu/Makefile diff --git a/externals/mct/mpeu/README b/src/externals/mct/mpeu/README similarity index 100% rename from externals/mct/mpeu/README rename to src/externals/mct/mpeu/README diff --git a/externals/mct/mpeu/assertmpeu.H b/src/externals/mct/mpeu/assertmpeu.H similarity index 100% rename from externals/mct/mpeu/assertmpeu.H rename to src/externals/mct/mpeu/assertmpeu.H diff --git a/externals/mct/mpeu/get_zeits.c b/src/externals/mct/mpeu/get_zeits.c similarity index 100% rename from externals/mct/mpeu/get_zeits.c rename to src/externals/mct/mpeu/get_zeits.c diff --git a/externals/mct/mpeu/m_FcComms.F90 b/src/externals/mct/mpeu/m_FcComms.F90 similarity index 98% rename from externals/mct/mpeu/m_FcComms.F90 rename to src/externals/mct/mpeu/m_FcComms.F90 index bdd76ea95ec..0bd675c2b75 100644 --- a/externals/mct/mpeu/m_FcComms.F90 +++ b/src/externals/mct/mpeu/m_FcComms.F90 @@ -13,10 +13,13 @@ ! ! !INTERFACE: ! -! Workaround for performance issue with rsend on cray systems with -! gemini interconnect +! Disable the use of the MPI ready send protocol by default, to +! address recurrent issues with poor performance or incorrect +! functionality in MPI libraries. When support is known to be robust, +! or for experimentation, can be re-enabled by defining the CPP token +! _USE_MPI_RSEND during the build process. ! -#ifdef _NO_MPI_RSEND +#ifndef _USE_MPI_RSEND #define MPI_RSEND MPI_SEND #define mpi_rsend mpi_send #define MPI_IRSEND MPI_ISEND diff --git a/externals/mct/mpeu/m_FileResolv.F90 b/src/externals/mct/mpeu/m_FileResolv.F90 similarity index 100% rename from externals/mct/mpeu/m_FileResolv.F90 rename to src/externals/mct/mpeu/m_FileResolv.F90 diff --git a/externals/mct/mpeu/m_Filename.F90 b/src/externals/mct/mpeu/m_Filename.F90 similarity index 100% rename from externals/mct/mpeu/m_Filename.F90 rename to src/externals/mct/mpeu/m_Filename.F90 diff --git a/externals/mct/mpeu/m_IndexBin_char.F90 b/src/externals/mct/mpeu/m_IndexBin_char.F90 similarity index 100% rename from externals/mct/mpeu/m_IndexBin_char.F90 rename to src/externals/mct/mpeu/m_IndexBin_char.F90 diff --git a/externals/mct/mpeu/m_IndexBin_integer.F90 b/src/externals/mct/mpeu/m_IndexBin_integer.F90 similarity index 100% rename from externals/mct/mpeu/m_IndexBin_integer.F90 rename to src/externals/mct/mpeu/m_IndexBin_integer.F90 diff --git a/externals/mct/mpeu/m_IndexBin_logical.F90 b/src/externals/mct/mpeu/m_IndexBin_logical.F90 similarity index 100% rename from externals/mct/mpeu/m_IndexBin_logical.F90 rename to src/externals/mct/mpeu/m_IndexBin_logical.F90 diff --git a/externals/mct/mpeu/m_List.F90 b/src/externals/mct/mpeu/m_List.F90 similarity index 100% rename from externals/mct/mpeu/m_List.F90 rename to src/externals/mct/mpeu/m_List.F90 diff --git a/externals/mct/mpeu/m_MergeSorts.F90 b/src/externals/mct/mpeu/m_MergeSorts.F90 similarity index 100% rename from externals/mct/mpeu/m_MergeSorts.F90 rename to src/externals/mct/mpeu/m_MergeSorts.F90 diff --git a/externals/mct/mpeu/m_Permuter.F90 b/src/externals/mct/mpeu/m_Permuter.F90 similarity index 100% rename from externals/mct/mpeu/m_Permuter.F90 rename to src/externals/mct/mpeu/m_Permuter.F90 diff --git a/externals/mct/mpeu/m_SortingTools.F90 b/src/externals/mct/mpeu/m_SortingTools.F90 similarity index 100% rename from externals/mct/mpeu/m_SortingTools.F90 rename to src/externals/mct/mpeu/m_SortingTools.F90 diff --git a/externals/mct/mpeu/m_StrTemplate.F90 b/src/externals/mct/mpeu/m_StrTemplate.F90 similarity index 100% rename from externals/mct/mpeu/m_StrTemplate.F90 rename to src/externals/mct/mpeu/m_StrTemplate.F90 diff --git a/externals/mct/mpeu/m_String.F90 b/src/externals/mct/mpeu/m_String.F90 similarity index 100% rename from externals/mct/mpeu/m_String.F90 rename to src/externals/mct/mpeu/m_String.F90 diff --git a/externals/mct/mpeu/m_StringLinkedList.F90 b/src/externals/mct/mpeu/m_StringLinkedList.F90 similarity index 100% rename from externals/mct/mpeu/m_StringLinkedList.F90 rename to src/externals/mct/mpeu/m_StringLinkedList.F90 diff --git a/externals/mct/mpeu/m_TraceBack.F90 b/src/externals/mct/mpeu/m_TraceBack.F90 similarity index 100% rename from externals/mct/mpeu/m_TraceBack.F90 rename to src/externals/mct/mpeu/m_TraceBack.F90 diff --git a/externals/mct/mpeu/m_chars.F90 b/src/externals/mct/mpeu/m_chars.F90 similarity index 100% rename from externals/mct/mpeu/m_chars.F90 rename to src/externals/mct/mpeu/m_chars.F90 diff --git a/externals/mct/mpeu/m_die.F90 b/src/externals/mct/mpeu/m_die.F90 similarity index 100% rename from externals/mct/mpeu/m_die.F90 rename to src/externals/mct/mpeu/m_die.F90 diff --git a/externals/mct/mpeu/m_dropdead.F90 b/src/externals/mct/mpeu/m_dropdead.F90 similarity index 100% rename from externals/mct/mpeu/m_dropdead.F90 rename to src/externals/mct/mpeu/m_dropdead.F90 diff --git a/externals/mct/mpeu/m_flow.F90 b/src/externals/mct/mpeu/m_flow.F90 similarity index 100% rename from externals/mct/mpeu/m_flow.F90 rename to src/externals/mct/mpeu/m_flow.F90 diff --git a/externals/mct/mpeu/m_inpak90.F90 b/src/externals/mct/mpeu/m_inpak90.F90 similarity index 100% rename from externals/mct/mpeu/m_inpak90.F90 rename to src/externals/mct/mpeu/m_inpak90.F90 diff --git a/externals/mct/mpeu/m_ioutil.F90 b/src/externals/mct/mpeu/m_ioutil.F90 similarity index 100% rename from externals/mct/mpeu/m_ioutil.F90 rename to src/externals/mct/mpeu/m_ioutil.F90 diff --git a/externals/mct/mpeu/m_mall.F90 b/src/externals/mct/mpeu/m_mall.F90 similarity index 100% rename from externals/mct/mpeu/m_mall.F90 rename to src/externals/mct/mpeu/m_mall.F90 diff --git a/externals/mct/mpeu/m_mpif.F90 b/src/externals/mct/mpeu/m_mpif.F90 similarity index 100% rename from externals/mct/mpeu/m_mpif.F90 rename to src/externals/mct/mpeu/m_mpif.F90 diff --git a/externals/mct/mpeu/m_mpif90.F90 b/src/externals/mct/mpeu/m_mpif90.F90 similarity index 100% rename from externals/mct/mpeu/m_mpif90.F90 rename to src/externals/mct/mpeu/m_mpif90.F90 diff --git a/externals/mct/mpeu/m_mpout.F90 b/src/externals/mct/mpeu/m_mpout.F90 similarity index 100% rename from externals/mct/mpeu/m_mpout.F90 rename to src/externals/mct/mpeu/m_mpout.F90 diff --git a/externals/mct/mpeu/m_rankMerge.F90 b/src/externals/mct/mpeu/m_rankMerge.F90 similarity index 100% rename from externals/mct/mpeu/m_rankMerge.F90 rename to src/externals/mct/mpeu/m_rankMerge.F90 diff --git a/externals/mct/mpeu/m_realkinds.F90 b/src/externals/mct/mpeu/m_realkinds.F90 similarity index 100% rename from externals/mct/mpeu/m_realkinds.F90 rename to src/externals/mct/mpeu/m_realkinds.F90 diff --git a/externals/mct/mpeu/m_stdio.F90 b/src/externals/mct/mpeu/m_stdio.F90 similarity index 100% rename from externals/mct/mpeu/m_stdio.F90 rename to src/externals/mct/mpeu/m_stdio.F90 diff --git a/externals/mct/mpeu/m_zeit.F90 b/src/externals/mct/mpeu/m_zeit.F90 similarity index 100% rename from externals/mct/mpeu/m_zeit.F90 rename to src/externals/mct/mpeu/m_zeit.F90 diff --git a/externals/mct/mpi-serial/.gitignore b/src/externals/mct/mpi-serial/.gitignore similarity index 100% rename from externals/mct/mpi-serial/.gitignore rename to src/externals/mct/mpi-serial/.gitignore diff --git a/externals/mct/mpi-serial/Makefile b/src/externals/mct/mpi-serial/Makefile similarity index 97% rename from externals/mct/mpi-serial/Makefile rename to src/externals/mct/mpi-serial/Makefile index 024d9f86241..0b1ca1db6c2 100644 --- a/externals/mct/mpi-serial/Makefile +++ b/src/externals/mct/mpi-serial/Makefile @@ -18,6 +18,8 @@ SRCS_C = mpi.c \ list.c \ handles.c \ comm.c \ + error.c \ + ic_merge.c \ group.c \ time.c \ pack.c \ diff --git a/externals/mct/mpi-serial/Makefile.conf.in b/src/externals/mct/mpi-serial/Makefile.conf.in similarity index 100% rename from externals/mct/mpi-serial/Makefile.conf.in rename to src/externals/mct/mpi-serial/Makefile.conf.in diff --git a/externals/mct/mpi-serial/README b/src/externals/mct/mpi-serial/README similarity index 71% rename from externals/mct/mpi-serial/README rename to src/externals/mct/mpi-serial/README index 20e377602bc..aaa728501f9 100644 --- a/externals/mct/mpi-serial/README +++ b/src/externals/mct/mpi-serial/README @@ -64,6 +64,9 @@ List of MPI calls supported mpi_abort mpi_error_string mpi_initialized + mpi_get_processor_name + mpi_get_library_version + mpi_wtime comm and group ops mpi_comm_free @@ -74,17 +77,36 @@ List of MPI calls supported mpi_comm_split mpi_comm_group mpi_group_incl + mpi_group_range_incl + mpi_group_union + mpi_group_intersection + mpi_group_difference + mpi_group_translate_ranks mpi_group_free + mpi_cart_create + mpi_cart_coords + mpi_dims_create send/receive ops mpi_irecv mpi_recv mpi_test + mpi_testany + mpi_testall + mpi_testsome mpi_wait mpi_waitany mpi_waitall + mpi_waitsome mpi_isend mpi_send + mpi_ssend + mpi_rsend + mpi_irsend + mpi_sendrecv + mpi_iprobe + mpi_probe + mpi_request_free collective operations mpi_barrier @@ -92,11 +114,27 @@ List of MPI calls supported mpi_gather mpi_gatherv mpi_allgather + mpi_scatter mpi_scatterv mpi_reduce mpi_allreduce - - + mpi_reduce_scatter + mpi_scan + mpi_alltoall + mpi_alltoallv + mpi_alltoallw + mpi_op_create + mpi_op_free + + data types and info objects + mpi_get_count + mpi_get_elements + mpi_pack + mpi_pack_size + mpi_unpack + mpi_info_create + mpi_info_set + mpi_info_free ----- EOF diff --git a/externals/mct/mpi-serial/aclocal.m4 b/src/externals/mct/mpi-serial/aclocal.m4 similarity index 100% rename from externals/mct/mpi-serial/aclocal.m4 rename to src/externals/mct/mpi-serial/aclocal.m4 diff --git a/externals/mct/mpi-serial/cart.c b/src/externals/mct/mpi-serial/cart.c similarity index 100% rename from externals/mct/mpi-serial/cart.c rename to src/externals/mct/mpi-serial/cart.c diff --git a/externals/mct/mpi-serial/collective.c b/src/externals/mct/mpi-serial/collective.c similarity index 97% rename from externals/mct/mpi-serial/collective.c rename to src/externals/mct/mpi-serial/collective.c index cead061130f..9a9736b4b64 100644 --- a/externals/mct/mpi-serial/collective.c +++ b/src/externals/mct/mpi-serial/collective.c @@ -488,27 +488,6 @@ int MPI_Alltoallw(void *sendbuf, int *sendcounts, } -/*********/ - - -FC_FUNC( mpi_op_create , MPI_OP_CREATE ) - ( void *function, int *commute, int *op, int *ierror ) -{ - *ierror=MPI_Op_create(function,*commute,op); -} - - - -int MPI_Op_create(MPI_User_function *function, int commute, MPI_Op *op) -{ - *op=MPI_OP_NULL; - - return(MPI_SUCCESS); - -} - - - /*********/ diff --git a/externals/mct/mpi-serial/comm.c b/src/externals/mct/mpi-serial/comm.c similarity index 100% rename from externals/mct/mpi-serial/comm.c rename to src/externals/mct/mpi-serial/comm.c diff --git a/externals/mct/mpi-serial/config.h.in b/src/externals/mct/mpi-serial/config.h.in similarity index 100% rename from externals/mct/mpi-serial/config.h.in rename to src/externals/mct/mpi-serial/config.h.in diff --git a/externals/mct/mpi-serial/configure b/src/externals/mct/mpi-serial/configure similarity index 100% rename from externals/mct/mpi-serial/configure rename to src/externals/mct/mpi-serial/configure diff --git a/externals/mct/mpi-serial/configure.in b/src/externals/mct/mpi-serial/configure.in similarity index 100% rename from externals/mct/mpi-serial/configure.in rename to src/externals/mct/mpi-serial/configure.in diff --git a/externals/mct/mpi-serial/copy.c b/src/externals/mct/mpi-serial/copy.c similarity index 100% rename from externals/mct/mpi-serial/copy.c rename to src/externals/mct/mpi-serial/copy.c diff --git a/src/externals/mct/mpi-serial/error.c b/src/externals/mct/mpi-serial/error.c new file mode 100644 index 00000000000..d26cfd164f9 --- /dev/null +++ b/src/externals/mct/mpi-serial/error.c @@ -0,0 +1,13 @@ + +#include "mpiP.h" + +/* + * Error handling code + * Just a stub for now to support the MPI interface without actually + * doing anything + */ + + int MPI_Errhandler_set(MPI_Comm comm, MPI_Errhandler handle) + { + return(MPI_SUCCESS); + } diff --git a/externals/mct/mpi-serial/fort.F90 b/src/externals/mct/mpi-serial/fort.F90 similarity index 100% rename from externals/mct/mpi-serial/fort.F90 rename to src/externals/mct/mpi-serial/fort.F90 diff --git a/externals/mct/mpi-serial/getcount.c b/src/externals/mct/mpi-serial/getcount.c similarity index 100% rename from externals/mct/mpi-serial/getcount.c rename to src/externals/mct/mpi-serial/getcount.c diff --git a/externals/mct/mpi-serial/group.c b/src/externals/mct/mpi-serial/group.c similarity index 100% rename from externals/mct/mpi-serial/group.c rename to src/externals/mct/mpi-serial/group.c diff --git a/externals/mct/mpi-serial/handles.c b/src/externals/mct/mpi-serial/handles.c similarity index 100% rename from externals/mct/mpi-serial/handles.c rename to src/externals/mct/mpi-serial/handles.c diff --git a/src/externals/mct/mpi-serial/ic_merge.c b/src/externals/mct/mpi-serial/ic_merge.c new file mode 100644 index 00000000000..ea19b387155 --- /dev/null +++ b/src/externals/mct/mpi-serial/ic_merge.c @@ -0,0 +1,15 @@ + +#include "mpiP.h" + +/* + * MPI_Intercomm_merge - Creates an intracommunicator from an intercommunicator + * This is just a stub for now to support mpi function calls even in Serial + * applications. In the case of a serial program, this function is a no-op and + * only ever returns MPI_SUCCESS + */ + +int MPI_Intercomm_merge( MPI_Comm intercomm, int high, MPI_Comm *newintracomm ) +{ + newintracomm = (MPI_Comm *)intercomm; + return(MPI_SUCCESS); +} diff --git a/externals/mct/mpi-serial/info.c b/src/externals/mct/mpi-serial/info.c similarity index 100% rename from externals/mct/mpi-serial/info.c rename to src/externals/mct/mpi-serial/info.c diff --git a/externals/mct/mpi-serial/list.c b/src/externals/mct/mpi-serial/list.c similarity index 100% rename from externals/mct/mpi-serial/list.c rename to src/externals/mct/mpi-serial/list.c diff --git a/externals/mct/mpi-serial/list.h b/src/externals/mct/mpi-serial/list.h similarity index 100% rename from externals/mct/mpi-serial/list.h rename to src/externals/mct/mpi-serial/list.h diff --git a/externals/mct/mpi-serial/listP.h b/src/externals/mct/mpi-serial/listP.h similarity index 100% rename from externals/mct/mpi-serial/listP.h rename to src/externals/mct/mpi-serial/listP.h diff --git a/externals/mct/mpi-serial/listops.h b/src/externals/mct/mpi-serial/listops.h similarity index 100% rename from externals/mct/mpi-serial/listops.h rename to src/externals/mct/mpi-serial/listops.h diff --git a/externals/mct/mpi-serial/m4/README b/src/externals/mct/mpi-serial/m4/README similarity index 100% rename from externals/mct/mpi-serial/m4/README rename to src/externals/mct/mpi-serial/m4/README diff --git a/externals/mct/mpi-serial/m4/ax_fc_version.m4 b/src/externals/mct/mpi-serial/m4/ax_fc_version.m4 similarity index 100% rename from externals/mct/mpi-serial/m4/ax_fc_version.m4 rename to src/externals/mct/mpi-serial/m4/ax_fc_version.m4 diff --git a/externals/mct/mpi-serial/mpi.c b/src/externals/mct/mpi-serial/mpi.c similarity index 100% rename from externals/mct/mpi-serial/mpi.c rename to src/externals/mct/mpi-serial/mpi.c diff --git a/externals/mct/mpi-serial/mpi.h b/src/externals/mct/mpi-serial/mpi.h similarity index 96% rename from externals/mct/mpi-serial/mpi.h rename to src/externals/mct/mpi-serial/mpi.h index 529b57b853a..9183bf89d20 100644 --- a/externals/mct/mpi-serial/mpi.h +++ b/src/externals/mct/mpi-serial/mpi.h @@ -1,4 +1,3 @@ - #ifndef _MPI_H_ #define _MPI_H_ @@ -48,7 +47,6 @@ typedef int MPI_Group; #define MPI_ERR_IN_STATUS (-1) #define MPI_ERR_LASTCODE (-1) - /* * MPI_UNDEFINED * @@ -191,6 +189,14 @@ typedef struct /* Fortran: INTEGER status(MPI_STATUS_SIZE) */ #define MPI_STATUSES_IGNORE ((MPI_Status *)0) +/* + * MPI Errhandling stubs (Not functional currently) + */ +typedef int MPI_Errhandler; + +#define MPI_ERRORS_ARE_FATAL ((MPI_Errhandler)0) +#define MPI_ERRORS_RETURN ((MPI_Errhandler)-1) + /* * Collective operations @@ -246,7 +252,8 @@ typedef int MPI_Info; /* handle */ extern int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm *newintercomm); - +extern int MPI_Intercomm_merge(MPI_Comm intercomm, int high, + MPI_Comm *newintercomm); extern int MPI_Cart_create(MPI_Comm comm_old, int ndims, int *dims, int *periods, int reorder, MPI_Comm *comm_cart); extern int MPI_Cart_get(MPI_Comm comm, int maxdims, int *dims, @@ -389,6 +396,9 @@ extern int MPI_Iprobe(int source, int tag, MPI_Comm comm, extern int MPI_Pack_size(int incount, MPI_Datatype type, MPI_Comm comm, MPI_Aint * size); +/* Error handling stub, not currently functional */ +extern int MPI_Errhandler_set(MPI_Comm comm, MPI_Errhandler handle); + /* new type functions */ extern int MPI_Get_count(MPI_Status *status, MPI_Datatype datatype, int *count); extern int MPI_Get_elements(MPI_Status *status, MPI_Datatype datatype, int *count); @@ -400,6 +410,9 @@ extern int MPI_Type_vector(int count, int blocklen, int stride, MPI_Datatype old extern int MPI_Type_hvector(int count, int blocklen, MPI_Aint stride, MPI_Datatype oldtype, MPI_Datatype *newtype); +extern int MPI_Type_create_hvector(int count, int blocklen, MPI_Aint stride, + MPI_Datatype oldtype, MPI_Datatype *newtype); + extern int MPI_Type_indexed(int count, int *blocklens, int *displacements, MPI_Datatype oldtype, MPI_Datatype *newtype); diff --git a/externals/mct/mpi-serial/mpiP.h b/src/externals/mct/mpi-serial/mpiP.h similarity index 100% rename from externals/mct/mpi-serial/mpiP.h rename to src/externals/mct/mpi-serial/mpiP.h diff --git a/externals/mct/mpi-serial/mpif.F90 b/src/externals/mct/mpi-serial/mpif.F90 similarity index 100% rename from externals/mct/mpi-serial/mpif.F90 rename to src/externals/mct/mpi-serial/mpif.F90 diff --git a/src/externals/mct/mpi-serial/mpif.h b/src/externals/mct/mpi-serial/mpif.h new file mode 100644 index 00000000000..b4537b5d4a2 --- /dev/null +++ b/src/externals/mct/mpi-serial/mpif.h @@ -0,0 +1,327 @@ + +! +! MPI_COMM_WORLD +! + +INTEGER MPI_COMM_WORLD +parameter (mpi_comm_world=1) + +! +! +! + +integer MPI_BOTTOM +parameter (MPI_BOTTOM=0) + + +! +! source,tag + ! + + integer MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_TAG_UB + parameter (mpi_any_source=-1, mpi_any_tag= -1, mpi_tag_ub=1681915906) + + integer MPI_PROC_NULL, MPI_ROOT + parameter (MPI_PROC_NULL=-2, MPI_ROOT=-3) + + integer MPI_COMM_NULL, MPI_REQUEST_NULL + parameter (MPI_COMM_NULL=0, MPI_REQUEST_NULL=0) + + integer MPI_GROUP_NULL, MPI_GROUP_EMPTY + parameter (MPI_GROUP_NULL=0, MPI_GROUP_EMPTY= -1) + + integer MPI_MAX_ERROR_STRING + parameter (MPI_MAX_ERROR_STRING=128) + + integer MPI_MAX_PROCESSOR_NAME + parameter (MPI_MAX_PROCESSOR_NAME=128) + + ! + ! Return codes + ! + + integer MPI_SUCCESS + parameter (MPI_SUCCESS=0) + + integer MPI_ERR_BUFFER + parameter (MPI_ERR_BUFFER= -1) + + integer MPI_ERR_COUNT + parameter (MPI_ERR_COUNT= -1) + + integer MPI_ERR_TYPE + parameter (MPI_ERR_TYPE= -1) + + integer MPI_ERR_TAG + parameter (MPI_ERR_TAG= -1) + + integer MPI_ERR_COMM + parameter (MPI_ERR_COMM= -1) + + integer MPI_ERR_RANK + parameter (MPI_ERR_RANK= -1) + + integer MPI_ERR_REQUEST + parameter (MPI_ERR_REQUEST= -1) + + integer MPI_ERR_ROOT + parameter (MPI_ERR_ROOT= -1) + + integer MPI_ERR_GROUP + parameter (MPI_ERR_GROUP= -1) + + integer MPI_ERR_OP + parameter (MPI_ERR_OP= -1) + + integer MPI_ERR_TOPOLOGY + parameter (MPI_ERR_TOPOLOGY= -1) + + integer MPI_ERR_DIMS + parameter (MPI_ERR_DIMS= -1) + + integer MPI_ERR_ARG + parameter (MPI_ERR_ARG= -1) + + integer MPI_ERR_UNKNOWN + parameter (MPI_ERR_UNKNOWN= -1) + + integer MPI_ERR_TRUNCATE + parameter (MPI_ERR_TRUNCATE= -1) + + integer MPI_ERR_OTHER + parameter (MPI_ERR_OTHER= -1) + + integer MPI_ERR_INTERN + parameter (MPI_ERR_INTERN= -1) + + integer MPI_PENDING + parameter (MPI_PENDING= -1) + + integer MPI_ERR_IN_STATUS + parameter (MPI_ERR_IN_STATUS= -1) + + integer MPI_ERR_LASTCODE + parameter (MPI_ERR_LASTCODE= -1) + + integer MPI_ERRORS_RETURN + parameter (MPI_ERRORS_RETURN= -1) + + ! + ! + + + integer MPI_UNDEFINED + parameter (MPI_UNDEFINED= -1) + + + ! + ! MPI_Status + ! + ! The values in this section MUST match the struct definition + ! in mpi.h + ! + + + INTEGER MPI_STATUS_SIZE + PARAMETER (MPI_STATUS_SIZE=4) + + INTEGER MPI_SOURCE, MPI_TAG, MPI_ERROR + PARAMETER(MPI_SOURCE=1, MPI_TAG=2, MPI_ERROR=3) + ! There is a 4th value only used internally + + INTEGER MPI_STATUS_IGNORE(MPI_STATUS_SIZE) + INTEGER MPI_STATUSES_IGNORE(MPI_STATUS_SIZE,1) + COMMON /MPISERIAL/ MPI_STATUS_IGNORE + COMMON /MPISERIAL/ MPI_STATUSES_IGNORE + + ! + ! MPI_IN_PLACE + ! + + INTEGER MPI_IN_PLACE + COMMON /MPISERIAL/ MPI_IN_PLACE + + SAVE /MPISERIAL/ ! Technically needed in case goes out of scope + + + ! + ! MPI_Datatype values + ! + ! New datatype values + ! Type constants represent integer handles, matching up to the index of the + ! type array equal to the absolute value of the constant plus one. For + ! example, MPI_BYTE=-12, corresponding to type index 11. + ! (Array in type_const.c) + ! + + + INTEGER MPI_DATATYPE_NULL + PARAMETER (MPI_DATATYPE_NULL=0) + + INTEGER MPI_BYTE + PARAMETER (MPI_BYTE=-12) + + INTEGER MPI_PACKED + PARAMETER (MPI_PACKED=-13) + + INTEGER MPI_LB + PARAMETER (MPI_LB=-14) + + INTEGER MPI_UB + PARAMETER (MPI_UB=-15) + + INTEGER MPI_INTEGER + PARAMETER (MPI_INTEGER=-16) + + INTEGER MPI_REAL + PARAMETER (MPI_REAL=-17) + + INTEGER MPI_DOUBLE_PRECISION + PARAMETER (MPI_DOUBLE_PRECISION=-18) + + INTEGER MPI_COMPLEX + PARAMETER (MPI_COMPLEX=-19) + + INTEGER MPI_DOUBLE_COMPLEX + PARAMETER (MPI_DOUBLE_COMPLEX=-20) + + INTEGER MPI_LOGICAL + PARAMETER (MPI_LOGICAL=-21) + + INTEGER MPI_CHARACTER + PARAMETER (MPI_CHARACTER=-22) + + integer MPI_2REAL + parameter (MPI_2REAL= -23) + + integer MPI_2DOUBLE_PRECISION + parameter (MPI_2DOUBLE_PRECISION= -24) + + integer MPI_2INTEGER + parameter (MPI_2INTEGER= -25) + + + ! + ! Size-specific types + ! + + INTEGER MPI_INTEGER1 + PARAMETER (MPI_INTEGER1= -32 ) + + INTEGER MPI_INTEGER2 + PARAMETER (MPI_INTEGER2= -33 ) + + INTEGER MPI_INTEGER4 + PARAMETER (MPI_INTEGER4= -34 ) + + INTEGER MPI_INTEGER8 + PARAMETER (MPI_INTEGER8= -35 ) + + INTEGER MPI_INTEGER16 + PARAMETER (MPI_INTEGER16= -36 ) + + + INTEGER MPI_REAL4 + PARAMETER (MPI_REAL4= -37 ) + + INTEGER MPI_REAL8 + PARAMETER (MPI_REAL8= -38 ) + + INTEGER MPI_REAL16 + PARAMETER (MPI_REAL16= -39 ) + + + integer MPI_COMPLEX8 + parameter (MPI_COMPLEX8= -40 ) + + integer MPI_COMPLEX16 + parameter (MPI_COMPLEX16= -41 ) + + integer MPI_COMPLEX32 + parameter (MPI_COMPLEX32= -42 ) + + integer MPI_LONG_LONG_INT + parameter (MPI_LONG_LONG_INT= -43) + + integer MPI_LONG_LONG + parameter (MPI_LONG_LONG= MPI_LONG_LONG_INT) + + integer MPI_UNSIGNED_LONG_LONG + parameter (MPI_UNSIGNED_LONG_LONG= -44) + + integer MPI_OFFSET + parameter (MPI_OFFSET= -45) + + ! + ! MPI_Op values + ! + ! (All are handled as no-op so no value is necessary; but provide one + ! anyway just in case.) + ! + + INTEGER MPI_SUM + PARAMETER (MPI_SUM=0) + INTEGER MPI_MAX + PARAMETER (MPI_MAX=0) + INTEGER MPI_MIN + PARAMETER (MPI_MIN=0) + INTEGER MPI_PROD + PARAMETER (MPI_PROD=0) + INTEGER MPI_LAND + PARAMETER (MPI_LAND=0) + INTEGER MPI_BAND + PARAMETER (MPI_BAND=0) + INTEGER MPI_LOR + PARAMETER (MPI_LOR=0) + INTEGER MPI_BOR + PARAMETER (MPI_BOR=0) + INTEGER MPI_LXOR + PARAMETER (MPI_LXOR=0) + INTEGER MPI_BXOR + PARAMETER (MPI_BXOR=0) + INTEGER MPI_MINLOC + PARAMETER (MPI_MINLOC=0) + INTEGER MPI_MAXLOC + PARAMETER (MPI_MAXLOC=0) + INTEGER MPI_OP_NULL + PARAMETER (MPI_OP_NULL=0) + + ! + ! MPI_Wtime + ! + + DOUBLE PRECISION MPI_WTIME + EXTERNAL MPI_WTIME + + + ! + ! Kinds + ! + + INTEGER MPI_OFFSET_KIND + PARAMETER (MPI_OFFSET_KIND=selected_int_kind(13)) + + INTEGER MPI_MODE_RDONLY + PARAMETER (MPI_MODE_RDONLY=0) + + INTEGER MPI_MODE_CREATE + PARAMETER (MPI_MODE_CREATE=1) + + INTEGER MPI_MODE_RDWR + PARAMETER (MPI_MODE_RDWR=2) + + + ! + ! Info + ! + + INTEGER MPI_INFO_NULL + PARAMETER (MPI_INFO_NULL=0) + + + ! + ! Library version string (must match C value) + ! + + INTEGER MPI_MAX_LIBRARY_VERSION_STRING + PARAMETER (MPI_MAX_LIBRARY_VERSION_STRING=80) diff --git a/externals/mct/mpi-serial/op.c b/src/externals/mct/mpi-serial/op.c similarity index 100% rename from externals/mct/mpi-serial/op.c rename to src/externals/mct/mpi-serial/op.c diff --git a/externals/mct/mpi-serial/pack.c b/src/externals/mct/mpi-serial/pack.c similarity index 100% rename from externals/mct/mpi-serial/pack.c rename to src/externals/mct/mpi-serial/pack.c diff --git a/externals/mct/mpi-serial/probe.c b/src/externals/mct/mpi-serial/probe.c similarity index 100% rename from externals/mct/mpi-serial/probe.c rename to src/externals/mct/mpi-serial/probe.c diff --git a/externals/mct/mpi-serial/protify.awk b/src/externals/mct/mpi-serial/protify.awk similarity index 100% rename from externals/mct/mpi-serial/protify.awk rename to src/externals/mct/mpi-serial/protify.awk diff --git a/externals/mct/mpi-serial/recv.c b/src/externals/mct/mpi-serial/recv.c similarity index 100% rename from externals/mct/mpi-serial/recv.c rename to src/externals/mct/mpi-serial/recv.c diff --git a/externals/mct/mpi-serial/req.c b/src/externals/mct/mpi-serial/req.c similarity index 100% rename from externals/mct/mpi-serial/req.c rename to src/externals/mct/mpi-serial/req.c diff --git a/externals/mct/mpi-serial/send.c b/src/externals/mct/mpi-serial/send.c similarity index 100% rename from externals/mct/mpi-serial/send.c rename to src/externals/mct/mpi-serial/send.c diff --git a/externals/mct/mpi-serial/tests/.gitignore b/src/externals/mct/mpi-serial/tests/.gitignore similarity index 100% rename from externals/mct/mpi-serial/tests/.gitignore rename to src/externals/mct/mpi-serial/tests/.gitignore diff --git a/externals/mct/mpi-serial/tests/Makefile b/src/externals/mct/mpi-serial/tests/Makefile similarity index 100% rename from externals/mct/mpi-serial/tests/Makefile rename to src/externals/mct/mpi-serial/tests/Makefile diff --git a/externals/mct/mpi-serial/tests/ctest.c b/src/externals/mct/mpi-serial/tests/ctest.c similarity index 100% rename from externals/mct/mpi-serial/tests/ctest.c rename to src/externals/mct/mpi-serial/tests/ctest.c diff --git a/externals/mct/mpi-serial/tests/ctest_old.c b/src/externals/mct/mpi-serial/tests/ctest_old.c similarity index 100% rename from externals/mct/mpi-serial/tests/ctest_old.c rename to src/externals/mct/mpi-serial/tests/ctest_old.c diff --git a/externals/mct/mpi-serial/tests/ftest.F90 b/src/externals/mct/mpi-serial/tests/ftest.F90 similarity index 100% rename from externals/mct/mpi-serial/tests/ftest.F90 rename to src/externals/mct/mpi-serial/tests/ftest.F90 diff --git a/externals/mct/mpi-serial/tests/ftest_internal.F90 b/src/externals/mct/mpi-serial/tests/ftest_internal.F90 similarity index 100% rename from externals/mct/mpi-serial/tests/ftest_internal.F90 rename to src/externals/mct/mpi-serial/tests/ftest_internal.F90 diff --git a/externals/mct/mpi-serial/tests/ftest_old.F90 b/src/externals/mct/mpi-serial/tests/ftest_old.F90 similarity index 98% rename from externals/mct/mpi-serial/tests/ftest_old.F90 rename to src/externals/mct/mpi-serial/tests/ftest_old.F90 index 93075219def..938d4472a94 100644 --- a/externals/mct/mpi-serial/tests/ftest_old.F90 +++ b/src/externals/mct/mpi-serial/tests/ftest_old.F90 @@ -99,7 +99,7 @@ program test call mpi_waitall(5,rreq,status,ier) print *,'recvs on MPI_COMM_WORLD done' - + do i=1,5 print *, 'Status source=',status(MPI_SOURCE,i), & ' tag=',status(MPI_TAG,i) @@ -137,13 +137,13 @@ program test MPI_COMM_WORLD) print *,temp end do - + do i=1,5 if (rbuf(i) .ne. sbuf(i)) then errcount = errcount + 1 print *,"Error for pack/send/unpack" print *,"found ",rbuf(i)," should be ",sbuf(i) - end if + end if end do ! @@ -160,6 +160,6 @@ program test else print *,"No errors" end if - + end diff --git a/externals/mct/mpi-serial/time.c b/src/externals/mct/mpi-serial/time.c similarity index 100% rename from externals/mct/mpi-serial/time.c rename to src/externals/mct/mpi-serial/time.c diff --git a/externals/mct/mpi-serial/type.c b/src/externals/mct/mpi-serial/type.c similarity index 97% rename from externals/mct/mpi-serial/type.c rename to src/externals/mct/mpi-serial/type.c index 22e3d305b38..8dd93f27414 100644 --- a/externals/mct/mpi-serial/type.c +++ b/src/externals/mct/mpi-serial/type.c @@ -448,6 +448,23 @@ int MPI_Type_hvector(int count, int blocklen, MPI_Aint stride, return Type_hvector(count, blocklen, stride, old_ptr, new_ptr); } +FC_FUNC( mpi_type_create_hvector, MPI_TYPE_CREATE_HVECTOR ) + (int * count, long * blocklen, long * stride, + int * oldtype, int * newtype, int * ierr) +{ + *ierr = MPI_Type_create_hvector(*count, *blocklen, *stride, *oldtype, newtype); +} + +int MPI_Type_create_hvector(int count, int blocklen, MPI_Aint stride, + MPI_Datatype oldtype, MPI_Datatype * newtype) +{ + Datatype old_ptr = *(Datatype*) mpi_handle_to_datatype(oldtype); + Datatype * new_ptr; + + mpi_alloc_handle(newtype, (void**) &new_ptr); + return Type_hvector(count, blocklen, stride, old_ptr, new_ptr); +} + int Type_hvector(int count, int blocklen, MPI_Aint stride, Datatype oldtype, Datatype *newtype) diff --git a/externals/mct/mpi-serial/type.h b/src/externals/mct/mpi-serial/type.h similarity index 100% rename from externals/mct/mpi-serial/type.h rename to src/externals/mct/mpi-serial/type.h diff --git a/externals/mct/mpi-serial/type_const.c b/src/externals/mct/mpi-serial/type_const.c similarity index 100% rename from externals/mct/mpi-serial/type_const.c rename to src/externals/mct/mpi-serial/type_const.c diff --git a/externals/mct/protex/protex b/src/externals/mct/protex/protex similarity index 100% rename from externals/mct/protex/protex rename to src/externals/mct/protex/protex diff --git a/externals/mct/testsystem/Makefile b/src/externals/mct/testsystem/Makefile similarity index 100% rename from externals/mct/testsystem/Makefile rename to src/externals/mct/testsystem/Makefile diff --git a/externals/mct/testsystem/testall/.gitignore b/src/externals/mct/testsystem/testall/.gitignore similarity index 100% rename from externals/mct/testsystem/testall/.gitignore rename to src/externals/mct/testsystem/testall/.gitignore diff --git a/externals/mct/testsystem/testall/Makefile b/src/externals/mct/testsystem/testall/Makefile similarity index 100% rename from externals/mct/testsystem/testall/Makefile rename to src/externals/mct/testsystem/testall/Makefile diff --git a/externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 b/src/externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 similarity index 98% rename from externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 rename to src/externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 index b865e44ddaf..a0ce00128b8 100644 --- a/externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 +++ b/src/externals/mct/testsystem/testall/ReadSparseMatrixAsc.F90 @@ -25,6 +25,7 @@ subroutine ReadSparseMatrixAsc(sMat, fileID, src_dims, dst_dims) use m_List, only : List_init => init use m_List, only : List_clean => clean + use m_AttrVect, only : Attrvect_zero => zero use m_SparseMatrix, only : SparseMatrix use m_SparseMatrix, only : SparseMatrix_Init => init use m_SparseMatrix, only : SparseMatrix_Clean => clean @@ -155,6 +156,7 @@ subroutine ReadSparseMatrixAsc(sMat, fileID, src_dims, dst_dims) nRows = dst_dims(1) * dst_dims(2) nColumns = src_dims(1) * src_dims(2) call SparseMatrix_init(sMat, nRows, nColumns, num_elements) + call AttrVect_zero(sMat%data) ! ...and store them. diff --git a/externals/mct/testsystem/testall/UNTESTED b/src/externals/mct/testsystem/testall/UNTESTED similarity index 100% rename from externals/mct/testsystem/testall/UNTESTED rename to src/externals/mct/testsystem/testall/UNTESTED diff --git a/externals/mct/testsystem/testall/ccm.F90 b/src/externals/mct/testsystem/testall/ccm.F90 similarity index 100% rename from externals/mct/testsystem/testall/ccm.F90 rename to src/externals/mct/testsystem/testall/ccm.F90 diff --git a/externals/mct/testsystem/testall/convertPOPT.F90 b/src/externals/mct/testsystem/testall/convertPOPT.F90 similarity index 100% rename from externals/mct/testsystem/testall/convertPOPT.F90 rename to src/externals/mct/testsystem/testall/convertPOPT.F90 diff --git a/externals/mct/testsystem/testall/convertgauss.F90 b/src/externals/mct/testsystem/testall/convertgauss.F90 similarity index 100% rename from externals/mct/testsystem/testall/convertgauss.F90 rename to src/externals/mct/testsystem/testall/convertgauss.F90 diff --git a/externals/mct/testsystem/testall/cpl.F90 b/src/externals/mct/testsystem/testall/cpl.F90 similarity index 99% rename from externals/mct/testsystem/testall/cpl.F90 rename to src/externals/mct/testsystem/testall/cpl.F90 index a709dd7d329..0a1235d9d0f 100644 --- a/externals/mct/testsystem/testall/cpl.F90 +++ b/src/externals/mct/testsystem/testall/cpl.F90 @@ -875,7 +875,7 @@ subroutine cpl (CPL_World) call zeit_co('COcnRouterInit') ! rml print router info - call MCT_Router_print(Atm2Cpl,CPL_World,90) + if(myProc==0)call MCT_Router_print(Atm2Cpl,CPL_World,90) close(90) call Router_test(Atm2Cpl,"CPL::Atm2Cpl",7000+myProc) diff --git a/externals/mct/testsystem/testall/job.ut-all.jaguar b/src/externals/mct/testsystem/testall/job.ut-all.jaguar similarity index 100% rename from externals/mct/testsystem/testall/job.ut-all.jaguar rename to src/externals/mct/testsystem/testall/job.ut-all.jaguar diff --git a/externals/mct/testsystem/testall/m_ACTEST.F90 b/src/externals/mct/testsystem/testall/m_ACTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_ACTEST.F90 rename to src/externals/mct/testsystem/testall/m_ACTEST.F90 diff --git a/externals/mct/testsystem/testall/m_AVTEST.F90 b/src/externals/mct/testsystem/testall/m_AVTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_AVTEST.F90 rename to src/externals/mct/testsystem/testall/m_AVTEST.F90 diff --git a/externals/mct/testsystem/testall/m_GGRIDTEST.F90 b/src/externals/mct/testsystem/testall/m_GGRIDTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_GGRIDTEST.F90 rename to src/externals/mct/testsystem/testall/m_GGRIDTEST.F90 diff --git a/externals/mct/testsystem/testall/m_GMAPTEST.F90 b/src/externals/mct/testsystem/testall/m_GMAPTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_GMAPTEST.F90 rename to src/externals/mct/testsystem/testall/m_GMAPTEST.F90 diff --git a/externals/mct/testsystem/testall/m_GSMAPTEST.F90 b/src/externals/mct/testsystem/testall/m_GSMAPTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_GSMAPTEST.F90 rename to src/externals/mct/testsystem/testall/m_GSMAPTEST.F90 diff --git a/externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 b/src/externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 rename to src/externals/mct/testsystem/testall/m_MCTWORLDTEST.F90 diff --git a/externals/mct/testsystem/testall/m_ROUTERTEST.F90 b/src/externals/mct/testsystem/testall/m_ROUTERTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_ROUTERTEST.F90 rename to src/externals/mct/testsystem/testall/m_ROUTERTEST.F90 diff --git a/externals/mct/testsystem/testall/m_SMATTEST.F90 b/src/externals/mct/testsystem/testall/m_SMATTEST.F90 similarity index 100% rename from externals/mct/testsystem/testall/m_SMATTEST.F90 rename to src/externals/mct/testsystem/testall/m_SMATTEST.F90 diff --git a/externals/mct/testsystem/testall/master.F90 b/src/externals/mct/testsystem/testall/master.F90 similarity index 100% rename from externals/mct/testsystem/testall/master.F90 rename to src/externals/mct/testsystem/testall/master.F90 diff --git a/externals/mct/testsystem/testall/mph.F90 b/src/externals/mct/testsystem/testall/mph.F90 similarity index 100% rename from externals/mct/testsystem/testall/mph.F90 rename to src/externals/mct/testsystem/testall/mph.F90 diff --git a/externals/mct/testsystem/testall/pop.F90 b/src/externals/mct/testsystem/testall/pop.F90 similarity index 100% rename from externals/mct/testsystem/testall/pop.F90 rename to src/externals/mct/testsystem/testall/pop.F90 diff --git a/externals/mct/testsystem/testall/processors_map.in b/src/externals/mct/testsystem/testall/processors_map.in similarity index 100% rename from externals/mct/testsystem/testall/processors_map.in rename to src/externals/mct/testsystem/testall/processors_map.in diff --git a/externals/mct/testsystem/testall/script.jag b/src/externals/mct/testsystem/testall/script.jag similarity index 100% rename from externals/mct/testsystem/testall/script.jag rename to src/externals/mct/testsystem/testall/script.jag diff --git a/externals/mct/testsystem/testall/ut_SparseMatrix.rc b/src/externals/mct/testsystem/testall/ut_SparseMatrix.rc similarity index 100% rename from externals/mct/testsystem/testall/ut_SparseMatrix.rc rename to src/externals/mct/testsystem/testall/ut_SparseMatrix.rc diff --git a/externals/mct/testunit/.gitignore b/src/externals/mct/testunit/.gitignore similarity index 100% rename from externals/mct/testunit/.gitignore rename to src/externals/mct/testunit/.gitignore diff --git a/externals/mct/testunit/AttrVect_Test.F90 b/src/externals/mct/testunit/AttrVect_Test.F90 similarity index 100% rename from externals/mct/testunit/AttrVect_Test.F90 rename to src/externals/mct/testunit/AttrVect_Test.F90 diff --git a/externals/mct/testunit/Makefile b/src/externals/mct/testunit/Makefile similarity index 100% rename from externals/mct/testunit/Makefile rename to src/externals/mct/testunit/Makefile diff --git a/externals/mct/testunit/master.F90 b/src/externals/mct/testunit/master.F90 similarity index 100% rename from externals/mct/testunit/master.F90 rename to src/externals/mct/testunit/master.F90 diff --git a/externals/pio1/CMakeLists.txt b/src/externals/pio1/CMakeLists.txt similarity index 100% rename from externals/pio1/CMakeLists.txt rename to src/externals/pio1/CMakeLists.txt diff --git a/externals/pio1/ChangeLog b/src/externals/pio1/ChangeLog similarity index 100% rename from externals/pio1/ChangeLog rename to src/externals/pio1/ChangeLog diff --git a/externals/pio1/ChangeLog_template b/src/externals/pio1/ChangeLog_template similarity index 100% rename from externals/pio1/ChangeLog_template rename to src/externals/pio1/ChangeLog_template diff --git a/externals/pio1/Doxyfile b/src/externals/pio1/Doxyfile similarity index 100% rename from externals/pio1/Doxyfile rename to src/externals/pio1/Doxyfile diff --git a/externals/pio1/DoxygenLayout.xml b/src/externals/pio1/DoxygenLayout.xml similarity index 100% rename from externals/pio1/DoxygenLayout.xml rename to src/externals/pio1/DoxygenLayout.xml diff --git a/externals/pio1/SVN_EXTERNAL_DIRECTORIES b/src/externals/pio1/SVN_EXTERNAL_DIRECTORIES similarity index 100% rename from externals/pio1/SVN_EXTERNAL_DIRECTORIES rename to src/externals/pio1/SVN_EXTERNAL_DIRECTORIES diff --git a/externals/pio1/customdoxygen.css b/src/externals/pio1/customdoxygen.css similarity index 100% rename from externals/pio1/customdoxygen.css rename to src/externals/pio1/customdoxygen.css diff --git a/externals/pio1/doc/CAMexample.txt b/src/externals/pio1/doc/CAMexample.txt similarity index 100% rename from externals/pio1/doc/CAMexample.txt rename to src/externals/pio1/doc/CAMexample.txt diff --git a/externals/pio1/doc/Decomp.txt b/src/externals/pio1/doc/Decomp.txt similarity index 100% rename from externals/pio1/doc/Decomp.txt rename to src/externals/pio1/doc/Decomp.txt diff --git a/externals/pio1/doc/DoxygenLayout.xml b/src/externals/pio1/doc/DoxygenLayout.xml similarity index 100% rename from externals/pio1/doc/DoxygenLayout.xml rename to src/externals/pio1/doc/DoxygenLayout.xml diff --git a/externals/pio1/doc/Error.txt b/src/externals/pio1/doc/Error.txt similarity index 100% rename from externals/pio1/doc/Error.txt rename to src/externals/pio1/doc/Error.txt diff --git a/externals/pio1/doc/Examples.txt b/src/externals/pio1/doc/Examples.txt similarity index 100% rename from externals/pio1/doc/Examples.txt rename to src/externals/pio1/doc/Examples.txt diff --git a/externals/pio1/doc/GettingStarted.txt b/src/externals/pio1/doc/GettingStarted.txt similarity index 100% rename from externals/pio1/doc/GettingStarted.txt rename to src/externals/pio1/doc/GettingStarted.txt diff --git a/externals/pio1/doc/Installing.txt b/src/externals/pio1/doc/Installing.txt similarity index 100% rename from externals/pio1/doc/Installing.txt rename to src/externals/pio1/doc/Installing.txt diff --git a/externals/pio1/doc/api.txt b/src/externals/pio1/doc/api.txt similarity index 100% rename from externals/pio1/doc/api.txt rename to src/externals/pio1/doc/api.txt diff --git a/externals/pio1/doc/base.txt b/src/externals/pio1/doc/base.txt similarity index 100% rename from externals/pio1/doc/base.txt rename to src/externals/pio1/doc/base.txt diff --git a/externals/pio1/doc/example/errorhandle b/src/externals/pio1/doc/example/errorhandle similarity index 100% rename from externals/pio1/doc/example/errorhandle rename to src/externals/pio1/doc/example/errorhandle diff --git a/externals/pio1/doc/example/simple-bc b/src/externals/pio1/doc/example/simple-bc similarity index 100% rename from externals/pio1/doc/example/simple-bc rename to src/externals/pio1/doc/example/simple-bc diff --git a/externals/pio1/doc/example/simple-bc-rearr b/src/externals/pio1/doc/example/simple-bc-rearr similarity index 100% rename from externals/pio1/doc/example/simple-bc-rearr rename to src/externals/pio1/doc/example/simple-bc-rearr diff --git a/externals/pio1/doc/example/simple-bc-rearr-pe1 b/src/externals/pio1/doc/example/simple-bc-rearr-pe1 similarity index 100% rename from externals/pio1/doc/example/simple-bc-rearr-pe1 rename to src/externals/pio1/doc/example/simple-bc-rearr-pe1 diff --git a/externals/pio1/doc/example/simple-bc-rearr-pe2 b/src/externals/pio1/doc/example/simple-bc-rearr-pe2 similarity index 100% rename from externals/pio1/doc/example/simple-bc-rearr-pe2 rename to src/externals/pio1/doc/example/simple-bc-rearr-pe2 diff --git a/externals/pio1/doc/example/simple-dof b/src/externals/pio1/doc/example/simple-dof similarity index 100% rename from externals/pio1/doc/example/simple-dof rename to src/externals/pio1/doc/example/simple-dof diff --git a/externals/pio1/doc/example/simple-dof-rearr b/src/externals/pio1/doc/example/simple-dof-rearr similarity index 100% rename from externals/pio1/doc/example/simple-dof-rearr rename to src/externals/pio1/doc/example/simple-dof-rearr diff --git a/externals/pio1/doc/faq.txt b/src/externals/pio1/doc/faq.txt similarity index 100% rename from externals/pio1/doc/faq.txt rename to src/externals/pio1/doc/faq.txt diff --git a/externals/pio1/doc/footer.html b/src/externals/pio1/doc/footer.html similarity index 100% rename from externals/pio1/doc/footer.html rename to src/externals/pio1/doc/footer.html diff --git a/externals/pio1/doc/header.html b/src/externals/pio1/doc/header.html similarity index 100% rename from externals/pio1/doc/header.html rename to src/externals/pio1/doc/header.html diff --git a/externals/pio1/doc/images/baseimage.graffle b/src/externals/pio1/doc/images/baseimage.graffle similarity index 100% rename from externals/pio1/doc/images/baseimage.graffle rename to src/externals/pio1/doc/images/baseimage.graffle diff --git a/externals/pio1/doc/images/block-cyclic-rearr.eps b/src/externals/pio1/doc/images/block-cyclic-rearr.eps similarity index 100% rename from externals/pio1/doc/images/block-cyclic-rearr.eps rename to src/externals/pio1/doc/images/block-cyclic-rearr.eps diff --git a/externals/pio1/doc/images/block-cyclic-rearr.graffle b/src/externals/pio1/doc/images/block-cyclic-rearr.graffle similarity index 100% rename from externals/pio1/doc/images/block-cyclic-rearr.graffle rename to src/externals/pio1/doc/images/block-cyclic-rearr.graffle diff --git a/externals/pio1/doc/images/block-cyclic-rearr.png b/src/externals/pio1/doc/images/block-cyclic-rearr.png similarity index 100% rename from externals/pio1/doc/images/block-cyclic-rearr.png rename to src/externals/pio1/doc/images/block-cyclic-rearr.png diff --git a/externals/pio1/doc/images/block-cyclic.eps b/src/externals/pio1/doc/images/block-cyclic.eps similarity index 100% rename from externals/pio1/doc/images/block-cyclic.eps rename to src/externals/pio1/doc/images/block-cyclic.eps diff --git a/externals/pio1/doc/images/block-cyclic.graffle b/src/externals/pio1/doc/images/block-cyclic.graffle similarity index 100% rename from externals/pio1/doc/images/block-cyclic.graffle rename to src/externals/pio1/doc/images/block-cyclic.graffle diff --git a/externals/pio1/doc/images/block-cyclic.png b/src/externals/pio1/doc/images/block-cyclic.png similarity index 100% rename from externals/pio1/doc/images/block-cyclic.png rename to src/externals/pio1/doc/images/block-cyclic.png diff --git a/externals/pio1/doc/images/dof-rearr.eps b/src/externals/pio1/doc/images/dof-rearr.eps similarity index 100% rename from externals/pio1/doc/images/dof-rearr.eps rename to src/externals/pio1/doc/images/dof-rearr.eps diff --git a/externals/pio1/doc/images/dof-rearr.graffle b/src/externals/pio1/doc/images/dof-rearr.graffle similarity index 100% rename from externals/pio1/doc/images/dof-rearr.graffle rename to src/externals/pio1/doc/images/dof-rearr.graffle diff --git a/externals/pio1/doc/images/dof-rearr.png b/src/externals/pio1/doc/images/dof-rearr.png similarity index 100% rename from externals/pio1/doc/images/dof-rearr.png rename to src/externals/pio1/doc/images/dof-rearr.png diff --git a/externals/pio1/doc/images/dof.eps b/src/externals/pio1/doc/images/dof.eps similarity index 100% rename from externals/pio1/doc/images/dof.eps rename to src/externals/pio1/doc/images/dof.eps diff --git a/externals/pio1/doc/images/dof.graffle b/src/externals/pio1/doc/images/dof.graffle similarity index 100% rename from externals/pio1/doc/images/dof.graffle rename to src/externals/pio1/doc/images/dof.graffle diff --git a/externals/pio1/doc/images/dof.png b/src/externals/pio1/doc/images/dof.png similarity index 100% rename from externals/pio1/doc/images/dof.png rename to src/externals/pio1/doc/images/dof.png diff --git a/externals/pio1/doc/testpio_example.txt b/src/externals/pio1/doc/testpio_example.txt similarity index 100% rename from externals/pio1/doc/testpio_example.txt rename to src/externals/pio1/doc/testpio_example.txt diff --git a/externals/pio1/doxygen.sty b/src/externals/pio1/doxygen.sty similarity index 100% rename from externals/pio1/doxygen.sty rename to src/externals/pio1/doxygen.sty diff --git a/externals/pio1/pio/CMakeLists.txt b/src/externals/pio1/pio/CMakeLists.txt similarity index 96% rename from externals/pio1/pio/CMakeLists.txt rename to src/externals/pio1/pio/CMakeLists.txt index 3b10c1a2cbd..d2c162bb71d 100644 --- a/externals/pio1/pio/CMakeLists.txt +++ b/src/externals/pio1/pio/CMakeLists.txt @@ -9,7 +9,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8.5) IF (USER_CMAKE_MODULE_PATH) SET (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${USER_CMAKE_MODULE_PATH}) ELSE() - SET (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../pio2/cmake") + SET (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") ENDIF() find_file( TESTFILE NAMES TryCSizeOf.f90 PATHS ${CMAKE_MODULE_PATH} NO_DEFAULT_PATH) get_filename_component( TESTFILEPATH ${TESTFILE} PATH) @@ -29,9 +29,6 @@ IF(${WITH_CSIZEOF} STREQUAL FALSE) endif() # Netcdf is required - -SET (CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../pio2/cmake" ${CMAKE_MODULE_PATH}) - #SET (NETCDF_FIND_COMPONENTS F90) FIND_PACKAGE(NetCDF "4.3.3" COMPONENTS C Fortran) IF (${NetCDF_Fortran_FOUND}) @@ -46,7 +43,7 @@ ENDIF () # PNetcdf is optional but used by default OPTION(WITH_PNETCDF "Whether to build with PnetCDF" TRUE) IF (${WITH_PNETCDF}) - FIND_PACKAGE(PnetCDF REQUIRED) + FIND_PACKAGE(PnetCDF REQUIRED COMPONENTS Fortran) ELSE () MESSAGE(WARNING "Warning: Not building with PNetcdf - cannot run all regression tests.") ENDIF () @@ -84,8 +81,8 @@ IF(NetCDF_Fortran_FOUND) ELSE() SET(bld_PIO_DEFINITIONS ${bld_PIO_DEFINITIONS} -D_NONETCDF) ENDIF() -IF(PnetCDF_C_FOUND) - SET(pio_include_dirs_ ${pio_include_dirs_} ${PNetCDF_INCLUDE_DIR}) +IF(PnetCDF_Fortran_FOUND) + SET(pio_include_dirs_ ${pio_include_dirs_} ${PnetCDF_Fortran_INCLUDE_DIRS}) SET(bld_PIO_DEFINITIONS ${bld_PIO_DEFINITIONS} -D_PNETCDF) ELSE() SET(bld_PIO_DEFINITIONS ${bld_PIO_DEFINITIONS} -D_NOPNETCDF) @@ -192,5 +189,3 @@ endif() if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/../testpio) ADD_SUBDIRECTORY(../testpio testpio) endif() - - diff --git a/externals/pio1/pio/C_interface_mod.F90 b/src/externals/pio1/pio/C_interface_mod.F90 similarity index 100% rename from externals/pio1/pio/C_interface_mod.F90 rename to src/externals/pio1/pio/C_interface_mod.F90 diff --git a/externals/pio1/pio/Makefile.conf.in b/src/externals/pio1/pio/Makefile.conf.in similarity index 100% rename from externals/pio1/pio/Makefile.conf.in rename to src/externals/pio1/pio/Makefile.conf.in diff --git a/externals/pio1/pio/README.config b/src/externals/pio1/pio/README.config similarity index 100% rename from externals/pio1/pio/README.config rename to src/externals/pio1/pio/README.config diff --git a/externals/pio1/pio/alloc_mod.F90.in b/src/externals/pio1/pio/alloc_mod.F90.in similarity index 100% rename from externals/pio1/pio/alloc_mod.F90.in rename to src/externals/pio1/pio/alloc_mod.F90.in diff --git a/externals/pio1/pio/box_rearrange.F90.in b/src/externals/pio1/pio/box_rearrange.F90.in similarity index 100% rename from externals/pio1/pio/box_rearrange.F90.in rename to src/externals/pio1/pio/box_rearrange.F90.in diff --git a/externals/pio1/pio/calcdecomp.F90 b/src/externals/pio1/pio/calcdecomp.F90 similarity index 100% rename from externals/pio1/pio/calcdecomp.F90 rename to src/externals/pio1/pio/calcdecomp.F90 diff --git a/externals/pio1/pio/calcdisplace_mod.F90 b/src/externals/pio1/pio/calcdisplace_mod.F90 similarity index 100% rename from externals/pio1/pio/calcdisplace_mod.F90 rename to src/externals/pio1/pio/calcdisplace_mod.F90 diff --git a/externals/pio1/pio/config.h.in b/src/externals/pio1/pio/config.h.in similarity index 100% rename from externals/pio1/pio/config.h.in rename to src/externals/pio1/pio/config.h.in diff --git a/externals/pio1/pio/dtypes.h b/src/externals/pio1/pio/dtypes.h similarity index 100% rename from externals/pio1/pio/dtypes.h rename to src/externals/pio1/pio/dtypes.h diff --git a/externals/pio1/pio/fdepends.awk b/src/externals/pio1/pio/fdepends.awk similarity index 100% rename from externals/pio1/pio/fdepends.awk rename to src/externals/pio1/pio/fdepends.awk diff --git a/externals/pio1/pio/iompi_mod.F90.in b/src/externals/pio1/pio/iompi_mod.F90.in similarity index 100% rename from externals/pio1/pio/iompi_mod.F90.in rename to src/externals/pio1/pio/iompi_mod.F90.in diff --git a/externals/pio1/pio/ionf_mod.F90 b/src/externals/pio1/pio/ionf_mod.F90 similarity index 100% rename from externals/pio1/pio/ionf_mod.F90 rename to src/externals/pio1/pio/ionf_mod.F90 diff --git a/externals/pio1/pio/nf_mod.F90 b/src/externals/pio1/pio/nf_mod.F90 similarity index 100% rename from externals/pio1/pio/nf_mod.F90 rename to src/externals/pio1/pio/nf_mod.F90 diff --git a/externals/pio1/pio/pio.F90 b/src/externals/pio1/pio/pio.F90 similarity index 98% rename from externals/pio1/pio/pio.F90 rename to src/externals/pio1/pio/pio.F90 index bcc41a32e37..b8e3a828a30 100644 --- a/externals/pio1/pio/pio.F90 +++ b/src/externals/pio1/pio/pio.F90 @@ -12,7 +12,7 @@ module pio ! only pio_offset is intended for export from kinds use pio_kinds, only : pio_offset - use piolib_mod, only : pio_initdecomp, & + use piolib_mod, only : pio_initdecomp, pio_set_rearr_opts, & pio_openfile, pio_closefile, pio_createfile, pio_setdebuglevel, & pio_seterrorhandling, pio_setframe, pio_init, pio_get_local_array_size, & pio_freedecomp, pio_syncfile,pio_numtowrite,pio_numtoread,pio_setiotype, & diff --git a/externals/pio1/pio/pio_kinds.F90 b/src/externals/pio1/pio/pio_kinds.F90 similarity index 100% rename from externals/pio1/pio/pio_kinds.F90 rename to src/externals/pio1/pio/pio_kinds.F90 diff --git a/externals/pio1/pio/pio_mpi_utils.F90 b/src/externals/pio1/pio/pio_mpi_utils.F90 similarity index 100% rename from externals/pio1/pio/pio_mpi_utils.F90 rename to src/externals/pio1/pio/pio_mpi_utils.F90 diff --git a/externals/pio1/pio/pio_msg_callbacks.F90 b/src/externals/pio1/pio/pio_msg_callbacks.F90 similarity index 100% rename from externals/pio1/pio/pio_msg_callbacks.F90 rename to src/externals/pio1/pio/pio_msg_callbacks.F90 diff --git a/externals/pio1/pio/pio_msg_getput_callbacks.F90.in b/src/externals/pio1/pio/pio_msg_getput_callbacks.F90.in similarity index 100% rename from externals/pio1/pio/pio_msg_getput_callbacks.F90.in rename to src/externals/pio1/pio/pio_msg_getput_callbacks.F90.in diff --git a/externals/pio1/pio/pio_msg_mod.F90 b/src/externals/pio1/pio/pio_msg_mod.F90 similarity index 100% rename from externals/pio1/pio/pio_msg_mod.F90 rename to src/externals/pio1/pio/pio_msg_mod.F90 diff --git a/externals/pio1/pio/pio_nf_utils.F90 b/src/externals/pio1/pio/pio_nf_utils.F90 similarity index 100% rename from externals/pio1/pio/pio_nf_utils.F90 rename to src/externals/pio1/pio/pio_nf_utils.F90 diff --git a/externals/pio1/pio/pio_spmd_utils.F90.in b/src/externals/pio1/pio/pio_spmd_utils.F90.in similarity index 100% rename from externals/pio1/pio/pio_spmd_utils.F90.in rename to src/externals/pio1/pio/pio_spmd_utils.F90.in diff --git a/externals/pio1/pio/pio_support.F90 b/src/externals/pio1/pio/pio_support.F90 similarity index 100% rename from externals/pio1/pio/pio_support.F90 rename to src/externals/pio1/pio/pio_support.F90 diff --git a/externals/pio1/pio/pio_types.F90 b/src/externals/pio1/pio/pio_types.F90 similarity index 99% rename from externals/pio1/pio/pio_types.F90 rename to src/externals/pio1/pio/pio_types.F90 index 3c6f0c78af6..02b0491a273 100644 --- a/externals/pio1/pio/pio_types.F90 +++ b/src/externals/pio1/pio/pio_types.F90 @@ -79,7 +79,7 @@ module pio_types !! @defgroup PIO_rearr_comm_fc_options PIO_rearr_comm_fc_options !! @brief Type that defines the PIO rearranger options !! @details -!! - enable_hs : Enable handshake (true/false) +!! - enable_hs : Enable handshake (true/false) !! - enable_isend : Enable Isends (true/false) !! - max_pend_req : Maximum pending requests (To indicated unlimited !! number of requests use PIO_REARR_COMM_UNLIMITED_PEND_REQ) diff --git a/externals/pio1/pio/pio_utils.F90 b/src/externals/pio1/pio/pio_utils.F90 similarity index 100% rename from externals/pio1/pio/pio_utils.F90 rename to src/externals/pio1/pio/pio_utils.F90 diff --git a/externals/pio1/pio/piodarray.F90.in b/src/externals/pio1/pio/piodarray.F90.in similarity index 100% rename from externals/pio1/pio/piodarray.F90.in rename to src/externals/pio1/pio/piodarray.F90.in diff --git a/externals/pio1/pio/piolib_mod.F90 b/src/externals/pio1/pio/piolib_mod.F90 similarity index 96% rename from externals/pio1/pio/piolib_mod.F90 rename to src/externals/pio1/pio/piolib_mod.F90 index 85993aa9e38..46a8738b366 100644 --- a/externals/pio1/pio/piolib_mod.F90 +++ b/src/externals/pio1/pio/piolib_mod.F90 @@ -58,6 +58,7 @@ module piolib_mod public :: PIO_init, & PIO_finalize, & PIO_initdecomp, & + PIO_set_rearr_opts,& PIO_openfile, & PIO_syncfile, & PIO_createfile, & @@ -1602,7 +1603,7 @@ subroutine init_iosystem_rearr_options(iosystem) #ifdef _NO_FLOW_CONTROL iosystem%rearr_opts%fcd = PIO_rearr_comm_fc_2d_disable #else - ! We ignore the following flags + ! We ignore the following flags ! 1) _MPISERIAL : The flow control code is never used when _MPISERIAL is set ! 2) _USE_COMP2IO_FC/_USE_IO2COMP_FC : These flags are not currently used ! (These were experimental flags). The user can explicitly control @@ -1622,6 +1623,78 @@ subroutine init_iosystem_rearr_options(iosystem) end subroutine init_iosystem_rearr_options + function PIO_set_rearr_opts(iosystem, comm_type, fcd,& + enable_hs_c2i, enable_isend_c2i,& + max_pend_req_c2i,& + enable_hs_i2c, enable_isend_i2c,& + max_pend_req_i2c) result(ierr) + + use pio_types + + type (iosystem_desc_t), intent(inout) :: iosystem + integer, intent(in) :: comm_type, fcd + logical, intent(in) :: enable_hs_c2i, enable_hs_i2c + logical, intent(in) :: enable_isend_c2i, enable_isend_i2c + integer, intent(in) :: max_pend_req_c2i, max_pend_req_i2c + + integer :: ierr + + ierr = PIO_NOERR + + if(max_pend_req_c2i < 0) then + if(max_pend_req_c2i /= PIO_REARR_COMM_UNLIMITED_PEND_REQ) then + call piodie(__PIO_FILE__,__LINE__,& + "Invalid max pend req (comp to io) specified") + end if + end if + if(max_pend_req_i2c < 0) then + if(max_pend_req_i2c /= PIO_REARR_COMM_UNLIMITED_PEND_REQ) then + call piodie(__PIO_FILE__,__LINE__,& + "Invalid max pend req (io to comp) specified") + end if + end if + + iosystem%rearr_opts%comm_type = comm_type + + ! Reset to defaults + iosystem%rearr_opts%comm_fc_opts_comp2io%enable_hs = .false. + iosystem%rearr_opts%comm_fc_opts_comp2io%enable_isend = .false. + iosystem%rearr_opts%comm_fc_opts_comp2io%max_pend_req = DEF_P2P_MAXREQ + + iosystem%rearr_opts%comm_fc_opts_io2comp%enable_hs = .false. + iosystem%rearr_opts%comm_fc_opts_io2comp%enable_isend = .false. + iosystem%rearr_opts%comm_fc_opts_io2comp%max_pend_req = DEF_P2P_MAXREQ + if(iosystem%rearr_opts%comm_type == PIO_REARR_COMM_COLL) then + ! Init/Reset rest of the structure to valid values + iosystem%rearr_opts%fcd = PIO_REARR_COMM_FC_2D_DISABLE + else if(iosystem%rearr_opts%comm_type == PIO_REARR_COMM_P2P) then + iosystem%rearr_opts%fcd = fcd + if(iosystem%rearr_opts%fcd == PIO_REARR_COMM_FC_2D_DISABLE) then + ! Nothing to do here - the opts are already reset to defaults above + else if(iosystem%rearr_opts%fcd == PIO_REARR_COMM_FC_1D_COMP2IO) then + iosystem%rearr_opts%comm_fc_opts_comp2io%enable_hs = enable_hs_c2i + iosystem%rearr_opts%comm_fc_opts_comp2io%enable_isend = enable_isend_c2i + iosystem%rearr_opts%comm_fc_opts_comp2io%max_pend_req = max_pend_req_c2i + else if(iosystem%rearr_opts%fcd == PIO_REARR_COMM_FC_1D_IO2COMP) then + iosystem%rearr_opts%comm_fc_opts_io2comp%enable_hs = enable_hs_i2c + iosystem%rearr_opts%comm_fc_opts_io2comp%enable_isend = enable_isend_i2c + iosystem%rearr_opts%comm_fc_opts_io2comp%max_pend_req = max_pend_req_i2c + else if(iosystem%rearr_opts%fcd == PIO_REARR_COMM_FC_2D_ENABLE) then + iosystem%rearr_opts%comm_fc_opts_comp2io%enable_hs = enable_hs_c2i + iosystem%rearr_opts%comm_fc_opts_comp2io%enable_isend = enable_isend_c2i + iosystem%rearr_opts%comm_fc_opts_comp2io%max_pend_req = max_pend_req_c2i + + iosystem%rearr_opts%comm_fc_opts_io2comp%enable_hs = enable_hs_i2c + iosystem%rearr_opts%comm_fc_opts_io2comp%enable_isend = enable_isend_i2c + iosystem%rearr_opts%comm_fc_opts_io2comp%max_pend_req = max_pend_req_i2c + else + call piodie(__PIO_FILE__,__LINE__, "Invalid flow control dir specified") + end if + else + call piodie(__PIO_FILE__,__LINE__, "Invalid comm type specified") + end if + + end function PIO_set_rearr_opts !> !! @public diff --git a/externals/pio1/pio/pionfatt_mod.F90.in b/src/externals/pio1/pio/pionfatt_mod.F90.in similarity index 100% rename from externals/pio1/pio/pionfatt_mod.F90.in rename to src/externals/pio1/pio/pionfatt_mod.F90.in diff --git a/externals/pio1/pio/pionfget_mod.F90.in b/src/externals/pio1/pio/pionfget_mod.F90.in similarity index 100% rename from externals/pio1/pio/pionfget_mod.F90.in rename to src/externals/pio1/pio/pionfget_mod.F90.in diff --git a/externals/pio1/pio/pionfput_mod.F90.in b/src/externals/pio1/pio/pionfput_mod.F90.in similarity index 99% rename from externals/pio1/pio/pionfput_mod.F90.in rename to src/externals/pio1/pio/pionfput_mod.F90.in index 1ce9353ba2a..48cd56d0df6 100644 --- a/externals/pio1/pio/pionfput_mod.F90.in +++ b/src/externals/pio1/pio/pionfput_mod.F90.in @@ -125,7 +125,6 @@ contains ierr = nfmpi_begin_indep_data(File%fh) if(Ios%io_rank==0 .and. (ierr==NF_EINDEP .or. ierr==PIO_NOERR)) then - print *,__PIO_FILE__,__LINE__,index,count,trim(ival) ierr = nfmpi_put_vara (File%fh, varid, int(index,kind=PIO_OFFSET), & int(count,kind=PIO_OFFSET), ival, int(count,kind=PIO_OFFSET), & MPI_CHARACTER) @@ -697,9 +696,11 @@ contains #endif #endif #ifdef _NETCDF +#ifdef _NETCDF4 case(pio_iotype_netcdf4p) ierr=nf90_var_par_access(File%fh, varid, NF90_COLLECTIVE) ierr = nf90_put_var(File%fh, varid, ival, start=int(pstart), count=int(pcount)) +#endif case(pio_iotype_netcdf, pio_iotype_netcdf4c) ! Only io proc 0 will do writing if (Ios%io_rank == 0) then diff --git a/externals/pio1/pio/pionfread_mod.F90.in b/src/externals/pio1/pio/pionfread_mod.F90.in similarity index 100% rename from externals/pio1/pio/pionfread_mod.F90.in rename to src/externals/pio1/pio/pionfread_mod.F90.in diff --git a/externals/pio1/pio/pionfwrite_mod.F90.in b/src/externals/pio1/pio/pionfwrite_mod.F90.in similarity index 100% rename from externals/pio1/pio/pionfwrite_mod.F90.in rename to src/externals/pio1/pio/pionfwrite_mod.F90.in diff --git a/externals/pio1/pio/piovdc.F90 b/src/externals/pio1/pio/piovdc.F90 similarity index 100% rename from externals/pio1/pio/piovdc.F90 rename to src/externals/pio1/pio/piovdc.F90 diff --git a/externals/pio1/pio/rearr_options.h b/src/externals/pio1/pio/rearr_options.h similarity index 100% rename from externals/pio1/pio/rearr_options.h rename to src/externals/pio1/pio/rearr_options.h diff --git a/externals/pio1/pio/rearrange.F90.in b/src/externals/pio1/pio/rearrange.F90.in similarity index 100% rename from externals/pio1/pio/rearrange.F90.in rename to src/externals/pio1/pio/rearrange.F90.in diff --git a/externals/pio1/pio/topology.c b/src/externals/pio1/pio/topology.c similarity index 100% rename from externals/pio1/pio/topology.c rename to src/externals/pio1/pio/topology.c diff --git a/externals/pio1/scripts/Utils.pm b/src/externals/pio1/scripts/Utils.pm similarity index 100% rename from externals/pio1/scripts/Utils.pm rename to src/externals/pio1/scripts/Utils.pm diff --git a/externals/pio1/scripts/config.pl b/src/externals/pio1/scripts/config.pl similarity index 100% rename from externals/pio1/scripts/config.pl rename to src/externals/pio1/scripts/config.pl diff --git a/externals/pio1/scripts/testpio_yellowstone.pl b/src/externals/pio1/scripts/testpio_yellowstone.pl similarity index 100% rename from externals/pio1/scripts/testpio_yellowstone.pl rename to src/externals/pio1/scripts/testpio_yellowstone.pl diff --git a/externals/pio1/tests/testpio/CAM05.csh b/src/externals/pio1/tests/testpio/CAM05.csh similarity index 100% rename from externals/pio1/tests/testpio/CAM05.csh rename to src/externals/pio1/tests/testpio/CAM05.csh diff --git a/externals/pio1/tests/testpio/CMakeLists.txt b/src/externals/pio1/tests/testpio/CMakeLists.txt similarity index 100% rename from externals/pio1/tests/testpio/CMakeLists.txt rename to src/externals/pio1/tests/testpio/CMakeLists.txt diff --git a/externals/pio1/tests/testpio/MPASA30km.csh b/src/externals/pio1/tests/testpio/MPASA30km.csh similarity index 100% rename from externals/pio1/tests/testpio/MPASA30km.csh rename to src/externals/pio1/tests/testpio/MPASA30km.csh diff --git a/externals/pio1/tests/testpio/MPASA60km.csh b/src/externals/pio1/tests/testpio/MPASA60km.csh similarity index 100% rename from externals/pio1/tests/testpio/MPASA60km.csh rename to src/externals/pio1/tests/testpio/MPASA60km.csh diff --git a/externals/pio1/tests/testpio/POPB.csh b/src/externals/pio1/tests/testpio/POPB.csh similarity index 100% rename from externals/pio1/tests/testpio/POPB.csh rename to src/externals/pio1/tests/testpio/POPB.csh diff --git a/externals/pio1/tests/testpio/POPC.csh b/src/externals/pio1/tests/testpio/POPC.csh similarity index 100% rename from externals/pio1/tests/testpio/POPC.csh rename to src/externals/pio1/tests/testpio/POPC.csh diff --git a/externals/pio1/tests/testpio/POPD.csh b/src/externals/pio1/tests/testpio/POPD.csh similarity index 100% rename from externals/pio1/tests/testpio/POPD.csh rename to src/externals/pio1/tests/testpio/POPD.csh diff --git a/externals/pio1/tests/testpio/POPDv0.csh b/src/externals/pio1/tests/testpio/POPDv0.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv0.csh rename to src/externals/pio1/tests/testpio/POPDv0.csh diff --git a/externals/pio1/tests/testpio/POPDv1.csh b/src/externals/pio1/tests/testpio/POPDv1.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv1.csh rename to src/externals/pio1/tests/testpio/POPDv1.csh diff --git a/externals/pio1/tests/testpio/POPDv2.csh b/src/externals/pio1/tests/testpio/POPDv2.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv2.csh rename to src/externals/pio1/tests/testpio/POPDv2.csh diff --git a/externals/pio1/tests/testpio/POPDv3.csh b/src/externals/pio1/tests/testpio/POPDv3.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv3.csh rename to src/externals/pio1/tests/testpio/POPDv3.csh diff --git a/externals/pio1/tests/testpio/POPDv4.csh b/src/externals/pio1/tests/testpio/POPDv4.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv4.csh rename to src/externals/pio1/tests/testpio/POPDv4.csh diff --git a/externals/pio1/tests/testpio/POPDv5.csh b/src/externals/pio1/tests/testpio/POPDv5.csh similarity index 100% rename from externals/pio1/tests/testpio/POPDv5.csh rename to src/externals/pio1/tests/testpio/POPDv5.csh diff --git a/externals/pio1/tests/testpio/README.testpio b/src/externals/pio1/tests/testpio/README.testpio similarity index 100% rename from externals/pio1/tests/testpio/README.testpio rename to src/externals/pio1/tests/testpio/README.testpio diff --git a/externals/pio1/tests/testpio/WRFB.csh b/src/externals/pio1/tests/testpio/WRFB.csh similarity index 100% rename from externals/pio1/tests/testpio/WRFB.csh rename to src/externals/pio1/tests/testpio/WRFB.csh diff --git a/externals/pio1/tests/testpio/build_defaults.xml b/src/externals/pio1/tests/testpio/build_defaults.xml similarity index 100% rename from externals/pio1/tests/testpio/build_defaults.xml rename to src/externals/pio1/tests/testpio/build_defaults.xml diff --git a/externals/pio1/tests/testpio/check_mod.F90 b/src/externals/pio1/tests/testpio/check_mod.F90 similarity index 100% rename from externals/pio1/tests/testpio/check_mod.F90 rename to src/externals/pio1/tests/testpio/check_mod.F90 diff --git a/externals/pio1/tests/testpio/config_bench.xml b/src/externals/pio1/tests/testpio/config_bench.xml similarity index 100% rename from externals/pio1/tests/testpio/config_bench.xml rename to src/externals/pio1/tests/testpio/config_bench.xml diff --git a/externals/pio1/tests/testpio/fdepends.awk b/src/externals/pio1/tests/testpio/fdepends.awk similarity index 100% rename from externals/pio1/tests/testpio/fdepends.awk rename to src/externals/pio1/tests/testpio/fdepends.awk diff --git a/externals/pio1/tests/testpio/gdecomp_mod.F90 b/src/externals/pio1/tests/testpio/gdecomp_mod.F90 similarity index 100% rename from externals/pio1/tests/testpio/gdecomp_mod.F90 rename to src/externals/pio1/tests/testpio/gdecomp_mod.F90 diff --git a/externals/pio1/tests/testpio/kinds_mod.F90 b/src/externals/pio1/tests/testpio/kinds_mod.F90 similarity index 100% rename from externals/pio1/tests/testpio/kinds_mod.F90 rename to src/externals/pio1/tests/testpio/kinds_mod.F90 diff --git a/externals/pio1/tests/testpio/kraken.128.csh b/src/externals/pio1/tests/testpio/kraken.128.csh similarity index 100% rename from externals/pio1/tests/testpio/kraken.128.csh rename to src/externals/pio1/tests/testpio/kraken.128.csh diff --git a/externals/pio1/tests/testpio/kraken.1K.csh b/src/externals/pio1/tests/testpio/kraken.1K.csh similarity index 100% rename from externals/pio1/tests/testpio/kraken.1K.csh rename to src/externals/pio1/tests/testpio/kraken.1K.csh diff --git a/externals/pio1/tests/testpio/kraken.256.csh b/src/externals/pio1/tests/testpio/kraken.256.csh similarity index 100% rename from externals/pio1/tests/testpio/kraken.256.csh rename to src/externals/pio1/tests/testpio/kraken.256.csh diff --git a/externals/pio1/tests/testpio/kraken.512.csh b/src/externals/pio1/tests/testpio/kraken.512.csh similarity index 100% rename from externals/pio1/tests/testpio/kraken.512.csh rename to src/externals/pio1/tests/testpio/kraken.512.csh diff --git a/externals/pio1/tests/testpio/kraken.64.csh b/src/externals/pio1/tests/testpio/kraken.64.csh similarity index 100% rename from externals/pio1/tests/testpio/kraken.64.csh rename to src/externals/pio1/tests/testpio/kraken.64.csh diff --git a/externals/pio1/tests/testpio/namelist_mod.F90 b/src/externals/pio1/tests/testpio/namelist_mod.F90 similarity index 100% rename from externals/pio1/tests/testpio/namelist_mod.F90 rename to src/externals/pio1/tests/testpio/namelist_mod.F90 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.apb05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.apb05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.apb05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.apb05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.asb01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.asb01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.asb01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.asb01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.asb04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.asb04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.asb04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.asb04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b06 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b06 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b06 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b06 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b07 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b07 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b07 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b07 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b08 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b08 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b08 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b08 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b09 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b09 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b09 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b09 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b10 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b10 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b10 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b10 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b11 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b11 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b11 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b11 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b12 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b12 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b12 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b12 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.b13 b/src/externals/pio1/tests/testpio/namelists/testpio_in.b13 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.b13 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.b13 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb06 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb06 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb06 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb06 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb07 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb07 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb07 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb07 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bb08 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bb08 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bb08 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bb08 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bn01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bn01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bn01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bn01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bn02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bn02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bn02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bn02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.bn03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.bn03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.bn03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.bn03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b06 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b06 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b06 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b06 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b07 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b07 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b07 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b07 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4b08 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4b08 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4b08 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4b08 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4n01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4n01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4n01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4n01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4n02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4n02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4n02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4n02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.n4n03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.n4n03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.n4n03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.n4n03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb06 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb06 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb06 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb06 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb07 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb07 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb07 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb07 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pb08 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pb08 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pb08 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pb08 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pn01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pn01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pn01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pn01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pn02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pn02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pn02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pn02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.pn03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.pn03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.pn03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.pn03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb04 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb04 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb04 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb04 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb05 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb05 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb05 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb05 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb06 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb06 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb06 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb06 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb07 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb07 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb07 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb07 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sb08 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sb08 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sb08 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sb08 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sn01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sn01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sn01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sn01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sn02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sn02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sn02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sn02 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.sn03 b/src/externals/pio1/tests/testpio/namelists/testpio_in.sn03 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.sn03 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.sn03 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.wr01 b/src/externals/pio1/tests/testpio/namelists/testpio_in.wr01 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.wr01 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.wr01 diff --git a/externals/pio1/tests/testpio/namelists/testpio_in.wr02 b/src/externals/pio1/tests/testpio/namelists/testpio_in.wr02 similarity index 100% rename from externals/pio1/tests/testpio/namelists/testpio_in.wr02 rename to src/externals/pio1/tests/testpio/namelists/testpio_in.wr02 diff --git a/externals/pio1/tests/testpio/perl5lib/ChangeLog b/src/externals/pio1/tests/testpio/perl5lib/ChangeLog similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/ChangeLog rename to src/externals/pio1/tests/testpio/perl5lib/ChangeLog diff --git a/externals/pio1/tests/testpio/perl5lib/README b/src/externals/pio1/tests/testpio/perl5lib/README similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/README rename to src/externals/pio1/tests/testpio/perl5lib/README diff --git a/externals/pio1/tests/testpio/perl5lib/XML/Changes b/src/externals/pio1/tests/testpio/perl5lib/XML/Changes similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/Changes rename to src/externals/pio1/tests/testpio/perl5lib/XML/Changes diff --git a/externals/pio1/tests/testpio/perl5lib/XML/Lite.pm b/src/externals/pio1/tests/testpio/perl5lib/XML/Lite.pm similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/Lite.pm rename to src/externals/pio1/tests/testpio/perl5lib/XML/Lite.pm diff --git a/externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm b/src/externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm rename to src/externals/pio1/tests/testpio/perl5lib/XML/Lite/Element.pm diff --git a/externals/pio1/tests/testpio/perl5lib/XML/README b/src/externals/pio1/tests/testpio/perl5lib/XML/README similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/README rename to src/externals/pio1/tests/testpio/perl5lib/XML/README diff --git a/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 b/src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 rename to src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite.3 diff --git a/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 b/src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 similarity index 100% rename from externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 rename to src/externals/pio1/tests/testpio/perl5lib/XML/man3/XML::Lite::Element.3 diff --git a/externals/pio1/tests/testpio/test.csh b/src/externals/pio1/tests/testpio/test.csh similarity index 100% rename from externals/pio1/tests/testpio/test.csh rename to src/externals/pio1/tests/testpio/test.csh diff --git a/externals/pio1/tests/testpio/test_lib.F90 b/src/externals/pio1/tests/testpio/test_lib.F90 similarity index 100% rename from externals/pio1/tests/testpio/test_lib.F90 rename to src/externals/pio1/tests/testpio/test_lib.F90 diff --git a/externals/pio1/tests/testpio/testdecomp.F90 b/src/externals/pio1/tests/testpio/testdecomp.F90 similarity index 100% rename from externals/pio1/tests/testpio/testdecomp.F90 rename to src/externals/pio1/tests/testpio/testdecomp.F90 diff --git a/externals/pio1/tests/testpio/testdecomp.bluefire.run b/src/externals/pio1/tests/testpio/testdecomp.bluefire.run similarity index 100% rename from externals/pio1/tests/testpio/testdecomp.bluefire.run rename to src/externals/pio1/tests/testpio/testdecomp.bluefire.run diff --git a/externals/pio1/tests/testpio/testdecomp_in b/src/externals/pio1/tests/testpio/testdecomp_in similarity index 100% rename from externals/pio1/tests/testpio/testdecomp_in rename to src/externals/pio1/tests/testpio/testdecomp_in diff --git a/externals/pio1/tests/testpio/testpio.F90 b/src/externals/pio1/tests/testpio/testpio.F90 similarity index 100% rename from externals/pio1/tests/testpio/testpio.F90 rename to src/externals/pio1/tests/testpio/testpio.F90 diff --git a/externals/pio1/tests/testpio/testpio_bench.pl b/src/externals/pio1/tests/testpio/testpio_bench.pl similarity index 100% rename from externals/pio1/tests/testpio/testpio_bench.pl rename to src/externals/pio1/tests/testpio/testpio_bench.pl diff --git a/externals/pio1/tests/testpio/testpio_build.pl b/src/externals/pio1/tests/testpio/testpio_build.pl similarity index 100% rename from externals/pio1/tests/testpio/testpio_build.pl rename to src/externals/pio1/tests/testpio/testpio_build.pl diff --git a/externals/pio1/tests/testpio/testpio_run.pl b/src/externals/pio1/tests/testpio/testpio_run.pl similarity index 100% rename from externals/pio1/tests/testpio/testpio_run.pl rename to src/externals/pio1/tests/testpio/testpio_run.pl diff --git a/externals/pio1/tests/testpio/utils_mod.F90 b/src/externals/pio1/tests/testpio/utils_mod.F90 similarity index 100% rename from externals/pio1/tests/testpio/utils_mod.F90 rename to src/externals/pio1/tests/testpio/utils_mod.F90 diff --git a/externals/pio1/tests/testpio/ystest.sh b/src/externals/pio1/tests/testpio/ystest.sh similarity index 100% rename from externals/pio1/tests/testpio/ystest.sh rename to src/externals/pio1/tests/testpio/ystest.sh diff --git a/externals/pio1/tests/unittests/CMakeLists.txt b/src/externals/pio1/tests/unittests/CMakeLists.txt similarity index 100% rename from externals/pio1/tests/unittests/CMakeLists.txt rename to src/externals/pio1/tests/unittests/CMakeLists.txt diff --git a/externals/pio1/tests/unittests/Levy_Notes b/src/externals/pio1/tests/unittests/Levy_Notes similarity index 100% rename from externals/pio1/tests/unittests/Levy_Notes rename to src/externals/pio1/tests/unittests/Levy_Notes diff --git a/externals/pio1/tests/unittests/README b/src/externals/pio1/tests/unittests/README similarity index 100% rename from externals/pio1/tests/unittests/README rename to src/externals/pio1/tests/unittests/README diff --git a/externals/pio1/tests/unittests/basic_tests.F90 b/src/externals/pio1/tests/unittests/basic_tests.F90 similarity index 100% rename from externals/pio1/tests/unittests/basic_tests.F90 rename to src/externals/pio1/tests/unittests/basic_tests.F90 diff --git a/externals/pio1/tests/unittests/driver.F90 b/src/externals/pio1/tests/unittests/driver.F90 similarity index 100% rename from externals/pio1/tests/unittests/driver.F90 rename to src/externals/pio1/tests/unittests/driver.F90 diff --git a/externals/pio1/tests/unittests/global_vars.F90 b/src/externals/pio1/tests/unittests/global_vars.F90 similarity index 100% rename from externals/pio1/tests/unittests/global_vars.F90 rename to src/externals/pio1/tests/unittests/global_vars.F90 diff --git a/externals/pio1/tests/unittests/input.nl b/src/externals/pio1/tests/unittests/input.nl similarity index 100% rename from externals/pio1/tests/unittests/input.nl rename to src/externals/pio1/tests/unittests/input.nl diff --git a/externals/pio1/tests/unittests/nc_set_log_level2.c b/src/externals/pio1/tests/unittests/nc_set_log_level2.c similarity index 100% rename from externals/pio1/tests/unittests/nc_set_log_level2.c rename to src/externals/pio1/tests/unittests/nc_set_log_level2.c diff --git a/externals/pio1/tests/unittests/ncdf_tests.F90 b/src/externals/pio1/tests/unittests/ncdf_tests.F90 similarity index 100% rename from externals/pio1/tests/unittests/ncdf_tests.F90 rename to src/externals/pio1/tests/unittests/ncdf_tests.F90 diff --git a/externals/pio1/tests/unittests/not_netcdf.ieee b/src/externals/pio1/tests/unittests/not_netcdf.ieee similarity index 100% rename from externals/pio1/tests/unittests/not_netcdf.ieee rename to src/externals/pio1/tests/unittests/not_netcdf.ieee diff --git a/externals/pio1/timing/CMakeLists.txt b/src/externals/pio1/timing/CMakeLists.txt similarity index 100% rename from externals/pio1/timing/CMakeLists.txt rename to src/externals/pio1/timing/CMakeLists.txt diff --git a/src/externals/pio1/timing/COPYING b/src/externals/pio1/timing/COPYING new file mode 100644 index 00000000000..324ce86b24c --- /dev/null +++ b/src/externals/pio1/timing/COPYING @@ -0,0 +1,17 @@ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the “Software”), to deal +in the Software for any noncommercial purposes without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the following +conditions: The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. Any +commercial use (including sale) of the software, and derivative development +towards commercial use, requires written permission of the copyright +holder. THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES +OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/externals/pio1/timing/ChangeLog b/src/externals/pio1/timing/ChangeLog similarity index 100% rename from externals/pio1/timing/ChangeLog rename to src/externals/pio1/timing/ChangeLog diff --git a/externals/pio1/timing/GPTLget_memusage.c b/src/externals/pio1/timing/GPTLget_memusage.c similarity index 100% rename from externals/pio1/timing/GPTLget_memusage.c rename to src/externals/pio1/timing/GPTLget_memusage.c diff --git a/externals/pio1/timing/GPTLprint_memusage.c b/src/externals/pio1/timing/GPTLprint_memusage.c similarity index 100% rename from externals/pio1/timing/GPTLprint_memusage.c rename to src/externals/pio1/timing/GPTLprint_memusage.c diff --git a/externals/pio1/timing/GPTLutil.c b/src/externals/pio1/timing/GPTLutil.c similarity index 100% rename from externals/pio1/timing/GPTLutil.c rename to src/externals/pio1/timing/GPTLutil.c diff --git a/externals/pio1/timing/Makefile b/src/externals/pio1/timing/Makefile similarity index 100% rename from externals/pio1/timing/Makefile rename to src/externals/pio1/timing/Makefile diff --git a/externals/pio1/timing/README b/src/externals/pio1/timing/README similarity index 100% rename from externals/pio1/timing/README rename to src/externals/pio1/timing/README diff --git a/externals/pio1/timing/XXXdotF/perf_mod.F b/src/externals/pio1/timing/XXXdotF/perf_mod.F similarity index 100% rename from externals/pio1/timing/XXXdotF/perf_mod.F rename to src/externals/pio1/timing/XXXdotF/perf_mod.F diff --git a/externals/pio1/timing/XXXdotF/perf_utils.F b/src/externals/pio1/timing/XXXdotF/perf_utils.F similarity index 100% rename from externals/pio1/timing/XXXdotF/perf_utils.F rename to src/externals/pio1/timing/XXXdotF/perf_utils.F diff --git a/externals/pio1/timing/f_wrappers.c b/src/externals/pio1/timing/f_wrappers.c similarity index 100% rename from externals/pio1/timing/f_wrappers.c rename to src/externals/pio1/timing/f_wrappers.c diff --git a/externals/pio1/timing/gptl.c b/src/externals/pio1/timing/gptl.c similarity index 100% rename from externals/pio1/timing/gptl.c rename to src/externals/pio1/timing/gptl.c diff --git a/externals/pio1/timing/gptl.h b/src/externals/pio1/timing/gptl.h similarity index 100% rename from externals/pio1/timing/gptl.h rename to src/externals/pio1/timing/gptl.h diff --git a/externals/pio1/timing/gptl.inc b/src/externals/pio1/timing/gptl.inc similarity index 100% rename from externals/pio1/timing/gptl.inc rename to src/externals/pio1/timing/gptl.inc diff --git a/externals/pio1/timing/gptl_papi.c b/src/externals/pio1/timing/gptl_papi.c similarity index 100% rename from externals/pio1/timing/gptl_papi.c rename to src/externals/pio1/timing/gptl_papi.c diff --git a/externals/pio1/timing/perf_mod.F90 b/src/externals/pio1/timing/perf_mod.F90 similarity index 100% rename from externals/pio1/timing/perf_mod.F90 rename to src/externals/pio1/timing/perf_mod.F90 diff --git a/externals/pio1/timing/perf_utils.F90 b/src/externals/pio1/timing/perf_utils.F90 similarity index 100% rename from externals/pio1/timing/perf_utils.F90 rename to src/externals/pio1/timing/perf_utils.F90 diff --git a/externals/pio1/timing/private.h b/src/externals/pio1/timing/private.h similarity index 100% rename from externals/pio1/timing/private.h rename to src/externals/pio1/timing/private.h diff --git a/externals/pio1/timing/threadutil.c b/src/externals/pio1/timing/threadutil.c similarity index 100% rename from externals/pio1/timing/threadutil.c rename to src/externals/pio1/timing/threadutil.c diff --git a/externals/pio2/.gitignore b/src/externals/pio2/.gitignore similarity index 100% rename from externals/pio2/.gitignore rename to src/externals/pio2/.gitignore diff --git a/src/externals/pio2/CMakeLists.txt b/src/externals/pio2/CMakeLists.txt new file mode 100644 index 00000000000..514384a5a9e --- /dev/null +++ b/src/externals/pio2/CMakeLists.txt @@ -0,0 +1,198 @@ +cmake_minimum_required (VERSION 2.8.12) +project (PIO C Fortran) +#cmake_policy(VERSION 3.5.2) + +# The project version number. +set(VERSION_MAJOR 2 CACHE STRING "Project major version number.") +set(VERSION_MINOR 0 CACHE STRING "Project minor version number.") +set(VERSION_PATCH 28 CACHE STRING "Project patch version number.") +mark_as_advanced(VERSION_MAJOR VERSION_MINOR VERSION_PATCH) + +#============================================================================== +# USER-DEFINED OPTIONS (set with "-DOPT=VAL" from command line) +#============================================================================== + +#===== Library Options ===== +option (PIO_ENABLE_FORTRAN "Enable the Fortran library builds" ON) +option (PIO_ENABLE_TIMING "Enable the use of the GPTL timing library" ON) +option (PIO_ENABLE_LOGGING "Enable debug logging (large output possible)" OFF) +option (PIO_ENABLE_DOC "Enable building PIO documentation" ON) +option (PIO_ENABLE_COVERAGE "Enable code coverage" OFF) +option (PIO_ENABLE_EXAMPLES "Enable PIO examples" ON) +option (PIO_INTERNAL_DOC "Enable PIO developer documentation" OFF) +option (PIO_TEST_BIG_ENDIAN "Enable test to see if machine is big endian" ON) +option (PIO_USE_MPIIO "Enable support for MPI-IO auto detect" ON) +option (PIO_USE_MPISERIAL "Enable mpi-serial support (instead of MPI)" OFF) +option (PIO_USE_MALLOC "Use native malloc (instead of bget package)" OFF) +option (WITH_PNETCDF "Require the use of PnetCDF" ON) + +# Set a variable that appears in the config.h.in file. +if(PIO_USE_MALLOC) + set(USE_MALLOC 1) +else() + set(USE_MALLOC 0) +endif() + +# Set a variable that appears in the config.h.in file. +if(PIO_ENABLE_LOGGING) + set(ENABLE_LOGGING 1) +else() + set(ENABLE_LOGGING 0) +endif() + +if(PIO_USE_MPISERIAL) + set(USE_MPI_SERIAL 1) +else() + set(USE_MPI_SERIAL 0) +endif() + +#============================================================================== +# PREPEND TO CMAKE MODULE PATH +#============================================================================== + +#===== Local modules ===== +list (APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake) + +#===== External modules ===== +if (NOT DEFINED USER_CMAKE_MODULE_PATH) + message (STATUS "Importing CMake_Fortran_utils") + execute_process( + COMMAND git clone https://github.com/CESM-Development/CMake_Fortran_utils + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + OUTPUT_QUIET + ERROR_QUIET) + find_path (USER_CMAKE_MODULE_PATH + NAMES mpiexec.cmake + HINTS ${CMAKE_BINARY_DIR}/CMake_Fortran_utils) + if (USER_CMAKE_MODULE_PATH) + message (STATUS "Importing CMake_Fortran_utils - success") + else () + message (FATAL_ERROR "Failed to import CMake_Fortran_utils") + endif () +endif () +set (USER_CMAKE_MODULE_PATH ${USER_CMAKE_MODULE_PATH} + CACHE STRING "Location of the CMake_Fortran_utils") +list (APPEND CMAKE_MODULE_PATH ${USER_CMAKE_MODULE_PATH}) + +INCLUDE (CheckTypeSize) + +#===== MPI ===== +if (PIO_USE_MPISERIAL) + find_package (MPISERIAL COMPONENTS C REQUIRED) + if (MPISERIAL_C_FOUND) + set (CMAKE_REQUIRED_INCLUDES ${MPISERIAL_C_INCLUDE_DIRS}) + endif () +else () + find_package (MPI REQUIRED) + set (CMAKE_REQUIRED_INCLUDES ${MPI_INCLUDE_PATH}) +endif () + +SET(CMAKE_EXTRA_INCLUDE_FILES "mpi.h") +check_type_size("MPI_Offset" SIZEOF_MPI_OFFSET) +SET(CMAKE_EXTRA_INCLUDE_FILES) + +#===== Library Variables ===== +set (PIO_FILESYSTEM_HINTS IGNORE CACHE STRING "Filesystem hints (lustre or gpfs)") + +#===== Testing Options ===== +option (PIO_ENABLE_TESTS "Enable the testing builds" ON) +option (PIO_VALGRIND_CHECK "Enable memory leak check using valgrind" OFF) + +#============================================================================== +# BACKWARDS COMPATIBILITY +#============================================================================== + +# Old NETCDF_DIR variable --> NetCDF_PATH +if (DEFINED NETCDF_DIR) + set (NetCDF_PATH ${NETCDF_DIR} + CACHE STRING "Location of the NetCDF library installation") +endif () + +# Old PNETCDF_DIR variable --> PnetCDF_PATH +if (DEFINED PNETCDF_DIR) + set (PnetCDF_PATH ${PNETCDF_DIR} + CACHE STRING "Location of the PnetCDF library installation") +endif () + +#============================================================================== +# HELPFUL GLOBAL VARIABLES +#============================================================================== + +# System Name +string (TOUPPER "${CMAKE_SYSTEM_NAME}" CMAKE_SYSTEM_NAME_CAPS) +set (CMAKE_SYSTEM_DIRECTIVE "${CMAKE_SYSTEM_NAME_CAPS}" + CACHE STRING "System name preprocessor directive") + +# C Compiler Name +string (TOUPPER "${CMAKE_C_COMPILER_ID}" CMAKE_C_COMPILER_NAME) +if (CMAKE_C_COMPILER_NAME STREQUAL "XL") + set (CMAKE_C_COMPILER_NAME "IBM") +endif () +set (CMAKE_C_COMPILER_DIRECTIVE "CPR${CMAKE_C_COMPILER_NAME}" + CACHE STRING "C compiler name preprocessor directive") + +# Fortran Compiler Name +string (TOUPPER "${CMAKE_Fortran_COMPILER_ID}" CMAKE_Fortran_COMPILER_NAME) +if (CMAKE_Fortran_COMPILER_NAME STREQUAL "XL") + set (CMAKE_Fortran_COMPILER_NAME "IBM") +endif () +set (CMAKE_Fortran_COMPILER_DIRECTIVE "CPR${CMAKE_Fortran_COMPILER_NAME}" + CACHE STRING "Fortran compiler name preprocessor directive") + +# configure a header file to pass some of the CMake settings +# to the source code +configure_file ( + "${PROJECT_SOURCE_DIR}/src/clib/config.h.in" + "${PROJECT_BINARY_DIR}/src/clib/config.h" + ) + +#============================================================================== +# SET CODE COVERAGE COMPILER FLAGS +#============================================================================== + +# Only support GNU compilers at this time +if (PIO_ENABLE_COVERAGE) + if (CMAKE_C_COMPILER_NAME STREQUAL "GNU") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fprofile-arcs -ftest-coverage") + else () + message (WARNING "The C compiler is non-GNU: coverage of C code could NOT be enabled") + endif () + if (CMAKE_Fortran_COMPILER_NAME STREQUAL "GNU") + set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fprofile-arcs -ftest-coverage") + else () + message (WARNING "The Fortran compiler is non-GNU: coverage of Fortran code could NOT be enabled") + endif () +endif () + +#============================================================================== +# INCLUDE SOURCE DIRECTORIES +#============================================================================== + +# Libraries +add_subdirectory (src) + +#============================================================================== +# TESTING TARGET +#============================================================================== + +# Custom "piotests" target (builds the test executables) +add_custom_target (tests) + +# Custom "check" target that depends upon "tests" +add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND}) +add_dependencies (check tests) + +# Tests +if (PIO_ENABLE_TESTS) + enable_testing() + include (CTest) + add_subdirectory (tests) + if (PIO_ENABLE_EXAMPLES) + add_subdirectory (examples) + endif () +endif () + +# Documentation +if (PIO_ENABLE_DOC) + add_subdirectory (doc) +endif () diff --git a/externals/pio2/CTestConfig.cmake b/src/externals/pio2/CTestConfig.cmake similarity index 54% rename from externals/pio2/CTestConfig.cmake rename to src/externals/pio2/CTestConfig.cmake index 3a7e8835ac9..cd7099ae6bb 100644 --- a/externals/pio2/CTestConfig.cmake +++ b/src/externals/pio2/CTestConfig.cmake @@ -10,6 +10,14 @@ set (CTEST_PROJECT_NAME "PIO") set (CTEST_NIGHTLY_START_TIME "00:00:00 EST") set (CTEST_DROP_METHOD "http") -set (CTEST_DROP_SITE "my.cdash.org") -set (CTEST_DROP_LOCATION "/submit.php?project=PIO") +if (DEFINED ENV{PIO_DASHBOARD_DROP_SITE}) + set (CTEST_DROP_SITE "$ENV{PIO_DASHBOARD_DROP_SITE}") +else () + set (CTEST_DROP_SITE "my.cdash.org") +endif () +if (DEFINED ENV{PIO_DASHBOARD_PROJECT_NAME}) + set (CTEST_DROP_LOCATION "/submit.php?project=$ENV{PIO_DASHBOARD_PROJECT_NAME}") +else () + set (CTEST_DROP_LOCATION "/submit.php?project=PIO") +endif () set (CTEST_DROP_SITE_CDASH TRUE) diff --git a/externals/pio2/CTestScript.cmake b/src/externals/pio2/CTestScript.cmake similarity index 79% rename from externals/pio2/CTestScript.cmake rename to src/externals/pio2/CTestScript.cmake index b313f1a24f5..e817f422d15 100644 --- a/externals/pio2/CTestScript.cmake +++ b/src/externals/pio2/CTestScript.cmake @@ -18,7 +18,7 @@ else () set (CTEST_DASHBOARD_ROOT "$ENV{HOME}/pio-dashboard") endif () -## -- Compiler ID +## -- Compiler ID if (DEFINED ENV{PIO_COMPILER_ID}) set (compid "$ENV{PIO_COMPILER_ID}") else () @@ -40,6 +40,7 @@ execute_process (COMMAND ${HOSTNAME_CMD} OUTPUT_STRIP_TRAILING_WHITESPACE) ## -- Set hostname ID (e.g., alcf, nwsc, nersc, ...) +message ("hostname is ${HOSTNAME}") # UCAR/NWSC Machines if (HOSTNAME MATCHES "^yslogin" OR @@ -47,13 +48,17 @@ if (HOSTNAME MATCHES "^yslogin" OR HOSTNAME MATCHES "^caldera" OR HOSTNAME MATCHES "^pronghorn") set (HOSTNAME_ID "nwsc") +# New UCAR/NWSC SGI Machines +elseif (HOSTNAME MATCHES "^laramie" OR + HOSTNAME MATCHES "^chadmin") + set (HOSTNAME_ID "nwscla") # ALCF/Argonne Machines elseif (HOSTNAME MATCHES "^mira" OR HOSTNAME MATCHES "^cetus" OR HOSTNAME MATCHES "^vesta" OR HOSTNAME MATCHES "^cooley") set (HOSTNAME_ID "alcf") -# ALCF/Argonne Machines +# NERSC Machines elseif (HOSTNAME MATCHES "^edison" OR HOSTNAME MATCHES "^cori" OR HOSTNAME MATCHES "^nid") @@ -64,6 +69,19 @@ elseif (HOSTNAME MATCHES "^h2ologin" ) # CGD local linux cluster elseif (HOSTNAME MATCHES "^hobart") set (HOSTNAME_ID "cgd") +# Argonne Linux workstations +elseif (HOSTNAME MATCHES "^compute001" OR + HOSTNAME MATCHES "^thwomp" OR + HOSTNAME MATCHES "^stomp" OR + HOSTNAME MATCHES "^crush" OR + HOSTNAME MATCHES "^crank" OR + HOSTNAME MATCHES "^steamroller" OR + HOSTNAME MATCHES "^grind" OR + HOSTNAME MATCHES "^churn" OR + HOSTNAME MATCHES "^trounce" OR + HOSTNAME MATCHES "^thrash" OR + HOSTNAME MATCHES "^vanquish") + set (HOSTNAME_ID "anlworkstation") else () if (CMAKE_SYSTEM_NAME MATCHES "Catamount") set (HOSTNAME_ID "ncsa") @@ -92,9 +110,9 @@ find_program (CTEST_GIT_COMMAND NAMES git) ## -- make command find_program (MAKE NAMES make) -#----------------------------------------------------------- +#----------------------------------------------------------- #-- Generate build-specific information -#----------------------------------------------------------- +#----------------------------------------------------------- ## -- CTest Site Name @@ -107,25 +125,25 @@ set (CTEST_BUILD_NAME "${osname}-${osrel}-${cpu}-${compid}") ## -- SRC Dir (where this script exists) set (CTEST_SOURCE_DIRECTORY "${CTEST_SCRIPT_DIRECTORY}") -## -- BIN Dir +## -- BIN Dir set (CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}/build-${CTEST_BUILD_NAME}-${CTEST_BUILD_GROUP}") ## -- Add the CTest script directory to the module path set (CTEST_EXTRA_SCRIPT_PATH "${CTEST_SOURCE_DIRECTORY}/ctest") list (APPEND CMAKE_MODULE_PATH ${CTEST_EXTRA_SCRIPT_PATH}) -# ----------------------------------------------------------- +# ----------------------------------------------------------- # -- Store Build-Specific Info (environment variables) -# ----------------------------------------------------------- +# ----------------------------------------------------------- set (ENV{PIO_DASHBOARD_SITE} ${CTEST_SITE}) set (ENV{PIO_DASHBOARD_BUILD_NAME} ${CTEST_BUILD_NAME}) set (ENV{PIO_DASHBOARD_SOURCE_DIR} ${CTEST_SOURCE_DIRECTORY}) set (ENV{PIO_DASHBOARD_BINARY_DIR} ${CTEST_BINARY_DIRECTORY}) -# ----------------------------------------------------------- +# ----------------------------------------------------------- # -- Run CTest -# ----------------------------------------------------------- +# ----------------------------------------------------------- ## -- Empty the binary directory ctest_empty_binary_directory(${CTEST_BINARY_DIRECTORY}) @@ -140,7 +158,7 @@ message (" -- Update source - ${CTEST_BUILD_NAME} --") set (CTEST_UPDATE_COMMAND "${CTEST_GIT_COMMAND}") ctest_update () -## -- Configure +## -- Configure message (" -- Configure build - ${CTEST_BUILD_NAME} --") include (CTestEnvironment-${HOSTNAME_ID}) set (CTEST_CONFIGURE_COMMAND "${CMAKE_COMMAND} ${CTEST_CONFIGURE_OPTIONS} ${CTEST_SOURCE_DIRECTORY}") @@ -162,9 +180,9 @@ message (" -- Submit to dashboard - ${CTEST_BUILD_NAME} --") message ("** -- PIO_DASHBOARD_SITE=$ENV{PIO_DASHBOARD_SITE}") ctest_submit () -# ----------------------------------------------------------- +# ----------------------------------------------------------- # -- Clear environment -# ----------------------------------------------------------- +# ----------------------------------------------------------- unset (ENV{PIO_DASHBOARD_SITE}) unset (ENV{PIO_DASHBOARD_BUILD_NAME}) diff --git a/externals/pio2/README.md b/src/externals/pio2/README.md similarity index 100% rename from externals/pio2/README.md rename to src/externals/pio2/README.md diff --git a/externals/pio2/cmake/FindGPTL.cmake b/src/externals/pio2/cmake/FindGPTL.cmake similarity index 100% rename from externals/pio2/cmake/FindGPTL.cmake rename to src/externals/pio2/cmake/FindGPTL.cmake diff --git a/externals/pio2/cmake/FindHDF5.cmake b/src/externals/pio2/cmake/FindHDF5.cmake similarity index 100% rename from externals/pio2/cmake/FindHDF5.cmake rename to src/externals/pio2/cmake/FindHDF5.cmake diff --git a/externals/pio2/cmake/FindLIBRT.cmake b/src/externals/pio2/cmake/FindLIBRT.cmake similarity index 100% rename from externals/pio2/cmake/FindLIBRT.cmake rename to src/externals/pio2/cmake/FindLIBRT.cmake diff --git a/externals/pio2/cmake/FindLIBZ.cmake b/src/externals/pio2/cmake/FindLIBZ.cmake similarity index 100% rename from externals/pio2/cmake/FindLIBZ.cmake rename to src/externals/pio2/cmake/FindLIBZ.cmake diff --git a/externals/pio2/cmake/FindMPE.cmake b/src/externals/pio2/cmake/FindMPE.cmake similarity index 100% rename from externals/pio2/cmake/FindMPE.cmake rename to src/externals/pio2/cmake/FindMPE.cmake diff --git a/externals/pio2/cmake/FindMPISERIAL.cmake b/src/externals/pio2/cmake/FindMPISERIAL.cmake similarity index 100% rename from externals/pio2/cmake/FindMPISERIAL.cmake rename to src/externals/pio2/cmake/FindMPISERIAL.cmake diff --git a/externals/pio2/cmake/FindNetCDF.cmake b/src/externals/pio2/cmake/FindNetCDF.cmake similarity index 96% rename from externals/pio2/cmake/FindNetCDF.cmake rename to src/externals/pio2/cmake/FindNetCDF.cmake index 54c5cd7d6e9..344714b18ab 100644 --- a/externals/pio2/cmake/FindNetCDF.cmake +++ b/src/externals/pio2/cmake/FindNetCDF.cmake @@ -71,6 +71,11 @@ foreach (NCDFcomp IN LISTS NetCDF_FIND_VALID_COMPONENTS) DEFINITIONS -I${NetCDF_C_INCLUDE_DIR} COMMENT "whether NetCDF has parallel support") + # Check if logging enabled + set(CMAKE_REQUIRED_INCLUDES ${NetCDF_C_INCLUDE_DIR}) + set(CMAKE_REQUIRED_LIBRARIES ${NetCDF_C_LIBRARIES}) + CHECK_FUNCTION_EXISTS(nc_set_log_level NetCDF_C_LOGGING_ENABLED) + endif () # Dependencies diff --git a/externals/pio2/cmake/FindPAPI.cmake b/src/externals/pio2/cmake/FindPAPI.cmake similarity index 100% rename from externals/pio2/cmake/FindPAPI.cmake rename to src/externals/pio2/cmake/FindPAPI.cmake diff --git a/externals/pio2/cmake/FindPnetCDF.cmake b/src/externals/pio2/cmake/FindPnetCDF.cmake similarity index 100% rename from externals/pio2/cmake/FindPnetCDF.cmake rename to src/externals/pio2/cmake/FindPnetCDF.cmake diff --git a/externals/pio2/cmake/FindSZIP.cmake b/src/externals/pio2/cmake/FindSZIP.cmake similarity index 100% rename from externals/pio2/cmake/FindSZIP.cmake rename to src/externals/pio2/cmake/FindSZIP.cmake diff --git a/externals/pio2/cmake/LibCheck.cmake b/src/externals/pio2/cmake/LibCheck.cmake similarity index 95% rename from externals/pio2/cmake/LibCheck.cmake rename to src/externals/pio2/cmake/LibCheck.cmake index 6d7647e49ec..3f12bdf7965 100644 --- a/externals/pio2/cmake/LibCheck.cmake +++ b/src/externals/pio2/cmake/LibCheck.cmake @@ -1,5 +1,5 @@ include (CMakeParseArguments) - +include (CheckFunctionExists) #============================================================================== # # FUNCTIONS TO HELP WITH Check* MODULES @@ -14,17 +14,17 @@ include (CMakeParseArguments) # HINTS ... # DEFINITIONS ... # COMMENT ) -# +# function (check_macro VARIABLE) # Parse the input arguments set (oneValueArgs COMMENT NAME) set (multiValueArgs HINTS DEFINITIONS) - cmake_parse_arguments (${VARIABLE} "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - + cmake_parse_arguments (${VARIABLE} "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + # If the return variable is defined, already, don't continue if (NOT DEFINED ${VARIABLE}) - + message (STATUS "Checking ${${VARIABLE}_COMMENT}") find_file (${VARIABLE}_TRY_FILE NAMES ${${VARIABLE}_NAME} @@ -40,17 +40,17 @@ function (check_macro VARIABLE) else () message (STATUS "Checking ${${VARIABLE}_COMMENT} - no") endif () - + set (${VARIABLE} ${COMPILE_RESULT} CACHE BOOL "${${VARIABLE}_COMMENT}") - + else () message (STATUS "Checking ${${VARIABLE}_COMMENT} - failed") endif () - + unset (${VARIABLE}_TRY_FILE CACHE) endif () - + endfunction () #______________________________________________________________________________ @@ -60,17 +60,17 @@ endfunction () # NAME # HINTS ... # DEFINITIONS ...) -# +# function (check_version PKG) # Parse the input arguments set (oneValueArgs NAME MACRO_REGEX) set (multiValueArgs HINTS) - cmake_parse_arguments (${PKG} "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + cmake_parse_arguments (${PKG} "" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) # If the return variable is defined, already, don't continue if (NOT DEFINED ${PKG}_VERSION) - + message (STATUS "Checking ${PKG} version") find_file (${PKG}_VERSION_HEADER NAMES ${${PKG}_NAME} @@ -96,9 +96,9 @@ function (check_version PKG) else () message (STATUS "Checking ${PKG} version - failed") endif () - + unset (${PKG}_VERSION_HEADER CACHE) - + endif () endfunction () \ No newline at end of file diff --git a/externals/pio2/cmake/LibFind.cmake b/src/externals/pio2/cmake/LibFind.cmake similarity index 100% rename from externals/pio2/cmake/LibFind.cmake rename to src/externals/pio2/cmake/LibFind.cmake diff --git a/externals/pio2/cmake/LibMPI.cmake b/src/externals/pio2/cmake/LibMPI.cmake similarity index 75% rename from externals/pio2/cmake/LibMPI.cmake rename to src/externals/pio2/cmake/LibMPI.cmake index 9e30717e4c4..2dd9a7d27a8 100644 --- a/externals/pio2/cmake/LibMPI.cmake +++ b/src/externals/pio2/cmake/LibMPI.cmake @@ -1,5 +1,16 @@ include (CMakeParseArguments) +# Find Valgrind to perform memory leak check +if (PIO_VALGRIND_CHECK) + find_program (VALGRIND_COMMAND NAMES valgrind) + if (VALGRIND_COMMAND) + set (VALGRIND_COMMAND_OPTIONS --leak-check=full --show-reachable=yes) + else () + message (WARNING "Valgrind not found: memory leak check could not be performed") + set (VALGRIND_COMMAND "") + endif () +endif () + # # - Functions for parallel testing with CTest # @@ -20,6 +31,11 @@ function (platform_name RETURN_VARIABLE) SITENAME MATCHES "^caldera") set (${RETURN_VARIABLE} "nwsc" PARENT_SCOPE) + + # New NWSC SGI machine + elseif (SITENAME MATCHES "^laramie") + + set (${RETURN_VARIABLE} "nwscla" PARENT_SCOPE) # ALCF/Argonne Machines elseif (SITENAME MATCHES "^mira" OR @@ -29,7 +45,7 @@ function (platform_name RETURN_VARIABLE) set (${RETURN_VARIABLE} "alcf" PARENT_SCOPE) - # ALCF/Argonne Machines + # NERSC Machines elseif (SITENAME MATCHES "^edison" OR SITENAME MATCHES "^cori") @@ -39,6 +55,12 @@ function (platform_name RETURN_VARIABLE) elseif (SITENAME MATCHES "^h2ologin") set (${RETURN_VARIABLE} "ncsa" PARENT_SCOPE) + + # OLCF/Oak Ridge Machines + elseif (SITENAME MATCHES "^eos" OR + SITENAME MATCHES "^titan") + + set (${RETURN_VARIABLE} "olcf" PARENT_SCOPE) else () @@ -78,14 +100,14 @@ function (add_mpi_test TESTNAME) # Run tests directly from the command line set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs} - ${MPIEXEC_PREFLAGS} ${exec_file} + ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${MPIEXEC_POSTFLAGS} ${exec_args}) else () # Run tests from the platform-specific executable set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM} - ${num_procs} ${exec_file} ${exec_args}) + ${num_procs} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${exec_args}) endif () diff --git a/externals/pio2/cmake/TryHDF5_HAS_SZIP.c b/src/externals/pio2/cmake/TryHDF5_HAS_SZIP.c similarity index 100% rename from externals/pio2/cmake/TryHDF5_HAS_SZIP.c rename to src/externals/pio2/cmake/TryHDF5_HAS_SZIP.c diff --git a/externals/pio2/cmake/TryNetCDF_DAP.c b/src/externals/pio2/cmake/TryNetCDF_DAP.c similarity index 100% rename from externals/pio2/cmake/TryNetCDF_DAP.c rename to src/externals/pio2/cmake/TryNetCDF_DAP.c diff --git a/externals/pio2/cmake/TryNetCDF_PARALLEL.c b/src/externals/pio2/cmake/TryNetCDF_PARALLEL.c similarity index 100% rename from externals/pio2/cmake/TryNetCDF_PARALLEL.c rename to src/externals/pio2/cmake/TryNetCDF_PARALLEL.c diff --git a/externals/pio2/cmake/TryNetCDF_PNETCDF.c b/src/externals/pio2/cmake/TryNetCDF_PNETCDF.c similarity index 100% rename from externals/pio2/cmake/TryNetCDF_PNETCDF.c rename to src/externals/pio2/cmake/TryNetCDF_PNETCDF.c diff --git a/externals/pio2/cmake/mpiexec.alcf b/src/externals/pio2/cmake/mpiexec.alcf similarity index 100% rename from externals/pio2/cmake/mpiexec.alcf rename to src/externals/pio2/cmake/mpiexec.alcf diff --git a/externals/pio2/cmake/mpiexec.ncsa b/src/externals/pio2/cmake/mpiexec.ncsa similarity index 100% rename from externals/pio2/cmake/mpiexec.ncsa rename to src/externals/pio2/cmake/mpiexec.ncsa diff --git a/externals/pio2/cmake/mpiexec.nersc b/src/externals/pio2/cmake/mpiexec.nersc similarity index 100% rename from externals/pio2/cmake/mpiexec.nersc rename to src/externals/pio2/cmake/mpiexec.nersc diff --git a/externals/pio2/cmake/mpiexec.nwsc b/src/externals/pio2/cmake/mpiexec.nwsc similarity index 100% rename from externals/pio2/cmake/mpiexec.nwsc rename to src/externals/pio2/cmake/mpiexec.nwsc diff --git a/src/externals/pio2/cmake/mpiexec.nwscla b/src/externals/pio2/cmake/mpiexec.nwscla new file mode 100755 index 00000000000..bb7018bf92c --- /dev/null +++ b/src/externals/pio2/cmake/mpiexec.nwscla @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Arguments: +# +# $1 - Number of MPI Tasks +# $2+ - Executable and its arguments +# + +NP=$1 +shift + +mpiexec_mpt -n $NP $@ diff --git a/src/externals/pio2/cmake/mpiexec.olcf b/src/externals/pio2/cmake/mpiexec.olcf new file mode 100755 index 00000000000..2bb0d1c8468 --- /dev/null +++ b/src/externals/pio2/cmake/mpiexec.olcf @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Arguments: +# +# $1 - Number of MPI Tasks +# $2+ - Executable and its arguments +# + +NP=$1 +shift + +aprun -n $NP $@ diff --git a/externals/pio2/ctest/CTestEnvironment-alcf.cmake b/src/externals/pio2/ctest/CTestEnvironment-alcf.cmake similarity index 100% rename from externals/pio2/ctest/CTestEnvironment-alcf.cmake rename to src/externals/pio2/ctest/CTestEnvironment-alcf.cmake diff --git a/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake b/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake new file mode 100644 index 00000000000..ddf04f063a6 --- /dev/null +++ b/src/externals/pio2/ctest/CTestEnvironment-anlworkstation.cmake @@ -0,0 +1,30 @@ +#============================================================================== +# +# This file sets the environment variables needed to configure and build +# on Argonne Linux workstations +# +#============================================================================== + +# Assume all package locations (NetCDF, PnetCDF, HDF5, etc) are already +# set with existing environment variables: NETCDF, PNETCDF, HDF5, etc. + +# Define the extra CMake configure options +set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE") +set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DNetCDF_PATH=$ENV{NETCDFROOT}") +set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPnetCDF_PATH=$ENV{PNETCDFROOT}") +set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DHDF5_PATH=$ENV{HDF5ROOT}") + +# If ENABLE_COVERAGE environment variable is set, then enable code coverage +if (DEFINED ENV{ENABLE_COVERAGE}) + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_ENABLE_COVERAGE=ON") +endif () + +# If VALGRIND_CHECK environment variable is set, then enable memory leak check using Valgrind +if (DEFINED ENV{VALGRIND_CHECK}) + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_VALGRIND_CHECK=ON") +endif () + +# If USE_MALLOC environment variable is set, then use native malloc (instead of bget package) +if (DEFINED ENV{USE_MALLOC}) + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MALLOC=ON") +endif () diff --git a/externals/pio2/ctest/CTestEnvironment-cgd.cmake b/src/externals/pio2/ctest/CTestEnvironment-cgd.cmake similarity index 100% rename from externals/pio2/ctest/CTestEnvironment-cgd.cmake rename to src/externals/pio2/ctest/CTestEnvironment-cgd.cmake diff --git a/externals/pio2/ctest/CTestEnvironment-ncsa.cmake b/src/externals/pio2/ctest/CTestEnvironment-ncsa.cmake similarity index 100% rename from externals/pio2/ctest/CTestEnvironment-ncsa.cmake rename to src/externals/pio2/ctest/CTestEnvironment-ncsa.cmake diff --git a/externals/pio2/ctest/CTestEnvironment-nersc.cmake b/src/externals/pio2/ctest/CTestEnvironment-nersc.cmake similarity index 100% rename from externals/pio2/ctest/CTestEnvironment-nersc.cmake rename to src/externals/pio2/ctest/CTestEnvironment-nersc.cmake diff --git a/externals/pio2/ctest/CTestEnvironment-nwsc.cmake b/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake similarity index 77% rename from externals/pio2/ctest/CTestEnvironment-nwsc.cmake rename to src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake index 29884997dc8..4a0d6fd3acd 100644 --- a/externals/pio2/ctest/CTestEnvironment-nwsc.cmake +++ b/src/externals/pio2/ctest/CTestEnvironment-nwsc.cmake @@ -1,7 +1,7 @@ #============================================================================== # # This file sets the environment variables needed to configure and build -# on the NCAR Wyoming Supercomputing Center systems +# on the NCAR Wyoming Supercomputing Center systems # (yellowstone/caldera/geyser). # #============================================================================== @@ -10,9 +10,9 @@ # set with existing environment variables: NETCDF, PNETCDF, HDF5, etc. # Define the extra CMake configure options -set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE") +set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_DOC=OFF") # If MPISERIAL environment variable is set, then enable MPISERIAL if (DEFINED ENV{MPISERIAL}) - set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MPISERIAL=ON") + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MPISERIAL=ON -DPIO_ENABLE_EXAMPLES=OFF ") endif () diff --git a/src/externals/pio2/ctest/CTestEnvironment-nwscla.cmake b/src/externals/pio2/ctest/CTestEnvironment-nwscla.cmake new file mode 100644 index 00000000000..b7f1d1c9aef --- /dev/null +++ b/src/externals/pio2/ctest/CTestEnvironment-nwscla.cmake @@ -0,0 +1,18 @@ +#============================================================================== +# +# This file sets the environment variables needed to configure and build +# on the new NCAR Wyoming Supercomputing Center systems +# (laramie/cheyenne). +# +#============================================================================== + +# Assume all package locations (NetCDF, PnetCDF, HDF5, etc) are already +# set with existing environment variables: NETCDF, PNETCDF, HDF5, etc. + +# Define the extra CMake configure options +set (CTEST_CONFIGURE_OPTIONS "-DCMAKE_VERBOSE_MAKEFILE=TRUE -DPIO_ENABLE_ASYNC=TRUE") + +# If MPISERIAL environment variable is set, then enable MPISERIAL +if (DEFINED ENV{MPISERIAL}) + set (CTEST_CONFIGURE_OPTIONS "${CTEST_CONFIGURE_OPTIONS} -DPIO_USE_MPISERIAL=ON") +endif () diff --git a/externals/pio2/ctest/CTestEnvironment-unknown.cmake b/src/externals/pio2/ctest/CTestEnvironment-unknown.cmake similarity index 100% rename from externals/pio2/ctest/CTestEnvironment-unknown.cmake rename to src/externals/pio2/ctest/CTestEnvironment-unknown.cmake diff --git a/externals/pio2/ctest/CTestScript-Test.cmake b/src/externals/pio2/ctest/CTestScript-Test.cmake similarity index 100% rename from externals/pio2/ctest/CTestScript-Test.cmake rename to src/externals/pio2/ctest/CTestScript-Test.cmake diff --git a/externals/pio2/ctest/runcdash-alcf-ibm.sh b/src/externals/pio2/ctest/runcdash-alcf-ibm.sh similarity index 89% rename from externals/pio2/ctest/runcdash-alcf-ibm.sh rename to src/externals/pio2/ctest/runcdash-alcf-ibm.sh index 3df9f86e333..9f36996bdaf 100755 --- a/externals/pio2/ctest/runcdash-alcf-ibm.sh +++ b/src/externals/pio2/ctest/runcdash-alcf-ibm.sh @@ -28,8 +28,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - $GIT clone https://github.com/PARALLELIO/ParallelIO src + $GIT clone https://github.com/PARALLELIO/ParallelIO src fi cd src +git checkout develop +git pull origin develop $CTEST -S CTestScript.cmake,${model} -VV diff --git a/src/externals/pio2/ctest/runcdash-anlworkstation.sh b/src/externals/pio2/ctest/runcdash-anlworkstation.sh new file mode 100755 index 00000000000..44651b2f3e6 --- /dev/null +++ b/src/externals/pio2/ctest/runcdash-anlworkstation.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +# Get/Generate the Dashboard Model +if [ $# -eq 0 ]; then + model=Experimental +else + model=$1 +fi + +source /software/common/adm/packages/softenv-1.6.2/etc/softenv-load.sh +source /software/common/adm/packages/softenv-1.6.2/etc/softenv-aliases.sh + +soft add +gcc-6.2.0 +soft add +mpich-3.2-gcc-6.2.0 +soft add +cmake-3.5.1 + +export NETCDFROOT=/soft/apps/packages/climate/netcdf/4.4.1c-4.2cxx-4.4.4f-parallel/gcc-6.2.0 +export PNETCDFROOT=/soft/apps/packages/climate/pnetcdf/1.7.0/gcc-6.2.0 +export HDF5ROOT=/soft/apps/packages/climate/hdf5/1.8.16-parallel/gcc-6.2.0 + +export CC=mpicc +export FC=mpifort + +export PIO_DASHBOARD_SITE=anlworkstation-`hostname` +export PIO_DASHBOARD_ROOT=/sandbox/dashboard +export CTEST_SCRIPT_DIRECTORY=${PIO_DASHBOARD_ROOT}/src +export PIO_DASHBOARD_SOURCE_DIR=${CTEST_SCRIPT_DIRECTORY} +export PIO_COMPILER_ID=gcc-`gcc --version | head -n 1 | cut -d' ' -f3` + +if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then + mkdir "$PIO_DASHBOARD_ROOT" +fi +cd "$PIO_DASHBOARD_ROOT" + +echo "CTEST_SCRIPT_DIRECTORY="${CTEST_SCRIPT_DIRECTORY} +echo "PIO_DASHBOARD_SOURCE_DIR="${PIO_DASHBOARD_SOURCE_DIR} + +if [ ! -d src ]; then + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src +fi +cd src +git checkout develop +git pull origin develop + +ctest -S CTestScript.cmake,${model} -VV diff --git a/src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh b/src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh new file mode 100755 index 00000000000..56407d9c425 --- /dev/null +++ b/src/externals/pio2/ctest/runcdash-cgd-gnu-openmpi.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +# Get/Generate the Dashboard Model +if [ $# -eq 0 ]; then + model=Experimental +else + model=$1 +fi + +module purge +module load compiler/gnu/5.4.0 +module load tool/parallel-netcdf/1.8.1/gnu-5.4.0/openmpi + +export CC=mpicc +export FC=mpif90 +export PIO_DASHBOARD_SITE="cgd" +export PIO_DASHBOARD_ROOT=/scratch/cluster/jedwards/dashboard +export CTEST_SCRIPT_DIRECTORY=${PIO_DASHBOARD_ROOT}/src +export PIO_DASHBOARD_SOURCE_DIR=${CTEST_SCRIPT_DIRECTORY} +export PIO_COMPILER_ID=gcc-`gcc --version | head -n 1 | cut -d' ' -f3` + +if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then + mkdir "$PIO_DASHBOARD_ROOT" +fi +cd "$PIO_DASHBOARD_ROOT" + +echo "CTEST_SCRIPT_DIRECTORY="${CTEST_SCRIPT_DIRECTORY} +echo "PIO_DASHBOARD_SOURCE_DIR="${PIO_DASHBOARD_SOURCE_DIR} + +if [ ! -d src ]; then + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src +fi +cd src +git checkout develop +git pull origin develop + + +ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runcdash-cgd-nag.sh b/src/externals/pio2/ctest/runcdash-cgd-nag.sh similarity index 65% rename from externals/pio2/ctest/runcdash-cgd-nag.sh rename to src/externals/pio2/ctest/runcdash-cgd-nag.sh index 44ced786200..e413186131a 100755 --- a/externals/pio2/ctest/runcdash-cgd-nag.sh +++ b/src/externals/pio2/ctest/runcdash-cgd-nag.sh @@ -8,16 +8,16 @@ else fi module purge -module load compiler/nag/6.0 -module load tool/parallel-netcdf/1.6.1/nag/openmpi +module load compiler/nag/6.1 +module load tool/parallel-netcdf/1.7.0/nag/mvapich2 export CC=mpicc export FC=mpif90 export PIO_DASHBOARD_SITE="cgd" -export PIO_DASHBOARD_ROOT=/scratch/cluster/katec/dashboard +export PIO_DASHBOARD_ROOT=/scratch/cluster/jedwards/dashboard export CTEST_SCRIPT_DIRECTORY=${PIO_DASHBOARD_ROOT}/src export PIO_DASHBOARD_SOURCE_DIR=${CTEST_SCRIPT_DIRECTORY} -export PIO_COMPILER_ID=Nag-6.0-gcc-`gcc --version | head -n 1 | cut -d' ' -f3` +export PIO_COMPILER_ID=Nag-6.1-gcc-`gcc --version | head -n 1 | cut -d' ' -f3` if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then mkdir "$PIO_DASHBOARD_ROOT" @@ -28,9 +28,11 @@ echo "CTEST_SCRIPT_DIRECTORY="${CTEST_SCRIPT_DIRECTORY} echo "PIO_DASHBOARD_SOURCE_DIR="${PIO_DASHBOARD_SOURCE_DIR} if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src fi - cd src +git checkout develop +git pull origin develop + ctest -S CTestScript.cmake,${model} -VV diff --git a/src/externals/pio2/ctest/runcdash-nersc-cray.sh b/src/externals/pio2/ctest/runcdash-nersc-cray.sh new file mode 100755 index 00000000000..d3516cea7d5 --- /dev/null +++ b/src/externals/pio2/ctest/runcdash-nersc-cray.sh @@ -0,0 +1,74 @@ +#!/bin/sh + +# Get/Generate the Dashboard Model +if [ $# -eq 0 ]; then + model=Experimental +else + model=$1 +fi + +module rm PrgEnv-intel +module rm PrgEnv-cray +module rm PrgEnv-gnu +module rm intel +module rm cce +module rm cray-parallel-netcdf +module rm cray-parallel-hdf5 +module rm pmi +module rm cray-libsci +module rm cray-mpich2 +module rm cray-mpich +module rm cray-netcdf +module rm cray-hdf5 +module rm cray-netcdf-hdf5parallel +module rm craype-sandybridge +module rm craype-ivybridge +module rm craype-haswell +module rm craype +module load PrgEnv-cray + +case "$NERSC_HOST" in + edison) + cd $CSCRATCH/dashboard + module switch cce cce/8.5.1 + module load craype-ivybridge + module load git/2.4.6 + module load cmake/3.3.2 + module load cray-hdf5-parallel/1.8.16 + module load cray-netcdf-hdf5parallel/4.3.3.1 + module load cray-parallel-netcdf/1.7.0 + ;; + cori) + cd $SCRATCH/dashboard + module switch cce cce/8.5.4 + module load craype-mic-knl + module load git/2.9.1 + module load cmake/3.3.2 + module load cray-hdf5-parallel/1.8.16 + module load cray-netcdf-hdf5parallel/4.3.3.1 + module load cray-parallel-netcdf/1.7.0 + ;; + +esac + +export CC=cc +export FC=ftn + +export PIO_DASHBOARD_ROOT=`pwd`/dashboard +export PIO_COMPILER_ID=Cray-`$CC -V 2>&1 | cut -d' ' -f5` + +if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then + mkdir "$PIO_DASHBOARD_ROOT" +fi +cd "$PIO_DASHBOARD_ROOT" + +if [ ! -d src ]; then + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src +fi +cd src +git checkout develop +git pull origin develop + +export HDF5_DISABLE_VERSION_CHECK=2 + +ctest -S CTestScript.cmake,${model} -VV diff --git a/src/externals/pio2/ctest/runcdash-nersc-intel.sh b/src/externals/pio2/ctest/runcdash-nersc-intel.sh new file mode 100755 index 00000000000..55c80559b60 --- /dev/null +++ b/src/externals/pio2/ctest/runcdash-nersc-intel.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +# Get/Generate the Dashboard Model +if [ $# -eq 0 ]; then + model=Experimental +else + model=$1 +fi + +module rm PrgEnv-intel +module rm PrgEnv-cray +module rm PrgEnv-gnu +module rm intel +module rm cce +module rm cray-parallel-netcdf +module rm cray-parallel-hdf5 +module rm pmi +module rm cray-libsci +module rm cray-mpich2 +module rm cray-mpich +module rm cray-netcdf +module rm cray-hdf5 +module rm cray-netcdf-hdf5parallel +module rm craype-sandybridge +module rm craype-ivybridge +module rm craype-haswell +module rm craype +module load PrgEnv-intel + +case "$NERSC_HOST" in + edison) + cd $CSCRATCH/dashboard + module switch intel intel/16.0.0.109 + module load craype-ivybridge + module load git/2.4.6 + module load cmake/3.3.2 + module load cray-hdf5-parallel/1.8.16 + module load cray-netcdf-hdf5parallel/4.3.3.1 + module load cray-parallel-netcdf/1.7.0 + ;; + cori) + cd $SCRATCH/dashboard + module switch intel intel/17.0.1.132 + module load craype-mic-knl + module load git/2.9.1 + module load cmake/3.3.2 + module load cray-hdf5-parallel/1.8.16 + module load cray-netcdf-hdf5parallel/4.3.3.1 + module load cray-parallel-netcdf/1.7.0 + ;; + +esac + +export CC=cc +export FC=ftn + +export PIO_DASHBOARD_ROOT=`pwd`/dashboard +export PIO_COMPILER_ID=Intel-`$CC --version | head -n 1 | cut -d' ' -f3` + +if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then + mkdir "$PIO_DASHBOARD_ROOT" +fi +cd "$PIO_DASHBOARD_ROOT" + +if [ ! -d src ]; then + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src +fi +cd src +git checkout develop +git pull origin develop + +export HDF5_DISABLE_VERSION_CHECK=2 +ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runcdash-nwsc-gnu.sh b/src/externals/pio2/ctest/runcdash-nwsc-gnu.sh similarity index 72% rename from externals/pio2/ctest/runcdash-nwsc-gnu.sh rename to src/externals/pio2/ctest/runcdash-nwsc-gnu.sh index 1049103dab5..d406138bcfd 100755 --- a/externals/pio2/ctest/runcdash-nwsc-gnu.sh +++ b/src/externals/pio2/ctest/runcdash-nwsc-gnu.sh @@ -9,11 +9,11 @@ fi module reset module unload netcdf -module swap intel gnu/5.3.0 +module swap intel gnu/6.1.0 module load git/2.3.0 module load cmake/3.0.2 -module load netcdf-mpi/4.3.3.1 -module load pnetcdf/1.6.1 +module load netcdf-mpi/4.4.1 +module load pnetcdf/1.7.0 export CC=mpicc export FC=mpif90 @@ -27,8 +27,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src fi cd src +git checkout develop +git pull origin develop ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh b/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh similarity index 75% rename from externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh rename to src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh index 6f6ec545c23..68ac5826be9 100755 --- a/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh +++ b/src/externals/pio2/ctest/runcdash-nwsc-intel-mpiserial.sh @@ -14,7 +14,7 @@ module load git/2.3.0 module load cmake/3.0.2 module load netcdf/4.3.3.1 -export MPISERIAL=/glade/p/work/katec/installs/intel_15.0.3 +export MPISERIAL=/glade/u/home/jedwards/mpi-serial/intel15.0.3/ export CC=icc export FC=ifort @@ -28,8 +28,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src fi cd src +git checkout develop +git pull origin develop ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runcdash-nwsc-intel.sh b/src/externals/pio2/ctest/runcdash-nwsc-intel.sh similarity index 71% rename from externals/pio2/ctest/runcdash-nwsc-intel.sh rename to src/externals/pio2/ctest/runcdash-nwsc-intel.sh index 53e2e85ebea..1e72e9a1b31 100755 --- a/externals/pio2/ctest/runcdash-nwsc-intel.sh +++ b/src/externals/pio2/ctest/runcdash-nwsc-intel.sh @@ -9,11 +9,11 @@ fi module reset module unload netcdf -module swap intel intel/16.0.0 +module swap intel intel/16.0.3 module load git/2.3.0 module load cmake/3.0.2 -module load netcdf-mpi/4.3.3.1 -module load pnetcdf/1.6.1 +module load netcdf-mpi/4.4.1 +module load pnetcdf/1.7.0 export CC=mpicc export FC=mpif90 @@ -27,8 +27,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src fi cd src +git checkout develop +git pull origin develop ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runcdash-nwsc-pgi.sh b/src/externals/pio2/ctest/runcdash-nwsc-pgi.sh similarity index 72% rename from externals/pio2/ctest/runcdash-nwsc-pgi.sh rename to src/externals/pio2/ctest/runcdash-nwsc-pgi.sh index 284e605acb3..20c09d619e3 100755 --- a/externals/pio2/ctest/runcdash-nwsc-pgi.sh +++ b/src/externals/pio2/ctest/runcdash-nwsc-pgi.sh @@ -9,11 +9,11 @@ fi module reset module unload netcdf -module swap intel pgi/15.10 +module swap intel pgi/16.5 module load git/2.3.0 module load cmake/3.0.2 -module load netcdf-mpi/4.3.3.1 -module load pnetcdf/1.6.1 +module load netcdf-mpi/4.4.1 +module load pnetcdf/1.7.0 export CC=mpicc export FC=mpif90 @@ -27,8 +27,10 @@ fi cd "$PIO_DASHBOARD_ROOT" if [ ! -d src ]; then - git clone https://github.com/PARALLELIO/ParallelIO src + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src fi cd src +git checkout develop +git pull origin develop ctest -S CTestScript.cmake,${model} -VV diff --git a/src/externals/pio2/ctest/runcdash-nwscla-intel.sh b/src/externals/pio2/ctest/runcdash-nwscla-intel.sh new file mode 100755 index 00000000000..630ac838085 --- /dev/null +++ b/src/externals/pio2/ctest/runcdash-nwscla-intel.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +# Get/Generate the Dashboard Model +if [ $# -eq 0 ]; then + model=Experimental +else + model=$1 +fi + +source /etc/profile.d/modules.sh + +module reset +module unload netcdf +module swap intel intel/17.0.1 +module load cmake/3.7.2 +module load netcdf-mpi/4.4.1.1 +module load pnetcdf/1.8.0 +module switch mpt mpt/2.15 +echo "MODULE LIST..." +module list + +export CC=mpicc +export FC=mpif90 + +export PIO_DASHBOARD_ROOT=/glade/scratch/jedwards/dashboard +export PIO_COMPILER_ID=Intel-`$CC --version | head -n 1 | cut -d' ' -f3` + +if [ ! -d "$PIO_DASHBOARD_ROOT" ]; then + mkdir "$PIO_DASHBOARD_ROOT" +fi +cd "$PIO_DASHBOARD_ROOT" + +if [ ! -d src ]; then + git clone --branch develop https://github.com/PARALLELIO/ParallelIO src +fi +cd src +git checkout develop +git pull origin develop + +ctest -S CTestScript.cmake,${model} -VV diff --git a/externals/pio2/ctest/runctest-alcf.sh b/src/externals/pio2/ctest/runctest-alcf.sh similarity index 100% rename from externals/pio2/ctest/runctest-alcf.sh rename to src/externals/pio2/ctest/runctest-alcf.sh diff --git a/src/externals/pio2/ctest/runctest-anlworkstation.sh b/src/externals/pio2/ctest/runctest-anlworkstation.sh new file mode 100755 index 00000000000..9718a834804 --- /dev/null +++ b/src/externals/pio2/ctest/runctest-anlworkstation.sh @@ -0,0 +1,18 @@ +#!/bin/sh +#============================================================================== +# +# This script defines how to run CTest on the Argonne Linux workstations. +# +# This assumes the CTest model name (e.g., "Nightly") is passed to it when +# run. +# +#============================================================================== + +# Get the CTest script directory +scrdir=$1 + +# Get the CTest model name +model=$2 + +# Run the "ctest" command in another process +ctest -S ${scrdir}/CTestScript-Test.cmake,${model} -V diff --git a/externals/pio2/ctest/runctest-cgd.sh b/src/externals/pio2/ctest/runctest-cgd.sh similarity index 86% rename from externals/pio2/ctest/runctest-cgd.sh rename to src/externals/pio2/ctest/runctest-cgd.sh index 02c99d10b4a..bbd31ccf5d0 100755 --- a/externals/pio2/ctest/runctest-cgd.sh +++ b/src/externals/pio2/ctest/runctest-cgd.sh @@ -1,8 +1,8 @@ #!/bin/sh #============================================================================== # -# This script defines how to run CTest on the Argonne Leadership Computing -# Facility systems (mira/cetus/vesta/cooley). +# This script defines how to run CTest on the NCAR CGD local cluster +# Hobart. # # This assumes the CTest model name (e.g., "Nightly") is passed to it when # run. @@ -29,7 +29,7 @@ echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh chmod +x runctest.sh # Submit the job to the queue -jobid=`/usr/local/bin/qsub -l nodes=1:ppn=4 runctest.sh` +jobid=`/usr/local/bin/qsub -l nodes=1:ppn=8 runctest.sh -q short` # Wait for the job to complete before exiting while true; do diff --git a/externals/pio2/ctest/runctest-ncsa.sh b/src/externals/pio2/ctest/runctest-ncsa.sh similarity index 100% rename from externals/pio2/ctest/runctest-ncsa.sh rename to src/externals/pio2/ctest/runctest-ncsa.sh diff --git a/externals/pio2/ctest/runctest-nersc.sh b/src/externals/pio2/ctest/runctest-nersc.sh similarity index 68% rename from externals/pio2/ctest/runctest-nersc.sh rename to src/externals/pio2/ctest/runctest-nersc.sh index 8c58fba0a1d..a84d26bbeb8 100755 --- a/externals/pio2/ctest/runctest-nersc.sh +++ b/src/externals/pio2/ctest/runctest-nersc.sh @@ -1,8 +1,8 @@ #!/bin/sh #============================================================================== # -# This script defines how to run CTest on the Argonne Leadership Computing -# Facility systems (mira/cetus/vesta/cooley). +# This script defines how to run CTest on the National Energy Research +# Scientific Computing Center systems (edison/cori). # # This assumes the CTest model name (e.g., "Nightly") is passed to it when # run. @@ -19,8 +19,17 @@ model=$2 echo "#!/bin/sh" > runctest.slurm echo "#SBATCH --partition debug" >> runctest.slurm echo "#SBATCH --nodes=1" >> runctest.slurm -echo "#SBATCH --ntasks-per-node=32" >> runctest.slurm -echo "#SBATCH --time=00:15:00" >> runctest.slurm +case "$NERSC_HOST" in + edison) + echo "#SBATCH --ntasks-per-node=32" >> runctest.slurm + ;; + cori) + echo "#SBATCH --ntasks-per-node=68" >> runctest.slurm + echo "#SBATCH -C knl" >> runctest.slurm + ;; +esac + +echo "#SBATCH --time=01:00:00" >> runctest.slurm echo "#SBATCH --export PIO_DASHBOARD_SITE,PIO_DASHBOARD_BUILD_NAME,PIO_DASHBOARD_SOURCE_DIR,PIO_DASHBOARD_BINARY_DIR" >> runctest.slurm #echo "cd \$PBS_O_WORKDIR" >> runctest.pbs @@ -29,7 +38,14 @@ echo "\$CTEST_CMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.s chmod +x runctest.slurm # Submit the job to the queue #jobid=`sbatch runctest.slurm| egrep -o -e "\b[0-9]+$"` -salloc -N 1 ./runctest.slurm +case "$NERSC_HOST" in + edison) + salloc -N 1 ./runctest.slurm + ;; + cori) + salloc -N 1 -C knl ./runctest.slurm + ;; +esac # Wait for the job to complete before exiting #while true; do # status=`squeue -j $jobid` diff --git a/externals/pio2/ctest/runctest-nwsc.sh b/src/externals/pio2/ctest/runctest-nwsc.sh similarity index 100% rename from externals/pio2/ctest/runctest-nwsc.sh rename to src/externals/pio2/ctest/runctest-nwsc.sh diff --git a/src/externals/pio2/ctest/runctest-nwscla.sh b/src/externals/pio2/ctest/runctest-nwscla.sh new file mode 100755 index 00000000000..d3e252317d0 --- /dev/null +++ b/src/externals/pio2/ctest/runctest-nwscla.sh @@ -0,0 +1,44 @@ +#!/bin/sh +#============================================================================== +# +# This script defines how to run CTest on the NCAR Wyoming Supercomputing +# Center systems (cheyenne/laramie). +# +# This assumes the CTest model name (e.g., "Nightly") is passed to it when +# run. +# +#============================================================================== + +# Get the CTest script directory +scrdir=$1 + +# Get the CTest model name +model=$2 + +# Write QSUB submission script with the test execution command +echo "#!/bin/sh" > runctest.sh +echo "#PBS -l walltime=01:00:00" >> runctest.sh +echo "#PBS -l select=1:ncpus=8:mpiprocs=8" >> runctest.sh +echo "#PBS -A P93300606" >> runctest.sh +echo "#PBS -q regular" >> runctest.sh +echo "export PIO_DASHBOARD_SITE=nwscla-${HOSTNAME}" >> runctest.sh +echo "CTESTCMD=`which ctest`" >> runctest.sh +echo "\$CTESTCMD -S ${scrdir}/CTestScript-Test.cmake,${model} -V" >> runctest.sh + +# Make the QSUB script executable +chmod +x runctest.sh + +# Submit the job to the queue +jobid=`qsub -l walltime=01:00:00 runctest.sh` + +# Wait for the job to complete before exiting +while true; do + qstat $jobid + if [ $? -eq 0 ]; then + sleep 30 + else + break; + fi +done + +exit 0 diff --git a/externals/pio2/ctest/runctest-unknown.sh b/src/externals/pio2/ctest/runctest-unknown.sh similarity index 100% rename from externals/pio2/ctest/runctest-unknown.sh rename to src/externals/pio2/ctest/runctest-unknown.sh diff --git a/externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake b/src/externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake similarity index 100% rename from externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake rename to src/externals/pio2/doc/CMakeFiles/3.2.3/CMakeSystem.cmake diff --git a/externals/pio2/doc/CMakeFiles/CMakeOutput.log b/src/externals/pio2/doc/CMakeFiles/CMakeOutput.log similarity index 100% rename from externals/pio2/doc/CMakeFiles/CMakeOutput.log rename to src/externals/pio2/doc/CMakeFiles/CMakeOutput.log diff --git a/externals/pio2/doc/CMakeFiles/cmake.check_cache b/src/externals/pio2/doc/CMakeFiles/cmake.check_cache similarity index 100% rename from externals/pio2/doc/CMakeFiles/cmake.check_cache rename to src/externals/pio2/doc/CMakeFiles/cmake.check_cache diff --git a/src/externals/pio2/doc/CMakeLists.txt b/src/externals/pio2/doc/CMakeLists.txt new file mode 100644 index 00000000000..982b445b66a --- /dev/null +++ b/src/externals/pio2/doc/CMakeLists.txt @@ -0,0 +1,43 @@ +#============================================================================== +# +# API documentation with Doxygen +# +#============================================================================== + +find_package(Doxygen) + +if(DOXYGEN_FOUND) + # This supports the build with/witout internal documentation. + if (PIO_INTERNAL_DOC) + SET(C_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/../src/clib") + else () + SET(C_SRC_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pioc.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_nc4.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_darray.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_get_nc.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_put_nc.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_varm.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_file.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio.h \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pio_nc.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/topology.c \\ +${CMAKE_CURRENT_SOURCE_DIR}/../src/clib/pioc_sc.c" ) + endif () + + # Process the Doxyfile using options set during configure. + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in + ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY) + + # Copy necessary files. + add_custom_target(doc + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/customdoxygen.css + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/DoxygenLayout.xml + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/doxygen.sty + ${CMAKE_CURRENT_BINARY_DIR} + COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating API documentation with Doxygen" VERBATIM) +endif(DOXYGEN_FOUND) diff --git a/externals/pio2/doc/Doxyfile.in b/src/externals/pio2/doc/Doxyfile.in similarity index 99% rename from externals/pio2/doc/Doxyfile.in rename to src/externals/pio2/doc/Doxyfile.in index 31dc3ffd113..d9fe2ca3260 100644 --- a/externals/pio2/doc/Doxyfile.in +++ b/src/externals/pio2/doc/Doxyfile.in @@ -769,11 +769,14 @@ WARN_LOGFILE = # Note: If this tag is empty the current directory is searched. INPUT = @CMAKE_CURRENT_SOURCE_DIR@/source \ - @CMAKE_CURRENT_SOURCE_DIR@/../src/clib \ @CMAKE_CURRENT_SOURCE_DIR@/../src/flib \ @CMAKE_CURRENT_SOURCE_DIR@/../examples/c \ @CMAKE_CURRENT_SOURCE_DIR@/../examples/f03 \ - @CMAKE_BINARY_DIR@/src/flib + @CMAKE_BINARY_DIR@/src/flib \ + @C_SRC_FILES@ + +# Uncomment this after the async code is fully merged into PIO. +# @CMAKE_CURRENT_SOURCE_DIR@/../src/clib # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses diff --git a/externals/pio2/doc/DoxygenLayout.xml b/src/externals/pio2/doc/DoxygenLayout.xml similarity index 100% rename from externals/pio2/doc/DoxygenLayout.xml rename to src/externals/pio2/doc/DoxygenLayout.xml diff --git a/externals/pio2/doc/customdoxygen.css b/src/externals/pio2/doc/customdoxygen.css similarity index 100% rename from externals/pio2/doc/customdoxygen.css rename to src/externals/pio2/doc/customdoxygen.css diff --git a/externals/pio2/doc/doxygen.sty b/src/externals/pio2/doc/doxygen.sty similarity index 100% rename from externals/pio2/doc/doxygen.sty rename to src/externals/pio2/doc/doxygen.sty diff --git a/externals/pio2/doc/images/baseimage.graffle b/src/externals/pio2/doc/images/baseimage.graffle similarity index 100% rename from externals/pio2/doc/images/baseimage.graffle rename to src/externals/pio2/doc/images/baseimage.graffle diff --git a/externals/pio2/doc/images/block-cyclic-rearr.eps b/src/externals/pio2/doc/images/block-cyclic-rearr.eps similarity index 100% rename from externals/pio2/doc/images/block-cyclic-rearr.eps rename to src/externals/pio2/doc/images/block-cyclic-rearr.eps diff --git a/externals/pio2/doc/images/block-cyclic-rearr.graffle b/src/externals/pio2/doc/images/block-cyclic-rearr.graffle similarity index 100% rename from externals/pio2/doc/images/block-cyclic-rearr.graffle rename to src/externals/pio2/doc/images/block-cyclic-rearr.graffle diff --git a/externals/pio2/doc/images/block-cyclic-rearr.png b/src/externals/pio2/doc/images/block-cyclic-rearr.png similarity index 100% rename from externals/pio2/doc/images/block-cyclic-rearr.png rename to src/externals/pio2/doc/images/block-cyclic-rearr.png diff --git a/externals/pio2/doc/images/block-cyclic.eps b/src/externals/pio2/doc/images/block-cyclic.eps similarity index 100% rename from externals/pio2/doc/images/block-cyclic.eps rename to src/externals/pio2/doc/images/block-cyclic.eps diff --git a/externals/pio2/doc/images/block-cyclic.graffle b/src/externals/pio2/doc/images/block-cyclic.graffle similarity index 100% rename from externals/pio2/doc/images/block-cyclic.graffle rename to src/externals/pio2/doc/images/block-cyclic.graffle diff --git a/externals/pio2/doc/images/block-cyclic.png b/src/externals/pio2/doc/images/block-cyclic.png similarity index 100% rename from externals/pio2/doc/images/block-cyclic.png rename to src/externals/pio2/doc/images/block-cyclic.png diff --git a/externals/pio2/doc/images/dof-rearr.eps b/src/externals/pio2/doc/images/dof-rearr.eps similarity index 100% rename from externals/pio2/doc/images/dof-rearr.eps rename to src/externals/pio2/doc/images/dof-rearr.eps diff --git a/externals/pio2/doc/images/dof-rearr.graffle b/src/externals/pio2/doc/images/dof-rearr.graffle similarity index 100% rename from externals/pio2/doc/images/dof-rearr.graffle rename to src/externals/pio2/doc/images/dof-rearr.graffle diff --git a/externals/pio2/doc/images/dof-rearr.png b/src/externals/pio2/doc/images/dof-rearr.png similarity index 100% rename from externals/pio2/doc/images/dof-rearr.png rename to src/externals/pio2/doc/images/dof-rearr.png diff --git a/externals/pio2/doc/images/dof.eps b/src/externals/pio2/doc/images/dof.eps similarity index 100% rename from externals/pio2/doc/images/dof.eps rename to src/externals/pio2/doc/images/dof.eps diff --git a/externals/pio2/doc/images/dof.graffle b/src/externals/pio2/doc/images/dof.graffle similarity index 100% rename from externals/pio2/doc/images/dof.graffle rename to src/externals/pio2/doc/images/dof.graffle diff --git a/externals/pio2/doc/images/dof.png b/src/externals/pio2/doc/images/dof.png similarity index 100% rename from externals/pio2/doc/images/dof.png rename to src/externals/pio2/doc/images/dof.png diff --git a/externals/pio2/doc/source/CAMexample.txt b/src/externals/pio2/doc/source/CAMexample.txt similarity index 100% rename from externals/pio2/doc/source/CAMexample.txt rename to src/externals/pio2/doc/source/CAMexample.txt diff --git a/externals/pio2/doc/source/Decomp.txt b/src/externals/pio2/doc/source/Decomp.txt similarity index 100% rename from externals/pio2/doc/source/Decomp.txt rename to src/externals/pio2/doc/source/Decomp.txt diff --git a/externals/pio2/doc/source/Error.txt b/src/externals/pio2/doc/source/Error.txt similarity index 88% rename from externals/pio2/doc/source/Error.txt rename to src/externals/pio2/doc/source/Error.txt index 4664919cbc8..72c0da23e20 100644 --- a/externals/pio2/doc/source/Error.txt +++ b/src/externals/pio2/doc/source/Error.txt @@ -21,9 +21,6 @@ describing the error and then calling mpi_abort. Application developers can change this behaivior with a call to \ref PIO_seterrorhandling -For example, if a developer wanted -to see if an input netcdf file contained the variable 'U' they might do the following: - \verbinclude errorhandle \copydoc PIO_error_method diff --git a/externals/pio2/doc/source/Examples.txt b/src/externals/pio2/doc/source/Examples.txt similarity index 99% rename from externals/pio2/doc/source/Examples.txt rename to src/externals/pio2/doc/source/Examples.txt index f2af4ebcf66..e18d2926c7c 100644 --- a/externals/pio2/doc/source/Examples.txt +++ b/src/externals/pio2/doc/source/Examples.txt @@ -41,6 +41,8 @@ The Fortran examples are in the examples/f03 subdirectory. - examplePio.f90 A simple example showing a write, then read, of a 1D variable. +### Other Examples + PIO has been implemented in several geophysical component models, including the Community Atmosphere Model (CAM), the Community Land Model (CLM), the Parallel Ocean Program (POP), the Community Ice CodE (CICE), and coupler for used by CCSM4.0 (CPL7). We also provide diff --git a/externals/pio2/doc/source/GettingStarted.txt b/src/externals/pio2/doc/source/GettingStarted.txt similarity index 100% rename from externals/pio2/doc/source/GettingStarted.txt rename to src/externals/pio2/doc/source/GettingStarted.txt diff --git a/externals/pio2/doc/source/Installing.txt b/src/externals/pio2/doc/source/Installing.txt similarity index 99% rename from externals/pio2/doc/source/Installing.txt rename to src/externals/pio2/doc/source/Installing.txt index 0c0020d39ac..6f281152248 100644 --- a/externals/pio2/doc/source/Installing.txt +++ b/src/externals/pio2/doc/source/Installing.txt @@ -20,7 +20,7 @@ The PIO code is currently stored on github at setenv DAV_CORES 4 + > execca ctest + +## PIO2 Performance Test + +To run the performance tests, you will need to add two files to the **tests/performance** subdirectory of the PIO build directory. First, you will need a decomp file. You can download one from our google code page here: +https://svn-ccsm-piodecomps.cgd.ucar.edu/trunk/ . +You can use any of these files, and save them to your home or base work directory. Secondly, you will need to add a namelist file, named "pioperf.nl". Save this file in the directory with your **pioperf** executable (this is found in the **tests/performance** subdirectory of the PIO build directory). + + +The contents of the namelist file should look like: + + &pioperf + + decompfile = "/u/home/user/piodecomp30tasks01dims06.dat" + + pio_typenames = 'pnetcdf' + + niotasks = 30 + + rearrangers = 1 + + nvars = 2 + + / + +Here, the second line ("decompfile") points to the path for your decomp file (wherever you saved it). For the rest of the lines, each item added to the list adds another test to be run. For instance, to test all of the types of supported IO, your pio_typenames would look like: + + pio_typenames = 'pnetcdf','netcdf','netcdf4p','netcdf4c' + +HDF5 is netcdf4p, and Parallel-Netcdf is pnetcdf. + +To test with different numbers of IO tasks, you could do: + + niotasks = 30,15,5 + +(These tasks are the subset of the run tasks that are designated IO tasks) + +To test with both of the rearranger algorithms: + + rearrangers = 1,2 + +(Each rearranger is a different algorithm for converting from data in memory to data in a file on disk. The first one, BOX, is the older method from PIO1, the second, SUBSET, is a newer method that seems to be more efficient in large numbers of tasks) + +To test with different numbers of variables: + + nvars = 8,5,3,2 + +(The more variables you use, the higher data throughput goes, usually) + +To run, submit a job with 'pioperf' as the executable, and at least as many tasks as you have specified in the decomposition file. On yellowstone, a submit script could look like: + + #!/bin/tcsh + + #BSUB -P P00000000 # project code + #BSUB -W 00:10 # wall-clock time (hrs:mins) + #BSUB -n 30 # number of tasks in job + #BSUB -R "span[ptile=16]" # run 16 MPI tasks per node + #BSUB -J pio_perftest # job name + #BSUB -o pio_perftest.%J.out # output file name in which %J is replaced by the job ID + #BSUB -e pio_perftest.%J.err # error file name in which %J is replaced by the job ID + #BSUB -q small # queue + + #run the executable + mpirun.lsf /glade/p/work/katec/pio_work/pio_build/tests/performance/pioperf + +The result(s) will look like a line in the output file such as: +~~~~~~~~~~~~~~ +RESULT: write BOX 4 30 2 16.9905924688 +~~~~~~~~~~~~~~ + +You can decode this as: +1. Read/write describes the io operation performed +2. BOX/SUBSET is the algorithm for the rearranger (as described above) +3. 4 [1-4] is the io library used for the operation. The options here are [1] Parallel-netcdf [2] NetCDF3 [3] NetCDF4-Compressed [4] NetCDF4-Parallel +4. 30 [any number] is the number of io-specific tasks used in the operation. Must be less than the number of MPI tasks used in the test. +5. 2 [any number] is the number of variables read or written during the operation +6. 16.9905924688 [any number] is the Data Rate of the operation in MB/s. This is the important value for determining performance of the system. The higher this numbre is, the better the PIO2 library is performing for the given operation. + +_Last updated: 05-17-2016_ +*/ diff --git a/externals/pio2/doc/source/api.txt b/src/externals/pio2/doc/source/api.txt similarity index 100% rename from externals/pio2/doc/source/api.txt rename to src/externals/pio2/doc/source/api.txt diff --git a/externals/pio2/doc/source/base.txt b/src/externals/pio2/doc/source/base.txt similarity index 96% rename from externals/pio2/doc/source/base.txt rename to src/externals/pio2/doc/source/base.txt index 84079df79eb..9a0401076a9 100644 --- a/externals/pio2/doc/source/base.txt +++ b/src/externals/pio2/doc/source/base.txt @@ -40,9 +40,12 @@ Please review the ChangeLog that is included with the distribution for up-to-dat - \ref intro - \ref install + - \ref mach_walkthrough - \ref decomp - \ref error + - \ref test - \ref examp - \ref faq - \ref api + - \ref contributing_code */ diff --git a/src/externals/pio2/doc/source/contributing_code.txt b/src/externals/pio2/doc/source/contributing_code.txt new file mode 100644 index 00000000000..c811777a02b --- /dev/null +++ b/src/externals/pio2/doc/source/contributing_code.txt @@ -0,0 +1,268 @@ +/****************************************************************************** + * + * + * + * Copyright (C) 2016 + * + * Permission to use, copy, modify, and distribute this software and its + * documentation under the terms of the GNU General Public License is hereby + * granted. No representations are made about the suitability of this software + * for any purpose. It is provided "as is" without express or implied warranty. + * See the GNU General Public License for more details. + * + * Documents produced by Doxygen are derivative works derived from the + * input used in their production; they are not affected by this license. + * + */ /*! \page contributing_code Guide for Contributors + +# Introduction # + +This document describes the code style to be used when contributing to +the PIO library. + +Consistency of style allows bugs to be spotted more quickly and +contributes to code quality. + +# C Code # + +## Warnings ## + +

The C library compiles under GNU gcc without warnings. No code will +be merged with the C library which causes warnings during compile. + +## Backward Compatibility ## + +

We cannot remove or change a function in the public C API. We can +add to it, carefully. + +## Brevity ## + +

Consider: C is a terse language. + +

    +
  • Use the fact that NULL == 0, and 0 == false, to shorten code. +
  • Assign and check return values in the same if statement. +
+ +## Indentation and Spacing## + +
    +
  • Use spaces not tabs. +
  • 4 spaces is the unit of intendation. +
  • Indentation as defined by the "linux" style in emacs (see below). +
  • Use spaces around most operators (=+-*/) not pointer or prefix/postfile (*++--) +
  • Spaces after most keywords (if, for, while, etc.) +
  • No spaces after function name. +
+ +## Braces ## + +

Put braces on their own line, avoiding their use if possible. + +## Documentation ## + +

    +
  • Every function must be documented using doxygen. +
  • Keep internal functions in separate code files, so that Doxygen +can easily build user and development builds of the documentation. +
  • Use the doxygen \@ingroup to put public functions in the correct +group. +
  • Code must be reasonably documented as to intention. +
  • Documentation quality and quantity are part of code review +process. +
  • Document in complete sentences. +
  • Use C (not C++) comment delimiters. +
+ +## Emacs ## + +

Put this in your .emacs file: + +

+(setq c-default-style "linux"
+          c-basic-offset 4)
+
+ +

The tab key (used anywhere on the line) will indent a line +correctly. M-x indent-region will indent a selected region of code. + +

M-x untabify will convert all the tabs in a file to spaces. + +## Code Review ## + +

    + +
  • All code is subject to review. + +
  • Pull requests will be focused on one issue. + +
  • Pull requests may not be submitted until all tests pass. + +
  • All non-trivial pull requests are associated with a github +issue. The issue is where discussion of requirements and +implementation details can be worked out. + +
  • Pull requests will be left up on github for about a day. Request +more time if you need it and are actively reviewing the code. (Note +that pull request can also be reviewed after they are merged, if you +miss one). + +
  • Jim will identify key stakeholders in changed code and ensure they +accept code changes. + +
  • Reviewers are open-minded and ready to accept improvements to the +library. + +
  • Reviewers will make comments on the pull request. All comments +must be resolved. + +
  • If chages are dictated, they happen on the branch, so code +reviewers can see the updated code. + +
  • The pull request is only merged when all programmers agree that +all issues have been resolved. + +
+ +## Merge Proceedure + +
    + +
  • Programmers begin work on a feature or fix by branching from +develop. + +
  • When a branch is ready, it is submitted to code review. + +
  • When code review is complete, and the changes are approved, the PR +is merged into the develop branch. + +
  • Mutliple merges into the develop branch may take place between +test cycles. (???) + +
  • The develop branch is tested automatically by Jenkins. + +
  • The develop branch is tested periodically by CDash (every ~6 +hours). + +
  • After all jenkins and Cdash builds complete successfully, with all +tests passing, and no warnings, the PR is merged into master by the +integrator. + +
  • Multiple PRs may be merged to master between test cycles. (???) + +
  • The branch is then deleted by whomever merged it to master. + +
  • The master branch is then tested on Jenkins. + +
  • The master branch is tested on CDash. Any test failures and the +merge to master will be rolled back. + +
+ +## Formatting Example ## + +
+/** 
+ * \@ingroup PIOc_inq_attname
+ * The PIO-C interface for the NetCDF function nc_inq_attname.
+ *
+ * This routine is called collectively by all tasks in the communicator 
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at: 
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * \@param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * \@param varid the variable ID.
+ * \@param attnum the attribute ID.
+ * \@return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_attname(int ncid, int varid, int attnum, char *name) 
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_attname ncid = %d varid = %d attnum = %d", ncid, varid,
+         attnum));
+
+    /* Find the info about this file. */
+    if (!(file = pio_get_file_from_id(ncid)))
+        return PIO_EBADID;
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async_interface)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_ATTNAME;
+            char name_present = name ? true : false;
+
+            if(ios->compmaster) 
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+            
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);
+#endif /* _PNETCDF */
+#ifdef _NETCDF
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_attname(file->fh, varid, attnum, name);
+#endif /* _NETCDF */
+        LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+    {
+        check_mpi(file, mpierr, __FILE__, __LINE__);            
+        return PIO_EIO;
+    }
+    check_netcdf(file, ierr, __FILE__, __LINE__);
+    
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (!ierr)
+        if (name)
+        {
+            int namelen = strlen(name);
+            if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+                check_mpi(file, mpierr, __FILE__, __LINE__);
+            if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->ioroot,
+                                    ios->my_comm)))
+                check_mpi(file, mpierr, __FILE__, __LINE__);
+        }
+
+    return ierr;
+}
+
+ +## Further Information ## + +

For style issues not already covered in this document, see this style +guide. + +_Last updated: 05-16-2016_ +*/ diff --git a/externals/pio2/doc/source/example/errorhandle b/src/externals/pio2/doc/source/example/errorhandle similarity index 100% rename from externals/pio2/doc/source/example/errorhandle rename to src/externals/pio2/doc/source/example/errorhandle diff --git a/externals/pio2/doc/source/example/simple-bc b/src/externals/pio2/doc/source/example/simple-bc similarity index 100% rename from externals/pio2/doc/source/example/simple-bc rename to src/externals/pio2/doc/source/example/simple-bc diff --git a/externals/pio2/doc/source/example/simple-bc-rearr b/src/externals/pio2/doc/source/example/simple-bc-rearr similarity index 100% rename from externals/pio2/doc/source/example/simple-bc-rearr rename to src/externals/pio2/doc/source/example/simple-bc-rearr diff --git a/externals/pio2/doc/source/example/simple-bc-rearr-pe1 b/src/externals/pio2/doc/source/example/simple-bc-rearr-pe1 similarity index 100% rename from externals/pio2/doc/source/example/simple-bc-rearr-pe1 rename to src/externals/pio2/doc/source/example/simple-bc-rearr-pe1 diff --git a/externals/pio2/doc/source/example/simple-bc-rearr-pe2 b/src/externals/pio2/doc/source/example/simple-bc-rearr-pe2 similarity index 100% rename from externals/pio2/doc/source/example/simple-bc-rearr-pe2 rename to src/externals/pio2/doc/source/example/simple-bc-rearr-pe2 diff --git a/externals/pio2/doc/source/example/simple-dof b/src/externals/pio2/doc/source/example/simple-dof similarity index 100% rename from externals/pio2/doc/source/example/simple-dof rename to src/externals/pio2/doc/source/example/simple-dof diff --git a/externals/pio2/doc/source/example/simple-dof-rearr b/src/externals/pio2/doc/source/example/simple-dof-rearr similarity index 100% rename from externals/pio2/doc/source/example/simple-dof-rearr rename to src/externals/pio2/doc/source/example/simple-dof-rearr diff --git a/externals/pio2/doc/source/faq.txt b/src/externals/pio2/doc/source/faq.txt similarity index 100% rename from externals/pio2/doc/source/faq.txt rename to src/externals/pio2/doc/source/faq.txt diff --git a/externals/pio2/doc/source/mach_walkthrough.txt b/src/externals/pio2/doc/source/mach_walkthrough.txt similarity index 87% rename from externals/pio2/doc/source/mach_walkthrough.txt rename to src/externals/pio2/doc/source/mach_walkthrough.txt index 7741c44e8e4..820a6a3f288 100644 --- a/externals/pio2/doc/source/mach_walkthrough.txt +++ b/src/externals/pio2/doc/source/mach_walkthrough.txt @@ -44,7 +44,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load git/2.3.0
%> module load cmake/3.0.2
%> module load netcdf-mpi/4.3.3.1
- %> module load pnetcdf/1.6.0
+ %> module load pnetcdf/1.6.1
+ GNU @@ -54,7 +54,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load git/2.3.0
%> module load cmake/3.0.2
%> module load netcdf-mpi/4.3.3.1
- %> module load pnetcdf/1.6.0
+ %> module load pnetcdf/1.6.1
+ PGI @@ -64,7 +64,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load git/2.3.0
%> module load cmake/3.0.2
%> module load netcdf-mpi/4.3.3.1
- %> module load pnetcdf/1.6.0
+ %> module load pnetcdf/1.6.1

  • Environment Variables @@ -103,7 +103,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake/3.0.0
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
    + GNU @@ -117,7 +117,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake/3.0.0
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
    + Cray @@ -132,7 +132,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake/3.0.0
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
  • Environment Variables @@ -178,7 +178,7 @@ And then set the following environment variables to add in the rest of the libra %> setenv LIBZ /soft/libraries/alcf/current/xl/ZLIB
    %> setenv HDF5 /soft/libraries/hdf5/1.8.14/cnk-xl/V1R2M2-20150213
    %> setenv NETCDF /soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/V1R2M2-20150213
    - %> setenv PNETCDF /soft/libraries/pnetcdf/1.6.0/cnk-xl/V1R2M2-20150213
    + %> setenv PNETCDF /soft/libraries/pnetcdf/1.6.1/cnk-xl/V1R2M2-20150213
    %> setenv CC /soft/compilers/wrappers/xl/mpixlc_r
    %> setenv FC /soft/compilers/wrappers/xl/mpixlf90_r
    @@ -209,7 +209,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
    + PGI @@ -219,7 +219,7 @@ Modules required for installation depend on your prefered compiler. Issue the co %> module load cmake
    %> module load cray-hdf5-parallel/1.8.14
    %> module load cray-netcdf-hdf5parallel/4.3.3.1
    - %> module load cray-parallel-netcdf/1.6.0
    + %> module load cray-parallel-netcdf/1.6.1
  • Environment Variables @@ -311,6 +311,13 @@ downloads page. (These instructions were tested using version

    Now you should be able to access mpicc, mpifort, and mpirun from the command line. (If not, make sure /usr/local/bin is in path.) +

    Note that if you wish to use valgrind on your programs, you should +configure MPICH like this: +

    CPPFLAGS=-I/usr/include/valgrind ./configure --prefix=/usr/local --enable-g=mem,meminit
    + +

    For this to work you must have packages valgrind and valgrind-devel +installed. +

  • Installing Zlib

    Download from the zlib @@ -343,26 +350,31 @@ sudo make install

    Download parallel-netcdf from the parallel-netcdf download -page. (These instructions were tested using version 1.6.1). Untar -with: 

    tar zxf parallel-netcdf-1.6.1.tar.gz
    +page. (These instructions were tested using version 1.7.0). Untar +with: 
    tar zxf parallel-netcdf-1.7.0.tar.gz

    Build with: -

    cd parallel-netcdf-1.6.1
-FC=mpifort CC=mpicc CFLAGS=-fPIC ./configure --prefix=/usr/local
+
    cd parallel-netcdf-1.7.0
+FC=mpifort CC=mpicc CFLAGS=-fPIC ./configure --prefix=/usr/local/pnetcdf-1.7.0
 make all check
 sudo make install 
    
 
+(This was also tested with version 1.6.1.)
+
 
  • Installing HDF5
 
 
    Download HDF5 from the HDF5 download
-page. (These instructions were tested using version 1.8.15-patch1). Untar
-with: 
    tar zxf hdf5-1.8.15-patch1.tar.gz
    
+page.
 
+
    1.10.0-patch1
    
+
+
    These instructions were tested using version 1.10.0-patch1. Untar
+with: 
    tar zxf hdf5-1.10.0-patch1.tar.gz
    
 
 
    Build with:
-
    cd hdf5-1.8.15-patch1
+
    cd hdf5-1.10.0-patch1
 CC=mpicc ./configure --with-zlib=/usr/local --prefix=/usr/local --with-szlib=/usr/local --enable-parallel
 make all check
 sudo make install 
    
@@ -376,8 +388,8 @@ with: 
    tar zxf netcdf-c-4.4.0.tar.gz
    
 
 
 
    Build with:
-
    cd netcdf-c-4.4.0
-CC=mpicc ./configure --with-zlib=/usr/local --prefix=/usr/local --with-szlib=/usr/local --with-hdf5=/usr/local --enable-pnetcdf --enable-parallel-tests --enable-logging
+
    cd netcdf-c-4.4.1
+CC=mpicc ./configure --enable-parallel-tests --prefix=/usr/local/netcdf-c-4.4.1 --with-hdf5=/usr/local --with-zlib=/usr/local --with-szlib=/usr/local
 make all check
 sudo make install 
    
 
@@ -391,8 +403,7 @@ with: 
    tar zxf netcdf-fortran-4.4.3.tar.gz
    
 
 
    Build with:
 
    cd netcdf-fortran-4.4.3
-F77=mpif77 F90=mpif90  ./configure --with-zlib=/usr/local --prefix=/usr/local --with-szlib=/usr/local --with-hdf5=/usr/local --enable-pnetcdf --enable-parallel-tests --enable-logging
-make all check
+make distclean; LD_LIBRARY_PATH=/usr/local/netcdf-c-4.4.1/lib FC=mpifort CPPFLAGS=-I/usr/local/netcdf-c-4.4.1/include LDFLAGS=-L/usr/local/netcdf-c-4.4.1/lib CC=mpicc ./configure --enable-parallel-tests --prefix=/usr/local/netcdf-c-4.4.1 && make check
 sudo make install 
    
 
 
  • Installing ParallelIO Library
@@ -415,12 +426,14 @@ build from the command line, including tests:
 
 
     cd build
-rm -rf * && CFLAGS=-g FFLAGS=-g CC=mpicc FC=mpifort cmake -DNetCDF_C_PATH=/usr/local -DNetCDF_Fortran_PATH=/usr/local -DPnetCDF_PATH=/usr/local -DPIO_HDF5_LOGGING=On .. && make VERBOSE=1 all tests check
+rm -rf * && CFLAGS='-Wall -g' FFLAGS=-g CC=mpicc FC=mpifort cmake -DNetCDF_C_PATH=/usr/local/netcdf-4.4.1 -DNetCDF_Fortran_PATH=/usr/local/netcdf-fortran-4.4.4 -DPnetCDF_PATH=/usr/local/pnetcdf-1.8.1 -DPIO_ENABLE_LOGGING=On .. && make VERBOSE=1 all tests check
 
    
 
-
    (Note the optional CFLAGS=-g which allows the use of a debugger
+
    Note the optional CFLAGS=-g which allows the use of a debugger
 with this code. Also note the optional VERBOSE=1 provided to make,
-which causes the build commands to be made visible.)
+which causes the build commands to be made visible.
+
+
    Note also the -DPIO_ENABLE_LOGGING=On which is helpful for debugging but should probably not be used in production builds.
 
 
  • Building and Running Performance Tests
 
@@ -477,5 +490,5 @@ variables to these tests will increase the time significantly.
 
 
 
-_Last updated: 11-18-2015_
+_Last updated: 05-16-2016_
 */
diff --git a/externals/pio2/doc/source/testpio_example.txt b/src/externals/pio2/doc/source/testpio_example.txt
similarity index 100%
rename from externals/pio2/doc/source/testpio_example.txt
rename to src/externals/pio2/doc/source/testpio_example.txt
diff --git a/externals/pio2/examples/CMakeLists.txt b/src/externals/pio2/examples/CMakeLists.txt
similarity index 64%
rename from externals/pio2/examples/CMakeLists.txt
rename to src/externals/pio2/examples/CMakeLists.txt
index ec8f36bded1..cd4fc6c23df 100644
--- a/externals/pio2/examples/CMakeLists.txt
+++ b/src/externals/pio2/examples/CMakeLists.txt
@@ -2,13 +2,17 @@
 ### CMakeList.txt for examples using pio
 ###-------------------------------------------------------------------------###
 
-if ("${PLATFORM}" STREQUAL "goldbach" )
+if (PIO_ENABLE_FORTRAN)
+  if ("${PLATFORM}" STREQUAL "goldbach" )
     ADD_SUBDIRECTORY(f03)
-elseif ("${PLATFORM}" STREQUAL "yellowstone" )
+  elseif ("${PLATFORM}" STREQUAL "yellowstone" )
     ADD_SUBDIRECTORY(f03)
     ADD_SUBDIRECTORY(c)
-else()
+  else()
     ADD_SUBDIRECTORY(f03)
     ADD_SUBDIRECTORY(c)
-#    ADD_SUBDIRECTORY(cxx)
+    #    ADD_SUBDIRECTORY(cxx)
+  endif()
+else()
+  ADD_SUBDIRECTORY(c)
 endif()
diff --git a/externals/pio2/examples/basic/CAM05.csh b/src/externals/pio2/examples/basic/CAM05.csh
similarity index 100%
rename from externals/pio2/examples/basic/CAM05.csh
rename to src/externals/pio2/examples/basic/CAM05.csh
diff --git a/externals/pio2/examples/basic/CMakeLists.txt b/src/externals/pio2/examples/basic/CMakeLists.txt
similarity index 100%
rename from externals/pio2/examples/basic/CMakeLists.txt
rename to src/externals/pio2/examples/basic/CMakeLists.txt
diff --git a/externals/pio2/examples/basic/MPASA30km.csh b/src/externals/pio2/examples/basic/MPASA30km.csh
similarity index 100%
rename from externals/pio2/examples/basic/MPASA30km.csh
rename to src/externals/pio2/examples/basic/MPASA30km.csh
diff --git a/externals/pio2/examples/basic/MPASA60km.csh b/src/externals/pio2/examples/basic/MPASA60km.csh
similarity index 100%
rename from externals/pio2/examples/basic/MPASA60km.csh
rename to src/externals/pio2/examples/basic/MPASA60km.csh
diff --git a/externals/pio2/examples/basic/POPB.csh b/src/externals/pio2/examples/basic/POPB.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPB.csh
rename to src/externals/pio2/examples/basic/POPB.csh
diff --git a/externals/pio2/examples/basic/POPC.csh b/src/externals/pio2/examples/basic/POPC.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPC.csh
rename to src/externals/pio2/examples/basic/POPC.csh
diff --git a/externals/pio2/examples/basic/POPD.csh b/src/externals/pio2/examples/basic/POPD.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPD.csh
rename to src/externals/pio2/examples/basic/POPD.csh
diff --git a/externals/pio2/examples/basic/POPDv0.csh b/src/externals/pio2/examples/basic/POPDv0.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv0.csh
rename to src/externals/pio2/examples/basic/POPDv0.csh
diff --git a/externals/pio2/examples/basic/POPDv1.csh b/src/externals/pio2/examples/basic/POPDv1.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv1.csh
rename to src/externals/pio2/examples/basic/POPDv1.csh
diff --git a/externals/pio2/examples/basic/POPDv2.csh b/src/externals/pio2/examples/basic/POPDv2.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv2.csh
rename to src/externals/pio2/examples/basic/POPDv2.csh
diff --git a/externals/pio2/examples/basic/POPDv3.csh b/src/externals/pio2/examples/basic/POPDv3.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv3.csh
rename to src/externals/pio2/examples/basic/POPDv3.csh
diff --git a/externals/pio2/examples/basic/POPDv4.csh b/src/externals/pio2/examples/basic/POPDv4.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv4.csh
rename to src/externals/pio2/examples/basic/POPDv4.csh
diff --git a/externals/pio2/examples/basic/POPDv5.csh b/src/externals/pio2/examples/basic/POPDv5.csh
similarity index 100%
rename from externals/pio2/examples/basic/POPDv5.csh
rename to src/externals/pio2/examples/basic/POPDv5.csh
diff --git a/externals/pio2/examples/basic/README.testpio b/src/externals/pio2/examples/basic/README.testpio
similarity index 100%
rename from externals/pio2/examples/basic/README.testpio
rename to src/externals/pio2/examples/basic/README.testpio
diff --git a/externals/pio2/examples/basic/WRFB.csh b/src/externals/pio2/examples/basic/WRFB.csh
similarity index 100%
rename from externals/pio2/examples/basic/WRFB.csh
rename to src/externals/pio2/examples/basic/WRFB.csh
diff --git a/externals/pio2/examples/basic/alloc_mod.F90.in b/src/externals/pio2/examples/basic/alloc_mod.F90.in
similarity index 100%
rename from externals/pio2/examples/basic/alloc_mod.F90.in
rename to src/externals/pio2/examples/basic/alloc_mod.F90.in
diff --git a/externals/pio2/examples/basic/build_defaults.xml b/src/externals/pio2/examples/basic/build_defaults.xml
similarity index 100%
rename from externals/pio2/examples/basic/build_defaults.xml
rename to src/externals/pio2/examples/basic/build_defaults.xml
diff --git a/externals/pio2/examples/basic/check_mod.F90 b/src/externals/pio2/examples/basic/check_mod.F90
similarity index 100%
rename from externals/pio2/examples/basic/check_mod.F90
rename to src/externals/pio2/examples/basic/check_mod.F90
diff --git a/externals/pio2/examples/basic/config_bench.xml b/src/externals/pio2/examples/basic/config_bench.xml
similarity index 100%
rename from externals/pio2/examples/basic/config_bench.xml
rename to src/externals/pio2/examples/basic/config_bench.xml
diff --git a/externals/pio2/examples/basic/fdepends.awk b/src/externals/pio2/examples/basic/fdepends.awk
similarity index 100%
rename from externals/pio2/examples/basic/fdepends.awk
rename to src/externals/pio2/examples/basic/fdepends.awk
diff --git a/externals/pio2/examples/basic/gdecomp_mod.F90 b/src/externals/pio2/examples/basic/gdecomp_mod.F90
similarity index 100%
rename from externals/pio2/examples/basic/gdecomp_mod.F90
rename to src/externals/pio2/examples/basic/gdecomp_mod.F90
diff --git a/externals/pio2/examples/basic/kinds_mod.F90 b/src/externals/pio2/examples/basic/kinds_mod.F90
similarity index 100%
rename from externals/pio2/examples/basic/kinds_mod.F90
rename to src/externals/pio2/examples/basic/kinds_mod.F90
diff --git a/externals/pio2/examples/basic/kraken.128.csh b/src/externals/pio2/examples/basic/kraken.128.csh
similarity index 100%
rename from externals/pio2/examples/basic/kraken.128.csh
rename to src/externals/pio2/examples/basic/kraken.128.csh
diff --git a/externals/pio2/examples/basic/kraken.1K.csh b/src/externals/pio2/examples/basic/kraken.1K.csh
similarity index 100%
rename from externals/pio2/examples/basic/kraken.1K.csh
rename to src/externals/pio2/examples/basic/kraken.1K.csh
diff --git a/externals/pio2/examples/basic/kraken.256.csh b/src/externals/pio2/examples/basic/kraken.256.csh
similarity index 100%
rename from externals/pio2/examples/basic/kraken.256.csh
rename to src/externals/pio2/examples/basic/kraken.256.csh
diff --git a/externals/pio2/examples/basic/kraken.512.csh b/src/externals/pio2/examples/basic/kraken.512.csh
similarity index 100%
rename from externals/pio2/examples/basic/kraken.512.csh
rename to src/externals/pio2/examples/basic/kraken.512.csh
diff --git a/externals/pio2/examples/basic/kraken.64.csh b/src/externals/pio2/examples/basic/kraken.64.csh
similarity index 100%
rename from externals/pio2/examples/basic/kraken.64.csh
rename to src/externals/pio2/examples/basic/kraken.64.csh
diff --git a/externals/pio2/examples/basic/namelist_mod.F90 b/src/externals/pio2/examples/basic/namelist_mod.F90
similarity index 100%
rename from externals/pio2/examples/basic/namelist_mod.F90
rename to src/externals/pio2/examples/basic/namelist_mod.F90
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.apb05 b/src/externals/pio2/examples/basic/namelists/testpio_in.apb05
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.apb05
rename to src/externals/pio2/examples/basic/namelists/testpio_in.apb05
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.asb01 b/src/externals/pio2/examples/basic/namelists/testpio_in.asb01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.asb01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.asb01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.asb04 b/src/externals/pio2/examples/basic/namelists/testpio_in.asb04
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.asb04
rename to src/externals/pio2/examples/basic/namelists/testpio_in.asb04
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b01 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b02 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b03 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b04 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b04
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b04
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b04
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b05 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b05
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b05
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b05
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b06 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b06
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b06
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b06
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b07 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b07
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b07
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b07
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4b08 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4b08
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4b08
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4b08
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4n01 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4n01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4n01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4n01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4n02 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4n02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4n02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4n02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.n4n03 b/src/externals/pio2/examples/basic/namelists/testpio_in.n4n03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.n4n03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.n4n03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb01 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb02 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb03 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb04 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb04
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb04
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb04
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb05 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb05
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb05
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb05
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb06 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb06
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb06
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb06
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb07 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb07
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb07
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb07
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pb08 b/src/externals/pio2/examples/basic/namelists/testpio_in.pb08
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pb08
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pb08
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pn01 b/src/externals/pio2/examples/basic/namelists/testpio_in.pn01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pn01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pn01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pn02 b/src/externals/pio2/examples/basic/namelists/testpio_in.pn02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pn02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pn02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.pn03 b/src/externals/pio2/examples/basic/namelists/testpio_in.pn03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.pn03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.pn03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps01 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps02 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps03 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps04 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps04
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps04
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps04
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps05 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps05
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps05
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps05
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps06 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps06
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps06
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps06
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps07 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps07
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps07
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps07
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.ps08 b/src/externals/pio2/examples/basic/namelists/testpio_in.ps08
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.ps08
rename to src/externals/pio2/examples/basic/namelists/testpio_in.ps08
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb01 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb02 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb03 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb04 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb04
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb04
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb04
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb05 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb05
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb05
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb05
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb06 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb06
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb06
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb06
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb07 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb07
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb07
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb07
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sb08 b/src/externals/pio2/examples/basic/namelists/testpio_in.sb08
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sb08
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sb08
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sn01 b/src/externals/pio2/examples/basic/namelists/testpio_in.sn01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sn01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sn01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sn02 b/src/externals/pio2/examples/basic/namelists/testpio_in.sn02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sn02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sn02
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.sn03 b/src/externals/pio2/examples/basic/namelists/testpio_in.sn03
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.sn03
rename to src/externals/pio2/examples/basic/namelists/testpio_in.sn03
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.wr01 b/src/externals/pio2/examples/basic/namelists/testpio_in.wr01
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.wr01
rename to src/externals/pio2/examples/basic/namelists/testpio_in.wr01
diff --git a/externals/pio2/examples/basic/namelists/testpio_in.wr02 b/src/externals/pio2/examples/basic/namelists/testpio_in.wr02
similarity index 100%
rename from externals/pio2/examples/basic/namelists/testpio_in.wr02
rename to src/externals/pio2/examples/basic/namelists/testpio_in.wr02
diff --git a/externals/pio2/examples/basic/perl5lib/ChangeLog b/src/externals/pio2/examples/basic/perl5lib/ChangeLog
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/ChangeLog
rename to src/externals/pio2/examples/basic/perl5lib/ChangeLog
diff --git a/externals/pio2/examples/basic/perl5lib/README b/src/externals/pio2/examples/basic/perl5lib/README
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/README
rename to src/externals/pio2/examples/basic/perl5lib/README
diff --git a/externals/pio2/examples/basic/perl5lib/XML/Changes b/src/externals/pio2/examples/basic/perl5lib/XML/Changes
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/Changes
rename to src/externals/pio2/examples/basic/perl5lib/XML/Changes
diff --git a/externals/pio2/examples/basic/perl5lib/XML/Lite.pm b/src/externals/pio2/examples/basic/perl5lib/XML/Lite.pm
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/Lite.pm
rename to src/externals/pio2/examples/basic/perl5lib/XML/Lite.pm
diff --git a/externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm b/src/externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm
rename to src/externals/pio2/examples/basic/perl5lib/XML/Lite/Element.pm
diff --git a/externals/pio2/examples/basic/perl5lib/XML/README b/src/externals/pio2/examples/basic/perl5lib/XML/README
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/README
rename to src/externals/pio2/examples/basic/perl5lib/XML/README
diff --git a/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3 b/src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3
rename to src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite.3
diff --git a/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3 b/src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3
similarity index 100%
rename from externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3
rename to src/externals/pio2/examples/basic/perl5lib/XML/man3/XML::Lite::Element.3
diff --git a/externals/pio2/examples/basic/test.csh b/src/externals/pio2/examples/basic/test.csh
similarity index 100%
rename from externals/pio2/examples/basic/test.csh
rename to src/externals/pio2/examples/basic/test.csh
diff --git a/externals/pio2/examples/basic/testdecomp.F90 b/src/externals/pio2/examples/basic/testdecomp.F90
similarity index 100%
rename from externals/pio2/examples/basic/testdecomp.F90
rename to src/externals/pio2/examples/basic/testdecomp.F90
diff --git a/externals/pio2/examples/basic/testdecomp.bluefire.run b/src/externals/pio2/examples/basic/testdecomp.bluefire.run
similarity index 100%
rename from externals/pio2/examples/basic/testdecomp.bluefire.run
rename to src/externals/pio2/examples/basic/testdecomp.bluefire.run
diff --git a/externals/pio2/examples/basic/testdecomp_in b/src/externals/pio2/examples/basic/testdecomp_in
similarity index 100%
rename from externals/pio2/examples/basic/testdecomp_in
rename to src/externals/pio2/examples/basic/testdecomp_in
diff --git a/externals/pio2/examples/basic/testpio.F90 b/src/externals/pio2/examples/basic/testpio.F90
similarity index 100%
rename from externals/pio2/examples/basic/testpio.F90
rename to src/externals/pio2/examples/basic/testpio.F90
diff --git a/externals/pio2/examples/basic/testpio_bench.pl b/src/externals/pio2/examples/basic/testpio_bench.pl
similarity index 100%
rename from externals/pio2/examples/basic/testpio_bench.pl
rename to src/externals/pio2/examples/basic/testpio_bench.pl
diff --git a/externals/pio2/examples/basic/testpio_build.pl b/src/externals/pio2/examples/basic/testpio_build.pl
similarity index 100%
rename from externals/pio2/examples/basic/testpio_build.pl
rename to src/externals/pio2/examples/basic/testpio_build.pl
diff --git a/externals/pio2/examples/basic/testpio_run.pl b/src/externals/pio2/examples/basic/testpio_run.pl
similarity index 100%
rename from externals/pio2/examples/basic/testpio_run.pl
rename to src/externals/pio2/examples/basic/testpio_run.pl
diff --git a/externals/pio2/examples/basic/utils_mod.F90 b/src/externals/pio2/examples/basic/utils_mod.F90
similarity index 100%
rename from externals/pio2/examples/basic/utils_mod.F90
rename to src/externals/pio2/examples/basic/utils_mod.F90
diff --git a/externals/pio2/examples/basic/wstest.c b/src/externals/pio2/examples/basic/wstest.c
similarity index 100%
rename from externals/pio2/examples/basic/wstest.c
rename to src/externals/pio2/examples/basic/wstest.c
diff --git a/externals/pio2/examples/basic/ystest.sh b/src/externals/pio2/examples/basic/ystest.sh
similarity index 100%
rename from externals/pio2/examples/basic/ystest.sh
rename to src/externals/pio2/examples/basic/ystest.sh
diff --git a/src/externals/pio2/examples/c/CMakeLists.txt b/src/externals/pio2/examples/c/CMakeLists.txt
new file mode 100644
index 00000000000..e35562b11cc
--- /dev/null
+++ b/src/externals/pio2/examples/c/CMakeLists.txt
@@ -0,0 +1,68 @@
+###-------------------------------------------------------------------------###
+### CMakeList.txt for examples using pio
+###-------------------------------------------------------------------------###
+
+# Include PIO include and lib directories
+INCLUDE_DIRECTORIES(${PIO_INCLUDE_DIRS})
+include_directories("${CMAKE_SOURCE_DIR}/examples/c")
+LINK_DIRECTORIES(${PIO_LIB_DIR})
+
+set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -O0")
+# Compiler-specific compiler options
+if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+endif()
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
+
+ADD_EXECUTABLE(examplePio EXCLUDE_FROM_ALL examplePio.c)
+TARGET_LINK_LIBRARIES(examplePio pioc)
+add_dependencies(tests examplePio)
+
+ADD_EXECUTABLE(example1 example1.c)
+TARGET_LINK_LIBRARIES(example1 pioc)
+add_dependencies(tests example1)
+
+ADD_EXECUTABLE(darray_no_async darray_no_async.c)
+TARGET_LINK_LIBRARIES(darray_no_async pioc)
+add_dependencies(tests darray_no_async)
+
+ADD_EXECUTABLE(darray_async darray_async.c)
+TARGET_LINK_LIBRARIES(darray_async pioc)
+add_dependencies(tests darray_async)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME examplePio COMMAND examplePio)
+  add_test(NAME example1 COMMAND example1)
+else ()
+  add_mpi_test(examplePio EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/examplePio NUMPROCS 4 TIMEOUT 60)
+  add_mpi_test(example1 EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/example1 NUMPROCS 4 TIMEOUT 60)
+  #add_mpi_test(darray_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_async NUMPROCS 5 TIMEOUT 60)
+  add_mpi_test(darray_no_async EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/darray_no_async NUMPROCS 4 TIMEOUT 60)
+endif ()
+
+
+#===== MPE =====
+find_package (MPE "2.4.8" COMPONENTS C)
+if (PIO_ENABLE_FORTRAN)
+  if (MPE_C_FOUND)
+    SET(SRC example2.c)
+    ADD_EXECUTABLE(example2 ${SRC})
+    TARGET_LINK_LIBRARIES(example2 pioc)
+    target_include_directories (example2
+      PUBLIC ${MPE_C_INCLUDE_DIRS})
+    target_compile_definitions (example2
+      PUBLIC HAVE_MPE)
+    target_link_libraries (example2 pioc lmpe mpe pthread m)
+  endif ()
+endif ()
diff --git a/src/externals/pio2/examples/c/darray_async.c b/src/externals/pio2/examples/c/darray_async.c
new file mode 100644
index 00000000000..1b15607f0f6
--- /dev/null
+++ b/src/externals/pio2/examples/c/darray_async.c
@@ -0,0 +1,387 @@
+/*
+ * @file
+ * @brief A simple C example for the ParallelIO Library.
+ *
+ * This example creates a netCDF output file with three dimensions
+ * (one unlimited) and one variable. It first writes, then reads the
+ * sample file using distributed arrays.
+ *
+ * This example can be run in parallel for 4 processors.
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#ifdef TIMING
+#include 
+#endif
+
+/* The name of this program. */
+#define TEST_NAME "darray_async"
+
+/* The number of possible output netCDF output flavors available to
+ * the ParallelIO library. */
+#define NUM_NETCDF_FLAVORS 4
+
+/* The number of dimensions in the example data. */
+#define NDIM3 3
+
+/* The number of timesteps of data. */
+#define NUM_TIMESTEPS 2
+
+/* The length of our sample data in X dimension.*/
+#define DIM_LEN_X 4
+
+/* The length of our sample data in Y dimension.*/
+#define DIM_LEN_Y 4
+
+/* The name of the variable in the netCDF output file. */
+#define VAR_NAME "foo"
+
+/* Return code when netCDF output file does not match
+ * expectations. */
+#define ERR_BAD 1001
+
+/* The meaning of life, the universe, and everything. */
+#define START_DATA_VAL 42
+
+/* Number of tasks this example runs on. */
+#define TARGET_NTASKS 5
+
+/* Logging level. */
+#define LOG_LEVEL 3
+
+/* Number of IO processors. */
+#define NUM_IO_TASKS 1
+
+/* Number of computation processors. */
+#define NUM_COMP_TASKS 4
+
+/* Number of computation components. */
+#define COMPONENT_COUNT 1
+        
+/* Lengths of dimensions. */
+int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
+
+/* Names of dimensions. */
+char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"};
+
+/* Handle MPI errors. This should only be used with MPI library
+ * function calls. */
+#define MPIERR(e) do {                                                  \
+	MPI_Error_string(e, err_buffer, &resultlen);			\
+	printf("MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
+	MPI_Finalize();							\
+	return 2;							\
+    } while (0)
+
+/* Handle non-MPI errors by finalizing the MPI library and exiting
+ * with an exit code. */
+#define ERR(e) do {				\
+	MPI_Finalize();				\
+	return e;				\
+    } while (0)
+
+/* Global err buffer for MPI. When there is an MPI error, this buffer
+ * is used to store the error message that is associated with the MPI
+ * error. */
+char err_buffer[MPI_MAX_ERROR_STRING];
+
+/* This is the length of the most recent MPI error message, stored
+ * int the global error string. */
+int resultlen;
+
+/* @brief Check the output file.
+ *
+ *  Use netCDF to check that the output is as expected.
+ *
+ * @param ntasks The number of processors running the example.
+ * @param filename The name of the example file to check.
+ *
+ * @return 0 if example file is correct, non-zero otherwise. */
+/* int check_file(int iosysid, int ntasks, char *filename, int iotype, */
+/*                int elements_per_pe, int my_rank, int ioid) */
+/* { */
+
+/*     int ncid;         /\* File ID from netCDF. *\/ */
+/*     int ndims;        /\* Number of dimensions. *\/ */
+/*     int nvars;        /\* Number of variables. *\/ */
+/*     int ngatts;       /\* Number of global attributes. *\/ */
+/*     int unlimdimid;   /\* ID of unlimited dimension. *\/ */
+/*     int natts;        /\* Number of variable attributes. *\/ */
+/*     nc_type xtype;    /\* NetCDF data type of this variable. *\/ */
+/*     int ret;          /\* Return code for function calls. *\/ */
+/*     int dimids[NDIM3]; /\* Dimension ids for this variable. *\/ */
+/*     char var_name[NC_MAX_NAME];   /\* Name of the variable. *\/ */
+/*     /\* size_t start[NDIM3];           /\\* Zero-based index to start read. *\\/ *\/ */
+/*     /\* size_t count[NDIM3];           /\\* Number of elements to read. *\\/ *\/ */
+/*     /\* int buffer[DIM_LEN_X];          /\\* Buffer to read in data. *\\/ *\/ */
+/*     /\* int expected[DIM_LEN_X];        /\\* Data values we expect to find. *\\/ *\/ */
+
+/*     /\* Open the file. *\/ */
+/*     if ((ret = PIOc_openfile_retry(iosysid, &ncid, &iotype, filename, 0, 0))) */
+/*         return ret; */
+/*     printf("opened file %s ncid = %d\n", filename, ncid); */
+
+/*     /\* Check the metadata. *\/ */
+/*     if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid))) */
+/*         return ret; */
+
+/*     /\* Check the dimensions. *\/ */
+/*     if (ndims != NDIM3 || nvars != 1 || ngatts != 0 || unlimdimid != 0) */
+/*         return ERR_BAD; */
+/*     for (int d = 0; d < NDIM3; d++) */
+/*     { */
+/*         char my_dim_name[NC_MAX_NAME]; */
+/*         PIO_Offset dimlen;  */
+        
+/*         if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen))) */
+/*             return ret; */
+/*         if (dimlen != (d ? dim_len[d] : NUM_TIMESTEPS) || strcmp(my_dim_name, dim_name[d])) */
+/*             return ERR_BAD; */
+/*     } */
+
+/*     /\* Check the variable. *\/ */
+/*     if ((ret = PIOc_inq_var(ncid, 0, var_name, &xtype, &ndims, dimids, &natts))) */
+/*         return ret; */
+/*     if (xtype != NC_INT || ndims != NDIM3 || dimids[0] != 0 || dimids[1] != 1 || */
+/*             dimids[2] != 2 || natts != 0) */
+/*         return ERR_BAD; */
+
+/*     /\* Allocate storage for sample data. *\/ */
+/*     int buffer[elements_per_pe]; */
+/*     int buffer_in[elements_per_pe]; */
+
+/*     /\* Check each timestep. *\/ */
+/*     for (int t = 0; t < NUM_TIMESTEPS; t++) */
+/*     { */
+/*         int varid = 0; /\* There's only one var in sample file. *\/ */
+        
+/*         /\* This is the data we expect for this timestep. *\/ */
+/*         for (int i = 0; i < elements_per_pe; i++) */
+/*             buffer[i] = 100 * t + START_DATA_VAL + my_rank; */
+
+/*         /\* Read one record. *\/ */
+/*         if ((ret = PIOc_setframe(ncid, varid, t))) */
+/*             ERR(ret); */
+/*         if ((ret = PIOc_read_darray(ncid, varid, ioid, elements_per_pe, buffer_in))) */
+/*             return ret; */
+
+/*         /\* Check the results. *\/ */
+/*         for (int i = 0; i < elements_per_pe; i++) */
+/*             if (buffer_in[i] != buffer[i]) */
+/*                 return ERR_BAD; */
+/*     } */
+
+/*     /\* Close the file. *\/ */
+/*     if ((ret = PIOc_closefile(ncid))) */
+/*         return ret; */
+
+/*     /\* Everything looks good! *\/ */
+/*     return 0; */
+/* } */
+
+/* Write, then read, a simple example with darrays.
+
+    The sample file created by this program is a small netCDF file. It
+    has the following contents (as shown by ncdump):
+
+    
    +netcdf darray_no_async_iotype_1 {
+dimensions:
+	unlimted = UNLIMITED ; // (2 currently)
+	x = 4 ;
+	y = 4 ;
+variables:
+	int foo(unlimted, x, y) ;
+data:
+
+ foo =
+  42, 42, 42, 42,
+  43, 43, 43, 43,
+  44, 44, 44, 44,
+  45, 45, 45, 45,
+  142, 142, 142, 142,
+  143, 143, 143, 143,
+  144, 144, 144, 144,
+  145, 145, 145, 145 ;
+}
+    
    
+
+*/
+    int main(int argc, char* argv[])
+    {
+	int my_rank;  /* Zero-based rank of processor. */
+	int ntasks;   /* Number of processors involved in current execution. */
+        int iosysid; /* The ID for the parallel I/O system. */
+	/* int ncid;     /\* The ncid of the netCDF file. *\/ */
+	/* int dimid[NDIM3];    /\* The dimension ID. *\/ */
+	/* int varid;    /\* The ID of the netCDF varable. *\/ */
+        /* char filename[NC_MAX_NAME + 1]; /\* Test filename. *\/ */
+        /* int num_flavors = 0;            /\* Number of iotypes available in this build. *\/ */
+	/* int format[NUM_NETCDF_FLAVORS]; /\* Different output flavors. *\/ */
+	int ret;                        /* Return value. */
+
+#ifdef TIMING
+	/* Initialize the GPTL timing library. */
+	if ((ret = GPTLinitialize ()))
+	    return ret;
+#endif
+        
+	/* Initialize MPI. */
+	if ((ret = MPI_Init(&argc, &argv)))
+	    MPIERR(ret);
+	if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
+	    MPIERR(ret);
+
+	/* Learn my rank and the total number of processors. */
+	if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
+	    MPIERR(ret);
+	if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
+	    MPIERR(ret);
+
+	/* Check that a valid number of processors was specified. */
+        printf("%d: ParallelIO Library darray_async example running on %d processors.\n",
+               my_rank, ntasks);
+	if (ntasks != TARGET_NTASKS)
+        {
+	    fprintf(stderr, "Number of processors must be %d!\n", TARGET_NTASKS);
+            return ERR_BAD;
+        }
+
+        /* Turn on logging. */
+        if ((ret = PIOc_set_log_level(LOG_LEVEL)))
+            return ret;
+
+        /* Num procs for computation. */
+        int num_procs2[COMPONENT_COUNT] = {4};
+        
+        /* Is the current process a computation task? */
+        int comp_task = my_rank < NUM_IO_TASKS ? 0 : 1;
+
+        /* Initialize the IO system. */
+        if ((ret = PIOc_init_async(MPI_COMM_WORLD, NUM_IO_TASKS, NULL, COMPONENT_COUNT,
+                                   num_procs2, NULL, NULL, NULL, PIO_REARR_BOX, &iosysid)))
+            ERR(ret);
+
+
+        /* The rest of the code executes on computation tasks only. As
+         * PIO functions are called on the computation tasks, the
+         * async system will call them on the IO task. When the
+         * computation tasks call PIO_finalize(), the IO task will get
+         * a message to shut itself down. */
+        if (comp_task)
+        {
+            /* PIO_Offset elements_per_pe; /\* Array elements per processing unit. *\/ */
+            /* int ioid;     /\* The I/O description ID. *\/ */
+            
+            /* /\* How many elements on each computation task? *\/ */
+            /* elements_per_pe = DIM_LEN_X * DIM_LEN_Y / NUM_COMP_TASKS; */
+
+            /* /\* Allocate and initialize array of decomposition mapping. *\/ */
+            /* PIO_Offset compdof[elements_per_pe]; */
+            /* for (int i = 0; i < elements_per_pe; i++) */
+            /*     compdof[i] = my_rank * elements_per_pe + i; */
+
+            /* /\* Create the PIO decomposition for this example. Since */
+            /*    this is a variable with an unlimited dimension, we want */
+            /*    to create a 2-D composition which represents one */
+            /*    record. *\/ */
+            /* printf("rank: %d Creating decomposition...\n", my_rank); */
+            /* if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe, */
+            /*                             compdof, &ioid, 0, NULL, NULL))) */
+            /*     ERR(ret); */
+
+/*         /\* The number of favors may change with the build parameters. *\/ */
+/* #ifdef _PNETCDF */
+/*         format[num_flavors++] = PIO_IOTYPE_PNETCDF; */
+/* #endif */
+/*         format[num_flavors++] = PIO_IOTYPE_NETCDF; */
+/* #ifdef _NETCDF4 */
+/*         format[num_flavors++] = PIO_IOTYPE_NETCDF4C; */
+/*         format[num_flavors++] = PIO_IOTYPE_NETCDF4P; */
+/* #endif */
+
+/* 	/\* Use PIO to create the example file in each of the four */
+/* 	 * available ways. *\/ */
+/* 	for (int fmt = 0; fmt < num_flavors; fmt++) */
+/* 	{ */
+/*             /\* Create a filename. *\/ */
+/*             sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]); */
+
+/* 	    /\* Create the netCDF output file. *\/ */
+/*             printf("rank: %d Creating sample file %s with format %d...\n", */
+/*                    my_rank, filename, format[fmt]); */
+/* 	    if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER))) */
+/* 		ERR(ret); */
+
+/* 	    /\* Define netCDF dimension and variable. *\/ */
+/*             printf("rank: %d Defining netCDF metadata...\n", my_rank); */
+/*             for (int d = 0; d < NDIM3; d++) */
+/*                 if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d]))) */
+/*                     ERR(ret); */
+/* 	    if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid))) */
+/* 	        ERR(ret); */
+/* 	    if ((ret = PIOc_enddef(ncid))) */
+/* 	        ERR(ret); */
+
+/* 	    /\* Allocate storage for sample data. *\/ */
+/*             int buffer[elements_per_pe]; */
+
+/*             /\* Write each timestep. *\/ */
+/*             for (int t = 0; t < NUM_TIMESTEPS; t++) */
+/*             { */
+/*                 /\* Create some data for this timestep. *\/ */
+/*                 for (int i = 0; i < elements_per_pe; i++) */
+/*                     buffer[i] = 100 * t + START_DATA_VAL + my_rank; */
+                
+/*                 /\* Write data to the file. *\/ */
+/*                 printf("rank: %d Writing sample data...\n", my_rank); */
+/*                 if ((ret = PIOc_setframe(ncid, varid, t))) */
+/*                     ERR(ret); */
+/*                 if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL))) */
+/*                     ERR(ret); */
+/*             } */
+
+/*             /\* THis will cause all data to be written to disk. *\/ */
+/*             if ((ret = PIOc_sync(ncid))) */
+/* 	        ERR(ret); */
+
+/* 	    /\* Close the netCDF file. *\/ */
+/*             printf("rank: %d Closing the sample data file...\n", my_rank); */
+/* 	    if ((ret = PIOc_closefile(ncid))) */
+/* 		ERR(ret); */
+
+/*             /\* Check the output file. *\/ */
+/*             /\* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, *\/ */
+/*             /\*                       my_rank, ioid))) *\/ */
+/*             /\*     ERR(ret); *\/ */
+/* 	} */
+
+            /* Free the PIO decomposition. */
+            /* printf("rank: %d Freeing PIO decomposition...\n", my_rank); */
+            /* if ((ret = PIOc_freedecomp(iosysid, ioid))) */
+            /*     ERR(ret); */
+
+            /* Finalize the IO system. Only call this from the computation tasks. */
+            printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME);
+            if ((ret = PIOc_finalize(iosysid)))
+                ERR(ret);
+        } /* endif comp_task */
+
+	/* Finalize the MPI library. */
+	MPI_Finalize();
+
+#ifdef TIMING
+	/* Finalize the GPTL timing library. */
+	if ((ret = GPTLfinalize ()))
+	    return ret;
+#endif
+
+        printf("rank: %d SUCCESS!\n", my_rank);
+	return 0;
+    }
diff --git a/src/externals/pio2/examples/c/darray_no_async.c b/src/externals/pio2/examples/c/darray_no_async.c
new file mode 100644
index 00000000000..14228ab422c
--- /dev/null
+++ b/src/externals/pio2/examples/c/darray_no_async.c
@@ -0,0 +1,358 @@
+/*
+ * @file
+ * @brief A simple C example for the ParallelIO Library.
+ *
+ * This example creates a netCDF output file with three dimensions
+ * (one unlimited) and one variable. It first writes, then reads the
+ * sample file using distributed arrays.
+ *
+ * This example can be run in parallel for 4 processors.
+ */
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#ifdef TIMING
+#include 
+#endif
+
+/* The number of possible output netCDF output flavors available to
+ * the ParallelIO library. */
+#define NUM_NETCDF_FLAVORS 4
+
+/* The number of dimensions in the example data. */
+#define NDIM3 3
+
+/* The number of timesteps of data. */
+#define NUM_TIMESTEPS 2
+
+/* The length of our sample data in X dimension.*/
+#define DIM_LEN_X 4
+
+/* The length of our sample data in Y dimension.*/
+#define DIM_LEN_Y 4
+
+/* The name of the variable in the netCDF output file. */
+#define VAR_NAME "foo"
+
+/* Return code when netCDF output file does not match
+ * expectations. */
+#define ERR_BAD 1001
+
+/* The meaning of life, the universe, and everything. */
+#define START_DATA_VAL 42
+
+/* Number of tasks this example runs on. */
+#define TARGET_NTASKS 4
+
+/* Logging level. */
+#define LOG_LEVEL 3
+
+/* Lengths of dimensions. */
+int dim_len[NDIM3] = {NC_UNLIMITED, DIM_LEN_X, DIM_LEN_Y};
+
+/* Names of dimensions. */
+char dim_name[NDIM3][PIO_MAX_NAME + 1] = {"unlimted", "x", "y"};
+
+/* Handle MPI errors. This should only be used with MPI library
+ * function calls. */
+#define MPIERR(e) do {                                                  \
+	MPI_Error_string(e, err_buffer, &resultlen);			\
+	printf("MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
+	MPI_Finalize();							\
+	return 2;							\
+    } while (0)
+
+/* Handle non-MPI errors by finalizing the MPI library and exiting
+ * with an exit code. */
+#define ERR(e) do {				\
+	MPI_Finalize();				\
+	return e;				\
+    } while (0)
+
+/* Global err buffer for MPI. When there is an MPI error, this buffer
+ * is used to store the error message that is associated with the MPI
+ * error. */
+char err_buffer[MPI_MAX_ERROR_STRING];
+
+/* This is the length of the most recent MPI error message, stored
+ * int the global error string. */
+int resultlen;
+
+/* @brief Check the output file.
+ *
+ *  Use netCDF to check that the output is as expected.
+ *
+ * @param ntasks The number of processors running the example.
+ * @param filename The name of the example file to check.
+ *
+ * @return 0 if example file is correct, non-zero otherwise. */
+int check_file(int iosysid, int ntasks, char *filename, int iotype,
+               int elements_per_pe, int my_rank, int ioid)
+{
+
+    int ncid;         /* File ID from netCDF. */
+    int ndims;        /* Number of dimensions. */
+    int nvars;        /* Number of variables. */
+    int ngatts;       /* Number of global attributes. */
+    int unlimdimid;   /* ID of unlimited dimension. */
+    int natts;        /* Number of variable attributes. */
+    nc_type xtype;    /* NetCDF data type of this variable. */
+    int ret;          /* Return code for function calls. */
+    int dimids[NDIM3]; /* Dimension ids for this variable. */
+    char var_name[NC_MAX_NAME];   /* Name of the variable. */
+    /* size_t start[NDIM3];           /\* Zero-based index to start read. *\/ */
+    /* size_t count[NDIM3];           /\* Number of elements to read. *\/ */
+    /* int buffer[DIM_LEN_X];          /\* Buffer to read in data. *\/ */
+    /* int expected[DIM_LEN_X];        /\* Data values we expect to find. *\/ */
+
+    /* Open the file. */
+    if ((ret = PIOc_openfile_retry(iosysid, &ncid, &iotype, filename, 0, 0)))
+        return ret;
+    printf("opened file %s ncid = %d\n", filename, ncid);
+
+    /* Check the metadata. */
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ret;
+
+    /* Check the dimensions. */
+    if (ndims != NDIM3 || nvars != 1 || ngatts != 0 || unlimdimid != 0)
+        return ERR_BAD;
+    for (int d = 0; d < NDIM3; d++)
+    {
+        char my_dim_name[NC_MAX_NAME];
+        PIO_Offset dimlen; 
+        
+        if ((ret = PIOc_inq_dim(ncid, d, my_dim_name, &dimlen)))
+            return ret;
+        if (dimlen != (d ? dim_len[d] : NUM_TIMESTEPS) || strcmp(my_dim_name, dim_name[d]))
+            return ERR_BAD;
+    }
+
+    /* Check the variable. */
+    if ((ret = PIOc_inq_var(ncid, 0, var_name, &xtype, &ndims, dimids, &natts)))
+        return ret;
+    if (xtype != NC_INT || ndims != NDIM3 || dimids[0] != 0 || dimids[1] != 1 ||
+            dimids[2] != 2 || natts != 0)
+        return ERR_BAD;
+
+    /* Allocate storage for sample data. */
+    int buffer[elements_per_pe];
+    int buffer_in[elements_per_pe];
+
+    /* Check each timestep. */
+    for (int t = 0; t < NUM_TIMESTEPS; t++)
+    {
+        int varid = 0; /* There's only one var in sample file. */
+        
+        /* This is the data we expect for this timestep. */
+        for (int i = 0; i < elements_per_pe; i++)
+            buffer[i] = 100 * t + START_DATA_VAL + my_rank;
+
+        /* Read one record. */
+        if ((ret = PIOc_setframe(ncid, varid, t)))
+            ERR(ret);
+        if ((ret = PIOc_read_darray(ncid, varid, ioid, elements_per_pe, buffer_in)))
+            return ret;
+
+        /* Check the results. */
+        for (int i = 0; i < elements_per_pe; i++)
+            if (buffer_in[i] != buffer[i])
+                return ERR_BAD;
+    }
+
+    /* Close the file. */
+    if ((ret = PIOc_closefile(ncid)))
+        return ret;
+
+    /* Everything looks good! */
+    return 0;
+}
+
+/* Write, then read, a simple example with darrays.
+
+    The sample file created by this program is a small netCDF file. It
+    has the following contents (as shown by ncdump):
+
+    
    +netcdf darray_no_async_iotype_1 {
+dimensions:
+	unlimted = UNLIMITED ; // (2 currently)
+	x = 4 ;
+	y = 4 ;
+variables:
+	int foo(unlimted, x, y) ;
+data:
+
+ foo =
+  42, 42, 42, 42,
+  43, 43, 43, 43,
+  44, 44, 44, 44,
+  45, 45, 45, 45,
+  142, 142, 142, 142,
+  143, 143, 143, 143,
+  144, 144, 144, 144,
+  145, 145, 145, 145 ;
+}
+    
    
+
+*/
+    int main(int argc, char* argv[])
+    {
+	int my_rank;  /* Zero-based rank of processor. */
+	int ntasks;   /* Number of processors involved in current execution. */
+	int ioproc_stride = 1;	    /* Stride in the mpi rank between io tasks. */
+	int ioproc_start = 0; 	    /* Rank of first task to be used for I/O. */
+        PIO_Offset elements_per_pe; /* Array elements per processing unit. */
+	int iosysid;  /* The ID for the parallel I/O system. */	
+	int ncid;     /* The ncid of the netCDF file. */
+	int dimid[NDIM3];    /* The dimension ID. */
+	int varid;    /* The ID of the netCDF varable. */
+	int ioid;     /* The I/O description ID. */
+        char filename[NC_MAX_NAME + 1]; /* Test filename. */
+        int num_flavors = 0;            /* Number of iotypes available in this build. */
+	int format[NUM_NETCDF_FLAVORS]; /* Different output flavors. */
+	int ret;                        /* Return value. */
+
+#ifdef TIMING
+	/* Initialize the GPTL timing library. */
+	if ((ret = GPTLinitialize ()))
+	    return ret;
+#endif
+
+	/* Initialize MPI. */
+	if ((ret = MPI_Init(&argc, &argv)))
+	    MPIERR(ret);
+	if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
+	    MPIERR(ret);
+
+	/* Learn my rank and the total number of processors. */
+	if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, &my_rank)))
+	    MPIERR(ret);
+	if ((ret = MPI_Comm_size(MPI_COMM_WORLD, &ntasks)))
+	    MPIERR(ret);
+
+	/* Check that a valid number of processors was specified. */
+	if (ntasks != TARGET_NTASKS)
+	    fprintf(stderr, "Number of processors must be 4!\n");
+        printf("%d: ParallelIO Library darray_no_async example running on %d processors.\n",
+               my_rank, ntasks);
+
+        /* Turn on logging. */
+        if ((ret = PIOc_set_log_level(LOG_LEVEL)))
+            return ret;
+        
+	/* Initialize the PIO IO system. This specifies how many and
+	 * which processors are involved in I/O. */
+	if ((ret = PIOc_Init_Intracomm(MPI_COMM_WORLD, 1, ioproc_stride,
+				       ioproc_start, PIO_REARR_BOX, &iosysid)))
+	    ERR(ret);
+
+	/* Describe the decomposition. */
+	elements_per_pe = DIM_LEN_X * DIM_LEN_Y / TARGET_NTASKS;
+
+        /* Allocate and initialize array of decomposition mapping. */
+	PIO_Offset compdof[elements_per_pe];
+	for (int i = 0; i < elements_per_pe; i++)
+	    compdof[i] = my_rank * elements_per_pe + i;
+
+	/* Create the PIO decomposition for this example. Since this
+         * is a variable with an unlimited dimension, we want to
+         * create a 2-D composition which represents one record. */
+        printf("rank: %d Creating decomposition...\n", my_rank);
+	if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3 - 1, &dim_len[1], elements_per_pe,
+				   compdof, &ioid, 0, NULL, NULL)))
+	    ERR(ret);
+
+        /* The number of favors may change with the build parameters. */
+#ifdef _PNETCDF
+        format[num_flavors++] = PIO_IOTYPE_PNETCDF;
+#endif
+        format[num_flavors++] = PIO_IOTYPE_NETCDF;
+#ifdef _NETCDF4
+        format[num_flavors++] = PIO_IOTYPE_NETCDF4C;
+        format[num_flavors++] = PIO_IOTYPE_NETCDF4P;
+#endif
+
+	/* Use PIO to create the example file in each of the four
+	 * available ways. */
+	for (int fmt = 0; fmt < num_flavors; fmt++)
+	{
+            /* Create a filename. */
+            sprintf(filename, "darray_no_async_iotype_%d.nc", format[fmt]);
+
+	    /* Create the netCDF output file. */
+            printf("rank: %d Creating sample file %s with format %d...\n",
+                   my_rank, filename, format[fmt]);
+	    if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename, PIO_CLOBBER)))
+		ERR(ret);
+
+	    /* Define netCDF dimension and variable. */
+            printf("rank: %d Defining netCDF metadata...\n", my_rank);
+            for (int d = 0; d < NDIM3; d++)
+                if ((ret = PIOc_def_dim(ncid, dim_name[d], dim_len[d], &dimid[d])))
+                    ERR(ret);
+	    if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM3, dimid, &varid)))
+	        ERR(ret);
+	    if ((ret = PIOc_enddef(ncid)))
+	        ERR(ret);
+
+	    /* Allocate storage for sample data. */
+            int buffer[elements_per_pe];
+
+            /* Write each timestep. */
+            for (int t = 0; t < NUM_TIMESTEPS; t++)
+            {
+                /* Create some data for this timestep. */
+                for (int i = 0; i < elements_per_pe; i++)
+                    buffer[i] = 100 * t + START_DATA_VAL + my_rank;
+                
+                /* Write data to the file. */
+                printf("rank: %d Writing sample data...\n", my_rank);
+                if ((ret = PIOc_setframe(ncid, varid, t)))
+                    ERR(ret);
+                if ((ret = PIOc_write_darray(ncid, varid, ioid, elements_per_pe, buffer, NULL)))
+                    ERR(ret);
+            }
+
+            /* THis will cause all data to be written to disk. */
+            if ((ret = PIOc_sync(ncid)))
+	        ERR(ret);
+
+	    /* Close the netCDF file. */
+            printf("rank: %d Closing the sample data file...\n", my_rank);
+	    if ((ret = PIOc_closefile(ncid)))
+		ERR(ret);
+
+            /* Check the output file. */
+            /* if ((ret = check_file(iosysid, ntasks, filename, format[fmt], elements_per_pe, */
+            /*                       my_rank, ioid))) */
+            /*     ERR(ret); */
+	}
+
+	/* Free the PIO decomposition. */
+        printf("rank: %d Freeing PIO decomposition...\n", my_rank);
+	if ((ret = PIOc_freedecomp(iosysid, ioid)))
+	    ERR(ret);
+
+	/* Finalize the IO system. */
+        printf("rank: %d Freeing PIO resources...\n", my_rank);
+	if ((ret = PIOc_finalize(iosysid)))
+	    ERR(ret);
+
+	/* Finalize the MPI library. */
+	MPI_Finalize();
+
+#ifdef TIMING
+	/* Finalize the GPTL timing library. */
+	if ((ret = GPTLfinalize ()))
+	    return ret;
+#endif
+
+        printf("rank: %d SUCCESS!\n", my_rank);
+	return 0;
+    }
diff --git a/externals/pio2/examples/c/example1.c b/src/externals/pio2/examples/c/example1.c
similarity index 90%
rename from externals/pio2/examples/c/example1.c
rename to src/externals/pio2/examples/c/example1.c
index 8402c826125..ad650f0b1fa 100644
--- a/externals/pio2/examples/c/example1.c
+++ b/src/externals/pio2/examples/c/example1.c
@@ -195,21 +195,8 @@ int check_file(int ntasks, char *filename) {
 	 * classic format file (but with different libraries). The
 	 * last two produce netCDF4/HDF5 format files, written with
 	 * and without using netCDF-4 parallel I/O. */
-	int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, 
-					  PIO_IOTYPE_NETCDF,
-					  PIO_IOTYPE_NETCDF4C,
-					  PIO_IOTYPE_NETCDF4P};
-
-	/** Names for the output files. Two of them (pnetcdf and
-	 * classic) will be in classic netCDF format, the others
-	 * (serial4 and parallel4) will be in netCDF-4/HDF5
-	 * format. All four can be read by the netCDF library, and all
-	 * will contain the same contents. */
-	char filename[NUM_NETCDF_FLAVORS][NC_MAX_NAME] = {"example1_pnetcdf.nc",
-							  "example1_classic.nc",
-							  "example1_serial4.nc",
-							  "example1_parallel4.nc"};
-	
+	int format[NUM_NETCDF_FLAVORS];
+
 	/** Number of processors that will do IO. In this example we
 	 * will do IO from all processors. */
 	int niotasks;
@@ -218,15 +205,9 @@ int check_file(int ntasks, char *filename) {
 	 * example. */
 	int ioproc_stride = 1;
 
-	/** Number of the aggregator? Always 0 in this example. */
-	int numAggregator = 0;
-
 	/** Zero based rank of first processor to be used for I/O. */
 	int ioproc_start = 0;
 
-	/** Specifies the flavor of netCDF output format. */
-	int iotype;
-
 	/** The dimension ID. */
 	int dimid;
 
@@ -265,12 +246,6 @@ int check_file(int ntasks, char *filename) {
 	 * as elements_per_pe.*/
 	int *buffer;
 
-	/** A buffer for reading data back from the file. The size of
-	 * this array will vary depending on how many processors are
-	 * involved in the execution of the example code. It's length
-	 * will be the same as elements_per_pe.*/
-	int *read_buffer;
-
 	/** A 1-D array which holds the decomposition mapping for this
 	 * example. The size of this array will vary depending on how
 	 * many processors are involved in the execution of the
@@ -278,6 +253,12 @@ int check_file(int ntasks, char *filename) {
 	 * elements_per_pe. */
 	PIO_Offset *compdof;
 
+        /** Test filename. */
+        char filename[NC_MAX_NAME + 1];
+
+        /** The number of netCDF flavors available in this build. */
+        int num_flavors = 0;
+            
 	/** Used for command line processing. */
 	int c;
 
@@ -304,7 +285,7 @@ int check_file(int ntasks, char *filename) {
 	/* Initialize MPI. */
 	if ((ret = MPI_Init(&argc, &argv)))
 	    MPIERR(ret);
-	if ((ret = MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
+	if ((ret = MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN)))
 	    MPIERR(ret);
 
 	/* Learn my rank and the total number of processors. */
@@ -344,16 +325,29 @@ int check_file(int ntasks, char *filename) {
 				   compdof, &ioid, NULL, NULL, NULL)))
 	    ERR(ret);
 	free(compdof);
+
+        /* The number of favors may change with the build parameters. */
+#ifdef _PNETCDF
+        format[num_flavors++] = PIO_IOTYPE_PNETCDF;
+#endif
+        format[num_flavors++] = PIO_IOTYPE_NETCDF;
+#ifdef _NETCDF4
+        format[num_flavors++] = PIO_IOTYPE_NETCDF4C;
+        format[num_flavors++] = PIO_IOTYPE_NETCDF4P;
+#endif
 	
 	/* Use PIO to create the example file in each of the four
 	 * available ways. */
-	for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) 
+	for (int fmt = 0; fmt < num_flavors; fmt++) 
 	{
+            /* Create a filename. */
+            sprintf(filename, "example1_%d.nc", fmt);
+            
 	    /* Create the netCDF output file. */
 	    if (verbose)
 		printf("rank: %d Creating sample file %s with format %d...\n",
-		       my_rank, filename[fmt], format[fmt]);
-	    if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt],
+		       my_rank, filename, format[fmt]);
+	    if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename,
 				       PIO_CLOBBER)))
 		ERR(ret);
 	
@@ -406,9 +400,12 @@ int check_file(int ntasks, char *filename) {
 
 	/* Check the output file. */
 	if (!my_rank)
-	    for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) 
-		if ((ret = check_file(ntasks, filename[fmt])))
+	    for (int fmt = 0; fmt < num_flavors; fmt++)
+            {
+                sprintf(filename, "example1_%d.nc", fmt);
+		if ((ret = check_file(ntasks, filename)))
 		    ERR(ret);
+            }
 
 	/* Finalize the MPI library. */
 	MPI_Finalize();
diff --git a/externals/pio2/examples/c/example2.c b/src/externals/pio2/examples/c/example2.c
similarity index 99%
rename from externals/pio2/examples/c/example2.c
rename to src/externals/pio2/examples/c/example2.c
index f4b75d075b7..becbcaccd29 100644
--- a/externals/pio2/examples/c/example2.c
+++ b/src/externals/pio2/examples/c/example2.c
@@ -48,8 +48,8 @@
  * responsibilty for writing and reading them will be spread between
  * all the processors used to run this example. */
 /**@{*/
-#define X_DIM_LEN 400
-#define Y_DIM_LEN 400
+#define X_DIM_LEN 20
+#define Y_DIM_LEN 30
 /**@}*/
 
 /** The number of timesteps of data to write. */
@@ -98,8 +98,7 @@ char dim_name[NDIM][NC_MAX_NAME + 1] = {"timestep", "x", "y"};
 int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
 
 /** Length of chunksizes to use in netCDF-4 files. */
-size_t chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2};
-
+PIO_Offset chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2};
 
 /** Number of MPE events. The start and stop of each event will be
  * tracked, and graphed. This value is used outside of HAVE_MPE ifdefs.*/
diff --git a/externals/pio2/examples/c/examplePio.c b/src/externals/pio2/examples/c/examplePio.c
similarity index 97%
rename from externals/pio2/examples/c/examplePio.c
rename to src/externals/pio2/examples/c/examplePio.c
index d0820a7327d..297f3923ce5 100644
--- a/externals/pio2/examples/c/examplePio.c
+++ b/src/externals/pio2/examples/c/examplePio.c
@@ -163,18 +163,15 @@ typedef struct examplePioClass
  */
 struct examplePioClass* epc_init( struct examplePioClass* this )
 {
-    int ierr;
-    int argc;
-    char *argv;
     int i, localVal;
 
     /*
     ** initialize MPI
     */
     
-    ierr = MPI_Init(NULL, NULL);
-    ierr = MPI_Comm_rank(MPI_COMM_WORLD, &this->myRank);
-    ierr = MPI_Comm_size(MPI_COMM_WORLD, &this->ntasks);
+    MPI_Init(NULL, NULL);
+    MPI_Comm_rank(MPI_COMM_WORLD, &this->myRank);
+    MPI_Comm_size(MPI_COMM_WORLD, &this->ntasks);
 
     /* Check that a valid number of processors was specified. */
     if (!(this->ntasks == 1 || this->ntasks == 2 || this->ntasks == 4 ||
@@ -235,10 +232,11 @@ struct examplePioClass* epc_init( struct examplePioClass* this )
         localVal++;
     }
 
-    if (this->verbose) {
-	printf("rank: %d length: %d [", this->myRank, this->arrIdxPerPe);
+    if (this->verbose)
+    {
+	printf("rank: %d length: %lld [", this->myRank, this->arrIdxPerPe);
 	for (i = 0; i < this->arrIdxPerPe; i++ ) {
-	    printf("%d", this->compdof[i]);
+	    printf("%lld", this->compdof[i]);
 	    if (i < this->arrIdxPerPe - 1)
 		printf(", ");
 	}
@@ -451,17 +449,15 @@ struct examplePioClass* epc_closeFile( struct examplePioClass* this )
  */
 struct examplePioClass* epc_cleanUp( struct examplePioClass* this )
 {
-    int ierr;
-    
     if (this->verbose)
 	printf("rank: %d Freeing local and library resources...\n", this->myRank);
     free(this->dataBuffer);
     free(this->readBuffer);
     free(this->compdof);
     
-    ierr = PIOc_freedecomp(this->pioIoSystem, this->iodescNCells);
-    ierr = PIOc_finalize(this->pioIoSystem);
-    ierr = MPI_Finalize();
+    PIOc_freedecomp(this->pioIoSystem, this->iodescNCells);
+    PIOc_finalize(this->pioIoSystem);
+    MPI_Finalize();
     
     return this;
 }
diff --git a/src/externals/pio2/examples/c/valsupp_example1.supp b/src/externals/pio2/examples/c/valsupp_example1.supp
new file mode 100644
index 00000000000..63f3e073836
--- /dev/null
+++ b/src/externals/pio2/examples/c/valsupp_example1.supp
@@ -0,0 +1,15 @@
+{
+   cond_jump_1
+   Memcheck:Cond
+   fun:MPIC_Waitall
+   fun:MPIR_Alltoallw_intra
+   fun:MPIR_Alltoallw
+   fun:MPIR_Alltoallw_impl
+   fun:PMPI_Alltoallw
+   fun:pio_swapm
+   fun:rearrange_comp2io
+   fun:PIOc_write_darray_multi
+   fun:flush_buffer
+   fun:PIOc_sync
+   fun:main
+}
\ No newline at end of file
diff --git a/externals/pio2/examples/cxx/CMakeLists.txt b/src/externals/pio2/examples/cxx/CMakeLists.txt
similarity index 100%
rename from externals/pio2/examples/cxx/CMakeLists.txt
rename to src/externals/pio2/examples/cxx/CMakeLists.txt
diff --git a/externals/pio2/examples/cxx/examplePio.cxx b/src/externals/pio2/examples/cxx/examplePio.cxx
similarity index 100%
rename from externals/pio2/examples/cxx/examplePio.cxx
rename to src/externals/pio2/examples/cxx/examplePio.cxx
diff --git a/externals/pio2/examples/f03/CMakeLists.txt b/src/externals/pio2/examples/f03/CMakeLists.txt
similarity index 96%
rename from externals/pio2/examples/f03/CMakeLists.txt
rename to src/externals/pio2/examples/f03/CMakeLists.txt
index e362ceacd13..a299c7cde8c 100644
--- a/externals/pio2/examples/f03/CMakeLists.txt
+++ b/src/externals/pio2/examples/f03/CMakeLists.txt
@@ -12,10 +12,9 @@ LINK_DIRECTORIES(${PIO_LIB_DIR})
 set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -O0")
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
-if(${PIO_BUILD_TIMING}) 
+if(${PIO_BUILD_TIMING})
   SET(TIMING_LINK_LIB timing)
 endif()
 SET(SRC examplePio.f90)
 ADD_EXECUTABLE(examplePio_f90 ${SRC})
 TARGET_LINK_LIBRARIES(examplePio_f90 piof pioc ${TIMING_LINK_LIB})
-
diff --git a/externals/pio2/examples/f03/examplePio.f90 b/src/externals/pio2/examples/f03/examplePio.f90
similarity index 99%
rename from externals/pio2/examples/f03/examplePio.f90
rename to src/externals/pio2/examples/f03/examplePio.f90
index f5eede9e26a..d2baddf2096 100644
--- a/externals/pio2/examples/f03/examplePio.f90
+++ b/src/externals/pio2/examples/f03/examplePio.f90
@@ -10,7 +10,7 @@ module pioExample
     use pio, only : PIO_nowrite, PIO_openfile
 
     implicit none
-    save
+
     private
 
     include 'mpif.h'
diff --git a/externals/pio2/scripts/prune_decomps.pl b/src/externals/pio2/scripts/prune_decomps.pl
similarity index 100%
rename from externals/pio2/scripts/prune_decomps.pl
rename to src/externals/pio2/scripts/prune_decomps.pl
diff --git a/externals/pio2/src/CMakeLists.txt b/src/externals/pio2/src/CMakeLists.txt
similarity index 72%
rename from externals/pio2/src/CMakeLists.txt
rename to src/externals/pio2/src/CMakeLists.txt
index ba6863d8509..5c5c079fa1e 100644
--- a/externals/pio2/src/CMakeLists.txt
+++ b/src/externals/pio2/src/CMakeLists.txt
@@ -2,11 +2,11 @@
 #  PRELIMINARIES
 #==============================================================================
 
-# Test for big-endian nature 
-if (PIO_TEST_BIG_ENDIAN) 
+# Test for big-endian nature
+if (PIO_TEST_BIG_ENDIAN)
   include (TestBigEndian)
   test_big_endian (PIO_BIG_ENDIAN_TEST_RESULT)
-  if (PIO_BIG_ENDIAN_TEST_RESULT) 
+  if (PIO_BIG_ENDIAN_TEST_RESULT)
     set (PIO_BIG_ENDIAN ON  CACHE BOOL "Whether machine is big endian")
   else ()
     set (PIO_BIG_ENDIAN OFF CACHE BOOL "Whether machine is big endian")
@@ -17,8 +17,8 @@ endif ()
 
 # Look for GPTL
 if (PIO_ENABLE_TIMING)
-    find_package (GPTL COMPONENTS C QUIET)
-    find_package (GPTL COMPONENTS Fortran_Perf QUIET)
+  find_package (GPTL COMPONENTS C QUIET)
+  find_package (GPTL COMPONENTS Fortran_Perf QUIET)
 endif ()
 
 #==============================================================================
@@ -27,13 +27,15 @@ endif ()
 
 # Build GPTL only if necessary
 if (PIO_ENABLE_TIMING)
-    if (NOT GPTL_C_FOUND OR NOT GPTL_Fortran_Perf_FOUND)
-        add_subdirectory (gptl)
-    endif ()
+  if (NOT GPTL_C_FOUND OR NOT GPTL_Fortran_Perf_FOUND)
+    add_subdirectory (gptl)
+  endif ()
 endif ()
 
 # Build the C library
 add_subdirectory (clib)
 
 # Build the Fortran library
-add_subdirectory (flib)
+if (PIO_ENABLE_FORTRAN)
+  add_subdirectory (flib)
+endif ()
diff --git a/src/externals/pio2/src/clib/CMakeLists.txt b/src/externals/pio2/src/clib/CMakeLists.txt
new file mode 100644
index 00000000000..7fac8891a8e
--- /dev/null
+++ b/src/externals/pio2/src/clib/CMakeLists.txt
@@ -0,0 +1,162 @@
+cmake_minimum_required (VERSION 2.8.12)
+include (CheckFunctionExists)
+project (PIOC C)
+
+#==============================================================================
+#  DEFINE THE TARGET
+#==============================================================================
+
+add_library (pioc topology.c pio_file.c pioc_support.c pio_lists.c
+  pioc.c pioc_sc.c pio_spmd.c pio_rearrange.c pio_nc4.c bget.c
+  pio_nc.c pio_put_nc.c pio_get_nc.c pio_getput_int.c pio_msg.c pio_varm.c
+  pio_darray.c pio_darray_int.c)
+
+# set up include-directories
+include_directories(
+  "${PROJECT_SOURCE_DIR}"   # to find foo/foo.h
+  "${PROJECT_BINARY_DIR}")  # to find foo/config.h
+
+# Include the clib source directory
+target_include_directories (pioc
+  PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+
+# System and compiler CPP directives
+target_compile_definitions (pioc
+  PUBLIC ${CMAKE_SYSTEM_DIRECTIVE})
+target_compile_definitions (pioc
+  PUBLIC ${CMAKE_C_COMPILER_DIRECTIVE})
+
+# Compiler-specific compiler options
+if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+  target_compile_options (pioc
+    PRIVATE -std=c99)
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI")
+  target_compile_options (pioc
+    PRIVATE -c99)
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
+  target_compile_options (pioc
+    PRIVATE -std=c99)
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
+  target_compile_options (pioc
+    PRIVATE -std=c99)
+endif()
+
+#==============================================================================
+#  DEFINE THE INSTALL
+#==============================================================================
+
+# Library
+install (TARGETS pioc DESTINATION lib)
+
+# Include/Header File
+install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/pio.h DESTINATION include)
+
+#==============================================================================
+#  DEFINE THE DEPENDENCIES
+#==============================================================================
+# MPI test done at top level
+if (MPISERIAL_C_FOUND)
+    target_compile_definitions (pioc
+      PRIVATE MPI_SERIAL)
+    target_include_directories (pioc
+      PUBLIC ${MPISERIAL_C_INCLUDE_DIRS})
+    target_link_libraries (pioc
+      PUBLIC ${MPISERIAL_C_LIBRARIES})
+    set (WITH_PNETCDF FALSE)
+endif ()
+
+
+
+#===== GPTL =====
+if (PIO_ENABLE_TIMING)
+  find_package (GPTL COMPONENTS C QUIET)
+  if (GPTL_C_FOUND)
+    message (STATUS "Found GPTL C: ${GPTL_C_LIBRARIES}")
+    target_include_directories (pioc
+      PUBLIC ${GPTL_C_INCLUDE_DIRS})
+    target_link_libraries (pioc
+      PUBLIC ${GPTL_C_LIBRARIES})
+  else ()
+    message (STATUS "Using internal GPTL C library for timing")
+    target_link_libraries (pioc
+      PUBLIC gptl)
+  endif ()
+  target_compile_definitions (pioc
+    PUBLIC TIMING)
+endif ()
+
+#===== NetCDF-C =====
+find_package (NetCDF "4.3.3" COMPONENTS C)
+if (NetCDF_C_FOUND)
+  target_include_directories (pioc
+    PUBLIC ${NetCDF_C_INCLUDE_DIRS})
+  target_compile_definitions (pioc
+    PUBLIC _NETCDF)
+  target_link_libraries (pioc
+    PUBLIC ${NetCDF_C_LIBRARIES})
+  if (${NetCDF_C_HAS_PARALLEL})
+    target_compile_definitions (pioc
+      PUBLIC _NETCDF4)
+  endif ()
+  if (${NetCDF_C_LOGGING_ENABLED})
+    target_compile_definitions (pioc
+      PUBLIC NETCDF_C_LOGGING_ENABLED)
+    # netcdf.h needs this to be defined to use netCDF logging.
+    target_compile_definitions (pioc
+      PUBLIC LOGGING)
+  endif()
+else ()
+  target_compile_definitions (pioc
+    PUBLIC _NONETCDF)
+endif ()
+
+#===== PnetCDF-C =====
+if (WITH_PNETCDF)
+  find_package (PnetCDF "1.7.0" COMPONENTS C)
+endif ()
+if (PnetCDF_C_FOUND)
+  target_include_directories (pioc
+    PUBLIC ${PnetCDF_C_INCLUDE_DIRS})
+  target_compile_definitions (pioc
+    PUBLIC _PNETCDF)
+  target_link_libraries (pioc
+    PUBLIC ${PnetCDF_C_LIBRARIES})
+
+  # Check library for varn functions
+  set (CMAKE_REQUIRED_LIBRARIES ${PnetCDF_C_LIBRARY})
+  check_function_exists (ncmpi_get_varn PnetCDF_C_HAS_VARN)
+  if (PnetCDF_C_HAS_VARN)
+    target_compile_definitions(pioc
+      PUBLIC USE_PNETCDF_VARN
+      PUBLIC USE_PNETCDF_VARN_ON_READ)
+  endif()
+else ()
+  target_compile_definitions (pioc
+    PUBLIC _NOPNETCDF)
+endif ()
+
+#===== Add EXTRAs =====
+target_include_directories (pioc
+  PUBLIC ${PIO_C_EXTRA_INCLUDE_DIRS})
+target_link_libraries (pioc
+  PUBLIC ${PIO_C_EXTRA_LIBRARIES})
+target_compile_options (pioc
+  PRIVATE ${PIO_C_EXTRA_COMPILE_OPTIONS})
+target_compile_definitions (pioc
+  PUBLIC ${PIO_C_EXTRA_COMPILE_DEFINITIONS})
+if (PIO_C_EXTRA_LINK_FLAGS)
+  set_target_properties(pioc PROPERTIES
+    LINK_FLAGS ${PIO_C_EXTRA_LINK_FLAGS})
+endif ()
+
+#===== Check for necessities =====
+if (NOT PnetCDF_C_FOUND AND NOT NetCDF_C_FOUND)
+  message (FATAL_ERROR "Must have PnetCDF and/or NetCDF C libraries")
+endif ()
+
+include(CheckTypeSize)
+check_type_size("size_t" SIZEOF_SIZE_T)
+CHECK_TYPE_SIZE("long long" SIZEOF_LONG_LONG)
+if (NOT ${SIZEOF_SIZE_T} EQUAL ${SIZEOF_LONG_LONG})
+  message (FATAL_ERROR "size_t and long long must be the same size!")
+endif ()
diff --git a/src/externals/pio2/src/clib/bget.c b/src/externals/pio2/src/clib/bget.c
new file mode 100644
index 00000000000..98bfc36893d
--- /dev/null
+++ b/src/externals/pio2/src/clib/bget.c
@@ -0,0 +1,1677 @@
+/*
+
+  B G E T
+
+  Buffer allocator
+
+  Designed and implemented in April of 1972 by John Walker, based on the
+  Case Algol OPRO$ algorithm implemented in 1966.
+
+  Reimplemented in 1975 by John Walker for the Interdata 70.
+  Reimplemented in 1977 by John Walker for the Marinchip 9900.
+  Reimplemented in 1982 by Duff Kurland for the Intel 8080.
+
+  Portable C version implemented in September of 1990 by an older, wiser
+  instance of the original implementor.
+
+  Souped up and/or weighed down  slightly  shortly  thereafter  by  Greg
+  Lutz.
+
+  AMIX  edition, including the new compaction call-back option, prepared
+  by John Walker in July of 1992.
+
+  Bug in built-in test program fixed, ANSI compiler warnings eradicated,
+  buffer pool validator  implemented,  and  guaranteed  repeatable  test
+  added by John Walker in October of 1995.
+
+  This program is in the public domain.
+
+  1. This is the book of the generations of Adam.   In the day that God
+  created man, in the likeness of God made he him;
+  2. Male and female created he them;  and  blessed     them,  and  called
+  their name Adam, in the day when they were created.
+  3. And  Adam  lived  an hundred and thirty years,     and begat a son in
+  his own likeness, and after his image; and called his name Seth:
+  4. And the days of  Adam  after  he  had  begotten  Seth  were  eight
+  hundred years: and he begat sons and daughters:
+  5. And  all  the  days  that Adam lived were nine     hundred and thirty
+  years: and he died.
+  6. And Seth lived an hundred and five years, and begat Enos:
+  7. And Seth lived after he begat Enos eight hundred and seven  years,
+  and begat sons and daughters:
+  8.  And  all the days of Seth were nine hundred and twelve years: and
+  he died.
+  9. And Enos lived ninety years, and begat Cainan:
+  10. And Enos lived after he begat  Cainan eight  hundred  and  fifteen
+  years, and begat sons and daughters:
+  11. And  all  the days of Enos were nine hundred  and five years:  and
+  he died.
+  12. And Cainan lived seventy years and begat Mahalaleel:
+  13. And Cainan lived  after he  begat  Mahalaleel  eight  hundred  and
+  forty years, and begat sons and daughters:
+  14. And  all the days of Cainan were nine  hundred and ten years:  and
+  he died.
+  15. And Mahalaleel lived sixty and five years, and begat Jared:
+  16. And Mahalaleel lived  after  he  begat    Jared  eight  hundred  and
+  thirty years, and begat sons and daughters:
+  17. And  all  the  days  of Mahalaleel  were eight hundred    ninety and
+  five years: and he died.
+  18. And Jared lived an hundred sixty and  two  years,   and  he  begat
+  Enoch:
+  19. And  Jared  lived  after he begat Enoch  eight hundred years,  and
+  begat sons and daughters:
+  20. And all the days of Jared  were nine hundred sixty and two  years:
+  and he died.
+  21. And Enoch lived sixty and five years, and begat Methuselah:
+  22. And  Enoch  walked   with  God    after  he  begat Methuselah  three
+  hundred years, and begat sons and daughters:
+  23. And all the days of  Enoch  were  three  hundred  sixty  and  five
+  years:
+  24. And Enoch walked with God: and he was not; for God took him.
+  25. And  Methuselah  lived    an  hundred  eighty and  seven years,  and
+  begat Lamech.
+  26. And Methuselah lived after he  begat Lamech seven  hundred  eighty
+  and two years, and begat sons and daughters:
+  27. And  all the days of Methuselah  were nine hundred  sixty and nine
+  years: and he died.
+  28. And Lamech lived an hundred eighty  and two  years,  and  begat  a
+  son:
+  29. And  he called his name Noah, saying,  This same shall    comfort us
+  concerning  our  work and toil of our hands, because of the ground
+  which the LORD hath cursed.
+  30. And  Lamech  lived  after  he begat Noah  five hundred    ninety and
+  five years, and begat sons and daughters:
+  31. And all the days of Lamech were  seven hundred seventy    and  seven
+  years: and he died.
+  32. And  Noah  was five hundred years old:    and Noah begat Shem,  Ham,
+  and Japheth.
+
+  And buffers begat buffers, and links begat    links,  and  buffer  pools
+  begat  links  to chains of buffer pools containing buffers, and lo the
+  buffers and links and pools of buffers and pools of links to chains of
+  pools  of  buffers were fruitful and they multiplied and the Operating
+  System looked down upon them and said that it was Good.
+
+
+  INTRODUCTION
+  ============
+
+  BGET  is a comprehensive memory allocation package which is easily
+  configured to the needs of an application.    BGET is  efficient  in
+  both  the  time  needed to allocate and release buffers and in the
+  memory  overhead  required    for  buffer   pool   management.    It
+  automatically    consolidates   contiguous     space   to   minimise
+  fragmentation.  BGET is configured    by  compile-time  definitions,
+  Major options include:
+
+  *   A  built-in  test  program        to  exercise  BGET   and
+  demonstrate how the various functions are used.
+
+  *   Allocation  by  either the "first fit" or "best fit"
+  method.
+
+  *   Wiping buffers at release time to catch  code  which
+  references previously released storage.
+
+  *   Built-in  routines to dump individual buffers or the
+  entire buffer pool.
+
+  *   Retrieval of allocation and pool size statistics.
+
+  *   Quantisation of buffer sizes to a power  of  two  to
+  satisfy hardware alignment constraints.
+
+  *   Automatic  pool compaction, growth, and shrinkage by
+  means of call-backs to user defined functions.
+
+  Applications  of  BGET  can  range    from  storage  management   in
+  ROM-based  embedded programs to providing the framework upon which
+  a  multitasking  system  incorporating   garbage   collection   is
+  constructed.   BGET  incorporates  extensive  internal consistency
+  checking using the  mechanism; all these checks  can  be
+  turned off by compiling with NDEBUG defined, yielding a version of
+  BGET with minimal size and maximum speed.
+
+  The  basic    algorithm  underlying  BGET  has withstood the test of
+  time;  more  than  25  years   have   passed   since   the     first
+  implementation  of    this  code.  And yet, it is substantially more
+  efficient than the native allocation  schemes  of  many  operating
+  systems: the Macintosh and Microsoft Windows to name two, on which
+  programs have obtained substantial speed-ups by layering  BGET  as
+  an application level memory manager atop the underlying system's.
+
+  BGET has been implemented on the largest mainframes and the lowest
+  of    microprocessors.   It  has served as the core for multitasking
+  operating systems, multi-thread applications, embedded software in
+  data  network switching processors, and a host of C programs.  And
+  while it has accreted flexibility and additional options over  the
+  years,  it    remains  fast, memory efficient, portable, and easy to
+  integrate into your program.
+
+
+  BGET IMPLEMENTATION ASSUMPTIONS
+  ===============================
+
+  BGET is written in as portable a dialect of C  as  possible.   The
+  only   fundamental     assumption   about  the  underlying  hardware
+  architecture is that memory is allocated is a linear  array  which
+  can  be  addressed  as a vector of C "char" objects.  On segmented
+  address space architectures, this generally means that BGET should
+  be used to allocate storage within a single segment (although some
+  compilers     simulate   linear   address   spaces   on    segmented
+  architectures).   On  segmented  architectures,  then, BGET buffer
+  pools  may not be larger than a segment, but since BGET allows any
+  number of separate buffer pools, there is no limit    on  the  total
+  storage  which  can  be  managed,  only  on the largest individual
+  object which can be allocated.  Machines  with  a  linear  address
+  architecture,  such  as  the VAX, 680x0, Sparc, MIPS, or the Intel
+  80386 and above in native mode, may use BGET without restriction.
+
+
+  GETTING STARTED WITH BGET
+  =========================
+
+  Although BGET can be configured in a multitude of fashions,  there
+  are  three    basic  ways  of  working  with  BGET.   The  functions
+  mentioned below are documented in the following  section.     Please
+  excuse  the  forward  references which are made in the interest of
+  providing a roadmap to guide you  to  the  BGET  functions  you're
+  likely to need.
+
+  Embedded Applications
+  ---------------------
+
+  Embedded applications  typically  have  a  fixed  area  of    memory
+  dedicated  to  buffer  allocation (often in a separate RAM address
+  space distinct from the ROM that contains  the  executable    code).
+  To    use  BGET in such an environment, simply call bpool() with the
+  start address and length of the buffer  pool  area    in  RAM,  then
+  allocate  buffers  with  bget()  and  release  them  with  brel().
+  Embedded applications with very limited RAM but abundant CPU speed
+  may  benefit  by configuring BGET for BestFit allocation (which is
+  usually not worth it in other environments).
+
+  Malloc() Emulation
+  ------------------
+
+  If the C library malloc() function is too  slow,  not  present  in
+  your  development environment (for example, an a native Windows or
+  Macintosh program), or otherwise unsuitable, you  can  replace  it
+  with  BGET.  Initially define a buffer pool of an appropriate size
+  with bpool()--usually obtained by making a call to    the  operating
+  system's  low-level  memory allocator.  Then allocate buffers with
+  bget(), bgetz(), and bgetr() (the last two permit  the  allocation
+  of    buffers initialised to zero and [inefficient] re-allocation of
+  existing buffers for  compatibility  with  C  library  functions).
+  Release buffers by calling brel().    If a buffer allocation request
+  fails, obtain more storage from the underlying  operating  system,
+  add it to the buffer pool by another call to bpool(), and continue
+  execution.
+
+  Automatic Storage Management
+  ----------------------------
+
+  You can use BGET as your application's native memory  manager  and
+  implement  automatic  storage  pool  expansion,  contraction,  and
+  optionally application-specific  memory  compaction  by  compiling
+  BGET  with    the  BECtl  variable defined, then calling bectl() and
+  supplying  functions  for  storage    compaction,  acquisition,  and
+  release,  as  well as a standard pool expansion increment.    All of
+  these functions are optional (although it doesn't make much  sense
+  to    provide  a  release  function without an acquisition function,
+  does it?).    Once the call-back functions have  been  defined  with
+  bectl(),  you simply use bget() and brel() to allocate and release
+  storage as before.    You can supply an  initial  buffer  pool  with
+  bpool()  or  rely  on  automatic  allocation to acquire the entire
+  pool.  When a call on  bget()  cannot  be  satisfied,  BGET  first
+  checks  if    a compaction function has been supplied.  If so, it is
+  called (with the space required to satisfy the allocation  request
+  and a sequence number to allow the compaction routine to be called
+  successively without looping).  If the compaction function is able
+  to  free any storage (it needn't know whether the storage it freed
+  was adequate) it should return a  nonzero  value,  whereupon  BGET
+  will retry the allocation request and, if it fails again, call the
+  compaction function again with the next-higher sequence number.
+
+  If    the  compaction  function  returns zero, indicating failure to
+  free space, or no compaction function is defined, BGET next  tests
+  whether  a    non-NULL  allocation function was supplied to bectl().
+  If so, that function is called with  an  argument  indicating  how
+  many  bytes  of  additional  space are required.  This will be the
+  standard pool expansion increment supplied in the call to  bectl()
+  unless  the  original  bget()  call requested a buffer larger than
+  this; buffers larger than the standard pool block can  be  managed
+  "off  the books" by BGET in this mode.  If the allocation function
+  succeeds in obtaining the storage, it returns a pointer to the new
+  block  and    BGET  expands  the  buffer  pool;  if  it  fails,  the
+  allocation request fails and returns NULL to  the  caller.     If  a
+  non-NULL  release  function  is  supplied,    expansion blocks which
+  become totally empty are released  to  the    global  free  pool  by
+  passing their addresses to the release function.
+
+  Equipped  with  appropriate  allocation,  release,    and compaction
+  functions, BGET can be used as part of very  sophisticated    memory
+  management     strategies,  including  garbage  collection.   (Note,
+  however, that BGET is *not* a garbage  collector  by  itself,  and
+  that  developing  such a system requires much additional logic and
+  careful design of the application's memory allocation strategy.)
+
+
+  BGET FUNCTION DESCRIPTIONS
+  ==========================
+
+  Functions implemented in this file (some are enabled by certain of
+  the optional settings below):
+
+  void bpool(void *buffer, bufsize len);
+
+  Create a buffer pool of  bytes, using the storage starting at
+  .     You  can  call  bpool()  subsequently  to   contribute
+  additional storage to the overall buffer pool.
+
+  void *bget(bufsize size);
+
+  Allocate  a  buffer of  bytes.  The address of the buffer is
+  returned, or NULL if insufficient memory was available to allocate
+  the buffer.
+
+  void *bgetz(bufsize size);
+
+  Allocate a buffer of  bytes and clear it to all zeroes.  The
+  address of the buffer is returned, or NULL if insufficient    memory
+  was available to allocate the buffer.
+
+  void *bgetr(void *buffer, bufsize newsize);
+
+  Reallocate a buffer previously allocated by bget(),  changing  its
+  size  to    and  preserving  all  existing data.  NULL is
+  returned if insufficient memory is    available  to  reallocate  the
+  buffer, in which case the original buffer remains intact.
+
+  void brel(void *buf);
+
+  Return  the  buffer  , previously allocated by bget(), to the
+  free space pool.
+
+  void bectl(int (*compact)(bufsize sizereq, int sequence),
+  void *(*acquire)(bufsize size),
+  void (*release)(void *buf),
+  bufsize pool_incr);
+
+  Expansion control: specify functions through which the package may
+  compact  storage  (or  take  other    appropriate  action)  when  an
+  allocation    request  fails,  and  optionally automatically acquire
+  storage for expansion blocks  when    necessary,  and  release  such
+  blocks when they become empty.  If  is non-NULL, whenever
+  a buffer allocation request fails, the  function will  be
+  called with arguments specifying the number of bytes (total buffer
+  size,  including  header  overhead)  required   to     satisfy   the
+  allocation request, and a sequence number indicating the number of
+  consecutive  calls    on    attempting  to   satisfy   this
+  allocation    request.   The sequence number is 1 for the first call
+  on  for a given allocation  request,  and    increments  on
+  subsequent    calls,  permitting  the    function  to  take
+  increasingly dire measures in an attempt to free up  storage.   If
+  the    function  returns  a nonzero value, the allocation
+  attempt is re-tried.  If  returns 0 (as  it  must    if  it
+  isn't  able  to  release  any  space  or add storage to the buffer
+  pool), the allocation request fails, which can  trigger  automatic
+  pool expansion if the  argument is non-NULL.  At the time
+  the    function  is  called,  the    state  of  the  buffer
+  allocator  is  identical  to  that    at  the  moment the allocation
+  request was made; consequently, the   function  may  call
+  brel(), bpool(), bstats(), and/or directly manipulate  the    buffer
+  pool  in  any  manner which would be valid were the application in
+  control.  This does not, however, relieve the    function
+  of the need to ensure that whatever actions it takes do not change
+  things   underneath  the  application  that  made  the  allocation
+  request.  For example, a  function that released a buffer
+  in    the  process  of  being reallocated with bgetr() would lead to
+  disaster.  Implementing a safe and effective    mechanism
+  requires  careful  design of an application's memory architecture,
+  and cannot generally be easily retrofitted into existing code.
+
+  If  is non-NULL, that function will be called whenever an
+  allocation    request  fails.  If the  function succeeds in
+  allocating the requested space and returns a pointer  to  the  new
+  area,  allocation will proceed using the expanded buffer pool.  If
+   cannot obtain the requested space, it should return NULL
+  and   the     entire  allocation  process  will  fail.   
+  specifies the normal expansion block size.    Providing an 
+  function will cause subsequent bget()  requests  for  buffers  too
+  large  to  be  managed in the linked-block scheme (in other words,
+  larger than  minus the buffer overhead) to be satisfied
+  directly by calls to the  function.  Automatic release of
+  empty pool blocks will occur only if all pool blocks in the system
+  are the size given by .
+
+  void bstats(bufsize *curalloc, bufsize *totfree,
+  bufsize *maxfree, long *nget, long *nrel);
+
+  The amount    of  space  currently  allocated  is  stored  into  the
+  variable  pointed  to by .  The total free space (sum of
+  all free blocks in the pool) is stored into the  variable  pointed
+  to    by  , and the size of the largest single block in the
+  free space    pool  is  stored  into  the  variable  pointed  to  by
+  .     The  variables  pointed  to  by  and  are
+  filled, respectively, with    the  number  of  successful  (non-NULL
+  return) bget() calls and the number of brel() calls.
+
+  void bstatse(bufsize *pool_incr, long *npool,
+  long *npget, long *nprel,
+  long *ndget, long *ndrel);
+
+  Extended  statistics: The expansion block size will be stored into
+  the variable pointed to by , or the negative thereof if
+  automatic  expansion  block  releases are disabled.  The number of
+  currently active pool blocks will  be  stored  into  the  variable
+  pointed  to  by  .  The variables pointed to by  and
+   will be filled with, respectively, the number of expansion
+  block   acquisitions   and    releases  which  have  occurred.   The
+  variables pointed to by  and  will  be  filled  with
+  the  number  of  bget()  and  brel()  calls, respectively, managed
+  through blocks directly allocated by the acquisition  and  release
+  functions.
+
+  void bufdump(void *buf);
+
+  The buffer pointed to by  is dumped on standard output.
+
+  void bpoold(void *pool, int dumpalloc, int dumpfree);
+
+  All buffers in the buffer pool , previously initialised by a
+  call on bpool(), are listed in ascending memory address order.  If
+   is nonzero, the  contents  of  allocated  buffers  are
+  dumped;  if  is nonzero, the contents of free blocks are
+  dumped.
+
+  int bpoolv(void *pool);
+
+  The  named    buffer  pool,  previously  initialised  by  a  call on
+  bpool(), is validated for bad pointers, overwritten data, etc.  If
+  compiled with NDEBUG not defined, any error generates an assertion
+  failure.  Otherwise 1 is returned if the pool is valid,  0    if  an
+  error is found.
+
+
+  BGET CONFIGURATION
+  ==================
+*/
+#include 
+#include 
+#include 
+#if PIO_USE_MALLOC
+#include 
+#endif
+
+#define TestProg    20000             /* Generate built-in test program
+                                         if defined.  The value specifies
+                                         how many buffer allocation attempts
+                                         the test program should make. */
+#undef TestProg
+
+#define SizeQuant   4                 /* Buffer allocation size quantum:
+                                         all buffers allocated are a
+                                         multiple of this size.  This
+                                         MUST be a power of two. */
+
+#define BufDump     1                 /* Define this symbol to enable the
+                                         bpoold() function which dumps the
+                                         buffers in a buffer pool. */
+
+#define BufValid    1                 /* Define this symbol to enable the
+                                         bpoolv() function for validating
+                                         a buffer pool. */
+
+#define DumpData    1                 /* Define this symbol to enable the
+                                         bufdump() function which allows
+                                         dumping the contents of an allocated
+                                         or free buffer. */
+
+#define BufStats    1                 /* Define this symbol to enable the
+                                         bstats() function which calculates
+                                         the total free space in the buffer
+                                         pool, the largest available
+                                         buffer, and the total space
+                                         currently allocated. */
+
+#define FreeWipe    1                 /* Wipe free buffers to a guaranteed
+                                         pattern of garbage to trip up
+                                         miscreants who attempt to use
+                                         pointers into released buffers. */
+
+//#define BestFit     1
+#undef BestFit
+/* Use a best fit algorithm when
+   searching for space for an
+   allocation request.  This uses
+   memory more efficiently, but
+   allocation will be much slower. */
+
+#define BECtl       1                 /* Define this symbol to enable the
+                                         bectl() function for automatic
+                                         pool space control.  */
+
+#include 
+
+#ifdef lint
+#define NDEBUG                        /* Exits in asserts confuse lint */
+/* LINTLIBRARY */                     /* Don't complain about def, no ref */
+extern char *sprintf();               /* Sun includes don't define sprintf */
+#endif
+
+#include 
+#include 
+
+#ifdef BufDump                        /* BufDump implies DumpData */
+#ifndef DumpData
+#define DumpData    1
+#endif
+#endif
+
+#ifdef DumpData
+#include 
+#endif
+
+/*  Declare the interface, including the requested buffer size type,
+    bufsize.  */
+
+#include "bget.h"
+
+#define MemSize     size_t            /* Type for size arguments to memxxx()
+                                         functions such as memcmp(). */
+
+/* Queue links */
+
+struct qlinks {
+    struct bfhead *flink;             /* Forward link */
+    struct bfhead *blink;             /* Backward link */
+};
+
+/* Header in allocated and free buffers */
+
+struct bhead {
+    bufsize prevfree;                 /* Relative link back to previous
+                                         free buffer in memory or 0 if
+                                         previous buffer is allocated.  */
+    bufsize bsize;                    /* Buffer size: positive if free,
+                                         negative if allocated. */
+};
+#define BH(p)   ((struct bhead *) (p))
+
+/*  Header in directly allocated buffers (by acqfcn) */
+
+struct bdhead {
+    bufsize tsize;                    /* Total size, including overhead */
+    struct bhead bh;                  /* Common header */
+};
+#define BDH(p)  ((struct bdhead *) (p))
+
+/* Header in free buffers */
+
+struct bfhead {
+    struct bhead bh;                  /* Common allocated/free header */
+    struct qlinks ql;                 /* Links on free list */
+};
+#define BFH(p)  ((struct bfhead *) (p))
+
+static struct bfhead freelist = {     /* List of free buffers */
+    {0, 0},
+    {&freelist, &freelist}
+};
+
+
+#ifdef BufStats
+static bufsize totalloc = 0;          /* Total space currently allocated */
+static long numget = 0, numrel = 0;   /* Number of bget() and brel() calls */
+#ifdef BECtl
+static long numpblk = 0;              /* Number of pool blocks */
+static long numpget = 0, numprel = 0; /* Number of block gets and rels */
+static long numdget = 0, numdrel = 0; /* Number of direct gets and rels */
+#endif /* BECtl */
+#endif /* BufStats */
+
+#ifdef BECtl
+
+/* Automatic expansion block management functions */
+
+static int (*compfcn) _((bufsize sizereq, int sequence)) = NULL;
+static void *(*acqfcn) _((bufsize size)) = NULL;
+static void (*relfcn) _((void *buf)) = NULL;
+
+static bufsize exp_incr = 0;          /* Expansion block size */
+static bufsize pool_len = 0;          /* 0: no bpool calls have been made
+                                         -1: not all pool blocks are
+                                         the same size
+                                         >0: (common) block size for all
+                                         bpool calls made so far
+                                      */
+#endif
+
+/*  Minimum allocation quantum: */
+
+#define QLSize  (sizeof(struct qlinks))
+#define SizeQ   ((SizeQuant > QLSize) ? SizeQuant : QLSize)
+
+#define V   (void)                    /* To denote unwanted returned values */
+
+/* End sentinel: value placed in bsize field of dummy block delimiting
+   end of pool block.  The most negative number which will  fit  in  a
+   bufsize, defined in a way that the compiler will accept. */
+
+#define ESent   ((bufsize) (-(((1L << (sizeof(bufsize) * 8 - 2)) - 1) * 2) - 2))
+
+/* added for PIO so that a bpool can be freed and another allocated */
+void bpoolrelease()
+{
+    LOG((2, "bpoolrelease"));
+    freelist.bh.prevfree=0;
+    freelist.bh.bsize=0;
+    freelist.ql.flink=&freelist;
+    freelist.ql.blink=&freelist;
+    LOG((2, "bpoolrelease"));
+
+#ifdef BufStats
+    totalloc = 0;             /* Total space currently allocated */
+    numget = 0;
+    numrel = 0;   /* Number of bget() and brel() calls */
+#ifdef BECtl
+    numpblk = 0;              /* Number of pool blocks */
+    numpget = 0;
+    numprel = 0; /* Number of block gets and rels */
+    numdget = 0;
+    numdrel = 0; /* Number of direct gets and rels */
+#endif /* BECtl */
+#endif /* BufStats */
+    LOG((2, "bpoolrelease"));
+
+#ifdef BECtl
+/* Automatic expansion block management functions */
+    compfcn = NULL;
+    acqfcn = NULL;
+    relfcn = NULL;
+    exp_incr = 0;
+    pool_len = 0;
+#endif
+    LOG((2, "bpoolrelease"));
+
+}
+
+
+
+/*  BGET  --  Allocate a buffer.  */
+
+void *bget(requested_size)
+bufsize requested_size;
+{
+    bufsize size = requested_size;
+    struct bfhead *b;
+#ifdef BestFit
+    struct bfhead *best;
+#endif
+    void *buf;
+#ifdef BECtl
+    int compactseq = 0;
+#endif
+
+#if PIO_USE_MALLOC
+    //    if(requested_size>maxsize){
+    //   maxsize=requested_size;
+    //   printf("%s %d %d\n",__FILE__,__LINE__,maxsize);
+    // }
+    buf = malloc(requested_size);
+    //    printf("bget allocate %ld %x\n",requested_size,buf);
+    return(buf);
+#endif
+
+
+    if(size<=0)
+        print_trace(NULL);
+
+    assert(size > 0);
+
+    if (size < SizeQ) {               /* Need at least room for the */
+        size = SizeQ;                 /*    queue links.  */
+    }
+#ifdef SizeQuant
+#if SizeQuant > 1
+    size = (size + (SizeQuant - 1)) & (~(SizeQuant - 1));
+#endif
+#endif
+
+    size += sizeof(struct bhead);     /* Add overhead in allocated buffer
+                                         to size required. */
+
+#ifdef BECtl
+    /* If a compact function was provided in the call to bectl(), wrap
+       a loop around the allocation process  to  allow  compaction  to
+       intervene in case we don't find a suitable buffer in the chain. */
+
+    while (1) {
+#endif
+        b = freelist.ql.flink;
+#ifdef BestFit
+        best = &freelist;
+#endif
+
+
+        /* Scan the free list searching for the first buffer big enough
+           to hold the requested size buffer. */
+
+#ifdef BestFit
+        while (b != &freelist) {
+            //    printf("%s %d %X %X %X %ld \n",__FILE__,__LINE__,b,&freelist,best,b->bh.bsize);
+            if (b->bh.bsize >= size) {
+                if ((best == &freelist) || (b->bh.bsize < best->bh.bsize)) {
+                    best = b;
+                }
+            }
+            b = b->ql.flink;              /* Link to next buffer */
+        }
+        b = best;
+#endif /* BestFit */
+
+        while (b != &freelist) {
+            if ((bufsize) b->bh.bsize >= size) {
+
+                /* Buffer  is big enough to satisfy  the request.  Allocate it
+                   to the caller.  We must decide whether the buffer is  large
+                   enough  to  split  into  the part given to the caller and a
+                   free buffer that remains on the free list, or  whether  the
+                   entire  buffer  should  be  removed  from the free list and
+                   given to the caller in its entirety.   We  only  split  the
+                   buffer if enough room remains for a header plus the minimum
+                   quantum of allocation. */
+
+                if ((b->bh.bsize - size) > (SizeQ + (sizeof(struct bhead)))) {
+                    struct bhead *ba, *bn;
+
+                    ba = BH(((char *) b) + (b->bh.bsize - size));
+                    bn = BH(((char *) ba) + size);
+                    assert(bn->prevfree == b->bh.bsize);
+                    /* Subtract size from length of free block. */
+                    b->bh.bsize -= size;
+                    /* Link allocated buffer to the previous free buffer. */
+                    ba->prevfree = b->bh.bsize;
+                    /* Plug negative size into user buffer. */
+                    ba->bsize = -(bufsize) size;
+                    /* Mark buffer after this one not preceded by free block. */
+                    bn->prevfree = 0;
+
+#ifdef BufStats
+                    totalloc += size;
+                    numget++;             /* Increment number of bget() calls */
+#endif
+                    buf = (void *) ((((char *) ba) + sizeof(struct bhead)));
+                    return buf;
+                } else {
+                    struct bhead *ba;
+
+                    ba = BH(((char *) b) + b->bh.bsize);
+                    assert(ba->prevfree == b->bh.bsize);
+
+                    /* The buffer isn't big enough to split.  Give  the  whole
+                       shebang to the caller and remove it from the free list. */
+
+                    assert(b->ql.blink->ql.flink == b);
+                    assert(b->ql.flink->ql.blink == b);
+                    b->ql.blink->ql.flink = b->ql.flink;
+                    b->ql.flink->ql.blink = b->ql.blink;
+
+#ifdef BufStats
+                    totalloc += b->bh.bsize;
+                    numget++;             /* Increment number of bget() calls */
+#endif
+                    /* Negate size to mark buffer allocated. */
+                    b->bh.bsize = -(b->bh.bsize);
+
+                    /* Zero the back pointer in the next buffer in memory
+                       to indicate that this buffer is allocated. */
+                    ba->prevfree = 0;
+
+                    /* Give user buffer starting at queue links. */
+                    buf =  (void *) &(b->ql);
+                    return buf;
+                }
+            }
+            b = b->ql.flink;              /* Link to next buffer */
+        }
+#ifdef BECtl
+
+        /* We failed to find a buffer.  If there's a compact  function
+           defined,  notify  it  of the size requested.  If it returns
+           TRUE, try the allocation again. */
+
+        if ((compfcn == NULL) || (!(*compfcn)(size, ++compactseq))) {
+            break;
+        }
+    }
+
+    /* No buffer available with requested size free. */
+
+    /* Don't give up yet -- look in the reserve supply. */
+
+    if (acqfcn != NULL) {
+        if (size > exp_incr - sizeof(struct bhead)) {
+
+            /* Request  is  too  large  to  fit in a single expansion
+               block.  Try to satisy it by a direct buffer acquisition. */
+
+            struct bdhead *bdh;
+
+            size += sizeof(struct bdhead) - sizeof(struct bhead);
+            if ((bdh = BDH((*acqfcn)((bufsize) size))) != NULL) {
+
+                /*  Mark the buffer special by setting the size field
+                    of its header to zero.  */
+                bdh->bh.bsize = 0;
+                bdh->bh.prevfree = 0;
+                bdh->tsize = size;
+#ifdef BufStats
+                totalloc += size;
+                numget++;             /* Increment number of bget() calls */
+                numdget++;            /* Direct bget() call count */
+#endif
+                buf =  (void *) (bdh + 1);
+
+                /*only let this happen once */
+                printf("%s %d memory request exceeds block size %ld %ld\n",__FILE__,__LINE__,
+                       size,exp_incr);
+                exp_incr = size+sizeof(struct bhead);
+
+                return buf;
+            }
+
+        } else {
+
+            /*  Try to obtain a new expansion block */
+
+            void *newpool;
+
+            if ((newpool = (*acqfcn)((bufsize) exp_incr)) != NULL) {
+                bpool(newpool, exp_incr);
+                buf =  bget(requested_size);  /* This can't, I say, can't
+                                                 get into a loop. */
+                //              printf("%s %d new memory block of size %d\n",__FILE__,__LINE__,exp_incr);
+                return buf;
+            }
+        }
+    }
+
+    /*  Still no buffer available */
+
+#endif /* BECtl */
+
+    return NULL;
+}
+
+/*  BGETZ  --  Allocate a buffer and clear its contents to zero.  We clear
+    the  entire  contents  of  the buffer to zero, not just the
+    region requested by the caller. */
+
+void *bgetz(size)
+bufsize size;
+{
+    char *buf = (char *) bget(size);
+
+    if (buf != NULL) {
+        struct bhead *b;
+        bufsize rsize;
+
+        b = BH(buf - sizeof(struct bhead));
+        rsize = -(b->bsize);
+        if (rsize == 0) {
+            struct bdhead *bd;
+
+            bd = BDH(buf - sizeof(struct bdhead));
+            rsize = bd->tsize - sizeof(struct bdhead);
+        } else {
+            rsize -= sizeof(struct bhead);
+        }
+        assert(rsize >= size);
+        V memset(buf, 0, (MemSize) rsize);
+    }
+    return ((void *) buf);
+}
+
+/*  BGETR  --  Reallocate a buffer.  This is a minimal implementation,
+    simply in terms of brel()  and  bget().      It  could  be
+    enhanced to allow the buffer to grow into adjacent free
+    blocks and to avoid moving data unnecessarily.  */
+
+void *bgetr(buf, size)
+void *buf;
+bufsize size;
+{
+    void *nbuf;
+    bufsize osize;                    /* Old size of buffer */
+    struct bhead *b;
+
+#if PIO_USE_MALLOC
+    return(realloc(buf, size));
+#endif
+    if ((nbuf = bget(size)) == NULL) { /* Acquire new buffer */
+        return NULL;
+    }
+    if (buf == NULL) {
+        return nbuf;
+    }
+    b = BH(((char *) buf) - sizeof(struct bhead));
+    osize = -b->bsize;
+#ifdef BECtl
+    if (osize == 0) {
+        /*  Buffer acquired directly through acqfcn. */
+        struct bdhead *bd;
+
+        bd = BDH(((char *) buf) - sizeof(struct bdhead));
+        osize = bd->tsize - sizeof(struct bdhead);
+    } else
+#endif
+        osize -= sizeof(struct bhead);
+    assert(osize > 0);
+    V memcpy((char *) nbuf, (char *) buf, /* Copy the data */
+             (MemSize) ((size < osize) ? size : osize));
+    brel(buf);
+    return nbuf;
+}
+
+/*  BREL  --  Release a buffer.  */
+
+void brel(buf)
+void *buf;
+{
+    struct bfhead *b, *bn;
+
+#if PIO_USE_MALLOC
+    //    printf("bget free %d %x\n",__LINE__,buf);
+    free(buf);
+    return;
+#endif
+
+
+    if(buf==NULL) return;       /* allow for null buffer */
+
+    b = BFH(((char *) buf) - sizeof(struct bhead));
+#ifdef BufStats
+    numrel++;                         /* Increment number of brel() calls */
+#endif
+    assert(buf != NULL);
+
+#ifdef BECtl
+    if (b->bh.bsize == 0) {           /* Directly-acquired buffer? */
+        struct bdhead *bdh;
+
+        bdh = BDH(((char *) buf) - sizeof(struct bdhead));
+        assert(b->bh.prevfree == 0);
+#ifdef BufStats
+        totalloc -= bdh->tsize;
+        assert(totalloc >= 0);
+        numdrel++;                    /* Number of direct releases */
+#endif /* BufStats */
+#ifdef FreeWipe
+        V memset((char *) buf, 0x55,
+                 (MemSize) (bdh->tsize - sizeof(struct bdhead)));
+#endif /* FreeWipe */
+        assert(relfcn != NULL);
+        (*relfcn)((void *) bdh);      /* Release it directly. */
+        return;
+    }
+#endif /* BECtl */
+
+    /* Buffer size must be negative, indicating that the buffer is
+       allocated. */
+
+    if (b->bh.bsize >= 0) {
+        bn = NULL;
+    }
+    assert(b->bh.bsize < 0);
+
+    /*  Back pointer in next buffer must be zero, indicating the
+        same thing: */
+
+    assert(BH((char *) b - b->bh.bsize)->prevfree == 0);
+
+#ifdef BufStats
+    totalloc += b->bh.bsize;
+    assert(totalloc >= 0);
+#endif
+
+    /* If the back link is nonzero, the previous buffer is free.  */
+
+    if (b->bh.prevfree != 0) {
+
+        /* The previous buffer is free.  Consolidate this buffer  with  it
+           by  adding  the  length  of  this  buffer  to the previous free
+           buffer.  Note that we subtract the size  in  the  buffer  being
+           released,  since  it's  negative to indicate that the buffer is
+           allocated. */
+
+        register bufsize size = b->bh.bsize;
+
+        /* Make the previous buffer the one we're working on. */
+        assert(BH((char *) b - b->bh.prevfree)->bsize == b->bh.prevfree);
+        b = BFH(((char *) b) - b->bh.prevfree);
+        b->bh.bsize -= size;
+    } else {
+
+        /* The previous buffer isn't allocated.  Insert this buffer
+           on the free list as an isolated free block. */
+
+        assert(freelist.ql.blink->ql.flink == &freelist);
+        assert(freelist.ql.flink->ql.blink == &freelist);
+        b->ql.flink = &freelist;
+        b->ql.blink = freelist.ql.blink;
+        freelist.ql.blink = b;
+        b->ql.blink->ql.flink = b;
+        b->bh.bsize = -b->bh.bsize;
+    }
+
+    /* Now we look at the next buffer in memory, located by advancing from
+       the  start  of  this  buffer  by its size, to see if that buffer is
+       free.  If it is, we combine  this  buffer  with  the  next  one  in
+       memory, dechaining the second buffer from the free list. */
+
+    bn =  BFH(((char *) b) + b->bh.bsize);
+    if (bn->bh.bsize > 0) {
+
+        /* The buffer is free.  Remove it from the free list and add
+           its size to that of our buffer. */
+
+        assert(BH((char *) bn + bn->bh.bsize)->prevfree == bn->bh.bsize);
+        assert(bn->ql.blink->ql.flink == bn);
+        assert(bn->ql.flink->ql.blink == bn);
+        bn->ql.blink->ql.flink = bn->ql.flink;
+        bn->ql.flink->ql.blink = bn->ql.blink;
+        b->bh.bsize += bn->bh.bsize;
+
+        /* Finally,  advance  to   the  buffer  that   follows  the  newly
+           consolidated free block.  We must set its  backpointer  to  the
+           head  of  the  consolidated free block.  We know the next block
+           must be an allocated block because the process of recombination
+           guarantees  that  two  free  blocks will never be contiguous in
+           memory.  */
+
+        bn = BFH(((char *) b) + b->bh.bsize);
+    }
+#ifdef FreeWipe
+    V memset(((char *) b) + sizeof(struct bfhead), 0x55,
+             (MemSize) (b->bh.bsize - sizeof(struct bfhead)));
+#endif
+    assert(bn->bh.bsize < 0);
+
+    /* The next buffer is allocated.  Set the backpointer in it  to  point
+       to this buffer; the previous free buffer in memory. */
+
+    bn->bh.prevfree = b->bh.bsize;
+
+#ifdef BECtl
+
+    /*  If  a  block-release function is defined, and this free buffer
+        constitutes the entire block, release it.  Note that  pool_len
+        is  defined  in  such a way that the test will fail unless all
+        pool blocks are the same size.  */
+
+    if (relfcn != NULL &&
+        ((bufsize) b->bh.bsize) == (pool_len - sizeof(struct bhead))) {
+
+        assert(b->bh.prevfree == 0);
+        assert(BH((char *) b + b->bh.bsize)->bsize == ESent);
+        assert(BH((char *) b + b->bh.bsize)->prevfree == b->bh.bsize);
+        /*  Unlink the buffer from the free list  */
+        b->ql.blink->ql.flink = b->ql.flink;
+        b->ql.flink->ql.blink = b->ql.blink;
+        //      printf("%s %d calling direct release for %x\n",__FILE__,__LINE__,b);
+        (*relfcn)(b);
+        //      printf("%s %d completed direct release \n",__FILE__,__LINE__);
+#ifdef BufStats
+        numprel++;                    /* Nr of expansion block releases */
+        numpblk--;                    /* Total number of blocks */
+        assert(numpblk == numpget - numprel);
+#endif /* BufStats */
+    }
+#endif /* BECtl */
+}
+
+#ifdef BECtl
+
+/*  BECTL  --  Establish automatic pool expansion control  */
+
+void bectl(compact, acquire, release, pool_incr)
+    int (*compact) _((bufsize sizereq, int sequence));
+void *(*acquire) _((bufsize size));
+void (*release) _((void *buf));
+bufsize pool_incr;
+{
+    compfcn = compact;
+    acqfcn = acquire;
+    relfcn = release;
+    exp_incr = pool_incr;
+}
+#endif
+
+/*  BPOOL  --  Add a region of memory to the buffer pool.  */
+
+void bpool(buf, len)
+void *buf;
+bufsize len;
+{
+    struct bfhead *b = BFH(buf);
+    struct bhead *bn;
+
+#ifdef SizeQuant
+    len &= ~(SizeQuant - 1);
+#endif
+#ifdef BECtl
+    if (pool_len == 0) {
+        pool_len = len;
+    } else if (len != pool_len) {
+        pool_len = -1;
+    }
+#ifdef BufStats
+    numpget++;                        /* Number of block acquisitions */
+    numpblk++;                        /* Number of blocks total */
+    assert(numpblk == numpget - numprel);
+#endif /* BufStats */
+#endif /* BECtl */
+
+    /* Since the block is initially occupied by a single free  buffer,
+       it  had  better  not  be  (much) larger than the largest buffer
+       whose size we can store in bhead.bsize. */
+
+    assert(len - sizeof(struct bhead) <= -((bufsize) ESent + 1));
+
+    /* Clear  the  backpointer at  the start of the block to indicate that
+       there  is  no  free  block  prior  to  this   one.    That   blocks
+       recombination when the first block in memory is released. */
+
+    b->bh.prevfree = 0;
+
+    /* Chain the new block to the free list. */
+
+    assert(freelist.ql.blink->ql.flink == &freelist);
+    assert(freelist.ql.flink->ql.blink == &freelist);
+    b->ql.flink = &freelist;
+    b->ql.blink = freelist.ql.blink;
+    freelist.ql.blink = b;
+    b->ql.blink->ql.flink = b;
+
+    /* Create a dummy allocated buffer at the end of the pool.  This dummy
+       buffer is seen when a buffer at the end of the pool is released and
+       blocks  recombination  of  the last buffer with the dummy buffer at
+       the end.  The length in the dummy buffer  is  set  to  the  largest
+       negative  number  to  denote  the  end  of  the pool for diagnostic
+       routines (this specific value is  not  counted  on  by  the  actual
+       allocation and release functions). */
+
+    len -= sizeof(struct bhead);
+    b->bh.bsize = (bufsize) len;
+#ifdef FreeWipe
+    V memset(((char *) b) + sizeof(struct bfhead), 0x55,
+             (MemSize) (len - sizeof(struct bfhead)));
+#endif
+    bn = BH(((char *) b) + len);
+    bn->prevfree = (bufsize) len;
+    /* Definition of ESent assumes two's complement! */
+    assert((~0) == -1);
+    bn->bsize = ESent;
+}
+
+#ifdef BufStats
+
+void bfreespace(bufsize *totfree, bufsize *maxfree)
+{
+    struct bfhead *b = freelist.ql.flink;
+    *totfree = 0;
+    *maxfree = -1;
+    while (b != &freelist) {
+        assert(b->bh.bsize > 0);
+        *totfree += b->bh.bsize;
+        if (b->bh.bsize > *maxfree) {
+            *maxfree = b->bh.bsize;
+        }
+        b = b->ql.flink;              /* Link to next buffer */
+    }
+}
+
+/*  BSTATS  --  Return buffer allocation free space statistics.  */
+
+void bstats(curalloc, totfree, maxfree, nget, nrel)
+    bufsize *curalloc, *totfree, *maxfree;
+long *nget, *nrel;
+{
+    *nget = numget;
+    *nrel = numrel;
+    *curalloc = totalloc;
+    bfreespace(totfree, maxfree);
+}
+
+#ifdef BECtl
+
+/*  BSTATSE  --  Return extended statistics  */
+
+void bstatse(pool_incr, npool, npget, nprel, ndget, ndrel)
+bufsize *pool_incr;
+long *npool, *npget, *nprel, *ndget, *ndrel;
+{
+    *pool_incr = (pool_len < 0) ? -exp_incr : exp_incr;
+    *npool = numpblk;
+    *npget = numpget;
+    *nprel = numprel;
+    *ndget = numdget;
+    *ndrel = numdrel;
+}
+#endif /* BECtl */
+#endif /* BufStats */
+
+#ifdef DumpData
+
+/*  BUFDUMP  --  Dump the data in a buffer.  This is called with the  user
+    data pointer, and backs up to the buffer header.  It will
+    dump either a free block or an allocated one.       */
+
+void bufdump(buf)
+void *buf;
+{
+    struct bfhead *b;
+    unsigned char *bdump;
+    bufsize bdlen;
+
+    b = BFH(((char *) buf) - sizeof(struct bhead));
+    assert(b->bh.bsize != 0);
+    if (b->bh.bsize < 0) {
+        bdump = (unsigned char *) buf;
+        bdlen = (-b->bh.bsize) - sizeof(struct bhead);
+    } else {
+        bdump = (unsigned char *) (((char *) b) + sizeof(struct bfhead));
+        bdlen = b->bh.bsize - sizeof(struct bfhead);
+    }
+
+    while (bdlen > 0) {
+        int i, dupes = 0;
+        bufsize l = bdlen;
+        char bhex[50], bascii[20];
+
+        if (l > 16) {
+            l = 16;
+        }
+
+        for (i = 0; i < l; i++) {
+            V sprintf(bhex + i * 3, "%02X ", bdump[i]);
+            bascii[i] = isprint(bdump[i]) ? bdump[i] : ' ';
+        }
+        bascii[i] = 0;
+        V printf("%-48s   %s\n", bhex, bascii);
+        bdump += l;
+        bdlen -= l;
+        while ((bdlen > 16) && (memcmp((char *) (bdump - 16),
+                                       (char *) bdump, 16) == 0)) {
+            dupes++;
+            bdump += 16;
+            bdlen -= 16;
+        }
+        if (dupes > 1) {
+            V printf(
+                "     (%d lines [%d bytes] identical to above line skipped)\n",
+                dupes, dupes * 16);
+        } else if (dupes == 1) {
+            bdump -= 16;
+            bdlen += 16;
+        }
+    }
+}
+#endif
+
+#ifdef BufDump
+
+/*  BPOOLD  --  Dump a buffer pool.  The buffer headers are always listed.
+    If DUMPALLOC is nonzero, the contents of allocated buffers
+    are  dumped.   If  DUMPFREE  is  nonzero,  free blocks are
+    dumped as well.  If FreeWipe  checking      is  enabled,  free
+    blocks      which  have  been clobbered will always be dumped. */
+
+void bpoold(buf, dumpalloc, dumpfree)
+void *buf;
+int dumpalloc, dumpfree;
+{
+    struct bfhead *b = BFH(buf);
+
+    while (b->bh.bsize != ESent) {
+        bufsize bs = b->bh.bsize;
+
+        if (bs < 0) {
+            bs = -bs;
+            V printf("Allocated buffer: size %6ld bytes.\n", (long) bs);
+            if (dumpalloc) {
+                bufdump((void *) (((char *) b) + sizeof(struct bhead)));
+            }
+        } else {
+            char *lerr = "";
+
+            assert(bs > 0);
+            if ((b->ql.blink->ql.flink != b) ||
+                (b->ql.flink->ql.blink != b)) {
+                lerr = "  (Bad free list links)";
+            }
+            V printf("Free block:       size %6ld bytes.%s\n",
+                     (long) bs, lerr);
+#ifdef FreeWipe
+            lerr = ((char *) b) + sizeof(struct bfhead);
+            if ((bs > sizeof(struct bfhead)) && ((*lerr != 0x55) ||
+                                                 (memcmp(lerr, lerr + 1,
+                                                         (MemSize) (bs - (sizeof(struct bfhead) + 1))) != 0))) {
+                V printf(
+                    "(Contents of above free block have been overstored.)\n");
+                bufdump((void *) (((char *) b) + sizeof(struct bhead)));
+            } else
+#endif
+                if (dumpfree) {
+                    bufdump((void *) (((char *) b) + sizeof(struct bhead)));
+                }
+        }
+        b = BFH(((char *) b) + bs);
+    }
+}
+#endif /* BufDump */
+
+#ifdef BufValid
+
+/*  BPOOLV  --  Validate a buffer pool.  If NDEBUG isn't defined,
+    any error generates an assertion failure.  */
+
+int bpoolv(buf)
+void *buf;
+{
+    struct bfhead *b = BFH(buf);
+
+    while (b->bh.bsize != ESent) {
+        bufsize bs = b->bh.bsize;
+
+        if (bs < 0) {
+            bs = -bs;
+        } else {
+            char *lerr = "";
+
+            assert(bs > 0);
+            if (bs <= 0) {
+                return 0;
+            }
+            if ((b->ql.blink->ql.flink != b) ||
+                (b->ql.flink->ql.blink != b)) {
+                V printf("Free block: size %6ld bytes.  (Bad free list links)\n",
+                         (long) bs);
+                assert(0);
+                return 0;
+            }
+#ifdef FreeWipe
+            lerr = ((char *) b) + sizeof(struct bfhead);
+            if ((bs > sizeof(struct bfhead)) && ((*lerr != 0x55) ||
+                                                 (memcmp(lerr, lerr + 1,
+                                                         (MemSize) (bs - (sizeof(struct bfhead) + 1))) != 0))) {
+                V printf(
+                    "(Contents of above free block have been overstored.)\n");
+                bufdump((void *) (((char *) b) + sizeof(struct bhead)));
+                assert(0);
+                return 0;
+            }
+#endif
+        }
+        b = BFH(((char *) b) + bs);
+    }
+    return 1;
+}
+#endif /* BufValid */
+
+/***********************\
+ *                      *
+ * Built-in test program *
+ *                      *
+        \***********************/
+
+#ifdef TestProg
+
+#define Repeatable  1                 /* Repeatable pseudorandom sequence */
+                                      /* If Repeatable is not defined, a
+                                         time-seeded pseudorandom sequence
+                                         is generated, exercising BGET with
+                                         a different pattern of calls on each
+                                         run. */
+#define OUR_RAND                      /* Use our own built-in version of
+                                         rand() to guarantee the test is
+                                         100% repeatable. */
+
+#ifdef BECtl
+#define PoolSize    300000            /* Test buffer pool size */
+#else
+#define PoolSize    50000             /* Test buffer pool size */
+#endif
+#define ExpIncr     32768             /* Test expansion block size */
+#define CompactTries 10               /* Maximum tries at compacting */
+
+#define dumpAlloc   0                 /* Dump allocated buffers ? */
+#define dumpFree    0                 /* Dump free buffers ? */
+
+#ifndef Repeatable
+extern long time();
+#endif
+
+extern char *malloc();
+extern int free _((char *));
+
+static char *bchain = NULL;           /* Our private buffer chain */
+static char *bp = NULL;               /* Our initial buffer pool */
+
+#include 
+
+#ifdef OUR_RAND
+
+static unsigned long int next = 1;
+
+/* Return next random integer */
+
+int rand()
+{
+    next = next * 1103515245L + 12345;
+    return (unsigned int) (next / 65536L) % 32768L;
+}
+
+/* Set seed for random generator */
+
+void srand(seed)
+unsigned int seed;
+{
+    next = seed;
+}
+#endif
+
+/*  STATS  --  Edit statistics returned by bstats() or bstatse().  */
+
+static void stats(when)
+char *when;
+{
+    bufsize cural, totfree, maxfree;
+    long nget, nfree;
+#ifdef BECtl
+    bufsize pincr;
+    long totblocks, npget, nprel, ndget, ndrel;
+#endif
+
+    bstats(&cural, &totfree, &maxfree, &nget, &nfree);
+    V printf(
+        "%s: %ld gets, %ld releases.  %ld in use, %ld free, largest = %ld\n",
+        when, nget, nfree, (long) cural, (long) totfree, (long) maxfree);
+#ifdef BECtl
+    bstatse(&pincr, &totblocks, &npget, &nprel, &ndget, &ndrel);
+    V printf(
+        "  Blocks: size = %ld, %ld (%ld bytes) in use, %ld gets, %ld frees\n",
+        (long)pincr, totblocks, pincr * totblocks, npget, nprel);
+    V printf("  %ld direct gets, %ld direct frees\n", ndget, ndrel);
+#endif /* BECtl */
+}
+
+#ifdef BECtl
+static int protect = 0;               /* Disable compaction during bgetr() */
+
+/*  BCOMPACT  --  Compaction call-back function.  */
+
+static int bcompact(bsize, seq)
+bufsize bsize;
+int seq;
+{
+#ifdef CompactTries
+    char *bc = bchain;
+    int i = rand() & 0x3;
+
+#ifdef COMPACTRACE
+    V printf("Compaction requested.  %ld bytes needed, sequence %d.\n",
+             (long) bsize, seq);
+#endif
+
+    if (protect || (seq > CompactTries)) {
+#ifdef COMPACTRACE
+        V printf("Compaction gave up.\n");
+#endif
+        return 0;
+    }
+
+    /* Based on a random cast, release a random buffer in the list
+       of allocated buffers. */
+
+    while (i > 0 && bc != NULL) {
+        bc = *((char **) bc);
+        i--;
+    }
+    if (bc != NULL) {
+        char *fb;
+
+        fb = *((char **) bc);
+        if (fb != NULL) {
+            *((char **) bc) = *((char **) fb);
+            brel((void *) fb);
+            return 1;
+        }
+    }
+
+#ifdef COMPACTRACE
+    V printf("Compaction bailed out.\n");
+#endif
+#endif /* CompactTries */
+    return 0;
+}
+
+/*  BEXPAND  --  Expand pool call-back function.  */
+
+static void *bexpand(size)
+bufsize size;
+{
+    void *np = NULL;
+    bufsize cural, totfree, maxfree;
+    long nget, nfree;
+
+    /* Don't expand beyond the total allocated size given by PoolSize. */
+
+    bstats(&cural, &totfree, &maxfree, &nget, &nfree);
+
+    if (cural < PoolSize) {
+        np = (void *) malloc((unsigned) size);
+    }
+#ifdef EXPTRACE
+    V printf("Expand pool by %ld -- %s.\n", (long) size,
+             np == NULL ? "failed" : "succeeded");
+#endif
+    return np;
+}
+
+/*  BSHRINK  --  Shrink buffer pool call-back function.  */
+
+static void bshrink(buf)
+void *buf;
+{
+    if (((char *) buf) == bp) {
+#ifdef EXPTRACE
+        V printf("Initial pool released.\n");
+#endif
+        bp = NULL;
+    }
+#ifdef EXPTRACE
+    V printf("Shrink pool.\n");
+#endif
+    free((char *) buf);
+}
+
+#endif /* BECtl */
+
+/*  Restrict buffer requests to those large enough to contain our pointer and
+    small enough for the CPU architecture.  */
+
+static bufsize blimit(bs)
+bufsize bs;
+{
+    if (bs < sizeof(char *)) {
+        bs = sizeof(char *);
+    }
+
+    /* This is written out in this ugly fashion because the
+       cool expression in sizeof(int) that auto-configured
+       to any length int befuddled some compilers. */
+
+    if (sizeof(int) == 2) {
+        if (bs > 32767) {
+            bs = 32767;
+        }
+    } else {
+        if (bs > 200000) {
+            bs = 200000;
+        }
+    }
+    return bs;
+}
+
+int main()
+{
+    int i;
+    double x;
+
+    /* Seed the random number generator.  If Repeatable is defined, we
+       always use the same seed.  Otherwise, we seed from the clock to
+       shake things up from run to run. */
+
+#ifdef Repeatable
+    V srand(1234);
+#else
+    V srand((int) time((long *) NULL));
+#endif
+
+    /*  Compute x such that pow(x, p) ranges between 1 and 4*ExpIncr as
+        p ranges from 0 to ExpIncr-1, with a concentration in the lower
+        numbers.  */
+
+    x = 4.0 * ExpIncr;
+    x = log(x);
+    x = exp(log(4.0 * ExpIncr) / (ExpIncr - 1.0));
+
+#ifdef BECtl
+    bectl(bcompact, bexpand, bshrink, (bufsize) ExpIncr);
+    bp = malloc(ExpIncr);
+    assert(bp != NULL);
+    bpool((void *) bp, (bufsize) ExpIncr);
+#else
+    bp = malloc(PoolSize);
+    assert(bp != NULL);
+    bpool((void *) bp, (bufsize) PoolSize);
+#endif
+
+    stats("Create pool");
+    V bpoolv((void *) bp);
+    bpoold((void *) bp, dumpAlloc, dumpFree);
+
+    for (i = 0; i < TestProg; i++) {
+        char *cb;
+        bufsize bs = pow(x, (double) (rand() & (ExpIncr - 1)));
+
+        assert(bs <= (((bufsize) 4) * ExpIncr));
+        bs = blimit(bs);
+        if (rand() & 0x400) {
+            cb = (char *) bgetz(bs);
+        } else {
+            cb = (char *) bget(bs);
+        }
+        if (cb == NULL) {
+#ifdef EasyOut
+            break;
+#else
+            char *bc = bchain;
+
+            if (bc != NULL) {
+                char *fb;
+
+                fb = *((char **) bc);
+                if (fb != NULL) {
+                    *((char **) bc) = *((char **) fb);
+                    brel((void *) fb);
+                }
+                continue;
+            }
+#endif
+        }
+        *((char **) cb) = (char *) bchain;
+        bchain = cb;
+
+        /* Based on a random cast, release a random buffer in the list
+           of allocated buffers. */
+
+        if ((rand() & 0x10) == 0) {
+            char *bc = bchain;
+            int i = rand() & 0x3;
+
+            while (i > 0 && bc != NULL) {
+                bc = *((char **) bc);
+                i--;
+            }
+            if (bc != NULL) {
+                char *fb;
+
+                fb = *((char **) bc);
+                if (fb != NULL) {
+                    *((char **) bc) = *((char **) fb);
+                    brel((void *) fb);
+                }
+            }
+        }
+
+        /* Based on a random cast, reallocate a random buffer in the list
+           to a random size */
+
+        if ((rand() & 0x20) == 0) {
+            char *bc = bchain;
+            int i = rand() & 0x3;
+
+            while (i > 0 && bc != NULL) {
+                bc = *((char **) bc);
+                i--;
+            }
+            if (bc != NULL) {
+                char *fb;
+
+                fb = *((char **) bc);
+                if (fb != NULL) {
+                    char *newb;
+
+                    bs = pow(x, (double) (rand() & (ExpIncr - 1)));
+                    bs = blimit(bs);
+#ifdef BECtl
+                    protect = 1;      /* Protect against compaction */
+#endif
+                    newb = (char *) bgetr((void *) fb, bs);
+#ifdef BECtl
+                    protect = 0;
+#endif
+                    if (newb != NULL) {
+                        *((char **) bc) = newb;
+                    }
+                }
+            }
+        }
+    }
+    stats("\nAfter allocation");
+    if (bp != NULL) {
+        V bpoolv((void *) bp);
+        bpoold((void *) bp, dumpAlloc, dumpFree);
+    }
+
+    while (bchain != NULL) {
+        char *buf = bchain;
+
+        bchain = *((char **) buf);
+        brel((void *) buf);
+    }
+    stats("\nAfter release");
+#ifndef BECtl
+    if (bp != NULL) {
+        V bpoolv((void *) bp);
+        bpoold((void *) bp, dumpAlloc, dumpFree);
+    }
+#endif
+
+    return 0;
+}
+#endif
diff --git a/src/externals/pio2/src/clib/bget.h b/src/externals/pio2/src/clib/bget.h
new file mode 100644
index 00000000000..2b5be7dcdc0
--- /dev/null
+++ b/src/externals/pio2/src/clib/bget.h
@@ -0,0 +1,41 @@
+/*
+
+  Interface definitions for bget.c, the memory management package.
+
+*/
+#ifndef _BGET_H_
+#define _BGET_H_
+
+/* in PIO we use DEBUG bget uses NDEBUG */
+//#ifndef DEBUG
+//#undef NDEBUG
+//#endif
+
+#ifndef _
+#ifdef PROTOTYPES
+#define  _(x)  x                      /* If compiler knows prototypes */
+#else
+#define  _(x)  ()                     /* It it doesn't */
+#endif /* PROTOTYPES */
+#endif
+
+typedef long bufsize;
+void    bpool       _((void *buffer, bufsize len));
+void   *bget        _((bufsize size));
+void   *bgetz       _((bufsize size));
+void   *bgetr       _((void *buffer, bufsize newsize));
+void    brel        _((void *buf));
+void    bectl       _((int (*compact)(bufsize sizereq, int sequence),
+                       void *(*acquire)(bufsize size),
+                       void (*release)(void *buf), bufsize pool_incr));
+void    bstats      _((bufsize *curalloc, bufsize *totfree, bufsize *maxfree,
+                       long *nget, long *nrel));
+void    bstatse     _((bufsize *pool_incr, long *npool, long *npget,
+                       long *nprel, long *ndget, long *ndrel));
+void    bufdump     _((void *buf));
+void    bpoold      _((void *pool, int dumpalloc, int dumpfree));
+int     bpoolv      _((void *pool));
+void bpoolrelease  _();
+void bfreespace  _((bufsize *maxfree, bufsize *totfree));
+
+#endif
diff --git a/externals/pio2/src/clib/config.h.in b/src/externals/pio2/src/clib/config.h.in
similarity index 76%
rename from externals/pio2/src/clib/config.h.in
rename to src/externals/pio2/src/clib/config.h.in
index e4be61d9e0c..8d80d37fd51 100644
--- a/externals/pio2/src/clib/config.h.in
+++ b/src/externals/pio2/src/clib/config.h.in
@@ -19,4 +19,10 @@
  * will use the included bget() package for memory management. */
 #define PIO_USE_MALLOC @USE_MALLOC@
 
+/** Set to non-zero to turn on logging. Output may be large. */
+#define PIO_ENABLE_LOGGING @ENABLE_LOGGING@
+
+/** Size of MPI_Offset type. */
+#define SIZEOF_MPI_OFFSET @SIZEOF_MPI_OFFSET@
+
 #endif /* _PIO_CONFIG_ */
diff --git a/externals/pio2/src/clib/dtypes.h b/src/externals/pio2/src/clib/dtypes.h
similarity index 100%
rename from externals/pio2/src/clib/dtypes.h
rename to src/externals/pio2/src/clib/dtypes.h
diff --git a/externals/pio2/src/clib/ncparser.pl b/src/externals/pio2/src/clib/ncparser.pl
similarity index 100%
rename from externals/pio2/src/clib/ncparser.pl
rename to src/externals/pio2/src/clib/ncparser.pl
diff --git a/externals/pio2/src/clib/ncputgetparser.pl b/src/externals/pio2/src/clib/ncputgetparser.pl
similarity index 100%
rename from externals/pio2/src/clib/ncputgetparser.pl
rename to src/externals/pio2/src/clib/ncputgetparser.pl
diff --git a/src/externals/pio2/src/clib/pio.h b/src/externals/pio2/src/clib/pio.h
new file mode 100644
index 00000000000..9dace3782c9
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio.h
@@ -0,0 +1,1189 @@
+/**
+ * @file
+ * Public headers for the PIO C interface.
+ * @author Jim Edwards
+ * @date  2014
+ *
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#ifndef _PIO_H_
+#define _PIO_H_
+#include 
+#include 
+#include 
+#include  /* memcpy */
+#include 
+
+#ifdef _NETCDF
+#include 
+#ifdef _NETCDF4
+#include 
+#endif
+#endif
+#ifdef _PNETCDF
+#include 
+#endif
+
+#ifndef MPI_OFFSET
+/** MPI_OFFSET is an integer type of size sufficient to represent the
+ * size (in bytes) of the largest file supported by MPI. In some MPI
+ * implementations MPI_OFFSET is not properly defined.  */
+#define MPI_OFFSET  MPI_LONG_LONG
+#endif
+
+/** PIO_OFFSET is an integer type of size sufficient to represent the
+ * size (in bytes) of the largest file supported by MPI. */
+#define PIO_OFFSET MPI_OFFSET
+#define PIO_Offset MPI_Offset
+
+/** The maximum number of variables allowed in a netCDF file. */
+#define PIO_MAX_VARS NC_MAX_VARS
+
+/** The maximum number of dimensions allowed in a netCDF file. */
+#define PIO_MAX_DIMS NC_MAX_DIMS
+
+/** Pass this to PIOc_set_iosystem_error_handling() as the iosysid in
+ * order to set default error handling. */
+#define PIO_DEFAULT (-1)
+
+/** Used in the decomposition netCDF file. */
+
+/* Holds the version of the decomposition file. */
+#define DECOMP_VERSION_ATT_NAME "version"
+
+/* Holds the maximum length of any task map. */
+#define DECOMP_MAX_MAPLEN_ATT_NAME "max_maplen"
+
+/* Name of title attribute. */
+#define DECOMP_TITLE_ATT_NAME "title"
+
+/* Name of history attribute. */
+#define DECOMP_HISTORY_ATT_NAME "history"
+
+/* Name of source attribute. */
+#define DECOMP_SOURCE_ATT_NAME "source"
+
+/* Name of array order (C or Fortran) attribute. */
+#define DECOMP_ORDER_ATT_NAME "array_order"
+
+/* Name of backtrace attribute. */
+#define DECOMP_BACKTRACE_ATT_NAME "backtrace"
+
+/* Name for the dim dim in decomp file. */
+#define DECOMP_DIM_DIM "dims"
+
+/* Name for the npes dim in decomp file. */
+#define DECOMP_TASK_DIM_NAME "task"
+
+/* Name for the npes dim in decomp file. */
+#define DECOMP_MAPELEM_DIM_NAME "map_element"
+
+#define DECOMP_NDIMS "ndims"
+
+/* Name of var in decomp file that holds global array sizes. */
+#define DECOMP_GLOBAL_SIZE_VAR_NAME "global_size"
+
+/* Name of var in decomp file that holds the length of the map for
+ * each task. */
+#define DECOMP_MAPLEN_VAR_NAME "maplen"
+
+/* Name of var in decomp file that holds map. */
+#define DECOMP_MAP_VAR_NAME "map"
+
+/* String used to indicate a decomposition file is in C
+ * array-order. */
+#define DECOMP_C_ORDER_STR "C"
+
+/* String used to indicate a decomposition file is in Fortran
+ * array-order. */
+#define DECOMP_FORTRAN_ORDER_STR "Fortran"
+
+
+/**
+ * Variable description structure.
+ */
+typedef struct var_desc_t
+{
+    /** The unlimited dimension in the netCDF file (typically the time
+     * dimension). -1 if there is no unlimited dimension. */
+    int record;
+
+    /** Number of dimensions for this variable. */
+    int ndims;
+
+    /** ID of each outstanding pnetcdf request for this variable. */
+    int *request;
+
+    /** Number of requests bending with pnetcdf. */
+    int nreqs;
+
+    /* Holds the fill value of this var. */
+    void *fillvalue;
+
+    /* The PIO data type (PIO_INT, PIO_FLOAT, etc.) */
+    int pio_type;
+
+    /* The size of the data type (2 for PIO_SHORT, 4 for PIO_INT, etc.) */
+    PIO_Offset type_size;
+
+    /** Non-zero if fill mode is turned on for this var. */
+    int use_fill;
+
+    /** Buffer that contains the holegrid fill values used to fill in
+     * missing sections of data when using the subset rearranger. */
+    void *fillbuf;
+
+    /** Data buffer for this variable. */
+    void *iobuf;
+} var_desc_t;
+
+/**
+ * IO region structure.
+ *
+ * Each IO region is a unit of data which can be described using start
+ * and count arrays. Each IO task may in general have multiple io
+ * regions per variable.  The box rearranger will have at most one io
+ * region per variable.
+ *
+ * The write from a particular IO task is divided into 1 or more
+ * regions each of which can be described using start and count. The
+ * io_region typedef is a linked list of those regions.
+ */
+typedef struct io_region
+{
+    /** The offset from the beginning of the data buffer to the
+     * beginning of this region.  */
+    int loffset;
+
+    /** Start array for this region. */
+    PIO_Offset *start;
+
+    /** Count array for this region. */
+    PIO_Offset *count;
+
+    /** Pointer to the next io_region in the list. */
+    struct io_region *next;
+} io_region;
+
+/**
+ * Rearranger comm type. The rearranger option values must match the
+ * definitions in the fortran interface.
+ */
+enum PIO_REARR_COMM_TYPE
+{
+    /** Point to point */
+    PIO_REARR_COMM_P2P = (0),
+
+    /** Collective */
+    PIO_REARR_COMM_COLL
+};
+
+/**
+ * Rearranger comm flow control direction. The rearranger option
+ * values must match the definitions in the fortran interface.
+ */
+enum PIO_REARR_COMM_FC_DIR
+{
+    /** Comp procs to io procs and vice versa */
+    PIO_REARR_COMM_FC_2D_ENABLE = (0),
+
+    /** Comp procs to io procs only */
+    PIO_REARR_COMM_FC_1D_COMP2IO,
+
+    /** IO procs to comp procs only */
+    PIO_REARR_COMM_FC_1D_IO2COMP,
+
+    /** Disable flow control */
+    PIO_REARR_COMM_FC_2D_DISABLE
+};
+
+/* Constant to indicate unlimited requests. */
+#define PIO_REARR_COMM_UNLIMITED_PEND_REQ -1
+
+/**
+ * Rearranger comm flow control options.
+ */
+typedef struct rearr_comm_fc_opt
+{
+    /** Enable handshake */
+    bool hs;
+
+    /** Enable isends - if false use blocking sends */
+    bool isend;
+
+    /** Max pending requests
+     * (PIO_REARR_COMM_UNLIMITED_PEND_REQ => unlimited pend req).
+     * This is the number of messages allowed to be in flight at one
+     * time. On some systems posting all messages at once creates a
+     * significant bottleneck in communications and throttling in this
+     * manner improves overall performance. */
+    int max_pend_req;
+} rearr_comm_fc_opt_t;
+
+/**
+ * Rearranger options
+ */
+typedef struct rearr_opt
+{
+    /** Comm type - see PIO_REARR_COMM_TYPE */
+    int comm_type;
+
+    /** Comm flow control dir - see PIO_REARR_COMM_FC_DIR */
+    int fcd;
+
+    /** flow control opts, comp to io procs */
+    rearr_comm_fc_opt_t comp2io;
+
+    /** flow control opts, io to comp procs */
+    rearr_comm_fc_opt_t io2comp;
+} rearr_opt_t;
+
+/**
+ * IO descriptor structure.
+ *
+ * This structure defines the mapping for a given variable between
+ * compute and IO decomposition.
+ */
+typedef struct io_desc_t
+{
+    /** The ID of this io_desc_t. */
+    int ioid;
+
+    /** The length of the decomposition map. */
+    int maplen;
+
+    /** A 1-D array with iodesc->maplen elements, which are the
+     * 1-based mappings to the global array for that task. */
+    PIO_Offset *map;
+
+    /** Number of tasks involved in the communication between comp and
+     * io tasks. */
+    int nrecvs;
+
+    /** Local size of the decomposition array on the compute node. */
+    int ndof;
+
+    /** All vars included in this io_desc_t have the same number of
+     * dimensions. */
+    int ndims;
+
+    /** An array of size ndims with the length of each dimension. */
+    int *dimlen;
+
+    /** The actual number of IO tasks participating. */
+    int num_aiotasks;
+
+    /** The rearranger in use for this variable. */
+    int rearranger;
+
+    /** Maximum number of regions in the decomposition. */
+    int maxregions;
+
+    /** Does this decomp leave holes in the field (true) or write
+     * everywhere (false) */
+    bool needsfill;
+
+    /** The maximum number of bytes of this iodesc before flushing. */
+    int maxbytes;
+
+    /** The MPI type of the data. */
+    MPI_Datatype basetype;
+
+    /** The size in bytes of a datum of MPI type basetype. */
+    int basetype_size;
+
+    /** Length of the iobuffer on this task for a single field on the
+     * IO node. The arrays from compute nodes gathered and rearranged
+     * to the io-nodes (which are sometimes collocated with compute
+     * nodes), each io task contains data from the compmap of one or
+     * more compute tasks in the iomap array. */
+    PIO_Offset llen;
+
+    /** Maximum llen participating. */
+    int maxiobuflen;
+
+    /** Array (length nrecvs) of computation tasks received from. */
+    int *rfrom;
+
+    /** Array (length nrecvs) of counts of data to be received from
+     * each computation task by the IO tasks. */
+    int *rcount;
+
+    /** Array (length numiotasks) of data counts to send to each task
+     * in the communication in pio_swapm(). */
+    int *scount;
+
+    /** Array (length ndof) for the BOX rearranger with the index
+     * for computation taks (send side during writes). */
+    PIO_Offset *sindex;
+
+    /** Index for the IO tasks (receive side during writes). */
+    PIO_Offset *rindex;
+
+    /** Array (of length nrecvs) of receive MPI types in pio_swapm() call. */
+    MPI_Datatype *rtype;
+
+    /** Array of send MPI types in pio_swapm() call. */
+    MPI_Datatype *stype;
+
+    /** Number of send MPI types in pio_swapm() call. */
+    int num_stypes;
+
+    /** Used when writing fill data. */
+    int holegridsize;
+
+    /** max holegridsize across all io tasks, needed for netcdf and netcdf4c serial */
+    int maxholegridsize;
+
+    /** Used when writing fill data. */
+    int maxfillregions;
+
+    /** Linked list of regions. */
+    io_region *firstregion;
+
+    /** Used when writing fill data. */
+    io_region *fillregion;
+
+    /** Rearranger flow control options
+     *  (handshake, non-blocking sends, pending requests)
+     */
+    rearr_opt_t rearr_opts;
+
+    /** In the subset communicator each io task is associated with a
+     * unique group of comp tasks this is the communicator for that
+     * group. */
+    MPI_Comm subset_comm;
+
+    /** Pointer to the next io_desc_t in the list. */
+    struct io_desc_t *next;
+} io_desc_t;
+
+/**
+ * IO system descriptor structure.
+ *
+ * This structure contains the general IO subsystem data and MPI
+ * structure
+ */
+typedef struct iosystem_desc_t
+{
+    /** The ID of this iosystem_desc_t. This will be obtained by
+     * calling PIOc_Init_Intercomm() or PIOc_Init_Intracomm(). */
+    int iosysid;
+
+    /** This is an MPI intra communicator that includes all the tasks in
+     * both the IO and the computation communicators. */
+    MPI_Comm union_comm;
+
+    /** This is an MPI intra communicator that includes all the tasks
+     * involved in IO. */
+    MPI_Comm io_comm;
+
+    /** This is an MPI intra communicator that includes all the tasks
+     * involved in computation. */
+    MPI_Comm comp_comm;
+
+    /** This is an MPI inter communicator between IO communicator and
+     * computation communicator. */
+    MPI_Comm intercomm;
+
+    /** This is a copy (but not an MPI copy) of either the comp (for
+     * non-async) or the union (for async) communicator. */
+    MPI_Comm my_comm;
+
+    /** This MPI group contains the processors involved in
+     * computation. */
+    MPI_Group compgroup;
+
+    /** This MPI group contains the processors involved in I/O. */
+    MPI_Group iogroup;
+
+    /** The number of tasks in the IO communicator. */
+    int num_iotasks;
+
+    /** The number of tasks in the computation communicator. */
+    int num_comptasks;
+
+    /** The number of tasks in the union communicator (will be
+     * num_comptasks for non-async, num_comptasks + num_iotasks for
+     * async). */
+    int num_uniontasks;
+
+    /** Rank of this task in the union communicator. */
+    int union_rank;
+
+    /** The rank of this process in the computation communicator, or -1
+     * if this process is not part of the computation communicator. */
+    int comp_rank;
+
+    /** The rank of this process in the IO communicator, or -1 if this
+     * process is not part of the IO communicator. */
+    int io_rank;
+
+    /** Set to MPI_ROOT if this task is the master of IO communicator, 0
+     * otherwise. */
+    int iomaster;
+
+    /** Set to MPI_ROOT if this task is the master of comp communicator, 0
+     * otherwise. */
+    int compmaster;
+
+    /** Rank of IO root task (which is rank 0 in io_comm) in the union
+     * communicator. Will always be 0 for async situations. */
+    int ioroot;
+
+    /** Rank of computation root task (which is rank 0 in
+     * comm_comms[cmp]) in the union communicator. Will always = number
+     * of IO tasks in async situations. */
+    int comproot;
+
+    /** An array of the ranks of all IO tasks within the union
+     * communicator. */
+    int *ioranks;
+
+    /** An array of the ranks of all computation tasks within the
+     * union communicator. */
+    int *compranks;
+
+    /** Controls handling errors. */
+    int error_handler;
+
+    /** The rearranger decides which parts of a distributed array are
+     * handled by which IO tasks. */
+    int default_rearranger;
+
+    /** True if asynchronous interface is in use. */
+    bool async;
+
+    /** True if this task is a member of the IO communicator. */
+    bool ioproc;
+
+    /** True if this task is a member of a computation
+     * communicator. */
+    bool compproc;
+
+    /** MPI Info object. */
+    MPI_Info info;
+
+    /** Index of this component in the list of components. */
+    int comp_idx;
+
+    /** Rearranger options. */
+    rearr_opt_t rearr_opts;
+
+    /** Pointer to the next iosystem_desc_t in the list. */
+    struct iosystem_desc_t *next;
+} iosystem_desc_t;
+
+/**
+ * The multi buffer holds data from one or more variables. Data are
+ * accumulated in the multi-buffer.
+ */
+typedef struct wmulti_buffer
+{
+    /** The ID that describes the decomposition, as returned from
+     * PIOc_Init_Decomp().  */
+    int ioid;
+
+    /** Non-zero if this is a buffer for a record var. */
+    int recordvar;
+
+    /** Number of arrays of data in the multibuffer. Each array had
+     * data for one var or record. When multibuffer is flushed, all
+     * arrays are written and num_arrays returns to zero. */
+    int num_arrays;
+
+    /** Size of this variables data on local task. All vars in the
+     * multi-buffer have the same size. */
+    int arraylen;
+
+    /** Array of varids. */
+    int *vid;
+
+    /** An array of current record numbers, for record vars. One
+     * element per variable. */
+    int *frame;
+
+    /** Array of fill values used for each var. */
+    void *fillvalue;
+
+    /** Pointer to the data. */
+    void *data;
+
+    /** Pointer to the next multi-buffer in the list. */
+    struct wmulti_buffer *next;
+} wmulti_buffer;
+
+/**
+ * File descriptor structure.
+ *
+ * This structure holds information associated with each open file
+ */
+typedef struct file_desc_t
+{
+    /** The IO system ID used to open this file. */
+    iosystem_desc_t *iosystem;
+
+    /** The ncid returned for this file by the underlying library
+     * (netcdf or pnetcdf). */
+    int fh;
+
+    /** The ncid that will be returned to the user. */
+    int pio_ncid;
+
+    /** The PIO_TYPE value that was used to open this file. */
+    int iotype;
+
+    /** List of variables in this file. */
+    struct var_desc_t varlist[PIO_MAX_VARS];
+
+    /** ??? */
+    int mode;
+
+    /** The wmulti_buffer is used to aggregate multiple variables with
+     * the same communication pattern prior to a write. */
+    struct wmulti_buffer buffer;
+
+    /** Pointer to the next file_desc_t in the list of open files. */
+    struct file_desc_t *next;
+
+    /** True if this task should participate in IO (only true for one
+     * task with netcdf serial files. */
+    int do_io;
+} file_desc_t;
+
+/**
+ * These are the supported methods of reading/writing netCDF
+ * files. (Not all methods can be used with all netCDF files.)
+ */
+enum PIO_IOTYPE
+{
+    /** Parallel Netcdf  (parallel) */
+    PIO_IOTYPE_PNETCDF = 1,
+
+    /** Netcdf3 Classic format (serial) */
+    PIO_IOTYPE_NETCDF = 2,
+
+    /**  NetCDF4 (HDF5) compressed format (serial) */
+    PIO_IOTYPE_NETCDF4C = 3,
+
+    /** NetCDF4 (HDF5) parallel */
+    PIO_IOTYPE_NETCDF4P = 4
+};
+
+/**
+ * These are the supported output data rearrangement methods.
+ */
+enum PIO_REARRANGERS
+{
+    /** Box rearranger. */
+    PIO_REARR_BOX = 1,
+
+    /** Subset rearranger. */
+    PIO_REARR_SUBSET = 2
+};
+
+/**
+ * These are the supported error handlers.
+ */
+enum PIO_ERROR_HANDLERS
+{
+    /** Errors cause abort. */
+    PIO_INTERNAL_ERROR = (-51),
+
+    /** Error codes are broadcast to all tasks. */
+    PIO_BCAST_ERROR = (-52),
+
+    /** Errors are returned to caller with no internal action. */
+    PIO_RETURN_ERROR = (-53)
+};
+
+#if defined( _PNETCDF) || defined(_NETCDF)
+
+#define PIO_GLOBAL NC_GLOBAL
+#define PIO_UNLIMITED NC_UNLIMITED
+
+/* NetCDF types. */
+#define PIO_BYTE   NC_BYTE
+#define PIO_CHAR   NC_CHAR
+#define PIO_SHORT  NC_SHORT
+#define PIO_INT    NC_INT
+#define PIO_FLOAT  NC_FLOAT
+#define PIO_REAL   NC_FLOAT
+#define PIO_DOUBLE NC_DOUBLE
+#define PIO_UBYTE  NC_UBYTE
+#define PIO_USHORT NC_USHORT
+#define PIO_UINT   NC_UINT
+#define PIO_INT64  NC_INT64
+#define PIO_UINT64 NC_UINT64
+#define PIO_STRING NC_STRING
+
+/* NetCDF flags. */
+#define PIO_WRITE  NC_WRITE
+#define PIO_NOWRITE  NC_NOWRITE
+#define PIO_CLOBBER NC_CLOBBER
+#define PIO_NOCLOBBER NC_NOCLOBBER
+#define PIO_FILL NC_FILL
+#define PIO_NOFILL NC_NOFILL
+#define PIO_MAX_NAME NC_MAX_NAME
+#define PIO_MAX_VAR_DIMS NC_MAX_VAR_DIMS
+#define PIO_64BIT_OFFSET NC_64BIT_OFFSET
+
+/** NC_64BIT_DATA This is a problem - need to define directly instead
+ * of using include file. */
+#define PIO_64BIT_DATA 0x0010
+
+/** Define the netCDF-based error codes. */
+#define PIO_NOERR  NC_NOERR
+#define PIO_EBADID NC_EBADID
+#define PIO_ENFILE NC_ENFILE
+#define PIO_EEXIST NC_EEXIST
+#define PIO_EINVAL NC_EINVAL
+#define PIO_EPERM NC_EPERM
+#define PIO_ENOTINDEFINE NC_ENOTINDEFINE
+#define PIO_EINDEFINE NC_EINDEFINE
+#define PIO_EINVALCOORDS NC_EINVALCOORDS
+#define PIO_EMAXDIMS NC_EMAXDIMS
+#define PIO_ENAMEINUSE NC_ENAMEINUSE
+#define PIO_ENOTATT NC_ENOTATT
+#define PIO_EMAXATTS NC_EMAXATTS
+#define PIO_EBADTYPE NC_EBADTYPE
+#define PIO_EBADDIM NC_EBADDIM
+#define PIO_EUNLIMPOS NC_EUNLIMPOS
+#define PIO_EMAXVARS NC_EMAXVARS
+#define PIO_ENOTVAR NC_ENOTVAR
+#define PIO_EGLOBAL NC_EGLOBAL
+#define PIO_ENOTNC NC_ENOTNC
+#define PIO_ESTS NC_ESTS
+#define PIO_EMAXNAME NC_EMAXNAME
+#define PIO_EUNLIMIT NC_EUNLIMIT
+#define PIO_ENORECVARS NC_ENORECVARS
+#define PIO_ECHAR NC_ECHAR
+#define PIO_EEDGE NC_EEDGE
+#define PIO_ESTRIDE NC_ESTRIDE
+#define PIO_EBADNAME NC_EBADNAME
+#define PIO_ERANGE NC_ERANGE
+#define PIO_ENOMEM NC_ENOMEM
+#define PIO_EVARSIZE NC_EVARSIZE
+#define PIO_EDIMSIZE NC_EDIMSIZE
+#define PIO_ETRUNC NC_ETRUNC
+#define PIO_EAXISTYPE NC_EAXISTYPE
+#define PIO_EDAP NC_EDAP
+#define PIO_ECURL NC_ECURL
+#define PIO_EIO NC_EIO
+#define PIO_ENODATA NC_ENODATA
+#define PIO_EDAPSVC NC_EDAPSVC
+#define PIO_EDAS NC_EDAS
+#define PIO_EDDS NC_EDDS
+#define PIO_EDATADDS NC_EDATADDS
+#define PIO_EDAPURL NC_EDAPURL
+#define PIO_EDAPCONSTRAINT NC_EDAPCONSTRAINT
+#define PIO_ETRANSLATION NC_ETRANSLATION
+#define PIO_EHDFERR NC_EHDFERR
+#define PIO_ECANTREAD NC_ECANTREAD
+#define PIO_ECANTWRITE NC_ECANTWRITE
+#define PIO_ECANTCREATE NC_ECANTCREATE
+#define PIO_EFILEMETA NC_EFILEMETA
+#define PIO_EDIMMETA NC_EDIMMETA
+#define PIO_EATTMETA NC_EATTMETA
+#define PIO_EVARMETA NC_EVARMETA
+#define PIO_ENOCOMPOUND NC_ENOCOMPOUND
+#define PIO_EATTEXISTS NC_EATTEXISTS
+#define PIO_ENOTNC4 NC_ENOTNC4
+#define PIO_ESTRICTNC3 NC_ESTRICTNC3
+#define PIO_ENOTNC3 NC_ENOTNC3
+#define PIO_ENOPAR NC_ENOPAR
+#define PIO_EPARINIT NC_EPARINIT
+#define PIO_EBADGRPID NC_EBADGRPID
+#define PIO_EBADTYPID NC_EBADTYPID
+#define PIO_ETYPDEFINED NC_ETYPDEFINED
+#define PIO_EBADFIELD NC_EBADFIELD
+#define PIO_EBADCLASS NC_EBADCLASS
+#define PIO_EMAPTYPE NC_EMAPTYPE
+#define PIO_ELATEFILL NC_ELATEFILL
+#define PIO_ELATEDEF NC_ELATEDEF
+#define PIO_EDIMSCALE NC_EDIMSCALE
+#define PIO_ENOGRP NC_ENOGRP
+#define PIO_ESTORAGE NC_ESTORAGE
+#define PIO_EBADCHUNK NC_EBADCHUNK
+#define PIO_ENOTBUILT NC_ENOTBUILT
+#define PIO_EDISKLESS NC_EDISKLESS
+
+/* These are the netCDF default fill values. */
+#define PIO_FILL_BYTE NC_FILL_BYTE
+#define PIO_FILL_CHAR NC_FILL_CHAR
+#define PIO_FILL_SHORT NC_FILL_SHORT
+#define PIO_FILL_INT NC_FILL_INT
+#define PIO_FILL_FLOAT NC_FILL_FLOAT
+#define PIO_FILL_DOUBLE NC_FILL_DOUBLE
+#define PIO_FILL_UBYTE NC_FILL_UBYTE
+#define PIO_FILL_USHORT NC_FILL_USHORT
+#define PIO_FILL_UINT NC_FILL_UINT
+#define PIO_FILL_INT64 NC_FILL_INT64
+#define PIO_FILL_UINT64 NC_FILL_UINT64
+#endif /*  defined( _PNETCDF) || defined(_NETCDF) */
+
+/** Define the extra error codes for the parallel-netcdf library. */
+#ifdef _PNETCDF
+#define PIO_EINDEP  NC_EINDEP
+#else  /* _PNETCDF */
+#define PIO_EINDEP  (-203)
+#endif /* _PNETCDF */
+
+/** Define error codes for PIO. */
+#define PIO_FIRST_ERROR_CODE (-500)
+#define PIO_EBADIOTYPE  (-500)
+
+/** ??? */
+#define PIO_REQ_NULL (NC_REQ_NULL-1)
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+    /* Error handling. */
+    int PIOc_strerror(int pioerr, char *errstr);
+    int PIOc_set_log_level(int level);
+
+    /* Decomposition. */
+
+    /* Init decomposition with 1-based compmap array. */
+    int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
+                        const PIO_Offset *compmap, int *ioidp, const int *rearr,
+                        const PIO_Offset *iostart, const PIO_Offset *iocount);
+    int PIOc_InitDecomp_bc(int iosysid, int basetype, int ndims, const int *gdimlen,
+                           const long int *start, const long int *count, int *ioidp);
+
+    /* Init decomposition with 0-based compmap array. */
+    int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
+                         const PIO_Offset *compmap, int *ioidp, int rearranger,
+                         const PIO_Offset *iostart, const PIO_Offset *iocount);
+
+    /* Free resources associated with a decomposition. */
+    int PIOc_freedecomp(int iosysid, int ioid);
+    
+    int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
+                     PIO_Offset **map, MPI_Comm comm);
+    int PIOc_readmap_from_f90(const char *file,int *ndims, int **gdims, PIO_Offset *maplen,
+                              PIO_Offset **map, int f90_comm);
+    int PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset maplen,
+                      PIO_Offset *map, MPI_Comm comm);
+    int PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims,
+                               PIO_Offset maplen, const PIO_Offset *map, int f90_comm);
+
+    /* Write a decomposition file. */
+    int PIOc_write_decomp(const char *file, int iosysid, int ioid, MPI_Comm comm);
+
+    /* Write a decomposition file using netCDF. */
+    int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
+                             char *title, char *history, int fortran_order);
+
+    /* Read a netCDF decomposition file. */
+    int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioid, MPI_Comm comm,
+                            int pio_type, char *title, char *history, int *fortran_order);
+
+    /* Initializing IO system for async. */
+    int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list, int component_count,
+                        int *num_procs_per_comp, int **proc_list, MPI_Comm *io_comm, MPI_Comm *comp_comm,
+                        int rearranger, int *iosysidp);
+    
+    int PIOc_Init_Intercomm(int component_count, MPI_Comm peer_comm, MPI_Comm *comp_comms,
+                            MPI_Comm io_comm, int *iosysidp);
+    int PIOc_get_numiotasks(int iosysid, int *numiotasks);
+    int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base, int rearr,
+                            int *iosysidp);
+    int PIOc_finalize(int iosysid);
+
+    /* Set error handling for entire io system. */
+    int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method);
+
+    /* Set error handling for entire io system. */
+    int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method);
+
+    int PIOc_iam_iotask(int iosysid, bool *ioproc);
+    int PIOc_iotask_rank(int iosysid, int *iorank);
+    int PIOc_iosystem_is_active(int iosysid, bool *active);
+    int PIOc_iotype_available(int iotype);
+    int PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd,
+                            bool enable_hs_c2i, bool enable_isend_c2i,
+                            int max_pend_req_c2i,
+                            bool enable_hs_i2c, bool enable_isend_i2c,
+                            int max_pend_req_i2c);
+    /* Distributed data. */
+    int PIOc_advanceframe(int ncid, int varid);
+    int PIOc_setframe(int ncid, int varid, int frame);
+    int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
+                          void *fillvalue);
+    int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars, PIO_Offset arraylen,
+                                void *array, const int *frame, void **fillvalue, bool flushtodisk);
+    int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array);
+    int PIOc_get_local_array_size(int ioid);
+
+    /* Handling files. */
+    int PIOc_redef(int ncid);
+    int PIOc_enddef(int ncid);
+    int PIOc_sync(int ncid);
+    int PIOc_deletefile(int iosysid, const char *filename);
+    int PIOc_createfile(int iosysid, int *ncidp,  int *iotype, const char *fname, int mode);
+    int PIOc_create(int iosysid, const char *path, int cmode, int *ncidp);
+    int PIOc_openfile(int iosysid, int *ncidp, int *iotype, const char *fname, int mode);
+    int PIOc_open(int iosysid, const char *path, int mode, int *ncidp);
+    int PIOc_closefile(int ncid);
+    int PIOc_inq_format(int ncid, int *formatp);
+    int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp);
+    int PIOc_inq_ndims(int ncid, int *ndimsp);
+    int PIOc_inq_nvars(int ncid, int *nvarsp);
+    int PIOc_inq_natts(int ncid, int *ngattsp);
+    int PIOc_inq_unlimdim(int ncid, int *unlimdimidp);
+    int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep);
+    int PIOc_set_blocksize(int newblocksize);
+    int PIOc_File_is_Open(int ncid);
+
+    /* Set the IO node data buffer size limit. */
+    PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit);
+
+    /* Set the error hanlding for a file. */
+    int PIOc_Set_File_Error_Handling(int ncid, int method);
+
+    int PIOc_set_hint(int iosysid, const char *hint, const char *hintval);
+    int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems,
+                             float preemption);
+    int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp,
+                             float *preemptionp);
+
+    /* Dimensions. */
+    int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp);
+    int PIOc_inq_dimid(int ncid, const char *name, int *idp);
+    int PIOc_inq_dimname(int ncid, int dimid, char *name);
+    int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp);
+    int PIOc_rename_dim(int ncid, int dimid, const char *name);
+    int PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp);
+
+    /* Variables. */
+    int PIOc_inq_varid(int ncid, const char *name, int *varidp);
+    int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp, int *dimidsp,
+                     int *nattsp);
+    int PIOc_inq_varname(int ncid, int varid, char *name);
+    int PIOc_inq_vartype(int ncid, int varid, nc_type *xtypep);
+    int PIOc_inq_varndims(int ncid, int varid, int *ndimsp);
+    int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp);
+    int PIOc_inq_varnatts(int ncid, int varid, int *nattsp);
+    int PIOc_def_var(int ncid, const char *name, nc_type xtype,  int ndims,
+                     const int *dimidsp, int *varidp);
+    int PIOc_set_fill(int ncid, int fillmode, int *old_modep);
+    int PIOc_def_var_fill(int ncid, int varid, int no_fill, const void *fill_value);
+    int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep);
+    int PIOc_rename_var(int ncid, int varid, const char *name);
+
+    /* These variable settings only apply to netCDF-4 files. */
+    int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
+                             int deflate_level);
+    int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
+                             int *deflate_levelp);
+    int PIOc_inq_var_szip(int ncid, int varid, int *options_maskp, int *pixels_per_blockp);
+    int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp);
+    int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp);
+    int PIOc_def_var_endian(int ncid, int varid, int endian);
+    int PIOc_inq_var_endian(int ncid, int varid, int *endianp);
+    int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
+                                 float preemption);
+    int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
+                                 float *preemptionp);
+
+    /* Attributes - misc. */
+    int PIOc_rename_att(int ncid, int varid, const char *name, const char *newname);
+    int PIOc_del_att(int ncid, int varid, const char *name);
+
+    /* Attributes - inquiry functions. */
+    int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep,
+                     PIO_Offset *lenp);
+    int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp);
+    int PIOc_inq_attlen(int ncid, int varid, const char *name, PIO_Offset *lenp);
+    int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep);
+    int PIOc_inq_attname(int ncid, int varid, int attnum, char *name);
+
+    /* Attributes - writing. */
+    int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len, const void *op);
+    int PIOc_put_att_text(int ncid, int varid, const char *name, PIO_Offset len, const char *op);
+    int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                           const signed char *op);
+    int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                           const short *op);
+    int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                         const int *op);
+    int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                          const long *op);
+    int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                           const float *op);
+    int PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                            const double *op);
+    int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                           const unsigned char *op);
+    int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                            const unsigned short *op);
+    int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                          const unsigned int *op);
+    int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype, PIO_Offset len,
+                              const long long *op);
+    int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype,
+                               PIO_Offset len, const unsigned long long *op);
+
+    /* Attributes - reading. */
+    int PIOc_get_att(int ncid, int varid, const char *name, void *ip);
+    int PIOc_get_att_text(int ncid, int varid, const char *name, char *ip);
+    int PIOc_get_att_schar(int ncid, int varid, const char *name, signed char *ip);
+    int PIOc_get_att_short(int ncid, int varid, const char *name, short *ip);
+    int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip);
+    int PIOc_get_att_long(int ncid, int varid, const char *name, long *ip);
+    int PIOc_get_att_float(int ncid, int varid, const char *name, float *ip);
+    int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip);
+    int PIOc_get_att_uchar(int ncid, int varid, const char *name, unsigned char *ip);
+    int PIOc_get_att_ushort(int ncid, int varid, const char *name, unsigned short *ip);
+    int PIOc_get_att_uint(int ncid, int varid, const char *name, unsigned int *ip);
+    int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip);
+    int PIOc_get_att_ulonglong(int ncid, int varid, const char *name, unsigned long long *ip);
+
+    /* Data reads - var. */
+    int PIOc_get_var(int ncid, int varid, void *buf);
+    int PIOc_get_var_text(int ncid, int varid, char *buf);
+    int PIOc_get_var_schar(int ncid, int varid, signed char *buf);
+    int PIOc_get_var_short(int ncid, int varid, short *buf);
+    int PIOc_get_var_int(int ncid, int varid, int *buf);
+    int PIOc_get_var_long(int ncid, int varid, long *buf);
+    int PIOc_get_var_float(int ncid, int varid, float *buf);
+    int PIOc_get_var_double(int ncid, int varid, double *buf);
+    int PIOc_get_var_uchar(int ncid, int varid, unsigned char *buf);
+    int PIOc_get_var_ushort(int ncid, int varid, unsigned short *buf);
+    int PIOc_get_var_uint(int ncid, int varid, unsigned int *buf);
+    int PIOc_get_var_longlong(int ncid, int varid, long long *buf);
+    int PIOc_get_var_ulonglong(int ncid, int varid, unsigned long long *buf);
+
+    /* Data writes - var. */
+    int PIOc_put_var(int ncid, int varid, const void *buf);
+    int PIOc_put_var_text(int ncid, int varid, const char *op);
+    int PIOc_put_var_schar(int ncid, int varid, const signed char *op);
+    int PIOc_put_var_short(int ncid, int varid, const short *op);
+    int PIOc_put_var_int(int ncid, int varid, const int *op);
+    int PIOc_put_var_long(int ncid, int varid, const long *op);
+    int PIOc_put_var_float(int ncid, int varid, const float *op);
+    int PIOc_put_var_double(int ncid, int varid, const double *op);
+    int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op);
+    int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op);
+    int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op);
+    int PIOc_put_var_longlong(int ncid, int varid, const long long *op);
+    int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op);
+
+    /* Data reads - var1. */
+    int PIOc_get_var1(int ncid, int varid, const PIO_Offset *index, void *buf);
+    int PIOc_get_var1_text(int ncid, int varid, const PIO_Offset *index, char *buf);
+    int PIOc_get_var1_schar(int ncid, int varid, const PIO_Offset *index, signed char *buf);
+    int PIOc_get_var1_short(int ncid, int varid, const PIO_Offset *index, short *buf);
+    int PIOc_get_var1_int(int ncid, int varid, const PIO_Offset *index, int *buf);
+    int PIOc_get_var1_long(int ncid, int varid, const PIO_Offset *index, long *buf);
+    int PIOc_get_var1_float(int ncid, int varid, const PIO_Offset *index, float *buf);
+    int PIOc_get_var1_double(int ncid, int varid, const PIO_Offset *index, double *buf);
+    int PIOc_get_var1_uchar(int ncid, int varid, const PIO_Offset *index, unsigned char *buf);
+    int PIOc_get_var1_ushort(int ncid, int varid, const PIO_Offset *index, unsigned short *buf);
+    int PIOc_get_var1_uint(int ncid, int varid, const PIO_Offset *index, unsigned int *buf);
+    int PIOc_get_var1_longlong(int ncid, int varid, const PIO_Offset *index, long long *buf);
+    int PIOc_get_var1_ulonglong(int ncid, int varid, const PIO_Offset *index, unsigned long long *buf);
+
+    /* Data writes - var1. */
+    int PIOc_put_var1(int ncid, int varid, const PIO_Offset *index, const void *buf);
+    int PIOc_put_var1_text(int ncid, int varid, const PIO_Offset *index, const char *op);
+    int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset *index, const signed char *op);
+    int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset *index, const short *op);
+    int PIOc_put_var1_int(int ncid, int varid, const PIO_Offset *index, const int *op);
+    int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset *index, const long *ip);
+    int PIOc_put_var1_float(int ncid, int varid, const PIO_Offset *index, const float *op);
+    int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset *index, const double *op);
+    int PIOc_put_var1_uchar(int ncid, int varid, const PIO_Offset *index,
+                            const unsigned char *op);
+    int PIOc_put_var1_ushort(int ncid, int varid, const PIO_Offset *index,
+                             const unsigned short *op);
+    int PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset *index,
+                           const unsigned int *op);
+    int PIOc_put_var1_longlong(int ncid, int varid, const PIO_Offset *index, const long long *op);
+    int PIOc_put_var1_ulonglong(int ncid, int varid, const PIO_Offset *index,
+                                const unsigned long long *op);
+
+    /* Data reads - vara. */
+    int PIOc_get_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count, void *buf);
+    int PIOc_get_vara_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           char *buf);
+    int PIOc_get_vara_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            signed char *buf);
+    int PIOc_get_vara_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            short *buf);
+    int PIOc_get_vara_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                          int *buf);
+    int PIOc_get_vara_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            float *buf);
+    int PIOc_get_vara_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           long *buf);
+    int PIOc_get_vara_double(int ncid, int varid, const PIO_Offset *start,
+                             const PIO_Offset *count, double *buf);
+    int PIOc_get_vara_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            unsigned char *buf);
+    int PIOc_get_vara_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             unsigned short *buf);
+    int PIOc_get_vara_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           unsigned int *buf);
+    int PIOc_get_vara_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                               long long *buf);
+    int PIOc_get_vara_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                                unsigned long long *buf);
+
+    /* Data writes - vara. */
+    int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const void *buf);
+    int PIOc_put_vara_text(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const char *op);
+    int PIOc_put_vara_schar(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const signed char *op);
+    int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const short *op);
+    int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                          const int *op);
+    int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const long *op);
+    int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const float *op);
+    int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const double *op);
+    int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const unsigned char *op);
+    int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start,
+                             const PIO_Offset *count, const unsigned short *op);
+    int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const unsigned int *op);
+    int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start,
+                               const PIO_Offset *count, const long long *op);
+    int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                                const PIO_Offset *count, const unsigned long long *op);
+
+    /* Data reads - vars. */
+    int PIOc_get_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, void *buf);
+    int PIOc_get_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, char *buf);
+    int PIOc_get_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, signed char *buf);
+    int PIOc_get_vars_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, short *buf);
+    int PIOc_get_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                          const PIO_Offset *stride, int *buf);
+    int PIOc_get_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, long *buf);
+    int PIOc_get_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, float *buf);
+    int PIOc_get_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, double *buf);
+    int PIOc_get_vars_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, unsigned char *buf);
+    int PIOc_get_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, unsigned short *buf);
+    int PIOc_get_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, unsigned int *buf);
+    int PIOc_get_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                               const PIO_Offset *stride, long long *buf);
+    int PIOc_get_vars_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                                const PIO_Offset *count, const PIO_Offset *stride,
+                                unsigned long long *buf);
+
+    /* Data writes - vars. */
+    int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, const void *buf);
+    int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const PIO_Offset *stride, const char *op);
+    int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const PIO_Offset *stride,
+                            const signed char *op);
+    int PIOc_put_vars_short(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const PIO_Offset *stride, const short *op);
+    int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                          const PIO_Offset *stride, const int *op);
+    int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const float *op);
+    int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start,
+                             const PIO_Offset *count, const PIO_Offset *stride, const double *op);
+    int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const long *op);
+    int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const unsigned char *op);
+    int PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, const unsigned short *op);
+    int PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const unsigned int *op);
+    int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                               const PIO_Offset *stride, const long long *op);
+    int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                                const PIO_Offset *count, const PIO_Offset *stride,
+                                const unsigned long long *op);
+
+    /* Varm functions are deprecated and should be used with extreme
+     * caution or not at all. Varm functions are not supported in
+     * async mode. */
+    int PIOc_put_varm(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, const PIO_Offset *imap, const void *buf,
+                      PIO_Offset bufcount, MPI_Datatype buftype);
+    int PIOc_get_varm_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap, signed char *buf);
+    int PIOc_put_varm_uchar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap,
+                            const unsigned char *op);
+    int PIOc_put_varm_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap, const short *op);
+    int PIOc_get_varm_short(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap, short *buf);
+    int PIOc_get_varm_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                                const PIO_Offset *stride, const PIO_Offset *imap, unsigned long long *buf);
+    int PIOc_get_varm_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, const PIO_Offset *imap, unsigned short *buf);
+    int PIOc_get_varm_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                               const PIO_Offset *stride, const PIO_Offset *imap, long long *buf);
+    int PIOc_put_varm_text(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const PIO_Offset *stride,
+                           const PIO_Offset *imap, const char *op);
+    int PIOc_put_varm_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, const PIO_Offset *imap, const unsigned short *op);
+    int PIOc_put_varm_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                                const PIO_Offset *count, const PIO_Offset *stride,
+                                const PIO_Offset *imap, const unsigned long long *op);
+    int PIOc_put_varm_int(int ncid, int varid, const PIO_Offset *start,
+                          const PIO_Offset *count, const PIO_Offset *stride,
+                          const PIO_Offset *imap, const int *op);
+    int PIOc_put_varm_float(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const PIO_Offset *stride,
+                            const PIO_Offset *imap, const float *op);
+    int PIOc_put_varm_long(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const PIO_Offset *stride,
+                           const PIO_Offset *imap, const long *op);
+    int PIOc_put_varm_uint(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const PIO_Offset *stride,
+                           const PIO_Offset *imap, const unsigned int *op);
+    int PIOc_put_varm_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, const PIO_Offset *imap, const double *op);
+    int PIOc_put_varm_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap, const signed char *op);
+    int PIOc_put_varm_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                               const PIO_Offset *stride, const PIO_Offset *imap, const long long *op);
+    int PIOc_get_varm_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                             const PIO_Offset *stride, const PIO_Offset *imap, double *buf);
+    int PIOc_get_varm_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const PIO_Offset *imap, char *buf);
+    int PIOc_get_varm_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                          const PIO_Offset *stride, const PIO_Offset *imap, int *buf);
+    int PIOc_get_varm_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const PIO_Offset *imap, unsigned int *buf);
+    int PIOc_get_varm(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, const PIO_Offset *imap, void *buf,
+                      PIO_Offset bufcount, MPI_Datatype buftype);
+    int PIOc_get_varm_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const PIO_Offset *imap, float *buf);
+    int PIOc_get_varm_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const PIO_Offset *imap, long *buf);
+#if defined(__cplusplus)
+}
+#endif
+
+#endif  // _PIO_H_
diff --git a/externals/pio2/src/clib/pio_c_get_template.c b/src/externals/pio2/src/clib/pio_c_get_template.c
similarity index 100%
rename from externals/pio2/src/clib/pio_c_get_template.c
rename to src/externals/pio2/src/clib/pio_c_get_template.c
diff --git a/externals/pio2/src/clib/pio_c_put_template.c b/src/externals/pio2/src/clib/pio_c_put_template.c
similarity index 100%
rename from externals/pio2/src/clib/pio_c_put_template.c
rename to src/externals/pio2/src/clib/pio_c_put_template.c
diff --git a/externals/pio2/src/clib/pio_c_template.c b/src/externals/pio2/src/clib/pio_c_template.c
similarity index 100%
rename from externals/pio2/src/clib/pio_c_template.c
rename to src/externals/pio2/src/clib/pio_c_template.c
diff --git a/src/externals/pio2/src/clib/pio_darray.c b/src/externals/pio2/src/clib/pio_darray.c
new file mode 100644
index 00000000000..1d6f15c7d30
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_darray.c
@@ -0,0 +1,690 @@
+/** @file
+ *
+ * Public functions that read and write distributed arrays in PIO.
+ *
+ * When arrays are distributed, each processor holds some of the
+ * array. Only by combining the distributed arrays from all processor
+ * can the full array be obtained.
+ *
+ * @author Jim Edwards
+ */
+#include 
+#include 
+#include 
+
+/* 10MB default limit. */
+PIO_Offset pio_buffer_size_limit = 10485760;
+
+/* Global buffer pool pointer. */
+void *CN_bpool = NULL;
+
+/* Maximum buffer usage. */
+PIO_Offset maxusage = 0;
+
+/**
+ * Set the PIO IO node data buffer size limit.
+ *
+ * The pio_buffer_size_limit will only apply to files opened after
+ * the setting is changed.
+ *
+ * @param limit the size of the buffer on the IO nodes
+ * @return The previous limit setting.
+ */
+PIO_Offset PIOc_set_buffer_size_limit(PIO_Offset limit)
+{
+    PIO_Offset oldsize = pio_buffer_size_limit;
+
+    /* If the user passed a valid size, use it. */
+    if (limit > 0)
+        pio_buffer_size_limit = limit;
+
+    return oldsize;
+}
+
+/**
+ * Write one or more arrays with the same IO decomposition to the
+ * file.
+ *
+ * This funciton is similar to PIOc_write_darray(), but allows the
+ * caller to use their own data buffering (instead of using the
+ * buffering implemented in PIOc_write_darray()).
+ *
+ * When the user calls PIOc_write_darray() one or more times, then
+ * PIO_write_darray_multi() will be called when the buffer is flushed.
+ *
+ * Internally, this function will:
+ * 
      
+ * 
    • Find info about file, decomposition, and variable.
+ * 
    • Do a special flush for pnetcdf if needed.
+ * 
    • Allocates a buffer big enough to hold all the data in the
+ * multi-buffer, for all tasks.
+ * 
    • Calls rearrange_comp2io() to move data from compute to IO
+ * tasks.
+ * 
    • For parallel iotypes (pnetcdf and netCDF-4 parallel) call
+ * pio_write_darray_multi_nc().
+ * 
    • For serial iotypes (netcdf classic and netCDF-4 serial) call
+ * write_darray_multi_serial().
+ * 
    • For subset rearranger, create holegrid to write missing
+ * data. Then call pio_write_darray_multi_nc() or
+ * write_darray_multi_serial() to write the holegrid.
+ * 
    • Special buffer flush for pnetcdf.
+ * 
    
+ *
+ * @param ncid identifies the netCDF file.
+ * @param varids an array of length nvars containing the variable ids to
+ * be written.
+ * @param ioid the I/O description ID as passed back by
+ * PIOc_InitDecomp().
+ * @param nvars the number of variables to be written with this
+ * call.
+ * @param arraylen the length of the array to be written. This is the
+ * length of the distrubited array. That is, the length of the portion
+ * of the data that is on the processor. The same arraylen is used for
+ * all variables in the call.
+ * @param array pointer to the data to be written. This is a pointer
+ * to an array of arrays with the distributed portion of the array
+ * that is on this processor. There are nvars arrays of data, and each
+ * array of data contains one record worth of data for that variable.
+ * @param frame an array of length nvars with the frame or record
+ * dimension for each of the nvars variables in IOBUF. NULL if this
+ * iodesc contains non-record vars.
+ * @param fillvalue pointer an array (of length nvars) of pointers to
+ * the fill value to be used for missing data.
+ * @param flushtodisk non-zero to cause buffers to be flushed to disk.
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ */
+int PIOc_write_darray_multi(int ncid, const int *varids, int ioid, int nvars,
+                            PIO_Offset arraylen, void *array, const int *frame,
+                            void **fillvalue, bool flushtodisk)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    io_desc_t *iodesc;     /* Pointer to IO description information. */
+    int rlen;              /* total data buffer size. */
+    var_desc_t *vdesc0;    /* pointer to var_desc structure for each var. */
+    int ierr;              /* Return code. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Check inputs. */
+    if (nvars <= 0 || !varids)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+    for (int v = 0; v < nvars; v++)
+        if (varids[v] < 0 || varids[v] > PIO_MAX_VARS)
+            return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_write_darray_multi ncid = %d ioid = %d nvars = %d arraylen = %ld "
+         "flushtodisk = %d",
+         ncid, ioid, nvars, arraylen, flushtodisk));
+
+    /* Check that we can write to this file. */
+    if (!(file->mode & PIO_WRITE))
+        return pio_err(ios, file, PIO_EPERM, __FILE__, __LINE__);
+
+    /* Get iodesc. */
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__);
+    pioassert(iodesc->rearranger == PIO_REARR_BOX || iodesc->rearranger == PIO_REARR_SUBSET,
+              "unknown rearranger", __FILE__, __LINE__);
+
+    /* Get a pointer to the variable info for the first variable. */
+    vdesc0 = &file->varlist[varids[0]];
+
+    /* if the buffer is already in use in pnetcdf we need to flush first */
+    if (file->iotype == PIO_IOTYPE_PNETCDF && vdesc0->iobuf)
+	flush_output_buffer(file, 1, 0);
+
+    pioassert(!vdesc0->iobuf, "buffer overwrite",__FILE__, __LINE__);
+
+    /* Determine total size of aggregated data (all vars/records).
+     * For netcdf serial writes we collect the data on io nodes and
+     * then move that data one node at a time to the io master node
+     * and write (or read). The buffer size on io task 0 must be as
+     * large as the largest used to accommodate this serial io
+     * method.  */
+    rlen = 0;
+    if (iodesc->llen > 0)
+        rlen = iodesc->maxiobuflen * nvars;
+
+    /* Allocate iobuf. */
+    if (rlen > 0)
+    {
+        /* Allocate memory for the buffer for all vars/records. */
+        if (!(vdesc0->iobuf = bget(iodesc->basetype_size * (size_t)rlen)))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        LOG((3, "allocated %lld bytes for variable buffer", (size_t)rlen * iodesc->basetype_size));
+
+        /* If fill values are desired, and we're using the BOX
+         * rearranger, insert fill values. */
+        if (iodesc->needsfill && iodesc->rearranger == PIO_REARR_BOX)
+            for (int nv = 0; nv < nvars; nv++)
+                for (int i = 0; i < iodesc->maxiobuflen; i++)
+                    memcpy(&((char *)vdesc0->iobuf)[iodesc->basetype_size * (i + nv * iodesc->maxiobuflen)],
+                           &((char *)fillvalue)[nv * iodesc->basetype_size], iodesc->basetype_size);
+    }
+    else if (file->iotype == PIO_IOTYPE_PNETCDF && ios->ioproc)
+    {
+	/* this assures that iobuf is allocated on all iotasks thus
+	 assuring that the flush_output_buffer call above is called
+	 collectively (from all iotasks) */
+        if (!(vdesc0->iobuf = bget(1)))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        LOG((3, "allocated token for variable buffer"));
+    }
+
+    /* Move data from compute to IO tasks. */
+    if ((ierr = rearrange_comp2io(ios, iodesc, array, vdesc0->iobuf, nvars)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Write the darray based on the iotype. */
+    LOG((2, "about to write darray for iotype = %d", file->iotype));
+    switch (file->iotype)
+    {
+    case PIO_IOTYPE_NETCDF4P:
+    case PIO_IOTYPE_PNETCDF:
+        if ((ierr = pio_write_darray_multi_nc(file, nvars, varids, iodesc->ndims, iodesc->basetype,
+                                              iodesc->maxregions, iodesc->firstregion, iodesc->llen,
+                                              iodesc->num_aiotasks, vdesc0->iobuf, frame)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+        break;
+    case PIO_IOTYPE_NETCDF4C:
+    case PIO_IOTYPE_NETCDF:
+        if ((ierr = write_darray_multi_serial(file, nvars, varids, iodesc, 0, frame)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+        break;
+    default:
+        return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__);
+    }
+
+    /* For PNETCDF the iobuf is freed in flush_output_buffer() */
+    if (file->iotype != PIO_IOTYPE_PNETCDF)
+    {
+        /* Release resources. */
+        if (vdesc0->iobuf)
+        {
+	    LOG((3,"freeing variable buffer in pio_darray"));
+            brel(vdesc0->iobuf);
+            vdesc0->iobuf = NULL;
+        }
+    }
+
+    /* The box rearranger will always have data (it could be fill
+     * data) to fill the entire array - that is the aggregate start
+     * and count values will completely describe one unlimited
+     * dimension unit of the array. For the subset method this is not
+     * necessarily the case, areas of missing data may never be
+     * written. In order to make sure that these areas are given the
+     * missing value a 'holegrid' is used to describe the missing
+     * points. This is generally faster than the netcdf method of
+     * filling the entire array with missing values before overwriting
+     * those values later. */
+    if (iodesc->rearranger == PIO_REARR_SUBSET && iodesc->needsfill)
+    {
+        LOG((2, "nvars = %d holegridsize = %ld iodesc->needsfill = %d\n", nvars,
+             iodesc->holegridsize, iodesc->needsfill));
+
+	pioassert(!vdesc0->fillbuf, "buffer overwrite",__FILE__, __LINE__);
+
+        /* Get a buffer. */
+	if (ios->io_rank == 0)
+	    vdesc0->fillbuf = bget(iodesc->maxholegridsize * iodesc->basetype_size * nvars);
+	else if (iodesc->holegridsize > 0)
+	    vdesc0->fillbuf = bget(iodesc->holegridsize * iodesc->basetype_size * nvars);
+
+        /* copying the fill value into the data buffer for the box
+         * rearranger. This will be overwritten with data where
+         * provided. */
+        for (int nv = 0; nv < nvars; nv++)
+            for (int i = 0; i < iodesc->holegridsize; i++)
+                memcpy(&((char *)vdesc0->fillbuf)[iodesc->basetype_size * (i + nv * iodesc->holegridsize)],
+                       &((char *)fillvalue)[iodesc->basetype_size * nv], iodesc->basetype_size);
+
+        /* Write the darray based on the iotype. */
+        switch (file->iotype)
+        {
+        case PIO_IOTYPE_PNETCDF:
+        case PIO_IOTYPE_NETCDF4P:
+            if ((ierr = pio_write_darray_multi_nc(file, nvars, varids,
+                                                  iodesc->ndims, iodesc->basetype, iodesc->maxfillregions,
+                                                  iodesc->fillregion, iodesc->holegridsize,
+                                                  iodesc->num_aiotasks, vdesc0->fillbuf, frame)))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+        case PIO_IOTYPE_NETCDF:
+            if ((ierr = write_darray_multi_serial(file, nvars, varids, iodesc, 1, frame)))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+            break;
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+
+        /* For PNETCDF fillbuf is freed in flush_output_buffer() */
+        if (file->iotype != PIO_IOTYPE_PNETCDF)
+        {
+            /* Free resources. */
+            if (vdesc0->fillbuf)
+            {
+                brel(vdesc0->fillbuf);
+                vdesc0->fillbuf = NULL;
+            }
+        }
+    }
+
+    /* Flush data to disk for pnetcdf. */
+    if (ios->ioproc && file->iotype == PIO_IOTYPE_PNETCDF)
+        if ((ierr = flush_output_buffer(file, flushtodisk, 0)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Find the fillvalue that should be used for a variable.
+ *
+ * @param file Info about file we are writing to. 
+ * @param varid the variable ID.
+ * @param vdesc pointer to var_desc_t info for this var.
+ * @returns 0 for success, non-zero error code for failure.
+ * @ingroup PIO_write_darray
+ */
+int find_var_fillvalue(file_desc_t *file, int varid, var_desc_t *vdesc)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */    
+    int no_fill;
+    int ierr;
+
+    /* Check inputs. */
+    pioassert(file && file->iosystem && vdesc, "invalid input", __FILE__, __LINE__);
+    ios = file->iosystem;
+    
+    LOG((3, "find_var_fillvalue file->pio_ncid = %d varid = %d", file->pio_ncid, varid));
+    
+    /* Find out PIO data type of var. */
+    if ((ierr = PIOc_inq_vartype(file->pio_ncid, varid, &vdesc->pio_type)))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+    
+    /* Find out length of type. */
+    if ((ierr = PIOc_inq_type(file->pio_ncid, vdesc->pio_type, NULL, &vdesc->type_size)))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+    LOG((3, "getting fill value for varid = %d pio_type = %d type_size = %d",
+         varid, vdesc->pio_type, vdesc->type_size));
+    
+    /* Allocate storage for the fill value. */
+    if (!(vdesc->fillvalue = malloc(vdesc->type_size)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    
+    /* Get the fill value. */
+    if ((ierr = PIOc_inq_var_fill(file->pio_ncid, varid, &no_fill, vdesc->fillvalue)))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+    vdesc->use_fill = no_fill ? 0 : 1;
+    LOG((3, "vdesc->use_fill = %d", vdesc->use_fill));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Write a distributed array to the output file.
+ *
+ * This routine aggregates output on the compute nodes and only sends
+ * it to the IO nodes when the compute buffer is full or when a flush
+ * is triggered.
+ *
+ * Internally, this function will:
+ * 
      
+ * 
    • Locate info about this file, decomposition, and variable.
+ * 
    • If we don't have a fillvalue for this variable, determine one
+ * and remember it for future calls.
+ * 
    • Initialize or find the multi_buffer for this record/var.
+ * 
    • Find out how much free space is available in the multi buffer
+ * and flush if needed.
+ * 
    • Store the new user data in the mutli buffer.
+ * 
    • If needed (only for subset rearranger), fill in gaps in data
+ * with fillvalue.
+ * 
    • Remember the frame value (i.e. record number) of this data if
+ * there is one.
+ * 
    
+ *
+ * NOTE: The write multi buffer wmulti_buffer is the cache on compute
+ * nodes that will collect and store multiple variables before sending
+ * them to the io nodes. Aggregating variables in this way leads to a
+ * considerable savings in communication expense. Variables in the wmb
+ * array must have the same decomposition and base data size and we
+ * also need to keep track of whether each is a recordvar (has an
+ * unlimited dimension) or not.
+ *
+ * @param ncid the ncid of the open netCDF file.
+ * @param varid the ID of the variable that these data will be written
+ * to.
+ * @param ioid the I/O description ID as passed back by
+ * PIOc_InitDecomp().
+ * @param arraylen the length of the array to be written. This should
+ * be at least the length of the local component of the distrubited
+ * array. (Any values beyond length of the local component will be
+ * ignored.)
+ * @param array pointer to an array of length arraylen with the data
+ * to be written. This is a pointer to the distributed portion of the
+ * array that is on this task.
+ * @param fillvalue pointer to the fill value to be used for missing
+ * data.
+ * @returns 0 for success, non-zero error code for failure.
+ * @ingroup PIO_write_darray
+ */
+int PIOc_write_darray(int ncid, int varid, int ioid, PIO_Offset arraylen, void *array,
+                      void *fillvalue)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Info about file we are writing to. */
+    io_desc_t *iodesc;     /* The IO description. */
+    var_desc_t *vdesc;     /* Info about the var being written. */
+    void *bufptr;          /* A data buffer. */
+    MPI_Datatype vtype;    /* The MPI type of the variable. */
+    wmulti_buffer *wmb;    /* The write multi buffer for one or more vars. */
+    int recordvar;         /* Non-zero if this is a record variable. */
+    int needsflush = 0;    /* True if we need to flush buffer. */
+    bufsize totfree;       /* Amount of free space in the buffer. */
+    bufsize maxfree;       /* Max amount of free space in buffer. */
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI functions. */
+    int ierr = PIO_NOERR;  /* Return code. */
+
+    LOG((1, "PIOc_write_darray ncid = %d varid = %d ioid = %d arraylen = %d",
+         ncid, varid, ioid, arraylen));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Can we write to this file? */
+    if (!(file->mode & PIO_WRITE))
+        return pio_err(ios, file, PIO_EPERM, __FILE__, __LINE__);
+
+    /* Get decomposition information. */
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check that the local size of the variable passed in matches the
+     * size expected by the io descriptor. Fail if arraylen is too
+     * small, just put a warning in the log if it is too big (the
+     * excess values will be ignored.) */
+    if (arraylen < iodesc->ndof)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+    LOG((2, "%s arraylen = %d iodesc->ndof = %d",
+         (iodesc->ndof != arraylen) ? "WARNING: iodesc->ndof != arraylen" : "",
+         arraylen, iodesc->ndof));
+
+    /* Get var description. */
+    vdesc = &(file->varlist[varid]);
+    LOG((2, "vdesc record %d ndims %d nreqs %d", vdesc->record, vdesc->ndims,
+         vdesc->nreqs));
+
+    /* If we don't know the fill value for this var, get it. */
+    if (!vdesc->fillvalue)
+        if ((ierr = find_var_fillvalue(file, varid, vdesc)))
+            return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__);            
+
+    /* Is this a record variable? The user must set the vdesc->record
+     * value by calling PIOc_setframe() before calling this
+     * function. */
+    recordvar = vdesc->record >= 0 ? 1 : 0;
+    LOG((3, "recordvar = %d", recordvar));
+
+    /* Move to end of list or the entry that matches this ioid. */
+    for (wmb = &file->buffer; wmb->next; wmb = wmb->next)
+        if (wmb->ioid == ioid && wmb->recordvar == recordvar)
+            break;
+
+    /* If we did not find an existing wmb entry, create a new wmb. */
+    if (wmb->ioid != ioid || wmb->recordvar != recordvar)
+    {
+        /* Allocate a buffer. */
+        if (!(wmb->next = bget((bufsize)sizeof(wmulti_buffer))))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        LOG((3, "allocated multi-buffer"));
+
+        /* Set pointer to newly allocated buffer and initialize.*/
+        wmb = wmb->next;
+        wmb->recordvar = recordvar;
+        wmb->next = NULL;
+        wmb->ioid = ioid;
+        wmb->num_arrays = 0;
+        wmb->arraylen = arraylen;
+        wmb->vid = NULL;
+        wmb->data = NULL;
+        wmb->frame = NULL;
+        wmb->fillvalue = NULL;
+    }
+    LOG((2, "wmb->num_arrays = %d arraylen = %d iodesc->basetype_size = %d\n",
+         wmb->num_arrays, arraylen, iodesc->basetype_size));
+
+    /* Find out how much free, contiguous space is available. */
+    bfreespace(&totfree, &maxfree);
+
+    /* maxfree is the available memory. If that is < 10% greater than
+     * the size of the current request needsflush is true. */
+    if (needsflush == 0)
+        needsflush = (maxfree <= 1.1 * (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size);
+
+    /* Tell all tasks on the computation communicator whether we need
+     * to flush data. */
+    if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &needsflush, 1,  MPI_INT,  MPI_MAX,
+                                ios->comp_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    LOG((2, "needsflush = %d", needsflush));
+
+    /* Flush data if needed. */
+    if (needsflush > 0)
+    {
+#ifdef PIO_ENABLE_LOGGING
+        /* Collect a debug report about buffer. */
+        cn_buffer_report(ios, true);
+        LOG((2, "maxfree = %ld wmb->num_arrays = %d (1 + wmb->num_arrays) *"
+             " arraylen * iodesc->basetype_size = %ld totfree = %ld\n", maxfree, wmb->num_arrays,
+             (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size, totfree));
+#endif /* PIO_ENABLE_LOGGING */
+
+        /* If needsflush == 2 flush to disk otherwise just flush to io node. */
+        if ((ierr = flush_buffer(ncid, wmb, needsflush == 2)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+    }
+
+    /* Get memory for data. */
+    if (arraylen > 0)
+    {
+        if (!(wmb->data = bgetr(wmb->data, (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size)))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+        LOG((2, "got %ld bytes for data", (1 + wmb->num_arrays) * arraylen * iodesc->basetype_size));
+    }
+
+    /* vid is an array of variable ids in the wmb list, grow the list
+     * and add the new entry. */
+    if (!(wmb->vid = bgetr(wmb->vid, sizeof(int) * (1 + wmb->num_arrays))))
+        return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* wmb->frame is the record number, we assume that the variables
+     * in the wmb list may not all have the same unlimited dimension
+     * value although they usually do. */
+    if (vdesc->record >= 0)
+        if (!(wmb->frame = bgetr(wmb->frame, sizeof(int) * (1 + wmb->num_arrays))))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* If we need a fill value, get it. If we are using the subset
+     * rearranger and not using the netcdf fill mode then we need to
+     * do an extra write to fill in the holes with the fill value. */
+    if (iodesc->needsfill)
+    {
+        /* Get memory to hold fill value. */
+        if (!(wmb->fillvalue = bgetr(wmb->fillvalue, iodesc->basetype_size * (1 + wmb->num_arrays))))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+        /* If the user passed a fill value, use that, otherwise use
+         * the default fill value of the netCDF type. Copy the fill
+         * value to the buffer. */
+        if (fillvalue)
+        {
+            memcpy((char *)wmb->fillvalue + iodesc->basetype_size * wmb->num_arrays,
+                   fillvalue, iodesc->basetype_size);
+            LOG((3, "copied user-provided fill value iodesc->basetype_size = %d",
+                 iodesc->basetype_size));
+        }
+        else
+        {
+            void *fill;
+            signed char byte_fill = PIO_FILL_BYTE;
+            char char_fill = PIO_FILL_CHAR;
+            short short_fill = PIO_FILL_SHORT;
+            int int_fill = PIO_FILL_INT;
+            float float_fill = PIO_FILL_FLOAT;
+            double double_fill = PIO_FILL_DOUBLE;
+#ifdef _NETCDF4
+            unsigned char ubyte_fill = PIO_FILL_UBYTE;
+            unsigned short ushort_fill = PIO_FILL_USHORT;
+            unsigned int uint_fill = PIO_FILL_UINT;
+            long long int64_fill = PIO_FILL_INT64;
+            long long uint64_fill = PIO_FILL_UINT64;
+#endif /* _NETCDF4 */
+            vtype = (MPI_Datatype)iodesc->basetype;
+            LOG((3, "caller did not provide fill value vtype = %d", vtype));
+
+            /* This must be done with an if statement, not a case, or
+             * openmpi will not build. */
+            if (vtype == MPI_BYTE)
+                fill = &byte_fill;
+            else if (vtype == MPI_CHAR)
+                fill = &char_fill;
+            else if (vtype == MPI_SHORT)
+                fill = &short_fill;
+            else if (vtype == MPI_INT)
+                fill = &int_fill;
+            else if (vtype == MPI_FLOAT)
+                fill = &float_fill;
+            else if (vtype == MPI_DOUBLE)
+                fill = &double_fill;
+#ifdef _NETCDF4
+            else if (vtype == MPI_UNSIGNED_CHAR)
+                fill = &ubyte_fill;
+            else if (vtype == MPI_UNSIGNED_SHORT)
+                fill = &ushort_fill;
+            else if (vtype == MPI_UNSIGNED)
+                fill = &uint_fill;
+            else if (vtype == MPI_LONG_LONG)
+                fill = &int64_fill;
+            else if (vtype == MPI_UNSIGNED_LONG_LONG)
+                fill = &uint64_fill;
+#endif /* _NETCDF4 */
+            else
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+
+            memcpy((char *)wmb->fillvalue + iodesc->basetype_size * wmb->num_arrays,
+                   fill, iodesc->basetype_size);
+            LOG((3, "copied fill value"));
+        }
+    }
+
+    /* Tell the buffer about the data it is getting. */
+    wmb->arraylen = arraylen;
+    wmb->vid[wmb->num_arrays] = varid;
+    LOG((3, "wmb->num_arrays = %d wmb->vid[wmb->num_arrays] = %d", wmb->num_arrays,
+         wmb->vid[wmb->num_arrays]));
+
+    /* Copy the user-provided data to the buffer. */
+    bufptr = (void *)((char *)wmb->data + arraylen * iodesc->basetype_size * wmb->num_arrays);
+    if (arraylen > 0)
+    {
+        memcpy(bufptr, array, arraylen * iodesc->basetype_size);
+        LOG((3, "copied %ld bytes of user data", arraylen * iodesc->basetype_size));
+    }
+
+    /* Add the unlimited dimension value of this variable to the frame
+     * array in wmb. */
+    if (wmb->frame)
+        wmb->frame[wmb->num_arrays] = vdesc->record;
+    wmb->num_arrays++;
+
+    LOG((2, "wmb->num_arrays = %d iodesc->maxbytes / iodesc->basetype_size = %d "
+         "iodesc->ndof = %d iodesc->llen = %d", wmb->num_arrays,
+         iodesc->maxbytes / iodesc->basetype_size, iodesc->ndof, iodesc->llen));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Read a field from a file to the IO library.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID to be read
+ * @param ioid: the I/O description ID as passed back by
+ * PIOc_InitDecomp().
+ * @param arraylen: the length of the array to be read. This
+ * is the length of the distrubited array. That is, the length of
+ * the portion of the data that is on the processor.
+ * @param array: pointer to the data to be read. This is a
+ * pointer to the distributed portion of the array that is on this
+ * processor.
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_read_darray
+ */
+int PIOc_read_darray(int ncid, int varid, int ioid, PIO_Offset arraylen,
+                     void *array)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    io_desc_t *iodesc;     /* Pointer to IO description information. */
+    void *iobuf = NULL;    /* holds the data as read on the io node. */
+    size_t rlen = 0;       /* the length of data in iobuf. */
+    int ierr;           /* Return code. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Get the iodesc. */
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, file, PIO_EBADID, __FILE__, __LINE__);
+    pioassert(iodesc->rearranger == PIO_REARR_BOX || iodesc->rearranger == PIO_REARR_SUBSET,
+              "unknown rearranger", __FILE__, __LINE__);
+
+    /* ??? */
+    if (ios->iomaster == MPI_ROOT)
+        rlen = iodesc->maxiobuflen;
+    else
+        rlen = iodesc->llen;
+
+    /* Allocate a buffer for one record. */
+    if (ios->ioproc && rlen > 0)
+        if (!(iobuf = bget(iodesc->basetype_size * rlen)))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Call the correct darray read function based on iotype. */
+    switch (file->iotype)
+    {
+    case PIO_IOTYPE_NETCDF:
+    case PIO_IOTYPE_NETCDF4C:
+        if ((ierr = pio_read_darray_nc_serial(file, iodesc, varid, iobuf)))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+        break;
+    case PIO_IOTYPE_PNETCDF:
+    case PIO_IOTYPE_NETCDF4P:
+        if ((ierr = pio_read_darray_nc(file, iodesc, varid, iobuf)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+        break;
+    default:
+        return pio_err(NULL, NULL, PIO_EBADIOTYPE, __FILE__, __LINE__);
+    }
+
+    /* Rearrange the data. */
+    if ((ierr = rearrange_io2comp(ios, iodesc, iobuf, array)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Free the buffer. */
+    if (rlen > 0)
+        brel(iobuf);
+
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pio_darray_int.c b/src/externals/pio2/src/clib/pio_darray_int.c
new file mode 100644
index 00000000000..79572219b91
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_darray_int.c
@@ -0,0 +1,1522 @@
+/** @file
+ *
+ * Private functions to help read and write distributed arrays in PIO.
+ *
+ * When arrays are distributed, each processor holds some of the
+ * array. Only by combining the distributed arrays from all processor
+ * can the full array be obtained.
+ *
+ * @author Jim Edwards
+ */
+
+#include 
+#include 
+#include 
+
+/* 10MB default limit. */
+extern PIO_Offset pio_buffer_size_limit;
+
+/* Initial size of compute buffer. */
+bufsize pio_cnbuffer_limit = 33554432;
+
+/* Global buffer pool pointer. */
+extern void *CN_bpool;
+
+/* Maximum buffer usage. */
+extern PIO_Offset maxusage;
+
+/* handler for freeing the memory buffer pool */
+void bpool_free(void *p)
+{
+  free(p);
+  if(p == CN_bpool){
+    CN_bpool = NULL;
+  }
+}
+
+
+/**
+ * Initialize the compute buffer to size pio_cnbuffer_limit.
+ *
+ * This routine initializes the compute buffer pool if the bget memory
+ * management is used. If malloc is used (that is, PIO_USE_MALLOC is
+ * non zero), this function does nothing.
+ *
+ * @param ios pointer to the iosystem descriptor which will use the
+ * new buffer.
+ * @returns 0 for success, error code otherwise.
+ */
+int compute_buffer_init(iosystem_desc_t *ios)
+{
+#if !PIO_USE_MALLOC
+
+    if (!CN_bpool)
+    {
+        if (!(CN_bpool = malloc(pio_cnbuffer_limit)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        bpool(CN_bpool, pio_cnbuffer_limit);
+        if (!CN_bpool)
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        bectl(NULL, malloc, bpool_free, pio_cnbuffer_limit);
+    }
+#endif
+    LOG((2, "compute_buffer_init complete"));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Write a set of one or more aggregated arrays to output file. This
+ * function is only used with parallel-netcdf and netcdf-4 parallel
+ * iotypes. Serial io types use write_darray_multi_serial().
+ *
+ * @param file a pointer to the open file descriptor for the file
+ * that will be written to
+ * @param nvars the number of variables to be written with this
+ * decomposition.
+ * @param vid: an array of the variable ids to be written.
+ * @param iodesc_ndims: the number of dimensions explicitly in the
+ * iodesc.
+ * @param basetype the basic type of the minimal data unit
+ * @param maxregions max number of blocks to be written from
+ * this iotask.
+ * @param firstregion pointer to the first element of a linked
+ * list of region descriptions. May be NULL.
+ * @param llen length of the iobuffer on this task for a single
+ * field.
+ * @param num_aiotasks actual number of iotasks participating.
+ * @param iobuf the buffer to be written from this mpi task. May be
+ * null. for example we have 8 ionodes and a distributed array with
+ * global size 4, then at least 4 nodes will have a null iobuf. In
+ * practice the box rearranger trys to have at least blocksize bytes
+ * on each io task and so if the total number of bytes to write is
+ * less than blocksize*numiotasks then some iotasks will have a NULL
+ * iobuf.
+ * @param frame the frame or record dimension for each of the nvars
+ * variables in iobuf.  NULL if this iodesc contains non-record vars.
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ */
+int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int iodesc_ndims,
+                              MPI_Datatype basetype, int maxregions, io_region *firstregion,
+                              PIO_Offset llen, int num_aiotasks, void *iobuf,
+                              const int *frame)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    var_desc_t *vdesc;     /* Pointer to var info struct. */
+    int fndims;            /* Number of dims for this var in the file. */
+    int dsize;             /* Data size (for one region). */
+    int tsize;             /* Size of MPI type. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr = PIO_NOERR;
+
+    /* Check inputs. */
+    pioassert(file && file->iosystem && vid && vid[0] >= 0 && vid[0] <= PIO_MAX_VARS,
+              "invalid input", __FILE__, __LINE__);
+
+    LOG((1, "pio_write_darray_multi_nc nvars = %d iodesc_ndims = %d basetype = %d "
+         "maxregions = %d llen = %d num_aiotasks = %d", nvars, iodesc_ndims,
+         basetype, maxregions, llen, num_aiotasks));
+
+#ifdef TIMING
+    /* Start timing this function. */
+    GPTLstart("PIO:write_darray_multi_nc");
+#endif
+
+    /* Get pointer to iosystem. */
+    ios = file->iosystem;
+
+    /* Point to var description scruct for first var. */
+    vdesc = file->varlist + vid[0];
+
+    /* If async is in use, send message to IO master task. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = 0;
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->pio_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Find out how many dims this variable has. */
+    if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid[0], &fndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Find out the size of the MPI type. */
+    if ((mpierr = MPI_Type_size(basetype, &tsize)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    LOG((2, "fndims = %d tsize = %d", fndims, tsize));
+
+    /* If this is an IO task write the data. */
+    if (ios->ioproc)
+    {
+        io_region *region = firstregion;
+        int rrcnt = 0; /* Number of subarray requests (pnetcdf only). */
+        void *bufptr;
+        size_t start[fndims];
+        size_t count[fndims];
+        int ndims = iodesc_ndims;
+        PIO_Offset *startlist[maxregions]; /* Array of start arrays for ncmpi_iput_varn(). */
+        PIO_Offset *countlist[maxregions]; /* Array of count  arrays for ncmpi_iput_varn(). */
+
+        LOG((3, "maxregions = %d", maxregions));
+
+        /* Process each region of data to be written. */
+        for (int regioncnt = 0; regioncnt < maxregions; regioncnt++)
+        {
+            /* Init start/count arrays to zero. */
+            for (int i = 0; i < fndims; i++)
+            {
+                start[i] = 0;
+                count[i] = 0;
+            }
+
+            if (region)
+            {
+                if (vdesc->record >= 0)
+                {
+                    /* This is a record based multidimensional
+                     * array. Figure out start/count for all but the
+                     * record dimension (dimid 0). */
+                    for (int i = fndims - ndims; i < fndims; i++)
+                    {
+                        start[i] = region->start[i - (fndims - ndims)];
+                        count[i] = region->count[i - (fndims - ndims)];
+                    }
+
+                    /* Now figure out start/count for record dimension. */
+                    if (fndims > 1 && ndims < fndims && count[1] > 0)
+                    {
+                        count[0] = 1;
+                        start[0] = frame[0];
+                    }
+                    else if (fndims == ndims)
+                    {
+                        /* ??? */
+                        start[0] += vdesc->record;
+                    }
+                }
+                else
+                {
+                    /* This is a non record variable. */
+                    for (int i = 0; i < ndims; i++)
+                    {
+                        start[i] = region->start[i];
+                        count[i] = region->count[i];
+                    }
+                }
+#if PIO_ENABLE_LOGGING
+                /* Log arrays for debug purposes. */
+                for (int i = 0; i < ndims; i++)
+                    LOG((3, "start[%d] = %d count[%d] = %d", i, start[i], i, count[i]));
+#endif /* PIO_ENABLE_LOGGING */
+            }
+
+            /* IO tasks will run the netCDF/pnetcdf functions to write the data. */
+            switch (file->iotype)
+            {
+#ifdef _NETCDF4
+            case PIO_IOTYPE_NETCDF4P:
+                /* For each variable to be written. */
+                for (int nv = 0; nv < nvars; nv++)
+                {
+                    /* Set the start of the record dimension. (Hasn't
+                     * this already been set above ???) */
+                    if (vdesc->record >= 0 && ndims < fndims)
+                        start[0] = frame[nv];
+
+                    /* If there is data for this region, get a pointer to it. */
+                    if (region)
+                        bufptr = (void *)((char *)iobuf + tsize * (nv * llen + region->loffset));
+
+                    /* Ensure collective access. */
+                    ierr = nc_var_par_access(file->fh, vid[nv], NC_COLLECTIVE);
+
+                    /* Write the data for this variable. */
+                    if (!ierr)
+                        ierr = nc_put_vara(file->fh, vid[nv], (size_t *)start, (size_t *)count, bufptr);
+                }
+                break;
+#endif
+#ifdef _PNETCDF
+            case PIO_IOTYPE_PNETCDF:
+                /* Get the total number of data elements we are
+                 * writing for this region. */
+                dsize = 1;
+                for (int i = 0; i < fndims; i++)
+                    dsize *= count[i];
+                LOG((3, "dsize = %d", dsize));
+
+                /* For pnetcdf's ncmpi_iput_varn() function, we need
+                 * to provide arrays of arrays for start/count. */
+                if (dsize > 0)
+                {
+                    /* Allocate storage for start/count arrays for
+                     * this region. */
+                    if (!(startlist[rrcnt] = calloc(fndims, sizeof(PIO_Offset))))
+                        return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+                    if (!(countlist[rrcnt] = calloc(fndims, sizeof(PIO_Offset))))
+                        return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+                    /* Copy the start/count arrays for this region. */
+                    for (int i = 0; i < fndims; i++)
+                    {
+                        startlist[rrcnt][i] = start[i];
+                        countlist[rrcnt][i] = count[i];
+                        LOG((3, "startlist[%d][%d] = %d countlist[%d][%d] = %d", rrcnt, i,
+                             startlist[rrcnt][i], rrcnt, i, countlist[rrcnt][i]));
+                    }
+                    rrcnt++;
+                }
+
+                /* Do this when we reach the last region. */
+                if (regioncnt == maxregions - 1)
+                {
+                    /* For each variable to be written. */
+                    for (int nv = 0; nv < nvars; nv++)
+                    {
+                        /* Get the var info. */
+                        vdesc = file->varlist + vid[nv];
+
+                        /* If this is a record var, set the start for
+                         * the record dimension. */
+                        if (vdesc->record >= 0 && ndims < fndims)
+                            for (int rc = 0; rc < rrcnt; rc++)
+                                startlist[rc][0] = frame[nv];
+
+                        /* Get a pointer to the data. */
+                        bufptr = (void *)((char *)iobuf + nv * tsize * llen);
+
+                        /* ??? */
+                        int reqn = 0;
+                        if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0 )
+                        {
+                            if (!(vdesc->request = realloc(vdesc->request, sizeof(int) *
+                                                           (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK))))
+                                return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+                            for (int i = vdesc->nreqs; i < vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK; i++)
+                                vdesc->request[i] = NC_REQ_NULL;
+                            reqn = vdesc->nreqs;
+                        }
+                        else
+                            while(vdesc->request[reqn] != NC_REQ_NULL)
+                                reqn++;
+
+                        /* Write, in non-blocking fashion, a list of subarrays. */
+                        LOG((3, "about to call ncmpi_iput_varn() vid[%d] = %d rrcnt = %d, llen = %d",
+                             nv, vid[nv], rrcnt, llen));
+                        ierr = ncmpi_iput_varn(file->fh, vid[nv], rrcnt, startlist, countlist,
+                                               bufptr, llen, basetype, vdesc->request + reqn);
+
+                        /* keeps wait calls in sync */
+                        if (vdesc->request[reqn] == NC_REQ_NULL)
+                            vdesc->request[reqn] = PIO_REQ_NULL;
+
+                        vdesc->nreqs += reqn + 1;
+
+                    }
+
+                    /* Free resources. */
+                    for (int i = 0; i < rrcnt; i++)
+                    {
+                        free(startlist[i]);
+                        free(countlist[i]);
+                    }
+                }
+                break;
+#endif
+            default:
+                return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+            }
+
+            /* Go to next region. */
+            if (region)
+                region = region->next;
+        } /* next regioncnt */
+    } /* endif (ios->ioproc) */
+
+    /* Check the return code from the netCDF/pnetcdf call. */
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+#ifdef TIMING
+    /* Stop timing this function. */
+    GPTLstop("PIO:write_darray_multi_nc");
+#endif
+
+    return ierr;
+}
+
+/**
+ * Fill the tmp_start and tmp_count arrays, which contain the start
+ * and count arrays for all regions.
+ *
+ * This is an internal function which is only called on io tasks. It
+ * is called by write_darray_multi_serial().
+ *
+ * @param region pointer to the first in a linked list of regions.
+ * @param maxregions the number of regions in the list.
+ * @param fndims the number of dimensions in the file.
+ * @param iodesc_ndims the number of dimensions in the decomposition.
+ * @param vdesc pointer to an array of var_desc_t for the vars being
+ * written.
+ * @param tmp_start pointer to an already allocaed array of length
+ * fndims * maxregions. This array will get the start values for all
+ * regions.
+ * @param tmp_count pointer to an already allocaed array of length
+ * fndims * maxregions. This array will get the count values for all
+ * regions.
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_read_darray
+ **/
+int find_all_start_count(io_region *region, int maxregions, int fndims,
+                         int iodesc_ndims, var_desc_t *vdesc, size_t *tmp_start,
+                         size_t *tmp_count)
+{
+    /* Check inputs. */
+    pioassert(maxregions >= 0 && fndims > 0 && iodesc_ndims >= 0 && vdesc &&
+              tmp_start && tmp_count, "invalid input", __FILE__, __LINE__);
+
+    /* Find the start/count arrays for each region in the list. */
+    for (int r = 0; r < maxregions; r++)
+    {
+        /* Initialize the start/count arrays for this region to 0. */
+        for (int d = 0; d < fndims; d++)
+        {
+            tmp_start[d + r * fndims] = 0;
+            tmp_count[d + r * fndims] = 0;
+        }
+
+        if (region)
+        {
+            if (vdesc->record >= 0)
+            {
+                /* This is a record based multidimensional
+                 * array. Copy start/count for non-record
+                 * dimensions. */
+                for (int i = fndims - iodesc_ndims; i < fndims; i++)
+                {
+                    tmp_start[i + r * fndims] = region->start[i - (fndims - iodesc_ndims)];
+                    tmp_count[i + r * fndims] = region->count[i - (fndims - iodesc_ndims)];
+                    LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + r * fndims,
+                         tmp_start[i + r * fndims], i + r * fndims,
+                         tmp_count[i + r * fndims]));
+                }
+            }
+            else
+            {
+                /* This is not a record based multidimensional array. */
+                for (int i = 0; i < iodesc_ndims; i++)
+                {
+                    tmp_start[i + r * fndims] = region->start[i];
+                    tmp_count[i + r * fndims] = region->count[i];
+                    LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + r * fndims,
+                         tmp_start[i + r * fndims], i + r * fndims,
+                         tmp_count[i + r * fndims]));
+                }
+            }
+
+            /* Move to next region. */
+            region = region->next;
+
+        } /* endif region */
+    } /* next r */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Internal function called by IO tasks other than IO task 0 to send
+ * their tmp_start/tmp_count arrays to IO task 0.
+ *
+ * This is an internal function which is only called on io tasks other
+ * than IO task 0. It is called by write_darray_multi_serial().
+ *
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ **/
+int send_all_start_count(iosystem_desc_t *ios, io_desc_t *iodesc, PIO_Offset llen,
+                         int maxregions, int nvars, int fndims, size_t *tmp_start,
+                         size_t *tmp_count, void *iobuf)
+{
+    MPI_Status status;     /* Recv status for MPI. */
+    int mpierr;  /* Return code from MPI function codes. */
+    int ierr;    /* Return code. */
+
+    /* Check inputs. */
+    pioassert(ios && ios->ioproc && ios->io_rank > 0 && maxregions >= 0,
+              "invalid inputs", __FILE__, __LINE__);
+
+    /* Do a handshake. */
+    if ((mpierr = MPI_Recv(&ierr, 1, MPI_INT, 0, 0, ios->io_comm, &status)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Send local length of iobuffer for each field (all
+     * fields are the same length). */
+    if ((mpierr = MPI_Send((void *)&llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((3, "sent llen = %d", llen));
+
+    /* Send the number of data regions, the start/count for
+     * all regions, and the data buffer with all the data. */
+    if (llen > 0)
+    {
+        if ((mpierr = MPI_Send((void *)&maxregions, 1, MPI_INT, 0, ios->io_rank + ios->num_iotasks,
+                               ios->io_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Send(tmp_start, maxregions * fndims, MPI_OFFSET, 0,
+                               ios->io_rank + 2 * ios->num_iotasks, ios->io_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Send(tmp_count, maxregions * fndims, MPI_OFFSET, 0,
+                               ios->io_rank + 3 * ios->num_iotasks, ios->io_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Send(iobuf, nvars * llen, iodesc->basetype, 0,
+                               ios->io_rank + 4 * ios->num_iotasks, ios->io_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((3, "sent data for maxregions = %d", maxregions));
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * This is an internal function that is run only on IO proc 0. It
+ * receives data from all the other IO tasks, and write that data to
+ * disk. This is called from write_darray_multi_serial().
+ *
+ * @param file a pointer to the open file descriptor for the file
+ * that will be written to.
+ * @param vid an array of the variable ids to be written
+ * @param frame the record dimension for each of the nvars variables
+ * in iobuf.  NULL if this iodesc contains non-record vars.
+ * @param iodesc pointer to the decomposition info.
+ * @param llen length of the iobuffer on this task for a single
+ * field.
+ * @param maxregions max number of blocks to be written from this
+ * iotask.
+ * @param nvars the number of variables to be written with this
+ * decomposition.
+ * @param fndims the number of dimensions in the file.
+ * @param tmp_start pointer to an already allocaed array of length
+ * fndims * maxregions. This array will get the start values for all
+ * regions.
+ * @param tmp_count pointer to an already allocaed array of length
+ * fndims * maxregions. This array will get the count values for all
+ * regions.
+ * @param iobuf the buffer to be written from this mpi task. May be
+ * null. for example we have 8 ionodes and a distributed array with
+ * global size 4, then at least 4 nodes will have a null iobuf. In
+ * practice the box rearranger trys to have at least blocksize bytes
+ * on each io task and so if the total number of bytes to write is
+ * less than blocksize*numiotasks then some iotasks will have a NULL
+ * iobuf.
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ */
+int recv_and_write_data(file_desc_t *file, const int *vid, const int *frame,
+                        io_desc_t *iodesc, PIO_Offset llen, int maxregions, int nvars,
+                        int fndims, size_t *tmp_start, size_t *tmp_count, void *iobuf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    size_t rlen;    /* Length of IO buffer on this task. */
+    int rregions;   /* Number of regions in buffer for this task. */
+    size_t start[fndims], count[fndims];
+    size_t loffset;
+    void *bufptr;
+    var_desc_t *vdesc;     /* Contains info about the variable. */
+    MPI_Status status;     /* Recv status for MPI. */
+    int mpierr;  /* Return code from MPI function codes. */
+    int ierr;    /* Return code. */
+
+    ios = file->iosystem;
+
+    /* For each of the other tasks that are using this task
+     * for IO. */
+    for (int rtask = 0; rtask < ios->num_iotasks; rtask++)
+    {
+        /* From the remote tasks, we send information about
+         * the data regions. and also the data. */
+        if (rtask)
+        {
+            /* handshake - tell the sending task I'm ready */
+            if ((mpierr = MPI_Send(&ierr, 1, MPI_INT, rtask, 0, ios->io_comm)))
+                return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+            /* Get length of iobuffer for each field on this
+             * task (all fields are the same length). */
+            if ((mpierr = MPI_Recv(&rlen, 1, MPI_OFFSET, rtask, rtask, ios->io_comm,
+                                   &status)))
+                return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+            LOG((3, "received rlen = %d", rlen));
+
+            /* Get the number of regions, the start/count
+             * values for all regions, and the data buffer. */
+            if (rlen > 0)
+            {
+                if ((mpierr = MPI_Recv(&rregions, 1, MPI_INT, rtask, rtask + ios->num_iotasks,
+                                       ios->io_comm, &status)))
+                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Recv(tmp_start, rregions * fndims, MPI_OFFSET, rtask,
+                                       rtask + 2 * ios->num_iotasks, ios->io_comm, &status)))
+                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Recv(tmp_count, rregions * fndims, MPI_OFFSET, rtask,
+                                       rtask + 3 * ios->num_iotasks, ios->io_comm, &status)))
+                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Recv(iobuf, nvars * rlen, iodesc->basetype, rtask,
+                                       rtask + 4 * ios->num_iotasks, ios->io_comm, &status)))
+                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+                LOG((3, "received data rregions = %d fndims = %d", rregions, fndims));
+            }
+        }
+        else /* task 0 */
+        {
+            rlen = llen;
+            rregions = maxregions;
+        }
+        LOG((3, "rtask = %d rlen = %d rregions = %d", rtask, rlen, rregions));
+
+        /* If there is data from this task, write it. */
+        if (rlen > 0)
+        {
+            loffset = 0;
+            for (int regioncnt = 0; regioncnt < rregions; regioncnt++)
+            {
+                LOG((3, "writing data for region with regioncnt = %d", regioncnt));
+
+                /* Get the start/count arrays for this region. */
+                for (int i = 0; i < fndims; i++)
+                {
+                    start[i] = tmp_start[i + regioncnt * fndims];
+                    count[i] = tmp_count[i + regioncnt * fndims];
+                    LOG((3, "start[%d] = %d count[%d] = %d", i, start[i], i, count[i]));
+                }
+
+                /* Process each variable in the buffer. */
+                for (int nv = 0; nv < nvars; nv++)
+                {
+                    LOG((3, "writing buffer var %d", nv));
+                    vdesc = file->varlist + vid[0];
+
+                    /* Get a pointer to the correct part of the buffer. */
+                    bufptr = (void *)((char *)iobuf + iodesc->basetype_size * (nv * rlen + loffset));
+
+                    /* If this var has an unlimited dim, set
+                     * the start on that dim to the frame
+                     * value for this variable. */
+                    if (vdesc->record >= 0)
+                    {
+                        if (fndims > 1 && iodesc->ndims < fndims && count[1] > 0)
+                        {
+                            count[0] = 1;
+                            start[0] = frame[nv];
+                        }
+                        else if (fndims == iodesc->ndims)
+                        {
+                            start[0] += vdesc->record;
+                        }
+                    }
+
+                    /* Call the netCDF functions to write the data. */
+                    if ((ierr = nc_put_vara(file->fh, vid[nv], start, count, bufptr)))
+                        return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
+
+                } /* next var */
+
+                /* Calculate the total size. */
+                size_t tsize = 1;
+                for (int i = 0; i < fndims; i++)
+                    tsize *= count[i];
+
+                /* Keep track of where we are in the buffer. */
+                loffset += tsize;
+
+                LOG((3, " at bottom of loop regioncnt = %d tsize = %d loffset = %d", regioncnt,
+                     tsize, loffset));
+            } /* next regioncnt */
+        } /* endif (rlen > 0) */
+    } /* next rtask */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Write a set of one or more aggregated arrays to output file in
+ * serial mode. This function is called for netCDF classic and
+ * netCDF-4 serial iotypes. Parallel iotypes use
+ * pio_write_darray_multi_nc().
+ *
+ * @param file a pointer to the open file descriptor for the file
+ * that will be written to.
+ * @param nvars the number of variables to be written with this
+ * decomposition.
+ * @param vid an array of the variable ids to be written
+ * @param iodesc pointer to the decomposition info.
+ * @param maxregions max number of blocks to be written from this
+ * iotask.
+ * @param firstregion pointer to the first element of a linked
+ * list of region descriptions. May be NULL.
+ * @param llen length of the iobuffer on this task for a single
+ * field.
+ * @param iobuf the buffer to be written from this mpi task. May be
+ * null. for example we have 8 ionodes and a distributed array with
+ * global size 4, then at least 4 nodes will have a null iobuf. In
+ * practice the box rearranger trys to have at least blocksize bytes
+ * on each io task and so if the total number of bytes to write is
+ * less than blocksize*numiotasks then some iotasks will have a NULL
+ * iobuf.
+ * @param frame the record dimension for each of the nvars variables
+ * in iobuf.  NULL if this iodesc contains non-record vars.
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ */
+int write_darray_multi_serial(file_desc_t *file, int nvars, const int *vid,
+                              io_desc_t *iodesc, int fill, const int *frame)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    var_desc_t *vdesc;     /* Contains info about the variable. */
+    int fndims;            /* Number of dims in the var in the file. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr;              /* Return code. */
+
+    /* Check inputs. */
+    pioassert(file && file->iosystem && file->varlist && vid && vid[0] >= 0 &&
+              vid[0] <= PIO_MAX_VARS && iodesc, "invalid input", __FILE__, __LINE__);
+
+    LOG((1, "write_darray_multi_serial nvars = %d iodesc->ndims = %d iodesc->basetype = %d",
+         nvars, iodesc->ndims, iodesc->basetype));
+
+    /* Get the iosystem info. */
+    ios = file->iosystem;
+
+    /* Get the var info. */
+    vdesc = file->varlist + vid[0];
+    LOG((2, "vdesc record %d ndims %d nreqs %d ios->async = %d", vdesc->record,
+         vdesc->ndims, vdesc->nreqs, ios->async));
+
+    /* Set these differently for data and fill writing. */
+    int num_regions = fill ? iodesc->maxfillregions: iodesc->maxregions;
+    io_region *region = fill ? iodesc->fillregion : iodesc->firstregion;
+    PIO_Offset llen = fill ? iodesc->holegridsize : iodesc->llen;
+    void *iobuf = fill ? vdesc->fillbuf : vdesc->iobuf;
+
+#ifdef TIMING
+    /* Start timing this function. */
+    GPTLstart("PIO:write_darray_multi_nc_serial");
+#endif
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = 0;
+
+            if (ios->comp_rank == 0)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->pio_ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Get the number of dimensions. */
+    if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid[0], &fndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Only IO tasks participate in this code. */
+    if (ios->ioproc)
+    {
+        size_t tmp_start[fndims * num_regions]; /* A start array for each region. */
+        size_t tmp_count[fndims * num_regions]; /* A count array for each region. */
+
+        LOG((3, "num_regions = %d", num_regions));
+
+        /* Fill the tmp_start and tmp_count arrays, which contain the
+         * start and count arrays for all regions. */
+        if ((ierr = find_all_start_count(region, num_regions, fndims, iodesc->ndims, vdesc,
+                                         tmp_start, tmp_count)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+        /* Tasks other than 0 will send their data to task 0. */
+        if (ios->io_rank > 0)
+        {
+            /* Send the tmp_start and tmp_count arrays from this IO task
+             * to task 0. */
+            if ((ierr = send_all_start_count(ios, iodesc, llen, num_regions, nvars, fndims,
+                                             tmp_start, tmp_count, iobuf)))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+        }
+        else
+        {
+            /* Task 0 will receive data from all other IO tasks. */
+
+            if ((ierr = recv_and_write_data(file, vid, frame, iodesc, llen, num_regions, nvars, fndims,
+                                            tmp_start, tmp_count, iobuf)))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+        }
+    }
+
+#ifdef TIMING
+    /* Stop timing this function. */
+    GPTLstop("PIO:write_darray_multi_nc_serial");
+#endif
+
+    return PIO_NOERR;
+}
+
+/**
+ * Read an array of data from a file to the (parallel) IO library.
+ *
+ * @param file a pointer to the open file descriptor for the file
+ * that will be written to
+ * @param iodesc a pointer to the defined iodescriptor for the buffer
+ * @param vid the variable id to be read
+ * @param iobuf the buffer to be read into from this mpi task. May be
+ * null. for example we have 8 ionodes and a distributed array with
+ * global size 4, then at least 4 nodes will have a null iobuf. In
+ * practice the box rearranger trys to have at least blocksize bytes
+ * on each io task and so if the total number of bytes to write is
+ * less than blocksize*numiotasks then some iotasks will have a NULL
+ * iobuf.
+ * @return 0 on success, error code otherwise.
+ * @ingroup PIO_read_darray
+ */
+int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    var_desc_t *vdesc;     /* Information about the variable. */
+    int ndims;             /* Number of dims in decomposition. */
+    int fndims;            /* Number of dims for this var in file. */
+    int ierr;              /* Return code from netCDF functions. */
+
+    /* Check inputs. */
+    pioassert(file && file->iosystem && iodesc && vid <= PIO_MAX_VARS, "invalid input",
+              __FILE__, __LINE__);
+
+#ifdef TIMING
+    /* Start timing this function. */
+    GPTLstart("PIO:read_darray_nc");
+#endif
+
+    /* Get the IO system info. */
+    ios = file->iosystem;
+
+    /* Get the variable info. */
+    vdesc = file->varlist + vid;
+
+    /* Get the number of dimensions in the decomposition. */
+    ndims = iodesc->ndims;
+
+    /* Get the number of dims for this var in the file. */
+    if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid, &fndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Is this a non-record var? */
+    if (fndims == ndims)
+        vdesc->record = -1;
+
+    /* IO procs will actially read the data. */
+    if (ios->ioproc)
+    {
+        io_region *region;
+        size_t start[fndims];
+        size_t count[fndims];
+        size_t tmp_bufsize = 1;
+        void *bufptr;
+        int rrlen = 0;
+        PIO_Offset *startlist[iodesc->maxregions];
+        PIO_Offset *countlist[iodesc->maxregions];
+
+        /* buffer is incremented by byte and loffset is in terms of
+           the iodessc->basetype so we need to multiply by the size of
+           the basetype. */
+        region = iodesc->firstregion;
+
+        /* ??? */
+        if (fndims > ndims)
+        {
+            ndims++;
+            if (vdesc->record < 0)
+                vdesc->record = 0;
+        }
+
+        /* For each regions, read the data. */
+        for (int regioncnt = 0; regioncnt < iodesc->maxregions; regioncnt++)
+        {
+            tmp_bufsize = 1;
+            if (region == NULL || iodesc->llen == 0)
+            {
+                /* No data for this region. */
+                for (int i = 0; i < fndims; i++)
+                {
+                    start[i] = 0;
+                    count[i] = 0;
+                }
+                bufptr = NULL;
+            }
+            else
+            {
+                /* Get a pointer where we should put the data we read. */
+                if (regioncnt == 0 || region == NULL)
+                    bufptr = iobuf;
+                else
+                    bufptr=(void *)((char *)iobuf + iodesc->basetype_size * region->loffset);
+
+                LOG((2, "%d %d %d", iodesc->llen - region->loffset, iodesc->llen, region->loffset));
+
+                /* Get the start/count arrays. */
+                if (vdesc->record >= 0 && fndims > 1)
+                {
+                    /* This is a record var. The unlimited dimension
+                     * (0) is handled specially. */
+                    start[0] = vdesc->record;
+                    for (int i = 1; i < ndims; i++)
+                    {
+                        start[i] = region->start[i-1];
+                        count[i] = region->count[i-1];
+                    }
+
+                    /* Read one record. */
+                    if (count[1] > 0)
+                        count[0] = 1;
+                }
+                else
+                {
+                    /* Non-time dependent array */
+                    for (int i = 0; i < ndims; i++)
+                    {
+                        start[i] = region->start[i];
+                        count[i] = region->count[i];
+                    }
+                }
+            }
+
+            /* Do the read. */
+            switch (file->iotype)
+            {
+#ifdef _NETCDF4
+            case PIO_IOTYPE_NETCDF4P:
+                ierr = nc_get_vara(file->fh, vid, start, count, bufptr);
+                break;
+#endif
+#ifdef _PNETCDF
+            case PIO_IOTYPE_PNETCDF:
+            {
+                tmp_bufsize = 1;
+                for (int j = 0; j < fndims; j++)
+                    tmp_bufsize *= count[j];
+
+                if (tmp_bufsize > 0)
+                {
+                    startlist[rrlen] = bget(fndims * sizeof(PIO_Offset));
+                    countlist[rrlen] = bget(fndims * sizeof(PIO_Offset));
+
+                    for (int j = 0; j < fndims; j++)
+                    {
+                        startlist[rrlen][j] = start[j];
+                        countlist[rrlen][j] = count[j];
+                    }
+                    rrlen++;
+                }
+
+                /* Is this is the last region to process? */
+                if (regioncnt == iodesc->maxregions - 1)
+                {
+                    /* Read a list of subarrays. */
+                    ierr = ncmpi_get_varn_all(file->fh, vid, rrlen, startlist,
+                                              countlist, iobuf, iodesc->llen, iodesc->basetype);
+
+                    /* Release the start and count arrays. */
+                    for (int i = 0; i < rrlen; i++)
+                    {
+                        brel(startlist[i]);
+                        brel(countlist[i]);
+                    }
+                }
+            }
+            break;
+#endif
+            default:
+                return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+            }
+
+            /* Check return code. */
+            if (ierr)
+                return check_netcdf(file, ierr, __FILE__,__LINE__);
+
+            /* Move to next region. */
+            if (region)
+                region = region->next;
+        } /* next regioncnt */
+    }
+
+#ifdef TIMING
+    /* Stop timing this function. */
+    GPTLstop("PIO:read_darray_nc");
+#endif
+
+    return PIO_NOERR;
+}
+
+/**
+ * Read an array of data from a file to the (serial) IO library. This
+ * function is only used with netCDF classic and netCDF-4 serial
+ * iotypes.
+ *
+ * @param file a pointer to the open file descriptor for the file
+ * that will be written to
+ * @param iodesc a pointer to the defined iodescriptor for the buffer
+ * @param vid the variable id to be read.
+ * @param iobuf the buffer to be written from this mpi task. May be
+ * null. for example we have 8 ionodes and a distributed array with
+ * global size 4, then at least 4 nodes will have a null iobuf. In
+ * practice the box rearranger trys to have at least blocksize bytes
+ * on each io task and so if the total number of bytes to write is
+ * less than blocksize * numiotasks then some iotasks will have a NULL
+ * iobuf.
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_read_darray
+ */
+int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid,
+                              void *iobuf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    var_desc_t *vdesc;     /* Information about the variable. */
+    int ndims;             /* Number of dims in decomposition. */
+    int fndims;            /* Number of dims for this var in file. */
+    MPI_Status status;
+    int mpierr;  /* Return code from MPI functions. */
+    int ierr;
+
+    /* Check inputs. */
+    pioassert(file && file->iosystem && iodesc && vid >= 0 && vid <= PIO_MAX_VARS,
+              "invalid input", __FILE__, __LINE__);
+
+#ifdef TIMING
+    /* Start timing this function. */
+    GPTLstart("PIO:read_darray_nc_serial");
+#endif
+    ios = file->iosystem;
+
+    /* Get var info for this var. */
+    vdesc = file->varlist + vid;
+
+    /* Get the number of dims in our decomposition. */
+    ndims = iodesc->ndims;
+
+    /* Get number of dims for this var. */
+    if ((ierr = PIOc_inq_varndims(file->pio_ncid, vid, &fndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Is this a non-record var? */
+    if (fndims == ndims)
+        vdesc->record = -1;
+
+    if (ios->ioproc)
+    {
+        io_region *region;
+        size_t start[fndims];
+        size_t count[fndims];
+        size_t tmp_start[fndims * iodesc->maxregions];
+        size_t tmp_count[fndims * iodesc->maxregions];
+        size_t tmp_bufsize;
+        void *bufptr;
+
+        /* buffer is incremented by byte and loffset is in terms of
+           the iodessc->basetype so we need to multiply by the size of
+           the basetype. */
+        region = iodesc->firstregion;
+
+        if (fndims > ndims)
+        {
+            if (vdesc->record < 0)
+                vdesc->record = 0;
+        }
+
+        /* Put together start/count arrays for all regions. */
+        for (int regioncnt = 0; regioncnt < iodesc->maxregions; regioncnt++)
+        {
+            if (!region || iodesc->llen == 0)
+            {
+                /* Nothing to write for this region. */
+                for (int i = 0; i < fndims; i++)
+                {
+                    tmp_start[i + regioncnt * fndims] = 0;
+                    tmp_count[i + regioncnt * fndims] = 0;
+                }
+                bufptr = NULL;
+            }
+            else
+            {
+                if (vdesc->record >= 0 && fndims > 1)
+                {
+                    /* This is a record var. Find start for record dims. */
+                    tmp_start[regioncnt * fndims] = vdesc->record;
+
+                    /* Find start/count for all non-record dims. */
+                    for (int i = 1; i < fndims; i++)
+                    {
+                        tmp_start[i + regioncnt * fndims] = region->start[i - 1];
+                        tmp_count[i + regioncnt * fndims] = region->count[i - 1];
+                    }
+
+                    /* Set count for record dimension. */
+                    if (tmp_count[1 + regioncnt * fndims] > 0)
+                        tmp_count[regioncnt * fndims] = 1;
+                }
+                else
+                {
+                    /* Non-time dependent array */
+                    for (int i = 0; i < fndims; i++)
+                    {
+                        tmp_start[i + regioncnt * fndims] = region->start[i];
+                        tmp_count[i + regioncnt * fndims] = region->count[i];
+                    }
+                }
+            }
+
+#if PIO_ENABLE_LOGGING
+            /* Log arrays for debug purposes. */
+            LOG((3, "region = %d", region));
+            for (int i = 0; i < fndims; i++)
+                LOG((3, "tmp_start[%d] = %d tmp_count[%d] = %d", i + regioncnt * fndims, tmp_start[i + regioncnt * fndims],
+                     i + regioncnt * fndims, tmp_count[i + regioncnt * fndims]));
+#endif /* PIO_ENABLE_LOGGING */
+
+            /* Move to next region. */
+            if (region)
+                region = region->next;
+        } /* next regioncnt */
+
+        /* IO tasks other than 0 send their starts/counts and data to
+         * IO task 0. */
+        if (ios->io_rank > 0)
+        {
+            if ((mpierr = MPI_Send(&iodesc->llen, 1, MPI_OFFSET, 0, ios->io_rank, ios->io_comm)))
+                return check_mpi(file, mpierr, __FILE__, __LINE__);
+            LOG((3, "sent iodesc->llen = %d", iodesc->llen));
+
+            if (iodesc->llen > 0)
+            {
+                if ((mpierr = MPI_Send(&(iodesc->maxregions), 1, MPI_INT, 0,
+                                       ios->num_iotasks + ios->io_rank, ios->io_comm)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Send(tmp_count, iodesc->maxregions * fndims, MPI_OFFSET, 0,
+                                       2 * ios->num_iotasks + ios->io_rank, ios->io_comm)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Send(tmp_start, iodesc->maxregions * fndims, MPI_OFFSET, 0,
+                                       3 * ios->num_iotasks + ios->io_rank, ios->io_comm)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                LOG((3, "sent iodesc->maxregions = %d tmp_count and tmp_start arrays", iodesc->maxregions));
+
+                if ((mpierr = MPI_Recv(iobuf, iodesc->llen, iodesc->basetype, 0,
+                                       4 * ios->num_iotasks + ios->io_rank, ios->io_comm, &status)))
+                    return check_mpi(file, mpierr, __FILE__, __LINE__);
+                LOG((3, "received %d elements of data", iodesc->llen));
+            }
+        }
+        else if (ios->io_rank == 0)
+        {
+            /* This is IO task 0. Get starts/counts and data from
+             * other IO tasks. */
+            int maxregions = 0;
+            size_t loffset, regionsize;
+            size_t this_start[fndims * iodesc->maxregions];
+            size_t this_count[fndims * iodesc->maxregions];
+
+            for (int rtask = 1; rtask <= ios->num_iotasks; rtask++)
+            {
+                if (rtask < ios->num_iotasks)
+                {
+                    if ((mpierr = MPI_Recv(&tmp_bufsize, 1, MPI_OFFSET, rtask, rtask, ios->io_comm, &status)))
+                        return check_mpi(file, mpierr, __FILE__, __LINE__);
+                    LOG((3, "received tmp_bufsize = %d", tmp_bufsize));
+
+                    if (tmp_bufsize > 0)
+                    {
+                        if ((mpierr = MPI_Recv(&maxregions, 1, MPI_INT, rtask, ios->num_iotasks + rtask,
+                                               ios->io_comm, &status)))
+                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        if ((mpierr = MPI_Recv(this_count, maxregions * fndims, MPI_OFFSET, rtask,
+                                               2 * ios->num_iotasks + rtask, ios->io_comm, &status)))
+                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        if ((mpierr = MPI_Recv(this_start, maxregions * fndims, MPI_OFFSET, rtask,
+                                               3 * ios->num_iotasks + rtask, ios->io_comm, &status)))
+                            return check_mpi(file, mpierr, __FILE__, __LINE__);
+                        LOG((3, "received maxregions = %d this_count, this_start arrays ", maxregions));
+                    }
+                }
+                else
+                {
+                    maxregions = iodesc->maxregions;
+                    tmp_bufsize = iodesc->llen;
+                }
+                LOG((3, "maxregions = %d tmp_bufsize = %d", maxregions, tmp_bufsize));
+
+                /* Now get each region of data. */
+                loffset = 0;
+                for (int regioncnt = 0; regioncnt < maxregions; regioncnt++)
+                {
+                    /* Get pointer where data should go. */
+                    bufptr = (void *)((char *)iobuf + iodesc->basetype_size * loffset);
+                    regionsize = 1;
+
+                    /* ??? */
+                    if (rtask < ios->num_iotasks)
+                    {
+                        for (int m = 0; m < fndims; m++)
+                        {
+                            start[m] = this_start[m + regioncnt * fndims];
+                            count[m] = this_count[m + regioncnt * fndims];
+                            regionsize *= count[m];
+                        }
+                    }
+                    else
+                    {
+                        for (int m = 0; m < fndims; m++)
+                        {
+                            start[m] = tmp_start[m + regioncnt * fndims];
+                            count[m] = tmp_count[m + regioncnt * fndims];
+                            regionsize *= count[m];
+                        }
+                    }
+                    loffset += regionsize;
+
+                    /* Read the data. */
+                    ierr = nc_get_vara(file->fh, vid, start, count, bufptr);
+
+                    /* Check error code of netCDF call. */
+                    if (ierr)
+                        return check_netcdf(file, ierr, __FILE__, __LINE__);
+                }
+
+                /* The decomposition may not use all of the active io
+                 * tasks. rtask here is the io task rank and
+                 * ios->num_iotasks is the number of iotasks actually
+                 * used in this decomposition. */
+                if (rtask < ios->num_iotasks && tmp_bufsize > 0)
+                    if ((mpierr = MPI_Send(iobuf, tmp_bufsize, iodesc->basetype, rtask,
+                                           4 * ios->num_iotasks + rtask, ios->io_comm)))
+                        return check_mpi(file, mpierr, __FILE__, __LINE__);
+            }
+        }
+    }
+
+#ifdef TIMING
+    /* Stop timing this function. */
+    GPTLstop("PIO:read_darray_nc_serial");
+#endif
+
+    return PIO_NOERR;
+}
+
+/**
+ * Flush the output buffer. This is only relevant for files opened
+ * with pnetcdf.
+ *
+ * @param file a pointer to the open file descriptor for the file
+ * that will be written to
+ * @param force true to force the flushing of the buffer
+ * @param addsize additional size to add to buffer (in bytes)
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ */
+int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize)
+{
+    int mpierr;  /* Return code from MPI functions. */
+    int ierr = PIO_NOERR;
+
+#ifdef _PNETCDF
+    var_desc_t *vdesc;
+    PIO_Offset usage = 0;
+
+    /* Check inputs. */
+    pioassert(file, "invalid input", __FILE__, __LINE__);
+
+    /* Find out the buffer usage. */
+    if ((ierr = ncmpi_inq_buffer_usage(file->fh, &usage)))
+	/* allow the buffer to be undefined */
+	if (ierr != NC_ENULLABUF)
+	    return pio_err(NULL, file, PIO_EBADID, __FILE__, __LINE__);
+
+    /* If we are not forcing a flush, spread the usage to all IO
+     * tasks. */
+    if (!force && file->iosystem->io_comm != MPI_COMM_NULL)
+    {
+        usage += addsize;
+        if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &usage, 1,  MPI_OFFSET,  MPI_MAX,
+                                    file->iosystem->io_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Keep track of the maximum usage. */
+    if (usage > maxusage)
+        maxusage = usage;
+
+    /* If the user forces it, or the buffer has exceeded the size
+     * limit, then flush to disk. */
+    if (force || usage >= pio_buffer_size_limit)
+    {
+        int rcnt;
+        int  maxreq;
+        int reqcnt;
+        maxreq = 0;
+        reqcnt = 0;
+        rcnt = 0;
+        for (int i = 0; i < PIO_MAX_VARS; i++)
+        {
+            vdesc = file->varlist + i;
+            reqcnt += vdesc->nreqs;
+            if (vdesc->nreqs > 0)
+                maxreq = i;
+        }
+        int request[reqcnt];
+        int status[reqcnt];
+
+        for (int i = 0; i <= maxreq; i++)
+        {
+            vdesc = file->varlist + i;
+#ifdef MPIO_ONESIDED
+            /*onesided optimization requires that all of the requests in a wait_all call represent
+              a contiguous block of data in the file */
+            if (rcnt > 0 && (prev_record != vdesc->record || vdesc->nreqs==0))
+            {
+                ierr = ncmpi_wait_all(file->fh, rcnt, request, status);
+                rcnt = 0;
+            }
+            prev_record = vdesc->record;
+#endif
+            for (reqcnt = 0; reqcnt < vdesc->nreqs; reqcnt++)
+                request[rcnt++] = max(vdesc->request[reqcnt], NC_REQ_NULL);
+
+            if (vdesc->request != NULL)
+                free(vdesc->request);
+            vdesc->request = NULL;
+            vdesc->nreqs = 0;
+
+#ifdef FLUSH_EVERY_VAR
+            ierr = ncmpi_wait_all(file->fh, rcnt, request, status);
+            rcnt = 0;
+#endif
+        }
+
+        if (rcnt > 0)
+            ierr = ncmpi_wait_all(file->fh, rcnt, request, status);
+
+        /* Release resources. */
+        for (int i = 0; i < PIO_MAX_VARS; i++)
+        {
+            vdesc = file->varlist + i;
+            if (vdesc->iobuf)
+            {
+		LOG((3,"freeing variable buffer in flush_output_buffer"));
+                brel(vdesc->iobuf);
+                vdesc->iobuf = NULL;
+            }
+            if (vdesc->fillbuf)
+            {
+                brel(vdesc->fillbuf);
+                vdesc->fillbuf = NULL;
+            }
+        }
+    }
+
+#endif /* _PNETCDF */
+    return ierr;
+}
+
+/**
+ * Print out info about the buffer for debug purposes. This should
+ * only be called when logging is enabled.
+ *
+ * @param ios pointer to the IO system structure
+ * @param collective true if collective report is desired
+ * @ingroup PIO_write_darray
+ */
+void cn_buffer_report(iosystem_desc_t *ios, bool collective)
+{
+    int mpierr;  /* Return code from MPI functions. */
+
+    LOG((2, "cn_buffer_report ios->iossysid = %d collective = %d CN_bpool = %d",
+         ios->iosysid, collective, CN_bpool));
+    if (CN_bpool)
+    {
+        long bget_stats[5];
+        long bget_mins[5];
+        long bget_maxs[5];
+
+        bstats(bget_stats, bget_stats+1,bget_stats+2,bget_stats+3,bget_stats+4);
+        if (collective)
+        {
+            LOG((3, "cn_buffer_report calling MPI_Reduce ios->comp_comm = %d", ios->comp_comm));
+            if ((mpierr = MPI_Reduce(bget_stats, bget_maxs, 5, MPI_LONG, MPI_MAX, 0, ios->comp_comm)))
+                check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            LOG((3, "cn_buffer_report calling MPI_Reduce"));
+            if ((mpierr = MPI_Reduce(bget_stats, bget_mins, 5, MPI_LONG, MPI_MIN, 0, ios->comp_comm)))
+                check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            if (ios->compmaster == MPI_ROOT)
+            {
+                LOG((1, "Currently allocated buffer space %ld %ld", bget_mins[0], bget_maxs[0]));
+                LOG((1, "Currently available buffer space %ld %ld", bget_mins[1], bget_maxs[1]));
+                LOG((1, "Current largest free block %ld %ld", bget_mins[2], bget_maxs[2]));
+                LOG((1, "Number of successful bget calls %ld %ld", bget_mins[3], bget_maxs[3]));
+                LOG((1, "Number of successful brel calls  %ld %ld", bget_mins[4], bget_maxs[4]));
+            }
+        }
+        else
+        {
+            LOG((1, "Currently allocated buffer space %ld", bget_stats[0]));
+            LOG((1, "Currently available buffer space %ld", bget_stats[1]));
+            LOG((1, "Current largest free block %ld", bget_stats[2]));
+            LOG((1, "Number of successful bget calls %ld", bget_stats[3]));
+            LOG((1, "Number of successful brel calls  %ld", bget_stats[4]));
+        }
+    }
+}
+
+/**
+ * Free the buffer pool. If malloc is used (that is, PIO_USE_MALLOC is
+ * non zero), this function does nothing.
+ *
+ * @param ios pointer to the IO system structure.
+ * @ingroup PIO_write_darray
+ */
+void free_cn_buffer_pool(iosystem_desc_t *ios)
+{
+#if !PIO_USE_MALLOC
+    LOG((2, "free_cn_buffer_pool CN_bpool = %d", CN_bpool));
+    /* Note: it is possible that CN_bpool has been freed and set to NULL by bpool_free() */
+    if (CN_bpool)
+    {
+        cn_buffer_report(ios, false);
+        bpoolrelease(CN_bpool);
+        LOG((2, "free_cn_buffer_pool done!"));
+        free(CN_bpool);
+        CN_bpool = NULL;
+    }
+#endif /* !PIO_USE_MALLOC */
+}
+
+/**
+ * Flush the buffer.
+ *
+ * @param ncid identifies the netCDF file.
+ * @param wmb pointer to the wmulti_buffer structure.
+ * @param flushtodisk if true, then flush data to disk.
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_write_darray
+ */
+int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk)
+{
+    file_desc_t *file;
+    int ret;
+
+    /* Check input. */
+    pioassert(wmb, "invalid input", __FILE__, __LINE__);
+
+    /* Get the file info (to get error handler). */
+    if ((ret = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "flush_buffer ncid = %d flushtodisk = %d", ncid, flushtodisk));
+
+    /* If there are any variables in this buffer... */
+    if (wmb->num_arrays > 0)
+    {
+        /* Write any data in the buffer. */
+        ret = PIOc_write_darray_multi(ncid, wmb->vid,  wmb->ioid, wmb->num_arrays,
+                                      wmb->arraylen, wmb->data, wmb->frame,
+                                      wmb->fillvalue, flushtodisk);
+
+        wmb->num_arrays = 0;
+
+        /* Release the list of variable IDs. */
+        brel(wmb->vid);
+        wmb->vid = NULL;
+
+        /* Release the data memory. */
+        brel(wmb->data);
+        wmb->data = NULL;
+
+        /* If there is a fill value, release it. */
+        if (wmb->fillvalue)
+            brel(wmb->fillvalue);
+        wmb->fillvalue = NULL;
+
+        /* Release the record number. */
+        if (wmb->frame)
+            brel(wmb->frame);
+        wmb->frame = NULL;
+
+        if (ret)
+            return pio_err(NULL, file, ret, __FILE__, __LINE__);
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Compute the maximum aggregate number of bytes. This is called by
+ * subset_rearrange_create() and box_rearrange_create().
+ *
+ * @param ios pointer to the IO system structure.
+ * @param iodesc a pointer to decomposition description.
+ * @returns 0 for success, error code otherwise.
+ */
+int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc)
+{
+    int maxbytesoniotask = INT_MAX;
+    int maxbytesoncomputetask = INT_MAX;
+    int maxbytes;
+    int mpierr;  /* Return code from MPI functions. */
+
+    /* Check inputs. */
+    pioassert(iodesc, "invalid input", __FILE__, __LINE__);
+
+    LOG((2, "compute_maxaggregate_bytes iodesc->maxiobuflen = %d iodesc->ndof = %d",
+         iodesc->maxiobuflen, iodesc->ndof));
+
+    /* Determine the max bytes that can be held on IO task. */
+    if (ios->ioproc && iodesc->maxiobuflen > 0)
+        maxbytesoniotask = pio_buffer_size_limit / iodesc->maxiobuflen;
+
+    /* Determine the max bytes that can be held on computation task. */
+    if (ios->comp_rank >= 0 && iodesc->ndof > 0)
+        maxbytesoncomputetask = pio_cnbuffer_limit / iodesc->ndof;
+
+    /* Take the min of the max IO and max comp bytes. */
+    maxbytes = min(maxbytesoniotask, maxbytesoncomputetask);
+    LOG((2, "compute_maxaggregate_bytes maxbytesoniotask = %d maxbytesoncomputetask = %d",
+         maxbytesoniotask, maxbytesoncomputetask));
+
+    /* Get the min value of this on all tasks. */
+    LOG((3, "before allreaduce maxbytes = %d", maxbytes));
+    if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxbytes, 1, MPI_INT, MPI_MIN,
+                                ios->union_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((3, "after allreaduce maxbytes = %d", maxbytes));
+
+    /* Remember the result. */
+    iodesc->maxbytes = maxbytes;
+
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pio_file.c b/src/externals/pio2/src/clib/pio_file.c
new file mode 100644
index 00000000000..c0523997cfd
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_file.c
@@ -0,0 +1,422 @@
+/**
+ * @file
+ * PIO File Handling
+ */
+#include 
+#include 
+#include 
+
+/* This is the next ncid that will be used when a file is opened or
+   created. We start at 16 so that it will be easy for us to notice
+   that it's not netcdf (starts at 4), pnetcdf (starts at 0) or
+   netCDF-4/HDF5 (starts at 65xxx). */
+int pio_next_ncid = 16;
+
+/**
+ * Open an existing file using PIO library.
+ *
+ * If the open fails, try again as netCDF serial before giving
+ * up. Input parameters are read on comp task 0 and ignored elsewhere.
+ *
+ * Note that the file is opened with default fill mode, NOFILL for
+ * pnetcdf, and FILL for netCDF classic and netCDF-4 files.
+ *
+ * @param iosysid : A defined pio system descriptor (input)
+ * @param ncidp : A pio file descriptor (output)
+ * @param iotype : A pio output format (input)
+ * @param filename : The filename to open
+ * @param mode : The netcdf mode for the open operation
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_openfile
+ */
+int PIOc_openfile(int iosysid, int *ncidp, int *iotype, const char *filename,
+                  int mode)
+{
+    return PIOc_openfile_retry(iosysid, ncidp, iotype, filename, mode, 1);
+}
+
+/**
+ * Open an existing file using PIO library.
+ *
+ * Input parameters are read on comp task 0 and ignored elsewhere.
+ *
+ * @param iosysid A defined pio system descriptor
+ * @param path The filename to open
+ * @param mode The netcdf mode for the open operation
+ * @param ncidp pointer to int where ncid will go
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_openfile
+ */
+int PIOc_open(int iosysid, const char *path, int mode, int *ncidp)
+{
+    int iotype;
+
+    LOG((1, "PIOc_open iosysid = %d path = %s mode = %x", iosysid, path, mode));
+
+    /* Figure out the iotype. */
+    if (mode & NC_NETCDF4)
+    {
+        if (mode & NC_MPIIO || mode & NC_MPIPOSIX)
+            iotype = PIO_IOTYPE_NETCDF4P;
+        else
+            iotype = PIO_IOTYPE_NETCDF4C;
+    }
+    else
+    {
+        if (mode & NC_PNETCDF || mode & NC_MPIIO)
+            iotype = PIO_IOTYPE_PNETCDF;
+        else
+            iotype = PIO_IOTYPE_NETCDF;
+    }
+
+    /* Open the file. If the open fails, do not retry as serial
+     * netCDF. Just return the error code. */
+    return PIOc_openfile_retry(iosysid, ncidp, &iotype, path, mode, 0);
+}
+
+/**
+ * Create a new file using pio. Input parameters are read on comp task
+ * 0 and ignored elsewhere. NOFILL mode will be turned on in all
+ * cases.
+ *
+ * @param iosysid A defined pio system ID, obtained from
+ * PIOc_InitIntercomm() or PIOc_InitAsync().
+ * @param ncidp A pointer that gets the ncid of the newly created
+ * file.
+ * @param iotype A pointer to a pio output format. Must be one of
+ * PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, or
+ * PIO_IOTYPE_NETCDF4P.
+ * @param filename The filename to create.
+ * @param mode The netcdf mode for the create operation.
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_createfile
+ */
+int PIOc_createfile(int iosysid, int *ncidp, int *iotype, const char *filename,
+                    int mode)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    int ret;               /* Return code from function calls. */
+
+    /* Get the IO system info from the id. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Create the file. */
+    if ((ret = PIOc_createfile_int(iosysid, ncidp, iotype, filename, mode)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Run this on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. (Because otherwise, in async
+     * mode, set_fill would be called twice by each IO task, since
+     * PIOc_createfile() will already be called on each IO task.) */
+    if (!ios->async || !ios->ioproc)
+    {
+        /* Set the fill mode to NOFILL. */
+        if ((ret = PIOc_set_fill(*ncidp, NC_NOFILL, NULL)))
+            return ret;
+    }
+
+    return ret;
+}
+
+/**
+ * Open a new file using pio. The default fill mode will be used (FILL
+ * for netCDF and netCDF-4 formats, NOFILL for pnetcdf.) Input
+ * parameters are read on comp task 0 and ignored elsewhere.
+ *
+ * @param iosysid : A defined pio system descriptor (input)
+ * @param cmode : The netcdf mode for the create operation.
+ * @param filename : The filename to open
+ * @param ncidp : A pio file descriptor (output)
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_create
+ */
+int PIOc_create(int iosysid, const char *filename, int cmode, int *ncidp)
+{
+    int iotype;            /* The PIO IO type. */
+
+    /* Figure out the iotype. */
+    if (cmode & NC_NETCDF4)
+    {
+        if (cmode & NC_MPIIO || cmode & NC_MPIPOSIX)
+            iotype = PIO_IOTYPE_NETCDF4P;
+        else
+            iotype = PIO_IOTYPE_NETCDF4C;
+    }
+    else
+    {
+        if (cmode & NC_PNETCDF || cmode & NC_MPIIO)
+            iotype = PIO_IOTYPE_PNETCDF;
+        else
+            iotype = PIO_IOTYPE_NETCDF;
+    }
+
+    return PIOc_createfile_int(iosysid, ncidp, &iotype, filename, cmode);
+}
+
+/**
+ * Close a file previously opened with PIO.
+ *
+ * @param ncid: the file pointer
+ * @returns PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_closefile(int ncid)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_closefile ncid = %d", ncid));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Sync changes before closing on all tasks if async is not in
+     * use, but only on non-IO tasks if async is in use. */
+    if (!ios->async || !ios->ioproc)
+        if (file->mode & PIO_WRITE)
+            PIOc_sync(ncid);
+
+    /* If async is in use and this is a comp tasks, then the compmaster
+     * sends a msg to the pio_msg_handler running on the IO master and
+     * waiting for a message. Then broadcast the ncid over the intercomm
+     * to the IO tasks. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_CLOSE_FILE;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        switch (file->iotype)
+        {
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_close(file->fh);
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank == 0)
+                ierr = nc_close(file->fh);
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            if ((file->mode & PIO_WRITE)){
+                ierr = ncmpi_buffer_detach(file->fh);
+            }
+            ierr = ncmpi_close(file->fh);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Delete file from our list of open files. */
+    pio_delete_file_from_list(ncid);
+
+    return ierr;
+}
+
+/**
+ * Delete a file.
+ *
+ * @param iosysid a pio system handle.
+ * @param filename a filename.
+ * @returns PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_deletefile(int iosysid, const char *filename)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+     int msg = PIO_MSG_DELETE_FILE;
+    size_t len;
+
+    LOG((1, "PIOc_deletefile iosysid = %d filename = %s", iosysid, filename));
+
+    /* Get the IO system info from the id. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* If async is in use, send message to IO master task. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            if (ios->comp_rank==0)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            len = strlen(filename);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "Bcast len = %d filename = %s", len, filename));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((3, "done hanlding errors mpierr = %d", mpierr));
+    }
+
+    /* If this is an IO task, then call the netCDF function. The
+     * barriers are needed to assure that no task is trying to operate
+     * on the file while it is being deleted. IOTYPE is not known, but
+     * nc_delete() will delete any type of file. */
+    if (ios->ioproc)
+    {
+        mpierr = MPI_Barrier(ios->io_comm);
+
+        if (!mpierr && ios->io_rank == 0)
+             ierr = nc_delete(filename);
+
+        if (!mpierr)
+            mpierr = MPI_Barrier(ios->io_comm);
+    }
+    LOG((2, "PIOc_deletefile ierr = %d", ierr));
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
+
+    return ierr;
+}
+
+/**
+ * PIO interface to nc_sync This routine is called collectively by all
+ * tasks in the communicator ios.union_comm.
+ *
+ * Refer to the  netcdf  documentation.
+ *
+ * @param ncid the ncid of the file to sync.
+ * @returns PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_sync(int ncid)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    wmulti_buffer *wmb, *twmb;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+
+    /* Get the file info from the ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, send message to IO master tasks. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_SYNC;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (file->mode & PIO_WRITE)
+    {
+        LOG((3, "PIOc_sync checking buffers"));
+
+        /*  cn_buffer_report( *ios, true); */
+        wmb = &file->buffer;
+        while (wmb)
+        {
+            /* If there are any data arrays waiting in the
+             * multibuffer, flush it. */
+            if (wmb->num_arrays > 0)
+                flush_buffer(ncid, wmb, true);
+            twmb = wmb;
+            wmb = wmb->next;
+            if (twmb == &file->buffer)
+            {
+                twmb->ioid = -1;
+                twmb->next = NULL;
+            }
+            else
+            {
+                brel(twmb);
+            }
+        }
+
+        if (ios->ioproc)
+        {
+            switch(file->iotype)
+            {
+#ifdef _NETCDF4
+            case PIO_IOTYPE_NETCDF4P:
+                ierr = nc_sync(file->fh);
+                break;
+            case PIO_IOTYPE_NETCDF4C:
+#endif
+            case PIO_IOTYPE_NETCDF:
+                if (ios->io_rank == 0)
+                    ierr = nc_sync(file->fh);
+                break;
+#ifdef _PNETCDF
+            case PIO_IOTYPE_PNETCDF:
+                flush_output_buffer(file, true, 0);
+                ierr = ncmpi_sync(file->fh);
+                break;
+#endif
+            default:
+                return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+            }
+        }
+        LOG((2, "PIOc_sync ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
+
+    return ierr;
+}
diff --git a/src/externals/pio2/src/clib/pio_get_nc.c b/src/externals/pio2/src/clib/pio_get_nc.c
new file mode 100644
index 00000000000..2cd85d8f947
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_get_nc.c
@@ -0,0 +1,1119 @@
+/**
+ * @file
+ * PIO functions to get data (excluding varm functions).
+ *
+ * @author Ed Hartnett
+ * @date  2016
+ *
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#include 
+#include 
+#include 
+
+/**
+ * Get strided, muti-dimensional subset of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                       const PIO_Offset *stride, char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_CHAR, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_uchar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride, unsigned char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_UBYTE, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_schar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride, signed char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_BYTE, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned 16-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_ushort(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, const PIO_Offset *stride, unsigned short *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_USHORT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_short(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride, short *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_SHORT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_uint(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, const PIO_Offset *stride, unsigned int *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_UINT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, int *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_INT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit int variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_long(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, const PIO_Offset *stride, long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, PIO_LONG_INTERNAL, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_float(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride, float *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_FLOAT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit floating point
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_double(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, const PIO_Offset *stride, double *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_DOUBLE, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned 64-bit int
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const PIO_Offset *stride,
+                            unsigned long long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, PIO_LONG_INTERNAL, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit int variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_longlong(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const PIO_Offset *stride, long long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_INT64, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_text(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_CHAR, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_uchar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, unsigned char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_UBYTE, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_schar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, signed char *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_BYTE, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned 16-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_ushort(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, unsigned short *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_USHORT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_short(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, short *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_SHORT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_long(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, PIO_LONG_INTERNAL, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_uint(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, unsigned int *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_UINT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_int(int ncid, int varid, const PIO_Offset *start,
+                      const PIO_Offset *count, int *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_INT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_float(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, float *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_FLOAT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a 64-bit floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_double(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, double *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_DOUBLE, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of an unsigned 64-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, unsigned long long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_UINT64, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara_longlong(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, long long *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_INT64, buf);
+}
+
+/**
+ * Get all data of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_text(int ncid, int varid, char *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_CHAR, buf);
+}
+
+/**
+ * Get all data of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_uchar(int ncid, int varid, unsigned char *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_UBYTE, buf);
+}
+
+/**
+ * Get all data of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_schar(int ncid, int varid, signed char *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_BYTE, buf);
+}
+
+/**
+ * Get all data of an unsigned 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_ushort(int ncid, int varid, unsigned short *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_USHORT, buf);
+}
+
+/**
+ * Get all data of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_short(int ncid, int varid, short *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_SHORT, buf);
+}
+
+/**
+ * Get all data of an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_uint(int ncid, int varid, unsigned int *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_UINT, buf);
+}
+
+/**
+ * Get all data of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_int(int ncid, int varid, int *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_INT, buf);
+}
+
+/**
+ * Get all data of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_long (int ncid, int varid, long *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, PIO_LONG_INTERNAL, buf);
+}
+
+/**
+ * Get all data of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_float(int ncid, int varid, float *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_FLOAT, buf);
+}
+
+/**
+ * Get all data of a 64-bit floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_double(int ncid, int varid, double *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_DOUBLE, buf);
+}
+
+/**
+ * Get all data of an unsigned 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_ulonglong(int ncid, int varid, unsigned long long *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_UINT64, buf);
+}
+
+/**
+ * Get all data of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_longlong(int ncid, int varid, long long *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_INT64, buf);
+}
+
+/**
+ * Get one value of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_text(int ncid, int varid, const PIO_Offset *index, char *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_CHAR, buf);
+}
+
+/**
+ * Get one value of an unsinged char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_uchar (int ncid, int varid, const PIO_Offset *index, unsigned char *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_UBYTE, buf);
+}
+
+/**
+ * Get one value of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_schar(int ncid, int varid, const PIO_Offset *index, signed char *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_BYTE, buf);
+}
+
+/**
+ * Get one value of an unsigned 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_ushort(int ncid, int varid, const PIO_Offset *index, unsigned short *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_USHORT, buf);
+}
+
+/**
+ * Get one value of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_short(int ncid, int varid, const PIO_Offset *index, short *buf)
+{
+    int ret = PIOc_get_var1_tc(ncid, varid, index, NC_SHORT, buf);
+    LOG((1, "PIOc_get_var1_short returned %d", ret));
+    return ret;
+}
+
+/**
+ * Get one value of an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_uint(int ncid, int varid, const PIO_Offset *index, unsigned int *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_UINT, buf);
+}
+
+/**
+ * Get one value of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_long (int ncid, int varid, const PIO_Offset *index, long *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, PIO_LONG_INTERNAL, buf);
+}
+
+/**
+ * Get one value of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_int(int ncid, int varid, const PIO_Offset *index, int *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_INT, buf);
+}
+
+/**
+ * Get one value of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_float(int ncid, int varid, const PIO_Offset *index, float *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_FLOAT, buf);
+}
+
+/**
+ * Get one value of a 64-bit floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_double (int ncid, int varid, const PIO_Offset *index, double *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_DOUBLE, buf);
+}
+
+/**
+ * Get one value of an unsigned 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_ulonglong (int ncid, int varid, const PIO_Offset *index,
+                             unsigned long long *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_INT64, buf);
+}
+
+/**
+ * Get one value of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_longlong(int ncid, int varid, const PIO_Offset *index,
+                           long long *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_INT64, buf);
+}
+
+/**
+ * Get all data from a variable the same type as the variable in the
+ * file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var(int ncid, int varid, void *buf)
+{
+    return PIOc_get_var_tc(ncid, varid, NC_NAT, buf);
+}
+
+/**
+ * Get one value from a variable the same type as the variable in the
+ * file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1(int ncid, int varid, const PIO_Offset *index, void *buf)
+{
+    return PIOc_get_var1_tc(ncid, varid, index, NC_NAT, buf);
+}
+
+/**
+ * Get a muti-dimensional subset of a variable the same type
+ * as the variable in the file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                  void *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, NULL, NC_NAT, buf);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a variable of the same type
+ * as the variable in the file.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                  const PIO_Offset *stride, void *buf)
+{
+    return PIOc_get_vars_tc(ncid, varid, start, count, stride, NC_NAT, buf);
+}
diff --git a/src/externals/pio2/src/clib/pio_getput_int.c b/src/externals/pio2/src/clib/pio_getput_int.c
new file mode 100644
index 00000000000..187f74d058d
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_getput_int.c
@@ -0,0 +1,1329 @@
+/**
+ * @file
+ * Internal PIO functions to get and put attributes and data
+ * (excluding varm functions).
+ *
+ * @author Ed Hartnett
+ * @date  2016
+ *
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#include 
+#include 
+#include 
+
+/**
+ * Write a netCDF attribute of any type, converting to any type.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param atttype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype,
+                    PIO_Offset len, nc_type memtype, const void *op)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    PIO_Offset atttype_len;    /* Length (in bytes) of the att type in file. */
+    PIO_Offset memtype_len;    /* Length of the att data type in memory. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr;           /* Return code from function calls. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide some valid parameters. */
+    if (!name || !op || strlen(name) > NC_MAX_NAME || len < 0)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_put_att_tc ncid = %d varid = %d name = %s atttype = %d len = %d memtype = %d",
+         ncid, varid, name, atttype, len, memtype));
+
+    /* Run these on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. */
+    if (!ios->async || !ios->ioproc)
+    {
+        /* Get the length (in bytes) of the type in file. */
+        if ((ierr = PIOc_inq_type(ncid, atttype, NULL, &atttype_len)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        /* Get the length (in bytes) of the type in memory. */
+        if (memtype == PIO_LONG_INTERNAL)
+            memtype_len = sizeof(long int);
+        else
+        {
+            if ((ierr = PIOc_inq_type(ncid, memtype, NULL, &memtype_len)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+        }
+        LOG((2, "PIOc_put_att atttype_len = %d memtype_len = %d", ncid, atttype_len, memtype_len));
+    }
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_PUT_ATT;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            int namelen = strlen(name);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&atttype, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len, 1, MPI_OFFSET,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&memtype, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)op, len * memtype_len, MPI_BYTE, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "PIOc_put_att finished bcast ncid = %d varid = %d namelen = %d name = %s "
+                 "len = %d atttype_len = %d memtype = %d memtype_len = %d", ncid, varid, namelen,
+                 name, len, atttype_len, memtype, memtype_len));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        LOG((2, "PIOc_put_att bcast from comproot = %d atttype_len = %d", ios->comproot,
+             atttype_len, memtype_len));
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            switch(memtype)
+            {
+            case NC_BYTE:
+                ierr = ncmpi_put_att_schar(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_CHAR:
+                ierr = ncmpi_put_att_text(file->fh, varid, name, len, op);
+                break;
+            case NC_SHORT:
+                ierr = ncmpi_put_att_short(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_INT:
+                ierr = ncmpi_put_att_int(file->fh, varid, name, atttype, len, op);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = ncmpi_put_att_long(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_FLOAT:
+                ierr = ncmpi_put_att_float(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_DOUBLE:
+                ierr = ncmpi_put_att_double(file->fh, varid, name, atttype, len, op);
+                break;
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            switch(memtype)
+            {
+            case NC_CHAR:
+                ierr = nc_put_att_text(file->fh, varid, name, len, op);
+                break;
+            case NC_BYTE:
+                ierr = nc_put_att_schar(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_SHORT:
+                ierr = nc_put_att_short(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_INT:
+                ierr = nc_put_att_int(file->fh, varid, name, atttype, len, op);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = nc_put_att_long(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_FLOAT:
+                ierr = nc_put_att_float(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_DOUBLE:
+                ierr = nc_put_att_double(file->fh, varid, name, atttype, len, op);
+                break;
+#ifdef _NETCDF4
+            case NC_UBYTE:
+                ierr = nc_put_att_uchar(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_USHORT:
+                ierr = nc_put_att_ushort(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_UINT:
+                ierr = nc_put_att_uint(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_INT64:
+                LOG((3, "about to call nc_put_att_longlong"));
+                ierr = nc_put_att_longlong(file->fh, varid, name, atttype, len, op);
+                break;
+            case NC_UINT64:
+                ierr = nc_put_att_ulonglong(file->fh, varid, name, atttype, len, op);
+                break;
+                /* case NC_STRING: */
+                /*      ierr = nc_put_att_string(file->fh, varid, name, atttype, len, op); */
+                /*      break; */
+#endif /* _NETCDF4 */
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get the value of an attribute of any type, converting to any type.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param memtype the type of the data in memory (if different from
+ * the type of the attribute, the data will be converted to
+ * memtype). The ip pointer points to memory to hold att_len elements
+ * of type memtype.
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void *ip)
+{
+    iosystem_desc_t *ios;   /* Pointer to io system information. */
+    file_desc_t *file;      /* Pointer to file information. */
+    nc_type atttype;        /* The type of the attribute. */
+    PIO_Offset attlen;      /* Number of elements in the attribute array. */
+    PIO_Offset atttype_len; /* Length in bytes of one element of the attribute type. */
+    PIO_Offset memtype_len; /* Length in bytes of one element of the memory type. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function calls. */
+    int ierr;               /* Return code from function calls. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide a name and destination pointer. */
+    if (!name || !ip || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_get_att_tc ncid %d varid %d name %s memtype %d",
+         ncid, varid, name, memtype));
+
+    /* Run these on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. */
+    if (!ios->async || !ios->ioproc)
+    {
+        /* Get the type and length of the attribute. */
+        if ((ierr = PIOc_inq_att(ncid, varid, name, &atttype, &attlen)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+        LOG((2, "atttype = %d attlen = %d", atttype, attlen));
+
+        /* Get the length (in bytes) of the type of the attribute. */
+        if ((ierr = PIOc_inq_type(ncid, atttype, NULL, &atttype_len)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        /* Get the length (in bytes) of the type that the user wants
+         * the data converted to. */
+        if (memtype == PIO_LONG_INTERNAL)
+            memtype_len = sizeof(long int);
+        else
+        {
+            if ((ierr = PIOc_inq_type(ncid, memtype, NULL, &memtype_len)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+        }
+    }
+    LOG((2, "atttype_len = %d memtype_len = %d", atttype_len, memtype_len));
+
+    /* If async is in use, and this is not an IO task, bcast the
+     * parameters and the attribute and type information we fetched. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_GET_ATT;
+            LOG((2, "sending parameters"));
+
+            /* Send the message to IO master. */
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the function parameters. */
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            int namelen = strlen(name);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&atttype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&memtype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            LOG((2, "Bcast complete ncid = %d varid = %d namelen = %d name = %s iotype = %d "
+                 "atttype = %d attlen = %d atttype_len = %d", ncid, varid, namelen, name, file->iotype,
+                 atttype, attlen, atttype_len));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        LOG((2, "mpi errors handled"));
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        LOG((2, "PIOc_get_att_tc bcast from comproot = %d attlen = %d atttype_len = %d", ios->comproot, attlen, atttype_len));
+        if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        LOG((2, "PIOc_get_att_tc bcast complete attlen = %d atttype_len = %d memtype_len = %d", attlen, atttype_len,
+             memtype_len));
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        LOG((2, "calling pnetcdf/netcdf"));
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            switch(memtype)
+            {
+            case NC_BYTE:
+                ierr = ncmpi_get_att_schar(file->fh, varid, name, ip);
+                break;
+            case NC_CHAR:
+                ierr = ncmpi_get_att_text(file->fh, varid, name, ip);
+                break;
+            case NC_SHORT:
+                ierr = ncmpi_get_att_short(file->fh, varid, name, ip);
+                break;
+            case NC_INT:
+                ierr = ncmpi_get_att_int(file->fh, varid, name, ip);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = ncmpi_get_att_long(file->fh, varid, name, ip);
+                break;
+            case NC_FLOAT:
+                ierr = ncmpi_get_att_float(file->fh, varid, name, ip);
+                break;
+            case NC_DOUBLE:
+                ierr = ncmpi_get_att_double(file->fh, varid, name, ip);
+                break;
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            switch(memtype)
+            {
+            case NC_CHAR:
+                ierr = nc_get_att_text(file->fh, varid, name, ip);
+                break;
+            case NC_BYTE:
+                ierr = nc_get_att_schar(file->fh, varid, name, ip);
+                break;
+            case NC_SHORT:
+                ierr = nc_get_att_short(file->fh, varid, name, ip);
+                break;
+            case NC_INT:
+                ierr = nc_get_att_int(file->fh, varid, name, ip);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = nc_get_att_long(file->fh, varid, name, ip);
+                break;
+            case NC_FLOAT:
+                ierr = nc_get_att_float(file->fh, varid, name, ip);
+                break;
+            case NC_DOUBLE:
+                ierr = nc_get_att_double(file->fh, varid, name, ip);
+                break;
+#ifdef _NETCDF4
+            case NC_UBYTE:
+                ierr = nc_get_att_uchar(file->fh, varid, name, ip);
+                break;
+            case NC_USHORT:
+                ierr = nc_get_att_ushort(file->fh, varid, name, ip);
+                break;
+            case NC_UINT:
+                ierr = nc_get_att_uint(file->fh, varid, name, ip);
+                break;
+            case NC_INT64:
+                LOG((3, "about to call nc_get_att_longlong"));
+                ierr = nc_get_att_longlong(file->fh, varid, name, ip);
+                break;
+            case NC_UINT64:
+                ierr = nc_get_att_ulonglong(file->fh, varid, name, ip);
+                break;
+                /* case NC_STRING: */
+                /*      ierr = nc_get_att_string(file->fh, varid, name, ip); */
+                /*      break; */
+#endif /* _NETCDF4 */
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    LOG((2, "ierr = %d", ierr));
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    LOG((2, "bcasting att values attlen = %d memtype_len = %d", attlen, memtype_len));
+    if ((mpierr = MPI_Bcast(ip, (int)attlen * memtype_len, MPI_BYTE, ios->ioroot,
+                            ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "get_att_tc data bcast complete"));
+    return PIO_NOERR;
+}
+
+/**
+ * Internal PIO function which provides a type-neutral interface to
+ * nc_get_vars.
+ *
+ * Users should not call this function directly. Instead, call one of
+ * the derived functions, depending on the type of data you are
+ * reading: PIOc_get_vars_text(), PIOc_get_vars_uchar(),
+ * PIOc_get_vars_schar(), PIOc_get_vars_ushort(),
+ * PIOc_get_vars_short(), PIOc_get_vars_uint(), PIOc_get_vars_int(),
+ * PIOc_get_vars_long(), PIOc_get_vars_float(),
+ * PIOc_get_vars_double(), PIOc_get_vars_ulonglong(),
+ * PIOc_get_vars_longlong()
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param xtype the netCDF type of the data being passed in buf. Data
+ * will be automatically covnerted from the type of the variable being
+ * read from to this type. If NC_NAT then the variable's file type
+ * will be used. Use special PIO_LONG_INTERNAL for _long() functions.
+ * @param buf pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                     const PIO_Offset *stride, nc_type xtype, void *buf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ndims;             /* The number of dimensions in the variable. */
+    PIO_Offset typelen;    /* Size (in bytes) of the data type of data in buf. */
+    PIO_Offset num_elem = 1; /* Number of data elements in the buffer. */
+    nc_type vartype;         /* The type of the var we are reading from. */
+    char start_present = start ? true : false;
+    char count_present = count ? true : false;
+    char stride_present = stride ? true : false;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr;                           /* Return code. */
+
+    LOG((1, "PIOc_get_vars_tc ncid = %d varid = %d xtype = %d start_present = %d "
+         "count_present = %d stride_present = %d", ncid, varid, xtype, start_present,
+         count_present, stride_present));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide a place to put some data. */
+    if (!buf)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Run these on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. */
+    if (!ios->async || !ios->ioproc)
+    {
+        /* Get the type of this var. */
+        if ((ierr = PIOc_inq_vartype(ncid, varid, &vartype)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        /* If no type was specified, use the var type. */
+        if (xtype == NC_NAT)
+            xtype = vartype;
+
+        /* Handle _long() calls with an special type. */
+        if (xtype == PIO_LONG_INTERNAL)
+            typelen = sizeof(long int);
+        else
+        {
+            if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+        }
+
+        /* Get the number of dims for this var. */
+        if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+        LOG((3, "ndims = %d", ndims));
+
+        /* Only scalar vars can pass NULL for start/count. */
+        pioassert(ndims == 0 || (start && count), "need start/count", __FILE__, __LINE__);
+
+        /* How many elements in buf? (For scalars, ndims is 0 and
+         * num_elem will remain 1). */
+        for (int vd = 0; vd < ndims; vd++)
+            num_elem *= count[vd];
+        LOG((2, "PIOc_get_vars_tc num_elem = %d", num_elem));
+    }
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_GET_VARS;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the function parameters and associated informaiton
+             * to the msg handler. */
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && start_present)
+                mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && count_present)
+                mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && stride_present)
+                mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_get_vars_tc ncid = %d varid = %d ndims = %d start_present = %d "
+                 "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid,
+                 ndims, start_present, count_present, stride_present, xtype, num_elem));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        LOG((2, "file->iotype = %d xtype = %d file->do_io = %d", file->iotype, xtype, file->do_io));
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            LOG((2, "pnetcdf calling ncmpi_get_vars_*() file->fh = %d varid = %d", file->fh, varid));
+            /* Turn on independent access for pnetcdf file. */
+            if ((ierr = ncmpi_begin_indep_data(file->fh)))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+            /* Only the IO master does the IO, so we are not really
+             * getting parallel IO here. */
+            if (ios->iomaster == MPI_ROOT)
+            {
+                switch(xtype)
+                {
+                case NC_BYTE:
+                    ierr = ncmpi_get_vars_schar(file->fh, varid, start, count, stride, buf);
+                    break;
+                case NC_CHAR:
+                    ierr = ncmpi_get_vars_text(file->fh, varid, start, count, stride, buf);
+                    break;
+                case NC_SHORT:
+                    ierr = ncmpi_get_vars_short(file->fh, varid, start, count, stride, buf);
+                    break;
+                case NC_INT:
+                    ierr = ncmpi_get_vars_int(file->fh, varid, start, count, stride, buf);
+                    break;
+                case PIO_LONG_INTERNAL:
+                    ierr = ncmpi_get_vars_long(file->fh, varid, start, count, stride, buf);
+                    break;
+                case NC_FLOAT:
+                    ierr = ncmpi_get_vars_float(file->fh, varid, start, count, stride, buf);
+                    break;
+                case NC_DOUBLE:
+                    ierr = ncmpi_get_vars_double(file->fh, varid, start, count, stride, buf);
+                    break;
+                default:
+                    return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+                }
+            }
+
+            /* Turn off independent access for pnetcdf file. */
+            if ((ierr = ncmpi_end_indep_data(file->fh)))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            switch(xtype)
+            {
+            case NC_BYTE:
+                ierr = nc_get_vars_schar(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_CHAR:
+                ierr = nc_get_vars_text(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_SHORT:
+                ierr = nc_get_vars_short(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_INT:
+                ierr = nc_get_vars_int(file->fh, varid, (size_t *)start, (size_t *)count,
+                                       (ptrdiff_t *)stride, buf);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = nc_get_vars_long(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_FLOAT:
+                ierr = nc_get_vars_float(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_DOUBLE:
+                ierr = nc_get_vars_double(file->fh, varid, (size_t *)start, (size_t *)count,
+                                          (ptrdiff_t *)stride, buf);
+                break;
+#ifdef _NETCDF4
+            case NC_UBYTE:
+                ierr = nc_get_vars_uchar(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_USHORT:
+                ierr = nc_get_vars_ushort(file->fh, varid, (size_t *)start, (size_t *)count,
+                                          (ptrdiff_t *)stride, buf);
+                break;
+            case NC_UINT:
+                ierr = nc_get_vars_uint(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_INT64:
+                LOG((3, "about to call nc_get_vars_longlong"));
+                ierr = nc_get_vars_longlong(file->fh, varid, (size_t *)start, (size_t *)count,
+                                            (ptrdiff_t *)stride, buf);
+                break;
+            case NC_UINT64:
+                ierr = nc_get_vars_ulonglong(file->fh, varid, (size_t *)start, (size_t *)count,
+                                             (ptrdiff_t *)stride, buf);
+                break;
+                /* case NC_STRING: */
+                /*      ierr = nc_get_vars_string(file->fh, varid, (size_t *)start, (size_t *)count, */
+                /*                                (ptrdiff_t *)stride, (void *)buf); */
+                /*      break; */
+#endif /* _NETCDF4 */
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Send the data. */
+    LOG((2, "PIOc_get_vars_tc bcasting data num_elem = %d typelen = %d ios->ioroot = %d", num_elem,
+         typelen, ios->ioroot));
+    if ((mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    LOG((2, "PIOc_get_vars_tc bcasting data complete"));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get one value of a variable of any type. This is an internal
+ * function.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param xtype the netcdf type of the variable.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
+                     void *buf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ndims;   /* The number of dimensions in the variable. */
+    int ierr;    /* Return code from function calls. */
+
+    /* Find the info about this file. We need this for error handling. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Find the number of dimensions. */
+    if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Set up count array. */
+    PIO_Offset count[ndims];
+    for (int c = 0; c < ndims; c++)
+        count[c] = 1;
+
+    return PIOc_get_vars_tc(ncid, varid, index, count, NULL, xtype, buf);
+}
+
+/**
+ * Get a complete variable of any type. This is an internal function.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param xtype the netcdf type of the variable.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    PIO_Offset *startp = NULL; /* Pointer to start array. */
+    PIO_Offset *countp = NULL; /* Pointer to count array. */
+    int ndims;   /* The number of dimensions in the variable. */
+    PIO_Offset my_start[PIO_MAX_DIMS];
+    PIO_Offset dimlen[PIO_MAX_DIMS];
+    int ierr;    /* Return code from function calls. */
+
+    LOG((1, "PIOc_get_var_tc ncid = %d varid = %d xtype = %d", ncid, varid,
+         xtype));
+
+    /* Find the info about this file. We need this for error handling. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Find the number of dimensions. */
+    if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Scalar vars (which have ndims == 0) should just pass NULLs for
+     * start/count. */
+    if (ndims)
+    {
+        /* Find the dimension IDs. */
+        int dimids[ndims];
+        if ((ierr = PIOc_inq_vardimid(ncid, varid, dimids)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+        /* Find the dimension lengths. */
+        for (int d = 0; d < ndims; d++)
+            if ((ierr = PIOc_inq_dimlen(ncid, dimids[d], &dimlen[d])))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+        /* Set up start array. */
+        for (int d = 0; d < ndims; d++)
+        {
+            my_start[d] = 0;
+            LOG((3, "my_start[%d] = %d dimlen[%d] = %d", d, my_start[d], d,
+                 dimlen[d]));
+        }
+
+        /* Set the start/count arrays. */
+        startp = my_start;
+        countp = dimlen;
+    }
+
+    return PIOc_get_vars_tc(ncid, varid, startp, countp, NULL, xtype, buf);
+}
+
+/**
+ * Internal PIO function which provides a type-neutral interface to
+ * nc_put_vars.
+ *
+ * Users should not call this function directly. Instead, call one of
+ * the derived functions, depending on the type of data you are
+ * writing: PIOc_put_vars_text(), PIOc_put_vars_uchar(),
+ * PIOc_put_vars_schar(), PIOc_put_vars_ushort(),
+ * PIOc_put_vars_short(), PIOc_put_vars_uint(), PIOc_put_vars_int(),
+ * PIOc_put_vars_long(), PIOc_put_vars_float(),
+ * PIOc_put_vars_longlong(), PIOc_put_vars_double(),
+ * PIOc_put_vars_ulonglong().
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param xtype the netCDF type of the data being passed in buf. Data
+ * will be automatically covnerted from this type to the type of the
+ * variable being written to. If NC_NAT then the variable's file type
+ * will be used. Use special PIO_LONG_INTERNAL for _long() functions.
+ * @param buf pointer to the data to be written.
+ *
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                     const PIO_Offset *stride, nc_type xtype, const void *buf)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;  /* Pointer to file information. */
+    int ndims;          /* The number of dimensions in the variable. */
+    PIO_Offset typelen; /* Size (in bytes) of the data type of data in buf. */
+    PIO_Offset num_elem = 1; /* Number of data elements in the buffer. */
+    char start_present = start ? true : false;    /* Is start non-NULL? */
+    char count_present = count ? true : false;    /* Is count non-NULL? */
+    char stride_present = stride ? true : false;  /* Is stride non-NULL? */
+    var_desc_t *vdesc;
+    int *request;
+    nc_type vartype;   /* The type of the var we are reading from. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr;          /* Return code from function calls. */
+
+    LOG((1, "PIOc_put_vars_tc ncid = %d varid = %d start_present = %d "
+         "count_present = %d stride_present = %d xtype = %d", ncid, varid,
+         start_present, count_present, stride_present, xtype));
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide a place to put some data. */
+    if (!buf)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Run these on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. */
+    if (!ios->async || !ios->ioproc)
+    {
+        /* Get the type of this var. */
+        if ((ierr = PIOc_inq_vartype(ncid, varid, &vartype)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        /* If no type was specified, use the var type. */
+        if (xtype == NC_NAT)
+            xtype = vartype;
+
+        /* Get the number of dims for this var. */
+        if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        /* Get the length of the data type. */
+        if (xtype == PIO_LONG_INTERNAL)
+            typelen = sizeof(long int);
+        else
+        {
+            if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &typelen)))
+                return check_netcdf(file, ierr, __FILE__, __LINE__);
+        }
+
+        LOG((2, "ndims = %d typelen = %d", ndims, typelen));
+
+        /* How many elements of data? If no count array was passed,
+         * this is a scalar. */
+        if (count)
+            for (int vd = 0; vd < ndims; vd++)
+                num_elem *= count[vd];
+    }
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_PUT_VARS;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the function parameters and associated informaiton
+             * to the msg handler. */
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && start_present)
+                mpierr = MPI_Bcast((PIO_Offset *)start, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && count_present)
+                mpierr = MPI_Bcast((PIO_Offset *)count, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && stride_present)
+                mpierr = MPI_Bcast((PIO_Offset *)stride, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_put_vars_tc ncid = %d varid = %d ndims = %d start_present = %d "
+                 "count_present = %d stride_present = %d xtype = %d num_elem = %d", ncid, varid,
+                 ndims, start_present, count_present, stride_present, xtype, num_elem));
+
+            /* Send the data. */
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)buf, num_elem * typelen, MPI_BYTE, ios->compmaster,
+                                   ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        LOG((2, "PIOc_put_vars_tc checked mpierr = %d", mpierr));
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        LOG((2, "PIOc_put_vars_tc bcast from comproot"));
+        if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        LOG((2, "PIOc_put_vars_tc complete bcast from comproot ndims = %d", ndims));
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            /* Scalars have to be handled differently. */
+            if (ndims == 0)
+            {
+                /* This is a scalar var. */
+                LOG((2, "pnetcdf writing scalar with ncmpi_put_vars_*() file->fh = %d varid = %d",
+                     file->fh, varid));
+                pioassert(!start && !count && !stride, "expected NULLs", __FILE__, __LINE__);
+
+                /* Turn on independent access for pnetcdf file. */
+                if ((ierr = ncmpi_begin_indep_data(file->fh)))
+                    return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+                /* Only the IO master does the IO, so we are not really
+                 * getting parallel IO here. */
+                if (ios->iomaster == MPI_ROOT)
+                {
+                    switch(xtype)
+                    {
+                    case NC_BYTE:
+                        ierr = ncmpi_put_vars_schar(file->fh, varid, start, count, stride, buf);
+                        break;
+                    case NC_CHAR:
+                        ierr = ncmpi_put_vars_text(file->fh, varid, start, count, stride, buf);
+                        break;
+                    case NC_SHORT:
+                        ierr = ncmpi_put_vars_short(file->fh, varid, start, count, stride, buf);
+                        break;
+                    case NC_INT:
+                        ierr = ncmpi_put_vars_int(file->fh, varid, start, count, stride, buf);
+                        break;
+                    case PIO_LONG_INTERNAL:
+                        ierr = ncmpi_put_vars_long(file->fh, varid, start, count, stride, buf);
+                        break;
+                    case NC_FLOAT:
+                        ierr = ncmpi_put_vars_float(file->fh, varid, start, count, stride, buf);
+                        break;
+                    case NC_DOUBLE:
+                        ierr = ncmpi_put_vars_double(file->fh, varid, start, count, stride, buf);
+                        break;
+                    default:
+                        return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+                    }
+                }
+
+                /* Turn off independent access for pnetcdf file. */
+                if ((ierr = ncmpi_end_indep_data(file->fh)))
+                    return pio_err(ios, file, ierr, __FILE__, __LINE__);
+            }
+            else
+            {
+                /* This is not a scalar var. */
+                PIO_Offset *fake_stride;
+
+                if (!stride_present)
+                {
+                    LOG((2, "stride not present"));
+                    if (!(fake_stride = malloc(ndims * sizeof(PIO_Offset))))
+                        return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+                    for (int d = 0; d < ndims; d++)
+                        fake_stride[d] = 1;
+                }
+                else
+                    fake_stride = (PIO_Offset *)stride;
+
+                LOG((2, "PIOc_put_vars_tc calling pnetcdf function"));
+                vdesc = &file->varlist[varid];
+                if (vdesc->nreqs % PIO_REQUEST_ALLOC_CHUNK == 0)
+                    if (!(vdesc->request = realloc(vdesc->request,
+                                                   sizeof(int) * (vdesc->nreqs + PIO_REQUEST_ALLOC_CHUNK))))
+                        return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+                request = vdesc->request + vdesc->nreqs;
+                LOG((2, "PIOc_put_vars_tc request = %d", vdesc->request));
+
+                /* Only the IO master actually does the call. */
+                if (ios->iomaster == MPI_ROOT)
+                {
+                    switch(xtype)
+                    {
+                    case NC_BYTE:
+                        ierr = ncmpi_bput_vars_schar(file->fh, varid, start, count, fake_stride, buf, request);
+                        break;
+                    case NC_CHAR:
+                        ierr = ncmpi_bput_vars_text(file->fh, varid, start, count, fake_stride, buf, request);
+                        break;
+                    case NC_SHORT:
+                        ierr = ncmpi_bput_vars_short(file->fh, varid, start, count, fake_stride, buf, request);
+                        break;
+                    case NC_INT:
+                        ierr = ncmpi_bput_vars_int(file->fh, varid, start, count, fake_stride, buf, request);
+                        break;
+                    case PIO_LONG_INTERNAL:
+                        ierr = ncmpi_bput_vars_long(file->fh, varid, start, count, fake_stride, buf, request);
+                        break;
+                    case NC_FLOAT:
+                        ierr = ncmpi_bput_vars_float(file->fh, varid, start, count, fake_stride, buf, request);
+                        break;
+                    case NC_DOUBLE:
+                        ierr = ncmpi_bput_vars_double(file->fh, varid, start, count, fake_stride, buf, request);
+                        break;
+                    default:
+                        return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+                    }
+                    LOG((2, "PIOc_put_vars_tc io_rank 0 done with pnetcdf call, ierr=%d", ierr));
+                }
+                else
+                    *request = PIO_REQ_NULL;
+
+                vdesc->nreqs++;
+                flush_output_buffer(file, false, 0);
+                LOG((2, "PIOc_put_vars_tc flushed output buffer"));
+
+                /* Free malloced resources. */
+                if (!stride_present)
+                    free(fake_stride);
+            } /* endif ndims == 0 */
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            LOG((2, "PIOc_put_vars_tc calling netcdf function file->iotype = %d",
+                 file->iotype));
+            switch(xtype)
+            {
+            case NC_BYTE:
+                ierr = nc_put_vars_schar(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_CHAR:
+                ierr = nc_put_vars_text(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_SHORT:
+                ierr = nc_put_vars_short(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_INT:
+                ierr = nc_put_vars_int(file->fh, varid, (size_t *)start, (size_t *)count,
+                                       (ptrdiff_t *)stride, buf);
+                break;
+            case PIO_LONG_INTERNAL:
+                ierr = nc_put_vars_long(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_FLOAT:
+                ierr = nc_put_vars_float(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_DOUBLE:
+                ierr = nc_put_vars_double(file->fh, varid, (size_t *)start, (size_t *)count,
+                                          (ptrdiff_t *)stride, buf);
+                break;
+#ifdef _NETCDF4
+            case NC_UBYTE:
+                ierr = nc_put_vars_uchar(file->fh, varid, (size_t *)start, (size_t *)count,
+                                         (ptrdiff_t *)stride, buf);
+                break;
+            case NC_USHORT:
+                ierr = nc_put_vars_ushort(file->fh, varid, (size_t *)start, (size_t *)count,
+                                          (ptrdiff_t *)stride, buf);
+                break;
+            case NC_UINT:
+                ierr = nc_put_vars_uint(file->fh, varid, (size_t *)start, (size_t *)count,
+                                        (ptrdiff_t *)stride, buf);
+                break;
+            case NC_INT64:
+                ierr = nc_put_vars_longlong(file->fh, varid, (size_t *)start, (size_t *)count,
+                                            (ptrdiff_t *)stride, buf);
+                break;
+            case NC_UINT64:
+                ierr = nc_put_vars_ulonglong(file->fh, varid, (size_t *)start, (size_t *)count,
+                                             (ptrdiff_t *)stride, buf);
+                break;
+                /* case NC_STRING: */
+                /*      ierr = nc_put_vars_string(file->fh, varid, (size_t *)start, (size_t *)count, */
+                /*                                (ptrdiff_t *)stride, (void *)buf); */
+                /*      break; */
+#endif /* _NETCDF4 */
+            default:
+                return pio_err(ios, file, PIO_EBADTYPE, __FILE__, __LINE__);
+            }
+            LOG((2, "PIOc_put_vars_tc io_rank 0 done with netcdf call, ierr=%d", ierr));
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+    LOG((2, "PIOc_put_vars_tc bcast netcdf return code %d complete", ierr));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Internal PIO function which provides a type-neutral interface to
+ * nc_put_var1 calls.
+ *
+ * Users should not call this function directly. Instead, call one of
+ * the derived functions, depending on the type of data you are
+ * writing: PIOc_put_var1_text(), PIOc_put_var1_uchar(),
+ * PIOc_put_var1_schar(), PIOc_put_var1_ushort(),
+ * PIOc_put_var1_short(), PIOc_put_var1_uint(), PIOc_put_var1_int(),
+ * PIOc_put_var1_long(), PIOc_put_var1_float(),
+ * PIOc_put_var1_longlong(), PIOc_put_var1_double(),
+ * PIOc_put_var1_ulonglong().
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param index an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param xtype the netCDF type of the data being passed in buf. Data
+ * will be automatically covnerted from this type to the type of the
+ * variable being written to.
+ * @param op pointer to the data to be written.
+ *
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
+                     const void *op)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ndims;   /* The number of dimensions in the variable. */
+    int ierr;    /* Return code from function calls. */
+
+    /* Find the info about this file. We need this for error handling. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Find the number of dimensions. */
+    if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Set up count array. */
+    PIO_Offset count[ndims];
+    for (int c = 0; c < ndims; c++)
+        count[c] = 1;
+
+    return PIOc_put_vars_tc(ncid, varid, index, count, NULL, xtype, op);
+}
+
+/**
+ * Internal PIO function which provides a type-neutral interface to
+ * nc_put_var calls.
+ *
+ * Users should not call this function directly. Instead, call one of
+ * the derived functions, depending on the type of data you are
+ * writing: PIOc_put_var_text(), PIOc_put_var_uchar(),
+ * PIOc_put_var_schar(), PIOc_put_var_ushort(),
+ * PIOc_put_var_short(), PIOc_put_var_uint(), PIOc_put_var_int(),
+ * PIOc_put_var_long(), PIOc_put_var_float(),
+ * PIOc_put_var_longlong(), PIOc_put_var_double(),
+ * PIOc_put_var_ulonglong().
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param xtype the netCDF type of the data being passed in buf. Data
+ * will be automatically covnerted from this type to the type of the
+ * variable being written to.
+ * @param op pointer to the data to be written.
+ *
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    PIO_Offset *startp = NULL; /* Pointer to start array. */
+    PIO_Offset *countp = NULL; /* Pointer to count array. */
+    PIO_Offset start[PIO_MAX_DIMS];
+    PIO_Offset count[PIO_MAX_DIMS];
+    int ndims;   /* The number of dimensions in the variable. */
+    int ierr;    /* Return code from function calls. */
+
+    LOG((1, "PIOc_put_var_tc ncid = %d varid = %d xtype = %d", ncid,
+         varid, xtype));
+
+    /* Find the info about this file. We need this for error handling. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Find the number of dimensions. */
+    if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+        return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+    /* Scalar vars (which have ndims == 0) should just pass NULLs for
+     * start/count. */
+    if (ndims)
+    {
+        int dimid[ndims];
+
+        /* Set up start array. */
+        for (int d = 0; d < ndims; d++)
+            start[d] = 0;
+
+        /* Get the dimids for this var. */
+        if ((ierr = PIOc_inq_vardimid(ncid, varid, dimid)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+        /* Count array are the dimlens. */
+        for (int d = 0; d < ndims; d++)
+            if ((ierr = PIOc_inq_dimlen(ncid, dimid[d], &count[d])))
+                return pio_err(ios, file, ierr, __FILE__, __LINE__);
+
+        /* Set the array pointers. */
+        startp = start;
+        countp = count;
+    }
+
+    return PIOc_put_vars_tc(ncid, varid, startp, countp, NULL, xtype, op);
+}
diff --git a/src/externals/pio2/src/clib/pio_internal.h b/src/externals/pio2/src/clib/pio_internal.h
new file mode 100644
index 00000000000..9f1a4a18d5d
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_internal.h
@@ -0,0 +1,564 @@
+/**
+ * @file
+ * Private headers and defines for the PIO C interface.
+ * @author Jim Edwards
+ * @date  2014
+ *
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#ifndef __PIO_INTERNAL__
+#define __PIO_INTERNAL__
+
+#include 
+
+/* These are the sizes of types in netCDF files. Do not replace these
+ * constants with sizeof() calls for C types. They are not the
+ * same. Even on a system where sizeof(short) is 4, the size of a
+ * short in a netCDF file is 2 bytes. */
+#define NETCDF_CHAR_SIZE 1
+#define NETCDF_SHORT_SIZE 2
+#define NETCDF_INT_FLOAT_SIZE 4
+#define NETCDF_DOUBLE_INT64_SIZE 8
+
+/* It seems that some versions of openmpi fail to define
+ * MPI_OFFSET. */
+#ifdef OMPI_OFFSET_DATATYPE
+#ifndef MPI_OFFSET
+#define MPI_OFFSET OMPI_OFFSET_DATATYPE
+#endif
+#endif
+#ifndef MPI_Offset
+#define MPI_Offset long long
+#endif
+
+#if defined(MPT_VERSION) || defined(OPEN_MPI)
+/* Some MPI implementations do not allow passing MPI_DATATYPE_NULL to comm functions
+ * even though the send or recv length is 0, in these cases we use MPI_CHAR */
+#define PIO_DATATYPE_NULL MPI_CHAR
+#else
+#define PIO_DATATYPE_NULL MPI_DATATYPE_NULL
+#endif
+
+#include 
+#include 
+#include 
+#ifdef TIMING
+#include 
+#endif
+#include 
+
+#if PIO_ENABLE_LOGGING
+void pio_log(int severity, const char *fmt, ...);
+#define LOG(e) pio_log e
+#else
+#define LOG(e)
+#endif /* PIO_ENABLE_LOGGING */
+
+#define max(a,b)                                \
+    ({ __typeof__ (a) _a = (a);                 \
+        __typeof__ (b) _b = (b);                \
+        _a > _b ? _a : _b; })
+
+#define min(a,b)                                \
+    ({ __typeof__ (a) _a = (a);                 \
+        __typeof__ (b) _b = (b);                \
+        _a < _b ? _a : _b; })
+
+#define MAX_GATHER_BLOCK_SIZE 0
+#define PIO_REQUEST_ALLOC_CHUNK 16
+
+/** This is needed to handle _long() functions. It may not be used as
+ * a data type when creating attributes or varaibles, it is only used
+ * internally. */
+#define PIO_LONG_INTERNAL 13
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+    extern PIO_Offset pio_buffer_size_limit;
+
+    /** Used to sort map points in the subset rearranger. */
+    typedef struct mapsort
+    {
+        int rfrom;
+        PIO_Offset soffset;
+        PIO_Offset iomap;
+    } mapsort;
+
+    /** swapm defaults. */
+    typedef struct pio_swapm_defaults
+    {
+        int nreqs;
+        bool handshake;
+        bool isend;
+    } pio_swapm_defaults;
+
+    /* Handle an error in the PIO library. */
+    int pio_err(iosystem_desc_t *ios, file_desc_t *file, int err_num, const char *fname,
+                int line);
+
+    /* For async cases, this runs on IO tasks and listens for messages. */
+    int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
+                         MPI_Comm io_comm);
+
+    void pio_get_env(void);
+    int  pio_add_to_iodesc_list(io_desc_t *iodesc);
+    io_desc_t *pio_get_iodesc_from_id(int ioid);
+    int pio_delete_iodesc_from_list(int ioid);
+    int pio_num_iosystem(int *niosysid);
+
+    int pio_get_file(int ncid, file_desc_t **filep);
+    int pio_delete_file_from_list(int ncid);
+    void pio_add_to_file_list(file_desc_t *file);
+    void pio_push_request(file_desc_t *file, int request);
+
+    /* Create a file (internal function). */
+    int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename, int mode);
+
+    /* Open a file with optional retry as netCDF-classic if first
+     * iotype does not work. */
+    int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype,
+                            const char *filename, int mode, int retry);
+
+    iosystem_desc_t *pio_get_iosystem_from_id(int iosysid);
+    int pio_add_to_iosystem_list(iosystem_desc_t *ios);
+
+    /* Check the return code from a netCDF call. */
+    int check_netcdf(file_desc_t *file, int status, const char *fname, int line);
+
+    /* Check the return code from a netCDF call, with ios pointer. */
+    int check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status,
+                      const char *fname, int line);
+
+    /* Given PIO type, find MPI type and type size. */
+    int find_mpi_type(int pio_type, MPI_Datatype *mpi_type, int *type_size);
+
+    /* Check whether an IO type is valid for this build. */
+    int iotype_is_valid(int iotype);
+
+    /* Print error message and abort. */
+    void piodie(const char *msg, const char *fname, int line);
+
+    /* Assert that an expression is true. */
+    void pioassert(bool exp, const char *msg, const char *fname, int line);
+
+    /* Compute start and count values for each io task for a decomposition. */
+    int CalcStartandCount(int pio_type, int ndims, const int *gdims, int num_io_procs,
+                          int myiorank, PIO_Offset *start, PIO_Offset *count, int *num_aiotasks);
+
+    /* Completes the mapping for the box rearranger. */
+    int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc, const int *dest_ioproc,
+                       const PIO_Offset *dest_ioindex);
+
+    /* Create the MPI communicators needed by the subset rearranger. */
+    int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc);
+
+    /* Check return from MPI function and print error message. */
+    void CheckMPIReturn(int ierr, const char *file, int line);
+
+    /* Like MPI_Alltoallw(), but with flow control. */
+    int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendtypes,
+                  void *recvbuf, int *recvcounts, int *rdispls, MPI_Datatype *recvtypes,
+                  MPI_Comm comm, rearr_comm_fc_opt_t *fc);
+
+    long long lgcd_array(int nain, long long* ain);
+
+    void PIO_Offset_size(MPI_Datatype *dtype, int *tsize);
+    PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in);
+
+    /* Convert an index into dimension values. */
+    void idx_to_dim_list(int ndims, const int *gdims, PIO_Offset idx, PIO_Offset *dim_list);
+
+    /* Convert a global coordinate value into a local array index. */
+    PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count);
+
+    /* Determine whether fill values are needed. */
+    int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gsize,
+                       const PIO_Offset *compmap);
+
+    /* Set start and count so that they describe the first region in map.*/
+    PIO_Offset find_region(int ndims, const int *gdims, int maplen, const PIO_Offset *map,
+                           PIO_Offset *start, PIO_Offset *count);
+
+    /* Calculate start and count regions for the subset rearranger. */
+    int get_regions(int ndims, const int *gdimlen, int maplen, const PIO_Offset *map,
+                    int *maxregions, io_region *firstregion);
+
+    /* Expand a region along dimension dim, by incrementing count[i] as
+     * much as possible, consistent with the map. */
+    void expand_region(int dim, const int *gdims, int maplen, const PIO_Offset *map,
+                       int region_size, int region_stride, const int *max_size,
+                       PIO_Offset *count);
+
+    /* Compare sets of rearranger options. */
+    bool cmp_rearr_opts(const rearr_opt_t *rearr_opts, const rearr_opt_t *exp_rearr_opts);
+
+    /* Check and reset, if needed, rearranger opts to default values. */
+    int check_and_reset_rearr_opts(rearr_opt_t *rearr_opt);
+
+    /* Compare rearranger flow control options. */
+    bool cmp_rearr_comm_fc_opts(const rearr_comm_fc_opt_t *opt,
+                                const rearr_comm_fc_opt_t *exp_opt);
+
+    /* Create a subset rearranger. */
+    int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap, const int *gsize,
+                                int ndim, io_desc_t *iodesc);
+
+
+    /* Create a box rearranger. */
+    int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *compmap, const int *gsize,
+                             int ndim, io_desc_t *iodesc);
+
+
+    /* Move data from IO tasks to compute tasks. */
+    int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf);
+
+    /* Move data from compute tasks to IO tasks. */
+    int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf, void *rbuf,
+                          int nvars);
+
+    /* Allocate and initialize storage for decomposition information. */
+    int malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims, io_desc_t **iodesc);
+    void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc);
+
+    /* Flush contents of multi-buffer to disk. */
+    int flush_output_buffer(file_desc_t *file, bool force, PIO_Offset addsize);
+
+    /* Compute the size that the IO tasks will need to hold the data. */
+    int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc);
+
+    /* Allocation memory for a data region. */
+    int alloc_region2(iosystem_desc_t *ios, int ndims, io_region **region);
+
+    /* Delete an entry from the lost of open IO systems. */
+    int pio_delete_iosystem_from_list(int piosysid);
+
+    /* Find greatest commond divisor. */
+    int gcd(int a, int b);
+
+    /* Find greatest commond divisor for long long. */
+    long long lgcd (long long a, long long b );
+
+    /* Find greatest commond divisor in an array. */
+    int gcd_array(int nain, int *ain);
+
+    void free_region_list(io_region *top);
+
+    /* Convert a global coordinate value into a local array index. */
+    PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count);
+
+    int ceil2(int i);
+    int pair(int np, int p, int k);
+
+    /* Create MPI datatypes used for comp2io and io2comp data transfers. */
+    int define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc);
+
+    /* Create the derived MPI datatypes used for comp2io and io2comp
+     * transfers. */
+    int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt, const PIO_Offset *mindex,
+                             const int *mcount, int *mfrom, MPI_Datatype *mtype);
+    int compare_offsets(const void *a, const void *b) ;
+
+    /* Print a trace statement, for debugging. */
+    void print_trace (FILE *fp);
+
+    void cn_buffer_report(iosystem_desc_t *ios, bool collective);
+
+    /* Initialize the compute buffer. */
+    int compute_buffer_init(iosystem_desc_t *ios);
+
+    void free_cn_buffer_pool(iosystem_desc_t *ios);
+
+    /* Flush PIO's data buffer. */
+    int flush_buffer(int ncid, wmulti_buffer *wmb, bool flushtodisk);
+
+    int compute_maxaggregate_bytes(iosystem_desc_t *ios, io_desc_t *iodesc);
+
+    /* Compute an element of start/count arrays. */
+    void compute_one_dim(int gdim, int ioprocs, int rank, PIO_Offset *start,
+                         PIO_Offset *count);
+
+    /* Check the return code from an MPI function call. */
+    int check_mpi(file_desc_t *file, int mpierr, const char *filename, int line);
+
+    /* Check the return code from an MPI function call. */
+    int check_mpi2(iosystem_desc_t *ios, file_desc_t *file, int mpierr, const char *filename,
+                   int line);
+
+    /* Darray support functions. */
+
+    /* Write aggregated arrays to file using parallel I/O (netCDF-4 parallel/pnetcdf) */
+    int pio_write_darray_multi_nc(file_desc_t *file, int nvars, const int *vid, int iodesc_ndims,
+                                  MPI_Datatype basetype, int maxregions, io_region *firstregion,
+                                  PIO_Offset llen, int num_aiotasks, void *iobuf,
+                                  const int *frame);
+
+    /* Write aggregated arrays to file using serial I/O (netCDF-3/netCDF-4 serial) */
+    int write_darray_multi_serial(file_desc_t *file, int nvars, const int *vid,
+                                  io_desc_t *iodesc, int fill, const int *frame);
+
+    int pio_read_darray_nc(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf);
+    int pio_read_darray_nc_serial(file_desc_t *file, io_desc_t *iodesc, int vid, void *iobuf);
+
+    /* Read atts with type conversion. */
+    int PIOc_get_att_tc(int ncid, int varid, const char *name, nc_type memtype, void *ip);
+
+    /* Write atts with type conversion. */
+    int PIOc_put_att_tc(int ncid, int varid, const char *name, nc_type atttype,
+                        PIO_Offset len, nc_type memtype, const void *op);
+
+    /* Generalized get functions. */
+    int PIOc_get_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                         const PIO_Offset *stride, nc_type xtype, void *buf);
+    int PIOc_get_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
+                         void *buf);
+    int PIOc_get_var_tc(int ncid, int varid, nc_type xtype, void *buf);
+    
+
+    /* Generalized put functions. */
+    int PIOc_put_vars_tc(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                         const PIO_Offset *stride, nc_type xtype, const void *buf);
+    int PIOc_put_var1_tc(int ncid, int varid, const PIO_Offset *index, nc_type xtype,
+                         const void *op);
+    int PIOc_put_var_tc(int ncid, int varid, nc_type xtype, const void *op);
+    
+    /* An internal replacement for a function pnetcdf does not
+     * have. */
+    int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name,
+                              PIO_Offset *sizep);
+
+    /* Handle end and re-defs. */
+    int pioc_change_def(int ncid, int is_enddef);
+
+    /* Initialize and finalize logging. */
+    void pio_init_logging(void);
+    void pio_finalize_logging(void );
+
+    /* Write a netCDF decomp file. */
+    int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmode, int ndims,
+                                 int *global_dimlen, int num_tasks, int *task_maplen, int *map,
+                                 const char *title, const char *history, int fortran_order);
+
+    /* Read a netCDF decomp file. */
+    int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int **global_dimlen,
+                                int *num_tasks, int **task_maplen, int *max_maplen, int **map, char *title,
+                                char *history, char *source, char *version, int *fortran_order);
+
+#if defined(__cplusplus)
+}
+#endif
+
+/** These are the messages that can be sent over the intercomm when
+ * async is being used. */
+enum PIO_MSG
+{
+    PIO_MSG_OPEN_FILE,
+    PIO_MSG_CREATE_FILE,
+    PIO_MSG_INQ_ATT,
+    PIO_MSG_INQ_FORMAT,
+    PIO_MSG_INQ_VARID,
+    PIO_MSG_DEF_VAR,
+    PIO_MSG_INQ_VAR,
+    PIO_MSG_PUT_ATT_DOUBLE,
+    PIO_MSG_PUT_ATT_INT,
+    PIO_MSG_RENAME_ATT,
+    PIO_MSG_DEL_ATT,
+    PIO_MSG_INQ,
+    PIO_MSG_GET_ATT_TEXT,
+    PIO_MSG_GET_ATT_SHORT,
+    PIO_MSG_PUT_ATT_LONG,
+    PIO_MSG_REDEF,
+    PIO_MSG_SET_FILL,
+    PIO_MSG_ENDDEF,
+    PIO_MSG_RENAME_VAR,
+    PIO_MSG_PUT_ATT_SHORT,
+    PIO_MSG_PUT_ATT_TEXT,
+    PIO_MSG_INQ_ATTNAME,
+    PIO_MSG_GET_ATT_ULONGLONG,
+    PIO_MSG_GET_ATT_USHORT,
+    PIO_MSG_PUT_ATT_ULONGLONG,
+    PIO_MSG_GET_ATT_UINT,
+    PIO_MSG_GET_ATT_LONGLONG,
+    PIO_MSG_PUT_ATT_SCHAR,
+    PIO_MSG_PUT_ATT_FLOAT,
+    PIO_MSG_RENAME_DIM,
+    PIO_MSG_GET_ATT_LONG,
+    PIO_MSG_INQ_DIM,
+    PIO_MSG_INQ_DIMID,
+    PIO_MSG_PUT_ATT_USHORT,
+    PIO_MSG_GET_ATT_FLOAT,
+    PIO_MSG_SYNC,
+    PIO_MSG_PUT_ATT_LONGLONG,
+    PIO_MSG_PUT_ATT_UINT,
+    PIO_MSG_GET_ATT_SCHAR,
+    PIO_MSG_INQ_ATTID,
+    PIO_MSG_DEF_DIM,
+    PIO_MSG_GET_ATT_INT,
+    PIO_MSG_GET_ATT_DOUBLE,
+    PIO_MSG_PUT_ATT_UCHAR,
+    PIO_MSG_GET_ATT_UCHAR,
+    PIO_MSG_PUT_VARS_UCHAR,
+    PIO_MSG_GET_VAR1_SCHAR,
+    PIO_MSG_GET_VARS_ULONGLONG,
+    PIO_MSG_GET_VARM_UCHAR,
+    PIO_MSG_GET_VARM_SCHAR,
+    PIO_MSG_GET_VARS_SHORT,
+    PIO_MSG_GET_VAR_DOUBLE,
+    PIO_MSG_GET_VARA_DOUBLE,
+    PIO_MSG_GET_VAR_INT,
+    PIO_MSG_GET_VAR_USHORT,
+    PIO_MSG_PUT_VARS_USHORT,
+    PIO_MSG_GET_VARA_TEXT,
+    PIO_MSG_PUT_VARS_ULONGLONG,
+    PIO_MSG_GET_VARA_INT,
+    PIO_MSG_PUT_VARM,
+    PIO_MSG_GET_VAR1_FLOAT,
+    PIO_MSG_GET_VAR1_SHORT,
+    PIO_MSG_GET_VARS_INT,
+    PIO_MSG_PUT_VARS_UINT,
+    PIO_MSG_GET_VAR_TEXT,
+    PIO_MSG_GET_VARM_DOUBLE,
+    PIO_MSG_PUT_VARM_UCHAR,
+    PIO_MSG_PUT_VAR_USHORT,
+    PIO_MSG_GET_VARS_SCHAR,
+    PIO_MSG_GET_VARA_USHORT,
+    PIO_MSG_PUT_VAR1_LONGLONG,
+    PIO_MSG_PUT_VARA_UCHAR,
+    PIO_MSG_PUT_VARM_SHORT,
+    PIO_MSG_PUT_VAR1_LONG,
+    PIO_MSG_PUT_VARS_LONG,
+    PIO_MSG_GET_VAR1_USHORT,
+    PIO_MSG_PUT_VAR_SHORT,
+    PIO_MSG_PUT_VARA_INT,
+    PIO_MSG_GET_VAR_FLOAT,
+    PIO_MSG_PUT_VAR1_USHORT,
+    PIO_MSG_PUT_VARA_TEXT,
+    PIO_MSG_PUT_VARM_TEXT,
+    PIO_MSG_GET_VARS_UCHAR,
+    PIO_MSG_GET_VAR,
+    PIO_MSG_PUT_VARM_USHORT,
+    PIO_MSG_GET_VAR1_LONGLONG,
+    PIO_MSG_GET_VARS_USHORT,
+    PIO_MSG_GET_VAR_LONG,
+    PIO_MSG_GET_VAR1_DOUBLE,
+    PIO_MSG_PUT_VAR_ULONGLONG,
+    PIO_MSG_PUT_VAR_INT,
+    PIO_MSG_GET_VARA_UINT,
+    PIO_MSG_PUT_VAR_LONGLONG,
+    PIO_MSG_GET_VARS_LONGLONG,
+    PIO_MSG_PUT_VAR_SCHAR,
+    PIO_MSG_PUT_VAR_UINT,
+    PIO_MSG_PUT_VAR,
+    PIO_MSG_PUT_VARA_USHORT,
+    PIO_MSG_GET_VAR_LONGLONG,
+    PIO_MSG_GET_VARA_SHORT,
+    PIO_MSG_PUT_VARS_SHORT,
+    PIO_MSG_PUT_VARA_UINT,
+    PIO_MSG_PUT_VARA_SCHAR,
+    PIO_MSG_PUT_VARM_ULONGLONG,
+    PIO_MSG_PUT_VAR1_UCHAR,
+    PIO_MSG_PUT_VARM_INT,
+    PIO_MSG_PUT_VARS_SCHAR,
+    PIO_MSG_GET_VARA_LONG,
+    PIO_MSG_PUT_VAR1,
+    PIO_MSG_GET_VAR1_INT,
+    PIO_MSG_GET_VAR1_ULONGLONG,
+    PIO_MSG_GET_VAR_UCHAR,
+    PIO_MSG_PUT_VARA_FLOAT,
+    PIO_MSG_GET_VARA_UCHAR,
+    PIO_MSG_GET_VARS_FLOAT,
+    PIO_MSG_PUT_VAR1_FLOAT,
+    PIO_MSG_PUT_VARM_FLOAT,
+    PIO_MSG_PUT_VAR1_TEXT,
+    PIO_MSG_PUT_VARS_TEXT,
+    PIO_MSG_PUT_VARM_LONG,
+    PIO_MSG_GET_VARS_LONG,
+    PIO_MSG_PUT_VARS_DOUBLE,
+    PIO_MSG_GET_VAR1,
+    PIO_MSG_GET_VAR_UINT,
+    PIO_MSG_PUT_VARA_LONGLONG,
+    PIO_MSG_GET_VARA,
+    PIO_MSG_PUT_VAR_DOUBLE,
+    PIO_MSG_GET_VARA_SCHAR,
+    PIO_MSG_PUT_VAR_FLOAT,
+    PIO_MSG_GET_VAR1_UINT,
+    PIO_MSG_GET_VARS_UINT,
+    PIO_MSG_PUT_VAR1_ULONGLONG,
+    PIO_MSG_PUT_VARM_UINT,
+    PIO_MSG_PUT_VAR1_UINT,
+    PIO_MSG_PUT_VAR1_INT,
+    PIO_MSG_GET_VARA_FLOAT,
+    PIO_MSG_GET_VARM_TEXT,
+    PIO_MSG_PUT_VARS_FLOAT,
+    PIO_MSG_GET_VAR1_TEXT,
+    PIO_MSG_PUT_VARA_SHORT,
+    PIO_MSG_PUT_VAR1_SCHAR,
+    PIO_MSG_PUT_VARA_ULONGLONG,
+    PIO_MSG_PUT_VARM_DOUBLE,
+    PIO_MSG_GET_VARM_INT,
+    PIO_MSG_PUT_VARA,
+    PIO_MSG_PUT_VARA_LONG,
+    PIO_MSG_GET_VARM_UINT,
+    PIO_MSG_GET_VARM,
+    PIO_MSG_PUT_VAR1_DOUBLE,
+    PIO_MSG_GET_VARS_DOUBLE,
+    PIO_MSG_GET_VARA_LONGLONG,
+    PIO_MSG_GET_VAR_ULONGLONG,
+    PIO_MSG_PUT_VARM_SCHAR,
+    PIO_MSG_GET_VARA_ULONGLONG,
+    PIO_MSG_GET_VAR_SHORT,
+    PIO_MSG_GET_VARM_FLOAT,
+    PIO_MSG_PUT_VAR_TEXT,
+    PIO_MSG_PUT_VARS_INT,
+    PIO_MSG_GET_VAR1_LONG,
+    PIO_MSG_GET_VARM_LONG,
+    PIO_MSG_GET_VARM_USHORT,
+    PIO_MSG_PUT_VAR1_SHORT,
+    PIO_MSG_PUT_VARS_LONGLONG,
+    PIO_MSG_GET_VARM_LONGLONG,
+    PIO_MSG_GET_VARS_TEXT,
+    PIO_MSG_PUT_VARA_DOUBLE,
+    PIO_MSG_PUT_VARS,
+    PIO_MSG_PUT_VAR_UCHAR,
+    PIO_MSG_GET_VAR1_UCHAR,
+    PIO_MSG_PUT_VAR_LONG,
+    PIO_MSG_GET_VARS,
+    PIO_MSG_GET_VARM_SHORT,
+    PIO_MSG_GET_VARM_ULONGLONG,
+    PIO_MSG_PUT_VARM_LONGLONG,
+    PIO_MSG_GET_VAR_SCHAR,
+    PIO_MSG_GET_ATT_UBYTE,
+    PIO_MSG_PUT_ATT_STRING,
+    PIO_MSG_GET_ATT_STRING,
+    PIO_MSG_PUT_ATT_UBYTE,
+    PIO_MSG_INQ_VAR_FILL,
+    PIO_MSG_DEF_VAR_FILL,
+    PIO_MSG_DEF_VAR_DEFLATE,
+    PIO_MSG_INQ_VAR_DEFLATE,
+    PIO_MSG_INQ_VAR_SZIP,
+    PIO_MSG_DEF_VAR_FLETCHER32,
+    PIO_MSG_INQ_VAR_FLETCHER32,
+    PIO_MSG_DEF_VAR_CHUNKING,
+    PIO_MSG_INQ_VAR_CHUNKING,
+    PIO_MSG_DEF_VAR_ENDIAN,
+    PIO_MSG_INQ_VAR_ENDIAN,
+    PIO_MSG_SET_CHUNK_CACHE,
+    PIO_MSG_GET_CHUNK_CACHE,
+    PIO_MSG_SET_VAR_CHUNK_CACHE,
+    PIO_MSG_GET_VAR_CHUNK_CACHE,
+    PIO_MSG_INITDECOMP_DOF,
+    PIO_MSG_WRITEDARRAY,
+    PIO_MSG_READDARRAY,
+    PIO_MSG_SETERRORHANDLING,
+    PIO_MSG_FREEDECOMP,
+    PIO_MSG_CLOSE_FILE,
+    PIO_MSG_DELETE_FILE,
+    PIO_MSG_EXIT,
+    PIO_MSG_GET_ATT,
+    PIO_MSG_PUT_ATT,
+    PIO_MSG_INQ_TYPE
+};
+
+#endif /* __PIO_INTERNAL__ */
diff --git a/src/externals/pio2/src/clib/pio_lists.c b/src/externals/pio2/src/clib/pio_lists.c
new file mode 100644
index 00000000000..df79302cfd1
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_lists.c
@@ -0,0 +1,305 @@
+/**
+ * @file
+ * PIO list functions.
+ */
+#include 
+#include 
+#include 
+#include 
+#include 
+
+static io_desc_t *pio_iodesc_list=NULL;
+static io_desc_t *current_iodesc=NULL;
+static iosystem_desc_t *pio_iosystem_list=NULL;
+static file_desc_t *pio_file_list = NULL;
+static file_desc_t *current_file=NULL;
+
+/** Add a new entry to the global list of open files.
+ *
+ * @param file pointer to the file_desc_t struct for the new file.
+ */
+void pio_add_to_file_list(file_desc_t *file)
+{
+    file_desc_t *cfile;
+
+    assert(file);
+
+    /* This file will be at the end of the list, and have no next. */
+    file->next = NULL;
+
+    /* Get a pointer to the global list of files. */
+    cfile = pio_file_list;
+
+    /* Keep a global pointer to the current file. */
+    current_file = file;
+
+    /* If there is nothing in the list, then file will be the first
+     * entry. Otherwise, move to end of the list. */
+    if (!cfile)
+        pio_file_list = file;
+    else
+    {
+        while (cfile->next)
+            cfile = cfile->next;
+        cfile->next = file;
+    }
+}
+
+/** Given ncid, find the file_desc_t data for an open file. The ncid
+ * used is the interally generated pio_ncid.
+ *
+ * @param ncid the PIO assigned ncid of the open file.
+ * @param cfile1 pointer to a pointer to a file_desc_t. The pointer
+ * will get a copy of the pointer to the file info.
+ *
+ * @returns 0 for success, error code otherwise.
+ */
+int pio_get_file(int ncid, file_desc_t **cfile1)
+{
+    file_desc_t *cfile = NULL;
+
+    LOG((2, "pio_get_file ncid = %d", ncid));
+
+    /* Caller must provide this. */
+    if (!cfile1)
+        return PIO_EINVAL;
+
+    /* Find the file pointer. */
+    if (current_file && current_file->pio_ncid == ncid)
+        cfile = current_file;
+    else
+        for (cfile = pio_file_list; cfile; cfile = cfile->next)
+            if (cfile->pio_ncid == ncid)
+            {
+                current_file = cfile;
+                break;
+            }
+
+    /* If not found, return error. */
+    if (!cfile)
+        return PIO_EBADID;
+
+    /* We depend on every file having a pointer to the iosystem. */
+    if (!cfile->iosystem)
+        return PIO_EINVAL;
+
+    /* Let's just ensure we have a valid IO type. */
+    pioassert(iotype_is_valid(cfile->iotype), "invalid IO type", __FILE__, __LINE__);
+
+    /* Copy pointer to file info. */
+    *cfile1 = cfile;
+
+    return PIO_NOERR;
+}
+
+/** Delete a file from the list of open files.
+ *
+ * @param ncid ID of file to delete from list
+ * @returns 0 for success, error code otherwise
+ */
+int pio_delete_file_from_list(int ncid)
+{
+
+    file_desc_t *cfile, *pfile = NULL;
+
+    /* Look through list of open files. */
+    for (cfile = pio_file_list; cfile; cfile = cfile->next)
+    {
+        if (cfile->pio_ncid == ncid)
+        {
+            if (!pfile)
+                pio_file_list = cfile->next;
+            else
+                pfile->next = cfile->next;
+
+            if (current_file == cfile)
+                current_file = pfile;
+
+            /* Free any fill values that were allocated. */
+            for (int v = 0; v < PIO_MAX_VARS; v++)
+                if (cfile->varlist[v].fillvalue)
+                    free(cfile->varlist[v].fillvalue);
+
+            /* Free the memory used for this file. */
+            free(cfile);
+            
+            return PIO_NOERR;
+        }
+        pfile = cfile;
+    }
+
+    /* No file was found. */
+    return PIO_EBADID;
+}
+
+/** Delete iosystem info from list.
+ *
+ * @param piosysid the iosysid to delete
+ * @returns 0 on success, error code otherwise
+ */
+int pio_delete_iosystem_from_list(int piosysid)
+{
+    iosystem_desc_t *ciosystem, *piosystem = NULL;
+
+    LOG((1, "pio_delete_iosystem_from_list piosysid = %d", piosysid));
+
+    for (ciosystem = pio_iosystem_list; ciosystem; ciosystem = ciosystem->next)
+    {
+        if (ciosystem->iosysid == piosysid)
+        {
+            if (piosystem == NULL)
+                pio_iosystem_list = ciosystem->next;
+            else
+                piosystem->next = ciosystem->next;
+            free(ciosystem);
+            return PIO_NOERR;
+        }
+        piosystem = ciosystem;
+    }
+    return PIO_EBADID;
+}
+
+/** Add iosystem info to list.
+ *
+ * @param ios pointer to the iosystem_desc_t info to add.
+ * @returns 0 on success, error code otherwise
+ */
+int pio_add_to_iosystem_list(iosystem_desc_t *ios)
+{
+    iosystem_desc_t *cios;
+    int i = 1;
+
+    assert(ios);
+
+    ios->next = NULL;
+    cios = pio_iosystem_list;
+    if (!cios)
+        pio_iosystem_list = ios;
+    else
+    {
+        i++;
+        while (cios->next)
+        {
+            cios = cios->next;
+            i++;
+        }
+        cios->next = ios;
+    }
+
+    ios->iosysid = i << 16;
+
+    return ios->iosysid;
+}
+
+/** Get iosystem info from list.
+ *
+ * @param iosysid id of the iosystem
+ * @returns pointer to iosystem_desc_t, or NULL if not found.
+ */
+iosystem_desc_t *pio_get_iosystem_from_id(int iosysid)
+{
+    iosystem_desc_t *ciosystem;
+
+    LOG((2, "pio_get_iosystem_from_id iosysid = %d", iosysid));
+
+    for (ciosystem = pio_iosystem_list; ciosystem; ciosystem = ciosystem->next)
+        if (ciosystem->iosysid == iosysid)
+            return ciosystem;
+
+    return NULL;
+}
+
+/** Count the number of open iosystems.
+ *
+ * @param niosysid pointer that will get the number of open iosystems.
+ * @returns 0 for success.
+ */
+int
+pio_num_iosystem(int *niosysid)
+{
+    int count = 0;
+
+    /* Count the elements in the list. */
+    for (iosystem_desc_t *c = pio_iosystem_list; c; c = c->next)
+        count++;
+
+    /* Return count to caller via pointer. */
+    if (niosysid)
+        *niosysid = count;
+
+    return PIO_NOERR;
+}
+
+/** Add an iodesc.
+ *
+ * @param io_desc_t pointer to data to add to list.
+ * @returns the ioid of the newly added iodesc.
+ */
+int pio_add_to_iodesc_list(io_desc_t *iodesc)
+{
+    io_desc_t *ciodesc;
+    int imax = 512;
+
+    iodesc->next = NULL;
+    if (pio_iodesc_list == NULL)
+        pio_iodesc_list = iodesc;
+    else{
+        imax++;
+        for (ciodesc = pio_iodesc_list; ciodesc->next; ciodesc=ciodesc->next, imax=ciodesc->ioid + 1);
+        ciodesc->next = iodesc;
+    }
+    iodesc->ioid = imax;
+    current_iodesc = iodesc;
+
+    return iodesc->ioid;
+}
+
+/** Get an iodesc.
+ *
+ * @param ioid ID of iodesc to get.
+ * @returns pointer to the iodesc struc.
+ */
+io_desc_t *pio_get_iodesc_from_id(int ioid)
+{
+    io_desc_t *ciodesc = NULL;
+
+    if (current_iodesc != NULL && current_iodesc->ioid == abs(ioid))
+        ciodesc = current_iodesc;
+    else
+        for (ciodesc = pio_iodesc_list; ciodesc; ciodesc = ciodesc->next)
+            if (ciodesc->ioid == abs(ioid))
+            {
+                current_iodesc = ciodesc;
+                break;
+            }
+
+    return ciodesc;
+}
+
+/** Delete an iodesc.
+ *
+ * @param ioid ID of iodesc to delete.
+ * @returns 0 on success, error code otherwise.
+ */
+int pio_delete_iodesc_from_list(int ioid)
+{
+    io_desc_t *ciodesc, *piodesc = NULL;
+
+    for (ciodesc = pio_iodesc_list; ciodesc; ciodesc = ciodesc->next)
+    {
+        if (ciodesc->ioid == ioid)
+        {
+            if (piodesc == NULL)
+                pio_iodesc_list = ciodesc->next;
+            else
+                piodesc->next = ciodesc->next;
+
+            if (current_iodesc == ciodesc)
+                current_iodesc = pio_iodesc_list;
+            free(ciodesc);
+            return PIO_NOERR;
+        }
+        piodesc = ciodesc;
+    }
+    return PIO_EBADID;
+}
diff --git a/src/externals/pio2/src/clib/pio_msg.c b/src/externals/pio2/src/clib/pio_msg.c
new file mode 100644
index 00000000000..b8ca0e9917b
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_msg.c
@@ -0,0 +1,2530 @@
+/** @file
+ *
+ * PIO async message handling. This file contains the code which
+ * runs on the IO nodes when async is in use. This code waits for
+ * messages from the computation nodes, and responds to messages by
+ * running the appropriate netCDF function.
+ *
+ * @author Ed Hartnett
+ */
+
+#include 
+#include 
+#include 
+
+#ifdef PIO_ENABLE_LOGGING
+extern int my_rank;
+extern int pio_log_level;
+#endif /* PIO_ENABLE_LOGGING */
+
+/** This function is run on the IO tasks to handle nc_inq_type*()
+ * functions.
+ *
+ * @param ios pointer to the iosystem info.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_type_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int xtype;
+    char name_present, size_present;
+    char *namep = NULL, name[NC_MAX_NAME + 1];
+    PIO_Offset *sizep = NULL, size;
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_type_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Handle null pointer issues. */
+    if (name_present)
+        namep = name;
+    if (size_present)
+        sizep = &size;
+
+    /* Call the function. */
+    if ((ret = PIOc_inq_type(ncid, xtype, namep, sizep)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "inq_type_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to find netCDF file
+ * format.
+ *
+ * @param ios pointer to the iosystem info.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_format_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int *formatp = NULL, format;
+    char format_present;
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_format_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "inq_format_handler got parameters ncid = %d format_present = %d",
+         ncid, format_present));
+
+    /* Manage NULL pointers. */
+    if (format_present)
+        formatp = &format;
+
+    /* Call the function. */
+    if ((ret = PIOc_inq_format(ncid, formatp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    if (formatp)
+        LOG((2, "inq_format_handler format = %d", *formatp));
+    LOG((1, "inq_format_handler succeeded!"));
+
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to set the file fill mode.
+ *
+ * @param ios pointer to the iosystem info.
+ * @returns 0 for success, error code otherwise.
+ * @internal
+ */
+int set_fill_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int fillmode;
+    int old_modep_present;
+    int old_mode, *old_modep = NULL;
+    int mpierr;
+    int ret;
+
+    LOG((1, "set_fill_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&fillmode, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&old_modep_present, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "set_fill_handler got parameters ncid = %d fillmode = %d old_modep_present = %d",
+         ncid, fillmode, old_modep_present));
+
+    /* Manage NULL pointers. */
+    if (old_modep_present)
+        old_modep = &old_mode;
+
+    /* Call the function. */
+    if ((ret = PIOc_set_fill(ncid, fillmode, old_modep)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "set_fill_handler succeeded!"));
+
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to create a netCDF file.
+ *
+ * @param ios pointer to the iosystem info.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int create_file_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int len;
+    int iotype;
+    int mode;
+    int mpierr;
+    int ret;
+
+    LOG((1, "create_file_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Get space for the filename. */
+    char filename[len + 1];
+
+    if ((mpierr = MPI_Bcast(filename, len + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "create_file_handler got parameters len = %d filename = %s iotype = %d mode = %d",
+         len, filename, iotype, mode));
+
+    /* Call the create file function. */
+    if ((ret = PIOc_createfile(ios->iosysid, &ncid, &iotype, filename, mode)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "create_file_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to close a netCDF file. It is
+ * only ever run on the IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int close_file_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int mpierr;
+    int ret;
+
+    LOG((1, "close_file_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "create_file_handler got parameter ncid = %d", ncid));
+
+    /* Call the close file function. */
+    if ((ret = PIOc_closefile(ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "close_file_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to inq a netCDF file. It is
+ * only ever run on the IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int ndims, nvars, ngatts, unlimdimid;
+    int *ndimsp = NULL, *nvarsp = NULL, *ngattsp = NULL, *unlimdimidp = NULL;
+    char ndims_present, nvars_present, ngatts_present, unlimdimid_present;
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "inq_handler ndims_present = %d nvars_present = %d ngatts_present = %d unlimdimid_present = %d",
+         ndims_present, nvars_present, ngatts_present, unlimdimid_present));
+
+    /* NULLs passed in to any of the pointers in the original call
+     * need to be matched with NULLs here. Assign pointers where
+     * non-NULL pointers were passed in. */
+    if (ndims_present)
+        ndimsp = &ndims;
+    if (nvars_present)
+        nvarsp = &nvars;
+    if (ngatts_present)
+        ngattsp = &ngatts;
+    if (unlimdimid_present)
+        unlimdimidp = &unlimdimid;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq(ncid, ndimsp, nvarsp, ngattsp, unlimdimidp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Do an inq_dim on a netCDF dimension. This function is only run on
+ * IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_dim_handler(iosystem_desc_t *ios, int msg)
+{
+    int ncid;
+    int dimid;
+    char name_present, len_present;
+    char *dimnamep = NULL;
+    PIO_Offset *dimlenp = NULL;
+    char dimname[NC_MAX_NAME + 1];
+    PIO_Offset dimlen;
+
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_dim_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&dimid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "inq_handler name_present = %d len_present = %d", name_present,
+         len_present));
+
+    /* Set the non-null pointers. */
+    if (name_present)
+        dimnamep = dimname;
+    if (len_present)
+        dimlenp = &dimlen;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq_dim(ncid, dimid, dimnamep, dimlenp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Do an inq_dimid on a netCDF dimension name. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_dimid_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int *dimidp = NULL, dimid;
+    int mpierr;
+    int id_present;
+    int ret;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+
+    LOG((1, "inq_dimid_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "inq_dimid_handler ncid = %d namelen = %d name = %s id_present = %d",
+         ncid, namelen, name, id_present));
+
+    /* Set non-null pointer. */
+    if (id_present)
+        dimidp = &dimid;
+
+    /* Call the inq_dimid function. */
+    if ((ret = PIOc_inq_dimid(ncid, name, dimidp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Handle attribute inquiry operations. This code only runs on IO
+ * tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_att_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    int ret;
+    char name[PIO_MAX_NAME + 1];
+    int namelen;
+    nc_type xtype, *xtypep = NULL;
+    PIO_Offset len, *lenp = NULL;
+    char xtype_present, len_present;
+
+    LOG((1, "inq_att_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster,
+                            ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Match NULLs in collective function call. */
+    if (xtype_present)
+        xtypep = &xtype;
+    if (len_present)
+        lenp = &len;
+
+    /* Call the function to learn about the attribute. */
+    if ((ret = PIOc_inq_att(ncid, varid, name, xtypep, lenp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Handle attribute inquiry operations. This code only runs on IO
+ * tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_attname_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int attnum;
+    char name[NC_MAX_NAME + 1], *namep = NULL;
+    char name_present;
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_att_name_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&attnum, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "inq_attname_handler got ncid = %d varid = %d attnum = %d name_present = %d",
+         ncid, varid, attnum, name_present));
+
+    /* Match NULLs in collective function call. */
+    if (name_present)
+        namep = name;
+
+    /* Call the function to learn about the attribute. */
+    if ((ret = PIOc_inq_attname(ncid, varid, attnum, namep)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Handle attribute inquiry operations. This code only runs on IO
+ * tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_attid_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char name[PIO_MAX_NAME + 1];
+    int namelen;
+    int id, *idp = NULL;
+    char id_present;
+    int mpierr;
+    int ret;
+
+    LOG((1, "inq_attid_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "inq_attid_handler got ncid = %d varid = %d id_present = %d",
+         ncid, varid, id_present));
+
+    /* Match NULLs in collective function call. */
+    if (id_present)
+        idp = &id;
+
+    /* Call the function to learn about the attribute. */
+    if ((ret = PIOc_inq_attid(ncid, varid, name, idp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Handle attribute operations. This code only runs on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int att_put_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    int ret;
+    char name[PIO_MAX_NAME + 1];
+    int namelen;
+    PIO_Offset attlen;  /* Number of elements in att array. */
+    nc_type atttype;    /* Type of att in file. */
+    PIO_Offset atttype_len; /* Length in bytes of one elementy of type atttype. */
+    nc_type memtype;    /* Type of att data in memory. */
+    PIO_Offset memtype_len; /* Length of element of memtype. */
+    void *op;
+
+    LOG((1, "att_put_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&atttype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&memtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Allocate memory for the attribute data. */
+    if (!(op = malloc(attlen * memtype_len)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(op, attlen * memtype_len, MPI_BYTE, 0, ios->intercomm)))
+    {
+        free(op);
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+    LOG((1, "att_put_handler ncid = %d varid = %d namelen = %d name = %s"
+         "atttype = %d attlen = %d atttype_len = %d memtype = %d memtype_len = 5d",
+         ncid, varid, namelen, name, atttype, attlen, atttype_len, memtype, memtype_len));
+
+    /* Call the function to write the attribute. */
+    ret = PIOc_put_att_tc(ncid, varid, name, atttype, attlen, memtype, op);
+
+    /* Free resources. */
+    free(op);
+
+    /* Did it work? */
+    if (ret)
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((2, "att_put_handler complete!"));
+    return PIO_NOERR;
+}
+
+/** Handle attribute operations. This code only runs on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @param msg the message sent my the comp root task.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int att_get_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    char name[PIO_MAX_NAME + 1];
+    int namelen;
+    PIO_Offset attlen;
+    nc_type atttype;        /* Type of att in file. */
+    PIO_Offset atttype_len; /* Length in bytes of an element of attype. */
+    nc_type memtype;        /* Type of att in memory. */
+    PIO_Offset memtype_len; /* Length in bytes of an element of memype. */
+    int *ip;
+    int iotype;
+    int ret;
+
+    LOG((1, "att_get_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&atttype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&attlen, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&atttype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&memtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&memtype_len, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "att_get_handler ncid = %d varid = %d namelen = %d name = %s iotype = %d"
+         " atttype = %d attlen = %d atttype_len = %d memtype = %d memtype_len = %d",
+         ncid, varid, namelen, name, iotype, atttype, attlen, atttype_len, memtype, memtype_len));
+
+    /* Allocate space for the attribute data. */
+    if (!(ip = malloc(attlen * memtype_len)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Call the function to read the attribute. */
+    ret = PIOc_get_att_tc(ncid, varid, name, memtype, ip);
+
+    /* Free resources. */
+    free(ip);
+
+    /* Did it work? */
+    if (ret)
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Handle var put operations. This code only runs on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int put_vars_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    PIO_Offset typelen;  /* Length (in bytes) of this type. */
+    nc_type xtype;       /* Type of the data being written. */
+    char start_present;  /* Zero if user passed a NULL start. */
+    char count_present;  /* Zero if user passed a NULL count. */
+    char stride_present; /* Zero if user passed a NULL stride. */
+    PIO_Offset *startp = NULL;
+    PIO_Offset *countp = NULL;
+    PIO_Offset *stridep = NULL;
+    int ndims;           /* Number of dimensions. */
+    void *buf;           /* Buffer for data storage. */
+    PIO_Offset num_elem; /* Number of data elements in the buffer. */
+    int mpierr;          /* Error code from MPI function calls. */
+
+    LOG((1, "put_vars_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Now we know how big to make these arrays. */
+    PIO_Offset start[ndims], count[ndims], stride[ndims];
+
+    if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (start_present)    
+        if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "put_vars_handler getting start[0] = %d ndims = %d", start[0], ndims));
+    if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (count_present)    
+        if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (stride_present)
+        if ((mpierr = MPI_Bcast(stride, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "put_vars_handler ncid = %d varid = %d ndims = %d "
+         "start_present = %d count_present = %d stride_present = %d xtype = %d "
+         "num_elem = %d typelen = %d", ncid, varid, ndims, start_present, count_present,
+         stride_present, xtype, num_elem, typelen));
+
+    /* Allocate room for our data. */
+    if (!(buf = malloc(num_elem * typelen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Get the data. */
+    if ((mpierr = MPI_Bcast(buf, num_elem * typelen, MPI_BYTE, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Set the non-NULL pointers. */
+    if (start_present)
+        startp = start;
+    if (count_present)
+        countp = count;
+    if (stride_present)
+        stridep = stride;
+
+    /* Call the function to write the data. No need to check return
+     * values, they are bcast to computation tasks inside function. */
+    switch(xtype)
+    {
+    case NC_BYTE:
+        PIOc_put_vars_schar(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_CHAR:
+        PIOc_put_vars_text(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_SHORT:
+        PIOc_put_vars_short(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_INT:
+        PIOc_put_vars_int(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case PIO_LONG_INTERNAL:
+        PIOc_put_vars_long(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_FLOAT:
+        PIOc_put_vars_float(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_DOUBLE:
+        PIOc_put_vars_double(ncid, varid, startp, countp, stridep, buf);
+        break;
+#ifdef _NETCDF4
+    case NC_UBYTE:
+        PIOc_put_vars_uchar(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_USHORT:
+        PIOc_put_vars_ushort(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_UINT:
+        PIOc_put_vars_uint(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_INT64:
+        PIOc_put_vars_longlong(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_UINT64:
+        PIOc_put_vars_ulonglong(ncid, varid, startp, countp, stridep, buf);
+        break;
+        /* case NC_STRING: */
+        /*      PIOc_put_vars_string(ncid, varid, startp, countp, */
+        /*                                stridep, (void *)buf); */
+        /*      break; */
+        /*    default:*/
+        /* PIOc_put_vars(ncid, varid, startp, countp, */
+        /*                   stridep, buf); */
+#endif /* _NETCDF4 */
+    }
+
+    free(buf);
+
+    return PIO_NOERR;
+}
+
+/** Handle var get operations. This code only runs on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int get_vars_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    PIO_Offset typelen; /** Length (in bytes) of this type. */
+    nc_type xtype; /** Type of the data being written. */
+    PIO_Offset *start;
+    PIO_Offset *count;
+    PIO_Offset *stride;
+    char start_present;
+    char count_present;
+    char stride_present;
+    PIO_Offset *startp = NULL, *countp = NULL, *stridep = NULL;
+    int ndims; /** Number of dimensions. */
+    void *buf; /** Buffer for data storage. */
+    PIO_Offset num_elem; /** Number of data elements in the buffer. */
+
+    LOG((1, "get_vars_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&start_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (start_present)
+    {
+        if (!(start = malloc(ndims * sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);            
+        if ((mpierr = MPI_Bcast(start, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+    if ((mpierr = MPI_Bcast(&count_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (count_present)
+    {
+        if (!(count = malloc(ndims * sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);            
+        if ((mpierr = MPI_Bcast(count, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+    if ((mpierr = MPI_Bcast(&stride_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (stride_present)
+    {
+        if (!(stride = malloc(ndims * sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);            
+        if ((mpierr = MPI_Bcast(stride, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+    if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&num_elem, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&typelen, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "get_vars_handler ncid = %d varid = %d ndims = %d "
+         "stride_present = %d xtype = %d num_elem = %d typelen = %d",
+         ncid, varid, ndims, stride_present, xtype, num_elem, typelen));
+
+    /* Allocate room for our data. */
+    if (!(buf = malloc(num_elem * typelen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Set the non-NULL pointers. */
+    if (start_present)
+        startp = start;
+
+    if (count_present)
+        countp = count;
+    
+    if (stride_present)
+        stridep = stride;
+
+    /* Call the function to read the data. */
+    switch(xtype)
+    {
+    case NC_BYTE:
+        PIOc_get_vars_schar(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_CHAR:
+        PIOc_get_vars_text(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_SHORT:
+        PIOc_get_vars_short(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_INT:
+        PIOc_get_vars_int(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case PIO_LONG_INTERNAL:
+        PIOc_get_vars_long(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_FLOAT:
+        PIOc_get_vars_float(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_DOUBLE:
+        PIOc_get_vars_double(ncid, varid, startp, countp, stridep, buf);
+        break;
+#ifdef _NETCDF4
+    case NC_UBYTE:
+        PIOc_get_vars_uchar(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_USHORT:
+        PIOc_get_vars_ushort(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_UINT:
+        PIOc_get_vars_uint(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_INT64:
+        PIOc_get_vars_longlong(ncid, varid, startp, countp, stridep, buf);
+        break;
+    case NC_UINT64:
+        PIOc_get_vars_ulonglong(ncid, varid, startp, countp, stridep, buf);
+        break;
+        /* case NC_STRING: */
+        /*      PIOc_get_vars_string(ncid, varid, startp, countp, */
+        /*                                stridep, (void *)buf); */
+        /*      break; */
+        /*    default:*/
+        /* PIOc_get_vars(ncid, varid, startp, countp, */
+        /*                   stridep, buf); */
+#endif /* _NETCDF4 */
+    }
+
+    /* Free resourses. */
+    free(buf);
+    if (start_present)
+        free(start);
+    if (count_present)
+        free(count);
+    if (stride_present)
+        free(stride);
+    
+    LOG((1, "get_vars_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * Do an inq_var on a netCDF variable. This function is only run on
+ * IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    char name_present, xtype_present, ndims_present, dimids_present, natts_present;
+    char name[NC_MAX_NAME + 1], *namep = NULL;
+    nc_type xtype, *xtypep = NULL;
+    int *ndimsp = NULL, *dimidsp = NULL, *nattsp = NULL;
+    int ndims, dimids[NC_MAX_DIMS], natts;
+    int ret;
+
+    LOG((1, "inq_var_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2,"inq_var_handler ncid = %d varid = %d name_present = %d xtype_present = %d ndims_present = %d "
+         "dimids_present = %d natts_present = %d",
+         ncid, varid, name_present, xtype_present, ndims_present, dimids_present, natts_present));
+
+    /* Set the non-NULL pointers. */
+    if (name_present)
+        namep = name;
+    if (xtype_present)
+        xtypep = &xtype;
+    if (ndims_present)
+        ndimsp = &ndims;
+    if (dimids_present)
+        dimidsp = dimids;
+    if (natts_present)
+        nattsp = &natts;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq_var(ncid, varid, namep, xtypep, ndimsp, dimidsp, nattsp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    if (ndims_present)
+        LOG((2, "inq_var_handler ndims = %d", ndims));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Do an inq_var_chunking on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_chunking_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char storage_present, chunksizes_present;
+    int storage, *storagep = NULL;
+    PIO_Offset chunksizes[NC_MAX_DIMS], *chunksizesp = NULL;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "inq_var_chunking_handler"));
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&storage_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2,"inq_var_handler ncid = %d varid = %d storage_present = %d chunksizes_present = %d",
+         ncid, varid, storage_present, chunksizes_present));
+
+    /* Set the non-NULL pointers. */
+    if (storage_present)
+        storagep = &storage;
+    if (chunksizes_present)
+        chunksizesp = chunksizes;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq_var_chunking(ncid, varid, storagep, chunksizesp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Do an inq_var_fill on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_fill_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char fill_mode_present, fill_value_present;
+    PIO_Offset type_size;
+    int fill_mode, *fill_modep = NULL;
+    PIO_Offset *fill_value, *fill_valuep = NULL;
+    int mpierr;
+
+    assert(ios);
+    LOG((1, "inq_var_fill_handler"));
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&fill_mode_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2,"inq_var_fill_handler ncid = %d varid = %d type_size = %lld, fill_mode_present = %d fill_value_present = %d",
+         ncid, varid, type_size, fill_mode_present, fill_value_present));
+
+    /* If we need to, alocate storage for fill value. */
+    if (fill_value_present)
+        if (!(fill_value = malloc(type_size)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Set the non-NULL pointers. */
+    if (fill_mode_present)
+        fill_modep = &fill_mode;
+    if (fill_value_present)
+        fill_valuep = fill_value;
+
+    /* Call the inq function to get the values. */
+    PIOc_inq_var_fill(ncid, varid, fill_modep, fill_valuep);
+
+    /* Free fill value storage if we allocated some. */
+    if (fill_value_present)
+        free(fill_value);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Do an inq_var_endian on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_endian_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char endian_present;
+    int endian, *endianp = NULL;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "inq_var_endian_handler"));
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&endian_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2,"inq_var_endian_handler ncid = %d varid = %d endian_present = %d", ncid, varid,
+         endian_present));
+
+    /* Set the non-NULL pointers. */
+    if (endian_present)
+        endianp = &endian;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq_var_endian(ncid, varid, endianp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Do an inq_var_deflate on a netCDF variable. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int inq_var_deflate_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char shuffle_present;
+    char deflate_present;
+    char deflate_level_present;
+    int shuffle, *shufflep;
+    int deflate, *deflatep;
+    int deflate_level, *deflate_levelp;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "inq_var_deflate_handler"));
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&shuffle_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (shuffle_present && !mpierr)
+        if ((mpierr = MPI_Bcast(&shuffle, 1, MPI_INT, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&deflate_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (deflate_present && !mpierr)
+        if ((mpierr = MPI_Bcast(&deflate, 1, MPI_INT, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&deflate_level_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (deflate_level_present && !mpierr)
+        if ((mpierr = MPI_Bcast(&deflate_level, 1, MPI_INT, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "inq_var_handler ncid = %d varid = %d shuffle_present = %d deflate_present = %d "
+         "deflate_level_present = %d", ncid, varid, shuffle_present, deflate_present,
+         deflate_level_present));
+
+    /* Set the non-NULL pointers. */
+    if (shuffle_present)
+        shufflep = &shuffle;
+    if (deflate_present)
+        deflatep = &deflate;
+    if (deflate_level_present)
+        deflate_levelp = &deflate_level;
+
+    /* Call the inq function to get the values. */
+    if ((ret = PIOc_inq_var_deflate(ncid, varid, shufflep, deflatep, deflate_levelp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** Do an inq_varid on a netCDF variable name. This function is only
+ * run on IO tasks.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int inq_varid_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int mpierr;
+    int ret;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Call the inq_dimid function. */
+    if ((ret = PIOc_inq_varid(ncid, name, &varid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to sync a netCDF file.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int sync_file_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int mpierr;
+    int ret;
+
+    LOG((1, "sync_file_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "sync_file_handler got parameter ncid = %d", ncid));
+
+    /* Call the sync file function. */
+    if ((ret = PIOc_sync(ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((2, "sync_file_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to enddef a netCDF file.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int change_def_file_handler(iosystem_desc_t *ios, int msg)
+{
+    int ncid;
+    int mpierr;
+
+    LOG((1, "change_def_file_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Call the function. */
+    if (msg == PIO_MSG_ENDDEF)
+        PIOc_enddef(ncid);
+    else
+        PIOc_redef(ncid);
+
+    LOG((1, "change_def_file_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to define a netCDF
+ *  variable.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int def_var_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+    int varid;
+    nc_type xtype;
+    int ndims;
+    int *dimids;
+
+    LOG((1, "def_var_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (!(dimids = malloc(ndims * sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(dimids, ndims, MPI_INT, 0, ios->intercomm)))
+    {
+        free(dimids);
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+    LOG((1, "def_var_handler got parameters namelen = %d "
+         "name = %s ncid = %d", namelen, name, ncid));
+
+    /* Call the function. */
+    if ((ret = PIOc_def_var(ncid, name, xtype, ndims, dimids, &varid)))
+    {
+        free(dimids);
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    }
+
+    /* Free resources. */
+    free(dimids);
+
+    LOG((1, "def_var_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to define chunking for a
+ *  netCDF variable.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int def_var_chunking_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int ndims;
+    int storage;
+    char chunksizes_present;
+    PIO_Offset chunksizes[NC_MAX_DIMS], *chunksizesp = NULL;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "def_var_chunking_handler comproot = %d", ios->comproot));
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&storage, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (chunksizes_present)
+        if ((mpierr = MPI_Bcast(chunksizes, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "def_var_chunking_handler got parameters ncid = %d varid = %d storage = %d "
+         "ndims = %d chunksizes_present = %d", ncid, varid, storage, ndims, chunksizes_present));
+
+    /* Set the non-NULL pointers. */
+    if (chunksizes_present)
+        chunksizesp = chunksizes;
+
+    /* Call the function. */
+    if ((ret = PIOc_def_var_chunking(ncid, varid, storage, chunksizesp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "def_var_chunking_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to define fill mode and fill
+ * value.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int def_var_fill_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int fill_mode;
+    char fill_value_present;
+    PIO_Offset type_size;
+    PIO_Offset *fill_valuep = NULL;
+    int mpierr;
+
+    assert(ios);
+    LOG((1, "def_var_fill_handler comproot = %d", ios->comproot));
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&fill_mode, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (fill_value_present)
+    {
+        if (!(fill_valuep = malloc(type_size)))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(fill_valuep, type_size, MPI_CHAR, 0, ios->intercomm)))
+        {
+            free(fill_valuep);
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        }
+    }
+    LOG((1, "def_var_fill_handler got parameters ncid = %d varid = %d fill_mode = %d "
+         "type_size = %lld fill_value_present = %d", ncid, varid, fill_mode, type_size, fill_value_present));
+
+    /* Call the function. */
+    PIOc_def_var_fill(ncid, varid, fill_mode, fill_valuep);
+
+    /* Free memory allocated for the fill value. */
+    if (fill_valuep)
+        free(fill_valuep);
+
+    LOG((1, "def_var_fill_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to define endianness for a
+ * netCDF variable.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int def_var_endian_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int endian;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "def_var_endian_handler comproot = %d", ios->comproot));
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&endian, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "def_var_endian_handler got parameters ncid = %d varid = %d endain = %d ",
+         ncid, varid, endian));
+
+    /* Call the function. */
+    if ((ret = PIOc_def_var_endian(ncid, varid, endian)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "def_var_chunking_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to define deflate settings for
+ * a netCDF variable.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int def_var_deflate_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int shuffle;
+    int deflate;
+    int deflate_level;
+    int mpierr;
+    int ret;
+
+    assert(ios);
+    LOG((1, "def_var_deflate_handler comproot = %d", ios->comproot));
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&shuffle, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&deflate, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&deflate_level, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "def_var_deflate_handler got parameters ncid = %d varid = %d shuffle = %d ",
+         "deflate = %d deflate_level = %d", ncid, varid, shuffle, deflate, deflate_level));
+
+    /* Call the function. */
+    if ((ret = PIOc_def_var_deflate(ncid, varid, shuffle, deflate, deflate_level)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "def_var_deflate_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to define chunk cache settings
+ * for a netCDF variable.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int set_var_chunk_cache_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    PIO_Offset size;
+    PIO_Offset nelems;
+    float preemption;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    assert(ios);
+    LOG((1, "set_var_chunk_cache_handler comproot = %d", ios->comproot));
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "set_var_chunk_cache_handler got params ncid = %d varid = %d size = %d "
+         "nelems = %d preemption = %g", ncid, varid, size, nelems, preemption));
+
+    /* Call the function. */
+    if ((ret = PIOc_set_var_chunk_cache(ncid, varid, size, nelems, preemption)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "def_var_chunk_cache_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to define a netCDF
+ * dimension.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int def_dim_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int len, namelen;
+    char name[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+    int dimid;
+
+    LOG((1, "def_dim_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "def_dim_handler got parameters namelen = %d "
+         "name = %s len = %d ncid = %d", namelen, name, len, ncid));
+
+    /* Call the function. */
+    if ((ret = PIOc_def_dim(ncid, name, len, &dimid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "def_dim_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to rename a netCDF
+ * dimension.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int rename_dim_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+    int dimid;
+
+    LOG((1, "rename_dim_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&dimid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "rename_dim_handler got parameters namelen = %d "
+         "name = %s ncid = %d dimid = %d", namelen, name, ncid, dimid));
+
+    /* Call the function. */
+    if ((ret = PIOc_rename_dim(ncid, dimid, name)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "rename_dim_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to rename a netCDF
+ * dimension.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int rename_var_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+    int varid;
+
+    LOG((1, "rename_var_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "rename_var_handler got parameters namelen = %d "
+         "name = %s ncid = %d varid = %d", namelen, name, ncid, varid));
+
+    /* Call the function. */
+    if ((ret = PIOc_rename_var(ncid, varid, name)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "rename_var_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to rename a netCDF
+ * attribute.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int rename_att_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int namelen, newnamelen;
+    char name[PIO_MAX_NAME + 1], newname[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+
+    LOG((1, "rename_att_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(newname, newnamelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "rename_att_handler got parameters namelen = %d name = %s ncid = %d varid = %d "
+         "newnamelen = %d newname = %s", namelen, name, ncid, varid, newnamelen, newname));
+
+    /* Call the function. */
+    if ((ret = PIOc_rename_att(ncid, varid, name, newname)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "rename_att_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to delete a netCDF
+ * attribute.
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int delete_att_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    int namelen;
+    char name[PIO_MAX_NAME + 1];
+    int mpierr;
+    int ret;
+
+    LOG((1, "delete_att_handler"));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(name, namelen + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "delete_att_handler namelen = %d name = %s ncid = %d varid = %d ",
+         namelen, name, ncid, varid));
+
+    /* Call the function. */
+    if ((ret = PIOc_del_att(ncid, varid, name)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "delete_att_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to open a netCDF file.
+ *
+ *
+ * @param ios pointer to the iosystem_desc_t.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int open_file_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int len;
+    int iotype;
+    int mode;
+    int mpierr;
+
+    LOG((1, "open_file_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "open_file_handler got parameter len = %d", len));
+
+    /* Get space for the filename. */
+    char filename[len + 1];
+
+    if ((mpierr = MPI_Bcast(filename, len + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&mode, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "open_file_handler got parameters len = %d filename = %s iotype = %d mode = %d",
+         len, filename, iotype, mode));
+
+    /* Call the open file function. Errors are handling within
+     * function, so return code can be ignored. */
+    PIOc_openfile(ios->iosysid, &ncid, &iotype, filename, mode);
+
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to delete a netCDF file.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ *
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int delete_file_handler(iosystem_desc_t *ios)
+{
+    int len;
+    int mpierr;
+    int ret;
+
+    LOG((1, "delete_file_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&len, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Get space for the filename. */
+    char filename[len + 1];
+
+    if ((mpierr = MPI_Bcast(filename, len + 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "delete_file_handler got parameters len = %d filename = %s",
+         len, filename));
+
+    /* Call the delete file function. */
+    if ((ret = PIOc_deletefile(ios->iosysid, filename)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "delete_file_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** 
+ * This function is run on the IO tasks to initialize a decomposition.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ */
+int initdecomp_dof_handler(iosystem_desc_t *ios)
+{
+    int iosysid;
+    int pio_type;
+    int ndims;
+    int maplen;
+    int ioid;
+    char rearranger_present;
+    int rearranger;
+    int *rearrangerp = NULL;
+    char iostart_present;
+    PIO_Offset *iostartp = NULL;
+    char iocount_present;
+    PIO_Offset *iocountp = NULL;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    LOG((1, "initdecomp_dof_handler called"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&pio_type, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Now we know the size of these arrays. */
+    int dims[ndims];
+    PIO_Offset iostart[ndims];
+    PIO_Offset iocount[ndims];
+
+    if ((mpierr = MPI_Bcast(dims, ndims, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&maplen, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    PIO_Offset compmap[maplen];
+
+    if ((mpierr = MPI_Bcast(compmap, maplen, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if ((mpierr = MPI_Bcast(&rearranger_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if (rearranger_present)
+        if ((mpierr = MPI_Bcast(&rearranger, 1, MPI_INT, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if ((mpierr = MPI_Bcast(&iostart_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if (iostart_present)
+        if ((mpierr = MPI_Bcast(iostart, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if ((mpierr = MPI_Bcast(&iocount_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    if (iocount_present)
+        if ((mpierr = MPI_Bcast(iocount, ndims, MPI_OFFSET, 0, ios->intercomm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "initdecomp_dof_handler iosysid = %d pio_type = %d ndims = %d maplen = %d "
+         "rearranger_present = %d iostart_present = %d iocount_present = %d ",
+         iosysid, pio_type, ndims, maplen, rearranger_present, iostart_present, iocount_present));
+
+    if (rearranger_present)
+        rearrangerp = &rearranger;
+    if (iostart_present)
+        iostartp = iostart;
+    if (iocount_present)
+        iocountp = iocount;
+
+    /* Call the function. */
+    ret = PIOc_InitDecomp(iosysid, pio_type, ndims, dims, maplen, compmap, &ioid, rearrangerp,
+                          iostartp, iocountp);
+    
+    LOG((1, "PIOc_InitDecomp returned %d", ret));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to...
+ * NOTE: not yet implemented
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ *
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int writedarray_handler(iosystem_desc_t *ios)
+{
+    assert(ios);
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to...
+ * NOTE: not yet implemented
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ *
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int readdarray_handler(iosystem_desc_t *ios)
+{
+    assert(ios);
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to set the error handler.
+ * NOTE: not yet implemented
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ *
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int seterrorhandling_handler(iosystem_desc_t *ios)
+{
+    int method;
+    int old_method_present;
+    int old_method;
+    int *old_methodp = NULL;
+    int mpierr;
+    int ret;
+
+    LOG((1, "seterrorhandling_handler comproot = %d", ios->comproot));
+    assert(ios);
+
+    /* Get the parameters for this function that the he comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&method, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&old_method_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    LOG((1, "seterrorhandling_handler got parameters method = %d old_method_present = %d",
+         method, old_method_present));
+
+    if (old_method_present)
+        old_methodp = &old_method;
+
+    /* Call the function. */
+    if ((ret = PIOc_set_iosystem_error_handling(ios->iosysid, method, old_methodp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "seterrorhandling_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to set the chunk cache
+ * parameters for netCDF-4.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ */
+int set_chunk_cache_handler(iosystem_desc_t *ios)
+{
+    int iosysid;
+    int iotype;
+    PIO_Offset size;
+    PIO_Offset nelems;
+    float preemption;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    LOG((1, "set_chunk_cache_handler called"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "set_chunk_cache_handler got params iosysid = %d iotype = %d size = %d "
+         "nelems = %d preemption = %g", iosysid, iotype, size, nelems, preemption));
+
+    /* Call the function. */
+    if ((ret = PIOc_set_chunk_cache(iosysid, iotype, size, nelems, preemption)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "set_chunk_cache_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to get the chunk cache
+ * parameters for netCDF-4.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ */
+int get_chunk_cache_handler(iosystem_desc_t *ios)
+{
+    int iosysid;
+    int iotype;
+    char size_present, nelems_present, preemption_present;
+    PIO_Offset size, *sizep;
+    PIO_Offset nelems, *nelemsp;
+    float preemption, *preemptionp;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    LOG((1, "get_chunk_cache_handler called"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&iotype, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "get_chunk_cache_handler got params iosysid = %d iotype = %d size_present = %d "
+         "nelems_present = %d preemption_present = %g", iosysid, iotype, size_present,
+         nelems_present, preemption_present));
+
+    /* Set the non-NULL pointers. */
+    if (size_present)
+        sizep = &size;
+    if (nelems_present)
+        nelemsp = &nelems;
+    if (preemption_present)
+        preemptionp = &preemption;
+
+    /* Call the function. */
+    if ((ret = PIOc_get_chunk_cache(iosysid, iotype, sizep, nelemsp, preemptionp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "get_chunk_cache_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is run on the IO tasks to get the variable chunk
+ * cache parameters for netCDF-4.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ */
+int get_var_chunk_cache_handler(iosystem_desc_t *ios)
+{
+    int ncid;
+    int varid;
+    char size_present, nelems_present, preemption_present;
+    PIO_Offset size, *sizep;
+    PIO_Offset nelems, *nelemsp;
+    float preemption, *preemptionp;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    LOG((1, "get_var_chunk_cache_handler called"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&ncid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&varid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "get_var_chunk_cache_handler got params ncid = %d varid = %d size_present = %d "
+         "nelems_present = %d preemption_present = %g", ncid, varid, size_present,
+         nelems_present, preemption_present));
+
+    /* Set the non-NULL pointers. */
+    if (size_present)
+        sizep = &size;
+    if (nelems_present)
+        nelemsp = &nelems;
+    if (preemption_present)
+        preemptionp = &preemption;
+
+    /* Call the function. */
+    if ((ret = PIOc_get_var_chunk_cache(ncid, varid, sizep, nelemsp, preemptionp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "get_var_chunk_cache_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/** This function is run on the IO tasks to free the decomp hanlder.
+ *
+ * @param ios pointer to the iosystem_desc_t data.
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ */
+int freedecomp_handler(iosystem_desc_t *ios)
+{
+    int iosysid;
+    int ioid;
+    int mpierr = MPI_SUCCESS;  /* Return code from MPI function codes. */
+    int ret; /* Return code. */
+
+    LOG((1, "freedecomp_handler called"));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Bcast(&ioid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "freedecomp_handler iosysid = %d ioid = %d", iosysid, ioid));
+
+    /* Call the function. */
+    ret = PIOc_freedecomp(iosysid, ioid);
+    
+    LOG((1, "PIOc_freedecomp returned %d", ret));
+    return PIO_NOERR;
+}
+
+/** Handle the finalize call.
+ *
+ * @param ios pointer to the iosystem info
+ * @param index
+ * @returns 0 for success, PIO_EIO for MPI Bcast errors, or error code
+ * from netCDF base function.
+ * @internal
+ */
+int finalize_handler(iosystem_desc_t *ios, int index)
+{
+    int iosysid;
+    int mpierr;
+    int ret;
+
+    LOG((1, "finalize_handler called index = %d", index));
+    assert(ios);
+
+    /* Get the parameters for this function that the the comp master
+     * task is broadcasting. */
+    if ((mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, 0, ios->intercomm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((1, "finalize_handler got parameter iosysid = %d", iosysid));
+
+    /* Call the function. */
+    LOG((2, "finalize_handler calling PIOc_finalize for iosysid = %d",
+         iosysid));
+    if ((ret = PIOc_finalize(iosysid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    LOG((1, "finalize_handler succeeded!"));
+    return PIO_NOERR;
+}
+
+/**
+ * This function is called by the IO tasks.  This function will not
+ * return, unless there is an error.
+ *
+ * @param io_rank
+ * @param component_count number of computation components
+ * @param iosys pointer to pointer to iosystem info
+ * @param io_comm MPI communicator for IO
+ * @returns 0 for success, error code otherwise.
+ */
+int pio_msg_handler2(int io_rank, int component_count, iosystem_desc_t **iosys,
+                     MPI_Comm io_comm)
+{
+    iosystem_desc_t *my_iosys;
+    int msg = 0;
+    MPI_Request req[component_count];
+    MPI_Status status;
+    int index;
+    int mpierr;
+    int ret = PIO_NOERR;
+    int open_components = component_count;
+
+    LOG((1, "pio_msg_handler2 called"));
+    assert(iosys);
+
+    /* Have IO comm rank 0 (the ioroot) register to receive
+     * (non-blocking) for a message from each of the comproots. */
+    if (!io_rank)
+    {
+        for (int cmp = 0; cmp < component_count; cmp++)
+        {
+            my_iosys = iosys[cmp];
+            LOG((1, "about to call MPI_Irecv union_comm = %d", my_iosys->union_comm));
+            if ((mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG,
+                                    my_iosys->union_comm, &req[cmp])))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            LOG((1, "MPI_Irecv req[%d] = %d", cmp, req[cmp]));
+        }
+    }
+
+    /* If the message is not -1, keep processing messages. */
+    while (msg != -1)
+    {
+        LOG((3, "pio_msg_handler2 at top of loop"));
+
+        /* Wait until any one of the requests are complete. Once it
+         * returns, the Waitany function automatically sets the
+         * appropriate member of the req array to MPI_REQUEST_NULL. */
+        if (!io_rank)
+        {
+            LOG((1, "about to call MPI_Waitany req[0] = %d MPI_REQUEST_NULL = %d",
+                 req[0], MPI_REQUEST_NULL));
+            for (int c = 0; c < component_count; c++)
+                LOG((2, "req[%d] = %d", c, req[c]));
+            if ((mpierr = MPI_Waitany(component_count, req, &index, &status)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            LOG((3, "Waitany returned index = %d req[%d] = %d", index, index, req[index]));
+        }
+
+        /* Broadcast the index of the computational component that
+         * originated the request to the rest of the IO tasks. */
+        LOG((3, "About to do Bcast of index = %d io_comm = %d", index, io_comm));
+        if ((mpierr = MPI_Bcast(&index, 1, MPI_INT, 0, io_comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        LOG((3, "index MPI_Bcast complete index = %d", index));
+
+        /* Set the correct iosys depending on the index. */
+        my_iosys = iosys[index];
+
+        /* Broadcast the msg value to the rest of the IO tasks. */
+        LOG((3, "about to call msg MPI_Bcast my_iosys->io_comm = %d", my_iosys->io_comm));
+        if ((mpierr = MPI_Bcast(&msg, 1, MPI_INT, 0, my_iosys->io_comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        LOG((1, "pio_msg_handler2 msg MPI_Bcast complete msg = %d", msg));
+
+        /* Handle the message. This code is run on all IO tasks. */
+        switch (msg)
+        {
+        case PIO_MSG_INQ_TYPE:
+            inq_type_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_FORMAT:
+            inq_format_handler(my_iosys);
+            break;
+        case PIO_MSG_CREATE_FILE:
+            create_file_handler(my_iosys);
+            LOG((2, "returned from create_file_handler"));
+            break;
+        case PIO_MSG_SYNC:
+            sync_file_handler(my_iosys);
+            break;
+        case PIO_MSG_ENDDEF:
+        case PIO_MSG_REDEF:
+            LOG((2, "calling change_def_file_handler"));
+            change_def_file_handler(my_iosys, msg);
+            LOG((2, "returned from change_def_file_handler"));
+            break;
+        case PIO_MSG_OPEN_FILE:
+            open_file_handler(my_iosys);
+            break;
+        case PIO_MSG_CLOSE_FILE:
+            close_file_handler(my_iosys);
+            break;
+        case PIO_MSG_DELETE_FILE:
+            delete_file_handler(my_iosys);
+            break;
+        case PIO_MSG_RENAME_DIM:
+            rename_dim_handler(my_iosys);
+            break;
+        case PIO_MSG_RENAME_VAR:
+            rename_var_handler(my_iosys);
+            break;
+        case PIO_MSG_RENAME_ATT:
+            rename_att_handler(my_iosys);
+            break;
+        case PIO_MSG_DEL_ATT:
+            delete_att_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_DIM:
+            def_dim_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_VAR:
+            def_var_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_VAR_CHUNKING:
+            def_var_chunking_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_VAR_FILL:
+            def_var_fill_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_VAR_ENDIAN:
+            def_var_endian_handler(my_iosys);
+            break;
+        case PIO_MSG_DEF_VAR_DEFLATE:
+            def_var_deflate_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VAR_ENDIAN:
+            inq_var_endian_handler(my_iosys);
+            break;
+        case PIO_MSG_SET_VAR_CHUNK_CACHE:
+            set_var_chunk_cache_handler(my_iosys);
+            break;
+        case PIO_MSG_GET_VAR_CHUNK_CACHE:
+            get_var_chunk_cache_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ:
+            inq_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_DIM:
+            inq_dim_handler(my_iosys, msg);
+            break;
+        case PIO_MSG_INQ_DIMID:
+            inq_dimid_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VAR:
+            inq_var_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VAR_CHUNKING:
+            inq_var_chunking_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VAR_FILL:
+            inq_var_fill_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VAR_DEFLATE:
+            inq_var_deflate_handler(my_iosys);
+            break;
+        case PIO_MSG_GET_ATT:
+            ret = att_get_handler(my_iosys);
+            break;
+        case PIO_MSG_PUT_ATT:
+            ret = att_put_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_VARID:
+            inq_varid_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_ATT:
+            inq_att_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_ATTNAME:
+            inq_attname_handler(my_iosys);
+            break;
+        case PIO_MSG_INQ_ATTID:
+            inq_attid_handler(my_iosys);
+            break;
+        case PIO_MSG_GET_VARS:
+            get_vars_handler(my_iosys);
+            break;
+        case PIO_MSG_PUT_VARS:
+            put_vars_handler(my_iosys);
+            break;
+        case PIO_MSG_INITDECOMP_DOF:
+            initdecomp_dof_handler(my_iosys);
+            break;
+        case PIO_MSG_WRITEDARRAY:
+            writedarray_handler(my_iosys);
+            break;
+        case PIO_MSG_READDARRAY:
+            readdarray_handler(my_iosys);
+            break;
+        case PIO_MSG_SETERRORHANDLING:
+            seterrorhandling_handler(my_iosys);
+            break;
+        case PIO_MSG_SET_CHUNK_CACHE:
+            set_chunk_cache_handler(my_iosys);
+            break;
+        case PIO_MSG_GET_CHUNK_CACHE:
+            get_chunk_cache_handler(my_iosys);
+            break;
+        case PIO_MSG_FREEDECOMP:
+            freedecomp_handler(my_iosys);
+            break;
+        case PIO_MSG_SET_FILL:
+            set_fill_handler(my_iosys);
+            break;
+        case PIO_MSG_EXIT:
+            finalize_handler(my_iosys, index);
+            msg = -1;
+            break;
+        default:
+            LOG((0, "unknown message received %d", msg));
+            return PIO_EINVAL;
+        }
+
+        /* If an error was returned by the handler, do nothing! */
+        LOG((3, "pio_msg_handler2 checking error ret = %d", ret));
+
+        /* Listen for another msg from the component whose message we
+         * just handled. */
+        if (!io_rank && msg != -1)
+        {
+            my_iosys = iosys[index];
+            LOG((3, "pio_msg_handler2 about to Irecv index = %d comproot = %d union_comm = %d",
+                 index, my_iosys->comproot, my_iosys->union_comm));
+            if ((mpierr = MPI_Irecv(&msg, 1, MPI_INT, my_iosys->comproot, MPI_ANY_TAG, my_iosys->union_comm,
+                                    &req[index])))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            LOG((3, "pio_msg_handler2 called MPI_Irecv req[%d] = %d", index, req[index]));
+        }
+
+        LOG((3, "pio_msg_handler2 done msg = %d open_components = %d",
+             msg, open_components));
+
+        /* If there are no more open components, exit. */
+        if (msg == -1)
+            if (--open_components)
+                msg = PIO_MSG_EXIT;
+    }
+
+    LOG((3, "returning from pio_msg_handler2"));
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pio_nc.c b/src/externals/pio2/src/clib/pio_nc.c
new file mode 100644
index 00000000000..00932bdfd64
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_nc.c
@@ -0,0 +1,2756 @@
+/**
+ * @file
+ * PIO interfaces to
+ * [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html)
+ * support functions
+ *
+ *  This file provides an interface to the
+ *  [NetCDF](http://www.unidata.ucar.edu/software/netcdf/docs/modules.html)
+ *  support functions.  Each subroutine calls the underlying netcdf or
+ *  pnetcdf or netcdf4 functions from the appropriate subset of mpi
+ *  tasks (io_comm). Each routine must be called collectively from
+ *  union_comm.
+ *
+ * @author Jim Edwards (jedwards@ucar.edu), Ed Hartnett
+ * @date     Feburary 2014, April 2016
+ */
+#include 
+#include 
+#include 
+
+/**
+ * @ingroup PIO_inq
+ * The PIO-C interface for the NetCDF function nc_inq.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm. For more information on the underlying
+ * NetCDF commmand please read about this function in the NetCDF
+ * documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ *
+ * @return PIO_NOERR for success, error code otherwise. See
+ * PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq(int ncid, int *ndimsp, int *nvarsp, int *ngattsp, int *unlimdimidp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function calls. */
+
+    LOG((1, "PIOc_inq ncid = %d", ncid));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ; /* Message for async notification. */
+            char ndims_present = ndimsp ? true : false;
+            char nvars_present = nvarsp ? true : false;
+            char ngatts_present = ngattsp ? true : false;
+            char unlimdimid_present = unlimdimidp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nvars_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ngatts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&unlimdimid_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq ncid = %d ndims_present = %d nvars_present = %d ngatts_present = %d unlimdimid_present = %d",
+                 ncid, ndims_present, nvars_present, ngatts_present, unlimdimid_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            ierr = ncmpi_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);
+            if (unlimdimidp)
+                LOG((2, "PIOc_inq returned from ncmpi_inq unlimdimid = %d", *unlimdimidp));
+        }
+#endif /* _PNETCDF */
+        if (file->iotype == PIO_IOTYPE_NETCDF && file->do_io)
+        {
+            LOG((2, "PIOc_inq calling classic nc_inq"));
+            /* Should not be necessary to do this - nc_inq should
+             * handle null pointers. This has been reported as a bug
+             * to netCDF developers. */
+            int tmp_ndims, tmp_nvars, tmp_ngatts, tmp_unlimdimid;
+            LOG((2, "PIOc_inq calling classic nc_inq"));
+            ierr = nc_inq(file->fh, &tmp_ndims, &tmp_nvars, &tmp_ngatts, &tmp_unlimdimid);
+            LOG((2, "PIOc_inq calling classic nc_inq"));
+            if (unlimdimidp)
+                LOG((2, "classic tmp_unlimdimid = %d", tmp_unlimdimid));
+            if (ndimsp)
+                *ndimsp = tmp_ndims;
+            if (nvarsp)
+                *nvarsp = tmp_nvars;
+            if (ngattsp)
+                *ngattsp = tmp_ngatts;
+            if (unlimdimidp)
+                *unlimdimidp = tmp_unlimdimid;
+            if (unlimdimidp)
+                LOG((2, "classic unlimdimid = %d", *unlimdimidp));
+        }
+        else if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            LOG((2, "PIOc_inq calling netcdf-4 nc_inq"));
+            ierr = nc_inq(file->fh, ndimsp, nvarsp, ngattsp, unlimdimidp);
+        }
+
+        LOG((2, "PIOc_inq netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (ndimsp)
+        if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    if (nvarsp)
+        if ((mpierr = MPI_Bcast(nvarsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    if (ngattsp)
+        if ((mpierr = MPI_Bcast(ngattsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    if (unlimdimidp)
+        if ((mpierr = MPI_Bcast(unlimdimidp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_ndims
+ * Find out how many dimensions are defined in the file.
+ *
+ * @param ncid the ncid of the open file.
+ * @param ndimsp a pointer that will get the number of dimensions.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_ndims(int ncid, int *ndimsp)
+{
+    LOG((1, "PIOc_inq_ndims"));
+    return PIOc_inq(ncid, ndimsp, NULL, NULL, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_nvars
+ * Find out how many variables are defined in a file.
+ *
+ * @param ncid the ncid of the open file.
+ * @param nvarsp a pointer that will get the number of variables.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_nvars(int ncid, int *nvarsp)
+{
+    return PIOc_inq(ncid, NULL, nvarsp, NULL, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_natts
+ * Find out how many global attributes are defined in a file.
+ *
+ * @param ncid the ncid of the open file.
+ * @param nattsp a pointer that will get the number of attributes.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_natts(int ncid, int *ngattsp)
+{
+    return PIOc_inq(ncid, NULL, NULL, ngattsp, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_unlimdim
+ * Find out the dimension ids of any unlimited dimensions.
+ *
+ * @param ncid the ncid of the open file.
+ * @param nattsp a pointer that will get an array of unlimited
+ * dimension IDs.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_unlimdim(int ncid, int *unlimdimidp)
+{
+    LOG((1, "PIOc_inq_unlimdim ncid = %d", ncid));
+    return PIOc_inq(ncid, NULL, NULL, NULL, unlimdimidp);
+}
+
+/**
+ * @ingroup PIO_typelen
+ * Learn the name and size of a type.
+ *
+ * @param ncid the ncid of the open file.
+ * @param xtype the type to learn about
+ * @param name pointer that will get the name of the type.
+ * @param sizep pointer that will get the size of the type in bytes.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_type(int ncid, nc_type xtype, char *name, PIO_Offset *sizep)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_type ncid = %d xtype = %d", ncid, xtype));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_TYPE; /* Message for async notification. */
+            char name_present = name ? true : false;
+            char size_present = sizep ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = pioc_pnetcdf_inq_type(ncid, xtype, name, sizep);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_type(file->fh, xtype, name, (size_t *)sizep);
+        LOG((2, "PIOc_inq_type netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (name)
+    {
+        int slen;
+        if (ios->iomaster == MPI_ROOT)
+            slen = strlen(name);
+        if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if (!mpierr)
+            if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
+                return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+    if (sizep)
+        if ((mpierr = MPI_Bcast(sizep , 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_format
+ * Learn the netCDF format of an open file.
+ *
+ * @param ncid the ncid of an open file.
+ * @param formatp a pointer that will get the format.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_format(int ncid, int *formatp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq ncid = %d", ncid));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_FORMAT;
+            char format_present = formatp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&format_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_format(file->fh, formatp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_format(file->fh, formatp);
+        LOG((2, "PIOc_inq netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (formatp)
+        if ((mpierr = MPI_Bcast(formatp , 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_dim
+ * The PIO-C interface for the NetCDF function nc_inq_dim.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param lenp a pointer that will get the number of values
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_dim(int ncid, int dimid, char *name, PIO_Offset *lenp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_dim ncid = %d dimid = %d", ncid, dimid));
+
+    /* Get the file info, based on the ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_DIM;
+            char name_present = name ? true : false;
+            char len_present = lenp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq netcdf Bcast name_present = %d", name_present));
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq netcdf Bcast len_present = %d", len_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            LOG((2, "calling ncmpi_inq_dim"));
+            ierr = ncmpi_inq_dim(file->fh, dimid, name, lenp);;
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            LOG((2, "calling nc_inq_dim"));
+            ierr = nc_inq_dim(file->fh, dimid, name, (size_t *)lenp);;
+        }
+        LOG((2, "ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (name)
+    {
+        int slen;
+        LOG((2, "bcasting results my_comm = %d", ios->my_comm));
+        if (ios->iomaster == MPI_ROOT)
+            slen = strlen(name);
+        if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (lenp)
+        if ((mpierr = MPI_Bcast(lenp , 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "done with PIOc_inq_dim"));
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_dimname
+ * Find the name of a dimension.
+ *
+ * @param ncid the ncid of an open file.
+ * @param dimid the dimension ID.
+ * @param name a pointer that gets the name of the dimension. Igorned
+ * if NULL.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_dimname(int ncid, int dimid, char *name)
+{
+    LOG((1, "PIOc_inq_dimname ncid = %d dimid = %d", ncid, dimid));
+    return PIOc_inq_dim(ncid, dimid, name, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_dimlen
+ * Find the length of a dimension.
+ *
+ * @param ncid the ncid of an open file.
+ * @param dimid the dimension ID.
+ * @param lenp a pointer that gets the length of the dimension. Igorned
+ * if NULL.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_inq_dimlen(int ncid, int dimid, PIO_Offset *lenp)
+{
+    return PIOc_inq_dim(ncid, dimid, NULL, lenp);
+}
+
+/**
+ * @ingroup PIO_inq_dimid
+ * The PIO-C interface for the NetCDF function nc_inq_dimid.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param idp a pointer that will get the id of the variable or attribute.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_dimid(int ncid, const char *name, int *idp)
+{
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    int ierr;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info, based on the ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+    LOG((2, "iosysid = %d", ios->iosysid));
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_inq_dimid ncid = %d name = %s", ncid, name));
+
+    /* If using async, and not an IO task, then send parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_DIMID;
+            char id_present = idp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            int namelen = strlen(name);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* IO tasks call the netCDF functions. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_dimid(file->fh, name, idp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_dimid(file->fh, name, idp);
+    }
+    LOG((3, "nc_inq_dimid call complete ierr = %d", ierr));
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results. */
+    if (idp)
+        if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_var
+ * The PIO-C interface for the NetCDF function nc_inq_var.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param xtypep a pointer that will get the type of the attribute.
+ * @param nattsp a pointer that will get the number of attributes
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_var(int ncid, int varid, char *name, nc_type *xtypep, int *ndimsp,
+                 int *dimidsp, int *nattsp)
+{
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    int ndims = 0;    /* The number of dimensions for this variable. */
+    int ierr;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_var ncid = %d varid = %d", ncid, varid));
+
+    /* Get the file info, based on the ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR;
+            char name_present = name ? true : false;
+            char xtype_present = xtypep ? true : false;
+            char ndims_present = ndimsp ? true : false;
+            char dimids_present = dimidsp ? true : false;
+            char natts_present = nattsp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&dimids_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&natts_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq_var name_present = %d xtype_present = %d ndims_present = %d "
+                 "dimids_present = %d, natts_present = %d nattsp = %d",
+                 name_present, xtype_present, ndims_present, dimids_present, natts_present, nattsp));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Call the netCDF layer. */
+    if (ios->ioproc)
+    {
+        LOG((2, "Calling the netCDF layer"));
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            ierr = ncmpi_inq_varndims(file->fh, varid, &ndims);
+            LOG((2, "from pnetcdf ndims = %d", ndims));
+            if (!ierr)
+                ierr = ncmpi_inq_var(file->fh, varid, name, xtypep, ndimsp, dimidsp, nattsp);
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            ierr = nc_inq_varndims(file->fh, varid, &ndims);
+            if (!ierr)
+            {
+                char my_name[NC_MAX_NAME + 1];
+                nc_type my_xtype;
+                int my_ndims = 0, my_dimids[ndims], my_natts = 0;
+                ierr = nc_inq_var(file->fh, varid, my_name, &my_xtype, &my_ndims, my_dimids, &my_natts);
+                LOG((3, "my_name = %s my_xtype = %d my_ndims = %d my_natts = %d",  my_name, my_xtype, my_ndims, my_natts));
+                if (!ierr)
+                {
+                    if (name)
+                        strcpy(name, my_name);
+                    if (xtypep)
+                        *xtypep = my_xtype;
+                    if (ndimsp)
+                        *ndimsp = my_ndims;
+                    if (dimidsp)
+                    {
+                        for (int d = 0; d < ndims; d++)
+                            dimidsp[d] = my_dimids[d];
+                    }
+                    if (nattsp)
+                        *nattsp = my_natts;
+                }
+            }
+        }
+        if (ndimsp)
+            LOG((2, "PIOc_inq_var ndims = %d ierr = %d", *ndimsp, ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast the results for non-null pointers. */
+    if (name)
+    {
+        int slen;
+        if (ios->iomaster == MPI_ROOT)
+            slen = strlen(name);
+        if ((mpierr = MPI_Bcast(&slen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast((void *)name, slen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+    if (xtypep)
+        if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    if (ndimsp)
+    {
+        if (ios->ioroot)
+            LOG((2, "PIOc_inq_var about to Bcast ndims = %d ios->ioroot = %d", *ndimsp, ios->ioroot));
+        if ((mpierr = MPI_Bcast(ndimsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        file->varlist[varid].ndims = *ndimsp;
+        LOG((2, "PIOc_inq_var Bcast ndims = %d", *ndimsp));
+    }
+    if (dimidsp)
+    {
+        if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(dimidsp, ndims, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+    if (nattsp)
+        if ((mpierr = MPI_Bcast(nattsp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_varname
+ * Get the name of a variable.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the variable ID.
+ * @param name a pointer that will get the variable name.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_varname(int ncid, int varid, char *name)
+{
+    return PIOc_inq_var(ncid, varid, name, NULL, NULL, NULL, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_vartype
+ * Find the type of a variable.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the variable ID.
+ * @param xtypep a pointer that will get the type of the
+ * attribute. Ignored if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_vartype(int ncid, int varid, nc_type *xtypep)
+{
+    return PIOc_inq_var(ncid, varid, NULL, xtypep, NULL, NULL, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_varndims
+ * Find the number of dimensions of a variable.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the variable ID.
+ * @param ndimsp a pointer that will get the number of
+ * dimensions. Ignored if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_varndims(int ncid, int varid, int *ndimsp)
+{
+    return PIOc_inq_var(ncid, varid, NULL, NULL, ndimsp, NULL, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_vardimid
+ * Find the dimension IDs associated with a variable.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the variable ID.
+ * @param dimidsp a pointer that will get an array of dimids. Ignored
+ * if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_vardimid(int ncid, int varid, int *dimidsp)
+{
+    return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, dimidsp, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_varnatts
+ * Find the number of attributes associated with a variable.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the variable ID.
+ * @param nattsp a pointer that will get the number of attriburtes. Ignored
+ * if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_varnatts(int ncid, int varid, int *nattsp)
+{
+    return PIOc_inq_var(ncid, varid, NULL, NULL, NULL, NULL, nattsp);
+}
+
+/**
+ * @ingroup PIO_inq_varid
+ * The PIO-C interface for the NetCDF function nc_inq_varid.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param varidp a pointer that will get the variable id
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_varid(int ncid, const char *name, int *varidp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get file info based on ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Caller must provide name. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_inq_varid ncid = %d name = %s", ncid, name));
+
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VARID;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            int namelen;
+            namelen = strlen(name);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_varid(file->fh, name, varidp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_varid(file->fh, name, varidp);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (varidp)
+        if ((mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_att
+ * The PIO-C interface for the NetCDF function nc_inq_att.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param xtypep a pointer that will get the type of the attribute.
+ * @param lenp a pointer that will get the number of values
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_att(int ncid, int varid, const char *name, nc_type *xtypep,
+                 PIO_Offset *lenp)
+{
+    int msg = PIO_MSG_INQ_ATT;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr;
+
+    /* Find file based on ncid. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_inq_att ncid = %d varid = %d", ncid, varid));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            char xtype_present = xtypep ? true : false;
+            char len_present = lenp ? true : false;
+            int namelen = strlen(name);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_att(file->fh, varid, name, xtypep, lenp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_att(file->fh, varid, name, xtypep, (size_t *)lenp);
+        LOG((2, "PIOc_inq netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results. */
+    if (xtypep)
+        if ((mpierr = MPI_Bcast(xtypep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (lenp)
+        if ((mpierr = MPI_Bcast(lenp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_attlen
+ * Get the length of an attribute.
+ *
+ * @param ncid the ID of an open file.
+ * @param varid the variable ID, or NC_GLOBAL for global attributes.
+ * @param name the name of the attribute.
+ * @param lenp a pointer that gets the lenght of the attribute
+ * array. Ignored if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_attlen(int ncid, int varid, const char *name, PIO_Offset *lenp)
+{
+    return PIOc_inq_att(ncid, varid, name, NULL, lenp);
+}
+
+/**
+ * @ingroup PIO_inq_atttype
+ * Get the type of an attribute.
+ *
+ * @param ncid the ID of an open file.
+ * @param varid the variable ID, or NC_GLOBAL for global attributes.
+ * @param name the name of the attribute.
+ * @param xtypep a pointer that gets the type of the
+ * attribute. Ignored if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_inq_atttype(int ncid, int varid, const char *name, nc_type *xtypep)
+{
+    return PIOc_inq_att(ncid, varid, name, xtypep, NULL);
+}
+
+/**
+ * @ingroup PIO_inq_attname
+ * The PIO-C interface for the NetCDF function nc_inq_attname.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param attnum the attribute ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_attname(int ncid, int varid, int attnum, char *name)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_attname ncid = %d varid = %d attnum = %d", ncid, varid,
+         attnum));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_ATTNAME;
+            char name_present = name ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&attnum, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&name_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_attname(file->fh, varid, attnum, name);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_attname(file->fh, varid, attnum, name);
+        LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (name)
+    {
+        int namelen = strlen(name);
+        if ((mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        /* Casting to void to avoid warnings on some compilers. */
+        if ((mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_inq_attid
+ * The PIO-C interface for the NetCDF function nc_inq_attid.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param idp a pointer that will get the id of the variable or attribute.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_inq_attid(int ncid, int varid, const char *name, int *idp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_inq_attid ncid = %d varid = %d name = %s", ncid, varid, name));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_ATTID;
+            int namelen = strlen(name);
+            char id_present = idp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&id_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_inq_attid(file->fh, varid, name, idp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_inq_attid(file->fh, varid, name, idp);
+        LOG((2, "PIOc_inq_attname netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results. */
+    if (idp)
+        if ((mpierr = MPI_Bcast(idp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_rename_dim
+ * The PIO-C interface for the NetCDF function nc_rename_dim.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_rename_dim(int ncid, int dimid, const char *name)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_rename_dim ncid = %d dimid = %d name = %s", ncid, dimid, name));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_RENAME_DIM; /* Message for async notification. */
+            int namelen = strlen(name);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&dimid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_rename_dim Bcast file->fh = %d dimid = %d namelen = %d name = %s",
+                 file->fh, dimid, namelen, name));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_rename_dim(file->fh, dimid, name);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_rename_dim(file->fh, dimid, name);
+        LOG((2, "PIOc_inq netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_rename_var
+ * The PIO-C interface for the NetCDF function nc_rename_var.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See PIOc_Set_File_Error_Handling
+ */
+int PIOc_rename_var(int ncid, int varid, const char *name)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_rename_var ncid = %d varid = %d name = %s", ncid, varid, name));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_RENAME_VAR; /* Message for async notification. */
+            int namelen = strlen(name);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_rename_var Bcast file->fh = %d varid = %d namelen = %d name = %s",
+                 file->fh, varid, namelen, name));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_rename_var(file->fh, varid, name);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_rename_var(file->fh, varid, name);
+        LOG((2, "PIOc_inq netcdf call returned %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_rename_att
+ * The PIO-C interface for the NetCDF function nc_rename_att.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @return PIO_NOERR for success, error code otherwise.  See
+ * PIOc_Set_File_Error_Handling
+ */
+int PIOc_rename_att(int ncid, int varid, const char *name,
+                    const char *newname)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI functions. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide names of correct length. */
+    if (!name || strlen(name) > NC_MAX_NAME ||
+        !newname || strlen(newname) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_rename_att ncid = %d varid = %d name = %s newname = %s",
+         ncid, varid, name, newname));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_RENAME_ATT; /* Message for async notification. */
+            int namelen = strlen(name);
+            int newnamelen = strlen(newname);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&newnamelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((char *)newname, newnamelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_rename_att(file->fh, varid, name, newname);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_rename_att(file->fh, varid, name, newname);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    LOG((2, "PIOc_rename_att succeeded"));
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_del_att
+ * The PIO-C interface for the NetCDF function nc_del_att.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__attributes.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name of the attribute to delete.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_del_att(int ncid, int varid, const char *name)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI functions. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_del_att ncid = %d varid = %d name = %s", ncid, varid, name));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEL_ATT;
+            int namelen = strlen(name); /* Length of name string. */
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((char *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_del_att(file->fh, varid, name);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_del_att(file->fh, varid, name);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * The PIO-C interface for the NetCDF function nc_set_fill.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param fillmode either NC_FILL or NC_NOFILL.
+ * @param old_modep a pointer to an int that gets the old setting.
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_set_fill
+ */
+int PIOc_set_fill(int ncid, int fillmode, int *old_modep)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI functions. */
+
+    LOG((1, "PIOc_set_fill ncid = %d fillmode = %d", ncid, fillmode));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_SET_FILL;
+            int old_modep_present = old_modep ? 1 : 0;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&fillmode, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&old_modep_present, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_set_fill sent ncid = %d fillmode = %d old_modep_present = %d", ncid, fillmode,
+                 old_modep_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            LOG((3, "about to call ncmpi_set_fill() fillmode = %d", fillmode));
+            ierr = ncmpi_set_fill(file->fh, fillmode, old_modep);
+        }
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_set_fill(file->fh, fillmode, old_modep);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results. */
+    if (old_modep)
+    {
+        LOG((2, "old_mode = %d", *old_modep));
+        if ((mpierr = MPI_Bcast(old_modep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    LOG((2, "PIOc_set_fill succeeded"));
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_enddef
+ * The PIO-C interface for the NetCDF function nc_enddef.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_enddef(int ncid)
+{
+    return pioc_change_def(ncid, 1);
+}
+
+/**
+ * @ingroup PIO_redef
+ * The PIO-C interface for the NetCDF function nc_redef.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__datasets.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_redef(int ncid)
+{
+    return pioc_change_def(ncid, 0);
+}
+
+/**
+ * @ingroup PIO_def_dim
+ * The PIO-C interface for the NetCDF function nc_def_dim.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__dimensions.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param idp a pointer that will get the id of the variable or attribute.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_def_dim(int ncid, const char *name, PIO_Offset len, int *idp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name shorter than NC_MAX_NAME +1. */
+    if (!name || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_def_dim ncid = %d name = %s len = %d", ncid, name, len));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_DIM;
+            int namelen = strlen(name);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+        }
+
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_def_dim(file->fh, name, len, idp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_def_dim(file->fh, name, (size_t)len, idp);
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (idp)
+        if ((mpierr = MPI_Bcast(idp , 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "def_dim ierr = %d", ierr));
+    return PIO_NOERR;
+}
+
+/**
+ * The PIO-C interface for the NetCDF function nc_def_var.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param varidp a pointer that will get the variable id
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_def_var
+ */
+int PIOc_def_var(int ncid, const char *name, nc_type xtype, int ndims,
+                 const int *dimidsp, int *varidp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file information. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide name and storage for varid. */
+    if (!name || !varidp || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_def_var ncid = %d name = %s xtype = %d ndims = %d", ncid, name,
+         xtype, ndims));
+
+    /* If using async, and not an IO task, then send parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR;
+            int namelen = strlen(name);
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&(ncid), 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&namelen, 1, MPI_INT,  ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)name, namelen + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)dimidsp, ndims, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+            ierr = ncmpi_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);
+#endif /* _PNETCDF */
+
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+            ierr = nc_def_var(file->fh, name, xtype, ndims, dimidsp, varidp);
+#ifdef _NETCDF4
+        /* For netCDF-4 serial files, turn on compression for this variable. */
+        if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4C)
+            ierr = nc_def_var_deflate(file->fh, *varidp, 0, 1, 1);
+
+        /* For netCDF-4 parallel files, set parallel access to collective. */
+        if (!ierr && file->iotype == PIO_IOTYPE_NETCDF4P)
+            ierr = nc_var_par_access(file->fh, *varidp, NC_COLLECTIVE);
+#endif /* _NETCDF4 */
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results. */
+    if (varidp)
+        if ((mpierr = MPI_Bcast(varidp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set the fill value for a variable.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * When the fill mode for the file is NC_FILL, then fill values are
+ * used for missing data. This function sets the fill value to be used
+ * for a variable. If no specific fill value is set (as a _FillValue
+ * attribute), then the default fill values from netcdf.h are used.
+ *
+ * NetCDF-4 and pnetcdf files allow setting fill_mode (to NC_FILL or
+ * NC_NOFILL) on a per-variable basis. NetCDF classic only allows the
+ * fill_mode setting to be set for the whole file. For this function,
+ * the fill_mode parameter is ignored for classic files. Set the
+ * file-level fill mode with PIOc_set_fill().
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param fill_mode fill mode for this variable (NC_FILL or NC_NOFILL)
+ * @param fill_value pointer to the fill value to be used if fill_mode is set to NC_FILL.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_def_var_fill(int ncid, int varid, int fill_mode, const void *fill_valuep)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    nc_type xtype;         /* The type of the variable (and fill value att). */
+    PIO_Offset type_size;  /* Size in bytes of this variable's type. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_def_var_fill ncid = %d varid = %d fill_mode = %d\n", ncid, varid,
+         fill_mode));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Caller must provide correct values. */
+    if ((fill_mode != NC_FILL && fill_mode != NC_NOFILL) ||
+        (fill_mode == NC_FILL && !fill_valuep))
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Run this on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. Get the size of this vars
+     * type. */
+    if (!ios->async || !ios->ioproc)
+    {
+        if ((ierr = PIOc_inq_vartype(ncid, varid, &xtype)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+        if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &type_size)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+    }
+    LOG((2, "PIOc_def_var_fill type_size = %d", type_size));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_FILL;
+            char fill_value_present = fill_valuep ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&fill_mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && fill_value_present)
+                mpierr = MPI_Bcast((PIO_Offset *)fill_valuep, type_size, MPI_CHAR, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "PIOc_def_var_fill ncid = %d varid = %d fill_mode = %d type_size = %d fill_value_present = %d",
+                 ncid, varid, fill_mode, type_size, fill_value_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+#ifdef _PNETCDF
+            ierr = ncmpi_def_var_fill(file->fh, varid, fill_mode, (void *)fill_valuep);
+#endif /* _PNETCDF */
+        }
+        else if (file->iotype == PIO_IOTYPE_NETCDF)
+        {
+            LOG((2, "defining fill value attribute for netCDF classic file"));
+            if (file->do_io)            
+                ierr = nc_put_att(file->fh, varid, _FillValue, xtype, 1, fill_valuep);
+        }
+        else
+        {
+#ifdef _NETCDF4
+            if (file->do_io)
+                ierr = nc_def_var_fill(file->fh, varid, fill_mode, fill_valuep);
+#endif
+        }
+        LOG((2, "after def_var_fill ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * The PIO-C interface for the NetCDF function nc_inq_var_fill.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm. For more information on the underlying NetCDF commmand
+ * please read about this function in the NetCDF documentation at:
+ * http://www.unidata.ucar.edu/software/netcdf/docs/group__variables.html
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param no_fill a pointer to int that will get the fill
+ * mode. Ignored if NULL (except with pnetcdf, which seg-faults with
+ * NULL.)
+ * @param fill_valuep pointer to space that gets the fill value for
+ * this variable. Ignored if NULL.
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_inq_var_fill
+ */
+int PIOc_inq_var_fill(int ncid, int varid, int *no_fill, void *fill_valuep)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    nc_type xtype;         /* Type of variable and its _FillValue attribute. */
+    PIO_Offset type_size;  /* Size in bytes of this variable's type. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+
+    LOG((1, "PIOc_inq_var_fill ncid = %d varid = %d", ncid, varid));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+    LOG((2, "found file"));
+
+    /* Run this on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. Get the size of this vars
+     * type. */
+    if (!ios->async || !ios->ioproc)
+    {
+        if ((ierr = PIOc_inq_vartype(ncid, varid, &xtype)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+        if ((ierr = PIOc_inq_type(ncid, xtype, NULL, &type_size)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+        LOG((2, "PIOc_inq_var_fill type_size = %d", type_size));
+    }
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_FILL;
+            char no_fill_present = no_fill ? true : false;
+            char fill_value_present = fill_valuep ? true : false;
+
+            LOG((2, "sending msg type_size = %d", type_size));
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&no_fill_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&fill_value_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq_var_fill ncid = %d varid = %d type_size = %lld no_fill_present = %d fill_value_present = %d",
+                 ncid, varid, type_size, no_fill_present, fill_value_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&xtype, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(&type_size, 1, MPI_OFFSET, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        LOG((2, "calling inq_var_fill file->iotype = %d file->fh = %d varid = %d",
+             file->iotype, file->fh, varid));
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+#ifdef _PNETCDF
+            ierr = ncmpi_inq_var_fill(file->fh, varid, no_fill, fill_valuep);
+#endif /* _PNETCDF */
+        }
+        else if (file->iotype == PIO_IOTYPE_NETCDF && file->do_io)
+        {
+            /* Get the file-level fill mode. */
+            if (no_fill)
+            {
+                ierr = nc_set_fill(file->fh, NC_NOFILL, no_fill);
+                if (!ierr)
+                    ierr = nc_set_fill(file->fh, *no_fill, NULL);
+            }
+
+            if (!ierr && fill_valuep)
+            {
+                ierr = nc_get_att(file->fh, varid, _FillValue, fill_valuep);
+                if (ierr == NC_ENOTATT)
+                {
+                    char char_fill_value = NC_FILL_CHAR;
+                    signed char byte_fill_value = NC_FILL_BYTE;
+                    short short_fill_value = NC_FILL_SHORT;
+                    int int_fill_value = NC_FILL_INT;
+                    float float_fill_value = NC_FILL_FLOAT;
+                    double double_fill_value = NC_FILL_DOUBLE;
+                    switch (xtype)
+                    {
+                    case NC_BYTE:
+                        memcpy(fill_valuep, &byte_fill_value, sizeof(signed char));
+                        break;
+                    case NC_CHAR:
+                        memcpy(fill_valuep, &char_fill_value, sizeof(char));
+                        break;
+                    case NC_SHORT:
+                        memcpy(fill_valuep, &short_fill_value, sizeof(short));
+                        break;
+                    case NC_INT:
+                        memcpy(fill_valuep, &int_fill_value, sizeof(int));
+                        break;
+                    case NC_FLOAT:
+                        memcpy(fill_valuep, &float_fill_value, sizeof(float));
+                        break;
+                    case NC_DOUBLE:
+                        memcpy(fill_valuep, &double_fill_value, sizeof(double));
+                        break;
+                    default:
+                        return pio_err(ios, file, NC_EBADTYPE, __FILE__, __LINE__);
+                    }
+                    ierr = PIO_NOERR;
+                }
+            }
+        }
+        else
+        {
+#ifdef _NETCDF4
+            /* The inq_var_fill is not supported in classic-only builds. */
+            if (file->do_io)
+                ierr = nc_inq_var_fill(file->fh, varid, no_fill, fill_valuep);
+#endif /* _NETCDF */
+        }
+        LOG((2, "after call to inq_var_fill, ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if (no_fill)
+        if ((mpierr = MPI_Bcast(no_fill, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (fill_valuep)
+        if ((mpierr = MPI_Bcast(fill_valuep, type_size, MPI_CHAR, ios->ioroot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get the value of an attribute of any type, with no type conversion.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_get_att
+ */
+int PIOc_get_att(int ncid, int varid, const char *name, void *ip)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    nc_type atttype;       /* The type of the attribute. */
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* User must provide a name and destination pointer. */
+    if (!name || !ip || strlen(name) > NC_MAX_NAME)
+        return pio_err(ios, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_get_att ncid %d varid %d name %s", ncid, varid, name));
+
+    /* Get the type of the attribute. */
+    if ((ierr = PIOc_inq_att(ncid, varid, name, &atttype, NULL)))
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+    LOG((2, "atttype = %d", atttype));
+
+    return PIOc_get_att_tc(ncid, varid, name, atttype, ip);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute of any type.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att(int ncid, int varid, const char *name, nc_type xtype,
+                 PIO_Offset len, const void *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, xtype, op);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 64-bit floating point array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_double(int ncid, int varid, const char *name, double *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_DOUBLE, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 8-bit unsigned char array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_uchar(int ncid, int varid, const char *name, unsigned char *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_UBYTE, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 16-bit unsigned integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_ushort(int ncid, int varid, const char *name, unsigned short *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_USHORT, (void *)ip);
+}
+
+/**
+ * Get the value of an 32-bit unsigned integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_get_att
+ */
+int PIOc_get_att_uint(int ncid, int varid, const char *name, unsigned int *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_UINT, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 32-bit ingeger array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_long(int ncid, int varid, const char *name, long *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_LONG_INTERNAL, (void *)ip);
+}
+
+/**
+ * Get the value of an text attribute. There is no type conversion
+ * with this call. If the attribute is not of type NC_CHAR, then an
+ * error will be returned.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ * @ingroup PIO_get_att
+ */
+int PIOc_get_att_text(int ncid, int varid, const char *name, char *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_CHAR, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 8-bit signed char array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_schar(int ncid, int varid, const char *name, signed char *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_BYTE, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 64-bit unsigned integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_ulonglong(int ncid, int varid, const char *name, unsigned long long *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_UINT64, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 16-bit integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_short(int ncid, int varid, const char *name, short *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_SHORT, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 32-bit integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_int(int ncid, int varid, const char *name, int *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_INT, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 64-bit integer array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_longlong(int ncid, int varid, const char *name, long long *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_INT64, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_get_att
+ * Get the value of an 32-bit floating point array attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute to get
+ * @param ip a pointer that will get the attribute value.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_get_att_float(int ncid, int varid, const char *name, float *ip)
+{
+    return PIOc_get_att_tc(ncid, varid, name, PIO_FLOAT, (void *)ip);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 8-bit signed chars.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_schar(int ncid, int varid, const char *name, nc_type xtype,
+                       PIO_Offset len, const signed char *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_BYTE, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 32-bit signed integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_long(int ncid, int varid, const char *name, nc_type xtype,
+                      PIO_Offset len, const long *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_LONG_INTERNAL, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 32-bit signed integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_int(int ncid, int varid, const char *name, nc_type xtype,
+                     PIO_Offset len, const int *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_INT, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 8-bit unsigned chars.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_uchar(int ncid, int varid, const char *name, nc_type xtype,
+                       PIO_Offset len, const unsigned char *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_UBYTE, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 64-bit signed integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_longlong(int ncid, int varid, const char *name, nc_type xtype,
+                          PIO_Offset len, const long long *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_INT64, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 32-bit unsigned integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_uint(int ncid, int varid, const char *name, nc_type xtype,
+                      PIO_Offset len, const unsigned int *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_UINT, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 32-bit floating points.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_float(int ncid, int varid, const char *name, nc_type xtype,
+                       PIO_Offset len, const float *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_FLOAT, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 64-bit unsigned integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_ulonglong(int ncid, int varid, const char *name, nc_type xtype,
+                           PIO_Offset len, const unsigned long long *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_UINT64, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 16-bit unsigned integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_ushort(int ncid, int varid, const char *name, nc_type xtype,
+                        PIO_Offset len, const unsigned short *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_USHORT, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF text attribute.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_text(int ncid, int varid, const char *name,
+                      PIO_Offset len, const char *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, NC_CHAR, len, NC_CHAR, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 16-bit integers.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_short(int ncid, int varid, const char *name, nc_type xtype,
+                       PIO_Offset len, const short *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_SHORT, op);
+}
+
+/**
+ * @ingroup PIO_put_att
+ * Write a netCDF attribute array of 64-bit floating points.
+ *
+ * This routine is called collectively by all tasks in the communicator
+ * ios.union_comm.
+ *
+ * @param ncid the ncid of the open file, obtained from
+ * PIOc_openfile() or PIOc_createfile().
+ * @param varid the variable ID.
+ * @param name the name of the attribute.
+ * @param xtype the nc_type of the attribute.
+ * @param len the length of the attribute array.
+ * @param op a pointer with the attribute data.
+ * @return PIO_NOERR for success, error code otherwise.
+ */
+int PIOc_put_att_double(int ncid, int varid, const char *name, nc_type xtype,
+                        PIO_Offset len, const double *op)
+{
+    return PIOc_put_att_tc(ncid, varid, name, xtype, len, PIO_DOUBLE, op);
+}
diff --git a/src/externals/pio2/src/clib/pio_nc4.c b/src/externals/pio2/src/clib/pio_nc4.c
new file mode 100644
index 00000000000..0b1cdd0e6b7
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_nc4.c
@@ -0,0 +1,1031 @@
+/** @file
+ *
+ * Functions to wrap netCDF-4 functions for PIO.
+ *
+ * @author Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/**
+ * Set deflate (zlib) settings for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable.
+ * @param shuffle non-zero to turn on shuffle filter (can be good for
+ * integer data).
+ * @param deflate non-zero to turn on zlib compression for this
+ * variable.
+ * @param deflate_level 1 to 9, with 1 being faster and 9 being more
+ * compressed.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_def_var_deflate(int ncid, int varid, int shuffle, int deflate,
+                         int deflate_level)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_DEFLATE;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&shuffle, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&deflate, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&deflate_level, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors from computation tasks. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->iotype == PIO_IOTYPE_NETCDF4P)
+            ierr = NC_EINVAL;
+        else
+            if (file->do_io)
+                ierr = nc_def_var_deflate(file->fh, varid, shuffle, deflate, deflate_level);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * Inquire about deflate (zlib compression) settings for a variable.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param shufflep pointer to an int that will get the status of the
+ * shuffle filter. Ignored if NULL.
+ * @param deflatep pointer to an int that will be set to non-zero if
+ * deflation is in use for this variable. Ignored if NULL.
+ * @param deflate_levelp pointer to an int that will get the deflation
+ * level (from 1-9) if deflation is in use for this variable.  Ignored
+ * if NULL.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_inq_var
+ */
+int PIOc_inq_var_deflate(int ncid, int varid, int *shufflep, int *deflatep,
+                         int *deflate_levelp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_DEFLATE;
+            char shuffle_present = shufflep ? true : false;
+            char deflate_present = deflatep ? true : false;
+            char deflate_level_present = deflate_levelp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&shuffle_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (shuffle_present && !mpierr)
+                mpierr = MPI_Bcast(shufflep, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&deflate_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (deflate_present && !mpierr)
+                mpierr = MPI_Bcast(deflatep, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&deflate_level_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (deflate_level_present && !mpierr)
+                mpierr = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq_var_deflate ncid = %d varid = %d shuffle_present = %d deflate_present = %d "
+                 "deflate_level_present = %d", ncid, varid, shuffle_present, deflate_present,
+                 deflate_level_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_inq_var_deflate(file->fh, varid, shufflep, deflatep, deflate_levelp);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if (shufflep)
+        if ((mpierr = MPI_Bcast(shufflep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (deflatep)
+        if ((mpierr = MPI_Bcast(deflatep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (deflate_levelp)
+        if ((mpierr = MPI_Bcast(deflate_levelp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * @ingroup PIO_def_var
+ * Set chunksizes for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * Chunksizes have important performance repercussions. NetCDF
+ * attempts to choose sensible chunk sizes by default, but for best
+ * performance check chunking against access patterns.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param storage NC_CONTIGUOUS or NC_CHUNKED.
+ * @param chunksizep an array of chunksizes. Must have a chunksize for
+ * every variable dimension.
+ * @return PIO_NOERR for success, otherwise an error code.
+ */
+int PIOc_def_var_chunking(int ncid, int varid, int storage, const PIO_Offset *chunksizesp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ndims;             /* The number of dimensions for this var. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_def_var_chunking ncid = %d varid = %d storage = %d", ncid,
+         varid, storage));
+
+    /* Find the info about this file. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* Run this on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. Get the number of
+     * dimensions. */
+    if (!ios->async || !ios->ioproc)
+        if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+            return check_netcdf(file, ierr, __FILE__, __LINE__);
+    LOG((2, "PIOc_def_var_chunking first ndims = %d", ndims));
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_CHUNKING;
+            char chunksizes_present = chunksizesp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&storage, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr && chunksizes_present)
+                mpierr = MPI_Bcast((PIO_Offset *)chunksizesp, ndims, MPI_OFFSET, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "PIOc_def_var_chunking ncid = %d varid = %d storage = %d ndims = %d chunksizes_present = %d",
+                 ncid, varid, storage, ndims, chunksizes_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    LOG((2, "PIOc_def_var_chunking ndims = %d", ndims));
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+        {
+            size_t chunksizes_sizet[ndims];
+            for (int d = 0; d < ndims; d++)
+            {
+                if (chunksizesp[d] < 0)
+                {
+                    ierr = PIO_ERANGE;
+                    break;
+                }
+                chunksizes_sizet[d] = chunksizesp[d];
+            }
+            if (!ierr)
+                ierr = nc_def_var_chunking(file->fh, varid, storage, chunksizes_sizet);
+        }
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Inquire about chunksizes for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param storagep pointer to int which will be set to either
+ * NC_CONTIGUOUS or NC_CHUNKED.
+ * @param chunksizep pointer to memory where chunksizes will be
+ * set. There are the same number of chunksizes as there are
+ * dimensions.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_inq_var
+ */
+int PIOc_inq_var_chunking(int ncid, int varid, int *storagep, PIO_Offset *chunksizesp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ndims; /* The number of dimensions in the variable. */
+
+    LOG((1, "PIOc_inq_var_chunking ncid = %d varid = %d"));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* Run these on all tasks if async is not in use, but only on
+     * non-IO tasks if async is in use. */
+    if (!ios->async || !ios->ioproc)
+    {
+        /* Find the number of dimensions of this variable. */
+        if ((ierr = PIOc_inq_varndims(ncid, varid, &ndims)))
+            return pio_err(ios, file, ierr, __FILE__, __LINE__);
+        LOG((2, "ndims = %d", ndims));
+    }
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_CHUNKING;
+            char storage_present = storagep ? true : false;
+            char chunksizes_present = chunksizesp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&storage_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&chunksizes_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_inq_var_chunking ncid = %d varid = %d storage_present = %d chunksizes_present = %d",
+                 ncid, varid, storage_present, chunksizes_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+        /* Broadcast values currently only known on computation tasks to IO tasks. */
+        if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+        {
+            size_t chunksizes_sizet[ndims];
+            ierr = nc_inq_var_chunking(file->fh, varid, storagep, chunksizes_sizet);
+            if (!ierr && chunksizesp)
+                for (int d = 0; d < ndims; d++)
+                {
+                    if (chunksizes_sizet[d] > NC_MAX_INT64)
+                    {
+                        ierr = PIO_ERANGE;
+                        break;
+                    }
+                    chunksizesp[d] = chunksizes_sizet[d];
+                }
+        }
+#endif
+        LOG((2, "ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if ((mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (storagep)
+        if ((mpierr = MPI_Bcast(storagep, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (chunksizesp)
+        if ((mpierr = MPI_Bcast(chunksizesp, ndims, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set chunksizes for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * Chunksizes have important performance repercussions. NetCDF
+ * attempts to choose sensible chunk sizes by default, but for best
+ * performance check chunking against access patterns.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param storage NC_CONTIGUOUS or NC_CHUNKED.
+ * @param chunksizep an array of chunksizes. Must have a chunksize for
+ * every variable dimension.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_def_var_endian(int ncid, int varid, int endian)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_DEF_VAR_ENDIAN;
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&endian, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_def_var_endian(file->fh, varid, endian);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Inquire about chunksizes for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param endianp pointer to int which will be set to
+ * endianness. Ignored if NULL.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_inq_var
+ */
+int PIOc_inq_var_endian(int ncid, int varid, int *endianp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_inq_var_endian ncid = %d varid = %d"));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INQ_VAR_ENDIAN;
+            char endian_present = endianp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&endian_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_inq_var_endian(file->fh, varid, endianp);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if (endianp)
+        if ((mpierr = MPI_Bcast(endianp, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set chunk cache netCDF files to be opened/created.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * The file chunk cache for HDF5 can be set, and will apply for any
+ * files opened or created until the program ends, or the settings are
+ * changed again. The cache settings apply only to the open file. They
+ * do not persist with the file, and must be set each time the file is
+ * opened, before it is opened, if they are to have effect.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param iotype the iotype of files to be created or opened.
+ * @param size size of file cache.
+ * @param nelems number of elements in file cache.
+ * @param preemption preemption setting for file cache.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_set_chunk_cache(int iosysid, int iotype, PIO_Offset size, PIO_Offset nelems,
+                         float preemption)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_set_chunk_cache iosysid = %d iotype = %d size = %d nelems = %d preemption = %g",
+         iosysid, iotype, size, nelems, preemption));
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Only netCDF-4 files can use this feature. */
+    if (iotype != PIO_IOTYPE_NETCDF4P && iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, NULL, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_SET_CHUNK_CACHE; /* Message for async notification. */
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        LOG((2, "calling nc_chunk_cache"));
+        if (iotype == PIO_IOTYPE_NETCDF4P)
+            ierr = nc_set_chunk_cache(size, nelems, preemption);
+        else
+            if (!ios->io_rank)
+                ierr = nc_set_chunk_cache(size, nelems, preemption);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
+
+    LOG((2, "PIOc_set_chunk_cache complete!"));
+    return PIO_NOERR;
+}
+
+/**
+ * Get current file chunk cache settings from HDF5.
+ *
+ * This function has no effect on netCDF classic files. Calling this
+ * function with iotype of PIO_IOTYPE_PNETCDF or PIO_IOTYPE_NETCDF
+ * returns an error.
+ *
+ * The file chunk cache for HDF5 can be set, and will apply for any
+ * files opened or created until the program ends, or the settings are
+ * changed again. The cache settings apply only to the open file. They
+ * do not persist with the file, and must be set each time the file is
+ * opened, before it is opened, if they are to have effect.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * Chunksizes have important performance repercussions. NetCDF
+ * attempts to choose sensible chunk sizes by default, but for best
+ * performance check chunking against access patterns.
+ *
+ * @param iotype the iotype of files to be created or opened.
+ * @param sizep gets the size of file cache.
+ * @param nelemsp gets the number of elements in file cache.
+ * @param preemptionp gets the preemption setting for file cache.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_get_chunk_cache(int iosysid, int iotype, PIO_Offset *sizep, PIO_Offset *nelemsp,
+                         float *preemptionp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_get_chunk_cache iosysid = %d iotype = %d", iosysid, iotype));
+
+    /* Get the io system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Only netCDF-4 files can use this feature. */
+    if (iotype != PIO_IOTYPE_NETCDF4P && iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, NULL, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_GET_CHUNK_CACHE; /* Message for async notification. */
+            char size_present = sizep ? true : false;
+            char nelems_present = nelemsp ? true : false;
+            char preemption_present = preemptionp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "PIOc_get_chunk_cache size_present = %d nelems_present = %d "
+                 "preemption_present = %d ", size_present, nelems_present, preemption_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (iotype == PIO_IOTYPE_NETCDF4P)
+            ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp);
+        else
+            if (!ios->io_rank)
+                ierr = nc_get_chunk_cache((size_t *)sizep, (size_t *)nelemsp, preemptionp);
+#endif
+        LOG((2, "nc_get_chunk_cache called ierr = %d", ierr));
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "bcast complete ierr = %d sizep = %d", ierr, sizep));
+    if (ierr)
+        return check_netcdf(NULL, ierr, __FILE__, __LINE__);
+
+    if (sizep)
+    {
+        LOG((2, "bcasting size = %d ios->ioroot = %d", *sizep, ios->ioroot));
+        if ((mpierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((2, "bcast size = %d", *sizep));
+    }
+    if (nelemsp)
+    {
+        if ((mpierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((2, "bcast complete nelems = %d", *nelemsp));
+    }
+    if (preemptionp)
+    {
+        if ((mpierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((2, "bcast complete preemption = %d", *preemptionp));
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set chunksizes for a variable.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * Chunksizes have important performance repercussions. NetCDF
+ * attempts to choose sensible chunk sizes by default, but for best
+ * performance check chunking against access patterns.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param storage NC_CONTIGUOUS or NC_CHUNKED.
+ * @param chunksizep an array of chunksizes. Must have a chunksize for
+ * every variable dimension.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_def_var
+ */
+int PIOc_set_var_chunk_cache(int ncid, int varid, PIO_Offset size, PIO_Offset nelems,
+                             float preemption)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_SET_VAR_CHUNK_CACHE;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1,MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&size, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nelems, 1, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&preemption, 1, MPI_FLOAT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_set_var_chunk_cache(file->fh, varid, size, nelems, preemption);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get the variable chunk cache settings.
+ *
+ * This function only applies to netCDF-4 files. When used with netCDF
+ * classic files, the error PIO_ENOTNC4 will be returned.
+ *
+ * Note that these settings are not part of the data file - they apply
+ * only to the open file as long as it is open.
+ *
+ *  See the netCDF
+ * variable documentation for details about the operation of this
+ * function.
+ *
+ * @param ncid the ncid of the open file.
+ * @param varid the ID of the variable to set chunksizes for.
+ * @param sizep will get the size of the cache in bytes. Ignored if NULL.
+ * @param nelemsp will get the number of elements in the cache. Ignored if NULL.
+ * @param preemptionp will get the cache preemption value. Ignored if NULL.
+ * @return PIO_NOERR for success, otherwise an error code.
+ * @ingroup PIO_inq_var
+ */
+int PIOc_get_var_chunk_cache(int ncid, int varid, PIO_Offset *sizep, PIO_Offset *nelemsp,
+                             float *preemptionp)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr;              /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_get_var_chunk_cache ncid = %d varid = %d"));
+
+    /* Get the file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* Only netCDF-4 files can use this feature. */
+    if (file->iotype != PIO_IOTYPE_NETCDF4P && file->iotype != PIO_IOTYPE_NETCDF4C)
+        return pio_err(ios, file, PIO_ENOTNC4, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_GET_VAR_CHUNK_CACHE; /* Message for async notification. */
+            char size_present = sizep ? true : false;
+            char nelems_present = nelemsp ? true : false;
+            char preemption_present = preemptionp ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&varid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&size_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&nelems_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&preemption_present, 1, MPI_CHAR, ios->compmaster,
+                                   ios->intercomm);
+            LOG((2, "PIOc_get_var_chunk_cache size_present = %d nelems_present = %d "
+                 "preemption_present = %d ", size_present, nelems_present, preemption_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+#ifdef _NETCDF4
+        if (file->do_io)
+            ierr = nc_get_var_chunk_cache(file->fh, varid, (size_t *)sizep, (size_t *)nelemsp,
+                                          preemptionp);
+#endif
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+
+    /* Broadcast results to all tasks. */
+    if (sizep && !ierr)
+        if ((mpierr = MPI_Bcast(sizep, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (nelemsp && !ierr)
+        if ((mpierr = MPI_Bcast(nelemsp, 1, MPI_OFFSET, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (preemptionp && !ierr)
+        if ((mpierr = MPI_Bcast(preemptionp, 1, MPI_FLOAT, ios->ioroot, ios->my_comm)))
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pio_put_nc.c b/src/externals/pio2/src/clib/pio_put_nc.c
new file mode 100644
index 00000000000..7c74144da3b
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_put_nc.c
@@ -0,0 +1,1190 @@
+/**
+ * @file
+ * PIO functions to write data.
+ *
+ * @author Ed Hartnett
+ * @date  2016
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#include 
+#include 
+#include 
+
+/**
+ * Get strided, muti-dimensional subset of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_text(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                       const PIO_Offset *stride, const char *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_CHAR, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_uchar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride,
+                        const unsigned char *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UBYTE, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_schar(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                        const PIO_Offset *stride, const signed char *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_BYTE, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned 16-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_ushort(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                         const PIO_Offset *stride, const unsigned short *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_USHORT, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_short(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const PIO_Offset *stride, const short *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_SHORT, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_uint(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                       const PIO_Offset *stride, const unsigned int *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_int(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                      const PIO_Offset *stride, const int *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_long(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                       const PIO_Offset *stride, const long *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, PIO_LONG_INTERNAL, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_float(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                        const PIO_Offset *stride, const float *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_FLOAT, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit unsigned integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_longlong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                           const PIO_Offset *stride, const long long *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_INT64, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of a 64-bit floating point
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_double(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                         const PIO_Offset *stride, const double *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_DOUBLE, op);
+}
+
+/**
+ * Get strided, muti-dimensional subset of an unsigned 64-bit integer
+ * variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars_ulonglong(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                            const PIO_Offset *stride, const unsigned long long *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_UINT64, op);
+}
+
+/**
+ * Get one value from an text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_text(int ncid, int varid, const PIO_Offset *index, const char *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_CHAR, op);
+}
+
+/**
+ * Get one value from an text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_uchar(int ncid, int varid, const PIO_Offset *index,
+                        const unsigned char *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_UBYTE, op);
+}
+
+/**
+ * Get one value from an signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_schar(int ncid, int varid, const PIO_Offset *index,
+                        const signed char *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_BYTE, op);
+}
+
+/**
+ * Get one value from an unsigned 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_ushort(int ncid, int varid, const PIO_Offset *index,
+                         const unsigned short *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_USHORT, op);
+}
+
+/**
+ * Get one value from a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_short(int ncid, int varid, const PIO_Offset *index,
+                        const short *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_SHORT, op);
+}
+
+/**
+ * Get one value from an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_uint(int ncid, int varid, const PIO_Offset *index,
+                       const unsigned int *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_UINT, op);
+}
+
+/**
+ * Get one value from an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_int(int ncid, int varid, const PIO_Offset *index, const int *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_INT, op);
+}
+
+/**
+ * Get one value from an floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_float(int ncid, int varid, const PIO_Offset *index, const float *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_FLOAT, op);
+}
+
+/**
+ * Get one value from an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_long(int ncid, int varid, const PIO_Offset *index, const long *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, PIO_LONG_INTERNAL, op);
+}
+
+/**
+ * Get one value from an 64-bit floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_double(int ncid, int varid, const PIO_Offset *index,
+                         const double *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_DOUBLE, op);
+}
+
+/**
+ * Get one value from an unsigned 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_ulonglong(int ncid, int varid, const PIO_Offset *index,
+                            const unsigned long long *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_UINT64, op);
+}
+
+/**
+ * Get one value from a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1_longlong(int ncid, int varid, const PIO_Offset *index,
+                           const long long *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_INT64, op);
+}
+
+/**
+ * Put muti-dimensional subset of a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_text(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, const char *op)
+{
+    return PIOc_put_vars_text(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_uchar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const unsigned char *op)
+{
+    return PIOc_put_vars_uchar(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_schar(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const signed char *op)
+{
+    return PIOc_put_vars_schar(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an unsigned 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_ushort(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, const unsigned short *op)
+{
+    return PIOc_put_vars_ushort(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_short(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const short *op)
+{
+    return PIOc_put_vars_short(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_uint(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, const unsigned int *op)
+{
+    return PIOc_put_vars_uint(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_int(int ncid, int varid, const PIO_Offset *start,
+                      const PIO_Offset *count, const int *op)
+{
+    return PIOc_put_vars_int(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_long(int ncid, int varid, const PIO_Offset *start,
+                       const PIO_Offset *count, const long *op)
+{
+    return PIOc_put_vars_long(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_float(int ncid, int varid, const PIO_Offset *start,
+                        const PIO_Offset *count, const float *op)
+{
+    return PIOc_put_vars_float(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of an unsigned 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_ulonglong(int ncid, int varid, const PIO_Offset *start,
+                            const PIO_Offset *count, const unsigned long long *op)
+{
+    return PIOc_put_vars_ulonglong(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_longlong(int ncid, int varid, const PIO_Offset *start,
+                           const PIO_Offset *count, const long long *op)
+{
+    return PIOc_put_vars_longlong(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put muti-dimensional subset of a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara_double(int ncid, int varid, const PIO_Offset *start,
+                         const PIO_Offset *count, const double *op)
+{
+    return PIOc_put_vars_double(ncid, varid, start, count, NULL, op);
+}
+
+/**
+ * Put all data to a text variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_text(int ncid, int varid, const char *op)
+{
+    return PIOc_put_var_tc(ncid, varid, PIO_CHAR, op);
+}
+
+/**
+ * Put all data to an unsigned char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_uchar(int ncid, int varid, const unsigned char *op)
+{
+    return PIOc_put_var_tc(ncid, varid, PIO_UBYTE, op);
+}
+
+/**
+ * Put all data to a signed char variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_schar(int ncid, int varid, const signed char *op)
+{
+    return PIOc_put_var_tc(ncid, varid, PIO_BYTE, op);
+}
+
+/**
+ * Put all data to a 16-bit unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_ushort(int ncid, int varid, const unsigned short *op)
+{
+    return PIOc_put_var_tc(ncid, varid, NC_USHORT, op);
+}
+
+/**
+ * Put all data to a 16-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_short(int ncid, int varid, const short *op)
+{
+    return PIOc_put_var_tc(ncid, varid, PIO_SHORT, op);
+}
+
+/**
+ * Put all data to an unsigned integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_uint(int ncid, int varid, const unsigned int *op)
+{
+    return PIOc_put_var_tc(ncid, varid, PIO_UINT, op);
+}
+
+/**
+ * Put all data to an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_int(int ncid, int varid, const int *op)
+{
+    return PIOc_put_var_tc(ncid, varid, PIO_INT, op);
+}
+
+/**
+ * Put all data to an integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_long(int ncid, int varid, const long *op)
+{
+    return PIOc_put_var_tc(ncid, varid, PIO_LONG_INTERNAL, op);
+}
+
+/**
+ * Put all data to a floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_float(int ncid, int varid, const float *op)
+{
+    return PIOc_put_var_tc(ncid, varid, PIO_FLOAT, op);
+}
+
+/**
+ * Put all data to an unsigned 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_ulonglong(int ncid, int varid, const unsigned long long *op)
+{
+    return PIOc_put_var_tc(ncid, varid, PIO_UINT64, op);
+}
+
+/**
+ * Put all data to a 64-bit integer variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_longlong(int ncid, int varid, const long long *op)
+{
+    return PIOc_put_var_tc(ncid, varid, PIO_INT64, op);
+}
+
+/**
+ * Put all data to a 64-bit floating point variable.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param op pointer to the data to be written.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var_double(int ncid, int varid, const double *op)
+{
+    return PIOc_put_var_tc(ncid, varid, PIO_DOUBLE, op);
+}
+
+/**
+ * Put all data to a variable of any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var(int ncid, int varid, const void *op)
+{
+    return PIOc_put_var_tc(ncid, varid, NC_NAT, op);
+}
+
+/**
+ * Get one value from a variable of any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_var1(int ncid, int varid, const PIO_Offset *index, const void *op)
+{
+    return PIOc_put_var1_tc(ncid, varid, index, NC_NAT, op);
+}
+
+/**
+ * Put muti-dimensional subset of a variable of any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vara(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                  const void *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_NAT, op);
+}
+
+/**
+ * Write strided, muti-dimensional subset of a variable of any type.
+ *
+ * This routine is called collectively by all tasks in the
+ * communicator ios.union_comm.
+ *
+ * @param ncid identifies the netCDF file
+ * @param varid the variable ID number
+ * @param start an array of start indicies (must have same number of
+ * entries as variable has dimensions). If NULL, indices of 0 will be
+ * used.
+ * @param count an array of counts (must have same number of entries
+ * as variable has dimensions). If NULL, counts matching the size of
+ * the variable will be used.
+ * @param stride an array of strides (must have same number of
+ * entries as variable has dimensions). If NULL, strides of 1 will be
+ * used.
+ * @param buf pointer that will get the data.
+ * @return PIO_NOERR on success, error code otherwise.
+ */
+int PIOc_put_vars(int ncid, int varid, const PIO_Offset *start, const PIO_Offset *count,
+                  const PIO_Offset *stride, const void *op)
+{
+    return PIOc_put_vars_tc(ncid, varid, start, count, stride, NC_NAT, op);
+}
diff --git a/src/externals/pio2/src/clib/pio_rearrange.c b/src/externals/pio2/src/clib/pio_rearrange.c
new file mode 100644
index 00000000000..74b27d45b06
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_rearrange.c
@@ -0,0 +1,2103 @@
+/** @file
+ * Code to map IO to model decomposition.
+ *
+ * @author Jim Edwards
+ */
+#include 
+#include 
+#include 
+
+/**
+ * Convert a 1-D index into a coordinate value in an arbitrary
+ * dimension space. E.g., for index 4 into a array defined as a[3][2],
+ * will return 2,0.
+ *
+ * Sometimes (from box_rearranger_create()) this function is called
+ * with -1 for idx. Not clear if this makes sense.
+ *
+ * @param ndims number of dimensions.
+ * @param gdimlen array of length ndims with the dimension sizes.
+ * @param idx the index to convert. This is the index into a 1-D array
+ * of data.
+ * @param dim_list array of length ndims that will get the dimensions
+ * corresponding to this index.
+ */
+void idx_to_dim_list(int ndims, const int *gdimlen, PIO_Offset idx,
+                     PIO_Offset *dim_list)
+{
+    /* Check inputs. */
+    pioassert(ndims >= 0 && gdimlen && idx >= -1 && dim_list, "invalid input",
+              __FILE__, __LINE__);
+    LOG((2, "idx_to_dim_list ndims = %d idx = %d", ndims, idx));
+
+    /* Easiest to start from the right and move left. */
+    for (int i = ndims - 1; i >= 0; --i)
+    {
+        int next_idx;
+
+        /* This way of doing div/mod is slightly faster than using "/"
+         * and "%". */
+        next_idx = idx / gdimlen[i];
+        dim_list[i] = idx - (next_idx * gdimlen[i]);
+        LOG((3, "next_idx = %d idx = %d gdimlen[%d] = %d dim_list[%d] = %d",
+             next_idx, idx, i, gdimlen[i], i, dim_list[i]));
+        idx = next_idx;
+    }
+}
+
+/**
+ * Expand a region along dimension dim, by incrementing count[i] as
+ * much as possible, consistent with the map.
+ *
+ * Once max_size is reached, the map is exhausted, or the next entries fail
+ * to match, expand_region updates the count and calls itself with the next
+ * outermost dimension, until the region has been expanded as much as
+ * possible along all dimensions.
+ *
+ * Precondition: maplen >= region_size (thus loop runs at least
+ * once).
+ *
+ * @param dim the dimension number to start with.
+ * @param gdimlen array of global dimension lengths.
+ * @param maplen the length of the map.
+ * @param map array (length maplen) with the the 1-based compmap.
+ * @param region_size ???
+ * @param region_stride amount incremented along dimension.
+ * @param max_size array of size dim + 1 that contains the maximum
+ * sizes along that dimension.
+ * @param count array of size dim + 1 that gets the new counts.
+ */
+void expand_region(int dim, const int *gdimlen, int maplen, const PIO_Offset *map,
+                   int region_size, int region_stride, const int *max_size,
+                   PIO_Offset *count)
+{
+    /* Flag used to signal that we can no longer expand the region
+       along dimension dim. */
+    int expansion_done = 0;
+
+    /* Check inputs. */
+    pioassert(dim >= 0 && gdimlen && maplen >= 0 && map && region_size >= 0 &&
+              maplen >= region_size && region_stride >= 0 && max_size && count,
+              "invalid input", __FILE__, __LINE__);
+
+    /* Expand no greater than max_size along this dimension. */
+    for (int i = 1; i <= max_size[dim]; ++i)
+    {
+        /* Count so far is at least i. */
+        count[dim] = i;
+
+        /* Now see if we can expand to i+1 by checking that the next
+           region_size elements are ahead by exactly region_stride.
+           Assuming monotonicity in the map, we could skip this for the
+           innermost dimension, but it's necessary past that because the
+           region does not necessarily comprise contiguous values. */
+        for (int j = 0; j < region_size; j++)
+        {
+            int test_idx; /* Index we are testing. */
+
+            test_idx = j + i * region_size;
+
+            /* If we have exhausted the map, or the map no longer matches,
+               we are done, break out of both loops. */
+            if (test_idx >= maplen || map[test_idx] != map[j] + i * region_stride)
+            {
+                expansion_done = 1;
+                break;
+            }
+        }
+        if (expansion_done)
+            break;
+    }
+
+    /* Move on to next outermost dimension if there are more left,
+     * else return. */
+    if (dim > 0)
+        expand_region(dim - 1, gdimlen, maplen, map, region_size * count[dim],
+                      region_stride * gdimlen[dim], max_size, count);
+}
+
+/**
+ * Set start and count so that they describe the first region in
+ * map.
+ *
+ * This function is used when creating the subset rearranger (it
+ * is not used for the box rearranger). It is called by get_regions().
+ *
+ * Preconditions:
+ * 
      
+ * 
    • ndims is > 0
+ * 
    • maplen is > 0
+ * 
    • All elements of map are inside the bounds specified by gdimlen.
+ * 
    
+ *
+ * Note that the map array is 1 based, but calculations are 0 based.
+ *
+ * @param ndims the number of dimensions.
+ * @param gdimlen an array length ndims with the sizes of the global
+ * dimensions.
+ * @param maplen the length of the map.
+ * @param map
+ * @param start array (length ndims) that will get start indicies of
+ * found region.
+ * @param count array (length ndims) that will get counts of found
+ * region.
+ * @returns length of the region found.
+ */
+PIO_Offset find_region(int ndims, const int *gdimlen, int maplen, const PIO_Offset *map,
+                       PIO_Offset *start, PIO_Offset *count)
+{
+    PIO_Offset regionlen = 1;
+
+    /* Check inputs. */
+    pioassert(ndims > 0 && gdimlen && maplen > 0 && map && start && count,
+              "invalid input", __FILE__, __LINE__);
+    LOG((2, "find_region ndims = %d maplen = %d", ndims, maplen));
+
+    int max_size[ndims];
+
+    /* Convert the index which is the first element of map into global
+     * data space. */
+    idx_to_dim_list(ndims, gdimlen, map[0] - 1, start);
+
+    /* Can't expand beyond the array edge. Set up max_size array for
+     * expand_region call below. */
+    for (int dim = 0; dim < ndims; ++dim)
+    {
+        max_size[dim] = gdimlen[dim] - start[dim];
+        LOG((3, "max_size[%d] = %d", max_size[dim]));
+    }
+
+    /* For each dimension, figure out how far we can expand in that dimension
+       while staying contiguous in the input array.
+
+       Start with the innermost dimension (ndims-1), and it will recurse
+       through to the outermost dimensions. */
+    expand_region(ndims - 1, gdimlen, maplen, map, 1, 1, max_size, count);
+
+    /* Calculate the number of data elements in this region. */
+    for (int dim = 0; dim < ndims; dim++)
+        regionlen *= count[dim];
+
+    return regionlen;
+}
+
+/**
+ * Convert a global coordinate value into a local array index.
+ *
+ * @param ndims the number of dimensions.
+ * @param lcoord pointer to an offset.
+ * @param count array of counts.
+ * @returns the local array index.
+ */
+PIO_Offset coord_to_lindex(int ndims, const PIO_Offset *lcoord, const PIO_Offset *count)
+{
+    PIO_Offset lindex = 0;
+    PIO_Offset stride = 1;
+
+    /* Check inputs. */
+    pioassert(ndims > 0 && lcoord && count, "invalid input", __FILE__, __LINE__);
+
+    for (int i = ndims - 1; i >= 0; i--)
+    {
+        lindex += lcoord[i] * stride;
+        stride = stride * count[i];
+    }
+    return lindex;
+}
+
+/**
+ * Compute the max io buffer size needed for an iodesc. It is the
+ * combined size (in number of data elements) of all the regions of
+ * data stored in the buffer of this iodesc. The max size is then set
+ * in the iodesc.
+ *
+ * @param io_comm the IO communicator
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @returns 0 for success, error code otherwise.
+ */
+int compute_maxIObuffersize(MPI_Comm io_comm, io_desc_t *iodesc)
+{
+    PIO_Offset totiosize = 0;
+    int mpierr; /* Return code from MPI calls. */
+
+    pioassert(iodesc, "need iodesc", __FILE__, __LINE__);
+
+    /*  compute the max io buffer size, for conveneance it is the
+     *  combined size of all regions */
+    for (io_region *region = iodesc->firstregion; region; region = region->next)
+    {
+        if (region->count[0] > 0)
+        {
+            PIO_Offset iosize = 1;
+            for (int i = 0; i < iodesc->ndims; i++)
+                iosize *= region->count[i];
+            totiosize += iosize;
+        }
+    }
+    LOG((2, "compute_maxIObuffersize got totiosize = %lld", totiosize));
+
+    /* Share the max io buffer size with all io tasks. */
+    if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &totiosize, 1, MPI_OFFSET, MPI_MAX, io_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    pioassert(totiosize > 0, "totiosize <= 0", __FILE__, __LINE__);
+    LOG((2, "after allreduce compute_maxIObuffersize got totiosize = %lld", totiosize));
+
+    /* Remember the result. */
+    iodesc->maxiobuflen = totiosize;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Create the derived MPI datatypes used for comp2io and io2comp
+ * transfers. Used in define_iodesc_datatypes().
+ *
+ * @param basetype The MPI type of data (MPI_INT, etc.).
+ * @param msgcnt This is the number of MPI types that are created.
+ * @param mindex An array (length numinds) of indexes into the data
+ * array from the comp map. Will be NULL when count is zero.
+ * @param mcount An array (length msgcnt) with the number of indexes
+ * to be put on each mpi message/task.
+ * @param mfrom A pointer to the previous structure in the read/write
+ * list. This is always NULL for the BOX rearranger.
+ * @param mtype pointer to an array (length msgcnt) which gets the
+ * created datatypes. Will be NULL when iodesc->nrecvs == 0.
+ * @returns 0 on success, error code otherwise.
+ */
+int create_mpi_datatypes(MPI_Datatype basetype, int msgcnt,
+                         const PIO_Offset *mindex, const int *mcount, int *mfrom,
+                         MPI_Datatype *mtype)
+{
+    int blocksize;
+    int numinds = 0;
+    PIO_Offset *lindex = NULL;
+    int mpierr; /* Return code from MPI functions. */
+
+    /* Check inputs. */
+    pioassert(msgcnt > 0 && mcount, "invalid input", __FILE__, __LINE__);
+
+    PIO_Offset bsizeT[msgcnt];
+
+    LOG((1, "create_mpi_datatypes basetype = %d msgcnt = %d", basetype, msgcnt));
+    LOG((2, "MPI_BYTE = %d MPI_CHAR = %d MPI_SHORT = %d MPI_INT = %d MPI_DOUBLE = %d",
+         MPI_BYTE, MPI_CHAR, MPI_SHORT, MPI_INT, MPI_DOUBLE));
+
+    /* How many indicies in the array? */
+    for (int j = 0; j < msgcnt; j++)
+        numinds += mcount[j];
+    LOG((2, "numinds = %d", numinds));
+
+    if (mindex)
+    {
+        if (!(lindex = malloc(numinds * sizeof(PIO_Offset))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        memcpy(lindex, mindex, (size_t)(numinds * sizeof(PIO_Offset)));
+        LOG((3, "allocated lindex, copied mindex"));
+    }
+
+    bsizeT[0] = 0;
+    mtype[0] = PIO_DATATYPE_NULL;
+    int pos = 0;
+    int ii = 0;
+
+    if (mfrom == NULL)
+    {
+        LOG((3, "mfrom is NULL"));
+        for (int i = 0; i < msgcnt; i++)
+        {
+            if (mcount[i] > 0)
+            {
+                /* Look for the largest block of data for io which
+                 * can be expressed in terms of start and
+                 * count. */
+                bsizeT[ii] = GCDblocksize(mcount[i], lindex + pos);
+                ii++;
+                pos += mcount[i];
+            }
+        }
+        blocksize = (int)lgcd_array(ii, bsizeT);
+    }
+    else
+    {
+        blocksize = 1;
+    }
+    LOG((3, "blocksize = %d", blocksize));
+
+    /* pos is an index to the start of each message block. */
+    pos = 0;
+    for (int i = 0; i < msgcnt; i++)
+    {
+        if (mcount[i] > 0)
+        {
+            int len = mcount[i] / blocksize;
+            int displace[len];
+            if (blocksize == 1)
+            {
+                if (!mfrom)
+                {
+                    for (int j = 0; j < len; j++)
+                        displace[j] = (int)(lindex[pos + j]);
+                }
+                else
+                {
+                    int k = 0;
+                    for (int j = 0; j < numinds; j++)
+                        if (mfrom[j] == i)
+                            displace[k++] = (int)(lindex[j]);
+                }
+
+            }
+            else
+            {
+                for (int j = 0; j < mcount[i]; j++)
+                    (lindex + pos)[j]++;
+
+                for (int j = 0; j < len; j++)
+                    displace[j] = ((lindex + pos)[j * blocksize] - 1);
+            }
+
+#if PIO_ENABLE_LOGGING
+            for (int j = 0; j < len; j++)
+                LOG((3, "displace[%d] = %d", j, displace[j]));
+#endif /* PIO_ENABLE_LOGGING */
+
+            LOG((3, "calling MPI_Type_create_indexed_block len = %d blocksize = %d "
+                 "basetype = %d", len, blocksize, basetype));
+            /* Create an indexed datatype with constant-sized blocks. */
+            if ((mpierr = MPI_Type_create_indexed_block(len, blocksize, displace,
+                                                        basetype, &mtype[i])))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+            if (mtype[i] == PIO_DATATYPE_NULL)
+                return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+            /* Commit the MPI data type. */
+            LOG((3, "about to commit type"));
+            if ((mpierr = MPI_Type_commit(&mtype[i])))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            pos += mcount[i];
+        }
+    }
+
+    /* Free resources. */
+    if (lindex)
+        free(lindex);
+
+    LOG((3, "done with create_mpi_datatypes()"));
+    return PIO_NOERR;
+}
+
+/**
+ * If needed, create the derived MPI datatypes used for comp2io and
+ * io2comp transfers.
+ *
+ * If iodesc->stype and iodesc->rtype arrays already exist, this
+ * function does nothing. This function is called from
+ * rearrange_io2comp() and rearrange_comp2io().
+ *
+ * NOTE from Jim: I am always oriented toward write so recieve
+ * always means io tasks and send always means comp tasks. The
+ * opposite relationship is actually the case for reading. I've
+ * played with different ways of referring to things to get rid of
+ * this orientation bias in the documentation as well as in
+ * variable names, but I haven't found anything that I found more
+ * satisfactory.
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @returns 0 on success, error code otherwise.
+ */
+int define_iodesc_datatypes(iosystem_desc_t *ios, io_desc_t *iodesc)
+{
+    int ret; /* Return value. */
+
+    pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__);
+    LOG((1, "define_iodesc_datatypes ios->ioproc = %d", ios->ioproc));
+
+    /* Set up the to transfer data to and from the IO tasks. */
+    if (ios->ioproc)
+    {
+        /* If the types for the IO tasks have not been created, then
+         * create them. */
+        if (!iodesc->rtype)
+        {
+            if (iodesc->nrecvs > 0)
+            {
+                /* Allocate memory for array of MPI types for the IO tasks. */
+                if (!(iodesc->rtype = malloc(iodesc->nrecvs * sizeof(MPI_Datatype))))
+                    return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+                LOG((2, "allocated memory for IO task MPI types iodesc->nrecvs = %d "
+                     "iodesc->rearranger = %d", iodesc->nrecvs, iodesc->rearranger));
+
+                /* Initialize data types to NULL. */
+                for (int i = 0; i < iodesc->nrecvs; i++)
+                    iodesc->rtype[i] = PIO_DATATYPE_NULL;
+
+                /* The different rearrangers get different values for mfrom. */
+                int *mfrom = iodesc->rearranger == PIO_REARR_SUBSET ? iodesc->rfrom : NULL;
+
+                /* Create the MPI datatypes. */
+                if ((ret = create_mpi_datatypes(iodesc->basetype, iodesc->nrecvs, iodesc->rindex,
+                                                iodesc->rcount, mfrom, iodesc->rtype)))
+                    return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+            }
+        }
+    }
+
+    /* Define the datatypes for the computation components if they
+     * don't exist. (These will be the send side in a write
+     * operation.) */
+    if (!iodesc->stype)
+    {
+        int ntypes;
+
+        /* Subset rearranger gets one type; box rearranger gets one
+         * type per IO task. */
+        ntypes = iodesc->rearranger == PIO_REARR_SUBSET ? 1 : ios->num_iotasks;
+
+        /* Allocate memory for array of MPI types for the computation tasks. */
+        if (!(iodesc->stype = malloc(ntypes * sizeof(MPI_Datatype))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        LOG((3, "allocated memory for computation MPI types ntypes = %d", ntypes));
+
+        /* Initialize send types to NULL. */
+        for (int i = 0; i < ntypes; i++)
+            iodesc->stype[i] = PIO_DATATYPE_NULL;
+
+        /* Remember how many types we created for the send side. */
+        iodesc->num_stypes = ntypes;
+
+        /* Create the MPI data types. */
+        LOG((3, "about to call create_mpi_datatypes for computation MPI types"));
+        if ((ret = create_mpi_datatypes(iodesc->basetype, ntypes, iodesc->sindex,
+                                        iodesc->scount, NULL, iodesc->stype)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    }
+
+    LOG((3, "done with define_iodesc_datatypes()"));
+    return PIO_NOERR;
+}
+
+/**
+ * Completes the mapping for the box rearranger. This function is
+ * called from box_rearrange_create(). It is not used for the subset
+ * rearranger.
+ *
+ * This function:
+ * 
      
+ * 
    • Allocates and inits iodesc->scount, an array (length
+ * ios->num_iotasks) containing number of data elements sent to each
+ * IO task from current compute task.
+ * 
    • Uses pio_swapm() to send iodesc->scount array from each
+ * computation task to all IO tasks.
+ * 
    • On IO tasks, allocates and inits iodesc->rcount and
+ * iodesc->rfrom arrays (length max(1, nrecvs)) which holds the amount
+ * of data to expect from each compute task and the rank of that
+ * task. .
+ * 
    • Allocates and inits iodesc->sindex arrays (length iodesc->ndof)
+ * which holds indecies for computation tasks.
+ * 
    • On IO tasks, allocates and init iodesc->rindex (length
+ * totalrecv) with indices of the data to be sent/received from this
+ * io task to each compute task.
+ * 
    • Uses pio_swapm() to send list of indicies on each compute task
+ * to the IO tasks.
+ * 
    
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @param dest_ioproc an array (length maplen) of IO task numbers.
+ * @param dest_ioindex an array (length maplen) of IO indicies.
+ * @returns 0 on success, error code otherwise.
+ */
+int compute_counts(iosystem_desc_t *ios, io_desc_t *iodesc,
+                   const int *dest_ioproc, const PIO_Offset *dest_ioindex)
+{
+    int *recv_buf = NULL;
+    int nrecvs = 0;
+    int ierr;
+
+    /* Check inputs. */
+    pioassert(ios && iodesc && dest_ioproc && dest_ioindex &&
+              iodesc->rearranger == PIO_REARR_BOX && ios->num_uniontasks > 0,
+              "invalid input", __FILE__, __LINE__);
+    LOG((1, "compute_counts ios->num_uniontasks = %d", ios->num_uniontasks));
+
+    /* Arrays for swapm all to all gather calls. */
+    MPI_Datatype sr_types[ios->num_uniontasks];
+    int send_counts[ios->num_uniontasks];
+    int send_displs[ios->num_uniontasks];
+    int recv_counts[ios->num_uniontasks];
+    int recv_displs[ios->num_uniontasks];
+
+    /* The list of indeces on each compute task */
+    PIO_Offset s2rindex[iodesc->ndof];
+
+    /* Allocate memory for the array of counts and init to zero. */
+    if (!(iodesc->scount = calloc(ios->num_iotasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* iodesc->scount is the number of data elements sent to each IO
+     * task from the current compute task. dest_ioindex[i] may be
+     * -1. */
+    for (int i = 0; i < iodesc->ndof; i++)
+        if (dest_ioindex[i] >= 0)
+            (iodesc->scount[dest_ioproc[i]])++;
+
+    /* Initialize arrays used in swapm call. */
+    for (int i = 0; i < ios->num_uniontasks; i++)
+    {
+        send_counts[i] = 0;
+        send_displs[i] = 0;
+        recv_counts[i] = 0;
+        recv_displs[i] = 0;
+        sr_types[i] = MPI_INT;
+    }
+
+    /* Setup for the swapm call. iodesc->scount is the amount of data
+     * this compute task will transfer to/from each iotask. For the
+     * box rearranger there can be more than one IO task per compute
+     * task. This provides enough information to know the size of data
+     * on the iotask, so at line 557 we allocate arrays to hold the
+     * map on the iotasks. iodesc->rcount is an array of the amount of
+     * data to expect from each compute task and iodesc->rfrom is the
+     * rank of that task. */
+    if (ios->compproc)
+    {
+        for (int i = 0; i < ios->num_iotasks; i++)
+        {
+            send_counts[ios->ioranks[i]] = 1;
+            send_displs[ios->ioranks[i]] = i * sizeof(int);
+            LOG((3, "send_counts[%d] = %d send_displs[%d] = %d", ios->ioranks[i],
+                 send_counts[ios->ioranks[i]], ios->ioranks[i], send_displs[ios->ioranks[i]]));
+        }
+    }
+    
+    /* IO tasks need to know how many data elements they will receive
+     * from each compute task. Allocate space for that, and set up
+     * swapm call. */
+    if (ios->ioproc)
+    {
+        /* Allocate memory to hold array of the scounts from all
+         * computation tasks. */
+        if (!(recv_buf = calloc(ios->num_comptasks, sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        /* Initialize arrays that keep track of ???. */
+        for (int i = 0; i < ios->num_comptasks; i++)
+        {
+            recv_counts[ios->compranks[i]] = 1;
+            recv_displs[ios->compranks[i]] = i * sizeof(int);
+        }
+    }
+
+    LOG((2, "about to share scount from each compute task to all IO tasks."));
+    /* Share the iodesc->scount from each compute task to all IO
+     * tasks. The scounts will end up in array recv_buf. */
+    if ((ierr = pio_swapm(iodesc->scount, send_counts, send_displs, sr_types,
+                          recv_buf, recv_counts, recv_displs, sr_types, ios->union_comm,
+                          &iodesc->rearr_opts.comp2io)))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+
+    /* On IO tasks, set up data receives. */
+    if (ios->ioproc)
+    {
+        /* Count the number of non-zero scounts from the compute
+         * tasks. */
+        for (int i = 0; i < ios->num_comptasks; i++)
+        {
+            if (recv_buf[i] != 0)
+                nrecvs++;
+            LOG((3, "recv_buf[%d] = %d", i, recv_buf[i]));
+        }
+
+        /* Get memory to hold the count of data receives. */
+        if (!(iodesc->rcount = calloc(max(1, nrecvs), sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        /* Get memory to hold the list of task data was from. */
+        if (!(iodesc->rfrom = calloc(max(1, nrecvs), sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        LOG((3, "allocared rfrom max(1, nrecvs) = %d", max(1, nrecvs)));
+
+        nrecvs = 0;
+        for (int i = 0; i < ios->num_comptasks; i++)
+        {
+            if (recv_buf[i] != 0)
+            {
+                iodesc->rcount[nrecvs] = recv_buf[i];
+                iodesc->rfrom[nrecvs] = i;
+                nrecvs++;
+            }
+        }
+        free(recv_buf);
+    }
+
+    /* ??? */
+    iodesc->nrecvs = nrecvs;
+    LOG((3, "iodesc->nrecvs = %d", iodesc->nrecvs));
+
+    /* Allocate an array for indicies on the computation tasks (the
+     * send side when writing). */
+    if (iodesc->sindex == NULL && iodesc->ndof > 0)
+        if (!(iodesc->sindex = malloc(iodesc->ndof * sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    LOG((2, "iodesc->ndof = %d", iodesc->ndof));
+
+    int tempcount[ios->num_iotasks];
+    int spos[ios->num_iotasks];
+
+    /* ??? */
+    spos[0] = 0;
+    tempcount[0] = 0;
+    for (int i = 1; i < ios->num_iotasks; i++)
+    {
+        spos[i] = spos[i - 1] + iodesc->scount[i - 1];
+        tempcount[i] = 0;
+        LOG((3, "spos[%d] = %d tempcount[%d] = %d", i, spos[i], i, tempcount[i]));
+    }
+
+    /* ??? */
+    for (int i = 0; i < iodesc->ndof; i++)
+    {
+        int iorank;
+        int ioindex;
+
+        iorank = dest_ioproc[i];
+        ioindex = dest_ioindex[i];
+        if (iorank > -1)
+        {
+            /* this should be moved to create_box */
+            iodesc->sindex[spos[iorank] + tempcount[iorank]] = i;
+
+            s2rindex[spos[iorank] + tempcount[iorank]] = ioindex;
+            (tempcount[iorank])++;
+            LOG((3, "iorank = %d ioindex = %d tempcount[iorank] = %d", iorank, ioindex,
+                 tempcount[iorank]));
+        }
+    }
+
+    /* Initialize arrays to zeros. */
+    for (int i = 0; i < ios->num_uniontasks; i++)
+    {
+        send_counts[i] = 0;
+        send_displs[i] = 0;
+        recv_counts[i] = 0;
+        recv_displs[i] = 0;
+    }
+
+    /* ??? */
+    for (int i = 0; i < ios->num_iotasks; i++)
+    {
+        /* Subset rearranger needs one type, box rearranger needs one for
+         * each IO task. */
+        send_counts[ios->ioranks[i]] = iodesc->scount[i];
+        if (send_counts[ios->ioranks[i]] > 0)
+            send_displs[ios->ioranks[i]] = spos[i] * SIZEOF_MPI_OFFSET;
+        LOG((3, "ios->ioranks[i] = %d iodesc->scount[%d] = %d spos[%d] = %d",
+             ios->ioranks[i], i, iodesc->scount[i], i, spos[i]));
+    }
+
+    /* Only do this on IO tasks. */
+    if (ios->ioproc)
+    {
+        int totalrecv = 0;
+        for (int i = 0; i < nrecvs; i++)
+        {
+            recv_counts[iodesc->rfrom[i]] = iodesc->rcount[i];
+            totalrecv += iodesc->rcount[i];
+        }
+
+        recv_displs[0] = 0;
+        for (int i = 1; i < nrecvs; i++)
+        {
+            recv_displs[iodesc->rfrom[i]] = recv_displs[iodesc->rfrom[i - 1]] +
+                iodesc->rcount[i - 1] * SIZEOF_MPI_OFFSET;
+            LOG((3, "iodesc->rfrom[%d] = %d recv_displs[iodesc->rfrom[i]] = %d", i,
+                 iodesc->rfrom[i], recv_displs[iodesc->rfrom[i]]));
+        }
+
+        /* rindex is an array of the indices of the data to be sent from
+           this io task to each compute task. */
+        LOG((3, "totalrecv = %d", totalrecv));
+        if (totalrecv > 0)
+        {
+            totalrecv = iodesc->llen;  /* can reduce memory usage here */
+            if (!(iodesc->rindex = calloc(totalrecv, sizeof(PIO_Offset))))
+                return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+            LOG((3, "allocated totalrecv elements in rindex array"));
+        }
+    }
+
+    /* For the swapm call below, init the types to MPI_OFFSET. */
+    for (int i = 0; i < ios->num_uniontasks; i++)
+        sr_types[i] = MPI_OFFSET;
+
+    /* Here we are sending the mapping from the index on the compute
+     * task to the index on the io task. */
+    /* s2rindex is the list of indeces on each compute task */
+    LOG((3, "sending mapping"));
+    if ((ierr = pio_swapm(s2rindex, send_counts, send_displs, sr_types, iodesc->rindex,
+                          recv_counts, recv_displs, sr_types, ios->union_comm,
+                          &iodesc->rearr_opts.comp2io)))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Moves data from compute tasks to IO tasks. This is called from
+ * PIOc_write_darray_multi().
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @param sbuf send buffer. May be NULL.
+ * @param rbuf receive buffer. May be NULL.
+ * @param nvars number of variables.
+ * @returns 0 on success, error code otherwise.
+ */
+int rearrange_comp2io(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
+                      void *rbuf, int nvars)
+{
+    int ntasks;       /* Number of tasks in communicator. */
+    int niotasks;     /* Number of IO tasks. */
+    MPI_Comm mycomm;  /* Communicator that data is transferred over. */
+    int mpierr;       /* Return code from MPI calls. */
+    int ret;
+
+#ifdef TIMING
+    GPTLstart("PIO:rearrange_comp2io");
+#endif
+
+    /* Caller must provide these. */
+    pioassert(ios && iodesc && nvars > 0, "invalid input", __FILE__, __LINE__);
+
+    LOG((1, "rearrange_comp2io nvars = %d iodesc->rearranger = %d", nvars,
+         iodesc->rearranger));
+
+    /* Different rearraangers use different communicators. */
+    if (iodesc->rearranger == PIO_REARR_BOX)
+    {
+        mycomm = ios->union_comm;
+        niotasks = ios->num_iotasks;
+    }
+    else
+    {
+        mycomm = iodesc->subset_comm;
+        niotasks = 1;
+    }
+
+    /* Get the number of tasks. */
+    if ((mpierr = MPI_Comm_size(mycomm, &ntasks)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    /* These are parameters to pio_swapm to send data from compute to
+     * IO tasks. */
+    int sendcounts[ntasks];
+    int recvcounts[ntasks];
+    int sdispls[ntasks];
+    int rdispls[ntasks];
+    MPI_Datatype sendtypes[ntasks];
+    MPI_Datatype recvtypes[ntasks];
+
+    /* Initialize pio_swapm parameter arrays. */
+    for (int i = 0; i < ntasks; i++)
+    {
+        sendcounts[i] = 0;
+        recvcounts[i] = 0;
+        sdispls[i] = 0;
+        rdispls[i] = 0;
+        recvtypes[i] = PIO_DATATYPE_NULL;
+        sendtypes[i] =  PIO_DATATYPE_NULL;
+    }
+    LOG((3, "ntasks = %d iodesc->basetype_size = %d niotasks = %d", ntasks,
+         iodesc->basetype_size, niotasks));
+
+    /* If it has not already been done, define the MPI data types that
+     * will be used for this io_desc_t. */
+    if ((ret = define_iodesc_datatypes(ios, iodesc)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* If this io proc will exchange data with compute tasks create a
+     * MPI DataType for that exchange. */
+    LOG((2, "ios->ioproc %d iodesc->nrecvs = %d", ios->ioproc, iodesc->nrecvs));
+    if (ios->ioproc && iodesc->nrecvs > 0)
+    {
+        for (int i = 0; i < iodesc->nrecvs; i++)
+        {
+            if (iodesc->rtype[i] != PIO_DATATYPE_NULL)
+            {
+                LOG((3, "iodesc->rtype[%d] = %d iodesc->rearranger = %d", i, iodesc->rtype[i],
+                        iodesc->rearranger));
+                if (iodesc->rearranger == PIO_REARR_SUBSET)
+                {
+                    LOG((3, "exchanging data for subset rearranger"));
+                    recvcounts[i] = 1;
+
+                    /*  Create an MPI derived data type from equally
+                     *  spaced blocks of the same size. The block size
+                     *  is 1, the stride here is the length of the
+                     *  collected array (llen). */
+#if PIO_USE_MPISERIAL
+                    if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size,
+                                                   iodesc->rtype[i], &recvtypes[i])))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#else
+                    if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size,
+                                                          iodesc->rtype[i], &recvtypes[i])))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#endif /* PIO_USE_MPISERIAL */
+                    pioassert(recvtypes[i] != PIO_DATATYPE_NULL, "bad mpi type", __FILE__, __LINE__);
+
+                    if ((mpierr = MPI_Type_commit(&recvtypes[i])))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                }
+                else
+                {
+                    LOG((3, "exchanging data for box rearranger"));
+                    LOG((3, "i = %d iodesc->rfrom[i] = %d recvcounts[iodesc->rfrom[i]] = %d", i,
+                         iodesc->rfrom[i], recvcounts[iodesc->rfrom[i]]));
+                    recvcounts[iodesc->rfrom[i]] = 1;
+
+#if PIO_USE_MPISERIAL
+                    if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size,
+                                                   iodesc->rtype[i], &recvtypes[iodesc->rfrom[i]])))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#else
+                    if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->llen * iodesc->basetype_size,
+                                                          iodesc->rtype[i], &recvtypes[iodesc->rfrom[i]])))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#endif /* PIO_USE_MPISERIAL */
+                    pioassert(recvtypes[iodesc->rfrom[i]] != PIO_DATATYPE_NULL,  "bad mpi type",
+                              __FILE__, __LINE__);
+
+                    if ((mpierr = MPI_Type_commit(&recvtypes[iodesc->rfrom[i]])))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+                    rdispls[iodesc->rfrom[i]] = 0;
+                }
+            }
+        }
+    }
+
+    /* On compute tasks loop over iotasks and create a data type for
+     * each exchange.  */
+    for (int i = 0; i < niotasks; i++)
+    {
+        int io_comprank = ios->ioranks[i];
+        if (iodesc->rearranger == PIO_REARR_SUBSET)
+            io_comprank = 0;
+
+        LOG((3, "i = %d iodesc->scount[i] = %d", i, iodesc->scount[i]));
+        if (iodesc->scount[i] > 0 && sbuf)
+        {
+            LOG((3, "io task %d creating sendtypes[%d]", i, io_comprank));
+            sendcounts[io_comprank] = 1;
+#if PIO_USE_MPISERIAL
+            if ((mpierr = MPI_Type_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->basetype_size,
+                                           iodesc->stype[i], &sendtypes[io_comprank])))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#else
+            if ((mpierr = MPI_Type_create_hvector(nvars, 1, (MPI_Aint)iodesc->ndof * iodesc->basetype_size,
+                                                  iodesc->stype[i], &sendtypes[io_comprank])))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#endif /* PIO_USE_MPISERIAL */
+            pioassert(sendtypes[io_comprank] != PIO_DATATYPE_NULL,  "bad mpi type", __FILE__, __LINE__);
+
+            if ((mpierr = MPI_Type_commit(&sendtypes[io_comprank])))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        }
+        else
+        {
+            sendcounts[io_comprank] = 0;
+        }
+    }
+
+    /* Data in sbuf on the compute nodes is sent to rbuf on the ionodes */
+    LOG((2, "about to call pio_swapm for sbuf"));
+    if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes,
+                         rbuf, recvcounts, rdispls, recvtypes, mycomm,
+                         &iodesc->rearr_opts.comp2io)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Free the MPI types. */
+    for (int i = 0; i < ntasks; i++)
+    {
+        LOG((3, "freeing MPI types for task %d", i));
+        if (sendtypes[i] != PIO_DATATYPE_NULL)
+            if ((mpierr = MPI_Type_free(&sendtypes[i])))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+        if (recvtypes[i] != PIO_DATATYPE_NULL)
+            if ((mpierr = MPI_Type_free(&recvtypes[i])))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+#ifdef TIMING
+    GPTLstop("PIO:rearrange_comp2io");
+#endif
+
+    return PIO_NOERR;
+}
+
+/**
+ * Moves data from IO tasks to compute tasks. This function is used in
+ * PIOc_read_darray().
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @param sbuf send buffer.
+ * @param rbuf receive buffer.
+ * @returns 0 on success, error code otherwise.
+ */
+int rearrange_io2comp(iosystem_desc_t *ios, io_desc_t *iodesc, void *sbuf,
+                      void *rbuf)
+{
+    MPI_Comm mycomm;
+    int ntasks;
+    int niotasks;
+    int mpierr; /* Return code from MPI calls. */
+    int ret;
+
+    /* Check inputs. */
+    pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__);
+
+#ifdef TIMING
+    GPTLstart("PIO:rearrange_io2comp");
+#endif
+
+    /* Different rearrangers use different communicators and number of
+     * IO tasks. */
+    if (iodesc->rearranger == PIO_REARR_BOX)
+    {
+        mycomm = ios->union_comm;
+        niotasks = ios->num_iotasks;
+    }
+    else
+    {
+        mycomm = iodesc->subset_comm;
+        niotasks = 1;
+    }
+    LOG((3, "niotasks = %d", niotasks));
+
+    /* Get the size of this communicator. */
+    if ((mpierr = MPI_Comm_size(mycomm, &ntasks)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Define the MPI data types that will be used for this
+     * io_desc_t. */
+    if ((ret = define_iodesc_datatypes(ios, iodesc)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Allocate arrays needed by the pio_swapm() function. */
+    int sendcounts[ntasks];
+    int recvcounts[ntasks];
+    int sdispls[ntasks];
+    int rdispls[ntasks];
+    MPI_Datatype sendtypes[ntasks];
+    MPI_Datatype recvtypes[ntasks];
+
+    /* Initialize arrays. */
+    for (int i = 0; i < ntasks; i++)
+    {
+        sendcounts[i] = 0;
+        recvcounts[i] = 0;
+        sdispls[i] = 0;
+        rdispls[i] = 0;
+        sendtypes[i] = PIO_DATATYPE_NULL;
+        recvtypes[i] = PIO_DATATYPE_NULL;
+    }
+
+    /* In IO tasks set up sendcounts/sendtypes for pio_swapm() call
+     * below. */
+    if (ios->ioproc)
+    {
+        for (int i = 0; i < iodesc->nrecvs; i++)
+        {
+            if (iodesc->rtype[i] != PIO_DATATYPE_NULL)
+            {
+                if (iodesc->rearranger == PIO_REARR_SUBSET)
+                {
+                    if (sbuf)
+                    {
+                        sendcounts[i] = 1;
+                        sendtypes[i] = iodesc->rtype[i];
+                    }
+                }
+                else
+                {
+                    sendcounts[iodesc->rfrom[i]] = 1;
+                    sendtypes[iodesc->rfrom[i]] = iodesc->rtype[i];
+                }
+            }
+        }
+    }
+
+    /* In the box rearranger each comp task may communicate with
+     * multiple IO tasks here we are setting the count and data type
+     * of the communication of a given compute task with each io
+     * task. */
+    for (int i = 0; i < niotasks; i++)
+    {
+        int io_comprank = ios->ioranks[i];
+
+        if (iodesc->rearranger == PIO_REARR_SUBSET)
+            io_comprank = 0;
+
+        if (iodesc->scount[i] > 0 && iodesc->stype[i] != PIO_DATATYPE_NULL)
+        {
+            recvcounts[io_comprank] = 1;
+            recvtypes[io_comprank] = iodesc->stype[i];
+        }
+    }
+
+    /* Data in sbuf on the ionodes is sent to rbuf on the compute nodes */
+    if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, rbuf, recvcounts,
+                         rdispls, recvtypes, mycomm, &iodesc->rearr_opts.io2comp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+#ifdef TIMING
+    GPTLstop("PIO:rearrange_io2comp");
+#endif
+
+    return PIO_NOERR;
+}
+
+/**
+ * Determine whether fill values are needed. This function compares
+ * how much data we have to how much data is in a record (or
+ * non-record var). If we have enough data to completely fill the
+ * variable, then fill is not needed.
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @param gdimlen pointer to an array length iodesc->ndims with the
+ * global array sizes for one record (for record vars) or for the
+ * entire var (for non-record vars).
+ * @param compmap only used for the box communicator.
+ * @returns 0 on success, error code otherwise.
+ */
+int determine_fill(iosystem_desc_t *ios, io_desc_t *iodesc, const int *gdimlen,
+                   const PIO_Offset *compmap)
+{
+    PIO_Offset totalllen = 0;
+    PIO_Offset totalgridsize = 1;
+    int mpierr; /* Return code from MPI calls. */
+
+    /* Check inputs. */
+    pioassert(ios && iodesc && gdimlen && compmap, "invalid input",
+              __FILE__, __LINE__);
+
+    /* Determine size of data space. */
+    for (int i = 0; i < iodesc->ndims; i++)
+        totalgridsize *= gdimlen[i];
+
+    /* Determine how many values we have locally. */
+    if (iodesc->rearranger == PIO_REARR_SUBSET)
+        totalllen = iodesc->llen;
+    else
+        for (int i = 0; i < iodesc->ndof; i++)
+            if (compmap[i] > 0)
+                totalllen++;
+
+    /* Add results accross communicator. */
+    LOG((2, "determine_fill before allreduce totalllen = %d totalgridsize = %d",
+         totalllen, totalgridsize));
+    if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &totalllen, 1, PIO_OFFSET, MPI_SUM,
+                                ios->union_comm)))
+        check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "after allreduce totalllen = %d", totalllen));
+
+    /* If the total size of the data provided to be written is < the
+     * total data size then we need fill values. */
+    iodesc->needsfill = totalllen < totalgridsize;
+
+    /*  TURN OFF FILL for timing test
+        iodesc->needsfill=false; */
+
+    return PIO_NOERR;
+}
+
+/**
+ * The box rearranger computes a mapping between IO tasks and compute
+ * tasks such that the data on IO tasks can be written with a single
+ * call to the underlying netCDF library. This may involve an all to
+ * all rearrangement in the mapping, but should minimize data movement
+ * in lower level libraries.
+ *
+ * On each compute task the application program passes a compmap array
+ * of length ndof. This array describes the arrangement of data in
+ * memory on that compute task.
+ *
+ * These arrays are gathered and rearranged to the IO-tasks (which are
+ * sometimes collocated with compute tasks), each IO task contains
+ * data from the compmap of one or more compute tasks in the iomap
+ * array and the length of that array is llen.
+ *
+ * This function:
+ * 
      
+ * 
    • For IO tasks, determines llen.
+ * 
    • Determine whether fill values will be needed.
+ * 
    • Do an allgether of llen values into array iomaplen.
+ * 
    • For each IO task, send starts/counts to all compute tasks.
+ * 
    • Find dest_ioindex and dest_ioproc for each element in the map.
+ * 
    • Call compute_counts().
+ * 
    • On IO tasks, compute the max IO buffer size.
+ * 
    • Call compute_maxaggregate_bytes().
+ * 
    
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param maplen the length of the map. This is the number of data
+ * elements on the compute task.
+ * @param compmap a 1 based array of offsets into the global space. A
+ * 0 in this array indicates a value which should not be transfered.
+ * @param gdimlen an array length ndims with the sizes of the global
+ * dimensions.
+ * @param ndims the number of dimensions.
+ * @param iodesc a pointer to the io_desc_t struct, which must be
+ * allocated before this function is called.
+ * @returns 0 on success, error code otherwise.
+ */
+int box_rearrange_create(iosystem_desc_t *ios, int maplen, const PIO_Offset *compmap,
+                         const int *gdimlen, int ndims, io_desc_t *iodesc)
+{
+    int ret;
+
+    /* Check inputs. */
+    pioassert(ios && maplen >= 0 && compmap && gdimlen && ndims > 0 && iodesc,
+              "invalid input", __FILE__, __LINE__);
+    LOG((1, "box_rearrange_create maplen = %d ndims = %d ios->num_comptasks = %d "
+         "ios->num_iotasks = %d", maplen, ndims, ios->num_comptasks, ios->num_iotasks));
+
+    /* Allocate arrays needed for this function. */
+    int dest_ioproc[maplen]; /* Destination IO task for each data element on compute task. */
+    PIO_Offset dest_ioindex[maplen];    /* Offset into IO task array for each data element. */
+    int sendcounts[ios->num_uniontasks]; /* Send counts for swapm call. */
+    int sdispls[ios->num_uniontasks];    /* Send displacements for swapm. */
+    int recvcounts[ios->num_uniontasks]; /* Receive counts for swapm. */
+    int rdispls[ios->num_uniontasks];    /* Receive displacements for swapm. */
+    MPI_Datatype dtypes[ios->num_uniontasks]; /* Array of MPI_OFFSET types for swapm. */
+    PIO_Offset iomaplen[ios->num_iotasks];   /* Gets the llen of all IO tasks. */
+
+    /* This is the box rearranger. */
+    iodesc->rearranger = PIO_REARR_BOX;
+
+    /* Number of elements of data on compute node. */
+    iodesc->ndof = maplen;
+
+    /* Initialize array values. */
+    for (int i = 0; i < maplen; i++)
+    {
+        dest_ioproc[i] = -1;
+        dest_ioindex[i] = -1;
+    }
+
+    /* Initialize arrays used in swapm. */
+    for (int i = 0; i < ios->num_uniontasks; i++)
+    {
+        sendcounts[i] = 0;
+        sdispls[i] = 0;
+        recvcounts[i] = 0;
+        rdispls[i] = 0;
+        dtypes[i] = MPI_OFFSET;
+    }
+
+    /* For IO tasks, determine llen, the length of the data array on
+     * the IO task. For computation tasks, llen will remain at 0. Also
+     * set up arrays for the allgather which will give every IO task a
+     * complete list of llens for each IO task. */
+    LOG((3, "ios->ioproc = %d ios->num_uniontasks = %d", ios->ioproc,
+         ios->num_uniontasks));
+    pioassert(iodesc->llen == 0, "error", __FILE__, __LINE__);
+    if (ios->ioproc)
+    {
+        /* Set up send counts for sending llen in all to all
+         * gather. We are sending to all tasks, IO and computation. */
+        for (int i = 0; i < ios->num_comptasks; i++)
+            sendcounts[ios->compranks[i]] = 1;
+        for (int i = 0; i < ios->num_iotasks; i++)
+            sendcounts[ios->ioranks[i]] = 1;
+
+        /* Determine llen, the lenght of the data array on this IO
+         * node, by multipliying the counts in the
+         * iodesc->firstregion. */
+        iodesc->llen = 1;
+        for (int i = 0; i < ndims; i++)
+        {
+            iodesc->llen *= iodesc->firstregion->count[i];
+            LOG((3, "iodesc->firstregion->start[%d] = %d iodesc->firstregion->count[%d] = %d",
+                 i, iodesc->firstregion->start[i], i, iodesc->firstregion->count[i]));
+        }
+        LOG((2, "iodesc->llen = %d", iodesc->llen));
+    }
+
+    /* Determine whether fill values will be needed. */
+    if ((ret = determine_fill(ios, iodesc, gdimlen, compmap)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((2, "iodesc->needsfill = %d ios->num_iotasks = %d", iodesc->needsfill,
+         ios->num_iotasks));
+
+    /* Set up receive counts and displacements to for an AllToAll
+     * gather of llen. */
+    for (int i = 0; i < ios->num_iotasks; i++)
+    {
+        recvcounts[ios->ioranks[i]] = 1;
+        rdispls[ios->ioranks[i]] = i * SIZEOF_MPI_OFFSET;
+        LOG((3, "i = %d ios->ioranks[%d] = %d recvcounts[%d] = %d rdispls[%d] = %d",
+             i, i, ios->ioranks[i], ios->ioranks[i], recvcounts[ios->ioranks[i]],
+             ios->ioranks[i], rdispls[ios->ioranks[i]]));
+    }
+
+    /* All-gather the llen to all tasks into array iomaplen. */
+    LOG((3, "calling pio_swapm to allgather llen into array iomaplen, ndims = %d dtypes[0] = %d",
+         ndims, dtypes));
+    if ((ret = pio_swapm(&iodesc->llen, sendcounts, sdispls, dtypes, iomaplen, recvcounts,
+                         rdispls, dtypes, ios->union_comm, &iodesc->rearr_opts.io2comp)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((3, "iodesc->llen = %d", iodesc->llen));
+#if PIO_ENABLE_LOGGING
+    for (int i = 0; i < ios->num_iotasks; i++)
+        LOG((3, "iomaplen[%d] = %d", i, iomaplen[i]));
+#endif /* PIO_ENABLE_LOGGING */
+
+    /* For each IO task send starts/counts to all compute tasks. */
+    for (int i = 0; i < ios->num_iotasks; i++)
+    {
+        /* The ipmaplen contains the llen (number of data elements)
+         * for this IO task. */
+        LOG((2, "iomaplen[%d] = %d", i, iomaplen[i]));
+
+        /* If there is data for this IO task, send start/count to all
+         * compute tasks. */
+        if (iomaplen[i] > 0)
+        {
+            PIO_Offset start[ndims];
+            PIO_Offset count[ndims];
+
+            /* Set up send/recv parameters for all to all gather of
+             * counts and starts. */
+            for (int j = 0; j < ios->num_uniontasks; j++)
+            {
+                sendcounts[j] = 0;
+                sdispls[j] = 0;
+                rdispls[j] = 0;
+                recvcounts[j] = 0;
+                if (ios->union_rank == ios->ioranks[i])
+                    sendcounts[j] = ndims;
+            }
+            recvcounts[ios->ioranks[i]] = ndims;
+
+            /* The count array from iotask i is sent to all compute tasks. */
+            LOG((3, "about to call pio_swapm with count from iotask %d ndims = %d",
+                 i, ndims));
+            if ((ret = pio_swapm(iodesc->firstregion->count, sendcounts, sdispls, dtypes, count,
+                                 recvcounts, rdispls, dtypes, ios->union_comm,
+                                 &iodesc->rearr_opts.io2comp)))
+                return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+            /* The start array from iotask i is sent to all compute tasks. */
+            LOG((3, "about to call pio_swapm with start from iotask %d ndims = %d",
+                 i, ndims));
+            if ((ret = pio_swapm(iodesc->firstregion->start,  sendcounts, sdispls, dtypes,
+                                 start, recvcounts, rdispls, dtypes, ios->union_comm,
+                                 &iodesc->rearr_opts.io2comp)))
+                return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+#if PIO_ENABLE_LOGGING
+            for (int d = 0; d < ndims; d++)
+                LOG((3, "start[%d] = %lld count[%d] = %lld", d, start[d], d, count[d]));
+#endif /* PIO_ENABLE_LOGGING */
+
+            /* For each element of the data array on the compute task,
+             * find the IO task to send the data element to, and its
+             * offset into the global data array. */
+            for (int k = 0; k < maplen; k++)
+            {
+                PIO_Offset gcoord[ndims], lcoord[ndims];
+                bool found = true;
+
+                /* The compmap array is 1 based but calculations are 0 based */
+                LOG((3, "about to call idx_to_dim_list ndims = %d ", ndims));
+                idx_to_dim_list(ndims, gdimlen, compmap[k] - 1, gcoord);
+#if PIO_ENABLE_LOGGING
+                for (int d = 0; d < ndims; d++)
+                    LOG((3, "gcoord[%d] = %lld", d, gcoord[d]));
+#endif /* PIO_ENABLE_LOGGING */
+
+                /* Find a destination for each entry in the compmap. */
+                for (int j = 0; j < ndims; j++)
+                {
+                    if (gcoord[j] >= start[j] && gcoord[j] < start[j] + count[j])
+                    {
+                        lcoord[j] = gcoord[j] - start[j];
+                    }
+                    else
+                    {
+                        found = false;
+                        break;
+                    }
+                }
+
+                /* Did we find a destination IO task for this element
+                 * of the computation task data array? If so, remember
+                 * the destination IO task, and determine the index
+                 * for that element in the IO task data. */
+                if (found)
+                {
+                    dest_ioindex[k] = coord_to_lindex(ndims, lcoord, count);
+                    dest_ioproc[k] = i;
+                    LOG((3, "found dest_ioindex[%d] = %d dest_ioproc[%d] = %d", k, dest_ioindex[k],
+                         k, dest_ioproc[k]));
+                }
+            }
+        }
+    }
+
+    /* Check that a destination is found for each compmap entry. */
+    for (int k = 0; k < maplen; k++)
+        if (dest_ioproc[k] < 0 && compmap[k] > 0)
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Completes the mapping for the box rearranger. */
+    LOG((2, "calling compute_counts maplen = %d", maplen));
+    if ((ret = compute_counts(ios, iodesc, dest_ioproc, dest_ioindex)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Compute the max io buffer size needed for an iodesc. */
+    if (ios->ioproc)
+    {
+        if ((ret = compute_maxIObuffersize(ios->io_comm, iodesc)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+        LOG((3, "iodesc->maxiobuflen = %d", iodesc->maxiobuflen));
+    }
+
+    /* Using maxiobuflen compute the maximum number of bytes that the
+     * io task buffer can handle. */
+    if ((ret = compute_maxaggregate_bytes(ios, iodesc)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((3, "iodesc->maxbytes = %d", iodesc->maxbytes));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Compare offsets is used by the sort in the subset rearranger. This
+ * function is passed to qsort.
+ *
+ * @param a pointer to an offset.
+ * @param b pointer to another offset.
+ * @returns 0 if offsets are the same or either pointer is NULL.
+ */
+int compare_offsets(const void *a, const void *b)
+{
+    mapsort *x = (mapsort *)a;
+    mapsort *y = (mapsort *)b;
+    if (!x || !y)
+        return 0;
+    return (int)(x->iomap - y->iomap);
+}
+
+/**
+ * Calculate start and count regions for the subset rearranger. This
+ * function is not used in the box rearranger.
+ *
+ * Each region is a block of output which can be represented in a
+ * single call to the underlying netcdf library.  This can be as small
+ * as a single data point, but we hope we've aggragated better than
+ * that.
+ *
+ * @param ndims the number of dimensions
+ * @param gdimlen an array length ndims with the sizes of the global
+ * dimensions.
+ * @param maplen the length of the map
+ * @param map may be NULL (when ???).
+ * @param maxregions
+ * @param firstregion pointer to the first region.
+ * @returns 0 on success, error code otherwise.
+ */
+int get_regions(int ndims, const int *gdimlen, int maplen, const PIO_Offset *map,
+                int *maxregions, io_region *firstregion)
+{
+    int nmaplen = 0;
+    int regionlen;
+    io_region *region;
+    int ret;
+
+    /* Check inputs. */
+    pioassert(ndims >= 0 && gdimlen && maplen >= 0 && maxregions && firstregion,
+              "invalid input", __FILE__, __LINE__);
+    LOG((1, "get_regions ndims = %d maplen = %d", ndims, maplen));
+
+    region = firstregion;
+    if (map)
+    {
+        while (map[nmaplen++] <= 0)
+        {
+            LOG((3, "map[%d] = %d", nmaplen, map[nmaplen]));
+            ;
+        }
+        nmaplen--;
+    }
+    region->loffset = nmaplen;
+    LOG((2, "region->loffset = %d", region->loffset));
+
+    *maxregions = 1;
+
+    while (nmaplen < maplen)
+    {
+        /* Here we find the largest region from the current offset
+           into the iomap. regionlen is the size of that region and we
+           step to that point in the map array until we reach the
+           end. */
+        for (int i = 0; i < ndims; i++)
+            region->count[i] = 1;
+
+        /* Set start/count to describe first region in map. */
+        regionlen = find_region(ndims, gdimlen, maplen-nmaplen,
+                                &map[nmaplen], region->start, region->count);
+        pioassert(region->start[0] >= 0, "failed to find region", __FILE__, __LINE__);
+
+        nmaplen = nmaplen + regionlen;
+        LOG((2, "regionlen = %d nmaplen = %d", regionlen, nmaplen));
+
+        /* If we need to, allocate the next region. */
+        if (region->next == NULL && nmaplen < maplen)
+        {
+            LOG((2, "allocating next region"));
+            if ((ret = alloc_region2(NULL, ndims, ®ion->next)))
+                return ret;
+
+            /* The offset into the local array buffer is the sum of
+             * the sizes of all of the previous regions (loffset) */
+            region = region->next;
+            region->loffset = nmaplen;
+
+            /* The calls to the io library are collective and so we
+               must have the same number of regions on each io task
+               maxregions will be the total number of regions on this
+               task. */
+            (*maxregions)++;
+            LOG((2, "*maxregions = %d", *maxregions));
+        }
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Create the MPI communicators needed by the subset rearranger.
+ *
+ * The subset rearranger needs a mapping from compute tasks to IO
+ * task, the only requirement is that each compute task map to one and
+ * only one IO task.  This mapping groups by mpi task id others are
+ * possible and may be better for certain decompositions
+ *
+ * The as yet unrealized vision here is that the user would be able to
+ * supply an alternative subset partitioning function. Requirements of
+ * this function are that there be exactly one io task per compute
+ * task group.
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @returns 0 on success, error code otherwise.
+ */
+int default_subset_partition(iosystem_desc_t *ios, io_desc_t *iodesc)
+{
+    int color;
+    int key;
+    int mpierr; /* Return value from MPI functions. */
+
+    pioassert(ios && iodesc, "invalid input", __FILE__, __LINE__);
+    LOG((1, "default_subset_partition ios->ioproc = %d ios->io_rank = %d "
+         "ios->comp_rank = %d", ios->ioproc, ios->io_rank, ios->comp_rank));
+
+    /* Create a new comm for each subset group with the io task in
+       rank 0 and only 1 io task per group */
+    if (ios->ioproc)
+    {
+        key = 0;
+        color= ios->io_rank;
+    }
+    else
+    {
+        int taskratio = ios->num_comptasks / ios->num_iotasks;
+        key = max(1, ios->comp_rank % taskratio + 1);
+        color = min(ios->num_iotasks - 1, ios->comp_rank / taskratio);
+    }
+    LOG((3, "key = %d color = %d", key, color));
+
+    /* Create new communicators. */
+    if ((mpierr = MPI_Comm_split(ios->comp_comm, color, key, &iodesc->subset_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Create the subset rearranger.
+ *
+ * The subset rearranger computes a mapping between IO tasks and
+ * compute tasks such that each compute task communicates with one and
+ * only one IO task.
+ *
+ * This function is called from PIOc_InitDecomp().
+ *
+ * This function:
+ * 
      
+ * 
    • Calls default_subset_partition() to create subset_comm.
+ * 
    • For IO tasks, allocates iodesc->rcount array (length ntasks).
+ * 
    • Allocates iodesc->scount array (length 1)
+ * 
    • Determins value of iodesc->scount[0], the number of data
+ * elements on this compute task which are read/written.
+ * 
    • Allocated and inits iodesc->sindex (length iodesc->scount[0]),
+ * init it to contain indicies to data.
+ * 
    • Pass the reduced maplen (without holes) from each compute task
+ * to its associated IO task.
+ * 
    • On IO tasks, determine llen.
+ * 
    • Determine whether fill values will be needed.
+ * 
    • Pass iodesc->sindex from each compute task to its associated IO
+ * task.
+ * 
    • Create shrtmap, which is compmap without the holes.
+ * 
    • Gather shrtmaps from each task into iomap.
+ * 
    • On IO tasks, sort the mapping, this will transpose the data
+ * into IO order.
+ * 
    • On IO tasks, allocate and init iodesc->rindex and iodesc->rfrom
+ * (length iodesc->llen).
+ * 
    • On IO tasks, handle fill values, if needed.
+ * 
    • On IO tasks, scatter values of srcindex to subset communicator.
+ * 
    • On IO tasks, call get_regions() and distribute the max
+ * maxregions to all tasks in IO communicator.
+ * 
    • On IO tasks, call compute_maxIObuffersize().
+ * 
    • Call compute_maxaggregate_bytes().
+ * 
    
+ *
+ * @param ios pointer to the iosystem_desc_t struct.
+ * @param maplen the length of the map.
+ * @param compmap a 1 based array of offsets into the array record on
+ * file. A 0 in this array indicates a value which should not be
+ * transfered.
+ * @param gdimlen an array length ndims with the sizes of the global
+ * dimensions.
+ * @param ndims the number of dimensions.
+ * @param iodesc a pointer to the io_desc_t struct.
+ * @returns 0 on success, error code otherwise.
+ */
+int subset_rearrange_create(iosystem_desc_t *ios, int maplen, PIO_Offset *compmap,
+                            const int *gdimlen, int ndims, io_desc_t *iodesc)
+{
+    int i, j;
+    PIO_Offset *iomap = NULL;
+    mapsort *map = NULL;
+    PIO_Offset totalgridsize;
+    PIO_Offset *srcindex = NULL;
+    PIO_Offset *myfillgrid = NULL;
+    int maxregions;
+    int rank, ntasks;
+    int rcnt = 0;
+    int mpierr; /* Return call from MPI function calls. */
+    int ret;
+
+    /* Check inputs. */
+    pioassert(ios && maplen >= 0 && compmap && gdimlen && ndims >= 0 && iodesc,
+              "invalid input", __FILE__, __LINE__);
+
+    LOG((2, "subset_rearrange_create maplen = %d ndims = %d", maplen, ndims));
+
+    /* subset partitions each have exactly 1 io task which is task 0
+     * of that subset_comm */
+    /* TODO: introduce a mechanism for users to define partitions */
+    if ((ret = default_subset_partition(ios, iodesc)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    iodesc->rearranger = PIO_REARR_SUBSET;
+
+    /* Get size of this subset communicator and rank of this task in it. */
+    if ((mpierr = MPI_Comm_rank(iodesc->subset_comm, &rank)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_size(iodesc->subset_comm, &ntasks)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Check rank for correctness. */
+    if (ios->ioproc)
+        pioassert(rank == 0, "Bad io rank in subset create", __FILE__, __LINE__);
+    else
+        pioassert(rank > 0 && rank < ntasks, "Bad comp rank in subset create",
+                  __FILE__, __LINE__);
+
+    /* Remember the maplen for this computation task. */
+    iodesc->ndof = maplen;
+
+    if (ios->ioproc)
+    {
+        /* Allocate space to hold count of data to be received in pio_swapm(). */
+        if (!(iodesc->rcount = malloc(ntasks * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        rcnt = 1;
+    }
+
+    /* Allocate space to hold count of data to be sent in pio_swapm(). */
+    if (!(iodesc->scount = malloc(sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    iodesc->scount[0] = 0;
+
+    /* Find the total size of the global data array. */
+    totalgridsize = 1;
+    for (i = 0; i < ndims; i++)
+        totalgridsize *= gdimlen[i];
+
+    /* Determine scount[0], the number of data elements in the
+     * computation task that are to be written, by looking at
+     * compmap. */
+    for (i = 0; i < maplen; i++)
+    {
+        /*  turns out this can be allowed in some cases
+            pioassert(compmap[i]>=0 && compmap[i]<=totalgridsize, "Compmap value out of bounds",
+            __FILE__,__LINE__); */
+        if (compmap[i] > 0)
+            (iodesc->scount[0])++;
+    }
+
+    /* Allocate an array for indicies on the computation tasks (the
+     * send side when writing). */
+    if (iodesc->scount[0] > 0)
+        if (!(iodesc->sindex = calloc(iodesc->scount[0], sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    j = 0;
+    for (i = 0; i < maplen; i++)
+        if (compmap[i] > 0)
+            iodesc->sindex[j++] = i;
+
+    /* Pass the reduced maplen (without holes) from each compute task
+     * to its associated IO task. */
+    if ((mpierr = MPI_Gather(iodesc->scount, 1, MPI_INT, iodesc->rcount, rcnt,
+                             MPI_INT, 0, iodesc->subset_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    iodesc->llen = 0;
+
+    int rdispls[ntasks];
+    int recvcounts[ntasks];
+
+    /* On IO tasks determine llen. */
+    if (ios->ioproc)
+    {
+        for (i = 0; i < ntasks; i++)
+        {
+            iodesc->llen += iodesc->rcount[i];
+            rdispls[i] = 0;
+            recvcounts[i] = iodesc->rcount[i];
+            if (i > 0)
+                rdispls[i] = rdispls[i - 1] + iodesc->rcount[i - 1];
+        }
+
+        if (iodesc->llen > 0)
+        {
+            if (!(srcindex = calloc(iodesc->llen, sizeof(PIO_Offset))))
+                return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+            for (i = 0; i < iodesc->llen; i++)
+                srcindex[i] = 0;
+        }
+    }
+    else
+    {
+        for (i = 0; i < ntasks; i++)
+        {
+            recvcounts[i] = 0;
+            rdispls[i] = 0;
+        }
+    }
+
+    /* Determine whether fill values will be needed. */
+    if ((ret = determine_fill(ios, iodesc, gdimlen, compmap)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Pass the sindex from each compute task to its associated IO task. */
+    if ((mpierr = MPI_Gatherv(iodesc->sindex, iodesc->scount[0], PIO_OFFSET,
+                              srcindex, recvcounts, rdispls, PIO_OFFSET, 0,
+                              iodesc->subset_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    /* On IO tasks which need it, allocate memory for the map and the
+     * iomap. */
+    if (ios->ioproc && iodesc->llen > 0)
+    {
+        if (!(map = calloc(iodesc->llen, sizeof(mapsort))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        if (!(iomap = calloc(iodesc->llen, sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    }
+
+    /* Now pass the compmap, skipping the holes. */
+    PIO_Offset *shrtmap;
+    if (maplen > iodesc->scount[0] && iodesc->scount[0] > 0)
+    {
+        if (!(shrtmap = calloc(iodesc->scount[0], sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        j = 0;
+        for (int i = 0; i < maplen; i++)
+            if (compmap[i] > 0)
+                shrtmap[j++] = compmap[i];
+    }
+    else
+    {
+        shrtmap = compmap;
+    }
+
+    /* Gather shrtmap from each task in the subset communicator, and
+     * put gathered results into iomap. */
+    if ((mpierr = MPI_Gatherv(shrtmap, iodesc->scount[0], PIO_OFFSET, iomap, recvcounts,
+                              rdispls, PIO_OFFSET, 0, iodesc->subset_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    if (shrtmap != compmap)
+        free(shrtmap);
+
+    /* On IO tasks that have data in the local array ??? */
+    if (ios->ioproc && iodesc->llen > 0)
+    {
+        int pos = 0;
+        int k = 0;
+        mapsort *mptr;
+        for (i = 0; i < ntasks; i++)
+        {
+            for (j = 0; j < iodesc->rcount[i]; j++)
+            {
+                mptr = &map[k];
+                mptr->rfrom = i;
+                mptr->soffset = srcindex[pos + j];
+                mptr->iomap = iomap[pos + j];
+                k++;
+            }
+            pos += iodesc->rcount[i];
+        }
+
+        /* sort the mapping, this will transpose the data into IO order */
+        qsort(map, iodesc->llen, sizeof(mapsort), compare_offsets);
+
+        if (!(iodesc->rindex = calloc(1, iodesc->llen * sizeof(PIO_Offset))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        if (!(iodesc->rfrom = calloc(1, iodesc->llen * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    }
+
+    int cnt[ntasks];
+    for (i = 0; i < ntasks; i++)
+    {
+        cnt[i] = rdispls[i];
+    }
+
+    /* For IO tasks init rfrom and rindex arrays (compute tasks have
+     * llen of 0). */
+    for (i = 0; i < iodesc->llen; i++)
+    {
+        mapsort *mptr = &map[i];
+        iodesc->rfrom[i] = mptr->rfrom;
+        iodesc->rindex[i] = i;
+        iomap[i] = mptr->iomap;
+        srcindex[(cnt[iodesc->rfrom[i]])++] = mptr->soffset;
+    }
+
+    /* Handle fill values if needed. */
+    if (ios->ioproc && iodesc->needsfill)
+    {
+        /* we need the list of offsets which are not in the union of iomap */
+        PIO_Offset thisgridsize[ios->num_iotasks];
+        PIO_Offset thisgridmin[ios->num_iotasks], thisgridmax[ios->num_iotasks];
+        int nio;
+        PIO_Offset *myusegrid = NULL;
+        int gcnt[ios->num_iotasks];
+        int displs[ios->num_iotasks];
+
+        thisgridmin[0] = 1;
+        thisgridsize[0] =  totalgridsize / ios->num_iotasks;
+        thisgridmax[0] = thisgridsize[0];
+        int xtra = totalgridsize - thisgridsize[0] * ios->num_iotasks;
+
+        for (nio = 0; nio < ios->num_iotasks; nio++)
+        {
+            int cnt = 0;
+            int imin = 0;
+            if (nio > 0)
+            {
+                thisgridsize[nio] =  totalgridsize / ios->num_iotasks;
+                if (nio >= ios->num_iotasks - xtra)
+                    thisgridsize[nio]++;
+                thisgridmin[nio] = thisgridmax[nio - 1] + 1;
+                thisgridmax[nio]= thisgridmin[nio] + thisgridsize[nio] - 1;
+            }
+            for (int i = 0; i < iodesc->llen; i++)
+            {
+                if (iomap[i] >= thisgridmin[nio] && iomap[i] <= thisgridmax[nio])
+                {
+                    cnt++;
+                    if (cnt == 1)
+                        imin = i;
+                }
+            }
+
+            /* Gather cnt from all tasks in the IO communicator into array gcnt. */
+            if ((mpierr = MPI_Gather(&cnt, 1, MPI_INT, gcnt, 1, MPI_INT, nio, ios->io_comm)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+            if (nio == ios->io_rank)
+            {
+                displs[0] = 0;
+                for (i = 1; i < ios->num_iotasks; i++)
+                    displs[i] = displs[i - 1] + gcnt[i - 1];
+
+                /* Allocate storage for the grid. */
+                if (!(myusegrid = malloc(thisgridsize[nio] * sizeof(PIO_Offset))))
+                    return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+                /* Initialize the grid to all -1. */
+                for (i = 0; i < thisgridsize[nio]; i++)
+                    myusegrid[i] = -1;
+            }
+
+            if ((mpierr = MPI_Gatherv(&iomap[imin], cnt, PIO_OFFSET, myusegrid, gcnt,
+                                      displs, PIO_OFFSET, nio, ios->io_comm)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        }
+
+        /* Allocate and initialize a grid to fill in missing values. ??? */
+        PIO_Offset grid[thisgridsize[ios->io_rank]];
+        for (i = 0; i < thisgridsize[ios->io_rank]; i++)
+            grid[i] = 0;
+
+        int cnt = 0;
+        for (i = 0; i < thisgridsize[ios->io_rank]; i++)
+        {
+            int j = myusegrid[i] - thisgridmin[ios->io_rank];
+            pioassert(j < thisgridsize[ios->io_rank], "out of bounds array index",
+                      __FILE__, __LINE__);
+            if (j >= 0)
+            {
+                grid[j] = 1;
+                cnt++;
+            }
+        }
+        if (myusegrid)
+            free(myusegrid);
+
+        iodesc->holegridsize = thisgridsize[ios->io_rank] - cnt;
+        if (iodesc->holegridsize > 0)
+        {
+            /* Allocate space for the fillgrid. */
+            if (!(myfillgrid = malloc(iodesc->holegridsize * sizeof(PIO_Offset))))
+                return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        }
+
+        /* Initialize the fillgrid. */
+        for (i = 0; i < iodesc->holegridsize; i++)
+            myfillgrid[i] = -1;
+
+        j = 0;
+        for (i = 0; i < thisgridsize[ios->io_rank]; i++)
+        {
+            if (grid[i] == 0)
+            {
+                if (myfillgrid[j] == -1)
+                    myfillgrid[j++] = thisgridmin[ios->io_rank] + i;
+                else
+                    return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+            }
+        }
+        maxregions = 0;
+        iodesc->maxfillregions = 0;
+        if (myfillgrid)
+        {
+            /* Allocate a data region to hold fill values. */
+            if ((ret = alloc_region2(ios, iodesc->ndims, &iodesc->fillregion)))
+                return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+            if ((ret = get_regions(iodesc->ndims, gdimlen, iodesc->holegridsize, myfillgrid,
+                                   &iodesc->maxfillregions, iodesc->fillregion)))
+                return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+            free(myfillgrid);
+            maxregions = iodesc->maxfillregions;
+        }
+
+        /* Get the max maxregions, and distribute it to all tasks in
+         * the IO communicator. */
+        if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX,
+                                    ios->io_comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        iodesc->maxfillregions = maxregions;
+
+        /* Get the max maxholegridsize, and distribute it to all tasks
+         * in the IO communicator. */
+	iodesc->maxholegridsize = iodesc->holegridsize;
+        if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &(iodesc->maxholegridsize), 1, MPI_INT,
+                                    MPI_MAX, ios->io_comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Scatter values of srcindex to subset communicator. ??? */
+    if ((mpierr = MPI_Scatterv((void *)srcindex, recvcounts, rdispls, PIO_OFFSET,
+                               (void *)iodesc->sindex, iodesc->scount[0],  PIO_OFFSET,
+                               0, iodesc->subset_comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    if (ios->ioproc)
+    {
+        iodesc->maxregions = 0;
+        if ((ret = get_regions(iodesc->ndims, gdimlen, iodesc->llen, iomap,
+                               &iodesc->maxregions, iodesc->firstregion)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+        maxregions = iodesc->maxregions;
+
+        /* Get the max maxregions, and distribute it to all tasks in
+         * the IO communicator. */
+        if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &maxregions, 1, MPI_INT, MPI_MAX, ios->io_comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        iodesc->maxregions = maxregions;
+
+        /* Free resources. */
+        if (iomap)
+            free(iomap);
+
+        if (map)
+            free(map);
+
+        if (srcindex)
+            free(srcindex);
+
+        /* Compute the max io buffer size needed for an iodesc. */
+        if ((ret = compute_maxIObuffersize(ios->io_comm, iodesc)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+        iodesc->nrecvs = ntasks;
+    }
+
+    /* Using maxiobuflen compute the maximum number of vars of this type that the io
+       task buffer can handle. */
+    if ((ret = compute_maxaggregate_bytes(ios, iodesc)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Performance tuning rearranger.
+ *
+ * @param ios pointer to the iosystem description struct.
+ * @param iodesc pointer to the IO description struct.
+ * @returns 0 on success, error code otherwise.
+ */
+void performance_tune_rearranger(iosystem_desc_t *ios, io_desc_t *iodesc)
+{
+#ifdef TIMING
+#ifdef PERFTUNE
+    double *wall, usr[2], sys[2];
+    void *cbuf, *ibuf;
+    int tsize;
+    int myrank;
+    int mpierr; /* Return code for MPI calls. */
+
+    assert(iodesc);
+
+    if ((mpierr = MPI_Type_size(iodesc->basetype, &tsize)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    cbuf = NULL;
+    ibuf = NULL;
+    if (iodesc->ndof > 0)
+        if (!(cbuf = bget(iodesc->ndof * tsize)))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (iodesc->llen > 0)
+        if (!(ibuf = bget(iodesc->llen * tsize)))
+            return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if (iodesc->rearranger == PIO_REARR_BOX)
+        mycomm = ios->union_comm;
+    else
+        mycomm = iodesc->subset_comm;
+
+    if ((mpierr = MPI_Comm_size(mycomm, &nprocs)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_rank(mycomm, &myrank)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    int log2 = log(nprocs) / log(2) + 1;
+    if (!(wall = bget(2 * 4 * log2 * sizeof(double))))
+        return pio_err(ios, file, PIO_ENOMEM, __FILE__, __LINE__);
+    double mintime;
+    int k = 0;
+
+    if ((mpierr = MPI_Barrier(mycomm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    GPTLstamp(&wall[0], &usr[0], &sys[0]);
+    rearrange_comp2io(ios, iodesc, cbuf, ibuf, 1);
+    rearrange_io2comp(ios, iodesc, ibuf, cbuf);
+    GPTLstamp(&wall[1], &usr[1], &sys[1]);
+    mintime = wall[1]-wall[0];
+    if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, &mintime, 1, MPI_DOUBLE, MPI_MAX, mycomm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    handshake = iodesc->rearr_opts.comp2io.hs;
+    isend = iodesc->isend;
+    maxreqs = iodesc->max_requests;
+
+    for (int i = 0; i < 2; i++)
+    {
+        if (i == 0)
+            iodesc->rearr_opts.comp2io.hs = false;
+        else
+            iodesc->rearr_opts.comp2io.hs = true;
+
+        for (int j = 0; j < 2; j++)
+        {
+            if (j == 0)
+                iodesc->isend = false;
+            else
+                iodesc->isend = true;
+
+            iodesc->max_requests = 0;
+
+            for (nreqs = nprocs; nreqs >= 2; nreqs /= 2)
+            {
+                iodesc->max_requests = nreqs;
+                if ((mpierr = MPI_Barrier(mycomm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                GPTLstamp(wall, usr, sys);
+                rearrange_comp2io(ios, iodesc, cbuf, ibuf, 1);
+                rearrange_io2comp(ios, iodesc, ibuf, cbuf);
+                GPTLstamp(wall+1, usr, sys);
+                wall[1] -= wall[0];
+                if ((mpierr = MPI_Allreduce(MPI_IN_PLACE, wall + 1, 1, MPI_DOUBLE, MPI_MAX,
+                                            mycomm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+                if (wall[1] < mintime * 0.95)
+                {
+                    handshake = iodesc->rearr_opts.comp2io.hs;
+                    isend = iodesc->isend;
+                    maxreqs = nreqs;
+                    mintime = wall[1];
+                }
+                else if (wall[1] > mintime * 1.05)
+                {
+                    exit;
+                }
+            }
+        }
+    }
+
+    iodesc->rearr_opts.comp2io.hs = handshake;
+    iodesc->isend = isend;
+    iodesc->max_requests = maxreqs;
+
+    LOG((1, "spmd optimization: maxreqs: %d handshake:%d isend:%d mintime=%f\n",
+         maxreqs,handshake,isend,mintime));
+
+    /* Free memory. */
+    brel(wall);
+    brel(cbuf);
+    brel(ibuf);
+#endif
+#endif
+}
diff --git a/src/externals/pio2/src/clib/pio_spmd.c b/src/externals/pio2/src/clib/pio_spmd.c
new file mode 100644
index 00000000000..c42fd60eff9
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_spmd.c
@@ -0,0 +1,494 @@
+/**
+ * @file
+ * Algorithms modeled after spmd_utils in the Community
+ * Atmosphere Model; C translation. This includes MPI_Gather,
+ * MPI_Gatherv, and MPI_Alltoallw with flow control options.
+ *
+ * @author Jim Edwards
+ * @date 2014
+ */
+#include 
+#include 
+#include 
+
+/**
+ * Returns the smallest power of 2 greater than
+ * or equal to i.
+ *
+ * @param i input number
+ * @returns the smallest power of 2 greater than
+ * or equal to i.
+ */
+int ceil2(int i)
+{
+    int p = 1;
+    while (p < i)
+        p *= 2;
+
+    return(p);
+}
+
+/**
+ * Given integers p and k between 0 and np-1,
+ * if (p+1)^k <= np-1 then return (p+1)^k else -1.
+ *
+ * @param np
+ * @param p integer between 0 and np - 1.
+ * @param k integer between 0 and np - 1.
+ * @returns (p + 1) ^ k else -1.
+ */
+int pair(int np, int p, int k)
+{
+    int q = (p + 1) ^ k ;
+    int pair = (q > np - 1) ? -1 : q;
+    return pair;
+}
+
+/**
+ * Provides the functionality of MPI_Alltoallw with flow control
+ * options. Generalized all-to-all communication allowing different
+ * datatypes, counts, and displacements for each partner
+ *
+ * @param sendbuf starting address of send buffer
+ * @param sendcounts integer array equal to the number of tasks in
+ * communicator comm (ntasks). It specifies the number of elements to
+ * send to each processor
+ * @param sdispls integer array (of length ntasks). Entry j
+ * specifies the displacement in bytes (relative to sendbuf) from
+ * which to take the outgoing data destined for process j.
+ * @param sendtypes array of datatypes (of length ntasks). Entry j
+ * specifies the type of data to send to process j.
+ * @param recvbuf address of receive buffer.
+ * @param recvcounts integer array (of length ntasks) specifying the
+ * number of elements that can be received from each processor.
+ * @param rdispls integer array (of length ntasks). Entry i
+ * specifies the displacement in bytes (relative to recvbuf) at which
+ * to place the incoming data from process i.
+ * @param recvtypes array of datatypes (of length ntasks). Entry i
+ * specifies the type of data received from process i.
+ * @param comm MPI communicator for the MPI_Alltoallw call.
+ * @param fc pointer to the struct that provided flow control options.
+ * @returns 0 for success, error code otherwise.
+ */
+int pio_swapm(void *sendbuf, int *sendcounts, int *sdispls, MPI_Datatype *sendtypes,
+              void *recvbuf, int *recvcounts, int *rdispls, MPI_Datatype *recvtypes,
+              MPI_Comm comm, rearr_comm_fc_opt_t *fc)
+{
+    int ntasks;  /* Number of tasks in communicator comm. */
+    int my_rank; /* Rank of this task in comm. */
+    int tag;
+    int offset_t;
+    int steps;
+    int istep;
+    int rstep;
+    int p;
+    int maxreq;
+    int maxreqh;
+    int hs = 1; /* Used for handshaking. */
+    void *ptr;
+    MPI_Status status; /* Not actually used - replace with MPI_STATUSES_IGNORE. */
+    int mpierr;  /* Return code from MPI functions. */
+
+    LOG((2, "pio_swapm fc->hs = %d fc->isend = %d fc->max_pend_req = %d", fc->hs,
+         fc->isend, fc->max_pend_req));
+
+    /* Get my rank and size of communicator. */
+    if ((mpierr = MPI_Comm_size(comm, &ntasks)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_rank(comm, &my_rank)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    LOG((2, "ntasks = %d my_rank = %d", ntasks, my_rank));
+
+    /* Now we know the size of these arrays. */
+    int swapids[ntasks];
+    MPI_Request rcvids[ntasks];
+    MPI_Request sndids[ntasks];
+    MPI_Request hs_rcvids[ntasks];
+
+    /* Print some debugging info, if logging is enabled. */
+#if PIO_ENABLE_LOGGING
+    {
+        for (int p = 0; p < ntasks; p++)
+            LOG((3, "sendcounts[%d] = %d sdispls[%d] = %d sendtypes[%d] = %d recvcounts[%d] = %d "
+                 "rdispls[%d] = %d recvtypes[%d] = %d", p, sendcounts[p], p, sdispls[p], p,
+                 sendtypes[p], p, recvcounts[p], p, rdispls[p], p, recvtypes[p]));
+    }
+#endif /* PIO_ENABLE_LOGGING */
+
+    /* If fc->max_pend_req == 0 no throttling is requested and the default
+     * mpi_alltoallw function is used. */
+    if (fc->max_pend_req == 0)
+    {
+        /* Call the MPI alltoall without flow control. */
+        LOG((3, "Calling MPI_Alltoallw without flow control."));
+        if ((mpierr = MPI_Alltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf,
+                                    recvcounts, rdispls, recvtypes, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        return PIO_NOERR;
+    }
+
+    /* an index for communications tags */
+    offset_t = ntasks;
+
+    /* Send to self. */
+    if (sendcounts[my_rank] > 0)
+    {
+        void *sptr, *rptr;
+        tag = my_rank + offset_t;
+        sptr = (char *)sendbuf + sdispls[my_rank];
+        rptr = (char *)recvbuf + rdispls[my_rank];
+
+        /*
+          MPI_Type_get_extent(sendtypes[my_rank], &lb, &extent);
+          printf("%s %d %d %d\n",__FILE__,__LINE__,extent, lb);
+          MPI_Type_get_extent(recvtypes[my_rank], &lb, &extent);
+          printf("%s %d %d %d\n",__FILE__,__LINE__,extent, lb);
+        */
+
+#ifdef ONEWAY
+        /* If ONEWAY is true we will post mpi_sendrecv comms instead
+         * of irecv/send. */
+        if ((mpierr = MPI_Sendrecv(sptr, sendcounts[my_rank],sendtypes[my_rank],
+                                   my_rank, tag, rptr, recvcounts[my_rank], recvtypes[my_rank],
+                                   my_rank, tag, comm, &status)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#else
+        if ((mpierr = MPI_Irecv(rptr, recvcounts[my_rank], recvtypes[my_rank],
+                                my_rank, tag, comm, rcvids)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Send(sptr, sendcounts[my_rank], sendtypes[my_rank],
+                               my_rank, tag, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+        if ((mpierr = MPI_Wait(rcvids, &status)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#endif
+    }
+
+    LOG((2, "Done sending to self... sending to other procs"));
+
+    /* When send to self is complete there is nothing left to do if
+     * ntasks==1. */
+    if (ntasks == 1)
+        return PIO_NOERR;
+
+    for (int i = 0; i < ntasks; i++)
+    {
+        rcvids[i] = MPI_REQUEST_NULL;
+        swapids[i] = 0;
+    }
+
+    if (fc->isend)
+        for (int i = 0; i < ntasks; i++)
+            sndids[i] = MPI_REQUEST_NULL;
+
+    if (fc->hs)
+        for (int i = 0; i < ntasks; i++)
+            hs_rcvids[i] = MPI_REQUEST_NULL;
+
+    steps = 0;
+    for (istep = 0; istep < ceil2(ntasks) - 1; istep++)
+    {
+        p = pair(ntasks, istep, my_rank);
+        if (p >= 0 && (sendcounts[p] > 0 || recvcounts[p] > 0))
+            swapids[steps++] = p;
+    }
+
+    LOG((3, "steps=%d", steps));
+
+    if (steps == 0)
+        return PIO_NOERR;
+
+    if (steps == 1)
+    {
+        maxreq = 1;
+        maxreqh = 1;
+    }
+    else
+    {
+        if (fc->max_pend_req == PIO_REARR_COMM_UNLIMITED_PEND_REQ)
+        {
+            maxreq = steps;
+            maxreqh = steps;
+        }
+        else if (fc->max_pend_req > 1 && fc->max_pend_req < steps)
+        {
+            maxreq = fc->max_pend_req;
+            maxreqh = maxreq / 2;
+        }
+        else if (fc->max_pend_req == 1)
+        {
+            /* Note that steps >= 2 here */
+            maxreq = 2;
+            maxreqh = 1;
+        }
+        else
+        {
+            maxreq = steps;
+            maxreqh = steps;
+        }
+    }
+
+    LOG((2, "fc->max_pend_req=%d, maxreq=%d, maxreqh=%d", fc->max_pend_req, maxreq, maxreqh));
+
+    /* If handshaking is in use, do a nonblocking recieve to listen
+     * for it. */
+    if (fc->hs)
+    {
+        for (istep = 0; istep < maxreq; istep++)
+        {
+            p = swapids[istep];
+            if (sendcounts[p] > 0)
+            {
+                tag = my_rank + offset_t;
+                if ((mpierr = MPI_Irecv(&hs, 1, MPI_INT, p, tag, comm, hs_rcvids + istep)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            }
+        }
+    }
+
+    /* Post up to maxreq irecv's. */
+    for (istep = 0; istep < maxreq; istep++)
+    {
+        p = swapids[istep];
+        if (recvcounts[p] > 0)
+        {
+            tag = p + offset_t;
+            ptr = (char *)recvbuf + rdispls[p];
+
+            if ((mpierr = MPI_Irecv(ptr, recvcounts[p], recvtypes[p], p, tag, comm,
+                                    rcvids + istep)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+            if (fc->hs)
+                if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, tag, comm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        }
+    }
+
+    /* Tell the paired task that this tasks' has posted it's irecvs'. */
+    rstep = maxreq;
+    for (istep = 0; istep < steps; istep++)
+    {
+        p = swapids[istep];
+        if (sendcounts[p] > 0)
+        {
+            tag = my_rank + offset_t;
+            /* If handshake is enabled don't post sends until the
+             * receiving task has posted recvs. */
+            if (fc->hs)
+            {
+                if ((mpierr = MPI_Wait(hs_rcvids + istep, &status)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                hs_rcvids[istep] = MPI_REQUEST_NULL;
+            }
+            ptr = (char *)sendbuf + sdispls[p];
+
+            /* On some software stacks MPI_Irsend() is either not available, not
+             * a major issue anymore, or is buggy. With PIO1 we have found that
+             * although the code correctly posts receives before the irsends,
+             * on some systems (software stacks) the code hangs. However the
+             * code works fine with isends. The USE_MPI_ISEND_FOR_FC macro should be
+             * used to choose between mpi_irsends and mpi_isends - the default
+             * is still mpi_irsend
+             */
+            if (fc->hs && fc->isend)
+            {
+#ifdef USE_MPI_ISEND_FOR_FC
+                if ((mpierr = MPI_Isend(ptr, sendcounts[p], sendtypes[p], p, tag, comm,
+                                        sndids + istep)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#else
+                if ((mpierr = MPI_Irsend(ptr, sendcounts[p], sendtypes[p], p, tag, comm,
+                                         sndids + istep)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+#endif
+            }
+            else if (fc->isend)
+            {
+                if ((mpierr = MPI_Isend(ptr, sendcounts[p], sendtypes[p], p, tag, comm,
+                                         sndids + istep)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            }
+            else
+            {
+                if ((mpierr = MPI_Send(ptr, sendcounts[p], sendtypes[p], p, tag, comm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            }
+        }
+
+        /* We did comms in sets of size max_reqs, if istep > maxreqh-1
+         * then there is a remainder that must be handled. */
+        if (istep > maxreqh - 1)
+        {
+            p = istep - maxreqh;
+            if (rcvids[p] != MPI_REQUEST_NULL)
+            {
+                if ((mpierr = MPI_Wait(rcvids + p, &status)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                rcvids[p] = MPI_REQUEST_NULL;
+            }
+            if (rstep < steps)
+            {
+                p = swapids[rstep];
+                if (fc->hs && sendcounts[p] > 0)
+                {
+                    tag = my_rank + offset_t;
+                    if ((mpierr = MPI_Irecv(&hs, 1, MPI_INT, p, tag, comm, hs_rcvids+rstep)))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                }
+                if (recvcounts[p] > 0)
+                {
+                    tag = p + offset_t;
+
+                    ptr = (char *)recvbuf + rdispls[p];
+                    if ((mpierr = MPI_Irecv(ptr, recvcounts[p], recvtypes[p], p, tag, comm, rcvids + rstep)))
+                        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    if (fc->hs)
+                        if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, tag, comm)))
+                            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                }
+                rstep++;
+            }
+        }
+    }
+
+    /* If steps > 0 there could still be outstanding messages, wait for
+     * them here. */
+    if (steps > 0)
+    {
+        LOG((2, "Waiting for outstanding msgs"));
+        if ((mpierr = MPI_Waitall(steps, rcvids, MPI_STATUSES_IGNORE)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if (fc->isend)
+            if ((mpierr = MPI_Waitall(steps, sndids, MPI_STATUSES_IGNORE)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Provides the functionality of MPI_Gatherv with flow control
+ * options. This function is not currently used, but we hope it will
+ * be useful in future optimizations.
+ *
+ * @param sendbuf starting address of send buffer.
+ * @param sendcnt number of elements in send buffer.
+ * @param sendtype data type of send buffer elements.
+ * @param recvbuf address of receive buffer.
+ * @param recvcnts integer array (of length group size) containing the
+ * number of elements that are received from each process (significant
+ * only at root).
+ * @param displs integer array (of length group size). Entry i
+ * specifies the displacement relative to recvbuf at which to place
+ * the incoming data from process i (significant only at root).
+ * @param recvtype data type of recv buffer elements (significant only
+ * at root).
+ * @param root rank of receiving process.
+ * @param comm communicator.
+ * @param flow_cntl if non-zero, flow control will be used.
+ * @returns 0 for success, error code otherwise.
+ */
+int pio_fc_gatherv(const void *sendbuf, int sendcnt, MPI_Datatype sendtype,
+                   void *recvbuf, const int *recvcnts, const int *displs,
+                   MPI_Datatype recvtype, int root, MPI_Comm comm, int flow_cntl)
+{
+    bool fc_gather;
+    int gather_block_size;
+    int mytask, nprocs;
+    int mtag;
+    MPI_Status status;
+    int hs;
+    int dsize;
+    int mpierr;  /* Return code from MPI functions. */
+
+    if (flow_cntl > 0)
+    {
+        fc_gather = true;
+        gather_block_size = min(flow_cntl, MAX_GATHER_BLOCK_SIZE);
+    }
+    else
+    {
+        fc_gather = false;
+    }
+
+    if (fc_gather)
+    {
+        if ((mpierr = MPI_Comm_rank(comm, &mytask)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Comm_size(comm, &nprocs)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+        mtag = 2 * nprocs;
+        hs = 1;
+
+        if (mytask == root)
+        {
+            int preposts = min(nprocs-1, gather_block_size);
+            int head = 0;
+            int count = 0;
+            int tail = 0;
+            MPI_Request rcvid[gather_block_size];
+
+            if ((mpierr = MPI_Type_size(recvtype, &dsize)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+            for (int p = 0; p < nprocs; p++)
+            {
+                if (p != root)
+                {
+                    if (recvcnts[p] > 0)
+                    {
+                        count++;
+                        if (count > preposts)
+                        {
+                            if ((mpierr = MPI_Wait(rcvid + tail, &status)))
+                                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                            tail = (tail + 1) % preposts;
+                        }
+
+                        void *ptr = (void *)((char *)recvbuf + dsize * displs[p]);
+
+                        if ((mpierr = MPI_Irecv(ptr, recvcnts[p], recvtype, p, mtag, comm, rcvid + head)))
+                            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                        head = (head + 1) % preposts;
+                        if ((mpierr = MPI_Send(&hs, 1, MPI_INT, p, mtag, comm)))
+                            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                    }
+                }
+            }
+
+            /* copy local data */
+            if ((mpierr = MPI_Type_size(sendtype, &dsize)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            if ((mpierr = MPI_Sendrecv(sendbuf, sendcnt, sendtype, mytask, 102, recvbuf, recvcnts[mytask],
+                                       recvtype, mytask, 102, comm, &status)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+            count = min(count, preposts);
+            if (count > 0)
+                if ((mpierr = MPI_Waitall(count, rcvid, MPI_STATUSES_IGNORE)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        }
+        else
+        {
+            if (sendcnt > 0)
+            {
+                if ((mpierr = MPI_Recv(&hs, 1, MPI_INT, root, mtag, comm, &status)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Send(sendbuf, sendcnt, sendtype, root, mtag, comm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            }
+        }
+    }
+    else
+    {
+        if ((mpierr = MPI_Gatherv(sendbuf, sendcnt, sendtype, recvbuf, recvcnts,
+                                  displs, recvtype, root, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pio_varm.c b/src/externals/pio2/src/clib/pio_varm.c
new file mode 100644
index 00000000000..f02cb196c91
--- /dev/null
+++ b/src/externals/pio2/src/clib/pio_varm.c
@@ -0,0 +1,1811 @@
+#include 
+#include 
+#include 
+
+///
+/// PIO interface to nc_put_varm
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const void *buf, PIO_Offset bufcount, MPI_Datatype buftype)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_uchar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned char *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_uchar(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_short
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const short *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_short(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+///
+/// PIO interface to nc_put_varm_text
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const char *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_text(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_ushort
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned short *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_ushort(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_ulonglong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned long long *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_ulonglong(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+///
+/// PIO interface to nc_put_varm_int
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const int *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_int(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_float
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const float *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_float(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+///
+/// PIO interface to nc_put_varm_long
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_long(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_uint
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const unsigned int *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_uint(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_double
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const double *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_double(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+///
+/// PIO interface to nc_put_varm_schar
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const signed char *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_schar(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+///
+/// PIO interface to nc_put_varm_longlong
+///
+/// This routine is called collectively by all tasks in the communicator ios.union_comm.
+///
+/// Refer to the  netcdf documentation. 
+///
+int PIOc_put_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], const long long *op)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    var_desc_t *vdesc;
+    int *request;
+
+    ierr = PIO_NOERR;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_var_par_access(file->fh, varid, NC_COLLECTIVE);
+            ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank==0){
+                ierr = nc_put_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap, op);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            vdesc = file->varlist + varid;
+
+            if (vdesc->nreqs%PIO_REQUEST_ALLOC_CHUNK == 0 ){
+                vdesc->request = realloc(vdesc->request,
+                                         sizeof(int)*(vdesc->nreqs+PIO_REQUEST_ALLOC_CHUNK));
+            }
+            request = vdesc->request+vdesc->nreqs;
+
+            if (ios->io_rank==0){
+                ierr = ncmpi_bput_varm_longlong(file->fh, varid, start, count, stride, imap, op, request);;
+            }else{
+                *request = PIO_REQ_NULL;
+            }
+            vdesc->nreqs++;
+            flush_output_buffer(file, false, 0);
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    return ierr;
+}
+
+int PIOc_get_varm_uchar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned char *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_UNSIGNED_CHAR;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_uchar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_uchar(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_uchar_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_schar (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], signed char *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_CHAR;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_schar(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_schar(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_schar_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_double (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], double *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_DOUBLE;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_double(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_double(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_double_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_text (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], char *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_CHAR;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_text(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_text(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_text_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_int (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], int *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_INT;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_int(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_int(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_int_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_uint (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned int *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_UNSIGNED;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_uint(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_uint(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_uint_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], void *buf, PIO_Offset bufcount, MPI_Datatype buftype)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibufcnt = bufcount;
+    ibuftype = buftype;
+    ierr = PIO_NOERR;
+
+    /* Sorry, but varm functions are not supported by the async interface. */
+    if (ios->async)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,   buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_all(file->fh, varid, start, count, stride, imap, buf, bufcount, buftype);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_float (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], float *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_FLOAT;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_float(file->fh, varid,(size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_float(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_float_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_long (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_LONG;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_long(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_long(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_long_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_ushort (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned short *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_UNSIGNED_SHORT;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_ushort(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_ushort(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_ushort_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_longlong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], long long *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_LONG_LONG;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_longlong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_longlong(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_longlong_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_short (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], short *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_SHORT;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_short(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_short(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_short_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
+
+int PIOc_get_varm_ulonglong (int ncid, int varid, const PIO_Offset start[], const PIO_Offset count[], const PIO_Offset stride[], const PIO_Offset imap[], unsigned long long *buf)
+{
+    int ierr;
+    iosystem_desc_t *ios;
+    file_desc_t *file;
+    MPI_Datatype ibuftype;
+    int ndims;
+    int ibufcnt;
+    bool bcast = false;
+
+    /* Get file info. */
+    if ((ierr = pio_get_file(ncid, &file)))
+        return ierr;
+    ios = file->iosystem;
+    ibuftype = MPI_UNSIGNED_LONG_LONG;
+    ierr = PIOc_inq_varndims(ncid, varid, &ndims);
+    ibufcnt = 1;
+    for(int i=0;iasync)
+        return PIO_EINVAL;
+
+    if (ios->ioproc){
+        switch(file->iotype){
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+#endif
+        case PIO_IOTYPE_NETCDF:
+            bcast = true;
+            if (ios->iomaster == MPI_ROOT){
+                ierr = nc_get_varm_ulonglong(file->fh, varid, (size_t *) start, (size_t *) count, (ptrdiff_t *) stride, (ptrdiff_t *) imap,  buf);;
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+#ifdef PNET_READ_AND_BCAST
+            ncmpi_begin_indep_data(file->fh);
+            if (ios->iomaster == MPI_ROOT){
+                ierr = ncmpi_get_varm_ulonglong(file->fh, varid, start, count, stride, imap,  buf);;
+            };
+            ncmpi_end_indep_data(file->fh);
+            bcast=true;
+#else
+            ierr = ncmpi_get_varm_ulonglong_all(file->fh, varid, start, count, stride, imap,  buf);;
+#endif
+            break;
+#endif
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+    }
+
+    ierr = check_netcdf(file, ierr, __FILE__,__LINE__);
+
+    if (ios->async || bcast ||
+        (ios->num_iotasks < ios->num_comptasks)){
+        MPI_Bcast(buf, ibufcnt, ibuftype, ios->ioroot, ios->my_comm);
+    }
+
+    return ierr;
+}
diff --git a/src/externals/pio2/src/clib/pioc.c b/src/externals/pio2/src/clib/pioc.c
new file mode 100644
index 00000000000..0dc4fed89fd
--- /dev/null
+++ b/src/externals/pio2/src/clib/pioc.c
@@ -0,0 +1,1541 @@
+/**
+ * @file
+ * Some initialization and support functions.
+ * @author Jim Edwards
+ * @date  2014
+ *
+ * @see http://code.google.com/p/parallelio/
+ */
+
+#include 
+#include 
+#include 
+
+/** The default error handler used when iosystem cannot be located. */
+int default_error_handler = PIO_INTERNAL_ERROR;
+
+/** The target blocksize for each io task when the box rearranger is
+ * used (see pio_sc.c). */
+extern int blocksize;
+
+/**
+ * Check to see if PIO has been initialized.
+ *
+ * @param iosysid the IO system ID
+ * @param active pointer that gets true if IO system is active, false
+ * otherwise.
+ * @returns 0 on success, error code otherwise
+ */
+int PIOc_iosystem_is_active(int iosysid, bool *active)
+{
+    iosystem_desc_t *ios;
+
+    /* Get the ios if there is one. */
+    ios = pio_get_iosystem_from_id(iosysid);
+
+    if (active)
+    {
+        if (!ios || (ios->comp_comm == MPI_COMM_NULL && ios->io_comm == MPI_COMM_NULL))
+            *active = false;
+        else
+            *active = true;
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Check to see if PIO file is open.
+ *
+ * @param ncid the ncid of an open file
+ * @returns 1 if file is open, 0 otherwise.
+ */
+int PIOc_File_is_Open(int ncid)
+{
+    file_desc_t *file;
+
+    /* If get file returns non-zero, then this file is not open. */
+    if (pio_get_file(ncid, &file))
+        return 0;
+    else
+        return 1;
+}
+
+/**
+ * Set the error handling method to be used for subsequent pio library
+ * calls, returns the previous method setting. Note that this changes
+ * error handling for the IO system that was used when this file was
+ * opened. Other files opened with the same IO system will also he
+ * affected by this call. This function is supported but
+ * deprecated. New code should use PIOc_set_iosystem_error_handling().
+ * This method has no way to return an error, so any failure will
+ * result in MPI_Abort.
+ *
+ * @param ncid the ncid of an open file
+ * @param method the error handling method
+ * @returns old error handler
+ * @ingroup PIO_error_method
+ */
+int PIOc_Set_File_Error_Handling(int ncid, int method)
+{
+    file_desc_t *file;
+    int oldmethod;
+
+    /* Get the file info. */
+    if (pio_get_file(ncid, &file))
+        piodie("Could not find file", __FILE__, __LINE__);
+
+    /* Check that valid error handler was provided. */
+    if (method != PIO_INTERNAL_ERROR && method != PIO_BCAST_ERROR &&
+        method != PIO_RETURN_ERROR)
+        piodie("Invalid error hanlder method", __FILE__, __LINE__);
+
+    /* Get the old method. */
+    oldmethod = file->iosystem->error_handler;
+
+    /* Set the error hanlder. */
+    file->iosystem->error_handler = method;
+
+    return oldmethod;
+}
+
+/**
+ * Increment the unlimited dimension of the given variable.
+ *
+ * @param ncid the ncid of the open file
+ * @param varid the variable ID
+ * @returns 0 on success, error code otherwise
+ */
+int PIOc_advanceframe(int ncid, int varid)
+{
+    file_desc_t *file;
+    int ret;
+
+    /* Get the file info. */
+    if ((ret = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+
+    /* Check inputs. */
+    if (varid < 0 || varid >= PIO_MAX_VARS)
+        return pio_err(NULL, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    file->varlist[varid].record++;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set the unlimited dimension of the given variable
+ *
+ * @param ncid the ncid of the file.
+ * @param varid the varid of the variable
+ * @param frame the value of the unlimited dimension.  In c 0 for the
+ * first record, 1 for the second
+ * @return PIO_NOERR for no error, or error code.
+ * @ingroup PIO_setframe
+ */
+int PIOc_setframe(int ncid, int varid, int frame)
+{
+    file_desc_t *file;
+    int ret;
+
+    /* Get file info. */
+    if ((ret = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+
+    /* Check inputs. */
+    if (varid < 0 || varid >= PIO_MAX_VARS)
+        return pio_err(NULL, file, PIO_EINVAL, __FILE__, __LINE__);
+
+    file->varlist[varid].record = frame;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get the number of IO tasks set.
+ *
+ * @param iosysid the IO system ID
+ * @param numiotasks a pointer taht gets the number of IO
+ * tasks. Ignored if NULL.
+ * @returns 0 on success, error code otherwise
+ */
+int PIOc_get_numiotasks(int iosysid, int *numiotasks)
+{
+    iosystem_desc_t *ios;
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    if (numiotasks)
+        *numiotasks = ios->num_iotasks;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Get the local size of the variable.
+ *
+ * @param ioid IO descrption ID.
+ * @returns the size of the array.
+ */
+int PIOc_get_local_array_size(int ioid)
+{
+    io_desc_t *iodesc;
+
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        piodie("Could not get iodesc", __FILE__, __LINE__);
+
+    return iodesc->ndof;
+}
+
+/**
+ * Set the error handling method used for subsequent calls. This
+ * function is deprecated. New code should use
+ * PIOc_set_iosystem_error_handling(). This method has no way to
+ * return an error, so any failure will result in MPI_Abort.
+ *
+ * @param iosysid the IO system ID
+ * @param method the error handling method
+ * @returns old error handler
+ * @ingroup PIO_error_method
+ */
+int PIOc_Set_IOSystem_Error_Handling(int iosysid, int method)
+{
+    iosystem_desc_t *ios;
+    int oldmethod;
+
+    /* Get the iosystem info. */
+    if (iosysid != PIO_DEFAULT)
+        if (!(ios = pio_get_iosystem_from_id(iosysid)))
+            piodie("Could not find IO system.", __FILE__, __LINE__);
+
+    /* Set the error handler. */
+    if (PIOc_set_iosystem_error_handling(iosysid, method, &oldmethod))
+        piodie("Could not set the IOSystem error hanlder", __FILE__, __LINE__);
+
+    return oldmethod;
+}
+
+/**
+ * Set the error handling method used for subsequent calls for this IO
+ * system.
+ *
+ * @param iosysid the IO system ID. Passing PIO_DEFAULT instead
+ * changes the default error handling for the library.
+ * @param method the error handling method
+ * @param old_method pointer to int that will get old method. Ignored
+ * if NULL.
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_error_method
+ */
+int PIOc_set_iosystem_error_handling(int iosysid, int method, int *old_method)
+{
+    iosystem_desc_t *ios = NULL;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+
+    LOG((1, "PIOc_set_iosystem_error_handling iosysid = %d method = %d", iosysid,
+         method));
+
+    /* Find info about this iosystem. */
+    if (iosysid != PIO_DEFAULT)
+        if (!(ios = pio_get_iosystem_from_id(iosysid)))
+            return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check that valid error handler was provided. */
+    if (method != PIO_INTERNAL_ERROR && method != PIO_BCAST_ERROR &&
+        method != PIO_RETURN_ERROR)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* If using async, and not an IO task, then send parameters. */
+    if (iosysid != PIO_DEFAULT)
+        if (ios->async)
+        {
+            if (!ios->ioproc)
+            {
+                int msg = PIO_MSG_SETERRORHANDLING;
+                char old_method_present = old_method ? true : false;
+
+                if (ios->compmaster == MPI_ROOT)
+                    mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+                if (!mpierr)
+                    mpierr = MPI_Bcast(&method, 1, MPI_INT, ios->compmaster, ios->intercomm);
+                if (!mpierr)
+                    mpierr = MPI_Bcast(&old_method_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            }
+
+            /* Handle MPI errors. */
+            if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+                check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+            if (mpierr)
+                return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        }
+
+    /* Return the current handler. */
+    if (old_method)
+        *old_method = iosysid == PIO_DEFAULT ? default_error_handler : ios->error_handler;
+
+    /* Set new error handler. */
+    if (iosysid == PIO_DEFAULT)
+        default_error_handler = method;
+    else
+        ios->error_handler = method;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Initialize the decomposition used with distributed arrays. The
+ * decomposition describes how the data will be distributed between
+ * tasks.
+ *
+ * Internally, this function will:
+ * 
      
+ * 
    • Allocate and initialize an iodesc struct for this
+ * decomposition. (This also allocates an io_region struct for the
+ * first region.)
+ * 
    • (Box rearranger only) If iostart or iocount are NULL, call
+ * CalcStartandCount() to determine starts/counts. Then call
+ * compute_maxIObuffersize() to compute the max IO buffer size needed.
+ * 
    • Create the rearranger.
+ * 
    • Assign an ioid and add this decomposition to the list of open
+ * decompositions.
+ * 
    
+ *
+ * @param iosysid the IO system ID.
+ * @param pio_type the basic PIO data type used.
+ * @param ndims the number of dimensions in the variable, not
+ * including the unlimited dimension.
+ * @param gdimlen an array length ndims with the sizes of the global
+ * dimensions.
+ * @param maplen the local length of the compmap array.
+ * @param compmap a 1 based array of offsets into the array record on
+ * file. A 0 in this array indicates a value which should not be
+ * transfered.
+ * @param ioidp pointer that will get the io description ID.
+ * @param rearranger pointer to the rearranger to be used for this
+ * decomp or NULL to use the default.
+ * @param iostart An array of start values for block cyclic
+ * decompositions for the SUBSET rearranger. Ignored if block
+ * rearranger is used. If NULL and SUBSET rearranger is used, the
+ * iostarts are generated.
+ * @param iocount An array of count values for block cyclic
+ * decompositions for the SUBSET rearranger. Ignored if block
+ * rearranger is used. If NULL and SUBSET rearranger is used, the
+ * iostarts are generated.
+ * @returns 0 on success, error code otherwise
+ * @ingroup PIO_initdecomp
+ */
+int PIOc_InitDecomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
+                    const PIO_Offset *compmap, int *ioidp, const int *rearranger,
+                    const PIO_Offset *iostart, const PIO_Offset *iocount)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    io_desc_t *iodesc;     /* The IO description. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function calls. */
+    int ierr;              /* Return code. */
+
+    LOG((1, "PIOc_InitDecomp iosysid = %d pio_type = %d ndims = %d maplen = %d",
+         iosysid, pio_type, ndims, maplen));
+
+    /* Get IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Caller must provide these. */
+    if (!gdimlen || !compmap || !ioidp)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Check the dim lengths. */
+    for (int i = 0; i < ndims; i++)
+        if (gdimlen[i] <= 0)
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_INITDECOMP_DOF; /* Message for async notification. */
+            char rearranger_present = rearranger ? true : false;
+            char iostart_present = iostart ? true : false;
+            char iocount_present = iocount ? true : false;
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&pio_type, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ndims, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((int *)gdimlen, ndims, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&maplen, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((PIO_Offset *)compmap, maplen, MPI_OFFSET, ios->compmaster, ios->intercomm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&rearranger_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (rearranger_present && !mpierr)
+                mpierr = MPI_Bcast((int *)rearranger, 1, MPI_INT, ios->compmaster, ios->intercomm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iostart_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (iostart_present && !mpierr)
+                mpierr = MPI_Bcast((PIO_Offset *)iostart, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iocount_present, 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (iocount_present && !mpierr)
+                mpierr = MPI_Bcast((PIO_Offset *)iocount, ndims, MPI_OFFSET, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_InitDecomp iosysid = %d pio_type = %d ndims = %d maplen = %d rearranger_present = %d iostart_present = %d "
+                 "iocount_present = %d ", iosysid, pio_type, ndims, maplen, rearranger_present, iostart_present, iocount_present));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Allocate space for the iodesc info. This also allocates the
+     * first region and copies the rearranger opts into this
+     * iodesc. */
+    if ((ierr = malloc_iodesc(ios, pio_type, ndims, &iodesc)))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+
+    /* Remember the maplen. */
+    iodesc->maplen = maplen;
+
+    /* Remember the map. */
+    if (!(iodesc->map = malloc(sizeof(PIO_Offset) * maplen)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int m = 0; m < maplen; m++)
+        iodesc->map[m] = compmap[m];
+
+    /* Remember the dim sizes. */
+    if (!(iodesc->dimlen = malloc(sizeof(int) * ndims)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int d = 0; d < ndims; d++)
+        iodesc->dimlen[d] = gdimlen[d];
+
+    /* Set the rearranger. */
+    if (!rearranger)
+        iodesc->rearranger = ios->default_rearranger;
+    else
+        iodesc->rearranger = *rearranger;
+    LOG((2, "iodesc->rearranger = %d", iodesc->rearranger));
+
+    /* Is this the subset rearranger? */
+    if (iodesc->rearranger == PIO_REARR_SUBSET)
+    {
+        iodesc->num_aiotasks = ios->num_iotasks;
+        LOG((2, "creating subset rearranger iodesc->num_aiotasks = %d",
+             iodesc->num_aiotasks));
+        if ((ierr = subset_rearrange_create(ios, maplen, (PIO_Offset *)compmap, gdimlen,
+                                            ndims, iodesc)))
+            return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+    }
+    else /* box rearranger */
+    {
+        if (ios->ioproc)
+        {
+            /*  Unless the user specifies the start and count for each
+             *  IO task compute it. */
+            if (iostart && iocount)
+            {
+                LOG((3, "iostart and iocount provided"));
+                for (int i = 0; i < ndims; i++)
+                {
+                    iodesc->firstregion->start[i] = iostart[i];
+                    iodesc->firstregion->count[i] = iocount[i];
+                }
+                iodesc->num_aiotasks = ios->num_iotasks;
+            }
+            else
+            {
+                /* Compute start and count values for each io task. */
+                LOG((2, "about to call CalcStartandCount pio_type = %d ndims = %d", pio_type, ndims));
+                if ((ierr = CalcStartandCount(pio_type, ndims, gdimlen, ios->num_iotasks,
+                                             ios->io_rank, iodesc->firstregion->start,
+                                             iodesc->firstregion->count, &iodesc->num_aiotasks)))
+                    return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+            }
+
+            /* Compute the max io buffer size needed for an iodesc. */
+            if ((ierr = compute_maxIObuffersize(ios->io_comm, iodesc)))
+                return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+            LOG((3, "compute_maxIObuffersize called iodesc->maxiobuflen = %d",
+                 iodesc->maxiobuflen));
+        }
+
+        /* Depending on array size and io-blocksize the actual number
+         * of io tasks used may vary. */
+        if ((mpierr = MPI_Bcast(&(iodesc->num_aiotasks), 1, MPI_INT, ios->ioroot,
+                                ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((3, "iodesc->num_aiotasks = %d", iodesc->num_aiotasks));
+
+        /* Compute the communications pattern for this decomposition. */
+        if (iodesc->rearranger == PIO_REARR_BOX)
+            if ((ierr = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc)))
+                return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+    }
+
+    /* Add this IO description to the list. */
+    *ioidp = pio_add_to_iodesc_list(iodesc);
+
+#if PIO_ENABLE_LOGGING
+    /* Log results. */
+    LOG((2, "iodesc ioid = %d nrecvs = %d ndof = %d ndims = %d num_aiotasks = %d "
+         "rearranger = %d maxregions = %d needsfill = %d llen = %d maxiobuflen  = %d",
+         iodesc->ioid, iodesc->nrecvs, iodesc->ndof, iodesc->ndims, iodesc->num_aiotasks,
+         iodesc->rearranger, iodesc->maxregions, iodesc->needsfill, iodesc->llen,
+         iodesc->maxiobuflen));
+    for (int j = 0; j < iodesc->llen; j++)
+        LOG((3, "rindex[%d] = %lld", j, iodesc->rindex[j]));
+#endif /* PIO_ENABLE_LOGGING */            
+
+    /* This function only does something if pre-processor macro
+     * PERFTUNE is set. */
+    performance_tune_rearranger(ios, iodesc);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Initialize the decomposition used with distributed arrays. The
+ * decomposition describes how the data will be distributed between
+ * tasks.
+ *
+ * @param iosysid the IO system ID.
+ * @param pio_type the basic PIO data type used.
+ * @param ndims the number of dimensions in the variable, not
+ * including the unlimited dimension.
+ * @param gdimlen an array length ndims with the sizes of the global
+ * dimensions.
+ * @param maplen the local length of the compmap array.
+ * @param compmap a 0 based array of offsets into the array record on
+ * file. A -1 in this array indicates a value which should not be
+ * transfered.
+ * @param ioidp pointer that will get the io description ID.
+ * @param rearranger the rearranger to be used for this decomp or 0 to
+ * use the default. Valid rearrangers are PIO_REARR_BOX and
+ * PIO_REARR_SUBSET.
+ * @param iostart An array of start values for block cyclic
+ * decompositions. If NULL ???
+ * @param iocount An array of count values for block cyclic
+ * decompositions. If NULL ???
+ * @returns 0 on success, error code otherwise
+ * @ingroup PIO_initdecomp
+ */
+int PIOc_init_decomp(int iosysid, int pio_type, int ndims, const int *gdimlen, int maplen,
+                     const PIO_Offset *compmap, int *ioidp, int rearranger,
+                     const PIO_Offset *iostart, const PIO_Offset *iocount)
+{
+    PIO_Offset compmap_1_based[maplen];
+    int *rearrangerp = NULL;
+
+    LOG((1, "PIOc_init_decomp iosysid = %d pio_type = %d ndims = %d maplen = %d",
+         iosysid, pio_type, ndims, maplen));
+
+    /* If the user specified a non-default rearranger, use it. */
+    if (rearranger)
+        rearrangerp = &rearranger;
+
+    /* Add 1 to all elements in compmap. */
+    for (int e = 0; e < maplen; e++)
+    {
+        LOG((3, "zero-based compmap[%d] = %d", e, compmap[e]));
+        compmap_1_based[e] = compmap[e] + 1;
+    }
+
+    /* Call the legacy version of the function. */
+    return PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap_1_based,
+                           ioidp, rearrangerp, iostart, iocount);
+}
+
+/**
+ * This is a simplified initdecomp which can be used if the memory
+ * order of the data can be expressed in terms of start and count on
+ * the file. In this case we compute the compdof.
+ *
+ * @param iosysid the IO system ID
+ * @param pio_type
+ * @param ndims the number of dimensions
+ * @param gdimlen an array length ndims with the sizes of the global
+ * dimensions.
+ * @param start start array
+ * @param count count array
+ * @param pointer that gets the IO ID.
+ * @returns 0 for success, error code otherwise
+ * @ingroup PIO_initdecomp
+ */
+int PIOc_InitDecomp_bc(int iosysid, int pio_type, int ndims, const int *gdimlen,
+                       const long int *start, const long int *count, int *ioidp)
+
+{
+    iosystem_desc_t *ios;
+    int n, i, maplen = 1;
+    PIO_Offset prod[ndims], loc[ndims];
+    int rearr = PIO_REARR_SUBSET;
+
+    LOG((1, "PIOc_InitDecomp_bc iosysid = %d pio_type = %d ndims = %d"));
+
+    /* Get the info about the io system. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check for required inputs. */
+    if (!gdimlen || !start || !count || !ioidp)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Check that dim, start, and count values are not obviously
+     * incorrect. */
+    for (int i = 0; i < ndims; i++)
+        if (gdimlen[i] <= 0 || start[i] < 0 || count[i] < 0 || (start[i] + count[i]) > gdimlen[i])
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Find the maplen. */
+    for (i = 0; i < ndims; i++)
+        maplen *= count[i];
+
+    /* Get storage for the compmap. */
+    PIO_Offset compmap[maplen];
+
+    /* Find the compmap. */
+    prod[ndims - 1] = 1;
+    loc[ndims - 1] = 0;
+    for (n = ndims - 2; n >= 0; n--)
+    {
+        prod[n] = prod[n + 1] * gdimlen[n + 1];
+        loc[n] = 0;
+    }
+    for (i = 0; i < maplen; i++)
+    {
+        compmap[i] = 0;
+        for (n = ndims - 1; n >= 0; n--)
+            compmap[i] += (start[n] + loc[n]) * prod[n];
+
+        n = ndims - 1;
+        loc[n] = (loc[n] + 1) % count[n];
+        while (loc[n] == 0 && n > 0)
+        {
+            n--;
+            loc[n] = (loc[n] + 1) % count[n];
+        }
+    }
+
+    return PIOc_InitDecomp(iosysid, pio_type, ndims, gdimlen, maplen, compmap, ioidp,
+                           &rearr, NULL, NULL);
+}
+
+/**
+ * Library initialization used when IO tasks are a subset of compute
+ * tasks.
+ *
+ * This function creates an MPI intracommunicator between a set of IO
+ * tasks and one or more sets of computational tasks.
+ *
+ * The caller must create all comp_comm and the io_comm MPI
+ * communicators before calling this function.
+ *
+ * Internally, this function does the following:
+ *
+ * 
      
+ * 
    • Initialize logging system (if PIO_ENABLE_LOGGING is set).
+ * 
    • Allocates and initializes the iosystem_desc_t struct (ios).
+ * 
    • MPI duplicated user comp_comm to ios->comp_comm and
+ * ios->union_comm.
+ * 
    • Set ios->my_comm to be ios->comp_comm. (Not an MPI
+ * duplication.)
+ * 
    • Find MPI rank in comp_comm, determine ranks of IO tasks,
+ * determine whether this task is one of the IO tasks.
+ * 
    • Identify the root IO tasks.
+ * 
    • Create MPI groups for IO tasks, and for computation tasks.
+ * 
    • On IO tasks, create an IO communicator (ios->io_comm).
+ * 
    • Assign an iosystemid, and put this iosystem_desc_t into the
+ * list of open iosystems.
+ * 
    • Initialize the bget buffer, unless PIO_USE_MALLOC was used.
+ * 
    
+ *
+ * When complete, there are three MPI communicators (ios->comp_comm,
+ * ios->union_comm, and ios->io_comm), and two MPI groups
+ * (ios->compgroup and ios->iogroup) that must be freed by MPI.
+ *
+ * @param comp_comm the MPI_Comm of the compute tasks.
+ * @param num_iotasks the number of io tasks to use.
+ * @param stride the offset between io tasks in the comp_comm.
+ * @param base the comp_comm index of the first io task.
+ * @param rearr the rearranger to use by default, this may be
+ * overriden in the PIO_init_decomp(). The rearranger is not used
+ * until the decomposition is initialized.
+ * @param iosysidp index of the defined system descriptor.
+ * @return 0 on success, otherwise a PIO error code.
+ * @ingroup PIO_init
+ */
+int PIOc_Init_Intracomm(MPI_Comm comp_comm, int num_iotasks, int stride, int base,
+                        int rearr, int *iosysidp)
+{
+    iosystem_desc_t *ios;
+    int ustride;
+    int num_comptasks; /* The size of the comp_comm. */
+    int mpierr;        /* Return value for MPI calls. */
+    int ret;           /* Return code for function calls. */
+
+    /* Turn on the logging system. */
+    pio_init_logging();
+
+    /* Find the number of computation tasks. */
+    if ((mpierr = MPI_Comm_size(comp_comm, &num_comptasks)))
+        return check_mpi2(NULL, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Check the inputs. */
+    if (!iosysidp || num_iotasks < 1 || num_iotasks * stride > num_comptasks)
+        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_Init_Intracomm comp_comm = %d num_iotasks = %d stride = %d base = %d "
+         "rearr = %d", comp_comm, num_iotasks, stride, base, rearr));
+
+    /* Allocate memory for the iosystem info. */
+    if (!(ios = calloc(1, sizeof(iosystem_desc_t))))
+        return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    ios->io_comm = MPI_COMM_NULL;
+    ios->intercomm = MPI_COMM_NULL;
+    ios->error_handler = default_error_handler;
+    ios->default_rearranger = rearr;
+    ios->num_iotasks = num_iotasks;
+    ios->num_comptasks = num_comptasks;
+
+    /* For non-async, the IO tasks are a subset of the comptasks. */
+    ios->num_uniontasks = num_comptasks;
+
+    /* Initialize the rearranger options. */
+    ios->rearr_opts.comm_type = PIO_REARR_COMM_COLL;
+    ios->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE;
+    
+    /* Copy the computation communicator into union_comm. */
+    if ((mpierr = MPI_Comm_dup(comp_comm, &ios->union_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Copy the computation communicator into comp_comm. */
+    if ((mpierr = MPI_Comm_dup(comp_comm, &ios->comp_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "union_comm = %d comp_comm = %d", ios->union_comm, ios->comp_comm));
+
+    ios->my_comm = ios->comp_comm;
+    ustride = stride;
+
+    /* Find MPI rank in comp_comm communicator. */
+    if ((mpierr = MPI_Comm_rank(ios->comp_comm, &ios->comp_rank)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* With non-async, all tasks are part of computation component. */
+    ios->compproc = true;
+
+    /* Create an array that holds the ranks of the tasks to be used
+     * for computation. */
+    if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < ios->num_comptasks; i++)
+        ios->compranks[i] = i;
+
+    /* Is this the comp master? */
+    if (ios->comp_rank == 0)
+        ios->compmaster = MPI_ROOT;
+    LOG((2, "comp_rank = %d num_comptasks = %d", ios->comp_rank, ios->num_comptasks));
+
+    /* Create an array that holds the ranks of the tasks to be used
+     * for IO. */
+    if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < ios->num_iotasks; i++)
+    {
+        ios->ioranks[i] = (base + i * ustride) % ios->num_comptasks;
+        if (ios->ioranks[i] == ios->comp_rank)
+            ios->ioproc = true;
+        LOG((3, "ios->ioranks[%d] = %d", i, ios->ioranks[i]));
+    }
+    ios->ioroot = ios->ioranks[0];
+
+    /* We are not providing an info object. */
+    ios->info = MPI_INFO_NULL;
+
+    /* Identify the task that will be the root of the IO communicator. */
+    if (ios->comp_rank == ios->ioranks[0])
+        ios->iomaster = MPI_ROOT;
+
+    /* Create a group for the computation tasks. */
+    if ((mpierr = MPI_Comm_group(ios->comp_comm, &ios->compgroup)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Create a group for the IO tasks. */
+    if ((mpierr = MPI_Group_incl(ios->compgroup, ios->num_iotasks, ios->ioranks,
+                                 &ios->iogroup)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Create an MPI communicator for the IO tasks. */
+    if ((mpierr = MPI_Comm_create(ios->comp_comm, ios->iogroup, &ios->io_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* For the tasks that are doing IO, get their rank within the IO
+     * communicator. If they are not doing IO, set their io_rank to
+     * -1. */
+    if (ios->ioproc)
+    {
+        if ((mpierr = MPI_Comm_rank(ios->io_comm, &ios->io_rank)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    }
+    else
+        ios->io_rank = -1;
+    LOG((3, "ios->io_comm = %d ios->io_rank = %d", ios->io_comm, ios->io_rank));
+
+    /* Rank in the union comm is the same as rank in the comp comm. */
+    ios->union_rank = ios->comp_rank;
+
+    /* Add this ios struct to the list in the PIO library. */
+    *iosysidp = pio_add_to_iosystem_list(ios);
+
+    /* Allocate buffer space for compute nodes. */
+    if ((ret = compute_buffer_init(ios)))
+        return ret;
+
+    LOG((2, "Init_Intracomm complete iosysid = %d", *iosysidp));
+
+    return PIO_NOERR;
+}
+
+/**
+ * Interface to call from pio_init from fortran.
+ *
+ * @param f90_comp_comm
+ * @param num_iotasks the number of IO tasks
+ * @param stride the stride to use assigning tasks
+ * @param base the starting point when assigning tasks
+ * @param rearr the rearranger
+ * @param rearr_opts the rearranger options
+ * @param iosysidp a pointer that gets the IO system ID
+ * @returns 0 for success, error code otherwise
+ */
+int PIOc_Init_Intracomm_from_F90(int f90_comp_comm,
+                                 const int num_iotasks, const int stride,
+                                 const int base, const int rearr,
+                                 rearr_opt_t *rearr_opts, int *iosysidp)
+{
+    int ret = PIO_NOERR;
+    ret = PIOc_Init_Intracomm(MPI_Comm_f2c(f90_comp_comm), num_iotasks,
+                              stride, base, rearr,
+                              iosysidp);
+    if (ret != PIO_NOERR)
+    {
+        LOG((1, "PIOc_Init_Intracomm failed"));
+        return ret;
+    }
+
+    if (rearr_opts)
+    {
+        LOG((1, "Setting rearranger options, iosys=%d", *iosysidp));
+        return PIOc_set_rearr_opts(*iosysidp, rearr_opts->comm_type,
+                                   rearr_opts->fcd,
+                                   rearr_opts->comp2io.hs,
+                                   rearr_opts->comp2io.isend,
+                                   rearr_opts->comp2io.max_pend_req,
+                                   rearr_opts->io2comp.hs,
+                                   rearr_opts->io2comp.isend,
+                                   rearr_opts->io2comp.max_pend_req);
+    }
+    return ret;
+}
+
+/**
+ * Send a hint to the MPI-IO library.
+ *
+ * @param iosysid the IO system ID
+ * @param hint the hint for MPI
+ * @param hintval the value of the hint
+ * @returns 0 for success, or PIO_BADID if iosysid can't be found.
+ */
+int PIOc_set_hint(int iosysid, const char *hint, const char *hintval)
+{
+    iosystem_desc_t *ios;
+    int mpierr; /* Return value for MPI calls. */
+
+    /* Get the iosysid. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* User must provide these. */
+    if (!hint || !hintval)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_set_hint hint = %s hintval = %s", hint, hintval));
+
+    /* Make sure we have an info object. */
+    if (ios->info == MPI_INFO_NULL)
+        if ((mpierr = MPI_Info_create(&ios->info)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Set the MPI hint. */
+    if (ios->ioproc)
+        if ((mpierr = MPI_Info_set(ios->info, hint, hintval)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Clean up internal data structures, free MPI resources, and exit the
+ * pio library.
+ *
+ * @param iosysid: the io system ID provided by PIOc_Init_Intracomm().
+ * @returns 0 for success or non-zero for error.
+ * @ingroup PIO_finalize
+ */
+int PIOc_finalize(int iosysid)
+{
+    iosystem_desc_t *ios;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr = PIO_NOERR;
+
+    LOG((1, "PIOc_finalize iosysid = %d MPI_COMM_NULL = %d", iosysid,
+         MPI_COMM_NULL));
+
+    /* Find the IO system information. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* If asynch IO is in use, send the PIO_MSG_EXIT message from the
+     * comp master to the IO processes. This may be called by
+     * componets for other components iosysid. So don't send unless
+     * there is a valid union_comm. */
+    if (ios->async && ios->union_comm != MPI_COMM_NULL)
+    {
+        int msg = PIO_MSG_EXIT;
+
+        LOG((3, "found iosystem info comproot = %d union_comm = %d comp_idx = %d",
+             ios->comproot, ios->union_comm, ios->comp_idx));
+        if (!ios->ioproc)
+        {
+            LOG((2, "sending msg = %d ioroot = %d union_comm = %d", msg,
+                 ios->ioroot, ios->union_comm));
+
+            /* Send the message to the message handler. */
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the parameters of the function call. */
+            if (!mpierr)
+                mpierr = MPI_Bcast((int *)&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        LOG((3, "handling async errors mpierr = %d my_comm = %d", mpierr, ios->my_comm));
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi2(ios, NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+        LOG((3, "async errors bcast"));
+    }
+
+    /* Free this memory that was allocated in init_intracomm. */
+    if (ios->ioranks)
+        free(ios->ioranks);
+    LOG((3, "Freed ioranks."));
+    if (ios->compranks)
+        free(ios->compranks);
+    LOG((3, "Freed compranks."));
+
+    /* Free the buffer pool. */
+    int niosysid;
+    if ((ierr = pio_num_iosystem(&niosysid)))
+        return pio_err(ios, NULL, ierr, __FILE__, __LINE__);
+    LOG((2, "%d iosystems are still open.", niosysid));
+
+    /* Only free the buffer pool if this is the last open iosysid. */
+    if (niosysid == 1)
+    {
+        free_cn_buffer_pool(ios);
+        LOG((2, "Freed buffer pool."));
+    }
+
+    /* Free the MPI groups. */
+    if (ios->compgroup != MPI_GROUP_NULL)
+        MPI_Group_free(&ios->compgroup);
+
+    if (ios->iogroup != MPI_GROUP_NULL)
+        MPI_Group_free(&(ios->iogroup));
+
+    /* Free the MPI communicators. my_comm is just a copy (but not an
+     * MPI copy), so does not have to have an MPI_Comm_free()
+     * call. comp_comm and io_comm are MPI duplicates of the comms
+     * handed into init_intercomm. So they need to be freed by MPI. */
+    if (ios->intercomm != MPI_COMM_NULL)
+        MPI_Comm_free(&ios->intercomm);
+    if (ios->union_comm != MPI_COMM_NULL)
+        MPI_Comm_free(&ios->union_comm);
+    if (ios->io_comm != MPI_COMM_NULL)
+        MPI_Comm_free(&ios->io_comm);
+    if (ios->comp_comm != MPI_COMM_NULL)
+        MPI_Comm_free(&ios->comp_comm);
+    if (ios->my_comm != MPI_COMM_NULL)
+        ios->my_comm = MPI_COMM_NULL;
+
+    /* Free the MPI Info object. */
+    if (ios->info != MPI_INFO_NULL)
+        MPI_Info_free(&ios->info);
+
+    /* Delete the iosystem_desc_t data associated with this id. */
+    LOG((2, "About to delete iosysid %d.", iosysid));
+    if ((ierr = pio_delete_iosystem_from_list(iosysid)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+
+    pio_finalize_logging();
+
+    LOG((2, "PIOc_finalize completed successfully"));
+    return PIO_NOERR;
+}
+
+/**
+ * Return a logical indicating whether this task is an IO task.
+ *
+ * @param iosysid the io system ID
+ * @param ioproc a pointer that gets 1 if task is an IO task, 0
+ * otherwise. Ignored if NULL.
+ * @returns 0 for success, or PIO_BADID if iosysid can't be found.
+ */
+int PIOc_iam_iotask(int iosysid, bool *ioproc)
+{
+    iosystem_desc_t *ios;
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    if (ioproc)
+        *ioproc = ios->ioproc;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Return the rank of this task in the IO communicator or -1 if this
+ * task is not in the communicator.
+ *
+ * @param iosysid the io system ID
+ * @param iorank a pointer that gets the io rank, or -1 if task is not
+ * in the IO communicator. Ignored if NULL.
+ * @returns 0 for success, or PIO_BADID if iosysid can't be found.
+ */
+int PIOc_iotask_rank(int iosysid, int *iorank)
+{
+    iosystem_desc_t *ios;
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    if (iorank)
+        *iorank = ios->io_rank;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Return true if this iotype is supported in the build, 0 otherwise.
+ *
+ * @param iotype the io type to check
+ * @returns 1 if iotype is in build, 0 if not.
+ */
+int PIOc_iotype_available(int iotype)
+{
+    switch(iotype)
+    {
+#ifdef _NETCDF4
+    case PIO_IOTYPE_NETCDF4P:
+    case PIO_IOTYPE_NETCDF4C:
+        return 1;
+#endif
+    case PIO_IOTYPE_NETCDF:
+        return 1;
+#ifdef _PNETCDF
+    case PIO_IOTYPE_PNETCDF:
+        return 1;
+        break;
+#endif
+    default:
+        return 0;
+    }
+}
+
+/**
+ * Library initialization used when IO tasks are distinct from compute
+ * tasks.
+ *
+ * This is a collective call.  Input parameters are read on
+ * comp_rank=0 values on other tasks are ignored.  This variation of
+ * PIO_init sets up a distinct set of tasks to handle IO, these tasks
+ * do not return from this call.  Instead they go to an internal loop
+ * and wait to receive further instructions from the computational
+ * tasks.
+ *
+ * Sequence of Events to do Asynch I/O
+ * -----------------------------------
+ *
+ * Here is the sequence of events that needs to occur when an IO
+ * operation is called from the collection of compute tasks.  I'm
+ * going to use pio_put_var because write_darray has some special
+ * characteristics that make it a bit more complicated...
+ *
+ * Compute tasks call pio_put_var with an integer argument
+ *
+ * The MPI_Send sends a message from comp_rank=0 to io_rank=0 on
+ * union_comm (a comm defined as the union of io and compute tasks)
+ * msg is an integer which indicates the function being called, in
+ * this case the msg is PIO_MSG_PUT_VAR_INT
+ *
+ * The iotasks now know what additional arguments they should expect
+ * to receive from the compute tasks, in this case a file handle, a
+ * variable id, the length of the array and the array itself.
+ *
+ * The iotasks now have the information they need to complete the
+ * operation and they call the pio_put_var routine.  (In pio1 this bit
+ * of code is in pio_get_put_callbacks.F90.in)
+ *
+ * After the netcdf operation is completed (in the case of an inq or
+ * get operation) the result is communicated back to the compute
+ * tasks.
+ *
+ * @param world the communicator containing all the available tasks.
+ *
+ * @param num_io_procs the number of processes for the IO component.
+ *
+ * @param io_proc_list an array of lenth num_io_procs with the
+ * processor number for each IO processor. If NULL then the IO
+ * processes are assigned starting at processes 0.
+ *
+ * @param component_count number of computational components
+ *
+ * @param num_procs_per_comp an array of int, of length
+ * component_count, with the number of processors in each computation
+ * component.
+ *
+ * @param proc_list an array of arrays containing the processor
+ * numbers for each computation component. If NULL then the
+ * computation components are assigned processors sequentially
+ * starting with processor num_io_procs.
+ *
+ * @param user_io_comm pointer to an MPI_Comm. If not NULL, it will
+ * get an MPI duplicate of the IO communicator. (It is a full
+ * duplicate and later must be freed with MPI_Free() by the caller.)
+ *
+ * @param user_comp_comm pointer to an array of pointers to MPI_Comm;
+ * the array is of length component_count. If not NULL, it will get an
+ * MPI duplicate of each computation communicator. (These are full
+ * duplicates and each must later be freed with MPI_Free() by the
+ * caller.)
+ *
+ * @param rearranger the default rearranger to use for decompositions
+ * in this IO system. Must be either PIO_REARR_BOX or
+ * PIO_REARR_SUBSET.
+ *
+ * @param iosysidp pointer to array of length component_count that
+ * gets the iosysid for each component.
+ *
+ * @return PIO_NOERR on success, error code otherwise.
+ * @ingroup PIO_init
+ */
+int PIOc_init_async(MPI_Comm world, int num_io_procs, int *io_proc_list,
+                    int component_count, int *num_procs_per_comp, int **proc_list,
+                    MPI_Comm *user_io_comm, MPI_Comm *user_comp_comm, int rearranger,
+                    int *iosysidp)
+{
+    int my_rank;          /* Rank of this task. */
+    int **my_proc_list;   /* Array of arrays of procs for comp components. */
+    int *my_io_proc_list; /* List of processors in IO component. */
+    int mpierr;           /* Return code from MPI functions. */
+    int ret;              /* Return code. */
+
+    /* Check input parameters. */
+    if (num_io_procs < 1 || component_count < 1 || !num_procs_per_comp || !iosysidp ||
+        (rearranger != PIO_REARR_BOX && rearranger != PIO_REARR_SUBSET))
+        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Temporarily limit to one computational component. */
+    if (component_count > 1)
+        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* Turn on the logging system for PIO. */
+    pio_init_logging();
+    LOG((1, "PIOc_Init_Async num_io_procs = %d component_count = %d", num_io_procs,
+         component_count));
+
+    /* If the user did not supply a list of process numbers to use for
+     * IO, create it. */
+    if (!io_proc_list)
+    {
+        LOG((3, "calculating processors for IO component"));
+        if (!(my_io_proc_list = malloc(num_io_procs * sizeof(int))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for (int p = 0; p < num_io_procs; p++)
+        {
+            my_io_proc_list[p] = p;
+            LOG((3, "my_io_proc_list[%d] = %d", p, my_io_proc_list[p]));
+        }
+    }
+    else
+        my_io_proc_list = io_proc_list;
+
+    /* If the user did not provide a list of processes for each
+     * component, create one. */
+    if (!proc_list)
+    {
+        int last_proc = num_io_procs;
+
+        /* Allocate space for array of arrays. */
+        if (!(my_proc_list = malloc((component_count) * sizeof(int *))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+        /* Fill the array of arrays. */
+        for (int cmp = 0; cmp < component_count; cmp++)
+        {
+            LOG((3, "calculating processors for component %d num_procs_per_comp[cmp] = %d", cmp, num_procs_per_comp[cmp]));
+
+            /* Allocate space for each array. */
+            if (!(my_proc_list[cmp] = malloc(num_procs_per_comp[cmp] * sizeof(int))))
+                return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+            int proc;
+            for (proc = last_proc; proc < num_procs_per_comp[cmp] + last_proc; proc++)
+            {
+                my_proc_list[cmp][proc - last_proc] = proc;
+                LOG((3, "my_proc_list[%d][%d] = %d", cmp, proc - last_proc, proc));
+            }
+            last_proc = proc;
+        }
+    }
+    else
+        my_proc_list = proc_list;
+
+    /* Get rank of this task in world. */
+    if ((ret = MPI_Comm_rank(world, &my_rank)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+    /* Is this process in the IO component? */
+    int pidx;
+    for (pidx = 0; pidx < num_io_procs; pidx++)
+        if (my_rank == my_io_proc_list[pidx])
+            break;
+    int in_io = (pidx == num_io_procs) ? 0 : 1;
+    LOG((3, "in_io = %d", in_io));
+
+    /* Allocate struct to hold io system info for each computation component. */
+    iosystem_desc_t *iosys[component_count], *my_iosys;
+    for (int cmp1 = 0; cmp1 < component_count; cmp1++)
+        if (!(iosys[cmp1] = (iosystem_desc_t *)calloc(1, sizeof(iosystem_desc_t))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Create group for world. */
+    MPI_Group world_group;
+    if ((ret = MPI_Comm_group(world, &world_group)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+    LOG((3, "world group created\n"));
+
+    /* We will create a group for the IO component. */
+    MPI_Group io_group;
+
+    /* The shared IO communicator. */
+    MPI_Comm io_comm;
+
+    /* Rank of current process in IO communicator. */
+    int io_rank = -1;
+
+    /* Set to MPI_ROOT on master process, MPI_PROC_NULL on other
+     * processes. */
+    int iomaster;
+
+    /* Create a group for the IO component. */
+    if ((ret = MPI_Group_incl(world_group, num_io_procs, my_io_proc_list, &io_group)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+    LOG((3, "created IO group - io_group = %d MPI_GROUP_EMPTY = %d", io_group, MPI_GROUP_EMPTY));
+
+    /* There is one shared IO comm. Create it. */
+    if ((ret = MPI_Comm_create(world, io_group, &io_comm)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+    LOG((3, "created io comm io_comm = %d", io_comm));
+
+    /* Does the user want a copy of the IO communicator? */
+    if (user_io_comm)
+    {
+        *user_io_comm = MPI_COMM_NULL;
+        if (in_io)
+            if ((mpierr = MPI_Comm_dup(io_comm, user_io_comm)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* For processes in the IO component, get their rank within the IO
+     * communicator. */
+    if (in_io)
+    {
+        LOG((3, "about to get io rank"));
+        if ((ret = MPI_Comm_rank(io_comm, &io_rank)))
+            return check_mpi(NULL, ret, __FILE__, __LINE__);
+        iomaster = !io_rank ? MPI_ROOT : MPI_PROC_NULL;
+        LOG((3, "intracomm created for io_comm = %d io_rank = %d IO %s",
+             io_comm, io_rank, iomaster == MPI_ROOT ? "MASTER" : "SERVANT"));
+    }
+
+    /* We will create a group for each computational component. */
+    MPI_Group group[component_count];
+
+    /* We will also create a group for each component and the IO
+     * component processes (i.e. a union of computation and IO
+     * processes. */
+    MPI_Group union_group[component_count];
+
+    /* For each computation component. */
+    for (int cmp = 0; cmp < component_count; cmp++)
+    {
+        LOG((3, "processing component %d", cmp));
+
+        /* Get pointer to current iosys. */
+        my_iosys = iosys[cmp];
+
+        /* Initialize some values. */
+        my_iosys->io_comm = MPI_COMM_NULL;
+        my_iosys->comp_comm = MPI_COMM_NULL;
+        my_iosys->union_comm = MPI_COMM_NULL;
+        my_iosys->intercomm = MPI_COMM_NULL;
+        my_iosys->my_comm = MPI_COMM_NULL;
+        my_iosys->async = 1;
+        my_iosys->error_handler = default_error_handler;
+        my_iosys->num_comptasks = num_procs_per_comp[cmp];
+        my_iosys->num_iotasks = num_io_procs;
+        my_iosys->num_uniontasks = my_iosys->num_comptasks + my_iosys->num_iotasks;
+        my_iosys->compgroup = MPI_GROUP_NULL;
+        my_iosys->iogroup = MPI_GROUP_NULL;
+        my_iosys->default_rearranger = rearranger;
+
+        /* Initialize the rearranger options. */
+        my_iosys->rearr_opts.comm_type = PIO_REARR_COMM_COLL;
+        my_iosys->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE;
+        
+        /* The rank of the computation leader in the union comm. */
+        my_iosys->comproot = num_io_procs;
+        LOG((3, "my_iosys->comproot = %d", my_iosys->comproot));
+
+        /* We are not providing an info object. */
+        my_iosys->info = MPI_INFO_NULL;
+
+        /* Create a group for this component. */
+        if ((ret = MPI_Group_incl(world_group, num_procs_per_comp[cmp], my_proc_list[cmp],
+                                  &group[cmp])))
+            return check_mpi(NULL, ret, __FILE__, __LINE__);
+        LOG((3, "created component MPI group - group[%d] = %d", cmp, group[cmp]));
+
+        /* For all the computation components create a union group
+         * with their processors and the processors of the (shared) IO
+         * component. */
+
+        /* How many processors in the union comm? */
+        int nprocs_union = num_io_procs + num_procs_per_comp[cmp];
+
+        /* This will hold proc numbers from both computation and IO
+         * components. */
+        int proc_list_union[nprocs_union];
+
+        /* Add proc numbers from IO. */
+        for (int p = 0; p < num_io_procs; p++)
+            proc_list_union[p] = my_io_proc_list[p];
+
+        /* Add proc numbers from computation component. */
+        for (int p = 0; p < num_procs_per_comp[cmp]; p++)
+            proc_list_union[p + num_io_procs] = my_proc_list[cmp][p];
+
+        /* Allocate space for computation task ranks. */
+        if (!(my_iosys->compranks = calloc(my_iosys->num_comptasks, sizeof(int))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        
+        /* Remember computation task ranks. */
+        for (int p = 0; p < num_procs_per_comp[cmp]; p++)
+            my_iosys->compranks[p] = my_proc_list[cmp][p];
+
+        /* Create the union group. */
+        if ((ret = MPI_Group_incl(world_group, nprocs_union, proc_list_union, &union_group[cmp])))
+            return check_mpi(NULL, ret, __FILE__, __LINE__);
+        LOG((3, "created union MPI_group - union_group[%d] = %d with %d procs", cmp,
+             union_group[cmp], nprocs_union));
+
+        /* Remember whether this process is in the IO component. */
+        my_iosys->ioproc = in_io;
+
+        /* With async, tasks are either in a computation component or
+         * the IO component. */
+        my_iosys->compproc = !in_io;
+
+        /* Is this process in this computation component? */
+        int in_cmp = 0;
+        for (pidx = 0; pidx < num_procs_per_comp[cmp]; pidx++)
+            if (my_rank == my_proc_list[cmp][pidx])
+                break;
+        in_cmp = (pidx == num_procs_per_comp[cmp]) ? 0 : 1;
+        LOG((3, "pidx = %d num_procs_per_comp[%d] = %d in_cmp = %d",
+             pidx, cmp, num_procs_per_comp[cmp], in_cmp));
+
+        /* Create an intracomm for this component. Only processes in
+         * the component need to participate in the intracomm create
+         * call. */
+        LOG((3, "creating intracomm cmp = %d from group[%d] = %d", cmp, cmp, group[cmp]));
+        if ((ret = MPI_Comm_create(world, group[cmp], &my_iosys->comp_comm)))
+            return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+        if (in_cmp)
+        {
+            /* Does the user want a copy? */
+            if (user_comp_comm)
+                if ((mpierr = MPI_Comm_dup(my_iosys->comp_comm, &user_comp_comm[cmp])))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+            /* Get the rank in this comp comm. */
+            if ((ret = MPI_Comm_rank(my_iosys->comp_comm, &my_iosys->comp_rank)))
+                return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+            /* Set comp_rank 0 to be the compmaster. It will have a
+             * setting of MPI_ROOT, all other tasks will have a
+             * setting of MPI_PROC_NULL. */
+            my_iosys->compmaster = my_iosys->comp_rank ? MPI_PROC_NULL : MPI_ROOT;
+
+            LOG((3, "intracomm created for cmp = %d comp_comm = %d comp_rank = %d comp %s",
+                 cmp, my_iosys->comp_comm, my_iosys->comp_rank,
+                 my_iosys->compmaster == MPI_ROOT ? "MASTER" : "SERVANT"));
+        }
+
+        /* If this is the IO component, make a copy of the IO comm for
+         * each computational component. */
+        if (in_io)
+        {
+            LOG((3, "making a dup of io_comm = %d io_rank = %d", io_comm, io_rank));
+            if ((ret = MPI_Comm_dup(io_comm, &my_iosys->io_comm)))
+                return check_mpi(NULL, ret, __FILE__, __LINE__);
+            LOG((3, "dup of io_comm = %d io_rank = %d", my_iosys->io_comm, io_rank));
+            my_iosys->iomaster = iomaster;
+            my_iosys->io_rank = io_rank;
+            my_iosys->ioroot = 0;
+            my_iosys->comp_idx = cmp;
+        }
+
+        /* Create an array that holds the ranks of the tasks to be used
+         * for IO. */
+        if (!(my_iosys->ioranks = calloc(my_iosys->num_iotasks, sizeof(int))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for (int i = 0; i < my_iosys->num_iotasks; i++)
+            my_iosys->ioranks[i] = my_io_proc_list[i];
+        my_iosys->ioroot = my_iosys->ioranks[0];
+
+        /* All the processes in this component, and the IO component,
+         * are part of the union_comm. */
+        if (in_io || in_cmp)
+        {
+            LOG((3, "my_iosys->io_comm = %d group = %d", my_iosys->io_comm, union_group[cmp]));
+            /* Create a group for the union of the IO component
+             * and one of the computation components. */
+            if ((ret = MPI_Comm_create(world, union_group[cmp], &my_iosys->union_comm)))
+                return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+            if ((ret = MPI_Comm_rank(my_iosys->union_comm, &my_iosys->union_rank)))
+                return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+            /* Set my_comm to union_comm for async. */
+            my_iosys->my_comm = my_iosys->union_comm;
+            LOG((3, "intracomm created for union cmp = %d union_rank = %d union_comm = %d",
+                 cmp, my_iosys->union_rank, my_iosys->union_comm));
+
+            if (in_io)
+            {
+                LOG((3, "my_iosys->io_comm = %d", my_iosys->io_comm));
+                /* Create the intercomm from IO to computation component. */
+                LOG((3, "about to create intercomm for IO component to cmp = %d "
+                     "my_iosys->io_comm = %d", cmp, my_iosys->io_comm));
+                if ((ret = MPI_Intercomm_create(my_iosys->io_comm, 0, my_iosys->union_comm,
+                                                my_proc_list[cmp][0], 0, &my_iosys->intercomm)))
+                    return check_mpi(NULL, ret, __FILE__, __LINE__);
+            }
+            else
+            {
+                /* Create the intercomm from computation component to IO component. */
+                LOG((3, "about to create intercomm for cmp = %d my_iosys->comp_comm = %d", cmp,
+                     my_iosys->comp_comm));
+                if ((ret = MPI_Intercomm_create(my_iosys->comp_comm, 0, my_iosys->union_comm,
+                                                my_io_proc_list[0], 0, &my_iosys->intercomm)))
+                    return check_mpi(NULL, ret, __FILE__, __LINE__);
+            }
+            LOG((3, "intercomm created for cmp = %d", cmp));
+        }
+
+        /* Add this id to the list of PIO iosystem ids. */
+        iosysidp[cmp] = pio_add_to_iosystem_list(my_iosys);
+        LOG((2, "new iosys ID added to iosystem_list iosysid = %d", iosysidp[cmp]));
+    } /* next computational component */
+
+    /* Now call the function from which the IO tasks will not return
+     * until the PIO_MSG_EXIT message is sent. This will handle all
+     * components. */
+    if (in_io)
+    {
+        LOG((2, "Starting message handler io_rank = %d component_count = %d",
+             io_rank, component_count));
+        if ((ret = pio_msg_handler2(io_rank, component_count, iosys, io_comm)))
+            return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+        LOG((2, "Returned from pio_msg_handler2() ret = %d", ret));
+    }
+
+    /* Free resources if needed. */
+    if (!io_proc_list)
+        free(my_io_proc_list);
+
+    if (in_io)
+        if ((mpierr = MPI_Comm_free(&io_comm)))
+            return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+    if (!proc_list)
+    {
+        for (int cmp = 0; cmp < component_count; cmp++)
+            free(my_proc_list[cmp]);
+        free(my_proc_list);
+    }
+
+    /* Free MPI groups. */
+    if ((ret = MPI_Group_free(&io_group)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+    for (int cmp = 0; cmp < component_count; cmp++)
+    {
+        if ((ret = MPI_Group_free(&group[cmp])))
+            return check_mpi(NULL, ret, __FILE__, __LINE__);
+        if ((ret = MPI_Group_free(&union_group[cmp])))
+            return check_mpi(NULL, ret, __FILE__, __LINE__);
+    }
+
+    if ((ret = MPI_Group_free(&world_group)))
+        return check_mpi(NULL, ret, __FILE__, __LINE__);
+
+    LOG((2, "successfully done with PIO_Init_Async"));
+    return PIO_NOERR;
+}
+
+/**
+ * Set the target blocksize for the box rearranger.
+ *
+ * @param newblocksize the new blocksize.
+ * @returns 0 for success.
+ * @ingroup PIO_set_blocksize
+ */
+int PIOc_set_blocksize(int newblocksize)
+{
+    if (newblocksize > 0)
+        blocksize = newblocksize;
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pioc_sc.c b/src/externals/pio2/src/clib/pioc_sc.c
new file mode 100644
index 00000000000..98e3c6aa11a
--- /dev/null
+++ b/src/externals/pio2/src/clib/pioc_sc.c
@@ -0,0 +1,440 @@
+/**
+ * @file
+ * Compute start and count arrays for the box rearranger
+ *
+ * @author Jim Edwards
+ * @date  2014
+ */
+#include 
+#include 
+#include 
+
+/** The default target blocksize for each io task when the box
+ * rearranger is used. */
+#define DEFAULT_BLOCKSIZE 1024
+
+/** The target blocksize for each io task when the box rearranger is
+ * used. */
+int blocksize = DEFAULT_BLOCKSIZE;
+
+/**
+ * Recursive Standard C Function: Greatest Common Divisor.
+ *
+ * @param a
+ * @param b
+ * @returns greates common divisor.
+ */
+int gcd(int a, int b )
+{
+    if (a == 0)
+        return b;
+    return gcd(b % a, a);
+}
+
+/**
+ * Recursive Standard C Function: Greatest Common Divisor for 64 bit
+ * ints.
+ *
+ * @param a
+ * @param b
+ * @returns greates common divisor.
+ */
+long long lgcd(long long a, long long b)
+{
+    if (a == 0)
+        return b;
+    return lgcd(b % a, a);
+}
+
+/**
+ * Return the gcd of elements in an int array.
+ *
+ * @param nain length of the array
+ * @param ain an array of length nain
+ * @returns greatest common divisor.
+ */
+int gcd_array(int nain, int *ain)
+{
+    int i;
+    int bsize = 1;
+
+    for (i = 0; i < nain; i++)
+        if (ain[i] <= 1)
+            return bsize;
+
+    bsize = ain[0];
+    i = 1;
+    while (i < nain && bsize > 1)
+    {
+        bsize = gcd(bsize, ain[i]);
+        i++;
+    }
+
+    return bsize;
+}
+
+/**
+ * Return the greatest common devisor of array ain as int_64.
+ *
+ * @param nain number of elements in ain.
+ * @param ain array of length nain.
+ * @returns GCD of elements in ain.
+ */
+long long lgcd_array(int nain, long long *ain)
+{
+    int i;
+    long long bsize = 1;
+
+    for (i = 0; i < nain; i++)
+        if(ain[i] <= 1)
+            return bsize;
+
+    bsize = ain[0];
+    i = 1;
+    while (i < nain && bsize > 1)
+    {
+        bsize = gcd(bsize, ain[i]);
+        i++;
+    }
+
+    return bsize;
+}
+
+/**
+ * Compute one element (dimension) of start and count arrays. This
+ * function is used by CalcStartandCount().
+ *
+ * @param gdim global size of one dimension.
+ * @param ioprocs number of io tasks.
+ * @param rank IO rank of this task.
+ * @param start pointer to PIO_Offset that will get the start value.
+ * @param count pointer to PIO_Offset that will get the count value.
+ */
+void compute_one_dim(int gdim, int ioprocs, int rank, PIO_Offset *start,
+                     PIO_Offset *count)
+{
+    int irank;     /* The IO rank for this task. */
+    int remainder;
+    int adds;
+    PIO_Offset lstart, lcount;
+
+    /* Check inputs. */
+    pioassert(gdim >= 0 && ioprocs > 0 && rank >= 0 && start && count,
+              "invalid input", __FILE__, __LINE__);
+
+    /* Determin which IO task to use. */
+    irank = rank % ioprocs;
+
+    /* Each IO task will have its share of the global dim. */
+    lcount = (long int)(gdim / ioprocs);
+
+    /* Find the start for this task. */
+    lstart = (long int)(lcount * irank);
+
+    /* Is there anything left over? */
+    remainder = gdim - lcount * ioprocs;
+
+    /* Distribute left over data to some IO tasks. */
+    if (remainder >= ioprocs - irank)
+    {
+        lcount++;
+        if ((adds = irank + remainder - ioprocs) > 0)
+            lstart += adds;
+    }
+
+    /* Return results to caller. */
+    *start = lstart;
+    *count = lcount;
+}
+
+/**
+ * Look for the largest block of data for io which can be expressed in
+ * terms of start and count.
+ *
+ * @param arrlen
+ * @param arr_in
+ * @returns the size of the block
+ */
+PIO_Offset GCDblocksize(int arrlen, const PIO_Offset *arr_in)
+{
+    int numblks = 0;  /* Number of blocks. */
+    int numtimes = 0; /* Number of times adjacent arr_in elements differ by != 1. */
+    int numgaps = 0;  /* Number of gaps. */
+    int j;  /* Loop counter. */
+    int ii; /* Loop counter. */
+    int n;
+    PIO_Offset bsize;     /* Size of the block. */
+    PIO_Offset bsizeg;    /* Size of gap block. */
+    PIO_Offset blklensum; /* Sum of all block lengths. */
+    PIO_Offset del_arr[arrlen - 1]; /* Array of deltas between adjacent elements in arr_in. */
+    PIO_Offset loc_arr[arrlen - 1];
+
+    /* Check inputs. */
+    pioassert(arrlen > 0 && arr_in, "invalid input", __FILE__, __LINE__);
+
+    /* Count the number of contiguous blocks in arr_in. If any if
+       these blocks is of size 1, we are done and can return.
+       Otherwise numtimes is the number of blocks. */
+    for (int i = 0; i < arrlen - 1; i++)
+    {
+        del_arr[i] = arr_in[i + 1] - arr_in[i];
+        if (del_arr[i] != 1)
+        {
+            numtimes++;
+            if ( i > 0 && del_arr[i - 1] > 1)
+                return(1);
+        }
+    }
+
+    /* If numtimes is 0 the all of the data in arr_in is contiguous
+     * and numblks=1. Not sure why I have three different variables
+     * here, seems like n,numblks and numtimes could be combined. */
+    numblks = numtimes + 1;
+    if (numtimes == 0)
+        n = numblks;
+    else
+        n = numtimes;
+
+    /* If numblks==1 then the result is arrlen and you can return. */
+    bsize = (PIO_Offset)arrlen;
+    if (numblks > 1)
+    {
+        PIO_Offset blk_len[numblks];
+        PIO_Offset gaps[numtimes];
+
+        /* If numblks > 1 then numtimes must be > 0 and this if block
+         * isn't needed. */
+        if (numtimes > 0)
+        {
+            ii = 0;
+            for (int i = 0; i < arrlen - 1; i++)
+                if (del_arr[i] > 1)
+                    gaps[ii++] = del_arr[i] - 1;
+            numgaps = ii;
+        }
+
+        j = 0;
+        for (int i = 0; i < n; i++)
+            loc_arr[i] = 1;
+
+        for (int i = 0; i < arrlen - 1; i++)
+            if(del_arr[i] != 1)
+                loc_arr[j++] = i;
+
+        blk_len[0] = loc_arr[0];
+        blklensum = blk_len[0];
+        for(int i = 1; i < numblks - 1; i++)
+        {
+            blk_len[i] = loc_arr[i] - loc_arr[i - 1];
+            blklensum += blk_len[i];
+        }
+        blk_len[numblks - 1] = arrlen - blklensum;
+
+        /* Get the GCD in blk_len array. */
+        bsize = lgcd_array(numblks, blk_len);
+
+        /* I don't recall why i needed these next two blocks, I
+         * remember struggling to get this right in all cases and I'm
+         * afraid that the end result is that bsize is almost always
+         * 1. */
+        if (numgaps > 0)
+        {
+            bsizeg = lgcd_array(numgaps, gaps);
+            bsize = lgcd(bsize, bsizeg);
+        }
+
+        /* ??? */
+        if (arr_in[0] > 0)
+            bsize = lgcd(bsize, arr_in[0]);
+    }
+    
+    return bsize;
+}
+
+/**
+ * Compute start and count values for each io task. This is used in
+ * PIOc_InitDecomp() for the box rearranger only.
+ *
+ * @param pio_type the PIO data type used in this decompotion.
+ * @param ndims the number of dimensions in the variable, not
+ * including the unlimited dimension.
+ * @param gdims an array of global size of each dimension.
+ * @param num_io_procs the number of IO tasks.
+ * @param myiorank rank of this task in IO communicator.
+ * @param start array of length ndims with data start values.
+ * @param count array of length ndims with data count values.
+ * @param num_aiotasks the number of IO tasks used(?)
+ * @returns 0 for success, error code otherwise.
+ */
+int CalcStartandCount(int pio_type, int ndims, const int *gdims, int num_io_procs,
+                      int myiorank, PIO_Offset *start, PIO_Offset *count, int *num_aiotasks)
+{
+    int minbytes; 
+    int maxbytes;
+    int minblocksize; /* Like minbytes, but in data elements. */
+    int basesize;     /* Size in bytes of base data type. */
+    int use_io_procs;
+    int i;
+    long int pgdims;
+    bool converged;
+    int iorank;
+    int ldims;
+    int tiorank;
+    int ioprocs;
+    int tioprocs;
+    int mystart[ndims], mycount[ndims];
+    long int pknt;
+    long int tpsize = 0;
+    int ret;
+
+    /* Check inputs. */
+    pioassert(pio_type > 0 && ndims > 0 && gdims && num_io_procs > 0 && start && count,
+              "invalid input", __FILE__, __LINE__);
+    LOG((1, "CalcStartandCount pio_type = %d ndims = %d num_io_procs = %d myiorank = %d",
+         pio_type, ndims, num_io_procs, myiorank));
+
+    /* We are trying to find start and count indices for each iotask
+     * such that each task has approximately blocksize data to write
+     * (read). The number of iotasks participating in the operation is
+     * blocksize/global_size. */
+    minbytes = blocksize - 256;
+    maxbytes = blocksize + 256;
+
+    /* Determine the size of the data type. */
+    if ((ret = find_mpi_type(pio_type, NULL, &basesize)))
+        return ret;
+
+    /* Determine the minimum block size. */
+    minblocksize = minbytes / basesize;
+
+    /* Find the total size of the data. */
+    pgdims = 1;
+    for (i = 0; i < ndims; i++)
+        pgdims *= (long int)gdims[i];
+
+    /* Find the number of ioprocs that are needed so that we have
+     * blocksize data on each iotask*/
+    use_io_procs = max(1, min((int)((float)pgdims / (float)minblocksize + 0.5), num_io_procs));
+
+    /* Initialize to 0. */
+    converged = 0;
+    for (i = 0; i < ndims; i++)
+    {
+        mystart[i] = 0;
+        mycount[i] = 0;
+    }
+
+    /* Use_io_procs is the number of ioprocs that are needed so that
+     * we have blocksize data on each iotask, now find start and count
+     * values needed on each of these tasks. */
+    while (!converged)
+    {
+        long int p;
+
+        for (iorank = 0; iorank < use_io_procs; iorank++)
+        {
+            for (i = 0; i < ndims; i++)
+            {
+                start[i] = 0;
+                count[i] = gdims[i];
+            }
+            ldims = ndims - 1;
+            p = basesize;
+            for (i = ndims - 1; i >= 0; i--)
+            {
+                p = p * gdims[i];
+                if (p / use_io_procs > maxbytes)
+                {
+                    ldims = i;
+                    break;
+                }
+            }
+
+            if (gdims[ldims] < use_io_procs)
+            {
+                if (ldims > 0 && gdims[ldims - 1] > use_io_procs)
+                    ldims--;
+                else
+                    use_io_procs -= (use_io_procs % gdims[ldims]);
+            }
+
+            ioprocs = use_io_procs;
+            tiorank = iorank;
+            for (i = 0; i <= ldims; i++)
+            {
+                if (gdims[i] >= ioprocs)
+                {
+                    compute_one_dim(gdims[i], ioprocs, tiorank, &start[i], &count[i]);
+                    if (start[i] + count[i] > gdims[i] + 1)
+                    {
+                        piodie("Start plus count exceeds dimension bound",__FILE__,__LINE__);
+                    }
+                }
+                else if(gdims[i] > 1)
+                {
+                    tioprocs = gdims[i];
+                    tiorank = (iorank * tioprocs) / ioprocs;
+                    compute_one_dim(gdims[i], tioprocs, tiorank, &start[i], &count[i]);
+                    ioprocs = ioprocs / tioprocs;
+                    tiorank  = iorank % ioprocs;
+                }
+
+            }
+
+            if (myiorank == iorank)
+            {
+                for (i = 0; i < ndims; i++)
+                {
+                    mystart[i] = start[i];
+                    mycount[i] = count[i];
+                }
+            }
+            pknt = 1;
+
+            for(i = 0; i < ndims; i++)
+                pknt *= count[i];
+
+            tpsize += pknt;
+
+            if (tpsize == pgdims && use_io_procs == iorank + 1)
+            {
+                converged = true;
+                break;
+            }
+            else if(tpsize >= pgdims)
+            {
+                break;
+            }
+        }
+
+        if (!converged)
+        {
+            tpsize = 0;
+            use_io_procs--;
+        }
+    }
+
+    /* On IO tasks, set the start/count arrays to computed values. On
+     * non-io tasks set start/count to zero. */
+    if (myiorank < use_io_procs)
+    {
+        for (i = 0; i < ndims; i++)
+        {
+            start[i] = mystart[i];
+            count[i] = mycount[i];
+        }
+    }
+    else
+    {
+        for (i = 0; i < ndims; i++)
+        {
+            start[i] = 0;
+            count[i] = 0;
+        }
+    }
+
+    /* Return the number of IO procs used to the caller. */
+    *num_aiotasks = use_io_procs;
+    
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/pioc_support.c b/src/externals/pio2/src/clib/pioc_support.c
new file mode 100644
index 00000000000..fba1ad9070a
--- /dev/null
+++ b/src/externals/pio2/src/clib/pioc_support.c
@@ -0,0 +1,2280 @@
+/** @file
+ * Support functions for the PIO library.
+ */
+#include 
+#if PIO_ENABLE_LOGGING
+#include 
+#include 
+#endif /* PIO_ENABLE_LOGGING */
+#include 
+#include 
+
+#include 
+
+#define VERSNO 2001
+
+/* Some logging constants. */
+#if PIO_ENABLE_LOGGING
+#define MAX_LOG_MSG 1024
+#define MAX_RANK_STR 12
+#define ERROR_PREFIX "ERROR: "
+#define NC_LEVEL_DIFF 3
+int pio_log_level = 0;
+int pio_log_ref_cnt = 0;
+int my_rank;
+FILE *LOG_FILE = NULL;
+#endif /* PIO_ENABLE_LOGGING */
+
+/**
+ * The PIO library maintains its own set of ncids. This is the next
+ * ncid number that will be assigned.
+ */
+extern int pio_next_ncid;
+
+/** The default error handler used when iosystem cannot be located. */
+extern int default_error_handler;
+
+/**
+ * Return a string description of an error code. If zero is passed,
+ * the errmsg will be "No error".
+ *
+ * @param pioerr the error code returned by a PIO function call.
+ * @param errmsg Pointer that will get the error message. The message
+ * will be PIO_MAX_NAME chars or less.
+ * @return 0 on success.
+ */
+int PIOc_strerror(int pioerr, char *errmsg)
+{
+    LOG((1, "PIOc_strerror pioerr = %d", pioerr));
+
+    /* Caller must provide this. */
+    pioassert(errmsg, "pointer to errmsg string must be provided", __FILE__, __LINE__);
+
+    /* System error? NetCDF and pNetCDF errors are always negative. */
+    if (pioerr > 0)
+    {
+        const char *cp = (const char *)strerror(pioerr);
+        if (cp)
+            strncpy(errmsg, cp, PIO_MAX_NAME);
+        else
+            strcpy(errmsg, "Unknown Error");
+    }
+    else if (pioerr == PIO_NOERR)
+        strcpy(errmsg, "No error");
+#if defined(_NETCDF)
+    else if (pioerr <= NC2_ERR && pioerr >= NC4_LAST_ERROR)     /* NetCDF error? */
+        strncpy(errmsg, nc_strerror(pioerr), NC_MAX_NAME);
+#endif /* endif defined(_NETCDF) */
+#if defined(_PNETCDF)
+    else if (pioerr > PIO_FIRST_ERROR_CODE)     /* pNetCDF error? */
+        strncpy(errmsg, ncmpi_strerror(pioerr), NC_MAX_NAME);
+#endif /* defined( _PNETCDF) */
+    else
+        /* Handle PIO errors. */
+        switch(pioerr)
+        {
+        case PIO_EBADIOTYPE:
+            strcpy(errmsg, "Bad IO type");
+            break;
+        default:
+            strcpy(errmsg, "Unknown Error: Unrecognized error code");
+        }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Set the logging level if PIO was built with
+ * PIO_ENABLE_LOGGING. Set to -1 for nothing, 0 for errors only, 1 for
+ * important logging, and so on. Log levels below 1 are only printed
+ * on the io/component root.
+ *
+ * A log file is also produced for each task. The file is called
+ * pio_log_X.txt, where X is the (0-based) task number.
+ *
+ * If the library is not built with logging, this function does
+ * nothing.
+ *
+ * @param level the logging level, 0 for errors only, 5 for max
+ * verbosity.
+ * @returns 0 on success, error code otherwise.
+ */
+int PIOc_set_log_level(int level)
+{
+
+#if PIO_ENABLE_LOGGING
+    /* Set the log level. */
+    pio_log_level = level;
+
+#if NETCDF_C_LOGGING_ENABLED
+    int ret;
+    
+    /* If netcdf logging is available turn it on starting at level = 4. */
+    if (level > NC_LEVEL_DIFF)
+        if ((ret = nc_set_log_level(level - NC_LEVEL_DIFF)))
+            return pio_err(NULL, NULL, ret, __FILE__, __LINE__);
+#endif /* NETCDF_C_LOGGING_ENABLED */
+#endif /* PIO_ENABLE_LOGGING */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Initialize logging.  Open log file, if not opened yet, or increment
+ * ref count if already open.
+ */
+void pio_init_logging(void)
+{
+#if PIO_ENABLE_LOGGING
+    char log_filename[NC_MAX_NAME];
+
+    if (!LOG_FILE)
+    {
+        /* Create a filename with the rank in it. */
+        MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
+        sprintf(log_filename, "pio_log_%d.txt", my_rank);
+
+        /* Open a file for this rank to log messages. */
+        LOG_FILE = fopen(log_filename, "w");
+
+        pio_log_ref_cnt = 1;
+    }
+    else
+    {
+        pio_log_ref_cnt++;
+    }
+#endif /* PIO_ENABLE_LOGGING */
+}
+
+/**
+ * Finalize logging - close log files, if open.
+ */
+void pio_finalize_logging(void)
+{
+#if PIO_ENABLE_LOGGING
+    pio_log_ref_cnt -= 1;
+    if (LOG_FILE)
+    {
+        if (pio_log_ref_cnt == 0)
+        {
+            fclose(LOG_FILE);
+            LOG_FILE = NULL;
+        }
+        else
+            LOG((2, "pio_finalize_logging, postpone close, ref_cnt = %d",
+                 pio_log_ref_cnt));
+    }
+#endif /* PIO_ENABLE_LOGGING */
+}
+
+#if PIO_ENABLE_LOGGING
+/**
+ * This function prints out a message, if the severity of the message
+ * is lower than the global pio_log_level. To use it, do something
+ * like this:
+ *
+ * pio_log(0, "this computer will explode in %d seconds", i);
+ *
+ * After the first arg (the severity), use the rest like a normal
+ * printf statement. Output will appear on stdout.
+ * This function is heavily based on the function in section 15.5 of
+ * the C FAQ.
+ *
+ * In code this functions should be wrapped in the LOG(()) macro.
+ *
+ * @param severity the severity of the message, 0 for error messages,
+ * then increasing levels of verbosity.
+ * @param fmt the format string.
+ * @param ... the arguments used in format string.
+ */
+void pio_log(int severity, const char *fmt, ...)
+{
+    va_list argp;
+    int t;
+    int rem_len = MAX_LOG_MSG;
+    char msg[MAX_LOG_MSG];
+    char *ptr = msg;
+    char rank_str[MAX_RANK_STR];
+
+    /* If the severity is greater than the log level, we don't print
+       this message. */
+    if (severity > pio_log_level)
+        return;
+
+    /* If the severity is 0, only print on rank 0. */
+    if (severity < 1 && my_rank != 0)
+        return;
+
+    /* If the severity is zero, this is an error. Otherwise insert that
+       many tabs before the message. */
+    if (!severity)
+    {
+        strncpy(ptr, ERROR_PREFIX, (rem_len > 0) ? rem_len : 0);
+        ptr += strlen(ERROR_PREFIX);
+        rem_len -= strlen(ERROR_PREFIX);
+    }
+    for (t = 0; t < severity; t++)
+    {
+        strncpy(ptr++, "\t", (rem_len > 0) ? rem_len : 0);
+        rem_len--;
+    }
+
+    /* Show the rank. */
+    snprintf(rank_str, MAX_RANK_STR, "%d ", my_rank);
+    strncpy(ptr, rank_str, (rem_len > 0) ? rem_len : 0);
+    ptr += strlen(rank_str);
+    rem_len -= strlen(rank_str);
+
+    /* Print out the variable list of args with vprintf. */
+    va_start(argp, fmt);
+    vsnprintf(ptr, ((rem_len > 0) ? rem_len : 0), fmt, argp);
+    va_end(argp);
+
+    /* Put on a final linefeed. */
+    ptr = msg + strlen(msg);
+    rem_len = MAX_LOG_MSG - strlen(msg);
+    strncpy(ptr, "\n\0", (rem_len > 0) ? rem_len : 0);
+
+    /* Send message to stdout. */
+    fprintf(stdout, "%s", msg);
+
+    /* Send message to log file. */
+    if (LOG_FILE)
+        fprintf(LOG_FILE, "%s", msg);
+
+    /* Ensure an immediate flush of stdout. */
+    fflush(stdout);
+    if (LOG_FILE)
+        fflush(LOG_FILE);
+}
+#endif /* PIO_ENABLE_LOGGING */
+
+/**
+ * Obtain a backtrace and print it to stderr. This is appended to the
+ * text decomposition file.
+ *
+ * Note from Jim:
+ *
+ * The stack trace can be used to identify the usage in
+ * the model code of the particular decomposition in question and so
+ * if using the pio performance tool leads to tuning that could be
+ * applied in the model you know more or less where to do it.
+ *
+ * It's also useful if you have a model bug - then you have 20 or so
+ * decomp files and you need to identify the one that was problematic.
+ * So it's used as an add to the developer and not used at all by any
+ * automated process or tools.
+ *
+ * @param fp file pointer to send output to
+ */
+void print_trace(FILE *fp)
+{
+    void *array[10];
+    size_t size;
+    char **strings;
+    size_t i;
+
+    /* Note that this won't actually work. */
+    if (fp == NULL)
+        fp = stderr;
+
+    size = backtrace(array, 10);
+    strings = backtrace_symbols(array, size);
+
+    fprintf(fp,"Obtained %zd stack frames.\n", size);
+
+    for (i = 0; i < size; i++)
+        fprintf(fp,"%s\n", strings[i]);
+
+    free(strings);
+}
+
+/**
+ * Abort program and call MPI_Abort().
+ *
+ * @param msg an error message
+ * @param fname name of code file where error occured
+ * @param line the line of code where the error occurred.
+ */
+void piodie(const char *msg, const char *fname, int line)
+{
+    fprintf(stderr,"Abort with message %s in file %s at line %d\n",
+            msg ? msg : "_", fname ? fname : "_", line);
+
+    print_trace(stderr);
+#ifdef MPI_SERIAL
+    abort();
+#else
+    MPI_Abort(MPI_COMM_WORLD, -1);
+#endif
+}
+
+/**
+ * Perform an assert. Note that this function does nothing if NDEBUG
+ * is defined.
+ *
+ * @param expression the expression to be evaluated
+ * @param msg an error message
+ * @param fname name of code file where error occured
+ * @param line the line of code where the error occurred.
+ */
+void pioassert(_Bool expression, const char *msg, const char *fname, int line)
+{
+#ifndef NDEBUG
+    if (!expression)
+        piodie(msg, fname, line);
+#endif
+}
+
+/**
+ * Handle MPI errors. An error message is sent to stderr, then the
+ * check_netcdf() function is called with PIO_EIO.
+ *
+ * @param file pointer to the file_desc_t info. Ignored if NULL.
+ * @param mpierr the MPI return code to handle
+ * @param filename the name of the code file where error occured.
+ * @param line the line of code where error occured.
+ * @return PIO_NOERR for no error, otherwise PIO_EIO.
+ */
+int check_mpi(file_desc_t *file, int mpierr, const char *filename,
+              int line)
+{
+    return check_mpi2(NULL, file, mpierr, filename, line);
+}
+
+/**
+ * Handle MPI errors. An error message is sent to stderr, then the
+ * check_netcdf() function is called with PIO_EIO. This version of the
+ * function accepts an ios parameter, for the (rare) occasions where
+ * we have an ios but not a file.
+ *
+ * @param ios pointer to the iosystem_info_t. May be NULL.
+ * @param file pointer to the file_desc_t info. May be NULL.
+ * @param mpierr the MPI return code to handle
+ * @param filename the name of the code file where error occured.
+ * @param line the line of code where error occured.
+ * @return PIO_NOERR for no error, otherwise PIO_EIO.
+ */
+int check_mpi2(iosystem_desc_t *ios, file_desc_t *file, int mpierr,
+               const char *filename, int line)
+{
+    if (mpierr)
+    {
+        char errstring[MPI_MAX_ERROR_STRING];
+        int errstrlen;
+
+        /* If we can get an error string from MPI, print it to stderr. */
+        if (!MPI_Error_string(mpierr, errstring, &errstrlen))
+            fprintf(stderr, "MPI ERROR: %s in file %s at line %d\n",
+                    errstring, filename ? filename : "_", line);
+
+        /* Handle all MPI errors as PIO_EIO. */
+        return pio_err(ios, file, PIO_EIO, filename, line);
+    }
+    return PIO_NOERR;
+}
+
+/**
+ * Check the result of a netCDF API call.
+ *
+ * @param file pointer to the PIO structure describing this
+ * file. Ignored if NULL.
+ * @param status the return value from the netCDF call.
+ * @param fname the name of the code file.
+ * @param line the line number of the netCDF call in the code.
+ * @return the error code
+ */
+int check_netcdf(file_desc_t *file, int status, const char *fname, int line)
+{
+    return check_netcdf2(NULL, file, status, fname, line);
+}
+
+/**
+ * Check the result of a netCDF API call. This is the same as
+ * check_netcdf() except for the extra iosystem_desc_t pointer, which
+ * is used to determine error handling when there is no file_desc_t
+ * pointer.
+ *
+ * @param ios pointer to the iosystem description struct. Ignored if NULL.
+ * @param file pointer to the PIO structure describing this file. Ignored if NULL.
+ * @param status the return value from the netCDF call.
+ * @param fname the name of the code file.
+ * @param line the line number of the netCDF call in the code.
+ * @return the error code
+ */
+int check_netcdf2(iosystem_desc_t *ios, file_desc_t *file, int status,
+                  const char *fname, int line)
+{
+    int eh = default_error_handler; /* Error handler that will be used. */
+    char errmsg[PIO_MAX_NAME + 1];  /* Error message. */
+
+    /* User must provide this. */
+    pioassert(fname, "code file name must be provided", __FILE__, __LINE__);
+
+    /* No harm, no foul. */
+    if (status == PIO_NOERR)
+        return PIO_NOERR;
+
+    LOG((1, "check_netcdf2 status = %d fname = %s line = %d", status, fname, line));
+
+    /* Pick an error handler. */
+    if (ios)
+        eh = ios->error_handler;
+    if (file)
+        eh = file->iosystem->error_handler;
+    pioassert(eh == PIO_INTERNAL_ERROR || eh == PIO_BCAST_ERROR || eh == PIO_RETURN_ERROR,
+              "invalid error handler", __FILE__, __LINE__);
+    LOG((2, "check_netcdf2 chose error handler = %d", eh));
+
+    /* Get an error message. */
+    if (eh != PIO_BCAST_ERROR && !PIOc_strerror(status, errmsg))
+    {
+        fprintf(stderr, "%s\n", errmsg);
+        LOG((1, "check_netcdf2 errmsg = %s", errmsg));
+    }
+
+    /* Decide what to do based on the error handler. */
+    if (eh == PIO_INTERNAL_ERROR)
+        piodie(errmsg, fname, line);        /* Die! */
+    else if (eh == PIO_BCAST_ERROR)
+    {
+	if (ios)
+	    MPI_Bcast(&status, 1, MPI_INT, ios->ioroot, ios->my_comm);
+	else if (file)
+	    MPI_Bcast(&status, 1, MPI_INT, file->iosystem->ioroot, file->iosystem->my_comm);
+    }
+
+    /* For PIO_RETURN_ERROR, just return the error. */
+    return status;
+}
+
+/**
+ * Handle an error in PIO. This will consult the error handler
+ * settings and either call MPI_Abort() or return an error code.
+ *
+ * The error hanlder has three settings:
+ *
+ * Errors cause abort: PIO_INTERNAL_ERROR.
+ *
+ * Error codes are broadcast to all tasks: PIO_BCAST_ERROR.
+ *
+ * Errors are returned to caller with no internal action:
+ * PIO_RETURN_ERROR.
+ *
+ * @param ios pointer to the IO system info. Ignored if NULL.
+ * @param file pointer to the file description data. Ignored if
+ * NULL. If provided file->iosystem is used as ios pointer.
+ * @param err_num the error code
+ * @param fname name of code file where error occured.
+ * @param line the line of code where the error occurred.
+ * @returns err_num if abort is not called.
+ */
+int pio_err(iosystem_desc_t *ios, file_desc_t *file, int err_num, const char *fname,
+            int line)
+{
+    char err_msg[PIO_MAX_NAME + 1];
+    int err_handler = default_error_handler; /* Default error handler. */
+    int ret;
+
+    /* User must provide this. */
+    pioassert(fname, "file name must be provided", __FILE__, __LINE__);
+
+    /* No harm, no foul. */
+    if (err_num == PIO_NOERR)
+        return PIO_NOERR;
+
+    /* Get the error message. */
+    if ((ret = PIOc_strerror(err_num, err_msg)))
+        return ret;
+
+    /* If logging is in use, log an error message. */
+    LOG((0, "%s err_num = %d fname = %s line = %d", err_msg, err_num, fname ? fname : '\0', line));
+
+    /* What error handler should we use? */
+    if (file)
+        err_handler = file->iosystem->error_handler;
+    else if (ios)
+        err_handler = ios->error_handler;
+
+    LOG((2, "pio_err chose error handler = %d", err_handler));
+
+    /* Should we abort? */
+    if (err_handler == PIO_INTERNAL_ERROR)
+    {
+        /* For debugging only, this will print a traceback of the call tree.  */
+        print_trace(stderr);
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    /* What should we do here??? */
+    if (err_handler == PIO_BCAST_ERROR)
+    {
+        /* ??? */
+    }
+
+    /* If abort was not called, we'll get here. */
+    return err_num;
+}
+
+/**
+ * Allocate a region struct, and initialize it.
+ *
+ * @param ios pointer to the IO system info, used for error
+ * handling. Ignored if NULL.
+ * @param ndims the number of dimensions for the data in this region.
+ * @param a pointer that gets a pointer to the newly allocated
+ * io_region struct.
+ * @returns 0 for success, error code otherwise.
+ */
+int alloc_region2(iosystem_desc_t *ios, int ndims, io_region **regionp)
+{
+    io_region *region;
+
+    /* Check inputs. */
+    pioassert(ndims >= 0 && regionp, "invalid input", __FILE__, __LINE__);
+    LOG((1, "alloc_region2 ndims = %d sizeof(io_region) = %d", ndims,
+         sizeof(io_region)));
+    
+    /* Allocate memory for the io_region struct. */
+    if (!(region = calloc(1, sizeof(io_region))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Allocate memory for the array of start indicies. */
+    if (!(region->start = calloc(ndims, sizeof(PIO_Offset))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Allocate memory for the array of counts. */
+    if (!(region->count = calloc(ndims, sizeof(PIO_Offset))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Return pointer to new region to caller. */
+    *regionp = region;
+    
+    return PIO_NOERR;
+}
+
+/**
+ * Given a PIO type, find the MPI type and the type size.
+ *
+ * @param pio_type a PIO type, PIO_INT, PIO_FLOAT, etc.
+ * @param mpi_type a pointer to MPI_Datatype that will get the MPI
+ * type that coresponds to the PIO type. Ignored if NULL.
+ * @param type_size a pointer to int that will get the size of the
+ * type, in bytes. (For example, 4 for PIO_INT). Ignored if NULL.
+ * @returns 0 for success, error code otherwise.
+ */
+int find_mpi_type(int pio_type, MPI_Datatype *mpi_type, int *type_size)
+{
+    MPI_Datatype my_mpi_type;
+    int my_type_size;
+
+    /* Decide on the base type. */
+    switch(pio_type)
+    {
+    case PIO_BYTE:
+        my_mpi_type = MPI_BYTE;
+        my_type_size = NETCDF_CHAR_SIZE;
+        break;
+    case PIO_CHAR:
+        my_mpi_type = MPI_CHAR;
+        my_type_size = NETCDF_CHAR_SIZE;
+        break;
+    case PIO_SHORT:
+        my_mpi_type = MPI_SHORT;
+        my_type_size = NETCDF_SHORT_SIZE;
+        break;
+    case PIO_INT:
+        my_mpi_type = MPI_INT;
+        my_type_size = NETCDF_INT_FLOAT_SIZE;
+        break;
+    case PIO_FLOAT:
+        my_mpi_type = MPI_FLOAT;
+        my_type_size = NETCDF_INT_FLOAT_SIZE;
+        break;
+    case PIO_DOUBLE:
+        my_mpi_type = MPI_DOUBLE;
+        my_type_size = NETCDF_DOUBLE_INT64_SIZE;
+        break;
+#ifdef _NETCDF4
+    case PIO_UBYTE:
+        my_mpi_type = MPI_UNSIGNED_CHAR;
+        my_type_size = NETCDF_CHAR_SIZE;
+        break;
+    case PIO_USHORT:
+        my_mpi_type = MPI_UNSIGNED_SHORT;
+        my_type_size = NETCDF_SHORT_SIZE;
+        break;
+    case PIO_UINT:
+        my_mpi_type = MPI_UNSIGNED;
+        my_type_size = NETCDF_INT_FLOAT_SIZE;
+        break;
+    case PIO_INT64:
+        my_mpi_type = MPI_LONG_LONG;
+        my_type_size = NETCDF_DOUBLE_INT64_SIZE;
+        break;
+    case PIO_UINT64:
+        my_mpi_type = MPI_UNSIGNED_LONG_LONG;
+        my_type_size = NETCDF_DOUBLE_INT64_SIZE;
+        break;
+    case PIO_STRING:
+        my_mpi_type = MPI_CHAR;
+        my_type_size = NETCDF_CHAR_SIZE;
+        break;
+#endif /* _NETCDF4 */
+    default:
+        return PIO_EBADTYPE;
+    }
+
+    /* If caller wants MPI type, set it. */
+    if (mpi_type)
+        *mpi_type = my_mpi_type;
+
+    /* If caller wants type size, set it. */
+    if (type_size)
+        *type_size = my_type_size;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Allocate space for an IO description struct, and initialize it.
+ *
+ * @param ios pointer to the IO system info, used for error
+ * handling.
+ * @param piotype the PIO data type (ex. PIO_FLOAT, PIO_INT, etc.).
+ * @param ndims the number of dimensions.
+ * @param iodesc pointer that gets the newly allocated io_desc_t.
+ * @returns 0 for success, error code otherwise.
+ */
+int malloc_iodesc(iosystem_desc_t *ios, int piotype, int ndims,
+                  io_desc_t **iodesc)
+{
+    MPI_Datatype mpi_type;
+    int mpierr;
+    int ret;
+
+    /* Check input. */
+    pioassert(ios && piotype > 0 && ndims >= 0 && iodesc,
+              "invalid input", __FILE__, __LINE__);
+
+    LOG((1, "malloc_iodesc piotype = %d ndims = %d", piotype, ndims));
+
+    /* Get the MPI type corresponding with the PIO type. */
+    if ((ret = find_mpi_type(piotype, &mpi_type, NULL)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Allocate space for the io_desc_t struct. */
+    if (!(*iodesc = calloc(1, sizeof(io_desc_t))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Remember the MPI type. */
+    (*iodesc)->basetype = mpi_type;
+
+    /* Get the size of the type. */
+    if ((mpierr = MPI_Type_size((*iodesc)->basetype, &(*iodesc)->basetype_size)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Initialize some values in the struct. */
+    (*iodesc)->maxregions = 1;
+    (*iodesc)->ioid = -1;
+    (*iodesc)->ndims = ndims;
+
+    /* Allocate space for, and initialize, the first region. */
+    if ((ret = alloc_region2(ios, ndims, &((*iodesc)->firstregion))))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);        
+
+    /* Set the swap memory settings to defaults for this IO system. */
+    (*iodesc)->rearr_opts = ios->rearr_opts;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Free a region list.
+ *
+ * top a pointer to the start of the list to free.
+ */
+void free_region_list(io_region *top)
+{
+    io_region *ptr, *tptr;
+
+    ptr = top;
+    while (ptr)
+    {
+        if (ptr->start)
+            free(ptr->start);
+        if (ptr->count)
+            free(ptr->count);
+        tptr = ptr;
+        ptr = ptr->next;
+        free(tptr);
+    }
+}
+
+/**
+ * Free a decomposition map.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition map to free.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_freedecomp(int iosysid, int ioid)
+{
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function calls. */
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = PIO_MSG_FREEDECOMP; /* Message for async notification. */
+
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&iosysid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ioid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            LOG((2, "PIOc_freedecomp iosysid = %d ioid = %d", iosysid, ioid));
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(NULL, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    }
+
+    /* Free the map. */
+    free(iodesc->map);
+
+    /* Free the dimlens. */
+    free(iodesc->dimlen);
+
+    if (iodesc->rfrom)
+        free(iodesc->rfrom);
+
+    if (iodesc->rtype)
+    {
+        for (int i = 0; i < iodesc->nrecvs; i++)
+            if (iodesc->rtype[i] != PIO_DATATYPE_NULL)
+                if ((mpierr = MPI_Type_free(&iodesc->rtype[i])))
+                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+        free(iodesc->rtype);
+    }
+
+    if (iodesc->stype)
+    {
+        for (int i = 0; i < iodesc->num_stypes; i++)
+            if (iodesc->stype[i] != PIO_DATATYPE_NULL)
+                if ((mpierr = MPI_Type_free(iodesc->stype + i)))
+                    return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+        iodesc->num_stypes = 0;
+        free(iodesc->stype);
+    }
+
+    if (iodesc->scount)
+        free(iodesc->scount);
+
+    if (iodesc->rcount)
+        free(iodesc->rcount);
+
+    if (iodesc->sindex)
+        free(iodesc->sindex);
+
+    if (iodesc->rindex)
+        free(iodesc->rindex);
+
+    if (iodesc->firstregion)
+        free_region_list(iodesc->firstregion);
+
+    if (iodesc->fillregion)
+        free_region_list(iodesc->fillregion);
+
+    if (iodesc->rearranger == PIO_REARR_SUBSET)
+        if ((mpierr = MPI_Comm_free(&iodesc->subset_comm)))
+            return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    return pio_delete_iodesc_from_list(ioid);
+}
+
+/**
+ * Read a decomposition map from a file. The decomp file is only read
+ * by task 0 in the communicator.
+ *
+ * @param file the filename
+ * @param ndims pointer to an int with the number of dims.
+ * @param gdims pointer to an array of dimension ids.
+ * @param fmaplen
+ * @param map
+ * @param comm
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_readmap(const char *file, int *ndims, int **gdims, PIO_Offset *fmaplen,
+                 PIO_Offset **map, MPI_Comm comm)
+{
+    int npes, myrank;
+    int rnpes, rversno;
+    int j;
+    int *tdims;
+    PIO_Offset *tmap;
+    MPI_Status status;
+    PIO_Offset maplen;
+    int mpierr; /* Return code for MPI calls. */
+
+    /* Check inputs. */
+    if (!file || !ndims || !gdims || !fmaplen || !map)
+        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    if ((mpierr = MPI_Comm_size(comm, &npes)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_rank(comm, &myrank)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    if (myrank == 0)
+    {
+        FILE *fp = fopen(file, "r");
+        if (!fp)
+            pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+        fscanf(fp,"version %d npes %d ndims %d\n",&rversno, &rnpes, ndims);
+
+        if (rversno != VERSNO)
+            return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+        if (rnpes < 1 || rnpes > npes)
+            return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+        if ((mpierr = MPI_Bcast(&rnpes, 1, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(ndims, 1, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if (!(tdims = calloc(*ndims, sizeof(int))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for (int i = 0; i < *ndims; i++)
+            fscanf(fp,"%d ", tdims + i);
+
+        if ((mpierr = MPI_Bcast(tdims, *ndims, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+        for (int i = 0; i < rnpes; i++)
+        {
+            fscanf(fp, "%d %lld", &j, &maplen);
+            if (j != i)  // Not sure how this could be possible
+                return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);
+            if (!(tmap = malloc(maplen * sizeof(PIO_Offset))))
+                return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+            for (j = 0; j < maplen; j++)
+                fscanf(fp, "%lld ", tmap+j);
+
+            if (i > 0)
+            {
+                if ((mpierr = MPI_Send(&maplen, 1, PIO_OFFSET, i, i + npes, comm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                if ((mpierr = MPI_Send(tmap, maplen, PIO_OFFSET, i, i, comm)))
+                    return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+                free(tmap);
+            }
+            else
+            {
+                *map = tmap;
+                *fmaplen = maplen;
+            }
+        }
+        fclose(fp);
+    }
+    else
+    {
+        if ((mpierr = MPI_Bcast(&rnpes, 1, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(ndims, 1, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        if (!(tdims = calloc(*ndims, sizeof(int))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        if ((mpierr = MPI_Bcast(tdims, *ndims, MPI_INT, 0, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+        if (myrank < rnpes)
+        {
+            if ((mpierr = MPI_Recv(&maplen, 1, PIO_OFFSET, 0, myrank + npes, comm, &status)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            if (!(tmap = malloc(maplen * sizeof(PIO_Offset))))
+                return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+            if ((mpierr = MPI_Recv(tmap, maplen, PIO_OFFSET, 0, myrank, comm, &status)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            *map = tmap;
+        }
+        else
+        {
+            tmap = NULL;
+            maplen = 0;
+        }
+        *fmaplen = maplen;
+    }
+    *gdims = tdims;
+    return PIO_NOERR;
+}
+
+/**
+ * Read a decomposition map from file.
+ *
+ * @param file the filename
+ * @param ndims pointer to the number of dimensions
+ * @param gdims pointer to an array of dimension ids
+ * @param maplen pointer to the length of the map
+ * @param map pointer to the map array
+ * @param f90_comm
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_readmap_from_f90(const char *file, int *ndims, int **gdims, PIO_Offset *maplen,
+                          PIO_Offset **map, int f90_comm)
+{
+    return PIOc_readmap(file, ndims, gdims, maplen, map, MPI_Comm_f2c(f90_comm));
+}
+
+/**
+ * Write the decomposition map to a file using netCDF, everyones
+ * favorite data format.
+ *
+ * @param iosysid the IO system ID.
+ * @param filename the filename to be used.
+ * @param cmode for PIOc_create(). Will be bitwise or'd with NC_WRITE.
+ * @param ioid the ID of the IO description.
+ * @param title optial title attribute for the file. Must be less than
+ * NC_MAX_NAME + 1 if provided. Ignored if NULL.
+ * @param history optial history attribute for the file. Must be less
+ * than NC_MAX_NAME + 1 if provided. Ignored if NULL.
+ * @param fortran_order set to non-zero if fortran array ordering is
+ * used, or to zero if C array ordering is used.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_write_nc_decomp(int iosysid, const char *filename, int cmode, int ioid,
+                         char *title, char *history, int fortran_order)
+{
+    iosystem_desc_t *ios; /* IO system info. */
+    io_desc_t *iodesc;    /* Decomposition info. */
+    int max_maplen;       /* The maximum maplen used for any task. */
+    int mpierr;
+    int ret;
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check inputs. */
+    if (!filename)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+    if (title)
+        if (strlen(title) > PIO_MAX_NAME)
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+    if (history)
+        if (strlen(history) > PIO_MAX_NAME)
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_write_nc_decomp filename = %s iosysid = %d ioid = %d "
+         "ios->num_comptasks = %d", filename, iosysid, ioid, ios->num_comptasks));
+
+    /* Get the IO desc, which describes the decomposition. */
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Allocate memory for array which will contain the length of the
+     * map on each task, for all computation tasks. */
+    int task_maplen[ios->num_comptasks];
+    LOG((3, "ios->num_comptasks = %d", ios->num_comptasks));
+
+    /* Gather maplens from all computation tasks and fill the
+     * task_maplen array on all tasks. */
+    if ((mpierr = MPI_Allgather(&iodesc->maplen, 1, MPI_INT, task_maplen, 1, MPI_INT,
+                                ios->comp_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+
+    /* Find the max maxplen. */
+    if ((mpierr = MPI_Allreduce(&iodesc->maplen, &max_maplen, 1, MPI_INT, MPI_MAX,
+                                ios->comp_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((3, "max_maplen = %d", max_maplen));
+
+    /* 2D array that will hold all the map information for all
+     * tasks. */
+    int full_map[ios->num_comptasks][max_maplen];
+
+    /* Fill local array with my map. Use the fill value for unused */
+    /* elements at the end if max_maplen is longer than maplen. Also
+     * subtract 1 because the iodesc->map is 1-based. */
+    int my_map[max_maplen];
+    for (int e = 0; e < max_maplen; e++)
+    {
+        my_map[e] = e < iodesc->maplen ? iodesc->map[e] - 1 : NC_FILL_INT;
+        LOG((3, "my_map[%d] = %d", e, my_map[e]));
+    }
+    
+    /* Gather my_map from all computation tasks and fill the full_map array. */
+    if ((mpierr = MPI_Allgather(&my_map, max_maplen, MPI_INT, full_map, max_maplen,
+                                MPI_INT, ios->comp_comm)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    
+    for (int p = 0; p < ios->num_comptasks; p++)
+        for (int e = 0; e < max_maplen; e++)
+            LOG((3, "full_map[%d][%d] = %d", p, e, full_map[p][e]));
+
+    /* Write the netCDF decomp file. */
+    if ((ret = pioc_write_nc_decomp_int(ios, filename, cmode, iodesc->ndims, iodesc->dimlen,
+                                        ios->num_comptasks, task_maplen, (int *)full_map, title,
+                                        history, fortran_order)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/**
+ * Read the decomposition map from a netCDF decomp file produced by
+ * PIOc_write_nc_decomp().
+ *
+ * @param iosysid the IO system ID.
+ * @param filename the name of the decomp file.
+ * @param ioid pointer that will get the newly-assigned ID of the IO
+ * description. The ioid is needed to later free the decomposition.
+ * @param comm an MPI communicator.
+ * @param pio_type the PIO type to be used as the type for the data.
+ * @param title pointer that will get optial title attribute for the
+ * file. Will be less than NC_MAX_NAME + 1 if provided. Ignored if
+ * NULL.
+ * @param history pointer that will get optial history attribute for
+ * the file. Will be less than NC_MAX_NAME + 1 if provided. Ignored if
+ * NULL.
+ * @param fortran_order pointer that gets set to 1 if fortran array
+ * ordering is used, or to zero if C array ordering is used.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_read_nc_decomp(int iosysid, const char *filename, int *ioidp, MPI_Comm comm,
+                        int pio_type, char *title, char *history, int *fortran_order)
+{
+    iosystem_desc_t *ios; /* Pointer to the IO system info. */
+    int ndims;            /* The number of data dims (except unlim). */
+    int max_maplen;       /* The max maplen of any task. */
+    int *global_dimlen;   /* An array with sizes of global dimensions. */
+    int *task_maplen;     /* A map of one tasks mapping to global data. */
+    int *full_map;        /* A map with the task maps of every task. */
+    int num_tasks_decomp; /* The number of tasks for this decomp. */
+    int size;             /* Size of comm. */
+    int my_rank;          /* Task rank in comm. */
+    char source_in[PIO_MAX_NAME + 1];  /* Text metadata in decomp file. */
+    char version_in[PIO_MAX_NAME + 1]; /* Text metadata in decomp file. */
+    int mpierr;
+    int ret;
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check inputs. */
+    if (!filename || !ioidp)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_read_nc_decomp filename = %s iosysid = %d pio_type = %d",
+         filename, iosysid, pio_type));
+
+    /* Get the communicator size and task rank. */
+    if ((mpierr = MPI_Comm_size(comm, &size)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_rank(comm, &my_rank)))
+        return check_mpi2(ios, NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "size = %d my_rank = %d", size, my_rank));
+
+    /* Read the file. This allocates three arrays that we have to
+     * free. */
+    if ((ret = pioc_read_nc_decomp_int(iosysid, filename, &ndims, &global_dimlen, &num_tasks_decomp,
+                                       &task_maplen, &max_maplen, &full_map, title, history,
+                                       source_in, version_in, fortran_order)))
+        return ret;
+    LOG((2, "ndims = %d num_tasks_decomp = %d max_maplen = %d", ndims, num_tasks_decomp,
+         max_maplen));
+
+    /* If the size does not match the number of tasks in the decomp,
+     * that's an error. */
+    if (size != num_tasks_decomp)
+        ret = PIO_EINVAL;
+
+    /* Now initialize the iodesc on each task for this decomposition. */
+    if (!ret)
+    {
+        PIO_Offset compmap[task_maplen[my_rank]];
+
+        /* Copy array into PIO_Offset array. Make it 1 based. */
+        for (int e = 0; e < task_maplen[my_rank]; e++)
+            compmap[e] = full_map[my_rank * max_maplen + e] + 1;
+
+        /* Initialize the decomposition. */
+        ret = PIOc_InitDecomp(iosysid, pio_type, ndims, global_dimlen, task_maplen[my_rank],
+                              compmap, ioidp, NULL, NULL, NULL);
+    }
+
+    /* Free resources. */
+    free(global_dimlen);
+    free(task_maplen);
+    free(full_map);
+
+    return ret;
+}
+
+/* Write the decomp information in netCDF. This is an internal
+ * function.
+ *
+ * @param ios pointer to io system info.
+ * @param filename the name the decomp file will have.
+ * @param cmode for PIOc_create(). Will be bitwise or'd with NC_WRITE.
+ * @param ndims number of dims in the data being described.
+ * @param global_dimlen an array, of size ndims, with the size of the
+ * global array in each dimension.
+ * @param num_tasks the number of tasks the data are decomposed over.
+ * @param task_maplen array of size num_tasks with the length of the
+ * map for each task.
+ * @param map pointer to a 2D array of size [num_tasks][max_maplen]
+ * with the 0-based mapping from local to global array elements.
+ * @param title null-terminated string that will be written as an
+ * attribute. If provided, length must be < PIO_MAX_NAME + 1. Ignored
+ * if NULL.
+ * @param history null-terminated string that will be written as an
+ * attribute. If provided, length must be < PIO_MAX_NAME + 1. Ignored
+ * if NULL.
+ * @param fortran_order set to non-zero if using fortran array
+ * ordering, 0 for C array ordering.
+ * @returns 0 for success, error code otherwise.
+ */
+int pioc_write_nc_decomp_int(iosystem_desc_t *ios, const char *filename, int cmode, int ndims,
+                             int *global_dimlen, int num_tasks, int *task_maplen, int *map,
+                             const char *title, const char *history, int fortran_order)
+{
+    int max_maplen = 0;
+    int ncid;
+    int ret;
+
+    /* Check inputs. */
+    pioassert(ios && filename && global_dimlen && task_maplen &&
+              (!title || strlen(title) <= PIO_MAX_NAME) &&
+              (!history || strlen(history) <= PIO_MAX_NAME), "invalid input",
+              __FILE__, __LINE__);
+
+    LOG((2, "pioc_write_nc_decomp_int filename = %s ndims = %d num_tasks = %d", filename,
+         ndims, num_tasks));
+
+    /* Find the maximum maplen. */
+    for (int t = 0; t < num_tasks; t++)
+        if (task_maplen[t] > max_maplen)
+            max_maplen = task_maplen[t];
+    LOG((3, "max_maplen = %d", max_maplen));
+
+    /* Create the netCDF decomp file. */
+    if ((ret = PIOc_create(ios->iosysid, filename, cmode | NC_WRITE, &ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write an attribute with the version of this file. */
+    char version[PIO_MAX_NAME + 1];
+    sprintf(version, "%d.%d.%d", PIO_VERSION_MAJOR, PIO_VERSION_MINOR, PIO_VERSION_PATCH);
+    if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_VERSION_ATT_NAME,
+                                 strlen(version) + 1, version)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write an attribute with the max map len. */
+    if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, DECOMP_MAX_MAPLEN_ATT_NAME,
+                                PIO_INT, 1, &max_maplen)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write title attribute, if the user provided one. */
+    if (title)
+        if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_TITLE_ATT_NAME,
+                                     strlen(title) + 1, title)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write history attribute, if the user provided one. */
+    if (history)
+        if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_HISTORY_ATT_NAME,
+                                     strlen(history) + 1, history)))
+            return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write a source attribute. */
+    char source[] = "Decomposition file produced by PIO library.";
+    if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_SOURCE_ATT_NAME,
+                                 strlen(source) + 1, source)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write an attribute with array ordering (C or Fortran). */
+    char c_order_str[] = DECOMP_C_ORDER_STR;
+    char fortran_order_str[] = DECOMP_FORTRAN_ORDER_STR;
+    char *my_order_str = fortran_order ? fortran_order_str : c_order_str;
+    if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_ORDER_ATT_NAME,
+                                 strlen(my_order_str) + 1, my_order_str)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Write an attribute with the stack trace. This can be helpful
+     * for debugging. */
+    #define MAX_BACKTRACE 10
+    void *bt[MAX_BACKTRACE];
+    size_t bt_size;
+    char **bt_strings;
+    bt_size = backtrace(bt, MAX_BACKTRACE);
+    bt_strings = backtrace_symbols(bt, bt_size);
+
+    /* Find the max size. */
+    int max_bt_size = 0;
+    for (int b = 0; b < bt_size; b++)
+        if (strlen(bt_strings[b]) > max_bt_size)
+            max_bt_size = strlen(bt_strings[b]);
+    if (max_bt_size > NC_MAX_NAME)
+        max_bt_size = NC_MAX_NAME;
+
+    /* Copy the backtrace into one long string. */
+    char full_bt[max_bt_size * bt_size + bt_size + 1];
+    full_bt[0] = '\0';
+    for (int b = 0; b < bt_size; b++)
+    {
+        strncat(full_bt, bt_strings[b], max_bt_size);
+        strcat(full_bt, "\n");
+    }
+    free(bt_strings);
+    printf("full_bt = %s", full_bt);
+
+    /* Write the stack trace as an attribute. */
+    if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, DECOMP_BACKTRACE_ATT_NAME,
+                                 strlen(full_bt) + 1, full_bt)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* We need a dimension for the dimensions in the data. (Example:
+     * for 4D data we will need to store 4 dimension IDs.) */
+    int dim_dimid;
+    if ((ret = PIOc_def_dim(ncid, DECOMP_DIM_DIM, ndims, &dim_dimid)))
+        return ret;
+
+    /* We need a dimension for tasks. If we have 4 tasks, we need to
+     * store an array of length 4 with the size of the local array on
+     * each task. */
+    int task_dimid;
+    if ((ret = PIOc_def_dim(ncid, DECOMP_TASK_DIM_NAME, num_tasks, &task_dimid)))
+        return ret;
+
+    /* We need a dimension for the map. It's length may vary, we will
+     * use the max_maplen for the dimension size. */
+    int mapelem_dimid;
+    if ((ret = PIOc_def_dim(ncid, DECOMP_MAPELEM_DIM_NAME, max_maplen,
+                            &mapelem_dimid)))
+        return ret;
+
+    /* Define a var to hold the global size of the array for each
+     * dimension. */
+    int gsize_varid;
+    if ((ret = PIOc_def_var(ncid, DECOMP_GLOBAL_SIZE_VAR_NAME, NC_INT, 1,
+                            &dim_dimid, &gsize_varid)))
+        return ret;
+
+    /* Define a var to hold the length of the local array on each task. */
+    int maplen_varid;
+    if ((ret = PIOc_def_var(ncid, DECOMP_MAPLEN_VAR_NAME, NC_INT, 1, &task_dimid,
+                            &maplen_varid)))
+        return ret;
+
+    /* Define a 2D var to hold the length of the local array on each task. */
+    int map_varid;
+    int map_dimids[2] = {task_dimid, mapelem_dimid};
+    if ((ret = PIOc_def_var(ncid, DECOMP_MAP_VAR_NAME, NC_INT, 2, map_dimids,
+                            &map_varid)))
+        return ret;
+
+    /* End define mode, to write data. */
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+
+    /* Write the global dimension sizes. */
+    if ((PIOc_put_var_int(ncid, gsize_varid, global_dimlen)))
+        return ret;
+
+    /* Write the size of the local array on each task. */
+    if ((PIOc_put_var_int(ncid, maplen_varid, task_maplen)))
+        return ret;
+
+    /* Write the map. */
+    if ((PIOc_put_var_int(ncid, map_varid, map)))
+        return ret;
+
+    /* Close the netCDF decomp file. */
+    if ((ret = PIOc_closefile(ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/* Read the decomp information from a netCDF decomp file. This is an
+ * internal function.
+ *
+ * @param iosysid the IO system ID.
+ * @param filename the name the decomp file will have.
+ * @param ndims pointer to int that will get number of dims in the
+ * data being described. Ignored if NULL.
+ * @param global_dimlen a pointer that gets an array, of size ndims,
+ * that will have the size of the global array in each
+ * dimension. Ignored if NULL, otherwise must be freed by caller.
+ * @param num_tasks pointer to int that gets the number of tasks the
+ * data are decomposed over. Ignored if NULL.
+ * @param task_maplen pointer that gets array of size num_tasks that
+ * gets the length of the map for each task. Ignored if NULL,
+ * otherwise must be freed by caller.
+ * @param max_maplen pointer to int that gets the maximum maplen for
+ * any task. Ignored if NULL.
+ * @param map pointer that gets a 2D array of size [num_tasks][max_maplen]
+ * that will have the 0-based mapping from local to global array
+ * elements. Ignored if NULL, otherwise must be freed by caller.
+ * @param title pointer that will get the contents of title attribute,
+ * if present. If present, title will be < PIO_MAX_NAME + 1 in
+ * length. Ignored if NULL.
+ * @param history pointer that will get the contents of history attribute,
+ * if present. If present, history will be < PIO_MAX_NAME + 1 in
+ * length. Ignored if NULL.
+ * @param source pointer that will get the contents of source
+ * attribute. Source will be < PIO_MAX_NAME + 1 in length. Ignored if
+ * NULL.
+ * @param version pointer that will get the contents of version
+ * attribute. It will be < PIO_MAX_NAME + 1 in length. Ignored if
+ * NULL.
+ * @param fortran_order int pointer that will get a 0 if this
+ * decomposition file uses C array ordering, 1 if it uses Fortran
+ * array ordering.
+ * @returns 0 for success, error code otherwise.
+ */
+int pioc_read_nc_decomp_int(int iosysid, const char *filename, int *ndims, int **global_dimlen,
+                            int *num_tasks, int **task_maplen, int *max_maplen, int **map, char *title,
+                            char *history, char *source, char *version, int *fortran_order)
+{
+    iosystem_desc_t *ios;
+    int ncid;
+    int ret;
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check inputs. */
+    if (!filename)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "pioc_read_nc_decomp_int iosysid = %d filename = %s", iosysid, filename));
+
+    /* Open the netCDF decomp file. */
+    if ((ret = PIOc_open(iosysid, filename, NC_WRITE, &ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Read version attribute. */
+    char version_in[PIO_MAX_NAME + 1];
+    if ((ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_VERSION_ATT_NAME, version_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((3, "version_in = %s", version_in));
+    if (version)
+        strncpy(version, version_in, PIO_MAX_NAME + 1);
+
+    /* Read order attribute. */
+    char order_in[PIO_MAX_NAME + 1];
+    if ((ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_ORDER_ATT_NAME, order_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((3, "order_in = %s", order_in));
+    if (fortran_order)
+    {
+        if (!strncmp(order_in, DECOMP_C_ORDER_STR, PIO_MAX_NAME + 1))
+            *fortran_order = 0;
+        else if (!strncmp(order_in, DECOMP_FORTRAN_ORDER_STR, PIO_MAX_NAME + 1))
+            *fortran_order = 1;
+        else
+            return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+    }
+
+    /* Read attribute with the max map len. */
+    int max_maplen_in;
+    if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, DECOMP_MAX_MAPLEN_ATT_NAME, &max_maplen_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    LOG((3, "max_maplen_in = %d", max_maplen_in));
+    if (max_maplen)
+        *max_maplen = max_maplen_in;
+
+    /* Read title attribute, if it is in the file. */
+    char title_in[NC_MAX_NAME + 1];
+    ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_TITLE_ATT_NAME, title_in);
+    if (ret == PIO_NOERR)
+    {
+        /* If the caller wants it, copy the title for them. */
+        if (title)
+            strncpy(title, title_in, PIO_MAX_NAME + 1);
+    }
+    else if (ret == PIO_ENOTATT)
+    {
+        /* No title attribute. */
+        if (title)
+            title[0] = '\0';
+    }
+    else
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Read history attribute, if it is in the file. */
+    char history_in[NC_MAX_NAME + 1];
+    ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_HISTORY_ATT_NAME, history_in);
+    if (ret == PIO_NOERR)
+    {
+        /* If the caller wants it, copy the history for them. */
+        if (history)
+            strncpy(history, history_in, PIO_MAX_NAME + 1);
+    }
+    else if (ret == PIO_ENOTATT)
+    {
+        /* No history attribute. */
+        if (history)
+            history[0] = '\0';
+    }
+    else
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    /* Read source attribute. */
+    char source_in[NC_MAX_NAME + 1];
+    if ((ret = PIOc_get_att_text(ncid, NC_GLOBAL, DECOMP_SOURCE_ATT_NAME, source_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (source)
+        strncpy(source, source_in, PIO_MAX_NAME + 1);
+
+    /* Read dimension for the dimensions in the data. (Example: for 4D
+     * data we will need to store 4 dimension IDs.) */
+    int dim_dimid;
+    PIO_Offset ndims_in;
+    if ((ret = PIOc_inq_dimid(ncid, DECOMP_DIM_DIM, &dim_dimid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if ((ret = PIOc_inq_dim(ncid, dim_dimid, NULL, &ndims_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (ndims)
+        *ndims = ndims_in;
+
+    /* Read the global sizes of the array. */
+    int gsize_varid;
+    int global_dimlen_in[ndims_in];
+    if ((ret = PIOc_inq_varid(ncid, DECOMP_GLOBAL_SIZE_VAR_NAME, &gsize_varid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if ((ret = PIOc_get_var_int(ncid, gsize_varid, global_dimlen_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (global_dimlen)
+    {
+        if (!(*global_dimlen = malloc(ndims_in * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for (int d = 0; d < ndims_in; d++)
+            (*global_dimlen)[d] = global_dimlen_in[d];
+    }
+
+    /* Read dimension for tasks. If we have 4 tasks, we need to store
+     * an array of length 4 with the size of the local array on each
+     * task. */
+    int task_dimid;
+    PIO_Offset num_tasks_in;
+    if ((ret = PIOc_inq_dimid(ncid, DECOMP_TASK_DIM_NAME, &task_dimid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if ((ret = PIOc_inq_dim(ncid, task_dimid, NULL, &num_tasks_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (num_tasks)
+        *num_tasks = num_tasks_in;
+
+    /* Read the length if the local array on each task. */
+    int maplen_varid;
+    int task_maplen_in[num_tasks_in];
+    if ((ret = PIOc_inq_varid(ncid, DECOMP_MAPLEN_VAR_NAME, &maplen_varid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if ((ret = PIOc_get_var_int(ncid, maplen_varid, task_maplen_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (task_maplen)
+    {
+        if (!(*task_maplen = malloc(num_tasks_in * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for (int t = 0; t < num_tasks_in; t++)
+            (*task_maplen)[t] = task_maplen_in[t];
+    }
+
+    /* Read the map. */
+    int map_varid;
+    int map_in[num_tasks_in][max_maplen_in];
+    if ((ret = PIOc_inq_varid(ncid, DECOMP_MAP_VAR_NAME, &map_varid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if ((ret = PIOc_get_var_int(ncid, map_varid, (int *)map_in)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+    if (map)
+    {
+        if (!(*map = malloc(num_tasks_in * max_maplen_in * sizeof(int))))
+            return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+        for (int t = 0; t < num_tasks_in; t++)
+            for (int l = 0; l < max_maplen_in; l++)
+                (*map)[t * max_maplen_in + l] = map_in[t][l];
+    }
+
+    /* Close the netCDF decomp file. */
+    if ((ret = PIOc_closefile(ncid)))
+        return pio_err(ios, NULL, ret, __FILE__, __LINE__);
+
+    return PIO_NOERR;
+}
+
+/**
+ * Write the decomposition map to a file.
+ *
+ * @param file the filename to be used.
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the IO description.
+ * @param comm an MPI communicator.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_write_decomp(const char *file, int iosysid, int ioid, MPI_Comm comm)
+{
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+
+    LOG((1, "PIOc_write_decomp file = %s iosysid = %d ioid = %d", file, iosysid, ioid));
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    if (!(iodesc = pio_get_iodesc_from_id(ioid)))
+        return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    return PIOc_writemap(file, iodesc->ndims, iodesc->dimlen, iodesc->maplen, iodesc->map,
+                         comm);
+}
+
+/**
+ * Write the decomposition map to a file.
+ *
+ * @param file the filename
+ * @param ndims the number of dimensions
+ * @param gdims an array of dimension ids
+ * @param maplen the length of the map
+ * @param map the map array
+ * @param comm an MPI communicator.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_writemap(const char *file, int ndims, const int *gdims, PIO_Offset maplen,
+                  PIO_Offset *map, MPI_Comm comm)
+{
+    int npes, myrank;
+    PIO_Offset *nmaplen = NULL;
+    MPI_Status status;
+    int i;
+    PIO_Offset *nmap;
+    int mpierr; /* Return code for MPI calls. */
+
+    LOG((1, "PIOc_writemap file = %s ndims = %d maplen = %d", file, ndims, maplen));
+
+    if ((mpierr = MPI_Comm_size(comm, &npes)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    if ((mpierr = MPI_Comm_rank(comm, &myrank)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    LOG((2, "npes = %d myrank = %d", npes, myrank));
+
+    /* Allocate memory for the nmaplen. */
+    if (myrank == 0)
+        if (!(nmaplen = malloc(npes * sizeof(PIO_Offset))))
+            return pio_err(NULL, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    if ((mpierr = MPI_Gather(&maplen, 1, PIO_OFFSET, nmaplen, 1, PIO_OFFSET, 0, comm)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+
+    /* Only rank 0 writes the file. */
+    if (myrank == 0)
+    {
+        FILE *fp;
+
+        /* Open the file to write. */
+        if (!(fp = fopen(file, "w")))
+            return pio_err(NULL, NULL, PIO_EIO, __FILE__, __LINE__);
+
+        /* Write the version and dimension info. */
+        fprintf(fp,"version %d npes %d ndims %d \n", VERSNO, npes, ndims);
+        for (i = 0; i < ndims; i++)
+            fprintf(fp, "%d ", gdims[i]);
+        fprintf(fp, "\n");
+
+        /* Write the map. */
+        fprintf(fp, "0 %lld\n", nmaplen[0]);
+        for (i = 0; i < nmaplen[0]; i++)
+            fprintf(fp, "%lld ", map[i]);
+        fprintf(fp,"\n");
+
+        for (i = 1; i < npes; i++)
+        {
+            LOG((2, "creating nmap for i = %d", i));
+            nmap = (PIO_Offset *)malloc(nmaplen[i] * sizeof(PIO_Offset));
+
+            if ((mpierr = MPI_Send(&i, 1, MPI_INT, i, npes + i, comm)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            if ((mpierr = MPI_Recv(nmap, nmaplen[i], PIO_OFFSET, i, i, comm, &status)))
+                return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+            LOG((2,"MPI_Recv map complete"));
+
+            fprintf(fp, "%d %lld\n", i, nmaplen[i]);
+            for (int j = 0; j < nmaplen[i]; j++)
+                fprintf(fp, "%lld ", nmap[j]);
+            fprintf(fp, "\n");
+
+            free(nmap);
+        }
+        /* Free memory for the nmaplen. */
+        free(nmaplen);
+        fprintf(fp, "\n");
+        print_trace(fp);
+
+        /* Close the file. */
+        fclose(fp);
+        LOG((2,"decomp file closed."));
+    }
+    else
+    {
+        LOG((2,"ready to MPI_Recv..."));
+        if ((mpierr = MPI_Recv(&i, 1, MPI_INT, 0, npes+myrank, comm, &status)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        LOG((2,"MPI_Recv got %d", i));
+        if ((mpierr = MPI_Send(map, maplen, PIO_OFFSET, 0, myrank, comm)))
+            return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+        LOG((2,"MPI_Send map complete"));
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Write the decomposition map to a file for F90.
+ *
+ * @param file the filename
+ * @param ndims the number of dimensions
+ * @param gdims an array of dimension ids
+ * @param maplen the length of the map
+ * @param map the map array
+ * @param comm an MPI communicator.
+ * @returns 0 for success, error code otherwise.
+ */
+int PIOc_writemap_from_f90(const char *file, int ndims, const int *gdims,
+                           PIO_Offset maplen, const PIO_Offset *map, int f90_comm)
+{
+    return PIOc_writemap(file, ndims, gdims, maplen, (PIO_Offset *)map,
+                         MPI_Comm_f2c(f90_comm));
+}
+
+/**
+ * Create a new file using pio. This is an internal function that is
+ * called by both PIOc_create() and PIOc_createfile(). Input
+ * parameters are read on comp task 0 and ignored elsewhere.
+ *
+ * @param iosysid A defined pio system ID, obtained from
+ * PIOc_InitIntercomm() or PIOc_InitAsync().
+ * @param ncidp A pointer that gets the ncid of the newly created
+ * file.
+ * @param iotype A pointer to a pio output format. Must be one of
+ * PIO_IOTYPE_PNETCDF, PIO_IOTYPE_NETCDF, PIO_IOTYPE_NETCDF4C, or
+ * PIO_IOTYPE_NETCDF4P.
+ * @param filename The filename to create.
+ * @param mode The netcdf mode for the create operation.
+ * @returns 0 for success, error code otherwise.
+ * @ingroup PIO_createfile
+ */
+int PIOc_createfile_int(int iosysid, int *ncidp, int *iotype, const char *filename,
+                        int mode)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI function codes. */
+    int ierr;              /* Return code from function calls. */
+
+    /* Get the IO system info from the iosysid. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* User must provide valid input for these parameters. */
+    if (!ncidp || !iotype || !filename || strlen(filename) > NC_MAX_NAME)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    /* A valid iotype must be specified. */
+    if (!iotype_is_valid(*iotype))
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((1, "PIOc_createfile iosysid = %d iotype = %d filename = %s mode = %d",
+         iosysid, *iotype, filename, mode));
+
+    /* Allocate space for the file info. */
+    if (!(file = calloc(sizeof(file_desc_t), 1)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Fill in some file values. */
+    file->fh = -1;
+    file->iosystem = ios;
+    file->iotype = *iotype;
+    file->buffer.ioid = -1;
+    for (int i = 0; i < PIO_MAX_VARS; i++)
+    {
+        file->varlist[i].record = -1;
+        file->varlist[i].ndims = -1;
+	file->varlist[i].iobuf = NULL;
+	file->varlist[i].fillbuf = NULL;
+    }
+    file->mode = mode;
+
+    /* Set to true if this task should participate in IO (only true for
+     * one task with netcdf serial files. */
+    if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF ||
+        ios->io_rank == 0)
+        file->do_io = 1;
+
+    LOG((2, "file->do_io = %d ios->async = %d", file->do_io, ios->async));
+
+    /* If async is in use, and this is not an IO task, bcast the
+     * parameters. */
+    if (ios->async)
+    {
+        int msg = PIO_MSG_CREATE_FILE;
+        size_t len = strlen(filename);
+
+        if (!ios->ioproc)
+        {
+            /* Send the message to the message handler. */
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the parameters of the function call. */
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            LOG((2, "len = %d filename = %s iotype = %d mode = %d", len, filename,
+                 file->iotype, file->mode));
+        }
+
+        /* Handle MPI errors. */
+        LOG((2, "handling mpi errors mpierr = %d", mpierr));
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this task is in the IO component, do the IO. */
+    if (ios->ioproc)
+    {
+        switch (file->iotype)
+        {
+#ifdef _NETCDF4
+        case PIO_IOTYPE_NETCDF4P:
+            file->mode = file->mode |  NC_MPIIO | NC_NETCDF4;
+            LOG((2, "Calling nc_create_par io_comm = %d mode = %d fh = %d",
+                 ios->io_comm, file->mode, file->fh));
+            ierr = nc_create_par(filename, file->mode, ios->io_comm, ios->info, &file->fh);
+            LOG((2, "nc_create_par returned %d file->fh = %d", ierr, file->fh));
+            break;
+        case PIO_IOTYPE_NETCDF4C:
+            file->mode = file->mode | NC_NETCDF4;
+#endif
+        case PIO_IOTYPE_NETCDF:
+            if (!ios->io_rank)
+            {
+                LOG((2, "Calling nc_create mode = %d", file->mode));
+                ierr = nc_create(filename, file->mode, &file->fh);
+            }
+            break;
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            LOG((2, "Calling ncmpi_create mode = %d", file->mode));
+            ierr = ncmpi_create(ios->io_comm, filename, file->mode, ios->info, &file->fh);
+            if (!ierr)
+                ierr = ncmpi_buffer_attach(file->fh, pio_buffer_size_limit);
+            break;
+#endif
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    /* If there was an error, free the memory we allocated and handle error. */
+    if (ierr)
+    {
+        free(file);
+        return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
+    }
+
+    /* Broadcast mode to all tasks. */
+    if ((mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->union_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    /* This flag is implied by netcdf create functions but we need
+       to know if its set. */
+    file->mode = file->mode | PIO_WRITE;
+
+    /* Assign the PIO ncid, necessary because files may be opened
+     * on mutilple iosystems, causing the underlying library to
+     * reuse ncids. Hilarious confusion ensues. */
+    file->pio_ncid = pio_next_ncid++;
+    LOG((2, "file->fh = %d file->pio_ncid = %d", file->fh, file->pio_ncid));
+
+    /* Return the ncid to the caller. */
+    *ncidp = file->pio_ncid;
+
+    /* Add the struct with this files info to the global list of
+     * open files. */
+    pio_add_to_file_list(file);
+
+    LOG((2, "Created file %s file->fh = %d file->pio_ncid = %d", filename,
+         file->fh, file->pio_ncid));
+
+    return ierr;
+}
+
+/**
+ * Open an existing file using PIO library. This is an internal
+ * function. Depending on the value of the retry parameter, a failed
+ * open operation will be handled differently. If retry is non-zero,
+ * then a failed attempt to open a file with netCDF-4 (serial or
+ * parallel), or parallel-netcdf will be followed by an attempt to
+ * open the file as a serial classic netCDF file. This is an important
+ * feature to some NCAR users. The functionality is exposed to the
+ * user as PIOc_openfile() (which does the retry), and PIOc_open()
+ * (which does not do the retry).
+ *
+ * Input parameters are read on comp task 0 and ignored elsewhere.
+ *
+ * @param iosysid: A defined pio system descriptor (input)
+ * @param ncidp: A pio file descriptor (output)
+ * @param iotype: A pio output format (input)
+ * @param filename: The filename to open
+ * @param mode: The netcdf mode for the open operation
+ * @param retry: non-zero to automatically retry with netCDF serial
+ * classic.
+ *
+ * @return 0 for success, error code otherwise.
+ * @ingroup PIO_openfile
+ */
+int PIOc_openfile_retry(int iosysid, int *ncidp, int *iotype, const char *filename,
+                        int mode, int retry)
+{
+    iosystem_desc_t *ios;  /** Pointer to io system information. */
+    file_desc_t *file;     /** Pointer to file information. */
+    int imode;  /** internal mode val for netcdf4 file open */
+    int mpierr = MPI_SUCCESS, mpierr2;  /** Return code from MPI function codes. */
+    int ierr = PIO_NOERR;  /** Return code from function calls. */
+
+    /* Get the IO system info from the iosysid. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* User must provide valid input for these parameters. */
+    if (!ncidp || !iotype || !filename)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+    if (*iotype < PIO_IOTYPE_PNETCDF || *iotype > PIO_IOTYPE_NETCDF4P)
+        return pio_err(ios, NULL, PIO_EINVAL, __FILE__, __LINE__);
+
+    LOG((2, "PIOc_openfile_retry iosysid = %d iotype = %d filename = %s mode = %d retry = %d",
+         iosysid, *iotype, filename, mode, retry));
+
+    /* Allocate space for the file info. */
+    if (!(file = calloc(sizeof(*file), 1)))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+
+    /* Fill in some file values. */
+    file->fh = -1;
+    file->iotype = *iotype;
+    file->iosystem = ios;
+    file->mode = mode;
+
+    for (int i = 0; i < PIO_MAX_VARS; i++)
+    {
+        file->varlist[i].record = -1;
+        file->varlist[i].ndims = -1;
+    }
+
+    /* Set to true if this task should participate in IO (only true
+     * for one task with netcdf serial files. */
+    if (file->iotype == PIO_IOTYPE_NETCDF4P || file->iotype == PIO_IOTYPE_PNETCDF ||
+        ios->io_rank == 0)
+        file->do_io = 1;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        int msg = PIO_MSG_OPEN_FILE;
+        size_t len = strlen(filename);
+
+        if (!ios->ioproc)
+        {
+            /* Send the message to the message handler. */
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            /* Send the parameters of the function call. */
+            if (!mpierr)
+                mpierr = MPI_Bcast(&len, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast((void *)filename, len + 1, MPI_CHAR, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->iotype, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            if (!mpierr)
+                mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->compmaster, ios->intercomm);
+        }
+
+        /* Handle MPI errors. */
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            return check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    if (ios->ioproc)
+    {
+        switch (file->iotype)
+        {
+#ifdef _NETCDF4
+
+        case PIO_IOTYPE_NETCDF4P:
+#ifdef _MPISERIAL
+            ierr = nc_open(filename, file->mode, &file->fh);
+#else
+            imode = file->mode |  NC_MPIIO;
+            ierr = nc_open_par(filename, imode, ios->io_comm, ios->info, &file->fh);
+            if (ierr == PIO_NOERR)
+                file->mode = imode;
+            LOG((2, "PIOc_openfile_retry:nc_open_par filename = %s mode = %d imode = %d ierr = %d",
+                 filename, file->mode, imode, ierr));
+#endif
+            break;
+
+        case PIO_IOTYPE_NETCDF4C:
+            if (ios->io_rank == 0)
+            {
+                imode = file->mode | NC_NETCDF4;
+                ierr = nc_open(filename, imode, &file->fh);
+                if (ierr == PIO_NOERR)
+                    file->mode = imode;
+            }
+            break;
+
+#endif
+
+        case PIO_IOTYPE_NETCDF:
+            if (ios->io_rank == 0)
+                ierr = nc_open(filename, file->mode, &file->fh);
+            break;
+
+#ifdef _PNETCDF
+        case PIO_IOTYPE_PNETCDF:
+            ierr = ncmpi_open(ios->io_comm, filename, file->mode, ios->info, &file->fh);
+
+            // This should only be done with a file opened to append
+            if (ierr == PIO_NOERR && (file->mode & PIO_WRITE))
+            {
+                if (ios->iomaster == MPI_ROOT)
+                    LOG((2, "%d Setting IO buffer %ld", __LINE__, pio_buffer_size_limit));
+                ierr = ncmpi_buffer_attach(file->fh, pio_buffer_size_limit);
+            }
+            LOG((2, "ncmpi_open(%s) : fd = %d", filename, file->fh));
+            break;
+#endif
+
+        default:
+            return pio_err(ios, file, PIO_EBADIOTYPE, __FILE__, __LINE__);
+        }
+
+        /* If the caller requested a retry, and we failed to open a
+           file due to an incompatible type of NetCDF, try it once
+           with just plain old basic NetCDF. */
+        if (retry)
+        {
+            LOG((2, "retry error code ierr = %d io_rank %d", ierr, ios->io_rank));
+            if ((ierr == NC_ENOTNC || ierr == NC_EINVAL) && (file->iotype != PIO_IOTYPE_NETCDF))
+            {
+                if (ios->iomaster == MPI_ROOT)
+                    printf("PIO2 pio_file.c retry NETCDF\n");
+
+                /* reset ierr on all tasks */
+                ierr = PIO_NOERR;
+
+                /* reset file markers for NETCDF on all tasks */
+                file->iotype = PIO_IOTYPE_NETCDF;
+
+                /* open netcdf file serially on main task */
+                if (ios->io_rank == 0)
+                    ierr = nc_open(filename, file->mode, &file->fh);
+                else
+                    file->do_io = 0;
+            }
+            LOG((2, "retry nc_open(%s) : fd = %d, iotype = %d, do_io = %d, ierr = %d", filename, file->fh, file->iotype, file->do_io, ierr));
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    LOG((2, "Bcasting error code ierr = %d ios->ioroot = %d ios->my_comm = %d", ierr, ios->ioroot,
+         ios->my_comm));
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    LOG((2, "Bcast error code ierr = %d", ierr));
+
+    /* If there was an error, free allocated memory and deal with the error. */
+    if (ierr)
+    {
+        free(file);
+        return check_netcdf2(ios, NULL, ierr, __FILE__, __LINE__);
+    }
+    LOG((2, "error code Bcast complete ierr = %d ios->my_comm = %d", ierr, ios->my_comm));
+
+    /* Broadcast results to all tasks. Ignore NULL parameters. */
+    if ((mpierr = MPI_Bcast(&file->mode, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+
+    /* Create the ncid that the user will see. This is necessary
+     * because otherwise ncids will be reused if files are opened
+     * on multiple iosystems. */
+    file->pio_ncid = pio_next_ncid++;
+
+    /* Return the PIO ncid to the user. */
+    *ncidp = file->pio_ncid;
+
+    /* Add this file to the list of currently open files. */
+    pio_add_to_file_list(file);
+
+    LOG((2, "Opened file %s file->pio_ncid = %d file->fh = %d ierr = %d",
+         filename, file->pio_ncid, file->fh, ierr));
+
+    return ierr;
+}
+
+/**
+ * Internal function to provide inq_type function for pnetcdf.
+ *
+ * @param ncid ignored because pnetcdf does not have user-defined
+ * types.
+ * @param xtype type to learn about.
+ * @param name pointer that gets name of type. Ignored if NULL.
+ * @param sizep pointer that gets size of type. Ignored if NULL.
+ * @returns 0 on success, error code otherwise.
+ */
+int pioc_pnetcdf_inq_type(int ncid, nc_type xtype, char *name,
+                          PIO_Offset *sizep)
+{
+    int typelen;
+
+    switch (xtype)
+    {
+    case NC_UBYTE:
+    case NC_BYTE:
+    case NC_CHAR:
+        typelen = 1;
+        break;
+    case NC_SHORT:
+    case NC_USHORT:
+        typelen = 2;
+        break;
+    case NC_UINT:
+    case NC_INT:
+    case NC_FLOAT:
+        typelen = 4;
+        break;
+    case NC_UINT64:
+    case NC_INT64:
+    case NC_DOUBLE:
+        typelen = 8;
+        break;
+    default:
+        return PIO_EBADTYPE;
+    }
+
+    /* If pointers were supplied, copy results. */
+    if (sizep)
+        *sizep = typelen;
+    if (name)
+        strcpy(name, "some type");
+
+    return PIO_NOERR;
+}
+
+/**
+ * This is an internal function that handles both PIOc_enddef and
+ * PIOc_redef.
+ *
+ * @param ncid the ncid of the file to enddef or redef
+ * @param is_enddef set to non-zero for enddef, 0 for redef.
+ * @returns PIO_NOERR on success, error code on failure. */
+int pioc_change_def(int ncid, int is_enddef)
+{
+    iosystem_desc_t *ios;  /* Pointer to io system information. */
+    file_desc_t *file;     /* Pointer to file information. */
+    int ierr = PIO_NOERR;  /* Return code from function calls. */
+    int mpierr = MPI_SUCCESS, mpierr2;  /* Return code from MPI functions. */
+
+    LOG((2, "pioc_change_def ncid = %d is_enddef = %d", ncid, is_enddef));
+
+    /* Find the info about this file. When I check the return code
+     * here, some tests fail. ???*/
+    if ((ierr = pio_get_file(ncid, &file)))
+        return pio_err(NULL, NULL, ierr, __FILE__, __LINE__);
+    ios = file->iosystem;
+
+    /* If async is in use, and this is not an IO task, bcast the parameters. */
+    if (ios->async)
+    {
+        if (!ios->ioproc)
+        {
+            int msg = is_enddef ? PIO_MSG_ENDDEF : PIO_MSG_REDEF;
+            if (ios->compmaster == MPI_ROOT)
+                mpierr = MPI_Send(&msg, 1, MPI_INT, ios->ioroot, 1, ios->union_comm);
+
+            if (!mpierr)
+                mpierr = MPI_Bcast(&ncid, 1, MPI_INT, ios->compmaster, ios->intercomm);
+            LOG((3, "pioc_change_def ncid = %d mpierr = %d", ncid, mpierr));
+        }
+
+        /* Handle MPI errors. */
+        LOG((3, "pioc_change_def handling MPI errors"));
+        if ((mpierr2 = MPI_Bcast(&mpierr, 1, MPI_INT, ios->comproot, ios->my_comm)))
+            check_mpi(file, mpierr2, __FILE__, __LINE__);
+        if (mpierr)
+            return check_mpi(file, mpierr, __FILE__, __LINE__);
+    }
+
+    /* If this is an IO task, then call the netCDF function. */
+    LOG((3, "pioc_change_def ios->ioproc = %d", ios->ioproc));
+    if (ios->ioproc)
+    {
+        LOG((3, "pioc_change_def calling netcdf function file->fh = %d file->do_io = %d iotype = %d",
+             file->fh, file->do_io, file->iotype));
+#ifdef _PNETCDF
+        if (file->iotype == PIO_IOTYPE_PNETCDF)
+        {
+            if (is_enddef)
+                ierr = ncmpi_enddef(file->fh);
+            else
+                ierr = ncmpi_redef(file->fh);
+        }
+#endif /* _PNETCDF */
+        if (file->iotype != PIO_IOTYPE_PNETCDF && file->do_io)
+        {
+            if (is_enddef)
+            {
+                LOG((3, "pioc_change_def calling nc_enddef file->fh = %d", file->fh));
+                ierr = nc_enddef(file->fh);
+            }
+            else
+                ierr = nc_redef(file->fh);
+        }
+    }
+
+    /* Broadcast and check the return code. */
+    LOG((3, "pioc_change_def bcasting return code ierr = %d", ierr));
+    if ((mpierr = MPI_Bcast(&ierr, 1, MPI_INT, ios->ioroot, ios->my_comm)))
+        return check_mpi(file, mpierr, __FILE__, __LINE__);
+    if (ierr)
+        return check_netcdf(file, ierr, __FILE__, __LINE__);
+    LOG((3, "pioc_change_def succeeded"));
+
+    return ierr;
+}
+
+/**
+ * Check whether an IO type is valid for the build.
+ *
+ * @param iotype the IO type to check
+ * @returns 0 if valid, non-zero otherwise.
+ */
+int iotype_is_valid(int iotype)
+{
+    /* Assume it's not valid. */
+    int ret = 0;
+
+    /* All builds include netCDF. */
+    if (iotype == PIO_IOTYPE_NETCDF)
+        ret++;
+
+    /* Some builds include netCDF-4. */
+#ifdef _NETCDF4
+    if (iotype == PIO_IOTYPE_NETCDF4C || iotype == PIO_IOTYPE_NETCDF4P)
+        ret++;
+#endif /* _NETCDF4 */
+
+    /* Some builds include pnetcdf. */
+    if (iotype == PIO_IOTYPE_PNETCDF)
+        ret++;
+#ifdef _PNETCDF
+#endif /* _PNETCDF */
+
+    return ret;
+}
+
+/**
+ * Set the rearranger options associated with an iosystem
+ *
+ * @param comm_type Type of communication (pt2pt/coll) used
+ * by the rearranger. See PIO_REARR_COMM_TYPE for more detail.
+ * Possible values are :
+ * PIO_REARR_COMM_P2P (Point to point communication)
+ * PIO_REARR_COMM_COLL (Collective communication)
+ * @param fcd Flow control direction for the rearranger.
+ * See PIO_REARR_COMM_FC_DIR for more detail.
+ * Possible values are :
+ * PIO_REARR_COMM_FC_2D_ENABLE : Enable flow control from
+ * compute processes to io processes and vice versa
+ * PIO_REARR_COMM_FC_1D_COMP2IO : Enable flow control from
+ * compute processes to io processes (only)
+ * PIO_REARR_COMM_FC_1D_IO2COMP : Enable flow control from
+ * io processes to compute processes (only)
+ * PIO_REARR_COMM_FC_2D_DISABLE : Disable flow control from
+ * compute processes to io processes and vice versa.
+ * @param enable_hs_c2i Enable handshake while rearranging
+ * data, from compute to io processes
+ * @param enable_isend_c2i Enable isends while rearranging
+ * data, from compute to io processes
+ * @param max_pend_req_c2i Maximum pending requests during
+ * data rearragment from compute processes to io processes
+ * @param enable_hs_i2c Enable handshake while rearranging
+ * data, from io to compute processes
+ * @param enable_isend_i2c Enable isends while rearranging
+ * data, from io to compute processes
+ * @param max_pend_req_i2c Maximum pending requests during
+ * data rearragment from io processes to compute processes
+ * @param iosysidp index of the defined system descriptor
+ * @return 0 on success, otherwise a PIO error code.
+ */
+int PIOc_set_rearr_opts(int iosysid, int comm_type, int fcd, bool enable_hs_c2i,
+                        bool enable_isend_c2i, int max_pend_req_c2i,
+                        bool enable_hs_i2c, bool enable_isend_i2c,
+                        int max_pend_req_i2c)
+{
+    iosystem_desc_t *ios;
+    rearr_opt_t user_rearr_opts = {
+        comm_type, fcd,
+        {enable_hs_c2i,enable_isend_c2i, max_pend_req_c2i},
+        {enable_hs_i2c, enable_isend_i2c, max_pend_req_i2c}
+    };
+
+    /* Check inputs. */
+    if ((comm_type != PIO_REARR_COMM_P2P && comm_type != PIO_REARR_COMM_FC_1D_COMP2IO) ||
+        (fcd < 0 || fcd > PIO_REARR_COMM_FC_2D_DISABLE) ||
+        (max_pend_req_c2i != PIO_REARR_COMM_UNLIMITED_PEND_REQ && max_pend_req_c2i < 0) ||
+        (max_pend_req_i2c != PIO_REARR_COMM_UNLIMITED_PEND_REQ && max_pend_req_i2c < 0))
+        return pio_err(NULL, NULL, PIO_EINVAL, __FILE__, __LINE__);        
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Set the options. */
+    ios->rearr_opts = user_rearr_opts;
+
+    return PIO_NOERR;
+}
diff --git a/src/externals/pio2/src/clib/topology.c b/src/externals/pio2/src/clib/topology.c
new file mode 100644
index 00000000000..f8e1fc016be
--- /dev/null
+++ b/src/externals/pio2/src/clib/topology.c
@@ -0,0 +1,363 @@
+#if defined(BGQDONT)
+
+#include 
+#include 
+
+#include 
+
+#include 
+#include 
+#include 
+#include 
+
+#define   Personality                    Personality_t
+
+int rank;
+int np;
+int my_name_len;
+char my_name[255];
+
+void identity(const MPI_Comm comm, int *iotask)
+{
+    MPIX_Hardware_t hw;
+    char message[100];
+    /* Number of MPI tasks per Pset */
+    int coreId;
+    int *TasksPerPset;
+    int *tmp;
+    int i,ierr;
+    Personality_t pers;
+
+    MPI_Comm_rank(comm,&rank);
+    MPI_Comm_size(comm,&np);
+    MPI_Get_processor_name(my_name, &my_name_len);
+
+
+    MPIX_Hardware(&hw);
+
+    Kernel_GetPersonality(&pers, sizeof(pers));
+
+    int numIONodes,numPsets,numNodesInPset,rankInPset;
+    int numpsets, psetID, psetsize, psetrank;
+
+    bgq_pset_info (comm, &numpsets, &psetID, &psetsize, &psetrank);
+
+    numIONodes = numpsets;
+    numNodesInPset = psetsize;
+    rankInPset = rank;
+    numPsets = numpsets;
+
+    if(rank == 0) { printf("number of IO nodes in block: %i \n",numIONodes);}
+    if(rank == 0) { printf("number of Psets in block : %i \n",numPsets);}
+    if(rank == 0) { printf("number of compute nodes in Pset: %i \n",numNodesInPset);}
+
+    int psetNum;
+    psetNum = psetID;
+
+#ifdef DEBUG
+    if((*iotask)>0) {
+        printf( "%04i (%-50s %s) %i yes\n", rank, my_name, message, psetNum );
+    } else {
+        printf( "%04i (%-50s %s) %i --\n", rank, my_name, message, psetNum);
+    }
+    printf("MPI task %6i is rank %3i in Pset: %3i \n",rank, rankInPset,psetNum);
+#endif
+    /* Determine which core on node....  I don't want to put more than one io-task per node */
+    coreId = Kernel_PhysicalProcessorID ();
+
+    TasksPerPset = malloc(numPsets*sizeof(int));
+    tmp = malloc(numPsets*sizeof(int));
+    for(i=0;i 0) {
+
+
+        Kernel_GetPersonality(&pers, sizeof(pers));
+
+        int numIONodes,numPsets,numNodesInPset,rankInPset;
+        int numiotasks_per_node,remainder,numIONodes_per_pset;
+        int lstride;
+
+        /* Number of computational nodes in processor set */
+
+        int numpsets, psetID, psetsize, psetrank;
+        bgq_pset_info (comm,&numpsets, &psetID, &psetsize, &psetrank);
+        numIONodes = numpsets;
+        numNodesInPset = psetsize;
+
+        /*     printf("Determine io tasks: me %i : nodes in pset= %i ionodes = %i\n", rank, numNodesInPset, numIONodes); */
+
+        if((*numiotasks) < 0 ) {
+            /* negative numiotasks value indicates that this is the number per IO-node */
+            (*numiotasks) = - (*numiotasks);
+            if((*numiotasks) > numNodesInPset) {
+                numiotasks_per_node = numNodesInPset;
+            } else  {
+                numiotasks_per_node = (*numiotasks);
+            }
+            remainder = 0;
+        } else if ((*numiotasks) > 0 ) {
+            /* balance the number of iotasks to number of IO nodes */
+            numiotasks_per_node = floor((float)(*numiotasks)/ (float) numIONodes);
+            /* put a minumum here so that we have a chance  - though this may be too low */
+            if (numiotasks_per_node < 1) {
+                numiotasks_per_node = 1;
+                *numiotasks = numIONodes;
+            }
+            remainder = (*numiotasks) - numiotasks_per_node * numIONodes;
+        } else if ((*numiotasks) == 0 ) {
+            if((*stride) > 0) {
+                numiotasks_per_node = numNodesInPset/(*stride);
+                if (numiotasks_per_node < 1) {
+                    numiotasks_per_node = 1;
+                    *numiotasks = numIONodes;
+                }
+            } else {
+                numiotasks_per_node = 8;  /* default number of IO-client per IO-node is not otherwise specificied */
+            }
+            remainder = 0;
+        }
+
+        /* number of IO nodes with a larger number of io-client per io-node */
+        if(remainder > 0) {
+            if(rank ==0) {printf("Unbalanced IO-configuration: %i IO-nodes have %i IO-clients : %i IO-nodes have %i IO-clients \n",
+                                 remainder, numiotasks_per_node+1, numIONodes-remainder,numiotasks_per_node);}
+            lstride = min(np,floor((float)numNodesInPset/(float)(numiotasks_per_node+1)));
+        } else {
+            if(rank == 0) {
+                printf("Balanced IO-configuration: %i IO-nodes have %i IO-clients\n",numIONodes-remainder, numiotasks_per_node);
+            }
+            lstride = min(np,floor((float)numNodesInPset/(float)numiotasks_per_node));
+        }
+
+        /* Number of processor sets */
+        numPsets = numpsets;
+
+        /* number of IO nodes in processor set (I need to add
+           code to deal with the case where numIONodes_per_pset != 1 works
+           correctly) */
+        numIONodes_per_pset = numIONodes/numPsets;
+
+        /* Determine which core on node....  I don't want to put more than one io-task per node */
+        coreId = Kernel_PhysicalProcessorID ();
+
+        /* What is the rank of this node in the processor set */
+
+        psetNum = psetID;
+        rankInPset = psetrank;
+
+        /* printf("Pset #: %i has %i nodes in Pset; base = %i\n",psetNum,numNodesInPset, *base); */
+
+        (*iamIOtask) = false;   /* initialize to zero */
+
+        if (numiotasks_per_node == numNodesInPset)(*base) = 0;  /* Reset the base to 0 if we are using all tasks */
+
+
+        /* start stridding MPI tasks from base task */
+        iam = max(0,rankInPset-(*base));
+        if (iam >= 0)  {
+            /* mark tasks that will be IO-tasks  or IO-clients */
+            /*    printf("iam = %d lstride = %d coreID = %d\n",iam,lstride,coreId); */
+            if((iam % lstride == 0) && (coreId == 0) ) {  /* only io tasks indicated by stride and coreId = 0 */
+                if((iam/lstride) < numiotasks_per_node) {
+                    /* only set the first (numiotasks_per_node - 1) tasks */
+                    (*iamIOtask) = true;
+                } else if ((iam/lstride) == numiotasks_per_node) {
+                    /*  If there is an uneven number of io-clients to io-nodes
+                        allocate the first remainder - 1 processor sets to
+                        have a total of numiotasks_per_node */
+                    if(psetNum < remainder) {(*iamIOtask) = true;
+                    };
+                }
+            }
+            /*       printf("comm = %d iam = %d lstride = %d coreID = %d iamIOtask = %i \n",comm, iam,lstride,coreId,(*iamIOtask)); */
+        }
+    }else{
+        /* We are not doing rearrangement.... so all tasks are io-tasks */
+        (*iamIOtask) = true;
+    }
+
+    /*printf("comm = %d myrank = %i iotask = %i \n", comm, rank, (*iamIOtask));*/
+
+    /* now we need to correctly determine the numiotasks */
+    MPI_Allreduce(iamIOtask, &task_count, 1, MPI_INT, MPI_SUM, comm);
+
+    (*numiotasks) = task_count;
+
+
+}
+
+int bgq_ion_id (void)
+{
+    int iA,  iB,  iC,  iD,  iE;                /* The local node's coordinates  */
+    int nA,  nB,  nC,  nD,  nE;                /* Size of each torus dimension  */
+    int brA, brB, brC, brD, brE;               /* The bridge node's coordinates */
+    int io_node_route_id;
+
+    Personality_t personality;
+
+    Kernel_GetPersonality(&personality, sizeof(personality));
+
+    iA  = personality.Network_Config.Acoord;
+    iB  = personality.Network_Config.Bcoord;
+    iC  = personality.Network_Config.Ccoord;
+    iD  = personality.Network_Config.Dcoord;
+    iE  = personality.Network_Config.Ecoord;
+
+    nA  = personality.Network_Config.Anodes;
+    nB  = personality.Network_Config.Bnodes;
+    nC  = personality.Network_Config.Cnodes;
+    nD  = personality.Network_Config.Dnodes;
+    nE  = personality.Network_Config.Enodes;
+
+    brA = personality.Network_Config.cnBridge_A;
+    brB = personality.Network_Config.cnBridge_B;
+    brC = personality.Network_Config.cnBridge_C;
+    brD = personality.Network_Config.cnBridge_D;
+    brE = personality.Network_Config.cnBridge_E;
+
+/*
+ * This is the bridge node, numbered in ABCDE order, E increments first.
+ * It is considered the unique "io node route identifer" because each
+ * bridge node only has one torus link to one io node.
+ */
+
+    io_node_route_id = brE + brD*nE + brC*nD*nE + brB*nC*nD*nE + brA*nB*nC*nD*nE;
+
+    return io_node_route_id;
+
+}
+
+
+
+int bgq_pset_info (MPI_Comm comm, int* tot_pset, int* psetID, int* pset_size, int* rank_in_pset)
+{
+    MPI_Comm comp_comm, pset_comm, bridge_comm;
+    int comp_rank, status, key, bridge_root, tot_bridges, cur_pset, itr, t_buf;
+    int temp_id, rem_psets;
+    MPI_Status mpi_status;
+
+    MPI_Comm_rank (comm, &comp_rank);
+
+    status = MPI_Comm_dup ( comm, &comp_comm);
+    if ( MPI_SUCCESS != status)
+    {
+        printf(" Error duplicating communicator \n");
+        MPI_Abort(comm, status);
+    }
+
+    // Compute the ION BridgeNode ID
+    key = bgq_ion_id ();
+
+    // Create the pset_comm per bridge node
+    status = MPI_Comm_split ( comp_comm, key, comp_rank, &pset_comm);
+    if ( MPI_SUCCESS != status)
+    {
+        printf(" Error splitting communicator \n");
+        MPI_Abort(comm, status);
+    }
+
+    // Calculate the rank in pset and pset size
+    MPI_Comm_rank (pset_comm, rank_in_pset);
+    MPI_Comm_size (pset_comm, pset_size);
+
+    // Create the Bridge root nodes communicator
+    bridge_root = 0;
+    if (0 == *rank_in_pset)
+        bridge_root = 1;
+
+    // Calculate the total number of bridge nodes / psets
+    tot_bridges = 0;
+    MPI_Allreduce (&bridge_root, &tot_bridges, 1, MPI_INT, MPI_SUM, comm);
+
+    *tot_pset = tot_bridges;
+
+    // Calculate the Pset ID
+    cur_pset = 0;
+    rem_psets = tot_bridges;
+    if ((0 == comp_rank) && (bridge_root ==1))
+    {
+        *psetID = 0;
+        rem_psets = tot_bridges-1;
+        cur_pset++;
+    }
+
+    t_buf = 0; // Dummy value
+    if (0 == comp_rank)
+    {
+        for (itr = 0; itr < rem_psets; itr++)
+        {
+            MPI_Recv (&t_buf,1, MPI_INT,  MPI_ANY_SOURCE, MPI_ANY_TAG,comm, &mpi_status);
+            MPI_Send (&cur_pset, 1, MPI_INT, mpi_status.MPI_SOURCE, 0, comm);
+            cur_pset++;
+        }
+    }
+
+    if ((1 == bridge_root) && ( 0 != comp_rank))
+    {
+        MPI_Send (&t_buf, 1, MPI_INT, 0, 0, comm);
+        MPI_Recv (&temp_id,1, MPI_INT, 0, 0, comm, &mpi_status);
+
+        *psetID = temp_id;
+        /*printf (" Pset ID is %d \n", *psetID);*/
+    }
+    // Broadcast the PSET ID to all ranks in the psetcomm
+    MPI_Bcast ( psetID, 1, MPI_INT, 0, pset_comm);
+
+    // Free the split comm
+    MPI_Comm_free (&pset_comm);
+
+    MPI_Barrier (comm);
+
+    return 0;
+}
+
+#endif
diff --git a/src/externals/pio2/src/flib/CMakeLists.txt b/src/externals/pio2/src/flib/CMakeLists.txt
new file mode 100644
index 00000000000..3d2c1711252
--- /dev/null
+++ b/src/externals/pio2/src/flib/CMakeLists.txt
@@ -0,0 +1,268 @@
+cmake_minimum_required (VERSION 2.8.12)
+project (PIOF Fortran)
+include (CheckFunctionExists)
+include (ExternalProject)
+include (LibCheck)
+
+#==============================================================================
+#  DEFINE THE TARGET
+#==============================================================================
+
+set (PIO_Fortran_SRCS pio_nf.F90
+  pio.F90
+  pio_kinds.F90
+  pio_types.F90
+  piolib_mod.F90
+  pio_support.F90)
+
+set (PIO_GenF90_SRCS pionfatt_mod.F90
+  pionfput_mod.F90
+  pionfget_mod.F90
+  piodarray.F90)
+
+set (PIO_Fortran_MODS ${CMAKE_CURRENT_BINARY_DIR}/pio.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pio_nf.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pio_types.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/piolib_mod.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pionfget_mod.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pio_kinds.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pio_support.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/piodarray.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pionfatt_mod.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/pionfput_mod.mod)
+
+add_library (piof ${PIO_Fortran_SRCS} ${PIO_GenF90_SRCS})
+if (NOT PIO_ENABLE_FORTRAN)
+  set_target_properties(piof PROPERTIES EXCLUDE_FROM_ALL TRUE)
+endif ()
+
+# Include flib source and binary directories (for Fortran modules)
+target_include_directories (piof
+  PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
+  PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+
+# System and compiler CPP directives
+target_compile_definitions (piof
+  PUBLIC ${CMAKE_SYSTEM_DIRECTIVE})
+target_compile_definitions (piof
+  PUBLIC ${CMAKE_Fortran_COMPILER_DIRECTIVE})
+
+# Compiler-specific compile options
+if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
+  target_compile_options (piof
+    PRIVATE -ffree-line-length-none)
+elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
+  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
+  #    target_compile_options (piof
+  #        PRIVATE -mismatch_all)
+endif()
+
+# Look for c_sizeof capability
+check_macro (Fortran_CSIZEOF
+  NAME TryCSizeOf.f90
+  HINTS ${CMAKE_MODULE_PATH}
+  COMMENT "whether Fortran compiler supports c_sizeof")
+if (NOT Fortran_CSIZEOF)
+  target_compile_definitions (piof
+    PUBLIC NO_C_SIZEOF)
+endif()
+
+# Look for filesystem hints
+if (PIO_FILESYSTEM_HINTS)
+  if (PIO_FILESYSTEM_HINTS STREQUAL lustre)
+    message (STATUS "PIO will use lustre filesystem hints")
+    target_compile_definitions (piof
+      PUBLIC PIO_LUSTRE_HINTS)
+  elseif (PIO_FILESYSTEM_HINTS STREQUAL gpfs)
+    message (STATUS "PIO will use gpfs filesystem hints")
+    target_compile_definitions (piof
+      PUBLIC PIO_GPFS_HINTS)
+  else ()
+    message (WARNING "${PIO_FILESYSTEM_HINTS} not valid option for PIO_FILESYSTEM_HINTS; use gpfs or lustre.")
+  endif ()
+endif ()
+
+#==============================================================================
+#  DEFINE THE INSTALL
+#==============================================================================
+
+# Library
+install (TARGETS piof DESTINATION lib)
+
+# Fortran Modules
+install (FILES ${PIO_Fortran_MODS} DESTINATION include)
+
+#==============================================================================
+#  DEFINE THE DEPENDENCIES
+#==============================================================================
+
+#===== pioc =====
+target_link_libraries(piof
+  PUBLIC pioc)
+
+#===== genf90 =====
+if (DEFINED GENF90_PATH)
+  add_custom_target(genf90
+    DEPENDS ${GENF90_PATH}/genf90.pl)
+else ()
+  ExternalProject_Add (genf90
+    PREFIX ${CMAKE_CURRENT_BINARY_DIR}/genf90
+    GIT_REPOSITORY https://github.com/PARALLELIO/genf90
+    GIT_TAG genf90_140121
+    UPDATE_COMMAND ""
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+    INSTALL_COMMAND "")
+  ExternalProject_Get_Property (genf90 SOURCE_DIR)
+  set (GENF90_PATH ${SOURCE_DIR})
+  unset (SOURCE_DIR)
+endif ()
+add_dependencies (piof genf90)
+
+#===== Fortran Source Generation with GenF90 =====
+foreach (SRC_FILE IN LISTS PIO_GenF90_SRCS)
+  add_custom_command (OUTPUT ${SRC_FILE}
+    COMMAND ${GENF90_PATH}/genf90.pl
+    ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in > ${SRC_FILE}
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in genf90)
+endforeach ()
+
+#===== MPI =====
+if (PIO_USE_MPISERIAL)
+  find_package (MPISERIAL COMPONENTS Fortran REQUIRED)
+  if (MPISERIAL_Fortran_FOUND)
+    target_compile_definitions (piof
+      PRIVATE _MPISERIAL)
+    target_include_directories (piof
+      PUBLIC ${MPISERIAL_Fortran_INCLUDE_DIRS})
+    target_link_libraries (piof
+      PUBLIC ${MPISERIAL_Fortran_LIBRARIES})
+
+    set (WITH_PNETCDF FALSE)
+    set (MPI_Fortran_INCLUDE_PATH ${MPISERIAL_Fortran_INCLUDE_DIRS})
+  endif ()
+else ()
+  find_package (MPI REQUIRED)
+endif ()
+
+# Check MPI I/O capabilities
+find_path(MPIF_H_PATH
+  NAMES mpif.h
+  PATHS ${MPI_Fortran_INCLUDE_PATH}
+  NO_DEFAULT_PATH)
+if (MPIF_H_PATH)
+  check_macro (MPI_HAS_MPIIO
+    NAME TryMPIIO.f90
+    HINTS ${CMAKE_MODULE_PATH}
+    DEFINITIONS -I{MPIF_H_PATH}
+    COMMENT "whether MPIIO is supported")
+  if (${MPI_HAS_MPIIO})
+    message (STATUS "MPIIO verified and enabled.")
+    target_compile_definitions (piof
+      PUBLIC USEMPIIO)
+  else ()
+    message (STATUS "MPIIO failed verification and therefore disabled.")
+  endif ()
+else ()
+  message (STATUS "MPIIO not detected and therefore disabled.")
+endif ()
+
+# Check for MPI Fortran module
+find_path(MPIMOD_PATH
+  NAMES mpi.mod MPI.mod
+  HINTS ${MPI_Fortran_INCLUDE_PATH})
+if (PIO_ENABLE_TIMING)
+  find_package (GPTL COMPONENTS Fortran_Perf QUIET)
+endif ()
+
+check_macro (MPI_HAS_Fortran_MOD
+  NAME TryMPIMod.f90
+  HINTS ${CMAKE_MODULE_PATH}
+  DEFINITIONS -I${MPIMOD_PATH}
+  COMMENT "whether MPI Fortran module is supported")
+if (${MPI_HAS_Fortran_MOD})
+  message (STATUS "MPI Fortran module verified and enabled.")
+else ()
+  message (STATUS "MPI Fortran module failed verification and therefore disabled.")
+  if (PIO_ENABLE_TIMING AND NOT GPTL_Fortran_Perf_FOUND)
+    target_compile_definitions (gptl
+      PUBLIC NO_MPIMOD)
+  endif()
+endif ()
+
+#===== GPTL =====
+if (PIO_ENABLE_TIMING)
+  if (GPTL_Fortran_Perf_FOUND)
+    message (STATUS "Found GPTL Fortran Perf: ${GPTL_Fortran_Perf_LIBRARIES}")
+    target_include_directories (piof
+      PUBLIC ${GPTL_Fortran_INCLUDE_DIRS})
+    target_link_libraries (piof
+      PUBLIC ${GPTL_Fortran_LIBRARIES})
+  else ()
+    message (STATUS "Using internal GPTL Fortran library for timing")
+    target_link_libraries (piof
+      PUBLIC gptl)
+  endif ()
+endif ()
+
+#===== NetCDF-Fortran =====
+find_package (NetCDF "4.3.3" COMPONENTS Fortran)
+if (NetCDF_Fortran_FOUND)
+  target_include_directories (piof
+    PUBLIC ${NetCDF_Fortran_INCLUDE_DIRS})
+  target_compile_definitions (piof
+    PUBLIC _NETCDF)
+  target_link_libraries (piof
+    PUBLIC ${NetCDF_Fortran_LIBRARIES})
+  if (EXISTS ${NetCDF_Fortran_INCLUDE_DIR}/netcdf_par.h)
+    target_compile_definitions (piof
+      PUBLIC _NETCDF4)
+  endif ()
+else ()
+  target_compile_definitions (piof
+    PUBLIC _NONETCDF)
+endif ()
+
+#===== PnetCDF =====
+if (WITH_PNETCDF)
+  find_package (PnetCDF "1.6" COMPONENTS Fortran REQUIRED)
+endif ()
+if (PnetCDF_Fortran_FOUND)
+  target_include_directories (piof
+    PUBLIC ${PnetCDF_Fortran_INCLUDE_DIRS})
+  target_compile_definitions (piof
+    PUBLIC _PNETCDF)
+  target_link_libraries (piof
+    PUBLIC ${PnetCDF_Fortran_LIBRARIES})
+
+  # Check library for varn functions
+  set (CMAKE_REQUIRED_LIBRARIES ${PnetCDF_Fortran_LIBRARY})
+  check_function_exists (ncmpi_get_varn PnetCDF_Fortran_HAS_VARN)
+  if (PnetCDF_Fortran_HAS_VARN)
+    target_compile_definitions(piof
+      PUBLIC USE_PNETCDF_VARN
+      PUBLIC USE_PNETCDF_VARN_ON_READ)
+  endif()
+else ()
+  target_compile_definitions (piof
+    PUBLIC _NOPNETCDF)
+endif ()
+
+#===== Add EXTRAs =====
+target_include_directories (piof
+  PUBLIC ${PIO_Fortran_EXTRA_INCLUDE_DIRS})
+target_link_libraries (piof
+  PUBLIC ${PIO_Fortran_EXTRA_LIBRARIES})
+target_compile_options (piof
+  PRIVATE ${PIO_Fortran_EXTRA_COMPILE_OPTIONS})
+target_compile_definitions (piof
+  PUBLIC ${PIO_Fortran_EXTRA_COMPILE_DEFINITIONS})
+if (PIO_Fortran_EXTRA_LINK_FLAGS)
+  set_target_properties(piof PROPERTIES
+    LINK_FLAGS ${PIO_Fortran_EXTRA_LINK_FLAGS})
+endif ()
+
+#===== Check for necessities =====
+if (NOT PnetCDF_Fortran_FOUND AND NOT NetCDF_Fortran_FOUND)
+  message (FATAL_ERROR "Must have PnetCDF and/or NetCDF Fortran libraries")
+endif ()
diff --git a/externals/pio2/src/flib/pio.F90 b/src/externals/pio2/src/flib/pio.F90
similarity index 90%
rename from externals/pio2/src/flib/pio.F90
rename to src/externals/pio2/src/flib/pio.F90
index 0609c506c53..cf8c96105b0 100644
--- a/externals/pio2/src/flib/pio.F90
+++ b/src/externals/pio2/src/flib/pio.F90
@@ -1,7 +1,7 @@
 !>
-!! @file
+!! @file 
 !! @brief User interface Module for PIO, this is the only file a user program should 'use'
-!!
+!! 
 !<
 
 module pio
@@ -16,9 +16,14 @@ module pio
        pio_seterrorhandling, pio_setframe, pio_init, pio_get_local_array_size, &
        pio_freedecomp, pio_syncfile, &
        pio_finalize, pio_set_hint, pio_getnumiotasks, pio_file_is_open, &
-       PIO_deletefile, PIO_get_numiotasks, PIO_iotype_available
+       PIO_deletefile, PIO_get_numiotasks, PIO_iotype_available, &
+       pio_set_rearr_opts
 
   use pio_types, only : io_desc_t, file_desc_t, var_desc_t, iosystem_desc_t, &
+       pio_rearr_opt_t, pio_rearr_comm_fc_opt_t, pio_rearr_comm_fc_2d_enable,&
+       pio_rearr_comm_fc_1d_comp2io, pio_rearr_comm_fc_1d_io2comp,&
+       pio_rearr_comm_fc_2d_disable, pio_rearr_comm_unlimited_pend_req,&
+       pio_rearr_comm_p2p, pio_rearr_comm_coll,&
        pio_int, pio_real, pio_double, pio_noerr, iotype_netcdf, &
        iotype_pnetcdf,  pio_iotype_netcdf4p, pio_iotype_netcdf4c, &
        pio_iotype_pnetcdf,pio_iotype_netcdf, &
@@ -28,9 +33,9 @@ module pio
        pio_nofill, pio_unlimited, pio_fill_int, pio_fill_double, pio_fill_float, &
 #endif
        pio_64bit_offset, pio_64bit_data, &
-       pio_internal_error, pio_bcast_error, pio_return_error, pio_rearr_opt_t
+       pio_internal_error, pio_bcast_error, pio_return_error, pio_default
 
-  use piodarray, only : pio_read_darray, pio_write_darray, pio_set_buffer_size_limit
+  use piodarray, only : pio_read_darray, pio_write_darray, pio_set_buffer_size_limit  
 
   use pio_nf, only:        &
        PIO_enddef,            &
@@ -53,12 +58,14 @@ module pio
        PIO_def_var   ,        &
        PIO_def_var_deflate   ,        &
        PIO_redef     ,          &
+       PIO_set_log_level,          &
        PIO_inquire_variable , &
        PIO_inquire_dimension, &
        PIO_set_chunk_cache, &
        PIO_get_chunk_cache, &
        PIO_set_var_chunk_cache, &
-       PIO_get_var_chunk_cache
+       PIO_get_var_chunk_cache, &
+       PIO_strerror
 
   use pionfatt_mod, only : PIO_put_att   => put_att,        &
        PIO_get_att   => get_att
@@ -111,11 +118,11 @@ integer(C_INT) function PIOc_iam_iotask(iosysid, iotask) &
          logical(C_BOOL), intent(out) :: iotask
        end function PIOc_iam_iotask
     end interface
-
+    
     ierr = PIOc_iam_iotask(iosystem%iosysid, ctask)
     task = ctask
   end function pio_iam_iotask
-
+  
 !>
 !! @public
 !! @brief Integer function returns rank of IO task.
@@ -131,7 +138,7 @@ integer(C_INT) function PIOc_iotask_rank(iosysid, rank) &
          integer(C_INT), intent(out) :: rank
        end function PIOc_iotask_rank
     end interface
-
+    
     ierr = PIOc_iotask_rank(iosystem%iosysid, rank)
   end function pio_iotask_rank
 
diff --git a/externals/pio2/src/flib/pio_kinds.F90 b/src/externals/pio2/src/flib/pio_kinds.F90
similarity index 100%
rename from externals/pio2/src/flib/pio_kinds.F90
rename to src/externals/pio2/src/flib/pio_kinds.F90
diff --git a/externals/pio2/src/flib/pio_nf.F90 b/src/externals/pio2/src/flib/pio_nf.F90
similarity index 94%
rename from externals/pio2/src/flib/pio_nf.F90
rename to src/externals/pio2/src/flib/pio_nf.F90
index 5c798f834ec..1b19beec4b9 100644
--- a/externals/pio2/src/flib/pio_nf.F90
+++ b/src/externals/pio2/src/flib/pio_nf.F90
@@ -14,7 +14,7 @@ module pio_nf
        pio_def_var_deflate                                  , &
        pio_def_var_chunking                                 , &
        pio_def_dim                                          , &
-       pio_inq_attname                                      , & 
+       pio_inq_attname                                      , &
        pio_inq_att                                          , &
        pio_inq_attlen                                       , &
        pio_inq_varid                                        , &
@@ -23,7 +23,7 @@ module pio_nf
        pio_inq_varndims                                     , &
        pio_inq_vardimid                                     , &
        pio_inq_varnatts                                     , &
-       pio_inq_var_deflate                                  , &       
+       pio_inq_var_deflate                                  , &
        pio_inquire_variable                                 , &
        pio_inquire_dimension                                , &
        pio_inq_dimname                                      , &
@@ -36,7 +36,9 @@ module pio_nf
        pio_get_chunk_cache                                  , &
        pio_set_var_chunk_cache                              , &
        pio_get_var_chunk_cache                              , &
-       pio_redef
+       pio_redef                                            , &
+       pio_set_log_level                                    , &
+       pio_strerror
 !       pio_copy_att    to be done
 
   interface pio_def_var
@@ -44,11 +46,11 @@ module pio_nf
           def_var_0d_desc                                   , &
           def_var_md_desc                                   , &
           def_var_0d_id                                     , &
-          def_var_md_id 
+          def_var_md_id
   end interface
   interface pio_def_var_deflate
      module procedure &
-          def_var_deflate_desc                              , &         
+          def_var_deflate_desc                              , &
           def_var_deflate_id
   end interface
   interface pio_def_var_chunking
@@ -180,13 +182,24 @@ module pio_nf
      module procedure &
           enddef_desc                                       , &
           enddef_id
-  end interface
+  end interface pio_enddef
+
   interface pio_redef
      module procedure &
           redef_desc                                        , &
           redef_id
   end interface
 
+  interface pio_set_log_level
+     module procedure &
+          set_log_level
+  end interface pio_set_log_level
+
+  interface pio_strerror
+     module procedure &
+          strerror
+  end interface pio_strerror
+
   interface pio_inquire
      module procedure &
           inquire_desc                                      , &
@@ -221,7 +234,7 @@ module pio_nf
 !! @defgroup PIO_inq_dimid PIO_inq_dimid
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimid
 !! @brief Returns the netcdf dimension id for the name.
 !! @details
@@ -239,7 +252,7 @@ integer function inq_dimid_desc(File                      ,name,dimid) result(ie
     ierr = inq_dimid_id(file%fh                             ,name,dimid)
   end function inq_dimid_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimid
 !! @brief Returns the netcdf dimension id for the name.
 !<
@@ -264,9 +277,9 @@ end function inq_dimid_id
 !! @defgroup PIO_inquire_dimension PIO_inquire_dimension
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire_dimension
-!! @brief  Get information about a particular dimension in netcdf file 
+!! @brief  Get information about a particular dimension in netcdf file
 !! @details
 !! @param ncid : A netcdf file descriptor returned by \ref PIO_openfile or \ref PIO_createfile.
 !! @param dimid : The netcdf dimension ID.
@@ -282,9 +295,9 @@ integer function inquire_dimension_desc(file              , dimid, name, len) re
     ierr = Inquire_dimension_id(file%fh                     , dimid, name, len)
   end function inquire_dimension_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire_dimension
-!! @brief  Get information about a particular dimension in netcdf file 
+!! @brief  Get information about a particular dimension in netcdf file
 !<
   integer function inquire_dimension_id(ncid                , dimid, name, len) result(ierr)
     integer                                                 , intent(in) :: ncid
@@ -305,9 +318,9 @@ end function inquire_dimension_id
 !! @defgroup PIO_inq_dimlen PIO_inq_dimlen
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file 
+!! @brief  Get information about the length of a particular dimension in netcdf file
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param dimid : The netcdf dimension ID.
@@ -321,9 +334,9 @@ integer function inq_dimlen_desc(File                     , dimid, len) result(i
     ierr = inq_dimlen_id(file%fh                            ,dimid,len)
   end function inq_dimlen_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file 
+!! @brief  Get information about the length of a particular dimension in netcdf file
 !<
   integer function inq_dimlen_desc_long(File                , dimid, len) result(ierr)
     type(file_desc_t)                                       , intent(in) :: File
@@ -332,9 +345,9 @@ integer function inq_dimlen_desc_long(File                , dimid, len) result(i
     ierr = inq_dimlen_id_long(file%fh                       ,dimid,len)
   end function inq_dimlen_desc_long
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file 
+!! @brief  Get information about the length of a particular dimension in netcdf file
 !<
   integer function inq_dimlen_id(ncid                       , dimid, len) result(ierr)
     integer                                                 , intent(in) :: ncid
@@ -345,9 +358,9 @@ integer function inq_dimlen_id(ncid                       , dimid, len) result(i
     len = int(llen)
   end function inq_dimlen_id
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimlen
-!! @brief  Get information about the length of a particular dimension in netcdf file 
+!! @brief  Get information about the length of a particular dimension in netcdf file
 !<
   integer function inq_dimlen_id_long(ncid                  , dimid, len) result(ierr)
     integer                                                 , intent(in) :: ncid
@@ -362,7 +375,7 @@ integer(C_INT) function PIOc_inq_dimlen(ncid         ,dimid,len) &
          integer(c_size_t) :: len
        end function PIOc_inq_dimlen
     end interface
-    
+
     ierr = PIOc_inq_dimlen(ncid                             ,dimid-1,len)
   end function inq_dimlen_id_long
 
@@ -371,9 +384,9 @@ end function inq_dimlen_id_long
 !! @defgroup PIO_inq_dimname PIO_inq_dimname
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimname
-!! @brief  Get information about the name of of a dimension. 
+!! @brief  Get information about the name of of a dimension.
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param dimid : The netcdf dimension ID.
@@ -387,9 +400,9 @@ integer function inq_dimname_desc(File                    , dimid, name) result(
     ierr = inq_dimname_id(file%fh                           ,dimid,name)
   end function inq_dimname_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_dimname
-!! @brief  Get information about the name of of a dimension. 
+!! @brief  Get information about the name of of a dimension.
 !<
   integer function inq_dimname_id(ncid                      , dimid, name) result(ierr)
     integer                                                 , intent(in) :: ncid
@@ -416,7 +429,7 @@ end function inq_dimname_id
 !! @defgroup PIO_inq_ndims PIO_inq_ndims
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_ndims
 !! @brief  Get information about the number of dimensions of a file or group.
 !! @details
@@ -430,7 +443,7 @@ integer function inq_ndims_desc(File                      , ndims) result(ierr)
     ierr = inq_ndims_id(file%fh                             , ndims)
   end function inq_ndims_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_ndims
 !! @brief  Get information about the number of dimensions of a file or group.
 !<
@@ -452,7 +465,7 @@ end function inq_ndims_id
 !! @defgroup PIO_inq_nvars PIO_inq_nvars
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_nvars
 !! @brief  Get information about the number of variables in a file or group.
 !! @details
@@ -466,7 +479,7 @@ integer function inq_nvars_desc(File                      , nvars) result(ierr)
     ierr = inq_nvars_id(file%fh                             , nvars)
   end function inq_nvars_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_nvars
 !! @brief  Get information about the number of variables in a file or group.
 !<
@@ -488,7 +501,7 @@ end function inq_nvars_id
 !! @defgroup PIO_inq_natts PIO_inq_natts
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_natts
 !! @brief  Get information about the number of global attributes in a file or group.
 !! @details
@@ -502,7 +515,7 @@ integer function inq_natts_desc(File                      , natts) result(ierr)
     ierr = inq_natts_id(file%fh                             , natts)
   end function inq_natts_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_natts
 !! @brief  Get information about the number of global attributes in a file or group.
 !<
@@ -524,7 +537,7 @@ end function inq_natts_id
 !! @defgroup PIO_inq_unlimdim PIO_inq_unlimdim
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_unlimdm
 !! @brief  Get information about the unlimited dimension in a file.
 !! @details
@@ -538,7 +551,7 @@ integer function inq_unlimdim_desc(File                   , unlimdim) result(ier
     ierr = inq_unlimdim_id(file%fh                          , unlimdim)
   end function inq_unlimdim_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_unlimdm
 !! @brief  Get information about the unlimited dimension in a file.
 !<
@@ -561,13 +574,13 @@ end function inq_unlimdim_id
 !! @defgroup PIO_inquire PIO_inquire
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire
 !! @brief Gets metadata information for netcdf file.
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param nDimensions :  Number of dimensions defined for the netcdf file
-!! @param nVariables : Number of variables defined for the netcdf file 
+!! @param nVariables : Number of variables defined for the netcdf file
 !! @param nAttributes : Number of attributes defined for the netcdf file
 !! @param unlimitedDimID : the Unlimited dimension ID
 !! @retval ierr @copydoc error_return
@@ -580,11 +593,11 @@ integer function inquire_desc(File                        ,nDimensions,nVariable
          nVariables                                         ,   &! number of variables
          nAttributes                                        ,  & ! number of global attributes
          unlimitedDimID ! ID of unlimited dimension
-    
+
     ierr = inquire_id(file%fh                               ,ndimensions,nvariables,nattributes,unlimitedDimID)
   end function inquire_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -605,7 +618,7 @@ end function inquire_id
 !>
 !! @defgroup PIO_enddef PIO_enddef
 !<
-!> 
+!>
 !! @public
 !! @ingroup PIO_enddef
 !! @brief Exits netcdf define mode.
@@ -617,7 +630,7 @@ integer function enddef_desc(File) result(ierr)
     type (File_desc_t)                                      , intent(inout) :: File
     ierr = enddef_id(file%fh)
   end function enddef_desc
-!> 
+!>
 !! @public
 !! @ingroup PIO_enddef
 !! @brief Wrapper for the C function \ref PIOc_enddef .
@@ -636,7 +649,7 @@ end function enddef_id
 !>
 !! @defgroup PIO_redef PIO_redef
 !<
-!> 
+!>
 !! @public
 !! @ingroup PIO_redef
 !! @brief Exits netcdf define mode.
@@ -648,18 +661,69 @@ integer function redef_desc(File) result(ierr)
     type (File_desc_t)                                      , intent(inout) :: File
     ierr = redef_id(file%fh)
   end function redef_desc
-!> 
+
+!>
+!! @defgroup PIO_set_log_level
+!<
+!>
+!! @ingroup PIO_set_log_level
+!! Sets the logging level. Only takes effect if PIO was built with
+!! PIO_ENABLE_LOGGING=On
+!!
+!! @param log_level the logging level.
+!! @retval ierr @copydoc error_return
+!<
+  integer function set_log_level(log_level) result(ierr)
+    integer, intent(in) :: log_level
+    interface
+       integer(C_INT) function PIOc_set_log_level(log_level) &
+            bind(C, name="PIOc_set_log_level")
+         use iso_c_binding
+         integer(C_INT), value :: log_level
+       end function PIOc_set_log_level
+    end interface
+    ierr = PIOc_set_log_level(log_level)
+  end function set_log_level
+
+  !>
+  !! @defgroup PIO_strerror
+  !<
+  !>
+  !! @ingroup PIO_strerror
+  !! Returns a descriptive string for an error code.
+  !!
+  !! @param errcode the error code
+  !! @retval a description of the error
+  !<
+  integer function strerror(errcode, errmsg) result(ierr)
+    integer, intent(in) :: errcode
+    character(len=*), intent(out) :: errmsg
+    interface
+       integer(C_INT) function PIOc_strerror(errcode, errmsg) &
+            bind(C, name="PIOc_strerror")
+         use iso_c_binding
+         integer(C_INT), value :: errcode
+         character(C_CHAR) :: errmsg(*)
+       end function PIOc_strerror
+    end interface
+    errmsg = C_NULL_CHAR
+    ierr = PIOc_strerror(errcode, errmsg)
+    call replace_c_null(errmsg)
+
+  end function strerror
+
+!>
 !! @public
 !! @ingroup PIO_redef
 !! @brief Wrapper for the C function \ref PIOc_redef .
 !<
   integer function redef_id(ncid) result(ierr)
-    integer                                                 ,intent(in) :: ncid
+    integer, intent(in) :: ncid
     interface
        integer(C_INT) function PIOc_redef(ncid) &
-            bind(C                                          ,name="PIOc_redef")
+            bind(C, name="PIOc_redef")
          use iso_c_binding
-         integer(C_INT)                                     , value :: ncid
+         integer(C_INT), value :: ncid
        end function PIOc_redef
     end interface
     ierr = PIOc_redef(ncid)
@@ -669,7 +733,7 @@ end function redef_id
 !! @defgroup PIO_def_dim PIO_def_dim
 !! @brief A set of functions to define dimensions and their attributes in NetCDF files.
 !<
-!> 
+!>
 !! @public
 !! @ingroup PIO_def_dim
 !! @brief Defines the netcdf dimension.
@@ -688,7 +752,7 @@ integer function def_dim_int_desc(File                    ,name,len,dimid) resul
 
     ierr = def_dim_id(file%fh                               ,name,int(len,pio_offset_kind),dimid)
   end function def_dim_int_desc
-!> 
+!>
 !! @public
 !! @ingroup PIO_def_dim
 !! @brief  Defines the netcdf dimension.
@@ -701,7 +765,7 @@ integer function def_dim_int_id(ncid                      ,name,len,dimid) resul
 
     ierr = def_dim_id(ncid                                  ,name,int(len,pio_offset_kind),dimid)
   end function def_dim_int_id
-!> 
+!>
 !! @public
 !! @ingroup PIO_def_dim
 !! @brief  Defines the netcdf dimension.
@@ -715,7 +779,7 @@ integer function def_dim_desc(File                        ,name,len,dimid) resul
 
     ierr = def_dim_id(file%fh                               ,name,len,dimid)
   end function def_dim_desc
-!> 
+!>
 !! @public
 !! @ingroup PIO_def_dim
 !! @brief  Defines the netcdf dimension.
@@ -745,9 +809,9 @@ end function def_dim_id
 !! @defgroup PIO_inquire_variable PIO_inquire_variable
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire_variable
-!! @brief Inquires if a NetCDF variable is present and returns its attributes  
+!! @brief Inquires if a NetCDF variable is present and returns its attributes
 !! @details
 !! @param ncid : A netcdf file descriptor returned by \ref PIO_openfile or \ref PIO_createfile.
 !! @param vardesc @copydoc var_desc_t
@@ -769,9 +833,9 @@ integer function inquire_variable_desc(file               , vardesc, name, xtype
     ierr = pio_inquire_variable(file%fh                     ,vardesc%varid,name,xtype,ndims,dimids,natts)
   end function inquire_variable_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire_variable
-!! @brief Inquires if a NetCDF variable is present and returns its attributes  
+!! @brief Inquires if a NetCDF variable is present and returns its attributes
 !<
   integer function inquire_variable_vid(file                , varid, name, xtype, ndims, dimids, natts) result(ierr)
     type(file_desc_t)                                       ,               intent(in) :: file
@@ -784,9 +848,9 @@ integer function inquire_variable_vid(file                , varid, name, xtype,
     ierr = pio_inquire_variable(file%fh                     ,varid,name,xtype,ndims,dimids,natts)
   end function inquire_variable_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inquire_variable
-!! @brief Inquires if a NetCDF variable is present and returns its attributes  
+!! @brief Inquires if a NetCDF variable is present and returns its attributes
 !<
   integer function inquire_variable_id(ncid                 , varid, name, xtype, ndims, dimids, natts) result(ierr)
     integer                                                 ,                intent( in) :: ncid
@@ -807,7 +871,7 @@ end function inquire_variable_id
 !! @defgroup PIO_inq_vardimid PIO_inq_vardimid
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vardimid
 !! @brief returns the dimids of the variable as an interger array
 !! @details
@@ -826,7 +890,7 @@ integer function inq_vardimid_desc(File                   ,vardesc,dimids) resul
     ierr = pio_inq_vardimid(File%fh                         , vardesc%varid, dimids)
   end function inq_vardimid_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vardimid
 !! @brief returns the dimids of the variable as an interger array
 !<
@@ -840,10 +904,10 @@ integer function inq_vardimid_vid(File                    ,varid,dimids) result(
     ierr = pio_inq_vardimid(File%fh                         , varid, dimids)
   end function inq_vardimid_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vardimid
 !! @brief returns the dimids of the variable as an interger array
-!<    
+!<
   integer function inq_vardimid_id(ncid                     ,varid,dimids) result(ierr)
     integer                                                 , intent(in) :: ncid
     integer                                                 , intent(in) :: varid
@@ -859,7 +923,7 @@ integer(C_INT) function PIOc_inq_vardimid(ncid       ,varid,dimids) &
        end function PIOc_inq_vardimid
     end interface
     integer :: i                                            , ndims
-    
+
     ierr = inq_varndims_id(ncid                             ,varid,ndims)
     allocate(cdimids(ndims))
 
@@ -875,13 +939,13 @@ end function inq_vardimid_id
 !! @defgroup PIO_inq_varndims PIO_inq_varndims
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varndims
 !! @brief Gets the number of dimension associated with a netcdf variable
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param vardesc @copydoc var_desc_t
-!! @param ndims : The number of dimensions for the variable 
+!! @param ndims : The number of dimensions for the variable
 !! @retval ierr @copydoc error_return
 !<
   integer function inq_varndims_desc(File                   ,vardesc,ndims) result(ierr)
@@ -893,7 +957,7 @@ integer function inq_varndims_desc(File                   ,vardesc,ndims) result
     ierr = pio_inq_varndims(File%fh                         , vardesc%varid, ndims)
   end function inq_varndims_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varndims
 !! @brief Gets the number of dimension associated with a netcdf variable
 !<
@@ -906,7 +970,7 @@ integer function inq_varndims_vid(File                    ,varid,ndims) result(i
     ierr = pio_inq_varndims(File%fh                         , varid, ndims)
   end function inq_varndims_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varndims
 !! @brief Gets the number of dimension associated with a netcdf variable
 !<
@@ -930,7 +994,7 @@ end function inq_varndims_id
 !! @defgroup PIO_inq_vartype PIO_inq_vartype
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vartype
 !! @brief Gets metadata information for netcdf file.
 !! @details
@@ -948,7 +1012,7 @@ integer function inq_vartype_desc(File                    ,vardesc,type) result(
     ierr = pio_inq_vartype(File%fh                          , vardesc%varid, type)
   end function inq_vartype_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vartype
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -961,7 +1025,7 @@ integer function inq_vartype_vid(File                     ,varid,type) result(ie
     ierr = pio_inq_vartype(File%fh                          , varid, type)
   end function inq_vartype_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_vartype
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -987,7 +1051,7 @@ end function inq_vartype_id
 !!  @defgroup PIO_inq_varnatts PIO_inq_varnatts
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varnatts
 !! @brief Gets metadata information for netcdf file.
 !! @details
@@ -1005,7 +1069,7 @@ integer function inq_varnatts_desc(File                   ,vardesc,natts) result
     ierr = pio_inq_varnatts(File%fh                         , vardesc%varid,natts)
   end function inq_varnatts_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varnatts
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -1018,7 +1082,7 @@ integer function inq_varnatts_vid(File                    ,varid,natts) result(i
     ierr = pio_inq_varnatts(File%fh                         , varid, natts)
   end function inq_varnatts_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varnatts
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -1044,7 +1108,7 @@ end function inq_varnatts_id
 !!  @defgroup PIO_inq_var_deflate PIO_inq_var_deflate
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_var_deflate
 !! @brief Gets metadata information for netcdf file.
 !! @details
@@ -1066,7 +1130,7 @@ integer function inq_var_deflate_desc(File, vardesc, shuffle, deflate, &
   end function inq_var_deflate_desc
 
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_var_deflate
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -1082,7 +1146,7 @@ integer function inq_var_deflate_vid(File, varid, shuffle, deflate, deflate_leve
   end function inq_var_deflate_vid
 
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_var_deflate
 !! @brief Gets metadata information for netcdf file.
 !<
@@ -1108,12 +1172,12 @@ end function PIOc_inq_var_deflate
 
     ierr = PIOc_inq_var_deflate(ncid, varid-1, shuffle, deflate, deflate_level)
   end function inq_var_deflate_id
-  
+
 !>
 !! @defgroup PIO_inq_varname
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varname
 !! @brief Get the name associated with a variable
 !! @details
@@ -1127,12 +1191,12 @@ integer function inq_varname_desc(File                    ,vardesc,name) result(
     type (File_desc_t)                                      , intent(in)   :: File
     type (Var_desc_t)                                       , intent(in)    :: vardesc
     character(len=*)                                        , intent(out)    :: name
-    
+
     ierr = pio_inq_varname(file%fh                          ,vardesc%varid,name)
 
   end function inq_varname_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varname
 !! @brief Get the name associated with a variable
 !<
@@ -1141,12 +1205,12 @@ integer function inq_varname_vid(File                     ,varid,name) result(ie
     type (File_desc_t)                                      , intent(in)   :: File
     integer                                                 , intent(in)    :: varid
     character(len=*)                                        , intent(out)    :: name
-    
+
     ierr = pio_inq_varname(file%fh                          ,varid,name)
 
   end function inq_varname_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_varname
 !! @brief Get the name associated with a variable
 !<
@@ -1173,10 +1237,10 @@ end function inq_varname_id
 !>
 !! @defgroup PIO_inq_varid
 !<
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_inq_varid
-!! @brief Returns the ID of a netcdf variable given its name 
+!! @brief Returns the ID of a netcdf variable given its name
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param name   : Name of the returned attribute
@@ -1191,10 +1255,10 @@ integer function inq_varid_desc(File,name,vardesc) result(ierr)
 
     ierr = pio_inq_varid(File%fh, name, vardesc%varid)
   end function inq_varid_desc
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_inq_varid
-!! @brief Returns the ID of a netcdf variable given its name 
+!! @brief Returns the ID of a netcdf variable given its name
 !<
   integer function inq_varid_vid(File,name,varid) result(ierr)
 
@@ -1204,10 +1268,10 @@ integer function inq_varid_vid(File,name,varid) result(ierr)
 
     ierr = pio_inq_varid(File%fh, name, varid)
   end function inq_varid_vid
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_inq_varid
-!! @brief Returns the ID of a netcdf variable given its name 
+!! @brief Returns the ID of a netcdf variable given its name
 !<
   integer function inq_varid_id(ncid,name,varid) result(ierr)
 
@@ -1234,9 +1298,9 @@ end function inq_varid_id
 !! @defgroup PIO_inq_attlen
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attlen
-!! @brief  Gets the attribute length 
+!! @brief  Gets the attribute length
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param vardesc @copydoc var_desc_t
@@ -1255,9 +1319,9 @@ integer function inq_attlen_desc(File,vardesc,name,len) result(ierr)
 
   end function inq_attlen_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attlen
-!! @brief  Gets the attribute length 
+!! @brief  Gets the attribute length
 !<
   integer function inq_attlen_vid(File,varid,name,len) result(ierr)
 
@@ -1270,9 +1334,9 @@ integer function inq_attlen_vid(File,varid,name,len) result(ierr)
 
   end function inq_attlen_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attlen
-!! @brief  Gets the attribute length 
+!! @brief  Gets the attribute length
 !<
   integer function inq_attlen_id(ncid,varid,name,len) result(ierr)
     integer, intent(in) :: ncid
@@ -1298,15 +1362,15 @@ end function inq_attlen_id
 !! @defgroup PIO_inq_att PIO_inq_att
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_att
 !! @brief  Gets information about attributes
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param vardesc @copydoc var_desc_t
-!! @param name : Name of the attribute 
+!! @param name : Name of the attribute
 !! @param xtype : The type of attribute
-!! @param len : The length of the attribute 
+!! @param len : The length of the attribute
 !! @retval ierr @copydoc error_return
 !<
   integer function inq_att_desc(File,vardesc,name,xtype,len) result(ierr)
@@ -1321,7 +1385,7 @@ integer function inq_att_desc(File,vardesc,name,xtype,len) result(ierr)
 
   end function inq_att_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_att
 !! @brief  Gets information about attributes
 !<
@@ -1337,7 +1401,7 @@ integer function inq_att_vid(File,varid,name,xtype,len) result(ierr)
 
   end function inq_att_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_att
 !! @brief  Gets information about attributes
 !<
@@ -1363,17 +1427,17 @@ integer(C_INT) function PIOc_inq_att(ncid,varid,name,xtype,len) &
          integer(C_SIZE_T) :: len
        end function PIOc_inq_att
     end interface
-    
+
     ierr = PIOc_inq_att(ncid,varid-1,trim(name)//C_NULL_CHAR,ixtype,xlen)
     if(present(len)) len=xlen
     if(present(xtype)) xtype = ixtype
 
   end function inq_att_id
-!> 
+!>
 !! @defgroup PIO_inq_attname
 !<
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attname
 !! @brief  Gets the name of an attribute
 !<
@@ -1387,7 +1451,7 @@ integer function inq_attname_desc(File,vdesc,attnum,name) result(ierr)
 
   end function inq_attname_desc
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attname
 !! @brief   Gets the name of an attribute
 !<
@@ -1396,12 +1460,12 @@ integer function inq_attname_vid(File,varid,attnum,name) result(ierr)
     integer, intent(in)           :: varid
     integer, intent(in)              :: attnum !Attribute number
     character(len=*), intent(out)     :: name
-    
+
     ierr = inq_attname_id(file%fh,varid,attnum,name)
 
   end function inq_attname_vid
 !>
-!! @public 
+!! @public
 !! @ingroup PIO_inq_attname
 !! @brief   Gets the name of an attribute
 !<
@@ -1432,14 +1496,14 @@ end function inq_attname_id
 !! @defgroup PIO_def_var PIO_def_var
 !<
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var
 !! @brief Defines a netcdf variable
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param name : The name of the variable to define
-!! @param type : The type of variable 
+!! @param type : The type of variable
 !! @param vardesc @copydoc var_desc_t
 !! @retval ierr @copydoc error_return
 !<
@@ -1454,8 +1518,8 @@ integer function def_var_0d_desc(File,name,type,vardesc) result(ierr)
     ierr = def_var_md_id(File%fh,name,type,dimids,vardesc%varid)
 
   end function def_var_0d_desc
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var
 !! @brief Defines a netcdf variable
 !<
@@ -1471,14 +1535,14 @@ integer function def_var_0d_id(ncid,name,type,varid) result(ierr)
 
   end function def_var_0d_id
 
-!> 
+!>
 !! @public
 !! @ingroup PIO_def_var
 !! @brief Defines the a netcdf variable
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param name : The name of the variable to define
-!! @param type : The type of variable 
+!! @param type : The type of variable
 !! @param dimids : The dimension identifier returned by \ref PIO_def_dim
 !! @param vardesc @copydoc var_desc_t
 !! @retval ierr @copydoc error_return
@@ -1492,8 +1556,8 @@ integer function def_var_md_desc(File,name,type,dimids,vardesc) result(ierr)
 
     ierr = def_var_md_id(file%fh,name,type,dimids,vardesc%varid)
   end function def_var_md_desc
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var
 !! @brief Defines a netcdf variable
 !<
@@ -1526,8 +1590,8 @@ end function PIOc_def_var
     varid = varid+1
   end function def_var_md_id
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var_deflate
 !! @brief Changes compression settings for a netCDF-4/HDF5 variable.
 !<
@@ -1554,8 +1618,8 @@ end function PIOc_def_var_deflate
     ierr = PIOc_def_var_deflate(file%fh, varid-1, shuffle, deflate, deflate_level)
   end function def_var_deflate_id
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var_deflate
 !! @brief Changes compression settings for a netCDF-4/HDF5 variable.
 !<
@@ -1570,8 +1634,8 @@ integer function def_var_deflate_desc(file, vardesc, shuffle, deflate, deflate_l
     ierr = def_var_deflate_id(file, vardesc%varid, shuffle, deflate, deflate_level)
   end function def_var_deflate_desc
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_def_var_chunking
 !! @brief Changes chunking settings for a netCDF-4/HDF5 variable.
 !<
@@ -1581,7 +1645,7 @@ integer function def_var_chunking(file, vardesc, storage, chunksizes) result(ier
     integer, intent(in) :: storage
     integer, intent(in) :: chunksizes(:)
     integer(C_INT) :: cchunksizes(PIO_MAX_VAR_DIMS)
-    integer :: ndims, i    
+    integer :: ndims, i
 
     interface
        integer (C_INT) function PIOc_def_var_chunking(ncid, varid, storage, chunksizes) &
@@ -1601,15 +1665,15 @@ end function PIOc_def_var_chunking
     ierr = PIOc_def_var_chunking(file%fh, vardesc%varid-1, storage, cchunksizes)
   end function def_var_chunking
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_set_chunk_cache
 !! @brief Changes chunk cache settings for netCDF-4/HDF5 files created after this call.
 !<
   integer function set_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
        chunk_cache_preemption) result(ierr)
     integer, intent(in) :: iosysid
-    integer, intent(in) :: iotype 
+    integer, intent(in) :: iotype
     integer(kind=PIO_OFFSET_KIND), intent(in) :: chunk_cache_size
     integer(kind=PIO_OFFSET_KIND), intent(in) :: chunk_cache_nelems
     real, intent(in) :: chunk_cache_preemption
@@ -1631,9 +1695,9 @@ end function PIOc_set_chunk_cache
          chunk_cache_preemption)
   end function set_chunk_cache
 
-!> 
+!>
 !! @public
-!! @ingroup PIO_set_chunk_cache  
+!! @ingroup PIO_get_chunk_cache
 !! @brief Gets current settings for chunk cache (only relevant for netCDF4/HDF5 files.)
 !<
   integer function get_chunk_cache(iosysid, iotype, chunk_cache_size, chunk_cache_nelems, &
@@ -1661,9 +1725,9 @@ end function PIOc_get_chunk_cache
          chunk_cache_preemption)
   end function get_chunk_cache
 
-!> 
-!! @public 
-!! @ingroup PIO_set_chunk_cache
+!>
+!! @public
+!! @ingroup PIO_set_var_chunk_cache
 !! @brief Changes chunk cache settings for a variable in a netCDF-4/HDF5 file.
 !<
   integer function set_var_chunk_cache_id(file, varid, chunk_cache_size, &
@@ -1691,8 +1755,8 @@ end function PIOc_set_var_chunk_cache
          chunk_cache_nelems, chunk_cache_preemption)
   end function set_var_chunk_cache_id
 
-  !> 
-!! @public 
+  !>
+!! @public
 !! @ingroup PIO_set_var_chunk_cache
 !! @brief Changes chunk cacne for a variable.
 !<
@@ -1708,8 +1772,8 @@ integer function set_var_chunk_cache_desc(file, vardesc, chunk_cache_size, &
          chunk_cache_nelems, chunk_cache_preemption)
   end function set_var_chunk_cache_desc
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_get_var_chunk_cache
 !! @brief Get the chunk cache settings for a variable.
 !<
@@ -1725,8 +1789,8 @@ integer function get_var_chunk_cache_desc(file, vardesc, chunk_cache_size, &
          chunk_cache_nelems, chunk_cache_preemption)
   end function get_var_chunk_cache_desc
 
-!> 
-!! @public 
+!>
+!! @public
 !! @ingroup PIO_get_var_chunk_cache
 !! @brief Get the chunk cache settings for a variable.
 !<
diff --git a/externals/pio2/src/flib/pio_support.F90 b/src/externals/pio2/src/flib/pio_support.F90
similarity index 100%
rename from externals/pio2/src/flib/pio_support.F90
rename to src/externals/pio2/src/flib/pio_support.F90
diff --git a/externals/pio2/src/flib/pio_types.F90 b/src/externals/pio2/src/flib/pio_types.F90
similarity index 76%
rename from externals/pio2/src/flib/pio_types.F90
rename to src/externals/pio2/src/flib/pio_types.F90
index c3482ffbb26..d8bb4a3f019 100644
--- a/externals/pio2/src/flib/pio_types.F90
+++ b/src/externals/pio2/src/flib/pio_types.F90
@@ -13,7 +13,7 @@ module pio_types
     !-------------------------------------------
     type, public :: DecompMap_t
 #ifdef SEQUENCE
-	sequence
+        sequence
 #endif
         integer(i4) :: start
         integer(i4) :: length
@@ -73,7 +73,7 @@ module pio_types
 !<
    type, public :: io_desc_t
 #ifdef SEQUENCE
-	sequence
+        sequence
 #endif
         integer(i4)         :: ioid
     end type
@@ -140,6 +140,14 @@ module pio_types
   integer(i4), public, parameter :: PIO_BCAST_ERROR = -52
   integer(i4), public, parameter :: PIO_RETURN_ERROR = -53
 
+!>
+!! @public
+!! @defgroup PIO_error_method error_methods
+!! @details
+!! Use this instead of ios to set error handling for the library.
+!<
+  integer(i4), public, parameter :: PIO_DEFAULT = -1
+
 !>
 !! @public
 !! @defgroup error_return error return codes
@@ -229,17 +237,69 @@ module pio_types
 #endif
    integer, public, parameter :: PIO_num_OST =  16
 
-   type, public :: PIO_rearr_comm_fc_opt_t
-      logical :: enable_hs            ! Enable handshake?
-      logical :: enable_isend         ! Enable isends?
-      integer :: max_pend_req         ! Maximum pending requests
+!>
+!! @defgroup PIO_rearr_comm_t PIO_rearr_comm_t
+!! @public 
+!! @brief The two choices for rearranger communication
+!! @details
+!!  - PIO_rearr_comm_p2p : Point to point
+!!  - PIO_rearr_comm_coll : Collective
+!>
+    enum, bind(c)
+      enumerator :: PIO_rearr_comm_p2p = 0
+      enumerator :: PIO_rearr_comm_coll
+    end enum
+
+!>
+!! @defgroup PIO_rearr_comm_dir PIO_rearr_comm_dir
+!! @public 
+!! @brief The four choices for rearranger communication direction
+!! @details
+!!  - PIO_rearr_comm_fc_2d_enable : COMM procs to IO procs and vice versa
+!!  - PIO_rearr_comm_fc_1d_comp2io: COMM procs to IO procs only
+!!  - PIO_rearr_comm_fc_1d_io2comp: IO procs to COMM procs only
+!!  - PIO_rearr_comm_fc_2d_disable: Disable flow control
+!>
+    enum, bind(c)
+      enumerator :: PIO_rearr_comm_fc_2d_enable = 0
+      enumerator :: PIO_rearr_comm_fc_1d_comp2io
+      enumerator :: PIO_rearr_comm_fc_1d_io2comp
+      enumerator :: PIO_rearr_comm_fc_2d_disable
+    end enum
+
+!>
+!! @defgroup PIO_rearr_comm_fc_options PIO_rearr_comm_fc_options
+!! @brief Type that defines the PIO rearranger options
+!! @details
+!!  - enable_hs : Enable handshake (true/false) 
+!!  - enable_isend : Enable Isends (true/false)
+!!  - max_pend_req : Maximum pending requests (To indicated unlimited
+!!                    number of requests use PIO_REARR_COMM_UNLIMITED_PEND_REQ)
+!>
+   type, bind(c), public :: PIO_rearr_comm_fc_opt_t
+      logical(c_bool) :: enable_hs            ! Enable handshake?
+      logical(c_bool) :: enable_isend         ! Enable isends?
+      integer(c_int) :: max_pend_req         ! Maximum pending requests
     end type PIO_rearr_comm_fc_opt_t
 
-    type, public :: PIO_rearr_opt_t
-      integer                         :: comm_type
-      integer                         :: fcd       ! Flow control direction
+    integer, public, parameter :: PIO_REARR_COMM_UNLIMITED_PEND_REQ = -1
+!>
+!! @defgroup PIO_rearr_options PIO_rearr_options
+!! @brief Type that defines the PIO rearranger options
+!! @details
+!!  - comm_type : @copydoc PIO_rearr_comm_t
+!!  - fcd : @copydoc PIO_rearr_comm_dir
+!!  - comm_fc_opts : @copydoc PIO_rearr_comm_fc_options
+!>
+    type, bind(c), public :: PIO_rearr_opt_t
+      integer(c_int)                         :: comm_type
+      integer(c_int)                         :: fcd       ! Flow control direction
       type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_comp2io
       type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_io2comp
     end type PIO_rearr_opt_t
 
+    public :: PIO_rearr_comm_p2p, PIO_rearr_comm_coll,&
+              PIO_rearr_comm_fc_2d_enable, PIO_rearr_comm_fc_1d_comp2io,&
+              PIO_rearr_comm_fc_1d_io2comp, PIO_rearr_comm_fc_2d_disable
+
 end module pio_types
diff --git a/externals/pio2/src/flib/piodarray.F90.in b/src/externals/pio2/src/flib/piodarray.F90.in
similarity index 97%
rename from externals/pio2/src/flib/piodarray.F90.in
rename to src/externals/pio2/src/flib/piodarray.F90.in
index f39926f96d7..7c80e89df8a 100644
--- a/externals/pio2/src/flib/piodarray.F90.in
+++ b/src/externals/pio2/src/flib/piodarray.F90.in
@@ -1,6 +1,6 @@
 #define __PIO_FILE__ 'piodarray'
 !>
-!! @file 
+!! @file
 !! @brief Read and write routines for decomposed data.
 !<
 module piodarray
@@ -16,8 +16,8 @@ module piodarray
   private
   public :: pio_read_darray, pio_write_darray, pio_set_buffer_size_limit
 
-  
-!> 
+
+!>
 !! @defgroup PIO_write_darray PIO_write_darray
 !! @brief The overloaded PIO_write_darray writes a distributed array to disk.
 !<
@@ -30,7 +30,7 @@ module piodarray
   end interface
 
 
-!> 
+!>
 !! @defgroup PIO_read_darray PIO_read_darray
 !! @brief The overloaded PIO_read_darray function reads a distributed array from disk.
 !<
@@ -90,9 +90,9 @@ end interface
 contains
 
   subroutine pio_set_buffer_size_limit(limit)
-    integer(PIO_OFFSET_KIND), intent(in) :: limit    
+    integer(PIO_OFFSET_KIND), intent(in) :: limit
     integer(PIO_OFFSET_KIND) :: oldval
-    interface 
+    interface
        integer(C_LONG_LONG) function PIOc_set_buffer_size_limit(limit) &
             bind(C,name="PIOc_set_buffer_size_limit")
          use iso_c_binding
@@ -103,7 +103,7 @@ contains
        call piodie(__PIO_FILE__,__LINE__,' bad value to buffer_size_limit: ',int(limit))
     end if
     oldval = PIOc_set_buffer_size_limit(limit)
-       
+
   end subroutine pio_set_buffer_size_limit
 
 ! TYPE real,int,double
@@ -138,7 +138,7 @@ contains
     carraylen = int(arraylen,C_SIZE_T)
     cptr = C_LOC(array)
 #ifdef TIMING
-    call t_startf("PIO:write_darray_{TYPE}")    
+    call t_startf("PIO:write_darray_{TYPE}")
 #endif
     if(present(fillval)) then
        iostat = PIOc_write_darray(file%fh, varDesc%varid-1, iodesc%ioid, carraylen,cptr, C_LOC(fillval))
@@ -146,7 +146,7 @@ contains
        iostat = PIOc_write_darray(file%fh, varDesc%varid-1, iodesc%ioid, carraylen, cptr, C_NULL_PTR)
     endif
 #ifdef TIMING
-    call t_stopf("PIO:write_darray_{TYPE}")    
+    call t_stopf("PIO:write_darray_{TYPE}")
 #endif
   end subroutine write_darray_1d_cinterface_{TYPE}
 
@@ -183,7 +183,7 @@ contains
     type(C_PTR) :: cptr
     integer :: i
     carraylen = int(arraylen,C_SIZE_T)
-    
+
     cptr = C_LOC(array)
     do i=1,nvars
        varid(i) = vardesc(i)%varid-1
@@ -198,10 +198,10 @@ contains
   end subroutine write_darray_multi_1d_cinterface_{TYPE}
 
 ! TYPE real,int,double
-!> 
+!>
 !! @public
 !! @ingroup PIO_write_darray
-!! @brief Writes a 1D array of type {TYPE}. 
+!! @brief Writes a 1D array of type {TYPE}.
 !! @details
 !! @param File    \ref file_desc_t
 !! @param varDesc \ref var_desc_t
@@ -209,7 +209,7 @@ contains
 !! @param array  : The data to be written
 !! @param iostat : The status returned from this routine (see \ref PIO_seterrorhandling for details)
 !! @param fillval : An optional fill value to fill holes in the data written
-!<  
+!<
   subroutine write_darray_multi_1d_{TYPE} (File,varDesc,ioDesc, array, iostat, fillval)
     ! !DESCRIPTION:
     !  Writes a block of TYPE to a netcdf file.
@@ -238,7 +238,7 @@ contains
     integer :: nvars
 
     nvars = size(vardesc)
-    
+
     call write_darray_multi_1d_cinterface_{TYPE} (file, varDesc, iodesc, nvars, size(array), array, iostat, fillval)
 
   end subroutine write_darray_multi_1d_{TYPE}
@@ -269,17 +269,17 @@ contains
     integer(i4), intent(out) :: iostat
 
     character(len=*), parameter :: subName=modName//'::write_darray_{TYPE}'
-    
+
     call write_darray_1d_cinterface_{TYPE} (file, varDesc, iodesc, size(array), array, iostat, fillval)
 
   end subroutine write_darray_1d_{TYPE}
 
 ! TYPE real,int,double
 ! DIMS 2,3,4,5,6,7
-!> 
+!>
 !! @public
 !! @ingroup PIO_write_darray
-!! @brief Writes a {DIMS}D array of type {TYPE}. 
+!! @brief Writes a {DIMS}D array of type {TYPE}.
 !! @details
 !! @param File @ref file_desc_t
 !! @param varDesc @ref var_desc_t
@@ -287,7 +287,7 @@ contains
 !! @param array  : The data to be written
 !! @param iostat : The status returned from this routine (see \ref PIO_seterrorhandling for details)
 !! @param fillval : An optional fill value to fill holes in the data written
-!<  
+!<
   subroutine write_darray_{DIMS}d_{TYPE} (File,varDesc,ioDesc, array, iostat, fillval)
     ! !INPUT PARAMETERS:
 
@@ -323,7 +323,7 @@ contains
 ! cannot call transfer function with a 0 sized array
     if(size(array)==0) then
        call write_darray_1d_{TYPE} (File, varDesc, iodesc, dumbvar, iostat)
-    else 
+    else
        call write_darray_1d_{TYPE} (File, varDesc, iodesc, transfer(array,transvar), iostat, fillval)
     end if
 #endif
@@ -331,7 +331,7 @@ contains
 
 ! TYPE real,int,double
 ! DIMS 1,2,3,4,5,6,7
-!> 
+!>
 !! @public
 !! @ingroup PIO_read_darray
 !! @brief Read distributed array of type {TYPE} from a netCDF variable of {DIMS} dimension(s).
@@ -386,7 +386,7 @@ contains
     ! !INPUT PARAMETERS:
     integer, intent(in) :: ncid, varid, ioid
     integer(C_SIZE_T), intent(in) :: alen
-    
+
     {VTYPE}, target ::  array(*)                 ! array to be read
 
     integer(i4), intent(out) :: iostat
diff --git a/externals/pio2/src/flib/piolib_mod.F90 b/src/externals/pio2/src/flib/piolib_mod.F90
similarity index 91%
rename from externals/pio2/src/flib/piolib_mod.F90
rename to src/externals/pio2/src/flib/piolib_mod.F90
index 70497c53cf8..e704dab4508 100644
--- a/externals/pio2/src/flib/piolib_mod.F90
+++ b/src/externals/pio2/src/flib/piolib_mod.F90
@@ -12,10 +12,11 @@ module piolib_mod
   use pio_kinds
   !--------------
   use pio_types, only : file_desc_t, iosystem_desc_t, var_desc_t, io_desc_t, &
-	pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4p, pio_iotype_netcdf4c, &
-        pio_noerr, pio_rearr_subset
+        pio_iotype_netcdf, pio_iotype_pnetcdf, pio_iotype_netcdf4p, pio_iotype_netcdf4c, &
+        pio_noerr, pio_rearr_subset, pio_rearr_opt_t
   !--------------
   use pio_support, only : piodie, debug, debugio, debugasync, checkmpireturn
+  use pio_nf, only : pio_set_log_level
   !
 
 
@@ -50,7 +51,8 @@ module piolib_mod
        PIO_FILE_IS_OPEN, &
        PIO_deletefile, &
        PIO_get_numiotasks, &
-       PIO_iotype_available
+       PIO_iotype_available, &
+       PIO_set_rearr_opts
 
 #ifdef MEMCHK
 !> this is an internal variable for memory leak debugging
@@ -142,7 +144,7 @@ module piolib_mod
 !! @defgroup PIO_initdecomp PIO_initdecomp
 !! @brief PIO_initdecomp is an overload interface the models decomposition to pio.
 !! @details initdecomp_1dof_bin_i8, initdecomp_1dof_nf_i4, initdecomp_2dof_bin_i4,
-!! and initdecomp_2dof_nf_i4 are all deprecated, but supported for backwards
+!! and initdecomp_2dof_nf_i4 are all depreciated, but supported for backwards
 !! compatibility.
 !<
   interface PIO_initdecomp
@@ -198,6 +200,7 @@ module piolib_mod
   interface PIO_seterrorhandling
      module procedure seterrorhandlingf
      module procedure seterrorhandlingi
+     module procedure seterrorhandlingg
   end interface
 
 !>
@@ -329,6 +332,7 @@ end subroutine setframe
 !<
   subroutine setdebuglevel(level)
     integer(i4), intent(in) :: level
+    integer :: ierr
     if(level.eq.0) then
        debug=.false.
        debugio=.false.
@@ -357,7 +361,11 @@ subroutine setdebuglevel(level)
        debug=.true.
        debugio=.true.
        debugasync=.true.
-
+    end if
+    ierr = PIO_set_log_level(level)
+    if(ierr /= PIO_NOERR) then
+      ! This is not a fatal error
+      print *, __PIO_FILE__, __LINE__, "Setting log level failed, ierr =",ierr
     end if
   end subroutine setdebuglevel
 
@@ -406,6 +414,35 @@ end function PIOc_Set_IOSystem_Error_Handling
 
   end subroutine seterrorhandlingi
 
+!>
+!! @ingroup PIO_seterrorhandling
+!! @public
+!! @brief set the pio error handling method for the iosystem
+!! @param iosystem : a defined pio system descriptor, see PIO_types
+!! @param method :
+!! @copydoc PIO_error_method
+!<
+  subroutine seterrorhandlingg(global, method, oldmethod)
+    integer, intent(in) :: global
+    integer, intent(in) :: method
+    integer, intent(out), optional :: oldmethod
+
+    interface
+       integer(c_int) function PIOc_Set_IOSystem_Error_Handling(global, method) &
+            bind(C,name="PIOc_Set_IOSystem_Error_Handling")
+         use iso_c_binding
+         integer(c_int), value :: global
+         integer(c_int), value :: method
+       end function PIOc_Set_IOSystem_Error_Handling
+    end interface
+    integer(c_int) ::  loldmethod
+
+    loldmethod = PIOc_Set_IOSystem_Error_Handling(global, method)
+    if(present(oldmethod)) oldmethod = loldmethod
+
+
+  end subroutine seterrorhandlingg
+
 
 !>
 !! @public
@@ -844,8 +881,10 @@ end subroutine PIO_initdecomp_dof_i8
 !! @param iosystem a derived type which can be used in subsequent pio operations (defined in PIO_types).
 !! @param base @em optional argument can be used to offset the first io task - default base is task 1.
 !<
-  subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride,  rearr, iosystem,base)
-    use pio_types, only : pio_internal_error
+  subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride,  rearr, iosystem,base, rearr_opts)
+    use pio_types, only : pio_internal_error, pio_rearr_opt_t
+    use iso_c_binding
+
     integer(i4), intent(in) :: comp_rank
     integer(i4), intent(in) :: comp_comm
     integer(i4), intent(in) :: num_iotasks
@@ -854,17 +893,21 @@ subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stri
     integer(i4), intent(in) :: rearr
     type (iosystem_desc_t), intent(out)  :: iosystem  ! io descriptor to initalize
     integer(i4), intent(in),optional :: base
+    type (pio_rearr_opt_t), intent(in), optional :: rearr_opts
+
     integer :: lbase
     integer :: ierr
     interface
-       integer(c_int) function PIOc_Init_Intracomm_from_F90(f90_comp_comm, num_iotasks, stride,base,rearr,iosysidp) &
+       integer(c_int) function PIOc_Init_Intracomm_from_F90(f90_comp_comm, num_iotasks, stride,base,rearr,rearr_opts,iosysidp) &
             bind(C,name="PIOc_Init_Intracomm_from_F90")
          use iso_c_binding
+         use pio_types
          integer(C_INT), value :: f90_comp_comm
          integer(C_INT), value :: num_iotasks
          integer(C_INT), value :: stride
          integer(C_INT), value :: base
          integer(C_INT), value :: rearr
+          type(pio_rearr_opt_t) :: rearr_opts
          integer(C_INT) :: iosysidp
        end function PIOc_Init_Intracomm_from_F90
     end interface
@@ -874,7 +917,7 @@ end function PIOc_Init_Intracomm_from_F90
 #endif
     lbase=0
     if(present(base)) lbase=base
-    ierr = PIOc_Init_Intracomm_from_F90(comp_comm,num_iotasks,stride,lbase,rearr,iosystem%iosysid)
+    ierr = PIOc_Init_Intracomm_from_F90(comp_comm,num_iotasks,stride,lbase,rearr,rearr_opts,iosystem%iosysid)
 
     call CheckMPIReturn("Bad Initialization in PIO_Init_Intracomm:  ", ierr,__FILE__,__LINE__)
 #ifdef TIMING
@@ -1212,7 +1255,7 @@ end function pio_iotype_available
 !! @brief  Create a NetCDF or PNetCDF file using PIO.
 !! @details  Input parameters are read on comp task 0 and ignored elsewhere
 !! @param iosystem : A defined pio system descriptor created by a call to @ref PIO_init (see PIO_types)
-!! @param file	:  The returned file descriptor
+!! @param file  :  The returned file descriptor
 !! @param iotype : @copydoc PIO_iotype
 !! @param fname : The name of the file to open
 !! @param amode_in : The NetCDF creation mode flag. the following flags are available:
@@ -1267,7 +1310,7 @@ end function createfile
 !! @brief open an existing file using pio
 !! @details  Input parameters are read on comp task 0 and ignored elsewhere.
 !! @param iosystem : a defined pio system descriptor created by a call to @ref PIO_init (see PIO_types)
-!! @param file	:  the returned file descriptor
+!! @param file  :  the returned file descriptor
 !! @param iotype : @copybrief PIO_iotype
 !! @param fname : the name of the file to open
 !! @param mode : a zero value (or PIO_nowrite) specifies the default
@@ -1482,6 +1525,64 @@ end function PIOc_deletefile
 
   end subroutine pio_deletefile
 
+!>
+!! @public
+!! @ingroup PIO_set_rearr_opts
+!! @brief Set the rerranger options
+!! @details
+!! @param ios : handle to pio iosystem
+!! @param comm_type : @copydoc PIO_rearr_comm_t
+!! @param fcd : @copydoc PIO_rearr_comm_dir
+!! @param enable_hs_c2i : Enable handshake (compute procs to io procs)
+!! @param enable_isend_c2i : Enable isends (compute procs to io procs)
+!! @param max_pend_req_c2i: Maximum pending requests (compute procs to io procs)
+!! @param enable_hs_i2c : Enable handshake (io procs to compute procs)
+!! @param enable_isend_i2c : Enable isends (io procs to compute procs)
+!! @param max_pend_req_i2c: Maximum pending requests (io procs to compute procs)
+!! @copydoc PIO_rearr_comm_fc_options
+!<
+  function pio_set_rearr_opts(ios, comm_type, fcd,&
+                              enable_hs_c2i, enable_isend_c2i,&
+                              max_pend_req_c2i,&
+                              enable_hs_i2c, enable_isend_i2c,&
+                              max_pend_req_i2c) result(ierr)
+
+    type(iosystem_desc_t), intent(inout) :: ios
+    integer, intent(in) :: comm_type, fcd
+    logical, intent(in) :: enable_hs_c2i, enable_hs_i2c
+    logical, intent(in) :: enable_isend_c2i, enable_isend_i2c
+    integer, intent(in) :: max_pend_req_c2i, max_pend_req_i2c
+    integer :: ierr
+    interface
+      integer(c_int) function PIOc_set_rearr_opts(iosysid, comm_type, fcd,&
+                                                  enable_hs_c2i, enable_isend_c2i,&
+                                                  max_pend_req_c2i,&
+                                                  enable_hs_i2c, enable_isend_i2c,&
+                                                  max_pend_req_i2c)&
+        bind(C,name="PIOc_set_rearr_opts")
+        use iso_c_binding
+        integer(C_INT), intent(in), value :: iosysid
+        integer(C_INT), intent(in), value :: comm_type
+        integer(C_INT), intent(in), value :: fcd
+        logical(C_BOOL), intent(in), value :: enable_hs_c2i
+        logical(C_BOOL), intent(in), value :: enable_isend_c2i
+        integer(C_INT), intent(in), value :: max_pend_req_c2i
+        logical(C_BOOL), intent(in), value :: enable_hs_i2c
+        logical(C_BOOL), intent(in), value :: enable_isend_i2c
+        integer(C_INT), intent(in), value :: max_pend_req_i2c
+      end function PIOc_set_rearr_opts
+    end interface
+
+    ierr = PIOc_set_rearr_opts(ios%iosysid, comm_type, fcd,&
+                                logical(enable_hs_c2i, kind=c_bool),&
+                                logical(enable_isend_c2i, kind=c_bool),&
+                                max_pend_req_c2i,&
+                                logical(enable_hs_i2c, kind=c_bool),&
+                                logical(enable_isend_i2c, kind=c_bool),&
+                                max_pend_req_i2c)
+
+  end function pio_set_rearr_opts
+
 
 end module piolib_mod
 
diff --git a/externals/pio2/src/flib/pionfatt_mod.F90.in b/src/externals/pio2/src/flib/pionfatt_mod.F90.in
similarity index 95%
rename from externals/pio2/src/flib/pionfatt_mod.F90.in
rename to src/externals/pio2/src/flib/pionfatt_mod.F90.in
index 4066b96c073..daaca6dca81 100644
--- a/externals/pio2/src/flib/pionfatt_mod.F90.in
+++ b/src/externals/pio2/src/flib/pionfatt_mod.F90.in
@@ -1,6 +1,6 @@
 #define __PIO_FILE__ "pionfatt_mod.F90"
 !>
-!! @file 
+!! @file
 !! @brief NetCDF attribute interface to PIO
 !<
 module pionfatt_mod
@@ -37,14 +37,14 @@ module pionfatt_mod
   end interface
 
   !>
-  !!  @public 
+  !!  @public
   !!  @defgroup PIO_put_att PIO_put_att
-  !!  @brief Writes an netcdf attribute to a file 
+  !!  @brief Writes an netcdf attribute to a file
   !<
   !>
-  !! @public 
+  !! @public
   !! @defgroup PIO_get_att PIO_get_att
-  !! @brief Reads  an netcdf attribute from a file 
+  !! @brief Reads  an netcdf attribute from a file
   !<
 
   private :: modName
@@ -121,7 +121,7 @@ module pionfatt_mod
        integer(C_INT), intent(out)   :: op
      end function PIOc_get_att_int
   end interface
-    
+
   interface
      integer(C_INT) function PIOc_get_att_float (ncid, varid, name, op) &
           bind(C,name="PIOc_get_att_float")
@@ -132,7 +132,7 @@ module pionfatt_mod
        real(C_FLOAT), intent(out) :: op
      end function PIOc_get_att_float
   end interface
-    
+
   interface
      integer(C_INT) function PIOc_get_att_double (ncid, varid, name, op) &
           bind(C,name="PIOc_get_att_double")
@@ -143,19 +143,19 @@ module pionfatt_mod
        real(C_DOUBLE), intent(out)   :: op
      end function PIOc_get_att_double
   end interface
-    
+
 
 contains
 
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_put_att
   !! @brief Writes an netcdf attribute to a file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varid : The netcdf variable identifier
   !! @param name : name of the attribute to add
-  !! @param var : The value for the netcdf attribute 
+  !! @param var : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function put_att_desc_{TYPE} (File, vdesc, name, values) result(ierr)
@@ -208,7 +208,7 @@ contains
     deallocate(cvar)
   end function put_att_id_text
 
-  integer function put_att_1d_id_text (ncid, varid, name, value) result(ierr)  
+  integer function put_att_1d_id_text (ncid, varid, name, value) result(ierr)
     use iso_c_binding
     integer, intent(in) :: ncid
     integer, intent(in) :: varid
@@ -220,8 +220,8 @@ contains
     slen = len(value(1))
     alen = size(value)
     allocate(nvalue(slen*alen))
-    
-    do i=1,alen 
+
+    do i=1,alen
        j= len_trim(value(i))
        do k=1,j
           nvalue(k+(i-1)*slen) = value(i)(k:k)
@@ -244,9 +244,9 @@ contains
     character(len=*), intent(in) :: values(arrlen)
     integer :: vallen
 
-    
+
     ierr = PIOc_put_att_text (ncid,varid-1,trim(name)//C_NULL_CHAR, int(arrlen,C_SIZE_T),values(1))
-        
+
 
   end function put_att_1d_id_text_internal
 
@@ -267,14 +267,14 @@ contains
   !pl The next line is needed by genf90.pl, do not remove it.
   ! TYPE real,double,int
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_put_att
   !! @brief Writes an netcdf attribute to a file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varid : The netcdf variable identifier
   !! @param name : name of the attribute to add
-  !! @param values : The value for the netcdf attribute 
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function put_att_1d_id_{TYPE} (ncid, varid, name, values) result(ierr)
@@ -305,14 +305,14 @@ contains
 
   ! TYPE real,int,double
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_put_att
   !! @brief Writes an netcdf attribute to a file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varDesc @copydoc var_desc_t
   !! @param name : name of the attribute to add
-  !! @param var : The value for the netcdf attribute 
+  !! @param var : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function put_att_1d_desc_{TYPE} (File,varDesc,name,values) result(ierr)
@@ -339,14 +339,14 @@ contains
 
 
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_get_att
   !! @brief Reads an netcdf attribute from a  file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varDesc @copydoc var_desc_t
   !! @param name : name of the attribute to get
-  !! @param values : The value for the netcdf attribute 
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function get_att_desc_{TYPE} (File,varDesc,name,values) result(ierr)
@@ -361,14 +361,14 @@ contains
   end function get_att_desc_{TYPE}
 
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_get_att
   !! @brief Reads an netcdf attribute from a  file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varDesc @copydoc var_desc_t
   !! @param name : name of the attribute to get
-  !! @param values : The value for the netcdf attribute 
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   ! TYPE int,real,double
@@ -386,14 +386,14 @@ contains
   end function get_att_desc_1d_{TYPE}
 
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_get_att
   !! @brief Reads an netcdf attribute from a  file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varid : The netcdf variable identifier
   !! @param name : name of the attribute to get
-  !! @param values : The value for the netcdf attribute 
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   ! TYPE int,real,double
@@ -410,7 +410,7 @@ contains
   end function get_att_id_{TYPE}
 
 
-  
+
   integer function get_att_{TYPE} (File,varid,name,values) result(ierr)
     type (File_desc_t), intent(in) , target :: File
     integer(i4), intent(in)        :: varid
@@ -425,14 +425,14 @@ contains
 
   ! TYPE real,int,double
   !>
-  !! @public 
+  !! @public
   !! @ingroup PIO_get_att
   !! @brief Reads an netcdf attribute from a  file
   !! @details
   !! @param File @copydoc file_desc_t
   !! @param varid : The netcdf variable identifier
   !! @param name : name of the attribute to get
-  !! @param values : The value for the netcdf attribute 
+  !! @param values : The value for the netcdf attribute
   !! @retval ierr @copydoc error_return
   !<
   integer function get_att_1d_{TYPE} (File,varid,name,values) result(ierr)
@@ -460,4 +460,3 @@ contains
 
 
 end module pionfatt_mod
-
diff --git a/externals/pio2/src/flib/pionfget_mod.F90.in b/src/externals/pio2/src/flib/pionfget_mod.F90.in
similarity index 98%
rename from externals/pio2/src/flib/pionfget_mod.F90.in
rename to src/externals/pio2/src/flib/pionfget_mod.F90.in
index 3cbca7e6bed..ebd335dde2e 100644
--- a/externals/pio2/src/flib/pionfget_mod.F90.in
+++ b/src/externals/pio2/src/flib/pionfget_mod.F90.in
@@ -1,6 +1,6 @@
 #define __PIO_FILE__ "pionfget_mod.F90"
 !>
-!! @file 
+!! @file
 !! @brief Read Routines for non-decomposed NetCDF data.
 !<
 module pionfget_mod
@@ -17,8 +17,8 @@ module pionfget_mod
 !! @defgroup PIO_get_var PIO_get_var
 !! @brief Reads non-decomposed data from a NetCDF file
 !! @details The get_var interface is provided as a simplified interface to
-!!  read variables from a NetCDF format file.   The variable is read on the 
-!!  root IO task and broadcast in its entirety to all tasks.  
+!!  read variables from a NetCDF format file.   The variable is read on the
+!!  root IO task and broadcast in its entirety to all tasks.
 !<
   public :: get_var
   interface get_var
@@ -284,7 +284,7 @@ CONTAINS
 !!   the variable's dimensions. Hence, if the variable is a record
 !!   variable, the first element of count corresponds to a count of the
 !!   number of records to read.
-!! Note: setting any element of the count array to zero causes the function to exit without error, and without doing anything. 
+!! Note: setting any element of the count array to zero causes the function to exit without error, and without doing anything.
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @ref error_return
 !<
@@ -332,7 +332,7 @@ CONTAINS
 !!   the variable's dimensions. Hence, if the variable is a record
 !!   variable, the first element of count corresponds to a count of the
 !!   number of records to read.
-!! Note: setting any element of the count array to zero causes the function to exit without error, and without doing anything. 
+!! Note: setting any element of the count array to zero causes the function to exit without error, and without doing anything.
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @ref error_return
 !<
@@ -379,11 +379,11 @@ CONTAINS
     integer, intent(in) :: varid
     {VTYPE}, intent(out) :: ival
     {VTYPE}  :: aival(1)
-    
+
 
     ierr = PIOc_get_var_{NCTYPE} (File%fh, varid-1, aival)
     ival = aival(1)
-       
+
   end function Get_var_0d_{TYPE}
 
 ! DIMS 1,2,3,4,5
@@ -395,7 +395,7 @@ CONTAINS
     ierr = get_var_text_internal(File%fh, varid, size(ival), ival)
 
   end function get_var_{DIMS}d_text
- 
+
   integer function get_var_text_internal (ncid,varid, nstrs, ival) result(ierr)
     integer, intent(in) :: ncid
     integer, intent(in) :: varid
diff --git a/externals/pio2/src/flib/pionfput_mod.F90.in b/src/externals/pio2/src/flib/pionfput_mod.F90.in
similarity index 98%
rename from externals/pio2/src/flib/pionfput_mod.F90.in
rename to src/externals/pio2/src/flib/pionfput_mod.F90.in
index fdc3e624197..d4b54220a94 100644
--- a/externals/pio2/src/flib/pionfput_mod.F90.in
+++ b/src/externals/pio2/src/flib/pionfput_mod.F90.in
@@ -1,6 +1,6 @@
 #define __PIO_FILE__ "pionfput_mod.F90"
 !>
-!! @file 
+!! @file
 !! @brief Write routines for non-decomposed NetCDF data.
 !<
 module pionfput_mod
@@ -18,17 +18,17 @@ module pionfput_mod
 !! @defgroup PIO_put_var PIO_put_var
 !! @brief Writes data to a netCDF file.
 !! @details The put_var interface is provided as a simplified interface to
-!!  write variables to a netcdf format file.   
-!! @warning Although this is a collective call the variable is written from the 
+!!  write variables to a netcdf format file.
+!! @warning Although this is a collective call the variable is written from the
 !!  root IO task, no consistancy check is made with data passed on other tasks.
-!! 
+!!
 !<
   public :: put_var
   interface put_var
      !  DIMS 0,1,2,3,4,5
      module procedure put_var_{DIMS}d_{TYPE}, put_var_vdesc_{DIMS}d_{TYPE}
      !  DIMS 1,2,3,4,5
-     module procedure put_vara_{DIMS}d_{TYPE} 
+     module procedure put_vara_{DIMS}d_{TYPE}
      !  DIMS 1,2,3,4,5
      module procedure put_vara_vdesc_{DIMS}d_{TYPE}
      module procedure put_var1_{TYPE}, put_var1_vdesc_{TYPE}
@@ -99,7 +99,7 @@ contains
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param varid : The netcdf variable identifier
-!! @param index : 
+!! @param index :
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @copydoc error_return
 !<
@@ -111,7 +111,7 @@ contains
     integer :: i
     integer, allocatable :: count(:)
     integer :: ndims
-    
+
     ndims = size(index)
     allocate(count(ndims))
     count = 1
@@ -134,7 +134,7 @@ contains
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param varid : The netcdf variable identifier
-!! @param index : 
+!! @param index :
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @copydoc error_return
 !<
@@ -157,18 +157,18 @@ contains
 
 #ifdef TIMING
     call t_startf("PIO:put_var1_{TYPE}")
-#endif 
+#endif
     clen = size(index)
     allocate(cindex(clen))
     do i=1,clen
        cindex(i) = index(clen-i+1)-1
     enddo
-    
+
     ierr = PIOc_put_var1_{NCTYPE} (file%fh, varid-1, cindex, ival)
     deallocate(cindex)
 #ifdef TIMING
     call t_stopf("PIO:put_var1_{TYPE}")
-#endif 
+#endif
   end function put_var1_{TYPE}
 
 !>
@@ -178,7 +178,7 @@ contains
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param vardesc @copydoc var_desc_t
-!! @param start : 
+!! @param start :
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @copydoc error_return
 !<
@@ -220,7 +220,7 @@ contains
     do i=1,len_trim(ival)
        cval(i) = ival(i:i)
     end do
-    
+
     ierr = PIOc_put_var_text(file%fh, varid-1,  cval)
 
     deallocate(cval)
@@ -276,7 +276,7 @@ contains
     integer, intent(in) :: ncid
     integer, intent(in) :: varid
     {VTYPE}, intent(in) :: ival(*)
-    
+
     interface
        integer(C_INT) function PIOc_put_var_{NCTYPE}(ncid, varid, op) &
             bind(C,name="PIOc_put_var_{NCTYPE}")
@@ -338,13 +338,13 @@ contains
     ierr=PIO_NOERR
 #ifdef TIMING
     call t_startf("PIO:put_var_0d_{TYPE}")
-#endif 
+#endif
 
     ierr = put_var_internal_{TYPE} (File%fh, varid, (/ival/))
 
 #ifdef TIMING
     call t_stopf("PIO:put_var_0d_{TYPE}")
-#endif 
+#endif
   end function put_var_0d_{TYPE}
 
 
@@ -419,7 +419,7 @@ contains
     integer(C_SIZE_T), allocatable :: cstart(:), ccount(:)
     integer :: i
     integer :: ndims
-   
+
     do i=1,size(count)
        if(count(i)<=0) then
           ndims=i-1
@@ -528,7 +528,7 @@ contains
     enddo
 #endif
 
-    
+
   end subroutine Fstring2Cstring_{DIMS}d
 
 
@@ -554,13 +554,13 @@ contains
     {VTYPE}, intent(in) :: ival{DIMSTR}
 #ifdef TIMING
     call t_startf("PIO:put_vara_{DIMS}d_{TYPE}")
-#endif 
+#endif
 
     ierr = put_vara_internal_{TYPE} (File%fh, varid, start, count, ival)
 
 #ifdef TIMING
     call t_stopf("PIO:put_vara_{DIMS}d_{TYPE}")
-#endif 
+#endif
   end function put_vara_{DIMS}d_{TYPE}
 
 ! DIMS 1,2,3,4,5
@@ -571,8 +571,8 @@ contains
 !! @details
 !! @param File @copydoc file_desc_t
 !! @param vardesc @copydoc var_desc_t
-!! @param start : 
-!! @param count : 
+!! @param start :
+!! @param count :
 !! @param ival : The value for the netcdf metadata
 !! @retval ierr @copydoc error_return
 !<
diff --git a/src/externals/pio2/src/gptl/CMakeLists.txt b/src/externals/pio2/src/gptl/CMakeLists.txt
new file mode 100644
index 00000000000..5f73687f23d
--- /dev/null
+++ b/src/externals/pio2/src/gptl/CMakeLists.txt
@@ -0,0 +1,156 @@
+cmake_minimum_required (VERSION 2.8.12)
+project (GPTL C Fortran)
+include (CheckFunctionExists)
+include (FortranCInterface)
+FortranCInterface_HEADER(cmake_fortran_c_interface.h
+  MACRO_NAMESPACE "FCI_")
+include (LibCheck)
+
+#==============================================================================
+#  DEFINE THE TARGET
+#==============================================================================
+
+set (GPTL_C_SRCS GPTLget_memusage.c
+  GPTLprint_memusage.c
+  GPTLutil.c
+  f_wrappers.c
+  gptl.c
+  gptl_papi.c
+  threadutil.c)
+
+set (GPTL_Fortran_SRCS perf_mod.F90
+  perf_utils.F90)
+
+set (GPTL_Fortran_MODS ${CMAKE_CURRENT_BINARY_DIR}/perf_mod.mod
+  ${CMAKE_CURRENT_BINARY_DIR}/perf_utils.mod)
+
+add_library (gptl ${GPTL_Fortran_SRCS} ${GPTL_C_SRCS})
+
+target_include_directories (gptl
+  PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}
+  PUBLIC ${CMAKE_CURRENT_BINARY_DIR})
+
+target_compile_definitions (gptl
+  PUBLIC INCLUDE_CMAKE_FCI)
+
+if (CMAKE_SYSTEM_NAME MATCHES "AIX")
+  target_compile_definitions (gptl
+    PUBLIC _AIX)
+else ()
+  target_compile_definitions (gptl
+    PUBLIC ${CMAKE_SYSTEM_DIRECTIVE})
+endif ()
+
+target_compile_definitions (gptl
+  PUBLIC ${CMAKE_Fortran_COMPILER_DIRECTIVE})
+
+if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
+  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
+  #    target_compile_options (gptl
+  #        PRIVATE -mismatch_all)
+endif ()
+
+#==============================================================================
+#  DEFINE THE INSTALL
+#==============================================================================
+
+# Library
+install (TARGETS gptl DESTINATION lib)
+
+# Header/Include File
+install (FILES ${CMAKE_CURRENT_SOURCE_DIR}/gptl.h DESTINATION include)
+
+# Fortran Modules
+install (FILES ${GPTL_Fortran_MODS} DESTINATION include)
+
+#==============================================================================
+#  DEFINE THE DEPENDENCIES
+#==============================================================================
+
+#===== PAPI =====
+if (ENABLE_PAPI)
+  find_package (PAPI)
+  if (PAPI_FOUND)
+    target_compile_definitions (gptl
+      PUBLIC HAVE_PAPI)
+    target_include_directories (gptl
+      PUBLIC ${PAPI_INCLUDE_DIRECTORIES})
+    target_link_libraries (gptl
+      PUBLIC ${PAPI_LIBRARIES})
+  endif ()
+endif ()
+
+#===== LIBRT =====
+if (ENABLE_LIBRT)
+  find_package (LIBRT)
+  if (LIBRT_FOUND)
+    target_compile_definitions (gptl
+      PUBLIC HAVE_LIBRT)
+    target_include_directories (gptl
+      PUBLIC ${LIBRT_INCLUDE_DIRECTORIES})
+    target_link_libraries (gptl
+      PUBLIC ${LIBRT_LIBRARIES})
+  endif ()
+endif ()
+
+#===== MPI =====
+if (PIO_USE_MPISERIAL)
+  find_package (MPISERIAL COMPONENTS C Fortran REQUIRED)
+  if (MPISERIAL_C_FOUND AND MPISERIAL_Fortran_FOUND)
+    target_compile_definitions (gptl
+      PRIVATE HAVE_MPI)
+    target_include_directories (gptl
+      PUBLIC ${MPISERIAL_C_INCLUDE_DIRS}
+      PUBLIC ${MPISERIAL_Fortran_INCLUDE_DIRS})
+    target_link_libraries (gptl
+      PUBLIC ${MPISERIAL_C_LIBRARIES}
+      PUBLIC ${MPISERIAL_Fortran_LIBRARIES})
+
+    set (MPI_C_LIBRARIES ${MPISERIAL_C_LIBRARIES})
+    set (MPI_Fortran_INCLUDE_PATH ${MPISERIAL_Fortran_INCLUDE_DIRS})
+  endif ()
+else ()
+  find_package (MPI REQUIRED)
+  if (MPI_C_FOUND AND MPI_Fortran_FOUND)
+    target_compile_definitions (gptl
+      PUBLIC HAVE_MPI)
+  endif ()
+endif ()
+
+# Check MPI library for Comm_f2c function
+set (CMAKE_REQUIRED_LIBRARIES ${MPI_C_LIBRARIES})
+check_function_exists (MPI_Comm_f2c MPI_HAS_COMM_F2C)
+if (NOT MPI_HAS_COMM_F2C)
+  target_compile_definitions(gptl
+    PRIVATE NO_COMM_F2C)
+endif ()
+
+# Check for MPI Fortran module
+find_path(MPIMOD_PATH
+  NAMES mpi.mod MPI.mod
+  PATHS ${MPI_Fortran_INCLUDE_PATH})
+check_macro (MPI_HAS_Fortran_MOD
+  NAME TryMPIMod.f90
+  HINTS ${CMAKE_MODULE_PATH}
+  DEFINITIONS -I${MPIMOD_PATH}
+  COMMENT "whether MPI Fortran module is supported")
+if (${MPI_HAS_Fortran_MOD})
+  message (STATUS "MPI Fortran module verified and enabled.")
+else ()
+  message (STATUS "MPI Fortran module failed verification and therefore disabled.")
+  target_compile_definitions (gptl
+    PUBLIC NO_MPIMOD)
+endif ()
+
+#===== GetTimeOfDay =====
+if (NOT DEFINED SYSTEM_HAS_GETTIMEOFDAY)
+  get_target_property (GPTL_LINK_LIBRARIES gptl LINK_LIBRARIES)
+  set (CMAKE_REQUIRED_LIBRARIES ${GPTL_LINK_LIBRARIES})
+  check_function_exists (gettimeofday GETTIMEOFDAY)
+  if (NOT GETTIMEOFDAY)
+    target_compile_definitions(gptl
+      PUBLIC NO_GETTIMEOFDAY)
+  endif ()
+  set (SYSTEM_HAS_GETTIMEOFDAY ${GETTIMEOFDAY}
+    CACHE INTERNAL "Whether the gettimeofday function could be found")
+endif ()
diff --git a/src/externals/pio2/src/gptl/COPYING b/src/externals/pio2/src/gptl/COPYING
new file mode 100644
index 00000000000..324ce86b24c
--- /dev/null
+++ b/src/externals/pio2/src/gptl/COPYING
@@ -0,0 +1,17 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the “Software”), to deal
+in the Software for any noncommercial purposes without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to permit
+persons to whom the Software is furnished to do so, subject to the following
+conditions: The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.  Any
+commercial use (including sale) of the software, and derivative development
+towards commercial use, requires written permission of the copyright
+holder. THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
+EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES
+OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/externals/pio2/src/gptl/ChangeLog b/src/externals/pio2/src/gptl/ChangeLog
similarity index 100%
rename from externals/pio2/src/gptl/ChangeLog
rename to src/externals/pio2/src/gptl/ChangeLog
diff --git a/externals/pio2/src/gptl/GPTLget_memusage.c b/src/externals/pio2/src/gptl/GPTLget_memusage.c
similarity index 100%
rename from externals/pio2/src/gptl/GPTLget_memusage.c
rename to src/externals/pio2/src/gptl/GPTLget_memusage.c
diff --git a/externals/pio2/src/gptl/GPTLprint_memusage.c b/src/externals/pio2/src/gptl/GPTLprint_memusage.c
similarity index 100%
rename from externals/pio2/src/gptl/GPTLprint_memusage.c
rename to src/externals/pio2/src/gptl/GPTLprint_memusage.c
diff --git a/externals/pio2/src/gptl/GPTLutil.c b/src/externals/pio2/src/gptl/GPTLutil.c
similarity index 100%
rename from externals/pio2/src/gptl/GPTLutil.c
rename to src/externals/pio2/src/gptl/GPTLutil.c
diff --git a/externals/pio2/src/gptl/README b/src/externals/pio2/src/gptl/README
similarity index 100%
rename from externals/pio2/src/gptl/README
rename to src/externals/pio2/src/gptl/README
diff --git a/externals/pio2/src/gptl/f_wrappers.c b/src/externals/pio2/src/gptl/f_wrappers.c
similarity index 100%
rename from externals/pio2/src/gptl/f_wrappers.c
rename to src/externals/pio2/src/gptl/f_wrappers.c
diff --git a/externals/pio2/src/gptl/gptl.c b/src/externals/pio2/src/gptl/gptl.c
similarity index 99%
rename from externals/pio2/src/gptl/gptl.c
rename to src/externals/pio2/src/gptl/gptl.c
index 6346bf1b999..d06d33f0883 100644
--- a/externals/pio2/src/gptl/gptl.c
+++ b/src/externals/pio2/src/gptl/gptl.c
@@ -195,8 +195,8 @@ static int update_ll_hash (Timer *, const int, const unsigned int);
 static inline int update_ptr (Timer *, const int);
 static int construct_tree (Timer *, Method);
 
-static int cmp (const char **, const char **);
-static int ncmp (const char **, const char **);
+static int cmp (const void *, const void *);
+static int ncmp (const void *, const void *);
 static int get_index ( const char *, const char *);
 
 typedef struct {
@@ -2933,9 +2933,9 @@ int get_index( const char * list,
 ** cmp: returns value from strcmp. for use with qsort
 */
 
-static int cmp(const char **x, const char **y)
+static int cmp(const void *x, const void *y)
 {
-  return strcmp(*x, *y);
+  return strcmp(*(char**)x, *(char**)y);
 }
 
 
@@ -2943,15 +2943,17 @@ static int cmp(const char **x, const char **y)
 ** ncmp: compares values of memory adresses pointed to by a pointer. for use with qsort
 */
 
-static int ncmp( const char **x, const char **y )
+static int ncmp( const void *x, const void *y )
 {
   static const char *thisfunc = "GPTLsetoption";
+  const char **ix = (const char **)x;
+  const char **iy = (const char **)y;
 
-  if( *x > *y )
+  if( *ix > *iy )
     return 1;
-  if( *x < *y )
+  if( *ix < *iy )
     return -1;
-  if( *x == *y )
+  if( *ix == *iy )
     GPTLerror("%s: shared memory address between timers\n", thisfunc);
 }
 
diff --git a/externals/pio2/src/gptl/gptl.h b/src/externals/pio2/src/gptl/gptl.h
similarity index 100%
rename from externals/pio2/src/gptl/gptl.h
rename to src/externals/pio2/src/gptl/gptl.h
diff --git a/externals/pio2/src/gptl/gptl.inc b/src/externals/pio2/src/gptl/gptl.inc
similarity index 100%
rename from externals/pio2/src/gptl/gptl.inc
rename to src/externals/pio2/src/gptl/gptl.inc
diff --git a/externals/pio2/src/gptl/gptl_papi.c b/src/externals/pio2/src/gptl/gptl_papi.c
similarity index 100%
rename from externals/pio2/src/gptl/gptl_papi.c
rename to src/externals/pio2/src/gptl/gptl_papi.c
diff --git a/externals/pio2/src/gptl/perf_mod.F90 b/src/externals/pio2/src/gptl/perf_mod.F90
similarity index 99%
rename from externals/pio2/src/gptl/perf_mod.F90
rename to src/externals/pio2/src/gptl/perf_mod.F90
index 2e38c491c67..e62059de98e 100644
--- a/externals/pio2/src/gptl/perf_mod.F90
+++ b/src/externals/pio2/src/gptl/perf_mod.F90
@@ -30,7 +30,7 @@ module perf_mod
 !-----------------------------------------------------------------------
    implicit none
    private                   ! Make the default access private
-   save
+
 
 !-----------------------------------------------------------------------
 ! Public interfaces ----------------------------------------------------
diff --git a/externals/pio2/src/gptl/perf_utils.F90 b/src/externals/pio2/src/gptl/perf_utils.F90
similarity index 100%
rename from externals/pio2/src/gptl/perf_utils.F90
rename to src/externals/pio2/src/gptl/perf_utils.F90
diff --git a/externals/pio2/src/gptl/private.h b/src/externals/pio2/src/gptl/private.h
similarity index 100%
rename from externals/pio2/src/gptl/private.h
rename to src/externals/pio2/src/gptl/private.h
diff --git a/externals/pio2/src/gptl/threadutil.c b/src/externals/pio2/src/gptl/threadutil.c
similarity index 100%
rename from externals/pio2/src/gptl/threadutil.c
rename to src/externals/pio2/src/gptl/threadutil.c
diff --git a/externals/pio2/tests/CMakeLists.txt b/src/externals/pio2/tests/CMakeLists.txt
similarity index 58%
rename from externals/pio2/tests/CMakeLists.txt
rename to src/externals/pio2/tests/CMakeLists.txt
index 3a210e3c12f..115c732b51c 100644
--- a/externals/pio2/tests/CMakeLists.txt
+++ b/src/externals/pio2/tests/CMakeLists.txt
@@ -5,21 +5,24 @@ project (PIOTests C Fortran)
 #  TESTING TARGET
 #==============================================================================
 
-# Custom "piotests" target (builds the test executables)
-add_custom_target (tests)
+# # Custom "piotests" target (builds the test executables)
+# add_custom_target (tests)
 
-# Custom "check" target that depends upon "tests"
-add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND})
-add_dependencies (check tests)
+# # Custom "check" target that depends upon "tests"
+# add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND})
+# add_dependencies (check tests)
 
 #==============================================================================
 #  INCLUDE SOURCE DIRECTORIES
 #==============================================================================
+add_subdirectory (cunit)
 
-add_subdirectory (unit)
-add_subdirectory (general)
-if (PIO_ENABLE_TIMING)
+if (PIO_ENABLE_FORTRAN)
+  add_subdirectory (unit)
+  add_subdirectory (general)
+  if (PIO_ENABLE_TIMING)
     add_subdirectory (performance)
-else ()
+  else ()
     message (STATUS "Cannot build performance test without gptl timing library")
-endif ()
+  endif ()
+endif()
diff --git a/src/externals/pio2/tests/cunit/CMakeLists.txt b/src/externals/pio2/tests/cunit/CMakeLists.txt
new file mode 100644
index 00000000000..418340b7916
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/CMakeLists.txt
@@ -0,0 +1,216 @@
+include (LibMPI)
+
+include_directories("${CMAKE_SOURCE_DIR}/tests/cunit")
+
+# Compiler-specific compiler options
+if ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "PGI")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c99")
+endif()
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
+
+#==============================================================================
+#  PREPARE FOR TESTING
+#==============================================================================
+
+# Don't run these tests if we are using MPI SERIAL.
+if (NOT PIO_USE_MPISERIAL)
+  add_executable (test_intercomm2 EXCLUDE_FROM_ALL test_intercomm2.c test_common.c)
+  target_link_libraries (test_intercomm2 pioc)
+  add_dependencies (tests test_intercomm2)
+  add_executable (test_async_simple EXCLUDE_FROM_ALL test_async_simple.c test_common.c)
+  target_link_libraries (test_async_simple pioc)
+  add_dependencies (tests test_async_simple)
+  add_executable (test_async_3proc EXCLUDE_FROM_ALL test_async_3proc.c test_common.c)
+  target_link_libraries (test_async_3proc pioc)
+  add_dependencies (tests test_async_3proc)
+  add_executable (test_async_4proc EXCLUDE_FROM_ALL test_async_4proc.c test_common.c)
+  target_link_libraries (test_async_4proc pioc)
+  add_dependencies (tests test_async_4proc)
+  add_executable (test_iosystem2_simple EXCLUDE_FROM_ALL test_iosystem2_simple.c test_common.c)
+  target_link_libraries (test_iosystem2_simple pioc)
+  add_dependencies (tests test_iosystem2_simple)
+  add_executable (test_iosystem2_simple2 EXCLUDE_FROM_ALL test_iosystem2_simple2.c test_common.c)
+  target_link_libraries (test_iosystem2_simple2 pioc)
+  add_dependencies (tests test_iosystem2_simple2)
+  add_executable (test_iosystem2 EXCLUDE_FROM_ALL test_iosystem2.c test_common.c)
+  target_link_libraries (test_iosystem2 pioc)
+  add_dependencies (tests test_iosystem2)
+  add_executable (test_iosystem3_simple EXCLUDE_FROM_ALL test_iosystem3_simple.c test_common.c)
+  target_link_libraries (test_iosystem3_simple pioc)
+  add_dependencies (tests test_iosystem3_simple)
+  add_executable (test_iosystem3_simple2 EXCLUDE_FROM_ALL test_iosystem3_simple2.c test_common.c)
+  target_link_libraries (test_iosystem3_simple2 pioc)
+  add_dependencies (tests test_iosystem3_simple2)
+  add_executable (test_iosystem3 EXCLUDE_FROM_ALL test_iosystem3.c test_common.c)
+  target_link_libraries (test_iosystem3 pioc)
+  add_dependencies (tests test_iosystem3)
+  add_executable (test_pioc EXCLUDE_FROM_ALL test_pioc.c test_common.c test_shared.c)
+  target_link_libraries (test_pioc pioc)
+  add_executable (test_pioc_unlim EXCLUDE_FROM_ALL test_pioc_unlim.c test_common.c test_shared.c)
+  target_link_libraries (test_pioc_unlim pioc)
+  add_executable (test_pioc_putget EXCLUDE_FROM_ALL test_pioc_putget.c test_common.c test_shared.c)
+  target_link_libraries (test_pioc_putget pioc)
+  add_executable (test_pioc_fill EXCLUDE_FROM_ALL test_pioc_fill.c test_common.c test_shared.c)
+  target_link_libraries (test_pioc_fill pioc)
+  add_executable (test_darray EXCLUDE_FROM_ALL test_darray.c test_common.c)
+  target_link_libraries (test_darray pioc)
+  add_executable (test_darray_multi EXCLUDE_FROM_ALL test_darray_multi.c test_common.c)
+  target_link_libraries (test_darray_multi pioc)
+  add_executable (test_darray_multivar EXCLUDE_FROM_ALL test_darray_multivar.c test_common.c)
+  target_link_libraries (test_darray_multivar pioc)
+  add_executable (test_darray_multivar2 EXCLUDE_FROM_ALL test_darray_multivar2.c test_common.c)
+  target_link_libraries (test_darray_multivar2 pioc)
+  add_executable (test_darray_1d EXCLUDE_FROM_ALL test_darray_1d.c test_common.c)
+  target_link_libraries (test_darray_1d pioc)  
+  add_executable (test_darray_3d EXCLUDE_FROM_ALL test_darray_3d.c test_common.c)
+  target_link_libraries (test_darray_3d pioc)  
+  add_executable (test_decomp_uneven EXCLUDE_FROM_ALL test_decomp_uneven.c test_common.c)
+  target_link_libraries (test_decomp_uneven pioc)  
+  add_executable (test_decomps EXCLUDE_FROM_ALL test_decomps.c test_common.c)
+  target_link_libraries (test_decomps pioc)
+  add_executable (test_rearr EXCLUDE_FROM_ALL test_rearr.c test_common.c)
+  target_link_libraries (test_rearr pioc)
+endif ()
+add_executable (test_spmd EXCLUDE_FROM_ALL test_spmd.c test_common.c)
+target_link_libraries (test_spmd pioc)
+add_dependencies (tests test_spmd)
+add_dependencies (tests test_rearr)
+add_dependencies (tests test_pioc)
+add_dependencies (tests test_pioc_unlim)
+add_dependencies (tests test_pioc_putget)
+add_dependencies (tests test_pioc_fill)
+add_dependencies (tests test_darray)
+add_dependencies (tests test_darray_multi)
+add_dependencies (tests test_darray_multivar)
+add_dependencies (tests test_darray_multivar2)
+add_dependencies (tests test_darray_1d)
+add_dependencies (tests test_darray_3d)
+add_dependencies (tests test_decomp_uneven)
+add_dependencies (tests test_decomps)
+
+# Test Timeout in seconds.
+if (PIO_VALGRIND_CHECK)
+  set (DEFAULT_TEST_TIMEOUT 240)
+else ()
+  set (DEFAULT_TEST_TIMEOUT 120)
+endif ()
+
+# All tests need a certain number of tasks, but they should be able to
+# run successfully with more than they need. Test this by providing an
+# extra processor for each C test.
+set (AT_LEAST_TWO_TASKS 3)
+set (AT_LEAST_THREE_TASKS 4)
+set (AT_LEAST_FOUR_TASKS 5)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME test_pioc
+    COMMAND test_pioc)
+else ()
+  add_mpi_test(test_spmd
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_spmd
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_rearr
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_rearr
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_intercomm2
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_intercomm2
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_async_simple
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_async_simple
+    NUMPROCS ${AT_LEAST_TWO_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_async_3proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_async_3proc
+    NUMPROCS ${AT_LEAST_THREE_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_async_4proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_async_4proc
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem2_simple
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem2_simple
+    NUMPROCS ${AT_LEAST_TWO_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem2_simple2
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem2_simple2
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem2
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem2
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem3_simple
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem3_simple
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem3_simple2
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem3_simple2
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_iosystem3
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_iosystem3
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_pioc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_pioc
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_pioc_unlim
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_pioc_unlim
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_pioc_putget
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_pioc_putget
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_pioc_fill
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_pioc_fill
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray_multi
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multi
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray_multivar
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multivar
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray_multivar2
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_multivar2
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray_1d
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_1d
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_darray_3d
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_darray_3d
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_decomp_uneven
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_decomp_uneven
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(test_decomps
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/test_decomps
+    NUMPROCS ${AT_LEAST_FOUR_TASKS}
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
diff --git a/src/externals/pio2/tests/cunit/pio_tests.h b/src/externals/pio2/tests/cunit/pio_tests.h
new file mode 100644
index 00000000000..5a601ea6df1
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/pio_tests.h
@@ -0,0 +1,103 @@
+/**
+ * @file
+ * Include file for tests for the Parallel IO library.
+ * @author Ed Hartnett
+ * @date 9/13/2016
+ */
+
+#ifndef _PIO_TESTS_H
+#define _PIO_TESTS_H
+
+#include  /* Include this for the sleep function. */
+#include 
+
+/* Timing header may need to be included. */
+#ifdef TIMING
+#include 
+#endif
+
+/** The number of possible output netCDF output flavors available to
+ * the ParallelIO library. */
+#define NUM_FLAVORS 4
+
+/** Number of netCDF types. */
+#define NUM_NETCDF_TYPES 12
+
+/* Number of NetCDF classic types. */
+#define NUM_CLASSIC_TYPES 6
+
+/* Number of NetCDF-4 types. */
+#define NUM_NETCDF4_TYPES 12
+
+/* Number of PIO rearrangers. */
+#define NUM_REARRANGERS 2
+
+/* Number of sample files constructed for these tests. */
+#define NUM_SAMPLES 3
+
+/** Error code for when things go wrong. */
+#define ERR_CHECK 1109
+#define ERR_INIT 1110
+#define ERR_AWFUL 1111
+#define ERR_WRONG 1112
+#define ERR_GPTL 1113
+#define ERR_MPI 1114
+
+/** The meaning of life, the universe, and everything. */
+#define TEST_VAL_42 42
+
+/** Handle MPI errors. This should only be used with MPI library
+ * function calls. */
+#define MPIERR(e) do {                                                  \
+        MPI_Error_string(e, err_buffer, &resultlen);                    \
+        fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
+        MPI_Finalize();                                                 \
+        return ERR_AWFUL;                                               \
+    } while (0)
+
+/** Handle non-MPI errors by finalizing the MPI library and exiting
+ * with an exit code. */
+#define ERR(e) do {                                                     \
+        fprintf(stderr, "%d Error %d in %s, line %d\n", my_rank, e, __FILE__, __LINE__); \
+        MPI_Finalize();                                                 \
+        return e;                                                       \
+    } while (0)
+
+/** Global err buffer for MPI. When there is an MPI error, this buffer
+ * is used to store the error message that is associated with the MPI
+ * error. */
+char err_buffer[MPI_MAX_ERROR_STRING];
+
+/** This is the length of the most recent MPI error message, stored
+ * int the global error string. */
+int resultlen;
+
+/* Function prototypes. */
+int pio_test_init(int argc, char **argv, int *my_rank, int *ntasks, int target_ntasks, MPI_Comm *test_comm);
+int pio_test_init2(int argc, char **argv, int *my_rank, int *ntasks, int min_ntasks,
+                   int max_ntasks, int log_level, MPI_Comm *test_comm);
+int create_nc_sample(int sample, int iosysid, int format, char *filename, int my_rank, int *ncid);
+int check_nc_sample(int sample, int iosysid, int format, char *filename, int my_rank, int *ncid);
+int create_nc_sample_0(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int check_nc_sample_0(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int create_nc_sample_1(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int check_nc_sample_1(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int create_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *ncid);
+int get_iotypes(int *num_flavors, int *flavors);
+int get_iotype_name(int iotype, char *name);
+int pio_test_finalize(MPI_Comm *test_comm);
+int test_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm,
+                int component_count, int num_io_procs, int target_ntasks, char *test_name);
+int test_no_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm, int target_ntasks,
+                   int x_dim_len, int y_dim_len);
+int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm,
+             int async);
+int run_test_main(int argc, char **argv, int min_ntasks, int max_ntasks,
+                  int log_level, char *test_name, int *dim_len, int component_count,
+                  int num_io_procs);
+
+/* Create a 2D decomposition used in some tests. */
+int create_decomposition_2d(int ntasks, int my_rank, int iosysid, int *dim_len_2d, int *ioid,
+                            int pio_type);
+#endif /* _PIO_TESTS_H */
diff --git a/src/externals/pio2/tests/cunit/test_async_3proc.c b/src/externals/pio2/tests/cunit/test_async_3proc.c
new file mode 100644
index 00000000000..4a2f8435fe2
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_async_3proc.c
@@ -0,0 +1,128 @@
+/*
+ * Tests for PIOc_Intercomm. This tests basic asynch I/O capability.
+ *
+ * This very simple test runs on 32 ranks. Eight are used for IO, the
+ * other 24 for computation. The netCDF sample files are created and
+ * checked.
+ *
+ * To run with valgrind, use this command:
+ * 
    mpiexec -n 4 valgrind -v --leak-check=full --suppressions=../../../tests/unit/valsupp_test.supp
+ * --error-exitcode=99 --track-origins=yes ./test_async_8io_24comp
    
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 3
+
+/* The name of this test. */
+#define TEST_NAME "test_async_3proc"
+
+/* Number of different combonations of IO and computation processor
+ * numbers we will try in this test. */
+#define NUM_COMBOS 2
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* Run async tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm; /* comm for test */
+
+    /* Num procs for IO and computation. */
+    int num_procs[NUM_COMBOS][COMPONENT_COUNT] = {{1}, {2}};
+
+    /* Number of processors that will do IO. */
+    int num_io_procs[NUM_COMBOS] = {2, 1};
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+			     &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        for (int combo = 0; combo < NUM_COMBOS; combo++)
+        {
+            /* Is the current process a computation task? */
+            int comp_task = my_rank < num_io_procs[combo] ? 0 : 1;
+
+            /* Initialize the IO system. */
+            if ((ret = PIOc_init_async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT,
+                                       num_procs[combo], NULL, NULL, NULL, PIO_REARR_BOX, iosysid)))
+                ERR(ERR_INIT);
+
+            for (int c = 0; c < COMPONENT_COUNT; c++)
+                printf("%d iosysid[%d] = %d\n", my_rank, c, iosysid[c]);
+
+            /* All the netCDF calls are only executed on the computation
+             * tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
+             * and when the do, they should go straight to finalize. */
+            if (comp_task)
+            {
+                for (int flv = 0; flv < num_flavors; flv++)
+                {
+                    char filename[NC_MAX_NAME + 1]; /* Test filename. */
+                    int my_comp_idx = 0; /* Index in iosysid array. */
+
+                    for (int sample = 0; sample < NUM_SAMPLES; sample++)
+                    {
+                        char iotype_name[NC_MAX_NAME + 1];
+
+                        /* Create a filename. */
+                        if ((ret = get_iotype_name(flavor[flv], iotype_name)))
+                            return ret;
+                        sprintf(filename, "%s_%s_%d_%d.nc", TEST_NAME, iotype_name, sample, my_comp_idx);
+
+                        /* Create sample file. */
+                        printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename);
+                        if ((ret = create_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                            ERR(ret);
+
+                        /* Check the file for correctness. */
+                        if ((ret = check_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                            ERR(ret);
+                    }
+                } /* next netcdf flavor */
+
+                /* Finalize the IO system. Only call this from the computation tasks. */
+                printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME);
+                for (int c = 0; c < COMPONENT_COUNT; c++)
+                {
+                    if ((ret = PIOc_finalize(iosysid[c])))
+                        ERR(ret);
+                    printf("%d %s PIOc_finalize completed for iosysid = %d\n", my_rank, TEST_NAME,
+                           iosysid[c]);
+                }
+            } /* endif comp_task */
+
+            /* Wait for everyone to catch up. */
+            printf("%d %s waiting for all processes!\n", my_rank, TEST_NAME);
+            MPI_Barrier(test_comm);
+        } /* next combo */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_async_4proc.c b/src/externals/pio2/tests/cunit/test_async_4proc.c
new file mode 100644
index 00000000000..db0197ab9be
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_async_4proc.c
@@ -0,0 +1,121 @@
+/*
+ * Tests for PIOc_Intercomm. This tests basic asynch I/O capability.
+ *
+ * This very simple test runs on 4 ranks.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_async_4proc"
+
+/* Number of different combonations of IO and computation processor
+ * numbers we will try in this test. */
+#define NUM_COMBOS 3
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* Run async tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Num procs for computation. */
+    int num_procs2[NUM_COMBOS][COMPONENT_COUNT] = {{1}, {2}, {3}};
+
+    /* Number of processors that will do IO. */
+    int num_io_procs[NUM_COMBOS] = {3, 2, 1};
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, &test_comm)))
+        ERR(ERR_INIT);
+    
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        for (int combo = 0; combo < NUM_COMBOS; combo++)
+        {
+            /* Is the current process a computation task? */
+            int comp_task = my_rank < num_io_procs[combo] ? 0 : 1;
+
+            /* Initialize the IO system. */
+            if ((ret = PIOc_init_async(test_comm, num_io_procs[combo], NULL, COMPONENT_COUNT,
+                                       num_procs2[combo], NULL, NULL, NULL, PIO_REARR_BOX, iosysid)))
+                ERR(ERR_INIT);
+
+            for (int c = 0; c < COMPONENT_COUNT; c++)
+                printf("%d iosysid[%d] = %d\n", my_rank, c, iosysid[c]);
+
+            /* All the netCDF calls are only executed on the computation
+             * tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
+             * and when the do, they should go straight to finalize. */
+            if (comp_task)
+            {
+                for (int flv = 0; flv < num_flavors; flv++)
+                {
+                    char filename[NC_MAX_NAME + 1]; /* Test filename. */
+                    int my_comp_idx = 0; /* Index in iosysid array. */
+
+                    for (int sample = 0; sample < NUM_SAMPLES; sample++)
+                    {
+                        char iotype_name[NC_MAX_NAME + 1];
+
+                        /* Create a filename. */
+                        if ((ret = get_iotype_name(flavor[flv], iotype_name)))
+                            return ret;
+                        sprintf(filename, "%s_%s_%d_%d.nc", TEST_NAME, iotype_name, sample, my_comp_idx);
+
+                        /* Create sample file. */
+                        printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename);
+                        if ((ret = create_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                            ERR(ret);
+
+                        /* Check the file for correctness. */
+                        if ((ret = check_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                            ERR(ret);
+                    }
+                } /* next netcdf flavor */
+
+                /* Finalize the IO system. Only call this from the computation tasks. */
+                printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME);
+                for (int c = 0; c < COMPONENT_COUNT; c++)
+                {
+                    if ((ret = PIOc_finalize(iosysid[c])))
+                        ERR(ret);
+                    printf("%d %s PIOc_finalize completed for iosysid = %d\n", my_rank, TEST_NAME,
+                           iosysid[c]);
+                }
+            } /* endif comp_task */
+
+            /* Wait for everyone to catch up. */
+            printf("%d %s waiting for all processes!\n", my_rank, TEST_NAME);
+            MPI_Barrier(test_comm);
+        } /* next combo */
+    }/* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_async_simple.c b/src/externals/pio2/tests/cunit/test_async_simple.c
new file mode 100644
index 00000000000..4b4836bcb02
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_async_simple.c
@@ -0,0 +1,130 @@
+/*
+ * Tests for PIOc_Intercomm. This tests basic asynch I/O capability.
+ *
+ * This very simple test runs on two ranks. One is used for
+ * computation, the other for IO. A sample netCDF file is created and
+ * checked.
+ *
+ * To run with valgrind, use this command:
+ * 
    mpiexec -n 4 valgrind -v --leak-check=full --suppressions=../../../tests/unit/valsupp_test.supp
+ * --error-exitcode=99 --track-origins=yes ./test_async_simple
    
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 2
+
+/* The name of this test. */
+#define TEST_NAME "test_async_simple"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* Run simple async test. */
+int main(int argc, char **argv)
+{
+#define NUM_IO_PROCS 1
+#define NUM_COMP_PROCS 1
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid[COMPONENT_COUNT]; /* The ID for the parallel I/O system. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    int num_procs[COMPONENT_COUNT] = {1}; /* Num procs for IO and computation. */
+    int io_proc_list[NUM_IO_PROCS] = {0};
+    int comp_proc_list[NUM_COMP_PROCS] = {1};
+    int *proc_list[COMPONENT_COUNT] = {comp_proc_list};
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+			     &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Only do something on TARGET_NTASKS tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Is the current process a computation task? */
+        int comp_task = my_rank < NUM_IO_PROCS ? 0 : 1;
+
+        /* Check for invalid values. */
+        if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT,
+                            num_procs, NULL, NULL, NULL, PIO_REARR_BOX, NULL) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT,
+                            num_procs, NULL, NULL, NULL, TEST_VAL_42, iosysid) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, -1,
+                            num_procs, NULL, NULL, NULL, PIO_REARR_BOX, iosysid) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT,
+                            NULL, NULL, NULL, NULL, PIO_REARR_BOX, iosysid) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+
+        /* Initialize the IO system. */
+        if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, io_proc_list, COMPONENT_COUNT,
+                                   num_procs, (int **)proc_list, NULL, NULL, PIO_REARR_BOX, iosysid)))
+            ERR(ERR_INIT);
+
+        /* All the netCDF calls are only executed on the computation
+         * tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
+         * and when the do, they should go straight to finalize. */
+        if (comp_task)
+        {
+            for (int flv = 0; flv < num_flavors; flv++)
+            {
+                int my_comp_idx = my_rank - 1; /* Index in iosysid array. */
+
+                for (int sample = 0; sample < NUM_SAMPLES; sample++)
+                {
+                    char filename[NC_MAX_NAME + 1]; /* Test filename. */
+                    char iotype_name[NC_MAX_NAME + 1];
+
+                    /* Create a filename. */
+                    if ((ret = get_iotype_name(flavor[flv], iotype_name)))
+                        return ret;
+                    sprintf(filename, "%s_%s_%d_%d.nc", TEST_NAME, iotype_name, sample, my_comp_idx);
+
+                    /* Create sample file. */
+                    printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename);
+                    if ((ret = create_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                        ERR(ret);
+
+                    /* Check the file for correctness. */
+                    if ((ret = check_nc_sample(sample, iosysid[my_comp_idx], flavor[flv], filename, my_rank, NULL)))
+                        ERR(ret);
+                }
+            } /* next netcdf flavor */
+
+            /* Finalize the IO system. Only call this from the computation tasks. */
+            printf("%d %s Freeing PIO resources\n", my_rank, TEST_NAME);
+            for (int c = 0; c < COMPONENT_COUNT; c++)
+            {
+                if ((ret = PIOc_finalize(iosysid[c])))
+                    ERR(ret);
+                printf("%d %s PIOc_finalize completed for iosysid = %d\n", my_rank, TEST_NAME,
+                       iosysid[c]);
+            }
+        } /* endif comp_task */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_common.c b/src/externals/pio2/tests/cunit/test_common.c
new file mode 100644
index 00000000000..e49e093e5f0
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_common.c
@@ -0,0 +1,926 @@
+/*
+ * Common test code for PIO C tests.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* The number of dimensions in the test data. */
+#define NDIM_S1 1
+
+/* The length of our test data. */
+#define DIM_LEN_S1 4
+
+/* The name of the dimension in the netCDF output file. */
+#define FIRST_DIM_NAME_S1 "jojo"
+#define DIM_NAME_S1 "dim_sample_s1"
+
+/* The name of the variable in the netCDF output file. */
+#define FIRST_VAR_NAME_S1 "bill"
+#define VAR_NAME_S1 "var_sample_s1"
+
+/* The number of dimensions in the sample 2 test data. */
+#define NDIM_S2 1
+
+/* The length of our sample 2 test data. */
+#define DIM_LEN_S2 4
+
+/* The name of the dimension in the sample 2 output file. */
+#define FIRST_DIM_NAME_S2 "jojo"
+#define DIM_NAME_S2 "dim_sample_s2"
+
+/* The name of the variable in the sample 2 output file. */
+#define FIRST_VAR_NAME_S2 "bill"
+#define VAR_NAME_S2 "var_sample_s2"
+
+/* The name of the global attribute in the sample 2 output file. */
+#define FIRST_ATT_NAME_S2 "willy_gatt_sample s2"
+#define ATT_NAME_S2 "gatt_sample s2"
+#define SHORT_ATT_NAME_S2 "short_gatt_sample s2"
+#define FLOAT_ATT_NAME_S2 "float_gatt_sample s2"
+#define DOUBLE_ATT_NAME_S2 "double_gatt_sample s2"
+
+/* The value of the global attribute in the sample 2 output file. */
+#define ATT_VALUE_S2 42
+
+/* For when we need 2D. */
+#define NDIM2 2
+
+/* How many flavors of netCDF are available? */
+int
+get_iotypes(int *num_flavors, int *flavors)
+{
+    int num = 0;
+    int fmtidx = 0;
+    int format[NUM_FLAVORS];
+
+#ifdef _PNETCDF
+    num++;
+    format[fmtidx++] = PIO_IOTYPE_PNETCDF;
+#endif
+#ifdef _NETCDF
+    num++;
+    format[fmtidx++] = PIO_IOTYPE_NETCDF;
+#endif
+#ifdef _NETCDF4
+    num += 2;
+    format[fmtidx++] = PIO_IOTYPE_NETCDF4C;
+    format[fmtidx] = PIO_IOTYPE_NETCDF4P;
+#endif
+
+    /* Pass results back to caller. */
+    if (num_flavors)
+        *num_flavors = num;
+    if (flavors)
+        for (int f = 0; f < num; f++)
+            flavors[f] = format[f];
+
+    return PIO_NOERR;
+}
+
+/* Name of each flavor.
+ *
+ * @param iotype the IO type
+ * @param name pointer that will get name of IO type. Must have enough
+ * memory allocated (NC_MAX_NAME + 1 works.)
+ * @returns 0 for success, error code otherwise.
+ * @internal
+ */
+int get_iotype_name(int iotype, char *name)
+{
+    char flavor_name[NUM_FLAVORS][NC_MAX_NAME + 1] = {"pnetcdf", "classic",
+                                                      "serial4", "parallel4"};
+
+    /* Check inputs. */
+    if (!name || iotype < PIO_IOTYPE_PNETCDF || iotype > PIO_IOTYPE_NETCDF4P)
+        return PIO_EINVAL;
+
+    /* Return name of iotype. They are numbered 1-4 in pio.h. */
+    strcpy(name, flavor_name[iotype - 1]);
+
+    return PIO_NOERR;
+}
+
+/* Initalize the test system.
+ *
+ * @param argc argument count from main().
+ * @param argv argument array from main().
+ * @param my_rank pointer that gets this tasks rank.
+ * @param ntasks pointer that gets the number of tasks in WORLD
+ * communicator.
+ * @param target_ntasks the number of tasks this test needs to run.
+ * @param comm a pointer to an MPI communicator that will be created
+ * for this test and contain target_ntasks tasks from WORLD.
+ * @returns 0 for success, error code otherwise.
+*/
+int pio_test_init(int argc, char **argv, int *my_rank, int *ntasks,
+		  int target_ntasks, MPI_Comm *comm)
+{
+    return pio_test_init2(argc,argv, my_rank, ntasks, target_ntasks,
+                          target_ntasks, 3, comm);
+}
+
+/* Initalize the test system.
+ *
+ * @param argc argument count from main().
+ * @param argv argument array from main().
+ * @param my_rank pointer that gets this tasks rank.
+ * @param ntasks pointer that gets the number of tasks in WORLD
+ * communicator.
+ * @param target_ntasks the number of tasks this test needs to run.
+ * @param log_level PIOc_set_log_level() will be called with this value.
+ * @param comm a pointer to an MPI communicator that will be created
+ * for this test and contain target_ntasks tasks from WORLD.
+ * @returns 0 for success, error code otherwise.
+*/
+int pio_test_init2(int argc, char **argv, int *my_rank, int *ntasks,
+                   int min_ntasks, int max_ntasks, int log_level, MPI_Comm *comm)
+{
+    int ret; /* Return value. */
+
+#ifdef TIMING
+    /* Initialize the GPTL timing library. */
+    if ((ret = GPTLinitialize()))
+        return ERR_GPTL;
+#endif
+
+    /* Initialize MPI. */
+    if ((ret = MPI_Init(&argc, &argv)))
+        MPIERR(ret);
+
+    /* Learn my rank and the total number of processors. */
+    if ((ret = MPI_Comm_rank(MPI_COMM_WORLD, my_rank)))
+        MPIERR(ret);
+    if ((ret = MPI_Comm_size(MPI_COMM_WORLD, ntasks)))
+        MPIERR(ret);
+    printf("%d has %d tasks\n", *my_rank, *ntasks);
+
+    /* Check that a valid number of processors was specified. */
+    if (*ntasks < min_ntasks)
+    {
+        fprintf(stderr, "ERROR: Number of processors must be at least %d for this test!\n",
+                min_ntasks);
+        return ERR_AWFUL;
+    }
+    else if (*ntasks > max_ntasks)
+    {
+	/* If more tasks are available than we need for this test,
+	 * create a communicator with exactly the number of tasks we
+	 * need. */
+        int color, key;
+        if (*my_rank < max_ntasks)
+        {
+            color = 0;
+	    key = *my_rank;
+        }
+        else
+        {
+            color = 1;
+            key = *my_rank - max_ntasks;
+        }
+	printf("%d splitting comm for test color = %d key = %d\n", *my_rank, color, key);
+        if ((ret = MPI_Comm_split(MPI_COMM_WORLD, color, key, comm)))
+            MPIERR(ret);
+    }
+    else
+    {
+	printf("%d using whole comm for test\n", *my_rank);
+        if ((ret = MPI_Comm_dup(MPI_COMM_WORLD, comm)))
+            MPIERR(ret);
+    }
+
+    /* Turn on logging. */
+    printf("%d setting log level to %d\n", *my_rank, log_level);
+    if ((ret = PIOc_set_log_level(log_level)))
+        return ret;
+    printf("%d done setting log level\n", *my_rank);
+
+    /* Change error handling so we can test inval parameters. */
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/* Finalize a PIO C test.
+*
+* @param test_comm pointer to the test communicator.
+* @returns 0 for success, error code otherwise.
+*/
+int pio_test_finalize(MPI_Comm *test_comm)
+{
+    int ret = PIO_NOERR; /* Return value. */
+
+    /* Wait for all processes to arrive here. */
+    if (MPI_Barrier(*test_comm))
+        return ERR_MPI;
+
+    /* Free communicator. */
+    if (MPI_Comm_free(test_comm))
+        return ERR_MPI;
+
+    /* Finalize MPI. */
+    MPI_Finalize();
+
+#ifdef TIMING
+    /* Finalize the GPTL timing library. */
+    if ((ret = GPTLfinalize()))
+        return ret;
+#endif
+
+    return ret;
+}
+
+/* Test the inq_format function. */
+int
+test_inq_format(int ncid, int format)
+{
+    int myformat;
+    int ret;
+
+    /* Get the format of an open file. */
+    if ((ret = PIOc_inq_format(ncid, &myformat)))
+        return ret;
+
+    /* Check the result. */
+    if ((format == PIO_IOTYPE_PNETCDF || format == PIO_IOTYPE_NETCDF) && myformat != 1)
+        return ERR_WRONG;
+    else if ((format == PIO_IOTYPE_NETCDF4C || format == PIO_IOTYPE_NETCDF4P) &&
+             myformat != 3)
+        return ERR_WRONG;
+
+    return PIO_NOERR;
+}
+
+/* Test the inq_type function for atomic types. */
+int
+test_inq_type(int ncid, int format)
+{
+#define NUM_TYPES 11
+    char type_name[NC_MAX_NAME + 1];
+    PIO_Offset type_size;
+    nc_type xtype[NUM_TYPES] = {NC_CHAR, NC_BYTE, NC_SHORT, NC_INT, NC_FLOAT, NC_DOUBLE,
+                                NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, NC_UINT64};
+    int type_len[NUM_TYPES] = {1, 1, 2, 4, 4, 8, 1, 2, 4, 8, 8};
+    int max_type = format == PIO_IOTYPE_NETCDF ? NC_DOUBLE : NC_UINT64;
+    int ret;
+
+    /* Check each type size. */
+    for (int i = 0; i < max_type; i++)
+    {
+        if ((ret = PIOc_inq_type(ncid, xtype[i], type_name, &type_size)))
+            return ret;
+        if (type_size != type_len[i])
+            return ERR_AWFUL;
+    }
+
+    return PIO_NOERR;
+}
+
+/* This creates a netCDF sample file in the specified format. */
+int
+create_nc_sample(int sample, int iosysid, int format, char *filename, int my_rank, int *ncid)
+{
+    switch(sample)
+    {
+    case 0:
+        return create_nc_sample_0(iosysid, format, filename, my_rank, ncid);
+        break;
+    case 1:
+        return create_nc_sample_1(iosysid, format, filename, my_rank, ncid);
+        break;
+    case 2:
+        return create_nc_sample_2(iosysid, format, filename, my_rank, ncid);
+        break;
+    }
+    return PIO_EINVAL;
+}
+
+/* This checks a netCDF sample file in the specified format. */
+int
+check_nc_sample(int sample, int iosysid, int format, char *filename, int my_rank, int *ncid)
+{
+    switch(sample)
+    {
+    case 0:
+        return check_nc_sample_0(iosysid, format, filename, my_rank, ncid);
+        break;
+    case 1:
+        return check_nc_sample_1(iosysid, format, filename, my_rank, ncid);
+        break;
+    case 2:
+        return check_nc_sample_2(iosysid, format, filename, my_rank, ncid);
+        break;
+    }
+    return PIO_EINVAL;
+}
+
+/* This creates an empty netCDF file in the specified format. */
+int
+create_nc_sample_0(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    int ncid;
+    int ret;
+
+    /* Create the file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &format, filename, NC_CLOBBER)))
+        return ret;
+    printf("%d file created ncid = %d\n", my_rank, ncid);
+
+    /* End define mode. */
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+
+    /* Test inq_format. */
+    if ((ret = test_inq_format(ncid, format)))
+        return ret;
+
+    /* Test inq_type. */
+    if ((ret = test_inq_type(ncid, format)))
+        return ret;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ret;
+        printf("%d closed file ncid = %d\n", my_rank, ncid);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Check sample file 1 for correctness. */
+int
+check_nc_sample_0(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    int ncid;
+    int ndims, nvars, ngatts, unlimdimid;
+    int ndims2, nvars2, ngatts2, unlimdimid2;
+    int ret;
+
+    /* Re-open the file to check it. */
+    printf("%d opening file %s format %d\n", my_rank, filename, format);
+    if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename,
+                             NC_NOWRITE)))
+        return ret;
+
+    /* Find the number of dimensions, variables, and global attributes.*/
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ret;
+    if (ndims != 0 || nvars != 0 || ngatts != 0 || unlimdimid != -1)
+        return ERR_WRONG;
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_ndims(ncid, &ndims2)))
+        return ret;
+    if (ndims2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_nvars(ncid, &nvars2)))
+        return ret;
+    if (nvars2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_natts(ncid, &ngatts2)))
+        return ret;
+    if (ngatts2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2)))
+        return ret;
+    if (unlimdimid != -1)
+        return ERR_WRONG;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file (again) ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* This creates a netCDF file in the specified format, with some
+ * sample values. */
+int
+create_nc_sample_1(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    /* The ncid of the netCDF file. */
+    int ncid;
+
+    /* The ID of the netCDF varable. */
+    int varid;
+
+    /* The ID of the netCDF dimension. */
+    int dimid;
+
+    /* Return code. */
+    int ret;
+
+    /* Start and count arrays for netCDF. */
+    PIO_Offset start[NDIM_S1], count[NDIM_S1] = {0};
+
+    /* The sample data. */
+    int data[DIM_LEN_S1];
+
+    /* Create the file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &format, filename, NC_CLOBBER)))
+        return ret;
+    printf("%d file created ncid = %d\n", my_rank, ncid);
+
+    /* /\* End define mode, then re-enter it. *\/ */
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    printf("%d calling redef\n", my_rank);
+    if ((ret = PIOc_redef(ncid)))
+        return ret;
+
+    /* Define a dimension. */
+    printf("%d defining dimension %s\n", my_rank, DIM_NAME_S1);
+    if ((ret = PIOc_def_dim(ncid, DIM_NAME_S1, DIM_LEN_S1, &dimid)))
+        return ret;
+
+    /* Define a 1-D variable. */
+    printf("%d defining variable %s\n", my_rank, VAR_NAME_S1);
+    if ((ret = PIOc_def_var(ncid, VAR_NAME_S1, NC_INT, NDIM_S1, &dimid, &varid)))
+        return ret;
+
+    /* End define mode. */
+    printf("%d ending define mode ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    printf("%d define mode ended ncid = %d\n", my_rank, ncid);
+
+    /* Write some data. For the PIOc_put/get functions, all data must
+     * be on compmaster before the function is called. Only
+     * compmaster's arguments are passed to the async msg handler. All
+     * other computation tasks are ignored. */
+    for (int i = 0; i < DIM_LEN_S1; i++)
+        data[i] = i;
+    printf("%d writing data\n", my_rank);
+    start[0] = 0;
+    count[0] = DIM_LEN_S1;
+    if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data)))
+        return ret;
+
+    /* Test inq_format. */
+    if ((ret = test_inq_format(ncid, format)))
+        return ret;
+
+    /* Test inq_type. */
+    if ((ret = test_inq_type(ncid, format)))
+        return ret;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ret;
+        printf("%d closed file ncid = %d\n", my_rank, ncid);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Check sample file 1 for correctness. */
+int
+check_nc_sample_1(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    int ncid;
+    int ret;
+    int ndims, nvars, ngatts, unlimdimid;
+    int ndims2, nvars2, ngatts2, unlimdimid2;
+    char dimname[NC_MAX_NAME + 1];
+    PIO_Offset dimlen;
+    char varname[NC_MAX_NAME + 1];
+    nc_type vartype;
+    int varndims, vardimids, varnatts;
+
+    /* Re-open the file to check it. */
+    printf("%d opening file %s format %d\n", my_rank, filename, format);
+    if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename,
+                             NC_NOWRITE)))
+        return ret;
+
+    /* Try to read the data. */
+    PIO_Offset start[NDIM_S1] = {0}, count[NDIM_S1] = {DIM_LEN_S1};
+    int data_in[DIM_LEN_S1];
+    if ((ret = PIOc_get_vars_tc(ncid, 0, start, count, NULL, NC_INT, data_in)))
+        return ret;
+    for (int i = 0; i < DIM_LEN_S1; i++)
+    {
+        printf("%d read data_in[%d] = %d\n", my_rank, i, data_in[i]);
+        if (data_in[i] != i)
+            return ERR_AWFUL;
+    }
+
+    /* Find the number of dimensions, variables, and global attributes.*/
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ret;
+    if (ndims != 1 || nvars != 1 || ngatts != 0 || unlimdimid != -1)
+        return ERR_WRONG;
+
+    /* This should return PIO_NOERR. */
+    if ((ret = PIOc_inq(ncid, NULL, NULL, NULL, NULL)))
+        return ret;
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_ndims(ncid, &ndims2)))
+        return ret;
+    if (ndims2 != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_nvars(ncid, &nvars2)))
+        return ret;
+    if (nvars2 != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_natts(ncid, &ngatts2)))
+        return ret;
+    if (ngatts2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2)))
+        return ret;
+    if (unlimdimid != -1)
+        return ERR_WRONG;
+
+    /* Check out the dimension. */
+    if ((ret = PIOc_inq_dim(ncid, 0, dimname, &dimlen)))
+        return ret;
+    if (strcmp(dimname, DIM_NAME_S1) || dimlen != DIM_LEN_S1)
+        return ERR_WRONG;
+
+    /* Check out the variable. */
+    if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts)))
+        return ret;
+    if (strcmp(varname, VAR_NAME_S1) || vartype != NC_INT || varndims != NDIM_S1 ||
+        vardimids != 0 || varnatts != 0)
+        return ERR_WRONG;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file (again) ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* This creates a netCDF file in the specified format, with some
+ * sample values. */
+int
+create_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    int ncid, varid, dimid;
+    PIO_Offset start[NDIM_S2], count[NDIM_S2] = {0};
+    int data[DIM_LEN_S2];
+    int ret;
+
+    /* Create a netCDF file with one dimension and one variable. */
+    printf("%d creating file %s\n", my_rank, filename);
+    if ((ret = PIOc_createfile(iosysid, &ncid, &format, filename, NC_CLOBBER)))
+        return ret;
+    printf("%d file created ncid = %d\n", my_rank, ncid);
+
+    /* End define mode, then re-enter it. */
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    if ((ret = PIOc_redef(ncid)))
+        return ret;
+
+    /* Define a dimension. */
+    char dimname2[NC_MAX_NAME + 1];
+    printf("%d defining dimension %s\n", my_rank, DIM_NAME_S2);
+    if ((ret = PIOc_def_dim(ncid, FIRST_DIM_NAME_S2, DIM_LEN_S2, &dimid)))
+        return ret;
+    if ((ret = PIOc_inq_dimname(ncid, 0, dimname2)))
+        return ret;
+    if (strcmp(dimname2, FIRST_DIM_NAME_S2))
+        return ERR_WRONG;
+    if ((ret = PIOc_rename_dim(ncid, 0, DIM_NAME_S2)))
+        return ret;
+
+    /* Define a 1-D variable. */
+    char varname2[NC_MAX_NAME + 1];
+    printf("%d defining variable %s\n", my_rank, VAR_NAME_S2);
+    if ((ret = PIOc_def_var(ncid, FIRST_VAR_NAME_S2, NC_INT, NDIM_S2, &dimid, &varid)))
+        return ret;
+    if ((ret = PIOc_inq_varname(ncid, 0, varname2)))
+        return ret;
+    if (strcmp(varname2, FIRST_VAR_NAME_S2))
+        return ERR_WRONG;
+    if ((ret = PIOc_rename_var(ncid, 0, VAR_NAME_S2)))
+        return ret;
+
+    /* Add a global attribute. */
+    printf("%d writing attributes %s\n", my_rank, ATT_NAME_S2);
+    int att_data = ATT_VALUE_S2;
+    short short_att_data = ATT_VALUE_S2;
+    float float_att_data = ATT_VALUE_S2;
+    double double_att_data = ATT_VALUE_S2;
+    char attname2[NC_MAX_NAME + 1];
+    /* Write an att and rename it. */
+    if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME_S2, NC_INT, 1, &att_data)))
+        return ret;
+    if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, attname2)))
+        return ret;
+    if (strcmp(attname2, FIRST_ATT_NAME_S2))
+        return ERR_WRONG;
+    if ((ret = PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME_S2, ATT_NAME_S2)))
+        return ret;
+
+    /* Write an att and delete it. */
+    if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME_S2, NC_INT, 1, &att_data)))
+        return ret;
+    if ((ret = PIOc_del_att(ncid, NC_GLOBAL, FIRST_ATT_NAME_S2)))
+        return ret;
+    /* if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, FIRST_ATT_NAME_S2, NULL, NULL)) != PIO_ENOTATT) */
+    /* { */
+    /*  printf("ret = %d\n", ret); */
+    /*  return ERR_AWFUL; */
+    /* } */
+
+    /* Write some atts of different types. */
+    if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME_S2, NC_SHORT, 1, &short_att_data)))
+        return ret;
+    if ((ret = PIOc_put_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME_S2, NC_FLOAT, 1, &float_att_data)))
+        return ret;
+    if ((ret = PIOc_put_att_double(ncid, NC_GLOBAL, DOUBLE_ATT_NAME_S2, NC_DOUBLE, 1, &double_att_data)))
+        return ret;
+
+    /* End define mode. */
+    printf("%d ending define mode ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    printf("%d define mode ended ncid = %d\n", my_rank, ncid);
+
+    /* Write some data. For the PIOc_put/get functions, all data must
+     * be on compmaster before the function is called. Only
+     * compmaster's arguments are passed to the async msg handler. All
+     * other computation tasks are ignored. */
+    for (int i = 0; i < DIM_LEN_S2; i++)
+        data[i] = i;
+    printf("%d writing data\n", my_rank);
+    printf("%d writing data\n", my_rank);
+    start[0] = 0;
+    count[0] = DIM_LEN_S2;
+    if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data)))
+        return ret;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ret;
+        printf("%d closed file ncid = %d\n", my_rank, ncid);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Check sample file 2 for correctness.
+ *
+ * @param ncidp if NULL, close file, otherwise return ncid of still-open file.
+ * @reaturns 0 for success and error code otherwise.
+ */
+int
+check_nc_sample_2(int iosysid, int format, char *filename, int my_rank, int *ncidp)
+{
+    int ncid;
+    int ret;
+    int ndims, nvars, ngatts, unlimdimid;
+    int ndims2, nvars2, ngatts2, unlimdimid2;
+    int dimid2;
+    char dimname[NC_MAX_NAME + 1];
+    PIO_Offset dimlen;
+    char dimname2[NC_MAX_NAME + 1];
+    PIO_Offset dimlen2;
+    char varname[NC_MAX_NAME + 1];
+    nc_type vartype;
+    int varndims, vardimids, varnatts;
+    char varname2[NC_MAX_NAME + 1];
+    nc_type vartype2;
+    int varndims2, vardimids2, varnatts2;
+    int varid2;
+    int att_data;
+    short short_att_data;
+    float float_att_data;
+    double double_att_data;
+    nc_type atttype;
+    PIO_Offset attlen;
+    char myattname[NC_MAX_NAME + 1];
+    int myid;
+    PIO_Offset start[NDIM_S2] = {0}, count[NDIM_S2] = {DIM_LEN_S2};
+    int data_in[DIM_LEN_S2];
+
+    /* Re-open the file to check it. */
+    printf("%d opening file %s format %d\n", my_rank, filename, format);
+    if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename, NC_NOWRITE)))
+        return ERR_CHECK;
+
+    /* Try to read the data. */
+    if ((ret = PIOc_get_vars_tc(ncid, 0, start, count, NULL, NC_INT, data_in)))
+        return ERR_CHECK;
+    for (int i = 0; i < DIM_LEN_S2; i++)
+    {
+        printf("%d read data_in[%d] = %d\n", my_rank, i, data_in[i]);
+        if (data_in[i] != i)
+            return ERR_AWFUL;
+    }
+
+    /* Find the number of dimensions, variables, and global attributes.*/
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ERR_CHECK;
+    if (ndims != 1 || nvars != 1 || ngatts != 4 || unlimdimid != -1)
+        return ERR_WRONG;
+
+    /* This should return PIO_NOERR. */
+    if ((ret = PIOc_inq(ncid, NULL, NULL, NULL, NULL)))
+        return ERR_CHECK;
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_ndims(ncid, &ndims2)))
+        return ERR_CHECK;
+    if (ndims2 != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_nvars(ncid, &nvars2)))
+        return ERR_CHECK;
+    if (nvars2 != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_natts(ncid, &ngatts2)))
+        return ERR_CHECK;
+    if (ngatts2 != 4)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2)))
+        return ERR_CHECK;
+    if (unlimdimid != -1)
+        return ERR_WRONG;
+    /* Should succeed, do nothing. */
+    if ((ret = PIOc_inq_unlimdim(ncid, NULL)))
+        return ERR_CHECK;
+
+    /* Check out the dimension. */
+    if ((ret = PIOc_inq_dim(ncid, 0, dimname, &dimlen)))
+        return ERR_CHECK;
+    if (strcmp(dimname, DIM_NAME_S2) || dimlen != DIM_LEN_S2)
+        return ERR_WRONG;
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_dimname(ncid, 0, dimname2)))
+        return ERR_CHECK;
+    if (strcmp(dimname2, DIM_NAME_S2))
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_dimlen(ncid, 0, &dimlen2)))
+        return ERR_CHECK;
+    if (dimlen2 != DIM_LEN_S2)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_dimid(ncid, DIM_NAME_S2, &dimid2)))
+        return ERR_CHECK;
+    if (dimid2 != 0)
+        return ERR_WRONG;
+
+    /* Check out the variable. */
+    if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts)))
+        return ERR_CHECK;
+    if (strcmp(varname, VAR_NAME_S2) || vartype != NC_INT || varndims != NDIM_S2 ||
+        vardimids != 0 || varnatts != 0)
+        return ERR_WRONG;
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_varname(ncid, 0, varname2)))
+        return ERR_CHECK;
+    if (strcmp(varname2, VAR_NAME_S2))
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_vartype(ncid, 0, &vartype2)))
+        return ERR_CHECK;
+    if (vartype2 != NC_INT)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_varndims(ncid, 0, &varndims2)))
+        return ERR_CHECK;
+    if (varndims2 != NDIM_S2)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_vardimid(ncid, 0, &vardimids2)))
+        return ERR_CHECK;
+    if (vardimids2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_varnatts(ncid, 0, &varnatts2)))
+        return ERR_CHECK;
+    if (varnatts2 != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_varid(ncid, VAR_NAME_S2, &varid2)))
+        return ERR_CHECK;
+    if (varid2 != 0)
+        return ERR_WRONG;
+
+    /* Check out the global attributes. */
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME_S2, &atttype, &attlen)))
+        return ERR_CHECK;
+    if (atttype != NC_INT || attlen != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME_S2, &attlen)))
+        return ERR_CHECK;
+    if (attlen != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, myattname)))
+        return ERR_CHECK;
+    if (strcmp(ATT_NAME_S2, myattname))
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_attid(ncid, NC_GLOBAL, ATT_NAME_S2, &myid)))
+        return ERR_CHECK;
+    if (myid != 0)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME_S2, &att_data)))
+        return ERR_CHECK;
+    printf("%d att_data = %d\n", my_rank, att_data);
+    if (att_data != ATT_VALUE_S2)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, SHORT_ATT_NAME_S2, &atttype, &attlen)))
+        return ERR_CHECK;
+    if (atttype != NC_SHORT || attlen != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME_S2, &short_att_data)))
+        return ERR_CHECK;
+    if (short_att_data != ATT_VALUE_S2)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME_S2, &float_att_data)))
+        return ERR_CHECK;
+    if (float_att_data != ATT_VALUE_S2)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_double(ncid, NC_GLOBAL, DOUBLE_ATT_NAME_S2, &double_att_data)))
+        return ERR_CHECK;
+    if (double_att_data != ATT_VALUE_S2)
+        return ERR_WRONG;
+
+    /* Close the file if ncidp was not provided. */
+    if (ncidp)
+        *ncidp = ncid;
+    else
+    {
+        printf("%d closing file (again) ncid = %d\n", my_rank, ncid);
+        if ((ret = PIOc_closefile(ncid)))
+            return ERR_CHECK;
+    }
+
+    return 0;
+}
+
+/* Create the decomposition to divide the 3-dimensional sample data
+ * between the 4 tasks. For the purposes of decomposition we are only
+ * concerned with 2 dimensions - we ignore the unlimited dimension.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim_len_2d an array of length 2 with the dim lengths.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @param pio_type the data type to use for the decomposition.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition_2d(int ntasks, int my_rank, int iosysid, int *dim_len_2d,
+                            int *ioid, int pio_type)
+{
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    int ret;
+
+    /* How many data elements per task? In this example we will end up
+     * with 4. */
+    elements_per_pe = dim_len_2d[0] * dim_len_2d[1] / ntasks;
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. This is a 1-based array, so add 1! */
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i + 1;
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe);
+    if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM2, dim_len_2d, elements_per_pe,
+                               compdof, ioid, NULL, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
diff --git a/src/externals/pio2/tests/cunit/test_darray.c b/src/externals/pio2/tests/cunit/test_darray.c
new file mode 100644
index 00000000000..e53e29bff4e
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_darray.c
@@ -0,0 +1,375 @@
+/*
+ * Tests for PIO distributed arrays.
+ *
+ * Ed Hartnett, 2/16/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* But sometimes we need arrays of the non-record dimensions. */
+#define NDIM2 2
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "foo"
+
+/* Test cases relating to PIOc_write_darray_multi(). */
+#define NUM_TEST_CASES_WRT_MULTI 3
+
+/* Test with and without specifying a fill value to
+ * PIOc_write_darray(). */
+#define NUM_TEST_CASES_FILLVALUE 2
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/** 
+ * Test the darray functionality. Create a netCDF file with 3
+ * dimensions and 1 PIO_INT variable, and use darray to write some
+ * data.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param pio_type the type of the data.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                int pio_type)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimids[NDIM];      /* The dimension IDs. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int ncid2;     /* The ncid of the re-opened netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    int ret;       /* Return code. */
+    PIO_Offset arraylen = 4;
+    void *fillvalue;
+    void *test_data;
+    void *test_data_in;
+    int fillvalue_int = NC_FILL_INT;
+    int test_data_int[arraylen];
+    int test_data_int_in[arraylen];
+    float fillvalue_float = NC_FILL_FLOAT;
+    float test_data_float[arraylen];
+    float test_data_float_in[arraylen];
+    double fillvalue_double = NC_FILL_DOUBLE;
+    double test_data_double[arraylen];
+    double test_data_double_in[arraylen];
+
+    /* Initialize some data. */
+    for (int f = 0; f < arraylen; f++)
+    {
+        test_data_int[f] = my_rank * 10 + f;
+        test_data_float[f] = my_rank * 10 + f + 0.5;
+        test_data_double[f] = my_rank * 100000 + f + 0.5;
+    }
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++) 
+    {
+
+        /* Add a couple of extra tests for the
+         * PIOc_write_darray_multi() function. */
+        for (int test_multi = 0; test_multi < NUM_TEST_CASES_WRT_MULTI; test_multi++)
+        {
+            /* Test with/without providing a fill value to PIOc_write_darray(). */
+            for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++)
+            {
+                /* Create the filename. */
+                sprintf(filename, "data_%s_iotype_%d_pio_type_%d_test_multi_%d_provide_fill_%d.nc", TEST_NAME,
+                        flavor[fmt], pio_type, test_multi, provide_fill);
+
+                /* Select the fill value and data. */
+                switch (pio_type)
+                {
+                case PIO_INT:
+                    fillvalue = provide_fill ? &fillvalue_int : NULL;
+                    test_data = test_data_int;
+                    test_data_in = test_data_int_in;
+                    break;
+                case PIO_FLOAT:
+                    fillvalue = provide_fill ? &fillvalue_float : NULL;
+                    test_data = test_data_float;
+                    test_data_in = test_data_float_in;
+                    break;
+                case PIO_DOUBLE:
+                    fillvalue = provide_fill ? &fillvalue_double : NULL;
+                    test_data = test_data_double;
+                    test_data_in = test_data_double_in;
+                    break;
+                default:
+                    ERR(ERR_WRONG);
+                }
+
+                /* Create the netCDF output file. */
+                printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename,
+                       flavor[fmt], pio_type);
+                if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+                    ERR(ret);
+
+                /* Define netCDF dimensions and variable. */
+                printf("%d Defining netCDF metadata...\n", my_rank);
+                for (int d = 0; d < NDIM; d++)
+                    if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                        ERR(ret);
+
+                /* Define a variable. */
+                if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, dimids, &varid)))
+                    ERR(ret);
+
+                /* End define mode. */
+                if ((ret = PIOc_enddef(ncid)))
+                    ERR(ret);
+
+                /* Set the value of the record dimension. */
+                if ((ret = PIOc_setframe(ncid, varid, 0)))
+                    ERR(ret);
+
+                int frame = 0;
+                int flushtodisk = test_multi - 1;
+                if (!test_multi)
+                {
+                    /* These should not work. */
+                    if (PIOc_write_darray(ncid + TEST_VAL_42, varid, ioid, arraylen, test_data, fillvalue) != PIO_EBADID)
+                        ERR(ERR_WRONG);
+                    if (PIOc_write_darray(ncid, varid, ioid + TEST_VAL_42, arraylen, test_data, fillvalue) != PIO_EBADID)
+                        ERR(ERR_WRONG);
+                    if (PIOc_write_darray(ncid, varid, ioid, arraylen - 1, test_data, fillvalue) != PIO_EINVAL)
+                        ERR(ERR_WRONG);
+
+                    /* Write the data. */
+                    if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
+                        ERR(ret);
+                }
+                else
+                {
+                    int varid_big = NC_MAX_VARS + TEST_VAL_42;
+
+                    /* These will not work. */
+                    if (PIOc_write_darray_multi(ncid + TEST_VAL_42, &varid, ioid, 1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EBADID)
+                        ERR(ERR_WRONG);
+                    if (PIOc_write_darray_multi(ncid, NULL, ioid, 1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EINVAL)
+                        ERR(ERR_WRONG);
+                    if (PIOc_write_darray_multi(ncid, &varid, ioid + TEST_VAL_42, 1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EBADID)
+                        ERR(ERR_WRONG);
+                    if (PIOc_write_darray_multi(ncid, &varid, ioid, -1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EINVAL)
+                        ERR(ERR_WRONG);
+                    /* if (PIOc_write_darray_multi(ncid, &varid, ioid, 1, arraylen, test_data, NULL, */
+                    /*                             fillvalue, flushtodisk) != PIO_EINVAL) */
+                    /*     ERR(ERR_WRONG); */
+                    if (PIOc_write_darray_multi(ncid, &varid_big, ioid, 1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EINVAL)
+                        ERR(ERR_WRONG);
+                
+                    /* Write the data with the _multi function. */
+                    if ((ret = PIOc_write_darray_multi(ncid, &varid, ioid, 1, arraylen, test_data, &frame,
+                                                       fillvalue, flushtodisk)))
+                        ERR(ret);
+                }
+
+                /* Close the netCDF file. */
+                if ((ret = PIOc_closefile(ncid)))
+                    ERR(ret);
+
+                /* Reopen the file. */
+                if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
+                    ERR(ret);
+
+                /* These should not work. */
+                if (PIOc_read_darray(ncid2 + TEST_VAL_42, varid, ioid, arraylen,
+                                     test_data_in) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_read_darray(ncid2, varid, ioid + TEST_VAL_42, arraylen,
+                                     test_data_in) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+        
+                /* Read the data. */
+                if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in)))
+                    ERR(ret);
+
+                /* Check the results. */
+                for (int f = 0; f < arraylen; f++)
+                {
+                    switch (pio_type)
+                    {
+                    case PIO_INT:
+                        if (test_data_int_in[f] != test_data_int[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_FLOAT:
+                        if (test_data_float_in[f] != test_data_float[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_DOUBLE:
+                        if (test_data_double_in[f] != test_data_double[f])
+                            return ERR_WRONG;
+                        break;
+                    default:
+                        ERR(ERR_WRONG);
+                    }
+                }
+
+                /* Try to write, but it won't work, because we opened file read-only. */
+                if (!test_multi)
+                {
+                    if (PIOc_write_darray(ncid2, varid, ioid, arraylen, test_data, fillvalue) != PIO_EPERM)
+                        ERR(ERR_WRONG);
+                }
+                else
+                {
+                    if (PIOc_write_darray_multi(ncid2, &varid, ioid, 1, arraylen, test_data, &frame,
+                                                fillvalue, flushtodisk) != PIO_EPERM)
+                        ERR(ERR_WRONG);
+                }
+        
+                /* Close the netCDF file. */
+                printf("%d Closing the sample data file...\n", my_rank);
+                if ((ret = PIOc_closefile(ncid2)))
+                    ERR(ret);
+            } /* next fillvalue test case */
+        } /* next test multi */
+    } /* next iotype */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Run all the tests. 
+ *
+ * @param iosysid the IO system ID.
+ * @param num_flavors number of available iotypes in the build.
+ * @param flavor pointer to array of the available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the communicator the test is running on.
+ * @returns 0 for success, error code otherwise. 
+ */
+int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
+                    MPI_Comm test_comm)
+{
+#define NUM_TYPES_TO_TEST 3
+    int ioid;
+    char filename[NC_MAX_NAME + 1];
+    int pio_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE};
+    int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
+    int ret; /* Return code. */
+
+    for (int t = 0; t < NUM_TYPES_TO_TEST; t++)
+    {        
+        /* This will be our file name for writing out decompositions. */
+        sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank,
+                *flavor, pio_type[t]);
+
+        /* Decompose the data over the tasks. */
+        if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d,
+                                           &ioid, pio_type[t])))
+        return ret;
+
+        /* Run a simple darray test. */
+        if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t])))
+            return ret;
+    
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+#define NUM_REARRANGERS_TO_TEST 2
+    int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              MIN_NTASKS, 3, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int iosysid;  /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+        int ret;      /* Return code. */
+        
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+        {
+        /* Initialize the PIO IO system. This specifies how
+         * many and which processors are involved in I/O. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+                                       ioproc_start, rearranger[r], &iosysid)))
+            return ret;
+
+        /* Run tests. */
+        printf("%d Running tests...\n", my_rank);
+        if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm)))
+            return ret;
+
+        /* Finalize PIO system. */
+        if ((ret = PIOc_finalize(iosysid)))
+            return ret;
+        } /* next rearranger */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_darray_1d.c b/src/externals/pio2/tests/cunit/test_darray_1d.c
new file mode 100644
index 00000000000..c5c6ac85012
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_darray_1d.c
@@ -0,0 +1,891 @@
+/*
+ * Tests for PIO distributed arrays. This test uses 1 dimension,
+ * everything very simple. ;-)
+ *
+ * Ed Hartnett, 2/27/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_1d"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 1
+
+/* The length of our sample data along the dimension. */
+#define DIM_LEN 8
+
+/* This is the length of the map for each task. */
+#define EXPECTED_MAPLEN 2
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "RedShirtSurvival"
+
+/* The dimension names. */
+#define DIM_NAME "episode"
+#define DIM_NAME_2 "phaser_draws"
+
+/* Create a 1D decomposition.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim_len an array of length 3 with the dimension sizes.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @param pio_type the type that will be used for basetype.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition_1d(int ntasks, int my_rank, int iosysid, int pio_type, int *ioid)
+{
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    int dim_len_1d[NDIM] = {DIM_LEN};
+    int ret;
+
+    /* How many data elements per task? In this example we will end up
+     * with 2. */
+    elements_per_pe = DIM_LEN / ntasks;
+
+    PIO_Offset compdof[elements_per_pe];
+
+    /* Don't forget to add 1! */
+    compdof[0] = my_rank + 1;
+
+    /* This means fill value will be used here. */
+    compdof[1] = 0;
+
+    /* Create the PIO decomposition for this test. */
+    if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM, dim_len_1d, elements_per_pe,
+                               compdof, ioid, NULL, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    return 0;
+}
+
+/**
+ * Test fill values and darrays.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param pio_type the type of the data.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the MPI communicator running the test.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_darray_fill(int iosysid, int ioid, int pio_type, int num_flavors, int *flavor,
+                     int my_rank, MPI_Comm test_comm)
+{
+#define NUM_FILLVALUE_PRESENT_TESTS 2
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimid;     /* The dimension ID. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    PIO_Offset arraylen = 2;
+    void *test_data;
+    void *fillvalue;
+    void *test_data_in;
+    void *expected_in;
+    PIO_Offset type_size;             /* Size of the data type. */
+    /* My rank as each type. */
+    signed char my_byte_rank = my_rank;   
+    char my_char_rank = my_rank;   
+    short my_short_rank = my_rank;   
+    float my_float_rank = my_rank;   
+    double my_double_rank = my_rank;
+#ifdef _NETCDF4
+    unsigned char my_ubyte_rank = my_rank;
+    unsigned short my_ushort_rank = my_rank;
+    unsigned int my_uint_rank = my_rank;
+    long long my_int64_rank = my_rank;
+    unsigned long long my_uint64_rank = my_rank;
+#endif /* _NETCDF4 */
+
+    /* Default fill value for each type. */
+    signed char byte_fill = NC_FILL_BYTE;
+    char char_fill = NC_FILL_CHAR;
+    short short_fill = NC_FILL_SHORT;
+    int int_fill = NC_FILL_INT;
+    float float_fill = NC_FILL_FLOAT;
+    double double_fill = NC_FILL_DOUBLE;
+#ifdef _NETCDF4
+    unsigned char ubyte_fill = NC_FILL_UBYTE;
+    unsigned short ushort_fill = NC_FILL_USHORT;
+    unsigned int uint_fill = NC_FILL_UINT;
+    long long int64_fill = NC_FILL_INT64;
+    unsigned long long uint64_fill = NC_FILL_UINT64;
+#endif /* _NETCDF4 */
+
+    void *bufr;
+    int ret; /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* BYTE and CHAR don't work with pnetcdf. Don't know why yet. */
+        if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR))
+            continue;
+
+        /* NetCDF-4 types only work with netCDF-4 formats. */
+        printf("pio_type = %d flavor[fmt] = %d\n", pio_type, flavor[fmt]);
+        if (pio_type > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C &&
+            flavor[fmt] != PIO_IOTYPE_NETCDF4P)
+            continue;
+        
+        for (int with_fillvalue = 0; with_fillvalue < NUM_FILLVALUE_PRESENT_TESTS; with_fillvalue++)
+        {
+            /* Create the filename. */
+            sprintf(filename, "data_%s_iotype_%d_pio_type_%d_with_fillvalue_%d.nc", TEST_NAME, flavor[fmt],
+                    pio_type, with_fillvalue);
+
+            /* Create the netCDF output file. */
+            printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename,
+                   flavor[fmt]);
+            if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+                ERR(ret);
+
+            /* Turn on fill mode. */
+            if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
+                ERR(ret);
+
+            /* Define netCDF dimensions and variable. */
+            if ((ret = PIOc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid)))
+                ERR(ret);
+
+            /* Define a variable. */
+            if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM, &dimid, &varid)))
+                ERR(ret);
+
+            /* End define mode. */
+            if ((ret = PIOc_enddef(ncid)))
+                ERR(ret);
+
+            /* Get the size of the type. */
+            if ((ret = PIOc_inq_type(ncid, pio_type, NULL, &type_size)))
+                return ret;
+
+            /* Initialize some data. */
+            signed char byte_test_data[2] = {my_rank, my_rank};
+            char char_test_data[2] = {my_rank, my_rank};
+            short short_test_data[2] = {my_rank, my_rank};
+            int int_test_data[2] = {my_rank, my_rank};
+            float float_test_data[2] = {my_rank, my_rank};
+            double double_test_data[2] = {my_rank, my_rank};
+#ifdef _NETCDF4
+            unsigned char ubyte_test_data[2] = {my_rank, my_rank};
+            unsigned short ushort_test_data[2] = {my_rank, my_rank};
+            unsigned int uint_test_data[2] = {my_rank, my_rank};
+            long long int64_test_data[2] = {my_rank, my_rank};
+            unsigned long long uint64_test_data[2] = {my_rank, my_rank};
+#endif /* _NETCDF4 */
+            
+            switch (pio_type)
+            {
+            case PIO_BYTE:
+                test_data = byte_test_data;
+                fillvalue = with_fillvalue ? &byte_fill : NULL;
+                expected_in = &my_byte_rank;
+                break;
+            case PIO_CHAR:
+                test_data = char_test_data;
+                fillvalue = with_fillvalue ? &char_fill : NULL;
+                expected_in = &my_char_rank;
+                break;
+            case PIO_SHORT:
+                test_data = short_test_data;
+                fillvalue = with_fillvalue ? &short_fill : NULL;
+                expected_in = &my_short_rank;
+                break;
+            case PIO_INT:
+                test_data = int_test_data;
+                fillvalue = with_fillvalue ? &int_fill : NULL;
+                expected_in = &my_rank;
+                break;
+            case PIO_FLOAT:
+                test_data = float_test_data;
+                fillvalue = with_fillvalue ? &float_fill : NULL;
+                expected_in = &my_float_rank;
+                break;
+            case PIO_DOUBLE:
+                test_data = double_test_data;
+                fillvalue = with_fillvalue ? &double_fill : NULL;
+                expected_in = &my_double_rank;
+                break;
+#ifdef _NETCDF4
+            case PIO_UBYTE:
+                test_data = ubyte_test_data;
+                fillvalue = with_fillvalue ? &ubyte_fill : NULL;
+                expected_in = &my_ubyte_rank;
+                break;
+            case PIO_USHORT:
+                test_data = ushort_test_data;
+                fillvalue = with_fillvalue ? &ushort_fill : NULL;
+                expected_in = &my_ushort_rank;
+                break;
+            case PIO_UINT:
+                test_data = uint_test_data;
+                fillvalue = with_fillvalue ? &uint_fill : NULL;
+                expected_in = &my_uint_rank;
+                break;
+            case PIO_INT64:
+                test_data = int64_test_data;
+                fillvalue = with_fillvalue ? &int64_fill : NULL;
+                expected_in = &my_int64_rank;
+                break;
+            case PIO_UINT64:
+                test_data = uint64_test_data;
+                fillvalue = with_fillvalue ? &uint64_fill : NULL;
+                expected_in = &my_uint64_rank;
+                break;
+#endif /* _NETCDF4 */
+            default:
+                return ERR_WRONG;
+            }
+
+            /* Write the data. Our test_data contains only one real
+             * value (instead of 2, as indicated by arraylen), but due
+             * to the decomposition, only the first value is used in
+             * the output. */
+            if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
+                ERR(ret);
+
+            /* Close the netCDF file. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+
+            /* Reopen the file. */
+            if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename, PIO_NOWRITE)))
+                ERR(ret);
+
+            /* Allocate space for data. */
+            if (!(test_data_in = malloc(type_size * arraylen)))
+                ERR(PIO_ENOMEM);
+
+            /* Read the data. */
+            if ((ret = PIOc_read_darray(ncid, varid, ioid, arraylen, test_data_in)))
+                ERR(ret);
+
+            /* Check the (first) result. */
+            if (memcmp(test_data_in, expected_in, type_size))
+                return ERR_WRONG;
+
+            /* Free resources. */
+            free(test_data_in);
+
+            /* Get a buffer big enough to hold the global array. */
+            if (!(bufr = malloc(DIM_LEN * type_size)))
+                return PIO_ENOMEM;
+
+            /* Get the whole array with good old get_var(). */
+            if ((ret = PIOc_get_var(ncid, varid, bufr)))
+                return ret;
+
+            /* Check the results. The first four values are 0, 1, 2, 3,
+             * and the rest are the default fill value of the type. */
+            for (int e = 0; e < DIM_LEN; e++)
+            {
+                switch (pio_type)
+                {
+                case PIO_BYTE:
+                    if (((signed char *)bufr)[e] != (e < 4 ? e : NC_FILL_BYTE))
+                        return ERR_WRONG;
+                    break;
+                case PIO_CHAR:
+                    if (((char *)bufr)[e] != (e < 4 ? e : NC_FILL_CHAR))
+                        return ERR_WRONG;
+                    break;
+                case PIO_SHORT:
+                    if (((short *)bufr)[e] != (e < 4 ? e : NC_FILL_SHORT))
+                        return ERR_WRONG;
+                    break;
+                case PIO_INT:
+                    if (((int *)bufr)[e] != (e < 4 ? e : NC_FILL_INT))
+                        return ERR_WRONG;
+                    break;
+                case PIO_FLOAT:
+                    if (((float *)bufr)[e] != (e < 4 ? e : NC_FILL_FLOAT))
+                        return ERR_WRONG;
+                    break;
+                case PIO_DOUBLE:
+                    if (((double *)bufr)[e] != (e < 4 ? e : NC_FILL_DOUBLE))
+                        return ERR_WRONG;
+                    break;
+#ifdef _NETCDF4
+                case PIO_UBYTE:
+                    if (((unsigned char *)bufr)[e] != (e < 4 ? e : NC_FILL_UBYTE))
+                        return ERR_WRONG;
+                    break;
+                case PIO_USHORT:
+                    if (((unsigned short *)bufr)[e] != (e < 4 ? e : NC_FILL_USHORT))
+                        return ERR_WRONG;
+                    break;
+                case PIO_UINT:
+                    if (((unsigned int *)bufr)[e] != (e < 4 ? e : NC_FILL_UINT))
+                        return ERR_WRONG;
+                    break;
+                case PIO_INT64:
+                    if (((long long *)bufr)[e] != (e < 4 ? e : NC_FILL_INT64))
+                        return ERR_WRONG;
+                    break;
+                case PIO_UINT64:
+                    if (((unsigned long long *)bufr)[e] != (e < 4 ? e : NC_FILL_UINT64))
+                        return ERR_WRONG;
+                    break;
+#endif /* _NETCDF4 */
+                default:
+                    return ERR_WRONG;
+                }
+            }
+
+            /* Release buffer. */
+            free(bufr);
+
+            /* Close the netCDF file. */
+            printf("%d Closing the sample data file...\n", my_rank);
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+        } /* with_fillvalue */
+    } /* next iotype */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Test fill values and darrays with an unlimited dim.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param pio_type the type of the data.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the MPI communicator running the test.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_darray_fill_unlim(int iosysid, int ioid, int pio_type, int num_flavors,
+                           int *flavor, int my_rank, MPI_Comm test_comm)
+{
+#define NDIM2 2
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimid[NDIM2];     /* The dimension ID. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    PIO_Offset arraylen = 2;
+    void *test_data;
+    void *fillvalue;
+    void *test_data_in;
+    void *expected_in;
+    PIO_Offset type_size;             /* Size of the data type. */
+
+    /* My rank as each type. */
+    signed char my_byte_rank = my_rank;   
+    char my_char_rank = my_rank;   
+    short my_short_rank = my_rank;   
+    float my_float_rank = my_rank;   
+    double my_double_rank = my_rank;
+#ifdef _NETCDF4
+    unsigned char my_ubyte_rank = my_rank;
+    unsigned short my_ushort_rank = my_rank;
+    unsigned int my_uint_rank = my_rank;
+    long long my_int64_rank = my_rank;
+    unsigned long long my_uint64_rank = my_rank;
+#endif /* _NETCDF4 */
+
+    /* Default fill value for each type. */
+    signed char byte_fill = NC_FILL_BYTE;
+    char char_fill = NC_FILL_CHAR;
+    short short_fill = NC_FILL_SHORT;
+    int int_fill = NC_FILL_INT;
+    float float_fill = NC_FILL_FLOAT;
+    double double_fill = NC_FILL_DOUBLE;
+#ifdef _NETCDF4
+    unsigned char ubyte_fill = NC_FILL_UBYTE;
+    unsigned short ushort_fill = NC_FILL_USHORT;
+    unsigned int uint_fill = NC_FILL_UINT;
+    long long int64_fill = NC_FILL_INT64;
+    unsigned long long uint64_fill = NC_FILL_UINT64;
+#endif /* _NETCDF4 */
+    void *bufr;
+    int ret;                        /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* BYTE and CHAR don't work with pnetcdf. Don't know why yet. */
+        if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR))
+            continue;
+
+        /* NetCDF-4 types only work with netCDF-4 formats. */
+        printf("pio_type = %d flavor[fmt] = %d\n", pio_type, flavor[fmt]);
+        if (pio_type > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C &&
+            flavor[fmt] != PIO_IOTYPE_NETCDF4P)
+            continue;
+        
+        /* Create the filename. */
+        sprintf(filename, "data_%s_iotype_%d_pio_type_%d_unlim.nc", TEST_NAME, flavor[fmt],
+                pio_type);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename,
+               flavor[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Turn on fill mode. */
+        if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
+            ERR(ret);
+
+        /* Define netCDF dimensions. */
+        if ((ret = PIOc_def_dim(ncid, DIM_NAME, NC_UNLIMITED, &dimid[0])))
+            ERR(ret);
+        if ((ret = PIOc_def_dim(ncid, DIM_NAME_2, DIM_LEN, &dimid[1])))
+            ERR(ret);
+
+        /* Define a variable. */
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, pio_type, NDIM2, dimid, &varid)))
+            ERR(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Get the size of the type. */
+        if ((ret = PIOc_inq_type(ncid, pio_type, NULL, &type_size)))
+            return ret;
+
+        /* Initialize some data. */
+        signed char byte_test_data[2] = {my_rank, my_rank};
+        char char_test_data[2] = {my_rank, my_rank};
+        short short_test_data[2] = {my_rank, my_rank};
+        int int_test_data[2] = {my_rank, my_rank};
+        float float_test_data[2] = {my_rank, my_rank};
+        double double_test_data[2] = {my_rank, my_rank};
+#ifdef _NETCDF4
+        unsigned char ubyte_test_data[2] = {my_rank, my_rank};
+        unsigned short ushort_test_data[2] = {my_rank, my_rank};
+        unsigned int uint_test_data[2] = {my_rank, my_rank};
+        long long int64_test_data[2] = {my_rank, my_rank};
+        unsigned long long uint64_test_data[2] = {my_rank, my_rank};
+#endif /* _NETCDF4 */
+        switch (pio_type)
+        {
+        case PIO_BYTE:
+            test_data = byte_test_data;
+            fillvalue = &byte_fill;
+            expected_in = &my_byte_rank;
+            break;
+        case PIO_CHAR:
+            test_data = char_test_data;
+            fillvalue = &char_fill;
+            expected_in = &my_char_rank;
+            break;
+        case PIO_SHORT:
+            test_data = short_test_data;
+            fillvalue = &short_fill;
+            expected_in = &my_short_rank;
+            break;
+        case PIO_INT:
+            test_data = int_test_data;
+            fillvalue = &int_fill;
+            expected_in = &my_rank;
+            break;
+        case PIO_FLOAT:
+            test_data = float_test_data;
+            fillvalue = &float_fill;
+            expected_in = &my_float_rank;
+            break;
+        case PIO_DOUBLE:
+            test_data = double_test_data;
+            fillvalue = &double_fill;
+            expected_in = &my_double_rank;
+            break;
+#ifdef _NETCDF4
+        case PIO_UBYTE:
+            test_data = ubyte_test_data;
+            fillvalue = &ubyte_fill;
+            expected_in = &my_ubyte_rank;
+            break;
+        case PIO_USHORT:
+            test_data = ushort_test_data;
+            fillvalue = &ushort_fill;
+            expected_in = &my_ushort_rank;
+            break;
+        case PIO_UINT:
+            test_data = uint_test_data;
+            fillvalue = &uint_fill;
+            expected_in = &my_uint_rank;
+            break;
+        case PIO_INT64:
+            test_data = int64_test_data;
+            fillvalue = &int64_fill;
+            expected_in = &my_int64_rank;
+            break;
+        case PIO_UINT64:
+            test_data = uint64_test_data;
+            fillvalue = &uint64_fill;
+            expected_in = &my_uint64_rank;
+            break;
+#endif /* _NETCDF4 */
+        default:
+            return ERR_WRONG;
+        }
+
+        /* Set the record number for the unlimited dimension. */
+        if ((ret = PIOc_setframe(ncid, varid, 0)))
+            ERR(ret);
+
+        /* Write the data. Our test_data contains only one real value
+         * (instead of 2, as indicated by arraylen), but due to the
+         * decomposition, only the first value is used in the
+         * output. */
+        if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
+            ERR(ret);
+
+        /* Set the record number for the unlimited dimension. */
+        if ((ret = PIOc_setframe(ncid, varid, 1)))
+            ERR(ret);
+
+        /* Write the data. Our test_data contains only one real value
+         * (instead of 2, as indicated by arraylen), but due to the
+         * decomposition, only the first value is used in the
+         * output. */
+        if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Reopen the file. */
+        if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename, PIO_NOWRITE)))
+            ERR(ret);
+
+        /* Allocate space for data. */
+        if (!(test_data_in = malloc(type_size * arraylen)))
+            ERR(PIO_ENOMEM);
+
+        /* Read the data. */
+        if ((ret = PIOc_read_darray(ncid, varid, ioid, arraylen, test_data_in)))
+            ERR(ret);
+
+        /* Check the (first) result. */
+        if (memcmp(test_data_in, expected_in, type_size))
+            return ERR_WRONG;
+
+        /* Free resources. */
+        free(test_data_in);
+
+        /* Get a buffer big enough to hold the global array. */
+        if (!(bufr = malloc(DIM_LEN * type_size * 2)))
+            return PIO_ENOMEM;
+
+        /* Get the whole array with good old get_var(). */
+        if ((ret = PIOc_get_var(ncid, varid, bufr)))
+            return ret;
+
+        /* Check the results. The first four values in each record are
+         * 0, 1, 2, 3, and the rest are the default fill value of the
+         * type. There are two records. */
+        for (int e = 0; e < DIM_LEN * 2; e++)
+        {
+            switch (pio_type)
+            {
+            case PIO_BYTE:
+                if (((signed char *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_BYTE))
+                    return ERR_WRONG;
+                break;
+            case PIO_CHAR:
+                if (((char *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_CHAR))
+                    return ERR_WRONG;
+                break;
+            case PIO_SHORT:
+                if (((short *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_SHORT))
+                    return ERR_WRONG;
+                break;
+            case PIO_INT:
+                if (((int *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_INT))
+                    return ERR_WRONG;
+                break;
+            case PIO_FLOAT:
+                if (((float *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_FLOAT))
+                    return ERR_WRONG;
+                break;
+            case PIO_DOUBLE:
+                if (((double *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_DOUBLE))
+                    return ERR_WRONG;
+                break;
+#ifdef _NETCDF4
+            case PIO_UBYTE:
+                if (((unsigned char *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_UBYTE))
+                    return ERR_WRONG;
+                break;
+            case PIO_USHORT:
+                if (((unsigned short *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_USHORT))
+                    return ERR_WRONG;
+                break;
+            case PIO_UINT:
+                if (((unsigned int *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_UINT))
+                    return ERR_WRONG;
+                break;
+            case PIO_INT64:
+                if (((long long *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_INT64))
+                    return ERR_WRONG;
+                break;
+            case PIO_UINT64:
+                if (((unsigned long long *)bufr)[e] != (e % 8 < 4 ? e % 8 : NC_FILL_UINT64))
+                    return ERR_WRONG;
+                break;
+#endif /* _NETCDF4 */
+            default:
+                return ERR_WRONG;
+            }
+        }
+
+        /* Release buffer. */
+        free(bufr);
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+    } /* next iotype */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Test the decomp read/write functionality.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param pio_type the type involved in this decompositon.
+ * @param rearranger the rearranger in use.
+ * @param test_comm the MPI communicator for this test.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                           int pio_type, int rearranger, MPI_Comm test_comm)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int ioid2;             /* ID for decomposition we will create from file. */
+    char title_in[PIO_MAX_NAME + 1];   /* Optional title. */
+    char history_in[PIO_MAX_NAME + 1]; /* Optional history. */
+    int fortran_order_in; /* Indicates fortran vs. c order. */
+    int ret;              /* Return code. */
+
+    /* Use PIO to create the decomp file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create the filename. */
+        sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]);
+
+        printf("writing decomp file %s\n", filename);
+        if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0)))
+            return ret;
+
+        /* Read the data. */
+        printf("reading decomp file %s\n", filename);
+        if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, pio_type,
+                                       title_in, history_in, &fortran_order_in)))
+            return ret;
+
+        /* Check the results. */
+        {
+            iosystem_desc_t *ios;
+            io_desc_t *iodesc;
+            MPI_Datatype expected_basetype;
+
+            switch (pio_type)
+            {
+            case PIO_BYTE:
+                expected_basetype = MPI_BYTE;
+                break;
+            case PIO_CHAR:
+                expected_basetype = MPI_CHAR;
+                break;
+            case PIO_SHORT:
+                expected_basetype = MPI_SHORT;
+                break;
+            case PIO_INT:
+                expected_basetype = MPI_INT;
+                break;
+            case PIO_FLOAT:
+                expected_basetype = MPI_FLOAT;
+                break;
+            case PIO_DOUBLE:
+                expected_basetype = MPI_DOUBLE;
+                break;
+#ifdef _NETCDF4
+            case PIO_UBYTE:
+                expected_basetype = MPI_UNSIGNED_CHAR;
+                break;
+            case PIO_USHORT:
+                expected_basetype = MPI_UNSIGNED_SHORT;
+                break;
+            case PIO_UINT:
+                expected_basetype = MPI_UNSIGNED;
+                break;
+            case PIO_INT64:
+                expected_basetype = MPI_LONG_LONG;
+                break;
+            case PIO_UINT64:
+                expected_basetype = MPI_UNSIGNED_LONG_LONG;
+                break;
+#endif /* _NETCDF4 */
+            default:
+                return ERR_WRONG;
+            }
+
+            /* Get the IO system info. */
+            if (!(ios = pio_get_iosystem_from_id(iosysid)))
+                return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+            /* Get the IO desc, which describes the decomposition. */
+            if (!(iodesc = pio_get_iodesc_from_id(ioid2)))
+                return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+            if (iodesc->ioid != ioid2 || iodesc->maplen != EXPECTED_MAPLEN || iodesc->ndims != NDIM)
+                return ERR_WRONG;
+            /* if (iodesc->nrecvs != 1) */
+            /*     return ERR_WRONG; */
+            /* if (iodesc->num_aiotasks != TARGET_NTASKS) */
+            /*     return ERR_WRONG; */
+            printf("iodesc->nrecvs = %d iodesc->num_aiotasks = %d\n", iodesc->nrecvs, iodesc->num_aiotasks);
+            if (iodesc->ndof != EXPECTED_MAPLEN)
+                return ERR_WRONG;
+            if (iodesc->rearranger != rearranger || iodesc->maxregions != 1)
+                return ERR_WRONG;
+            if (!iodesc->needsfill || iodesc->basetype != expected_basetype)
+                return ERR_WRONG;
+            /* Don't forget to add 1!! */
+            if (iodesc->map[0] != my_rank + 1 || iodesc->map[1] != 0)
+                return ERR_WRONG;
+            if (iodesc->dimlen[0] != DIM_LEN)
+                return ERR_WRONG;
+        }
+
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid2)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+#define NUM_REARRANGERS_TO_TEST 2
+    int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+#ifdef _NETCDF4
+#define NUM_TYPES_TO_TEST 11
+    int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
+                                        PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64};
+#else
+#define NUM_TYPES_TO_TEST 6
+    int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE};
+#endif /* _NETCDF4 */
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              MIN_NTASKS, 3, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int iosysid;  /* The ID for the parallel I/O system. */
+        int ioid;     /* Decomposition ID. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Rank of first processor to be used for I/O. */
+        int ret;      /* Return code. */
+
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+        {
+            /* Initialize the PIO IO system. This specifies how many and
+             * which processors are involved in I/O. */
+            if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+                                           ioproc_start, rearranger[r], &iosysid)))
+                return ret;
+
+            /* Run tests for each data type. */
+            for (int t = 0; t < NUM_TYPES_TO_TEST; t++)
+            {
+                /* Decompose the data over the tasks. */
+                if ((ret = create_decomposition_1d(TARGET_NTASKS, my_rank, iosysid, test_type[t],
+                                                   &ioid)))
+                    return ret;
+
+                /* Test decomposition read/write. */
+                if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank,
+                                                  test_type[t], rearranger[r], test_comm)))
+                    return ret;
+
+                /* Run tests. */
+                if ((ret = test_darray_fill(iosysid, ioid, test_type[t], num_flavors, flavor,
+                                            my_rank, test_comm)))
+                    return ret;
+
+                /* Run tests. */
+                if ((ret = test_darray_fill_unlim(iosysid, ioid, test_type[t], num_flavors,
+                                                  flavor, my_rank, test_comm)))
+                    return ret;
+
+                /* Free the PIO decomposition. */
+                if ((ret = PIOc_freedecomp(iosysid, ioid)))
+                    ERR(ret);
+            }
+
+            /* Finalize PIO system. */
+            if ((ret = PIOc_finalize(iosysid)))
+                return ret;
+        } /* next rearranger */
+
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_darray_3d.c b/src/externals/pio2/tests/cunit/test_darray_3d.c
new file mode 100644
index 00000000000..e261c0cec43
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_darray_3d.c
@@ -0,0 +1,422 @@
+/*
+ * Tests for PIO distributed arrays.
+ *
+ * Ed Hartnett, 2/21/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_3d"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 4
+
+/* But sometimes we need arrays of the non-record dimensions. */
+#define NDIM3 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+#define Z_DIM_LEN 4
+
+/* This is the length of the map for each task. */
+#define EXPECTED_MAPLEN 16
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "foo"
+
+/* Test with and without specifying a fill value to
+ * PIOc_write_darray(). */
+#define NUM_TEST_CASES_FILLVALUE 2
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y", "z"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN};
+
+#define DIM_NAME "dim"
+#define NDIM1 1
+
+/* Run test for each of the rearrangers. */
+#define NUM_REARRANGERS_TO_TEST 2
+
+/* Create the decomposition to divide the 4-dimensional sample data
+ * between the 4 tasks. For the purposes of decomposition we are only
+ * concerned with 3 dimensions - we ignore the unlimited dimension.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim_len an array of length 3 with the dimension sizes.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *ioid)
+{
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    int dim_len_3d[NDIM3] = {X_DIM_LEN, Y_DIM_LEN, Z_DIM_LEN};
+    int ret;
+
+    /* How many data elements per task? In this example we will end up
+     * with 4. */
+    elements_per_pe = X_DIM_LEN * Y_DIM_LEN * Z_DIM_LEN / ntasks;
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. */
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i;
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe);
+    if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM3, dim_len_3d, elements_per_pe,
+                                compdof, ioid, 0, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
+/**
+ * Test the darray functionality. Create a netCDF file with 4
+ * dimensions and 1 PIO_INT variable, and use darray to write some
+ * data.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param provide_fill 1 if fillvalue should be provided to PIOc_write_darray().
+ * @returns 0 for success, error code otherwise.
+*/
+int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                int provide_fill)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimids[NDIM];      /* The dimension IDs. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int ncid2;     /* The ncid of the re-opened netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    int ret;       /* Return code. */
+    PIO_Offset arraylen = 16;
+    int int_fillvalue = NC_FILL_INT;
+    void *fillvalue = NULL;
+    int test_data[arraylen];
+    int test_data2[arraylen];
+    int test_data_in[arraylen];
+
+    /* Initialize some data. */
+    for (int f = 0; f < arraylen; f++)
+    {
+        test_data[f] = my_rank * 10 + f;
+        test_data2[f] = 2 * (my_rank * 10 + f);
+    }
+
+    /* Are we providing a fill value? */
+    if (provide_fill)
+        fillvalue = &int_fillvalue;
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create the filename. */
+        sprintf(filename, "data_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename,
+               flavor[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Turn on fill mode. */
+        if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
+            ERR(ret);
+
+        /* Define netCDF dimensions and variable. */
+        printf("%d Defining netCDF metadata...\n", my_rank);
+        for (int d = 0; d < NDIM; d++)
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                ERR(ret);
+
+        /* Define a variable. */
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, dimids, &varid)))
+            ERR(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Set the value of the record dimension. */
+        if ((ret = PIOc_setframe(ncid, varid, 0)))
+            ERR(ret);
+
+        /* Write the data. */
+        if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvalue)))
+            ERR(ret);
+
+        /* Set the value of the record dimension to the second record. */
+        if ((ret = PIOc_setframe(ncid, varid, 1)))
+            ERR(ret);
+
+        /* Write the data for the second record. */
+        if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data2, fillvalue)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Reopen the file. */
+        if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
+            ERR(ret);
+
+        /* Read the data. */
+        if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in)))
+            ERR(ret);
+
+        /* Check the results. */
+        for (int f = 0; f < arraylen; f++)
+            if (test_data_in[f] != test_data[f])
+                return ERR_WRONG;
+
+        /* Set the value of the record dimension to the second record. */
+        if ((ret = PIOc_setframe(ncid2, varid, 1)))
+            ERR(ret);
+
+        /* Read the data. */
+        if ((ret = PIOc_read_darray(ncid2, varid, ioid, arraylen, test_data_in)))
+            ERR(ret);
+
+        /* Check the results. */
+        for (int f = 0; f < arraylen; f++)
+            if (test_data_in[f] != test_data2[f])
+                return ERR_WRONG;
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid2)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/**
+ * Test the decomp read/write functionality.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param rearranger the rearranger to use (PIO_REARR_BOX or
+ * PIO_REARR_SUBSET).
+ * @param test_comm the MPI communicator for this test.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                           int rearranger, MPI_Comm test_comm)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int ioid2;             /* ID for decomposition we will create from file. */
+    char title_in[PIO_MAX_NAME + 1];   /* Optional title. */
+    char history_in[PIO_MAX_NAME + 1]; /* Optional history. */
+    int fortran_order_in; /* Indicates fortran vs. c order. */
+    int ret;              /* Return code. */
+
+    /* Use PIO to create the decomp file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create the filename. */
+        sprintf(filename, "decomp_%s_iotype_%d.nc", TEST_NAME, flavor[fmt]);
+
+        printf("writing decomp file %s\n", filename);
+        if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0)))
+            return ret;
+
+        /* Read the data. */
+        printf("reading decomp file %s\n", filename);
+        if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, PIO_INT,
+                                       title_in, history_in, &fortran_order_in)))
+            return ret;
+
+        /* Check the results. */
+        {
+            iosystem_desc_t *ios;
+            io_desc_t *iodesc;
+
+            /* Get the IO system info. */
+            if (!(ios = pio_get_iosystem_from_id(iosysid)))
+                return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+            /* Get the IO desc, which describes the decomposition. */
+            if (!(iodesc = pio_get_iodesc_from_id(ioid2)))
+                return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+            if (iodesc->ioid != ioid2 || iodesc->maplen != EXPECTED_MAPLEN || iodesc->ndims != NDIM3 ||
+                iodesc->ndof != EXPECTED_MAPLEN)
+                return ERR_WRONG;
+            if (iodesc->rearranger != rearranger || iodesc->maxregions != 1 ||
+                iodesc->needsfill || iodesc->basetype != MPI_INT)
+                return ERR_WRONG;
+            for (int e = 0; e < iodesc->maplen; e++)
+                if (iodesc->map[e] != my_rank * iodesc->maplen + e + 1)
+                    return ERR_WRONG;
+            if (iodesc->dimlen[0] != X_DIM_LEN || iodesc->dimlen[1] != Y_DIM_LEN ||
+                iodesc->dimlen[2] != Z_DIM_LEN)
+                return ERR_WRONG;
+            if (rearranger == PIO_REARR_SUBSET)
+            {
+                if (iodesc->nrecvs != 1  || iodesc->num_aiotasks != TARGET_NTASKS)
+                    return ERR_WRONG;
+            }
+            else
+            {
+                /* I haven't figured out yet what these should be for
+                 * the box rearranger. */
+                printf("iodesc->nrecv = %d iodesc->num_aiotasks = %d\n", iodesc->nrecvs,
+                       iodesc->num_aiotasks);
+            }
+        }
+
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid2)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/**
+ * Run all the tests.
+ *
+ * @param iosysid the IO system ID.
+ * @param num_flavors number of available iotypes in the build.
+ * @param flavor pointer to array of the available iotypes.
+ * @param my_rank rank of this task.
+ * @param rearranger the rearranger to use (PIO_REARR_BOX or
+ * PIO_REARR_SUBSET).
+ * @param test_comm the communicator the test is running on.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
+                    int rearranger, MPI_Comm test_comm)
+{
+    int ioid;
+    int my_test_size;
+    int ret; /* Return code. */
+
+    if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
+        MPIERR(ret);
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition_3d(TARGET_NTASKS, my_rank, iosysid, &ioid)))
+        return ret;
+
+    /* Test decomposition read/write. */
+    if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank,
+                                      rearranger, test_comm)))
+        return ret;
+
+    /* Test with/without providing a fill value to PIOc_write_darray(). */
+    for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++)
+    {
+        /* Run a simple darray test. */
+        if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, provide_fill)))
+            return ret;
+    }
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              MIN_NTASKS, 3, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+        int iosysid;  /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+        int ret;      /* Return code. */
+
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+        {
+            /* Initialize the PIO IO system. This specifies how
+             * many and which processors are involved in I/O. */
+            if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+                                           ioproc_start, rearranger[r], &iosysid)))
+                return ret;
+
+            /* Run tests. */
+            printf("%d Running tests...\n", my_rank);
+            if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank,
+                                       rearranger[r], test_comm)))
+                return ret;
+
+            /* Finalize PIO system. */
+            if ((ret = PIOc_finalize(iosysid)))
+                return ret;
+
+        } /* next rearranger */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_darray_multi.c b/src/externals/pio2/tests/cunit/test_darray_multi.c
new file mode 100644
index 00000000000..206b783e121
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_darray_multi.c
@@ -0,0 +1,473 @@
+/*
+ * Tests for PIO distributed arrays. This program tests the
+ * PIOc_write_darray_multi() function with more than one variable.
+ *
+ * Ed Hartnett, 3/7/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_multi"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* But sometimes we need arrays of the non-record dimensions. */
+#define NDIM2 2
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* Number of variables. */
+#define NVAR 3
+
+/* For attributes. */
+#define NOTE_NAME "note"
+#define NOTE "This is a test file for the PIO library, and may be deleted."
+
+/* Who would have thought? */
+#define TOTAL_NUMBER_OF_STOOGES_NAME "Total_Number_of_Stooges"
+#define TOTAL_NUMBER_OF_STOOGES 6
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"year", "Stooge_popularity", "face_smacks"};
+
+/* The variable names. */
+char var_name[NVAR][PIO_MAX_NAME + 1] = {"Larry", "Curly", "Moe"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/** 
+ * Test the darray functionality. Create a netCDF file with 3
+ * dimensions and 3 variable, and use PIOc_write_darray_multi() to
+ * write one record of data to all three vars at once.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param pio_type the type of the data.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                int pio_type)
+{
+#define NUM_TEST_CASES_WRT_MULTI 2
+#define NUM_TEST_CASES_FILLVALUE 2
+
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimids[NDIM];      /* The dimension IDs. */
+    int ncid;              /* The ncid of the netCDF file. */
+    int ncid2;             /* The ncid of the re-opened netCDF file. */
+    int varid[NVAR];       /* The IDs of the netCDF varables. */
+    int ret;               /* Return code. */
+    PIO_Offset arraylen = 4; /* Amount of data from each task. */
+    void *fillvalue;       /* Pointer to fill value. */
+    void *test_data;       /* Pointer to test data we will write. */
+    void *test_data_in;    /* Pointer to buffer we will read into. */
+
+    /* Default fill value array for each type. */
+    signed char byte_fill[NVAR] = {NC_FILL_BYTE, NC_FILL_BYTE, NC_FILL_BYTE};
+    char char_fill[NVAR] = {NC_FILL_CHAR, NC_FILL_CHAR, NC_FILL_CHAR};
+    short short_fill[NVAR] = {NC_FILL_SHORT, NC_FILL_SHORT, NC_FILL_SHORT};
+    int int_fill[NVAR] = {NC_FILL_INT, NC_FILL_INT, NC_FILL_INT};
+    float float_fill[NVAR] = {NC_FILL_FLOAT, NC_FILL_FLOAT, NC_FILL_FLOAT};
+    double double_fill[NVAR] = {NC_FILL_DOUBLE, NC_FILL_DOUBLE, NC_FILL_DOUBLE};
+#ifdef _NETCDF4
+    unsigned char ubyte_fill[NVAR] = {NC_FILL_UBYTE, NC_FILL_UBYTE, NC_FILL_UBYTE};
+    unsigned short ushort_fill[NVAR] = {NC_FILL_USHORT, NC_FILL_USHORT, NC_FILL_USHORT};
+    unsigned int uint_fill[NVAR] = {NC_FILL_UINT, NC_FILL_UINT, NC_FILL_UINT};
+    long long int64_fill[NVAR] = {NC_FILL_INT64, NC_FILL_INT64, NC_FILL_INT64};
+    unsigned long long uint64_fill[NVAR] = {NC_FILL_UINT64, NC_FILL_UINT64, NC_FILL_UINT64};
+#endif /* _NETCDF4 */
+
+    /* Test data we will write. */
+    signed char test_data_byte[arraylen * NVAR];
+    char test_data_char[arraylen * NVAR];
+    short test_data_short[arraylen * NVAR];
+    int test_data_int[arraylen * NVAR];
+    float test_data_float[arraylen * NVAR];
+    double test_data_double[arraylen * NVAR];
+#ifdef _NETCDF4
+    unsigned char test_data_ubyte[arraylen * NVAR];
+    unsigned short test_data_ushort[arraylen * NVAR];
+    unsigned int test_data_uint[arraylen * NVAR];
+    long long test_data_int64[arraylen * NVAR];
+    unsigned long long test_data_uint64[arraylen * NVAR];
+#endif /* _NETCDF4 */
+
+    /* We will read test data into these buffers. */
+    signed char test_data_byte_in[arraylen];
+    char test_data_char_in[arraylen];
+    short test_data_short_in[arraylen];
+    int test_data_int_in[arraylen];
+    float test_data_float_in[arraylen];
+    double test_data_double_in[arraylen];
+#ifdef _NETCDF4
+    unsigned char test_data_ubyte_in[arraylen];
+    unsigned short test_data_ushort_in[arraylen];
+    unsigned int test_data_uint_in[arraylen];
+    long long test_data_int64_in[arraylen];
+    unsigned long long test_data_uint64_in[arraylen];
+#endif /* _NETCDF4 */
+
+    /* Initialize a big blob of test data for NVAR vars. */
+    for (int f = 0; f < arraylen * NVAR; f++)
+    {
+        test_data_byte[f] = my_rank * 1 + f;
+        test_data_char[f] = my_rank * 2 + f;
+        test_data_short[f] = my_rank * 5 + f;
+        test_data_int[f] = my_rank * 10 + f;
+        test_data_float[f] = my_rank * 10 + f + 0.5;
+        test_data_double[f] = my_rank * 100000 + f + 0.5;
+#ifdef _NETCDF4
+        test_data_ubyte[f] = my_rank * 3 + f;
+        test_data_ushort[f] = my_rank * 9 + f;
+        test_data_uint[f] = my_rank * 100 + f;
+        test_data_int64[f] = my_rank * 10000 + f;
+        test_data_uint64[f] = my_rank * 100000 + f;
+#endif /* _NETCDF4 */
+    }
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* 1-byte types not working with pnetcdf. */
+        if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR))
+            continue;
+
+        /* NetCDF-4 types only work with netCDF-4. */
+        if (pio_type > PIO_DOUBLE && (flavor[fmt] != PIO_IOTYPE_NETCDF4C &&
+                                      flavor[fmt] != PIO_IOTYPE_NETCDF4P))
+            continue;
+
+        /* Add a couple of extra tests for the
+         * PIOc_write_darray_multi() function. */
+        for (int test_multi = 0; test_multi < NUM_TEST_CASES_WRT_MULTI; test_multi++)
+        {
+            /* Test with/without providing a fill value to PIOc_write_darray(). */
+            for (int provide_fill = 0; provide_fill < NUM_TEST_CASES_FILLVALUE; provide_fill++)
+            {
+                /* Create the filename. */
+                sprintf(filename, "data_%s_iotype_%d_pio_type_%d_test_multi_%d_provide_fill_%d.nc", TEST_NAME,
+                        flavor[fmt], pio_type, test_multi, provide_fill);
+
+                /* Select the fill value and data. */
+                switch (pio_type)
+                {
+                case PIO_BYTE:
+                    fillvalue = provide_fill ? byte_fill : NULL;
+                    test_data = test_data_byte;
+                    test_data_in = test_data_byte_in;
+                    break;
+                case PIO_CHAR:
+                    fillvalue = provide_fill ? char_fill : NULL;
+                    test_data = test_data_char;
+                    test_data_in = test_data_char_in;
+                    break;
+                case PIO_SHORT:
+                    fillvalue = provide_fill ? short_fill : NULL;
+                    test_data = test_data_short;
+                    test_data_in = test_data_short_in;
+                    break;
+                case PIO_INT:
+                    fillvalue = provide_fill ? int_fill : NULL;
+                    test_data = test_data_int;
+                    test_data_in = test_data_int_in;
+                    break;
+                case PIO_FLOAT:
+                    fillvalue = provide_fill ? float_fill : NULL;
+                    test_data = test_data_float;
+                    test_data_in = test_data_float_in;
+                    break;
+                case PIO_DOUBLE:
+                    fillvalue = provide_fill ? double_fill : NULL;
+                    test_data = test_data_double;
+                    test_data_in = test_data_double_in;
+                    break;
+#ifdef _NETCDF4
+                case PIO_UBYTE:
+                    fillvalue = provide_fill ? ubyte_fill : NULL;
+                    test_data = test_data_ubyte;
+                    test_data_in = test_data_ubyte_in;
+                    break;
+                case PIO_USHORT:
+                    fillvalue = provide_fill ? ushort_fill : NULL;
+                    test_data = test_data_ushort;
+                    test_data_in = test_data_ushort_in;
+                    break;
+                case PIO_UINT:
+                    fillvalue = provide_fill ? uint_fill : NULL;
+                    test_data = test_data_uint;
+                    test_data_in = test_data_uint_in;
+                    break;
+                case PIO_INT64:
+                    fillvalue = provide_fill ? int64_fill : NULL;
+                    test_data = test_data_int64;
+                    test_data_in = test_data_int64_in;
+                    break;
+                case PIO_UINT64:
+                    fillvalue = provide_fill ? uint64_fill : NULL;
+                    test_data = test_data_uint64;
+                    test_data_in = test_data_uint64_in;
+                    break;
+#endif /* _NETCDF4 */
+                default:
+                    ERR(ERR_WRONG);
+                }
+
+                /* Create the netCDF output file. */
+                printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename,
+                       flavor[fmt], pio_type);
+                if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+                    ERR(ret);
+
+                /* Define netCDF dimensions and variable. */
+                printf("%d Defining netCDF metadata...\n", my_rank);
+                for (int d = 0; d < NDIM; d++)
+                    if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                        ERR(ret);
+
+                /* Define a variable. */
+                for (int v = 0; v < NVAR; v++)
+                    if ((ret = PIOc_def_var(ncid, var_name[v], pio_type, NDIM, dimids, &varid[v])))
+                        ERR(ret);
+
+                /* Leave a note. */
+                if ((ret = PIOc_put_att_text(ncid, NC_GLOBAL, NOTE_NAME, strlen(NOTE), NOTE)))
+                    ERR(ret);
+                int num_stooges = TOTAL_NUMBER_OF_STOOGES;
+                if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, TOTAL_NUMBER_OF_STOOGES_NAME, PIO_INT, 1, &num_stooges)))
+                    ERR(ret);
+
+                /* End define mode. */
+                if ((ret = PIOc_enddef(ncid)))
+                    ERR(ret);
+
+                /* Set the value of the record dimension. */
+                if ((ret = PIOc_setframe(ncid, varid[0], 0)))
+                    ERR(ret);
+
+                int frame[NVAR] = {0, 0, 0};
+                int flushtodisk = test_multi;
+
+                /* Write the data with the _multi function. */
+                if ((ret = PIOc_write_darray_multi(ncid, varid, ioid, NVAR, arraylen, test_data, frame,
+                                                   fillvalue, flushtodisk)))
+                    ERR(ret);
+
+                /* Close the netCDF file. */
+                if ((ret = PIOc_closefile(ncid)))
+                    ERR(ret);
+
+                /* Reopen the file. */
+                if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
+                    ERR(ret);
+
+                /* Now use read_darray on each var in turn and make
+                 * sure we get correct data. */
+                for (int v = 0; v < NVAR; v++)
+                {
+                    /* Read the data. */
+                    if ((ret = PIOc_read_darray(ncid2, varid[v], ioid, arraylen, test_data_in)))
+                        ERR(ret);
+                    
+                    /* Check the results. */
+                    for (int f = 0; f < arraylen; f++)
+                    {
+                        switch (pio_type)
+                        {
+                        case PIO_BYTE:
+                            if (test_data_byte_in[f] != test_data_byte[f + arraylen * v])
+                                return ERR_WRONG;
+                            break;
+                        case PIO_CHAR:
+                            if (test_data_char_in[f] != test_data_char[f + arraylen * v])
+                                return ERR_WRONG;
+                            break;
+                        case PIO_SHORT:
+                            if (test_data_short_in[f] != test_data_short[f + arraylen * v])
+                                return ERR_WRONG;
+                            break;
+                        case PIO_INT:
+                            if (test_data_int_in[f] != test_data_int[f + arraylen * v])
+                                return ERR_WRONG;
+                            break;
+                        case PIO_FLOAT:
+                            if (test_data_float_in[f] != test_data_float[f + arraylen * v])
+                                return ERR_WRONG;
+                            break;
+                        case PIO_DOUBLE:
+                            if (test_data_double_in[f] != test_data_double[f + arraylen * v])
+                                return ERR_WRONG;
+                            break;
+#ifdef _NETCDF4
+                        case PIO_UBYTE:
+                            if (test_data_ubyte_in[f] != test_data_ubyte[f + arraylen * v])
+                                return ERR_WRONG;
+                            break;
+                        case PIO_USHORT:
+                            if (test_data_ushort_in[f] != test_data_ushort[f + arraylen * v])
+                                return ERR_WRONG;
+                            break;
+                        case PIO_UINT:
+                            if (test_data_uint_in[f] != test_data_uint[f + arraylen * v])
+                                return ERR_WRONG;
+                            break;
+                        case PIO_INT64:
+                            if (test_data_int64_in[f] != test_data_int64[f + arraylen * v])
+                                return ERR_WRONG;
+                            break;
+                        case PIO_UINT64:
+                            if (test_data_uint64_in[f] != test_data_uint64[f + arraylen * v])
+                                return ERR_WRONG;
+                            break;
+#endif /* _NETCDF4 */
+                        default:
+                            ERR(ERR_WRONG);
+                        }
+                    }
+                }
+
+                /* Close the netCDF file. */
+                printf("%d Closing the sample data file...\n", my_rank);
+                if ((ret = PIOc_closefile(ncid2)))
+                    ERR(ret);
+            } /* next fillvalue test case */
+        } /* next test multi */
+    } /* next iotype */
+
+    return PIO_NOERR;
+}
+
+/**
+ * Run all the tests. 
+ *
+ * @param iosysid the IO system ID.
+ * @param num_flavors number of available iotypes in the build.
+ * @param flavor pointer to array of the available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the communicator the test is running on.
+ * @returns 0 for success, error code otherwise. 
+ */
+int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
+                    MPI_Comm test_comm)
+{
+#ifdef _NETCDF4
+#define NUM_TYPES_TO_TEST 11
+    int pio_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
+                                       PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64};
+#else
+#define NUM_TYPES_TO_TEST 6
+    int pio_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE};
+#endif /* _NETCDF4 */
+    int ioid;
+    char filename[NC_MAX_NAME + 1];
+    int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
+    int ret; /* Return code. */
+
+    for (int t = 0; t < NUM_TYPES_TO_TEST; t++)
+    {        
+        /* This will be our file name for writing out decompositions. */
+        sprintf(filename, "%s_decomp_rank_%d_flavor_%d_type_%d.nc", TEST_NAME, my_rank,
+                *flavor, pio_type[t]);
+
+        /* Decompose the data over the tasks. */
+        if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d,
+                                           &ioid, pio_type[t])))
+        return ret;
+
+        /* Run a simple darray test. */
+        if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, pio_type[t])))
+            return ret;
+    
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+#define NUM_REARRANGERS_TO_TEST 2
+    int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              MIN_NTASKS, 3, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int iosysid;  /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+        int ret;      /* Return code. */
+        
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+        {
+        /* Initialize the PIO IO system. This specifies how
+         * many and which processors are involved in I/O. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+                                       ioproc_start, rearranger[r], &iosysid)))
+            return ret;
+
+        /* Run tests. */
+        printf("%d Running tests...\n", my_rank);
+        if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm)))
+            return ret;
+
+        /* Finalize PIO system. */
+        if ((ret = PIOc_finalize(iosysid)))
+            return ret;
+        } /* next rearranger */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_darray_multivar.c b/src/externals/pio2/tests/cunit/test_darray_multivar.c
new file mode 100644
index 00000000000..773de9b5f60
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_darray_multivar.c
@@ -0,0 +1,604 @@
+/*
+ * Tests for PIO distributed arrays.
+ *
+ * Ed Hartnett, 2/16/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_multivar"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* But sometimes we need arrays of the non-record dimensions. */
+#define NDIM2 2
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The name of the variables in the netCDF output files. */
+#define VAR_NAME_1 "STICKS"
+#define VAR_NAME_2 "NIX"
+#define VAR_NAME_3 "HICK"
+#define VAR_NAME_4 "PIX"
+
+/* Number of variables in the test file. */
+#define NUM_VAR 4
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* The var names. */
+char var_name[NUM_VAR][PIO_MAX_NAME + 1] = {"STICKS", "NIX", "HICK", "PIX"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/** 
+ * Test the darray functionality. Create a netCDF file with 3
+ * dimensions and 4 variables, and use darray to write to one of them.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param pio_type the type of the data.
+ * @param test_comm the communicator that is running this test.
+ * @param rearranger the rearranger in use for this test.
+ * @param use_fill 1 if fill mode should be set.
+ * @param use_default 1 if default fill values should be used
+ * (ignored if use_fill is 0).
+ * @returns 0 for success, error code otherwise.
+*/
+int test_multivar_darray(int iosysid, int ioid, int num_flavors, int *flavor,
+                         int my_rank, int pio_type, MPI_Comm test_comm,
+                         int rearranger, int use_fill, int use_default)
+{
+#define NUM_MULTIVAR_TEST_CASES 2
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimids[NDIM];     /* The dimension IDs. */
+    int ncid;             /* The ncid of the netCDF file. */
+    int ncid2;            /* The ncid of the re-opened netCDF file. */
+    int varid[NUM_VAR];   /* The IDs of the netCDF varables. */
+    PIO_Offset arraylen = 4;
+    void *fillvalue;
+    void *test_data;
+    void *test_data_in;
+    signed char fillvalue_byte = NC_FILL_BYTE;
+    signed char custom_fillvalue_byte = -TEST_VAL_42;
+    signed char test_data_byte[arraylen];
+    signed char test_data_byte_in[arraylen];
+    char fillvalue_char = NC_FILL_CHAR;
+    char custom_fillvalue_char = -TEST_VAL_42;
+    char test_data_char[arraylen];
+    char test_data_char_in[arraylen];
+    short fillvalue_short = NC_FILL_SHORT;
+    short custom_fillvalue_short = -TEST_VAL_42;
+    short test_data_short[arraylen];
+    short test_data_short_in[arraylen];
+    int fillvalue_int = NC_FILL_INT;
+    int custom_fillvalue_int = -TEST_VAL_42;
+    int test_data_int[arraylen];
+    int test_data_int_in[arraylen];
+    float fillvalue_float = NC_FILL_FLOAT;
+    float custom_fillvalue_float = -TEST_VAL_42;
+    float test_data_float[arraylen];
+    float test_data_float_in[arraylen];
+    double fillvalue_double = NC_FILL_DOUBLE;
+    double custom_fillvalue_double = (-TEST_VAL_42 * 100);
+    double test_data_double[arraylen];
+    double test_data_double_in[arraylen];
+#ifdef _NETCDF4
+    unsigned char fillvalue_ubyte = NC_FILL_UBYTE;
+    unsigned char custom_fillvalue_ubyte = TEST_VAL_42;
+    unsigned char test_data_ubyte[arraylen];
+    unsigned char test_data_ubyte_in[arraylen];
+    unsigned short fillvalue_ushort = NC_FILL_USHORT;
+    unsigned short custom_fillvalue_ushort = (TEST_VAL_42 * 100);
+    unsigned short test_data_ushort[arraylen];
+    unsigned short test_data_ushort_in[arraylen];
+    unsigned int fillvalue_uint = NC_FILL_UINT;
+    unsigned int custom_fillvalue_uint = (TEST_VAL_42 * 100);
+    unsigned int test_data_uint[arraylen];
+    unsigned int test_data_uint_in[arraylen];
+    long long fillvalue_int64 = NC_FILL_INT64;
+    long long custom_fillvalue_int64 = (TEST_VAL_42 * 100);
+    long long test_data_int64[arraylen];
+    long long test_data_int64_in[arraylen];
+    unsigned long long fillvalue_uint64 = NC_FILL_UINT64;
+    unsigned long long custom_fillvalue_uint64 = (TEST_VAL_42 * 100);
+    unsigned long long test_data_uint64[arraylen];
+    unsigned long long test_data_uint64_in[arraylen];
+#endif /* _NETCDF4 */
+    int ret;       /* Return code. */
+
+    /* Initialize some data. */
+    for (int f = 0; f < arraylen; f++)
+    {
+        test_data_byte[f] = my_rank * 10 + f;
+        test_data_char[f] = my_rank * 10 + f;
+        test_data_short[f] = my_rank * 10 + f;
+        test_data_int[f] = my_rank * 10 + f;
+        test_data_float[f] = my_rank * 10 + f + 0.5;
+        test_data_double[f] = my_rank * 100000 + f + 0.5;
+#ifdef _NETCDF4
+        test_data_ubyte[f] = my_rank * 10 + f;        
+        test_data_ushort[f] = my_rank * 10 + f;        
+        test_data_uint[f] = my_rank * 10 + f;        
+        test_data_int64[f] = my_rank * 10 + f;        
+        test_data_uint64[f] = my_rank * 10 + f;        
+#endif /* _NETCDF4 */
+    }
+
+    /* Select the fill value and data. */
+    switch (pio_type)
+    {
+    case PIO_BYTE:
+        fillvalue = use_default ? &fillvalue_byte : &custom_fillvalue_byte;
+        test_data = test_data_byte;
+        test_data_in = test_data_byte_in;
+        break;
+    case PIO_CHAR:
+        fillvalue = use_default ? &fillvalue_char : &custom_fillvalue_char;
+        test_data = test_data_char;
+        test_data_in = test_data_char_in;
+        break;
+    case PIO_SHORT:
+        fillvalue = use_default ? &fillvalue_short : &custom_fillvalue_short;
+        test_data = test_data_short;
+        test_data_in = test_data_short_in;
+        break;
+    case PIO_INT:
+        fillvalue = use_default ? &fillvalue_int : &custom_fillvalue_int;
+        test_data = test_data_int;
+        test_data_in = test_data_int_in;
+        break;
+    case PIO_FLOAT:
+        fillvalue = use_default ? &fillvalue_float : &custom_fillvalue_float;
+        test_data = test_data_float;
+        test_data_in = test_data_float_in;
+        break;
+    case PIO_DOUBLE:
+        fillvalue = use_default ? &fillvalue_double : &custom_fillvalue_double;
+        test_data = test_data_double;
+        test_data_in = test_data_double_in;
+        break;
+#ifdef _NETCDF4
+    case PIO_UBYTE:
+        fillvalue = use_default ? &fillvalue_ubyte : &custom_fillvalue_ubyte;
+        test_data = test_data_ubyte;
+        test_data_in = test_data_ubyte_in;
+        break;
+    case PIO_USHORT:
+        fillvalue = use_default ? &fillvalue_ushort : &custom_fillvalue_ushort;
+        test_data = test_data_ushort;
+        test_data_in = test_data_ushort_in;
+        break;
+    case PIO_UINT:
+        fillvalue = use_default ? &fillvalue_uint : &custom_fillvalue_uint;
+        test_data = test_data_uint;
+        test_data_in = test_data_uint_in;
+        break;
+    case PIO_INT64:
+        fillvalue = use_default ? &fillvalue_int64 : &custom_fillvalue_int64;
+        test_data = test_data_int64;
+        test_data_in = test_data_int64_in;
+        break;
+    case PIO_UINT64:
+        fillvalue = use_default ? &fillvalue_uint64 : &custom_fillvalue_uint64;
+        test_data = test_data_uint64;
+        test_data_in = test_data_uint64_in;
+        break;
+#endif /* _NETCDF4 */
+    default:
+        ERR(ERR_WRONG);
+    }
+
+    /* Try in pnetcdf only. This code demonstrates that pnetcdf fill
+     * values do work. But not for PIO, so we have a bug somewhere. */
+    {
+#ifdef _PNETCDF
+        int ncid;
+        int varid;
+        int dimid;
+        char test_filename[] = "pnetcdf_test.nc";
+        int ret;
+
+        if ((ret = ncmpi_create(test_comm, test_filename, NC_CLOBBER, MPI_INFO_NULL, &ncid)))
+            return ret;
+        if ((ret = ncmpi_set_fill(ncid, NC_FILL, NULL)))
+            return ret;            
+        if ((ret = ncmpi_def_dim(ncid, "dim_name", 5, &dimid)))
+            return ret;
+        if ((ret = ncmpi_def_var(ncid, "dim_name", NC_INT, 1, &dimid, &varid)))
+            return ret;
+        if ((ret = ncmpi_enddef(ncid)))
+            return ret;
+        if ((ret = ncmpi_close(ncid)))
+            return ret;
+
+        /* Reopen and check. */
+        if ((ret = ncmpi_open(test_comm, test_filename, NC_NOWRITE, MPI_INFO_NULL, &ncid)))
+            return ret;
+        int datum;
+        MPI_Offset start[1] = {0};
+        ret = ncmpi_get_var1_int(ncid, varid, start, &datum);
+        printf("datum ret = %d\n", ret);
+
+        /* Not sure why this doesn't work. */
+        /* if ((ret = ncmpi_get_var1_int(ncid, varid, start, &datum))) */
+        /*     return ret; */
+        printf("datum = %d\n", datum);
+        if ((ret = ncmpi_close(ncid)))
+            return ret;
+#endif /* _PNETCDF */
+    }
+
+    for (int tc = 0; tc < NUM_MULTIVAR_TEST_CASES; tc++)
+    {
+        /* Use PIO to create the example file in each of the four
+         * available ways. */
+        for (int fmt = 0; fmt < num_flavors; fmt++)
+        {
+            /* BYTE and CHAR don't work with pnetcdf. Don't know why yet. */
+            if (flavor[fmt] == PIO_IOTYPE_PNETCDF && (pio_type == PIO_BYTE || pio_type == PIO_CHAR))
+                continue;
+            
+            /* NetCDF-4 types only work with netCDF-4 formats. */
+            printf("pio_type = %d flavor[fmt] = %d\n", pio_type, flavor[fmt]);
+            if (pio_type > PIO_DOUBLE && flavor[fmt] != PIO_IOTYPE_NETCDF4C &&
+                flavor[fmt] != PIO_IOTYPE_NETCDF4P)
+                continue;
+            
+            /* Create the filename. */
+            sprintf(filename, "data_%s_iotype_%d_tc_%d_pio_type_%d_use_fill_%d_default_fill_%d.nc",
+                    TEST_NAME, flavor[fmt], tc, pio_type, use_fill, use_default);
+            
+            /* Create the netCDF output file. */
+            printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename,
+                   flavor[fmt], pio_type);
+            if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+                ERR(ret);
+
+            /* Turn on fill mode if desired. */
+            if (use_fill)
+                if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
+                    ERR(ret);
+
+            /* Define netCDF dimensions and variable. */
+            printf("%d Defining netCDF metadata...\n", my_rank);
+            for (int d = 0; d < NDIM; d++)
+                if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                    ERR(ret);
+
+            /* Define the variables. */
+            for (int v = 0; v < NUM_VAR; v++)
+            {
+                if ((ret = PIOc_def_var(ncid, var_name[v], pio_type, NDIM, dimids, &varid[v])))
+                    ERR(ret);
+                if (use_fill && !use_default)
+                    if ((ret = PIOc_def_var_fill(ncid, varid[v], NC_FILL, fillvalue)))
+                        ERR(ret);
+            }
+
+            /* End define mode. */
+            if ((ret = PIOc_enddef(ncid)))
+                ERR(ret);
+
+            /* Set the value of the record dimension. */
+            for (int v = 0; v < NUM_VAR; v++)
+            {
+                if ((ret = PIOc_setframe(ncid, varid[v], 0)))
+                    ERR(ret);
+
+                /* For the first test case we just write the first variable. */
+                if (tc == 0)
+                    break;
+            }
+
+            /* Write the data. */
+            for (int v = 0; v < NUM_VAR; v++)
+            {
+                if ((ret = PIOc_write_darray(ncid, varid[v], ioid, arraylen, test_data, fillvalue)))
+                    ERR(ret);
+
+                /* For the first test case we just write the first variable. */
+                if (tc == 0)
+                    break;
+            }
+
+            /* Close the netCDF file. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+
+            /* Reopen the file. */
+            if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
+                ERR(ret);
+
+            for (int v = 0; v < NUM_VAR; v++)
+            {
+                /* Read the data. */
+                if ((ret = PIOc_read_darray(ncid2, varid[0], ioid, arraylen, test_data_in)))
+                    ERR(ret);
+                
+                /* Check the results. */
+                for (int f = 0; f < arraylen; f++)
+                {
+                    switch (pio_type)
+                    {
+                    case PIO_BYTE:
+                        if (test_data_byte_in[f] != test_data_byte[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_CHAR:
+                        if (test_data_char_in[f] != test_data_char[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_SHORT:
+                        if (test_data_short_in[f] != test_data_short[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_INT:
+                        if (test_data_int_in[f] != test_data_int[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_FLOAT:
+                        if (test_data_float_in[f] != test_data_float[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_DOUBLE:
+                        if (test_data_double_in[f] != test_data_double[f])
+                            return ERR_WRONG;
+                        break;
+#ifdef _NETCDF4
+                    case PIO_UBYTE:
+                        if (test_data_ubyte_in[f] != test_data_ubyte[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_USHORT:
+                        if (test_data_ushort_in[f] != test_data_ushort[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_UINT:
+                        if (test_data_uint_in[f] != test_data_uint[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_INT64:
+                        if (test_data_int64_in[f] != test_data_int64[f])
+                            return ERR_WRONG;
+                        break;
+                    case PIO_UINT64:
+                        if (test_data_uint64_in[f] != test_data_uint64[f])
+                            return ERR_WRONG;
+                        break;
+#endif /* _NETCDF4 */
+                    default:
+                        ERR(ERR_WRONG);
+                    }
+                }
+
+                /* For the first test case we just write the first variable. */
+                if (tc == 0)
+                    break;
+            } /* next var */
+
+            /* For the first test case, if fill mode is in use the other vars
+             * should have fill values. */
+            if (tc == 0 && use_fill && flavor[fmt] != PIO_IOTYPE_PNETCDF)
+            {
+                /* Read the data. */
+                if ((ret = PIOc_read_darray(ncid2, varid[1], ioid, arraylen, test_data_in)))
+                    ERR(ret);
+            
+                /* Check the results. */
+                for (int f = 0; f < arraylen; f++)
+                {
+                    switch (pio_type)
+                    {
+                    case PIO_BYTE:
+                        if (test_data_byte_in[f] != (use_default ? NC_FILL_BYTE : custom_fillvalue_byte))
+                            return ERR_WRONG;
+                        break;
+                    case PIO_CHAR:
+                        if (test_data_char_in[f] != (use_default ? NC_FILL_CHAR : custom_fillvalue_char))
+                            return ERR_WRONG;
+                        break;
+                    case PIO_SHORT:
+                        if (test_data_short_in[f] != (use_default ? NC_FILL_SHORT : custom_fillvalue_short))
+                            return ERR_WRONG;
+                        break;
+                    case PIO_INT:
+                        if (test_data_int_in[f] != (use_default ? NC_FILL_INT : custom_fillvalue_int))
+                            return ERR_WRONG;
+                        break;
+                    case PIO_FLOAT:
+                        if (test_data_float_in[f] != (use_default ? NC_FILL_FLOAT : custom_fillvalue_float))
+                            return ERR_WRONG;
+                        break;
+                    case PIO_DOUBLE:
+                        if (test_data_double_in[f] != (use_default ? NC_FILL_DOUBLE : custom_fillvalue_double))
+                            return ERR_WRONG;
+                        break;
+#ifdef _NETCDF4
+                    case PIO_UBYTE:
+                        if (test_data_ubyte_in[f] != (use_default ? NC_FILL_UBYTE : custom_fillvalue_ubyte))
+                            return ERR_WRONG;
+                        break;
+                    case PIO_USHORT:
+                        if (test_data_ushort_in[f] != (use_default ? NC_FILL_USHORT : custom_fillvalue_ushort))
+                            return ERR_WRONG;
+                        break;
+                    case PIO_UINT:
+                        if (test_data_uint_in[f] != (use_default ? NC_FILL_UINT : custom_fillvalue_uint))
+                            return ERR_WRONG;
+                        break;
+                    case PIO_INT64:
+                        if (test_data_int64_in[f] != (use_default ? NC_FILL_INT64 : custom_fillvalue_int64))
+                            return ERR_WRONG;
+                        break;
+                    case PIO_UINT64:
+                        if (test_data_uint64_in[f] != (use_default ? NC_FILL_UINT64 : custom_fillvalue_uint64))
+                            return ERR_WRONG;
+                        break;
+#endif /* _NETCDF4 */
+                    default:
+                        ERR(ERR_WRONG);
+                    }
+                }
+            }
+
+            /* Close the netCDF file. */
+            printf("%d Closing the sample data file...\n", my_rank);
+            if ((ret = PIOc_closefile(ncid2)))
+                ERR(ret);
+        }
+    } /* next test case */
+    
+    return PIO_NOERR;
+}
+
+/**
+ * Run all the tests. 
+ *
+ * @param iosysid the IO system ID.
+ * @param num_flavors number of available iotypes in the build.
+ * @param flavor pointer to array of the available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the communicator the test is running on.
+ * @param rearranger the rearranger in use in this test.
+ * @returns 0 for success, error code otherwise. 
+ */
+int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
+                    MPI_Comm test_comm, int rearranger)
+{
+#define NUM_FILL_TESTS 3
+#ifdef _NETCDF4
+#define NUM_TYPES_TO_TEST 11
+    int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
+                                        PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64};
+#else
+#define NUM_TYPES_TO_TEST 6
+    int test_type[NUM_TYPES_TO_TEST] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE};
+#endif /* _NETCDF4 */
+    int ioid;
+    int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
+    int ret; /* Return code. */
+
+    for (int t = 0; t < NUM_TYPES_TO_TEST; t++)
+    {        
+        int use_fill = 0;
+        int use_default = 0;
+        
+        /* Decompose the data over the tasks. */
+        if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d,
+                                           &ioid, test_type[t])))
+        return ret;
+
+        /* Run the different combinations of use_fill and use_default. */
+        for (int f = 0; f < NUM_FILL_TESTS; f++)
+        {
+            /* Set flags for this test case. */
+            if (f == 1)
+                use_fill++;
+            if (f == 2)
+                use_default++;
+            
+            /* Run the multivar darray tests. */
+            if ((ret = test_multivar_darray(iosysid, ioid, num_flavors, flavor, my_rank, test_type[t],
+                                            test_comm, rearranger, use_fill, use_default)))
+                return ret;
+        }
+        
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+#define NUM_REARRANGERS 2
+    int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+    int my_rank;
+    int ntasks;
+    int num_flavors;         /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm;      /* A communicator for this test. */
+    int ret;                 /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, MIN_NTASKS,
+                              3, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int iosysid;              /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+        int ret;                  /* Return code. */
+        
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        /* Test for both arrangers. */
+        for (int r = 0; r < NUM_REARRANGERS; r++)
+        {
+
+            /* Initialize the PIO IO system. This specifies how
+             * many and which processors are involved in I/O. */
+            if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+                                           ioproc_start, rearranger[r], &iosysid)))
+                return ret;
+            
+            /* Run tests. */
+            printf("%d Running tests...\n", my_rank);
+            if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm,
+                                       rearranger[r])))
+                return ret;
+            
+            /* Finalize PIO system. */
+            if ((ret = PIOc_finalize(iosysid)))
+                return ret;
+        }
+
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_darray_multivar2.c b/src/externals/pio2/tests/cunit/test_darray_multivar2.c
new file mode 100644
index 00000000000..b25ba890ab7
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_darray_multivar2.c
@@ -0,0 +1,287 @@
+/*
+ * Tests for PIO distributed arrays.
+ *
+ * Ed Hartnett, Jim Edwards, 4/20/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_multivar2"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* But sometimes we need arrays of the non-record dimensions. */
+#define NDIM2 2
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* Number of variables in the test file. */
+#define NUM_VAR 2
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* The var names. */
+char var_name[NUM_VAR][PIO_MAX_NAME + 1] = {"Aubery", "Martin"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/**
+ * Test the darray functionality. Create a netCDF file with 3
+ * dimensions and 2 variables. One of the vars uses the record
+ * dimension, the other does not. Then use darray to write to them.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param pio_type the type of the data.
+ * @param test_comm the communicator that is running this test.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_multivar_darray(int iosysid, int ioid, int num_flavors, int *flavor,
+                         int my_rank, int pio_type, MPI_Comm test_comm)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dimids[NDIM];     /* The dimension IDs. */
+    int ncid;             /* The ncid of the netCDF file. */
+    int varid[NUM_VAR];   /* The IDs of the netCDF varables. */
+    PIO_Offset arraylen = 4;
+    int custom_fillvalue_int = -TEST_VAL_42;
+    int test_data_int[arraylen];
+    int ret;       /* Return code. */
+
+    /* Initialize some data. */
+    for (int f = 0; f < arraylen; f++)
+        test_data_int[f] = my_rank * 10 + f;
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create the filename. */
+        sprintf(filename, "data_%s_iotype_%d_pio_type_%d.nc", TEST_NAME, flavor[fmt], pio_type);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d type %d\n", my_rank, filename,
+               flavor[fmt], pio_type);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Define netCDF dimensions and variable. */
+        printf("%d Defining netCDF metadata...\n", my_rank);
+        for (int d = 0; d < NDIM; d++)
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                ERR(ret);
+
+        /* Var 0 does not have a record dim, varid 1 is a record var. */
+        if ((ret = PIOc_def_var(ncid, var_name[0], pio_type, NDIM - 1, &dimids[1], &varid[0])))
+            ERR(ret);
+        if ((ret = PIOc_def_var(ncid, var_name[1], pio_type, NDIM, dimids, &varid[1])))
+            ERR(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Set the value of the record dimension for varid 1. */
+        if ((ret = PIOc_setframe(ncid, varid[1], 0)))
+            ERR(ret);
+
+        /* Write the data. */
+        for (int v = 0; v < NUM_VAR; v++)
+            if ((ret = PIOc_write_darray(ncid, varid[v], ioid, arraylen, test_data_int, &custom_fillvalue_int)))
+                ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Check the file contents. */
+        {
+            int ncid2;            /* The ncid of the re-opened netCDF file. */
+            int test_data_int_in[arraylen];
+                        
+            /* Reopen the file. */
+            if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[fmt], filename, PIO_NOWRITE)))
+                ERR(ret);
+            
+            for (int v = 0; v < NUM_VAR; v++)
+            {
+                /* Read the data. */
+                if ((ret = PIOc_read_darray(ncid2, varid[v], ioid, arraylen, test_data_int_in)))
+                    ERR(ret);
+                
+                /* Check the results. */
+                for (int f = 0; f < arraylen; f++)
+                    if (test_data_int_in[f] != test_data_int[f])
+                        return ERR_WRONG;
+            } /* next var */
+            
+            /* Close the netCDF file. */
+            if ((ret = PIOc_closefile(ncid2)))
+                ERR(ret);
+        }
+    }
+
+    return PIO_NOERR;
+}
+
+/* Create the decomposition to divide the 3-dimensional sample data
+ * between the 4 tasks. For the purposes of decomposition we are only
+ * concerned with 2 dimensions - we ignore the unlimited dimension.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim_len_2d an array of length 2 with the dim lengths.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @param pio_type the data type to use for the decomposition.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition_2d_2(int ntasks, int my_rank, int iosysid, int *dim_len_2d,
+                              int *ioid, int pio_type)
+{
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    int ret;
+
+    /* How many data elements per task? In this example we will end up
+     * with 4. */
+    elements_per_pe = dim_len_2d[0] * dim_len_2d[1] / ntasks;
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. This is a 1-based array, so add 1! */
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i + 1;
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe);
+    if ((ret = PIOc_InitDecomp(iosysid, pio_type, NDIM2, dim_len_2d, elements_per_pe,
+                               compdof, ioid, NULL, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
+/**
+ * Run all the tests.
+ *
+ * @param iosysid the IO system ID.
+ * @param num_flavors number of available iotypes in the build.
+ * @param flavor pointer to array of the available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the communicator the test is running on.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_all_darray(int iosysid, int num_flavors, int *flavor, int my_rank,
+                    MPI_Comm test_comm)
+{
+    int ioid;
+    int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
+    int ret; /* Return code. */
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition_2d_2(TARGET_NTASKS, my_rank, iosysid, dim_len_2d,
+                                         &ioid, PIO_INT)))
+        return ret;
+    
+    /* Run the multivar darray tests. */
+    if ((ret = test_multivar_darray(iosysid, ioid, num_flavors, flavor, my_rank, PIO_INT,
+                                    test_comm)))
+        return ret;
+    
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+    int my_rank;
+    int ntasks;
+    int num_flavors;         /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm;      /* A communicator for this test. */
+    int ret;                 /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS, MIN_NTASKS,
+                              3, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        int iosysid;              /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+        int ret;                  /* Return code. */
+
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        /* Initialize the PIO IO system. This specifies how
+         * many and which processors are involved in I/O. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+                                       ioproc_start, PIO_REARR_BOX, &iosysid)))
+            return ret;
+
+        /* Run tests. */
+        printf("%d Running tests...\n", my_rank);
+        if ((ret = test_all_darray(iosysid, num_flavors, flavor, my_rank, test_comm)))
+            return ret;
+        
+        /* Finalize PIO system. */
+        if ((ret = PIOc_finalize(iosysid)))
+            return ret;
+
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_decomp_uneven.c b/src/externals/pio2/tests/cunit/test_decomp_uneven.c
new file mode 100644
index 00000000000..b6d47004eff
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_decomp_uneven.c
@@ -0,0 +1,380 @@
+/*
+ * Tests for PIO distributed arrays. This tests cases when arrays do
+ * not distribute evenly over the processors.
+ *
+ * Ed Hartnett, 3/6/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_darray_uneven"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* This is for 3D data decompositions. */
+#define NDIM3 3
+
+/* Create the decomposition to divide the 4-dimensional sample data
+ * between the 4 tasks. For the purposes of decomposition we are only
+ * concerned with 3 dimensions - we ignore the unlimited dimension.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim_len an array of length 3 with the dimension sizes.
+ * @param pio_type the type for this decomposition.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * lengths.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition_3d(int ntasks, int my_rank, int iosysid, int *dim_len,
+                            int pio_type, int *ioid)
+{
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset remainder;     /* Left over array elements. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    PIO_Offset data_size = 1;
+    int ret;
+
+    /* How many data elements per task? In this example we will end up
+     * with 4. */
+    for (int d = 0; d < NDIM3; d++)
+        data_size *= dim_len[d];
+    elements_per_pe = data_size / ntasks;
+    remainder = data_size % ntasks;
+
+    /* Distribute the remaining elements. */
+    if (my_rank < remainder)
+        elements_per_pe++;
+    printf("%d elements_per_pe = %lld remainder = %lld\n", my_rank, elements_per_pe, remainder);
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. */
+    for (int i = 0; i < elements_per_pe; i++)
+    {
+        int my_remainder = 0;
+        if (my_rank >= remainder)
+            my_remainder = remainder;
+        compdof[i] = my_rank * elements_per_pe + i + my_remainder;
+        printf("%d my_remainder = %d compdof[%d] = %lld\n", my_rank, i, my_remainder, compdof[i]);
+    }
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe);
+    if ((ret = PIOc_init_decomp(iosysid, pio_type, NDIM3, dim_len, elements_per_pe,
+                                compdof, ioid, 0, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
+/**
+ * Test the decomp read/write functionality. Given an ioid for a 3D
+ * decomposition, this function will write a decomp file, then read it
+ * in to ensure the correct values are read.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param rearranger the rearranger to use (PIO_REARR_BOX or
+ * PIO_REARR_SUBSET).
+ * @param test_comm the MPI communicator for this test.
+ * @param dim_len array of length 3 with dim lengths.
+ * @param expected_maplen pointer to array of length TARGET_NTASKS
+ * with the maplen we expect to get for each of the tasks running this
+ * test.
+ * @param pio_type the type we expect to be associated with
+ * this decomposition.
+ * @param full_maplen the length of the full map.
+ * @param pointer to expected map, an array of TARGET_NTASKS *
+ * max_maplen.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                           int rearranger, MPI_Comm test_comm, int *dim_len, int *expected_maplen,
+                           int pio_type, int fill_maplen, int *expected_map)
+{
+#define TEST_DECOMP_TITLE "Decomposition data for test_darray_uneven.c in PIO library."
+#define TEST_DECOMP_HISTORY "This file may be deleted; it is for test purposes only: "
+    int ioid2;                         /* ID for decomp we read into. */
+    char filename[PIO_MAX_NAME + 1];   /* Name for the output files. */
+    char title[] = TEST_DECOMP_TITLE;
+    char history[PIO_MAX_NAME + 1] = TEST_DECOMP_HISTORY;
+    char title_in[PIO_MAX_NAME + 1];
+    char history_in[PIO_MAX_NAME + 1];
+    int fortran_order_in; /* Indicates fortran vs. c order. */
+    int ret;              /* Return code. */
+
+    /* Use PIO to create the decomp file in one of the four
+     * available ways. */
+    for (int fmt = 0; fmt < 1; fmt++)
+    {
+        /* Create the filename. */
+        sprintf(filename, "decomp_%s_pio_type_%d_dims_%d_x_%d_x_%d.nc", TEST_NAME, pio_type,
+                dim_len[0], dim_len[1], dim_len[2]);
+
+        /* Create history string. */
+        strncat(history, filename, NC_MAX_NAME - strlen(TEST_DECOMP_HISTORY));
+
+        printf("writing decomp file %s\n", filename);
+        if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, title, history, 0)))
+            return ret;
+        printf("about to check map with netCDF\n");
+
+        /* Open the decomposition file with netCDF. */
+        int ncid_in;
+        int iotype = PIO_IOTYPE_NETCDF;
+        if ((ret = PIOc_openfile(iosysid, &ncid_in, &iotype, filename, NC_NOWRITE)))
+            return ret;
+
+        /* Get the max maplen. */
+        int max_maplen;
+        if ((ret = PIOc_get_att_int(ncid_in, NC_GLOBAL, DECOMP_MAX_MAPLEN_ATT_NAME, &max_maplen)))
+            return ret;
+        printf("max_maplen = %d\n", max_maplen);
+
+        /* Check dims. */
+        PIO_Offset ndims_in;
+        if ((ret = PIOc_inq_dim(ncid_in, 0, NULL, &ndims_in)))
+            return ret;
+        if (ndims_in != NDIM3)
+            return ERR_WRONG;
+        PIO_Offset ntasks_in;
+        if ((ret = PIOc_inq_dim(ncid_in, 1, NULL, &ntasks_in)))
+            return ret;
+        if (ntasks_in != TARGET_NTASKS)
+            return ERR_WRONG;
+
+        /* Check the maplen. */
+        int maplen_varid;
+        int maplen_in[TARGET_NTASKS];
+        if ((ret = PIOc_inq_varid(ncid_in, DECOMP_MAPLEN_VAR_NAME, &maplen_varid)))
+            return ret;
+        if ((ret = PIOc_get_var(ncid_in, maplen_varid, &maplen_in)))
+            return ret;
+        for (int t = 0; t < TARGET_NTASKS; t++)
+        {
+            printf("%d maplen_in[%d] = %d expected_maplen[%d] = %d\n", my_rank, t, maplen_in[t], t, expected_maplen[t]);
+            if (maplen_in[t] != expected_maplen[t])
+                return ERR_WRONG;
+        }
+
+        /* Check the map. */
+        int map_varid;
+        int map_in[TARGET_NTASKS][max_maplen];
+        if ((ret = PIOc_inq_varid(ncid_in, DECOMP_MAP_VAR_NAME, &map_varid)))
+            return ret;
+        if ((ret = PIOc_get_var(ncid_in, map_varid, (int *)&map_in)))
+            return ret;
+        printf("about to check map\n");
+        for (int t = 0; t < TARGET_NTASKS; t++)
+        {
+            for (int e = 0; e < max_maplen; e++)
+            {
+                printf("%d t = %d e = %d map_in[t][e] = %d expected_map[t * max_maplen + e] = %d\n",
+                       my_rank, t, e, map_in[t][e], expected_map[t * max_maplen + e]);
+                if (map_in[t][e] != expected_map[t * max_maplen + e])
+                    return ERR_WRONG;
+            }
+        }
+
+        /* Close the decomposition file. */
+        if ((ret = PIOc_closefile(ncid_in)))
+            return ret;
+
+        /* Read the decomposition file into PIO. */
+        printf("reading decomp file %s\n", filename);
+        if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, pio_type,
+                                       title_in, history_in, &fortran_order_in)))
+            return ret;
+
+        /* Check the results. */
+        {
+            iosystem_desc_t *ios;
+            io_desc_t *iodesc;
+
+            /* Get the IO system info. */
+            if (!(ios = pio_get_iosystem_from_id(iosysid)))
+                return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+            /* Get the IO desc, which describes the decomposition. */
+            if (!(iodesc = pio_get_iodesc_from_id(ioid2)))
+                return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+            /* We need to find the MPI type we will expect to see in
+             * iodesc. */
+            MPI_Datatype expected_mpi_type;
+            if ((ret = find_mpi_type(pio_type, &expected_mpi_type, NULL)))
+                return ret;
+
+            /* Check values in iodesc. */
+            printf("ioid2 = %d iodesc->ioid = %d iodesc->maplen = %d iodesc->ndims = %d "
+                   "iodesc->ndof = %d iodesc->rearranger = %d iodesc->maxregions = %d "
+                   "iodesc->needsfill = %d iodesc->basetype = %d expected_mpi_type = %d\n",
+                   ioid2, iodesc->ioid, iodesc->maplen, iodesc->ndims, iodesc->ndof,
+                   iodesc->rearranger, iodesc->maxregions, iodesc->needsfill, iodesc->basetype,
+                   expected_mpi_type);
+            if (strcmp(title, title_in) || strcmp(history, history_in))
+                return ERR_WRONG;
+            if (iodesc->ioid != ioid2 || iodesc->rearranger != rearranger ||
+                iodesc->basetype != expected_mpi_type)
+                return ERR_WRONG;
+            if (iodesc->ndims != NDIM3)
+                return ERR_WRONG;
+            if (iodesc->maplen != expected_maplen[my_rank])
+                return ERR_WRONG;
+            if (iodesc->ndims != NDIM3 || iodesc->ndof != expected_maplen[my_rank])
+                return ERR_WRONG;
+            if (iodesc->needsfill)
+                return ERR_WRONG;
+            /* Don't forget to add 1! */
+            for (int e = 0; e < iodesc->maplen; e++)
+            {
+                printf("%d e = %d max_maplen = %d iodesc->map[e] = %lld expected_map[my_rank * max_maplen + e] = %d\n",
+                       my_rank, e, max_maplen, iodesc->map[e], expected_map[my_rank * max_maplen + e]);
+                if (iodesc->map[e] != expected_map[my_rank * max_maplen + e] + 1)
+                    return ERR_WRONG;
+            }
+            for (int d = 0; d < NDIM3; d++)
+                if (iodesc->dimlen[d] != dim_len[d])
+                    return ERR_WRONG;
+        }
+
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid2)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/* Run tests for darray functions. */
+int main(int argc, char **argv)
+{
+/* #define NUM_TYPES_TO_TEST 3 */
+/*     int test_type[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT, PIO_DOUBLE}; */
+#define NUM_TYPES_TO_TEST 1
+    int test_type[NUM_TYPES_TO_TEST] = {PIO_INT};
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              MIN_NTASKS, 3, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+#define NUM_REARRANGERS_TO_TEST 2
+        int rearranger[NUM_REARRANGERS_TO_TEST] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+        int iosysid;  /* The ID for the parallel I/O system. */
+        int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+        int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+        int map_1x4x4[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
+        int map_2x4x4[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+                           16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31};
+        int map_3x4x4[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+                           24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47};
+        int map_1x3x3[] = {0, 1, 2, 3, 4, PIO_FILL_INT, 5, 6, PIO_FILL_INT, 7, 8, PIO_FILL_INT};
+        int map_1x2x3[] = {0, 1, 2, 3, 4, PIO_FILL_INT, 5, PIO_FILL_INT};
+#define NUM_DIM_COMBOS_TO_TEST 5
+        int dim_len[NUM_DIM_COMBOS_TO_TEST][NDIM3] = {{1, 4, 4},
+                                                      {2, 4, 4},                                                      
+                                                      {3, 4, 4},
+                                                      {1, 3, 3},
+                                                      {1, 2, 3}};
+        int expected_maplen[NUM_DIM_COMBOS_TO_TEST][TARGET_NTASKS] = {{4, 4, 4, 4},
+                                                                      {8, 8, 8, 8},
+                                                                      {12, 12, 12, 12},
+                                                                      {3, 2, 2, 2},
+                                                                      {2, 2, 1, 1}};
+        int *expected_map[NUM_DIM_COMBOS_TO_TEST] = {map_1x4x4, map_2x4x4, map_3x4x4, map_1x3x3, map_1x2x3};
+        int ret;      /* Return code. */
+
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        for (int r = 0; r < NUM_REARRANGERS_TO_TEST; r++)
+        {
+            int ioid; /* Decomposition ID. */
+            
+            /* Initialize the PIO IO system. This specifies how
+             * many and which processors are involved in I/O. */
+            if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, ioproc_stride,
+                                           ioproc_start, rearranger[r], &iosysid)))
+                return ret;
+
+            /* Run tests for each data type. */
+            for (int t = 0; t < NUM_TYPES_TO_TEST; t++)
+            {
+                for (int dc = 0; dc < NUM_DIM_COMBOS_TO_TEST; dc++)
+                {
+                    /* What is length of map for this combo? */
+                    int full_maplen = 1;
+                    for (int d = 0; d < NDIM3; d++)
+                        full_maplen *= dim_len[dc][d];
+                    
+                    /* Decompose the data over the tasks. */
+                    if ((ret = create_decomposition_3d(TARGET_NTASKS, my_rank, iosysid, dim_len[dc],
+                                                       test_type[t], &ioid)))
+                        return ret;
+                    
+                    /* Test decomposition read/write. */
+                    if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank,
+                                                      rearranger[r], test_comm, dim_len[dc],
+                                                      expected_maplen[dc], test_type[t], full_maplen,
+                                                      expected_map[dc])))
+                        return ret;
+                    
+                    /* Free the PIO decomposition. */
+                    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+                        ERR(ret);
+                }
+            }
+            
+            /* Finalize PIO system. */
+            if ((ret = PIOc_finalize(iosysid)))
+                return ret;
+
+        } /* next rearranger */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_decomps.c b/src/externals/pio2/tests/cunit/test_decomps.c
new file mode 100644
index 00000000000..96dc1a386e2
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_decomps.c
@@ -0,0 +1,420 @@
+/*
+ * Tests for PIO data decompositons.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_decomps"
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* In decompositions we specify with 1 less dimension. */
+#define NDIM2 2
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 1
+
+/* Files of decompositions. */
+#define DECOMP_FILE "decomp.txt"
+#define DECOMP_BC_FILE "decomp.txt"
+
+/* Used when initializing PIO. */
+#define STRIDE1 1
+#define STRIDE2 2
+#define BASE0 0
+#define BASE1 1
+#define NUM_IO1 1
+#define NUM_IO2 2
+#define NUM_IO4 4
+#define REARRANGER 2
+
+/**
+ * Test some decomposition functions. 
+ *
+ * @param iosysid the IO system ID.
+ * @param my_rank the 0-based rank of this task.
+ * @param test_comm communicator that includes all tasks paticipating in test.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_decomp1(int iosysid, int my_rank, MPI_Comm test_comm)
+{
+    int ioid;                   /* The decomposition ID. */
+    PIO_Offset elements_per_pe; /* Array index per processing unit. */
+    PIO_Offset *compdof;        /* The decomposition mapping. */
+    int slice_dimlen[2];
+    int bad_slice_dimlen[2];    /* Invalid values. */
+    int ndims;
+    int *gdims;
+    PIO_Offset fmaplen;
+    PIO_Offset *map;
+    int ret;
+    
+    /* Describe the decomposition. This is a 1-based array, so add 1! */
+    slice_dimlen[0] = X_DIM_LEN;
+    slice_dimlen[1] = Y_DIM_LEN;
+    elements_per_pe = X_DIM_LEN * Y_DIM_LEN / TARGET_NTASKS;
+
+    /* The compdof array contains a mapping for this task into the
+     * global data array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i + 1;
+
+    /* These should not work. */
+    bad_slice_dimlen[1] = 0;
+    if (PIOc_InitDecomp(iosysid + TEST_VAL_42, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe,
+                        compdof, &ioid, NULL, NULL, NULL) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, bad_slice_dimlen, (PIO_Offset)elements_per_pe,
+                        compdof, &ioid, NULL, NULL, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, NULL, (PIO_Offset)elements_per_pe,
+                        compdof, &ioid, NULL, NULL, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe,
+                        NULL, &ioid, NULL, NULL, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe,
+                        compdof, NULL, NULL, NULL, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+        
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition...\n", my_rank);
+    if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe,
+                               compdof, &ioid, NULL, NULL, NULL)))
+        return ret;
+    free(compdof);
+
+    /* These should not work. */
+    if (PIOc_write_decomp(DECOMP_FILE, iosysid + TEST_VAL_42, ioid, test_comm) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_write_decomp(DECOMP_FILE, iosysid, ioid + TEST_VAL_42, test_comm) != PIO_EBADID)
+        return ERR_WRONG;
+
+    /* Write the decomp file. */
+    if ((ret = PIOc_write_decomp(DECOMP_FILE, iosysid, ioid, test_comm)))
+        return ret;
+
+    /* These should not work. */
+    if (PIOc_readmap(NULL, &ndims, (int **)&gdims, &fmaplen, (PIO_Offset **)&map,
+                     test_comm) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_readmap(DECOMP_FILE, NULL, (int **)&gdims, &fmaplen, (PIO_Offset **)&map,
+                     test_comm) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_readmap(DECOMP_FILE, &ndims, NULL, &fmaplen, (PIO_Offset **)&map,
+                     test_comm) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_readmap(DECOMP_FILE, &ndims, (int **)&gdims, NULL, (PIO_Offset **)&map,
+                     test_comm) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_readmap(DECOMP_FILE, &ndims, (int **)&gdims, &fmaplen, NULL, test_comm) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Read the decomp file and check results. */
+    if ((ret = PIOc_readmap(DECOMP_FILE, &ndims, (int **)&gdims, &fmaplen, (PIO_Offset **)&map,
+                            test_comm)))
+        return ret;
+    printf("ndims = %d fmaplen = %lld\n", ndims, fmaplen);
+    if (ndims != 2 || fmaplen != 4)
+        return ERR_WRONG;
+    for (int d = 0; d < ndims; d++)
+    {
+        printf("gdims[%d] = %d\n", d, gdims[d]);
+    }
+    for (int m = 0; m < fmaplen; m++)
+    {
+        printf("map[%d] = %lld\n", m, map[m]);
+    }
+
+    free(map);
+    free(gdims);
+
+    /* These should not work. */
+    if (PIOc_freedecomp(iosysid + TEST_VAL_42, ioid) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_freedecomp(iosysid, ioid + TEST_VAL_42) != PIO_EBADID)
+        return ERR_WRONG;
+        
+    /* Free the PIO decomposition. */
+    printf("%d Freeing PIO decomposition...\n", my_rank);
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        return ret;
+        
+    return 0;
+}
+
+/**
+ * Test PIOc_InitDecomp_bc(). 
+ *
+ * @param iosysid the IO system ID.
+ * @param my_rank the 0-based rank of this task.
+ * @param test_comm communicator that includes all tasks paticipating in test.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_decomp_bc(int iosysid, int my_rank, MPI_Comm test_comm)
+{
+    int ioid;                   /* The decomposition ID. */
+    long int start[NDIM2];
+    long int count[NDIM2];
+    long int bad_count[NDIM2] = {-1, 0};
+    long int bad_start[NDIM2] = {-1, 0};
+    int ndims;
+    int *gdims;
+    PIO_Offset fmaplen;
+    PIO_Offset *map;
+    int slice_dimlen[NDIM2];
+    int ret;
+    
+    /* Describe the decomposition. This is a 1-based array, so add 1! */
+    start[0] = my_rank;
+    start[1] = 0;
+    count[0] = 1;
+    count[1] = 4;
+    slice_dimlen[0] = X_DIM_LEN;
+    slice_dimlen[1] = Y_DIM_LEN;
+
+    /* These should not work. */
+    if (PIOc_InitDecomp_bc(iosysid + TEST_VAL_42, PIO_FLOAT, 2, slice_dimlen, start, count, &ioid) != PIO_EBADID)
+        return ERR_WRONG;
+    printf("ret = %d\n", PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, NULL, start, count, &ioid));
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, NULL, start, count, &ioid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, NULL, count, &ioid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, start, NULL, &ioid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, start, count, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, bad_start, count, &ioid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, start, bad_count, &ioid) != PIO_EINVAL)
+        return ERR_WRONG;
+        
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition...\n", my_rank);
+    if ((ret = PIOc_InitDecomp_bc(iosysid, PIO_FLOAT, 2, slice_dimlen, start, count, &ioid)))
+        return ret;
+
+    /* Write the decomp file. */
+    if ((ret = PIOc_write_decomp(DECOMP_BC_FILE, iosysid, ioid, test_comm)))
+        return ret;
+
+    /* Read the decomp file and check results. */
+    if ((ret = PIOc_readmap(DECOMP_BC_FILE, &ndims, (int **)&gdims, &fmaplen, (PIO_Offset **)&map,
+                            test_comm)))
+        return ret;
+    printf("ndims = %d fmaplen = %lld\n", ndims, fmaplen);
+    if (ndims != 2 || fmaplen != 4)
+        return ERR_WRONG;
+    for (int d = 0; d < ndims; d++)
+    {
+        printf("gdims[%d] = %d\n", d, gdims[d]);
+    }
+    for (int m = 0; m < fmaplen; m++)
+    {
+        printf("map[%d] = %lld\n", m, map[m]);
+    }
+
+    free(map);
+    free(gdims);
+        
+    /* Free the PIO decomposition. */
+    printf("%d Freeing PIO decomposition...\n", my_rank);
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        return ret;
+        
+    return 0;
+}
+
+/** 
+ * Test the decomp read/write functionality.
+ *
+ * @param iosysid the IO system ID.
+ * @param ioid the ID of the decomposition.
+ * @param num_flavors the number of IOTYPES available in this build.
+ * @param flavor array of available iotypes.
+ * @param my_rank rank of this task.
+ * @param test_comm the MPI communicator for this test.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_decomp_read_write(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                           MPI_Comm test_comm)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int ioid2;             /* ID for decomposition we will create from file. */
+    char title_in[PIO_MAX_NAME + 1];   /* Optional title. */
+    char history_in[PIO_MAX_NAME + 1]; /* Optional history. */
+    int fortran_order_in; /* Indicates fortran vs. c order. */
+    int num_decomp_file_types = 1;
+    int ret;              /* Return code. */
+
+#ifdef _NETCDF4
+    /* Two extra output methods to tests if NetCDF-4 is present. */
+    num_decomp_file_types = 3;
+#endif /* _NETCDF4 */
+    
+    for (int decomp_file_type = 0; decomp_file_type < num_decomp_file_types; decomp_file_type++)
+    {
+        int cmode = 0;
+            
+        /* Determine the create mode. */
+        if (decomp_file_type)
+            cmode |= NC_NETCDF4;
+        if (decomp_file_type == 2)
+            cmode |= NC_MPIIO;
+            
+        /* Use PIO to create the decomp file in each of the four
+         * available ways. */
+        for (int fmt = 0; fmt < num_flavors; fmt++) 
+        {
+            /* Create the filename. */
+            sprintf(filename, "decomp_%s_iotype_%d_deomp_type_%d.nc", TEST_NAME, flavor[fmt],
+                    decomp_file_type);
+
+            printf("writing decomp file %s\n", filename);
+            if ((ret = PIOc_write_nc_decomp(iosysid, filename, cmode, ioid, NULL, NULL, 0)))
+                return ret;
+    
+            /* Read the data. */
+            printf("reading decomp file %s\n", filename);
+            if ((ret = PIOc_read_nc_decomp(iosysid, filename, &ioid2, test_comm, PIO_INT,
+                                           title_in, history_in, &fortran_order_in)))
+                return ret;
+    
+            /* Check the results. */
+            {
+                iosystem_desc_t *ios;
+                io_desc_t *iodesc;
+            
+                /* Get the IO system info. */
+                if (!(ios = pio_get_iosystem_from_id(iosysid)))
+                    return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+                /* Get the IO desc, which describes the decomposition. */
+                if (!(iodesc = pio_get_iodesc_from_id(ioid2)))
+                    return pio_err(ios, NULL, PIO_EBADID, __FILE__, __LINE__);
+                if (iodesc->ioid != ioid2 || iodesc->maplen != TARGET_NTASKS || iodesc->ndims != NDIM2 ||
+                    iodesc->nrecvs != 1 || iodesc->ndof != TARGET_NTASKS || iodesc->num_aiotasks != TARGET_NTASKS
+                    || iodesc->rearranger != PIO_REARR_SUBSET || iodesc->maxregions != 1 ||
+                    iodesc->needsfill || iodesc->basetype != MPI_INT)
+                    return ERR_WRONG;
+                for (int e = 0; e < iodesc->maplen; e++)
+                    if (iodesc->map[e] != my_rank * iodesc->maplen + e + 1)
+                        return ERR_WRONG;
+                if (iodesc->dimlen[0] != X_DIM_LEN || iodesc->dimlen[1] != Y_DIM_LEN)
+                    return ERR_WRONG;
+                printf("%d in my test iodesc->maxiobuflen = %d\n", my_rank, iodesc->maxiobuflen);
+            }
+        
+
+            /* Free the PIO decomposition. */
+            if ((ret = PIOc_freedecomp(iosysid, ioid2)))
+                ERR(ret);
+        }
+    }
+    return PIO_NOERR;
+}
+
+/* Run decomp tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks;  /* Number of processors involved in current execution. */
+    int iosysid; /* The ID for the parallel I/O system. */
+    MPI_Group world_group;      /* An MPI group of world. */
+    MPI_Comm test_comm;
+    int num_flavors;            /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS];    /* iotypes for the supported netCDF IO flavors. */
+    int dim_len_2d[NDIM2] = {X_DIM_LEN, Y_DIM_LEN};
+    int ioid;
+    int ret;                    /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+                             &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Initialize PIO system on world. */
+        printf("%d about to call Init_Intracomm\n", my_rank);
+        if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO4, STRIDE1, BASE0, REARRANGER, &iosysid)))
+            ERR(ret);
+        printf("%d done with Init_Intracomm\n", my_rank);
+
+        /* Set the error handler. */
+        /*PIOc_Set_IOSystem_Error_Handling(iosysid, PIO_BCAST_ERROR);*/
+        printf("%d about to set iosystem error hanlder for world\n", my_rank);
+        if ((ret = PIOc_set_iosystem_error_handling(iosysid, PIO_BCAST_ERROR, NULL)))
+            ERR(ret);
+        printf("%d done setting iosystem error hanlder for world\n", my_rank);
+
+        /* Get MPI_Group of world comm. */
+        if ((ret = MPI_Comm_group(test_comm, &world_group)))
+            ERR(ret);
+
+        /* Test basic decomp stuff. */
+        if ((ret = test_decomp1(iosysid, my_rank, test_comm)))
+            return ret;
+
+        /* Test PIOc_InitDecomp_bc(). */
+        if ((ret = test_decomp_bc(iosysid, my_rank, test_comm)))
+            return ret;
+
+        /* Decompose the data over the tasks. */
+        if ((ret = create_decomposition_2d(TARGET_NTASKS, my_rank, iosysid, dim_len_2d, &ioid,
+                                           PIO_INT)))
+            return ret;
+
+        /* Test decomposition read/write. */
+        if ((ret = test_decomp_read_write(iosysid, ioid, num_flavors, flavor, my_rank, test_comm)))
+            return ret;
+    
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+        
+        /* Finalize PIO systems. */
+        printf("%d pio finalized\n", my_rank);
+        if ((ret = PIOc_finalize(iosysid)))
+            ERR(ret);
+
+        /* Free MPI resources used by test. */
+        if ((ret = MPI_Group_free(&world_group)))
+            ERR(ret);
+
+    } /* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_intercomm2.c b/src/externals/pio2/tests/cunit/test_intercomm2.c
new file mode 100644
index 00000000000..e55c6e1ff2e
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_intercomm2.c
@@ -0,0 +1,558 @@
+/*
+ * Tests for PIOc_Intercomm. This tests the Init_Intercomm()
+ * function, and basic asynch I/O capability.
+ *
+ * To run with valgrind, use this command:
+ * 
    mpiexec -n 4 valgrind -v --leak-check=full --suppressions=../../../tests/unit/valsupp_test.supp
+ * --error-exitcode=99 --track-origins=yes ./test_intercomm2
    
+ *
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_intercomm2"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 2
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the test data. */
+#define NDIM 1
+
+/* The length of our test data. */
+#define DIM_LEN 4
+
+/* Number of netCDF atomic types. */
+#define NUM_TYPES 11
+
+/* The name of the dimension in the netCDF output file. */
+#define FIRST_DIM_NAME "jojo"
+#define DIM_NAME "dim_test_intercomm2"
+
+/* The name of the variable in the netCDF output file. */
+#define FIRST_VAR_NAME "bill"
+#define VAR_NAME "var_test_intercomm2"
+
+/* The name of the global attribute in the netCDF output file. */
+#define FIRST_ATT_NAME "willy_gatt_test_intercomm2"
+#define ATT_NAME "gatt_test_intercomm2"
+#define SHORT_ATT_NAME "short_gatt_test_intercomm2"
+#define FLOAT_ATT_NAME "float_gatt_test_intercomm2"
+#define DOUBLE_ATT_NAME "double_gatt_test_intercomm2"
+
+/* The value of the global attribute in the netCDF output file. */
+#define ATT_VALUE 42
+
+/* Check the file for correctness. */
+int check_file(int iosysid, int format, char *filename, int my_rank)
+{
+    int ncid;
+    int ret;
+    int ndims, nvars, ngatts, unlimdimid;
+    int ndims2, nvars2, ngatts2, unlimdimid2;
+    int dimid2;
+    char dimname[NC_MAX_NAME + 1];
+    PIO_Offset dimlen;
+    char dimname2[NC_MAX_NAME + 1];
+    PIO_Offset dimlen2;
+    char varname[NC_MAX_NAME + 1];
+    nc_type vartype;
+    int varndims, vardimids, varnatts;
+    char varname2[NC_MAX_NAME + 1];
+    nc_type vartype2;
+    int varndims2, vardimids2, varnatts2;
+    int varid2;
+    int att_data;
+    short short_att_data;
+    float float_att_data;
+    double double_att_data;
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+
+    /* Create a name that is too long. */
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+
+    /* Re-open the file to check it. */
+    printf("%d test_intercomm2 opening file %s format %d\n", my_rank, filename, format);
+    if ((ret = PIOc_openfile(iosysid, &ncid, &format, filename,
+                             NC_NOWRITE)))
+        ERR(ret);
+
+    /* Try to read the data. */
+    PIO_Offset start[NDIM] = {0}, count[NDIM] = {DIM_LEN};
+    int data_in[DIM_LEN];
+    /* Check with various combinations of start/count */
+    for (int start_index = 0; start_index < DIM_LEN; start_index++)
+    {
+        start[0] = start_index;
+        count[0] = DIM_LEN - start_index;
+        if ((ret = PIOc_get_vars_tc(ncid, 0, start, count, NULL, NC_INT, data_in)))
+            ERR(ret);
+        for (int i = 0; i < count[0]; i++)
+        {
+            printf("%d test_intercomm2 read data_in[%d] = %d, start_index = %d\n", my_rank, i, data_in[i], start_index);
+            if (data_in[i] != (i + start_index))
+                ERR(ERR_AWFUL);
+        }
+    }
+
+    /* Find the number of dimensions, variables, and global attributes.*/
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        ERR(ret);
+    if (ndims != 1 || nvars != 1 || ngatts != 4 || unlimdimid != -1)
+        ERR(ERR_WRONG);
+
+    /* This should return PIO_NOERR. */
+    if ((ret = PIOc_inq(ncid, NULL, NULL, NULL, NULL)))
+        ERR(ret);
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_ndims(ncid, &ndims2)))
+        ERR(ret);
+    if (ndims2 != 1)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_nvars(ncid, &nvars2)))
+        ERR(ret);
+    if (nvars2 != 1)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_natts(ncid, &ngatts2)))
+        ERR(ret);
+    if (ngatts2 != 4)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_unlimdim(ncid, &unlimdimid2)))
+        ERR(ret);
+    if (unlimdimid != -1)
+        ERR(ERR_WRONG);
+    /* Should succeed, do nothing. */
+    if ((ret = PIOc_inq_unlimdim(ncid, NULL)))
+        ERR(ret);
+
+    /* Check out the dimension. */
+    if ((ret = PIOc_inq_dim(ncid, 0, dimname, &dimlen)))
+        ERR(ret);
+    if (strcmp(dimname, DIM_NAME) || dimlen != DIM_LEN)
+        ERR(ERR_WRONG);
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_dimname(ncid, 0, dimname2)))
+        ERR(ret);
+    if (strcmp(dimname2, DIM_NAME))
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_dimlen(ncid, 0, &dimlen2)))
+        ERR(ret);
+    if (dimlen2 != DIM_LEN)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_dimid(ncid, DIM_NAME, &dimid2)))
+        ERR(ret);
+    if (dimid2 != 0)
+        ERR(ERR_WRONG);
+
+    /* These should not work. */
+    if (PIOc_inq_dimid(ncid + TEST_VAL_42, DIM_NAME, &dimid2) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_inq_dimid(ncid, too_long_name, &dimid2) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+
+    /* Check out the variable. */
+    if ((ret = PIOc_inq_var(ncid, 0, varname, &vartype, &varndims, &vardimids, &varnatts)))
+        ERR(ret);
+    if (strcmp(varname, VAR_NAME) || vartype != NC_INT || varndims != NDIM ||
+        vardimids != 0 || varnatts != 0)
+        ERR(ERR_WRONG);
+
+    /* Check the other functions that get these values. */
+    if ((ret = PIOc_inq_varname(ncid, 0, varname2)))
+        ERR(ret);
+    if (strcmp(varname2, VAR_NAME))
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_vartype(ncid, 0, &vartype2)))
+        ERR(ret);
+    if (vartype2 != NC_INT)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_varndims(ncid, 0, &varndims2)))
+        ERR(ret);
+    if (varndims2 != NDIM)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_vardimid(ncid, 0, &vardimids2)))
+        ERR(ret);
+    if (vardimids2 != 0)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_varnatts(ncid, 0, &varnatts2)))
+        ERR(ret);
+    if (varnatts2 != 0)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_varid(ncid, VAR_NAME, &varid2)))
+        ERR(ret);
+    if (varid2 != 0)
+        ERR(ERR_WRONG);
+
+    /* These should not work. */
+    if (PIOc_inq_varid(ncid + TEST_VAL_42, VAR_NAME, &varid2) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_inq_varid(ncid, NULL, &varid2) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+    if (PIOc_inq_varid(ncid, too_long_name, &varid2) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+
+    /* Check out the global attributes. */
+    nc_type atttype;
+    PIO_Offset attlen;
+    char myattname[NC_MAX_NAME + 1];
+    int myid;
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &atttype, &attlen)))
+        ERR(ret);
+    if (atttype != NC_INT || attlen != 1)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_attlen(ncid, NC_GLOBAL, ATT_NAME, &attlen)))
+        ERR(ret);
+    if (attlen != 1)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, myattname)))
+        ERR(ret);
+    if (strcmp(ATT_NAME, myattname))
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_attid(ncid, NC_GLOBAL, ATT_NAME, &myid)))
+        ERR(ret);
+    if (PIOc_inq_attid(ncid + TEST_VAL_42, NC_GLOBAL, ATT_NAME, &myid) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (myid != 0)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_data)))
+        ERR(ret);
+    printf("%d test_intercomm2 att_data = %d\n", my_rank, att_data);
+    if (att_data != ATT_VALUE)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, SHORT_ATT_NAME, &atttype, &attlen)))
+        ERR(ret);
+    if (atttype != NC_SHORT || attlen != 1)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_get_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, &short_att_data)))
+        ERR(ret);
+    if (short_att_data != ATT_VALUE)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_get_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME, &float_att_data)))
+        ERR(ret);
+    if (float_att_data != ATT_VALUE)
+        ERR(ERR_WRONG);
+    if ((ret = PIOc_get_att_double(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, &double_att_data)))
+        ERR(ret);
+    if (double_att_data != ATT_VALUE)
+        ERR(ERR_WRONG);
+
+    /* These should not work. */
+    if (PIOc_inq_att(ncid + TEST_VAL_42, NC_GLOBAL, ATT_NAME, &atttype, &attlen) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_inq_att(ncid + TEST_VAL_42, NC_GLOBAL, NULL, &atttype, &attlen) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_inq_att(ncid + TEST_VAL_42, NC_GLOBAL, too_long_name, &atttype, &attlen) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_get_att(ncid, NC_GLOBAL, TEST_NAME, &att_data) != PIO_ENOTATT)
+        ERR(ERR_WRONG);
+    if (PIOc_get_att(ncid, NC_GLOBAL, NULL, &att_data) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+    if (PIOc_get_att(ncid, NC_GLOBAL, too_long_name, &att_data) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+
+    /* Close the file. */
+    printf("%d test_intercomm2 closing file (again) ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_closefile(ncid)))
+        ERR(ret);
+
+    return 0;
+}
+
+/* Run Tests for Init_Intercomm. */
+int main(int argc, char **argv)
+{
+    /* Zero-based rank of processor. */
+    int my_rank;
+
+    /* Number of processors involved in current execution. */
+    int ntasks;
+
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+
+    /* Names for the output files. */
+    char filename[NUM_FLAVORS][NC_MAX_NAME + 1];
+
+    /* The ID for the parallel I/O system. */
+    int iosysid[COMPONENT_COUNT];
+
+    /* Return code. */
+    int ret;
+
+    MPI_Comm test_comm;
+
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+
+    /* Create a name that is too long. */
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+    
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Figure out iotypes. */
+    if ((ret = get_iotypes(&num_flavors, flavor)))
+        ERR(ret);
+
+    if(my_rank < TARGET_NTASKS)
+    {
+	printf("%d: test_intercomm2 ParallelIO Library test_intercomm2 running on %d processors.\n",
+	       my_rank, ntasks);
+
+        /* Initialize the PIO IO system. This specifies how many and which
+         * processors are involved in I/O. */
+
+        /* Turn on logging. */
+        if ((ret = PIOc_set_log_level(3)))
+            ERR(ret);
+
+        /* How many processors will be used for our IO and 2 computation components. */
+        int num_procs[COMPONENT_COUNT] = {2};
+
+        /* Is the current process a computation task? */
+        int comp_task = my_rank < 2 ? 0 : 1;
+
+        /* Index of computation task in iosysid array. Varies by rank and
+         * does not apply to IO component processes. */
+        int my_comp_idx = comp_task ? 0 : -1;
+
+        /* Initialize the IO system. */
+        if ((ret = PIOc_init_async(test_comm, NUM_IO_PROCS, NULL, COMPONENT_COUNT,
+                                   num_procs, NULL, NULL, NULL, PIO_REARR_BOX, iosysid)))
+            ERR(ERR_AWFUL);
+
+	printf("%d: test_intercomm2 ParallelIO Library test_intercomm2 comp task returned.\n",
+	       my_rank);
+
+        /* All the netCDF calls are only executed on the computation
+         * tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
+         * and when the do, they should go straight to finalize. */
+        if (comp_task)
+        {
+            for (int fmt = 0; fmt < num_flavors; fmt++)
+            {
+                int ncid, varid, dimid;
+                PIO_Offset start[NDIM], count[NDIM] = {0};
+                int data[DIM_LEN];
+
+                /* Create the filename for this flavor. */
+                sprintf(filename[fmt], "test_intercomm2_%d.nc", flavor[fmt]);
+
+                /* Create a netCDF file with one dimension and one variable. */
+		printf("%d test_intercomm2 creating file %s\n", my_rank, filename[fmt]);
+                if ((ret = PIOc_createfile(iosysid[my_comp_idx], &ncid, &flavor[fmt], filename[fmt],
+                                           NC_CLOBBER)))
+                    ERR(ret);
+		printf("%d test_intercomm2 file created ncid = %d\n", my_rank, ncid);
+
+                /* End define mode, then re-enter it. */
+                if ((ret = PIOc_enddef(ncid)))
+                    ERR(ret);
+		printf("%d test_intercomm2 calling redef\n", my_rank);
+                if ((ret = PIOc_redef(ncid)))
+                    ERR(ret);
+
+                /* Test the inq_format function. */
+                int myformat;
+                if (PIOc_inq_format(ncid + TEST_VAL_42, &myformat) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_inq_format(ncid, &myformat)))
+                    ERR(ret);
+                if ((flavor[fmt] == PIO_IOTYPE_PNETCDF || flavor[fmt] == PIO_IOTYPE_NETCDF) &&
+                    myformat != 1)
+                    ERR(ERR_AWFUL);
+                else if ((flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P) &&
+                         myformat != 3)
+                    ERR(ERR_AWFUL);
+
+                /* Test the inq_type function for atomic types. */
+                char type_name[NC_MAX_NAME + 1];
+                PIO_Offset type_size;
+                nc_type xtype[NUM_TYPES] = {NC_CHAR, NC_BYTE, NC_SHORT, NC_INT, NC_FLOAT, NC_DOUBLE,
+                                            NC_UBYTE, NC_USHORT, NC_UINT, NC_INT64, NC_UINT64};
+                int type_len[NUM_TYPES] = {1, 1, 2, 4, 4, 8, 1, 2, 4, 8, 8};
+                int max_type = flavor[fmt] == PIO_IOTYPE_NETCDF ? NC_DOUBLE : NC_UINT64;
+
+                /* This should not work. */
+                if (PIOc_inq_type(ncid + TEST_VAL_42, xtype[0], type_name, &type_size) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+
+                /* These should work. */
+                for (int i = 0; i < max_type; i++)
+                {
+                    if ((ret = PIOc_inq_type(ncid, xtype[i], type_name, &type_size)))
+                        ERR(ret);
+                    if (type_size != type_len[i])
+                        ERR(ERR_AWFUL);
+                }
+
+                /* Define a dimension. */
+                char dimname2[NC_MAX_NAME + 1];
+		printf("%d test_intercomm2 defining dimension %s\n", my_rank, DIM_NAME);
+                if ((ret = PIOc_def_dim(ncid, FIRST_DIM_NAME, DIM_LEN, &dimid)))
+                    ERR(ret);
+                if ((ret = PIOc_inq_dimname(ncid, 0, dimname2)))
+                    ERR(ret);
+                if (strcmp(dimname2, FIRST_DIM_NAME))
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_rename_dim(ncid, 0, DIM_NAME)))
+                    ERR(ret);
+
+                /* These should not work. */
+                if (PIOc_rename_dim(ncid + TEST_VAL_42, 0, DIM_NAME) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_dim(ncid, 0, NULL) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_dim(ncid, 0, too_long_name) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+
+                /* Define a 1-D variable. */
+                char varname2[NC_MAX_NAME + 1];
+		printf("%d test_intercomm2 defining variable %s\n", my_rank, VAR_NAME);
+                if ((ret = PIOc_def_var(ncid, FIRST_VAR_NAME, NC_INT, NDIM, &dimid, &varid)))
+                    ERR(ret);
+                if ((ret = PIOc_inq_varname(ncid, 0, varname2)))
+                    ERR(ret);
+                if (strcmp(varname2, FIRST_VAR_NAME))
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_rename_var(ncid, 0, VAR_NAME)))
+                    ERR(ret);
+
+                /* These should not work. */
+                if (PIOc_rename_var(ncid + TEST_VAL_42, 0, VAR_NAME) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_var(ncid, 0, NULL) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_var(ncid, 0, too_long_name) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+
+                /* Add a global attribute. */
+		printf("%d test_intercomm2 writing attributes %s\n", my_rank, ATT_NAME);
+                int att_data = ATT_VALUE;
+                short short_att_data = ATT_VALUE;
+                float float_att_data = ATT_VALUE;
+                double double_att_data = ATT_VALUE;
+                char attname2[NC_MAX_NAME + 1];
+
+                /* Write an att and rename it. */
+                if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data)))
+                    ERR(ret);
+                if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, attname2)))
+                    ERR(ret);
+                if (strcmp(attname2, FIRST_ATT_NAME))
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, ATT_NAME)))
+                    ERR(ret);
+
+                /* These should not work. */
+                if (PIOc_inq_attname(ncid + TEST_VAL_42, NC_GLOBAL, 0, attname2) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_att(ncid + TEST_VAL_42, NC_GLOBAL, FIRST_ATT_NAME, ATT_NAME) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, NULL) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+                if (PIOc_rename_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, too_long_name) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+                if (PIOc_del_att(ncid + TEST_VAL_42, NC_GLOBAL, FIRST_ATT_NAME) != PIO_EBADID)
+                    ERR(ERR_WRONG);
+                if (PIOc_del_att(ncid, NC_GLOBAL, NULL) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+                if (PIOc_del_att(ncid, NC_GLOBAL, too_long_name) != PIO_EINVAL)
+                    ERR(ERR_WRONG);
+
+                /* Write an att and delete it. */
+                if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, FIRST_ATT_NAME, NC_INT, 1, &att_data)))
+                    ERR(ret);
+                if ((ret = PIOc_del_att(ncid, NC_GLOBAL, FIRST_ATT_NAME)))
+                    ERR(ret);
+                /* if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, FIRST_ATT_NAME, NULL, NULL)) != PIO_ENOTATT) */
+                /* { */
+                /*      printf("ret = %d\n", ret); */
+                /*      ERR(ERR_AWFUL); */
+                /* } */
+
+                /* Write some atts of different types. */
+                if ((ret = PIOc_put_att_short(ncid, NC_GLOBAL, SHORT_ATT_NAME, NC_SHORT, 1, &short_att_data)))
+                    ERR(ret);
+                if ((ret = PIOc_put_att_float(ncid, NC_GLOBAL, FLOAT_ATT_NAME, NC_FLOAT, 1, &float_att_data)))
+                    ERR(ret);
+                if ((ret = PIOc_put_att_double(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, NC_DOUBLE, 1, &double_att_data)))
+                    ERR(ret);
+
+                /* Check some att types. */
+                nc_type myatttype;
+                if ((ret = PIOc_inq_atttype(ncid, NC_GLOBAL, SHORT_ATT_NAME, &myatttype)))
+                    ERR(ret);
+                if (myatttype != NC_SHORT)
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_inq_atttype(ncid, NC_GLOBAL, FLOAT_ATT_NAME, &myatttype)))
+                    ERR(ret);
+                if (myatttype != NC_FLOAT)
+                    ERR(ERR_WRONG);
+                if ((ret = PIOc_inq_atttype(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, &myatttype)))
+                    ERR(ret);
+                if (myatttype != NC_DOUBLE)
+                    ERR(ERR_WRONG);
+
+                /* End define mode. */
+		printf("%d test_intercomm2 ending define mode ncid = %d\n", my_rank, ncid);
+                if ((ret = PIOc_enddef(ncid)))
+                    ERR(ret);
+
+                /* Write some data. For the PIOc_put/get functions, all
+                 * data must be on compmaster before the function is
+                 * called. Only compmaster's arguments are passed to the
+                 * async msg handler. All other computation tasks are
+                 * ignored. */
+                for (int i = 0; i < DIM_LEN; i++)
+                    data[i] = i;
+		printf("%d test_intercomm2 writing data\n", my_rank);
+                start[0] = 0;
+                count[0] = DIM_LEN;
+                if ((ret = PIOc_put_vars_tc(ncid, varid, start, count, NULL, NC_INT, data)))
+                    ERR(ret);
+
+                /* Close the file. */
+		printf("%d test_intercomm2 closing file ncid = %d\n", my_rank, ncid);
+                if ((ret = PIOc_closefile(ncid)))
+                    ERR(ret);
+
+                /* Check the file for correctness. */
+                if ((ret = check_file(iosysid[my_comp_idx], flavor[fmt], filename[fmt], my_rank)))
+                    ERR(ret);
+
+                /* Now delete the file. */
+                /* if ((ret = PIOc_deletefile(iosysid, filename[fmt]))) */
+                /*      ERR(ret); */
+                /* if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename[fmt], */
+                /*                           NC_NOWRITE)) != PIO_ENFILE) */
+                /*      ERR(ERR_AWFUL); */
+
+            } /* next netcdf flavor */
+
+            /* Finalize the IO system. Only call this from the computation tasks. */
+	    printf("%d test_intercomm2 Freeing PIO resources\n", my_rank);
+            if ((ret = PIOc_finalize(iosysid[my_comp_idx])))
+                ERR(ret);
+        }
+    } /* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem2.c b/src/externals/pio2/tests/cunit/test_iosystem2.c
new file mode 100644
index 00000000000..ca27acad514
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem2.c
@@ -0,0 +1,226 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran pio_iosystem_tests2.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem2"
+
+/* Used to define netcdf test file. */
+#define PIO_TF_MAX_STR_LEN 100
+#define ATTNAME "filename"
+#define DIMNAME "filename_dim"
+
+/* This creates a netCDF file in the specified format, with some
+ * sample values. */
+int create_file(MPI_Comm comm, int iosysid, int format, char *filename,
+                char *attname, char *dimname, int my_rank)
+{
+    int ncid, varid, dimid;
+    int ret;
+
+    /* Create the file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &format, filename, NC_CLOBBER)))
+        return ret;
+    printf("%d file created ncid = %d\n", my_rank, ncid);
+
+    /* Use the ncid to set the IO system error handler. This function
+     * is deprecated. */
+    PIOc_Set_File_Error_Handling(ncid, PIO_RETURN_ERROR);
+    int method = PIOc_Set_File_Error_Handling(ncid, PIO_RETURN_ERROR);
+    if (method != PIO_RETURN_ERROR)
+        return ERR_WRONG;
+
+    /* Define a dimension. */
+    printf("%d defining dimension %s\n", my_rank, dimname);
+    if ((ret = PIOc_def_dim(ncid, dimname, PIO_TF_MAX_STR_LEN, &dimid)))
+        return ret;
+
+    /* Define a 1-D variable. */
+    printf("%d defining variable %s\n", my_rank, attname);
+    if ((ret = PIOc_def_var(ncid, attname, NC_CHAR, 1, &dimid, &varid)))
+        return ret;
+
+    /* Write an attribute. */
+    if ((ret = PIOc_put_att_text(ncid, varid, attname, strlen(filename), filename)))
+        return ret;
+
+    /* End define mode. */
+    printf("%d ending define mode ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    printf("%d define mode ended ncid = %d\n", my_rank, ncid);
+
+    /* Close the file. */
+    printf("%d closing file ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_closefile(ncid)))
+        return ret;
+    printf("%d closed file ncid = %d\n", my_rank, ncid);
+
+    return PIO_NOERR;
+}
+
+/* This checks an already-open netCDF file. */
+int check_file(MPI_Comm comm, int iosysid, int format, int ncid, char *filename,
+               char *attname, char *dimname, int my_rank)
+{
+    int dimid;
+    int ret;
+
+    /* Check the file. */
+    if ((ret = PIOc_inq_dimid(ncid, dimname, &dimid)))
+        return ret;
+    printf("%d dimid = %d\n", my_rank, dimid);
+
+    return PIO_NOERR;
+}
+
+/* This opens and checks a netCDF file. */
+int open_and_check_file(MPI_Comm comm, int iosysid, int iotype, int *ncid, char *fname,
+                        char *attname, char *dimname, int disable_close, int my_rank)
+{
+    int mode = PIO_WRITE;
+    int ret;
+
+    /* Open the file. */
+    if ((ret = PIOc_openfile(iosysid, ncid, &iotype, fname, mode)))
+        return ret;
+
+    /* Check the file. */
+    if ((ret = check_file(comm, iosysid, iotype, *ncid, fname, attname, dimname, my_rank)))
+        return ret;
+
+    /* Close the file, maybe. */
+    if (!disable_close)
+        if ((ret = PIOc_closefile(*ncid)))
+            return ret;
+
+    return PIO_NOERR;
+}
+
+/* Run async tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid; /* The ID for the parallel I/O system. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    int ret; /* Return code. */
+    int num_flavors;
+    int iotypes[NUM_FLAVORS];
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+                             &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, iotypes)))
+            ERR(ret);
+
+        /* Split world into odd and even. */
+        MPI_Comm newcomm;
+        int even = my_rank % 2 ? 0 : 1;
+        if ((ret = MPI_Comm_split(test_comm, even, 0, &newcomm)))
+            MPIERR(ret);
+        printf("%d newcomm = %d even = %d\n", my_rank, newcomm, even);
+
+        /* Get rank in new communicator and its size. */
+        int new_rank, new_size;
+        if ((ret = MPI_Comm_rank(newcomm, &new_rank)))
+            MPIERR(ret);
+        if ((ret = MPI_Comm_size(newcomm, &new_size)))
+            MPIERR(ret);
+        printf("%d newcomm = %d new_rank = %d new_size = %d\n", my_rank, newcomm,
+               new_rank, new_size);
+
+        /* Initialize PIO system. */
+        if ((ret = PIOc_Init_Intracomm(newcomm, 2, 1, 0, 1, &iosysid)))
+            ERR(ret);
+
+        /* This should fail. */
+        if (PIOc_finalize(iosysid + TEST_VAL_42) != PIO_EBADID)
+            ERR(ERR_WRONG);
+
+        /* Initialize another PIO system. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, 4, 1, 0, 1, &iosysid_world)))
+            ERR(ret);
+
+        for (int i = 0; i < num_flavors; i++)
+        {
+            char fname0[] = "pio_iosys_test_file0.nc";
+            char fname1[] = "pio_iosys_test_file1.nc";
+            char fname2[] = "pio_iosys_test_file2.nc";
+            printf("\n\n%d i = %d\n", my_rank, i);
+
+            if ((ret = create_file(test_comm, iosysid_world, iotypes[i], fname0, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            if ((ret = create_file(test_comm, iosysid_world, iotypes[i], fname1, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            if ((ret = create_file(test_comm, iosysid_world, iotypes[i], fname2, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            MPI_Barrier(test_comm);
+
+            /* Now check the first file. */
+            int ncid;
+            if ((ret = open_and_check_file(test_comm, iosysid_world, iotypes[i], &ncid, fname0,
+                                           ATTNAME, DIMNAME, 1, my_rank)))
+                ERR(ret);
+
+            /* Now have the odd/even communicators each check one of the
+             * remaining files. */
+            int ncid2;
+            char *fname = even ? fname1 : fname2;
+            printf("\n***\n");
+            if ((ret = open_and_check_file(newcomm, iosysid, iotypes[i], &ncid2, fname,
+                                           ATTNAME, DIMNAME, 1, my_rank)))
+                ERR(ret);
+
+
+            /* Close the still-open files. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+            if ((ret = PIOc_closefile(ncid2)))
+                ERR(ret);
+        } /* next iotype */
+        if ((ret = MPI_Comm_free(&newcomm)))
+            MPIERR(ret);
+
+        /* Finalize PIO system. */
+        if ((ret = PIOc_finalize(iosysid)))
+            ERR(ret);
+
+        /* Finalize PIO system. */
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+    } /* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem2_simple.c b/src/externals/pio2/tests/cunit/test_iosystem2_simple.c
new file mode 100644
index 00000000000..66ae617372b
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem2_simple.c
@@ -0,0 +1,235 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran pio_iosystem_tests2.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 2
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem2_simple"
+
+/* Number of test files generated. */
+#define NUM_FILES 2
+
+/* Used to define netcdf test file. */
+#define DIMLEN 1
+
+/* Needed to init intracomm. */
+#define STRIDE 1
+#define BASE 0
+#define REARRANGER 1
+
+/* Ten megabytes. */
+#define TEN_MEG 10485760
+
+/* Run test. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid; /* The ID for the parallel I/O system. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    PIO_Offset oldlimit;
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS, &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if(my_rank < TARGET_NTASKS)
+    {
+        /* Try setting the blocksize. */
+        if ((ret = PIOc_set_blocksize(2048)))
+            ERR(ret);
+
+        /* Try setting the buffer size limit. */
+        oldlimit = PIOc_set_buffer_size_limit(200000);
+        if (oldlimit != TEN_MEG)
+            ERR(ERR_WRONG);
+        oldlimit = PIOc_set_buffer_size_limit(TEN_MEG);
+
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Split world into odd and even. */
+        MPI_Comm newcomm;
+        int even = my_rank % 2 ? 0 : 1;
+        if ((ret = MPI_Comm_split(test_comm, even, 0, &newcomm)))
+            MPIERR(ret);
+
+        /* Get size of new communicator. */
+        int new_size;
+        if ((ret = MPI_Comm_size(newcomm, &new_size)))
+            MPIERR(ret);
+        printf("%d newcomm = %d even = %d new_size = %d\n", my_rank, newcomm, even, new_size);
+
+        /* Check that some bad inputs are rejected. */
+        if (PIOc_Init_Intracomm(newcomm, new_size, STRIDE + 30, BASE, REARRANGER,
+                                &iosysid) != PIO_EINVAL)
+            return ERR_WRONG;
+        if (PIOc_Init_Intracomm(newcomm, new_size, STRIDE, BASE, REARRANGER, NULL) != PIO_EINVAL)
+            return ERR_WRONG;
+
+        /* Initialize an intracomm for evens/odds. */
+        if ((ret = PIOc_Init_Intracomm(newcomm, new_size, STRIDE, BASE, REARRANGER, &iosysid)))
+            ERR(ret);
+
+        /* Test some support functions. */
+        bool active;
+        if ((ret = PIOc_iosystem_is_active(iosysid, &active)))
+            ERR(ret);
+        if (!active)
+            ERR(ERR_WRONG);
+        if ((ret = PIOc_iosystem_is_active(iosysid + 1, &active)))
+            ERR(ret);
+        if (active)
+            ERR(ERR_WRONG);
+        if ((ret = PIOc_iosystem_is_active(iosysid, NULL)))
+            ERR(ret);
+
+        int numiotasks;
+        if (PIOc_get_numiotasks(iosysid + TEST_VAL_42, &numiotasks) != PIO_EBADID)
+            ERR(ERR_WRONG);
+        if ((ret = PIOc_get_numiotasks(iosysid, NULL)))
+            ERR(ret);
+        if ((ret = PIOc_get_numiotasks(iosysid, &numiotasks)))
+            ERR(ret);
+        printf("%d numiotasks = %d\n", my_rank, numiotasks);
+        if (numiotasks != 1)
+            ERR(ERR_WRONG);
+
+        int iorank;
+        if (PIOc_iotask_rank(iosysid + TEST_VAL_42, &iorank) != PIO_EBADID)
+            ERR(ERR_WRONG);
+        if ((ret = PIOc_iotask_rank(iosysid, NULL)))
+            ERR(ret);
+        if ((ret = PIOc_iotask_rank(iosysid, &iorank)))
+            ERR(ret);
+        printf("%d iorank = %d\n", my_rank, iorank);
+        /* Each of two tasks has an iosystem. On both iosystems, the
+         * single task has iorank of zero. */
+        if (iorank != 0)
+            ERR(ERR_WRONG);
+
+        /* Both tasks are IO tasks. */
+        bool ioproc;
+        if (PIOc_iam_iotask(iosysid + TEST_VAL_42, &ioproc) != PIO_EBADID)
+            ERR(ret);
+        if ((ret = PIOc_iam_iotask(iosysid, NULL)))
+            ERR(ret);
+        if ((ret = PIOc_iam_iotask(iosysid, &ioproc)))
+            ERR(ret);
+        if (!ioproc)
+            ERR(ERR_WRONG);
+
+        /* Initialize an intracomm for all processes. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, STRIDE, BASE, REARRANGER,
+                                       &iosysid_world)))
+            ERR(ret);
+
+        int ncid;
+        int ncid2;
+        for (int i = 0; i < num_flavors; i++)
+        {
+            char fn[NUM_FILES][NC_MAX_NAME + 1];
+            char dimname[NUM_FILES][NC_MAX_NAME + 1];
+
+            /* Create the test files. */
+            for (int f = 0; f < NUM_FILES; f++)
+            {
+                int lncid, dimid;
+
+                /* Creat file and dimension name. */
+                sprintf(fn[f], "pio_iosys_test_file%d.nc", f);
+                sprintf(dimname[f], "dim_%d", f);
+
+                if ((ret = PIOc_createfile(iosysid_world, &lncid, &flavor[i], fn[f], NC_CLOBBER)))
+                    return ret;
+                /* Define a dimension. */
+                if ((ret = PIOc_def_dim(lncid, dimname[f], DIMLEN, &dimid)))
+                    return ret;
+                if ((ret = PIOc_enddef(lncid)))
+                    return ret;
+                if ((ret = PIOc_closefile(lncid)))
+                    return ret;
+            }
+
+            /* These should not work. */
+            if (PIOc_openfile(iosysid_world + TEST_VAL_42, &ncid, &flavor[i], fn[0], PIO_WRITE) != PIO_EBADID)
+                return ERR_WRONG;
+            if (PIOc_openfile(iosysid_world, NULL, &flavor[i], fn[0], PIO_WRITE) != PIO_EINVAL)
+                return ERR_WRONG;
+            if (PIOc_openfile(iosysid_world, &ncid, NULL, fn[0], PIO_WRITE) != PIO_EINVAL)
+                return ERR_WRONG;
+            if (PIOc_openfile(iosysid_world, &ncid, &flavor[i], NULL, PIO_WRITE) != PIO_EINVAL)
+                return ERR_WRONG;
+            int bad_iotype = flavor[i] + TEST_VAL_42;
+            if (PIOc_openfile(iosysid_world, &ncid, &bad_iotype, fn[0], PIO_WRITE) != PIO_EINVAL)
+                return ERR_WRONG;
+
+            /* Open the first file with world iosystem. */
+            if ((ret = PIOc_openfile(iosysid_world, &ncid, &flavor[i], fn[0], PIO_WRITE)))
+                return ret;
+
+            /* Check the first file. */
+            char dimname_in[NC_MAX_NAME + 1];
+            if ((ret = PIOc_inq_dimname(ncid, 0, dimname_in)))
+                return ret;
+            printf("%d ncid dimname_in = %s should be %s\n", my_rank, dimname_in, dimname[0]);
+            if (strcmp(dimname_in, dimname[0]))
+                return ERR_WRONG;
+
+            /* Check the other file with the other IO. */
+            if (even)
+            {
+                if ((ret = PIOc_openfile(iosysid, &ncid2, &flavor[i], fn[1], PIO_WRITE)))
+                    return ret;
+                if ((ret = PIOc_inq_dimname(ncid2, 0, dimname_in)))
+                    return ret;
+                printf("%d ncid2 dimname_in = %s should be %s\n", my_rank, dimname_in, dimname[1]);
+                if (strcmp(dimname_in, dimname[1]))
+                    return ERR_WRONG;
+            }
+
+            /* Close the still-open files. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+            if (even)
+                if ((ret = PIOc_closefile(ncid2)))
+                    ERR(ret);
+        } /* next iotype */
+        if ((ret = MPI_Comm_free(&newcomm)))
+            MPIERR(ret);
+
+        /* Finalize PIO odd/even intracomm. */
+        if ((ret = PIOc_finalize(iosysid)))
+            ERR(ret);
+
+
+        /* Finalize PIO world intracomm. */
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+    }/* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem2_simple2.c b/src/externals/pio2/tests/cunit/test_iosystem2_simple2.c
new file mode 100644
index 00000000000..018caaedb9d
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem2_simple2.c
@@ -0,0 +1,142 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran pio_iosystem_tests2.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem2_simple2"
+
+/* Number of test files generated. */
+#define NUM_FILES 3
+
+/* Used to define netcdf test file. */
+#define PIO_TF_MAX_STR_LEN 100
+#define ATTNAME "filename"
+#define DIMNAME "filename_dim"
+
+/* Needed to init intracomm. */
+#define STRIDE 1
+#define BASE 0
+#define REARRANGER 1
+
+/* Run test. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid; /* The ID for the parallel I/O system. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+			     &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Only do something on the first TARGET_NTASKS tasks. */
+    if (my_rank < TARGET_NTASKS)
+    {
+	/* Figure out iotypes. */
+	if ((ret = get_iotypes(&num_flavors, flavor)))
+	    ERR(ret);
+
+        /* Split world into odd and even. */
+        MPI_Comm newcomm;
+        int even = my_rank % 2 ? 0 : 1;
+        if ((ret = MPI_Comm_split(test_comm, even, 0, &newcomm)))
+            MPIERR(ret);
+        printf("%d newcomm = %d even = %d\n", my_rank, newcomm, even);
+
+        /* Get size of new communicator. */
+        int new_size;
+        if ((ret = MPI_Comm_size(newcomm, &new_size)))
+            MPIERR(ret);
+
+        /* Initialize an intracomm for evens/odds. */
+        if ((ret = PIOc_Init_Intracomm(newcomm, new_size, STRIDE, BASE, REARRANGER, &iosysid)))
+            ERR(ret);
+
+        /* Initialize an intracomm for all processes. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, STRIDE, BASE, REARRANGER,
+                                       &iosysid_world)))
+            ERR(ret);
+
+        for (int flv = 0; flv < num_flavors; flv++)
+        {
+            char filename[NUM_SAMPLES][NC_MAX_NAME + 1]; /* Test filename. */
+            int sample_ncid[NUM_SAMPLES];
+
+            for (int sample = 0; sample < NUM_SAMPLES; sample++)
+            {
+                char iotype_name[NC_MAX_NAME + 1];
+
+                /* Create a filename. */
+                if ((ret = get_iotype_name(flavor[flv], iotype_name)))
+                    return ret;
+                sprintf(filename[sample], "%s_%s_%d_%d.nc", TEST_NAME, iotype_name, sample, 0);
+
+                /* Create sample file. */
+                printf("%d %s creating file %s\n", my_rank, TEST_NAME, filename[sample]);
+                if ((ret = create_nc_sample(sample, iosysid_world, flavor[flv], filename[sample],
+					    my_rank, NULL)))
+                    ERR(ret);
+
+                /* Check the file for correctness. */
+                if ((ret = check_nc_sample(sample, iosysid_world, flavor[flv], filename[sample],
+					   my_rank, &sample_ncid[sample])))
+                    ERR(ret);
+
+            }
+
+            /* Now check the files with the other iosysid. Even and odd
+             * processes will check different files. */
+            int this_sample = even ? 0 : 1;
+            char *file1 = filename[this_sample];
+            int ncid2;
+            if ((ret = check_nc_sample(this_sample, iosysid, flavor[flv], file1, my_rank, &ncid2)))
+                ERR(ret);
+
+            /* Now close the open files. */
+            for (int sample = 0; sample < NUM_SAMPLES; sample++)
+                if ((ret = PIOc_closefile(sample_ncid[sample])))
+                    ERR(ret);
+
+            if ((ret = PIOc_closefile(ncid2)))
+                ERR(ret);
+        } /* next iotype */
+        if ((ret = MPI_Comm_free(&newcomm)))
+            MPIERR(ret);
+
+        /* Finalize PIO odd/even intracomm. */
+        if ((ret = PIOc_finalize(iosysid)))
+            ERR(ret);
+
+        /* Finalize PIO world intracomm. */
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+    }  /* my_rank < TARGET_NTASKS */
+
+    /* Wait for all task before finalizing. */
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem3.c b/src/externals/pio2/tests/cunit/test_iosystem3.c
new file mode 100644
index 00000000000..53b0c63b67c
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem3.c
@@ -0,0 +1,375 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran
+ * pio_iosystem_tests3.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem3"
+
+/* Used to define netcdf test file. */
+#define PIO_TF_MAX_STR_LEN 100
+#define ATTNAME "filename"
+#define DIMNAME "filename_dim"
+
+/* Used to devide up the tasks into MPI groups. */
+#define OVERLAP_NUM_RANGES 2
+#define EVEN_NUM_RANGES 1
+
+/* Used when initializing PIO. */
+#define STRIDE1 1
+#define STRIDE2 2
+#define BASE0 0
+#define BASE1 1
+#define NUM_IO1 1
+#define NUM_IO2 2
+#define NUM_IO4 4
+#define REARRANGER 2
+
+/* This creates a netCDF file in the specified format, with some
+ * sample values. */
+int create_file(MPI_Comm comm, int iosysid, int format, char *filename,
+                char *attname, char *dimname, int my_rank)
+{
+    int ncid, varid, dimid;
+    int ret;
+
+    /* Create the file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &format, filename, NC_CLOBBER)))
+        return ret;
+    printf("%d file created ncid = %d\n", my_rank, ncid);
+
+    /* Define a dimension. */
+    printf("%d defining dimension %s\n", my_rank, dimname);
+    if ((ret = PIOc_def_dim(ncid, dimname, PIO_TF_MAX_STR_LEN, &dimid)))
+        return ret;
+
+    /* Define a 1-D variable. */
+    printf("%d defining variable %s\n", my_rank, attname);
+    if ((ret = PIOc_def_var(ncid, attname, NC_CHAR, 1, &dimid, &varid)))
+        return ret;
+
+    /* Write an attribute. */
+    if ((ret = PIOc_put_att_text(ncid, varid, attname, strlen(filename), filename)))
+        return ret;
+
+    /* End define mode. */
+    printf("%d ending define mode ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+    printf("%d define mode ended ncid = %d\n", my_rank, ncid);
+
+    /* Close the file. */
+    printf("%d closing file ncid = %d\n", my_rank, ncid);
+    if ((ret = PIOc_closefile(ncid)))
+        return ret;
+    printf("%d closed file ncid = %d\n", my_rank, ncid);
+
+    return PIO_NOERR;
+}
+
+/* This checks an already-open netCDF file. */
+int check_file(MPI_Comm comm, int iosysid, int format, int ncid, char *filename,
+               char *attname, char *dimname, int my_rank)
+{
+    int dimid;
+    int varid;
+    char *att_data;
+    int ret;
+
+    /* Check the dimid. */
+    if ((ret = PIOc_inq_dimid(ncid, dimname, &dimid)))
+        return ret;
+    if (dimid)
+        return ERR_WRONG;
+
+    /* Check the varid (it's got the same name as the att). */
+    if ((ret = PIOc_inq_varid(ncid, attname, &varid)))
+        return ret;
+    if (varid)
+        return ERR_WRONG;
+
+    /* Check the attribute. Null terminating byte deliberately ignored
+     * to match fortran code. */
+    if (!(att_data = malloc(strlen(filename) * sizeof(char))))
+        return PIO_ENOMEM;
+    if ((ret = PIOc_get_att(ncid, varid, attname, att_data)))
+        return ret;
+    printf("%d DONE with get_att!!!\n", my_rank);
+    if (strncmp(att_data, filename, strlen(filename)))
+        return ERR_WRONG;
+    free(att_data);
+    printf("%d DONE with get_att!!!\n", my_rank);
+
+    return PIO_NOERR;
+}
+
+/* This opens and checks a netCDF file. */
+int open_and_check_file(MPI_Comm comm, int iosysid, int iotype, int *ncid, char *fname,
+                        char *attname, char *dimname, int disable_close, int my_rank)
+{
+    int mode = PIO_WRITE;
+    int ret;
+
+    /* Open the file. */
+    if ((ret = PIOc_openfile(iosysid, ncid, &iotype, fname, mode)))
+        return ret;
+
+    /* Check the file. */
+    if ((ret = check_file(comm, iosysid, iotype, *ncid, fname, attname, dimname, my_rank)))
+        return ret;
+
+    /* Close the file, maybe. */
+    if (!disable_close)
+        if ((ret = PIOc_closefile(*ncid)))
+            return ret;
+
+    return PIO_NOERR;
+}
+
+/* Run async tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    int even_iosysid; /* The ID for iosystem of even_comm. */
+    int overlap_iosysid; /* The ID for iosystem of even_comm. */
+    MPI_Group world_group; /* An MPI group of world. */
+    MPI_Group even_group; /* An MPI group of 0 and 2. */
+    MPI_Group overlap_group; /* An MPI group of 0, 1, and 3. */
+    MPI_Comm even_comm = MPI_COMM_NULL; /* Communicator for tasks 0, 2 */
+    MPI_Comm overlap_comm = MPI_COMM_NULL; /* Communicator for tasks 0, 1, 2. */
+    int even_rank = -1, overlap_rank = -1; /* Tasks rank in communicator. */
+    int even_size = 0, overlap_size = 0; /* Size of communicator. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+                             &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Initialize PIO system on world. */
+        printf("%d about to call Init_Intracomm\n", my_rank);
+        if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO4, STRIDE1, BASE0, REARRANGER, &iosysid_world)))
+            ERR(ret);
+        printf("%d done with Init_Intracomm\n", my_rank);
+
+        /* Set the error handler. */
+        /*PIOc_Set_IOSystem_Error_Handling(iosysid_world, PIO_BCAST_ERROR);*/
+        printf("%d about to set iosystem error hanlder for world\n", my_rank);
+        if ((ret = PIOc_set_iosystem_error_handling(iosysid_world, PIO_BCAST_ERROR, NULL)))
+            ERR(ret);
+        printf("%d done setting iosystem error hanlder for world\n", my_rank);
+
+        /* Get MPI_Group of world comm. */
+        if ((ret = MPI_Comm_group(test_comm, &world_group)))
+            ERR(ret);
+
+        /* Create a group with tasks 0 and 2. */
+        int even_ranges[EVEN_NUM_RANGES][3] = {{0, 2, 2}};
+        if ((ret = MPI_Group_range_incl(world_group, EVEN_NUM_RANGES, even_ranges, &even_group)))
+            ERR(ret);
+
+        /* Create a communicator from the even_group. */
+        if ((ret = MPI_Comm_create(test_comm, even_group, &even_comm)))
+            ERR(ret);
+
+        /* Learn my rank and the total number of processors in even group. */
+        if (even_comm != MPI_COMM_NULL)
+        {
+            if ((ret = MPI_Comm_rank(even_comm, &even_rank)))
+                MPIERR(ret);
+            if ((ret = MPI_Comm_size(even_comm, &even_size)))
+                MPIERR(ret);
+        }
+        printf("%d even_comm = %d even_rank = %d even_size = %d\n", my_rank,
+               even_comm, even_rank, even_size);
+
+        /* Create a group with tasks 0, 1, and 3. */
+        int overlap_ranges[OVERLAP_NUM_RANGES][3] = {{0, 0, 1}, {1, 3, 2}};
+        if ((ret = MPI_Group_range_incl(world_group, OVERLAP_NUM_RANGES,
+                                        overlap_ranges, &overlap_group)))
+            ERR(ret);
+
+        /* Create a communicator from the overlap_group. */
+        if ((ret = MPI_Comm_create(test_comm, overlap_group, &overlap_comm)))
+            ERR(ret);
+
+        /* Learn my rank and the total number of processors in overlap
+         * group. */
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            if ((ret = MPI_Comm_rank(overlap_comm, &overlap_rank)))
+                MPIERR(ret);
+            if ((ret = MPI_Comm_size(overlap_comm, &overlap_size)))
+                MPIERR(ret);
+        }
+        printf("%d overlap_comm = %d overlap_rank = %d overlap_size = %d\n", my_rank,
+               overlap_comm, overlap_rank, overlap_size);
+
+        /* Initialize PIO system for even. */
+        if (even_comm != MPI_COMM_NULL)
+        {
+            if ((ret = PIOc_Init_Intracomm(even_comm, NUM_IO1, STRIDE1, BASE1, REARRANGER, &even_iosysid)))
+                ERR(ret);
+
+            /* These should not work. */
+            if (PIOc_set_hint(even_iosysid + TEST_VAL_42, NULL, NULL) != PIO_EBADID)
+                ERR(ERR_WRONG);
+            if (PIOc_set_hint(even_iosysid, NULL, NULL) != PIO_EINVAL)
+                ERR(ERR_WRONG);
+
+            /* Set the hint (which will be ignored). */
+            if ((ret = PIOc_set_hint(even_iosysid, "hint", "hint_value")))
+                ERR(ret);
+
+            /* Set the error handler. */
+            /*PIOc_Set_IOSystem_Error_Handling(even_iosysid, PIO_BCAST_ERROR);*/
+            printf("%d about to set iosystem error hanlder for even\n", my_rank);
+            if ((ret = PIOc_set_iosystem_error_handling(even_iosysid, PIO_BCAST_ERROR, NULL)))
+                ERR(ret);
+            printf("%d done setting iosystem error hanlder for even\n", my_rank);
+        }
+
+        /* Initialize PIO system for overlap comm. */
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            if ((ret = PIOc_Init_Intracomm(overlap_comm, NUM_IO2, STRIDE1, BASE1, REARRANGER,
+                                           &overlap_iosysid)))
+                ERR(ret);
+
+            printf("%d about to set iosystem error hanlder for overlap\n", my_rank);
+            /* Set the error handler. */
+            /* if ((ret = PIOc_set_iosystem_error_handling(overlap_iosysid, PIO_BCAST_ERROR))) */
+            /*     ERR(ret); */
+            PIOc_Set_IOSystem_Error_Handling(overlap_iosysid, PIO_BCAST_ERROR);
+            printf("%d done setting iosystem error hanlder for overlap\n", my_rank);
+        }
+
+        for (int i = 0; i < num_flavors; i++)
+        {
+            char fname0[] = "pio_iosys_test_file0.nc";
+            char fname1[] = "pio_iosys_test_file1.nc";
+            char fname2[] = "pio_iosys_test_file2.nc";
+            printf("\n\n%d i = %d\n", my_rank, i);
+
+            if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname0, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname1, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            if ((ret = create_file(test_comm, iosysid_world, flavor[i], fname2, ATTNAME,
+                                   DIMNAME, my_rank)))
+                ERR(ret);
+
+            /* Now check the first file from WORLD communicator. */
+            int ncid;
+            if ((ret = open_and_check_file(test_comm, iosysid_world, flavor[i], &ncid, fname0,
+                                           ATTNAME, DIMNAME, 1, my_rank)))
+                ERR(ret);
+
+            /* Now have the even communicators check the files. */
+            int ncid2;
+            if (even_comm != MPI_COMM_NULL)
+            {
+                printf("\n***\n%d Checking file for even_comm\n", my_rank);
+                if ((ret = open_and_check_file(even_comm, even_iosysid, flavor[i], &ncid2, fname2,
+                                               ATTNAME, DIMNAME, 1, my_rank)))
+                    ERR(ret);
+                if ((ret = check_file(even_comm, even_iosysid, flavor[i], ncid2, fname2,
+                                      ATTNAME, DIMNAME, my_rank)))
+                    ERR(ret);
+            }
+
+            /* Now have the overlap communicators check the files. */
+            int ncid3;
+            if (overlap_comm != MPI_COMM_NULL)
+            {
+                printf("\n***%d Checking file for overlap_comm\n", my_rank);
+                if ((ret = open_and_check_file(overlap_comm, overlap_iosysid, flavor[i], &ncid3, fname1,
+                                               ATTNAME, DIMNAME, 1, my_rank)))
+                    ERR(ret);
+                if ((ret = check_file(overlap_comm, overlap_iosysid, flavor[i], ncid3, fname1,
+                                      ATTNAME, DIMNAME, my_rank)))
+                    ERR(ret);
+            }
+
+            /* Close the still-open files. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+            if (even_comm != MPI_COMM_NULL)
+            {
+                if ((ret = PIOc_closefile(ncid2)))
+                    ERR(ret);
+            }
+            if (overlap_comm != MPI_COMM_NULL)
+            {
+                if ((ret = PIOc_closefile(ncid3)))
+                    ERR(ret);
+            }
+        } /* next iotype */
+        /* Finalize PIO systems. */
+        printf("%d pio finalizing %d\n", my_rank, even_iosysid);
+        if (even_comm != MPI_COMM_NULL)
+            if ((ret = PIOc_finalize(even_iosysid)))
+                ERR(ret);
+        printf("%d pio finalizing %d\n", my_rank, overlap_iosysid);
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            printf("%d calling PIOc_finalize with iosysid = %d\n", my_rank, overlap_iosysid);
+            if ((ret = PIOc_finalize(overlap_iosysid)))
+                ERR(ret);
+        }
+        printf("%d pio finalized\n", my_rank);
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+
+        /* Free MPI resources used by test. */
+        if ((ret = MPI_Group_free(&overlap_group)))
+            ERR(ret);
+        if ((ret = MPI_Group_free(&even_group)))
+            ERR(ret);
+        if ((ret = MPI_Group_free(&world_group)))
+            ERR(ret);
+        if (overlap_comm != MPI_COMM_NULL)
+            if ((ret = MPI_Comm_free(&overlap_comm)))
+                ERR(ret);
+        if (even_comm != MPI_COMM_NULL)
+            if ((ret = MPI_Comm_free(&even_comm)))
+                ERR(ret);
+
+    } /* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem3_simple.c b/src/externals/pio2/tests/cunit/test_iosystem3_simple.c
new file mode 100644
index 00000000000..7ac3ad2ebad
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem3_simple.c
@@ -0,0 +1,123 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran pio_iosystem_tests2.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem3_simple"
+
+/* Needed to init intracomm. */
+#define STRIDE 1
+#define BASE 0
+#define REARRANGER 1
+
+/* Used to devide up the tasks into MPI groups. */
+#define OVERLAP_NUM_RANGES 2
+
+/* Run test. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    int overlap_iosysid; /* The ID for iosystem of overlap_comm. */
+    MPI_Group world_group; /* An MPI group of world. */
+    MPI_Group overlap_group; /* An MPI group of 0, 1, and 3. */
+    MPI_Comm overlap_comm = MPI_COMM_NULL; /* Communicator for tasks 0, 1, 2. */
+    int overlap_rank = -1; /* Tasks rank in communicator. */
+    int overlap_size = 0; /* Size of communicator. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+			     &test_comm)))
+        ERR(ERR_INIT);
+    
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+
+        /* Turn on logging. */
+        if ((ret = PIOc_set_log_level(3)))
+            return ret;
+
+        /* Initialize PIO system on world. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, 4, 1, 0, 1, &iosysid_world)))
+            ERR(ret);
+
+        /* Get MPI_Group of world comm. */
+        if ((ret = MPI_Comm_group(test_comm, &world_group)))
+            ERR(ret);
+
+        /* Create a group with tasks 0, 1, 3. */
+        int overlap_ranges[OVERLAP_NUM_RANGES][3] = {{0, 0, 1}, {1, 3, 2}};
+        if ((ret = MPI_Group_range_incl(world_group, OVERLAP_NUM_RANGES,
+                                        overlap_ranges, &overlap_group)))
+            ERR(ret);
+
+        /* Create a communicator from the overlap_group. */
+        if ((ret = MPI_Comm_create(test_comm, overlap_group, &overlap_comm)))
+            ERR(ret);
+
+        /* Learn my rank and the total number of processors in overlap
+         * group. */
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            if ((ret = MPI_Comm_rank(overlap_comm, &overlap_rank)))
+                MPIERR(ret);
+            if ((ret = MPI_Comm_size(overlap_comm, &overlap_size)))
+                MPIERR(ret);
+        }
+        printf("%d overlap_comm = %d overlap_rank = %d overlap_size = %d\n", my_rank,
+               overlap_comm, overlap_rank, overlap_size);
+
+        /* Initialize PIO system for overlap comm. */
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            if ((ret = PIOc_Init_Intracomm(overlap_comm, 1, 1, 0, 1, &overlap_iosysid)))
+                ERR(ret);
+        }
+
+        printf("%d pio finalizing %d\n", my_rank, overlap_iosysid);
+        /* Finalize PIO system. */
+        if (overlap_comm != MPI_COMM_NULL)
+        {
+            printf("%d calling PIOc_finalize with iosysid = %d\n", my_rank, overlap_iosysid);
+            if ((ret = PIOc_finalize(overlap_iosysid)))
+                ERR(ret);
+        }
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+        printf("%d pio finalized\n", my_rank);
+
+        /* Free MPI resources used by test. */
+        if ((ret = MPI_Group_free(&overlap_group)))
+            ERR(ret);
+        if ((ret = MPI_Group_free(&world_group)))
+            ERR(ret);
+        if (overlap_comm != MPI_COMM_NULL)
+            if ((ret = MPI_Comm_free(&overlap_comm)))
+                ERR(ret);
+        printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+    }
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_iosystem3_simple2.c b/src/externals/pio2/tests/cunit/test_iosystem3_simple2.c
new file mode 100644
index 00000000000..973ef16c7eb
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_iosystem3_simple2.c
@@ -0,0 +1,101 @@
+/*
+ * Tests the PIO library with multiple iosysids in use at the
+ * same time.
+ *
+ * This is a simplified, C version of the fortran
+ * pio_iosystem_tests3.F90.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_iosystem3"
+
+/* Used when initializing PIO. */
+#define STRIDE1 1
+#define BASE0 0
+#define NUM_IO4 4
+#define REARRANGER 1
+
+/* Run async tests. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks; /* Number of processors involved in current execution. */
+    int iosysid_world; /* The ID for the parallel I/O system. */
+    char fname0[NC_MAX_NAME + 1];
+    int ncid;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    int ret; /* Return code. */
+    MPI_Comm test_comm;
+
+    /* Initialize test. */
+    if ((ret = pio_test_init(argc, argv, &my_rank, &ntasks, TARGET_NTASKS,
+                             &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+
+        /* Initialize PIO system on world. */
+        if ((ret = PIOc_Init_Intracomm(test_comm, NUM_IO4, STRIDE1, BASE0, REARRANGER, &iosysid_world)))
+            ERR(ret);
+
+        for (int i = 0; i < num_flavors; i++)
+        {
+            /* Create the file. */
+            sprintf(fname0, "test_iosystem3_simple2_%d.nc", i);
+            if ((ret = PIOc_createfile(iosysid_world, &ncid, &flavor[i], fname0, NC_CLOBBER)))
+                return ret;
+
+            /* End define mode. */
+            if ((ret = PIOc_enddef(ncid)))
+                return ret;
+
+            /* Close the file. */
+            if ((ret = PIOc_closefile(ncid)))
+                return ret;
+
+            /* Now check the first file from WORLD communicator. */
+            int mode = PIO_WRITE;
+
+            /* Open the file. */
+            if ((ret = PIOc_openfile(iosysid_world, &ncid, &flavor[i], fname0, mode)))
+                return ret;
+
+            /* Check the file. */
+            int ndims;
+            if ((ret = PIOc_inq(ncid, &ndims, NULL, NULL, NULL)))
+                return ret;
+
+            /* Close the file. */
+            if ((ret = PIOc_closefile(ncid)))
+                return ret;
+        } /* next iotype */
+
+        /* Finalize PIO systems. */
+        printf("%d pio finalizing\n", my_rank);
+        if ((ret = PIOc_finalize(iosysid_world)))
+            ERR(ret);
+    } /* my_rank < TARGET_NTASKS */
+
+    /* Finalize test. */
+    printf("%d %s finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ERR_AWFUL;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_pioc.c b/src/externals/pio2/tests/cunit/test_pioc.c
new file mode 100644
index 00000000000..466ed99f706
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_pioc.c
@@ -0,0 +1,2138 @@
+/*
+ * Tests for PIO Functions.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_pioc"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 40
+#define Y_DIM_LEN 40
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 1
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "foo"
+
+/* The name of the attribute in the netCDF output files. */
+#define ATT_NAME "foo"
+#define ATT_NAME2 "bar"
+
+/* Value to write to attributes. */
+#define ATT_VAL 42
+
+/* The meaning of life, the universe, and everything. */
+#define START_DATA_VAL 42
+
+/* Values for some netcdf-4 settings. */
+#define VAR_CACHE_SIZE (1024 * 1024)
+#define VAR_CACHE_NELEMS 10
+#define VAR_CACHE_PREEMPTION 0.5
+
+/* Number of NetCDF classic types. */
+#define NUM_CLASSIC_TYPES 6
+
+/* Number of NetCDF-4 types. */
+#define NUM_NETCDF4_TYPES 12
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/* Length of chunksizes to use in netCDF-4 files. */
+PIO_Offset chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2};
+
+#define DIM_NAME "dim"
+#define NDIM1 1
+#define DIM_LEN 4
+
+/* Length of the max maplen in decomp testing. */
+#define MAX_MAPLEN 1
+
+
+/* Create the decomposition to divide the 1-dimensional sample data
+ * between the 4 tasks.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim1_len the length of the dimension.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition(int ntasks, int my_rank, int iosysid, int dim1_len, int *ioid)
+{
+#define NDIM1 1
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    int dim_len[NDIM1] = {dim1_len};
+    int bad_dim_len[NDIM1] = {-50};
+    int ret;
+
+    /* How many data elements per task? */
+    elements_per_pe = dim1_len / ntasks;
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. The new init_decomp uses a 0-based
+     * array, so don't add 1! */
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i;
+
+    /* These should fail. */
+    if (PIOc_init_decomp(iosysid + TEST_VAL_42, PIO_FLOAT, NDIM1, dim_len, elements_per_pe,
+                         compdof, ioid, 0, NULL, NULL) != PIO_EBADID)
+        ERR(ERR_WRONG);
+    if (PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, bad_dim_len, elements_per_pe,
+                         compdof, ioid, 0, NULL, NULL) != PIO_EINVAL)
+        ERR(ERR_WRONG);
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe);
+    if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, dim_len, elements_per_pe,
+                                compdof, ioid, 0, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
+/* Check the contents of the test file. */
+int check_darray_file(int iosysid, int ntasks, int my_rank, char *filename)
+{
+    int ncid;
+    int ndims, nvars, ngatts, unlimdimid;
+    char dim_name_in[PIO_MAX_NAME + 1];
+    PIO_Offset dim_len_in;
+    PIO_Offset arraylen = 1;
+    float data_in;
+    int ioid;
+    int ret;
+
+    assert(filename);
+
+    /* Open the file. */
+    if ((ret = PIOc_open(iosysid, filename, NC_NOWRITE, &ncid)))
+        return ret;
+
+    /* Check metadata. */
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ret;
+    if (ndims != 1 || nvars != 1 || ngatts != 0 || unlimdimid != -1)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_dim(ncid, 0, dim_name_in, &dim_len_in)))
+        return ret;
+    if (strcmp(dim_name_in, DIM_NAME) || dim_len_in != DIM_LEN)
+        return ERR_WRONG;
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition(ntasks, my_rank, iosysid, DIM_LEN, &ioid)))
+        return ret;
+
+    /* Read data. */
+    if ((ret = PIOc_read_darray(ncid, 0, ioid, arraylen, &data_in)))
+        return ret;
+
+    /* Check data. */
+    if (data_in != my_rank * 10)
+        return ERR_WRONG;
+
+    /* Close the file. */
+    if ((ret = PIOc_closefile(ncid)))
+        return ret;
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return PIO_NOERR;
+}
+
+/* Test the darray functionality. */
+int test_darray(int iosysid, int ioid, int num_flavors, int *flavor, int my_rank,
+                int fv)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dim_len[NDIM1] = {DIM_LEN}; /* Length of the dimensions in the sample data. */
+    int dimids[NDIM1];      /* The dimension IDs. */
+    int ncid;      /* The ncid of the netCDF file. */
+    int varid;     /* The ID of the netCDF varable. */
+    int ret;       /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* Create the filename. */
+        sprintf(filename, "%s_flavor_%d_fv_%d.nc", TEST_NAME, flavor[fmt], fv);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename,
+               flavor[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Define netCDF dimensions and variable. */
+        printf("rank: %d Defining netCDF metadata...\n", my_rank);
+        if ((ret = PIOc_def_dim(ncid, DIM_NAME, (PIO_Offset)dim_len[0], &dimids[0])))
+            ERR(ret);
+
+        /* Define a variable. */
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM1, dimids, &varid)))
+            ERR(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Write some data. */
+        PIO_Offset arraylen = 1;
+        float fillvalue = 0.0;
+        float *fillvaluep = fv ? &fillvalue : NULL;
+        float test_data[arraylen];
+        for (int f = 0; f < arraylen; f++)
+            test_data[f] = my_rank * 10 + f;
+        if ((ret = PIOc_write_darray(ncid, varid, ioid, arraylen, test_data, fillvaluep)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("rank: %d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Check the file contents. */
+        if ((ret = check_darray_file(iosysid, TARGET_NTASKS, my_rank, filename)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/* Check the dimension names.
+ *
+ * @param my_rank rank of process
+ * @param ncid ncid of open netCDF file
+ * @returns 0 for success, error code otherwise. */
+int check_dim_names(int my_rank, int ncid, MPI_Comm test_comm)
+{
+    char dim_name[PIO_MAX_NAME + 1];
+    char zero_dim_name[PIO_MAX_NAME + 1];
+    int my_test_rank;
+    int ret;
+
+    /* Find rank in test communicator. */
+    if ((ret = MPI_Comm_rank(test_comm, &my_test_rank)))
+        MPIERR(ret);
+
+    for (int d = 0; d < NDIM; d++)
+    {
+        memset(dim_name, 0, sizeof(dim_name));
+        if ((ret = PIOc_inq_dimname(ncid, d, dim_name)))
+            return ret;
+        printf("my_rank %d my_test_rank %d dim %d name %s\n", my_rank, my_test_rank, d, dim_name);
+
+        /* Did other ranks get the same name? */
+        memset(zero_dim_name, 0, sizeof(zero_dim_name));
+        if (!my_test_rank)
+            strcpy(zero_dim_name, dim_name);
+        printf("rank %d dim_name %s zero_dim_name %s\n", my_rank, dim_name, zero_dim_name);
+        if ((ret = MPI_Bcast(&zero_dim_name, strlen(dim_name) + 1, MPI_CHAR, 0,
+                             test_comm)))
+            MPIERR(ret);
+        printf("%d zero_dim_name = %s dim_name = %s\n", my_rank, zero_dim_name, dim_name);
+        if (strcmp(dim_name, zero_dim_name))
+            return ERR_AWFUL;
+    }
+    return 0;
+}
+
+/* Check the variable name.
+ *
+ * @param my_rank rank of process
+ * @param ncid ncid of open netCDF file
+ *
+ * @returns 0 for success, error code otherwise. */
+int check_var_name(int my_rank, int ncid, MPI_Comm test_comm)
+{
+    char var_name[PIO_MAX_NAME + 1];
+    char zero_var_name[PIO_MAX_NAME + 1];
+    int my_test_rank;
+    int ret;
+
+    /* Find rank in test communicator. */
+    if ((ret = MPI_Comm_rank(test_comm, &my_test_rank)))
+        MPIERR(ret);
+
+    memset(var_name, 0, sizeof(var_name));
+    if ((ret = PIOc_inq_varname(ncid, 0, var_name)))
+        return ret;
+    printf("my_rank %d var name %s\n", my_rank, var_name);
+
+    /* Did other ranks get the same name? */
+    memset(zero_var_name, 0, sizeof(zero_var_name));
+    if (!my_test_rank)
+        strcpy(zero_var_name, var_name);
+    if ((ret = MPI_Bcast(&zero_var_name, strlen(var_name) + 1, MPI_CHAR, 0,
+                         test_comm)))
+        MPIERR(ret);
+    if (strcmp(var_name, zero_var_name))
+        return ERR_AWFUL;
+    return 0;
+}
+
+/* Check the attribute name.
+ *
+ * @param my_rank rank of process
+ * @param ncid ncid of open netCDF file
+ * @param flavor the iotype
+ * @param test_comm the MPI communicator of the test.
+ * @param async 1 if we are testing async, 0 otherwise.
+ * @returns 0 for success, error code otherwise.
+ */
+int check_atts(int my_rank, int ncid, int flavor, MPI_Comm test_comm, int async)
+{
+    char att_name[PIO_MAX_NAME + 1];
+    char zero_att_name[PIO_MAX_NAME + 1];
+    int my_test_rank;
+    nc_type att_type;
+    PIO_Offset att_len;
+    signed char att_schar_value;
+    short att_short_value;
+    int att_int_value;
+    float att_float_value;
+    double att_double_value;
+    unsigned char att_uchar_value;
+    unsigned short att_ushort_value;
+    unsigned int att_uint_value;
+    long long int att_int64_value;
+    unsigned long long int att_uint64_value;
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+    int ret;
+
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+
+    /* Find rank in test communicator. */
+    if ((ret = MPI_Comm_rank(test_comm, &my_test_rank)))
+        MPIERR(ret);
+
+    memset(att_name, 0, sizeof(att_name));
+    if ((ret = PIOc_inq_attname(ncid, NC_GLOBAL, 0, att_name)))
+        return ret;
+
+    /* Did all ranks get the same name? */
+    memset(zero_att_name, 0, sizeof(zero_att_name));
+    if (!my_test_rank)
+        strcpy(zero_att_name, att_name);
+    if ((ret = MPI_Bcast(&zero_att_name, strlen(att_name) + 1, MPI_CHAR, 0,
+                         test_comm)))
+        MPIERR(ret);
+    if (strcmp(att_name, zero_att_name))
+        return ERR_AWFUL;
+
+    /* These should not work. */
+    if (PIOc_get_att_int(ncid + TEST_VAL_42, NC_GLOBAL, ATT_NAME, &att_int_value) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_get_att_int(ncid, TEST_VAL_42, ATT_NAME, &att_int_value) != PIO_ENOTVAR)
+        return ERR_WRONG;
+    if (PIOc_get_att_int(ncid, NC_GLOBAL, NULL, &att_int_value) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* These should not work. */
+    if (PIOc_inq_att(ncid, NC_GLOBAL, too_long_name, &att_type, &att_len) != PIO_EINVAL)
+        return ERR_WRONG;
+    int tmp_attid;
+    if (PIOc_inq_attid(ncid, NC_GLOBAL, too_long_name, &tmp_attid) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Check first att. */
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME, &att_type, &att_len)))
+        return ret;
+    if (att_type != PIO_INT || att_len != 1)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME, &att_int_value)))
+        return ret;
+    if (att_int_value != ATT_VAL)
+        return ERR_WRONG;
+
+    /* Try the typeless get. */
+    int att_int_value2;
+    if ((ret = PIOc_get_att(ncid, NC_GLOBAL, ATT_NAME, &att_int_value2)))
+        ERR(ret);
+    if (att_int_value2 != ATT_VAL)
+        return ERR_WRONG;
+
+    /* Check second att. */
+    if ((ret = PIOc_inq_att(ncid, NC_GLOBAL, ATT_NAME2, &att_type, &att_len)))
+        return ret;
+    if (att_type != PIO_FLOAT || att_len != 1)
+        return ERR_WRONG;
+
+    /* Try converting to every type. */
+    if ((ret = PIOc_get_att_schar(ncid, NC_GLOBAL, ATT_NAME2, &att_schar_value)))
+        return ret;
+    if (att_schar_value != ATT_VAL)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_short(ncid, NC_GLOBAL, ATT_NAME2, &att_short_value)))
+        return ret;
+    if (att_short_value != ATT_VAL)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_int(ncid, NC_GLOBAL, ATT_NAME2, &att_int_value)))
+        return ret;
+    if (att_int_value != ATT_VAL)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_float(ncid, NC_GLOBAL, ATT_NAME2, &att_float_value)))
+        return ret;
+    if (att_float_value != ATT_VAL)
+        return ERR_WRONG;
+    if ((ret = PIOc_get_att_double(ncid, NC_GLOBAL, ATT_NAME2, &att_double_value)))
+        return ret;
+    if (att_double_value != ATT_VAL)
+        return ERR_WRONG;
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_att_uchar(ncid, NC_GLOBAL, ATT_NAME2, &att_uchar_value)))
+            return ret;
+        if (att_uchar_value != ATT_VAL)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_att_ushort(ncid, NC_GLOBAL, ATT_NAME2, &att_ushort_value)))
+            return ret;
+        if (att_ushort_value != ATT_VAL)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_att_uint(ncid, NC_GLOBAL, ATT_NAME2, &att_uint_value)))
+            return ret;
+        if (att_uint_value != ATT_VAL)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_att_longlong(ncid, NC_GLOBAL, ATT_NAME2, &att_int64_value)))
+            return ret;
+        if (att_int64_value != ATT_VAL)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_att_ulonglong(ncid, NC_GLOBAL, ATT_NAME2, &att_uint64_value)))
+            return ret;
+        if (att_uint64_value != ATT_VAL)
+            return ERR_WRONG;
+    }
+    return 0;
+}
+
+/*
+ * Check error strings.
+ *
+ * @param my_rank rank of this task.
+ * @param num_tries number of errcodes to try.
+ * @param errcode pointer to array of error codes, of length num_tries.
+ * @param expected pointer to an array of strings, with the expected
+ * error messages for each error code.
+ * @returns 0 for success, error code otherwise.
+ */
+int check_error_strings(int my_rank, int num_tries, int *errcode,
+                        const char **expected)
+{
+    int ret;
+
+    /* Try each test code. */
+    for (int try = 0; try < num_tries; try++)
+    {
+        char errstr[PIO_MAX_NAME + 1];
+
+        /* Get the error string for this errcode. */
+        if ((ret = PIOc_strerror(errcode[try], errstr)))
+            return ret;
+
+        printf("%d for errcode = %d message = %s\n", my_rank, errcode[try], errstr);
+
+        /* Check that it was as expected. */
+        if (strncmp(errstr, expected[try], strlen(expected[try])))
+        {
+            printf("%d expected %s got %s\n", my_rank, expected[try], errstr);
+            return ERR_AWFUL;
+        }
+        if (!my_rank)
+            printf("%d errcode = %d passed\n", my_rank, errcode[try]);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_iotype_available() function.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int test_iotypes(int my_rank)
+{
+    /* This is never present. */
+    if (PIOc_iotype_available(1000))
+        return ERR_WRONG;
+
+    /* NetCDF is always present. */
+    if (!PIOc_iotype_available(PIO_IOTYPE_NETCDF))
+        return ERR_WRONG;
+
+    /* Pnetcdf may or may not be present. */
+#ifdef _PNETCDF
+    if (!PIOc_iotype_available(PIO_IOTYPE_PNETCDF))
+        return ERR_WRONG;
+#else
+    if (PIOc_iotype_available(PIO_IOTYPE_PNETCDF))
+        return ERR_WRONG;
+#endif /* _PNETCDF */
+
+    /* NetCDF-4 may or may not be present. */
+#ifdef _NETCDF4
+    if (!PIOc_iotype_available(PIO_IOTYPE_NETCDF4C))
+        return ERR_WRONG;
+    if (!PIOc_iotype_available(PIO_IOTYPE_NETCDF4P))
+        return ERR_WRONG;
+#else
+    if (PIOc_iotype_available(PIO_IOTYPE_NETCDF4C))
+        return ERR_WRONG;
+    if (PIOc_iotype_available(PIO_IOTYPE_NETCDF4P))
+        return ERR_WRONG;
+#endif /* _NETCDF4 */
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_strerror() function for classic netCDF.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int check_strerror_netcdf(int my_rank)
+{
+#define NUM_NETCDF_TRIES 5
+    int errcode[NUM_NETCDF_TRIES] = {PIO_EBADID, NC4_LAST_ERROR - 1, 0, 1, -600};
+    const char *expected[NUM_NETCDF_TRIES] = {"NetCDF: Not a valid ID",
+                                              "Unknown Error: Unrecognized error code", "No error",
+                                              nc_strerror(1), "Unknown Error: Unrecognized error code"};
+    int ret;
+
+    if ((ret = check_error_strings(my_rank, NUM_NETCDF_TRIES, errcode, expected)))
+        return ret;
+
+    /* When called with a code of 0, these functions should do nothing
+     * and return 0. */
+    if (check_mpi(NULL, 0, __FILE__, __LINE__))
+        return ERR_WRONG;
+    if (check_mpi2(NULL, NULL, 0, __FILE__, __LINE__))
+        return ERR_WRONG;
+    if (pio_err(NULL, NULL, 0, __FILE__, __LINE__))
+        return ERR_WRONG;
+    if (check_netcdf(NULL, 0, __FILE__, __LINE__))
+        return ERR_WRONG;
+    if (check_netcdf2(NULL, NULL, 0, __FILE__, __LINE__))
+        return ERR_WRONG;
+
+    /* When called with other error messages, these functions should
+     * return PIO_EIO. */
+    if (check_mpi(NULL, MPI_ERR_OTHER, __FILE__, __LINE__) != PIO_EIO)
+        return ERR_WRONG;
+    if (check_mpi(NULL, MPI_ERR_UNKNOWN, __FILE__, __LINE__) != PIO_EIO)
+        return ERR_WRONG;
+
+    if (!my_rank)
+        printf("check_strerror_netcdf SUCCEEDED!\n");
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_strerror() function for netCDF-4.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int check_strerror_netcdf4(int my_rank)
+{
+#ifdef _NETCDF4
+#define NUM_NETCDF4_TRIES 2
+    int errcode[NUM_NETCDF4_TRIES] = {NC_ENOTNC3, NC_ENOPAR};
+    const char *expected[NUM_NETCDF4_TRIES] =
+        {"NetCDF: Attempting netcdf-3 operation on netcdf-4 file",
+         "NetCDF: Parallel operation on file opened for non-parallel access"};
+    int ret;
+
+    if ((ret = check_error_strings(my_rank, NUM_NETCDF4_TRIES, errcode, expected)))
+        return ret;
+
+    if (!my_rank)
+        printf("check_strerror_netcdf4 SUCCEEDED!\n");
+#endif /* _NETCDF4 */
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_strerror() function for parallel-netCDF.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int check_strerror_pnetcdf(int my_rank)
+{
+#ifdef _PNETCDF
+#define NUM_PNETCDF_TRIES 2
+    int errcode[NUM_PNETCDF_TRIES] = {NC_EMULTIDEFINE_VAR_NUM, NC_EMULTIDEFINE_ATTR_VAL};
+    const char *expected[NUM_PNETCDF_TRIES] =
+        {"Number of variables is",
+         "Attribute value is inconsistent among processes."};
+    int ret;
+
+    if ((ret = check_error_strings(my_rank, NUM_PNETCDF_TRIES, errcode, expected)))
+        return ret;
+
+    if (!my_rank)
+        printf("check_strerror_pnetcdf SUCCEEDED!\n");
+#endif /* _PNETCDF */
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_strerror() function for PIO.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int check_strerror_pio(int my_rank)
+{
+#define NUM_PIO_TRIES 6
+    int errcode[NUM_PIO_TRIES] = {PIO_EBADID,
+                                  NC_ENOTNC3, NC4_LAST_ERROR - 1, 0, 1,
+                                  PIO_EBADIOTYPE};
+    const char *expected[NUM_PIO_TRIES] = {"NetCDF: Not a valid ID",
+                                           "NetCDF: Attempting netcdf-3 operation on netcdf-4 file",
+                                           "Unknown Error: Unrecognized error code", "No error",
+                                           nc_strerror(1), "Bad IO type"};
+    int ret;
+
+    if ((ret = check_error_strings(my_rank, NUM_PIO_TRIES, errcode, expected)))
+        return ret;
+
+    if (!my_rank)
+        printf("check_strerror_pio SUCCEEDED!\n");
+
+    return PIO_NOERR;
+}
+
+/* Check the PIOc_strerror() function.
+ *
+ * @param my_rank the rank of this process.
+ * @return 0 for success, error code otherwise.
+ */
+int check_strerror(int my_rank)
+{
+    int ret;
+
+    printf("checking strerror for netCDF-classic error codes...\n");
+    if ((ret = check_strerror_netcdf(my_rank)))
+        return ret;
+
+    printf("checking strerror for netCDF-4 error codes...\n");
+    if ((ret = check_strerror_netcdf4(my_rank)))
+        return ret;
+
+    printf("checking strerror for pnetcdf error codes...\n");
+    if ((ret = check_strerror_pnetcdf(my_rank)))
+        return ret;
+
+    printf("checking strerror for PIO error codes...\n");
+    if ((ret = check_strerror_pio(my_rank)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/* Define metadata for the test file. */
+int define_metadata(int ncid, int my_rank, int flavor)
+{
+    int dimids[NDIM]; /* The dimension IDs. */
+    int varid; /* The variable ID. */
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+    int ret;
+
+    /* Check invalid parameters. */
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+    if (PIOc_def_dim(ncid + 1, dim_name[0], (PIO_Offset)dim_len[0], &dimids[0]) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_def_dim(ncid, NULL, (PIO_Offset)dim_len[0], &dimids[0]) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_dim(ncid, too_long_name, (PIO_Offset)dim_len[0], &dimids[0]) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Define dimensions. */
+    for (int d = 0; d < NDIM; d++)
+        if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+            return ret;
+
+    /* Check invalid parameters. */
+    if (PIOc_def_var(ncid + 1, VAR_NAME, PIO_INT, NDIM, dimids, &varid) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_def_var(ncid, NULL, PIO_INT, NDIM, dimids, &varid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, dimids, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_var(ncid, too_long_name, PIO_INT, NDIM, dimids, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_var(ncid, too_long_name, TEST_VAL_42, NDIM, dimids, &varid) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_var(ncid, too_long_name, PIO_LONG_INTERNAL, NDIM, dimids, &varid) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Define a variable. */
+    if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, NDIM, dimids, &varid)))
+        return ret;
+
+    /* Set the fill mode. */
+    int fillmode = PIO_NOFILL;
+    int temp_mode;
+    int old_mode;
+    if ((ret = PIOc_set_fill(ncid, fillmode, &old_mode)))
+        return ERR_WRONG;
+    if ((ret = PIOc_set_fill(ncid, fillmode, &temp_mode)))
+        return ERR_WRONG;
+    printf("%d old_mode = %d temp_mode = %d\n", my_rank, old_mode, temp_mode);
+    if (temp_mode != PIO_NOFILL)
+        return ERR_WRONG;
+    if ((ret = PIOc_set_fill(ncid, old_mode, NULL)))
+        return ret;
+
+    /* Set the fill value for netCDF-4 files. */
+    int int_fill = -999;
+    int int_fill_in;
+    int fill_mode;
+
+    /* These should not work. */
+    if (PIOc_def_var_fill(ncid + TEST_VAL_42, varid, NC_FILL, &int_fill) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_def_var_fill(ncid, varid + TEST_VAL_42, NC_FILL, &int_fill) != PIO_ENOTVAR)
+        return ERR_WRONG;
+    if (PIOc_def_var_fill(ncid, varid, NC_FILL + TEST_VAL_42, &int_fill) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_def_var_fill(ncid, varid, NC_FILL, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Set the fill value. */
+    if ((ret = PIOc_def_var_fill(ncid, varid, NC_FILL, &int_fill)))
+        return ret;
+
+    /* These should not work. */
+    if (PIOc_inq_var_fill(ncid + TEST_VAL_42, varid, &fill_mode, &int_fill_in) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_inq_var_fill(ncid, varid + TEST_VAL_42, &fill_mode, &int_fill_in) != PIO_ENOTVAR)
+        return ERR_WRONG;
+
+    /* Check the fill value. */
+    if ((ret = PIOc_inq_var_fill(ncid, varid, &fill_mode, &int_fill_in)))
+        return ret;
+    if (fill_mode != NC_FILL || int_fill_in != int_fill)
+        ERR(ERR_WRONG);
+
+    /* These should also work. */
+    int_fill_in = 0;
+
+    /* This does not work for pnetcdf, but probably should. */
+    if (flavor != PIO_IOTYPE_PNETCDF)
+    {
+        if ((ret = PIOc_inq_var_fill(ncid, varid, NULL, &int_fill_in)))
+            return ret;
+        if (int_fill_in != int_fill)
+            ERR(ERR_WRONG);
+        if ((ret = PIOc_inq_var_fill(ncid, varid, NULL, NULL)))
+            return ret;
+    }
+    if ((ret = PIOc_inq_var_fill(ncid, varid, &fill_mode, NULL)))
+        return ret;
+    if (fill_mode != NC_FILL)
+        ERR(ERR_WRONG);
+
+    return PIO_NOERR;
+}
+
+/* Check the metadata in the test file. */
+int check_metadata(int ncid, int my_rank, int flavor)
+{
+    int ndims, nvars, ngatts, unlimdimid, natts, dimid[NDIM];
+    PIO_Offset len_in;
+    char name_in[PIO_MAX_NAME + 1];
+    nc_type xtype_in;
+    int ret;
+
+    /* Check how many dims, vars, global atts there are, and the id of
+     * the unlimited dimension. */
+    if (PIOc_inq(ncid + 1, &ndims, &nvars, &ngatts, &unlimdimid) != PIO_EBADID)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq(ncid, NULL, NULL, NULL, NULL)))
+        return ret;
+    if ((ret = PIOc_inq(ncid, &ndims, &nvars, &ngatts, &unlimdimid)))
+        return ret;
+    if (ndims != NDIM || nvars != 1 || ngatts != 0 || unlimdimid != 0)
+        return ERR_AWFUL;
+
+    /* Check the dimensions. */
+    for (int d = 0; d < NDIM; d++)
+    {
+        if (PIOc_inq_dim(ncid + 1, d, name_in, &len_in) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_inq_dim(ncid, d + 40, name_in, &len_in) != PIO_EBADDIM)
+            return ERR_WRONG;
+        if ((ret = PIOc_inq_dim(ncid, d, NULL, NULL)))
+            return ret;
+        if ((ret = PIOc_inq_dim(ncid, d, name_in, &len_in)))
+            return ret;
+        if (len_in != dim_len[d] || strcmp(name_in, dim_name[d]))
+            return ERR_AWFUL;
+    }
+
+    /* Check the variable. */
+    if (PIOc_inq_var(ncid + 1, 0, name_in, &xtype_in, &ndims, dimid, &natts) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_inq_var(ncid, 45, name_in, &xtype_in, &ndims, dimid, &natts) != PIO_ENOTVAR)
+        return ERR_WRONG;
+    if ((ret = PIOc_inq_var(ncid, 0, name_in, NULL, NULL, NULL, NULL)))
+        return ret;
+    if ((ret = PIOc_inq_var(ncid, 0, name_in, &xtype_in, &ndims, dimid, &natts)))
+        return ret;
+    if (strcmp(name_in, VAR_NAME) || xtype_in != PIO_INT || ndims != NDIM ||
+        dimid[0] != 0 || dimid[1] != 1 || dimid[2] != 2 || natts != 1)
+        return ERR_AWFUL;
+
+    return PIO_NOERR;
+}
+
+/* Test file operations.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @param test_comm the MPI communicator of the test.
+ * @param async 1 if we are testing async, 0 otherwise.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_names(int iosysid, int num_flavors, int *flavor, int my_rank,
+               MPI_Comm test_comm, int async)
+{
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+    int ret;    /* Return code. */
+
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        int ncid;
+        int varid;
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        char iotype_name[PIO_MAX_NAME + 1];
+        int dimids[NDIM];        /* The dimension IDs. */
+        int att_val = ATT_VAL;
+        float float_att_val = ATT_VAL;
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "%s_%s_names.nc", TEST_NAME, iotype_name);
+
+        /* Create the netCDF output file. */
+        printf("rank: %d Creating sample file %s with format %d...\n",
+               my_rank, filename, flavor[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_CLOBBER)))
+            return ret;
+
+        /* Define netCDF dimensions and variable. */
+        printf("rank: %d Defining netCDF metadata...\n", my_rank);
+        for (int d = 0; d < NDIM; d++)
+        {
+            printf("rank: %d Defining netCDF dimension %s, length %d\n", my_rank,
+                   dim_name[d], dim_len[d]);
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                return ret;
+        }
+
+        /* These should not work. */
+        if (PIOc_setframe(ncid + TEST_VAL_42, 0, 0) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_setframe(ncid, -1, 0) != PIO_EINVAL)
+            return ERR_WRONG;
+        if (PIOc_setframe(ncid, NC_MAX_VARS + 1, 0) != PIO_EINVAL)
+            return ERR_WRONG;
+        if (PIOc_advanceframe(ncid + TEST_VAL_42, 0) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_advanceframe(ncid, -1) != PIO_EINVAL)
+            return ERR_WRONG;
+        if (PIOc_advanceframe(ncid, NC_MAX_VARS + 1) != PIO_EINVAL)
+            return ERR_WRONG;
+
+        /* Check the dimension names. */
+        if ((ret = check_dim_names(my_rank, ncid, test_comm)))
+            ERR(ret);
+
+        /* These should not work. */
+        if (PIOc_put_att_int(ncid + TEST_VAL_42, NC_GLOBAL, ATT_NAME, PIO_INT, 1, &att_val) != PIO_EBADID)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, TEST_VAL_42, ATT_NAME, PIO_INT, 1, &att_val) != PIO_ENOTVAR)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, NULL, PIO_INT, 1, &att_val) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, PIO_INT, 1, NULL) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, too_long_name, PIO_INT, 1, &att_val) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, PIO_INT, -1, &att_val) != PIO_EINVAL)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, TEST_VAL_42, 1, &att_val) != PIO_EBADTYPE)
+            ERR(ERR_WRONG);
+        if (PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, PIO_LONG_INTERNAL, 1, &att_val) != PIO_EBADTYPE)
+            ERR(ERR_WRONG);
+
+        /* Define a global attribute. */
+        if ((ret = PIOc_put_att_int(ncid, NC_GLOBAL, ATT_NAME, PIO_INT, 1, &att_val)))
+            ERR(ret);
+        if ((ret = PIOc_put_att_float(ncid, NC_GLOBAL, ATT_NAME2, PIO_FLOAT, 1, &float_att_val)))
+            ERR(ret);
+
+        /* Check the attribute name. */
+        if ((ret = check_atts(my_rank, ncid, flavor[fmt], test_comm, async)))
+            ERR(ret);
+
+        /* Define a variable. */
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid)))
+            ERR(ret);
+
+        /* Check the variable name. */
+        if ((ret = check_var_name(my_rank, ncid, test_comm)))
+            ERR(ret);
+
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("rank: %d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test file operations.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_files(int iosysid, int num_flavors, int *flavor, int my_rank)
+{
+    int ncid;
+    int ret;    /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        char iotype_name[PIO_MAX_NAME + 1];
+
+        /* Overwrite existing test file. */
+        int mode = PIO_CLOBBER;
+
+        /* If this is netCDF-4, add the netCDF4 flag. */
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            printf("%d adding NC_NETCDF4 flag\n", my_rank);
+            mode |= NC_NETCDF4;
+        }
+
+        /* If this is pnetcdf or netCDF-4 parallel, add the MPIIO flag. */
+        if (flavor[fmt] == PIO_IOTYPE_PNETCDF || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            printf("%d adding NC_MPIIO flag\n", my_rank);
+            mode |= NC_MPIIO;
+        }
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "%s_%s.nc", TEST_NAME, iotype_name);
+
+        /* Testing some invalid parameters. */
+        if (PIOc_create(iosysid + 1, filename, mode, &ncid) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_create(iosysid, filename, mode, NULL) != PIO_EINVAL)
+            return ERR_WRONG;
+        if (PIOc_create(iosysid, NULL, mode, &ncid) != PIO_EINVAL)
+            return ERR_WRONG;
+
+        /* Create the netCDF output file. */
+        printf("%d Creating sample file %s with format %d...\n", my_rank, filename, flavor[fmt]);
+        if ((ret = PIOc_create(iosysid, filename, mode, &ncid)))
+            ERR(ret);
+
+        /* Check this support function. */
+        if (!PIOc_File_is_Open(ncid))
+            ERR(ERR_WRONG);
+        if (PIOc_File_is_Open(ncid + 1))
+            ERR(ERR_WRONG);
+
+        /* Define the test file metadata. */
+        if ((ret = define_metadata(ncid, my_rank, flavor[fmt])))
+            ERR(ret);
+
+        /* End define mode. */
+        if (PIOc_enddef(ncid + 1) != PIO_EBADID)
+            return ERR_WRONG;
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if (PIOc_closefile(ncid + 1) != PIO_EBADID)
+            return ERR_WRONG;
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Check some invalid paramters. */
+        if (PIOc_open(iosysid + 1, filename, mode, &ncid) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_open(iosysid, NULL, mode, &ncid) != PIO_EINVAL)
+            return ERR_WRONG;
+        if (PIOc_open(iosysid, filename, mode, NULL) != PIO_EINVAL)
+            return ERR_WRONG;
+
+        /* Reopen the test file. */
+        printf("%d Re-opening sample file %s with format %d...\n",
+               my_rank, filename, flavor[fmt]);
+        if ((ret = PIOc_open(iosysid, filename, mode, &ncid)))
+            ERR(ret);
+
+        /* Check the test file metadata. */
+        if ((ret = check_metadata(ncid, my_rank, flavor[fmt])))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test the deletion of files.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_deletefile(int iosysid, int num_flavors, int *flavor, int my_rank)
+{
+    int ncid;
+    int ret;    /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        char iotype_name[PIO_MAX_NAME + 1];
+        int old_method;
+
+        /* These should fail. */
+        if (PIOc_set_iosystem_error_handling(iosysid + TEST_VAL_42, PIO_RETURN_ERROR, &old_method) != PIO_EBADID)
+            return ERR_WRONG;
+        if (PIOc_set_iosystem_error_handling(iosysid, PIO_RETURN_ERROR + TEST_VAL_42, &old_method) != PIO_EINVAL)
+            return ERR_WRONG;
+
+        /* Set error handling. */
+        if ((ret = PIOc_set_iosystem_error_handling(iosysid, PIO_RETURN_ERROR, &old_method)))
+            return ret;
+        if (old_method != PIO_INTERNAL_ERROR && old_method != PIO_RETURN_ERROR)
+            return ERR_WRONG;
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "delete_me_%s_%s.nc", TEST_NAME, iotype_name);
+
+        printf("%d testing delete for file %s with format %d...\n",
+               my_rank, filename, flavor[fmt]);
+        int bad_iotype = TEST_VAL_42;
+        if (PIOc_createfile(iosysid, &ncid, &bad_iotype, filename, PIO_CLOBBER) != PIO_EINVAL)
+            return ERR_WRONG;
+        if ((ret = PIOc_createfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* This should not work. */
+        if (PIOc_deletefile(iosysid + TEST_VAL_42, filename) != PIO_EBADID)
+            ERR(ERR_WRONG);
+
+        /* Now delete the file. */
+        printf("%d Deleting %s...\n", my_rank, filename);
+        if ((ret = PIOc_deletefile(iosysid, filename)))
+            ERR(ret);
+
+        /* Make sure it is gone. Openfile will now return an error
+         * code when I try to open the file. */
+        if (!PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_NOWRITE))
+            ERR(ERR_WRONG);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test the netCDF-4 optimization functions. */
+int test_nc4(int iosysid, int num_flavors, int *flavor, int my_rank)
+{
+    int ncid;    /* The ncid of the netCDF file. */
+    int dimids[NDIM];    /* The dimension IDs. */
+    int varid;    /* The ID of the netCDF varable. */
+
+    /* For setting the chunk cache. */
+    PIO_Offset chunk_cache_size = 1024*1024;
+    PIO_Offset chunk_cache_nelems = 1024;
+    float chunk_cache_preemption = 0.5;
+
+    /* For reading the chunk cache. */
+    PIO_Offset chunk_cache_size_in;
+    PIO_Offset chunk_cache_nelems_in;
+    float chunk_cache_preemption_in;
+
+    int storage = NC_CHUNKED; /* Storage of netCDF-4 files (contiguous vs. chunked). */
+    PIO_Offset my_chunksize[NDIM]; /* Chunksizes we get from file. */
+    int shuffle;    /* The shuffle filter setting in the netCDF-4 test file. */
+    int deflate;    /* Non-zero if deflate set for the variable in the netCDF-4 test file. */
+    int deflate_level;    /* The deflate level set for the variable in the netCDF-4 test file. */
+    int endianness;    /* Endianness of variable. */
+    PIO_Offset var_cache_size;    /* Size of the var chunk cache. */
+    PIO_Offset var_cache_nelems; /* Number of elements in var cache. */
+    float var_cache_preemption;     /* Var cache preemption. */
+    char varname_in[PIO_MAX_NAME];
+    int expected_ret; /* The return code we expect to get. */
+    int ret;    /* Return code. */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        char iotype_name[PIO_MAX_NAME + 1];
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "%s_%s.nc", TEST_NAME, iotype_name);
+
+        printf("%d Setting chunk cache for file %s with format %d...\n",
+               my_rank, filename, flavor[fmt]);
+
+        /* Try to set the chunk cache. */
+        chunk_cache_preemption = 0.5;
+        ret = PIOc_set_chunk_cache(iosysid, flavor[fmt], chunk_cache_size,
+                                   chunk_cache_nelems, chunk_cache_preemption);
+
+        /* What result did we expect to get? */
+        expected_ret = flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P ?
+            PIO_NOERR : PIO_ENOTNC4;
+        if (ret != expected_ret)
+            ERR(ERR_AWFUL);
+
+        /* Try to set the chunk cache for netCDF-4 iotypes. */
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+            if ((ret = PIOc_set_chunk_cache(iosysid, flavor[fmt], chunk_cache_size,
+                                            chunk_cache_nelems, chunk_cache_preemption)))
+                ERR(ERR_AWFUL);
+
+        /* Now check the chunk cache. */
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            if ((ret = PIOc_get_chunk_cache(iosysid, flavor[fmt], &chunk_cache_size_in,
+                                            &chunk_cache_nelems_in, &chunk_cache_preemption_in)))
+                ERR(ERR_AWFUL);
+
+            /* Check that we got the correct values. */
+            if (chunk_cache_size_in != chunk_cache_size || chunk_cache_nelems_in != chunk_cache_nelems ||
+                chunk_cache_preemption_in != chunk_cache_preemption)
+                ERR(ERR_AWFUL);
+        }
+
+        /* Create the netCDF output file. */
+        printf("%d Creating sample file %s with format %d...\n",
+               my_rank, filename, flavor[fmt]);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Define netCDF dimensions and variable. */
+        printf("%d Defining netCDF metadata...\n", my_rank);
+        for (int d = 0; d < NDIM; d++)
+        {
+            printf("%d Defining netCDF dimension %s, length %d\n", my_rank,
+                   dim_name[d], dim_len[d]);
+            if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+                ERR(ret);
+        }
+        printf("%d Defining netCDF variable %s, ndims %d\n", my_rank, VAR_NAME, NDIM);
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, &varid)))
+            ERR(ret);
+
+        /* Check that invalid arguments are properly rejected. */
+        if (PIOc_def_var_chunking(ncid + TEST_VAL_42, 0, NC_CHUNKED, chunksize) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_inq_var_chunking(ncid + TEST_VAL_42, 0, &storage, my_chunksize) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_inq_var_deflate(ncid + TEST_VAL_42, 0, &shuffle, &deflate, &deflate_level) != PIO_EBADID)
+            ERR(ret);
+        if (PIOc_def_var_endian(ncid + TEST_VAL_42, 0, 1) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_def_var_deflate(ncid + TEST_VAL_42, 0, 0, 0, 0) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_inq_var_endian(ncid + TEST_VAL_42, 0, &endianness) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_set_var_chunk_cache(ncid + TEST_VAL_42, 0, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
+                                     VAR_CACHE_PREEMPTION) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_get_var_chunk_cache(ncid + TEST_VAL_42, 0, &var_cache_size, &var_cache_nelems,
+                                     &var_cache_preemption) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_set_chunk_cache(iosysid + TEST_VAL_42, flavor[fmt], chunk_cache_size, chunk_cache_nelems,
+                                 chunk_cache_preemption) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+        if (PIOc_get_chunk_cache(iosysid + TEST_VAL_42, flavor[fmt], &chunk_cache_size,
+                                 &chunk_cache_nelems, &chunk_cache_preemption) != PIO_EBADID)
+            ERR(ERR_AWFUL);
+
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            if (PIOc_def_var_chunking(ncid, TEST_VAL_42, NC_CHUNKED, chunksize) != PIO_ENOTVAR)
+                ERR(ERR_AWFUL);
+            if (PIOc_inq_var_chunking(ncid, TEST_VAL_42, &storage, my_chunksize) != PIO_ENOTVAR)
+                ERR(ERR_AWFUL);
+            if (PIOc_set_var_chunk_cache(ncid, TEST_VAL_42, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
+                                         VAR_CACHE_PREEMPTION) != PIO_ENOTVAR)
+                ERR(ERR_AWFUL);
+            if (PIOc_get_var_chunk_cache(ncid, TEST_VAL_42, &var_cache_size, &var_cache_nelems,
+                                         &var_cache_preemption) != PIO_ENOTVAR)
+                ERR(ERR_AWFUL);
+        }
+        else
+        {
+            printf("my ret = %d\n", PIOc_def_var_chunking(ncid, TEST_VAL_42, NC_CHUNKED, chunksize));
+            if (PIOc_def_var_chunking(ncid, TEST_VAL_42, NC_CHUNKED, chunksize) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_inq_var_chunking(ncid, TEST_VAL_42, &storage, my_chunksize) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_set_var_chunk_cache(ncid, TEST_VAL_42, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
+                                         VAR_CACHE_PREEMPTION) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_get_var_chunk_cache(ncid, TEST_VAL_42, &var_cache_size, &var_cache_nelems,
+                                         &var_cache_preemption) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+        }
+
+        /* For netCDF-4 files, set the chunksize to improve performance. */
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            unsigned long long too_big_chunksize[NDIM] = {(unsigned long long)NC_MAX_INT64 + TEST_VAL_42, X_DIM_LEN/2, Y_DIM_LEN/2};
+            if (PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, (MPI_Offset *)too_big_chunksize) == PIO_NOERR)
+                ERR(ret);
+
+            printf("%d Defining chunksizes\n", my_rank);
+            if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)))
+                ERR(ret);
+
+            /* Setting deflate should not work with parallel iotype. */
+            printf("%d Defining deflate\n", my_rank);
+            ret = PIOc_def_var_deflate(ncid, 0, 0, 1, 1);
+            if (flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+            {
+                if (ret == PIO_NOERR)
+                    ERR(ERR_WRONG);
+            }
+            else
+            {
+                if (ret != PIO_NOERR)
+                    ERR(ERR_WRONG);
+            }
+
+            /* Check that the inq_varname function works. */
+            printf("%d Checking varname\n", my_rank);
+            if ((ret = PIOc_inq_varname(ncid, 0, NULL)))
+                ERR(ret);
+            if ((ret = PIOc_inq_varname(ncid, 0, varname_in)))
+                ERR(ret);
+
+            /* Check that the inq_var_chunking function works. */
+            printf("%d Checking chunksizes\n", my_rank);
+            if ((ret = PIOc_inq_var_chunking(ncid, 0, NULL, NULL)))
+                ERR(ret);
+            if ((ret = PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize)))
+                ERR(ret);
+
+            /* Check the answers. */
+            if (storage != NC_CHUNKED)
+                ERR(ERR_AWFUL);
+            for (int d1 = 0; d1 < NDIM; d1++)
+                if (my_chunksize[d1] != chunksize[d1])
+                    ERR(ERR_AWFUL);
+
+            /* Check that the inq_var_deflate functions works. */
+            if ((ret = PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level)))
+                ERR(ret);
+
+            /* For serial netCDF-4 deflate is turned on by default */
+            if (flavor[fmt] == PIO_IOTYPE_NETCDF4C)
+                if (shuffle || !deflate || deflate_level != 1)
+                    ERR(ERR_AWFUL);
+
+            /* For parallel netCDF-4, no compression available. :-( */
+            if (flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+                if (shuffle || deflate)
+                    ERR(ERR_AWFUL);
+
+            /* Check setting the chunk cache for the variable. */
+            printf("%d PIOc_set_var_chunk_cache...\n", my_rank);
+            if ((ret = PIOc_set_var_chunk_cache(ncid, 0, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
+                                                VAR_CACHE_PREEMPTION)))
+                ERR(ret);
+
+            /* Check getting the chunk cache values for the variable. */
+            printf("%d PIOc_get_var_chunk_cache...\n", my_rank);
+            if ((ret = PIOc_get_var_chunk_cache(ncid, 0, &var_cache_size, &var_cache_nelems,
+                                                &var_cache_preemption)))
+                ERR(ret);
+
+            /* Check that we got expected values. */
+            printf("%d var_cache_size = %lld\n", my_rank, var_cache_size);
+            if (var_cache_size != VAR_CACHE_SIZE)
+                ERR(ERR_AWFUL);
+            if (var_cache_nelems != VAR_CACHE_NELEMS)
+                ERR(ERR_AWFUL);
+            if (var_cache_preemption != VAR_CACHE_PREEMPTION)
+                ERR(ERR_AWFUL);
+
+            if ((ret = PIOc_def_var_endian(ncid, 0, 1)))
+                ERR(ERR_AWFUL);
+            if ((ret = PIOc_inq_var_endian(ncid, 0, &endianness)))
+                ERR(ERR_AWFUL);
+            if (endianness != 1)
+                ERR(ERR_WRONG);
+        }
+        else
+        {
+            /* Trying to set or inq netCDF-4 settings for non-netCDF-4
+             * files results in the PIO_ENOTNC4 error. */
+            if (PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level) != PIO_ENOTNC4)
+                ERR(ret);
+            if (PIOc_def_var_endian(ncid, 0, 1) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_inq_var_endian(ncid, 0, &endianness) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_set_var_chunk_cache(ncid, 0, VAR_CACHE_SIZE, VAR_CACHE_NELEMS,
+                                         VAR_CACHE_PREEMPTION) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_get_var_chunk_cache(ncid, 0, &var_cache_size, &var_cache_nelems,
+                                         &var_cache_preemption) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_set_chunk_cache(iosysid, flavor[fmt], chunk_cache_size, chunk_cache_nelems,
+                                     chunk_cache_preemption) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+            if (PIOc_get_chunk_cache(iosysid, flavor[fmt], &chunk_cache_size,
+                                     &chunk_cache_nelems, &chunk_cache_preemption) != PIO_ENOTNC4)
+                ERR(ERR_AWFUL);
+        }
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/* This function is part of test_scalar(). It tests the contents of
+ * the scalar var. */
+int check_scalar_var(int ncid, int varid, int flavor)
+{
+    char var_name_in[PIO_MAX_NAME + 1];
+    int var_type_in;
+    int ndims_in;
+    int natts_in;
+    int val_in;
+    int ret;
+
+    /* Learn the var metadata. */
+    if ((ret = PIOc_inq_var(ncid, varid, var_name_in, &var_type_in, &ndims_in, NULL,
+                            &natts_in)))
+        return ret;
+
+    /* Is the metadata correct? */
+    if (strcmp(var_name_in, VAR_NAME) || var_type_in != PIO_INT || ndims_in != 0 || natts_in != 0)
+        return ERR_WRONG;
+
+    /* Get the value. */
+    if ((ret = PIOc_get_var_int(ncid, varid, &val_in)))
+        return ret;
+    printf("val_in = %d\n", val_in);
+
+    /* Is the value correct? */
+    if (val_in != TEST_VAL_42)
+        return ERR_WRONG;
+
+    return 0;
+}
+
+/* Test scalar vars. */
+int test_scalar(int iosysid, int num_flavors, int *flavor, int my_rank, int async,
+                MPI_Comm test_comm)
+{
+    int ncid;    /* The ncid of the netCDF file. */
+    int varid;   /* The ID of the netCDF varable. */
+    int ret;     /* Return code. */
+
+    /* Use netCDF classic to create a file with a scalar var, then set
+     * and read the value. */
+    if (my_rank == 0)
+    {
+        char test_file[] = "netcdf_test.nc";
+        int test_val = TEST_VAL_42;
+        int test_val_in;
+
+        if ((ret = nc_create(test_file, NC_CLOBBER, &ncid)))
+            return ret;
+        if ((ret = nc_def_var(ncid, VAR_NAME, NC_INT, 0, NULL, &varid)))
+            return ret;
+        if ((ret = nc_enddef(ncid)))
+            return ret;
+        if ((ret = nc_put_var(ncid, varid, &test_val)))
+            return ret;
+        if ((ret = nc_close(ncid)))
+            return ret;
+        if ((ret = nc_open(test_file, NC_NOWRITE, &ncid)))
+            return ret;
+        /* if ((ret = nc_get_var(ncid, varid, &test_val_in))) */
+        /*     return ret; */
+        /* if (test_val_in != test_val) */
+        /*     return ERR_WRONG; */
+        if ((ret = nc_get_vars(ncid, varid, NULL, NULL, NULL, &test_val_in)))
+            return ret;
+        if (test_val_in != test_val)
+            return ERR_WRONG;
+        if ((ret = nc_close(ncid)))
+            return ret;
+    }
+
+    /* Use pnetCDF to create a file with a scalar var, then set and
+     * read the value. */
+#ifdef _PNETCDF
+    {
+        char test_file[] = "pnetcdf_test.nc";
+        int test_val = TEST_VAL_42;
+        int test_val_in;
+
+        if ((ret = ncmpi_create(test_comm, test_file, NC_CLOBBER, MPI_INFO_NULL, &ncid)))
+            return ret;
+        if ((ret = ncmpi_def_var(ncid, VAR_NAME, NC_INT, 0, NULL, &varid)))
+            return ret;
+        if ((ret = ncmpi_enddef(ncid)))
+            return ret;
+        if ((ret = ncmpi_put_var_int_all(ncid, varid, &test_val)))
+            return ret;
+        if ((ret = ncmpi_close(ncid)))
+            return ret;
+        if ((ret = ncmpi_open(test_comm, test_file, NC_NOWRITE, MPI_INFO_NULL, &ncid)))
+            return ret;
+        /* Turn on independent access for pnetcdf file. */
+        if ((ret = ncmpi_begin_indep_data(ncid)))
+            return ret;
+        /* if ((ret = ncmpi_get_var_int(ncid, varid, &test_val_in))) */
+        /*     return ret; */
+        if ((ret = ncmpi_get_vars_int(ncid, varid, NULL, NULL, NULL, &test_val_in)))
+            return ret;
+        if ((ret = ncmpi_end_indep_data(ncid)))
+            return ret;
+        if (test_val_in != test_val)
+            return ERR_WRONG;
+        printf("ret = %d test_val_in = %d\n", ret, test_val_in);
+        if (test_val_in != test_val)
+            return ERR_WRONG;
+        if ((ret = ncmpi_close(ncid)))
+            return ret;
+    }
+#endif /* _PNETCDF */
+
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        char iotype_name[PIO_MAX_NAME + 1];
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "%s_%s_scalar_async_%d.nc", TEST_NAME, iotype_name, async);
+
+        /* Create the netCDF output file. */
+        printf("%d Creating test file %s.\n", my_rank, filename);
+        if ((ret = PIOc_createfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_CLOBBER)))
+            ERR(ret);
+
+        /* Define a scalar variable. */
+        if ((ret = PIOc_def_var(ncid, VAR_NAME, PIO_INT, 0, NULL, &varid)))
+            ERR(ret);
+
+        /* End define mode. */
+        if ((ret = PIOc_enddef(ncid)))
+            ERR(ret);
+
+        /* Write a scalar value. */
+        int test_val = TEST_VAL_42;
+        if ((ret = PIOc_put_var_int(ncid, varid, &test_val)))
+            ERR(ret);
+
+        /* Check the scalar var. */
+        if ((ret = check_scalar_var(ncid, varid, flavor[fmt])))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        printf("%d Closing the sample data file...\n", my_rank);
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Reopen the file. */
+        if ((ret = PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_NOWRITE)))
+            ERR(ret);
+
+        /* Check the scalar var again. */
+        if ((ret = check_scalar_var(ncid, varid, flavor[fmt])))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+    }
+    return PIO_NOERR;
+}
+
+/** Test the malloc_iodesc() function.
+ *
+ * @param my_rank rank of this task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_malloc_iodesc2(int iosysid, int my_rank)
+{
+    /* More types are available for netCDF-4 builds. */
+#ifdef _NETCDF4
+    int num_types = NUM_NETCDF_TYPES;
+#else
+    int num_types = NUM_CLASSIC_TYPES;
+#endif /* _NETCDF4 */
+    int test_type[NUM_NETCDF_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT,
+                                       PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT,
+                                       PIO_UINT, PIO_INT64, PIO_UINT64, PIO_STRING};
+    MPI_Datatype mpi_type[NUM_NETCDF_TYPES] = {MPI_BYTE, MPI_CHAR, MPI_SHORT, MPI_INT,
+                                               MPI_FLOAT, MPI_DOUBLE, MPI_UNSIGNED_CHAR,
+                                               MPI_UNSIGNED_SHORT, MPI_UNSIGNED, MPI_LONG_LONG,
+                                               MPI_UNSIGNED_LONG_LONG, MPI_CHAR};
+    int ioid;
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    int ret;
+
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+    printf("test_malloc_iodesc2 num_types %d\n",num_types);
+    /* Test with each type. */
+    for (int t = 0; t < num_types; t++)
+    {
+
+        if ((ret = malloc_iodesc(ios, test_type[t], 1, &iodesc)))
+            return ret;
+        if (iodesc->basetype != mpi_type[t])
+            return ERR_WRONG;
+        if (iodesc->ndims != 1)
+            return ERR_WRONG;
+        ioid = pio_add_to_iodesc_list(iodesc);
+        if (iodesc->firstregion)
+            free_region_list(iodesc->firstregion);
+        if ((ret = pio_delete_iodesc_from_list(ioid)))
+            return ret;
+    }
+    return 0;
+}
+
+/* Test some decomp internal functions. */
+int test_decomp_internal(int my_test_size, int my_rank, int iosysid, int dim_len,
+                         MPI_Comm test_comm, int async)
+{
+    int ioid;
+    char filename[NC_MAX_NAME + 1];    /* Test decomp filename. */
+    char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
+    iosystem_desc_t *ios; /* IO system info. */
+    int ret;
+
+    /* This will be our file name for writing out decompositions. */
+    sprintf(filename, "decomp_%s_rank_%d_async_%d.txt", TEST_NAME, my_rank, async);
+    sprintf(nc_filename, "nc_decomp_internal_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async);
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition(my_test_size, my_rank, iosysid, dim_len, &ioid)))
+        return ret;
+
+    /* Write the decomp file (on appropriate tasks). */
+    if ((ret = PIOc_write_decomp(filename, iosysid, ioid, test_comm)))
+        return ret;
+
+    /* Some values for the netCDF decomp file for this iosystem. */
+    char *title = "Very Simple Test Decompositon";
+    char *history = "Added to PIO automatic testing by Ed in February 2017.";
+    int global_dimlen[] = {DIM_LEN};
+    int task_maplen[TARGET_NTASKS] = {1, 1, 1, 1};
+    int map[TARGET_NTASKS][1] = {{0},{1},{2},{3}};
+
+    /* Get the IO system info. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Write the decomposition file. */
+    if ((ret = pioc_write_nc_decomp_int(ios, nc_filename, 0, NDIM1, global_dimlen,
+                                        TARGET_NTASKS, task_maplen, (int *)map, title,
+                                        history, 0)))
+        return ret;
+
+    int ndims_in;
+    int num_tasks_in;
+    int max_maplen_in;
+    char title_in[PIO_MAX_NAME + 1];
+    char history_in[PIO_MAX_NAME + 1];
+    char source_in[PIO_MAX_NAME + 1];
+    char version_in[PIO_MAX_NAME + 1];
+    char expected_source[] = "Decomposition file produced by PIO library.";
+    int *global_dimlen_in;
+    int *task_maplen_in;
+    int *map_in;
+    int fortran_order_in;
+
+    /* These should not work. */
+    if (pioc_read_nc_decomp_int(iosysid + TEST_VAL_42, nc_filename, &ndims_in, &global_dimlen_in,
+                                &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                history_in, source_in, version_in, &fortran_order_in) != PIO_EBADID)
+        return ERR_WRONG;
+    if (pioc_read_nc_decomp_int(iosysid, NULL, &ndims_in, &global_dimlen_in,
+                                &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                history_in, source_in, version_in, &fortran_order_in) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (!pioc_read_nc_decomp_int(iosysid, "no_file", &ndims_in, &global_dimlen_in,
+                                &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                history_in, source_in, version_in, &fortran_order_in))
+        return ERR_WRONG;
+
+    /* Read the decomp file. */
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+
+    
+    /* Did we get the correct answers? */
+    printf("source_in = %s\n", source_in);
+    if (strcmp(title, title_in) || strcmp(history, history_in) ||
+        strcmp(source_in, expected_source))
+        return ERR_WRONG;
+    if (ndims_in != NDIM1 || num_tasks_in != TARGET_NTASKS || max_maplen_in != 1 ||
+        fortran_order_in)
+        return ERR_WRONG;
+    for (int d = 0; d < ndims_in; d++)
+        if (global_dimlen_in[d] != global_dimlen[d])
+            return ERR_WRONG;
+    for (int t = 0; t < num_tasks_in; t++)
+        if (task_maplen_in[t] != 1)
+            return ERR_WRONG;
+    for (int t = 0; t < num_tasks_in; t++)
+        for (int l = 0; l < max_maplen_in; l++)
+            if (map_in[t * max_maplen_in + l] != map[t][l])
+                return ERR_WRONG;
+
+    /* Free resources. */
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    /* These should also work. */
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, NULL, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, NULL,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       NULL, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, NULL, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, NULL, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, NULL, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, NULL,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       NULL, source_in, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, NULL, version_in, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, NULL, &fortran_order_in)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, NULL)))
+        return ret;
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return 0;
+}
+
+/* Test some decomp public API functions. */
+int test_decomp_public(int my_test_size, int my_rank, int iosysid, int dim_len,
+                       MPI_Comm test_comm, int async)
+{
+    int ioid;
+    char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
+    int ret;
+
+    /* This will be our file name for writing out decompositions. */
+    sprintf(nc_filename, "nc_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async);
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition(my_test_size, my_rank, iosysid, dim_len, &ioid)))
+        return ret;
+
+    /* We will document our decomp file with metadata, like good
+     * netCDF users should. */
+    char *title = "Very Simple Test Decompositon";
+    char *history = "Added to PIO automatic testing by Ed in February 2017.";
+
+    /* These should not work. */
+    char too_long_name[PIO_MAX_NAME * 5 + 1];
+    memset(too_long_name, 74, PIO_MAX_NAME * 5);
+    too_long_name[PIO_MAX_NAME * 5] = 0;
+
+    if (PIOc_write_nc_decomp(iosysid + TEST_VAL_42, nc_filename, 0, ioid,
+                             title, history, 0) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_write_nc_decomp(iosysid, NULL, 0, ioid, title, history, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid + TEST_VAL_42,
+                             title, history, 0) != PIO_EBADID)
+        return ERR_WRONG;
+
+    if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid,
+                             too_long_name, history, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid,
+                             title, too_long_name, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Write a netCDF decomp file for this iosystem. */
+    if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, title,
+                                    history, 0)))
+        return ret;
+
+    int ioid_in;
+    char title_in[PIO_MAX_NAME + 1];
+    char history_in[PIO_MAX_NAME + 1];
+    int fortran_order_in;
+
+    /* These should not work. */
+    if (PIOc_read_nc_decomp(iosysid + TEST_VAL_42, nc_filename, &ioid_in, test_comm,
+                            PIO_INT, title_in, history_in, &fortran_order_in) != PIO_EBADID)
+        return ret;
+    if (PIOc_read_nc_decomp(iosysid, NULL, &ioid_in, test_comm, PIO_INT, title_in,
+                            history_in, &fortran_order_in) != PIO_EINVAL)
+        return ret;
+    if (PIOc_read_nc_decomp(iosysid, nc_filename, NULL, test_comm, PIO_INT, title_in,
+                            history_in, &fortran_order_in) != PIO_EINVAL)
+        return ret;
+
+    /* Read it using the public read function. */
+    if ((ret = PIOc_read_nc_decomp(iosysid, nc_filename, &ioid_in, test_comm, PIO_INT,
+                                   title_in, history_in, &fortran_order_in)))
+        return ret;
+
+    /* Did we get expected results? */
+    if (strcmp(title, title_in) || strcmp(history, history_in))
+        return ERR_WRONG;
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid_in)))
+        ERR(ret);
+
+    /* These should also work. */
+    if ((ret = PIOc_read_nc_decomp(iosysid, nc_filename, &ioid_in, test_comm, PIO_CHAR, NULL,
+                                   history_in, &fortran_order_in)))
+        return ret;
+    if ((ret = PIOc_freedecomp(iosysid, ioid_in)))
+        ERR(ret);
+
+    if ((ret = PIOc_read_nc_decomp(iosysid, nc_filename, &ioid_in, test_comm, PIO_BYTE, title_in,
+                                   NULL, &fortran_order_in)))
+        return ret;
+    if ((ret = PIOc_freedecomp(iosysid, ioid_in)))
+        ERR(ret);
+
+    if ((ret = PIOc_read_nc_decomp(iosysid, nc_filename, &ioid_in, test_comm, PIO_SHORT, title_in,
+                                  history_in, NULL)))
+        return ret;
+    if ((ret = PIOc_freedecomp(iosysid, ioid_in)))
+        ERR(ret);
+
+    /* Read it using the intertal function. */
+    int ndims_in;
+    int num_tasks_in;
+    int max_maplen_in;
+    char source_in[PIO_MAX_NAME + 1];
+    char version_in[PIO_MAX_NAME + 1];
+    char expected_source[] = "Decomposition file produced by PIO library.";
+    int *global_dimlen_in;
+    int *task_maplen_in;
+    int *map_in;
+
+    /* Read the decomp file. */
+    if ((ret = pioc_read_nc_decomp_int(iosysid, nc_filename, &ndims_in, &global_dimlen_in,
+                                       &num_tasks_in, &task_maplen_in, &max_maplen_in, &map_in, title_in,
+                                       history_in, source_in, version_in, &fortran_order_in)))
+        return ret;
+
+    /* Did we get the correct answers? */
+    printf("source_in = %s\n", source_in);
+    if (strcmp(title, title_in) || strcmp(history, history_in) ||
+        strcmp(source_in, expected_source))
+        return ERR_WRONG;
+    if (ndims_in != NDIM1 || num_tasks_in != TARGET_NTASKS || max_maplen_in != 1 ||
+        fortran_order_in)
+        return ERR_WRONG;
+    for (int d = 0; d < ndims_in; d++)
+        if (global_dimlen_in[d] != DIM_LEN)
+            return ERR_WRONG;
+    for (int t = 0; t < num_tasks_in; t++)
+        if (task_maplen_in[t] != 1)
+            return ERR_WRONG;
+    for (int t = 0; t < num_tasks_in; t++)
+        for (int l = 0; l < max_maplen_in; l++)
+            if (map_in[t * max_maplen_in + l] != t)
+                return ERR_WRONG;
+
+    /* Free resources. */
+    free(global_dimlen_in);
+    free(task_maplen_in);
+    free(map_in);
+
+    /* /\* These should also work. *\/ */
+    /* if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, title, history, 0))) */
+    /*     return ret; */
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return 0;
+}
+
+/* Test some decomp public API functions. */
+int test_decomp_public_2(int my_test_size, int my_rank, int iosysid, int dim_len,
+                         MPI_Comm test_comm, int async)
+{
+    int ioid;
+    char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
+    int ret;
+
+    /* This will be our file name for writing out decompositions. */
+    sprintf(nc_filename, "nc_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async);
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition(my_test_size, my_rank, iosysid, dim_len, &ioid)))
+        return ret;
+
+    /* Write a netCDF decomp file for this iosystem. */
+    if ((ret = PIOc_write_nc_decomp(iosysid, nc_filename, 0, ioid, NULL, NULL, 0)))
+        return ret;
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return 0;
+}
+
+/* Test some decomp public API functions. */
+int test_decomp_2(int my_test_size, int my_rank, int iosysid, int dim_len,
+                  MPI_Comm test_comm, int async)
+{
+    int ioid;
+    char nc_filename[NC_MAX_NAME + 1]; /* Test decomp filename (netcdf version). */
+    int ret;
+
+    /* This will be our file name for writing out decompositions. */
+    sprintf(nc_filename, "nc_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async);
+
+    /* Decompose the data over the tasks. */
+    if ((ret = create_decomposition(my_test_size, my_rank, iosysid, dim_len, &ioid)))
+        return ret;
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return 0;
+}
+
+/* Test some decomp public API functions with async. */
+int test_decomp_public_async(int my_test_size, int my_rank, int iosysid, MPI_Comm test_comm,
+                             int async)
+{
+#define ELEM1 1
+#define LEN3 3
+    int ioid;
+    int dim_len = LEN3;
+    PIO_Offset elements_per_pe = ELEM1;
+    PIO_Offset compdof[ELEM1] = {my_rank + 1};
+    char filename[PIO_MAX_NAME + 1];
+    int ret;
+
+    sprintf(filename, "async_decomp_%s_rank_%d_async_%d.nc", TEST_NAME, my_rank, async);
+    
+    /* Create the PIO decomposition for this test. */
+    if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, NDIM1, &dim_len, elements_per_pe,
+                                compdof, &ioid, PIO_REARR_BOX, NULL, NULL)))
+        ERR(ret);
+
+    /* Write the decomp file (on appropriate tasks). */
+    if ((ret = PIOc_write_nc_decomp(iosysid, filename, 0, ioid, NULL, NULL, 0)))
+        return ret;
+
+    /* Free the PIO decomposition. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        ERR(ret);
+
+    return 0;
+}
+
+/* Run all the tests. */
+int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm,
+             int async)
+{
+    int ioid;
+    int my_test_size;
+    char filename[NC_MAX_NAME + 1];
+    char nc_filename[NC_MAX_NAME + 1];
+    int ret; /* Return code. */
+
+    if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
+        MPIERR(ret);
+
+    /* This will be our file name for writing out decompositions. */
+    sprintf(filename, "decomp_%d.txt", my_rank);
+    sprintf(nc_filename, "decomp_%d.nc", my_rank);
+
+    /* This is a simple test that just creates the decomp with
+     * async. */
+    if (async)
+        if ((ret = test_decomp_public_async(my_test_size, my_rank, iosysid, test_comm, async)))
+            return ret;
+    
+    /* Check iotypes. */
+    printf("%d Testing iotypes. async = %d\n", my_rank, async);
+    if ((ret = test_iotypes(my_rank)))
+        ERR(ret);
+
+    /* Test file deletes. */
+    printf("%d Testing deletefile. async = %d\n", my_rank, async);
+    if ((ret = test_deletefile(iosysid, num_flavors, flavor, my_rank)))
+        return ret;
+
+    /* Test file stuff. */
+    printf("%d Testing file creation. async = %d\n", my_rank, async);
+    if ((ret = test_files(iosysid, num_flavors, flavor, my_rank)))
+        return ret;
+
+    /* Test some misc stuff. */
+    if ((ret = test_malloc_iodesc2(iosysid, my_rank)))
+        return ret;
+
+    /* Test decomposition internal functions. */
+    if (!async)
+        if ((ret = test_decomp_internal(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async)))
+            return ret;
+    /* Test decomposition public API functions. */
+    if (!async)
+        if ((ret = test_decomp_public(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async)))
+            return ret;
+
+    /* This is a simple test that just creates a decomp. */
+    /* if ((ret = test_decomp_2(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async))) */
+    /*     return ret; */
+
+    /* This is a simple test that just writes the decomp. */
+    if (!async)
+        if ((ret = test_decomp_public_2(my_test_size, my_rank, iosysid, DIM_LEN, test_comm, async)))
+            return ret;
+
+    /* Decompose the data over the tasks. */
+    if (!async)
+    {
+        if ((ret = create_decomposition(my_test_size, my_rank, iosysid, DIM_LEN, &ioid)))
+            return ret;
+
+        /* Run the darray tests. */
+        for (int fv = 0; fv < 2; fv++)
+            if ((ret = test_darray(iosysid, ioid, num_flavors, flavor, my_rank, fv)))
+                return ret;
+
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+    }
+
+    /* Check the error string function. */
+    printf("%d Testing streror. async = %d\n", my_rank, async);
+    if ((ret = check_strerror(my_rank)))
+        ERR(ret);
+
+    /* Test name stuff. */
+    printf("%d Testing names. async = %d\n", my_rank, async);
+    if ((ret = test_names(iosysid, num_flavors, flavor, my_rank, test_comm, async)))
+        return ret;
+
+    /* Test netCDF-4 functions. */
+    printf("%d Testing nc4 functions. async = %d\n", my_rank, async);
+    if ((ret = test_nc4(iosysid, num_flavors, flavor, my_rank)))
+        return ret;
+    
+    /* Test scalar var. */
+    printf("%d Testing scalar var. async = %d\n", my_rank, async);
+    if ((ret = test_scalar(iosysid, num_flavors, flavor, my_rank, async, test_comm)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/* Run all tests. */
+int main(int argc, char **argv)
+{
+    /* Change the 5th arg to 3 to turn on logging. */
+    return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 3,
+                         TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS);
+}
diff --git a/src/externals/pio2/tests/cunit/test_pioc_fill.c b/src/externals/pio2/tests/cunit/test_pioc_fill.c
new file mode 100644
index 00000000000..c699a566be5
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_pioc_fill.c
@@ -0,0 +1,820 @@
+/*
+ * More tests for PIO data reading and writing routines.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_pioc_fill"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The names of the variable in the netCDF output files. */
+#define VAR_NAME "Laurel"
+#define VAR_NAME_2 "Hardy"
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/* 1D dim for fill_mode test. */
+#define DIM_NAME "SonsOfTheDesert"
+#define DIM_LEN 1
+
+/* Some sample data values to write. */
+char text[] = "hi";
+char char_data = 2;
+signed char byte_data = -42;
+short short_data = -300;
+int int_data = -10000;
+long int long_data = -20000;
+float float_data = -42.42;
+double double_data = -420000000000.5;
+unsigned char ubyte_data = 43;
+unsigned short ushort_data = 666;
+unsigned int uint_data = 666666;
+long long int64_data = -99999999999;
+unsigned long long uint64_data = 99999999999;
+
+char char_fill_value = 18;
+signed char byte_fill_value = -18;
+short short_fill_value = -1818;
+int int_fill_value = -18181818;
+float float_fill_value = 18.18;
+double double_fill_value = 1818181818.18;
+unsigned char ubyte_fill_value = 18;
+unsigned short ushort_fill_value = 1818;
+unsigned int uint_fill_value = 18181818;
+long long int64_fill_value = -181818181818;
+unsigned long long uint64_fill_value = 181818181818;
+
+char text_array[X_DIM_LEN/2][Y_DIM_LEN];
+char char_array[X_DIM_LEN/2][Y_DIM_LEN];
+signed char byte_array[X_DIM_LEN/2][Y_DIM_LEN];
+short short_array[X_DIM_LEN/2][Y_DIM_LEN];
+int int_array[X_DIM_LEN/2][Y_DIM_LEN];
+long int long_array[X_DIM_LEN/2][Y_DIM_LEN];
+float float_array[X_DIM_LEN/2][Y_DIM_LEN];
+double double_array[X_DIM_LEN/2][Y_DIM_LEN];
+unsigned char ubyte_array[X_DIM_LEN/2][Y_DIM_LEN];
+unsigned short ushort_array[X_DIM_LEN/2][Y_DIM_LEN];
+unsigned int uint_array[X_DIM_LEN/2][Y_DIM_LEN];
+long long int64_array[X_DIM_LEN/2][Y_DIM_LEN];
+unsigned long long uint64_array[X_DIM_LEN/2][Y_DIM_LEN];
+
+/* Fill up the data arrays with some values. */
+void init_arrays()
+{
+    for (int x = 0; x < X_DIM_LEN/2; x++)
+    {
+        strcpy(text_array[x], text);
+        for (int y = 0; y < Y_DIM_LEN; y++)
+        {
+            char_array[x][y] = char_data + y + x * Y_DIM_LEN;
+            byte_array[x][y] = byte_data + y + x * Y_DIM_LEN;
+            short_array[x][y] = short_data + y + x * Y_DIM_LEN;
+            int_array[x][y] = int_data + y + x * Y_DIM_LEN;
+            long_array[x][y] = int_data + y + x * Y_DIM_LEN;
+            float_array[x][y] = float_data + y + x * Y_DIM_LEN;
+            double_array[x][y] = double_data + y + x * Y_DIM_LEN;
+            ubyte_array[x][y] = ubyte_data + y + x * Y_DIM_LEN;
+            ushort_array[x][y] = ushort_data + y + x * Y_DIM_LEN;
+            uint_array[x][y] = uint_data + y + x * Y_DIM_LEN;
+            int64_array[x][y] = int64_data + y + x * Y_DIM_LEN;
+            uint64_array[x][y] = uint64_data + y + x * Y_DIM_LEN;
+        }
+    }
+}
+
+/* Use the vara functions to write some data to an open test file. */
+int putget_write_vara(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                      int flavor)
+{
+    int ret;
+    printf("now writing data\n");
+
+    if ((ret = PIOc_put_vara_schar(ncid, varid[0], start, count, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_text(ncid, varid[1], start, count, (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_short(ncid, varid[2], start, count, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_int(ncid, varid[3], start, count, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_float(ncid, varid[4], start, count, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_double(ncid, varid[5], start, count, (double *)double_array)))
+        return ret;
+
+    printf("now wrote classic data\n");
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        printf("now writing netcdf4 data\n");
+        if ((ret = PIOc_put_vara_uchar(ncid, varid[6], start, count, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_ushort(ncid, varid[7], start, count, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_uint(ncid, varid[8], start, count, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_longlong(ncid, varid[9], start, count, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_ulonglong(ncid, varid[10], start, count, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+int check_fill(int ncid, int *varid, int flavor, int default_fill)
+{
+    int fill_mode;
+    char char_fill_value_in;
+    signed char byte_fill_value_in;
+    short short_fill_value_in;
+    int int_fill_value_in;
+    float float_fill_value_in;
+    double double_fill_value_in;
+    unsigned char ubyte_fill_value_in;
+    unsigned short ushort_fill_value_in;
+    unsigned int uint_fill_value_in;
+    long long int64_fill_value_in;
+    unsigned long long uint64_fill_value_in;
+    int ret;
+
+    printf("checking fill values for flavor %d default_fill %d\n", flavor, default_fill);
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[0], &fill_mode, &byte_fill_value_in)))
+        return ret;
+    printf("byte_fill_value_in = %d\n", (int)byte_fill_value_in);
+    if (fill_mode != NC_FILL || byte_fill_value_in != (default_fill ? NC_FILL_BYTE : byte_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[1], &fill_mode, &char_fill_value_in)))
+        return ret;
+    if (fill_mode != NC_FILL || char_fill_value_in != (default_fill ? NC_FILL_CHAR : char_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[2], &fill_mode, &short_fill_value_in)))
+        return ret;
+    if (fill_mode != NC_FILL || short_fill_value_in != (default_fill ? NC_FILL_SHORT : short_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[3], &fill_mode, &int_fill_value_in)))
+        return ret;
+    printf("int_fill_value_in = %d\n", int_fill_value_in);
+    if (fill_mode != NC_FILL || int_fill_value_in != (default_fill ? NC_FILL_INT : int_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[4], &fill_mode, &float_fill_value_in)))
+        return ret;
+    if (fill_mode != NC_FILL || float_fill_value_in != (default_fill ? NC_FILL_FLOAT : float_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if ((ret = PIOc_inq_var_fill(ncid, varid[5], &fill_mode, &double_fill_value_in)))
+        return ret;
+    if (fill_mode != NC_FILL || double_fill_value_in != (default_fill ? NC_FILL_DOUBLE : double_fill_value))
+        return ERR_WRONG;
+    fill_mode = -99;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_inq_var_fill(ncid, varid[6], &fill_mode, &ubyte_fill_value_in)))
+            return ret;
+        if (fill_mode != NC_FILL || ubyte_fill_value_in != (default_fill ? NC_FILL_UBYTE : ubyte_fill_value))
+            return ERR_WRONG;
+        fill_mode = -99;
+
+        if ((ret = PIOc_inq_var_fill(ncid, varid[7], &fill_mode, &ushort_fill_value_in)))
+            return ret;
+        if (fill_mode != NC_FILL || ushort_fill_value_in != (default_fill ? NC_FILL_USHORT : ushort_fill_value))
+            return ERR_WRONG;
+        fill_mode = -99;
+
+        if ((ret = PIOc_inq_var_fill(ncid, varid[8], &fill_mode, &uint_fill_value_in)))
+            return ret;
+        if (fill_mode != NC_FILL || uint_fill_value_in != (default_fill ? NC_FILL_UINT : uint_fill_value))
+            return ERR_WRONG;
+        fill_mode = -99;
+
+        if ((ret = PIOc_inq_var_fill(ncid, varid[9], &fill_mode, &int64_fill_value_in)))
+            return ret;
+        if (fill_mode != NC_FILL || int64_fill_value_in != (default_fill ? NC_FILL_INT64 : int64_fill_value))
+            return ERR_WRONG;
+        fill_mode = -99;
+
+        if ((ret = PIOc_inq_var_fill(ncid, varid[10], &fill_mode, &uint64_fill_value_in)))
+            return ret;
+        if (fill_mode != NC_FILL || uint64_fill_value_in != (default_fill ? NC_FILL_UINT64 : uint64_fill_value))
+            return ERR_WRONG;
+        fill_mode = -99;
+    }
+
+    return 0;
+}
+
+/* Use the vara functions to read some data from an open test file. */
+int putget_read_vara(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                     int default_fill, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    /* Read the data we wrote. */
+    if ((ret = PIOc_get_vara_schar(ncid, varid[0], start, count, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_text(ncid, varid[1], start, count, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_short(ncid, varid[2], start, count, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_int(ncid, varid[3], start, count, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_float(ncid, varid[4], start, count, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_double(ncid, varid[5], start, count, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN/2; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+                return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vara_uchar(ncid, varid[6], start, count, (unsigned char *)ubyte_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_ushort(ncid, varid[7], start, count, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_uint(ncid, varid[8], start, count, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_longlong(ncid, varid[9], start, count, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_ulonglong(ncid, varid[10], start, count, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN/2; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    /* Check some fill value stuff. */
+    if ((ret = check_fill(ncid, varid, flavor, default_fill)))
+        return ret;
+
+    return 0;
+}
+
+/* Use the vara functions to read some data from an open test
+ * file. Expect only fill data. */
+int putget_read_vara_fill(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                          int default_fill, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN/2][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    /* Read the data we wrote. */
+    if ((ret = PIOc_get_vara_schar(ncid, varid[0], start, count, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_text(ncid, varid[1], start, count, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_short(ncid, varid[2], start, count, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_int(ncid, varid[3], start, count, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_float(ncid, varid[4], start, count, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_double(ncid, varid[5], start, count, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN/2; x++)
+    {
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != (default_fill ? NC_FILL_BYTE : byte_fill_value))
+                return ERR_WRONG;
+            if (text_array_in[x][y] != (default_fill ? NC_FILL_CHAR : char_fill_value))
+                return ERR_WRONG;
+            if (short_array_in[x][y] != (default_fill ? NC_FILL_SHORT : short_fill_value))
+                return ERR_WRONG;
+            if (int_array_in[x][y] != (default_fill ? NC_FILL_INT : int_fill_value))
+                return ERR_WRONG;
+            if (float_array_in[x][y] != (default_fill ? NC_FILL_FLOAT : float_fill_value))
+                return ERR_WRONG;
+            if (double_array_in[x][y] != (default_fill ? NC_FILL_DOUBLE : double_fill_value))
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vara_uchar(ncid, varid[6], start, count, (unsigned char *)ubyte_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_ushort(ncid, varid[7], start, count, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_uint(ncid, varid[8], start, count, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_longlong(ncid, varid[9], start, count, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_ulonglong(ncid, varid[10], start, count, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN/2; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != (default_fill ? NC_FILL_UBYTE : ubyte_fill_value))
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != (default_fill ? NC_FILL_USHORT : ushort_fill_value))
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != (default_fill ? NC_FILL_UINT : uint_fill_value))
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != (default_fill ? NC_FILL_INT64 : int64_fill_value))
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != (default_fill ? NC_FILL_UINT64 : uint64_fill_value))
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Create a test file for the tests to write data to and check by
+ * reading it back. In this function we create the file, define the
+ * dims and vars, and pass back the ncid.
+ *
+ * @param iosysid the IO system ID.
+ * @param try the number of the test run, 0 for var, 1 for var1, 2 for
+ * vara, 3 for vars.
+ * @param flavor the PIO IO type.
+ * @param dim_len array of length NDIM of the dimension lengths.
+ * @param varid array of varids for the variables in the test file.
+ * @param filename the name of the test file to create.
+ * @param default_fill true if default fill values are to be used.
+ * @param ncidp pointer that gets the ncid of the created file.
+ * @returns 0 for success, error code otherwise.
+ */
+int create_putget_file(int iosysid, int flavor, int *dim_len, int *varid, const char *filename,
+                       int default_fill, int *ncidp)
+{
+    int dimids[NDIM];        /* The dimension IDs. */
+    int num_vars = NUM_CLASSIC_TYPES;
+    int xtype[NUM_NETCDF_TYPES] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT, PIO_DOUBLE,
+                                    PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64, PIO_UINT64, PIO_STRING};
+    int ncid;
+    int old_mode;
+    int ret;
+
+    /* Create the netCDF output file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &flavor, filename, PIO_CLOBBER)))
+        return ret;
+
+    /* This should not work. */
+    if (PIOc_set_fill(ncid + TEST_VAL_42, NC_FILL, &old_mode) != PIO_EBADID)
+        return ret;
+
+    /* Turn on fill mode. */
+    if ((ret = PIOc_set_fill(ncid, NC_FILL, &old_mode)))
+        return ret;
+    printf("old_mode = %d\n", old_mode);
+    if (old_mode != NC_NOFILL)
+        return ERR_WRONG;
+
+    /* Define netCDF dimensions and variable. */
+    for (int d = 0; d < NDIM; d++)
+        if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+            return ret;
+
+    /* For netcdf-4, there are extra types. */
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+        num_vars = NUM_NETCDF_TYPES;
+
+    /* Define variables. */
+    for (int v = 0; v < num_vars; v++)
+    {
+        char var_name[PIO_MAX_NAME + 1];
+        snprintf(var_name, PIO_MAX_NAME, "%s_%d", VAR_NAME, xtype[v]);
+        printf("defining var %s\n", var_name);
+        if ((ret = PIOc_def_var(ncid, var_name, xtype[v], NDIM, dimids, &varid[v])))
+            return ret;
+    }
+
+    /* Maybe set fill values. */
+    if (!default_fill)
+    {
+        if ((ret = PIOc_def_var_fill(ncid, varid[0], NC_FILL, &byte_fill_value)))
+            return ret;
+        if ((ret = PIOc_def_var_fill(ncid, varid[1], NC_FILL, &char_fill_value)))
+            return ret;
+        if ((ret = PIOc_def_var_fill(ncid, varid[2], NC_FILL, &short_fill_value)))
+            return ret;
+        if ((ret = PIOc_def_var_fill(ncid, varid[3], NC_FILL, &int_fill_value)))
+            return ret;
+        if ((ret = PIOc_def_var_fill(ncid, varid[4], NC_FILL, &float_fill_value)))
+            return ret;
+        if ((ret = PIOc_def_var_fill(ncid, varid[5], NC_FILL, &double_fill_value)))
+            return ret;
+        if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+        {
+            if ((ret = PIOc_def_var_fill(ncid, varid[6], NC_FILL, &ubyte_fill_value)))
+                return ret;
+            if ((ret = PIOc_def_var_fill(ncid, varid[7], NC_FILL, &ushort_fill_value)))
+                return ret;
+            if ((ret = PIOc_def_var_fill(ncid, varid[8], NC_FILL, &uint_fill_value)))
+                return ret;
+            if ((ret = PIOc_def_var_fill(ncid, varid[9], NC_FILL, &int64_fill_value)))
+                return ret;
+            if ((ret = PIOc_def_var_fill(ncid, varid[10], NC_FILL, &uint64_fill_value)))
+                return ret;
+        }
+    }
+
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+
+    /* Pass back the ncid. */
+    *ncidp = ncid;
+
+    return 0;
+}
+
+/* Test data fill values.
+ *
+ * This function creates a file with 3 dimensions, with a var of each
+ * type. Then it uses the vara functions to write, and then read a
+ * subset of the variable data from the test file. The rest of the
+ * data are fill values.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @param test_comm the MPI communicator running the test.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_fill(int iosysid, int num_flavors, int *flavor, int my_rank,
+              MPI_Comm test_comm)
+{
+    int dim_len[NDIM] = {NUM_TIMESTEPS, X_DIM_LEN, Y_DIM_LEN};
+
+    printf("running tests for %d flavors\n", num_flavors);
+
+    /* Test with and without default fill values. */
+    for (int default_fill = 0; default_fill < 2; default_fill++)
+    {
+        /* Use PIO to create the example file in each of the four
+         * available ways. */
+        for (int fmt = 0; fmt < num_flavors; fmt++)
+        {
+            char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+            char iotype_name[PIO_MAX_NAME + 1];
+            int ncid;
+            int varid[NUM_NETCDF_TYPES];
+            int ret;    /* Return code. */
+
+            /* Create a filename. */
+            if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+                return ret;
+            snprintf(filename, PIO_MAX_NAME, "%s_default_fill_%d_%s.nc", TEST_NAME, default_fill, iotype_name);
+
+            /* Create test file with dims and vars defined. */
+            printf("%d creating test file %s for flavor = %d...\n", my_rank, filename, flavor[fmt]);
+            if ((ret = create_putget_file(iosysid, flavor[fmt], dim_len, varid, filename,
+                                          default_fill, &ncid)))
+                return ret;
+            printf("created file %s\n", filename);
+
+            /* Write some data. */
+            PIO_Offset start[NDIM] = {1, 0, 0};
+            PIO_Offset count[NDIM] = {1, X_DIM_LEN/2, Y_DIM_LEN};
+
+            printf("writing data to %s\n", filename);
+            /* Use the no-type vara functions to write some data. */
+            if ((ret = putget_write_vara(ncid, varid, start, count, flavor[fmt])))
+                return ret;
+
+            printf("wrote data to %s\n", filename);
+            /* Make sure all data are written (pnetcdf needs this). */
+            if ((ret = PIOc_sync(ncid)))
+                return ret;
+
+            /* Use the vara functions to read some data. */
+            if ((ret = putget_read_vara(ncid, varid, start, count, default_fill, flavor[fmt])))
+                return ret;
+
+            /* Close the netCDF file. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+
+            /* Access to read it. */
+            printf("about to try to open file %s\n", filename);
+            if ((ret = PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_WRITE)))
+                ERR(ret);
+
+            /* Use the vara functions to read some data. */
+            if ((ret = putget_read_vara(ncid, varid, start, count, default_fill, flavor[fmt])))
+                return ret;
+
+            /* Use the vara functions to read some data which are just fill values. */
+            start[0] = 0;
+            if ((ret = putget_read_vara_fill(ncid, varid, start, count, default_fill, flavor[fmt])))
+                return ret;
+
+            /* Close the netCDF file. */
+            if ((ret = PIOc_closefile(ncid)))
+                ERR(ret);
+
+        } /* next flavor */
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test fill mode in files. Create a file with no data, and make sure
+ * fill mode is working correctly for all iotypes.
+ *
+ * This function creates a file with 1 dim (of length 1), 1 var. Fill
+ * mode is turned on for the file. No data is written to the
+ * file. Then the file is closed and re-opened, and the data value
+ * checked.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @param test_comm the MPI communicator running the test.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_fill_mode(int iosysid, int num_flavors, int *flavor, int my_rank,
+                   MPI_Comm test_comm, int async)
+{
+#define NUM_TYPES_TO_TEST 2
+    int xtype[NUM_TYPES_TO_TEST] = {PIO_INT, PIO_FLOAT};
+
+    printf("test_fill_mode async = %d\n", async);
+
+    /* Test with and without default fill values. */
+    for (int default_fill = 0; default_fill < 2; default_fill++)
+    {
+        /* Test with and without extra var that does get data. */
+        for (int extra_var = 0; extra_var < 2; extra_var++)
+        {
+            /* Use PIO to create the example file in each of the four
+             * available ways. */
+            for (int fmt = 0; fmt < num_flavors; fmt++)
+            {
+                for (int t = 0; t < NUM_TYPES_TO_TEST; t++)
+                {
+                    char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+                    char iotype_name[PIO_MAX_NAME + 1];
+                    int ncid;
+                    int dimid;
+                    int varid;
+                    int varid2;
+                    void *fillvalue;
+                    PIO_Offset type_size;
+                    int default_int_fill = NC_FILL_INT;
+                    float default_float_fill = NC_FILL_FLOAT;
+                    void *extra_data;
+                    int extra_data_int = TEST_VAL_42;
+                    float extra_data_float = TEST_VAL_42;
+                    int ret;    /* Return code. */
+
+                    /* Create a filename. */
+                    if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+                        return ret;
+                    snprintf(filename, PIO_MAX_NAME, "%s_fill_mode_async_%d_default_fill_%d_extra_var_%d_%s.nc",
+                             TEST_NAME, async, default_fill, extra_var, iotype_name);
+
+                    /* Create the test file. */
+                    if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+                        return ret;
+
+                    /* Turn on fill mode. */
+                    if ((ret = PIOc_set_fill(ncid, NC_FILL, NULL)))
+                        return ret;
+
+                    /* Define a dimension. */
+                    if ((ret = PIOc_def_dim(ncid, DIM_NAME, DIM_LEN, &dimid)))
+                        return ret;
+
+                    /* Define a variable. */
+                    if ((ret = PIOc_def_var(ncid, VAR_NAME, xtype[t], 1, &dimid, &varid)))
+                        return ret;
+
+                    /* Do we want an extra variable? */
+                    if (extra_var)
+                        if ((ret = PIOc_def_var(ncid, VAR_NAME_2, xtype[t], 1, &dimid, &varid2)))
+                            return ret;
+
+                    /* Find the size of our type. */
+                    if ((ret = PIOc_inq_type(ncid, xtype[t], NULL, &type_size)))
+                        return ret;
+                    if ((xtype[t] == PIO_INT || xtype[t] == PIO_FLOAT) && type_size != 4)
+                        return ERR_WRONG;
+
+                    /* Determine fill value and extra data, depending on type. */
+                    switch (xtype[t])
+                    {
+                    case PIO_INT:
+                        fillvalue = (default_fill ? &default_int_fill: &int_fill_value);
+                        extra_data = &extra_data_int;
+                        break;
+                    case PIO_FLOAT:
+                        fillvalue = (default_fill ? &default_float_fill: &float_fill_value);
+                        extra_data = &extra_data_float;
+                        break;
+                    default:
+                        return ERR_WRONG;
+                    }
+                
+                    /* If not using a default fill value, set one. */
+                    if (!default_fill)
+                        if ((ret = PIOc_def_var_fill(ncid, varid, NC_FILL, fillvalue)))
+                            return ret;
+
+                    /* End define mode. */
+                    if ((ret = PIOc_enddef(ncid)))
+                        return ret;
+
+                    /* If there is an extra variable, write data to it. */
+                    if (extra_var)
+                        if ((ret = PIOc_put_var(ncid, varid2, extra_data)))
+                            return ret;
+                    
+                    /* Close the netCDF file. */
+                    if ((ret = PIOc_closefile(ncid)))
+                        ERR(ret);
+
+                    /* Access to read it. */
+                    printf("about to try to open file %s\n", filename);
+                    if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename, PIO_WRITE)))
+                        ERR(ret);
+
+                    /* Check the fill value. */
+                    void *fillvalue_in;
+                    int fill_mode_in;
+
+                    if (!(fillvalue_in = malloc(type_size)))
+                        return PIO_ENOMEM;
+                    if ((ret = PIOc_inq_var_fill(ncid, varid, &fill_mode_in, fillvalue_in)))
+                        return ret;
+                    if (fill_mode_in != NC_FILL)
+                        return ERR_WRONG;
+                    if (memcmp(fillvalue_in, fillvalue, type_size))
+                        return ERR_WRONG;
+                    free(fillvalue_in);
+
+                    /* Allocate space to read one element of data. */
+                    void *data_in;
+                    if (!(data_in = malloc(type_size)))
+                        return PIO_ENOMEM;
+
+                    /* Use the vara functions to read 1 datum from
+                     * the var we didn't write to. We should get a
+                     * fill value. */
+                    if ((ret = PIOc_get_var(ncid, varid, data_in)))
+                        return ret;
+                    if (memcmp(data_in, fillvalue, type_size))
+                        return ERR_WRONG;
+
+                    /* Use the vara functions to read 1 datum from the
+                     * var we did write to. We should get the value we
+                     * wrote. */
+                    if (extra_var)
+                    {
+                        if ((ret = PIOc_get_var(ncid, varid2, data_in)))
+                            return ret;
+                        if (memcmp(data_in, extra_data, type_size))
+                            return ERR_WRONG;
+                    }
+
+                    /* Free memory. */
+                    free(data_in);
+
+                    /* Close the netCDF file. */
+                    if ((ret = PIOc_closefile(ncid)))
+                        ERR(ret);
+                } /* next type */
+            } /* with or without extra var */
+        } /* next flavor */
+    }
+
+    return PIO_NOERR;
+}
+
+/* Run all the tests. */
+int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm,
+             int async)
+{
+    int my_test_size;
+    int ret; /* Return code. */
+
+    if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
+        MPIERR(ret);
+
+    /* Test that empty vars get fill values when fill mode is on. */
+    if ((ret = test_fill_mode(iosysid, num_flavors, flavor, my_rank, test_comm,
+                              async)))
+        ERR(ret);
+
+    /* Test read/write stuff. */
+    if ((ret = test_fill(iosysid, num_flavors, flavor, my_rank, test_comm)))
+        ERR(ret);
+
+    return PIO_NOERR;
+}
+
+/* Run Tests for NetCDF-4 Functions. */
+int main(int argc, char **argv)
+{
+    int ret;
+
+    /* Initialize data arrays with sample data. */
+    init_arrays();
+
+    /* Change the 5th arg to 3 to turn on logging. */
+    if ((ret = run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 0,
+                             TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS)))
+        return ret;
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_pioc_putget.c b/src/externals/pio2/tests/cunit/test_pioc_putget.c
new file mode 100644
index 00000000000..02e9177be54
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_pioc_putget.c
@@ -0,0 +1,2082 @@
+/*
+ * Tests for PIO data reading and writing routines.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_pioc_putget"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 40
+#define Y_DIM_LEN 40
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 1
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "foo"
+
+/* The name of the attribute in the netCDF output files. */
+#define ATT_NAME "bar"
+
+/* The meaning of life, the universe, and everything. */
+#define START_DATA_VAL 42
+
+/* Values for some netcdf-4 settings. */
+#define VAR_CACHE_SIZE (1024 * 1024)
+#define VAR_CACHE_NELEMS 10
+#define VAR_CACHE_PREEMPTION 0.5
+
+/* Number of NetCDF classic types. */
+#define NUM_CLASSIC_TYPES 6
+
+/* Number of NetCDF-4 types. */
+#define NUM_NETCDF4_TYPES 12
+
+/* Length of the attributes for each type. */
+#define ATT_LEN 3
+
+/* Text att (must be two chars long to fit in ATT_LEN with a NULL). */
+#define TEXT_ATT_VALUE "hi"
+
+/* Names for each type attribute. */
+#define TEXT_ATT_NAME "text_att"
+#define SCHAR_ATT_NAME "schar_att"
+#define SHORT_ATT_NAME "short_att"
+#define INT_ATT_NAME "int_att"
+#define LONG_ATT_NAME "long_att"
+#define FLOAT_ATT_NAME "float_att"
+#define DOUBLE_ATT_NAME "double_att"
+#define UCHAR_ATT_NAME "uchar_att"
+#define USHORT_ATT_NAME "ushort_att"
+#define UINT_ATT_NAME "uint_att"
+#define INT64_ATT_NAME "int64_att"
+#define UINT64_ATT_NAME "unit64_att"
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/* Length of chunksizes to use in netCDF-4 files. */
+PIO_Offset chunksize[NDIM] = {2, X_DIM_LEN/2, Y_DIM_LEN/2};
+
+/* Some sample data values to write. */
+char text[] = "hi";
+char char_data = 2;
+signed char byte_data = -42;
+short short_data = -300;
+int int_data = -10000;
+long int long_data = -20000;
+float float_data = -42.42;
+double double_data = -420000000000.5;
+unsigned char ubyte_data = 43;
+unsigned short ushort_data = 666;
+unsigned int uint_data = 666666;
+long long int64_data = -99999999999;
+unsigned long long uint64_data = 99999999999;
+
+char text_array[X_DIM_LEN][Y_DIM_LEN];
+char char_array[X_DIM_LEN][Y_DIM_LEN];
+signed char byte_array[X_DIM_LEN][Y_DIM_LEN];
+short short_array[X_DIM_LEN][Y_DIM_LEN];
+int int_array[X_DIM_LEN][Y_DIM_LEN];
+long int long_array[X_DIM_LEN][Y_DIM_LEN];
+float float_array[X_DIM_LEN][Y_DIM_LEN];
+double double_array[X_DIM_LEN][Y_DIM_LEN];
+unsigned char ubyte_array[X_DIM_LEN][Y_DIM_LEN];
+unsigned short ushort_array[X_DIM_LEN][Y_DIM_LEN];
+unsigned int uint_array[X_DIM_LEN][Y_DIM_LEN];
+long long int64_array[X_DIM_LEN][Y_DIM_LEN];
+unsigned long long uint64_array[X_DIM_LEN][Y_DIM_LEN];
+
+#define DIM_NAME "dim"
+#define NDIM1 1
+#define DIM_LEN 4
+
+/* Fill up the data arrays with some values. */
+void init_arrays()
+{
+    for (int x = 0; x < X_DIM_LEN; x++)
+    {
+        strcpy(text_array[x], text);
+        for (int y = 0; y < Y_DIM_LEN; y++)
+        {
+            char_array[x][y] = char_data;
+            byte_array[x][y] = byte_data;
+            short_array[x][y] = short_data;
+            int_array[x][y] = int_data;
+            long_array[x][y] = int_data;
+            float_array[x][y] = float_data;
+            double_array[x][y] = double_data;
+            ubyte_array[x][y] = ubyte_data;
+            ushort_array[x][y] = ushort_data;
+            uint_array[x][y] = uint_data;
+            int64_array[x][y] = int64_data;
+            uint64_array[x][y] = uint64_data;
+        }
+    }
+}
+
+/**
+ * Test the attribute conversions.
+ *
+ * @param ncid the ncid of the open test file.
+ * @param flavor the iotype of the test file.
+ * @param expected array of int, one per type. Expected return value.
+ * @param expected_data array of values we expect to find in attributes.
+ * @return 0 for success, error code otherwise.
+ */
+int test_att_conv_byte(int ncid, int flavor, char *name, int *expected, long long *expected_data)
+{
+    signed char byte_array_in[ATT_LEN];
+    short short_array_in[ATT_LEN];
+    unsigned char ubyte_array_in[ATT_LEN];
+    int int_array_in[ATT_LEN];
+    long long_array_in[ATT_LEN];
+    float float_array_in[ATT_LEN];
+    double double_array_in[ATT_LEN];
+    unsigned short ushort_array_in[ATT_LEN];
+    unsigned int uint_array_in[ATT_LEN];
+    long long int64_array_in[ATT_LEN];
+    unsigned long long uint64_array_in[ATT_LEN];
+
+    /* Read the att and check results. */
+    printf("expecting %d got %d\n", expected[PIO_BYTE], PIOc_get_att_schar(ncid, NC_GLOBAL, name, byte_array_in));
+
+    if (expected[PIO_BYTE] != PIOc_get_att_schar(ncid, NC_GLOBAL, name, byte_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_BYTE] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (byte_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (expected[PIO_SHORT] != PIOc_get_att_short(ncid, NC_GLOBAL, name, short_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_SHORT] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (short_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (expected[PIO_INT] != PIOc_get_att_int(ncid, NC_GLOBAL, name, int_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_INT] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (int_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (expected[PIO_INT] != PIOc_get_att_long(ncid, NC_GLOBAL, name, long_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_INT] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (long_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (expected[PIO_FLOAT] != PIOc_get_att_float(ncid, NC_GLOBAL, name, float_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_FLOAT] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (float_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (expected[PIO_DOUBLE] != PIOc_get_att_double(ncid, NC_GLOBAL, name, double_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_DOUBLE] == 0)
+        for (int x = 0; x < ATT_LEN; x++)
+            if (double_array_in[x] != expected_data[x])
+                return ERR_WRONG;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((expected[PIO_UBYTE] != PIOc_get_att_uchar(ncid, NC_GLOBAL, name, ubyte_array_in)))
+            return ERR_WRONG;
+
+        if (expected[PIO_UBYTE] == 0)
+            for (int x = 0; x < ATT_LEN; x++)
+                if (ubyte_array_in[x] != expected_data[x])
+                    return ERR_WRONG;
+
+        if ((expected[PIO_USHORT] != PIOc_get_att_ushort(ncid, NC_GLOBAL, name, ushort_array_in)))
+            return ERR_WRONG;
+
+        if (expected[PIO_USHORT] == 0)
+            for (int x = 0; x < ATT_LEN; x++)
+                if (ushort_array_in[x] != expected_data[x])
+                    return ERR_WRONG;
+
+        if ((expected[PIO_UINT] != PIOc_get_att_uint(ncid, NC_GLOBAL, name, uint_array_in)))
+            return ERR_WRONG;
+
+        if (expected[PIO_UINT] == 0)
+            for (int x = 0; x < ATT_LEN; x++)
+                if (uint_array_in[x] != expected_data[x])
+                    return ERR_WRONG;
+
+        if ((expected[PIO_INT64] != PIOc_get_att_longlong(ncid, NC_GLOBAL, name, int64_array_in)))
+            return ERR_WRONG;
+
+        if (expected[PIO_INT64] == 0)
+            for (int x = 0; x < ATT_LEN; x++)
+                if (int64_array_in[x] != expected_data[x])
+                    return ERR_WRONG;
+
+        if ((expected[PIO_UINT64] != PIOc_get_att_ulonglong(ncid, NC_GLOBAL, name, uint64_array_in)))
+            return ERR_WRONG;
+
+        if (expected[PIO_UINT64] == 0)
+            for (int x = 0; x < ATT_LEN; x++)
+                if (uint64_array_in[x] != expected_data[x])
+                    return ERR_WRONG;
+    }
+
+    return PIO_NOERR;
+}
+
+/**
+ * Test the attribute conversions.
+ *
+ * @param ncid the ncid of the open test file.
+ * @param flavor the iotype of the test file.
+ * @param expected array of int, one per type. Expected return value.
+ * @param expected_data array of values we expect to find in attributes.
+ * @return 0 for success, error code otherwise.
+ */
+int test_att_conv_int64(int ncid, int flavor, char *name, int *expected, long long *expected_data)
+{
+    float float_array_in[ATT_LEN];
+    double double_array_in[ATT_LEN];
+    unsigned char ubyte_array_in[ATT_LEN];
+    unsigned short ushort_array_in[ATT_LEN];
+    unsigned int uint_array_in[ATT_LEN];
+    long long int64_array_in[ATT_LEN];
+
+    /* Read the att and check results. */
+    if (expected[PIO_FLOAT] != PIOc_get_att_float(ncid, NC_GLOBAL, name, float_array_in))
+        return ERR_WRONG;
+
+    if (expected[PIO_DOUBLE] != PIOc_get_att_double(ncid, NC_GLOBAL, name, double_array_in))
+        return ERR_WRONG;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((expected[PIO_UBYTE] != PIOc_get_att_uchar(ncid, NC_GLOBAL, name, ubyte_array_in)))
+            return ERR_WRONG;
+        if ((expected[PIO_USHORT] != PIOc_get_att_ushort(ncid, NC_GLOBAL, name, ushort_array_in)))
+            return ERR_WRONG;
+        if ((expected[PIO_UINT] != PIOc_get_att_uint(ncid, NC_GLOBAL, name, uint_array_in)))
+            return ERR_WRONG;
+
+        if ((expected[PIO_INT64] != PIOc_get_att_longlong(ncid, NC_GLOBAL, name, int64_array_in)))
+            return ERR_WRONG;
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test attribute read/write operations with NC_BYTE data.
+ *
+ * This function creates a file with 3 dimensions, with a var of each
+ * type. Then it uses the var/var1/vars/vars functions to write, and
+ * then read data from the test file.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_atts_byte(int iosysid, int num_flavors, int *flavor, int my_rank,
+                   MPI_Comm test_comm)
+{
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        char iotype_name[PIO_MAX_NAME + 1];
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        int ncid;
+        int ret;    /* Return code. */
+
+        /* Create test file with dims and vars defined. */
+        printf("%d creating test file for flavor = %d...\n", my_rank, flavor[fmt]);
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "%s_att_byte_%s.nc", TEST_NAME, iotype_name);
+
+        /* Create the netCDF output file. */
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+            return ret;
+
+        if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, SCHAR_ATT_NAME, PIO_BYTE, ATT_LEN, (signed char *)byte_array)))
+            return ret;
+        if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, SHORT_ATT_NAME, PIO_SHORT, ATT_LEN, (signed char *)byte_array)))
+            return ret;
+        if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, INT_ATT_NAME, PIO_INT, ATT_LEN, (signed char *)byte_array)))
+            return ret;
+        if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, FLOAT_ATT_NAME, PIO_FLOAT, ATT_LEN, (signed char *)byte_array)))
+            return ret;
+        if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, PIO_DOUBLE, ATT_LEN, (signed char *)byte_array)))
+            return ret;
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+        {
+            if (PIOc_put_att_schar(ncid, NC_GLOBAL, UCHAR_ATT_NAME, PIO_UBYTE, ATT_LEN, (signed char *)byte_array) != PIO_ERANGE)
+                return ERR_WRONG;
+            if (PIOc_put_att_schar(ncid, NC_GLOBAL, USHORT_ATT_NAME, PIO_USHORT, ATT_LEN, (signed char *)byte_array) != PIO_ERANGE)
+                return ERR_WRONG;
+            if (PIOc_put_att_schar(ncid, NC_GLOBAL, UINT_ATT_NAME, PIO_UINT, ATT_LEN, (signed char *)byte_array) != PIO_ERANGE)
+                return ERR_WRONG;
+            if ((ret = PIOc_put_att_schar(ncid, NC_GLOBAL, INT64_ATT_NAME, PIO_INT64, ATT_LEN, (signed char *)byte_array)))
+                return ret;
+            if (PIOc_put_att_schar(ncid, NC_GLOBAL, UINT64_ATT_NAME, PIO_UINT64, ATT_LEN, (signed char *)byte_array) != PIO_ERANGE)
+                return ERR_WRONG;
+        }
+        if ((ret = PIOc_enddef(ncid)))
+            return ret;
+
+        /* Test the attribute conversions. */
+        long long expected_data[ATT_LEN];
+        for (int x = 0; x < ATT_LEN; x++)
+            expected_data[x] = byte_array[x][0];
+
+        int schar_expected[NUM_NETCDF_TYPES + 1] = {0, 0, PIO_ERANGE, 0, 0, 0, 0, PIO_ERANGE, PIO_ERANGE, PIO_ERANGE,
+                                                    0, PIO_ERANGE, PIO_ERANGE};
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], SCHAR_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], SHORT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], INT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], FLOAT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], DOUBLE_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+            if ((ret = test_att_conv_byte(ncid, flavor[fmt], INT64_ATT_NAME, schar_expected, expected_data)))
+                ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Reopen the file. */
+        if ((ret = PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_NOWRITE)))
+            ERR(ret);
+
+        /* Test the attribute conversions. */
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], SCHAR_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], SHORT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], INT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], FLOAT_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if ((ret = test_att_conv_byte(ncid, flavor[fmt], DOUBLE_ATT_NAME, schar_expected, expected_data)))
+            ERR(ret);
+        if (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P)
+            if ((ret = test_att_conv_byte(ncid, flavor[fmt], INT64_ATT_NAME, schar_expected, expected_data)))
+                ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+    } /* next flavor */
+
+    return PIO_NOERR;
+}
+
+/* Test attribute read/write operations with NC_INT64 data.
+ *
+ * This function creates a file with 3 dimensions, with a var of each
+ * type. Then it uses the var/var1/vars/vars functions to write, and
+ * then read data from the test file.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_atts_int64(int iosysid, int num_flavors, int *flavor, int my_rank,
+                    MPI_Comm test_comm)
+{
+    /* Use PIO to create the example file in each of the four
+     * available ways. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        char iotype_name[PIO_MAX_NAME + 1];
+        char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+        int ncid;
+        int ret;    /* Return code. */
+
+        /* Only do these tests on netCDF-4 files. */
+        if (flavor[fmt] != PIO_IOTYPE_NETCDF4C && flavor[fmt] != PIO_IOTYPE_NETCDF4P)
+            continue;
+
+        /* Create test file with dims and vars defined. */
+        printf("%d creating test file for flavor = %d...\n", my_rank, flavor[fmt]);
+
+        /* Create a filename. */
+        if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+            return ret;
+        sprintf(filename, "%s_att_int64_%s.nc", TEST_NAME, iotype_name);
+
+        /* Create the netCDF output file. */
+        if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, PIO_CLOBBER)))
+            return ret;
+
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, SCHAR_ATT_NAME, PIO_BYTE, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, SHORT_ATT_NAME, PIO_SHORT, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, INT_ATT_NAME, PIO_INT, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if ((ret = PIOc_put_att_longlong(ncid, NC_GLOBAL, FLOAT_ATT_NAME, PIO_FLOAT, ATT_LEN, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_att_longlong(ncid, NC_GLOBAL, DOUBLE_ATT_NAME, PIO_DOUBLE, ATT_LEN, (long long *)int64_array)))
+            return ret;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, UCHAR_ATT_NAME, PIO_UBYTE, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, USHORT_ATT_NAME, PIO_USHORT, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, UINT_ATT_NAME, PIO_UINT, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if ((ret = PIOc_put_att_longlong(ncid, NC_GLOBAL, INT64_ATT_NAME, PIO_INT64, ATT_LEN, (long long *)int64_array)))
+            return ret;
+        if (PIOc_put_att_longlong(ncid, NC_GLOBAL, UINT64_ATT_NAME, PIO_UINT64, ATT_LEN, (long long *)int64_array) != PIO_ERANGE)
+            return ERR_WRONG;
+        if ((ret = PIOc_enddef(ncid)))
+            return ret;
+
+        /* Test the attribute conversions. */
+        long long expected_data[ATT_LEN];
+        for (int x = 0; x < ATT_LEN; x++)
+            expected_data[x] = int64_array[x][0];
+
+        int int64_expected[NUM_NETCDF_TYPES + 1] = {0, PIO_ERANGE, PIO_ERANGE, PIO_ERANGE, PIO_ERANGE, 0, 0, 0, 0, 0,
+                                                    0, PIO_ERANGE, PIO_ERANGE};
+        if ((ret = test_att_conv_int64(ncid, flavor[fmt], SCHAR_ATT_NAME, int64_expected, expected_data)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+        /* Reopen the file. */
+        if ((ret = PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_NOWRITE)))
+            ERR(ret);
+
+        if ((ret = test_att_conv_int64(ncid, flavor[fmt], SCHAR_ATT_NAME, int64_expected, expected_data)))
+            ERR(ret);
+
+        /* Close the netCDF file. */
+        if ((ret = PIOc_closefile(ncid)))
+            ERR(ret);
+
+    } /* next flavor */
+
+    return PIO_NOERR;
+}
+
+/* Use the var1 functions to write some data to an open test file. */
+int putget_write_var1(int ncid, int *varid, PIO_Offset *index, int flavor)
+{
+    int ret;
+
+    /* These should not work. */
+    if (PIOc_put_var1_text(ncid, varid[1], index, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_put_var1_text(ncid + TEST_VAL_42, varid[1], index, text) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_put_var1_text(ncid, varid[1] + TEST_VAL_42, index, text) != PIO_ENOTVAR)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_put_var1_text(ncid, varid[1], index, text)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_schar(ncid, varid[0], index, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_short(ncid, varid[2], index, &short_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_int(ncid, varid[3], index, &int_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_long(ncid, varid[4], index, &long_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_float(ncid, varid[5], index, &float_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1_double(ncid, varid[6], index, &double_data)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_var1_uchar(ncid, varid[7], index, &ubyte_data)))
+            return ret;
+        if ((ret = PIOc_put_var1_ushort(ncid, varid[8], index, &ushort_data)))
+            return ret;
+        if ((ret = PIOc_put_var1_uint(ncid, varid[9], index, &uint_data)))
+            return ret;
+        if ((ret = PIOc_put_var1_longlong(ncid, varid[10], index, &int64_data)))
+            return ret;
+        if ((ret = PIOc_put_var1_ulonglong(ncid, varid[11], index, &uint64_data)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the no type versions of the var1 functions to write some data
+ * to an open test file. */
+int putget_write_var1_nt(int ncid, int *varid, PIO_Offset *index, int flavor)
+{
+    int ret;
+
+    /* These should not work. */
+    if (PIOc_put_var1(ncid, varid[1], index, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_put_var1(ncid + TEST_VAL_42, varid[1], index, text) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_put_var1(ncid, varid[1] + TEST_VAL_42, index, text) != PIO_ENOTVAR)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_put_var1(ncid, varid[0], index, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[1], index, text)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[2], index, &short_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[3], index, &int_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[4], index, &int_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[5], index, &float_data)))
+        return ret;
+
+    if ((ret = PIOc_put_var1(ncid, varid[6], index, &double_data)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_var1(ncid, varid[7], index, &ubyte_data)))
+            return ret;
+        if ((ret = PIOc_put_var1(ncid, varid[8], index, &ushort_data)))
+            return ret;
+        if ((ret = PIOc_put_var1(ncid, varid[9], index, &uint_data)))
+            return ret;
+        if ((ret = PIOc_put_var1(ncid, varid[10], index, &int64_data)))
+            return ret;
+        if ((ret = PIOc_put_var1(ncid, varid[11], index, &uint64_data)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the var functions to write some data to an open test file. */
+int putget_write_var(int ncid, int *varid, int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_var_text(ncid, varid[1], (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_schar(ncid, varid[0], (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_short(ncid, varid[2], (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_int(ncid, varid[3], (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_long(ncid, varid[4], (long int *)long_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_float(ncid, varid[5], (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var_double(ncid, varid[6], (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_var_uchar(ncid, varid[7], (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_var_ushort(ncid, varid[8], (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_var_uint(ncid, varid[9], (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_var_longlong(ncid, varid[10], (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_var_ulonglong(ncid, varid[11], (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the var functions to write some data to an open test file. */
+int putget_write_var_nt(int ncid, int *varid, int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[1], (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[0], (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[2], (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[3], (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[4], (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[5], (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_var(ncid, varid[6], (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_var(ncid, varid[7], (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_var(ncid, varid[8], (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_var(ncid, varid[9], (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_var(ncid, varid[10], (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_var(ncid, varid[11], (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the att functions to write some data to attributes in an open
+ * test file.
+ *
+ * @param ncid the ncid of the test file to read.
+ * @param varid an array of varids in the file.
+ * @param flavor the PIO IO type of the test file.
+ * @returns 0 for success, error code otherwise.
+*/
+int test_write_atts(int ncid, int *varid, int flavor)
+{
+    int ret;
+
+    /* Test some invalid parameters. (Type is irrelevant here.) */
+    if (PIOc_put_att_schar(ncid + 1, varid[0], SCHAR_ATT_NAME, PIO_BYTE,
+                           ATT_LEN, (signed char *)byte_array) != PIO_EBADID)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_put_att_text(ncid, varid[0], TEXT_ATT_NAME, ATT_LEN,
+                                 TEXT_ATT_VALUE)))
+        return ret;
+
+    /* Use put_att() for the schar. */
+    if ((ret = PIOc_put_att(ncid, varid[0], SCHAR_ATT_NAME, PIO_BYTE,
+                            ATT_LEN, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_att_text(ncid, varid[1], TEXT_ATT_NAME, ATT_LEN,
+                                 TEXT_ATT_VALUE)))
+        return ret;
+
+    if ((ret = PIOc_put_att_short(ncid, varid[2], SHORT_ATT_NAME, PIO_SHORT,
+                                  ATT_LEN, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_att_int(ncid, varid[3], INT_ATT_NAME, PIO_INT,
+                                  ATT_LEN, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_att_long(ncid, varid[4], LONG_ATT_NAME, PIO_INT,
+                                 ATT_LEN, (long int *)long_array)))
+        return ret;
+
+    if ((ret = PIOc_put_att_float(ncid, varid[5], FLOAT_ATT_NAME, PIO_FLOAT,
+                                  ATT_LEN, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_att_double(ncid, varid[6], DOUBLE_ATT_NAME, PIO_DOUBLE,
+                                  ATT_LEN, (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_att_uchar(ncid, varid[7], UCHAR_ATT_NAME, PIO_UBYTE,
+                                      ATT_LEN, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_att_ushort(ncid, varid[8], USHORT_ATT_NAME, PIO_SHORT,
+                                       ATT_LEN, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_att_uint(ncid, varid[9], UINT_ATT_NAME, PIO_UINT,
+                                     ATT_LEN, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_att_longlong(ncid, varid[10], INT64_ATT_NAME, PIO_INT64,
+                                         ATT_LEN, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_att_ulonglong(ncid, varid[11], UINT64_ATT_NAME, PIO_UINT64,
+                                          ATT_LEN, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the att functions to read some attributes from an open test
+ * file.
+ *
+ * @param ncid the ncid of the test file to read.
+ * @param varid an array of varids in the file.
+ * @param flavor the PIO IO type of the test file.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_read_att(int ncid, int *varid, int flavor)
+{
+    char text_in[ATT_LEN];
+    signed char byte_array_in[ATT_LEN];
+    short short_array_in[ATT_LEN];
+    unsigned char ubyte_array_in[ATT_LEN];
+    int int_array_in[ATT_LEN];
+    long int long_array_in[ATT_LEN];
+    float float_array_in[ATT_LEN];
+    double double_array_in[ATT_LEN];
+    unsigned short ushort_array_in[ATT_LEN];
+    unsigned int uint_array_in[ATT_LEN];
+    long long int64_array_in[ATT_LEN];
+    unsigned long long uint64_array_in[ATT_LEN];
+    int x;
+    int ret;
+
+    if ((ret = PIOc_get_att_schar(ncid, varid[0], SCHAR_ATT_NAME, byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_att_text(ncid, varid[1], TEXT_ATT_NAME, text_in)))
+        return ret;
+    if ((ret = PIOc_get_att_short(ncid, varid[2], SHORT_ATT_NAME, short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_att_int(ncid, varid[3], INT_ATT_NAME, int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_att_long(ncid, varid[4], LONG_ATT_NAME, long_array_in)))
+        return ret;
+    if ((ret = PIOc_get_att_float(ncid, varid[5], FLOAT_ATT_NAME, float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_att_double(ncid, varid[6], DOUBLE_ATT_NAME, double_array_in)))
+        return ret;
+    for (x = 0; x < ATT_LEN; x++)
+    {
+        if (strncmp(text_in, TEXT_ATT_VALUE, ATT_LEN))
+            return ERR_WRONG;
+        if (byte_array_in[x] != byte_array[x][0])
+            return ERR_WRONG;
+        if (short_array_in[x] != short_array[x][0])
+            return ERR_WRONG;
+        if (int_array_in[x] != int_array[x][0])
+            return ERR_WRONG;
+        if (long_array_in[x] != long_array[x][0])
+            return ERR_WRONG;
+        if (float_array_in[x] != float_array[x][0])
+            return ERR_WRONG;
+        if (double_array_in[x] != double_array[x][0])
+            return ERR_WRONG;
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_att_uchar(ncid, varid[7], UCHAR_ATT_NAME, ubyte_array_in)))
+            return ret;
+        if ((ret = PIOc_get_att_ushort(ncid, varid[8], USHORT_ATT_NAME, ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_att_uint(ncid, varid[9], UINT_ATT_NAME, uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_att_longlong(ncid, varid[10], INT64_ATT_NAME, int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_att_ulonglong(ncid, varid[11], UINT64_ATT_NAME, uint64_array_in)))
+            return ret;
+        for (x = 0; x < ATT_LEN; x++)
+        {
+            if (ubyte_array_in[x] != ubyte_array[x][0])
+                return ERR_WRONG;
+            if (ushort_array_in[x] != ushort_array[x][0])
+                return ERR_WRONG;
+            if (uint_array_in[x] != uint_array[x][0])
+                return ERR_WRONG;
+            if (int64_array_in[x] != int64_array[x][0])
+                return ERR_WRONG;
+            if (uint64_array_in[x] != uint64_array[x][0])
+                return ERR_WRONG;
+        }
+    }
+
+    return 0;
+}
+
+/* Use the vara functions to write some data to an open test file. */
+int putget_write_vara(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                      int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_vara_text(ncid, varid[1], start, count, (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_schar(ncid, varid[0], start, count, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_short(ncid, varid[2], start, count, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_int(ncid, varid[3], start, count, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_long(ncid, varid[4], start, count, (long int *)long_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_float(ncid, varid[5], start, count, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara_double(ncid, varid[6], start, count, (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_vara_uchar(ncid, varid[7], start, count, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_ushort(ncid, varid[8], start, count, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_uint(ncid, varid[9], start, count, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_longlong(ncid, varid[10], start, count, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_vara_ulonglong(ncid, varid[11], start, count, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the no-type vara functions to write some data to an open test file. */
+int putget_write_vara_nt(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                         int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[1], start, count, (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[0], start, count, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[2], start, count, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[3], start, count, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[4], start, count, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[5], start, count, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[6], start, count, (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_vara(ncid, varid[7], start, count, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_vara(ncid, varid[8], start, count, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_vara(ncid, varid[9], start, count, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_vara(ncid, varid[10], start, count, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_vara(ncid, varid[11], start, count, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the vars functions to write some data to an open test file. */
+int putget_write_vars(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                      PIO_Offset *stride, int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_vara_text(ncid, varid[1], start, count, (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_schar(ncid, varid[0], start, count, stride, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_short(ncid, varid[2], start, count, stride, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_int(ncid, varid[3], start, count, stride, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_long(ncid, varid[4], start, count, stride, (long int *)long_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_float(ncid, varid[5], start, count, stride, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars_double(ncid, varid[6], start, count, stride, (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_vars_uchar(ncid, varid[7], start, count, stride, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_vars_ushort(ncid, varid[8], start, count, stride, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_vars_uint(ncid, varid[9], start, count, stride, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_vars_longlong(ncid, varid[10], start, count, stride, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_vars_ulonglong(ncid, varid[11], start, count, stride, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the no-type vars functions to write some data to an open test file. */
+int putget_write_vars_nt(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                         PIO_Offset *stride, int flavor)
+{
+    int ret;
+
+    if ((ret = PIOc_put_vara(ncid, varid[1], start, count, (char *)text_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[0], start, count, stride, (signed char *)byte_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[2], start, count, stride, (short *)short_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[3], start, count, stride, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[4], start, count, stride, (int *)int_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[5], start, count, stride, (float *)float_array)))
+        return ret;
+
+    if ((ret = PIOc_put_vars(ncid, varid[6], start, count, stride, (double *)double_array)))
+        return ret;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_put_vars(ncid, varid[7], start, count, stride, (unsigned char *)ubyte_array)))
+            return ret;
+        if ((ret = PIOc_put_vars(ncid, varid[8], start, count, stride, (unsigned short *)ushort_array)))
+            return ret;
+        if ((ret = PIOc_put_vars(ncid, varid[9], start, count, stride, (unsigned int *)uint_array)))
+            return ret;
+        if ((ret = PIOc_put_vars(ncid, varid[10], start, count, stride, (long long *)int64_array)))
+            return ret;
+        if ((ret = PIOc_put_vars(ncid, varid[11], start, count, stride, (unsigned long long *)uint64_array)))
+            return ret;
+    }
+
+    return 0;
+}
+
+/* Use the var1 functions to read some data from an open test file. */
+int putget_read_var1(int ncid, int *varid, PIO_Offset *index, int flavor)
+{
+    signed char byte_data_in;
+    char text_data_in[PIO_MAX_NAME];
+    short short_data_in;
+    unsigned char ubyte_data_in;
+    int int_data_in;
+    long int long_data_in;
+    float float_data_in;
+    double double_data_in;
+    unsigned short ushort_data_in;
+    unsigned int uint_data_in;
+    long long int64_data_in;
+    unsigned long long uint64_data_in;
+    int ret;
+
+    /* These should not work. */
+    if (PIOc_get_var1_schar(ncid + TEST_VAL_42, varid[0], index, &byte_data_in) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_get_var1_schar(ncid, varid[0] + TEST_VAL_42, index, &byte_data_in) != PIO_ENOTVAR)
+        return ERR_WRONG;
+    if (PIOc_get_var1_schar(ncid, varid[0], index, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Get and check the data. */
+    if ((ret = PIOc_get_var1_schar(ncid, varid[0], index, &byte_data_in)))
+        return ret;
+    if (byte_data_in != byte_data)
+        return ERR_WRONG;
+
+    memset(text_data_in, 0, sizeof(text_data_in));
+    if ((ret = PIOc_get_var1_text(ncid, varid[1], index, text_data_in)))
+        return ret;
+    printf("text_data_in = %s\n", text_data_in);
+    if (strncmp(text_data_in, "h", 1))
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1_short(ncid, varid[2], index, &short_data_in)))
+        return ret;
+    if (short_data_in != short_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1_int(ncid, varid[3], index, &int_data_in)))
+        return ret;
+    if (int_data_in != int_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1_long(ncid, varid[4], index, &long_data_in)))
+        return ret;
+    if (long_data_in != long_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1_float(ncid, varid[5], index, &float_data_in)))
+        return ret;
+    if (float_data_in != float_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1_double(ncid, varid[6], index, &double_data_in)))
+        return ret;
+    if (double_data_in != double_data)
+        return ERR_WRONG;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_var1_uchar(ncid, varid[7], index, &ubyte_data_in)))
+            return ret;
+        if (ubyte_data_in != ubyte_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1_ushort(ncid, varid[8], index, &ushort_data_in)))
+            return ret;
+        if (ushort_data_in != ushort_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1_uint(ncid, varid[9], index, &uint_data_in)))
+            return ret;
+        if (uint_data_in != uint_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1_longlong(ncid, varid[10], index, &int64_data_in)))
+            return ret;
+        if (int64_data_in != int64_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1_ulonglong(ncid, varid[11], index, &uint64_data_in)))
+            return ret;
+        if (uint64_data_in != uint64_data)
+            return ERR_WRONG;
+    }
+
+    return 0;
+}
+
+/* Use the var1 functions to read some data from an open test file. */
+int putget_read_var1_nt(int ncid, int *varid, PIO_Offset *index, int flavor)
+{
+    signed char byte_data_in;
+    char text_data_in[PIO_MAX_NAME];
+    short short_data_in;
+    unsigned char ubyte_data_in;
+    int int_data_in;
+    float float_data_in;
+    double double_data_in;
+    unsigned short ushort_data_in;
+    unsigned int uint_data_in;
+    long long int64_data_in;
+    unsigned long long uint64_data_in;
+    int ret;
+
+    /* These should not work. */
+    if (PIOc_get_var1(ncid + TEST_VAL_42, varid[0], index, &byte_data_in) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_get_var1(ncid, varid[0] + TEST_VAL_42, index, &byte_data_in) != PIO_ENOTVAR)
+        return ERR_WRONG;
+    if (PIOc_get_var1(ncid, varid[0], index, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* Get and check the data. */
+    if ((ret = PIOc_get_var1(ncid, varid[0], index, &byte_data_in)))
+        return ret;
+    if (byte_data_in != byte_data)
+        return ERR_WRONG;
+
+    memset(text_data_in, 0, sizeof(text_data_in));
+    if ((ret = PIOc_get_var1(ncid, varid[1], index, text_data_in)))
+        return ret;
+    printf("text_data_in = %s\n", text_data_in);
+    if (strncmp(text_data_in, "h", 1))
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1(ncid, varid[2], index, &short_data_in)))
+        return ret;
+    if (short_data_in != short_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1(ncid, varid[3], index, &int_data_in)))
+        return ret;
+    if (int_data_in != int_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1(ncid, varid[4], index, &int_data_in)))
+        return ret;
+    if (int_data_in != int_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1(ncid, varid[5], index, &float_data_in)))
+        return ret;
+    if (float_data_in != float_data)
+        return ERR_WRONG;
+
+    if ((ret = PIOc_get_var1(ncid, varid[6], index, &double_data_in)))
+        return ret;
+    if (double_data_in != double_data)
+        return ERR_WRONG;
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_var1(ncid, varid[7], index, &ubyte_data_in)))
+            return ret;
+        if (ubyte_data_in != ubyte_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1(ncid, varid[8], index, &ushort_data_in)))
+            return ret;
+        if (ushort_data_in != ushort_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1(ncid, varid[9], index, &uint_data_in)))
+            return ret;
+        if (uint_data_in != uint_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1(ncid, varid[10], index, &int64_data_in)))
+            return ret;
+        if (int64_data_in != int64_data)
+            return ERR_WRONG;
+        if ((ret = PIOc_get_var1(ncid, varid[11], index, &uint64_data_in)))
+            return ret;
+        if (uint64_data_in != uint64_data)
+            return ERR_WRONG;
+    }
+
+    return 0;
+}
+
+/* Use the var functions to read some data from an open test file.
+ *
+ * @param ncid the ncid of the test file to read.
+ * @param varid an array of varids in the file.
+ * @param unlim non-zero if unlimited dimension is in use.
+ * @param flavor the PIO IO type of the test file.
+ * @returns 0 for success, error code otherwise.
+*/
+int putget_read_var(int ncid, int *varid, int unlim, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long int long_array_in[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    /* When using the unlimited dimension, no data are wrtten by the
+     * put_var_TYPE() functions, since the length of the unlimited
+     * dimension is still 0. */
+    if (unlim)
+    {
+        return 0;
+    }
+
+    if ((ret = PIOc_get_var_schar(ncid, varid[0], (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_text(ncid, varid[1], (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_short(ncid, varid[2], (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_int(ncid, varid[3], (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_long(ncid, varid[4], (long int *)long_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_float(ncid, varid[5], (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var_double(ncid, varid[6], (double *)double_array_in)))
+        return ret;
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+                return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (long_array_in[x][y] != long_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_var_uchar(ncid, varid[7], (unsigned char *)ubyte_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var_ushort(ncid, varid[8], (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var_uint(ncid, varid[9], (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var_longlong(ncid, varid[10], (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var_ulonglong(ncid, varid[11], (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Use the no-type var functions to read some data from an open test file.
+ *
+ * @param ncid the ncid of the test file to read.
+ * @param varid an array of varids in the file.
+ * @param unlim non-zero if unlimited dimension is in use.
+ * @param flavor the PIO IO type of the test file.
+ * @returns 0 for success, error code otherwise.
+*/
+int putget_read_var_nt(int ncid, int *varid, int unlim, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in2[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    /* When using the unlimited dimension, no data are wrtten by the
+     * put_var_TYPE() functions, since the length of the unlimited
+     * dimension is still 0. */
+    if (unlim)
+    {
+        return 0;
+    }
+
+    if ((ret = PIOc_get_var(ncid, varid[0], (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[1], (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[2], (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[3], (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[4], (int *)int_array_in2)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[5], (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_var(ncid, varid[6], (double *)double_array_in)))
+        return ret;
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+                return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in2[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_var(ncid, varid[7], (unsigned char *)ubyte_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var(ncid, varid[8], (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var(ncid, varid[9], (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var(ncid, varid[10], (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_var(ncid, varid[11], (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Use the vara functions to read some data from an open test file. */
+int putget_read_vara(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                     int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long int long_array_in[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    if ((ret = PIOc_get_vara_schar(ncid, varid[0], start, count, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_text(ncid, varid[1], start, count, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_short(ncid, varid[2], start, count, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_int(ncid, varid[3], start, count, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_long(ncid, varid[4], start, count, (long int *)long_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_float(ncid, varid[5], start, count, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara_double(ncid, varid[6], start, count, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+                return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (long_array_in[x][y] != long_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vara_uchar(ncid, varid[7], start, count, (unsigned char *)ubyte_array_in)))
+            return ret;
+
+        if ((ret = PIOc_get_vara_ushort(ncid, varid[8], start, count, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_uint(ncid, varid[9], start, count, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_longlong(ncid, varid[10], start, count, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara_ulonglong(ncid, varid[11], start, count, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Use the vars functions to read some data from an open test file. */
+int putget_read_vars(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                     PIO_Offset *stride, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long int long_array_in[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    if ((ret = PIOc_get_vars_schar(ncid, varid[0], start, count, stride, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_text(ncid, varid[1], start, count, stride, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_short(ncid, varid[2], start, count, stride, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_int(ncid, varid[3], start, count, stride, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_long(ncid, varid[4], start, count, stride, (long int *)long_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_float(ncid, varid[5], start, count, stride, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars_double(ncid, varid[6], start, count, stride, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+            return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (long_array_in[x][y] != long_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vars_uchar(ncid, varid[7], start, count, stride, (unsigned char *)ubyte_array_in)))
+            return ret;
+
+        if ((ret = PIOc_get_vars_ushort(ncid, varid[8], start, count, stride, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars_uint(ncid, varid[9], start, count, stride, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars_longlong(ncid, varid[10], start, count, stride, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars_ulonglong(ncid, varid[11], start, count, stride, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Use the no-type vara functions to read some data from an open test file. */
+int putget_read_vara_nt(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                        int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in2[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    if ((ret = PIOc_get_vara(ncid, varid[0], start, count, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[1], start, count, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[2], start, count, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[3], start, count, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[4], start, count, (int *)int_array_in2)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[5], start, count, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vara(ncid, varid[6], start, count, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+                return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in2[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vara(ncid, varid[7], start, count, (unsigned char *)ubyte_array_in)))
+            return ret;
+
+        if ((ret = PIOc_get_vara(ncid, varid[8], start, count, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara(ncid, varid[9], start, count, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara(ncid, varid[10], start, count, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vara(ncid, varid[11], start, count, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Use the vars functions to read some data from an open test file. */
+int putget_read_vars_nt(int ncid, int *varid, PIO_Offset *start, PIO_Offset *count,
+                        PIO_Offset *stride, int flavor)
+{
+    signed char byte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    char text_array_in[X_DIM_LEN][Y_DIM_LEN];
+    short short_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned char ubyte_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int int_array_in2[X_DIM_LEN][Y_DIM_LEN];
+    float float_array_in[X_DIM_LEN][Y_DIM_LEN];
+    double double_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned short ushort_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned int uint_array_in[X_DIM_LEN][Y_DIM_LEN];
+    long long int64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    unsigned long long uint64_array_in[X_DIM_LEN][Y_DIM_LEN];
+    int x, y;
+    int ret;
+
+    if ((ret = PIOc_get_vars(ncid, varid[0], start, count, stride, (signed char *)byte_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[1], start, count, stride, (char *)text_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[2], start, count, stride, (short *)short_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[3], start, count, stride, (int *)int_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[4], start, count, stride, (int *)int_array_in2)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[5], start, count, stride, (float *)float_array_in)))
+        return ret;
+    if ((ret = PIOc_get_vars(ncid, varid[6], start, count, stride, (double *)double_array_in)))
+        return ret;
+
+    for (x = 0; x < X_DIM_LEN; x++)
+    {
+        if (strncmp(text_array_in[x], text, strlen(text)))
+            return ERR_WRONG;
+        for (y = 0; y < Y_DIM_LEN; y++)
+        {
+            if (byte_array_in[x][y] != byte_array[x][y])
+                return ERR_WRONG;
+            if (short_array_in[x][y] != short_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (int_array_in2[x][y] != int_array[x][y])
+                return ERR_WRONG;
+            if (float_array_in[x][y] != float_array[x][y])
+                return ERR_WRONG;
+            if (double_array_in[x][y] != double_array[x][y])
+                return ERR_WRONG;
+        }
+    }
+
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+    {
+        if ((ret = PIOc_get_vars(ncid, varid[7], start, count, stride, (unsigned char *)ubyte_array_in)))
+            return ret;
+
+        if ((ret = PIOc_get_vars(ncid, varid[8], start, count, stride, (unsigned short *)ushort_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars(ncid, varid[9], start, count, stride, (unsigned int *)uint_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars(ncid, varid[10], start, count, stride, (long long *)int64_array_in)))
+            return ret;
+        if ((ret = PIOc_get_vars(ncid, varid[11], start, count, stride, (unsigned long long *)uint64_array_in)))
+            return ret;
+        for (x = 0; x < X_DIM_LEN; x++)
+            for (y = 0; y < Y_DIM_LEN; y++)
+            {
+                if (ubyte_array_in[x][y] != ubyte_array[x][y])
+                    return ERR_WRONG;
+                if (ushort_array_in[x][y] != ushort_array[x][y])
+                    return ERR_WRONG;
+                if (uint_array_in[x][y] != uint_array[x][y])
+                    return ERR_WRONG;
+                if (int64_array_in[x][y] != int64_array[x][y])
+                    return ERR_WRONG;
+                if (uint64_array_in[x][y] != uint64_array[x][y])
+                    return ERR_WRONG;
+            }
+    }
+
+    return 0;
+}
+
+/* Create a test file for the putget tests to write data to and check
+ * by reading it back. In this function we create the file, define the
+ * dims and vars, and pass back the ncid.
+ *
+ * @param iosysid the IO system ID.
+ * @param try the number of the test run, 0 for var, 1 for var1, 2 for
+ * vara, 3 for vars.
+ * @param unlim non-zero if unlimited dimension should be used.
+ * @param flavor the PIO IO type.
+ * @param dim_len array of length NDIM of the dimension lengths.
+ * @param varid array of varids for the variables in the test file.
+ * @param filename the name of the test file to create.
+ * @param ncidp pointer that gets the ncid of the created file.
+ * @returns 0 for success, error code otherwise.
+ */
+int create_putget_file(int iosysid, int access, int unlim, int flavor, int *dim_len,
+                       int *varid, const char *filename, int *ncidp)
+{
+    int dimids[NDIM];        /* The dimension IDs. */
+    int num_vars = NUM_CLASSIC_TYPES + 1;
+    int xtype[NUM_NETCDF4_TYPES + 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_LONG_INTERNAL,
+                                        PIO_FLOAT, PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64,
+                                        PIO_UINT64, PIO_STRING};
+    int ncid;
+    int ret;
+
+    /* This should not work. */
+    if (PIOc_createfile(iosysid + TEST_VAL_42, &ncid, &flavor, filename, PIO_CLOBBER) != PIO_EBADID)
+        return ERR_WRONG;
+
+    /* Create the netCDF output file. */
+    if ((ret = PIOc_createfile(iosysid, &ncid, &flavor, filename, PIO_CLOBBER)))
+        return ret;
+
+    /* Are we using unlimited dimension? */
+    if (!unlim)
+        dim_len[0] = NUM_TIMESTEPS;
+
+    printf("filename = %s\n", filename);
+    /* Define netCDF dimensions and variable. */
+    for (int d = 0; d < NDIM; d++)
+        if ((ret = PIOc_def_dim(ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+            return ret;
+
+    /* For netcdf-4, there are extra types. */
+    if (flavor == PIO_IOTYPE_NETCDF4C || flavor == PIO_IOTYPE_NETCDF4P)
+        num_vars = NUM_NETCDF4_TYPES + 1;
+
+    printf("filename = %s\n", filename);
+    /* Define variables. */
+    for (int v = 0; v < num_vars; v++)
+    {
+        char var_name[PIO_MAX_NAME + 1];
+        snprintf(var_name, PIO_MAX_NAME, "%s_%d", VAR_NAME, xtype[v]);
+        printf("defining var %s\n", var_name);
+        /*nc_type my_type = xtype[v] == PIO_LONG_INTERNAL ? PIO_INT : xtype[v];*/
+        nc_type my_type;
+        if (xtype[v] == PIO_LONG_INTERNAL)
+            my_type = PIO_INT;
+        else
+            my_type = xtype[v];
+        if ((ret = PIOc_def_var(ncid, var_name, my_type, NDIM, dimids, &varid[v])))
+            return ret;
+    }
+
+    printf("filename = %s\n", filename);
+    /* For the first access, also test attributes. */
+    if (access == 0)
+        if ((ret = test_write_atts(ncid, varid, flavor)))
+            return ret;
+
+    if ((ret = PIOc_enddef(ncid)))
+        return ret;
+
+    /* Pass back the ncid. */
+    *ncidp = ncid;
+
+    return 0;
+}
+
+/** Check the contents of the test file. */
+int check_file(int access, int ncid, int *varid, int flavor, PIO_Offset *index, PIO_Offset *start,
+               PIO_Offset *count, PIO_Offset *stride, int unlim)
+{
+    int ret;
+    
+    switch (access)
+    {
+    case 0:
+        /* Use the att functions to read some data. */
+        if ((ret = test_read_att(ncid, varid, flavor)))
+            return ret;
+
+        /* Use the vara functions to read some data. */
+        if ((ret = putget_read_var(ncid, varid, unlim, flavor)))
+            return ret;
+        break;
+
+    case 1:
+        /* Use the var1 functions to read some data. */
+        if ((ret = putget_read_var1(ncid, varid, index, flavor)))
+            return ret;
+        break;
+
+    case 2:
+        /* Use the vara functions to read some data. */
+        if ((ret = putget_read_vara(ncid, varid, start, count, flavor)))
+            return ret;
+        break;
+
+    case 3:
+        /* Use the vara functions to read some data. */
+        if ((ret = putget_read_vars(ncid, varid, start, count, stride, flavor)))
+            return ret;
+        break;
+
+    case 4:
+        /* Use the var1 functions to read some data. */
+        if ((ret = putget_read_var_nt(ncid, varid, unlim, flavor)))
+            return ret;
+        break;
+
+    case 5:
+        /* Use the var1 functions to read some data. */
+        if ((ret = putget_read_var1_nt(ncid, varid, index, flavor)))
+            return ret;
+        break;
+
+    case 6:
+        /* Use the vara functions to read some data. */
+        if ((ret = putget_read_vara_nt(ncid, varid, start, count, flavor)))
+            return ret;
+        break;
+
+    case 7:
+        /* Use the vara functions to read some data. */
+        if ((ret = putget_read_vars_nt(ncid, varid, start, count, stride, flavor)))
+            return ret;
+        break;
+
+    }
+
+    return PIO_NOERR;
+}
+
+/* Test data read/write operations.
+ *
+ * This function creates a file with 3 dimensions, with a var of each
+ * type. Then it uses the var/var1/vars/vars functions to write, and
+ * then read data from the test file.
+ *
+ * @param iosysid the iosystem ID that will be used for the test.
+ * @param num_flavors the number of different IO types that will be tested.
+ * @param flavor an array of the valid IO types.
+ * @param my_rank 0-based rank of task.
+ * @returns 0 for success, error code otherwise.
+ */
+int test_putget(int iosysid, int num_flavors, int *flavor, int my_rank,
+                MPI_Comm test_comm)
+{
+    int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+#define NUM_ACCESS 8
+    for (int unlim = 0; unlim < 2; unlim++)
+        for (int access = 0; access < NUM_ACCESS; access++)
+        {
+            /* Use PIO to create the example file in each of the four
+             * available ways. */
+            for (int fmt = 0; fmt < num_flavors; fmt++)
+            {
+                char filename[PIO_MAX_NAME + 1]; /* Test filename. */
+                char iotype_name[PIO_MAX_NAME + 1];
+                int ncid;
+                int varid[NUM_NETCDF4_TYPES + 1];
+                int ret;    /* Return code. */
+
+                /* Create a filename. */
+                if ((ret = get_iotype_name(flavor[fmt], iotype_name)))
+                    return ret;
+                snprintf(filename, PIO_MAX_NAME, "%s_putget_access_%d_unlim_%d_%s.nc", TEST_NAME,
+                         access, unlim, iotype_name);
+
+                /* Create test file with dims and vars defined. */
+                printf("%d Access %d creating test file %s for flavor = %d...\n", my_rank, access,
+                       filename, flavor[fmt]);
+                if ((ret = create_putget_file(iosysid, access, unlim, flavor[fmt], dim_len, varid,
+                                              filename, &ncid)))
+                    return ret;
+                printf("created file %s\n", filename);
+
+                /* Write some data. */
+                PIO_Offset index[NDIM] = {0, 0, 0};
+                PIO_Offset start[NDIM] = {0, 0, 0};
+                PIO_Offset count[NDIM] = {1, X_DIM_LEN, Y_DIM_LEN};
+                PIO_Offset stride[NDIM] = {1, 1, 1};
+
+                switch (access)
+                {
+                case 0:
+                    printf("%d Access %d writing data with var functions for flavor = %d...\n",
+                           my_rank, access, flavor[fmt]);
+                    /* Use the var functions to write some data. */
+                    if ((ret = putget_write_var(ncid, varid, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 1:
+                    /* Use the var1 functions to write some data. */
+                    if ((ret = putget_write_var1(ncid, varid, index, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 2:
+                    /* Use the vara functions to write some data. */
+                    if ((ret = putget_write_vara(ncid, varid, start, count, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 3:
+                    /* Use the vara functions to write some data. */
+                    if ((ret = putget_write_vars(ncid, varid, start, count, stride, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 4:
+                    /* Use the no-type var functions to write some data. */
+                    if ((ret = putget_write_var_nt(ncid, varid, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 5:
+                    /* Use the no-type var1 functions to write some data. */
+                    if ((ret = putget_write_var1_nt(ncid, varid, index, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 6:
+                    /* Use the no-type vara functions to write some data. */
+                    if ((ret = putget_write_vara_nt(ncid, varid, start, count, flavor[fmt])))
+                        return ret;
+                    break;
+
+                case 7:
+                    /* Use the no-type vars functions to write some data. */
+                    if ((ret = putget_write_vars_nt(ncid, varid, start, count, stride, flavor[fmt])))
+                        return ret;
+                    break;
+
+                }
+
+                /* Check for bad input handling. */
+                if (PIOc_sync(ncid + TEST_VAL_42) != PIO_EBADID)
+                    return ret;
+
+                /* Make sure all data are written (pnetcdf needs this). */
+                if ((ret = PIOc_sync(ncid)))
+                    return ret;
+
+                /* Check contents of the file. */
+                if ((ret = check_file(access, ncid, varid, flavor[fmt], index, start, count, stride, unlim)))
+                    return ret;
+
+                /* Close the netCDF file. */
+                if ((ret = PIOc_closefile(ncid)))
+                    ERR(ret);
+
+                /* /\* Access to read it. *\/ */
+                printf("about to try to open file %s\n", filename);
+                if ((ret = PIOc_openfile(iosysid, &ncid, &(flavor[fmt]), filename, PIO_NOWRITE)))
+                    ERR(ret);
+
+                /* Check contents of the file. */
+                if ((ret = check_file(access, ncid, varid, flavor[fmt], index, start, count, stride, unlim)))
+                    return ret;
+
+                /* Close the netCDF file. */
+                if ((ret = PIOc_closefile(ncid)))
+                    ERR(ret);
+
+            } /* next flavor */
+        } /* next access */
+
+    return PIO_NOERR;
+}
+
+/* Run all the tests. */
+int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm,
+             int async)
+{
+    int my_test_size;
+    int ret; /* Return code. */
+
+    if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
+        MPIERR(ret);
+
+    /* Test attribute stuff. */
+    printf("%d Testing attributes with NC_BYTE data, async = %d\n", my_rank, async);
+    if ((ret = test_atts_byte(iosysid, num_flavors, flavor, my_rank, test_comm)))
+        return ret;
+
+    printf("%d Testing attributes with NC_INT64 data, async = %d\n", my_rank, async);
+    if ((ret = test_atts_int64(iosysid, num_flavors, flavor, my_rank, test_comm)))
+        return ret;
+
+    /* Test read/write stuff. */
+    printf("%d Testing putget. async = %d\n", my_rank, async);
+    if ((ret = test_putget(iosysid, num_flavors, flavor, my_rank, test_comm)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/* Run Tests for NetCDF-4 Functions. */
+int main(int argc, char **argv)
+{
+    /* Initialize data arrays with sample data. */
+    init_arrays();
+
+    return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 0,
+                         TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_pioc_unlim.c b/src/externals/pio2/tests/cunit/test_pioc_unlim.c
new file mode 100644
index 00000000000..3b4a4df347c
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_pioc_unlim.c
@@ -0,0 +1,188 @@
+/*
+ * Tests for PIO Functions. In this test we use a simple 3D variable,
+ * with an unlimited dimension. The data will have two timesteps, and
+ * 4x4 elements each timestep.
+ *
+ * Ed Hartnett, 2/14/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 4
+
+/* The name of this test. */
+#define TEST_NAME "test_pioc_unlim"
+
+/* Number of processors that will do IO. */
+#define NUM_IO_PROCS 1
+
+/* Number of computational components to create. */
+#define COMPONENT_COUNT 1
+
+/* The number of dimensions in the example data. In this test, we
+ * are using three-dimensional data. */
+#define NDIM 3
+
+/* The length of our sample data along each dimension. */
+#define X_DIM_LEN 4
+#define Y_DIM_LEN 4
+
+/* The number of timesteps of data to write. */
+#define NUM_TIMESTEPS 2
+
+/* The name of the variable in the netCDF output files. */
+#define VAR_NAME "var_3D_with_unlim"
+
+/* The meaning of life, the universe, and everything. */
+#define START_DATA_VAL 42
+
+/* The dimension names. */
+char dim_name[NDIM][PIO_MAX_NAME + 1] = {"timestep", "x", "y"};
+
+/* Length of the dimensions in the sample data. */
+int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN};
+
+/* Create the decomposition to divide the 3-dimensional sample data
+ * between the 4 tasks.
+ *
+ * @param ntasks the number of available tasks
+ * @param my_rank rank of this task.
+ * @param iosysid the IO system ID.
+ * @param dim1_len the length of the dimension.
+ * @param ioid a pointer that gets the ID of this decomposition.
+ * @returns 0 for success, error code otherwise.
+ **/
+int create_decomposition(int ntasks, int my_rank, int iosysid, int dim1_len,
+                         int *ioid)
+{
+    PIO_Offset elements_per_pe;     /* Array elements per processing unit. */
+    PIO_Offset *compdof;  /* The decomposition mapping. */
+    int ret;
+
+    /* How many data elements per task? */
+    elements_per_pe = X_DIM_LEN * Y_DIM_LEN / ntasks;
+
+    /* Allocate space for the decomposition array. */
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+
+    /* Describe the decomposition. This is a 1-based array, so add 1! */
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i + 1;
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition elements_per_pe = %lld\n", my_rank, elements_per_pe);
+    if ((ret = PIOc_InitDecomp(iosysid, PIO_FLOAT, NDIM - 1, &dim_len[1], elements_per_pe,
+                               compdof, ioid, NULL, NULL, NULL)))
+        ERR(ret);
+
+    printf("%d decomposition initialized.\n", my_rank);
+
+    /* Free the mapping. */
+    free(compdof);
+
+    return 0;
+}
+
+/* Create the test file. */
+int create_test_file(int iosysid, int ioid, int iotype, int my_rank, int *ncid, int *varid)
+{
+    char filename[PIO_MAX_NAME + 1]; /* Name for the output files. */
+    int dim_len[NDIM] = {NC_UNLIMITED, X_DIM_LEN, Y_DIM_LEN}; /* Length of the dimensions in the sample data. */
+    int dimids[NDIM];      /* The dimension IDs. */
+    int ret;       /* Return code. */
+
+    /* Create the filename. */
+    sprintf(filename, "%s_iotype_%d.nc", TEST_NAME, iotype);
+    
+    /* Create the netCDF output file. */
+    printf("rank: %d Creating sample file %s with format %d...\n", my_rank, filename,
+           iotype);
+    if ((ret = PIOc_createfile(iosysid, ncid, &iotype, filename, PIO_CLOBBER)))
+        ERR(ret);
+    
+    /* Define netCDF dimensions and variable. */
+    printf("rank: %d Defining netCDF metadata...\n", my_rank);
+    for (int d = 0; d < NDIM; d++)
+        if ((ret = PIOc_def_dim(*ncid, dim_name[d], (PIO_Offset)dim_len[d], &dimids[d])))
+            ERR(ret);
+    
+    /* Define a variable. */
+    if ((ret = PIOc_def_var(*ncid, VAR_NAME, PIO_FLOAT, NDIM, dimids, varid)))
+        ERR(ret);
+    
+    /* End define mode. */
+    if ((ret = PIOc_enddef(*ncid)))
+        ERR(ret);
+    
+    return PIO_NOERR;
+}
+
+/* Run all the tests. */
+int test_all(int iosysid, int num_flavors, int *flavor, int my_rank, MPI_Comm test_comm,
+             int async)
+{
+    int ioid;
+    int ncid;
+    int varid;
+    int my_test_size;
+    char filename[NC_MAX_NAME + 1];
+    int ret; /* Return code. */
+
+    if ((ret = MPI_Comm_size(test_comm, &my_test_size)))
+        MPIERR(ret);
+    
+    /* This will be our file name for writing out decompositions. */
+    sprintf(filename, "decomp_%d.txt", my_rank);
+
+    if (!async)
+    {
+        printf("%d Testing darray. async = %d\n", my_rank, async);
+        
+        /* Decompose the data over the tasks. */
+        if ((ret = create_decomposition(my_test_size, my_rank, iosysid, X_DIM_LEN, &ioid)))
+            return ret;
+
+        /* Use PIO to create the example file in each of the four
+         * available ways. */
+        for (int fmt = 0; fmt < num_flavors; fmt++)
+        {
+            if ((ret = create_test_file(iosysid, ioid, flavor[fmt], my_rank, &ncid, &varid)))
+                return ret;
+            if ((ret = PIOc_setframe(ncid, varid, 0)))
+                return ret;
+            if ((ret = PIOc_advanceframe(ncid, varid)))
+                return ret;
+
+            /* Look at the internals to check that the frame commands
+             * worked. */
+            file_desc_t *file;            
+            if ((ret = pio_get_file(ncid, &file)))
+                return ret;
+            if (file->varlist[varid].record != 1)
+                return ERR_WRONG;
+            
+            if ((PIOc_closefile(ncid)))
+                return ret;
+        }
+
+        /* Free the PIO decomposition. */
+        if ((ret = PIOc_freedecomp(iosysid, ioid)))
+            ERR(ret);
+    }
+
+    return PIO_NOERR;
+}
+
+/* Run Tests for NetCDF-4 Functions. */
+int main(int argc, char **argv)
+{
+    /* Change the 5th arg to 3 to turn on logging. */
+    return run_test_main(argc, argv, MIN_NTASKS, TARGET_NTASKS, 3,
+                         TEST_NAME, dim_len, COMPONENT_COUNT, NUM_IO_PROCS);
+}
diff --git a/src/externals/pio2/tests/cunit/test_rearr.c b/src/externals/pio2/tests/cunit/test_rearr.c
new file mode 100644
index 00000000000..89e278bfc14
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_rearr.c
@@ -0,0 +1,1542 @@
+/*
+ * This program tests some internal functions in the library related
+ * to the box and subset rearranger, and the transfer of data betweeen
+ * IO and computation tasks.
+ *
+ * Ed Hartnett, 3/9/17
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 1
+
+/* The name of this test. */
+#define TEST_NAME "test_rearr"
+
+/* For 1-D use. */
+#define NDIM1 1
+
+/* For maplens of 2. */
+#define MAPLEN2 2
+
+/* Name of test var. (Name of a Welsh town.)*/
+#define VAR_NAME "Llanfairpwllgwyngyllgogerychwyrndrobwllllantysiliogogogoch"
+
+/* Test some of the rearranger utility functions. */
+int test_rearranger_opts1(int iosysid)
+{
+    iosystem_desc_t *ios;
+    int ret;
+
+    /* This should not work. */
+    if (PIOc_set_rearr_opts(TEST_VAL_42, 0, 0, false, false, 0, false,
+                            false, 0) != PIO_EBADID)
+        return ERR_WRONG;
+    if (PIOc_set_rearr_opts(iosysid, TEST_VAL_42, 0, false, false, 0, false,
+                            false, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_set_rearr_opts(iosysid, 0, TEST_VAL_42, false, false, 0, false,
+                            false, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_set_rearr_opts(iosysid, 0, 0, false, false,
+                            PIO_REARR_COMM_UNLIMITED_PEND_REQ - 1, false,
+                            false, 0) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (PIOc_set_rearr_opts(iosysid, 0, 0, false, false, 0, false,
+                            false, PIO_REARR_COMM_UNLIMITED_PEND_REQ - 1) !=
+        PIO_EINVAL)
+        return ERR_WRONG;
+
+    /* This should work. */
+    if ((ret = PIOc_set_rearr_opts(iosysid, PIO_REARR_COMM_P2P,
+                                   PIO_REARR_COMM_FC_1D_COMP2IO, true,
+                                   true, TEST_VAL_42, true, true, TEST_VAL_42 + 1)))
+        return ret;
+
+    /* Get the IO system info from the id. */
+    if (!(ios = pio_get_iosystem_from_id(iosysid)))
+        return pio_err(NULL, NULL, PIO_EBADID, __FILE__, __LINE__);
+
+    /* Check the rearranger settings. */
+    if (ios->rearr_opts.comm_type != PIO_REARR_COMM_P2P ||
+        ios->rearr_opts.fcd != PIO_REARR_COMM_FC_1D_COMP2IO ||
+        !ios->rearr_opts.comp2io.hs || !ios->rearr_opts.comp2io.isend ||
+        !ios->rearr_opts.io2comp.hs || !ios->rearr_opts.io2comp.isend ||
+        ios->rearr_opts.comp2io.max_pend_req != TEST_VAL_42 ||
+        ios->rearr_opts.io2comp.max_pend_req != TEST_VAL_42 + 1)
+        return ERR_WRONG;
+
+    return 0;
+}
+
+/* Test the compare_offsets() function. */
+int test_compare_offsets()
+{
+    mapsort m1, m2, m3;
+
+    m1.rfrom = 0;
+    m1.soffset = 0;
+    m1.iomap = 0;
+    m2.rfrom = 0;
+    m2.soffset = 0;
+    m2.iomap = 0;
+    m3.rfrom = 0;
+    m3.soffset = 0;
+    m3.iomap = 1;
+
+    /* Return 0 if either or both parameters are null. */
+    if (compare_offsets(NULL, &m2))
+        return ERR_WRONG;
+    if (compare_offsets(&m1, NULL))
+        return ERR_WRONG;
+    if (compare_offsets(NULL, NULL))
+        return ERR_WRONG;
+
+    /* m1 and m2 are the same. */
+    if (compare_offsets(&m1, &m2))
+        return ERR_WRONG;
+
+    /* m1 and m3 are different. */
+    if (compare_offsets(&m1, &m3) != -1)
+        return ERR_WRONG;
+    return 0;
+}
+
+/* Test the ceil2() and pair() functions. */
+int test_ceil2_pair()
+{
+    /* Test the ceil2() function. */
+    if (ceil2(1) != 1)
+        return ERR_WRONG;
+    if (ceil2(-100) != 1)
+        return ERR_WRONG;
+    if (ceil2(2) != 2)
+        return ERR_WRONG;
+    if (ceil2(3) != 4)
+        return ERR_WRONG;
+    if (ceil2(16) != 16)
+        return ERR_WRONG;
+    if (ceil2(17) != 32)
+        return ERR_WRONG;
+
+    /* Test the pair() function. */
+    if (pair(4, 0, 0) != 1)
+        return ERR_WRONG;
+    if (pair(4, 2, 2) != 1)
+        return ERR_WRONG;
+
+    return 0;
+}
+
+/* Test the create_mpi_datatypes() function.
+ * @returns 0 for success, error code otherwise.*/
+int test_create_mpi_datatypes()
+{
+    MPI_Datatype basetype = MPI_INT;
+    int *mfrom = NULL;
+    int mpierr;
+    int ret;
+
+    {
+        int msgcnt = 1;
+        PIO_Offset mindex[1] = {0};
+        int mcount[1] = {1};
+        MPI_Datatype mtype;
+
+        /* Create an MPI data type. */
+        if ((ret = create_mpi_datatypes(basetype, msgcnt, mindex, mcount, mfrom, &mtype)))
+            return ret;
+
+        /* Free the type. */
+        if ((mpierr = MPI_Type_free(&mtype)))
+            MPIERR(mpierr);
+    }
+
+    {
+        int msgcnt = 4;
+        PIO_Offset mindex[4] = {0, 0, 0, 0};
+        int mcount[4] = {1, 1, 1, 1};
+        MPI_Datatype mtype2[4];
+
+        /* Create 4 MPI data types. */
+        if ((ret = create_mpi_datatypes(basetype, msgcnt, mindex, mcount, mfrom, mtype2)))
+            return ret;
+
+        /* Check the size of the data types. It should be 4. */
+        MPI_Aint lb, extent;
+        for (int t = 0; t < 4; t++)
+        {
+            if ((mpierr = MPI_Type_get_extent(mtype2[t], &lb, &extent)))
+                MPIERR(mpierr);
+            printf("t = %d lb = %ld extent = %ld\n", t, lb, extent);
+            if (lb != 0 || extent != 4)
+                return ERR_WRONG;
+        }
+
+        /* Free them. */
+        for (int t = 0; t < 4; t++)
+            if ((mpierr = MPI_Type_free(&mtype2[t])))
+                return ERR_WRONG;
+    }
+
+    return 0;
+}
+
+/* Test the idx_to_dim_list() function. */
+int test_idx_to_dim_list()
+{
+    int ndims = 1;
+    int gdims[1] = {1};
+    PIO_Offset idx = 0;
+    PIO_Offset dim_list[1];
+
+    /* This simplest case. */
+    idx_to_dim_list(ndims, gdims, idx, dim_list);
+
+    if (dim_list[0] != 0)
+        return ERR_WRONG;
+
+    /* The case given in the function docs. */
+    int ndims2 = 2;
+    int gdims2[2] = {3, 2};
+    PIO_Offset idx2 = 4;
+    PIO_Offset dim_list2[2];
+
+    /* According to function docs, we should get 2,0 */
+    idx_to_dim_list(ndims2, gdims2, idx2, dim_list2);
+    printf("dim_list2[0] = %lld\n", dim_list2[0]);
+    printf("dim_list2[1] = %lld\n", dim_list2[1]);
+
+    /* This is the correct result! */
+    if (dim_list2[0] != 2 || dim_list2[1] != 0)
+        return ERR_WRONG;
+
+    return 0;
+}
+
+/* Test the coord_to_lindex() function. */
+int test_coord_to_lindex()
+{
+    int ndims = 1;
+    PIO_Offset lcoord[1] = {0};
+    PIO_Offset count[1] = {1};
+    PIO_Offset lindex;
+
+    /* Not sure what this function is really doing. */
+    lindex = coord_to_lindex(ndims, lcoord, count);
+    if (lindex != 0)
+        return ERR_WRONG;
+
+    int ndims2 = 2;
+    PIO_Offset lcoord2[2] = {0, 0};
+    PIO_Offset count2[2] = {1, 1};
+    PIO_Offset lindex2;
+
+    lindex2 = coord_to_lindex(ndims2, lcoord2, count2);
+    if (lindex2 != 0)
+        return ERR_WRONG;
+
+    int ndims3 = 2;
+    PIO_Offset lcoord3[2] = {1, 2};
+    PIO_Offset count3[2] = {1, 1};
+    PIO_Offset lindex3;
+
+    lindex3 = coord_to_lindex(ndims3, lcoord3, count3);
+    printf("lindex = %lld\n", lindex3);
+    if (lindex3 != 3)
+        return ERR_WRONG;
+
+    return 0;
+}
+
+/* Test compute_maxIObuffersize() function. */
+int test_compute_maxIObuffersize(MPI_Comm test_comm, int my_rank)
+{
+    int ret;
+
+    {
+        /* This is a simple test with one region containing 1 data
+         * element. */
+        io_desc_t iodesc;
+        io_region *ior1;
+        int ndims = 1;
+
+        /* This is how we allocate a region. */
+        if ((ret = alloc_region2(NULL, ndims, &ior1)))
+            return ret;
+        ior1->next = NULL;
+        ior1->count[0] = 1;
+
+        iodesc.firstregion = ior1;
+        iodesc.ndims = 1;
+
+        /* Run the function. Simplest possible case. */
+        if ((ret = compute_maxIObuffersize(test_comm, &iodesc)))
+            return ret;
+        if (iodesc.maxiobuflen != 1)
+            return ERR_WRONG;
+
+        /* Free resources for the region. */
+        free(ior1->start);
+        free(ior1->count);
+        free(ior1);
+
+    }
+
+    {
+        /* This also has a single region, but with 2 dims and count
+         * values > 1. */
+        io_desc_t iodesc;
+        io_region *ior2;
+        int ndims = 2;
+
+        /* This is how we allocate a region. */
+        if ((ret = alloc_region2(NULL, ndims, &ior2)))
+            return ret;
+
+        /* These should be 0. */
+        for (int i = 0; i < ndims; i++)
+            if (ior2->start[i] != 0 || ior2->count[i] != 0)
+                return ERR_WRONG;
+
+        ior2->next = NULL;
+        ior2->count[0] = 10;
+        ior2->count[1] = 2;
+
+        iodesc.firstregion = ior2;
+        iodesc.ndims = 2;
+
+        /* Run the function. */
+        if ((ret = compute_maxIObuffersize(test_comm, &iodesc)))
+            return ret;
+        if (iodesc.maxiobuflen != 20)
+            return ERR_WRONG;
+
+        /* Free resources for the region. */
+        free(ior2->start);
+        free(ior2->count);
+        free(ior2);
+    }
+
+    {
+        /* This test has two regions of different sizes. */
+        io_desc_t iodesc;
+        io_region *ior3;
+        io_region *ior4;
+        int ndims = 2;
+
+        /* This is how we allocate a region. */
+        if ((ret = alloc_region2(NULL, ndims, &ior4)))
+            return ret;
+        ior4->next = NULL;
+        ior4->count[0] = 10;
+        ior4->count[1] = 2;
+
+        if ((ret = alloc_region2(NULL, ndims, &ior3)))
+            return ret;
+        ior3->next = ior4;
+        ior3->count[0] = 100;
+        ior3->count[1] = 5;
+
+        iodesc.firstregion = ior3;
+        iodesc.ndims = 2;
+
+        /* Run the function. */
+        if ((ret = compute_maxIObuffersize(test_comm, &iodesc)))
+            return ret;
+        printf("iodesc.maxiobuflen = %d\n", iodesc.maxiobuflen);
+        if (iodesc.maxiobuflen != 520)
+            return ERR_WRONG;
+
+        /* Free resources for the region. */
+        free(ior4->start);
+        free(ior4->count);
+        free(ior4);
+        free(ior3->start);
+        free(ior3->count);
+        free(ior3);
+    }
+
+    return 0;
+}
+
+/* Tests for determine_fill() function. */
+int test_determine_fill(MPI_Comm test_comm)
+{
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    int gsize[1] = {4};
+    PIO_Offset compmap[1] = {1};
+    int ret;
+
+    /* Initialize ios. */
+    if (!(ios = calloc(1, sizeof(iosystem_desc_t))))
+        return PIO_ENOMEM;
+    ios->union_comm = test_comm;
+
+    /* Set up iodesc for test. */
+    if (!(iodesc = calloc(1, sizeof(io_desc_t))))
+        return PIO_ENOMEM;
+    iodesc->ndims = 1;
+    iodesc->rearranger = PIO_REARR_SUBSET;
+    iodesc->llen = 1;
+
+    /* We don't need fill. */
+    if ((ret = determine_fill(ios, iodesc, gsize, compmap)))
+        return ret;
+    if (iodesc->needsfill)
+        return ERR_WRONG;
+
+    /* Change settings, so now we do need fill. */
+    iodesc->llen = 0;
+    if ((ret = determine_fill(ios, iodesc, gsize, compmap)))
+        return ret;
+    if (!iodesc->needsfill)
+        return ERR_WRONG;
+
+    /* Free test resources. */
+    free(ios);
+    free(iodesc);
+
+    return 0;
+}
+
+/* Run tests for get_start_and_count_regions() funciton. */
+int test_get_regions(int my_rank)
+{
+#define MAPLEN 2
+    int ndims = NDIM1;
+    const int gdimlen[NDIM1] = {8};
+    /* Don't forget map is 1-based!! */
+    PIO_Offset map[MAPLEN] = {(my_rank * 2) + 1, ((my_rank + 1) * 2) + 1};
+    int maxregions;
+    io_region *ior1;
+    int ret;
+
+    /* This is how we allocate a region. */
+    if ((ret = alloc_region2(NULL, NDIM1, &ior1)))
+        return ret;
+    ior1->next = NULL;
+    ior1->count[0] = 1;
+
+    /* Call the function we are testing. */
+    if ((ret = get_regions(ndims, gdimlen, MAPLEN, map, &maxregions, ior1)))
+        return ret;
+    if (maxregions != 2)
+        return ERR_WRONG;
+
+    /* Free resources for the region. */
+    free(ior1->next->start);
+    free(ior1->next->count);
+    free(ior1->next);
+    free(ior1->start);
+    free(ior1->count);
+    free(ior1);
+
+    return 0;
+}
+
+/* Run tests for find_region() function. */
+int test_find_region()
+{
+    int ndims = NDIM1;
+    int gdimlen[NDIM1] = {4};
+    int maplen = 1;
+    PIO_Offset map[1] = {1};
+    PIO_Offset start[NDIM1];
+    PIO_Offset count[NDIM1];
+    PIO_Offset regionlen;
+
+    /* Call the function we are testing. */
+    regionlen = find_region(ndims, gdimlen, maplen, map, start, count);
+
+    /* Check results. */
+    printf("regionlen = %lld start[0] = %lld count[0] = %lld\n", regionlen, start[0], count[0]);
+    if (regionlen != 1 || start[0] != 0 || count[0] != 1)
+        return ERR_WRONG;
+
+    return 0;
+}
+
+/* Run tests for expand_region() function. */
+int test_expand_region()
+{
+    int dim = 0;
+    int gdims[NDIM1] = {1};
+    int maplen = 1;
+    PIO_Offset map[1] = {5};
+    int region_size = 1;
+    int region_stride = 1;
+    int max_size[NDIM1] = {10};
+    PIO_Offset count[NDIM1];
+
+    expand_region(dim, gdims, maplen, map, region_size, region_stride, max_size, count);
+    if (count[0] != 1)
+        return ERR_WRONG;
+    printf("max_size[0] = %d count[0] = %lld\n", max_size[0], count[0]);
+
+    return 0;
+}
+
+/* Test define_iodesc_datatypes() function. */
+int test_define_iodesc_datatypes()
+{
+#define NUM_REARRANGERS 2
+    int rearranger[NUM_REARRANGERS] = {PIO_REARR_BOX, PIO_REARR_SUBSET};
+    int mpierr;
+    int ret;
+
+    /* Run the functon. */
+    for (int r = 0; r < NUM_REARRANGERS; r++)
+    {
+        iosystem_desc_t ios;
+        io_desc_t iodesc;
+
+        /* Set up test for IO task with BOX rearranger to create one type. */
+        ios.ioproc = 1; /* this is IO proc. */
+        ios.num_iotasks = 4; /* The number of IO tasks. */
+        iodesc.rtype = NULL; /* Array of MPI types will be created here. */
+        iodesc.nrecvs = 1; /* Number of types created. */
+        iodesc.basetype = MPI_INT;
+        iodesc.stype = NULL; /* Array of MPI types will be created here. */
+
+        /* Allocate space for arrays in iodesc that will be filled in
+         * define_iodesc_datatypes(). */
+        if (!(iodesc.rcount = malloc(iodesc.nrecvs * sizeof(int))))
+            return PIO_ENOMEM;
+        if (!(iodesc.rfrom = malloc(iodesc.nrecvs * sizeof(int))))
+            return PIO_ENOMEM;
+        if (!(iodesc.rindex = malloc(1 * sizeof(PIO_Offset))))
+            return PIO_ENOMEM;
+        iodesc.rindex[0] = 0;
+        iodesc.rcount[0] = 1;
+
+        iodesc.rearranger = rearranger[r];
+
+        /* The two rearrangers create a different number of send types. */
+        int num_send_types = iodesc.rearranger == PIO_REARR_BOX ? ios.num_iotasks : 1;
+
+        if (!(iodesc.sindex = malloc(num_send_types * sizeof(PIO_Offset))))
+            return PIO_ENOMEM;
+        if (!(iodesc.scount = malloc(num_send_types * sizeof(int))))
+            return PIO_ENOMEM;
+        for (int st = 0; st < num_send_types; st++)
+        {
+            iodesc.sindex[st] = 0;
+            iodesc.scount[st] = 1;
+        }
+
+        /* Run the test function. */
+        if ((ret = define_iodesc_datatypes(&ios, &iodesc)))
+            return ret;
+
+        /* We created send types, so free them. */
+        for (int st = 0; st < num_send_types; st++)
+            if ((mpierr = MPI_Type_free(&iodesc.stype[st])))
+                MPIERR(mpierr);
+
+        /* We created one receive type, so free it. */
+        if ((mpierr = MPI_Type_free(&iodesc.rtype[0])))
+            MPIERR(mpierr);
+
+        /* Free resources. */
+        free(iodesc.rtype);
+        free(iodesc.sindex);
+        free(iodesc.scount);
+        free(iodesc.stype);
+        free(iodesc.rcount);
+        free(iodesc.rfrom);
+        free(iodesc.rindex);
+    }
+
+    return 0;
+}
+
+/* Test the compute_counts() function with the box rearranger. */
+int test_compute_counts(MPI_Comm test_comm, int my_rank)
+{
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    int dest_ioproc[TARGET_NTASKS] = {0, 1, 2, 3};
+    PIO_Offset dest_ioindex[TARGET_NTASKS] = {0, 1, 2, 3};
+    int ret;
+
+    /* Initialize ios. */
+    if (!(ios = calloc(1, sizeof(iosystem_desc_t))))
+        return PIO_ENOMEM;
+
+    ios->num_iotasks = TARGET_NTASKS;
+    ios->num_comptasks = TARGET_NTASKS;
+    ios->num_uniontasks = TARGET_NTASKS;
+    ios->ioproc = 1;
+    ios->compproc = 1;
+    ios->union_comm = test_comm;
+    if (!(ios->ioranks = malloc(TARGET_NTASKS * sizeof(int))))
+        return PIO_ENOMEM;
+    for (int t = 0; t < TARGET_NTASKS; t++)
+        ios->ioranks[t] = t;
+    if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < TARGET_NTASKS; i++)
+        ios->compranks[i] = i;
+
+    /* Initialize iodesc. */
+    if (!(iodesc = calloc(1, sizeof(io_desc_t))))
+        return PIO_ENOMEM;
+    iodesc->rearranger = PIO_REARR_BOX;
+    iodesc->ndof = TARGET_NTASKS;
+    iodesc->llen = TARGET_NTASKS;
+    iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL;
+    iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE;
+
+    /* Test the function. */
+    if ((ret = compute_counts(ios, iodesc, dest_ioproc, dest_ioindex)))
+        return ret;
+
+    /* Check results. */
+    for (int i = 0; i < ios->num_iotasks; i++)
+        if (iodesc->scount[i] != 1 || iodesc->sindex[i] != i)
+            return ERR_WRONG;
+
+    for (int i = 0; i < iodesc->ndof; i++)
+        if (iodesc->rcount[i] != 1 || iodesc->rfrom[i] != i ||
+            iodesc->rindex[i] != my_rank)
+            return ERR_WRONG;
+
+    /* Free resources allocated in compute_counts(). */
+    free(iodesc->scount);
+    free(iodesc->sindex);
+    free(iodesc->rcount);
+    free(iodesc->rfrom);
+    free(iodesc->rindex);
+
+    /* Free test resources. */
+    free(ios->ioranks);
+    free(ios->compranks);
+    free(iodesc);
+    free(ios);
+
+    return 0;
+}
+
+/* Call PIOc_InitDecomp() with parameters such that it calls
+ * box_rearrange_create() just like test_box_rearrange_create() will
+ * (see below). */
+int test_init_decomp(int iosysid, MPI_Comm test_comm, int my_rank)
+{
+    int ioid;
+    PIO_Offset compmap[MAPLEN2] = {my_rank * 2, (my_rank + 1) * 2};
+    const int gdimlen[NDIM1] = {8};
+    int ret;
+
+    /* Initialize a decomposition. */
+    if ((ret = PIOc_init_decomp(iosysid, PIO_INT, NDIM1, gdimlen, MAPLEN2,
+                                compmap, &ioid, PIO_REARR_BOX, NULL, NULL)))
+        return ret;
+
+    /* Free it. */
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        return ret;
+
+    return 0;
+}
+
+/* Test for the box_rearrange_create() function. */
+int test_box_rearrange_create(MPI_Comm test_comm, int my_rank)
+{
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    io_region *ior1;
+    int maplen = MAPLEN2;
+    PIO_Offset compmap[MAPLEN2] = {(my_rank * 2) + 1, ((my_rank + 1) * 2) + 1};
+    const int gdimlen[NDIM1] = {8};
+    int ndims = NDIM1;
+    int ret;
+
+    /* Allocate IO system info struct for this test. */
+    if (!(ios = calloc(1, sizeof(iosystem_desc_t))))
+        return PIO_ENOMEM;
+
+    /* Allocate IO desc struct for this test. */
+    if (!(iodesc = calloc(1, sizeof(io_desc_t))))
+        return PIO_ENOMEM;
+
+    /* Default rearranger options. */
+    iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL;
+    iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE;
+
+    /* Set up for determine_fill(). */
+    ios->union_comm = test_comm;
+    ios->io_comm = test_comm;
+    iodesc->ndims = NDIM1;
+    iodesc->rearranger = PIO_REARR_BOX;
+
+    /* Set up the IO task info for the test. */
+    ios->ioproc = 1;
+    ios->compproc = 1;
+    ios->union_rank = my_rank;
+    ios->num_iotasks = 4;
+    ios->num_comptasks = 4;
+    ios->num_uniontasks = 4;
+    if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < TARGET_NTASKS; i++)
+        ios->ioranks[i] = i;
+    if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < TARGET_NTASKS; i++)
+        ios->compranks[i] = i;
+
+    /* This is how we allocate a region. */
+    if ((ret = alloc_region2(NULL, NDIM1, &ior1)))
+        return ret;
+    if (my_rank == 0)
+        ior1->count[0] = 8;
+
+    iodesc->firstregion = ior1;
+
+    /* We are finally ready to run the code under test. */
+    if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc)))
+        return ret;
+
+    /* Check some results. */
+    if (iodesc->rearranger != PIO_REARR_BOX || iodesc->ndof != maplen ||
+        iodesc->llen != my_rank ? 0 : 8 || !iodesc->needsfill)
+        return ERR_WRONG;
+
+    /* for (int i = 0; i < ios->num_iotasks; i++) */
+    /* { */
+    /*     /\* sindex is only allocated if scount[i] > 0. *\/ */
+    /*     if (iodesc->scount[i] != i ? 0 : 1 || */
+    /*         (iodesc->scount[i] && iodesc->sindex[i] != 0)) */
+    /*         return ERR_WRONG; */
+    /* } */
+
+    /* for (int i = 0; i < iodesc->ndof; i++) */
+    /* { */
+    /*     /\* rcount is 1 for rank 0, 0 on other tasks. *\/ */
+    /*     if (iodesc->rcount[i] != my_rank ? 0 : 1) */
+    /*         return ERR_WRONG; */
+
+    /*     /\* rfrom is 0 everywhere, except task 0, array elemnt 1. *\/ */
+    /*     if (my_rank == 0 && i == 1) */
+    /*     { */
+    /*         if (iodesc->rfrom[i] != 1) */
+    /*             return ERR_WRONG; */
+    /*     } */
+    /*     else */
+    /*     { */
+    /*         if (iodesc->rfrom[i] != 0) */
+    /*             return ERR_WRONG; */
+    /*     } */
+
+    /*     /\* rindex is only allocated where there is a non-zero count. *\/ */
+    /*     if (iodesc->rcount[i]) */
+    /*         if (iodesc->rindex[i] != 0) */
+    /*             return ERR_WRONG; */
+    /* } */
+
+    /* Free resources allocated in compute_counts(). */
+    free(iodesc->scount);
+    free(iodesc->sindex);
+    free(iodesc->rcount);
+    free(iodesc->rfrom);
+    free(iodesc->rindex);
+
+    /* Free resources from test. */
+    free(ior1->start);
+    free(ior1->count);
+    free(ior1);
+    free(ios->ioranks);
+    free(ios->compranks);
+    free(iodesc);
+    free(ios);
+
+    return 0;
+}
+
+/* Test for the box_rearrange_create() function. */
+int test_box_rearrange_create_2(MPI_Comm test_comm, int my_rank)
+{
+#define MAPLEN2 2
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    io_region *ior1;
+    int maplen = MAPLEN2;
+    PIO_Offset compmap[MAPLEN2] = {1, 0};
+    const int gdimlen[NDIM1] = {8};
+    int ndims = NDIM1;
+    int ret;
+
+    /* Allocate IO system info struct for this test. */
+    if (!(ios = calloc(1, sizeof(iosystem_desc_t))))
+        return PIO_ENOMEM;
+
+    /* Allocate IO desc struct for this test. */
+    if (!(iodesc = calloc(1, sizeof(io_desc_t))))
+        return PIO_ENOMEM;
+
+    /* Default rearranger options. */
+    iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL;
+    iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE;
+
+    /* Set up for determine_fill(). */
+    ios->union_comm = test_comm;
+    ios->io_comm = test_comm;
+    iodesc->ndims = NDIM1;
+    iodesc->rearranger = PIO_REARR_BOX;
+
+    /* This is the size of the map in computation tasks. */
+    iodesc->ndof = 2;
+
+    /* Set up the IO task info for the test. */
+    ios->ioproc = 1;
+    ios->compproc = 1;
+    ios->union_rank = my_rank;
+    ios->num_iotasks = 4;
+    ios->num_comptasks = 4;
+    ios->num_uniontasks = 4;
+    if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < TARGET_NTASKS; i++)
+        ios->ioranks[i] = i;
+    if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < TARGET_NTASKS; i++)
+        ios->compranks[i] = i;
+
+    /* This is how we allocate a region. */
+    if ((ret = alloc_region2(NULL, NDIM1, &ior1)))
+        return ret;
+    ior1->next = NULL;
+    if (my_rank == 0)
+        ior1->count[0] = 8;
+
+    iodesc->firstregion = ior1;
+
+    /* We are finally ready to run the code under test. */
+    if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc)))
+        return ret;
+
+    /* Check some results. */
+    if (iodesc->rearranger != PIO_REARR_BOX || iodesc->ndof != maplen ||
+        iodesc->llen != my_rank ? 0 : 8 || !iodesc->needsfill)
+        return ERR_WRONG;
+
+    for (int i = 0; i < ios->num_iotasks; i++)
+    {
+        /* sindex is only allocated if scount[i] > 0. */
+        if (iodesc->scount[i] != i ? 0 : 1 ||
+            (iodesc->scount[i] && iodesc->sindex[i] != 0))
+            return ERR_WRONG;
+    }
+
+    if (my_rank == 0)
+    {
+        for (int i = 0; i < iodesc->ndof; i++)
+        {
+            /* rcount is 1 for rank 0, 0 on other tasks. */
+            if (iodesc->rcount[i] != 1)
+                return ERR_WRONG;
+
+            /* rfrom only matters if there is a non-zero count. */
+            if (iodesc->rcount[i])
+                if (iodesc->rfrom[i] != i ? 1 : 0)
+                    return ERR_WRONG;
+
+            /* rindex is only allocated where there is a non-zero count. */
+            if (iodesc->rcount[i])
+                if (iodesc->rindex[i] != 0)
+                    return ERR_WRONG;
+        }
+    }
+
+    /* Free resources allocated in compute_counts(). */
+    free(iodesc->scount);
+    free(iodesc->sindex);
+    free(iodesc->rcount);
+    free(iodesc->rfrom);
+    free(iodesc->rindex);
+
+    /* Free resources from test. */
+    free(ior1->start);
+    free(ior1->count);
+    free(ior1);
+    free(ios->ioranks);
+    free(ios->compranks);
+    free(iodesc);
+    free(ios);
+
+    return 0;
+}
+
+/* Test function default_subset_partition. */
+int test_default_subset_partition(MPI_Comm test_comm, int my_rank)
+{
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    int mpierr;
+    int ret;
+
+    /* Allocate IO system info struct for this test. */
+    if (!(ios = calloc(1, sizeof(iosystem_desc_t))))
+        return PIO_ENOMEM;
+
+    /* Allocate IO desc struct for this test. */
+    if (!(iodesc = calloc(1, sizeof(io_desc_t))))
+        return PIO_ENOMEM;
+
+    ios->ioproc = 1;
+    ios->io_rank = my_rank;
+    ios->comp_comm = test_comm;
+
+    /* Run the function to test. */
+    if ((ret = default_subset_partition(ios, iodesc)))
+        return ret;
+
+    /* Free the created communicator. */
+    if ((mpierr = MPI_Comm_free(&iodesc->subset_comm)))
+        MPIERR(mpierr);
+
+    /* Free resources from test. */
+    free(iodesc);
+    free(ios);
+
+    return 0;
+}
+
+/* Test function rearrange_comp2io. */
+int test_rearrange_comp2io(MPI_Comm test_comm, int my_rank)
+{
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    void *sbuf = NULL;
+    void *rbuf = NULL;
+    int nvars = 1;
+    io_region *ior1;
+    int maplen = 2;
+    PIO_Offset compmap[2] = {1, 0};
+    const int gdimlen[NDIM1] = {8};
+    int ndims = NDIM1;
+    int mpierr;
+    int ret;
+
+    /* Allocate some space for data. */
+    if (!(sbuf = calloc(4, sizeof(int))))
+        return PIO_ENOMEM;
+    if (!(rbuf = calloc(4, sizeof(int))))
+        return PIO_ENOMEM;
+
+    /* Allocate IO system info struct for this test. */
+    if (!(ios = calloc(1, sizeof(iosystem_desc_t))))
+        return PIO_ENOMEM;
+
+    /* Allocate IO desc struct for this test. */
+    if (!(iodesc = calloc(1, sizeof(io_desc_t))))
+        return PIO_ENOMEM;
+
+    ios->ioproc = 1;
+    ios->compproc = 1;
+    ios->io_rank = my_rank;
+    ios->union_comm = test_comm;
+    ios->num_iotasks = TARGET_NTASKS;
+    ios->num_uniontasks = TARGET_NTASKS;
+    iodesc->rearranger = PIO_REARR_BOX;
+    iodesc->basetype = MPI_INT;
+
+    /* Set up test for IO task with BOX rearranger to create one type. */
+    iodesc->rtype = NULL; /* Array of MPI types will be created here. */
+    iodesc->nrecvs = 1; /* Number of types created. */
+    iodesc->basetype = MPI_INT;
+    iodesc->stype = NULL; /* Array of MPI types will be created here. */
+
+    /* The two rearrangers create a different number of send types. */
+    int num_send_types = iodesc->rearranger == PIO_REARR_BOX ? ios->num_iotasks : 1;
+
+    /* Default rearranger options. */
+    iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL;
+    iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE;
+
+    /* Set up for determine_fill(). */
+    ios->union_comm = test_comm;
+    ios->io_comm = test_comm;
+    iodesc->ndims = NDIM1;
+    iodesc->rearranger = PIO_REARR_BOX;
+
+    iodesc->ndof = 4;
+
+    /* Set up the IO task info for the test. */
+    ios->union_rank = my_rank;
+    ios->num_comptasks = 4;
+    if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < TARGET_NTASKS; i++)
+        ios->ioranks[i] = i;
+    if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < TARGET_NTASKS; i++)
+        ios->compranks[i] = i;
+
+    /* This is how we allocate a region. */
+    if ((ret = alloc_region2(NULL, NDIM1, &ior1)))
+        return ret;
+    ior1->next = NULL;
+    if (my_rank == 0)
+        ior1->count[0] = 8;
+
+    iodesc->firstregion = ior1;
+
+    /* Create the box rearranger. */
+    if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc)))
+        return ret;
+
+    /* Run the function to test. */
+    if ((ret = rearrange_comp2io(ios, iodesc, sbuf, rbuf, nvars)))
+        return ret;
+    printf("returned from rearrange_comp2io\n");
+
+    /* We created send types, so free them. */
+    for (int st = 0; st < num_send_types; st++)
+        if (iodesc->stype[st] != PIO_DATATYPE_NULL)
+            if ((mpierr = MPI_Type_free(&iodesc->stype[st])))
+                MPIERR(mpierr);
+
+    /* We created one receive type, so free it. */
+    if (iodesc->rtype)
+        for (int r = 0; r < iodesc->nrecvs; r++)
+            if (iodesc->rtype[r] != PIO_DATATYPE_NULL)
+                if ((mpierr = MPI_Type_free(&iodesc->rtype[r])))
+                    MPIERR(mpierr);
+
+    /* Free resources allocated in library code. */
+    free(iodesc->rtype);
+    free(iodesc->sindex);
+    free(iodesc->scount);
+    free(iodesc->stype);
+    free(iodesc->rcount);
+    free(iodesc->rfrom);
+    free(iodesc->rindex);
+
+    /* Free resources from test. */
+    free(ior1->start);
+    free(ior1->count);
+    free(ior1);
+    free(ios->ioranks);
+    free(ios->compranks);
+    free(iodesc);
+    free(ios);
+    free(sbuf);
+    free(rbuf);
+
+    return 0;
+}
+
+/* Test function rearrange_io2comp. */
+int test_rearrange_io2comp(MPI_Comm test_comm, int my_rank)
+{
+    iosystem_desc_t *ios;
+    io_desc_t *iodesc;
+    void *sbuf = NULL;
+    void *rbuf = NULL;
+    io_region *ior1;
+    int maplen = 2;
+    PIO_Offset compmap[2] = {1, 0};
+    const int gdimlen[NDIM1] = {8};
+    int ndims = NDIM1;
+    int mpierr;
+    int ret;
+
+    /* Allocate some space for data. */
+    if (!(sbuf = calloc(4, sizeof(int))))
+        return PIO_ENOMEM;
+    if (!(rbuf = calloc(4, sizeof(int))))
+        return PIO_ENOMEM;
+
+    /* Allocate IO system info struct for this test. */
+    if (!(ios = calloc(1, sizeof(iosystem_desc_t))))
+        return PIO_ENOMEM;
+
+    /* Allocate IO desc struct for this test. */
+    if (!(iodesc = calloc(1, sizeof(io_desc_t))))
+        return PIO_ENOMEM;
+
+    ios->ioproc = 1;
+    ios->io_rank = my_rank;
+    ios->union_comm = test_comm;
+    ios->num_iotasks = TARGET_NTASKS;
+    iodesc->rearranger = PIO_REARR_BOX;
+    iodesc->basetype = MPI_INT;
+
+    /* Set up test for IO task with BOX rearranger to create one type. */
+    ios->ioproc = 1; /* this is IO proc. */
+    ios->num_iotasks = 4; /* The number of IO tasks. */
+    iodesc->rtype = NULL; /* Array of MPI types will be created here. */
+    iodesc->nrecvs = 1; /* Number of types created. */
+    iodesc->basetype = MPI_INT;
+    iodesc->stype = NULL; /* Array of MPI types will be created here. */
+
+    /* The two rearrangers create a different number of send types. */
+    int num_send_types = iodesc->rearranger == PIO_REARR_BOX ? ios->num_iotasks : 1;
+
+    /* Default rearranger options. */
+    iodesc->rearr_opts.comm_type = PIO_REARR_COMM_COLL;
+    iodesc->rearr_opts.fcd = PIO_REARR_COMM_FC_2D_DISABLE;
+
+    /* Set up for determine_fill(). */
+    ios->union_comm = test_comm;
+    ios->io_comm = test_comm;
+    iodesc->ndims = NDIM1;
+    iodesc->rearranger = PIO_REARR_BOX;
+
+    iodesc->ndof = 4;
+
+    /* Set up the IO task info for the test. */
+    ios->ioproc = 1;
+    ios->compproc = 1;
+    ios->union_rank = my_rank;
+    ios->num_iotasks = 4;
+    ios->num_comptasks = 4;
+    ios->num_uniontasks = 4;
+    if (!(ios->ioranks = calloc(ios->num_iotasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < TARGET_NTASKS; i++)
+        ios->ioranks[i] = i;
+    if (!(ios->compranks = calloc(ios->num_comptasks, sizeof(int))))
+        return pio_err(ios, NULL, PIO_ENOMEM, __FILE__, __LINE__);
+    for (int i = 0; i < TARGET_NTASKS; i++)
+        ios->compranks[i] = i;
+
+    /* This is how we allocate a region. */
+    if ((ret = alloc_region2(NULL, NDIM1, &ior1)))
+        return ret;
+    ior1->next = NULL;
+    if (my_rank == 0)
+        ior1->count[0] = 8;
+
+    iodesc->firstregion = ior1;
+
+    /* Create the box rearranger. */
+    if ((ret = box_rearrange_create(ios, maplen, compmap, gdimlen, ndims, iodesc)))
+        return ret;
+
+    /* Run the function to test. */
+    if ((ret = rearrange_io2comp(ios, iodesc, sbuf, rbuf)))
+        return ret;
+    printf("returned from rearrange_comp2io\n");
+
+    /* We created send types, so free them. */
+    for (int st = 0; st < num_send_types; st++)
+        if (iodesc->stype[st] != PIO_DATATYPE_NULL)
+            if ((mpierr = MPI_Type_free(&iodesc->stype[st])))
+                MPIERR(mpierr);
+
+    /* We created one receive type, so free it. */
+    if (iodesc->rtype)
+        for (int r = 0; r < iodesc->nrecvs; r++)
+            if (iodesc->rtype[r] != PIO_DATATYPE_NULL)
+                if ((mpierr = MPI_Type_free(&iodesc->rtype[r])))
+                    MPIERR(mpierr);
+
+    /* Free resources allocated in library code. */
+    free(iodesc->rtype);
+    free(iodesc->sindex);
+    free(iodesc->scount);
+    free(iodesc->stype);
+    free(iodesc->rcount);
+    free(iodesc->rfrom);
+    free(iodesc->rindex);
+
+    /* Free resources from test. */
+    free(ior1->start);
+    free(ior1->count);
+    free(ior1);
+    free(ios->ioranks);
+    free(ios->compranks);
+    free(iodesc);
+    free(ios);
+    free(sbuf);
+    free(rbuf);
+
+    return 0;
+}
+
+/* These tests do not need an iosysid. */
+int run_no_iosys_tests(int my_rank, MPI_Comm test_comm)
+{
+    int ret;
+
+    printf("%d running idx_to_dim_list tests\n", my_rank);
+    if ((ret = test_idx_to_dim_list()))
+        return ret;
+
+    printf("%d running coord_to_lindex tests\n", my_rank);
+    if ((ret = test_coord_to_lindex()))
+        return ret;
+
+    printf("%d running compute_maxIObuffersize tests\n", my_rank);
+    if ((ret = test_compute_maxIObuffersize(test_comm, my_rank)))
+        return ret;
+
+    printf("%d running determine_fill\n", my_rank);
+    if ((ret = test_determine_fill(test_comm)))
+        return ret;
+
+    printf("%d running tests for expand_region()\n", my_rank);
+    if ((ret = test_expand_region()))
+        return ret;
+
+    printf("%d running tests for find_region()\n", my_rank);
+    if ((ret = test_find_region()))
+        return ret;
+
+    printf("%d running tests for get_regions()\n", my_rank);
+    if ((ret = test_get_regions(my_rank)))
+        return ret;
+
+    printf("%d running create_mpi_datatypes tests\n", my_rank);
+    if ((ret = test_create_mpi_datatypes()))
+        return ret;
+
+    printf("%d running define_iodesc_datatypes tests\n", my_rank);
+    if ((ret = test_define_iodesc_datatypes()))
+        return ret;
+
+    printf("%d running compare_offsets tests\n", my_rank);
+    if ((ret = test_compare_offsets()))
+        return ret;
+
+    printf("%d running compute_counts tests for box rearranger\n", my_rank);
+    if ((ret = test_compute_counts(test_comm, my_rank)))
+        return ret;
+
+    printf("%d running tests for box_rearrange_create\n", my_rank);
+    if ((ret = test_box_rearrange_create(test_comm, my_rank)))
+        return ret;
+
+    printf("%d running more tests for box_rearrange_create\n", my_rank);
+    if ((ret = test_box_rearrange_create_2(test_comm, my_rank)))
+        return ret;
+
+    printf("%d running tests for default_subset_partition\n", my_rank);
+    if ((ret = test_default_subset_partition(test_comm, my_rank)))
+        return ret;
+
+    printf("%d running tests for rearrange_comp2io\n", my_rank);
+    if ((ret = test_rearrange_comp2io(test_comm, my_rank)))
+        return ret;
+
+    printf("%d running tests for rearrange_io2comp\n", my_rank);
+    if ((ret = test_rearrange_io2comp(test_comm, my_rank)))
+        return ret;
+
+     return 0;
+}
+
+/* Test scalar vars. */
+int test_scalar(int numio, int iosysid, MPI_Comm test_comm, int my_rank,
+                int num_flavors, int *flavor)
+{
+
+    int var_type[NUM_NETCDF4_TYPES - 1] = {PIO_BYTE, PIO_CHAR, PIO_SHORT, PIO_INT, PIO_FLOAT,
+                                           PIO_DOUBLE, PIO_UBYTE, PIO_USHORT, PIO_UINT, PIO_INT64,
+                                           PIO_UINT64};
+    char char_data = 2;
+    signed char byte_data = -42;
+    short short_data = -300;
+    int int_data = -10000;
+    float float_data = -42.42;
+    double double_data = -420000000000.5;
+    unsigned char ubyte_data = 43;
+    unsigned short ushort_data = 666;
+    unsigned int uint_data = 666666;
+    long long int64_data = -99999999999;
+    unsigned long long uint64_data = 99999999999;
+    char char_data_in;
+    signed char byte_data_in;
+    short short_data_in;
+    unsigned char ubyte_data_in;
+    int int_data_in;
+    float float_data_in;
+    double double_data_in;
+    unsigned short ushort_data_in;
+    unsigned int uint_data_in;
+    long long int64_data_in;
+    unsigned long long uint64_data_in;
+
+    int ret;
+
+    /* Run tests with all available iotypes. */
+    for (int fmt = 0; fmt < num_flavors; fmt++)
+    {
+        /* For netcdf-4, there are extra types. */
+        int num_types = (flavor[fmt] == PIO_IOTYPE_NETCDF4C || flavor[fmt] == PIO_IOTYPE_NETCDF4P) ?
+            NUM_NETCDF4_TYPES - 1 : NUM_CLASSIC_TYPES;
+
+        /* For each available type, create a file with a scalar var of
+         * that type. */
+        for (int t = 0; t < num_types; t++)
+        {
+            int ncid;
+            int varid;
+            char filename[PIO_MAX_NAME + 1];
+
+            printf("test with t = %d\n", t);
+
+            /* These iotypes only handle netCDF classic types. */
+            if (t >= NUM_CLASSIC_TYPES &&
+                (flavor[fmt] == PIO_IOTYPE_PNETCDF || flavor[fmt] == PIO_IOTYPE_NETCDF))
+                continue;
+
+            /* Create filename. */
+            sprintf(filename, "%s_scalar_numio_%d_iotype_%d_var_type_%d.nc", TEST_NAME,
+                    numio, flavor[fmt], var_type[t]);
+
+            /* Create the file. */
+            if ((ret = PIOc_createfile(iosysid, &ncid, &flavor[fmt], filename, NC_CLOBBER)))
+                return ret;
+
+            /* Define scalar var. */
+            if ((ret = PIOc_def_var(ncid, VAR_NAME, var_type[t], 0, NULL, &varid)))
+                return ret;
+
+            /* End define mode. */
+            if ((ret = PIOc_enddef(ncid)))
+                return ret;
+
+            /* Write a value. */
+            switch (var_type[t])
+            {
+            case PIO_BYTE:
+                if ((ret = PIOc_put_var_schar(ncid, varid, &byte_data)))
+                    return ret;
+                break;
+            case PIO_CHAR:
+                if ((ret = PIOc_put_var_text(ncid, varid, &char_data)))
+                    return ret;
+                break;
+            case PIO_SHORT:
+                if ((ret = PIOc_put_var_short(ncid, varid, &short_data)))
+                    return ret;
+                break;
+            case PIO_INT:
+                if ((ret = PIOc_put_var_int(ncid, varid, &int_data)))
+                    return ret;
+                break;
+            case PIO_FLOAT:
+                if ((ret = PIOc_put_var_float(ncid, varid, &float_data)))
+                    return ret;
+                break;
+            case PIO_DOUBLE:
+                if ((ret = PIOc_put_var_double(ncid, varid, &double_data)))
+                    return ret;
+                break;
+#ifdef _NETCDF4                
+            case PIO_UBYTE:
+                if ((ret = PIOc_put_var_uchar(ncid, varid, &ubyte_data)))
+                    return ret;
+                break;
+            case PIO_USHORT:
+                if ((ret = PIOc_put_var_ushort(ncid, varid, &ushort_data)))
+                    return ret;
+                break;
+            case PIO_UINT:
+                if ((ret = PIOc_put_var_uint(ncid, varid, &uint_data)))
+                    return ret;
+                break;
+            case PIO_INT64:
+                if ((ret = PIOc_put_var_longlong(ncid, varid, &int64_data)))
+                    return ret;
+                break;
+            case PIO_UINT64:
+                if ((ret = PIOc_put_var_ulonglong(ncid, varid, &uint64_data)))
+                    return ret;
+                break;
+#endif /* _NETCDF4 */                
+            default:
+                return ERR_WRONG;
+            }
+
+            /* Close the file. */
+            if ((ret = PIOc_closefile(ncid)))
+                return ret;
+
+            /* Reopen the file. */
+            if ((ret = PIOc_openfile(iosysid, &ncid, &flavor[fmt], filename, NC_NOWRITE)))
+                return ret;
+
+            /* Read the value. Is the value correct? */
+            switch (var_type[t])
+            {
+            case PIO_BYTE:
+                if ((ret = PIOc_get_var_schar(ncid, varid, &byte_data_in)))
+                    return ret;
+                if (byte_data_in != byte_data)
+                    return ERR_WRONG;
+                break;
+            case PIO_CHAR:
+                if ((ret = PIOc_get_var_text(ncid, varid, &char_data_in)))
+                    return ret;
+                if (char_data_in != char_data)
+                    return ERR_WRONG;
+                break;
+            case PIO_SHORT:
+                if ((ret = PIOc_get_var_short(ncid, varid, &short_data_in)))
+                    return ret;
+                if (short_data_in != short_data)
+                    return ERR_WRONG;
+                break;
+            case PIO_INT:
+                if ((ret = PIOc_get_var_int(ncid, varid, &int_data_in)))
+                    return ret;
+                if (int_data_in != int_data)
+                    return ERR_WRONG;
+                break;
+            case PIO_FLOAT:
+                if ((ret = PIOc_get_var_float(ncid, varid, &float_data_in)))
+                    return ret;
+                if (float_data_in != float_data)
+                    return ERR_WRONG;
+                break;
+            case PIO_DOUBLE:
+                if ((ret = PIOc_get_var_double(ncid, varid, &double_data_in)))
+                    return ret;
+                if (double_data_in != double_data)
+                    return ERR_WRONG;
+                break;
+#ifdef _NETCDF4                
+            case PIO_UBYTE:
+                if ((ret = PIOc_get_var_uchar(ncid, varid, &ubyte_data_in)))
+                    return ret;
+                if (ubyte_data_in != ubyte_data)
+                    return ERR_WRONG;
+                break;
+            case PIO_USHORT:
+                if ((ret = PIOc_get_var_ushort(ncid, varid, &ushort_data_in)))
+                    return ret;
+                if (ushort_data_in != ushort_data)
+                    return ERR_WRONG;
+                break;
+            case PIO_UINT:
+                if ((ret = PIOc_get_var_uint(ncid, varid, &uint_data_in)))
+                    return ret;
+                if (uint_data_in != uint_data)
+                    return ERR_WRONG;
+                break;
+            case PIO_INT64:
+                if ((ret = PIOc_get_var_longlong(ncid, varid, &int64_data_in)))
+                    return ret;
+                if (int64_data_in != int64_data)
+                    return ERR_WRONG;
+                break;
+            case PIO_UINT64:
+                if ((ret = PIOc_get_var_ulonglong(ncid, varid, &uint64_data_in)))
+                    return ret;
+                if (uint64_data_in != uint64_data)
+                    return ERR_WRONG;
+                break;
+#endif /* _NETCDF4 */                
+            default:
+                return ERR_WRONG;
+            }
+
+            /* Close the file. */
+            if ((ret = PIOc_closefile(ncid)))
+                return ret;
+
+        } /* next iotype */
+    } /* next type */
+
+    return 0;
+}
+
+/* These tests are run with different rearrangers and numbers of IO
+ * tasks. */
+int run_iosys_tests(int numio, int iosysid, int my_rank, MPI_Comm test_comm,
+                    int num_flavors, int *flavor)
+{
+    int ret;
+
+    printf("%d running rearranger opts tests 1\n", my_rank);
+    if ((ret = test_rearranger_opts1(iosysid)))
+        return ret;
+
+    printf("%d running test for init_decomp\n", my_rank);
+    if ((ret = test_init_decomp(iosysid, test_comm, my_rank)))
+        return ret;
+
+    printf("%d running test for init_decomp\n", my_rank);
+    if ((ret = test_scalar(numio, iosysid, test_comm, my_rank, num_flavors, flavor)))
+        return ret;
+
+    return 0;
+}
+
+/* Run Tests for pio_spmd.c functions. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks;  /* Number of processors involved in current execution. */
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;     /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              TARGET_NTASKS, 3, &test_comm)))
+        ERR(ERR_INIT);
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Figure out iotypes. */
+    if ((ret = get_iotypes(&num_flavors, flavor)))
+        ERR(ret);
+    printf("Runnings tests for %d flavors\n", num_flavors);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* Run the tests that don't need an iosysid. */
+        if ((ret = run_no_iosys_tests(my_rank, test_comm)))
+            return ret;
+
+        /* Test code with both rearrangers. */
+        for (int r = 0; r < NUM_REARRANGERS; r++)
+        {
+            /* Test code with 1, 2, 3, and 4 io tasks. */
+            for (int numio = 1; numio <= TARGET_NTASKS; numio++)
+            {
+                int iosysid;
+                int rearranger = r ? PIO_REARR_SUBSET : PIO_REARR_BOX;
+
+                if ((ret = PIOc_Init_Intracomm(test_comm, numio, 1, 0, rearranger,
+                                               &iosysid)))
+                    return ret;
+
+                /* Run the tests that need an iosysid. */
+                if ((ret = run_iosys_tests(numio, iosysid, my_rank, test_comm,
+                                           num_flavors, flavor)))
+                    return ret;
+
+                /* Finalize PIO system. */
+                if ((ret = PIOc_finalize(iosysid)))
+                    return ret;
+            } /* next numio */
+        } /* next rearranger */
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_shared.c b/src/externals/pio2/tests/cunit/test_shared.c
new file mode 100644
index 00000000000..0926e0be9fd
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_shared.c
@@ -0,0 +1,198 @@
+/*
+ * Common test code for PIO C tests.
+ *
+ * Ed Hartnett
+ */
+#include 
+#include 
+#include 
+
+/* Test with async.
+ *
+ * @param my_rank rank of the task.
+ * @param nprocs the size of the communicator.
+ * @param num_flavors the number of PIO IO types that will be tested.
+ * @param flavors array of the PIO IO types that will be tested.
+ * @param test_comm communicator with all test tasks.
+ * @returns 0 for success error code otherwise.
+ */
+int test_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm,
+                int component_count, int num_io_procs, int target_ntasks, char *test_name)
+{
+    int iosysid[component_count];  /* The ID for the parallel I/O system. */
+    int num_procs[component_count]; /* Num procs in each component. */
+    MPI_Comm io_comm;              /* Will get a duplicate of IO communicator. */
+    MPI_Comm comp_comm[component_count]; /* Will get duplicates of computation communicators. */
+    int mpierr;  /* Return code from MPI functions. */
+    int ret;     /* Return code. */
+
+    num_procs[0] = target_ntasks - 1;
+
+    /* Is the current process a computation task? */
+    int comp_task = my_rank < num_io_procs ? 0 : 1;
+    printf("%d comp_task = %d\n", my_rank, comp_task);
+
+    /* Initialize the IO system. */
+    if ((ret = PIOc_init_async(test_comm, num_io_procs, NULL, component_count,
+                               num_procs, NULL, &io_comm, comp_comm, PIO_REARR_BOX, iosysid)))
+        ERR(ERR_INIT);
+    for (int c = 0; c < component_count; c++)
+        printf("%d iosysid[%d] = %d\n", my_rank, c, iosysid[c]);
+
+    /* All the netCDF calls are only executed on the computation
+     * tasks. The IO tasks have not returned from PIOc_Init_Intercomm,
+     * and when the do, they should go straight to finalize. */
+    if (comp_task)
+    {
+        for (int c = 0; c < component_count; c++)
+        {
+            printf("%d Running tests...\n", my_rank);
+            if ((ret = test_all(iosysid[c], num_flavors, flavor, my_rank, comp_comm[0], 1)))
+                return ret;
+
+            /* Finalize the IO system. Only call this from the computation tasks. */
+            printf("%d %s Freeing PIO resources\n", my_rank, test_name);
+            if ((ret = PIOc_finalize(iosysid[c])))
+                ERR(ret);
+            printf("%d %s PIOc_finalize completed for iosysid = %d\n", my_rank, test_name,
+                   iosysid[c]);
+            if ((mpierr = MPI_Comm_free(&comp_comm[c])))
+                MPIERR(mpierr);
+        }
+    }
+    else
+    {
+        if ((mpierr = MPI_Comm_free(&io_comm)))
+            MPIERR(mpierr);
+    } /* endif comp_task */
+
+    return PIO_NOERR;
+}
+
+/* Test without async.
+ *
+ * @param my_rank rank of the task.
+ * @param num_flavors the number of PIO IO types that will be tested.
+ * @param flavors array of the PIO IO types that will be tested.
+ * @param test_comm communicator with all test tasks.
+ * @returns 0 for success error code otherwise.
+ */
+int test_no_async2(int my_rank, int num_flavors, int *flavor, MPI_Comm test_comm, int target_ntasks,
+                   int x_dim_len, int y_dim_len)
+{
+    int niotasks;    /* Number of processors that will do IO. */
+    int ioproc_stride = 1;    /* Stride in the mpi rank between io tasks. */
+    int ioproc_start = 0;     /* Zero based rank of first processor to be used for I/O. */
+    PIO_Offset elements_per_pe; /* Array index per processing unit. */
+    int iosysid;  /* The ID for the parallel I/O system. */
+    int ioid;     /* The I/O description ID. */
+    PIO_Offset *compdof; /* The decomposition mapping. */
+    int slice_dimlen[2];
+    int ret;      /* Return code. */
+
+    slice_dimlen[0] = x_dim_len;
+    slice_dimlen[1] = y_dim_len;
+
+    /* keep things simple - 1 iotask per MPI process */
+    niotasks = target_ntasks;
+
+    /* Initialize the PIO IO system. This specifies how
+     * many and which processors are involved in I/O. */
+    if ((ret = PIOc_Init_Intracomm(test_comm, niotasks, ioproc_stride,
+                                   ioproc_start, PIO_REARR_SUBSET, &iosysid)))
+        return ret;
+
+    /* Describe the decomposition. This is a 0-based array, so don't add 1! */
+    elements_per_pe = x_dim_len * y_dim_len / target_ntasks;
+    if (!(compdof = malloc(elements_per_pe * sizeof(PIO_Offset))))
+        return PIO_ENOMEM;
+    for (int i = 0; i < elements_per_pe; i++)
+        compdof[i] = my_rank * elements_per_pe + i;
+
+    /* Create the PIO decomposition for this test. */
+    printf("%d Creating decomposition...\n", my_rank);
+    if ((ret = PIOc_init_decomp(iosysid, PIO_FLOAT, 2, slice_dimlen, (PIO_Offset)elements_per_pe,
+                                compdof, &ioid, 0, NULL, NULL)))
+        return ret;
+    free(compdof);
+
+    /* Run tests. */
+    printf("%d Running tests...\n", my_rank);
+    if ((ret = test_all(iosysid, num_flavors, flavor, my_rank, test_comm, 0)))
+        return ret;
+
+    /* Free the PIO decomposition. */
+    printf("%d Freeing PIO decomposition...\n", my_rank);
+    if ((ret = PIOc_freedecomp(iosysid, ioid)))
+        return ret;
+
+    /* Finalize PIO system. */
+    if ((ret = PIOc_finalize(iosysid)))
+        return ret;
+
+    return PIO_NOERR;
+}
+
+/*
+ * Run the test code, executing the test_all() function without async,
+ * and then with async.
+ *
+ * @param argc argument count passed into main.
+ * @param argv argument array passed into main.
+ * @param min_ntasks the min number of tasks this code need to run.
+ * @param max_ntasks the code will run on this many tasks, extras will
+ * be ignored.
+ * @param log_level the level to set the logger at.
+ * @param test_name the name of the test being run.
+ * @param dim_len array of size 3 with the dimension sizes.
+ * @param component_count number of computation components.
+ * @param num_io_procs number of IO components.
+ * @returns 0 for success, error code otherwise.
+ */
+int run_test_main(int argc, char **argv, int min_ntasks, int max_ntasks,
+                  int log_level, char *test_name, int *dim_len, int component_count,
+                  int num_io_procs)
+{
+    int my_rank;
+    int ntasks;
+    int num_flavors; /* Number of PIO netCDF flavors in this build. */
+    int flavor[NUM_FLAVORS]; /* iotypes for the supported netCDF IO flavors. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+    int ret;         /* Return code. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, min_ntasks,
+                              max_ntasks, log_level, &test_comm)))
+        ERR(ERR_INIT);
+
+    if ((ret = PIOc_set_iosystem_error_handling(PIO_DEFAULT, PIO_RETURN_ERROR, NULL)))
+        return ret;
+
+    /* Only do something on max_ntasks tasks. */
+    if (my_rank < max_ntasks)
+    {
+        /* Figure out iotypes. */
+        if ((ret = get_iotypes(&num_flavors, flavor)))
+            ERR(ret);
+        printf("Runnings tests for %d flavors\n", num_flavors);
+
+        /* Run tests without async feature. */
+        if ((ret = test_no_async2(my_rank, num_flavors, flavor, test_comm, max_ntasks,
+                                  dim_len[1], dim_len[2])))
+            return ret;
+
+        /* Run tests with async. */
+        if ((ret = test_async2(my_rank, num_flavors, flavor, test_comm, component_count,
+                               num_io_procs, max_ntasks, test_name)))
+            return ret;
+
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, test_name);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, test_name);
+    return 0;
+}
diff --git a/src/externals/pio2/tests/cunit/test_spmd.c b/src/externals/pio2/tests/cunit/test_spmd.c
new file mode 100644
index 00000000000..2c51ce2ccab
--- /dev/null
+++ b/src/externals/pio2/tests/cunit/test_spmd.c
@@ -0,0 +1,532 @@
+/*
+ * This program tests some internal functions in the PIO library.
+ *
+ * Jim Edwards
+ * Ed Hartnett, 11/23/16
+ */
+#include 
+#include 
+#include 
+
+/* The number of tasks this test should run on. */
+#define TARGET_NTASKS 4
+
+/* The minimum number of tasks this test should run on. */
+#define MIN_NTASKS 1
+
+/* The name of this test. */
+#define TEST_NAME "test_spmd"
+
+/* Number of test cases in inner loop of test. */
+#define NUM_TEST_CASES 5
+
+/* Test MPI_Alltoallw by having processor i send different amounts of
+ * data to each processor.  The first test sends i items to processor
+ * i from all processors. */
+int run_spmd_tests(MPI_Comm test_comm)
+{
+    int my_rank;  /* 0-based rank in test_comm. */
+    int ntasks;   /* Number of tasks in test_comm. */
+    int num_elem; /* Number of elements in buffers. */
+    int type_size; /* Size in bytes of an element. */
+    int mpierr;   /* Return value from MPI calls. */
+    int ret;      /* Return value. */
+
+    /* Learn rank and size. */
+    if ((mpierr = MPI_Comm_size(test_comm, &ntasks)))
+        MPIERR(mpierr);
+    if ((mpierr = MPI_Comm_rank(test_comm, &my_rank)))
+        MPIERR(mpierr);
+
+    /* Determine size of buffers. */
+    num_elem = ntasks;
+
+    int sbuf[ntasks];    /* The send buffer. */
+    int rbuf[ntasks];    /* The receive buffer. */
+    int sendcounts[ntasks]; /* Number of elements of data being sent from each task. */
+    int recvcounts[ntasks]; /* Number of elements of data being sent from each task. */
+    int sdispls[ntasks]; /* Displacements for sending data. */
+    int rdispls[ntasks]; /* Displacements for receiving data. */
+    MPI_Datatype sendtypes[ntasks]; /* MPI types of data being sent. */
+    MPI_Datatype recvtypes[ntasks]; /* MPI types of data being received. */
+
+    /* Load up the send buffer. */
+    for (int i = 0; i < num_elem; i++)
+        sbuf[i] = my_rank;
+
+    /* Load up the receive buffer to make debugging easier. */
+    for (int i = 0; i < num_elem; i++)
+        rbuf[i] = -999;
+
+    /* Get the size of the int type for MPI. (Should always be 4.) */
+    if ((mpierr = MPI_Type_size(MPI_INT, &type_size)))
+        return check_mpi(NULL, mpierr, __FILE__, __LINE__);
+    assert(type_size == sizeof(int));
+
+    /* Initialize the arrays. */
+    for (int i = 0; i < ntasks; i++)
+    {
+        sendcounts[i] = 1;
+        sdispls[i] = 0;
+        sendtypes[i] = MPI_INT;
+        recvcounts[i] = 1;
+        rdispls[i] = i * type_size;
+        recvtypes[i] = MPI_INT;
+    }
+
+    /* Perform tests for different values of msg_cnt. (BTW it hangs
+     * with msg_cnt = 1!). */
+    for (int msg_cnt = 0; msg_cnt < TARGET_NTASKS; msg_cnt = msg_cnt ? msg_cnt * 2 : 4)
+    {
+        if (!my_rank)
+            printf("message count %d\n",msg_cnt);
+
+        for (int itest = 0; itest < NUM_TEST_CASES; itest++)
+        {
+            rearr_comm_fc_opt_t fc;            
+            fc.hs = false;
+            fc.isend = false;
+
+            /* Wait for all tasks. */
+            MPI_Barrier(test_comm);
+
+            /* Print results. */
+            if (!my_rank)
+                for (int e = 0; e < num_elem; e++)
+                    printf("sbuf[%d] = %d\n", e, sbuf[e]);
+
+            /* Set the parameters different for each test case. */
+            if (itest == 1)
+            {
+                fc.hs = true;
+                fc.isend = true;
+            }
+            else if (itest == 2)
+            {
+                fc.hs = false;
+                fc.isend = true;
+            }
+            else if (itest == 3)
+            {
+                fc.hs = false;
+                fc.isend = false;
+            }
+            else if (itest == 4)
+            {
+                fc.hs = true;
+                fc.isend = false;
+            }
+
+            /* Run the swapm function. */
+            if ((ret = pio_swapm(sbuf, sendcounts, sdispls, sendtypes, rbuf, recvcounts,
+                                 rdispls, recvtypes, test_comm, &fc)))
+                return ret;
+
+            /* Print results. */
+            /* MPI_Barrier(test_comm); */
+            /* for (int e = 0; e < num_elem; e++) */
+            /*     printf("%d sbuf[%d] = %d\n", my_rank, e, sbuf[e]); */
+            /* MPI_Barrier(test_comm); */
+            /* for (int e = 0; e < num_elem; e++) */
+            /*     printf("%d rbuf[%d] = %d\n", my_rank, e, rbuf[e]); */
+
+            /* Check that rbuf has 0, 1, ..., ntasks-1. */
+            for (int e = 0; e < num_elem; e++)
+                if (((int *)rbuf)[e] != e)
+                    return ERR_WRONG;
+        }
+    }
+
+    return 0;
+}
+
+/* Test some of the functions in the file pioc_sc.c. 
+ *
+ * @param test_comm the MPI communicator that the test code is running on. 
+ * @returns 0 for success, error code otherwise.
+ */
+int run_sc_tests(MPI_Comm test_comm)
+{
+#define SC_ARRAY_LEN 3
+    int my_rank;  /* 0-based rank in test_comm. */
+    int ntasks;   /* Number of tasks in test_comm. */
+    int mpierr;   /* Return value from MPI calls. */
+    int array1[SC_ARRAY_LEN] = {7, 42, 14};
+    int array2[SC_ARRAY_LEN] = {2, 3, 7};
+    int array3[SC_ARRAY_LEN] = {90, 180, 270};
+    int array4[SC_ARRAY_LEN] = {1, 180, 270};
+
+    /* Learn rank and size. */
+    if ((mpierr = MPI_Comm_size(test_comm, &ntasks)))
+        MPIERR(mpierr);
+    if ((mpierr = MPI_Comm_rank(test_comm, &my_rank)))
+        MPIERR(mpierr);
+
+    /* Test the gcd() function. */
+    if (gcd(0, 2) != 2)
+        return ERR_WRONG;
+    if (gcd(2, 2) != 2)
+        return ERR_WRONG;
+    if (gcd(42, 2) != 2)
+        return ERR_WRONG;
+
+    /* Test the long long version. */
+    if (lgcd(0, 2) != 2)
+        return ERR_WRONG;
+    if (lgcd(2, 2) != 2)
+        return ERR_WRONG;
+    if (lgcd(42, 2) != 2)
+        return ERR_WRONG;
+
+    /* Test the gcd_array() function. */
+    if (gcd_array(SC_ARRAY_LEN, array1) != 7)
+        return ERR_WRONG;
+    if (gcd_array(SC_ARRAY_LEN, array2) != 1)
+        return ERR_WRONG;
+    if (gcd_array(SC_ARRAY_LEN, array3) != 90)
+        return ERR_WRONG;
+    if (gcd_array(SC_ARRAY_LEN, array4) != 1)
+        return ERR_WRONG;
+
+    /* Test compute_one_dim. */
+    PIO_Offset start, count;
+    compute_one_dim(4, 4, my_rank, &start, &count);
+    if (start != my_rank || count != 1)
+        return ERR_WRONG;
+    compute_one_dim(400, 4, my_rank, &start, &count);
+    if (start != my_rank * 100 || count != 100)
+        return ERR_WRONG;
+    /* Left over data will go to task 3. */
+    compute_one_dim(5, 4, my_rank, &start, &count);
+    if (start != my_rank || count != (my_rank == 3 ? 2 : 1))
+        return ERR_WRONG;
+    printf("my_rank = %d start = %lld count = %lld\n", my_rank, start, count);
+    return 0;
+}
+
+/* Tesst some list stuff. */
+int test_lists()
+{
+    file_desc_t *fdesc;
+    
+    /* Test that bad input is correctly rejected. */
+    if (pio_delete_iodesc_from_list(42) != PIO_EBADID)
+        return ERR_WRONG;
+    if (pio_delete_iosystem_from_list(42) != PIO_EBADID)
+        return ERR_WRONG;
+    if (pio_delete_file_from_list(42) != PIO_EBADID)
+        return ERR_WRONG;
+    if (pio_get_file(42, NULL) != PIO_EINVAL)
+        return ERR_WRONG;
+    if (pio_get_file(42, &fdesc) != PIO_EBADID)
+        return ERR_WRONG;
+    return 0;
+}
+
+/* Test the ceil2() and pair() functions. */
+int test_ceil2_pair()
+{
+    /* Test the ceil2() function. */
+    if (ceil2(1) != 1)
+        return ERR_WRONG;
+    if (ceil2(-100) != 1)
+        return ERR_WRONG;
+    if (ceil2(2) != 2)
+        return ERR_WRONG;
+    if (ceil2(3) != 4)
+        return ERR_WRONG;
+    if (ceil2(16) != 16)
+        return ERR_WRONG;
+    if (ceil2(17) != 32)
+        return ERR_WRONG;
+
+    /* Test the pair() function. */
+    if (pair(4, 0, 0) != 1)
+        return ERR_WRONG;
+    if (pair(4, 2, 2) != 1)
+        return ERR_WRONG;
+    
+    return 0;
+}
+
+/* Test the function that finds an MPI type to match a PIO type. */
+int test_find_mpi_type()
+{
+    MPI_Datatype mpi_type;
+    int type_size;
+    int ret;
+
+    /* This should not work. */
+    if (find_mpi_type(PIO_BYTE + 42, &mpi_type, &type_size) != PIO_EBADTYPE)
+        return ERR_WRONG;
+
+    /* Try every atomic type. */
+    if ((ret = find_mpi_type(PIO_BYTE, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_BYTE || type_size != 1)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_CHAR, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_CHAR || type_size != 1)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_SHORT, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_SHORT || type_size != 2)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_INT, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_INT || type_size != 4)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_FLOAT, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_FLOAT || type_size != 4)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_DOUBLE, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_DOUBLE || type_size != 8)
+        return ERR_WRONG;
+
+    /* These should also work. */
+    if ((ret = find_mpi_type(PIO_INT, &mpi_type, NULL)))
+        return ret;
+    if (mpi_type != MPI_INT)
+        return ERR_WRONG;
+    if ((ret = find_mpi_type(PIO_INT, NULL, &type_size)))
+        return ret;
+    if (type_size != 4)
+        return ERR_WRONG;
+    if ((ret = find_mpi_type(PIO_INT, NULL, NULL)))
+        return ret;
+
+#ifdef _NETCDF4
+    if ((ret = find_mpi_type(PIO_UBYTE, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_UNSIGNED_CHAR || type_size != 1)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_USHORT, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_UNSIGNED_SHORT || type_size != 2)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_UINT, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_UNSIGNED || type_size != 4)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_INT64, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_LONG_LONG || type_size != 8)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_UINT64, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_UNSIGNED_LONG_LONG || type_size != 8)
+        return ERR_WRONG;
+
+    if ((ret = find_mpi_type(PIO_STRING, &mpi_type, &type_size)))
+        return ret;
+    if (mpi_type != MPI_CHAR || type_size != 1)
+        return ERR_WRONG;
+
+#endif /* _NETCDF4 */
+    return PIO_NOERR;
+}
+
+int test_misc()
+{
+    wmulti_buffer wmb;
+
+    /* This should not work. */
+    if (flush_buffer(TEST_VAL_42, &wmb, 0) != PIO_EBADID)
+        return ERR_WRONG;
+
+    return 0;
+}
+
+/* This test code was recovered from main() in pioc_sc.c. */
+int test_CalcStartandCount()
+{
+    int ndims = 2;
+    int gdims[2] = {31, 777602};
+    int num_io_procs = 24;
+    bool converged = false;
+    PIO_Offset start[ndims], kount[ndims];
+    int iorank, numaiotasks = 0;
+    long int tpsize = 0;
+    long int psize;
+    long int pgdims = 1;
+    int scnt;
+    int ret;
+
+    for (int i = 0; i < ndims; i++)
+        pgdims *= gdims[i];
+
+    while (!converged)
+    {
+        for (iorank = 0; iorank < num_io_procs; iorank++)
+        {
+            if ((ret = CalcStartandCount(PIO_DOUBLE, ndims, gdims, num_io_procs, iorank,
+                                         start, kount, &numaiotasks)))
+                return ret;
+            if (iorank < numaiotasks)
+                printf("iorank %d start %lld %lld count %lld %lld\n", iorank, start[0],
+                       start[1], kount[0], kount[1]);
+
+            if (numaiotasks < 0)
+                return numaiotasks;
+
+            psize = 1;
+            scnt = 0;
+            for (int i = 0; i < ndims; i++)
+            {
+                psize *= kount[i];
+                scnt += kount[i];
+            }
+            tpsize += psize;
+        }
+
+        if (tpsize == pgdims)
+            converged = true;
+        else
+        {
+            printf("Failed to converge %ld %ld %d\n", tpsize, pgdims, num_io_procs);
+            tpsize = 0;
+            num_io_procs--;
+        }
+    }
+
+    return 0;
+}
+
+/* Test the GDCblocksize() function. */
+int run_GDCblocksize_tests(MPI_Comm test_comm)
+{
+    {
+        int arrlen = 1;
+        PIO_Offset arr_in[1] = {0};
+        PIO_Offset blocksize;
+        
+        blocksize = GCDblocksize(arrlen, arr_in);
+        if (blocksize != 1)
+            return ERR_WRONG;
+    }
+
+    {
+        int arrlen = 4;
+        PIO_Offset arr_in[4] = {0, 1, 2, 3};
+        PIO_Offset blocksize;
+        
+        blocksize = GCDblocksize(arrlen, arr_in);
+        if (blocksize != 4)
+            return ERR_WRONG;
+    }
+    
+    {
+        int arrlen = 4;
+        PIO_Offset arr_in[4] = {0, 2, 3, 4};
+        PIO_Offset blocksize;
+
+        blocksize = GCDblocksize(arrlen, arr_in);
+        if (blocksize != 1)
+            return ERR_WRONG;
+    }
+    
+    {
+        int arrlen = 4;
+        PIO_Offset arr_in[4] = {0, 1, 3, 4};
+        PIO_Offset blocksize;
+
+        blocksize = GCDblocksize(arrlen, arr_in);
+        if (blocksize != 1)
+            return ERR_WRONG;
+    }
+    
+    {
+        int arrlen = 4;
+        PIO_Offset arr_in[4] = {0, 1, 2, 4};
+        PIO_Offset blocksize;
+
+        blocksize = GCDblocksize(arrlen, arr_in);
+        if (blocksize != 1)
+            return ERR_WRONG;
+    }
+    
+    return 0;
+}
+
+/* Run Tests for pio_spmd.c functions. */
+int main(int argc, char **argv)
+{
+    int my_rank; /* Zero-based rank of processor. */
+    int ntasks;  /* Number of processors involved in current execution. */
+    int ret;     /* Return code. */
+    MPI_Comm test_comm; /* A communicator for this test. */
+
+    /* Initialize test. */
+    if ((ret = pio_test_init2(argc, argv, &my_rank, &ntasks, MIN_NTASKS,
+                              TARGET_NTASKS, 3, &test_comm)))
+        ERR(ERR_INIT);
+
+    /* Test code runs on TARGET_NTASKS tasks. The left over tasks do
+     * nothing. */
+    if (my_rank < TARGET_NTASKS)
+    {
+        /* I don't need this iosystem, but it's the only way to get
+         * the logs to write. */
+        int iosysid;
+        if ((ret = PIOc_Init_Intracomm(test_comm, TARGET_NTASKS, 1, 0, PIO_REARR_BOX, &iosysid)))
+            return ret;
+
+        printf("%d running tests for functions in pioc_sc.c\n", my_rank);
+        if ((ret = run_sc_tests(test_comm)))
+            return ret;
+
+        printf("%d running tests for GCDblocksize()\n", my_rank);
+        if ((ret = run_GDCblocksize_tests(test_comm)))
+            return ret;
+
+        printf("%d running spmd test code\n", my_rank);
+        if ((ret = run_spmd_tests(test_comm)))
+            return ret;
+        
+        printf("%d running CalcStartandCount test code\n", my_rank);
+        if ((ret = test_CalcStartandCount()))
+            return ret;
+
+        printf("%d running list tests\n", my_rank);
+        if ((ret = test_lists()))
+            return ret;
+
+        printf("%d running ceil2/pair tests\n", my_rank);
+        if ((ret = test_ceil2_pair()))
+            return ret;
+
+        printf("%d running find_mpi_type tests\n", my_rank);
+        if ((ret = test_find_mpi_type()))
+            return ret;
+
+        printf("%d running misc tests\n", my_rank);
+        if ((ret = test_misc()))
+            return ret;
+
+        /* Finalize PIO system. */
+        if ((ret = PIOc_finalize(iosysid)))
+            return ret;
+
+    } /* endif my_rank < TARGET_NTASKS */
+
+    /* Finalize the MPI library. */
+    printf("%d %s Finalizing...\n", my_rank, TEST_NAME);
+    if ((ret = pio_test_finalize(&test_comm)))
+        return ret;
+
+    printf("%d %s SUCCESS!!\n", my_rank, TEST_NAME);
+
+    return 0;
+}
diff --git a/src/externals/pio2/tests/general/CMakeLists.txt b/src/externals/pio2/tests/general/CMakeLists.txt
new file mode 100644
index 00000000000..b71382d3064
--- /dev/null
+++ b/src/externals/pio2/tests/general/CMakeLists.txt
@@ -0,0 +1,832 @@
+include (LibMPI)
+
+#==============================================================================
+#  GENERATE TARGET SOURCES
+#==============================================================================
+
+SET(GENERATED_SRCS pio_file_simple_tests.F90
+  pio_init_finalize.F90
+  pio_fail.F90
+  pio_file_fail.F90
+  ncdf_simple_tests.F90
+  ncdf_get_put.F90
+  ncdf_fail.F90
+  ncdf_inq.F90
+  pio_rearr.F90
+  pio_rearr_opts.F90
+  pio_rearr_opts2.F90
+  pio_decomp_tests.F90
+  pio_decomp_tests_1d.F90
+  pio_decomp_tests_2d.F90
+  pio_decomp_tests_3d.F90
+  pio_decomp_frame_tests.F90
+  pio_decomp_fillval.F90
+  pio_iosystem_tests.F90
+  pio_iosystem_tests2.F90
+  pio_iosystem_tests3.F90)
+
+foreach (SRC_FILE IN LISTS GENERATED_SRCS)
+  add_custom_command (
+    OUTPUT ${SRC_FILE}
+    COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tf_f90gen.pl
+    --annotate-source
+    --out=${CMAKE_CURRENT_BINARY_DIR}/${SRC_FILE}
+    ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${SRC_FILE}.in)
+endforeach ()
+
+if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
+  add_definitions(-ffree-line-length-none)
+endif()
+
+if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
+  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
+  #    target_compile_options (gptl
+  #        PRIVATE -mismatch_all)
+endif ()
+
+#==============================================================================
+#  DEFINE THE TARGETS AND TESTS
+#==============================================================================
+
+# Test Timeout (4 min = 240 sec)
+set (DEFAULT_TEST_TIMEOUT 240)
+
+#===== pio_init_finalize =====
+add_executable (pio_init_finalize EXCLUDE_FROM_ALL
+  pio_init_finalize.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_init_finalize piof)
+add_dependencies (tests pio_init_finalize)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME init_finialize_1_proc
+    COMMAND pio_init_finalize)
+  set_tests_properties(init_finialize_1_proc
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME init_finialize_2_proc
+    COMMAND pio_init_finalize)
+  set_tests_properties(init_finialize_2_proc
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME init_finalize_2_proc_with_args
+    COMMAND pio_init_finalize --pio-tf-stride=2 --pio-tf-num-aggregators=2)
+  set_tests_properties(init_finalize_2_proc_with_args
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(init_finialize_1_proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_init_finalize
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(init_finialize_2_proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_init_finalize
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(init_finalize_2_proc_with_args
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_init_finalize
+    ARGUMENTS --pio-tf-stride=2 --pio-tf-num-aggregators=2
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_file_simple_tests =====
+add_executable (pio_file_simple_tests EXCLUDE_FROM_ALL
+  pio_file_simple_tests.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_file_simple_tests piof)
+add_dependencies (tests pio_file_simple_tests)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_file_simple_tests
+    COMMAND pio_file_simple_tests)
+  set_tests_properties(pio_file_simple_tests
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_file_simple_tests
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_file_simple_tests
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_file_fail =====
+add_executable (pio_file_fail EXCLUDE_FROM_ALL
+  pio_file_fail.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_file_fail piof)
+if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
+  target_compile_options (pio_init_finalize
+    PRIVATE -ffree-line-length-none)
+endif()
+add_dependencies (tests pio_file_fail)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_file_fail
+    COMMAND pio_file_fail)
+  set_tests_properties(pio_file_fail
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_file_fail
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_file_fail
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== ncdf_simple_tests =====
+add_executable (ncdf_simple_tests EXCLUDE_FROM_ALL
+  ncdf_simple_tests.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (ncdf_simple_tests piof)
+add_dependencies (tests ncdf_simple_tests)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME ncdf_simple_tests
+    COMMAND ncdf_simple_tests)
+  set_tests_properties(ncdf_simple_tests
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(ncdf_simple_tests
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_simple_tests
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== ncdf_get_put =====
+add_executable (ncdf_get_put EXCLUDE_FROM_ALL
+  ncdf_get_put.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (ncdf_get_put piof)
+add_dependencies (tests ncdf_get_put)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME ncdf_get_put_1proc
+    COMMAND ncdf_get_put)
+  set_tests_properties(ncdf_get_put_1proc
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME ncdf_get_put_2proc
+    COMMAND ncdf_get_put)
+  set_tests_properties(ncdf_get_put_2proc
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(ncdf_get_put_1proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_get_put
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(ncdf_get_put_2proc
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_get_put
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== ncdf_fail =====
+add_executable (ncdf_fail EXCLUDE_FROM_ALL
+  ncdf_fail.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (ncdf_fail piof)
+add_dependencies (tests ncdf_fail)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME ncdf_fail
+    COMMAND ncdf_fail)
+  set_tests_properties(ncdf_fail
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(ncdf_fail
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_fail
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== ncdf_inq =====
+add_executable (ncdf_inq EXCLUDE_FROM_ALL
+  ncdf_inq.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (ncdf_inq piof)
+add_dependencies (tests ncdf_inq)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME ncdf_inq
+    COMMAND ncdf_inq)
+  set_tests_properties(ncdf_inq
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(ncdf_inq
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ncdf_inq
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_rearr =====
+add_executable (pio_rearr EXCLUDE_FROM_ALL
+    pio_rearr.F90
+    ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_rearr piof)
+add_dependencies (tests pio_rearr)
+
+if (PIO_USE_MPISERIAL)
+    add_test(NAME pio_rearr
+        COMMAND pio_rearr)
+    set_tests_properties(pio_rearr
+        PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+    add_mpi_test(pio_rearr
+        EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_rearr
+        NUMPROCS 4
+        TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_rearr_opts =====
+add_executable (pio_rearr_opts EXCLUDE_FROM_ALL
+    pio_rearr_opts.F90
+    ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_rearr_opts piof)
+add_dependencies (tests pio_rearr_opts)
+
+if (PIO_USE_MPISERIAL)
+    add_test(NAME pio_rearr_opts
+        COMMAND pio_rearr_opts)
+    set_tests_properties(pio_rearr_opts
+        PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+    add_mpi_test(pio_rearr_opts
+        EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_rearr_opts
+        NUMPROCS 4
+        TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_rearr_opts2 =====
+add_executable (pio_rearr_opts2 EXCLUDE_FROM_ALL
+    pio_rearr_opts2.F90
+    ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_rearr_opts2 piof)
+add_dependencies (tests pio_rearr_opts2)
+
+if (PIO_USE_MPISERIAL)
+    add_test(NAME pio_rearr_opts2
+        COMMAND pio_rearr_opts2)
+    set_tests_properties(pio_rearr_opts2
+        PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+    add_mpi_test(pio_rearr_opts2_1p
+        EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_rearr_opts2
+        NUMPROCS 1
+        TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+    add_mpi_test(pio_rearr_opts2_3p
+        EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_rearr_opts2
+        NUMPROCS 3
+        TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+    add_mpi_test(pio_rearr_opts2_4p
+        EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_rearr_opts2
+        NUMPROCS 4
+        TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_tests =====
+add_executable (pio_decomp_tests EXCLUDE_FROM_ALL
+  pio_decomp_tests.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_tests piof)
+add_dependencies (tests pio_decomp_tests)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_tests_1p
+    COMMAND pio_decomp_tests)
+  set_tests_properties(pio_decomp_tests_1p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2p
+    COMMAND pio_decomp_tests)
+  set_tests_properties(pio_decomp_tests_2p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3p
+    COMMAND pio_decomp_tests)
+  set_tests_properties(pio_decomp_tests_3p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_4p_1agg
+    COMMAND pio_decomp_tests --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_4p_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_4p_2agg
+    COMMAND pio_decomp_tests --pio-tf-num-aggregators=2)
+  set_tests_properties(pio_decomp_tests_4p_2agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_4p_3agg
+    COMMAND pio_decomp_tests --pio-tf-num-aggregators=3)
+  set_tests_properties(pio_decomp_tests_4p_3agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_4p_1iop
+    COMMAND pio_decomp_tests --pio-tf-num-io-tasks=1)
+  set_tests_properties(pio_decomp_tests_4p_1iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_4p_2iop
+    COMMAND pio_decomp_tests --pio-tf-num-io-tasks=2)
+  set_tests_properties(pio_decomp_tests_4p_2iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_4p_3iop
+    COMMAND pio_decomp_tests --pio-tf-num-io-tasks=3)
+  set_tests_properties(pio_decomp_tests_4p_3iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_4p_2iop_2str
+    COMMAND pio_decomp_tests --pio-tf-num-io-tasks=2 --pio-tf-stride=2)
+  set_tests_properties(pio_decomp_tests_4p_2iop_2str
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_4p_2iop_1agg
+    COMMAND pio_decomp_tests --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_4p_2iop_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_tests_1p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    NUMPROCS 3
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_4p_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_4p_2agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-aggregators=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_4p_3agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-aggregators=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_4p_1iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-io-tasks=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_4p_2iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-io-tasks=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_4p_3iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-io-tasks=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_4p_2iop_2str
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-stride=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_4p_2iop_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_tests_1d =====
+add_executable (pio_decomp_tests_1d EXCLUDE_FROM_ALL
+  pio_decomp_tests_1d.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_tests_1d piof)
+add_dependencies (tests pio_decomp_tests_1d)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_tests_1d_1p
+    COMMAND pio_decomp_tests_1d)
+  set_tests_properties(pio_decomp_tests_1d_1p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_2p
+    COMMAND pio_decomp_tests_1d)
+  set_tests_properties(pio_decomp_tests_1d_2p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_3p
+    COMMAND pio_decomp_tests_1d)
+  set_tests_properties(pio_decomp_tests_1d_3p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_1d_4p_1agg
+    COMMAND pio_decomp_tests_1d --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_1d_4p_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_4p_2agg
+    COMMAND pio_decomp_tests_1d --pio-tf-num-aggregators=2)
+  set_tests_properties(pio_decomp_tests_1d_4p_2agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_4p_3agg
+    COMMAND pio_decomp_tests_1d --pio-tf-num-aggregators=3)
+  set_tests_properties(pio_decomp_tests_1d_4p_3agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_1d_4p_1iop
+    COMMAND pio_decomp_tests_1d --pio-tf-num-io-tasks=1)
+  set_tests_properties(pio_decomp_tests_1d_4p_1iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_4p_2iop
+    COMMAND pio_decomp_tests_1d --pio-tf-num-io-tasks=2)
+  set_tests_properties(pio_decomp_tests_1d_4p_2iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_4p_3iop
+    COMMAND pio_decomp_tests_1d --pio-tf-num-io-tasks=3)
+  set_tests_properties(pio_decomp_tests_1d_4p_3iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_1d_4p_2iop_2str
+    COMMAND pio_decomp_tests_1d --pio-tf-num-io-tasks=2 --pio-tf-stride=2)
+  set_tests_properties(pio_decomp_tests_1d_4p_2iop_2str
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_1d_4p_2iop_1agg
+    COMMAND pio_decomp_tests_1d --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_1d_4p_2iop_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_tests_1d_1p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_2p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_3p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    NUMPROCS 3
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_1d_4p_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_4p_2agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-aggregators=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_4p_3agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-aggregators=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_1d_4p_1iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-io-tasks=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_4p_2iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-io-tasks=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_4p_3iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-io-tasks=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_1d_4p_2iop_2str
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-stride=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_1d_4p_2iop_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_1d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_tests_2d =====
+add_executable (pio_decomp_tests_2d EXCLUDE_FROM_ALL
+  pio_decomp_tests_2d.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_tests_2d piof)
+add_dependencies (tests pio_decomp_tests_2d)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_tests_2d_1p
+    COMMAND pio_decomp_tests_2d)
+  set_tests_properties(pio_decomp_tests_2d_1p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_2p
+    COMMAND pio_decomp_tests_2d)
+  set_tests_properties(pio_decomp_tests_2d_2p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_3p
+    COMMAND pio_decomp_tests_2d)
+  set_tests_properties(pio_decomp_tests_2d_3p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_2d_4p_1agg
+    COMMAND pio_decomp_tests_2d --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_2d_4p_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_4p_2agg
+    COMMAND pio_decomp_tests_2d --pio-tf-num-aggregators=2)
+  set_tests_properties(pio_decomp_tests_2d_4p_2agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_4p_3agg
+    COMMAND pio_decomp_tests_2d --pio-tf-num-aggregators=3)
+  set_tests_properties(pio_decomp_tests_2d_4p_3agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_2d_4p_1iop
+    COMMAND pio_decomp_tests_2d --pio-tf-num-io-tasks=1)
+  set_tests_properties(pio_decomp_tests_2d_4p_1iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_4p_2iop
+    COMMAND pio_decomp_tests_2d --pio-tf-num-io-tasks=2)
+  set_tests_properties(pio_decomp_tests_2d_4p_2iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_4p_3iop
+    COMMAND pio_decomp_tests_2d --pio-tf-num-io-tasks=3)
+  set_tests_properties(pio_decomp_tests_2d_4p_3iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_2d_4p_2iop_2str
+    COMMAND pio_decomp_tests_2d --pio-tf-num-io-tasks=2 --pio-tf-stride=2)
+  set_tests_properties(pio_decomp_tests_2d_4p_2iop_2str
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_2d_4p_2iop_1agg
+    COMMAND pio_decomp_tests_2d --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_2d_4p_2iop_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_tests_2d_1p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_2p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_3p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    NUMPROCS 3
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_2d_4p_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_4p_2agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-aggregators=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_4p_3agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-aggregators=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_2d_4p_1iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-io-tasks=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_4p_2iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-io-tasks=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_4p_3iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-io-tasks=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_2d_4p_2iop_2str
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-stride=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_2d_4p_2iop_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_2d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_tests_3d =====
+add_executable (pio_decomp_tests_3d EXCLUDE_FROM_ALL
+  pio_decomp_tests_3d.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_tests_3d piof)
+add_dependencies (tests pio_decomp_tests_3d)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_tests_3d_1p
+    COMMAND pio_decomp_tests_3d)
+  set_tests_properties(pio_decomp_tests_3d_1p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_2p
+    COMMAND pio_decomp_tests_3d)
+  set_tests_properties(pio_decomp_tests_3d_2p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_3p
+    COMMAND pio_decomp_tests_3d)
+  set_tests_properties(pio_decomp_tests_3d_3p
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_3d_4p_1agg
+    COMMAND pio_decomp_tests_3d --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_3d_4p_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_4p_2agg
+    COMMAND pio_decomp_tests_3d --pio-tf-num-aggregators=2)
+  set_tests_properties(pio_decomp_tests_3d_4p_2agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_4p_3agg
+    COMMAND pio_decomp_tests_3d --pio-tf-num-aggregators=3)
+  set_tests_properties(pio_decomp_tests_3d_4p_3agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_3d_4p_1iop
+    COMMAND pio_decomp_tests_3d --pio-tf-num-io-tasks=1)
+  set_tests_properties(pio_decomp_tests_3d_4p_1iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_4p_2iop
+    COMMAND pio_decomp_tests_3d --pio-tf-num-io-tasks=2)
+  set_tests_properties(pio_decomp_tests_3d_4p_2iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_4p_3iop
+    COMMAND pio_decomp_tests_3d --pio-tf-num-io-tasks=3)
+  set_tests_properties(pio_decomp_tests_3d_4p_3iop
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_test(NAME pio_decomp_tests_3d_4p_2iop_2str
+    COMMAND pio_decomp_tests_3d --pio-tf-num-io-tasks=2 --pio-tf-stride=2)
+  set_tests_properties(pio_decomp_tests_3d_4p_2iop_2str
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_test(NAME pio_decomp_tests_3d_4p_2iop_1agg
+    COMMAND pio_decomp_tests_3d --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1)
+  set_tests_properties(pio_decomp_tests_3d_4p_2iop_1agg
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_tests_3d_1p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    NUMPROCS 1
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_2p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    NUMPROCS 2
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_3p
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    NUMPROCS 3
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_3d_4p_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_4p_2agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-aggregators=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_4p_3agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-aggregators=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_3d_4p_1iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-io-tasks=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_4p_2iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-io-tasks=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_4p_3iop
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-io-tasks=3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+  add_mpi_test(pio_decomp_tests_3d_4p_2iop_2str
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-stride=2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+  add_mpi_test(pio_decomp_tests_3d_4p_2iop_1agg
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_tests_3d
+    ARGUMENTS --pio-tf-num-io-tasks=2 --pio-tf-num-aggregators=1
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_frame_tests =====
+add_executable (pio_decomp_frame_tests EXCLUDE_FROM_ALL
+  pio_decomp_frame_tests.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_frame_tests piof)
+add_dependencies (tests pio_decomp_frame_tests)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_frame_tests
+    COMMAND pio_decomp_frame_tests)
+  set_tests_properties(pio_decomp_frame_tests
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_frame_tests
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_frame_tests
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_decomp_fillval =====
+add_executable (pio_decomp_fillval EXCLUDE_FROM_ALL
+  pio_decomp_fillval.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_decomp_fillval piof)
+add_dependencies (tests pio_decomp_fillval)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_decomp_fillval
+    COMMAND pio_decomp_fillval)
+  set_tests_properties(pio_decomp_fillval
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_decomp_fillval
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_decomp_fillval
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_iosystems_test =====
+add_executable (pio_iosystem_tests EXCLUDE_FROM_ALL
+  pio_iosystem_tests.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_iosystem_tests piof)
+add_dependencies (tests pio_iosystem_tests)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_iosystem_tests
+    COMMAND pio_iosystem_tests)
+  set_tests_properties(pio_iosystem_tests
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_iosystem_tests
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_iosystem_tests
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+
+#===== pio_iosystems_test2 =====
+add_executable (pio_iosystem_tests2 EXCLUDE_FROM_ALL
+  pio_iosystem_tests2.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_iosystem_tests2 piof)
+add_dependencies (tests pio_iosystem_tests2)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_iosystem_tests2
+    COMMAND pio_iosystem_tests2)
+  set_tests_properties(pio_iosystem_tests2
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_iosystem_tests2
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_iosystem_tests2
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+#===== pio_iosystems_test3 =====
+add_executable (pio_iosystem_tests3 EXCLUDE_FROM_ALL
+  pio_iosystem_tests3.F90
+  ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+target_link_libraries (pio_iosystem_tests3 piof)
+add_dependencies (tests pio_iosystem_tests3)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_iosystem_tests3
+    COMMAND pio_iosystem_tests3)
+  set_tests_properties(pio_iosystem_tests3
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_iosystem_tests3
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_iosystem_tests3
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
diff --git a/src/externals/pio2/tests/general/README.md b/src/externals/pio2/tests/general/README.md
new file mode 100644
index 00000000000..83539789881
--- /dev/null
+++ b/src/externals/pio2/tests/general/README.md
@@ -0,0 +1,142 @@
+# Using the testing framework
+ PIO uses a custom testing framework (`/tests/general/util/*`) to generate tests in the `/tests/general` directory. The testing framework provides,
+
+* Faster creation of tests : Easier creation of tests from simple stub source
+* Improved testcase code readability : Most of the boiler plate code is generated by the framework
+* Better error checking and reporting : User friendly parallel asserts
+* More testing with less code : Templated tests, auto generation of boiler plate PIO code
+* Easy integration with CTest 
+
+# Writing a test
+
+## Testcase format
+ All testcases recognized by the framework need to be subroutines with the following tags to indicate the start and end of the procedure,
+
+* `PIO_TF_AUTO_TEST_SUB_BEGIN` : This tag is used in the first line of the subroutine definition followed by the subroutine name. Consider this tag as the replacement of the `SUBROUTINE` keyword in Fortran
+* `PIO_TF_AUTO_TEST_SUB_END` : This tag marks the end of the subroutine, like the `"END SUBRTOUINE"` keyword in Fortran. This tag is followed by the subroutine name.
+
+ A simple hello world test program is shown below,
+
+    PIO_TF_AUTO_TEST_SUB_BEGIN hellow
+
+      print *, "Hello world"
+
+    PIO_TF_AUTO_TEST_SUB_END hellow
+
+ The testing framework converts this stub into a Fortran subroutine. The testing framework also adds a test driver that includes necessary code to initialize (and finalize) MPI, PIO and includes calls to all testcases tagged with the `PIO_TF_AUTO_TEST_SUB_BEGIN/PIO_TF_AUTO_TEST_SUB_END` tags.
+
+ Fortran code in the stub source file that don't have `PIO_TF_AUTO_TEST_SUB_BEGIN/PIO_TF_AUTO_TEST_SUB_END` tags are not modified by the framework. A test function cannot call other test functions but can call other Fortran utility functions and subroutines. It is however recommended that these utility functions return error codes that can be interpreted by a test function using a parallel assert provided by the framework (to signal a failure in these utility functions).
+
+## Runtime environment
+ The framework also adds code to read command line arguments to set,
+
+* PIO Stride : Set via "--pio-tf-stride" command line option
+* Number of PIO I/O tasks : Set via "--pio-tf-num-io-tasks" command line option
+* Number of PIO aggregators : Set via "--pio-tf-num-aggregators" command line option
+* PIO logging level : Set via "--pio-tf-log-level" command line option
+
+ The above options allow us to run the same test with different PIO parameters.
+
+## Testcase utils
+
+ The following utilities are available from the framework,
+
+*  PIO_TF_CHECK_VAL : Check if two variables are equal. This macro allows checking between two real/integer/double values or arrays. It also allows checking if all the values in an array is equal to a scalar value.
+
+        PIO_TF_CHECK_VAL(scalar_var1, scalar_var2)
+        PIO_TF_CHECK_VAL(array1, array2)
+        PIO_TF_CHECK_VAL(array1, scalar_var1)
+
+* `PIO_TF_CHECK_ERR` : Macro used to check the return code from a PIO function. The format for the macro is shown below,
+
+        PIO_TF_CHECK_ERR(pio_return_code, ``)
+
+* `PIO_TF_PASSERT` : A parallel assert macro. The format is shown below,
+
+        PIO_TF_PASSERT(``, ``)
+
+* `PIO_TF_ASSERT` : An assert (not parallel, local to each process) macro. The format is shown below,
+
+        PIO_TF_ASSERT(``, ``)
+
+* `PIO_TF_LOG` : Allows logging info into stdout
+
+        PIO_TF_LOG(*,*)(``)
+
+* `PIO_TF_ERROR` : Used to flag fatal errors
+
+        PIO_TF_ERROR(``)
+
+* `pio_tf_world_sz_, pio_tf_world_rank_` : Variables that provide size and rank of each process.
+
+## Templated testcases
+
+ Testcases can be templated for the different types supported by PIO. The framework provides a mapped collection of recognized fortran types and the corresponding PIO types. Templated testcases precede with a `PIO_TF_TEMPLATE<>` tag as show below,
+
+    PIO_TF_TEMPLATE
+    PIO_TF_AUTO_TEST_SUB_BEGIN templated_hellow
+
+     PIO_TF_FC_DATA_TYPE, dimension(10) :: buf
+     ...
+
+     call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc)
+
+     ... 
+
+    PIO_TF_AUTO_tEST_SUB_END templated_hellow
+
+ For the code shown above the framework generates multiple Fortran subroutines by replacing `PIO_TF_FC_DATA_TYPE` and `PIO_TF_DATA_TYPE` with matching Fortran and PIO types respectively.
+
+# Building a testcase
+## Generating the testcase
+
+ Testcases are generated by the '/tests/general/util/pio_tf_f90gen.pl' utility. It parses the stub source code provided as a command line argument and generates Fortran test cases (with all the boiler plate code necessary). The utility can be run as shown below,
+
+
+    ./util/pio_tf_f90gen.pl --annotate-source --out=hellow.F90 hellow.F90.in
+
+
+ In the above command hellow.F90.in is the stub test case created by the developer, and the pio_tf_f90gen.pl utility creates hellow.F90, the test case. The newly generated testcase (hellow.F90) is a Fortran program that depends only on '/tests/general/util/pio_tutil.F90' to build and contains all the code required to run the tests in it.
+
+## Building the generated testcase
+ The test case generated is a Fortran program that depends only on '/tests/general/util/pio_tutil.F90" to build. The test case can be built using any MPI compiler wrapper (mpif90).
+
+ The previous two steps are incorporated into CMakeLists.txt to add a new test to the testsuite.
+
+# Running the testcase
+
+## Running the test suite on your laptop
+ The test suite can be run by invoking the "ctest" command from the PIO build directory
+
+## Running the test suite on Supercomputers/Clusters
+
+ TODO
+
+## Running a single test on your laptop
+ A single test can be run by passing the test executable to an MPI job launcher (mpiexec), similar to running any MPI program.
+
+    mpiexec -n 4 ./hellow
+
+# Adding a test into the test suite
+ The build and running steps need to be incorporated into CMakeLists.txt to add a new testcase into the test suite. Please modify `/tests/general/CMakeLists.txt` as suggested below to add a new test to the test suite,
+
+* Add the testcase stub source into `GENERATED_SRCS`
+* Add a CMake build step : To generate the test executable
+
+        add_executable (pio_init_finalize EXCLUDE_FROM_ALL 
+            pio_init_finalize.F90
+            ${CMAKE_CURRENT_SOURCE_DIR}/util/pio_tutil.F90)
+        target_link_libraries (pio_init_finalize piof)
+        add_dependencies (tests pio_init_finalize)
+
+* Use CMake user defined macro add_mpi_test (or add_test() for mpi-serial) to run the test
+
+        add_mpi_test(init_finialize_1_proc
+                EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_init_finalize
+                NUMPROCS 1
+                TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+
+# Debugging failed tests
+ 
+* Run the test suite using ctest with the "--verbose" option. This outputs a lot of information from the testing framework (to stdout), including the various tests run and the reason for failure. All information in the output from the PIO testing framework is prepended with the `PIO_TF: ` tag. To run a single failing test use the "-R" option available with ctest (e.g. To only run "pio_iosystem_tests3", ctest --verbose -R pio_iosystem_tests3).
+* Run the failing test manually (using the MPI job launcher) and debug it. Also consider running the test with PIO log level > 0 (mpiexec -n 4 ./tests/general/pio_iosystem_tests3 --pio-tf-log-level=6)
diff --git a/externals/pio2/tests/general/ncdf_fail.F90.in b/src/externals/pio2/tests/general/ncdf_fail.F90.in
similarity index 98%
rename from externals/pio2/tests/general/ncdf_fail.F90.in
rename to src/externals/pio2/tests/general/ncdf_fail.F90.in
index f864dfada13..dfc09148a49 100644
--- a/externals/pio2/tests/general/ncdf_fail.F90.in
+++ b/src/externals/pio2/tests/general/ncdf_fail.F90.in
@@ -1,6 +1,6 @@
 MODULE ncdf_fail_tgv
   use pio_tutil
-  implicit none	
+  implicit none
 
   ! tgv = test global vars
   character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_fname = "pio_ncdf_test_file.nc"
@@ -13,12 +13,12 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_clob_then_no_clob
   type(file_desc_t) :: pio_file
   character(len=PIO_TF_MAX_STR_LEN), parameter :: clob_fname = "pio_clob_test_file.nc"
   integer :: ret
-  
+
   ret = PIO_createfile(pio_tf_iosystem_, pio_file, tgv_iotype, clob_fname, PIO_CLOBBER)
   PIO_TF_CHECK_ERR(ret, "Failed to create:" // trim(clob_fname))
 
   call PIO_closefile(pio_file)
-  
+
   ret = PIO_createfile(pio_tf_iosystem_, pio_file, tgv_iotype, clob_fname, PIO_NOCLOBBER)
   PIO_TF_PASSERT(ret /= PIO_NOERR, "Create file with clobber then no clobber did not fail as expected")
 
@@ -32,7 +32,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_redef_with_no_write
   Implicit none
   type(file_desc_t) :: pio_file
   integer :: ret
-  
+
   ret = PIO_openfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname, PIO_nowrite)
   PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(tgv_fname))
 
@@ -50,7 +50,7 @@ PIO_TF_AUTO_TEST_SUB_BEGIN test_redef_twice
   Implicit none
   type(file_desc_t) :: pio_file
   integer :: ret
-  
+
   ret = PIO_openfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname, PIO_write)
   PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(tgv_fname))
 
@@ -79,7 +79,7 @@ PIO_TF_TEST_DRIVER_BEGIN
   integer :: num_iotypes
 
   num_iotypes = 0
-  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes) 
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
   do i=1,num_iotypes
     tgv_iotype = iotypes(i)
     ret = PIO_createfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname)
@@ -92,7 +92,7 @@ PIO_TF_TEST_DRIVER_BEGIN
     PIO_TF_AUTO_TESTS_RUN(trim(iotype_descs(i)))
 
     call PIO_deletefile(pio_tf_iosystem_, tgv_fname)
-  end do 
+  end do
   if(allocated(iotypes)) then
     deallocate(iotypes)
     deallocate(iotype_descs)
diff --git a/src/externals/pio2/tests/general/ncdf_get_put.F90.in b/src/externals/pio2/tests/general/ncdf_get_put.F90.in
new file mode 100644
index 00000000000..ea8e5933b71
--- /dev/null
+++ b/src/externals/pio2/tests/general/ncdf_get_put.F90.in
@@ -0,0 +1,569 @@
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN test_put_1datt
+  Implicit none
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(var_desc_t)  :: pio_var, pio_cvar
+  integer :: pio_dim
+  integer, parameter :: DIM_LEN = 100
+  PIO_TF_FC_DATA_TYPE, dimension(DIM_LEN) :: val
+  CHARACTER(len=DIM_LEN) :: cval
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  
+  val = pio_tf_world_sz_
+  cval = "DUMMY_STRING"
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_ncdf_get_put.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename))
+
+    ! Since file is just created no need to enter redef
+    ret = PIO_def_dim(pio_file, 'dummy_dim_put_val', DIM_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_var_put_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_var_put_cval', PIO_char, (/pio_dim/), pio_cvar)
+    PIO_TF_CHECK_ERR(ret, "Failed to define char var:" // trim(filename))
+
+    ret = PIO_put_att(pio_file, pio_var, 'dummy_att_put_val', val);
+    PIO_TF_CHECK_ERR(ret, "Failed to put attribute:" // trim(filename))
+
+    ret = PIO_put_att(pio_file, pio_cvar, 'dummy_att_put_cval', cval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put char attribute:" // trim(filename))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename))
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_put_1datt
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1datt
+  Implicit none
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(var_desc_t)  :: pio_var, pio_cvar
+  integer :: pio_dim
+  integer, parameter :: DIM_LEN = 100
+  PIO_TF_FC_DATA_TYPE, dimension(DIM_LEN) :: pval, gval
+  PIO_TF_FC_DATA_TYPE :: init_val
+  CHARACTER(len=DIM_LEN) :: pcval, gcval
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  
+  init_val = pio_tf_world_sz_
+
+  pval = init_val
+  pcval = "DUMMY_STRING"
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_ncdf_get_put.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename))
+
+    ! Since file is just created no need to enter redef
+    ret = PIO_def_dim(pio_file, 'dummy_dim', DIM_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_cval', PIO_char, (/pio_dim/), pio_cvar)
+    PIO_TF_CHECK_ERR(ret, "Failed to define char var:" // trim(filename))
+
+    ret = PIO_put_att(pio_file, pio_var, 'dummy_att_val', pval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put attribute:" // trim(filename))
+
+    ret = PIO_put_att(pio_file, pio_cvar, 'dummy_att_cval', pcval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put char attribute:" // trim(filename))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename))
+
+    ret = PIO_get_att(pio_file, pio_var, 'dummy_att_val', gval);
+    PIO_TF_CHECK_ERR(ret, "Failed to get attribute:" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gval, init_val), "Got wrong value")
+
+    ! FIXME: Check the values are correct
+    ret = PIO_get_att(pio_file, pio_cvar, 'dummy_att_cval', gcval);
+    PIO_TF_CHECK_ERR(ret, "Failed to get attribute:" // trim(filename))
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_put_get_1datt
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_0dvar
+  Implicit none
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(var_desc_t)  :: pio_var, pio_cvar
+  PIO_TF_FC_DATA_TYPE, dimension(1) :: pval, gval
+  CHARACTER(len=1) :: pcval, gcval
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  
+  pval = pio_tf_world_sz_
+  pcval = "D"
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_ncdf_get_put.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename))
+
+    ! Since file is just created no need to enter redef
+    ret = PIO_def_var(pio_file, 'dummy_scalar_var_put_val', PIO_TF_DATA_TYPE, pio_var)
+    PIO_TF_CHECK_ERR(ret, "Failed to define scalar var:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_scalar_var_put_cval', PIO_char, pio_cvar)
+    PIO_TF_CHECK_ERR(ret, "Failed to define scalar char var:" // trim(filename))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename))
+
+    ret = PIO_put_var(pio_file, pio_var, pval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put scalar var:" // trim(filename))
+
+    ret = PIO_put_var(pio_file, pio_cvar, pcval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put scalar char var:" // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    ret = PIO_get_var(pio_file, pio_var, gval);
+    PIO_TF_CHECK_ERR(ret, "Failed to get scalar var:" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gval, pval), "Got wrong value")
+
+    ret = PIO_get_var(pio_file, pio_cvar, gcval);
+    PIO_TF_CHECK_ERR(ret, "Failed to get scalar char var:" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gcval, pcval), "Got wrong value")
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_put_get_0dvar
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar
+  Implicit none
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(var_desc_t)  :: pio_var, pio_cvar
+  integer :: pio_dim
+  integer, parameter :: DIM_LEN = 100
+  PIO_TF_FC_DATA_TYPE, dimension(DIM_LEN) :: pval, gval
+  CHARACTER(len=DIM_LEN) :: pcval, gcval
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  
+  pval = pio_tf_world_sz_
+  pcval = "DUMMY_STRING"
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_ncdf_get_put.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename))
+
+    ! Since file is just created no need to enter redef
+    ret = PIO_def_dim(pio_file, 'dummy_dim_put_val', DIM_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_var_put_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_var_put_cval', PIO_char, (/pio_dim/), pio_cvar)
+    PIO_TF_CHECK_ERR(ret, "Failed to define char var:" // trim(filename))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename))
+
+    ret = PIO_put_var(pio_file, pio_var, pval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put var:" // trim(filename))
+
+    ret = PIO_put_var(pio_file, pio_cvar, pcval);
+    PIO_TF_CHECK_ERR(ret, "Failed to put char var:" // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    ret = PIO_get_var(pio_file, pio_var, gval);
+    PIO_TF_CHECK_ERR(ret, "Failed to get var:" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gval, pval), "Got wrong value")
+
+    ret = PIO_get_var(pio_file, pio_cvar, gcval);
+    PIO_TF_CHECK_ERR(ret, "Failed to get char var:" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gcval, pcval), "Got wrong value")
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_put_get_1dvar
+
+! Write out a 1d var slice from a 2d var
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar_slice
+  Implicit none
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(var_desc_t)  :: pio_var, pio_cvar
+  integer :: pio_dim
+  integer, parameter :: MAX_ROW_DIM_LEN = 100
+  PIO_TF_FC_DATA_TYPE, dimension(MAX_ROW_DIM_LEN) :: gval, exp_val
+  integer, parameter :: MAX_COL_DIM_LEN = 4
+  ! Only COL_WRITE_DIM of MAX_COL_DIM_LEN columns in pval is written out
+  integer, parameter :: COL_WRITE_DIM = 2
+  PIO_TF_FC_DATA_TYPE, dimension(MAX_ROW_DIM_LEN, MAX_COL_DIM_LEN) :: pval
+  integer, dimension(:) :: start(4), count(4)
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  
+  pval = -1
+  pval(:,COL_WRITE_DIM) = pio_tf_world_sz_
+  exp_val = pio_tf_world_sz_
+  start = 0
+  count = 0
+  start(1) = 1
+  count(1) = MAX_ROW_DIM_LEN
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_ncdf_get_put_slice.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename))
+
+    ! Since file is just created no need to enter redef
+    ret = PIO_def_dim(pio_file, 'dummy_dim_put_val', MAX_ROW_DIM_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_var_put_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename))
+
+    ret = PIO_put_var(pio_file, pio_var, start, count, pval(:,COL_WRITE_DIM));
+    PIO_TF_CHECK_ERR(ret, "Failed to put var:" // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    ret = PIO_get_var(pio_file, pio_var, gval);
+    PIO_TF_CHECK_ERR(ret, "Failed to get var:" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gval, exp_val), "Got wrong value")
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_put_get_1dvar_slice
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_1dvar_4parts
+  Implicit none
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(var_desc_t)  :: pio_var, pio_cvar
+  integer :: pio_dim
+  integer, parameter :: DIM_LEN = 16
+  integer, parameter :: PART_LEN = DIM_LEN / 4
+  PIO_TF_FC_DATA_TYPE, dimension(DIM_LEN) :: pval, gval
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  integer, dimension(1) :: start, count = PART_LEN
+  
+  do i=1,DIM_LEN
+    pval(i) = i
+  end do
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_ncdf_get_put.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename))
+
+    ! Since file is just created no need to enter redef
+    ret = PIO_def_dim(pio_file, 'dummy_dim_put_val', DIM_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, 'dummy_var_put_val', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename))
+
+    start = 1
+    ret = PIO_put_var(pio_file, pio_var, start, count, pval(1 : PART_LEN))
+    PIO_TF_CHECK_ERR(ret, "Failed to put var (1st part):" // trim(filename))
+
+    start = PART_LEN + 1
+    ret = PIO_put_var(pio_file, pio_var, start, count, pval(PART_LEN + 1 : PART_LEN * 2))
+    PIO_TF_CHECK_ERR(ret, "Failed to put var (2nd part):" // trim(filename))
+
+    start = PART_LEN * 2 + 1
+    ret = PIO_put_var(pio_file, pio_var, start, count, pval(PART_LEN * 2 + 1 : PART_LEN * 3))
+    PIO_TF_CHECK_ERR(ret, "Failed to put var (3rd part):" // trim(filename))
+
+    start = PART_LEN * 3 + 1
+    ret = PIO_put_var(pio_file, pio_var, start, count, pval(PART_LEN * 3 + 1 : DIM_LEN))
+    PIO_TF_CHECK_ERR(ret, "Failed to put var (4th part):" // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    start = PART_LEN * 3 + 1
+    ret = PIO_get_var(pio_file, pio_var, start, count, gval(PART_LEN * 3 + 1 : DIM_LEN))
+    PIO_TF_CHECK_ERR(ret, "Failed to get var (4th part):" // trim(filename))
+
+    start = PART_LEN * 2 + 1
+    ret = PIO_get_var(pio_file, pio_var, start, count, gval(PART_LEN * 2 + 1 : PART_LEN * 3))
+    PIO_TF_CHECK_ERR(ret, "Failed to get var (3rd part):" // trim(filename))
+
+    start = PART_LEN + 1
+    ret = PIO_get_var(pio_file, pio_var, start, count, gval(PART_LEN + 1 : PART_LEN * 2))
+    PIO_TF_CHECK_ERR(ret, "Failed to get var (2nd part):" // trim(filename))
+
+    start = 1
+    ret = PIO_get_var(pio_file, pio_var, start, count, gval(1 : PART_LEN))
+    PIO_TF_CHECK_ERR(ret, "Failed to get var (1st part):" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gval, pval), "Got wrong value")
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_put_get_1dvar_4parts
+
+! Write out 2d/3d/4d vars, one time slice at a time
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN test_put_get_md2mdplus1_var
+  Implicit none
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  integer, parameter :: MAX_DIMS = 4
+  integer, parameter :: MAX_ROWS = 10
+  integer, parameter :: MAX_COLS = 10
+  integer, parameter :: MAX_LEVS = 3
+  integer, parameter :: MAX_TIMES = 3
+  integer, dimension(MAX_DIMS) :: pio_dims
+  type(var_desc_t)  :: pio_2dvar, pio_3dvar, pio_4dvar
+  PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS,MAX_TIMES) :: gval_2d, exp_val_2d
+  PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS,MAX_COLS,MAX_TIMES) :: gval_3d, exp_val_3d
+  PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS,MAX_COLS,MAX_LEVS,MAX_TIMES) ::&
+                                                      gval_4d, exp_val_4d
+  ! Only one slice is written out at a time
+  ! pval_1d is a 1d slice of gval_2d ...
+  PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS) :: pval_1d
+  PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS, MAX_COLS) :: pval_2d
+  PIO_TF_FC_DATA_TYPE, dimension(MAX_ROWS, MAX_COLS, MAX_LEVS) :: pval_3d
+  integer, dimension(:) :: start(MAX_DIMS), count(MAX_DIMS)
+  integer :: pval_start
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, k, l, m, n, tstep, ret
+  
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_ncdf_get_put_md_slice.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(filename))
+
+    ! Since file is just created no need to enter redef
+    ret = PIO_def_dim(pio_file, 'nrows', MAX_ROWS, pio_dims(1))
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_dim(pio_file, 'ncols', MAX_COLS, pio_dims(2))
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_dim(pio_file, 'nlevs', MAX_LEVS, pio_dims(3))
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_dim(pio_file, 'timesteps', MAX_TIMES, pio_dims(4))
+    PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, '2d_val', PIO_TF_DATA_TYPE,&
+              (/pio_dims(1),pio_dims(4)/), pio_2dvar)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, '3d_val', PIO_TF_DATA_TYPE,&
+              (/pio_dims(1),pio_dims(2),pio_dims(4)/), pio_3dvar)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_def_var(pio_file, '4d_val', PIO_TF_DATA_TYPE,&
+              pio_dims, pio_4dvar)
+    PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(filename))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(filename))
+
+    ! Put vals are for each timestep &
+    ! expected vals are combined for all timesteps
+    do k=1,MAX_ROWS
+      pval_1d(k) = k
+    end do
+    do tstep=1,MAX_TIMES
+      pval_start = (tstep - 1) * MAX_ROWS
+      exp_val_2d(:,tstep) = pval_1d + pval_start
+    end do
+    do l=1,MAX_COLS
+      do k=1,MAX_ROWS
+        pval_2d(k,l) = (l - 1)*MAX_ROWS + k
+      end do
+    end do
+    do tstep=1,MAX_TIMES
+      do l=1,MAX_COLS
+        do k=1,MAX_ROWS
+          pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS)
+          exp_val_3d(:,:,tstep) = pval_2d + pval_start
+        end do
+      end do
+    end do
+    do m=1,MAX_LEVS
+      do l=1,MAX_COLS
+        do k=1,MAX_ROWS
+          pval_3d(k,l,m) = ((m-1)*(MAX_COLS*MAX_ROWS)+(l - 1)*MAX_ROWS + k)
+        end do
+      end do
+    end do
+    do tstep=1,MAX_TIMES
+      do m=1,MAX_LEVS
+        do l=1,MAX_COLS
+          do k=1,MAX_ROWS
+            pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS * MAX_LEVS)
+            exp_val_4d(:,:,:,tstep) = pval_3d + pval_start
+          end do
+        end do
+      end do
+    end do
+    ! Put 2d/3d/4d vals, one timestep at a time
+    do tstep=1,MAX_TIMES
+      start = 0
+      count = 0
+
+      start(1) = 1
+      count(1) = MAX_ROWS
+      start(2) = tstep
+      count(2) = 1
+      pval_start = (tstep - 1) * MAX_ROWS
+      ret = PIO_put_var(pio_file, pio_2dvar, start, count,&
+              pval_1d(:)+pval_start)
+      PIO_TF_CHECK_ERR(ret, "Failed to put 2d var:" // trim(filename))
+
+      start(1) = 1
+      count(1) = MAX_ROWS
+      start(2) = 1
+      count(2) = MAX_COLS
+      start(3) = tstep
+      count(3) = 1
+      pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS)
+      ret = PIO_put_var(pio_file, pio_3dvar, start, count,&
+              pval_2d(:,:)+pval_start)
+      PIO_TF_CHECK_ERR(ret, "Failed to put 3d var:" // trim(filename))
+
+      start(1) = 1
+      count(1) = MAX_ROWS
+      start(2) = 1
+      count(2) = MAX_COLS
+      start(3) = 1
+      count(3) = MAX_LEVS
+      start(4) = tstep
+      count(4) = 1
+      pval_start = (tstep - 1) * (MAX_ROWS * MAX_COLS * MAX_LEVS)
+      ret = PIO_put_var(pio_file, pio_4dvar, start, count,&
+              pval_3d(:,:,:)+pval_start)
+      PIO_TF_CHECK_ERR(ret, "Failed to put 4d var:" // trim(filename))
+    end do
+
+    call PIO_syncfile(pio_file)
+
+    ret = PIO_get_var(pio_file, pio_2dvar, gval_2d)
+    PIO_TF_CHECK_ERR(ret, "Failed to get 2d var:" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gval_2d, exp_val_2d), "Got wrong value (2d var)")
+
+    ret = PIO_get_var(pio_file, pio_3dvar, gval_3d)
+    PIO_TF_CHECK_ERR(ret, "Failed to get 3d var:" // trim(filename))
+
+    PIO_TF_CHECK_VAL((gval_3d, exp_val_3d), "Got wrong value (3d var)")
+
+    ret = PIO_get_var(pio_file, pio_4dvar, gval_4d)
+    PIO_TF_CHECK_ERR(ret, "Failed to get 4d var:" // trim(filename))
+
+    ! Special code to handle 4d vals is required since the framework
+    ! currently does not support comparing 4d arrays
+    do tstep=1,MAX_TIMES
+      PIO_TF_CHECK_VAL((gval_4d(:,:,:,tstep), exp_val_4d(:,:,:,tstep)), "Got wrong value (4d var)")
+    end do
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_put_get_md2mdplus1_var
+
diff --git a/src/externals/pio2/tests/general/ncdf_inq.F90.in b/src/externals/pio2/tests/general/ncdf_inq.F90.in
new file mode 100644
index 00000000000..6ac3b1fd3ce
--- /dev/null
+++ b/src/externals/pio2/tests/general/ncdf_inq.F90.in
@@ -0,0 +1,192 @@
+MODULE ncdf_inq_tests_tgv
+  use pio_tutil
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_fname = "pio_ncdf_inq_test_file.nc"
+  integer :: tgv_iotype
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_var_name = "dummy_var"
+  integer, parameter :: tgv_var_ndims = 1
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_dim_name = "dummy_dim"
+  integer, parameter :: TGV_DIM_LEN = 100
+  integer, parameter :: tgv_var_natts = 2
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_iatt_name = "dummy_iatt"
+  integer, parameter :: tgv_iatt_val = 3
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: tgv_catt_name = "dummy_catt"
+  integer, parameter :: TGV_ATT_LEN = 100
+  character(len=TGV_ATT_LEN) :: tgv_catt_val = "DUMMY_STR"
+END MODULE ncdf_inq_tests_tgv
+
+SUBROUTINE test_setup(ret)
+  use pio_tutil
+  use ncdf_inq_tests_tgv
+  implicit none
+
+  integer, intent(out) :: ret
+
+  type(file_desc_t) :: pio_file
+  type(var_desc_t)  :: pio_var
+  integer :: pio_dim
+  real, dimension(TGV_DIM_LEN) :: val
+  integer, dimension(TGV_ATT_LEN) :: iatt
+  CHARACTER(len=TGV_ATT_LEN) :: catt
+
+  ret = PIO_createfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname, PIO_CLOBBER)
+  PIO_TF_CHECK_ERR(ret, "Failed to open:" // trim(tgv_fname))
+
+  ! Since file is just created no need to enter redef
+  ret = PIO_def_dim(pio_file, tgv_dim_name, TGV_DIM_LEN, pio_dim)
+  PIO_TF_CHECK_ERR(ret, "Failed to define dim:" // trim(tgv_fname))
+
+  ret = PIO_def_var(pio_file, tgv_var_name, pio_real, (/pio_dim/), pio_var)
+  PIO_TF_CHECK_ERR(ret, "Failed to define var:" // trim(tgv_fname))
+
+  iatt = tgv_iatt_val
+  ret = PIO_put_att(pio_file, pio_var, tgv_iatt_name, iatt);
+  PIO_TF_CHECK_ERR(ret, "Failed to put attribute:" // trim(tgv_fname))
+
+  catt = tgv_catt_val
+  ret = PIO_put_att(pio_file, pio_var, tgv_catt_name, catt);
+  PIO_TF_CHECK_ERR(ret, "Failed to put attribute:" // trim(tgv_fname))
+
+  ret = PIO_enddef(pio_file)
+  PIO_TF_CHECK_ERR(ret, "Failed to enddef:" // trim(tgv_fname))
+
+  val = pio_tf_world_rank_
+  ret = PIO_put_var(pio_file, pio_var, val)
+  PIO_TF_CHECK_ERR(ret, "Failed to put var: " // trim(tgv_fname))
+
+  call PIO_closefile(pio_file)
+END SUBROUTINE test_setup
+
+SUBROUTINE test_teardown(ret)
+  use pio_tutil
+  use ncdf_inq_tests_tgv
+  implicit none
+
+  integer, intent(out) :: ret
+
+  ret = PIO_NOERR
+  call PIO_deletefile(pio_tf_iosystem_, tgv_fname)  
+END SUBROUTINE test_teardown
+
+SUBROUTINE test_inq_var(pio_file, ret)
+  use pio_tutil
+  use ncdf_inq_tests_tgv
+  implicit none
+
+  type(file_desc_t), intent(in) :: pio_file
+  integer, intent(inout) :: ret
+
+  type(var_desc_t)  :: pio_var
+  integer :: var_id, var_type, var_ndims, var_natts
+  integer, dimension(:), allocatable :: var_dims
+  character(len=pio_max_name) :: var_name
+
+  ret = pio_inq_varid(pio_file, tgv_var_name, var_id)
+  PIO_TF_CHECK_ERR(ret, "Failed to inquire varid :"//trim(tgv_fname))
+
+  ret = pio_inq_varname(pio_file, var_id, var_name)
+  PIO_TF_PASSERT(var_name .eq. tgv_var_name, "Variable name is not the expected value")
+
+  ret = pio_inq_vartype(pio_file, var_id, var_type)
+  PIO_TF_PASSERT(var_type == PIO_real, "Variable tye is not the expected type")
+
+  ret = pio_inq_varndims(pio_file, var_id, var_ndims)
+  PIO_TF_PASSERT(var_ndims == tgv_var_ndims, "Num of dims for variable is not expected")
+
+  allocate(var_dims(var_ndims))
+  ret = pio_inq_vardimid(pio_file, var_id, var_dims)
+  PIO_TF_CHECK_ERR(ret, "Failed to get dim ids:"//trim(tgv_fname))
+  deallocate(var_dims)
+
+  ret = pio_inq_varnatts(pio_file, var_id, var_natts)
+  PIO_TF_PASSERT(var_natts == tgv_var_natts, "Num of atts for variable is not expected")
+
+END SUBROUTINE test_inq_var
+
+SUBROUTINE test_inq_dim(pio_file, ret)
+  use pio_tutil
+  use ncdf_inq_tests_tgv
+  implicit none
+
+  type(file_desc_t), intent(in) :: pio_file
+  integer, intent(inout) :: ret
+
+  integer :: dim_id
+  character(len=pio_max_name) :: dim_name
+  integer(kind=pio_offset_kind) :: dim_len
+
+  ret = pio_inq_dimid(pio_file, tgv_dim_name, dim_id)
+  PIO_TF_CHECK_ERR(ret, "Failed to inq dimid :"//trim(tgv_fname))
+
+  ret = pio_inq_dimname(pio_file, dim_id, dim_name)
+  PIO_TF_PASSERT(dim_name .eq. tgv_dim_name, "Dim name is not the expected value") 
+
+  ret = pio_inq_dimlen(pio_file, dim_id, dim_len)
+  PIO_TF_PASSERT(dim_len == TGV_DIM_LEN, "Dim length is not the expected value")
+
+END SUBROUTINE test_inq_dim
+
+SUBROUTINE test_inq_att(pio_file, ret)
+  use pio_tutil
+  use ncdf_inq_tests_tgv
+  implicit none
+
+  type(file_desc_t), intent(inout) :: pio_file
+  integer, intent(inout) :: ret
+
+  integer :: var_id
+  integer(kind=pio_offset_kind) :: att_len
+
+  ret = pio_inq_varid(pio_file, tgv_var_name, var_id)
+  PIO_TF_CHECK_ERR(ret, "Failed to inquire varid :"//trim(tgv_fname))
+
+  ret = pio_inq_attlen(pio_file, var_id, tgv_iatt_name, att_len)
+  PIO_TF_CHECK_ERR(ret, "Failed to inquire att len :"//trim(tgv_fname))
+  !PIO_TF_PASSERT(att_len == tgv_iatt_len, "Att length is not expected value")
+
+  ret = pio_inq_attlen(pio_file, var_id, tgv_catt_name, att_len)
+  PIO_TF_CHECK_ERR(ret, "Failed to inquire att len :"//trim(tgv_fname))
+  PIO_TF_PASSERT(att_len == len(trim(tgv_catt_val)), "Attribute length is not expected value")
+
+END SUBROUTINE test_inq_att
+
+PIO_TF_AUTO_TEST_SUB_BEGIN test_inq
+  use ncdf_inq_tests_tgv
+  Implicit none
+  type(file_desc_t) :: pio_file
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: i, ret
+  
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing type :", iotype_descs(i)
+    tgv_iotype = iotypes(i)
+
+    call test_setup(ret)
+    PIO_TF_CHECK_ERR(ret, "Test setup failed")
+
+    ret = PIO_openfile(pio_tf_iosystem_, pio_file, tgv_iotype, tgv_fname, PIO_nowrite)
+    PIO_TF_CHECK_ERR(ret, "Failed to open: "// trim(tgv_fname))
+
+    call test_inq_var(pio_file, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to inq var:" // trim(tgv_fname))
+
+    call test_inq_dim(pio_file, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to inq dim:" // trim(tgv_fname))
+
+    call test_inq_att(pio_file, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to inq att:" // trim(tgv_fname))
+
+    call PIO_closefile(pio_file)
+
+    call test_teardown(ret)
+    PIO_TF_CHECK_ERR(ret, "Test teardown failed")
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END test_inq
diff --git a/externals/pio2/tests/general/ncdf_simple_tests.F90.in b/src/externals/pio2/tests/general/ncdf_simple_tests.F90.in
similarity index 100%
rename from externals/pio2/tests/general/ncdf_simple_tests.F90.in
rename to src/externals/pio2/tests/general/ncdf_simple_tests.F90.in
diff --git a/externals/pio2/tests/general/pio_decomp_fillval.F90.in b/src/externals/pio2/tests/general/pio_decomp_fillval.F90.in
similarity index 100%
rename from externals/pio2/tests/general/pio_decomp_fillval.F90.in
rename to src/externals/pio2/tests/general/pio_decomp_fillval.F90.in
diff --git a/externals/pio2/tests/general/pio_decomp_fillval2.F90 b/src/externals/pio2/tests/general/pio_decomp_fillval2.F90
similarity index 100%
rename from externals/pio2/tests/general/pio_decomp_fillval2.F90
rename to src/externals/pio2/tests/general/pio_decomp_fillval2.F90
diff --git a/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in b/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in
new file mode 100644
index 00000000000..e75fdce61de
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_decomp_frame_tests.F90.in
@@ -0,0 +1,420 @@
+! Get a 3D column decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_HGT_SZ blocks of 
+! (VEC_COL_SZ rows x VEC_ROW_SZ columns) elements
+! # All odd procs have  VEC_HGT_SZ blocks of
+! (VEC_COL_SZ rows x VEC_ROW_SZ + 1 columns) elements
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2, VEC_HGT_SZ = 2
+! and ranks 0, 1, 2,
+! e.g. 1) |(1,1,1) (1,2,1) (2,1,1) (2,2,1)|
+!         |(1,1,2) (1,2,2) (2,1,2) (2,2,2)| ,
+!         |(1,3,1) (1,4,1) (1,5,1) (2,3,1) (2,4,1) (2,5,1)|
+!         |(1,3,2) (1,4,2) (1,5,2) (2,3,2) (2,4,2) (2,5,2)|,
+!         |(1,6,1) (1,7,1) (2,6,1) (2,7,1)|
+!         |(1,6,2) (1,7,2) (2,6,2) (2,7,2)|
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2, VEC_HGT_SZ = 2
+! and ranks 0, 1, 2,
+! e.g. 1  |(1,3,1) (1,4,1) (1,5,1) (2,3,1) (2,4,1) (2,5,1)|
+!         |(1,3,2) (1,4,2) (1,5,2) (2,3,2) (2,4,2) (2,5,2)|,
+!         |(1,1,1) (1,2,1) (2,1,1) (2,2,1)|
+!         |(1,1,2) (1,2,2) (2,1,2) (2,2,2)| ,
+!         |(1,6,1) (1,7,1) (2,6,1) (2,7,1)|
+!         |(1,6,2) (1,7,2) (2,6,2) (2,7,2)|
+! This for example can be used to force rearrangement when reading
+! or writing data.
+SUBROUTINE get_3d_col_decomp_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_ROW_SZ = 2
+  integer, parameter :: VEC_COL_SZ = 2
+  integer, parameter :: VEC_HGT_SZ = 2
+  integer, parameter :: NDIMS = 3
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(NDIMS), intent(out) :: dims
+  integer, dimension(NDIMS), intent(out) :: start
+  integer, dimension(NDIMS), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = VEC_COL_SZ
+  dims(2) = num_even_procs * VEC_ROW_SZ + num_odd_procs * (VEC_ROW_SZ + 1)
+  dims(3) = VEC_HGT_SZ
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+
+  ! Rows
+  start(1) = 1
+  count(1) = VEC_COL_SZ
+
+  ! Columns
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(2) = VEC_ROW_SZ + 1
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + (VEC_ROW_SZ) + 1
+      else
+        count(2) = VEC_ROW_SZ
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(2) = VEC_ROW_SZ
+      start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) - (VEC_ROW_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(2) = VEC_ROW_SZ
+    else
+      count(2) = VEC_ROW_SZ + 1
+    end if
+    start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+  end if
+
+  ! Height
+  start(3) = 1
+  count(3) = VEC_HGT_SZ
+END SUBROUTINE
+
+! Write with one decomp (to force rearrangement) and read with another (no
+! rearrangement)
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_4d_col_decomp
+  implicit none
+  integer, parameter :: NDIMS = 4
+  integer, parameter :: NFRAMES = 3
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(NDIMS) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:,:,:,:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(NDIMS-1) :: dims
+  integer, dimension(NDIMS) :: pio_dims
+  integer :: i, j, k, tmp_idx, ierr, lsz, nrows, ncols, nhgts
+  integer(kind=pio_offset_kind) :: f
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - forcing rearrangement
+  call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  nrows = count(1)
+  ncols = count(2)
+  nhgts = count(3)
+  
+  allocate(wbuf(nrows, ncols, nhgts, NFRAMES))
+  allocate(compdof(nrows * ncols * nhgts))
+  do f=1,NFRAMES
+    do k=1,nhgts
+      do j=1,ncols
+        do i=1,nrows
+          wbuf(i,j,k,f) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                        (start(2) - 1 + j - 1) * dims(1) + i
+          wbuf(i,j,k,f) = wbuf(i,j,k,f) + (f - 1) * (dims(1) * dims(2) * dims(3))
+          tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i
+          compdof(tmp_idx) = wbuf(i,j,k,1)
+        end do
+      end do
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  ! Set the decomposition for reading data - different from the write decomp
+  call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.)
+  nrows = count(1)
+  ncols = count(2)
+  nhgts = count(3)
+  
+  allocate(rbuf(nrows, ncols, nhgts, NFRAMES))
+  allocate(compdof(nrows * ncols * nhgts))
+  allocate(exp_val(nrows, ncols, nhgts, NFRAMES))
+
+  do f=1,NFRAMES
+    do k=1,nhgts
+      do j=1,ncols
+        do i=1,nrows
+          tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i
+          compdof(tmp_idx) =  (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                              (start(2) - 1 + j - 1) * dims(1) + i
+          exp_val(i,j,k,f) = compdof(tmp_idx) + (f - 1) * (dims(1) * dims(2) * dims(3))
+        end do
+      end do
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_hgt', dims(3), pio_dims(3))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_time', pio_unlimited, pio_dims(4))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    do f=1,NFRAMES
+      call PIO_setframe(pio_file, pio_var, f)
+      ! Write the current frame
+      call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf(:,:,:,f), ierr)
+      PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+    end do
+
+    call PIO_syncfile(pio_file)
+
+    do f=1,NFRAMES
+      call PIO_setframe(pio_file, pio_var, f)
+      call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf(:,:,:,f), ierr)
+      PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+    end do
+
+    do f=1,NFRAMES
+      PIO_TF_CHECK_VAL((rbuf(:,:,:,f), exp_val(:,:,:,f)), "Got wrong val, frame=", f)
+    end do
+
+    call PIO_closefile(pio_file)
+    
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_write_read_4d_col_decomp
+
+! Using a 3d decomp for writing out a 3d and a 4d var
+! Write with one decomp (to force rearrangement) and read with another (no
+! rearrangement)
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_reuse_3d_decomp
+  implicit none
+  integer, parameter :: NDIMS = 4
+  integer, parameter :: NFRAMES = 3
+  type(var_desc_t)  :: pio_var3d, pio_var4d
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(NDIMS) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:,:,:,:), allocatable :: rbuf4d, wbuf4d, exp_val4d
+  PIO_TF_FC_DATA_TYPE, dimension(:,:,:), allocatable :: rbuf3d, wbuf3d, exp_val3d
+  integer, dimension(NDIMS-1) :: dims
+  integer, dimension(NDIMS) :: pio_dims
+  integer :: i, j, k, tmp_idx, ierr, lsz, nrows, ncols, nhgts
+  integer(kind=pio_offset_kind) :: f
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - forcing rearrangement
+  call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  nrows = count(1)
+  ncols = count(2)
+  nhgts = count(3)
+  
+  ! Initialize the 4d var
+  allocate(wbuf4d(nrows, ncols, nhgts, NFRAMES))
+  do f=1,NFRAMES
+    do k=1,nhgts
+      do j=1,ncols
+        do i=1,nrows
+          wbuf4d(i,j,k,f) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                        (start(2) - 1 + j - 1) * dims(1) + i
+          wbuf4d(i,j,k,f) = wbuf4d(i,j,k,f) + (f - 1) * (dims(1) * dims(2) * dims(3))
+        end do
+      end do
+    end do
+  end do
+  allocate(compdof(nrows * ncols * nhgts))
+  do k=1,nhgts
+    do j=1,ncols
+      do i=1,nrows
+        tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i
+        compdof(tmp_idx) = wbuf4d(i,j,k,1)
+      end do
+    end do
+  end do
+  ! Initialize the 3d var
+  allocate(wbuf3d(nrows, ncols, nhgts)) 
+  do k=1,nhgts
+    do j=1,ncols
+      do i=1,nrows
+        wbuf3d(i,j,k) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                      (start(2) - 1 + j - 1) * dims(1) + i
+      end do
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  ! Set the decomposition for reading data - different from the write decomp
+  call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.)
+  nrows = count(1)
+  ncols = count(2)
+  nhgts = count(3)
+  
+  allocate(rbuf4d(nrows, ncols, nhgts, NFRAMES))
+  rbuf4d = 0
+  ! Expected val for 4d var
+  allocate(exp_val4d(nrows, ncols, nhgts, NFRAMES))
+  do f=1,NFRAMES
+    do k=1,nhgts
+      do j=1,ncols
+        do i=1,nrows
+          exp_val4d(i,j,k,f) =  (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                              (start(2) - 1 + j - 1) * dims(1) + i
+          exp_val4d(i,j,k,f) = exp_val4d(i,j,k,f)+(f - 1) * (dims(1) * dims(2) * dims(3))
+        end do
+      end do
+    end do
+  end do
+  allocate(compdof(nrows * ncols * nhgts))
+  do k=1,nhgts
+    do j=1,ncols
+      do i=1,nrows
+        tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i
+        compdof(tmp_idx) = exp_val4d(i,j,k,1)
+      end do
+    end do
+  end do
+
+  allocate(rbuf3d(nrows, ncols, nhgts))
+  rbuf3d = 0
+  ! Expected val for 3d var
+  allocate(exp_val3d(nrows, ncols, nhgts))
+  do k=1,nhgts
+    do j=1,ncols
+      do i=1,nrows
+        exp_val3d(i,j,k) =  (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                            (start(2) - 1 + j - 1) * dims(1) + i
+      end do
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_hgt', dims(3), pio_dims(3))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_time', pio_unlimited, pio_dims(4))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_3d_var', PIO_TF_DATA_TYPE, pio_dims(1:3), pio_var3d)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a 3d var : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_4d_var', PIO_TF_DATA_TYPE, pio_dims, pio_var4d)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a 4d var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    call PIO_write_darray(pio_file, pio_var3d, wr_iodesc, wbuf3d, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write 3d darray : " // trim(filename))
+
+    do f=1,NFRAMES
+      call PIO_setframe(pio_file, pio_var4d, f)
+      ! Write the current frame
+      call PIO_write_darray(pio_file, pio_var4d, wr_iodesc, wbuf4d(:,:,:,f), ierr)
+      PIO_TF_CHECK_ERR(ierr, "Failed to write 4d darray : " // trim(filename))
+    end do
+    call PIO_syncfile(pio_file)
+
+    rbuf4d = 0
+    rbuf3d = 0
+
+    call PIO_read_darray(pio_file, pio_var3d, rd_iodesc, rbuf3d, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read 3d darray : " // trim(filename))
+
+    do f=1,NFRAMES
+      call PIO_setframe(pio_file, pio_var4d, f)
+      call PIO_read_darray(pio_file, pio_var4d, rd_iodesc, rbuf4d(:,:,:,f), ierr)
+      PIO_TF_CHECK_ERR(ierr, "Failed to read 4d darray : " // trim(filename))
+    end do
+
+    do f=1,NFRAMES
+      PIO_TF_CHECK_VAL((rbuf4d(:,:,:,f), exp_val4d(:,:,:,f)), "Got wrong 4d val, frame=", f)
+    end do
+    PIO_TF_CHECK_VAL((rbuf3d, exp_val3d), "Got wrong 3dd val")
+
+    call PIO_closefile(pio_file)
+    
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+
+  deallocate(exp_val3d)
+  deallocate(rbuf3d)
+  deallocate(wbuf3d)
+
+  deallocate(exp_val4d)
+  deallocate(rbuf4d)
+  deallocate(wbuf4d)
+PIO_TF_AUTO_TEST_SUB_END nc_reuse_3d_decomp
diff --git a/src/externals/pio2/tests/general/pio_decomp_tests.F90.in b/src/externals/pio2/tests/general/pio_decomp_tests.F90.in
new file mode 100644
index 00000000000..40476f4740a
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_decomp_tests.F90.in
@@ -0,0 +1,305 @@
+PIO_TF_AUTO_TEST_SUB_BEGIN init_decomp_1d_get_loc_sz
+  implicit none
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  integer, dimension(:), allocatable :: data_types
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: data_type_descs
+  type(io_desc_t) :: iodesc
+  integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps
+  integer, dimension(1) :: dims
+  integer :: i, ntypes, lsz
+
+  do i=1,VEC_LOCAL_SZ
+    compdof_rel_disps(i) = i
+  end do
+  dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_
+  compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps
+
+  call PIO_TF_Get_data_types(data_types, data_type_descs, ntypes)
+  do i=1,ntypes
+    PIO_TF_LOG(0,*) "Testing type : ", data_type_descs(i)
+    call PIO_initdecomp(pio_tf_iosystem_, data_types(i), dims, compdof, iodesc)
+    lsz = PIO_get_local_array_size(iodesc)
+    PIO_TF_PASSERT(lsz == VEC_LOCAL_SZ, "Checking the local array size")
+    call PIO_freedecomp(pio_tf_iosystem_, iodesc)
+  end do
+
+  if(allocated(data_types)) then
+    deallocate(data_types)
+    deallocate(data_type_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END init_decomp_1d_get_loc_sz
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_1d_darray
+  implicit none
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: iodesc
+  integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps
+  PIO_TF_FC_DATA_TYPE, dimension(VEC_LOCAL_SZ) :: buf
+  integer, dimension(1) :: dims
+  integer :: pio_dim
+  integer :: i, ierr, lsz
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  do i=1,VEC_LOCAL_SZ
+    compdof_rel_disps(i) = i
+  end do
+  dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_
+  compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps
+  buf = pio_tf_world_rank_;
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, iodesc, buf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    ! FIXME: Verify the written output
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, iodesc)
+PIO_TF_AUTO_TEST_SUB_END nc_write_1d_darray
+
+! Write 1d array, although diff procs have different
+! number of elements to write locally they all use
+! the same buffer size (compdof size is different for
+! each rank but buf size is the same)
+! Odd procs write out VEC_LOCAL_SZ_ODD elements &
+! even procs write out VEC_LOCAL_SZ_EVEN elements, but
+! all procs use buf[MAX_VEC_SZ]
+! eg:
+! Elements in buffer on each proc with MAX_VEC_SZ = 2,
+! [0 1] [2 X] [3 4] [5 X] ...
+! The 'X'es in the buffer are not written out
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_const_buf_sz
+  implicit none
+  integer, parameter :: MAX_VEC_SZ = 2
+  integer, parameter :: VEC_LOCAL_SZ_ODD = MAX_VEC_SZ - 1
+  integer, parameter :: VEC_LOCAL_SZ_EVEN = MAX_VEC_SZ
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: iodesc
+  integer, dimension(:), allocatable :: compdof, compdof_rel_disps
+  integer :: compdof_rel_start
+  integer :: cdof_sz = VEC_LOCAL_SZ_ODD
+  PIO_TF_FC_DATA_TYPE, dimension(MAX_VEC_SZ) :: wbuf, rbuf
+  integer, dimension(1) :: dims
+  integer :: pio_dim
+  integer :: i, ierr, lsz
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  logical :: is_even = .false.
+  integer :: nodd_procs, nodd_procs_bfr, neven_procs, neven_procs_bfr
+
+  nodd_procs = pio_tf_world_sz_ / 2
+  ! Number of odd procs before this rank
+  nodd_procs_bfr = pio_tf_world_rank_ / 2
+  neven_procs = pio_tf_world_sz_ - nodd_procs
+  ! Number of even procs before this rank
+  neven_procs_bfr = pio_tf_world_rank_ - nodd_procs_bfr
+
+  ! Odd procs write out VEC_LOCAL_SZ_ODD elements &
+  ! even procs write out VEC_LOCAL_SZ_EVEN elements
+  if(mod(pio_tf_world_rank_, 2) == 0) then
+    is_even = .true.
+    cdof_sz = VEC_LOCAL_SZ_EVEN
+  end if
+  allocate(compdof(cdof_sz))
+  allocate(compdof_rel_disps(cdof_sz))
+  wbuf = 0
+  rbuf = 0
+  do i=1,cdof_sz
+    compdof_rel_disps(i) = i
+    wbuf(i) = i
+  end do
+  ! Find out where compdof starts for this rank
+  compdof_rel_start = nodd_procs_bfr * VEC_LOCAL_SZ_ODD +&
+                      neven_procs_bfr * VEC_LOCAL_SZ_EVEN
+  dims(1) = nodd_procs * VEC_LOCAL_SZ_ODD + neven_procs * VEC_LOCAL_SZ_EVEN
+  compdof = compdof_rel_start + compdof_rel_disps
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc)
+
+  deallocate(compdof)
+  deallocate(compdof_rel_disps)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val")
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+  call PIO_freedecomp(pio_tf_iosystem_, iodesc)
+PIO_TF_AUTO_TEST_SUB_END nc_wr_1d_const_buf_sz
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_1d_reuse_decomp
+  implicit none
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  type(var_desc_t)  :: pio_var1_file1, pio_var2_file1, pio_var1_file2
+  type(file_desc_t) :: pio_file1, pio_file2
+  character(len=PIO_TF_MAX_STR_LEN) :: filename1, filename2
+  type(io_desc_t) :: iodesc
+  integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps
+  PIO_TF_FC_DATA_TYPE, dimension(VEC_LOCAL_SZ) :: buf, rbuf
+  integer, dimension(1) :: dims
+  integer :: pio_dim_file1, pio_dim_file2
+  integer :: i, ierr, lsz
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  do i=1,VEC_LOCAL_SZ
+    compdof_rel_disps(i) = i
+  end do
+  dims(1) = VEC_LOCAL_SZ * pio_tf_world_sz_
+  compdof = VEC_LOCAL_SZ * pio_tf_world_rank_ + compdof_rel_disps
+  buf = pio_tf_world_rank_;
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, iodesc)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename1 = "test_pio_decomp_simple_tests.testfile1"
+  filename2 = "test_pio_decomp_simple_tests.testfile2"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file1, iotypes(i),&
+                            filename1, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename1))
+
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file2, iotypes(i),&
+                            filename2, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename1))
+
+    ierr = PIO_def_dim(pio_file1, 'PIO_TF_test_dim', dims(1), pio_dim_file1)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename1))
+
+    ierr = PIO_def_dim(pio_file2, 'PIO_TF_test_dim', dims(1), pio_dim_file2)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename2))
+
+    ierr = PIO_def_var(pio_file1, 'PIO_TF_test_var1', PIO_TF_DATA_TYPE,&
+                          (/pio_dim_file1/), pio_var1_file1)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define first var : " // trim(filename1))
+
+    ierr = PIO_def_var(pio_file1, 'PIO_TF_test_var2', PIO_TF_DATA_TYPE,&
+                          (/pio_dim_file1/), pio_var2_file1)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define second var : " // trim(filename1))
+
+    ierr = PIO_def_var(pio_file2, 'PIO_TF_test_var1', PIO_TF_DATA_TYPE,&
+                          (/pio_dim_file2/), pio_var1_file2)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define first var : " // trim(filename2))
+
+    ierr = PIO_enddef(pio_file1)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename1))
+
+    ierr = PIO_enddef(pio_file2)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename2))
+
+    ! Write the variables - file1
+    call PIO_write_darray(pio_file1, pio_var1_file1, iodesc, buf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write first darray : " // trim(filename1))
+
+    call PIO_write_darray(pio_file1, pio_var2_file1, iodesc, buf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write second darray : " // trim(filename1))
+
+    ! Write the variables - file2
+    call PIO_write_darray(pio_file2, pio_var1_file2, iodesc, buf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write first darray : " // trim(filename2))
+
+    call PIO_syncfile(pio_file1)
+    rbuf = 0
+    call PIO_read_darray(pio_file1, pio_var1_file1, iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read first darray : " // trim(filename1))
+
+    PIO_TF_CHECK_VAL((rbuf, buf), "Got wrong val")
+
+    rbuf = 0
+    call PIO_read_darray(pio_file1, pio_var2_file1, iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read second darray : " // trim(filename1))
+
+    PIO_TF_CHECK_VAL((rbuf, buf), "Got wrong val")
+
+    call PIO_syncfile(pio_file2)
+    rbuf = 0
+    call PIO_read_darray(pio_file2, pio_var1_file2, iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read first darray : " // trim(filename2))
+
+    PIO_TF_CHECK_VAL((rbuf, buf), "Got wrong val")
+
+    call PIO_closefile(pio_file1)
+    call PIO_closefile(pio_file2)
+    call PIO_deletefile(pio_tf_iosystem_, filename1);
+    call PIO_deletefile(pio_tf_iosystem_, filename2);
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, iodesc)
+PIO_TF_AUTO_TEST_SUB_END nc_write_1d_reuse_decomp
diff --git a/src/externals/pio2/tests/general/pio_decomp_tests_1d.F90.in b/src/externals/pio2/tests/general/pio_decomp_tests_1d.F90.in
new file mode 100644
index 00000000000..48d4f95a101
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_decomp_tests_1d.F90.in
@@ -0,0 +1,306 @@
+! Get a block cyclic decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_LOCAL_SZ elements
+! # All odd procs have VEC_LOCAL_SZ + 1 elements
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [1,2] [3,4,5] [6,7]
+! e.g. 2)    [1,2] [3,4,5] [6,7] [8,9,10]
+! e.g. 3)    [1,2] [3,4,5] [6,7] [8,9,10] [11,12]
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [3,4,5] [1,2] [6,7]
+! e.g. 2)    [3,4,5] [1,2] [8,9,10] [6,7]
+! e.g. 3)    [3,4,5] [1,2] [8,9,10] [6,7] [11,12]
+SUBROUTINE get_1d_bc_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(1), intent(out) :: dims
+  integer, dimension(1), intent(out) :: start
+  integer, dimension(1), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = num_even_procs * VEC_LOCAL_SZ + num_odd_procs * (VEC_LOCAL_SZ + 1)
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(1) = VEC_LOCAL_SZ + 1
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + (VEC_LOCAL_SZ) + 1
+      else
+        count(1) = VEC_LOCAL_SZ
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(1) = VEC_LOCAL_SZ
+      start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) - (VEC_LOCAL_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(1) = VEC_LOCAL_SZ
+    else
+      count(1) = VEC_LOCAL_SZ + 1
+    end if
+    start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+  end if
+
+END SUBROUTINE
+
+! Get a 1d block decomposition with holes
+! If has_hole is TRUE, the decomposition is such that
+! # All even procs have VEC_LOCAL_SZ * 2 elements
+! # (the even procs take all elems from the odd procs)
+! # All odd procs have 0 elements
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [1,2,3,4] [] [5,6]
+! e.g. 2)    [1,2,3,4] [] [5,6,7,8] []
+! If has_hole is FALSE, the data is evenly divided among all procs
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [1,2] [3,4] [5,6]
+! e.g. 2)    [1,2] [3,4] [5,6] [7,8]
+SUBROUTINE get_1d_bc_with_holes_info(rank, sz, dims, start, count, has_hole)
+  integer, parameter :: VEC_LOCAL_SZ = 7
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(1), intent(out) :: dims
+  integer, dimension(1), intent(out) :: start
+  integer, dimension(1), intent(out) :: count
+  logical, intent(in) :: has_hole
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = VEC_LOCAL_SZ * sz
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+  if(has_hole) then
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        count(1) = VEC_LOCAL_SZ * 2
+      else
+        count(1) = VEC_LOCAL_SZ
+      end if
+      start(1) = ieven * VEC_LOCAL_SZ * 2 + 1
+    else
+      count(1) = 0
+      start(1) = 0
+    end if
+  else
+    count(1) = VEC_LOCAL_SZ
+    start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ) + 1
+  end if
+
+END SUBROUTINE
+
+! Test block cyclic interface
+! Write with one decomp and read with another
+! Test all combs 
+! - no rearrage read + no rearrange write
+! - rearrage read + no rearrange write
+! - no rearrage read + rearrange write
+! - rearrage read + rearrange write
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_rd_1d_bc
+  implicit none
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(1) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(1) :: dims
+  integer :: pio_dim
+  integer :: i, ierr, lsz
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  logical, dimension(2) :: enable_rd_rearr = (/.false., .true./)
+  integer :: rd_rearr_opt_idx
+  logical, dimension(2) :: enable_wr_rearr = (/.false., .true./)
+  integer :: wr_rearr_opt_idx
+
+  do rd_rearr_opt_idx=1,size(enable_rd_rearr)
+    do wr_rearr_opt_idx=1,size(enable_wr_rearr)
+      PIO_TF_LOG(0, *) "Testing Rd rearr =", enable_rd_rearr(rd_rearr_opt_idx), ",Write rearr=", enable_wr_rearr(wr_rearr_opt_idx)
+      ! Set the decomposition for writing data - forcing rearrangement
+      call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims,&
+             start, count, enable_wr_rearr(wr_rearr_opt_idx))
+      allocate(wbuf(count(1)))
+      allocate(compdof(count(1)))
+      do i=1,count(1)
+        wbuf(i) = start(1) + i - 1
+        compdof(i) = start(1) + i - 1
+      end do
+
+      call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+      deallocate(compdof)
+
+      ! Set the decomposition for reading data - different from the write decomp
+      call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims,&
+             start, count, enable_rd_rearr(rd_rearr_opt_idx))
+      allocate(rbuf(count(1)))
+      allocate(compdof(count(1)))
+      allocate(exp_val(count(1)))
+      do i=1,count(1)
+        compdof(i) = start(1) + i -1
+        ! Expected value, after reading, is the same as the compdof
+        exp_val(i) = compdof(i)
+      end do
+
+      call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+      deallocate(compdof)
+
+      num_iotypes = 0
+      call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+      filename = "test_pio_decomp_simple_tests.testfile"
+      do i=1,num_iotypes
+        PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+        ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+        PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+        ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
+        PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+        ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+        PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+        ierr = PIO_enddef(pio_file)
+        PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+        ! Write the variable out
+        call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+        PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+        call PIO_syncfile(pio_file)
+
+        rbuf = 0
+        call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr)
+        PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+        PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+
+        call PIO_closefile(pio_file)
+        
+        call PIO_deletefile(pio_tf_iosystem_, filename);
+      end do
+
+      if(allocated(iotypes)) then
+        deallocate(iotypes)
+        deallocate(iotype_descs)
+      end if
+
+      call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+      call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+      deallocate(exp_val)
+      deallocate(rbuf)
+      deallocate(wbuf)
+    end do !wr_rearr_opt_idx=1,size(enable_wr_rearr)
+  end do !rd_rearr_opt_idx=1,size(enable_rd_rearr)
+PIO_TF_AUTO_TEST_SUB_END nc_wr_rd_1d_bc
+
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_wr_1d_bc_with_holes
+  implicit none
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(1) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(1) :: dims
+  integer :: pio_dim
+  integer :: i, ierr, lsz
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - has holes
+  call get_1d_bc_with_holes_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  allocate(wbuf(count(1)))
+  allocate(rbuf(count(1)))
+  allocate(compdof(count(1)))
+  allocate(exp_val(count(1)))
+  do i=1,count(1)
+    wbuf(i) = start(1) + i - 1
+    compdof(i) = wbuf(i)
+    exp_val(i) = wbuf(i)
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim', dims(1), pio_dim)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, wr_iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+
+    call PIO_closefile(pio_file)
+    
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_wr_1d_bc_with_holes
diff --git a/src/externals/pio2/tests/general/pio_decomp_tests_2d.F90.in b/src/externals/pio2/tests/general/pio_decomp_tests_2d.F90.in
new file mode 100644
index 00000000000..27dafcdfb0b
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_decomp_tests_2d.F90.in
@@ -0,0 +1,367 @@
+! Get a 2D column decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_COL_SZ rows of VEC_ROW_SZ elements
+! # All odd procs have  VEC_COL_SZ rows of VEC_ROW_SZ + 1 elements
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2 and ranks 0, 1, 2,
+! e.g. 1) |(1,1) (1,2)|  |(1,3) (1,4) (1,5)|  |(1,6) (1,7)|
+!         |(2,1) (2,2)|, |(2,3) (2,4) (2,5)|, |(2,6) (2,7)|
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2 and ranks 0, 1, 2
+! e.g. 1) |(1,3) (1,4) (1,5)|  |(1,1) (1,2)|   |(1,6) (1,7)|
+!         |(2,3) (2,4) (2,5)|, |(2,1) (2,2)|,  |(2,6) (2,7)|
+SUBROUTINE get_2d_col_decomp_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_ROW_SZ = 7
+  integer, parameter :: VEC_COL_SZ = 7
+  integer, parameter :: NDIMS = 2
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(NDIMS), intent(out) :: dims
+  integer, dimension(NDIMS), intent(out) :: start
+  integer, dimension(NDIMS), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = VEC_COL_SZ
+  dims(2) = num_even_procs * VEC_ROW_SZ + num_odd_procs * (VEC_ROW_SZ + 1)
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+
+  ! Rows
+  start(1) = 1
+  count(1) = VEC_COL_SZ
+
+  ! Columns
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(2) = VEC_ROW_SZ + 1
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + (VEC_ROW_SZ) + 1
+      else
+        count(2) = VEC_ROW_SZ
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(2) = VEC_ROW_SZ
+      start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) - (VEC_ROW_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(2) = VEC_ROW_SZ
+    else
+      count(2) = VEC_ROW_SZ + 1
+    end if
+    start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+  end if
+
+END SUBROUTINE
+
+! Get a 2D row decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_COL_SZ rows of VEC_ROW_SZ elements
+! # All odd procs have  VEC_COL_SZ+1 rows of VEC_ROW_SZ elements
+! e.g. For VEC_ROW_SZ = 6, VEC_COL_SZ = 1 and ranks 0, 1, 2,
+! e.g. 1) |(1,1) (1,2) (1,3) (1,4) (1,5) (1,6)|,
+!         |(2,1) (2,2) (2,3) (2,4) (2,5) (2,6)|
+!         |(3,1) (3,2) (3,3) (3,4) (3,5) (3,6)|,
+!         |(4,1) (4,2) (4,3) (4,4) (4,5) (4,6)|
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements (rows) with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_ROW_SZ = 6, VEC_COL_SZ = 1 and ranks 0, 1, 2
+! e.g. 1) |(2,1) (2,2) (2,3) (2,4) (2,5) (2,6)|
+!         |(3,1) (3,2) (3,3) (3,4) (3,5) (3,6)|,
+!         |(1,1) (1,2) (1,3) (1,4) (1,5) (1,6)|,
+!         |(4,1) (4,2) (4,3) (4,4) (4,5) (4,6)|
+SUBROUTINE get_2d_row_decomp_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_COL_SZ = 7
+  integer, parameter :: VEC_ROW_SZ = 7
+  integer, parameter :: NDIMS = 2
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(NDIMS), intent(out) :: dims
+  integer, dimension(NDIMS), intent(out) :: start
+  integer, dimension(NDIMS), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = num_even_procs * VEC_COL_SZ + num_odd_procs * (VEC_COL_SZ + 1)
+  dims(2) = VEC_ROW_SZ
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+
+  ! Rows
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(1) = VEC_COL_SZ + 1
+        start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + (VEC_COL_SZ) + 1
+      else
+        count(1) = VEC_COL_SZ
+        start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(1) = VEC_COL_SZ
+      start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) - (VEC_COL_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(1) = VEC_COL_SZ
+    else
+      count(1) = VEC_COL_SZ + 1
+    end if
+    start(1) = ieven * VEC_COL_SZ + iodd * (VEC_COL_SZ + 1) + 1
+  end if
+
+  ! Columns
+  start(2) = 1
+  count(2) = VEC_ROW_SZ
+
+END SUBROUTINE
+
+! Write with one decomp (to force rearrangement) and read with another (no
+! rearrangement)
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_2d_col_decomp
+  implicit none
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  integer, parameter :: NDIMS = 2
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(NDIMS) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(NDIMS) :: dims
+  integer, dimension(NDIMS) :: pio_dims
+  integer :: i, j, tmp_idx, ierr, lsz, nrows, ncols
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - forcing rearrangement
+  call get_2d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  nrows = count(1)
+  ncols = count(2)
+  
+  allocate(wbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  do j=1,ncols
+    do i=1,nrows
+      wbuf(i,j) = (start(2) - 1 + j - 1) * nrows + i
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = wbuf(i,j)
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  ! Set the decomposition for reading data - different from the write decomp
+  call get_2d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.)
+  nrows = count(1)
+  ncols = count(2)
+  
+  allocate(rbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  allocate(exp_val(nrows, ncols))
+  do j=1,ncols
+    do i=1,nrows
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = (start(2) - 1 + j - 1) * nrows + i
+      ! Expected value, after reading, is the same as the compdof
+      exp_val(i,j) = compdof(tmp_idx)
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+
+    call PIO_closefile(pio_file)
+    
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_write_read_2d_col_decomp
+
+! Write with one decomp (to force rearrangement) and read with another (no
+! rearrangement)
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_2d_row_decomp
+  implicit none
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  integer, parameter :: NDIMS = 2
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(NDIMS) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:,:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(NDIMS) :: dims
+  integer, dimension(NDIMS) :: pio_dims
+  integer :: i, j, tmp_idx, ierr, lsz, nrows, ncols
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - forcing rearrangement
+  call get_2d_row_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  nrows = count(1)
+  ncols = count(2)
+
+  allocate(wbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  do j=1,ncols
+    do i=1,nrows
+      wbuf(i,j) = (start(2) - 1 + j - 1) * dims(1) + start(1) + i - 1
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = wbuf(i,j)
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  ! Set the decomposition for reading data - different from the write decomp
+  call get_2d_row_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.)
+  nrows = count(1)
+  ncols = count(2)
+  
+  allocate(rbuf(nrows, ncols))
+  allocate(compdof(nrows * ncols))
+  allocate(exp_val(nrows, ncols))
+  do j=1,ncols
+    do i=1,nrows
+      tmp_idx = (j - 1) * nrows + i
+      compdof(tmp_idx) = (start(2) - 1 + j - 1) * dims(1) + start(1) + i - 1
+      ! Expected value, after reading, is the same as the compdof
+      exp_val(i,j) = compdof(tmp_idx)
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+
+    call PIO_closefile(pio_file)
+    
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_write_read_2d_row_decomp
diff --git a/src/externals/pio2/tests/general/pio_decomp_tests_3d.F90.in b/src/externals/pio2/tests/general/pio_decomp_tests_3d.F90.in
new file mode 100644
index 00000000000..22cdacfe98a
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_decomp_tests_3d.F90.in
@@ -0,0 +1,209 @@
+! Get a 3D column decomposition
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_HGT_SZ blocks of 
+! (VEC_COL_SZ rows x VEC_ROW_SZ columns) elements
+! # All odd procs have  VEC_HGT_SZ blocks of
+! (VEC_COL_SZ rows x VEC_ROW_SZ + 1 columns) elements
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2, VEC_HGT_SZ = 2
+! and ranks 0, 1, 2,
+! e.g. 1) |(1,1,1) (1,2,1) (2,1,1) (2,2,1)|
+!         |(1,1,2) (1,2,2) (2,1,2) (2,2,2)| ,
+!         |(1,3,1) (1,4,1) (1,5,1) (2,3,1) (2,4,1) (2,5,1)|
+!         |(1,3,2) (1,4,2) (1,5,2) (2,3,2) (2,4,2) (2,5,2)|,
+!         |(1,6,1) (1,7,1) (2,6,1) (2,7,1)|
+!         |(1,6,2) (1,7,2) (2,6,2) (2,7,2)|
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! e.g. For VEC_ROW_SZ = 2, VEC_COL_SZ = 2, VEC_HGT_SZ = 2
+! and ranks 0, 1, 2,
+! e.g. 1  |(1,3,1) (1,4,1) (1,5,1) (2,3,1) (2,4,1) (2,5,1)|
+!         |(1,3,2) (1,4,2) (1,5,2) (2,3,2) (2,4,2) (2,5,2)|,
+!         |(1,1,1) (1,2,1) (2,1,1) (2,2,1)|
+!         |(1,1,2) (1,2,2) (2,1,2) (2,2,2)| ,
+!         |(1,6,1) (1,7,1) (2,6,1) (2,7,1)|
+!         |(1,6,2) (1,7,2) (2,6,2) (2,7,2)|
+! This for example can be used to force rearrangement when reading
+! or writing data.
+SUBROUTINE get_3d_col_decomp_info(rank, sz, dims, start, count, force_rearrange)
+  integer, parameter :: VEC_ROW_SZ = 2
+  integer, parameter :: VEC_COL_SZ = 2
+  integer, parameter :: VEC_HGT_SZ = 2
+  integer, parameter :: NDIMS = 3
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(NDIMS), intent(out) :: dims
+  integer, dimension(NDIMS), intent(out) :: start
+  integer, dimension(NDIMS), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = VEC_COL_SZ
+  dims(2) = num_even_procs * VEC_ROW_SZ + num_odd_procs * (VEC_ROW_SZ + 1)
+  dims(3) = VEC_HGT_SZ
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+
+  ! Rows
+  start(1) = 1
+  count(1) = VEC_COL_SZ
+
+  ! Columns
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(2) = VEC_ROW_SZ + 1
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + (VEC_ROW_SZ) + 1
+      else
+        count(2) = VEC_ROW_SZ
+        start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(2) = VEC_ROW_SZ
+      start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) - (VEC_ROW_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(2) = VEC_ROW_SZ
+    else
+      count(2) = VEC_ROW_SZ + 1
+    end if
+    start(2) = ieven * VEC_ROW_SZ + iodd * (VEC_ROW_SZ + 1) + 1
+  end if
+
+  ! Height
+  start(3) = 1
+  count(3) = VEC_HGT_SZ
+END SUBROUTINE
+
+! Write with one decomp (to force rearrangement) and read with another (no
+! rearrangement)
+PIO_TF_TEMPLATE
+PIO_TF_AUTO_TEST_SUB_BEGIN nc_write_read_3d_col_decomp
+  implicit none
+  type(var_desc_t)  :: pio_var
+  type(file_desc_t) :: pio_file
+  integer, parameter :: NDIMS = 3
+  character(len=PIO_TF_MAX_STR_LEN) :: filename
+  type(io_desc_t) :: wr_iodesc, rd_iodesc
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(NDIMS) :: start, count
+  PIO_TF_FC_DATA_TYPE, dimension(:,:,:), allocatable :: rbuf, wbuf, exp_val
+  integer, dimension(NDIMS) :: dims
+  integer, dimension(NDIMS) :: pio_dims
+  integer :: i, j, k, tmp_idx, ierr, lsz, nrows, ncols, nhgts
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+
+  ! Set the decomposition for writing data - forcing rearrangement
+  call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  nrows = count(1)
+  ncols = count(2)
+  nhgts = count(3)
+  
+  allocate(wbuf(nrows, ncols, nhgts))
+  allocate(compdof(nrows * ncols * nhgts))
+  do k=1,nhgts
+    do j=1,ncols
+      do i=1,nrows
+        wbuf(i,j,k) = (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                      (start(2) - 1 + j - 1) * dims(1) + i
+        tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i
+        compdof(tmp_idx) = wbuf(i,j,k)
+      end do
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, wr_iodesc)
+  deallocate(compdof)
+
+  ! Set the decomposition for reading data - different from the write decomp
+  call get_3d_col_decomp_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .false.)
+  nrows = count(1)
+  ncols = count(2)
+  nhgts = count(3)
+  
+  allocate(rbuf(nrows, ncols, nhgts))
+  allocate(compdof(nrows * ncols * nhgts))
+  allocate(exp_val(nrows, ncols, nhgts))
+
+  do k=1,nhgts
+    do j=1,ncols
+      do i=1,nrows
+        tmp_idx = (k - 1) * (ncols * nrows) + (j - 1) * nrows + i
+        compdof(tmp_idx) =  (start(3) - 1 + k - 1) * (dims(1) * dims(2)) +&
+                            (start(2) - 1 + j - 1) * dims(1) + i
+        exp_val(i,j,k) = compdof(tmp_idx)
+      end do
+    end do
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_TF_DATA_TYPE, dims, compdof, rd_iodesc)
+  deallocate(compdof)
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  filename = "test_pio_decomp_simple_tests.testfile"
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : PIO_TF_DATA_TYPE : ", iotype_descs(i)
+    ierr = PIO_createfile(pio_tf_iosystem_, pio_file, iotypes(i), filename, PIO_CLOBBER) 
+    PIO_TF_CHECK_ERR(ierr, "Could not create file " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_row', dims(1), pio_dims(1))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_col', dims(2), pio_dims(2))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_dim(pio_file, 'PIO_TF_test_dim_hgt', dims(3), pio_dims(3))
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a dim : " // trim(filename))
+
+    ierr = PIO_def_var(pio_file, 'PIO_TF_test_var', PIO_TF_DATA_TYPE, pio_dims, pio_var)
+    PIO_TF_CHECK_ERR(ierr, "Failed to define a var : " // trim(filename))
+
+    ierr = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ierr, "Failed to end redef mode : " // trim(filename))
+
+    ! Write the variable out
+    call PIO_write_darray(pio_file, pio_var, wr_iodesc, wbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to write darray : " // trim(filename))
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, rd_iodesc, rbuf, ierr)
+    PIO_TF_CHECK_ERR(ierr, "Failed to read darray : " // trim(filename))
+
+    PIO_TF_CHECK_VAL((rbuf, exp_val), "Got wrong val")
+
+    call PIO_closefile(pio_file)
+    
+    call PIO_deletefile(pio_tf_iosystem_, filename);
+  end do
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+  call PIO_freedecomp(pio_tf_iosystem_, rd_iodesc)
+  call PIO_freedecomp(pio_tf_iosystem_, wr_iodesc)
+  deallocate(exp_val)
+  deallocate(rbuf)
+  deallocate(wbuf)
+PIO_TF_AUTO_TEST_SUB_END nc_write_read_3d_col_decomp
diff --git a/externals/pio2/tests/general/pio_fail.F90.in b/src/externals/pio2/tests/general/pio_fail.F90.in
similarity index 100%
rename from externals/pio2/tests/general/pio_fail.F90.in
rename to src/externals/pio2/tests/general/pio_fail.F90.in
diff --git a/externals/pio2/tests/general/pio_file_fail.F90.in b/src/externals/pio2/tests/general/pio_file_fail.F90.in
similarity index 100%
rename from externals/pio2/tests/general/pio_file_fail.F90.in
rename to src/externals/pio2/tests/general/pio_file_fail.F90.in
diff --git a/externals/pio2/tests/general/pio_file_simple_tests.F90.in b/src/externals/pio2/tests/general/pio_file_simple_tests.F90.in
similarity index 97%
rename from externals/pio2/tests/general/pio_file_simple_tests.F90.in
rename to src/externals/pio2/tests/general/pio_file_simple_tests.F90.in
index 05f90fd7f9a..af5604b1798 100644
--- a/externals/pio2/tests/general/pio_file_simple_tests.F90.in
+++ b/src/externals/pio2/tests/general/pio_file_simple_tests.F90.in
@@ -53,7 +53,7 @@ PIO_TF_TEST_DRIVER_BEGIN
   integer :: num_iotypes
 
   num_iotypes = 0
-  call PIO_TF_Get_iotypes(iotypes, iotype_descs, num_iotypes) 
+  call PIO_TF_Get_iotypes(iotypes, iotype_descs, num_iotypes)
   dummy_file = "test_pio_file_simple_tests.testfile"
   do i=1,num_iotypes
     PIO_TF_TEST_RUN(create_file_no_opts(iotypes(i), dummy_file), trim(iotype_descs(i)))
@@ -66,4 +66,3 @@ PIO_TF_TEST_DRIVER_BEGIN
   end if
 
 PIO_TF_TEST_DRIVER_END
-
diff --git a/externals/pio2/tests/general/pio_init_finalize.F90.in b/src/externals/pio2/tests/general/pio_init_finalize.F90.in
similarity index 100%
rename from externals/pio2/tests/general/pio_init_finalize.F90.in
rename to src/externals/pio2/tests/general/pio_init_finalize.F90.in
diff --git a/src/externals/pio2/tests/general/pio_iosystem_tests.F90.in b/src/externals/pio2/tests/general/pio_iosystem_tests.F90.in
new file mode 100644
index 00000000000..33e59a2c96b
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_iosystem_tests.F90.in
@@ -0,0 +1,309 @@
+! Split comm world into two comms (one with even procs and the other
+! with odd procs
+SUBROUTINE split_world_odd_even(new_comm, new_rank, new_size, is_even)
+  use mpi
+  use pio_tutil
+  implicit none
+  integer, intent(inout) :: new_comm
+  integer, intent(inout) :: new_rank
+  integer, intent(inout) :: new_size
+  logical, intent(inout) :: is_even
+
+  integer :: ierr
+  integer :: color
+
+  new_comm = MPI_COMM_NULL
+  new_rank = 0
+  new_size = 0
+
+  if(mod(pio_tf_world_rank_, 2) == 0) then
+    is_even = .true.
+    color = 1
+  else
+    is_even = .false.
+    color = 0
+  end if
+
+  call MPI_Comm_split(pio_tf_comm_, color, 0, new_comm, ierr)
+
+  call MPI_Comm_rank(new_comm, new_rank, ierr)
+  call MPI_Comm_size(new_comm, new_size, ierr)
+END SUBROUTINE split_world_odd_even
+
+SUBROUTINE split_world_only_even(new_comm, new_rank, new_size, is_even)
+  use mpi
+  use pio_tutil
+  implicit none
+  integer, intent(inout) :: new_comm
+  integer, intent(inout) :: new_rank
+  integer, intent(inout) :: new_size
+  logical, intent(inout) :: is_even
+
+  integer :: ierr
+  integer :: color
+
+  new_comm = MPI_COMM_NULL
+  new_rank = 0
+  new_size = 0
+
+  if(mod(pio_tf_world_rank_, 2) == 0) then
+    is_even = .true.
+    color = 1
+  else
+    is_even = .false.
+    color = MPI_UNDEFINED
+  end if
+
+  call MPI_Comm_split(pio_tf_comm_, color, 0, new_comm, ierr)
+
+  if(new_comm /= MPI_COMM_NULL) then
+    call MPI_Comm_rank(new_comm, new_rank, ierr)
+    call MPI_Comm_size(new_comm, new_size, ierr)
+  end if
+END SUBROUTINE split_world_only_even
+
+! Create a file with a global attribute (filename)
+SUBROUTINE create_file(comm, iosys, iotype, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    type(file_desc_t) :: pio_file
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+
+    ret = PIO_createfile(iosys, pio_file, iotype, fname, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to create dummy file :" // trim(fname))
+
+    ret = PIO_def_dim(pio_file, dimname, PIO_TF_MAX_STR_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_def_var(pio_file, attname, PIO_char, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_put_att(pio_file, pio_var, attname, fname)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to put att " // trim(attname) // " in file :" // trim(fname))
+
+    call PIO_closefile(pio_file)
+END SUBROUTINE create_file
+
+! Check the contents of file : Check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+    character(len=PIO_TF_MAX_STR_LEN) :: val
+
+    ret = PIO_inq_dimid(pio_file, dimname, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_inq_varid(pio_file, attname, pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_get_att(pio_file, pio_var, attname, val)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to get att " // trim(attname) // " in file :" // trim(fname))
+
+    PRINT *, "val = ", trim(val), ", fname =", trim(fname)
+    PIO_TF_PASSERT(val .eq. fname, comm, "Attribute value is not the expected value")
+END SUBROUTINE check_file
+
+! Open and check the contents of file : open it and check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE open_and_check_file(comm, iosys, iotype, pio_file, fname, &
+                      attname, dimname, disable_fclose, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    logical, intent(in) :: disable_fclose
+    integer, intent(inout) :: ret
+
+    ret = PIO_openfile(iosys, pio_file, iotype, fname, PIO_write)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to open:" // fname)
+
+    call check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, comm, "Checking contents of file failed:" // fname)
+
+    if(.not. disable_fclose) then
+      call PIO_closefile(pio_file)
+    end if
+END SUBROUTINE open_and_check_file
+
+! Create a file with one iosystem - with all procs, and open/read with
+! another iosystem - subset (odd/even) of procs
+PIO_TF_AUTO_TEST_SUB_BEGIN two_iosystems_odd_even
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname1 = "pio_iosys_test_file1.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname2 = "pio_iosys_test_file2.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  character(len=PIO_TF_MAX_STR_LEN), pointer :: fname
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, num_iotypes = 0
+  type(file_desc_t) :: pio_file
+
+  type(iosystem_desc_t) :: odd_even_iosys
+  integer :: odd_even_comm, odd_even_comm_rank, odd_even_comm_size
+  logical :: is_even
+  integer :: ret
+
+  ! Split world to odd and even procs
+  call split_world_odd_even(odd_even_comm, odd_even_comm_rank, odd_even_comm_size, is_even)
+
+  call PIO_init(odd_even_comm_rank, odd_even_comm, odd_even_comm_size, &
+                1, &! Num aggregators
+                1, &! Stride
+                PIO_rearr_subset, odd_even_iosys, base=0)
+  call PIO_seterrorhandling(odd_even_iosys, PIO_BCAST_ERROR)
+
+  ! Open two different files and close it with two different iosystems
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create two files to be opened later - world - all procs
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname1, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname1)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname2, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname2)
+
+    ! Open file1 from odd processes and file2 from even processes
+    if(is_even) then
+      fname => fname1
+    else
+      fname => fname2
+    end if
+
+    call open_and_check_file(odd_even_comm, odd_even_iosys, iotypes(i), &
+                    pio_file, fname, attname, dimname, .false., ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname)
+  end do
+
+  call PIO_finalize(odd_even_iosys, ret)
+  call MPI_Comm_free(odd_even_comm, ret)
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END two_iosystems_odd_even
+
+! Create a file with one iosystem - with all procs, and open/read with
+! two iosystems - (1) with all procs (2) subset (even) of procs
+PIO_TF_AUTO_TEST_SUB_BEGIN two_iosystems_even_all
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname1 = "pio_iosys_test_file1.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname2 = "pio_iosys_test_file2.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  character(len=PIO_TF_MAX_STR_LEN), pointer :: fname
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, num_iotypes = 0
+  type(file_desc_t) :: pio_file1, pio_file2
+
+  type(iosystem_desc_t) :: odd_even_iosys
+  integer :: odd_even_comm, odd_even_comm_rank, odd_even_comm_size
+  logical :: is_even
+  logical :: disable_fclose = .true.
+  integer :: ret
+
+  ! Split world to odd and even procs
+  call split_world_only_even(odd_even_comm, odd_even_comm_rank, odd_even_comm_size, is_even)
+
+  if(is_even) then
+    call PIO_init(odd_even_comm_rank, odd_even_comm, odd_even_comm_size, &
+                  1, &! Num aggregators
+                  1, &! Stride
+                  PIO_rearr_subset, odd_even_iosys, base=0)
+    call PIO_seterrorhandling(odd_even_iosys, PIO_BCAST_ERROR)
+  end if
+
+  ! Open two different files and close it with two different iosystems
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create two files to be opened later - world - all procs
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname1, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname1)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname2, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname2)
+
+    ! Open file1 from all even processes and file2 from all odd processes
+    ! - PIO called from all processes with the odd_even_iosys
+    if(is_even) then
+      call open_and_check_file(odd_even_comm, odd_even_iosys, iotypes(i), &
+                    pio_file1, fname1, attname, dimname, disable_fclose, ret)
+      PRINT *, "file1,", trim(fname1), "fh: ", pio_file1%fh
+    end if
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname1)
+
+    call open_and_check_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                    pio_file2, fname2, attname, dimname, disable_fclose, ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname2)
+    PRINT *, "file2,", trim(fname2), "fh: ", pio_file2%fh
+
+    ! Check contents of the files again
+    ! - PIO called from odd and even processes separately with odd_even_iosys
+    if(is_even) then
+      call check_file(odd_even_comm, odd_even_iosys, iotypes(i), pio_file1, &
+                      fname1, attname, dimname, ret)
+      !call PIO_closefile(pio_file1)
+    end if
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname1)
+
+    call check_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), pio_file2, &
+                    fname2, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname2)
+
+    if(disable_fclose) then
+      if(is_even) then
+        call PIO_closefile(pio_file1)
+      end if
+      call PIO_closefile(pio_file2)
+    end if
+  end do
+
+  if(is_even) then
+    call PIO_finalize(odd_even_iosys, ret)
+    call MPI_Comm_free(odd_even_comm, ret)
+  end if
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END two_iosystems_even_all
diff --git a/src/externals/pio2/tests/general/pio_iosystem_tests2.F90.in b/src/externals/pio2/tests/general/pio_iosystem_tests2.F90.in
new file mode 100644
index 00000000000..e8e3d8fe431
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_iosystem_tests2.F90.in
@@ -0,0 +1,238 @@
+! Split comm world into two comms (one with even procs and the other
+! with odd procs
+SUBROUTINE split_world_odd_even(new_comm, new_rank, new_size, is_even)
+  use mpi
+  use pio_tutil
+  implicit none
+  integer, intent(inout) :: new_comm
+  integer, intent(inout) :: new_rank
+  integer, intent(inout) :: new_size
+  logical, intent(inout) :: is_even
+
+  integer :: ierr
+  integer :: color
+
+  new_comm = MPI_COMM_NULL
+  new_rank = 0
+  new_size = 0
+
+  if(mod(pio_tf_world_rank_, 2) == 0) then
+    is_even = .true.
+    color = 1
+  else
+    is_even = .false.
+    color = 0
+  end if
+
+  call MPI_Comm_split(pio_tf_comm_, color, 0, new_comm, ierr)
+
+  call MPI_Comm_rank(new_comm, new_rank, ierr)
+  call MPI_Comm_size(new_comm, new_size, ierr)
+END SUBROUTINE split_world_odd_even
+
+SUBROUTINE split_world_only_even(new_comm, new_rank, new_size, is_even)
+  use mpi
+  use pio_tutil
+  implicit none
+  integer, intent(inout) :: new_comm
+  integer, intent(inout) :: new_rank
+  integer, intent(inout) :: new_size
+  logical, intent(inout) :: is_even
+
+  integer :: ierr
+  integer :: color
+
+  new_comm = MPI_COMM_NULL
+  new_rank = 0
+  new_size = 0
+
+  if(mod(pio_tf_world_rank_, 2) == 0) then
+    is_even = .true.
+    color = 1
+  else
+    is_even = .false.
+    color = MPI_UNDEFINED
+  end if
+
+  call MPI_Comm_split(pio_tf_comm_, color, 0, new_comm, ierr)
+
+  if(new_comm /= MPI_COMM_NULL) then
+    call MPI_Comm_rank(new_comm, new_rank, ierr)
+    call MPI_Comm_size(new_comm, new_size, ierr)
+  end if
+END SUBROUTINE split_world_only_even
+
+! Create a file with a global attribute (filename)
+SUBROUTINE create_file(comm, iosys, iotype, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    type(file_desc_t) :: pio_file
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+
+    ret = PIO_createfile(iosys, pio_file, iotype, fname, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to create dummy file :" // trim(fname))
+
+    ret = PIO_def_dim(pio_file, dimname, PIO_TF_MAX_STR_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_def_var(pio_file, attname, PIO_char, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_put_att(pio_file, pio_var, attname, fname)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to put att " // trim(attname) // " in file :" // trim(fname))
+
+    call PIO_closefile(pio_file)
+END SUBROUTINE create_file
+
+! Check the contents of file : Check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+    character(len=PIO_TF_MAX_STR_LEN) :: val
+
+    ret = PIO_inq_dimid(pio_file, dimname, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_inq_varid(pio_file, attname, pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_get_att(pio_file, pio_var, attname, val)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to get att " // trim(attname) // " in file :" // trim(fname))
+
+    PRINT *, "val = ", trim(val), ", fname =", trim(fname)
+    PIO_TF_PASSERT(val .eq. fname, comm, "Attribute value is not the expected value")
+END SUBROUTINE check_file
+
+! Open and check the contents of file : open it and check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE open_and_check_file(comm, iosys, iotype, pio_file, fname, &
+                      attname, dimname, disable_fclose, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    logical, intent(in) :: disable_fclose
+    integer, intent(inout) :: ret
+
+    ret = PIO_openfile(iosys, pio_file, iotype, fname, PIO_write)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to open:" // fname)
+
+    call check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, comm, "Checking contents of file failed:" // fname)
+
+    if(.not. disable_fclose) then
+      call PIO_closefile(pio_file)
+    end if
+END SUBROUTINE open_and_check_file
+
+! Create three files with one iosystem - with all procs, and open/read with
+! another iosystem - subset (odd/even) of procs
+PIO_TF_AUTO_TEST_SUB_BEGIN three_files_two_iosystems_odd_even
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname0 = "pio_iosys_test_file0.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname1 = "pio_iosys_test_file1.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname2 = "pio_iosys_test_file2.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  character(len=PIO_TF_MAX_STR_LEN), pointer :: fname
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, num_iotypes = 0
+  type(file_desc_t) :: pio_file0, pio_file
+
+  type(iosystem_desc_t) :: odd_even_iosys
+  integer :: odd_even_comm, odd_even_comm_rank, odd_even_comm_size
+  logical :: is_even
+  integer :: ret
+
+  ! Split world to odd and even procs
+  call split_world_odd_even(odd_even_comm, odd_even_comm_rank, odd_even_comm_size, is_even)
+
+  call PIO_init(odd_even_comm_rank, odd_even_comm, odd_even_comm_size, &
+                1, &! Num aggregators
+                1, &! Stride
+                PIO_rearr_subset, odd_even_iosys, base=0)
+  call PIO_seterrorhandling(odd_even_iosys, PIO_BCAST_ERROR)
+
+  ! Open two different files and close it with two different iosystems
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create three files to be opened later - world - all procs
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname0, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname0)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname1, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname1)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname2, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname2)
+
+    ! Open file0 from all procs - disable close
+    call open_and_check_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                    pio_file0, fname0, attname, dimname, .true., ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname0)
+
+    ! Open file1 from odd processes and file2 from even processes
+    ! - disable close
+    if(is_even) then
+      fname => fname1
+    else
+      fname => fname2
+    end if
+
+    call open_and_check_file(odd_even_comm, odd_even_iosys, iotypes(i), &
+                    pio_file, fname, attname, dimname, .true., ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname)
+
+    ! Make sure that we can still check the contents of the file
+    call check_file(odd_even_comm, odd_even_iosys, iotypes(i), pio_file, &
+                    fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Checking (second) contents of file failed :" // fname)
+
+    call PIO_closefile(pio_file)
+    call PIO_closefile(pio_file0)
+  end do
+
+  call PIO_finalize(odd_even_iosys, ret)
+  call MPI_Comm_free(odd_even_comm, ret)
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END three_files_two_iosystems_odd_even
diff --git a/src/externals/pio2/tests/general/pio_iosystem_tests3.F90.in b/src/externals/pio2/tests/general/pio_iosystem_tests3.F90.in
new file mode 100644
index 00000000000..66c25690e1e
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_iosystem_tests3.F90.in
@@ -0,0 +1,284 @@
+! Split comm world into two comms (even procs and odd procs) and
+! rank == overlapped_rank included in both comms
+SUBROUTINE split_world_two_with_overlap(new_comms, new_ranks, new_sizes, overlapped_rank)
+  use mpi
+  use pio_tutil
+  implicit none
+  integer, parameter :: NUM_COMMS = 2
+  integer, dimension(NUM_COMMS), intent(inout) :: new_comms
+  integer, dimension(NUM_COMMS), intent(inout) :: new_ranks
+  integer, dimension(NUM_COMMS), intent(inout) :: new_sizes
+  integer, intent(in) :: overlapped_rank
+
+  integer :: i, ierr
+  integer :: world_group
+  ! first group range (first rank, last rank, stride) for including rank=overlapped_rank
+  ! second group range (first rank, last rank, stride) for other procs
+  !   strided depending on number of comms
+  ! Note: NUM_GROUP_RANGES is always 2, indep of value of NUM_COMMS
+  integer, parameter :: NUM_GROUP_RANGES = 2
+  integer :: nranges
+  integer, dimension(3,NUM_GROUP_RANGES) :: new_group_ranges
+  integer, dimension(NUM_COMMS) :: new_groups
+
+  do i=1,NUM_COMMS
+    new_comms(i) = MPI_COMM_NULL
+    new_ranks(i) = -1
+    new_sizes(i) = 0
+  end do
+
+  call MPI_Comm_group(pio_tf_comm_, world_group, ierr)
+
+  do i=1,NUM_COMMS
+    if(pio_tf_world_sz_ == 1) then
+      nranges = 1
+      new_group_ranges(1,1) = overlapped_rank
+      new_group_ranges(2,1) = overlapped_rank
+      new_group_ranges(3,1) = 1
+    else if(overlapped_rank / NUM_COMMS /= i-1) then
+      nranges = 2
+      ! Include rank == overlapped_rank
+      new_group_ranges(1,1) = overlapped_rank
+      new_group_ranges(2,1) = overlapped_rank
+      new_group_ranges(3,1) = 1
+
+      ! Include other processes split evenly among NUM_COMMS
+      new_group_ranges(1,2) = i-1
+      new_group_ranges(2,2) = i-1 + (pio_tf_world_sz_/NUM_COMMS - 1) * NUM_COMMS
+      new_group_ranges(3,2) = NUM_COMMS
+    else
+      nranges = 1
+      ! Include processes split evenly among NUM_COMMS
+      ! rank == overlapped_rank is already included in this range
+      new_group_ranges(1,1) = i-1
+      new_group_ranges(2,1) = i-1 + (pio_tf_world_sz_/NUM_COMMS - 1) * NUM_COMMS
+      new_group_ranges(3,1) = NUM_COMMS
+    end if
+
+    call MPI_Group_range_incl(world_group, nranges, new_group_ranges,&
+          new_groups(i), ierr)
+  end do
+
+  call MPI_Group_free(world_group, ierr)
+
+  ! Create communicators corresponding to the groups
+  ! All the communicators will have rank == overlapped_rank and are
+  ! disjoint otherwise
+  do i=1,NUM_COMMS
+    call MPI_Comm_create(pio_tf_comm_, new_groups(i), new_comms(i), ierr)
+    call MPI_Group_free(new_groups(i), ierr)
+    if(new_comms(i) /= MPI_COMM_NULL) then
+      call MPI_Comm_rank(new_comms(i), new_ranks(i), ierr)
+      call MPI_Comm_size(new_comms(i), new_sizes(i), ierr)
+    else
+      new_ranks(i) = -1
+      new_sizes(i) = 0
+    end if
+  end do
+
+END SUBROUTINE split_world_two_with_overlap
+
+! Create a file with a global attribute (filename)
+SUBROUTINE create_file(comm, iosys, iotype, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    type(file_desc_t) :: pio_file
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+
+    ret = PIO_createfile(iosys, pio_file, iotype, fname, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to create dummy file :" // trim(fname))
+
+    ret = PIO_def_dim(pio_file, dimname, PIO_TF_MAX_STR_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_def_var(pio_file, attname, PIO_char, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_put_att(pio_file, pio_var, attname, fname)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to put att " // trim(attname) // " in file :" // trim(fname))
+
+    call PIO_closefile(pio_file)
+END SUBROUTINE create_file
+
+! Check the contents of file : Check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+    character(len=PIO_TF_MAX_STR_LEN) :: val
+
+    ret = PIO_inq_dimid(pio_file, dimname, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_inq_varid(pio_file, attname, pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to find var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_get_att(pio_file, pio_var, attname, val)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to get att " // trim(attname) // " in file :" // trim(fname))
+
+    PRINT *, "val = ", trim(val), ", fname =", trim(fname)
+    PIO_TF_PASSERT(val .eq. fname, comm, "Attribute value is not the expected value")
+END SUBROUTINE check_file
+
+! Open and check the contents of file : open it and check the
+! global attribute 'filename' (should be equal to the
+! name of the file, fname)
+SUBROUTINE open_and_check_file(comm, iosys, iotype, pio_file, fname, &
+                      attname, dimname, disable_fclose, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    logical, intent(in) :: disable_fclose
+    integer, intent(inout) :: ret
+
+    ret = PIO_openfile(iosys, pio_file, iotype, fname, PIO_write)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to open:" // fname)
+
+    call check_file(comm, iosys, iotype, pio_file, fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, comm, "Checking contents of file failed:" // fname)
+
+    if(.not. disable_fclose) then
+      call PIO_closefile(pio_file)
+    end if
+END SUBROUTINE open_and_check_file
+
+! Create three files with one iosystem - with all procs, and open/read with
+! two different iosystems - subset (odd/even) of procs
+! The two iosystems created overlap at rank=0 (and are disjoint otherwise)
+PIO_TF_AUTO_TEST_SUB_BEGIN three_files_two_iosystems_with_overlap
+  use mpi
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname0 = "pio_iosys_test_file0.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname1 = "pio_iosys_test_file1.nc"
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname2 = "pio_iosys_test_file2.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  character(len=PIO_TF_MAX_STR_LEN), pointer :: fname
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, j, num_iotypes = 0
+  type(file_desc_t) :: pio_file0
+  integer, parameter :: NUM_COMMS = 2
+  type(file_desc_t), dimension(NUM_COMMS) :: pio_files
+  integer :: pio_base_rank = 0, pio_num_iotasks = 0
+
+  type(iosystem_desc_t), dimension(NUM_COMMS) :: overlapped_iosys
+  integer, dimension(NUM_COMMS) :: overlapped_comms, overlapped_comm_ranks, overlapped_comm_sizes
+  integer :: ret
+
+  ! Split world to two disjoint comms with overlap only at overlapped_rank=0
+  call split_world_two_with_overlap(overlapped_comms, overlapped_comm_ranks, overlapped_comm_sizes, 0)
+
+  do i=1,NUM_COMMS
+    if(overlapped_comms(i) /= MPI_COMM_NULL) then
+      ! If we have more than 1 proc, make sure that the io tasks start from 1
+      ! since rank=0 is always shared by all the overlapped_comms above
+      if(overlapped_comm_sizes(i) > 1) then
+        pio_base_rank = 1
+        pio_num_iotasks = overlapped_comm_sizes(i) - 1
+      else
+        pio_base_rank = 0
+        pio_num_iotasks = overlapped_comm_sizes(i)
+      end if
+      call PIO_init(overlapped_comm_ranks(i), overlapped_comms(i), &
+                    pio_num_iotasks, &
+                    1, &! Num aggregators
+                    1, &! Stride
+                    PIO_rearr_subset, overlapped_iosys(i), base=pio_base_rank)
+      call PIO_seterrorhandling(overlapped_iosys(i), PIO_BCAST_ERROR)
+    end if
+  end do
+
+  ! Open two different files and close it with two different iosystems
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create three files to be opened later - world - all procs
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname0, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname0)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname1, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname1)
+
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname2, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, "Failed to create file :" // fname2)
+
+    ! Open file0 from all procs - disable close
+    call open_and_check_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                    pio_file0, fname0, attname, dimname, .true., ret)
+    PIO_TF_CHECK_ERR(ret, "Checking contents of file failed :" // fname0)
+
+    fname => fname1
+
+    do j=1,NUM_COMMS
+      ! The two comms operate on different files
+      if(fname == fname1) then
+        fname => fname2
+      else
+        fname => fname1
+      end if
+      if(overlapped_comms(j) /= MPI_COMM_NULL) then
+        call open_and_check_file(overlapped_comms(j), overlapped_iosys(j), iotypes(i), &
+                        pio_files(j), fname, attname, dimname, .true., ret)
+        PIO_TF_CHECK_ERR(ret, overlapped_comms(j), "Checking contents of file failed :" // fname)
+
+        ! Make sure that we can still check the contents of the file
+        call check_file(overlapped_comms(j), overlapped_iosys(j), iotypes(i), &
+                        pio_files(j), fname, attname, dimname, ret)
+        PIO_TF_CHECK_ERR(ret, overlapped_comms(j), "Checking (second) contents of file failed :" // fname)
+      end if
+    end do
+
+    do j=1,NUM_COMMS
+      if(overlapped_comms(j) /= MPI_COMM_NULL) then
+        call PIO_closefile(pio_files(j))
+      end if
+    end do
+    call PIO_closefile(pio_file0)
+  end do
+
+  do i=1,NUM_COMMS
+    if(overlapped_comms(i) /= MPI_COMM_NULL) then
+      call PIO_finalize(overlapped_iosys(i), ret)
+      call MPI_Comm_free(overlapped_comms(i), ret)
+    end if
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END three_files_two_iosystems_with_overlap
diff --git a/src/externals/pio2/tests/general/pio_rearr.F90.in b/src/externals/pio2/tests/general/pio_rearr.F90.in
new file mode 100644
index 00000000000..f3a28c4e128
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_rearr.F90.in
@@ -0,0 +1,252 @@
+! Create a file with a global attribute (filename)
+SUBROUTINE create_file(comm, iosys, iotype, fname, attname, dimname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    character(len=*), intent(in) :: fname
+    character(len=*), intent(in) :: attname
+    character(len=*), intent(in) :: dimname
+    integer, intent(inout) :: ret
+
+    type(file_desc_t) :: pio_file
+    integer :: pio_dim
+    type(var_desc_t) :: pio_var
+
+    ret = PIO_createfile(iosys, pio_file, iotype, fname, PIO_CLOBBER)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to create dummy file :" // trim(fname))
+
+    ret = PIO_def_dim(pio_file, dimname, PIO_TF_MAX_STR_LEN, pio_dim)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define dim "// trim(dimname) // "in file :" // trim(fname))
+
+    ret = PIO_def_var(pio_file, attname, PIO_char, (/pio_dim/), pio_var)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to define var " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_put_att(pio_file, pio_var, attname, fname)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to put att " // trim(attname) // " in file :" // trim(fname))
+
+    ret = PIO_enddef(pio_file)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to enddef, file :" // trim(fname))
+
+    call PIO_closefile(pio_file)
+END SUBROUTINE create_file
+
+! Open a file and perform read and write
+SUBROUTINE open_and_check_rdwr(comm, iosys, iotype, pio_file, fname, ret)
+    use pio_tutil
+    implicit none
+
+    integer, intent(in) :: comm
+    type(iosystem_desc_t), intent(inout) :: iosys
+    integer, intent(in) :: iotype
+    type(file_desc_t), intent(inout) :: pio_file
+    character(len=*), intent(in) :: fname
+    integer, intent(inout) :: ret
+
+    integer, parameter :: VEC_LOCAL_SZ = 3
+    character(len=PIO_TF_MAX_STR_LEN) :: var_name = "test_1d_var"
+    character(len=PIO_TF_MAX_STR_LEN) :: dim_name = "test_dim_1d_var"
+    type(var_desc_t) :: pio_var
+    type(io_desc_t) :: iodesc
+    integer, dimension(VEC_LOCAL_SZ) :: compdof, compdof_rel_disps
+    integer :: start_compdof
+    real, dimension(VEC_LOCAL_SZ) :: wbuf
+    real, dimension(:), allocatable :: rbuf
+    integer, dimension(1) :: dims
+    integer :: pio_dim
+    integer :: comm_rank, comm_sz
+    integer :: i
+
+    call MPI_Comm_size(comm, comm_sz, ret)
+    call MPI_Comm_rank(comm, comm_rank, ret)
+
+    dims(1) = VEC_LOCAL_SZ * comm_sz
+
+    ret = PIO_openfile(iosys, pio_file, iotype, fname, PIO_write)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to open:" // fname)
+
+    ret = pio_inq_dimid(pio_file, dim_name, pio_dim)
+    if(ret /= PIO_NOERR) then
+      ! Define the required dim/var
+      ret = PIO_redef(pio_file)
+      PIO_TF_CHECK_ERR(ret, comm, "Failed to redef:" // fname)
+
+      ret = PIO_def_dim(pio_file, dim_name, dims(1), pio_dim)
+      PIO_TF_CHECK_ERR(ret, comm, "Failed to define dim:" // fname)
+
+      ! Assume var is not defined either
+      ret = PIO_def_var(pio_file, var_name, pio_real, (/pio_dim/), pio_var)
+      PIO_TF_CHECK_ERR(ret, comm, "Failed to define var:" // fname)
+
+      ret = PIO_enddef(pio_file)
+      PIO_TF_CHECK_ERR(ret, comm, "Failed to enddef:" // fname)
+    else
+      ! Assume var was also defined
+      ret = PIO_inq_varid(pio_file, var_name, pio_var)
+      PIO_TF_CHECK_ERR(ret, comm, "Failed to inq var:" // fname)
+    end if
+
+    ! Compdof is a simple reverse (of chunks of VEC_LOCAL_SZ) of the file offsets
+    ! e.g. VEC_LOCAL_SZ = 3, with 2 procs
+    ! [4 5 6] [1 2 3]
+    start_compdof = (comm_sz - comm_rank - 1) * VEC_LOCAL_SZ
+    do i=1,VEC_LOCAL_SZ
+      compdof_rel_disps(i) = i
+    end do
+
+    compdof = start_compdof + compdof_rel_disps
+    wbuf = compdof
+    allocate(rbuf(size(wbuf)))
+    rbuf = 0
+
+    call PIO_initdecomp(iosys, pio_real, dims, compdof, iodesc)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to init decomp:" // fname)
+
+    ! Write and read back the data
+    call PIO_write_darray(pio_file, pio_var, iodesc, wbuf, ret)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to write darray: " // fname)
+
+    call PIO_syncfile(pio_file)
+
+    call PIO_read_darray(pio_file, pio_var, iodesc, rbuf, ret)
+    PIO_TF_CHECK_ERR(ret, comm, "Failed to read darray: " // fname)
+
+    PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val")
+
+    deallocate(rbuf)
+
+    call PIO_freedecomp(iosys, iodesc)
+
+    call PIO_closefile(pio_file)
+END SUBROUTINE open_and_check_rdwr
+
+PIO_TF_AUTO_TEST_SUB_BEGIN test_rearrs_base
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname = "pio_test_rearrs_base.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  integer, parameter :: NUM_REARRANGERS = 2
+  integer :: rearrs(NUM_REARRANGERS) = (/pio_rearr_subset,pio_rearr_box/)
+  character(len=PIO_TF_MAX_STR_LEN) :: rearrs_info(NUM_REARRANGERS) = (/"PIO_REARR_SUBSET","PIO_REARR_BOX   "/)
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, j, k, num_iotypes = 0
+  type(file_desc_t) :: pio_file
+
+  type(iosystem_desc_t) :: dup_iosys
+  integer :: dup_comm
+  integer :: dup_comm_rank, dup_comm_sz, dup_iosys_base
+  integer :: ret
+
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing iotype: ", iotype_descs(i)
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, pio_tf_comm_, "Failed to create file :" // fname)
+
+    call MPI_Comm_dup(pio_tf_comm_, dup_comm, ret)
+    call MPI_Comm_rank(dup_comm, dup_comm_rank, ret)
+    call MPI_Comm_size(dup_comm, dup_comm_sz, ret)
+    do j=1, NUM_REARRANGERS
+      PIO_TF_LOG(0,*) "Testing rearr : ", rearrs_info(j)
+      do k=1, dup_comm_sz
+        dup_iosys_base = k-1
+        PIO_TF_LOG(0,*) "Base = ", dup_iosys_base
+        call PIO_init(pio_tf_world_rank_, dup_comm, pio_tf_world_sz_, &
+                      1, &! Num aggregators
+                      1, &! Stride
+                      rearrs(j), dup_iosys, base=dup_iosys_base)
+        call PIO_seterrorhandling(dup_iosys, PIO_BCAST_ERROR)
+
+        call open_and_check_rdwr(dup_comm, dup_iosys, iotypes(i), &
+                        pio_file, fname, ret)
+        PIO_TF_CHECK_ERR(ret, dup_comm, "Checking rd+wr on file failed :" // fname)
+
+        call PIO_finalize(dup_iosys, ret)
+      end do
+    end do
+    call PIO_deletefile(pio_tf_iosystem_, fname)
+    call MPI_Comm_free(dup_comm, ret)
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END test_rearrs_base
+
+! Test different combinations of the rearrangers to test the compatibility
+! between different rearrangers
+! init/rd+wr/finalize with one rearranger followed by another and try
+! all combinations ((subset,box),(box_subset),(box,box),(subset,subset))
+PIO_TF_AUTO_TEST_SUB_BEGIN test_rearrs_combs
+  implicit none
+
+  character(len=PIO_TF_MAX_STR_LEN), target :: fname = "pio_rearrs_combs.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: attname = "filename"
+  character(len=PIO_TF_MAX_STR_LEN), parameter :: dimname = "filename_dim"
+  integer, parameter :: NUM_REARRANGERS = 2
+  integer :: rearrs(NUM_REARRANGERS) = (/pio_rearr_subset,pio_rearr_box/)
+  integer, parameter :: MAX_PERMS = 4
+  integer :: rearrs_perms(NUM_REARRANGERS,MAX_PERMS) = reshape(&
+                (/pio_rearr_subset, pio_rearr_box,&
+                  pio_rearr_box, pio_rearr_subset,&
+                  pio_rearr_subset, pio_rearr_subset,&
+                  pio_rearr_box, pio_rearr_box/),&
+                  (/NUM_REARRANGERS,MAX_PERMS/)&
+              ) 
+  character(len=PIO_TF_MAX_STR_LEN) :: rearrs_perms_info(NUM_REARRANGERS,MAX_PERMS) =&
+            reshape(&
+              (/"PIO_REARR_SUBSET", "PIO_REARR_BOX   ",&
+              "PIO_REARR_BOX   ", "PIO_REARR_SUBSET",&
+              "PIO_REARR_SUBSET", "PIO_REARR_SUBSET",&
+              "PIO_REARR_BOX   ", "PIO_REARR_BOX   "/),&
+              (/NUM_REARRANGERS,MAX_PERMS/)&
+            )
+  character(len=PIO_TF_MAX_STR_LEN) :: rearrs_info(NUM_REARRANGERS) = (/"PIO_REARR_SUBSET","PIO_REARR_BOX   "/)
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: i, j, k, num_iotypes = 0
+  type(file_desc_t) :: pio_file
+
+  type(iosystem_desc_t) :: dup_iosys
+  integer :: dup_comm
+  integer :: ret
+
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing iotype: ", iotype_descs(i)
+    call create_file(pio_tf_comm_, pio_tf_iosystem_, iotypes(i), &
+                      fname, attname, dimname, ret)
+    PIO_TF_CHECK_ERR(ret, pio_tf_comm_, "Failed to create file :" // fname)
+
+    call MPI_Comm_dup(pio_tf_comm_, dup_comm, ret)
+    ! Try different combinations of rearrangers
+    do k=1,MAX_PERMS
+      do j=1, NUM_REARRANGERS
+        PIO_TF_LOG(0,*) "Testing rearr : ", rearrs_perms_info(j,k)
+        call PIO_init(pio_tf_world_rank_, dup_comm, pio_tf_world_sz_, &
+                      1, &! Num aggregators
+                      1, &! Stride
+                      rearrs_perms(j,k), dup_iosys, base=0)
+        call PIO_seterrorhandling(dup_iosys, PIO_BCAST_ERROR)
+
+        call open_and_check_rdwr(dup_comm, dup_iosys, iotypes(i), &
+                        pio_file, fname, ret)
+        PIO_TF_CHECK_ERR(ret, dup_comm, "Checking rd+wr on file failed :" // fname)
+
+        call PIO_finalize(dup_iosys, ret)
+      end do
+    end do
+    call PIO_deletefile(pio_tf_iosystem_, fname)
+    call MPI_Comm_free(dup_comm, ret)
+  end do
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+PIO_TF_AUTO_TEST_SUB_END test_rearrs_combs
+
diff --git a/src/externals/pio2/tests/general/pio_rearr_opts.F90.in b/src/externals/pio2/tests/general/pio_rearr_opts.F90.in
new file mode 100644
index 00000000000..9d168ea0473
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_rearr_opts.F90.in
@@ -0,0 +1,483 @@
+MODULE pio_rearr_opts_tgv
+  use pio_tutil
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_fname ="pio_rearr_opts_test.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_var_name ="dummy_var"
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_dim_name ="dummy_dim"
+END MODULE pio_rearr_opts_tgv
+
+! Test init/finalize wiht a default set of rearranger options
+PIO_TF_AUTO_TEST_SUB_BEGIN init_fin_with_rearr_opts
+  implicit none
+
+  integer, parameter :: NUM_REARRANGERS = 2
+  integer :: rearrs(NUM_REARRANGERS) = (/pio_rearr_subset,pio_rearr_box/)
+  character(len=PIO_TF_MAX_STR_LEN) :: rearrs_info(NUM_REARRANGERS) = (/"PIO_REARR_SUBSET","PIO_REARR_BOX   "/)
+  type(pio_rearr_opt_t) :: pio_rearr_opts
+  ! Dummy val for max pend req
+  integer, parameter :: MAX_PEND_REQ = 10
+
+  type(iosystem_desc_t) :: dup_iosys
+  integer :: dup_comm
+  integer :: i, ret
+
+  call MPI_Comm_dup(pio_tf_comm_, dup_comm, ret)
+
+  do i=1,NUM_REARRANGERS
+    ! Some dummy values for rearranger options
+    pio_rearr_opts%comm_type = PIO_rearr_comm_p2p
+    pio_rearr_opts%fcd = PIO_rearr_comm_fc_2d_enable
+
+    pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req = PIO_REARR_COMM_UNLIMITED_PEND_REQ
+    pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = .true.
+    pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = .true.
+
+    pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req = MAX_PEND_REQ
+    pio_rearr_opts%comm_fc_opts_io2comp%enable_hs = .true.
+    pio_rearr_opts%comm_fc_opts_io2comp%enable_isend = .true.
+
+    call PIO_init(pio_tf_world_rank_, dup_comm, pio_tf_world_sz_, &
+                  1, &! Num aggregators
+                  1, &! Stride
+                  rearrs(i), dup_iosys, base=0, rearr_opts=pio_rearr_opts)
+    call PIO_seterrorhandling(dup_iosys, PIO_BCAST_ERROR)
+
+    call PIO_finalize(dup_iosys, ret)
+    PIO_TF_CHECK_ERR(ret, "Finalize failed")
+  end do
+
+  call MPI_Comm_free(dup_comm, ret)
+
+PIO_TF_AUTO_TEST_SUB_END init_fin_with_rearr_opts
+
+SUBROUTINE print_rearr_opts(pio_rearr_opts, ret)
+  use pio_tutil
+  implicit none
+
+  type(pio_rearr_opt_t), intent(in) :: pio_rearr_opts
+  integer, intent(inout) :: ret
+
+  ret = PIO_NOERR
+
+  PIO_TF_LOG(0,*) "PIO rearranger options :"
+  if(pio_rearr_opts%comm_type == PIO_rearr_comm_p2p) then
+    PIO_TF_LOG(0,*) " comm_type = PIO_rearr_comm_p2p"
+  else if(pio_rearr_opts%comm_type == PIO_rearr_comm_coll) then
+    PIO_TF_LOG(0,*) " comm_type = PIO_rearr_comm_coll"
+  else
+    PIO_TF_LOG(0,*) " comm_type = INVALID"
+  end if
+
+  if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_2d_enable) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_2d_enable"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_1d_comp2io) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_1d_comp2io"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_1d_io2comp) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_1d_io2comp"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_2d_disable) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_2d_disable"
+  else
+    PIO_TF_LOG(0,*) " fcd = INVALID"
+  end if
+
+  PIO_TF_LOG(0,*) " comp2io max_pend_req =", pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req
+  if(pio_rearr_opts%comm_fc_opts_comp2io%enable_hs) then
+    PIO_TF_LOG(0,*) " comp2io enable_hs = TRUE"
+  else
+    PIO_TF_LOG(0,*) " comp2io enable_hs = FALSE"
+  end if
+  if(pio_rearr_opts%comm_fc_opts_comp2io%enable_isend) then
+    PIO_TF_LOG(0,*) " comp2io enable_isend = TRUE"
+  else
+    PIO_TF_LOG(0,*) " comp2io enable_isend = FALSE"
+  end if
+
+  PIO_TF_LOG(0,*) " io2comp max_pend_req =", pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req
+  if(pio_rearr_opts%comm_fc_opts_io2comp%enable_hs) then
+    PIO_TF_LOG(0,*) " io2comp enable_hs = TRUE"
+  else
+    PIO_TF_LOG(0,*) " io2comp enable_hs = FALSE"
+  end if
+  if(pio_rearr_opts%comm_fc_opts_io2comp%enable_isend) then
+    PIO_TF_LOG(0,*) " io2comp enable_isend = TRUE"
+  else
+    PIO_TF_LOG(0,*) " io2comp enable_isend = FALSE"
+  end if
+
+END SUBROUTINE print_rearr_opts
+
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_LOCAL_SZ elements
+! # All odd procs have VEC_LOCAL_SZ + 1 elements
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [1,2] [3,4,5] [6,7]
+! e.g. 2)    [1,2] [3,4,5] [6,7] [8,9,10]
+! e.g. 3)    [1,2] [3,4,5] [6,7] [8,9,10] [11,12]
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [3,4,5] [1,2] [6,7]
+! e.g. 2)    [3,4,5] [1,2] [8,9,10] [6,7]
+! e.g. 3)    [3,4,5] [1,2] [8,9,10] [6,7] [11,12]
+SUBROUTINE get_1d_bc_info(rank, sz, dims, start, count, force_rearrange)
+  implicit none
+
+  integer, parameter :: VEC_LOCAL_SZ = 3
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(1), intent(out) :: dims
+  integer, dimension(1), intent(out) :: start
+  integer, dimension(1), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = num_even_procs * VEC_LOCAL_SZ + num_odd_procs * (VEC_LOCAL_SZ + 1)
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(1) = VEC_LOCAL_SZ + 1
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + (VEC_LOCAL_SZ) + 1
+      else
+        count(1) = VEC_LOCAL_SZ
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(1) = VEC_LOCAL_SZ
+      start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) - (VEC_LOCAL_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(1) = VEC_LOCAL_SZ
+    else
+      count(1) = VEC_LOCAL_SZ + 1
+    end if
+    start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+  end if
+
+END SUBROUTINE
+
+! Create a decomp that will be used by tests below
+! The iodesc needs to be freed by the caller
+SUBROUTINE create_decomp_and_init_buf(iosys, iocomm, iodesc, wbuf, dims, ret)
+  use pio_tutil
+  implicit none
+
+  type(iosystem_desc_t), intent(inout) :: iosys
+  integer, intent(in) :: iocomm
+  type(io_desc_t), intent(out) :: iodesc
+  real, dimension(:), allocatable, intent(inout) :: wbuf
+  integer, dimension(1), intent(out) :: dims
+  integer, intent(out) :: ret
+
+  integer :: pio_dim
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(1) :: start, count
+  integer :: i
+  integer :: rank, sz
+
+  ret = PIO_NOERR
+
+  call MPI_Comm_size(iocomm, sz, ret)
+  call MPI_Comm_rank(iocomm, rank, ret)
+
+  call get_1d_bc_info(rank, sz, dims, start, count, .true.)
+  allocate(wbuf(count(1)))
+  allocate(compdof(count(1)))
+  do i=1,count(1)
+    wbuf(i) = start(1) + i - 1
+    compdof(i) = wbuf(i)
+  end do
+
+  call PIO_initdecomp(iosys, PIO_real, dims, compdof, iodesc)
+  deallocate(compdof)
+
+END SUBROUTINE
+
+! Create file and var used in the tests below
+! All details are picked from pio_rearr_opts_tgv module
+SUBROUTINE create_file_and_var(iotype, ret)
+  use pio_tutil
+  use pio_rearr_opts_tgv
+  implicit none
+
+  integer, intent(in) :: iotype
+  integer, intent(out) :: ret
+
+  type(file_desc_t) :: pio_file
+  type(var_desc_t) :: pio_var
+  integer :: pio_dim
+  integer, dimension(1) :: dims
+  integer, dimension(1) :: start, count
+
+  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+
+  ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotype, tgv_fname, PIO_CLOBBER)
+  PIO_TF_CHECK_ERR(ret, "Could not create file " // trim(tgv_fname))
+
+  ret = PIO_def_dim(pio_file, tgv_dim_name, dims(1), pio_dim)
+  PIO_TF_CHECK_ERR(ret, "Failed to define dim file =" // trim(tgv_fname))
+
+  ret = PIO_def_var(pio_file, tgv_var_name, pio_real, (/pio_dim/), pio_var)
+  PIO_TF_CHECK_ERR(ret, "Failed to define var file =" // trim(tgv_fname))
+
+  ret = PIO_enddef(pio_file)
+  PIO_TF_CHECK_ERR(ret, "Failed to enddef file =" // trim(tgv_fname))
+
+  call PIO_closefile(pio_file)
+
+END SUBROUTINE
+
+! Open file and inq var
+! All details are picked from pio_rearr_opts_tgv module
+! Note: The file is kept open so the called needs to close it
+SUBROUTINE open_file_and_get_var(iosys, pio_file, iotype, pio_var, dims, ret)
+  use pio_tutil
+  use pio_rearr_opts_tgv
+  implicit none
+
+  type(iosystem_desc_t), intent(inout) :: iosys
+  type(file_desc_t), intent(out) :: pio_file
+  integer, intent(in) :: iotype
+  type(var_desc_t), intent(out) :: pio_var
+  integer, dimension(1), intent(in) :: dims
+  integer, intent(out) :: ret
+
+  integer :: pio_dim
+
+  ret = PIO_openfile(iosys, pio_file, iotype, tgv_fname, pio_write)
+  PIO_TF_CHECK_ERR(ret, "Could not create file " // trim(tgv_fname))
+
+  ret = PIO_inq_varid(pio_file, tgv_var_name, pio_var)
+  PIO_TF_CHECK_ERR(ret, "Could not inq var " // trim(tgv_fname))
+
+END SUBROUTINE
+
+
+! Test all possible combinations of rearranger options
+PIO_TF_AUTO_TEST_SUB_BEGIN write_with_rearr_opts
+  use pio_rearr_opts_tgv
+  implicit none
+  interface
+    subroutine create_decomp_and_init_buf(iosys, iocomm, iodesc, wbuf, dims, ret)
+      use pio_tutil
+      type(iosystem_desc_t), intent(inout) :: iosys
+      integer, intent(in) :: iocomm
+      type(io_desc_t), intent(out) :: iodesc
+      real, dimension(:), allocatable, intent(inout) :: wbuf
+      integer, dimension(1), intent(out) :: dims
+      integer, intent(out) :: ret
+    end subroutine create_decomp_and_init_buf
+  end interface 
+
+  integer, parameter :: NUM_REARRANGERS = 2
+  integer :: rearrs(NUM_REARRANGERS) = (/pio_rearr_subset,pio_rearr_box/)
+  character(len=PIO_TF_MAX_STR_LEN) :: rearrs_info(NUM_REARRANGERS) = (/"PIO_REARR_SUBSET","PIO_REARR_BOX   "/)
+  type(pio_rearr_opt_t) :: pio_rearr_opts
+
+  ! Different rearranger options that are tested here
+  integer, parameter :: NUM_COMM_TYPE_OPTS = 2
+  integer :: comm_type_opts(NUM_COMM_TYPE_OPTS) =&
+                  (/pio_rearr_comm_p2p,pio_rearr_comm_coll/)
+  integer, parameter :: NUM_FCD_OPTS = 4
+  integer :: fcd_opts(NUM_FCD_OPTS) = &
+                  (/pio_rearr_comm_fc_2d_disable,&
+                    pio_rearr_comm_fc_1d_comp2io,&
+                    pio_rearr_comm_fc_1d_io2comp,&
+                    pio_rearr_comm_fc_2d_enable/)
+  integer :: num_fcd_opts_comm_type
+  integer, parameter :: NUM_ENABLE_HS_OPTS = 2
+  logical :: enable_hs_opts(NUM_ENABLE_HS_OPTS) = (/.true.,.false./)
+  integer :: num_enable_hs_opts_comp2io, num_enable_hs_opts_io2comp
+  integer, parameter :: NUM_ENABLE_ISEND_OPTS = 2
+  logical :: enable_isend_opts(NUM_ENABLE_ISEND_OPTS) = (/.true.,.false./)
+  integer :: num_enable_isend_opts_comp2io, num_enable_isend_opts_io2comp
+  integer, parameter :: NUM_MAX_PEND_REQ_OPTS = 2
+  integer :: max_pend_req_opts(NUM_MAX_PEND_REQ_OPTS) = &
+                  (/pio_rearr_comm_unlimited_pend_req, 2/)
+  integer :: num_max_pend_req_opts_comp2io, num_max_pend_req_opts_io2comp
+
+  type(iosystem_desc_t) :: dup_iosys
+  integer :: dup_comm
+  integer :: cur_rearr, cur_comm_type_opt, cur_fcd_opt, cur_enable_hs_c2i, &
+              cur_enable_isend_c2i, cur_max_pend_req_c2i, &
+              cur_enable_hs_i2c, cur_enable_isend_i2c, cur_max_pend_req_i2c
+
+  type(file_desc_t) :: pio_file
+  type(io_desc_t) :: iodesc
+  type(var_desc_t) :: pio_var
+  integer, dimension(1) :: dims
+
+  real, dimension(:), allocatable :: rbuf, wbuf
+
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: ret, ierr, i
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create the file and decomp 
+    call create_file_and_var(iotypes(i), ret)
+    PIO_TF_CHECK_ERR(ret, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname))
+
+    call MPI_Comm_dup(pio_tf_comm_, dup_comm, ret)
+
+    ! Test all combinations of these flow control parameters
+    do cur_rearr=1,NUM_REARRANGERS
+      PIO_TF_LOG(0,*) "Testing ", rearrs_info(cur_rearr)
+      do cur_comm_type_opt=1,NUM_COMM_TYPE_OPTS
+        pio_rearr_opts%comm_type = comm_type_opts(cur_comm_type_opt)
+        if(pio_rearr_opts%comm_type == pio_rearr_comm_coll) then
+          ! For coll we only test pio_rearr_comm_fc_2d_disable
+          num_fcd_opts_comm_type = 1
+        else if(pio_rearr_opts%comm_type == pio_rearr_comm_p2p) then
+          ! for p2p we test all possible combinations
+          num_fcd_opts_comm_type = NUM_FCD_OPTS
+        else
+          PIO_TF_ERROR("Unexpected comm type")
+        end if
+        do cur_fcd_opt=1,num_fcd_opts_comm_type
+          pio_rearr_opts%fcd = fcd_opts(cur_fcd_opt)
+          if(pio_rearr_opts%fcd == pio_rearr_comm_fc_2d_enable) then
+            num_enable_hs_opts_comp2io = NUM_ENABLE_HS_OPTS
+            num_enable_hs_opts_io2comp = NUM_ENABLE_HS_OPTS
+            num_enable_isend_opts_comp2io = NUM_ENABLE_ISEND_OPTS
+            num_enable_isend_opts_io2comp = NUM_ENABLE_ISEND_OPTS
+            num_max_pend_req_opts_comp2io = NUM_MAX_PEND_REQ_OPTS
+            num_max_pend_req_opts_io2comp = NUM_MAX_PEND_REQ_OPTS
+          else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_1d_comp2io) then
+            ! Only test different rearr opts in comp2io direction
+            num_enable_hs_opts_comp2io = NUM_ENABLE_HS_OPTS
+            num_enable_hs_opts_io2comp = 1
+            num_enable_isend_opts_comp2io = NUM_ENABLE_ISEND_OPTS
+            num_enable_isend_opts_io2comp = 1
+            num_max_pend_req_opts_comp2io = NUM_MAX_PEND_REQ_OPTS
+            num_max_pend_req_opts_io2comp = 1
+          else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_1d_io2comp) then
+            ! Only test different rearr opts in io2comp direction
+            num_enable_hs_opts_comp2io = 1
+            num_enable_hs_opts_io2comp = NUM_ENABLE_HS_OPTS
+            num_enable_isend_opts_comp2io = 1
+            num_enable_isend_opts_io2comp = NUM_ENABLE_ISEND_OPTS
+            num_max_pend_req_opts_comp2io = 1
+            num_max_pend_req_opts_io2comp = NUM_MAX_PEND_REQ_OPTS
+          else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_2d_disable) then
+            ! Just test one default combination
+            num_enable_hs_opts_comp2io = 1
+            num_enable_hs_opts_io2comp = 1
+            num_enable_isend_opts_comp2io = 1
+            num_enable_isend_opts_io2comp = 1
+            num_max_pend_req_opts_comp2io = 1
+            num_max_pend_req_opts_io2comp = 1
+          else
+            PIO_TF_ERROR("Unexpected flow control option")
+          end if
+          do cur_enable_hs_c2i=1,num_enable_hs_opts_comp2io
+            pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = &
+                  enable_hs_opts(cur_enable_hs_c2i)
+
+            do cur_enable_isend_c2i=1,num_enable_isend_opts_comp2io
+              pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = &
+                    enable_isend_opts(cur_enable_isend_c2i)
+
+              do cur_max_pend_req_c2i=1,num_max_pend_req_opts_comp2io
+                pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req = &
+                    max_pend_req_opts(cur_max_pend_req_c2i)
+
+                do cur_enable_hs_i2c=1,num_enable_hs_opts_io2comp
+                  pio_rearr_opts%comm_fc_opts_io2comp%enable_hs =&
+                    enable_hs_opts(cur_enable_hs_i2c)
+                
+                  do cur_enable_isend_i2c=1,num_enable_isend_opts_io2comp
+                    pio_rearr_opts%comm_fc_opts_io2comp%enable_isend =&
+                      enable_isend_opts(cur_enable_isend_i2c)
+
+                    do cur_max_pend_req_i2c=1,num_max_pend_req_opts_io2comp
+                      pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req =&
+                        max_pend_req_opts(cur_max_pend_req_i2c)
+
+                      call print_rearr_opts(pio_rearr_opts, ret)
+                      ! Ignoring return value - just printing for info
+                      call PIO_init(pio_tf_world_rank_,&
+                                      dup_comm, pio_tf_world_sz_, &
+                                      1, &! Num aggregators
+                                      1, &! Stride
+                                      rearrs(cur_rearr),&
+                                      dup_iosys, base=0,&
+                                      rearr_opts=pio_rearr_opts)
+
+                      call PIO_seterrorhandling(dup_iosys, PIO_BCAST_ERROR)
+
+                      call create_decomp_and_init_buf(dup_iosys, dup_comm, iodesc, wbuf, dims, ret)
+                      PIO_TF_CHECK_ERR(ret, dup_comm, "Creating decomp failed")
+
+                      allocate(rbuf(size(wbuf)))
+                      rbuf = 0
+
+                      call open_file_and_get_var(dup_iosys, pio_file, iotypes(i),&
+                              pio_var, dims, ret)
+                      PIO_TF_CHECK_ERR(ret, dup_comm, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname))
+
+                      call PIO_write_darray(pio_file, pio_var, iodesc, wbuf, ret)
+                      PIO_TF_CHECK_ERR(ret, dup_comm, "Writing var failed fname="//trim(tgv_fname))
+
+                      call PIO_syncfile(pio_file)
+
+                      call PIO_read_darray(pio_file, pio_var, iodesc, rbuf, ret)
+                      PIO_TF_CHECK_ERR(ret, dup_comm, "Reading var failed fname="//trim(tgv_fname))
+
+                      PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val")
+
+                      call PIO_freedecomp(dup_iosys, iodesc)
+
+                      deallocate(rbuf)
+                      if(allocated(wbuf)) then
+                        deallocate(wbuf)
+                      end if
+
+                      call PIO_closefile(pio_file)
+
+                      call PIO_finalize(dup_iosys, ret)
+                      PIO_TF_CHECK_ERR(ret, "Finalize failed")
+                    end do ! cur_max_pend_req_i2c
+                  end do ! cur_enable_isend_i2c
+                end do ! cur_enable_hs_i2c
+              end do ! cur_max_pend_req_c2i    
+            end do ! cur_enable_isend_c2i
+          end do ! cur_enable_hs_c2i
+        end do ! cur_fcd_opt
+      end do ! cur_comm_type_opt
+    end do ! cur_rearr
+
+    call PIO_deletefile(pio_tf_iosystem_, trim(tgv_fname))
+
+    call MPI_Comm_free(dup_comm, ret)
+  end do ! iotypes
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END write_with_rearr_opts
diff --git a/src/externals/pio2/tests/general/pio_rearr_opts2.F90.in b/src/externals/pio2/tests/general/pio_rearr_opts2.F90.in
new file mode 100644
index 00000000000..9744a87cfc3
--- /dev/null
+++ b/src/externals/pio2/tests/general/pio_rearr_opts2.F90.in
@@ -0,0 +1,420 @@
+MODULE pio_rearr_opts_tgv
+  use pio_tutil
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_fname ="pio_rearr_opts2_test.nc"
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_var_name ="dummy_var"
+  character(len=PIO_TF_MAX_STR_LEN), parameter ::tgv_dim_name ="dummy_dim"
+END MODULE pio_rearr_opts_tgv
+
+SUBROUTINE print_rearr_opts(pio_rearr_opts, ret)
+  use pio_tutil
+  implicit none
+
+  type(pio_rearr_opt_t), intent(in) :: pio_rearr_opts
+  integer, intent(inout) :: ret
+
+  ret = PIO_NOERR
+
+  PIO_TF_LOG(0,*) "PIO rearranger options :"
+  if(pio_rearr_opts%comm_type == PIO_rearr_comm_p2p) then
+    PIO_TF_LOG(0,*) " comm_type = PIO_rearr_comm_p2p"
+  else if(pio_rearr_opts%comm_type == PIO_rearr_comm_coll) then
+    PIO_TF_LOG(0,*) " comm_type = PIO_rearr_comm_coll"
+  else
+    PIO_TF_LOG(0,*) " comm_type = INVALID"
+  end if
+
+  if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_2d_enable) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_2d_enable"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_1d_comp2io) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_1d_comp2io"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_1d_io2comp) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_1d_io2comp"
+  else if(pio_rearr_opts%fcd == PIO_rearr_comm_fc_2d_disable) then
+    PIO_TF_LOG(0,*) " fcd = PIO_rearr_comm_fc_2d_disable"
+  else
+    PIO_TF_LOG(0,*) " fcd = INVALID"
+  end if
+
+  PIO_TF_LOG(0,*) " comp2io max_pend_req =", pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req
+  if(pio_rearr_opts%comm_fc_opts_comp2io%enable_hs) then
+    PIO_TF_LOG(0,*) " comp2io enable_hs = TRUE"
+  else
+    PIO_TF_LOG(0,*) " comp2io enable_hs = FALSE"
+  end if
+  if(pio_rearr_opts%comm_fc_opts_comp2io%enable_isend) then
+    PIO_TF_LOG(0,*) " comp2io enable_isend = TRUE"
+  else
+    PIO_TF_LOG(0,*) " comp2io enable_isend = FALSE"
+  end if
+
+  PIO_TF_LOG(0,*) " io2comp max_pend_req =", pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req
+  if(pio_rearr_opts%comm_fc_opts_io2comp%enable_hs) then
+    PIO_TF_LOG(0,*) " io2comp enable_hs = TRUE"
+  else
+    PIO_TF_LOG(0,*) " io2comp enable_hs = FALSE"
+  end if
+  if(pio_rearr_opts%comm_fc_opts_io2comp%enable_isend) then
+    PIO_TF_LOG(0,*) " io2comp enable_isend = TRUE"
+  else
+    PIO_TF_LOG(0,*) " io2comp enable_isend = FALSE"
+  end if
+
+END SUBROUTINE print_rearr_opts
+
+! If force_rearrange is FALSE, the decomposition is such that
+! # All even procs have VEC_LOCAL_SZ elements
+! # All odd procs have VEC_LOCAL_SZ + 1 elements
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [1,2] [3,4,5] [6,7]
+! e.g. 2)    [1,2] [3,4,5] [6,7] [8,9,10]
+! e.g. 3)    [1,2] [3,4,5] [6,7] [8,9,10] [11,12]
+! If force_rearrange is TRUE, the decomposition is such that,
+! If possible, the even rank "exchanges" elements with the next
+! higher ranked odd proc.
+! This for example can be used to force rearrangement when reading
+! or writing data.
+! e.g. For VEC_LOCAL_SZ = 2,
+! e.g. 1)    [3,4,5] [1,2] [6,7]
+! e.g. 2)    [3,4,5] [1,2] [8,9,10] [6,7]
+! e.g. 3)    [3,4,5] [1,2] [8,9,10] [6,7] [11,12]
+SUBROUTINE get_1d_bc_info(rank, sz, dims, start, count, force_rearrange)
+  implicit none
+
+  integer, parameter :: VEC_LOCAL_SZ = 3
+  integer, intent(in) :: rank
+  integer, intent(in) :: sz
+  integer, dimension(1), intent(out) :: dims
+  integer, dimension(1), intent(out) :: start
+  integer, dimension(1), intent(out) :: count
+  logical, intent(in) :: force_rearrange
+
+  logical :: is_even_rank
+  integer :: num_odd_procs, num_even_procs
+  integer :: iodd, ieven
+
+  is_even_rank = .false.
+  if (mod(rank, 2) == 0) then
+    is_even_rank = .true.
+  end if
+  num_odd_procs = sz / 2
+  num_even_procs = sz - num_odd_procs
+  dims(1) = num_even_procs * VEC_LOCAL_SZ + num_odd_procs * (VEC_LOCAL_SZ + 1)
+  ! Number of odd and even procs before this rank
+  iodd = rank / 2
+  ieven = (rank + 1) / 2
+  if(force_rearrange) then
+    ! Make sure that we force rearrangement
+    if (is_even_rank) then
+      if(rank + 1 < sz) then
+        ! Force rearrangement
+        count(1) = VEC_LOCAL_SZ + 1
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + (VEC_LOCAL_SZ) + 1
+      else
+        count(1) = VEC_LOCAL_SZ
+        start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+      end if
+    else
+      ! For all odd procs there is an even lower ranked, rank-1, proc
+      ! So force rearrangement
+      count(1) = VEC_LOCAL_SZ
+      start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) - (VEC_LOCAL_SZ) + 1
+    end if
+  else
+    if (is_even_rank) then
+      count(1) = VEC_LOCAL_SZ
+    else
+      count(1) = VEC_LOCAL_SZ + 1
+    end if
+    start(1) = ieven * VEC_LOCAL_SZ + iodd * (VEC_LOCAL_SZ + 1) + 1
+  end if
+
+END SUBROUTINE
+
+! Create a decomp that will be used by tests below
+! The iodesc needs to be freed by the caller
+SUBROUTINE create_decomp_and_init_buf(iodesc, wbuf, dims, ret)
+  use pio_tutil
+  implicit none
+
+  type(io_desc_t), intent(out) :: iodesc
+  real, dimension(:), allocatable, intent(inout) :: wbuf
+  integer, dimension(1), intent(out) :: dims
+  integer, intent(out) :: ret
+
+  integer :: pio_dim
+  integer, dimension(:), allocatable :: compdof
+  integer, dimension(1) :: start, count
+  integer :: i
+
+  ret = PIO_NOERR
+
+  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+  allocate(wbuf(count(1)))
+  allocate(compdof(count(1)))
+  do i=1,count(1)
+    wbuf(i) = start(1) + i - 1
+    compdof(i) = wbuf(i)
+  end do
+
+  call PIO_initdecomp(pio_tf_iosystem_, PIO_real, dims, compdof, iodesc)
+  deallocate(compdof)
+
+END SUBROUTINE
+
+! Create file and var used in the tests below
+! All details are picked from pio_rearr_opts_tgv module
+SUBROUTINE create_file_and_var(iotype, ret)
+  use pio_tutil
+  use pio_rearr_opts_tgv
+  implicit none
+
+  integer, intent(in) :: iotype
+  integer, intent(out) :: ret
+
+  type(file_desc_t) :: pio_file
+  type(var_desc_t) :: pio_var
+  integer :: pio_dim
+  integer, dimension(1) :: dims
+  integer, dimension(1) :: start, count
+
+  call get_1d_bc_info(pio_tf_world_rank_, pio_tf_world_sz_, dims, start, count, .true.)
+
+  ret = PIO_createfile(pio_tf_iosystem_, pio_file, iotype, tgv_fname, PIO_CLOBBER)
+  PIO_TF_CHECK_ERR(ret, "Could not create file " // trim(tgv_fname))
+
+  ret = PIO_def_dim(pio_file, tgv_dim_name, dims(1), pio_dim)
+  PIO_TF_CHECK_ERR(ret, "Failed to define dim file =" // trim(tgv_fname))
+
+  ret = PIO_def_var(pio_file, tgv_var_name, pio_real, (/pio_dim/), pio_var)
+  PIO_TF_CHECK_ERR(ret, "Failed to define var file =" // trim(tgv_fname))
+
+  ret = PIO_enddef(pio_file)
+  PIO_TF_CHECK_ERR(ret, "Failed to enddef file =" // trim(tgv_fname))
+
+  call PIO_closefile(pio_file)
+
+END SUBROUTINE
+
+! Open file and inq var
+! All details are picked from pio_rearr_opts_tgv module
+! Note: The file is kept open so the called needs to close it
+SUBROUTINE open_file_and_get_var(pio_file, iotype, pio_var, ret)
+  use pio_tutil
+  use pio_rearr_opts_tgv
+  implicit none
+
+  type(file_desc_t), intent(out) :: pio_file
+  integer, intent(in) :: iotype
+  type(var_desc_t), intent(out) :: pio_var
+  integer, intent(out) :: ret
+
+  integer :: pio_dim
+
+  ret = PIO_openfile(pio_tf_iosystem_, pio_file, iotype, tgv_fname, pio_write)
+  PIO_TF_CHECK_ERR(ret, "Could not create file " // trim(tgv_fname))
+
+  ret = PIO_inq_varid(pio_file, tgv_var_name, pio_var)
+  PIO_TF_CHECK_ERR(ret, "Could not inq var " // trim(tgv_fname))
+
+END SUBROUTINE
+
+! Test all possible combinations of rearranger options
+! using the pio interface to explicitly set the rearranger options
+! after init.
+PIO_TF_AUTO_TEST_SUB_BEGIN set_rearr_opts_and_write
+  use pio_rearr_opts_tgv
+  implicit none
+  interface
+    subroutine create_decomp_and_init_buf(iodesc, wbuf, dims, ret)
+      use pio_tutil
+      type(io_desc_t), intent(out) :: iodesc
+      real, dimension(:), allocatable, intent(inout) :: wbuf
+      integer, dimension(1), intent(out) :: dims
+      integer, intent(out) :: ret
+    end subroutine create_decomp_and_init_buf
+  end interface 
+
+  type(pio_rearr_opt_t) :: pio_rearr_opts
+
+  ! Different rearranger options that are tested here
+  integer, parameter :: NUM_COMM_TYPE_OPTS = 2
+  integer :: comm_type_opts(NUM_COMM_TYPE_OPTS) =&
+                  (/pio_rearr_comm_p2p,pio_rearr_comm_coll/)
+  integer, parameter :: NUM_FCD_OPTS = 4
+  integer :: fcd_opts(NUM_FCD_OPTS) = &
+                  (/pio_rearr_comm_fc_2d_disable,&
+                    pio_rearr_comm_fc_1d_comp2io,&
+                    pio_rearr_comm_fc_1d_io2comp,&
+                    pio_rearr_comm_fc_2d_enable/)
+  integer :: num_fcd_opts_comm_type
+  integer, parameter :: NUM_ENABLE_HS_OPTS = 2
+  logical :: enable_hs_opts(NUM_ENABLE_HS_OPTS) = (/.true.,.false./)
+  integer :: num_enable_hs_opts_comp2io, num_enable_hs_opts_io2comp
+  integer, parameter :: NUM_ENABLE_ISEND_OPTS = 2
+  logical :: enable_isend_opts(NUM_ENABLE_ISEND_OPTS) = (/.true.,.false./)
+  integer :: num_enable_isend_opts_comp2io, num_enable_isend_opts_io2comp
+  integer, parameter :: NUM_MAX_PEND_REQ_OPTS = 3
+  integer :: max_pend_req_opts(NUM_MAX_PEND_REQ_OPTS) = &
+                  (/pio_rearr_comm_unlimited_pend_req, 1, 2/)
+  integer :: num_max_pend_req_opts_comp2io, num_max_pend_req_opts_io2comp
+
+  integer ::  cur_comm_type_opt, cur_fcd_opt, cur_enable_hs_c2i, &
+              cur_enable_isend_c2i, cur_max_pend_req_c2i, &
+              cur_enable_hs_i2c, cur_enable_isend_i2c, cur_max_pend_req_i2c
+
+  type(file_desc_t) :: pio_file
+  type(io_desc_t) :: iodesc
+  type(var_desc_t) :: pio_var
+  integer, dimension(1) :: dims
+
+  real, dimension(:), allocatable :: rbuf, wbuf
+
+  ! iotypes = valid io types
+  integer, dimension(:), allocatable :: iotypes
+  character(len=PIO_TF_MAX_STR_LEN), dimension(:), allocatable :: iotype_descs
+  integer :: num_iotypes
+  integer :: ret, ierr, i
+
+  num_iotypes = 0
+  call PIO_TF_Get_nc_iotypes(iotypes, iotype_descs, num_iotypes)
+  do i=1,num_iotypes
+    PIO_TF_LOG(0,*) "Testing : ", iotype_descs(i)
+    ! Create the file and decomp 
+    call create_file_and_var(iotypes(i), ret)
+    PIO_TF_CHECK_ERR(ret, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname))
+
+    call open_file_and_get_var(pio_file, iotypes(i), pio_var, ret)
+    PIO_TF_CHECK_ERR(ret, "Creating file/decomp/var reqd for test failed :" // trim(tgv_fname))
+
+    ! Test all combinations of these flow control parameters
+    do cur_comm_type_opt=1,NUM_COMM_TYPE_OPTS
+      pio_rearr_opts%comm_type = comm_type_opts(cur_comm_type_opt)
+      if(pio_rearr_opts%comm_type == pio_rearr_comm_coll) then
+        ! For coll we only test pio_rearr_comm_fc_2d_disable
+        num_fcd_opts_comm_type = 1
+      else if(pio_rearr_opts%comm_type == pio_rearr_comm_p2p) then
+        ! for p2p we test all possible combinations
+        num_fcd_opts_comm_type = NUM_FCD_OPTS
+      else
+        PIO_TF_ERROR("Unexpected comm type")
+      end if
+      do cur_fcd_opt=1,num_fcd_opts_comm_type
+        pio_rearr_opts%fcd = fcd_opts(cur_fcd_opt)
+        if(pio_rearr_opts%fcd == pio_rearr_comm_fc_2d_enable) then
+          num_enable_hs_opts_comp2io = NUM_ENABLE_HS_OPTS
+          num_enable_hs_opts_io2comp = NUM_ENABLE_HS_OPTS
+          num_enable_isend_opts_comp2io = NUM_ENABLE_ISEND_OPTS
+          num_enable_isend_opts_io2comp = NUM_ENABLE_ISEND_OPTS
+          num_max_pend_req_opts_comp2io = NUM_MAX_PEND_REQ_OPTS
+          num_max_pend_req_opts_io2comp = NUM_MAX_PEND_REQ_OPTS
+        else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_1d_comp2io) then
+          ! Only test different rearr opts in comp2io direction
+          num_enable_hs_opts_comp2io = NUM_ENABLE_HS_OPTS
+          num_enable_hs_opts_io2comp = 1
+          num_enable_isend_opts_comp2io = NUM_ENABLE_ISEND_OPTS
+          num_enable_isend_opts_io2comp = 1
+          num_max_pend_req_opts_comp2io = NUM_MAX_PEND_REQ_OPTS
+          num_max_pend_req_opts_io2comp = 1
+        else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_1d_io2comp) then
+          ! Only test different rearr opts in io2comp direction
+          num_enable_hs_opts_comp2io = 1
+          num_enable_hs_opts_io2comp = NUM_ENABLE_HS_OPTS
+          num_enable_isend_opts_comp2io = 1
+          num_enable_isend_opts_io2comp = NUM_ENABLE_ISEND_OPTS
+          num_max_pend_req_opts_comp2io = 1
+          num_max_pend_req_opts_io2comp = NUM_MAX_PEND_REQ_OPTS
+        else if(pio_rearr_opts%fcd == pio_rearr_comm_fc_2d_disable) then
+          ! Just test one default combination
+          num_enable_hs_opts_comp2io = 1
+          num_enable_hs_opts_io2comp = 1
+          num_enable_isend_opts_comp2io = 1
+          num_enable_isend_opts_io2comp = 1
+          num_max_pend_req_opts_comp2io = 1
+          num_max_pend_req_opts_io2comp = 1
+        else
+          PIO_TF_ERROR("Unexpected flow control option")
+        end if
+        do cur_enable_hs_c2i=1,num_enable_hs_opts_comp2io
+          pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = &
+                enable_hs_opts(cur_enable_hs_c2i)
+
+          do cur_enable_isend_c2i=1,num_enable_isend_opts_comp2io
+            pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = &
+                  enable_isend_opts(cur_enable_isend_c2i)
+
+            do cur_max_pend_req_c2i=1,num_max_pend_req_opts_comp2io
+              pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req = &
+                  max_pend_req_opts(cur_max_pend_req_c2i)
+
+              do cur_enable_hs_i2c=1,num_enable_hs_opts_io2comp
+                pio_rearr_opts%comm_fc_opts_io2comp%enable_hs =&
+                  enable_hs_opts(cur_enable_hs_i2c)
+              
+                do cur_enable_isend_i2c=1,num_enable_isend_opts_io2comp
+                  pio_rearr_opts%comm_fc_opts_io2comp%enable_isend =&
+                    enable_isend_opts(cur_enable_isend_i2c)
+
+                  do cur_max_pend_req_i2c=1,num_max_pend_req_opts_io2comp
+                    pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req =&
+                      max_pend_req_opts(cur_max_pend_req_i2c)
+
+                    call print_rearr_opts(pio_rearr_opts, ret)
+                    ! Ignoring return value - just printing for info
+
+                    ! Set the rearranger options
+                    ! rearr_opt_t descributes logicals as logical(kind=c_bool)
+                    ! but pio_set_rearr_opts() expects regular logicals, hence
+                    ! explicit type cast is reqd for logical params
+                    ret = PIO_set_rearr_opts(pio_tf_iosystem_,&
+                            pio_rearr_opts%comm_type,&
+                            pio_rearr_opts%fcd,&
+                            logical(pio_rearr_opts%comm_fc_opts_comp2io%enable_hs),&
+                            logical(pio_rearr_opts%comm_fc_opts_comp2io%enable_isend),&
+                            pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req,&
+                            logical(pio_rearr_opts%comm_fc_opts_io2comp%enable_hs),&
+                            logical(pio_rearr_opts%comm_fc_opts_io2comp%enable_isend),&
+                            pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req)
+                    PIO_TF_CHECK_ERR(ret, "Setting rearr opts failed fname="//trim(tgv_fname))
+
+                    call create_decomp_and_init_buf(iodesc, wbuf, dims, ret)
+                    PIO_TF_CHECK_ERR(ret, "Creating decomp failed")
+
+                    allocate(rbuf(size(wbuf)))
+                    rbuf = 0
+
+                    call PIO_write_darray(pio_file, pio_var, iodesc, wbuf, ret)
+                    PIO_TF_CHECK_ERR(ret, "Writing var failed fname="//trim(tgv_fname))
+
+                    call PIO_syncfile(pio_file)
+
+                    call PIO_read_darray(pio_file, pio_var, iodesc, rbuf, ret)
+                    PIO_TF_CHECK_ERR(ret, "Reading var failed fname="//trim(tgv_fname))
+
+                    PIO_TF_CHECK_VAL((rbuf, wbuf), "Got wrong val")
+
+                    call PIO_freedecomp(pio_tf_iosystem_, iodesc)
+
+                    deallocate(rbuf)
+                    if(allocated(wbuf)) then
+                      deallocate(wbuf)
+                    end if
+
+                  end do ! cur_max_pend_req_i2c
+                end do ! cur_enable_isend_i2c
+              end do ! cur_enable_hs_i2c
+            end do ! cur_max_pend_req_c2i    
+          end do ! cur_enable_isend_c2i
+        end do ! cur_enable_hs_c2i
+      end do ! cur_fcd_opt
+    end do ! cur_comm_type_opt
+
+    call PIO_closefile(pio_file)
+    call PIO_deletefile(pio_tf_iosystem_, trim(tgv_fname))
+  end do ! iotypes
+
+  if(allocated(iotypes)) then
+    deallocate(iotypes)
+    deallocate(iotype_descs)
+  end if
+
+PIO_TF_AUTO_TEST_SUB_END set_rearr_opts_and_write
diff --git a/externals/pio2/tests/general/test_memleak.c b/src/externals/pio2/tests/general/test_memleak.c
similarity index 93%
rename from externals/pio2/tests/general/test_memleak.c
rename to src/externals/pio2/tests/general/test_memleak.c
index 8d6b9bce2f8..3c81f477df1 100644
--- a/externals/pio2/tests/general/test_memleak.c
+++ b/src/externals/pio2/tests/general/test_memleak.c
@@ -1,5 +1,5 @@
 /**
- * @file 
+ * @file
  * Tests for handling of fillval.
  *
  * This test was added to track down memory leaks in
@@ -42,19 +42,19 @@
 /** Handle MPI errors. This should only be used with MPI library
  * function calls. */
 #define MPIERR(e) do {                                                  \
-	MPI_Error_string(e, err_buffer, &resultlen);			\
+	MPI_Error_string(e, err_buffer, &resultlen);                    \
 	fprintf(stderr, "MPI error, line %d, file %s: %s\n", __LINE__, __FILE__, err_buffer); \
-	MPI_Finalize();							\
-	return 2;							\
-    } while (0) 
+	MPI_Finalize();                                                 \
+	return 2;                                                       \
+    } while (0)
 
 /** Handle non-MPI errors by finalizing the MPI library and exiting
  * with an exit code. */
-#define ERR(e) do {				\
-        fprintf(stderr, "Error %d in %s, line %d\n", e, __FILE__, __LINE__); \
-	MPI_Finalize();				\
-	return e;				\
-    } while (0) 
+#define ERR(e) do {                             \
+	fprintf(stderr, "Error %d in %s, line %d\n", e, __FILE__, __LINE__); \
+	MPI_Finalize();                         \
+	return e;                               \
+    } while (0)
 
 /** Global err buffer for MPI. When there is an MPI error, this buffer
  * is used to store the error message that is associated with the MPI
@@ -86,7 +86,7 @@ int
 main(int argc, char **argv)
 {
     int verbose = 1;
-    
+
     /** Zero-based rank of processor. */
     int my_rank;
 
@@ -99,7 +99,7 @@ main(int argc, char **argv)
      * classic format file (but with different libraries). The
      * last two produce netCDF4/HDF5 format files, written with
      * and without using netCDF-4 parallel I/O. */
-    int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF, 
+    int format[NUM_NETCDF_FLAVORS] = {PIO_IOTYPE_PNETCDF,
 				      PIO_IOTYPE_NETCDF,
 				      PIO_IOTYPE_NETCDF4C,
 				      PIO_IOTYPE_NETCDF4P};
@@ -113,7 +113,7 @@ main(int argc, char **argv)
 							  "test_nc4_classic.nc",
 							  "test_nc4_serial4.nc",
 							  "test_nc4_parallel4.nc"};
-	
+
     /** Number of processors that will do IO. In this example we
      * will do IO from all processors. */
     int niotasks;
@@ -179,13 +179,13 @@ main(int argc, char **argv)
 
     /** Return code. */
     int ret;
-    
-#ifdef TIMING    
+
+#ifdef TIMING
     /* Initialize the GPTL timing library. */
     if ((ret = GPTLinitialize ()))
 	return ret;
-#endif    
-    
+#endif
+
     /* Initialize MPI. */
     if ((ret = MPI_Init(&argc, &argv)))
 	MPIERR(ret);
@@ -206,8 +206,8 @@ main(int argc, char **argv)
 	printf("%d: ParallelIO Library example1 running on %d processors.\n",
 	       my_rank, ntasks);
 
-    /* keep things simple - 1 iotask per MPI process */    
-    niotasks = ntasks; 
+    /* keep things simple - 1 iotask per MPI process */
+    niotasks = ntasks;
 
     /* Initialize the PIO IO system. This specifies how
      * many and which processors are involved in I/O. */
@@ -222,7 +222,7 @@ main(int argc, char **argv)
     for (int i = 0; i < elements_per_pe; i++) {
 	compdof[i] = my_rank * elements_per_pe + i + 1;
     }
-	
+
     /* Create the PIO decomposition for this test. */
     if (verbose)
 	printf("rank: %d Creating decomposition...\n", my_rank);
@@ -236,10 +236,10 @@ main(int argc, char **argv)
     if ((ret = MPE_Log_event(event_num[END][INIT], 0, "end init")))
 	MPIERR(ret);
 #endif /* HAVE_MPE */
-	
+
     /* Use PIO to create the example file in each of the four
      * available ways. */
-    for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++) 
+    for (int fmt = 0; fmt < NUM_NETCDF_FLAVORS; fmt++)
     {
 #ifdef HAVE_MPE
 	/* Log with MPE that we are starting CREATE. */
@@ -254,7 +254,7 @@ main(int argc, char **argv)
 	if ((ret = PIOc_createfile(iosysid, &ncid, &(format[fmt]), filename[fmt],
 				   PIO_CLOBBER)))
 	    ERR(ret);
-	
+
 	/* Define netCDF dimensions and variable. */
 	if (verbose)
 	    printf("rank: %d Defining netCDF metadata...\n", my_rank);
@@ -278,8 +278,8 @@ main(int argc, char **argv)
 	    int storage;
 	    size_t my_chunksize[NDIM];
 	    if ((ret = PIOc_inq_var_chunking(ncid, 0, &storage, my_chunksize)))
-	    	ERR(ret);
-	    
+		ERR(ret);
+
 	    /** For serial netCDF-4, only processor rank 0 gets the answers. */
 	    if (format[fmt] == PIO_IOTYPE_NETCDF4C && !my_rank ||
 		format[fmt] == PIO_IOTYPE_NETCDF4P)
@@ -288,7 +288,7 @@ main(int argc, char **argv)
 		    ERR(ERR_AWFUL);
 		for (int d = 0; d < NDIM; d++)
 		    if (my_chunksize[d] != chunksize[d])
-		    	ERR(ERR_AWFUL);
+			ERR(ERR_AWFUL);
 	    }
 
 	    /* Check that the inv_var_deflate functions works. */
@@ -296,7 +296,7 @@ main(int argc, char **argv)
 	    int deflate;
 	    int deflate_level;
 	    if ((ret = PIOc_inq_var_deflate(ncid, 0, &shuffle, &deflate, &deflate_level)))
-	    	ERR(ret);
+		ERR(ret);
 
 	    /** For serial netCDF-4, only processor rank 0 gets the
 	     * answers. Also deflate is turned on by default */
@@ -314,8 +314,8 @@ main(int argc, char **argv)
 	     * in the PIO_ENOTNC4 error. */
 	    if ((ret = PIOc_def_var_chunking(ncid, 0, NC_CHUNKED, chunksize)) != PIO_ENOTNC4)
 		ERR(ERR_AWFUL);
-	}	    
-	
+	}
+
 	if ((ret = PIOc_enddef(ncid)))
 	    ERR(ret);
 
@@ -325,13 +325,13 @@ main(int argc, char **argv)
 	if ((ret = PIOc_closefile(ncid)))
 	    ERR(ret);
     }
-	
+
     /* Free the PIO decomposition. */
     if (verbose)
 	printf("rank: %d Freeing PIO decomposition...\n", my_rank);
     if ((ret = PIOc_freedecomp(iosysid, ioid)))
 	ERR(ret);
-	
+
     /* Finalize the IO system. */
     if (verbose)
 	printf("rank: %d Freeing PIO resources...\n", my_rank);
@@ -341,11 +341,11 @@ main(int argc, char **argv)
     /* Finalize the MPI library. */
     MPI_Finalize();
 
-#ifdef TIMING    
+#ifdef TIMING
     /* Finalize the GPTL timing library. */
     if ((ret = GPTLfinalize ()))
 	return ret;
-#endif    
-    
+#endif
+
     return 0;
 }
diff --git a/externals/pio2/tests/general/util/pio_tf_f90gen.pl b/src/externals/pio2/tests/general/util/pio_tf_f90gen.pl
similarity index 90%
rename from externals/pio2/tests/general/util/pio_tf_f90gen.pl
rename to src/externals/pio2/tests/general/util/pio_tf_f90gen.pl
index f14b13480ef..f880cda1beb 100755
--- a/externals/pio2/tests/general/util/pio_tf_f90gen.pl
+++ b/src/externals/pio2/tests/general/util/pio_tf_f90gen.pl
@@ -69,7 +69,7 @@ sub generate_gen_templates
   my ($gen_templ_func_bodies) = "";
 
   # Find typenames for the template and corresponding concrete types
-  my (@typenames); 
+  my (@typenames);
   my ($ntypenames);
   my (@typename_counter, @typename_counter_max);
   my ($typename);
@@ -148,7 +148,7 @@ sub generate_gen_templates
 }
 
 # FIXME - Not used, kept around so that the logic is not lost
-# Could be used to generate templates with all permutations of the typenames 
+# Could be used to generate templates with all permutations of the typenames
 sub generate_gen_templates_all_permutes
 {
   my ($gen_templ_func_name, $ref_modif_gen_templ_func_names,
@@ -156,7 +156,7 @@ sub generate_gen_templates_all_permutes
   my ($gen_templ_func_bodies) = "";
 
   # Find typenames for the template and corresponding concrete types
-  my (@typenames); 
+  my (@typenames);
   my ($ntypenames);
   my (@typename_counter, @typename_counter_max);
   my ($typename);
@@ -298,11 +298,11 @@ sub transform_src
       $out_line = $out_line . $1 . "  END IF\n";
       $out_line = $out_line . $1 . "  IF (pio_tf_world_rank_ == 0) THEN\n";
       $out_line = $out_line . $1 . "    IF (pio_tf_retval_utest_ == 0) THEN\n";
-      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
       $out_line = $out_line . $1 . "        \"$_\",&\n";
       $out_line = $out_line . $1 . "        \"---------\",\"PASSED\"\n";
       $out_line = $out_line . $1 . "    ELSE\n";
-      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
       $out_line = $out_line . $1 . "        \"$_\",&\n";
       $out_line = $out_line . $1 . "        \"---------\",\"FAILED\"\n";
       $out_line = $out_line . $1 . "    END IF\n";
@@ -330,11 +330,11 @@ sub transform_src
       $out_line = $out_line . $1 . "  END IF\n";
       $out_line = $out_line . $1 . "  IF (pio_tf_world_rank_ == 0) THEN\n";
       $out_line = $out_line . $1 . "    IF (pio_tf_retval_utest_ == 0) THEN\n";
-      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
       $out_line = $out_line . $1 . "        pio_tf_tmp_log_str_,&\n";
       $out_line = $out_line . $1 . "        \"---------\",\"PASSED\"\n";
       $out_line = $out_line . $1 . "    ELSE\n";
-      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+      $out_line = $out_line . $1 . "      WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
       $out_line = $out_line . $1 . "        pio_tf_tmp_log_str_,&\n";
       $out_line = $out_line . $1 . "        \"---------\",\"FAILED\"\n";
       $out_line = $out_line . $1 . "    END IF\n";
@@ -361,20 +361,34 @@ sub transform_src
     $out_line = $out_line . $1 . "END IF\n";
     $out_line = $out_line . $1 . "IF (pio_tf_world_rank_ == 0) THEN\n";
     $out_line = $out_line . $1 . "  IF (pio_tf_retval_utest_ == 0) THEN\n";
-    $out_line = $out_line . $1 . "    WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+    $out_line = $out_line . $1 . "    WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
     $out_line = $out_line . $1 . "      pio_tf_tmp_log_str_,&\n";
     $out_line = $out_line . $1 . "      \"--------\", \"PASSED\"\n";
     $out_line = $out_line . $1 . "  ELSE\n";
-    $out_line = $out_line . $1 . "    WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+    $out_line = $out_line . $1 . "    WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
     $out_line = $out_line . $1 . "      pio_tf_tmp_log_str_,&\n";
     $out_line = $out_line . $1 . "      \"--------\", \"FAILED\"\n";
     $out_line = $out_line . $1 . "  END IF\n";
     $out_line = $out_line . $1 . "END IF";
     $cur_test_case_num += 1;
 }
+  elsif(/^(\s*)PIO_TF_PASSERT\((.+),([^,]+),([^)]+)\)(.*)$/s){
+    $out_line = $1 . $5 . "\n";
+    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_($2, $3))) THEN\n";
+    $out_line = $out_line . $1 . "  call MPI_COMM_RANK($3, pio_tf_tmp_comm_rank_, pio_tf_retval_utest_)\n";
+    $out_line = $out_line . $1 . "  pio_tf_retval_utest_ = -1\n";
+    $out_line = $out_line . $1 . "  IF (pio_tf_tmp_comm_rank_ == 0) THEN\n";
+    $out_line = $out_line . $1 . "    PRINT *, \"PIO_TF: Assertion failed :\",&\n";
+    $out_line = $out_line . $1 . "      " . $4 . ",&\n";
+    $out_line = $out_line . $1 . "       \":\", __FILE__, \":\", __LINE__,&\n";
+    $out_line = $out_line . $1 . "      " . "\"($template_fname:$template_line_no)\"\n";
+    $out_line = $out_line . $1 . "  END IF\n";
+    $out_line = $out_line . $1 . "  RETURN\n";
+    $out_line = $out_line . $1 . "END IF";
+  }
   elsif(/^(\s*)PIO_TF_PASSERT\((.+),([^)]+)\)(.*)$/s){
     $out_line = $1 . $4 . "\n";
-    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_($2))) THEN\n";
+    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_($2, pio_tf_comm_))) THEN\n";
     $out_line = $out_line . $1 . "  pio_tf_retval_utest_ = -1\n";
     $out_line = $out_line . $1 . "  IF (pio_tf_world_rank_ == 0) THEN\n";
     $out_line = $out_line . $1 . "    PRINT *, \"PIO_TF: Assertion failed :\",&\n";
@@ -396,9 +410,23 @@ sub transform_src
     $out_line = $out_line . $1 . "  RETURN\n";
     $out_line = $out_line . $1 . "END IF";
   }
+  elsif(/^(\s*)PIO_TF_CHECK_ERR\(([^,]+),([^,]+),(.+)\)(\s*)$/s){
+    $out_line = $1 . $5 . "\n";
+    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_(($2) == PIO_NOERR, $3))) THEN\n";
+    $out_line = $out_line . $1 . "  call MPI_COMM_RANK($3, pio_tf_tmp_comm_rank_, pio_tf_retval_utest_)\n";
+    $out_line = $out_line . $1 . "  pio_tf_retval_utest_ = -1\n";
+    $out_line = $out_line . $1 . "  IF (pio_tf_tmp_comm_rank_ == 0) THEN\n";
+    $out_line = $out_line . $1 . "    PRINT *, \"PIO_TF: PIO Function failed:\",&\n";
+    $out_line = $out_line . $1 . "      " . $4 . ",&\n";
+    $out_line = $out_line . $1 . "      \":\", __FILE__, \":\", __LINE__,&\n";
+    $out_line = $out_line . $1 . "      " . "\"($template_fname:$template_line_no)\"\n";
+    $out_line = $out_line . $1 . "  END IF\n";
+    $out_line = $out_line . $1 . "  RETURN\n";
+    $out_line = $out_line . $1 . "END IF";
+  }
   elsif(/^(\s*)PIO_TF_CHECK_ERR\(([^,]+),(.+)\)(\s*)$/s){
     $out_line = $1 . $4 . "\n";
-    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_(($2) == PIO_NOERR))) THEN\n";
+    $out_line = $out_line . $1 . "IF (.NOT. (PIO_TF_Passert_(($2) == PIO_NOERR, pio_tf_comm_))) THEN\n";
     $out_line = $out_line . $1 . "  pio_tf_retval_utest_ = -1\n";
     $out_line = $out_line . $1 . "  IF (pio_tf_world_rank_ == 0) THEN\n";
     $out_line = $out_line . $1 . "    PRINT *, \"PIO_TF: PIO Function failed:\",&\n";
@@ -455,8 +483,8 @@ sub transform_src
 # Returns 1 => if a generic template func is found, 0 otherwise
 sub find_gen_templ_funcs
 {
-  my($_, $ref_templ_typenames) = @_;
-  if(/^\s*PIO_TF_TEMPLATE\s*<(.*)>.*$/s){
+  my($str, $ref_templ_typenames) = @_;
+  if($str =~ /^\s*PIO_TF_TEMPLATE\s*<(.*)>.*$/s){
     if($1 eq ''){
       print "Error parsing template code, Check syntax of PIO_TF_TEMPLATE...\n";
       return 0;
@@ -555,7 +583,7 @@ sub update_auto_func_list_with_gen_templ
   my ($gen_templ_func_name);
   my (@modif_gen_templ_func_names);
   foreach $gen_templ_func_name (@{$ref_gen_templ_funcs_list}){
-    # Get the general template function names 
+    # Get the general template function names
     &generate_gen_templates($gen_templ_func_name, \@modif_gen_templ_func_names,
         $base_file_name, $ifline_num);
     my ($func_name);
@@ -573,20 +601,29 @@ sub get_default_test_main
   $out_line = $out_line . "  PROGRAM PIO_TF_Test_main_\n";
   $out_line = $out_line . "    USE pio_tutil\n";
   $out_line = $out_line . "    IMPLICIT NONE\n";
-  $out_line = $out_line . "    INTEGER ierr\n";
+  $out_line = $out_line . "    INTEGER, PARAMETER :: NREARRS = 2\n";
+  $out_line = $out_line . "    INTEGER :: rearrs(NREARRS) = (/pio_rearr_subset,pio_rearr_box/)\n";
+  $out_line = $out_line . "    CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: rearrs_info(NREARRS) = (/\"PIO_REARR_SUBSET\",\"PIO_REARR_BOX   \"/)\n";
+  $out_line = $out_line . "    INTEGER i, ierr\n";
   $out_line = $out_line . "\n";
   $out_line = $out_line . "    pio_tf_nerrs_total_=0\n";
   $out_line = $out_line . "    pio_tf_retval_utest_=0\n";
   $out_line = $out_line . "    CALL MPI_Init(ierr)\n";
-  $out_line = $out_line . "    CALL PIO_TF_Init_()\n";
-  $out_line = $out_line . "    CALL PIO_TF_Test_driver_()\n";
-  $out_line = $out_line . "    CALL PIO_TF_Finalize_()\n";
+  $out_line = $out_line . "    DO i=1,SIZE(rearrs)\n";
+  $out_line = $out_line . "      CALL PIO_TF_Init_(rearrs(i))\n";
+  $out_line = $out_line . "      IF (pio_tf_world_rank_ == 0) THEN\n";
+  $out_line = $out_line . "        WRITE(*,*) \"PIO_TF: Testing : \", trim(rearrs_info(i))\n";
+  $out_line = $out_line . "      END IF\n";
+  $out_line = $out_line . "      CALL PIO_TF_Test_driver_()\n";
+  $out_line = $out_line . "      CALL PIO_TF_Finalize_()\n";
+  $out_line = $out_line . "    END DO\n";
   $out_line = $out_line . "    IF (pio_tf_world_rank_ == 0) THEN\n";
   $out_line = $out_line . "      IF (pio_tf_nerrs_total_ == 0) THEN\n";
   $out_line = $out_line . "        IF (pio_tf_retval_utest_ == 0) THEN\n";
   $out_line = $out_line . "          WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF: \",&\n";
   $out_line = $out_line . "           \"All tests\", \"---------\", \"PASSED\"\n";
   $out_line = $out_line . "        ELSE\n";
+  $out_line = $out_line . "          pio_tf_nerrs_total_ = pio_tf_nerrs_total_ + 1\n";
   $out_line = $out_line . "          WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF: \",&\n";
   $out_line = $out_line . "           \"Test driver\", \"---------\", \"FAILED\"\n";
   $out_line = $out_line . "        END IF\n";
@@ -630,10 +667,10 @@ sub get_default_test_driver
     $out_line = $out_line . "    END IF\n";
     $out_line = $out_line . "    IF (pio_tf_world_rank_ == 0) THEN\n";
     $out_line = $out_line . "      IF (pio_tf_retval_utest_ == 0) THEN\n";
-    $out_line = $out_line . "        WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+    $out_line = $out_line . "        WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
     $out_line = $out_line . "          \"$_\",\"-----------\", \"PASSED\"\n";
     $out_line = $out_line . "      ELSE\n";
-    $out_line = $out_line . "        WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n"; 
+    $out_line = $out_line . "        WRITE(*,PIO_TF_TEST_RES_FMT) \"PIO_TF:Test $cur_test_case_num:\",&\n";
     $out_line = $out_line . "          \"$_\",\"-----------\", \"FAILED\"\n";
     $out_line = $out_line . "      END IF\n";
     $out_line = $out_line . "    END IF\n";
@@ -756,7 +793,7 @@ sub process_template_file
   $orig_line = "";
   while(){
     my ($out_line);
-  
+
     $orig_line = $orig_line . $_;
     chomp;
     $_ = $in_line . $_;
@@ -769,7 +806,7 @@ sub process_template_file
       # Strip off comments at the end of the line
       if(/^([^!]*)!(.*)$/s){
         if($verbose){ print "Stripping comment: $2\n"; }
-        $_ = $1; 
+        $_ = $1;
       }
       if(/^\s*PROGRAM.*/si){
         # Report error and exit if the template has a PROGRAM keyword
@@ -829,8 +866,8 @@ ()
 # Read input args
 GetOptions(
   # Annotate generated source with template line numbers etc
-  "annotate-source"		=>	\$annotate_source,
-  "out=s"				      =>	\$output_fname,
+  "annotate-source"             =>      \$annotate_source,
+  "out=s"                                     =>        \$output_fname,
   "verbose"           =>  \$verbose
 );
 
@@ -853,4 +890,3 @@ ()
 
 if($verbose){ print "Reading input args complete\n" }
 &process_template_file($template_fname, $output_fname);
-
diff --git a/externals/pio2/tests/general/util/pio_tutil.F90 b/src/externals/pio2/tests/general/util/pio_tutil.F90
similarity index 72%
rename from externals/pio2/tests/general/util/pio_tutil.F90
rename to src/externals/pio2/tests/general/util/pio_tutil.F90
index 398d9d2fbe5..e4a076f3a10 100644
--- a/externals/pio2/tests/general/util/pio_tutil.F90
+++ b/src/externals/pio2/tests/general/util/pio_tutil.F90
@@ -13,9 +13,10 @@ MODULE pio_tutil
     ENUMERATOR  ::  IARG_STRIDE_SIDX = 1
     ENUMERATOR  ::  IARG_NUM_IO_TASKS_SIDX
     ENUMERATOR  ::  IARG_NUM_AGGREGATORS_SIDX
+    ENUMERATOR  ::  IARG_LOG_LEVEL_SIDX
     ! Unfortunately since fortran starts with index 1 we need the
     ! hack below. Don't forget to update when adding more argvs
-    ENUMERATOR  ::  NUM_IARGS = IARG_NUM_AGGREGATORS_SIDX
+    ENUMERATOR  ::  NUM_IARGS = IARG_LOG_LEVEL_SIDX
     ENUMERATOR  ::  IARG_MAX_SIDX = NUM_IARGS
   END ENUM
 
@@ -25,6 +26,7 @@ MODULE pio_tutil
 
   ! MPI info
   INTEGER ::  pio_tf_world_rank_, pio_tf_world_sz_
+  INTEGER :: pio_tf_tmp_comm_rank_, pio_tf_tmp_comm_sz_
   INTEGER :: pio_tf_comm_
 
   ! REAL types
@@ -39,11 +41,11 @@ MODULE pio_tutil
   ! PIO_TF_TEST_RES_FMT is used for formatted test result output
   ! -- Useful for writes like
   ! HEADER_STRING, TEST_DESC, FOOTER_STRING, TEST_STATUS
-  ! "PIO_TF: Test no: 12", "Test name, desc etc", "-----", "PASSED" 
+  ! "PIO_TF: Test no: 12", "Test name, desc etc", "-----", "PASSED"
   CHARACTER(LEN=*), PARAMETER :: PIO_TF_TEST_RES_FMT = "(A20,T22,A40,T64,A6,T72,A6)"
   ! -- Useful for writes like
   ! HEADER_STRING, NUMBER_OF_TESTS, FOOTER_STRING, TEST_STATUS
-  ! "PIO_TF: [", 3, "] -----", "FAILED" 
+  ! "PIO_TF: [", 3, "] -----", "FAILED"
   CHARACTER(LEN=*), PARAMETER :: PIO_TF_TEST_RES_FMT2 = "(A20,T22,I5,T28,A10,T62,A16)"
   CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: pio_tf_tmp_log_str_
 
@@ -64,13 +66,23 @@ MODULE pio_tutil
   PUBLIC  :: PIO_TF_Get_data_types
   PUBLIC  :: PIO_TF_Check_val_
   ! Private functions
+  PRIVATE :: PIO_TF_Check_int_arr_arr_
   PRIVATE :: PIO_TF_Check_int_arr_val, PIO_TF_Check_int_arr_arr
   PRIVATE :: PIO_TF_Check_int_arr_arr_tol
+  PRIVATE :: PIO_TF_Check_2d_int_arr_arr
+  PRIVATE :: PIO_TF_Check_3d_int_arr_arr
   PRIVATE :: PIO_TF_Check_real_arr_val, PIO_TF_Check_real_arr_arr
+  PRIVATE :: PIO_TF_Check_real_arr_arr_tol_
   PRIVATE :: PIO_TF_Check_real_arr_arr_tol
+  PRIVATE :: PIO_TF_Check_2d_real_arr_arr
+  PRIVATE :: PIO_TF_Check_3d_real_arr_arr
   PRIVATE :: PIO_TF_Check_double_arr_val, PIO_TF_Check_double_arr_arr
+  PRIVATE :: PIO_TF_Check_double_arr_arr_tol_
   PRIVATE :: PIO_TF_Check_double_arr_arr_tol
+  PRIVATE :: PIO_TF_Check_2d_double_arr_arr
+  PRIVATE :: PIO_TF_Check_3d_double_arr_arr
   PRIVATE :: PIO_TF_Check_char_str_str
+  PRIVATE :: PIO_TF_Get_idx_from_1d_idx
 
   ! Note that the tolerance value provided is ignored when comparing two
   ! integer arrays
@@ -79,18 +91,24 @@ MODULE pio_tutil
         PIO_TF_Check_int_arr_val,     &
         PIO_TF_Check_int_arr_arr,     &
         PIO_TF_Check_int_arr_arr_tol, &
+        PIO_TF_Check_2d_int_arr_arr,  &
+        PIO_TF_Check_3d_int_arr_arr,  &
         PIO_TF_Check_real_arr_val,    &
         PIO_TF_Check_real_arr_arr,    &
+        PIO_TF_Check_2d_real_arr_arr, &
+        PIO_TF_Check_3d_real_arr_arr, &
         PIO_TF_Check_real_arr_arr_tol,&
         PIO_TF_Check_double_arr_val,  &
         PIO_TF_Check_double_arr_arr,  &
+        PIO_TF_Check_2d_double_arr_arr,&
+        PIO_TF_Check_3d_double_arr_arr,&
         PIO_TF_Check_double_arr_arr_tol,&
         PIO_TF_Check_char_str_str
   END INTERFACE
 
 CONTAINS
   ! Initialize Testing framework - Internal (Not directly used by unit tests)
-  SUBROUTINE  PIO_TF_Init_
+  SUBROUTINE  PIO_TF_Init_(rearr)
 #ifdef TIMING
    use perf_mod
 #endif
@@ -99,6 +117,7 @@ SUBROUTINE  PIO_TF_Init_
 #else
     include 'mpif.h'
 #endif
+    INTEGER, INTENT(IN) :: rearr
     INTEGER ierr
 
     CALL MPI_COMM_DUP(MPI_COMM_WORLD, pio_tf_comm_, ierr);
@@ -110,7 +129,7 @@ SUBROUTINE  PIO_TF_Init_
 
 
 
-    pio_tf_log_level_ = 0
+    pio_tf_log_level_ = 3
     pio_tf_num_aggregators_ = 0
     pio_tf_num_io_tasks_ = 0
     pio_tf_stride_ = 1
@@ -142,16 +161,25 @@ SUBROUTINE  PIO_TF_Init_
           pio_tf_num_io_tasks_,       &
           pio_tf_num_aggregators_,    &
           pio_tf_stride_,             &
-          PIO_rearr_subset,              &
+          rearr,                      &
           pio_tf_iosystem_,           &
           base=0)
 
     ! Set PIO to bcast error
     CALL PIO_seterrorhandling(pio_tf_iosystem_, PIO_BCAST_ERROR)
+
+    ! Set PIO logging level
+    ierr = PIO_set_log_level(pio_tf_log_level_)
+    if(ierr /= PIO_NOERR) then
+      PRINT *, "PIO_TF: Error setting PIO logging level"
+    end if
   END SUBROUTINE PIO_TF_Init_
 
   ! Finalize Testing framework - Internal (Not directly used by unit tests)
   SUBROUTINE  PIO_TF_Finalize_
+#ifdef TIMING
+   use perf_mod
+#endif
 #ifndef NO_MPIMOD
     use mpi
 #else
@@ -166,36 +194,45 @@ SUBROUTINE  PIO_TF_Finalize_
 
     ! Finalize PIO
     CALL PIO_finalize(pio_tf_iosystem_, ierr);
+    CALL MPI_COMM_FREE(pio_tf_comm_, ierr);
+
+#ifdef TIMING
+    call t_finalizef()
+#endif
   END SUBROUTINE PIO_TF_Finalize_
 
   ! Collective assert - Internal (Not directly used by unit tests)
   ! Each processes passes in its local assert condition and the function
   ! returns the global assert condition
-  LOGICAL FUNCTION PIO_TF_Passert_(local_result)
+  LOGICAL FUNCTION PIO_TF_Passert_(local_result, comm)
 #ifndef NO_MPIMOD
     use mpi
 #else
     include 'mpif.h'
 #endif
     LOGICAL, INTENT(IN) ::  local_result
+    INTEGER, INTENT(IN) ::  comm
     LOGICAL :: global_result
     LOGICAL :: failed, all_failed
     INTEGER :: rank, ierr
     LOGICAL, DIMENSION(:), ALLOCATABLE :: failed_ranks
 
-    CALL MPI_ALLREDUCE(local_result, global_result, 1, MPI_LOGICAL, MPI_LAND, pio_tf_comm_, ierr)
+    CALL MPI_COMM_RANK(comm, pio_tf_tmp_comm_rank_, ierr)
+    CALL MPI_COMM_SIZE(comm, pio_tf_tmp_comm_sz_, ierr)
+
+    CALL MPI_ALLREDUCE(local_result, global_result, 1, MPI_LOGICAL, MPI_LAND, comm, ierr)
     IF (.NOT. global_result) THEN
       failed = .NOT. local_result
 !      IF (pio_tf_world_rank_ == 0) THEN
-        ALLOCATE(failed_ranks(pio_tf_world_sz_))
+        ALLOCATE(failed_ranks(pio_tf_tmp_comm_sz_))
 !      END IF
       ! Gather the ranks where assertion failed
-      CALL MPI_GATHER(failed, 1, MPI_LOGICAL, failed_ranks, 1, MPI_LOGICAL, 0, pio_tf_comm_, ierr)
+      CALL MPI_GATHER(failed, 1, MPI_LOGICAL, failed_ranks, 1, MPI_LOGICAL, 0, comm, ierr)
 
       ! Display the ranks where the assertion failed
-      IF (pio_tf_world_rank_ == 0) THEN
+      IF (pio_tf_tmp_comm_rank_ == 0) THEN
         all_failed = .TRUE.
-        DO rank=1,pio_tf_world_sz_
+        DO rank=1,pio_tf_tmp_comm_sz_
           IF (.NOT. failed_ranks(rank)) THEN
             all_failed = .FALSE.
             ! Thank you - f90
@@ -207,7 +244,7 @@ LOGICAL FUNCTION PIO_TF_Passert_(local_result)
         ELSE
           PRINT *, "PIO_TF: Fatal Error: Assertion failed on following processes: "
           WRITE(*,"(A8)",ADVANCE="NO") "PIO_TF: "
-          DO rank=1,pio_tf_world_sz_
+          DO rank=1,pio_tf_tmp_comm_sz_
             IF (failed_ranks(rank)) THEN
               WRITE(*,"(I5,A)",ADVANCE="NO") rank-1, ","
             END IF
@@ -487,7 +524,52 @@ SUBROUTINE PIO_TF_Get_data_types(data_types, data_type_descs, num_data_types)
 
   END SUBROUTINE PIO_TF_Get_data_types
 
-  LOGICAL FUNCTION PIO_TF_Check_int_arr_arr(arr, exp_arr)
+  ! Get original (multi-d) index string from 1d (reshaped) index
+  SUBROUTINE PIO_TF_Get_idx_from_1d_idx(idx_1d, arr_shape, idx_str)
+    INTEGER, INTENT(IN) :: idx_1d
+    INTEGER, DIMENSION(:), INTENT(IN) :: arr_shape
+    CHARACTER(LEN=*), INTENT(OUT) :: idx_str
+
+    CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: fmt_str
+    ! Number of elems in that dim in "1D view"
+    INTEGER, DIMENSION(:), ALLOCATABLE :: dim_wgt
+    INTEGER, DIMENSION(:), ALLOCATABLE :: idx_md
+    INTEGER :: tmp_idx
+    INTEGER :: i, sz
+
+    idx_str = ""
+
+    sz = SIZE(arr_shape)
+    ALLOCATE(dim_wgt(sz))
+    ALLOCATE(idx_md(sz))
+
+    ! Assign place weights = num of elems in that dim in "1D view"
+    dim_wgt = 1
+    DO i=2,sz
+      dim_wgt(i) = dim_wgt(i-1) * arr_shape(i-1)
+    END DO
+
+    ! Convert 1d reshaped index to original multi-d index
+    tmp_idx = idx_1d - 1
+    idx_md = 0
+    DO i=sz,1,-1
+      idx_md(i) = tmp_idx / dim_wgt(i) + 1
+      tmp_idx = MOD(tmp_idx, dim_wgt(i))
+    END DO
+
+    IF(sz == 1) THEN
+      WRITE(fmt_str, *) "(",sz,"(I5)", ")"
+    ELSE
+      WRITE(fmt_str, *) "(",sz,"(I5,',')", ")"
+    END IF
+    WRITE(idx_str,fmt_str) idx_md
+
+    DEALLOCATE(idx_md)
+    DEALLOCATE(dim_wgt)
+
+  END SUBROUTINE PIO_TF_Get_idx_from_1d_idx
+
+  LOGICAL FUNCTION PIO_TF_Check_int_arr_arr_(arr, exp_arr, arr_shape)
 #ifndef NO_MPIMOD
     USE mpi
 #else
@@ -495,6 +577,8 @@ LOGICAL FUNCTION PIO_TF_Check_int_arr_arr(arr, exp_arr)
 #endif
     INTEGER, DIMENSION(:), INTENT(IN) :: arr
     INTEGER, DIMENSION(:), INTENT(IN) :: exp_arr
+    INTEGER, DIMENSION(:), INTENT(IN) :: arr_shape
+    CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: idx_str
     INTEGER :: arr_sz, i, ierr
     ! Not equal at id = nequal_idx
     INTEGER :: nequal_idx
@@ -536,14 +620,23 @@ LOGICAL FUNCTION PIO_TF_Check_int_arr_arr(arr, exp_arr)
       IF (pio_tf_world_rank_ == 0) THEN
          DO i=1,pio_tf_world_sz_
             IF(gfail_info(i) % idx /= -1) THEN
-               PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[", gfail_info(i) % idx, "]=", &
+               CALL PIO_TF_Get_idx_from_1d_idx(gfail_info(i) % idx, arr_shape, idx_str)
+               PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[",&
+                    trim(idx_str), "]=",&
                     gfail_info(i) % val, ", Expected = ", gfail_info(i) % exp_val
             END IF
          END DO
       END IF
       deallocate(gfail_info)
    end if
-    PIO_TF_Check_int_arr_arr = gequal
+    PIO_TF_Check_int_arr_arr_ = gequal
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_int_arr_arr(arr, exp_arr)
+    INTEGER, DIMENSION(:), INTENT(IN) :: arr
+    INTEGER, DIMENSION(:), INTENT(IN) :: exp_arr
+
+    PIO_TF_Check_int_arr_arr = PIO_TF_Check_int_arr_arr_(arr, exp_arr, SHAPE(arr))
   END FUNCTION
 
   ! Note that the tolerance value is ignored when comparing two integer arrays
@@ -568,7 +661,45 @@ LOGICAL FUNCTION PIO_TF_Check_int_arr_val(arr, val)
     DEALLOCATE(arr_val)
   END FUNCTION
 
-  LOGICAL FUNCTION PIO_TF_Check_real_arr_arr_tol(arr, exp_arr, tol)
+  LOGICAL FUNCTION PIO_TF_Check_2d_int_arr_arr(arr, exp_arr)
+    INTEGER, DIMENSION(:,:), INTENT(IN) :: arr
+    INTEGER, DIMENSION(:,:), INTENT(IN) :: exp_arr
+
+    INTEGER, DIMENSION(:), ALLOCATABLE :: arr_val
+    INTEGER, DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_2d_int_arr_arr = PIO_TF_Check_int_arr_arr_(arr_val, exp_arr_val,&
+                                    SHAPE(arr))
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_3d_int_arr_arr(arr, exp_arr)
+    INTEGER, DIMENSION(:,:,:), INTENT(IN) :: arr
+    INTEGER, DIMENSION(:,:,:), INTENT(IN) :: exp_arr
+
+    INTEGER, DIMENSION(:), ALLOCATABLE :: arr_val
+    INTEGER, DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_3d_int_arr_arr = PIO_TF_Check_int_arr_arr_(arr_val, exp_arr_val,&
+                                    SHAPE(arr))
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_real_arr_arr_tol_(arr, exp_arr, arr_shape, tol)
 #ifndef NO_MPIMOD
     USE mpi
 #else
@@ -576,8 +707,10 @@ LOGICAL FUNCTION PIO_TF_Check_real_arr_arr_tol(arr, exp_arr, tol)
 #endif
     REAL(KIND=fc_real), DIMENSION(:), INTENT(IN) :: arr
     REAL(KIND=fc_real), DIMENSION(:), INTENT(IN) :: exp_arr
+    INTEGER, DIMENSION(:), INTENT(IN) :: arr_shape
     REAL, INTENT(IN) :: tol
 
+    CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: idx_str
     INTEGER :: arr_sz, i, ierr
     ! Not equal at id = nequal_idx
     REAL(KIND=fc_real) :: nequal_idx
@@ -619,15 +752,25 @@ LOGICAL FUNCTION PIO_TF_Check_real_arr_arr_tol(arr, exp_arr, tol)
       IF (pio_tf_world_rank_ == 0) THEN
         DO i=1,pio_tf_world_sz_
           IF(INT(gfail_info(i) % idx) /= -1) THEN
-            PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[", &
-                 INT(gfail_info(i) % idx), "]=", gfail_info(i) % val, ", Expected = ", gfail_info(i) % exp_val
+            CALL PIO_TF_Get_idx_from_1d_idx(INT(gfail_info(i)%idx), arr_shape, idx_str)
+            PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[", trim(idx_str),&
+                 "]=", gfail_info(i) % val, ", Expected = ", gfail_info(i) % exp_val
           END IF
         END DO
       END IF
       DEALLOCATE(gfail_info)
    END IF
 
-    PIO_TF_Check_real_arr_arr_tol = gequal
+    PIO_TF_Check_real_arr_arr_tol_ = gequal
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_real_arr_arr_tol(arr, exp_arr, tol)
+    REAL(KIND=fc_real), DIMENSION(:), INTENT(IN) :: arr
+    REAL(KIND=fc_real), DIMENSION(:), INTENT(IN) :: exp_arr
+    REAL, INTENT(IN) :: tol
+
+    PIO_TF_Check_real_arr_arr_tol = PIO_TF_Check_real_arr_arr_tol_(arr, exp_arr,&
+                                SHAPE(arr), 0.0)
   END FUNCTION
 
   LOGICAL FUNCTION PIO_TF_Check_real_arr_arr(arr, exp_arr)
@@ -648,7 +791,45 @@ LOGICAL FUNCTION PIO_TF_Check_real_arr_val(arr, val)
     DEALLOCATE(arr_val)
   END FUNCTION
 
-  LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol(arr, exp_arr, tol)
+  LOGICAL FUNCTION PIO_TF_Check_2d_real_arr_arr(arr, exp_arr)
+    REAL(KIND=fc_real), DIMENSION(:,:), INTENT(IN) :: arr
+    REAL(KIND=fc_real), DIMENSION(:,:), INTENT(IN) :: exp_arr
+
+    REAL(KIND=fc_real), DIMENSION(:), ALLOCATABLE :: arr_val
+    REAL(KIND=fc_real), DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_2d_real_arr_arr = PIO_TF_Check_real_arr_arr_tol_(arr_val,&
+                                      exp_arr_val, SHAPE(arr), 0.0)
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_3d_real_arr_arr(arr, exp_arr)
+    REAL(KIND=fc_real), DIMENSION(:,:,:), INTENT(IN) :: arr
+    REAL(KIND=fc_real), DIMENSION(:,:,:), INTENT(IN) :: exp_arr
+
+    REAL(KIND=fc_real), DIMENSION(:), ALLOCATABLE :: arr_val
+    REAL(KIND=fc_real), DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_3d_real_arr_arr = PIO_TF_Check_real_arr_arr_tol_(arr_val,&
+                                      exp_arr_val, SHAPE(arr), 0.0)
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol_(arr, exp_arr, arr_shape, tol)
 #ifndef NO_MPIMOD
     USE mpi
 #else
@@ -659,7 +840,10 @@ LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol(arr, exp_arr, tol)
 #endif
     REAL(KIND=fc_double), DIMENSION(:), INTENT(IN) :: arr
     REAL(KIND=fc_double), DIMENSION(:), INTENT(IN) :: exp_arr
+    INTEGER, DIMENSION(:), INTENT(IN) :: arr_shape
     REAL, INTENT(IN) :: tol
+
+    CHARACTER(LEN=PIO_TF_MAX_STR_LEN) :: idx_str
     INTEGER :: arr_sz, i, ierr
     ! Not equal at id = nequal_idx
     REAL(KIND=fc_double) :: nequal_idx
@@ -703,8 +887,9 @@ LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol(arr, exp_arr, tol)
       IF (pio_tf_world_rank_ == 0) THEN
         DO i=1,pio_tf_world_sz_
           IF(INT(gfail_info(i) % idx) /= -1) THEN
-            PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[", &
-                  INT(gfail_info(i) % idx), "]=", gfail_info(i) % val, &
+            CALL PIO_TF_Get_idx_from_1d_idx(INT(gfail_info(i)%idx), arr_shape, idx_str)
+            PRINT *, "PIO_TF: Fatal Error: rank =", i, ", Val[",&
+                  trim(idx_str), "]=", gfail_info(i) % val,&
                   ", Expected = ", gfail_info(i) % exp_val
           END IF
         END DO
@@ -712,7 +897,16 @@ LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol(arr, exp_arr, tol)
       DEALLOCATE(gfail_info)
    END IF
 
-    PIO_TF_Check_double_arr_arr_tol = gequal
+    PIO_TF_Check_double_arr_arr_tol_ = gequal
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_double_arr_arr_tol(arr, exp_arr, tol)
+    REAL(KIND=fc_double), DIMENSION(:), INTENT(IN) :: arr
+    REAL(KIND=fc_double), DIMENSION(:), INTENT(IN) :: exp_arr
+    REAL, INTENT(IN) :: tol
+
+    PIO_TF_Check_double_arr_arr_tol = PIO_TF_Check_double_arr_arr_tol_(arr, exp_arr,&
+                                    SHAPE(arr), 0.0)
   END FUNCTION
 
   LOGICAL FUNCTION PIO_TF_Check_double_arr_arr(arr, exp_arr)
@@ -733,11 +927,53 @@ LOGICAL FUNCTION PIO_TF_Check_double_arr_val(arr, val)
     DEALLOCATE(arr_val)
   END FUNCTION
 
+  LOGICAL FUNCTION PIO_TF_Check_2d_double_arr_arr(arr, exp_arr)
+    REAL(KIND=fc_double), DIMENSION(:,:), INTENT(IN) :: arr
+    REAL(KIND=fc_double), DIMENSION(:,:), INTENT(IN) :: exp_arr
+
+    REAL(KIND=fc_double), DIMENSION(:), ALLOCATABLE :: arr_val
+    REAL(KIND=fc_double), DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_2d_double_arr_arr = PIO_TF_Check_double_arr_arr_tol_(arr_val,&
+                                      exp_arr_val, SHAPE(arr), 0.0)
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
+  LOGICAL FUNCTION PIO_TF_Check_3d_double_arr_arr(arr, exp_arr)
+    REAL(KIND=fc_double), DIMENSION(:,:,:), INTENT(IN) :: arr
+    REAL(KIND=fc_double), DIMENSION(:,:,:), INTENT(IN) :: exp_arr
+
+    REAL(KIND=fc_double), DIMENSION(:), ALLOCATABLE :: arr_val
+    REAL(KIND=fc_double), DIMENSION(:), ALLOCATABLE :: exp_arr_val
+    INTEGER, PARAMETER :: NDIMS = 2
+
+    ALLOCATE(arr_val(SIZE(arr)))
+    ALLOCATE(exp_arr_val(SIZE(exp_arr)))
+    arr_val = RESHAPE(arr,(/SIZE(arr)/))
+    exp_arr_val = RESHAPE(exp_arr,(/SIZE(exp_arr)/))
+
+    PIO_TF_Check_3d_double_arr_arr = PIO_TF_Check_double_arr_arr_tol_(arr_val,&
+                                      exp_arr_val, SHAPE(arr), 0.0)
+    DEALLOCATE(arr_val)
+    DEALLOCATE(exp_arr_val)
+  END FUNCTION
+
   LOGICAL FUNCTION PIO_TF_Check_char_str_str(str1, str2)
     CHARACTER(LEN=*), INTENT(IN)  :: str1
     CHARACTER(LEN=*), INTENT(IN)  :: str2
 
-    PIO_TF_Check_char_str_str = .TRUE.
+    IF (str1 == str2) THEN
+      PIO_TF_Check_char_str_str = .TRUE.
+    ELSE
+      PIO_TF_Check_char_str_str = .FALSE.
+    END IF
   END FUNCTION
 
   ! Parse and process input arguments like "--pio-tf-stride=2" passed
@@ -761,6 +997,8 @@ SUBROUTINE Parse_and_process_input(argv)
         READ(argv(pos+1:), *) pio_tf_num_aggregators_
       ELSE IF (argv(:pos) == "--pio-tf-stride=") THEN
         READ(argv(pos+1:), *) pio_tf_stride_
+      ELSE IF (argv(:pos) == "--pio-tf-log-level=") THEN
+        READ(argv(pos+1:), *) pio_tf_log_level_
       ELSE IF (argv(:pos) == "--pio-tf-input-file=") THEN
         PRINT *, "This option is not implemented yet"
       END IF
@@ -791,12 +1029,14 @@ SUBROUTINE Read_input()
       send_buf(IARG_STRIDE_SIDX) = pio_tf_stride_
       send_buf(IARG_NUM_IO_TASKS_SIDX) = pio_tf_num_io_tasks_
       send_buf(IARG_NUM_AGGREGATORS_SIDX) = pio_tf_num_aggregators_
+      send_buf(IARG_LOG_LEVEL_SIDX) = pio_tf_log_level_
     END IF
     ! Make sure all processes get the input args
     CALL MPI_BCAST(send_buf, NUM_IARGS, MPI_INTEGER, 0, pio_tf_comm_, ierr)
     pio_tf_stride_ = send_buf(IARG_STRIDE_SIDX)
     pio_tf_num_io_tasks_ = send_buf(IARG_NUM_IO_TASKS_SIDX)
     pio_tf_num_aggregators_ = send_buf(IARG_NUM_AGGREGATORS_SIDX)
+    pio_tf_log_level_ = send_buf(IARG_LOG_LEVEL_SIDX)
 
   END SUBROUTINE Read_input
 END MODULE pio_tutil
diff --git a/externals/pio2/tests/performance/CMakeLists.txt b/src/externals/pio2/tests/performance/CMakeLists.txt
similarity index 56%
rename from externals/pio2/tests/performance/CMakeLists.txt
rename to src/externals/pio2/tests/performance/CMakeLists.txt
index ea9efbf0a51..fce71424308 100644
--- a/externals/pio2/tests/performance/CMakeLists.txt
+++ b/src/externals/pio2/tests/performance/CMakeLists.txt
@@ -3,22 +3,22 @@
 #==============================================================================
 
 add_executable (pioperf EXCLUDE_FROM_ALL
-                pioperformance.F90)
+  pioperformance.F90)
 target_link_libraries (pioperf piof)
 add_dependencies (tests pioperf)
 
 if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
-    target_compile_options (pioperf
-        PRIVATE -ffree-line-length-none)
+  target_compile_options (pioperf
+    PRIVATE -ffree-line-length-none)
 endif()
 
 if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
-   set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
-#    target_compile_options (gptl
-#        PRIVATE -mismatch_all)
+  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
+  #    target_compile_options (gptl
+  #        PRIVATE -mismatch_all)
 endif ()
 
-if (PIO_HDF5_LOGGING) 
-    target_compile_definitions (pioperf 
-        PUBLIC LOGGING)
+if (PIO_HDF5_LOGGING)
+  target_compile_definitions (pioperf
+    PUBLIC LOGGING)
 endif ()
diff --git a/externals/pio2/tests/performance/gensimple.pl b/src/externals/pio2/tests/performance/gensimple.pl
similarity index 100%
rename from externals/pio2/tests/performance/gensimple.pl
rename to src/externals/pio2/tests/performance/gensimple.pl
diff --git a/externals/pio2/tests/performance/kt.PIO1.perfmakefile b/src/externals/pio2/tests/performance/kt.PIO1.perfmakefile
similarity index 100%
rename from externals/pio2/tests/performance/kt.PIO1.perfmakefile
rename to src/externals/pio2/tests/performance/kt.PIO1.perfmakefile
diff --git a/externals/pio2/tests/performance/pioperformance.F90 b/src/externals/pio2/tests/performance/pioperformance.F90
similarity index 100%
rename from externals/pio2/tests/performance/pioperformance.F90
rename to src/externals/pio2/tests/performance/pioperformance.F90
diff --git a/src/externals/pio2/tests/unit/CMakeLists.txt b/src/externals/pio2/tests/unit/CMakeLists.txt
new file mode 100644
index 00000000000..eaf7c0f5694
--- /dev/null
+++ b/src/externals/pio2/tests/unit/CMakeLists.txt
@@ -0,0 +1,63 @@
+include (LibMPI)
+
+include_directories("${CMAKE_SOURCE_DIR}/tests/unit")
+
+#==============================================================================
+#  PREPARE FOR TESTING
+#==============================================================================
+
+file (COPY "./input.nl"
+  DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file (COPY "./not_netcdf.ieee"
+  DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
+#==============================================================================
+#  DEFINE THE TARGETS AND TESTS
+#==============================================================================
+
+set (SRCS basic_tests.F90
+  driver.F90
+  global_vars.F90
+  ncdf_tests.F90)
+
+add_executable (pio_unit_test EXCLUDE_FROM_ALL ${SRCS})
+target_link_libraries (pio_unit_test piof)
+if ("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
+  target_compile_options (pio_unit_test
+    PRIVATE -ffree-line-length-none)
+endif()
+
+if (CMAKE_Fortran_COMPILER_ID STREQUAL "NAG")
+  set ( CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -mismatch_all" )
+  #    target_compile_options (gptl
+  #        PRIVATE -mismatch_all)
+endif ()
+
+add_dependencies (tests pio_unit_test)
+
+# Test Timeout in seconds.
+set (DEFAULT_TEST_TIMEOUT 60)
+
+# All tests need a certain number of tasks, but they should be able to
+# run successfully with more than they need. Test this by providing an
+# extra processor for each C test.
+set (AT_LEAST_TWO_TASKS 3)
+set (AT_LEAST_THREE_TASKS 4)
+set (AT_LEAST_FOUR_TASKS 5)
+
+if (PIO_USE_MPISERIAL)
+  add_test(NAME pio_unit_test
+    COMMAND pio_unit_test)
+  set_tests_properties(pio_unit_test
+    PROPERTIES TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+else ()
+  add_mpi_test(pio_unit_test
+    EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/pio_unit_test
+    NUMPROCS 4
+    TIMEOUT ${DEFAULT_TEST_TIMEOUT})
+endif ()
+
+if (PIO_HDF5_LOGGING)
+  target_compile_definitions (pio_unit_test
+    PUBLIC LOGGING)
+endif ()
diff --git a/externals/pio2/tests/unit/Levy_Notes b/src/externals/pio2/tests/unit/Levy_Notes
similarity index 100%
rename from externals/pio2/tests/unit/Levy_Notes
rename to src/externals/pio2/tests/unit/Levy_Notes
diff --git a/externals/pio2/tests/unit/README b/src/externals/pio2/tests/unit/README
similarity index 100%
rename from externals/pio2/tests/unit/README
rename to src/externals/pio2/tests/unit/README
diff --git a/externals/pio2/tests/unit/basic_tests.F90 b/src/externals/pio2/tests/unit/basic_tests.F90
similarity index 87%
rename from externals/pio2/tests/unit/basic_tests.F90
rename to src/externals/pio2/tests/unit/basic_tests.F90
index de41cef69d4..daad01babca 100644
--- a/externals/pio2/tests/unit/basic_tests.F90
+++ b/src/externals/pio2/tests/unit/basic_tests.F90
@@ -33,7 +33,7 @@ Subroutine test_create(test_id, err_msg)
 
       ! Local Vars
       character(len=str_len) :: filename
-      integer                :: iotype, ret_val, pio_dim
+      integer                :: iotype, ret_val, ret_val2, pio_dim
 
       err_msg = "no_error"
 
@@ -51,7 +51,7 @@ Subroutine test_create(test_id, err_msg)
         ! Error in PIO_createfile
          print *,' ret_val = ', ret_val
         err_msg = "Could not create " // trim(filename)
-        call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+        call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
       end if
       
       call mpi_barrier(mpi_comm_world,ret_val)
@@ -62,7 +62,7 @@ Subroutine test_create(test_id, err_msg)
           ! Error in PIO_enddef
           err_msg = "Could not end define mode"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
       end if
       call PIO_closefile(pio_file)
@@ -73,7 +73,7 @@ Subroutine test_create(test_id, err_msg)
       if (ret_val .ne. PIO_NOERR) then
         ! Error in PIO_openfile
         err_msg = "Could not open " // trim(filename)
-        call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+        call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
       end if
 
       ! Close file
@@ -85,7 +85,7 @@ Subroutine test_create(test_id, err_msg)
         if (ret_val .ne. PIO_NOERR) then
           ! Error in PIO_createfile
           err_msg = "Could not clobber " // trim(filename)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Leave define mode
@@ -94,7 +94,7 @@ Subroutine test_create(test_id, err_msg)
           ! Error in PIO_enddef
           err_msg = "Could not end define mode in clobbered file"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Close file
@@ -113,7 +113,7 @@ Subroutine test_create(test_id, err_msg)
           err_msg = "Was able to clobber file despite PIO_NOCLOBBER"
           ret_val = PIO_enddef(pio_file)
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
       end if
 
@@ -137,7 +137,7 @@ Subroutine test_open(test_id, err_msg)
 
       ! Local Vars
       character(len=str_len) :: filename
-      integer                :: iotype, ret_val
+      integer                :: iotype, ret_val, ret_val2
 
       ! Data used to test writing
       integer,          dimension(3) :: data_buffer, compdof
@@ -147,6 +147,12 @@ Subroutine test_open(test_id, err_msg)
       integer :: unlimdimid
       type(var_desc_t)               :: pio_var
 
+      ! These will be used to set chunk cache sizes in netCDF-4/HDF5
+      ! files.
+      integer(kind=PIO_OFFSET_KIND) :: chunk_cache_size
+      integer(kind=PIO_OFFSET_KIND) :: chunk_cache_nelems
+      real :: chunk_cache_preemption
+
       err_msg = "no_error"
       dims(1) = 3*ntasks
       compdof = 3*my_rank+(/1,2,3/)  ! Where in the global array each task writes
@@ -166,7 +172,7 @@ Subroutine test_open(test_id, err_msg)
         ! Error in PIO_openfile
         err_msg = "Successfully opened file that doesn't exist"
         call PIO_closefile(pio_file)
-        call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+        call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
       end if
 
       ! Open existing file, write data to it (for binary file, need to create new file)
@@ -178,17 +184,16 @@ Subroutine test_open(test_id, err_msg)
       if (ret_val .ne. PIO_NOERR) then
         ! Error in PIO_openfile (or PIO_createfile)
         err_msg = "Could not open " // trim(filename) // " in write mode"
-        call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+        call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
       end if
 
       ! Enter define mode for netcdf files
       if (is_netcdf(iotype)) then
         ret_val = PIO_redef(pio_file)
         if (ret_val .ne. PIO_NOERR) then
-          ! Error in PIO_redef
           err_msg = "Could not enter redef mode"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Define a new dimension N
@@ -197,7 +202,7 @@ Subroutine test_open(test_id, err_msg)
           ! Error in PIO_def_dim
           err_msg = "Could not define dimension N"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Define a new variable foo
@@ -207,7 +212,7 @@ Subroutine test_open(test_id, err_msg)
           ! Error in PIO_def_var
           err_msg = "Could not define variable foo"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Leave define mode
@@ -217,7 +222,7 @@ Subroutine test_open(test_id, err_msg)
            print *,__FILE__,__LINE__,ret_val
           err_msg = "Could not end define mode"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
       end if
 
@@ -229,7 +234,7 @@ Subroutine test_open(test_id, err_msg)
         ! Error in PIO_write_darray
         err_msg = "Could not write data"
         call PIO_closefile(pio_file)
-        call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+        call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
       end if
 
       ! Close file
@@ -242,7 +247,7 @@ Subroutine test_open(test_id, err_msg)
         if (ret_val .ne. PIO_NOERR) then
           ! Error opening file
           err_msg = "Could not open file in NoWrite mode"
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         ! Try to write (should fail)
@@ -254,7 +259,7 @@ Subroutine test_open(test_id, err_msg)
           ! Error in PIO_write_darray
           err_msg = "Wrote to file opened in NoWrite mode"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
         call mpi_barrier(MPI_COMM_WORLD,ret_val)
@@ -267,25 +272,25 @@ Subroutine test_open(test_id, err_msg)
           err_msg = "Error in read_darray"
           call PIO_closefile(pio_file)
           print *,__FILE__,__LINE__,err_msg
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
         if(any(data_buffer /= my_rank)) then
           err_msg = "Error reading data"
           call PIO_closefile(pio_file)
           print *,__FILE__,__LINE__,iotype, trim(err_msg), data_buffer
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
 
+        ret_val = PIO_set_log_level(3)
         ret_val = PIO_inq_unlimdim(pio_file, unlimdimid)
         if(unlimdimid /= -1) then
            err_msg = "Error in inq_unlimdim"
            call PIO_closefile(pio_file)
            print *,__FILE__,__LINE__,iotype, trim(err_msg)
-           call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+           call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
-
+        ret_val = PIO_set_log_level(0)
         
-
         ! Close file
         call PIO_closefile(pio_file)
       end if
@@ -303,7 +308,7 @@ Subroutine test_open(test_id, err_msg)
           ! Error in PIO_openfile
           err_msg = "Opened a non-netcdf file as netcdf"
           call PIO_closefile(pio_file)
-          call mpi_abort(MPI_COMM_WORLD,0,ret_val)
+          call mpi_abort(MPI_COMM_WORLD, 0, ret_val2)
         end if
       end if
 
diff --git a/externals/pio2/tests/unit/driver.F90 b/src/externals/pio2/tests/unit/driver.F90
similarity index 90%
rename from externals/pio2/tests/unit/driver.F90
rename to src/externals/pio2/tests/unit/driver.F90
index 7dbcfada7ed..a955339ee08 100644
--- a/externals/pio2/tests/unit/driver.F90
+++ b/src/externals/pio2/tests/unit/driver.F90
@@ -23,9 +23,10 @@ Program pio_unit_test_driver
        ltest_netcdf4c,     &
        ltest_pnetcdf,    &
        stride
-#if defined( _NETCDF4) && defined(LOGGING)
-  integer, external :: nc_set_log_level2
-#endif
+  integer ret_val
+  character(len=pio_max_name) :: errmsg
+  character(len=pio_max_name) :: expected
+  
   ! Set up MPI
   call MPI_Init(ierr)
   call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr)
@@ -62,7 +63,7 @@ Program pio_unit_test_driver
 
      ! Ignore namelist values if PIO not built with correct options
      ! (i.e. don't test pnetcdf if not built with pnetcdf)
-
+     ret_val = PIO_set_log_level(2)
 #ifndef _NETCDF
      if (ltest_netcdf) then
         write(*,"(A,1x,A)") "WARNING: can not test netcdf files because PIO", &
@@ -116,11 +117,22 @@ Program pio_unit_test_driver
        PIO_rearr_subset,  & ! rearr
        pio_iosystem, base=1)     ! iosystem
 
-  call PIO_seterrorhandling(pio_iosystem, PIO_BCAST_ERROR)
+  call PIO_seterrorhandling(pio_iosystem, PIO_RETURN_ERROR)
+  call PIO_seterrorhandling(PIO_DEFAULT, PIO_RETURN_ERROR)
 
   fail_cnt = 0
   test_cnt = 0
 
+  ! Test pio_strerror.
+  ret_val = PIO_strerror(-33, errmsg);
+  print *, 'errcode =', -33, ' strerror = ', errmsg
+  expected = 'NetCDF: Not a valid ID'
+  if (trim(errmsg) .ne. expected) then
+     err_msg = 'expected ' // trim(expected) // ' and got ' // trim(errmsg)
+     print *, err_msg
+     call parse(err_msg, fail_cnt)
+  end if
+     
   do test_id=1,ntest
      if (ltest(test_id)) then
         ! Make sure i is a valid test number
@@ -142,9 +154,7 @@ Program pio_unit_test_driver
                 write(*,"(A,I0)") "Error, not configured for test #", test_id
            call MPI_Abort(MPI_COMM_WORLD, 0, ierr)
         end select
-#if defined( _NETCDF4) && defined(LOGGING)
-        if(master_task) ierr = nc_set_log_level2(3)
-#endif
+        
         ! test_create()
         if (master_task) write(*,"(3x,A,1x)") "testing PIO_createfile..."
         call test_create(test_id, err_msg)
@@ -193,6 +203,9 @@ Program pio_unit_test_driver
   end if
 
   call PIO_finalize(pio_iosystem, ierr)
+#ifdef TIMING
+  call t_finalizef()
+#endif
   call MPI_Finalize(ierr)
   if(fail_cnt>0) then
      stop 1
diff --git a/externals/pio2/tests/unit/global_vars.F90 b/src/externals/pio2/tests/unit/global_vars.F90
similarity index 96%
rename from externals/pio2/tests/unit/global_vars.F90
rename to src/externals/pio2/tests/unit/global_vars.F90
index 9e8fced0353..f3347ba38c9 100644
--- a/externals/pio2/tests/unit/global_vars.F90
+++ b/src/externals/pio2/tests/unit/global_vars.F90
@@ -12,7 +12,7 @@ module global_vars
 
   include 'mpif.h' ! _EXTERNAL
 
-  integer, parameter :: str_len = 255, ntest=4
+  integer, parameter :: str_len = pio_max_name, ntest=4
    integer, parameter ::NETCDF =1, &
                         NETCDF4P=2, &
                         NETCDF4C=3, &
diff --git a/externals/pio2/tests/unit/input.nl b/src/externals/pio2/tests/unit/input.nl
similarity index 100%
rename from externals/pio2/tests/unit/input.nl
rename to src/externals/pio2/tests/unit/input.nl
diff --git a/externals/pio2/tests/unit/nc_set_log_level2.c b/src/externals/pio2/tests/unit/nc_set_log_level2.c
similarity index 64%
rename from externals/pio2/tests/unit/nc_set_log_level2.c
rename to src/externals/pio2/tests/unit/nc_set_log_level2.c
index deaf9087561..ad64d8e3309 100644
--- a/externals/pio2/tests/unit/nc_set_log_level2.c
+++ b/src/externals/pio2/tests/unit/nc_set_log_level2.c
@@ -3,8 +3,8 @@
 
 int nc_set_log_level2_(int *il)
 {
-  int i;
-  i = nc_set_log_level( *il );
-  return(i);
+    int i;
+    i = nc_set_log_level( *il );
+    return(i);
 }
 #endif
diff --git a/externals/pio2/tests/unit/ncdf_tests.F90 b/src/externals/pio2/tests/unit/ncdf_tests.F90
similarity index 99%
rename from externals/pio2/tests/unit/ncdf_tests.F90
rename to src/externals/pio2/tests/unit/ncdf_tests.F90
index 55459012de0..c811a6ddd0c 100644
--- a/externals/pio2/tests/unit/ncdf_tests.F90
+++ b/src/externals/pio2/tests/unit/ncdf_tests.F90
@@ -349,13 +349,14 @@ Subroutine test_nc4(test_id, err_msg)
        if (ret_val .ne. PIO_NOERR) then
           err_msg = "Could not set chunk cache"
           call PIO_closefile(pio_file)
+          return
        endif
        if (chunk_cache_size_in .ne. chunk_cache_size .or. chunk_cache_nelems_in .ne. chunk_cache_nelems .or. &
             chunk_cache_preemption_in .ne. chunk_cache_preemption) then
           err_msg = "Incorrect chunk cache values!"
           call PIO_closefile(pio_file)
+          return
        endif
-       return
     else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
        err_msg = "Did not get expected error when trying to set chunk cache for pnetcdf file"
        call PIO_closefile(pio_file)
@@ -441,14 +442,15 @@ Subroutine test_nc4(test_id, err_msg)
        if (ret_val .ne. PIO_NOERR) then
           err_msg = "Could not set variable chunk cache"
           call PIO_closefile(pio_file)
+          return
        endif
        if (chunk_cache_size_in .ne. chunk_cache_size .or. &
             chunk_cache_nelems_in .ne. chunk_cache_nelems .or. &
             chunk_cache_preemption_in .ne. chunk_cache_preemption) then
           err_msg = "Incorrect variable chunk cache values!"
           call PIO_closefile(pio_file)
+          return
        endif
-       return
     else if (iotype .eq. PIO_iotype_pnetcdf .and. ret_val .eq. PIO_NOERR) then
        err_msg = "Did not get expected error when trying to get variable chunk cache for pnetcdf file"
        call PIO_closefile(pio_file)
diff --git a/externals/pio2/tests/unit/not_netcdf.ieee b/src/externals/pio2/tests/unit/not_netcdf.ieee
similarity index 100%
rename from externals/pio2/tests/unit/not_netcdf.ieee
rename to src/externals/pio2/tests/unit/not_netcdf.ieee
diff --git a/externals/pio2/tests/unit/test_names.c b/src/externals/pio2/tests/unit/test_names.c
similarity index 100%
rename from externals/pio2/tests/unit/test_names.c
rename to src/externals/pio2/tests/unit/test_names.c
diff --git a/externals/pio2/tests/unit/test_nc4.c b/src/externals/pio2/tests/unit/test_nc4.c
similarity index 100%
rename from externals/pio2/tests/unit/test_nc4.c
rename to src/externals/pio2/tests/unit/test_nc4.c
diff --git a/share/csm_share/README b/src/share/README
similarity index 79%
rename from share/csm_share/README
rename to src/share/README
index 1315e06147a..dff6073505b 100644
--- a/share/csm_share/README
+++ b/src/share/README
@@ -5,10 +5,10 @@
 
                   A description of csm_share
 
-This module exists to collect code shared between various CCSM components.
-Excluding this "shared code" module, CCSM components are built using disjoint
+This module exists to collect code shared between various CIME components.
+Excluding this "shared code" module, CIME components are built using disjoint
 sets of source code.  The use of this shared code is similar to the use of
-object code libraries where each subdirectory of csm_share is equivalant to
+object code libraries where each subdirectory of share is equivalant to
 one library.  While object library routines are accessed by linking to libraries
 during the load phase, these shared code routines are accessed by including the
 appropriate source code directory path during the compile phase.
@@ -25,7 +25,7 @@ Motivation for this code sharing includes:
 
 Current subsets ("libraries") of shared code only include:
 
-shr - very generic, general-purpose code that is likely to be useful to all
-      CCSM components.  CCSM components may be explicitly required to use some
+util - very generic, general-purpose code that is likely to be useful to all
+      CIME components.  CIME components may be explicitly required to use some
       parts of this code, for example the physical constants module.
 
diff --git a/share/shr_RandNum/include/dSFMT-common.h b/src/share/RandNum/include/dSFMT-common.h
similarity index 100%
rename from share/shr_RandNum/include/dSFMT-common.h
rename to src/share/RandNum/include/dSFMT-common.h
diff --git a/share/shr_RandNum/include/dSFMT-params.h b/src/share/RandNum/include/dSFMT-params.h
similarity index 100%
rename from share/shr_RandNum/include/dSFMT-params.h
rename to src/share/RandNum/include/dSFMT-params.h
diff --git a/share/shr_RandNum/include/dSFMT-params19937.h b/src/share/RandNum/include/dSFMT-params19937.h
similarity index 100%
rename from share/shr_RandNum/include/dSFMT-params19937.h
rename to src/share/RandNum/include/dSFMT-params19937.h
diff --git a/share/shr_RandNum/include/dSFMT.h b/src/share/RandNum/include/dSFMT.h
similarity index 100%
rename from share/shr_RandNum/include/dSFMT.h
rename to src/share/RandNum/include/dSFMT.h
diff --git a/share/shr_RandNum/src/dsfmt_f03/dSFMT.c b/src/share/RandNum/src/dsfmt_f03/dSFMT.c
similarity index 100%
rename from share/shr_RandNum/src/dsfmt_f03/dSFMT.c
rename to src/share/RandNum/src/dsfmt_f03/dSFMT.c
diff --git a/share/shr_RandNum/src/dsfmt_f03/dSFMT_interface.F90 b/src/share/RandNum/src/dsfmt_f03/dSFMT_interface.F90
similarity index 100%
rename from share/shr_RandNum/src/dsfmt_f03/dSFMT_interface.F90
rename to src/share/RandNum/src/dsfmt_f03/dSFMT_interface.F90
diff --git a/share/shr_RandNum/src/dsfmt_f03/dSFMT_utils.c b/src/share/RandNum/src/dsfmt_f03/dSFMT_utils.c
similarity index 100%
rename from share/shr_RandNum/src/dsfmt_f03/dSFMT_utils.c
rename to src/share/RandNum/src/dsfmt_f03/dSFMT_utils.c
diff --git a/share/shr_RandNum/src/kissvec/kissvec.c b/src/share/RandNum/src/kissvec/kissvec.c
similarity index 100%
rename from share/shr_RandNum/src/kissvec/kissvec.c
rename to src/share/RandNum/src/kissvec/kissvec.c
diff --git a/share/shr_RandNum/src/kissvec/kissvec_mod.F90 b/src/share/RandNum/src/kissvec/kissvec_mod.F90
similarity index 100%
rename from share/shr_RandNum/src/kissvec/kissvec_mod.F90
rename to src/share/RandNum/src/kissvec/kissvec_mod.F90
diff --git a/share/shr_RandNum/src/mt19937/mersennetwister_mod.F90 b/src/share/RandNum/src/mt19937/mersennetwister_mod.F90
similarity index 100%
rename from share/shr_RandNum/src/mt19937/mersennetwister_mod.F90
rename to src/share/RandNum/src/mt19937/mersennetwister_mod.F90
diff --git a/share/shr_RandNum/src/shr_RandNum_mod.F90 b/src/share/RandNum/src/shr_RandNum_mod.F90
similarity index 100%
rename from share/shr_RandNum/src/shr_RandNum_mod.F90
rename to src/share/RandNum/src/shr_RandNum_mod.F90
diff --git a/share/shr_RandNum/test/bench/Makefile b/src/share/RandNum/test/bench/Makefile
similarity index 100%
rename from share/shr_RandNum/test/bench/Makefile
rename to src/share/RandNum/test/bench/Makefile
diff --git a/share/shr_RandNum/test/bench/test_shr_RandNum.F90 b/src/share/RandNum/test/bench/test_shr_RandNum.F90
similarity index 100%
rename from share/shr_RandNum/test/bench/test_shr_RandNum.F90
rename to src/share/RandNum/test/bench/test_shr_RandNum.F90
diff --git a/share/esmf_wrf_timemgr/CMakeLists.txt b/src/share/esmf_wrf_timemgr/CMakeLists.txt
similarity index 100%
rename from share/esmf_wrf_timemgr/CMakeLists.txt
rename to src/share/esmf_wrf_timemgr/CMakeLists.txt
diff --git a/share/esmf_wrf_timemgr/ESMF.F90 b/src/share/esmf_wrf_timemgr/ESMF.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF.F90
rename to src/share/esmf_wrf_timemgr/ESMF.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_AlarmClockMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_AlarmMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_AlarmMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_AlarmMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_AlarmMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_BaseMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_BaseMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_BaseMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_BaseMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_BaseTimeMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_CalendarMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_CalendarMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_CalendarMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_CalendarMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_ClockMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_ClockMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_ClockMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_ClockMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_FractionMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_FractionMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_FractionMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_FractionMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_Macros.inc b/src/share/esmf_wrf_timemgr/ESMF_Macros.inc
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_Macros.inc
rename to src/share/esmf_wrf_timemgr/ESMF_Macros.inc
diff --git a/share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_ShrTimeMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_Stubs.F90 b/src/share/esmf_wrf_timemgr/ESMF_Stubs.F90
similarity index 79%
rename from share/esmf_wrf_timemgr/ESMF_Stubs.F90
rename to src/share/esmf_wrf_timemgr/ESMF_Stubs.F90
index 5deced65f09..9267c8fa80b 100644
--- a/share/esmf_wrf_timemgr/ESMF_Stubs.F90
+++ b/src/share/esmf_wrf_timemgr/ESMF_Stubs.F90
@@ -11,39 +11,47 @@ MODULE ESMF_Stubs
 ! Bogus typedefs
    TYPE ESMF_Grid
       INTEGER :: dummy
-   END TYPE
+   END TYPE ESMF_Grid
 
    TYPE ESMF_GridComp
       INTEGER :: dummy
-   END TYPE
+   END TYPE ESMF_GridComp
 
    TYPE ESMF_State
       INTEGER :: dummy
-   END TYPE
+   END TYPE ESMF_State
 
    TYPE ESMF_VM
       INTEGER :: dummy
-   END TYPE
+   END TYPE ESMF_VM
 
    TYPE ESMF_END_FLAG
       INTEGER :: dummy
-   END TYPE
-   TYPE(ESMF_END_FLAG), PARAMETER ::  &
-      ESMF_END_ABORT   = ESMF_END_FLAG(1), &
-      ESMF_END_NORMAL  = ESMF_END_FLAG(2), &
+   END TYPE ESMF_END_FLAG
+   TYPE(ESMF_END_FLAG), PARAMETER ::                &
+      ESMF_END_ABORT   = ESMF_END_FLAG(1),          &
+      ESMF_END_NORMAL  = ESMF_END_FLAG(2),          &
       ESMF_END_KEEPMPI = ESMF_END_FLAG(3)
 
    TYPE ESMF_MsgType
       INTEGER :: mtype
-   END TYPE
-   TYPE(ESMF_MsgType), PARAMETER  ::      &
-      ESMF_LOG_INFO  =   ESMF_MsgType(1), &
-      ESMF_LOG_WARNING = ESMF_MsgType(2), &
+   END TYPE ESMF_MsgType
+   TYPE(ESMF_MsgType), PARAMETER  ::                &
+      ESMF_LOG_INFO  =   ESMF_MsgType(1),           &
+      ESMF_LOG_WARNING = ESMF_MsgType(2),           &
       ESMF_LOG_ERROR =   ESMF_MsgType(3)
 
    TYPE ESMF_LOG
       INTEGER :: dummy
-   END TYPE
+   END TYPE ESMF_LOG
+
+   TYPE ESMF_LogKind_Flag
+      INTEGER :: dummy
+   END TYPE ESMF_LogKind_Flag
+   TYPE(ESMF_LogKind_Flag), PARAMETER ::            &
+        ESMF_LOGKIND_NONE = ESMF_LogKind_Flag(1),   &
+        ESMF_LOGKIND_SINGLE = ESMF_LogKind_Flag(2), &
+        ESMF_LOGKIND_MULTI = ESMF_LogKind_Flag(3)
 
    LOGICAL, private, save :: initialized = .false.
 
@@ -52,17 +60,20 @@ MODULE ESMF_Stubs
    PUBLIC ESMF_LogWrite, ESMF_LOG, ESMF_MsgType, ESMF_END_FLAG
    PUBLIC ESMF_LOG_INFO, ESMF_LOG_WARNING, ESMF_LOG_ERROR
    PUBLIC ESMF_END_ABORT, ESMF_END_NORMAL, ESMF_END_KEEPMPI
+   PUBLIC ESMF_LogKind_Flag
+   PUBLIC ESMF_LOGKIND_NONE, ESMF_LOGKIND_SINGLE, ESMF_LOGKIND_MULTI
 
 CONTAINS
 
 
 ! NOOP
-   SUBROUTINE ESMF_Initialize( vm, defaultCalendar, rc )
+   SUBROUTINE ESMF_Initialize( vm, defaultCalendar, logkindflag, rc )
       USE ESMF_BaseMod
       USE ESMF_CalendarMod
 !     USE ESMF_TimeMod,     only: defaultCal
       TYPE(ESMF_VM),           INTENT(IN   ), OPTIONAL :: vm
       TYPE(ESMF_CalKind_Flag), INTENT(IN   ), OPTIONAL :: defaultCalendar
+      TYPE(ESMF_LogKind_Flag), INTENT(IN   ), OPTIONAL :: logkindflag
       INTEGER,                 INTENT(  OUT), OPTIONAL :: rc
 
       TYPE(ESMF_CalKind_Flag) :: defaultCalType
diff --git a/share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_TimeIntervalMod.F90
diff --git a/share/esmf_wrf_timemgr/ESMF_TimeMgr.inc b/src/share/esmf_wrf_timemgr/ESMF_TimeMgr.inc
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_TimeMgr.inc
rename to src/share/esmf_wrf_timemgr/ESMF_TimeMgr.inc
diff --git a/share/esmf_wrf_timemgr/ESMF_TimeMod.F90 b/src/share/esmf_wrf_timemgr/ESMF_TimeMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/ESMF_TimeMod.F90
rename to src/share/esmf_wrf_timemgr/ESMF_TimeMod.F90
diff --git a/share/esmf_wrf_timemgr/Makefile b/src/share/esmf_wrf_timemgr/Makefile
similarity index 100%
rename from share/esmf_wrf_timemgr/Makefile
rename to src/share/esmf_wrf_timemgr/Makefile
diff --git a/share/esmf_wrf_timemgr/MeatMod.F90 b/src/share/esmf_wrf_timemgr/MeatMod.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/MeatMod.F90
rename to src/share/esmf_wrf_timemgr/MeatMod.F90
diff --git a/share/esmf_wrf_timemgr/README b/src/share/esmf_wrf_timemgr/README
similarity index 100%
rename from share/esmf_wrf_timemgr/README
rename to src/share/esmf_wrf_timemgr/README
diff --git a/share/esmf_wrf_timemgr/unittests/Makefile b/src/share/esmf_wrf_timemgr/unittests/Makefile
similarity index 100%
rename from share/esmf_wrf_timemgr/unittests/Makefile
rename to src/share/esmf_wrf_timemgr/unittests/Makefile
diff --git a/share/esmf_wrf_timemgr/unittests/go.csh b/src/share/esmf_wrf_timemgr/unittests/go.csh
similarity index 100%
rename from share/esmf_wrf_timemgr/unittests/go.csh
rename to src/share/esmf_wrf_timemgr/unittests/go.csh
diff --git a/share/esmf_wrf_timemgr/unittests/test.F90 b/src/share/esmf_wrf_timemgr/unittests/test.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/unittests/test.F90
rename to src/share/esmf_wrf_timemgr/unittests/test.F90
diff --git a/share/esmf_wrf_timemgr/unittests/wrf_stuff.F90 b/src/share/esmf_wrf_timemgr/unittests/wrf_stuff.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/unittests/wrf_stuff.F90
rename to src/share/esmf_wrf_timemgr/unittests/wrf_stuff.F90
diff --git a/share/esmf_wrf_timemgr/wrf_error_fatal.F90 b/src/share/esmf_wrf_timemgr/wrf_error_fatal.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/wrf_error_fatal.F90
rename to src/share/esmf_wrf_timemgr/wrf_error_fatal.F90
diff --git a/share/esmf_wrf_timemgr/wrf_message.F90 b/src/share/esmf_wrf_timemgr/wrf_message.F90
similarity index 100%
rename from share/esmf_wrf_timemgr/wrf_message.F90
rename to src/share/esmf_wrf_timemgr/wrf_message.F90
diff --git a/share/csm_share/include/dynamic_vector_procdef.inc b/src/share/include/dynamic_vector_procdef.inc
similarity index 100%
rename from share/csm_share/include/dynamic_vector_procdef.inc
rename to src/share/include/dynamic_vector_procdef.inc
diff --git a/share/csm_share/include/dynamic_vector_typedef.inc b/src/share/include/dynamic_vector_typedef.inc
similarity index 100%
rename from share/csm_share/include/dynamic_vector_typedef.inc
rename to src/share/include/dynamic_vector_typedef.inc
diff --git a/src/share/include/shr_assert.h b/src/share/include/shr_assert.h
new file mode 100644
index 00000000000..b09e0d12715
--- /dev/null
+++ b/src/share/include/shr_assert.h
@@ -0,0 +1,22 @@
+#ifdef NDEBUG
+#define SHR_ASSERT(assert, msg)
+#define SHR_ASSERT_FL(assert, file, line)
+#define SHR_ASSERT_MFL(assert, msg, file, line)
+#define SHR_ASSERT_ALL(assert, msg)
+#define SHR_ASSERT_ALL_FL(assert, file, line)
+#define SHR_ASSERT_ALL_MFL(assert, msg, file, line)
+#define SHR_ASSERT_ANY(assert, msg)
+#define SHR_ASSERT_ANY_FL(assert, file, line)
+#define SHR_ASSERT_ANY_MFL(assert, msg, file, line)
+#else
+#define SHR_ASSERT(assert, my_msg) call shr_assert(assert, msg=my_msg)
+#define SHR_ASSERT_FL(assert, my_file, my_line) call shr_assert(assert, file=my_file, line=my_line)
+#define SHR_ASSERT_MFL(assert, my_msg, my_file, my_line) call shr_assert(assert, msg=my_msg, file=my_file, line=my_line)
+#define SHR_ASSERT_ALL(assert, my_msg) call shr_assert_all(assert, msg=my_msg)
+#define SHR_ASSERT_ALL_FL(assert, my_file, my_line) call shr_assert_all(assert, file=my_file, line=my_line)
+#define SHR_ASSERT_ALL_MFL(assert, my_msg, my_file, my_line) call shr_assert_all(assert, msg=my_msg, file=my_file, line=my_line)
+#define SHR_ASSERT_ANY(assert, my_msg) call shr_assert_any(assert, msg=my_msg)
+#define SHR_ASSERT_ANY_FL(assert, my_file, my_line) call shr_assert_any(assert, file=my_file, line=my_line)
+#define SHR_ASSERT_ANY_MFL(assert, my_msg, my_file, my_line) call shr_assert_any(assert, msg=my_msg, file=my_file, line=my_line)
+#endif
+use shr_assert_mod
diff --git a/share/csm_share/test/old_unit_testers/Makefile b/src/share/test/old_unit_testers/Makefile
similarity index 100%
rename from share/csm_share/test/old_unit_testers/Makefile
rename to src/share/test/old_unit_testers/Makefile
diff --git a/share/csm_share/test/old_unit_testers/Mkdepends b/src/share/test/old_unit_testers/Mkdepends
similarity index 100%
rename from share/csm_share/test/old_unit_testers/Mkdepends
rename to src/share/test/old_unit_testers/Mkdepends
diff --git a/share/csm_share/test/old_unit_testers/Mksrcfiles b/src/share/test/old_unit_testers/Mksrcfiles
similarity index 100%
rename from share/csm_share/test/old_unit_testers/Mksrcfiles
rename to src/share/test/old_unit_testers/Mksrcfiles
diff --git a/share/csm_share/test/old_unit_testers/bundle_expected.F90 b/src/share/test/old_unit_testers/bundle_expected.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/bundle_expected.F90
rename to src/share/test/old_unit_testers/bundle_expected.F90
diff --git a/share/csm_share/test/old_unit_testers/config.h b/src/share/test/old_unit_testers/config.h
similarity index 100%
rename from share/csm_share/test/old_unit_testers/config.h
rename to src/share/test/old_unit_testers/config.h
diff --git a/share/csm_share/test/old_unit_testers/make.Macros b/src/share/test/old_unit_testers/make.Macros
similarity index 100%
rename from share/csm_share/test/old_unit_testers/make.Macros
rename to src/share/test/old_unit_testers/make.Macros
diff --git a/share/csm_share/test/old_unit_testers/namelist b/src/share/test/old_unit_testers/namelist
similarity index 100%
rename from share/csm_share/test/old_unit_testers/namelist
rename to src/share/test/old_unit_testers/namelist
diff --git a/share/csm_share/test/old_unit_testers/nl/atm.stdin b/src/share/test/old_unit_testers/nl/atm.stdin
similarity index 100%
rename from share/csm_share/test/old_unit_testers/nl/atm.stdin
rename to src/share/test/old_unit_testers/nl/atm.stdin
diff --git a/share/csm_share/test/old_unit_testers/nl/cpl.stdin b/src/share/test/old_unit_testers/nl/cpl.stdin
similarity index 100%
rename from share/csm_share/test/old_unit_testers/nl/cpl.stdin
rename to src/share/test/old_unit_testers/nl/cpl.stdin
diff --git a/share/csm_share/test/old_unit_testers/nl/ice.stdin b/src/share/test/old_unit_testers/nl/ice.stdin
similarity index 100%
rename from share/csm_share/test/old_unit_testers/nl/ice.stdin
rename to src/share/test/old_unit_testers/nl/ice.stdin
diff --git a/share/csm_share/test/old_unit_testers/nl/lnd.stdin b/src/share/test/old_unit_testers/nl/lnd.stdin
similarity index 100%
rename from share/csm_share/test/old_unit_testers/nl/lnd.stdin
rename to src/share/test/old_unit_testers/nl/lnd.stdin
diff --git a/share/csm_share/test/old_unit_testers/nl/ocn.stdin b/src/share/test/old_unit_testers/nl/ocn.stdin
similarity index 100%
rename from share/csm_share/test/old_unit_testers/nl/ocn.stdin
rename to src/share/test/old_unit_testers/nl/ocn.stdin
diff --git a/share/csm_share/test/old_unit_testers/run_dshr_bundle_test b/src/share/test/old_unit_testers/run_dshr_bundle_test
similarity index 100%
rename from share/csm_share/test/old_unit_testers/run_dshr_bundle_test
rename to src/share/test/old_unit_testers/run_dshr_bundle_test
diff --git a/share/csm_share/test/old_unit_testers/run_file_test b/src/share/test/old_unit_testers/run_file_test
similarity index 100%
rename from share/csm_share/test/old_unit_testers/run_file_test
rename to src/share/test/old_unit_testers/run_file_test
diff --git a/share/csm_share/test/old_unit_testers/test_mod.F90 b/src/share/test/old_unit_testers/test_mod.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_mod.F90
rename to src/share/test/old_unit_testers/test_mod.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_file.F90 b/src/share/test/old_unit_testers/test_shr_file.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_file.F90
rename to src/share/test/old_unit_testers/test_shr_file.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_log.F90 b/src/share/test/old_unit_testers/test_shr_log.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_log.F90
rename to src/share/test/old_unit_testers/test_shr_log.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_mpi.F90 b/src/share/test/old_unit_testers/test_shr_mpi.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_mpi.F90
rename to src/share/test/old_unit_testers/test_shr_mpi.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_orb.F90 b/src/share/test/old_unit_testers/test_shr_orb.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_orb.F90
rename to src/share/test/old_unit_testers/test_shr_orb.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_scam.F90 b/src/share/test/old_unit_testers/test_shr_scam.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_scam.F90
rename to src/share/test/old_unit_testers/test_shr_scam.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_streams.F90 b/src/share/test/old_unit_testers/test_shr_streams.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_streams.F90
rename to src/share/test/old_unit_testers/test_shr_streams.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_sys.F90 b/src/share/test/old_unit_testers/test_shr_sys.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_sys.F90
rename to src/share/test/old_unit_testers/test_shr_sys.F90
diff --git a/share/csm_share/test/old_unit_testers/test_shr_tInterp.F90 b/src/share/test/old_unit_testers/test_shr_tInterp.F90
similarity index 100%
rename from share/csm_share/test/old_unit_testers/test_shr_tInterp.F90
rename to src/share/test/old_unit_testers/test_shr_tInterp.F90
diff --git a/share/csm_share/test/unit/CMakeLists.txt b/src/share/test/unit/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/CMakeLists.txt
rename to src/share/test/unit/CMakeLists.txt
diff --git a/share/csm_share/test/unit/dynamic_vector/CMakeLists.txt b/src/share/test/unit/dynamic_vector/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/CMakeLists.txt
rename to src/share/test/unit/dynamic_vector/CMakeLists.txt
diff --git a/share/csm_share/test/unit/dynamic_vector/character16_vector_tests.pf.in b/src/share/test/unit/dynamic_vector/character16_vector_tests.pf.in
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/character16_vector_tests.pf.in
rename to src/share/test/unit/dynamic_vector/character16_vector_tests.pf.in
diff --git a/share/csm_share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc b/src/share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc
rename to src/share/test/unit/dynamic_vector/dynamic_vector_base_tests.inc
diff --git a/share/csm_share/test/unit/dynamic_vector/dynamic_vector_character16.F90 b/src/share/test/unit/dynamic_vector/dynamic_vector_character16.F90
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/dynamic_vector_character16.F90
rename to src/share/test/unit/dynamic_vector/dynamic_vector_character16.F90
diff --git a/share/csm_share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90 b/src/share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90
rename to src/share/test/unit/dynamic_vector/dynamic_vector_int_ptr.F90
diff --git a/share/csm_share/test/unit/dynamic_vector/dynamic_vector_integer.F90 b/src/share/test/unit/dynamic_vector/dynamic_vector_integer.F90
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/dynamic_vector_integer.F90
rename to src/share/test/unit/dynamic_vector/dynamic_vector_integer.F90
diff --git a/share/csm_share/test/unit/dynamic_vector/dynamic_vector_r8.F90 b/src/share/test/unit/dynamic_vector/dynamic_vector_r8.F90
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/dynamic_vector_r8.F90
rename to src/share/test/unit/dynamic_vector/dynamic_vector_r8.F90
diff --git a/share/csm_share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in b/src/share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in
rename to src/share/test/unit/dynamic_vector/int_ptr_vector_tests.pf.in
diff --git a/share/csm_share/test/unit/dynamic_vector/integer_vector_tests.pf.in b/src/share/test/unit/dynamic_vector/integer_vector_tests.pf.in
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/integer_vector_tests.pf.in
rename to src/share/test/unit/dynamic_vector/integer_vector_tests.pf.in
diff --git a/share/csm_share/test/unit/dynamic_vector/ptr_wrapper.F90 b/src/share/test/unit/dynamic_vector/ptr_wrapper.F90
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/ptr_wrapper.F90
rename to src/share/test/unit/dynamic_vector/ptr_wrapper.F90
diff --git a/share/csm_share/test/unit/dynamic_vector/r8_vector_tests.pf.in b/src/share/test/unit/dynamic_vector/r8_vector_tests.pf.in
similarity index 100%
rename from share/csm_share/test/unit/dynamic_vector/r8_vector_tests.pf.in
rename to src/share/test/unit/dynamic_vector/r8_vector_tests.pf.in
diff --git a/share/csm_share/test/unit/mock/CMakeLists.txt b/src/share/test/unit/mock/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/mock/CMakeLists.txt
rename to src/share/test/unit/mock/CMakeLists.txt
diff --git a/share/csm_share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90 b/src/share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90
similarity index 100%
rename from share/csm_share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90
rename to src/share/test/unit/mock/shr_sys_mod.nompi_abortthrows.F90
diff --git a/share/csm_share/test/unit/shr_assert_test/CMakeLists.txt b/src/share/test/unit/shr_assert_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_assert_test/CMakeLists.txt
rename to src/share/test/unit/shr_assert_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_assert_test/test_assert.pf b/src/share/test/unit/shr_assert_test/test_assert.pf
similarity index 70%
rename from share/csm_share/test/unit/shr_assert_test/test_assert.pf
rename to src/share/test/unit/shr_assert_test/test_assert.pf
index 5bc7239e26f..0e05b0c8a75 100644
--- a/share/csm_share/test/unit/shr_assert_test/test_assert.pf
+++ b/src/share/test/unit/shr_assert_test/test_assert.pf
@@ -22,9 +22,15 @@ end subroutine assert_can_pass
 @Test
 subroutine assert_can_fail()
   call shr_assert(.false., "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_can_fail
 
+@Test
+subroutine assert_prints_file_and_line()
+  call shr_assert(.false., "Expected failure.", file='foo', line=42)
+  call assertExceptionRaised("ABORTED: ERROR in foo at line 42: Expected failure.")
+end subroutine assert_prints_file_and_line
+
 @Test
 subroutine assert_all_scalar_can_pass()
   call shr_assert_all(.true., "Assert unexpectedly aborted!")
@@ -33,7 +39,7 @@ end subroutine assert_all_scalar_can_pass
 @Test
 subroutine assert_all_scalar_can_fail()
   call shr_assert_all(.false., "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_all_scalar_can_fail
 
 @Test
@@ -44,7 +50,7 @@ end subroutine assert_any_scalar_can_pass
 @Test
 subroutine assert_any_scalar_can_fail()
   call shr_assert_any(.false., "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_any_scalar_can_fail
 
 end module test_assert
diff --git a/share/csm_share/test/unit/shr_assert_test/test_assert_array.pf b/src/share/test/unit/shr_assert_test/test_assert_array.pf
similarity index 95%
rename from share/csm_share/test/unit/shr_assert_test/test_assert_array.pf
rename to src/share/test/unit/shr_assert_test/test_assert_array.pf
index 219b030a2aa..c847d4626a7 100644
--- a/share/csm_share/test/unit/shr_assert_test/test_assert_array.pf
+++ b/src/share/test/unit/shr_assert_test/test_assert_array.pf
@@ -73,7 +73,7 @@ subroutine assert_all_can_fail(this)
   class(TestAssertArray), intent(inout) :: this
   call assert_all_wrapper([.false.], 1, this%rank, &
        "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_all_can_fail
 
 @Test
@@ -84,7 +84,7 @@ subroutine assert_all_partial_false_fails(this)
   test_array = [( mod(i,2) == 0, i = 1, size(test_array) )]
   call assert_all_wrapper(test_array, 2, this%rank, &
        "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_all_partial_false_fails
 
 @Test
@@ -92,7 +92,7 @@ subroutine assert_any_size_zero_fails(this)
   class(TestAssertArray), intent(inout) :: this
   call assert_any_wrapper([logical::], 0, this%rank, &
        "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_any_size_zero_fails
 
 @Test
@@ -107,7 +107,7 @@ subroutine assert_any_can_fail(this)
   class(TestAssertArray), intent(inout) :: this
   call assert_any_wrapper([.false.], 1, this%rank, &
        "Expected failure.")
-  call assertExceptionRaised("ABORTED: Expected failure.")
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
 end subroutine assert_any_can_fail
 
 @Test
diff --git a/src/share/test/unit/shr_assert_test/test_macro.pf b/src/share/test/unit/shr_assert_test/test_macro.pf
new file mode 100644
index 00000000000..d75f39dae14
--- /dev/null
+++ b/src/share/test/unit/shr_assert_test/test_macro.pf
@@ -0,0 +1,87 @@
+module test_macro
+
+! Test that if NDEBUG is not defined, shr_assert macros run assertions.
+
+use pfunit_mod
+
+#undef NDEBUG
+#include "shr_assert.h"
+
+contains
+
+@Test
+subroutine macro_assert_can_pass()
+  SHR_ASSERT(.true., "Assert macro unexpectedly aborted!")
+end subroutine macro_assert_can_pass
+
+@Test
+subroutine macro_assert_can_fail()
+  SHR_ASSERT(.false., "Expected failure.")
+  ! When this was written, the preprocessor did not recognize this assert,
+  ! so call it directly instead of using an "@".
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
+end subroutine macro_assert_can_fail
+
+@Test
+subroutine macro_assert_fl()
+  SHR_ASSERT_FL(.false., "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42")
+end subroutine macro_assert_fl
+
+@Test
+subroutine macro_assert_mfl()
+  SHR_ASSERT_MFL(.false., "Expected failure.", "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42: Expected failure.")
+end subroutine macro_assert_mfl
+
+@Test
+subroutine macro_assert_all_can_pass()
+  SHR_ASSERT_ALL(([.true., .true.]), "Assert macro unexpectedly aborted!")
+end subroutine macro_assert_all_can_pass
+
+@Test
+subroutine macro_assert_all_can_fail()
+  SHR_ASSERT_ALL(([.true., .false.]), "Expected failure.")
+  ! When this was written, the preprocessor did not recognize this assert,
+  ! so call it directly instead of using an "@".
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
+end subroutine macro_assert_all_can_fail
+
+@Test
+subroutine macro_assert_all_fl()
+  SHR_ASSERT_ALL_FL(([.true., .false.]), "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42")
+end subroutine macro_assert_all_fl
+
+@Test
+subroutine macro_assert_all_mfl()
+  SHR_ASSERT_ALL_MFL(([.true., .false.]), "Expected failure.", "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42: Expected failure.")
+end subroutine macro_assert_all_mfl
+
+@Test
+subroutine macro_assert_any_can_pass()
+  SHR_ASSERT_ANY(([.true., .false.]), "Assert macro unexpectedly aborted!")
+end subroutine macro_assert_any_can_pass
+
+@Test
+subroutine macro_assert_any_can_fail()
+  SHR_ASSERT_ANY(([.false., .false.]), "Expected failure.")
+  ! When this was written, the preprocessor did not recognize this assert,
+  ! so call it directly instead of using an "@".
+  call assertExceptionRaised("ABORTED: ERROR: Expected failure.")
+end subroutine macro_assert_any_can_fail
+
+@Test
+subroutine macro_assert_any_fl()
+  SHR_ASSERT_ANY_FL(([.false., .false.]), "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42")
+end subroutine macro_assert_any_fl
+
+@Test
+subroutine macro_assert_any_mfl()
+  SHR_ASSERT_ANY_MFL(([.false., .false.]), "Expected failure.", "my_file", 42)
+  call assertExceptionRaised("ABORTED: ERROR in my_file at line 42: Expected failure.")
+end subroutine macro_assert_any_mfl
+
+end module test_macro
diff --git a/src/share/test/unit/shr_assert_test/test_ndebug.pf b/src/share/test/unit/shr_assert_test/test_ndebug.pf
new file mode 100644
index 00000000000..ef1c6226447
--- /dev/null
+++ b/src/share/test/unit/shr_assert_test/test_ndebug.pf
@@ -0,0 +1,63 @@
+module test_ndebug
+
+! Test that if NDEBUG is defined, shr_assert macros do nothing.
+
+use pfunit_mod
+
+#define NDEBUG
+#include "shr_assert.h"
+
+contains
+
+  logical function unreachable_function(macro_name)
+    character(len=*), intent(in) :: macro_name
+
+    call throw("NDEBUG failed to turn off " // macro_name)
+  end function unreachable_function
+
+@Test
+subroutine ndebug_controls_assert_macro()
+  SHR_ASSERT(unreachable_function("SHR_ASSERT"), "Fake message.")
+end subroutine ndebug_controls_assert_macro
+
+@Test
+subroutine ndebug_controls_assert_fl_macro()
+  SHR_ASSERT_FL(unreachable_function("SHR_ASSERT_FL"), "my_file", 42)
+end subroutine ndebug_controls_assert_fl_macro
+
+@Test
+subroutine ndebug_controls_assert_mfl_macro()
+  SHR_ASSERT_MFL(unreachable_function("SHR_ASSERT_MFL"), "Fake message.", "my_file", 42)
+end subroutine ndebug_controls_assert_mfl_macro
+
+@Test
+subroutine ndebug_controls_assert_all_macro()
+  SHR_ASSERT_ALL(unreachable_function("SHR_ASSERT_ALL"), "Fake message.")
+end subroutine ndebug_controls_assert_all_macro
+
+@Test
+subroutine ndebug_controls_assert_all_fl_macro()
+  SHR_ASSERT_ALL_FL(unreachable_function("SHR_ASSERT_ALL_FL"), "my_file", 42)
+end subroutine ndebug_controls_assert_all_fl_macro
+
+@Test
+subroutine ndebug_controls_assert_all_mfl_macro()
+  SHR_ASSERT_ALL_MFL(unreachable_function("SHR_ASSERT_ALL_MFL"), "Fake message.", "my_file", 42)
+end subroutine ndebug_controls_assert_all_mfl_macro
+
+@Test
+subroutine ndebug_controls_assert_any_macro()
+  SHR_ASSERT_ANY(unreachable_function("SHR_ASSERT_ANY"), "Fake message.")
+end subroutine ndebug_controls_assert_any_macro
+
+@Test
+subroutine ndebug_controls_assert_any_fl_macro()
+  SHR_ASSERT_ANY_FL(unreachable_function("SHR_ASSERT_ANY_FL"), "my_file", 42)
+end subroutine ndebug_controls_assert_any_fl_macro
+
+@Test
+subroutine ndebug_controls_assert_any_mfl_macro()
+  SHR_ASSERT_ANY_MFL(unreachable_function("SHR_ASSERT_ANY_MFL"), "Fake message.", "my_file", 42)
+end subroutine ndebug_controls_assert_any_mfl_macro
+
+end module test_ndebug
diff --git a/share/csm_share/test/unit/shr_infnan_test/CMakeLists.txt b/src/share/test/unit/shr_infnan_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_infnan_test/CMakeLists.txt
rename to src/share/test/unit/shr_infnan_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_infnan_test/test_infnan.F90 b/src/share/test/unit/shr_infnan_test/test_infnan.F90
similarity index 100%
rename from share/csm_share/test/unit/shr_infnan_test/test_infnan.F90
rename to src/share/test/unit/shr_infnan_test/test_infnan.F90
diff --git a/share/csm_share/test/unit/shr_log_test/CMakeLists.txt b/src/share/test/unit/shr_log_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_log_test/CMakeLists.txt
rename to src/share/test/unit/shr_log_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_log_test/test_error_printers.pf b/src/share/test/unit/shr_log_test/test_error_printers.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_log_test/test_error_printers.pf
rename to src/share/test/unit/shr_log_test/test_error_printers.pf
diff --git a/share/csm_share/test/unit/shr_precip_test/CMakeLists.txt b/src/share/test/unit/shr_precip_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_precip_test/CMakeLists.txt
rename to src/share/test/unit/shr_precip_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_precip_test/test_shr_precip.pf b/src/share/test/unit/shr_precip_test/test_shr_precip.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_precip_test/test_shr_precip.pf
rename to src/share/test/unit/shr_precip_test/test_shr_precip.pf
diff --git a/share/csm_share/test/unit/shr_spfn_test/CMakeLists.txt b/src/share/test/unit/shr_spfn_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_spfn_test/CMakeLists.txt
rename to src/share/test/unit/shr_spfn_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_spfn_test/test_erf_r4.pf b/src/share/test/unit/shr_spfn_test/test_erf_r4.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_spfn_test/test_erf_r4.pf
rename to src/share/test/unit/shr_spfn_test/test_erf_r4.pf
diff --git a/share/csm_share/test/unit/shr_spfn_test/test_erf_r8.pf b/src/share/test/unit/shr_spfn_test/test_erf_r8.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_spfn_test/test_erf_r8.pf
rename to src/share/test/unit/shr_spfn_test/test_erf_r8.pf
diff --git a/share/csm_share/test/unit/shr_spfn_test/test_gamma_factorial.pf b/src/share/test/unit/shr_spfn_test/test_gamma_factorial.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_spfn_test/test_gamma_factorial.pf
rename to src/share/test/unit/shr_spfn_test/test_gamma_factorial.pf
diff --git a/share/csm_share/test/unit/shr_spfn_test/test_igamma.pf b/src/share/test/unit/shr_spfn_test/test_igamma.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_spfn_test/test_igamma.pf
rename to src/share/test/unit/shr_spfn_test/test_igamma.pf
diff --git a/share/csm_share/test/unit/shr_strconvert_test/CMakeLists.txt b/src/share/test/unit/shr_strconvert_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_strconvert_test/CMakeLists.txt
rename to src/share/test/unit/shr_strconvert_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_strconvert_test/test_toString.pf b/src/share/test/unit/shr_strconvert_test/test_toString.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_strconvert_test/test_toString.pf
rename to src/share/test/unit/shr_strconvert_test/test_toString.pf
diff --git a/share/csm_share/test/unit/shr_string_test/CMakeLists.txt b/src/share/test/unit/shr_string_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_string_test/CMakeLists.txt
rename to src/share/test/unit/shr_string_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_string_test/test_shr_string.pf b/src/share/test/unit/shr_string_test/test_shr_string.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_string_test/test_shr_string.pf
rename to src/share/test/unit/shr_string_test/test_shr_string.pf
diff --git a/share/csm_share/test/unit/shr_vmath_test/CMakeLists.txt b/src/share/test/unit/shr_vmath_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_vmath_test/CMakeLists.txt
rename to src/share/test/unit/shr_vmath_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_vmath_test/test_vmath.F90 b/src/share/test/unit/shr_vmath_test/test_vmath.F90
similarity index 100%
rename from share/csm_share/test/unit/shr_vmath_test/test_vmath.F90
rename to src/share/test/unit/shr_vmath_test/test_vmath.F90
diff --git a/share/csm_share/test/unit/shr_wv_sat_test/CMakeLists.txt b/src/share/test/unit/shr_wv_sat_test/CMakeLists.txt
similarity index 100%
rename from share/csm_share/test/unit/shr_wv_sat_test/CMakeLists.txt
rename to src/share/test/unit/shr_wv_sat_test/CMakeLists.txt
diff --git a/share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat.pf b/src/share/test/unit/shr_wv_sat_test/test_wv_sat.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat.pf
rename to src/share/test/unit/shr_wv_sat_test/test_wv_sat.pf
diff --git a/share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf b/src/share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf
similarity index 100%
rename from share/csm_share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf
rename to src/share/test/unit/shr_wv_sat_test/test_wv_sat_each_method.pf
diff --git a/share/timing/CMakeLists.txt b/src/share/timing/CMakeLists.txt
similarity index 100%
rename from share/timing/CMakeLists.txt
rename to src/share/timing/CMakeLists.txt
diff --git a/src/share/timing/COPYING b/src/share/timing/COPYING
new file mode 100644
index 00000000000..324ce86b24c
--- /dev/null
+++ b/src/share/timing/COPYING
@@ -0,0 +1,17 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the “Software”), to deal
+in the Software for any noncommercial purposes without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to permit
+persons to whom the Software is furnished to do so, subject to the following
+conditions: The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.  Any
+commercial use (including sale) of the software, and derivative development
+towards commercial use, requires written permission of the copyright
+holder. THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO
+EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES
+OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/share/timing/ChangeLog b/src/share/timing/ChangeLog
similarity index 100%
rename from share/timing/ChangeLog
rename to src/share/timing/ChangeLog
diff --git a/share/timing/GPTLget_memusage.c b/src/share/timing/GPTLget_memusage.c
similarity index 100%
rename from share/timing/GPTLget_memusage.c
rename to src/share/timing/GPTLget_memusage.c
diff --git a/share/timing/GPTLprint_memusage.c b/src/share/timing/GPTLprint_memusage.c
similarity index 100%
rename from share/timing/GPTLprint_memusage.c
rename to src/share/timing/GPTLprint_memusage.c
diff --git a/share/timing/GPTLutil.c b/src/share/timing/GPTLutil.c
similarity index 100%
rename from share/timing/GPTLutil.c
rename to src/share/timing/GPTLutil.c
diff --git a/share/timing/Makefile b/src/share/timing/Makefile
similarity index 100%
rename from share/timing/Makefile
rename to src/share/timing/Makefile
diff --git a/share/timing/README b/src/share/timing/README
similarity index 100%
rename from share/timing/README
rename to src/share/timing/README
diff --git a/share/timing/f_wrappers.c b/src/share/timing/f_wrappers.c
similarity index 100%
rename from share/timing/f_wrappers.c
rename to src/share/timing/f_wrappers.c
diff --git a/share/timing/gptl.c b/src/share/timing/gptl.c
similarity index 100%
rename from share/timing/gptl.c
rename to src/share/timing/gptl.c
diff --git a/share/timing/gptl.h b/src/share/timing/gptl.h
similarity index 100%
rename from share/timing/gptl.h
rename to src/share/timing/gptl.h
diff --git a/share/timing/gptl.inc b/src/share/timing/gptl.inc
similarity index 100%
rename from share/timing/gptl.inc
rename to src/share/timing/gptl.inc
diff --git a/share/timing/gptl_papi.c b/src/share/timing/gptl_papi.c
similarity index 100%
rename from share/timing/gptl_papi.c
rename to src/share/timing/gptl_papi.c
diff --git a/share/timing/perf_mod.F90 b/src/share/timing/perf_mod.F90
similarity index 100%
rename from share/timing/perf_mod.F90
rename to src/share/timing/perf_mod.F90
diff --git a/share/timing/perf_utils.F90 b/src/share/timing/perf_utils.F90
similarity index 100%
rename from share/timing/perf_utils.F90
rename to src/share/timing/perf_utils.F90
diff --git a/share/timing/private.h b/src/share/timing/private.h
similarity index 100%
rename from share/timing/private.h
rename to src/share/timing/private.h
diff --git a/share/csm_share/unit_test_stubs/README b/src/share/unit_test_stubs/README
similarity index 100%
rename from share/csm_share/unit_test_stubs/README
rename to src/share/unit_test_stubs/README
diff --git a/share/csm_share/unit_test_stubs/pio/CMakeLists.txt b/src/share/unit_test_stubs/pio/CMakeLists.txt
similarity index 100%
rename from share/csm_share/unit_test_stubs/pio/CMakeLists.txt
rename to src/share/unit_test_stubs/pio/CMakeLists.txt
diff --git a/share/csm_share/unit_test_stubs/pio/README b/src/share/unit_test_stubs/pio/README
similarity index 100%
rename from share/csm_share/unit_test_stubs/pio/README
rename to src/share/unit_test_stubs/pio/README
diff --git a/share/csm_share/unit_test_stubs/pio/pio.F90.in b/src/share/unit_test_stubs/pio/pio.F90.in
similarity index 93%
rename from share/csm_share/unit_test_stubs/pio/pio.F90.in
rename to src/share/unit_test_stubs/pio/pio.F90.in
index 22313cef88d..a855889396b 100644
--- a/share/csm_share/unit_test_stubs/pio/pio.F90.in
+++ b/src/share/unit_test_stubs/pio/pio.F90.in
@@ -23,7 +23,7 @@ module pio
 #ifndef NO_MPIMOD
    use mpi, only : MPI_OFFSET_KIND ! _EXTERNAL
 #endif
-
+   use iso_c_binding
   implicit none
   private
 
@@ -43,7 +43,17 @@ module pio
   type, public :: var_desc_t
   end type var_desc_t
 
+   type, bind(c), public :: PIO_rearr_comm_fc_opt_t
+      logical(c_bool) :: enable_hs            ! Enable handshake?
+      logical(c_bool) :: enable_isend         ! Enable isends?
+      integer(c_int) :: max_pend_req         ! Maximum pending requests
+    end type PIO_rearr_comm_fc_opt_t
+
   type, public :: PIO_rearr_opt_t
+      integer(c_int)                         :: comm_type
+      integer(c_int)                         :: fcd       ! Flow control direction
+      type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_comp2io
+      type(PIO_rearr_comm_fc_opt_t)   :: comm_fc_opts_io2comp
   end type PIO_rearr_opt_t
 
   integer, parameter, private ::          &
@@ -70,6 +80,16 @@ module pio
   integer, parameter, public :: pio_rearr_box = 1
   integer, parameter, public :: PIO_WRITE = 0
 
+  integer, parameter, public :: pio_rearr_comm_p2p = 0
+  integer, parameter, public :: pio_rearr_comm_coll = 1
+  integer,parameter,public :: PIO_rearr_comm_fc_2d_enable = 0
+  integer,parameter,public :: PIO_rearr_comm_fc_1d_comp2io = 1
+  integer,parameter,public :: PIO_rearr_comm_fc_1d_io2comp = 2
+  integer,parameter,public :: PIO_rearr_comm_fc_2d_disable = 3
+  integer, public, parameter :: PIO_REARR_COMM_UNLIMITED_PEND_REQ = -1
+  integer, public, parameter :: PIO_NOERR=0
+
+  public :: PIO_set_rearr_opts
   public :: PIO_def_dim
   public :: PIO_enddef
   public :: PIO_FILE_IS_OPEN
@@ -254,6 +274,18 @@ contains
     type (io_desc_t) :: iodesc
   end subroutine freedecomp_file
 
+  integer function PIO_set_rearr_opts(iosystem, comm_type, fcd,&
+                              enable_hs_c2i, enable_isend_c2i,&
+                              max_pend_req_c2i,&
+                              enable_hs_i2c, enable_isend_i2c,&
+                              max_pend_req_i2c) result(ierr)
+    type (iosystem_desc_t), intent(inout)  :: iosystem
+    integer, intent(in) :: comm_type, fcd
+    logical, intent(in) :: enable_hs_c2i, enable_hs_i2c
+    logical, intent(in) :: enable_isend_c2i, enable_isend_i2c
+    integer, intent(in) :: max_pend_req_c2i, max_pend_req_i2c
+  end function PIO_set_rearr_opts
+
   integer function get_att_desc_{TYPE} (File,varDesc,name,value) result(ierr)
     type (File_desc_t), intent(inout) , target :: File
     type (VAR_desc_t), intent(in)     :: varDesc
@@ -344,16 +376,18 @@ contains
     ierr = 0
   end function get_var_vdesc_{DIMS}d_{TYPE}
 
-  subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride,  rearr, iosystem,base)
+  subroutine init_intracom(comp_rank, comp_comm, num_iotasks, num_aggregator, stride,  rearr, iosystem,base, rearr_opts)
+
     integer(i4), intent(in) :: comp_rank
     integer(i4), intent(in) :: comp_comm
     integer(i4), intent(in) :: num_iotasks
     integer(i4), intent(in) :: num_aggregator
     integer(i4), intent(in) :: stride
     integer(i4), intent(in) :: rearr
-    type (iosystem_desc_t), intent(inout)  :: iosystem  ! io descriptor to initalize
-
+    type (iosystem_desc_t), intent(out)  :: iosystem  ! io descriptor to initalize
     integer(i4), intent(in),optional :: base
+    type (pio_rearr_opt_t), intent(in), optional :: rearr_opts
+
   end subroutine init_intracom
 
   subroutine init_intercom(component_count, peer_comm, comp_comms, io_comm, iosystem)
@@ -362,7 +396,7 @@ contains
     integer, intent(in) :: comp_comms(component_count)   !  The compute communicator
     integer, intent(in) :: io_comm     !  The io communicator
 
-    type (iosystem_desc_t), intent(inout)  :: iosystem(component_count)  ! io descriptor to initalize
+    type (iosystem_desc_t), intent(out)  :: iosystem(component_count)  ! io descriptor to initalize
   end subroutine init_intercom
 
   subroutine PIO_initdecomp_bc(iosystem,basepiotype,dims,compstart,compcount,iodesc,iostart,iocount)
diff --git a/share/csm_share/shr/CMakeLists.txt b/src/share/util/CMakeLists.txt
similarity index 100%
rename from share/csm_share/shr/CMakeLists.txt
rename to src/share/util/CMakeLists.txt
diff --git a/share/csm_share/shr/mct_mod.F90 b/src/share/util/mct_mod.F90
similarity index 99%
rename from share/csm_share/shr/mct_mod.F90
rename to src/share/util/mct_mod.F90
index 7f9dc3f6edb..80121dcfd49 100644
--- a/share/csm_share/shr/mct_mod.F90
+++ b/src/share/util/mct_mod.F90
@@ -1292,4 +1292,3 @@ end function mct_myindex
 !===============================================================================
 
 end module mct_mod
-
diff --git a/share/csm_share/shr/shr_assert_mod.F90.in b/src/share/util/shr_assert_mod.F90.in
similarity index 85%
rename from share/csm_share/shr/shr_assert_mod.F90.in
rename to src/share/util/shr_assert_mod.F90.in
index 1dce64b1157..fc62d64bae1 100644
--- a/share/csm_share/shr/shr_assert_mod.F90.in
+++ b/src/share/util/shr_assert_mod.F90.in
@@ -16,6 +16,8 @@ use shr_log_mod, only: &
 
 use shr_infnan_mod, only: shr_infnan_isnan
 
+use shr_strconvert_mod, only: toString
+
 implicit none
 private
 save
@@ -66,38 +68,64 @@ end interface
 
 contains
 
-subroutine shr_assert(var, msg)
+subroutine shr_assert(var, msg, file, line)
 
-  ! Logical being asserted.
+  ! Logical being asserted
   logical, intent(in) :: var
-  ! Optional error message if assert fails.
-  character(len=*), intent(in) :: msg
-
-  if (.not. var) call shr_sys_abort(msg)
+  ! Optional error message if assert fails
+  character(len=*), intent(in), optional :: msg
+  ! Optional file and line of the caller, written out if given
+  ! (line is ignored if file is absent)
+  character(len=*), intent(in), optional :: file
+  integer         , intent(in), optional :: line
+
+  character(len=:), allocatable :: full_msg
+
+  if (.not. var) then
+     full_msg = 'ERROR'
+     if (present(file)) then
+        full_msg = full_msg // ' in ' // trim(file)
+        if (present(line)) then
+           full_msg = full_msg // ' at line ' // toString(line)
+        end if
+     end if
+     if (present(msg)) then
+        full_msg = full_msg // ': ' // msg
+     end if
+     call shr_sys_abort(full_msg)
+  end if
 
 end subroutine shr_assert
 
 ! DIMS 1,2,3,4,5,6,7
-subroutine shr_assert_all_{DIMS}d(var, msg)
+subroutine shr_assert_all_{DIMS}d(var, msg, file, line)
 
-  ! Logical being asserted.
+  ! Logical being asserted
   logical, intent(in) :: var{DIMSTR}
-  ! Optional error message if assert fails.
-  character(len=*), intent(in) :: msg
+  ! Optional error message if assert fails
+  character(len=*), intent(in), optional :: msg
+  ! Optional file and line of the caller, written out if given
+  ! (line is ignored if file is absent)
+  character(len=*), intent(in), optional :: file
+  integer         , intent(in), optional :: line
 
-  call shr_assert(all(var), msg)
+  call shr_assert(all(var), msg=msg, file=file, line=line)
 
 end subroutine shr_assert_all_{DIMS}d
 
 ! DIMS 1,2,3,4,5,6,7
-subroutine shr_assert_any_{DIMS}d(var, msg)
+subroutine shr_assert_any_{DIMS}d(var, msg, file, line)
 
-  ! Logical being asserted.
+  ! Logical being asserted
   logical, intent(in) :: var{DIMSTR}
-  ! Optional error message if assert fails.
-  character(len=*), intent(in) :: msg
+  ! Optional error message if assert fails
+  character(len=*), intent(in), optional :: msg
+  ! Optional file and line of the caller, written out if given
+  ! (line is ignored if file is absent)
+  character(len=*), intent(in), optional :: file
+  integer         , intent(in), optional :: line
 
-  call shr_assert(any(var), msg)
+  call shr_assert(any(var), msg=msg, file=file, line=line)
 
 end subroutine shr_assert_any_{DIMS}d
 
diff --git a/share/csm_share/shr/shr_cal_mod.F90 b/src/share/util/shr_cal_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_cal_mod.F90
rename to src/share/util/shr_cal_mod.F90
diff --git a/share/csm_share/shr/shr_const_mod.F90 b/src/share/util/shr_const_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_const_mod.F90
rename to src/share/util/shr_const_mod.F90
diff --git a/share/csm_share/shr/shr_dmodel_mod.F90 b/src/share/util/shr_dmodel_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_dmodel_mod.F90
rename to src/share/util/shr_dmodel_mod.F90
diff --git a/share/csm_share/shr/shr_file_mod.F90 b/src/share/util/shr_file_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_file_mod.F90
rename to src/share/util/shr_file_mod.F90
diff --git a/share/csm_share/shr/shr_flux_mod.F90 b/src/share/util/shr_flux_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_flux_mod.F90
rename to src/share/util/shr_flux_mod.F90
diff --git a/share/csm_share/shr/shr_frz_mod.F90.in b/src/share/util/shr_frz_mod.F90.in
similarity index 100%
rename from share/csm_share/shr/shr_frz_mod.F90.in
rename to src/share/util/shr_frz_mod.F90.in
diff --git a/share/csm_share/shr/shr_infnan_mod.F90.in b/src/share/util/shr_infnan_mod.F90.in
similarity index 100%
rename from share/csm_share/shr/shr_infnan_mod.F90.in
rename to src/share/util/shr_infnan_mod.F90.in
diff --git a/share/csm_share/shr/shr_kind_mod.F90 b/src/share/util/shr_kind_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_kind_mod.F90
rename to src/share/util/shr_kind_mod.F90
diff --git a/share/csm_share/shr/shr_log_mod.F90 b/src/share/util/shr_log_mod.F90
similarity index 88%
rename from share/csm_share/shr/shr_log_mod.F90
rename to src/share/util/shr_log_mod.F90
index 230cf8aa889..e116df57e09 100644
--- a/share/csm_share/shr/shr_log_mod.F90
+++ b/src/share/util/shr_log_mod.F90
@@ -53,6 +53,13 @@ module shr_log_mod
 !     \newline
 !     errMsg = shr\_log\_errMsg(__FILE__, __LINE__)
 !
+! This is meant to be used when a routine expects a string argument for some message,
+! but you want to provide file and line information.
+!
+! However: Note that the performance of this function can be very bad. It is currently
+! maintained because it is used by old code, but you should probably avoid using this
+! in new code if possible.
+!
 ! !REVISION HISTORY:
 !     2013-July-23 - Bill Sacks
 !
diff --git a/share/csm_share/shr/shr_map_mod.F90 b/src/share/util/shr_map_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_map_mod.F90
rename to src/share/util/shr_map_mod.F90
diff --git a/share/csm_share/shr/shr_mct_mod.F90 b/src/share/util/shr_mct_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_mct_mod.F90
rename to src/share/util/shr_mct_mod.F90
diff --git a/share/csm_share/shr/shr_mem_mod.F90 b/src/share/util/shr_mem_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_mem_mod.F90
rename to src/share/util/shr_mem_mod.F90
diff --git a/share/csm_share/shr/shr_mpi_mod.F90 b/src/share/util/shr_mpi_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_mpi_mod.F90
rename to src/share/util/shr_mpi_mod.F90
diff --git a/share/csm_share/shr/shr_msg_mod.F90 b/src/share/util/shr_msg_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_msg_mod.F90
rename to src/share/util/shr_msg_mod.F90
diff --git a/share/csm_share/shr/shr_ncread_mod.F90 b/src/share/util/shr_ncread_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_ncread_mod.F90
rename to src/share/util/shr_ncread_mod.F90
diff --git a/share/csm_share/shr/shr_nl_mod.F90 b/src/share/util/shr_nl_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_nl_mod.F90
rename to src/share/util/shr_nl_mod.F90
diff --git a/share/csm_share/shr/shr_orb_mod.F90 b/src/share/util/shr_orb_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_orb_mod.F90
rename to src/share/util/shr_orb_mod.F90
diff --git a/share/csm_share/shr/shr_pcdf_mod.F90 b/src/share/util/shr_pcdf_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_pcdf_mod.F90
rename to src/share/util/shr_pcdf_mod.F90
diff --git a/share/csm_share/shr/shr_pio_mod.F90 b/src/share/util/shr_pio_mod.F90
similarity index 92%
rename from share/csm_share/shr/shr_pio_mod.F90
rename to src/share/util/shr_pio_mod.F90
index f4030f0a584..27894e9e73b 100644
--- a/share/csm_share/shr/shr_pio_mod.F90
+++ b/src/share/util/shr_pio_mod.F90
@@ -52,7 +52,11 @@ module shr_pio_mod
   integer, allocatable :: io_compid(:)
   integer :: pio_debug_level=0, pio_blocksize=0
   integer(kind=pio_offset_kind) :: pio_buffer_size_limit=-1
-  type(pio_rearr_opt_t)  :: pio_rearr_opts
+  integer :: pio_rearr_opt_comm_type, pio_rearr_opt_fcd
+  logical :: pio_rearr_opt_c2i_enable_hs, pio_rearr_opt_c2i_enable_isend
+  integer :: pio_rearr_opt_c2i_max_pend_req
+  logical :: pio_rearr_opt_i2c_enable_hs, pio_rearr_opt_i2c_enable_isend
+  integer :: pio_rearr_opt_i2c_max_pend_req
   integer :: total_comps=0
 
 #define DEBUGI 1
@@ -144,6 +148,7 @@ subroutine shr_pio_init2(comp_id, comp_name, comp_iamin, comp_comm, comp_comm_ia
     integer :: ncomps
     character(len=shr_kind_cl) :: nlfilename, cname
     type(iosystem_desc_t) :: iosys
+    integer :: ret
     character(*), parameter :: subName = '(shr_pio_init2) '
 
     if(pio_debug_level>0) then
@@ -176,12 +181,18 @@ subroutine shr_pio_init2(comp_id, comp_name, comp_iamin, comp_comm, comp_comm_ia
     allocate(iosystems(total_comps))
 
     if(pio_async_interface) then
-#ifdef PIO1
-       call pio_init(total_comps,mpi_comm_world, comp_comm, io_comm, iosystems, rearr_opts=pio_rearr_opts)
-
-#else
        call pio_init(total_comps,mpi_comm_world, comp_comm, io_comm, iosystems)
-#endif
+       do i=1,total_comps
+         ret =  pio_set_rearr_opts(iosystems(i), pio_rearr_opt_comm_type,&
+                  pio_rearr_opt_fcd,&
+                  pio_rearr_opt_c2i_enable_hs, pio_rearr_opt_c2i_enable_isend,&
+                  pio_rearr_opt_c2i_max_pend_req,&
+                  pio_rearr_opt_i2c_enable_hs, pio_rearr_opt_i2c_enable_isend,&
+                  pio_rearr_opt_i2c_max_pend_req)
+         if(ret /= PIO_NOERR) then
+            write(shr_log_unit,*) "ERROR: Setting rearranger options failed"
+         end if
+       end do
        i=1
     else
        do i=1,total_comps
@@ -196,25 +207,25 @@ subroutine shr_pio_init2(comp_id, comp_name, comp_iamin, comp_comm, comp_comm_ia
              call shr_pio_read_component_namelist(nlfilename , comp_comm(i), pio_comp_settings(i)%pio_stride, &
                   pio_comp_settings(i)%pio_root, pio_comp_settings(i)%pio_numiotasks, &
                   pio_comp_settings(i)%pio_iotype, pio_comp_settings(i)%pio_rearranger)
-#ifdef PIO1
              call pio_init(comp_comm_iam(i), comp_comm(i), pio_comp_settings(i)%pio_numiotasks, 0, &
                   pio_comp_settings(i)%pio_stride, &
                   pio_comp_settings(i)%pio_rearranger, iosystems(i), &
-                  base=pio_comp_settings(i)%pio_root, rearr_opts=pio_rearr_opts)
+                  base=pio_comp_settings(i)%pio_root)
+             ret = pio_set_rearr_opts(iosystems(i), pio_rearr_opt_comm_type,&
+                    pio_rearr_opt_fcd,&
+                    pio_rearr_opt_c2i_enable_hs, pio_rearr_opt_c2i_enable_isend,&
+                    pio_rearr_opt_c2i_max_pend_req,&
+                    pio_rearr_opt_i2c_enable_hs, pio_rearr_opt_i2c_enable_isend,&
+                    pio_rearr_opt_i2c_max_pend_req)
+             if(ret /= PIO_NOERR) then
+                write(shr_log_unit,*) "ERROR: Setting rearranger options failed"
+             end if
              if(comp_comm_iam(i)==0) then
                 write(shr_log_unit,*) io_compname(i),' : pio_numiotasks = ',pio_comp_settings(i)%pio_numiotasks
                 write(shr_log_unit,*) io_compname(i),' : pio_stride = ',pio_comp_settings(i)%pio_stride
                 write(shr_log_unit,*) io_compname(i),' : pio_root = ',pio_comp_settings(i)%pio_root
                 write(shr_log_unit,*) io_compname(i),' : pio_iotype = ',pio_comp_settings(i)%pio_iotype
              end if
-#else
-             call pio_init(comp_comm_iam(i), comp_comm(i), pio_comp_settings(i)%pio_numiotasks, 0, &
-                  pio_comp_settings(i)%pio_stride, &
-                  pio_comp_settings(i)%pio_rearranger, iosystems(i), &
-                  base=pio_comp_settings(i)%pio_root)
-#endif
-
-
           end if
        end do
     end if
@@ -383,9 +394,9 @@ subroutine shr_pio_read_default_namelist(nlfilename, Comm, pio_stride, pio_root,
 
     integer :: iam, ierr, npes, unitn
     logical :: iamroot
-    namelist /pio_default_inparm/ pio_stride, pio_root, pio_numiotasks, &
-          pio_typename, pio_async_interface, pio_debug_level, pio_blocksize, &
-          pio_buffer_size_limit, pio_rearranger, &
+    namelist /pio_default_inparm/  &
+          pio_async_interface, pio_debug_level, pio_blocksize, &
+          pio_buffer_size_limit, &
           pio_rearr_comm_type, pio_rearr_comm_fcd, &
           pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, &
           pio_rearr_comm_enable_isend_comp2io, &
@@ -417,6 +428,15 @@ subroutine shr_pio_read_default_namelist(nlfilename, Comm, pio_stride, pio_root,
     pio_async_interface = .false.   ! pio tasks are a subset of component tasks
     pio_rearranger = PIO_REARR_SUBSET
 
+    pio_rearr_comm_type = 'p2p'
+    pio_rearr_comm_fcd = '2denable'
+    pio_rearr_comm_max_pend_req_comp2io = 0
+    pio_rearr_comm_enable_hs_comp2io = .true.
+    pio_rearr_comm_enable_isend_comp2io = .false.
+    pio_rearr_comm_max_pend_req_io2comp = 0
+    pio_rearr_comm_enable_hs_io2comp = .true.
+    pio_rearr_comm_enable_isend_io2comp = .false.
+
     if(iamroot) then
        unitn=shr_file_getunit()
        open( unitn, file=trim(nlfilename), status='old' , iostat=ierr)
@@ -447,13 +467,11 @@ subroutine shr_pio_read_default_namelist(nlfilename, Comm, pio_stride, pio_root,
     call shr_mpi_bcast(pio_async_interface, Comm)
     call shr_mpi_bcast(pio_rearranger, Comm)
 
-#ifdef PIO1
      call shr_pio_rearr_opts_set(Comm, pio_rearr_comm_type, pio_rearr_comm_fcd, &
            pio_rearr_comm_max_pend_req_comp2io, pio_rearr_comm_enable_hs_comp2io, &
            pio_rearr_comm_enable_isend_comp2io, &
            pio_rearr_comm_max_pend_req_io2comp, pio_rearr_comm_enable_hs_io2comp, &
            pio_rearr_comm_enable_isend_io2comp, pio_numiotasks)
-#endif
 
   end subroutine shr_pio_read_default_namelist
 
@@ -645,7 +663,7 @@ subroutine shr_pio_namelist_set(npes,mycomm, pio_stride, pio_root, pio_numiotask
     end if
 
   end subroutine shr_pio_namelist_set
-#ifdef PIO1
+
   ! This subroutine sets the global PIO rearranger options
   ! The input args that represent the rearranger options are valid only
   ! on the root proc of comm
@@ -793,32 +811,31 @@ subroutine shr_pio_rearr_opts_set(comm, pio_rearr_comm_type, pio_rearr_comm_fcd,
     ! buf(6) = max_pend_req_io2comp
     ! buf(7) = enable_hs_io2comp
     ! buf(8) = enable_isend_io2comp
-    pio_rearr_opts%comm_type = buf(1)
-    pio_rearr_opts%fcd = buf(2)
-    pio_rearr_opts%comm_fc_opts_comp2io%max_pend_req = buf(3)
+    pio_rearr_opt_comm_type = buf(1)
+    pio_rearr_opt_fcd = buf(2)
+    pio_rearr_opt_c2i_max_pend_req = buf(3)
     if(buf(4) == 0) then
-      pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = .false.
+      pio_rearr_opt_c2i_enable_hs = .false.
     else
-      pio_rearr_opts%comm_fc_opts_comp2io%enable_hs = .true.
+      pio_rearr_opt_c2i_enable_hs = .true.
     end if
     if(buf(5) == 0) then
-      pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = .false.
+      pio_rearr_opt_c2i_enable_isend = .false.
     else
-      pio_rearr_opts%comm_fc_opts_comp2io%enable_isend = .true.
+      pio_rearr_opt_c2i_enable_isend = .true.
     end if
-    pio_rearr_opts%comm_fc_opts_io2comp%max_pend_req = buf(6)
+    pio_rearr_opt_i2c_max_pend_req = buf(6)
     if(buf(7) == 0) then
-      pio_rearr_opts%comm_fc_opts_io2comp%enable_hs = .false.
+      pio_rearr_opt_i2c_enable_hs = .false.
     else
-      pio_rearr_opts%comm_fc_opts_io2comp%enable_hs = .true.
+      pio_rearr_opt_i2c_enable_hs = .true.
     end if
     if(buf(8) == 0) then
-      pio_rearr_opts%comm_fc_opts_io2comp%enable_isend = .false.
+      pio_rearr_opt_i2c_enable_isend = .false.
     else
-      pio_rearr_opts%comm_fc_opts_io2comp%enable_isend = .true.
+      pio_rearr_opt_i2c_enable_isend = .true.
     end if
   end subroutine
-#endif
 !===============================================================================
 
 end module shr_pio_mod
diff --git a/share/csm_share/shr/shr_precip_mod.F90 b/src/share/util/shr_precip_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_precip_mod.F90
rename to src/share/util/shr_precip_mod.F90
diff --git a/share/csm_share/shr/shr_reprosum_mod.F90 b/src/share/util/shr_reprosum_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_reprosum_mod.F90
rename to src/share/util/shr_reprosum_mod.F90
diff --git a/share/csm_share/shr/shr_reprosumx86.c b/src/share/util/shr_reprosumx86.c
similarity index 100%
rename from share/csm_share/shr/shr_reprosumx86.c
rename to src/share/util/shr_reprosumx86.c
diff --git a/share/csm_share/shr/shr_scam_mod.F90 b/src/share/util/shr_scam_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_scam_mod.F90
rename to src/share/util/shr_scam_mod.F90
diff --git a/share/csm_share/shr/shr_spfn_mod.F90 b/src/share/util/shr_spfn_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_spfn_mod.F90
rename to src/share/util/shr_spfn_mod.F90
diff --git a/share/csm_share/shr/shr_strconvert_mod.F90 b/src/share/util/shr_strconvert_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_strconvert_mod.F90
rename to src/share/util/shr_strconvert_mod.F90
diff --git a/share/csm_share/shr/shr_strdata_mod.F90 b/src/share/util/shr_strdata_mod.F90
similarity index 99%
rename from share/csm_share/shr/shr_strdata_mod.F90
rename to src/share/util/shr_strdata_mod.F90
index 01dda274f6d..323e08fb942 100644
--- a/share/csm_share/shr/shr_strdata_mod.F90
+++ b/src/share/util/shr_strdata_mod.F90
@@ -1165,7 +1165,7 @@ subroutine shr_strdata_readnml(SDAT,file,rc,mpicom)
       call MPI_COMM_SIZE(mpicom,ntasks,rCode)
    endif
 
-!--master--task--
+   !--master--task--
    if (my_task == master_task) then
 
    !----------------------------------------------------------------------------
@@ -1241,15 +1241,21 @@ subroutine shr_strdata_readnml(SDAT,file,rc,mpicom)
    end do
 
    do n = 1,SDAT%nstreams
-      call shr_stream_parseInput(SDAT%streams(n),fileName,yearAlign,yearFirst,yearLast)
-      call shr_stream_init(SDAT%stream(n),fileName,yearFirst,yearLast,yearAlign, &
-                   trim(SDAT%taxMode(n)))
+      if (trim(SDAT%streams(n)) /= shr_strdata_nullstr) then
+
+         ! extract fileName (stream description text file), yearAlign, yearFirst, yearLast from SDAT%streams(n)
+         call shr_stream_parseInput(SDAT%streams(n), fileName, yearAlign, yearFirst, yearLast)
+
+         ! initialize stream datatype, read description text file
+         call shr_stream_init(SDAT%stream(n), fileName, yearFirst, yearLast, yearAlign, trim(SDAT%taxMode(n)))
+
+      end if
    enddo
 
-!   call shr_strdata_print(SDAT,trim(file)//' NML_ONLY')
+   !   call shr_strdata_print(SDAT,trim(file)//' NML_ONLY')
 
    endif   ! master_task
-!--master--task--
+   !--master--task--
 
    if (present(mpicom)) then
       call shr_strdata_bcastnml(SDAT,mpicom)
diff --git a/share/csm_share/shr/shr_stream_mod.F90 b/src/share/util/shr_stream_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_stream_mod.F90
rename to src/share/util/shr_stream_mod.F90
diff --git a/share/csm_share/shr/shr_string_mod.F90 b/src/share/util/shr_string_mod.F90
similarity index 99%
rename from share/csm_share/shr/shr_string_mod.F90
rename to src/share/util/shr_string_mod.F90
index 428f9ac881d..f50898227bc 100644
--- a/share/csm_share/shr/shr_string_mod.F90
+++ b/src/share/util/shr_string_mod.F90
@@ -1732,7 +1732,7 @@ function shr_string_listCreateField( numFields, strBase ) result ( retString )
    ! this assert isn't that accurate since it counts all integers as being one
    ! digit, but it should catch most errors and under rather than overestimates
    !
-   SHR_ASSERT( ( ( ( len(strBase) + 3 ) * numFields ) <= 1024 ) , errMsg(__FILE__, __LINE__) )
+   SHR_ASSERT_FL( ( ( ( len(strBase) + 3 ) * numFields ) <= 1024 ) , __FILE__, __LINE__)
 
    retString = ''
    do idx = 1,numFields
diff --git a/share/csm_share/shr/shr_sys_mod.F90 b/src/share/util/shr_sys_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_sys_mod.F90
rename to src/share/util/shr_sys_mod.F90
diff --git a/share/csm_share/shr/shr_tInterp_mod.F90 b/src/share/util/shr_tInterp_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_tInterp_mod.F90
rename to src/share/util/shr_tInterp_mod.F90
diff --git a/share/csm_share/shr/shr_timer_mod.F90 b/src/share/util/shr_timer_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_timer_mod.F90
rename to src/share/util/shr_timer_mod.F90
diff --git a/share/csm_share/shr/shr_vmath_mod.F90 b/src/share/util/shr_vmath_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_vmath_mod.F90
rename to src/share/util/shr_vmath_mod.F90
diff --git a/share/csm_share/shr/shr_wv_sat_mod.F90 b/src/share/util/shr_wv_sat_mod.F90
similarity index 100%
rename from share/csm_share/shr/shr_wv_sat_mod.F90
rename to src/share/util/shr_wv_sat_mod.F90
diff --git a/share/csm_share/shr/water_isotopes.F90 b/src/share/util/water_isotopes.F90
similarity index 100%
rename from share/csm_share/shr/water_isotopes.F90
rename to src/share/util/water_isotopes.F90
diff --git a/share/csm_share/shr/water_types.F90 b/src/share/util/water_types.F90
similarity index 100%
rename from share/csm_share/shr/water_types.F90
rename to src/share/util/water_types.F90
diff --git a/tools/Readme.configure b/tools/Readme.configure
index 6e1058e6472..36b310ee5f9 100644
--- a/tools/Readme.configure
+++ b/tools/Readme.configure
@@ -1,36 +1,60 @@
-SYNOPSIS
-     configure [options]
-OPTIONS
-     User supplied values are denoted in angle brackets (<>).  Any value that contains
-     white-space must be quoted.  Long option names may be supplied with either single
-     or double leading dashes.  A consequence of this is that single letter options may
-     NOT be bundled.
+usage: configure [-h] [-d] [-v] [-s] [--machine MACHINE]
+                 [--machines-dir MACHINES_DIR]
+                 [--macros-format {Makefile,CMake}] [--output-dir OUTPUT_DIR]
+                 [--compiler COMPILER] [--mpilib MPILIB]
 
-     -cimeroot            Specify the toplevel cime directory.
-                          default: use CIMEROOT environment variable
-     -mach          Specify a machine (required).
-     -compiler      Specify a compiler for the target machine (optional)
-                          default: default compiler for the target machine
-     -mpilib        Specify an mpi library for the target machine (optional)
-                          default: mpi-serial  (most of the tools are serial apps)
-     -mach_dir      Specify the locations of the Machines directory (optional).
+This script writes CIME build information to a directory. The pieces of
+information that will be written include: 1. Machine-specific build settings
+(i.e. the "Macros" file). 2. File-specific build settings (i.e. "Depends"
+files). 3. Environment variable loads (i.e. the env_mach_specific files). The
+.env_mach_specific.sh and .env_mach_specific.csh files are specific to a given
+compiler, MPI library, and DEBUG setting. By default, these will be the
+machine's default compiler, the machine's default MPI library, and FALSE,
+respectively. These can be changed by setting the environment variables
+COMPILER, MPILIB, and DEBUG, respectively.
 
-     -output_dir    default: current working directory
-     -output_format  Output format can be make or cmake.
-                          default: make
-     -help [or -h]        Print usage to STDOUT (optional).
-     -loglevel
+optional arguments:
+  -h, --help            show this help message and exit
+  -d, --debug           Print debug information (very verbose) to file /home/j
+                        edwards/cesm2_0_alpha/cime/tools/mapping/map_field/src
+                        /configure.log
+  -v, --verbose         Add additional context (time and file) to log messages
+  -s, --silent          Print only warnings and error messages
+  --machine MACHINE     The machine to create build information for.
+  --machines-dir MACHINES_DIR
+                        The machines directory to take build information from.
+                        Overrides the CIME_MODEL environment variable, and
+                        must be specified if that variable is not set.
+  --macros-format {Makefile,CMake}
+                        The format of Macros file to generate. If 'Makefile'
+                        is passed in, a file called 'Macros.make' is
+                        generated. If 'CMake' is passed in, a file called
+                        'Macros.cmake' is generated. This option can be
+                        specified multiple times to generate multiple files.
+                        If not used at all, Macros generation is skipped. Note
+                        that Depends files are currently always in Makefile
+                        format, regardless of this option.
+  --output-dir OUTPUT_DIR
+                        The directory to write files to. If not specified,
+                        defaults to the current working directory.
+  --compiler COMPILER, -compiler COMPILER
+                        Specify a compiler. To see list of supported compilers
+                        for each machine, use the utility manage_case in this
+                        directory
+  --mpilib MPILIB, -mpilib MPILIB
+                        Specify the mpilib. To see list of supported mpilibs
+                        for each machine, use the utility manage_case in this
+                        directory. The default is the first listing in MPILIBS
+                        in config_machines.xml
+  --clean               Remove old Macros and env files before attempting to
+                        create new ones
 
-EXAMPLES
 
-  ./configure -mach bluefire -compiler ibm -cimeroot /path/to/cime
 
-
-This tool is used to create Macros files to build CESM tools on supported machines.
 It will create a Macros file in make or cmake format along with the module support
 for the specified or default compiler.   It will also create two files
 .env_mach_specific.csh
 .env_mach_specific.sh
 
 Before you try to run make you should source the file above appropriate for your shell
-this will set the proper environment for supported systems which use modules.
\ No newline at end of file
+this will set the proper environment for supported systems which use modules.
diff --git a/tools/configure b/tools/configure
index 3632f599fca..997b1d3e524 100755
--- a/tools/configure
+++ b/tools/configure
@@ -20,7 +20,7 @@ import sys
 
 _CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
 sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools"))
-sys.path.append(os.path.join(_CIMEROOT, "scripts", "utils", "python"))
+sys.path.append(os.path.join(_CIMEROOT, "scripts", "lib"))
 
 from standard_script_setup import *
 from CIME.utils import expect, get_model
@@ -55,8 +55,21 @@ def parse_command_line(args):
                         help="The directory to write files to. If not "
                         "specified, defaults to the current working directory.")
 
+    parser.add_argument("--compiler", "-compiler",
+                        help="Specify a compiler. "
+                        "To see list of supported compilers for each machine, use the utility manage_case in this directory")
+
+    parser.add_argument("--mpilib", "-mpilib",
+                        help="Specify the mpilib. "
+                        "To see list of supported mpilibs for each machine, use the utility manage_case in this directory. "
+                        "The default is the first listing in MPILIBS in config_machines.xml")
+
+    parser.add_argument("--clean", action="store_true", 
+                        help="Remove old Macros and env files before attempting to create new ones")
+
+    argcnt = len(args)
     args = parser.parse_args()
-    CIME.utils.handle_standard_logging_options(args)
+    CIME.utils.parse_args_and_handle_standard_logging_options(args)
 
     opts = {}
     if args.machines_dir is not None:
@@ -83,7 +96,9 @@ def parse_command_line(args):
     opts['output_dir'] = args.output_dir
 
     # Set compiler.
-    if "COMPILER" in os.environ:
+    if args.compiler is not None:
+        compiler = args.compiler
+    elif "COMPILER" in os.environ:
         compiler = os.environ["COMPILER"]
     else:
         compiler = machobj.get_default_compiler()
@@ -94,11 +109,14 @@ def parse_command_line(args):
     opts['compiler'] = compiler
     opts['os'] = machobj.get_value('OS')
     # Set MPI library.
-    if "MPILIB" in os.environ:
+    if args.mpilib is not None:
+        mpilib = args.mpilib
+    elif "MPILIB" in os.environ:
         mpilib = os.environ["MPILIB"]
     else:
         mpilib = machobj.get_default_MPIlib()
         os.environ["MPILIB"] = mpilib
+
     expect(opts['machobj'].is_valid_MPIlib(mpilib),
            "Invalid MPI library name given in MPILIB environment variable: %s" %
            mpilib)
@@ -115,12 +133,25 @@ def parse_command_line(args):
         os.environ["DEBUG"] = "FALSE"
     opts['debug'] = debug
 
+
+    if args.clean:
+        files = ["Macros.make", "Macros.cmake", "env_mach_specific.xml", ".env_mach_specific.sh", 
+                 ".env_mach_specific.csh", "Depends.%s"%compiler, "Depends.%s"%args.machine,
+                  "Depends.%s.%s"%(args.machine,compiler)]
+        for file_ in files:
+            if os.path.isfile(file_):
+                logger.warn("Removing file %s"%file_)
+                os.remove(file_)
+        if argcnt == 2:
+            opts['clean_only'] = True
+
     return opts
 
 def _main():
     opts = parse_command_line(sys.argv)
-    configure(opts['machobj'], opts['output_dir'], opts['macros_format'],
-              opts['compiler'], opts['mpilib'], opts['debug'], opts['os'])
+    if "clean_only" not in opts or not opts["clean_only"]:
+        configure(opts['machobj'], opts['output_dir'], opts['macros_format'],
+                  opts['compiler'], opts['mpilib'], opts['debug'], opts['os'])
 
 if __name__ == "__main__":
     _main()
diff --git a/tools/cprnc/CMakeLists.txt b/tools/cprnc/CMakeLists.txt
index f923e6bbd3b..cdff9be18e0 100644
--- a/tools/cprnc/CMakeLists.txt
+++ b/tools/cprnc/CMakeLists.txt
@@ -1,7 +1,8 @@
 PROJECT(CPRNC C Fortran)
 
+ 	
 execute_process(COMMAND ../configure --macros-format=CMake WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR})
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "$ENV{CIMEROOT}/externals/CMake")
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "$ENV{CIMEROOT}/src/externals/CMake")
 
 INCLUDE(Macros.cmake)
 
@@ -9,7 +10,7 @@ ENABLE_LANGUAGE(Fortran)
 
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 
-SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} $ENV{CIMEROOT}/externals/pio2/cmake)
+SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} $ENV{CIMEROOT}/src/externals/pio2/cmake)
 
 # Defining NETCDF_LIBRARIES and NETCDF_INCLUDE_DIR enables bypassing
 #   the search for Netcdf
@@ -20,7 +21,7 @@ ENDIF ()
 
 ADD_CUSTOM_COMMAND(
   OUTPUT ${PROJECT_BINARY_DIR}/compare_vars_mod.F90
-  COMMAND perl ${PROJECT_SOURCE_DIR}/../../externals/genf90/genf90.pl
+  COMMAND perl ${PROJECT_SOURCE_DIR}/../../src/externals/genf90/genf90.pl
     ${PROJECT_SOURCE_DIR}/compare_vars_mod.F90.in > ${PROJECT_BINARY_DIR}/compare_vars_mod.F90
 )
 
diff --git a/tools/cprnc/Makefile b/tools/cprnc/Makefile
index 3957f8e14b7..ba5940d1e26 100644
--- a/tools/cprnc/Makefile
+++ b/tools/cprnc/Makefile
@@ -41,7 +41,7 @@ INC_NETCDF := $(NETCDF_PATH)/include
 UNAMES := $(shell uname -s)
 SNAME := $(shell uname -n | cut -c1-2)
 
-GENF90 = ../../externals/genf90/genf90.pl
+GENF90 = ../../src/externals/genf90/genf90.pl
 
 FC := $(SFC)
 FFLAGS += -I$(INC_NETCDF) -I.
diff --git a/tools/cprnc/README b/tools/cprnc/README
index 1ac0fd2e090..eceb705aafc 100644
--- a/tools/cprnc/README
+++ b/tools/cprnc/README
@@ -19,14 +19,14 @@ configure.
 
 Next, run make to build cprnc. For instance, using sh/bash as a login shell:
 
-CIMEROOT=../.. source .env_mach_specific.sh && make
+CIMEROOT=../.. source ./.env_mach_specific.sh && make
 
 Finally, put the resulting executable in CCSM_CPRNC as defined in
 config_machines.xml.
 
 You can also build cprnc using cmake:
 export CIMEROOT=../..
-MPILIB=mpi-serial source .env_mach_specific.sh
+MPILIB=mpi-serial source ./.env_mach_specific.sh
 ../configure --macros-format=CMake
 
 Next run cmake . to build the Makefile and then
diff --git a/tools/cprnc/summarize_cprnc_diffs b/tools/cprnc/summarize_cprnc_diffs
index cb8a3c8f830..8adaf87b75f 100755
--- a/tools/cprnc/summarize_cprnc_diffs
+++ b/tools/cprnc/summarize_cprnc_diffs
@@ -53,7 +53,8 @@ if (!$opts{'testid'}) {
 #    Dir => directory (gives test name)
 #    Filename  => cprnc filename
 #    Variable  => variable
-#    RMS       => normalized rms value [may or may not be present]
+#    RMS       => rms value            [may or may not be present]
+#    RMS_NORM  => normalized rms value [may or may not be present]
 #    FILLDIFF  => ' '                  [may or may not be present]
 #    DIMSIZEDIFF => ' '                [may or may not be present]
 my ($summary_hash) =
@@ -104,9 +105,12 @@ SYNOPSIS
     it looks for files of the form *.nc.cprnc.out.SUFFIX. (With this naming
     convention [i.e., looking for files of the form *.nc.cprnc.out], note that
     it only looks at output for baseline comparisons - NOT output from the test
-    itself, such as cprnc output files from the exact restart test.)
+    itself, such as cprnc output files from the exact restart test.) (Actually,
+    we also allow for files of the form *.nc_[0-9][0-9][0-9][0-9].cprnc.out,
+    such as *.nc_0001.cprnc.out and *.nc_0002.cprnc.out, to pick up
+    multi-instance files.)
 
-    Summaries of cprnc differences (normalized RMS differences, FILLDIFFs and DIMSIZEDIFFs)
+    Summaries of cprnc differences (RMS and normalized RMS differences, FILLDIFFs and DIMSIZEDIFFs)
     are placed in three output files beginning with the name 'cprnc.summary', in
     the current directory.  These files contain the same information, but one is
     sorted by test name, one is sorted by variable name, and is one sorted from
@@ -127,7 +131,7 @@ EOF
 
 
 # process_cprnc_output
-# Read through all cprnc files, and build hashes of instances of RMS, FILLDIFF and DIMSIZEDIFF
+# Read through all cprnc files, and build hashes of instances of RMS, normalized RMS, FILLDIFF and DIMSIZEDIFF
 # Inputs:
 #  - basedir
 #  - testid
@@ -147,10 +151,10 @@ sub process_cprnc_output {
 
       my @cprnc_files;
       if ($output_suffix) {
-         @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out.${output_suffix}";
+         @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out.${output_suffix} ${test_dir}/run/*.nc_[0-9][0-9][0-9][0-9].cprnc.out.${output_suffix}";
       }
       else {
-         @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out";
+         @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out ${test_dir}/run/*.nc_[0-9][0-9][0-9][0-9].cprnc.out";
       }
 
       foreach my $cprnc_file (@cprnc_files) {
@@ -214,13 +218,13 @@ sub process_line {
 
       # For RMS errors, keep the highest error found
       if ($diff_type eq "RMS") {
-         if (exists $diffs->{$key} && exists $diffs->{$key}{$diff_type}) {
-            if ($diffs->{$key}{$diff_type} > $rms_normalized) {
-               warn "WARNING: Ignoring lower RMS value:        $key : $rms_normalized < $diffs->{$key}{$diff_type}\n";
+         if (exists $diffs->{$key} && exists $diffs->{$key}{'RMS_NORM'}) {
+            if ($diffs->{$key}{'RMS_NORM'} > $rms_normalized) {
+               warn "WARNING: Ignoring lower RMS value:        $key : $rms_normalized < $diffs->{$key}{'RMS_NORM'}\n";
                return;
             }
             else {
-               warn "WARNING: Replacing RMS with higher value: $key : $rms_normalized > $diffs->{$key}{$diff_type}\n";
+               warn "WARNING: Replacing RMS with higher value: $key : $rms_normalized > $diffs->{$key}{'RMS_NORM'}\n";
             }
          }
       }
@@ -237,11 +241,16 @@ sub process_line {
       }
 
       # Whether or not the hash already contained the given key, we need to add
-      # the value of interest -- either the normalized RMS error or the fact
-      # that there is a FILLDIFF or DIMSIZEDIFF; in the latter cases, note that
-      # $rms_normalized is irrelevant, but we use it as the value of the hash
-      # anyway for simplicity.
-      $diffs->{$key}{$diff_type} = $rms_normalized;
+      # the value of interest -- either the RMS and normalized RMS errors, or
+      # the fact that there is a FILLDIFF or DIMSIZEDIFF.
+      if ($diff_type eq "RMS") {
+         $diffs->{$key}{'RMS'} = $rms;
+         $diffs->{$key}{'RMS_NORM'} = $rms_normalized;
+      } else {
+         # No meaningful value here - just record the fact that we saw a
+         # FILLDIFF or DIMSIZEDIFF
+         $diffs->{$key}{$diff_type} = "";
+      }
    } elsif ($diff_type ne '') {
       die "Unexpected diff_type: $diff_type";
    }
@@ -287,7 +296,7 @@ sub print_results_by_test {
    my $last_dir;
    my $last_filename;
 
-   my $separator_width = sum(values %$widths) + 30;
+   my $separator_width = sum(values %$widths) + 57;
 
    for my $key (@sorted_keys) {
 
@@ -327,7 +336,7 @@ sub print_results_by_varname {
 
    my $last_variable;
 
-   my $separator_width = sum(values %$widths) + 30;
+   my $separator_width = sum(values %$widths) + 57;
 
    for my $key (@sorted_keys) {
 
@@ -349,7 +358,7 @@ sub print_results_by_varname {
 
 
 
-# print_results_by_rms: Print sorted hash entries to a file, sorted by RMS
+# print_results_by_rms: Print sorted hash entries to a file, sorted by RMS_NORM
 # Inputs:
 #  - outfile: name of output file
 #  - summary_hash: hash reference containing results to print
@@ -359,7 +368,7 @@ sub print_results_by_rms {
 
    open OUT, ">", "$outfile" or die "ERROR opening $outfile";
 
-   my @sorted_keys = sort {$summary_hash->{$b}{'RMS'}      <=> $summary_hash->{$a}{'RMS'}
+   my @sorted_keys = sort {$summary_hash->{$b}{'RMS_NORM'} <=> $summary_hash->{$a}{'RMS_NORM'}
                         or $summary_hash->{$a}{'Dir'}      cmp $summary_hash->{$b}{'Dir'}
                         or $summary_hash->{$a}{'Filename'} cmp $summary_hash->{$b}{'Filename'}
                         or $summary_hash->{$a}{'Variable'} cmp $summary_hash->{$b}{'Variable'} }
@@ -381,6 +390,7 @@ sub print_results_by_rms {
 #   - Filename
 #   - Variable
 #   - RMS (optional)
+#   - RMS_NORM (optional)
 #   - FILLDIFF (optional)
 #   - DIMSIZEDIFF (optional)
 # - widths: hash reference giving widths of output strings
@@ -392,11 +402,15 @@ sub format_line {
    my $filename = $hash_ref->{'Filename'};
    my $variable = $hash_ref->{'Variable'};
    my $rms = "";
+   my $rms_normalized = "";
    my $filldiff = "";
    my $dimsizediff = "";
    if (exists $hash_ref->{'RMS'}) {
       $rms = sprintf(" : RMS %-16g", $hash_ref->{'RMS'});
    }
+   if (exists $hash_ref->{'RMS_NORM'}) {
+      $rms_normalized = sprintf(" : RMS_NORM %-16g", $hash_ref->{'RMS_NORM'});
+   }
    if (exists $hash_ref->{'FILLDIFF'}) {
       $filldiff = " : FILLDIFF";
    }
@@ -408,9 +422,9 @@ sub format_line {
    my $format = '%-' . $widths->{'Dir'} . '.' . $widths->{'Dir'} . 's : ' .
                 '%-' . $widths->{'Filename'} . '.' . $widths->{'Filename'} . 's : ' .
                 '%-' . $widths->{'Variable'} . '.' . $widths->{'Variable'} . 's' .
-                '%s%s%s';
+                '%s%s%s%s';
 
-   sprintf($format, $dir, $filename, $variable, $filldiff, $dimsizediff, $rms);
+   sprintf($format, $dir, $filename, $variable, $filldiff, $dimsizediff, $rms, $rms_normalized);
 }
 
 #=======================================================================
@@ -432,10 +446,10 @@ sub format_line {
 # process_line("FILLDIFF var1", "test_dir1", "file_b", \%diffs);
 
 # # add an RMS to existing filldiff
-# process_line("RMS var1 99 NORMALIZED 42", "test_dir1", "file_b", \%diffs);
+# process_line("RMS var1 4200 NORMALIZED 42", "test_dir1", "file_b", \%diffs);
 
 # # test basic rms error
-# process_line("RMS var17 99 NORMALIZED 3.14", "test_dir1", "file_b", \%diffs);
+# process_line("RMS var17 0.314 NORMALIZED 3.14", "test_dir1", "file_b", \%diffs);
 
 # # add a filldiff to existing rms error
 # process_line("FILLDIFF var17", "test_dir1", "file_b", \%diffs);
@@ -450,55 +464,60 @@ sub format_line {
 # process_line("RMS var100 99 NORMALIZED 100", "test_dir2", "file_d", \%diffs);
 
 # # test a warning: should issue a warning and replace the above setting
-# process_line("RMS var100 99 NORMALIZED 200", "test_dir2", "file_d", \%diffs);
+# process_line("RMS var100 9 NORMALIZED 200", "test_dir2", "file_d", \%diffs);
 
 # # test a warning: should issue a warning but NOT replace the above setting
-# process_line("RMS var100 99 NORMALIZED 50", "test_dir2", "file_d", \%diffs);
+# # (normalized RMS is smaller even though standard RMS is bigger: the normalized
+# # one should be considered in deciding whether to replace the previous setting)
+# process_line("RMS var100 999 NORMALIZED 50", "test_dir2", "file_d", \%diffs);
 
 # print Dumper(\%diffs);
 
 
-# THE ABOVE SHOULD PRINT:
+# THE ABOVE SHOULD PRINT SOMETHING LIKE THIS (though the output from Dumper will
+# likely appear in a different order):
 
 # WARNING: Replacing RMS with higher value: test_dir2 file_d var100 : 200 > 100
 # WARNING: Ignoring lower RMS value:        test_dir2 file_d var100 : 50 < 200
 # $VAR1 = {
 #           'test_dir1 file_b var17' => {
-#                                         'FILLDIFF' => '',
-#                                         'RMS' => '3.14',
-#                                         'Filename' => 'file_b',
+#                                         'RMS' => '0.314',
 #                                         'Variable' => 'var17',
-#                                         'Dir' => 'test_dir1'
-#                                       },
-#           'test_dir2 file_c var42' => {
+#                                         'Filename' => 'file_b',
 #                                         'FILLDIFF' => '',
-#                                         'Filename' => 'file_c',
-#                                         'Variable' => 'var42',
-#                                         'Dir' => 'test_dir2'
+#                                         'Dir' => 'test_dir1',
+#                                         'RMS_NORM' => '3.14'
 #                                       },
-#           'test_dir1 file_b var1' => {
-#                                        'FILLDIFF' => '',
-#                                        'RMS' => '42',
-#                                        'Filename' => 'file_b',
-#                                        'Variable' => 'var1',
-#                                        'Dir' => 'test_dir1'
-#                                      },
 #           'test_dir2 file_d var100' => {
-#                                          'RMS' => 200,
+#                                          'Dir' => 'test_dir2',
+#                                          'RMS_NORM' => 200,
 #                                          'Filename' => 'file_d',
 #                                          'Variable' => 'var100',
-#                                          'Dir' => 'test_dir2'
-#                                        }
+#                                          'RMS' => '9'
+#                                        },
+#           'test_dir1 file_b var1' => {
+#                                        'Filename' => 'file_b',
+#                                        'RMS_NORM' => '42',
+#                                        'FILLDIFF' => '',
+#                                        'Dir' => 'test_dir1',
+#                                        'RMS' => '4200',
+#                                        'Variable' => 'var1'
+#                                      },
 #           'test_dir2 file_c var43' => {
-#                                          'Variable' => 'var43',
-#                                          'DIMSIZEDIFF' => '',
-#                                          'Dir' => 'test_dir2',
-#                                          'Filename' => 'file_c'
-#                                        }
+#                                         'Variable' => 'var43',
+#                                         'DIMSIZEDIFF' => '',
+#                                         'Dir' => 'test_dir2',
+#                                         'Filename' => 'file_c'
+#                                       },
+#           'test_dir2 file_c var42' => {
+#                                         'Filename' => 'file_c',
+#                                         'Dir' => 'test_dir2',
+#                                         'FILLDIFF' => '',
+#                                         'Variable' => 'var42'
+#                                       }
 #         };
 
 
-
 #-----------------------------------------------------------------------
 # Testing the print routines
 #-----------------------------------------------------------------------
@@ -512,19 +531,19 @@ sub format_line {
 # This should give:
 
 # $ cat testout.by_rms
-# test_dir2                                : file_d                                   : var100                                   : RMS 200
-# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 42
-# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 3.14
+# test_dir2                                : file_d                                   : var100                                   : RMS 9                : RMS_NORM 200
+# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 4200             : RMS_NORM 42
+# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 0.314            : RMS_NORM 3.14
 # test_dir2                                : file_c                                   : var42                                    : FILLDIFF
 # test_dir2                                : file_c                                   : var43                                    : DIMSIZEDIFF
 # $ cat testout.by_test
-# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 42
-# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 3.14
-# ======================================================================================================================================================
+# test_dir1                                : file_b                                   : var1                                     : FILLDIFF : RMS 4200             : RMS_NORM 42
+# test_dir1                                : file_b                                   : var17                                    : FILLDIFF : RMS 0.314            : RMS_NORM 3.14
+# =================================================================================================================================================================================
 # test_dir2                                : file_c                                   : var42                                    : FILLDIFF
 # test_dir2                                : file_c                                   : var43                                    : DIMSIZEDIFF
-# ======================================================================================================================================================
-# test_dir2                                : file_d                                   : var100                                   : RMS 200
+# =================================================================================================================================================================================
+# test_dir2                                : file_d                                   : var100                                   : RMS 9                : RMS_NORM 200
 
 
 
diff --git a/tools/mapping/check_maps/.gitignore b/tools/mapping/check_maps/.gitignore
index cdfc1c16f6e..7d65634c86f 100644
--- a/tools/mapping/check_maps/.gitignore
+++ b/tools/mapping/check_maps/.gitignore
@@ -1,2 +1,4 @@
 ESMF_RegridWeightGenCheck
 src/*.o
+*.nc
+*.Log
diff --git a/tools/mapping/check_maps/README b/tools/mapping/check_maps/README
index f1731dddc39..0970d3fc28b 100644
--- a/tools/mapping/check_maps/README
+++ b/tools/mapping/check_maps/README
@@ -1,9 +1,3 @@
-==========================================================================
-$Id: README 46983 2013-05-09 22:08:12Z tcraig $
-$URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/check_maps/README $
-==========================================================================
-
-
 ===============
 ABOUT THIS TOOL
 ===============
@@ -42,23 +36,29 @@ BUILD
 =====
 
 To compile this tool, you need to have the ESMFMKFILE environment variable set.
-The easiest way to do that is to load the ESMF modules. To do this, you must
-already have the intel module loaded. Then load the version of ESMF you want
-to use (as of April 2013, the most recent version available is 6.1.1):
+The easiest way to do that is to load the ESMF modules. On yellowstone and
+cheyenne, you must already have the intel module loaded. Then load the version
+of ESMF you want to use (as of April 2017, the most recent version available is
+7.0.0):
 
+Yellowstone:
 $ module load esmf
-$ module load esmf-6.3.0r-ncdfio-uni-O
+$ module load esmf-7.0.0-ncdfio-uni-O
+
+Cheyenne:
+$ module load esmf_libs/7.0.0
+$ module load esmf-7.0.0-ncdfio-uni-O
 
-This will set ESMFMKFILE. With this variable set, enter the mapping/check_maps/src/
-directory and run gmake
+This will set ESMFMKFILE. With this variable set, enter the src/ directory and
+run gmake
 
-$ cd mapping/check_maps/src
+$ cd src
 $ gmake [VERBOSE=TRUE]
 
 VERBOSE=TRUE turns on some extra diagnostic output and is optional.
 
-Back in the mapping/check_maps/ directory, you should now have an executable
-file named ESMF_RegridWeightGenCheck.
+Back in this directory, you should now have an executable file named
+ESMF_RegridWeightGenCheck.
 
 Note that at this time, the tool only works in serial mode (building with the
 mpi-enabled version of the ESMF makefile leads to a segmentation fault when
@@ -78,6 +78,6 @@ listed in FILELIST
   --help, -h        Output this usage information
 
 Notes:
-  1) For use on yellowstone, geyser, or caldera only!
+  1) For use on cheyenne, yellowstone, geyser, or caldera only!
   2) Need to set ESMFMKFILE (see comments in Makefile) or compilation will fail
   3) If -rc option is not enabled, -v flag is ignored and verbose / concise will depend on previous compilation
diff --git a/tools/mapping/check_maps/src/ESMF_RegridWeightGenCheck.F90 b/tools/mapping/check_maps/src/ESMF_RegridWeightGenCheck.F90
index 189cc70ba6f..01c500ea1b5 100644
--- a/tools/mapping/check_maps/src/ESMF_RegridWeightGenCheck.F90
+++ b/tools/mapping/check_maps/src/ESMF_RegridWeightGenCheck.F90
@@ -716,6 +716,8 @@ program OfflineTester
               failCnt = failCnt + 1
               if (Verbose) then
                 print *, "FAILED: conservation error = ", totArea
+                print *, "        area1 = ", sum(grid1area)
+                print *, "        area2 = ", sum(grid2area)
               else
                 print *, "FAILED: conservation error = ", totArea, &
                          " in test ", j
diff --git a/tools/mapping/check_maps/src/Makefile b/tools/mapping/check_maps/src/Makefile
index 873af453426..2d1ddb3ba75 100644
--- a/tools/mapping/check_maps/src/Makefile
+++ b/tools/mapping/check_maps/src/Makefile
@@ -15,11 +15,18 @@
 ###       The executable should run on the yellowstone login node         ######
 ################################################################################
 
-ifneq ($(origin ESMFMKFILE), environment)
-$(error Environment variable ESMFMKFILE was not set.)
+# Don't require ESMF to be loaded to run "make clean":
+USE_ESMF=TRUE
+ifeq ($(MAKECMDGOALS),clean)
+  USE_ESMF=FALSE
 endif
 
-include $(ESMFMKFILE)
+ifeq ($(USE_ESMF),TRUE)
+  ifneq ($(origin ESMFMKFILE), environment)
+    $(error Environment variable ESMFMKFILE was not set.)
+  endif
+  include $(ESMFMKFILE)
+endif
 TARGET = ESMF_RegridWeightGenCheck
 
 # To compile with verbose output, run
diff --git a/tools/mapping/gen_domain_files/INSTALL b/tools/mapping/gen_domain_files/INSTALL
index a7b97822f2e..a244892d296 100644
--- a/tools/mapping/gen_domain_files/INSTALL
+++ b/tools/mapping/gen_domain_files/INSTALL
@@ -2,17 +2,16 @@
 HOW TO BUILD
 ============
 
-Prior to building, you must make sure environment variables CIMEROOT and CIME_MODEL are set.
-
 (1) $ cd src
-(2) $ $CIMEROOT/tools/configure --machine [machine name] --macros-format Makefile
-(3) Bash users:
-    $ . .env_mach_specific.sh
-    csh users:
-    $ source .env_mach_specific.csh
-(4) $ gmake
+(2) $ ../../../configure --macros-format Makefile --mpilib mpi-serial
+Bash users:
+(3) $ (. ./.env_mach_specific.sh ; gmake)
+csh users:
+(3) $ (source ./.env_mach_specific.csh ; gmake)
 
-Note: in the second step, replace [machine name] with the machine you are
-building on. Also, some machines have dedicated build nodes, so you might need
-to SSH to another node before the 'gmake' step.
+Note: in the second step, you may need to include "--machine [machine name]",
+where [machine name] is the name of the machine you are building on. In most
+cases configure can figure that out on its own, but if you get an error that is
+the first fix to try. Also, some machines have dedicated build nodes, so you
+might need to SSH to another node before the 'gmake' step.
 
diff --git a/tools/mapping/gen_domain_files/src/Makefile b/tools/mapping/gen_domain_files/src/Makefile
index 0a10703350c..82c749eb39a 100644
--- a/tools/mapping/gen_domain_files/src/Makefile
+++ b/tools/mapping/gen_domain_files/src/Makefile
@@ -36,7 +36,19 @@ default: $(EXENAME)
 $(MACROS):
 	@echo "use the configure script located in the Machines directory to create the Makefile $(MACROS) file"
 
--include $(MACROS)
+# Do not include Macros for clean or distclean targets
+USE_MACROS=TRUE
+ifeq ($(MAKECMDGOALS),clean)
+  USE_MACROS=FALSE
+endif
+ifeq ($(MAKECMDGOALS),distcleanclean)
+  USE_MACROS=FALSE
+endif
+
+ifeq ($(USE_MACROS),TRUE)
+  -include $(MACROS)
+endif
+
 # Check for the netcdf library and include directories
 ifdef NETCDF_PATH
   LIB_NETCDF:=$(NETCDF_PATH)/lib
@@ -133,7 +145,7 @@ $(EXENAME): $(OBJS)
 clean:
 	$(RM) -f $(OBJS) $(EXENAME) *.mod
 
-veryclean:
-	$(RM) -f $(OBJS) $(EXENAME) *.mod $(MACROS) env_mach* Depends* .env_mach*
+distclean: clean
+	../../../configure --clean
 
 gen_domain.o : gen_domain.F90
diff --git a/tools/mapping/gen_mapping_files/README b/tools/mapping/gen_mapping_files/README
index cd60a919454..a9594191d3a 100644
--- a/tools/mapping/gen_mapping_files/README
+++ b/tools/mapping/gen_mapping_files/README
@@ -120,8 +120,8 @@ Note: if rtm is specified and lnd is not, then this tool will
 
 You can also set the following env variables:
   ESMFBIN_PATH - Path to ESMF binaries
-                 (Leave unset on yellowstone and caldera and the tool
-                 will be loaded from modules)
+                 (Leave unset on cheyenne, yellowstone, and caldera and the
+                 tool will be loaded from modules)
   MPIEXEC ------ Name of mpirun executable
                  (default is mpirun.lsf on yellowstone and caldera; if
                  you run interactively on yellowstone, mpi is not used)
diff --git a/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/.gitignore b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/.gitignore
new file mode 100644
index 00000000000..4e025b24ca3
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/.gitignore
@@ -0,0 +1,2 @@
+*.Log
+*.nc
diff --git a/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/README b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/README
index c0f157a05c0..e0741bdb2d5 100644
--- a/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/README
+++ b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/README
@@ -82,8 +82,8 @@ where
 
 You can also set the following env variables:
   ESMFBIN_PATH - Path to ESMF binaries
-                 (Leave unset on yellowstone and caldera and the tool
-                 will be loaded from modules)
+                 (Leave unset on cheyenne, yellowstone, and caldera and the
+                 tool will be loaded from modules)
   MPIEXEC ------ Name of mpirun executable
                  (default is mpirun.lsf on yellowstone and caldera; if
                  you run interactively on yellowstone, mpi is not used)
diff --git a/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/create_ESMF_map.sh b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/create_ESMF_map.sh
index 012fbe9e417..60448bf1c8d 100755
--- a/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/create_ESMF_map.sh
+++ b/tools/mapping/gen_mapping_files/gen_ESMF_mapping_file/create_ESMF_map.sh
@@ -80,8 +80,8 @@ usage() {
   echo ''
   echo 'You can also set the following env variables:'
   echo '  ESMFBIN_PATH - Path to ESMF binaries '
-  echo '                 (Leave unset on yellowstone/caldera/pronghorn and the tool'
-  echo '                 will be loaded from modules)'
+  echo '                 (Leave unset on cheyenne/yellowstone/caldera/pronghorn'
+  echo '                 and the tool will be loaded from modules)'
   echo '  MPIEXEC ------ Name of mpirun executable'
   echo '                 (default is mpirun.lsf on yellowstone/caldera/pronghorn; if'
   echo '                 you run interactively on yellowstone, mpi is not used)'
@@ -225,6 +225,9 @@ if [ $MACH == "UNSET" ]; then
     ys* )
       MACH="yellowstone"
     ;;
+    cheyenne* )
+      MACH="cheyenne"
+    ;;
     geyser* )
       MACH="geyser"
     ;;
@@ -244,10 +247,13 @@ if [ $MACH == "UNSET" ]; then
 fi
 
 # Machine specific settings:
-# 1) can not run in parallel interactively on yellowstone
+# 1) can not run in parallel interactively on yellowstone or cheyenne
 if [ $MACH == "yellowstone" ] && [ $interactive == "YES" ]; then
   serial="TRUE"
 fi
+if [ $MACH == "cheyenne" ] && [ $interactive == "YES" ]; then
+  serial="TRUE"
+fi
 # 2) jaguar requires additional environment var
 if [ $MACH == "jaguar" ] && [ -z "$REGRID_PROC" ]; then
   REGRID_PROC=8
@@ -310,22 +316,26 @@ fi
 
 case $MACH in
   ## yellowstone, geyser, caldera, or pronghorn
-  "yellowstone" | "geyser" | "caldera" | "pronghorn" )
+  "cheyenne" | "yellowstone" | "geyser" | "caldera" | "pronghorn" )
     # From tcsh, script will not find module command
     # So check to see if module works, otherwise source an init file
     module list > /dev/null 2>&1 || source /etc/profile.d/modules.sh
     module purge
     module load intel
     module load nco
-    module load esmf
+    if [  $MACH == "cheyenne" ]; then
+      module load esmf_libs/7.0.0
+    else
+      module load esmf
+    fi
 
     if [ $serial == "TRUE" ]; then
-      module load esmf-6.3.0rp1-ncdfio-uni-O
+      module load esmf-7.0.0-ncdfio-uni-O
       if [ -z "$MPIEXEC" ]; then
         MPIEXEC=""
       fi
     else
-      module load esmf-6.3.0rp1-ncdfio-mpi-O
+      module load esmf-7.0.0-ncdfio-mpi-O
       if [ -z "$MPIEXEC" ]; then
         MPIEXEC="mpirun.lsf"
       fi
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL b/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL
index 098bcb55443..c4af5677228 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/INSTALL
@@ -2,17 +2,16 @@
 HOW TO BUILD
 ============
 
-Prior to building, you must make sure $CIMEROOT is set.
-
 (1) $ cd src
-(2) $ $CIMEROOT/tools/configure -mach [machine name]
-(3) Bash users:
-    $ . .env_mach_specific.sh
-    csh users:
-    $ source .env_mach_specific.csh
-(4) $ gmake
+(2) $ ../../../../configure --macros-format Makefile --mpilib mpi-serial
+Bash users:
+(3) $ (. ./.env_mach_specific.sh ; gmake)
+csh users:
+(3) $ (source ./.env_mach_specific.csh ; gmake)
 
-Note: in the second step, replace [machine name] with the machine you are
-building on. Also, some machines have dedicated build nodes, so you might need
-to SSH to another node before the 'gmake' step.
+Note: in the second step, you may need to include "--machine [machine name]",
+where [machine name] is the name of the machine you are building on. In most
+cases configure can figure that out on its own, but if you get an error that is
+the first fix to try. Also, some machines have dedicated build nodes, so you
+might need to SSH to another node before the 'gmake' step.
 
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/README b/tools/mapping/gen_mapping_files/runoff_to_ocn/README
index fe37d86ba3d..fe6a1f54e77 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/README
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/README
@@ -1,86 +1,159 @@
-===============
-ABOUT THIS TOOL
-===============
+=================
+ABOUT THESE TOOLS
+=================
 
-The src/ directory here contains F90 files and a Makefile to produce the
-runoff_map executable, which reads in a runoff file and an ocean grid file
-and outputs a map from the former to the latter:
+The CIME coupler can map two different types of runoff to the ocean: liquid
+(river) and ice; these tools are provided to help generate both maps. The src/
+directory contains F90 files and a Makefile to produce the runoff_map executable
+(see the INSTALL file for more details), and run_merge_mapping_files.sh is a
+bash script that provides a clean interface to an NCL script.
 
-1) Computes an initial nearest neighbor mapping from the rof grid cells to
-   active ocean grid cells.
-2) Computes a smoothing matrix on the ocean grid.
-3) The two matrices are multipled together to generate the final mapping file.
+======================
+HOW TO USE THESE TOOLS
+======================
 
-====================
-HOW TO USE THIS TOOL
-====================
+1) Build (see the INSTALL file in this directory for details)
+2) Setup a namelist file (see the "NAMELIST FILE FORMAT" section below or any
+   runoff_map_*.nml for an example)
+3) Run with the following command:
 
-1) Build (in the src/ directory)
-2) Setup a namelist file (runoff_map.nml)
-3) Run ./runoff_map
+$ ./runoff_map < [namelist file]
 
-============
-HOW TO BUILD
-============
+4) For some ocean models, it is useful to use combine two liquid runoff maps
+   so that river runoff sent to the open ocean is treated differently than
+   runoff sent to a marginal sea. This step is generally not needed for ice
+   runoff, but two maps can be merged with the following command:
+   
+$ ./run_merge_mapping_files.sh
 
-See INSTALL file in this directory
+See the "MERGE MAPPING FILES" section below for more details.
 
 ====================
 NAMELIST FILE FORMAT
 ====================
 
-The namelist file must be called "runoff_map.nml".  That name is hardwired
-into the executable.
-
 The namelist looks like this
 
-  &input_nml
-   gridtype     = 'scrip'
-   file_roff    = '/glade/p/cesm/cseg/mapping/grids/wr50a_090614.nc'
-   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/ar9v4_100920.nc'
-   file_nn      = 'map_wr50a_ar9v4_nn.nc '
-   file_smooth  = 'map_ar9v4_ar9v4_smoother.nc '
-   file_new     = 'map_wr50a_to_ar9v4_e1000r300_130507.nc'
-   title        = 'runoff map: wr50a -> ar9v4, smoothed '
+&input_nml
+   gridtype     = 'rtm'
+   file_roff    = '/glade/p/cesm/cseg/inputdata/lnd/clm2/rtmdata/rdirc.05.061026'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx1v7_151008.nc'
+   file_ocn_coastal_mask = '/glade/p/cesm/cseg/mapping/grids/gx1v7_coast_170322.nc'
+   file_nn      = 'map_r05_to_gx1v7_coast_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx1v7_coast_to_gx1v7_sm_e1000r300_170324.nc'
+   file_new     = 'map_r05_to_gx1v7_nnsm_e1000r300_170324.nc'
+   title        = 'runoff map: r05 -> gx1v7, nearest neighbor and smoothed'
    eFold        = 1000000.0
    rMax         =  300000.0
+   restrict_smooth_src_to_nn_dest = .true.
    step1 = .true.
-   step2 = .false.
-   step3 = .false.
-  /
-
-where
-
-Input grid files:
-  file_roff =   an ascii rdirc file OR an obs rtm file OR a scrip grid file
-  gridtype  =  type of file_roff file, "rtm" or "obs" or "scrip"
-     rtm is a 720 x 360 grid ascii file
-     obs is a netcdf file with xc, yc, xv, yv, mask and area variable names
-     scrip is a scrip type grid file (must contain grid_area along with typical
-       scrip grid variables)
-  file_ocn  =  a scrip ocean grid file (must contain grid_area along with
-    typical scrip grid variables)
-
-Input parameters:
-  eFold  = smoothing eFold distance in meters
-  rMax   = maximum radius of effect in meters
-
-Settings:
-  title = ascii string to add to mapping files
-  step1 = computes nearest neighbor map
-  step2 = computes smooth map
-  step3 = multiple two maps together
-
-Output fields:
-  file_nn     = nearest neighbor mapping file
-  file_smooth = smoother mapping file
-  file_new    = combined file
-
-
-==========
-HOW TO RUN
-==========
-
-Execute the binary ./runoff_map (again, this will read the runoff_map.nml
-namelist file - see above section for tips on creating the file)
+   step2 = .true.
+   step3 = .true.
+/
+
+Where the variables can be divided into four categories:
+
+1. Input grid files
+  gridtype =  type of file_roff file, "rtm" or "obs" or "scrip"
+              * rtm is a 720 x 360 grid ascii file
+              * obs is a netcdf file with xc, yc, xv, yv, mask and area variable names
+              * scrip is a scrip type grid file (must contain grid_area along with
+                typical scrip grid variables)
+  file_roff  =  an ascii rdirc file OR an obs rtm file OR a scrip grid file
+  file_ocn   =  a scrip ocean grid file where the mask is 1 for all ocean grid
+                cells (see note 3 below)
+  file_ocn_coastal_mask  =  a scrip ocean grid file where the mask is only 1
+                            for coastal grid cells (see note 3 below)
+
+NOTES:
+1) gridtype, file_roff, and file_ocn MUST be specified in the namelist
+2) if file_ocn_coastal_mask is not specified, file_ocn will be used
+3) The file_ocn and file_ocn_coast_mask must be standard scrip grid files that
+   include the cell area
+
+2. Input parameters
+  eFold = smoothing eFold distance in meters (default: 1000000)
+  rMax  = maximum radius of effect in meters (default: 300000)
+
+3. Settings
+  title                 = ascii string to add to mapping files (default: 'unset')
+  restrict_smooth_src_to_nn_dest = option to limit the source points for step2 to
+                                   just the points that get mapped to in step1; if
+                                   false, use all ocean points in
+                                   file_ocn_coastal_mask instead (default: .true.)
+  step1                 = computes nearest neighbor map (default: .true.)
+  step2                 = computes smooth map (default: .true.)
+  step3                 = multiply two maps together (default: .true.)
+
+4. Output fields
+  file_nn     = nearest neighbor mapping file (default: 'nn.nc')
+  file_smooth = smoother mapping file (default: 'smooth.nc')
+  file_new    = combined file (default: 'nnsm.nc')
+
+========================
+WHAT THESE TOOLS PROVIDE
+========================
+
+The runoff_map executable generates three maps:
+
+1) A nearest neighbor mapping from the rof grid to the ocean grid.
+   * For river runoff, it may be desireble to limit the destination grid so that
+     runoff only enters at coastal points. The optional file_ocn_coast_mask
+     variable in the namelist allows you to specify two ocean grid files; one for
+     the full ocean and one for just the coast. Typically this should not be
+     specified for ice runoff.
+2) A smoothing map from the ocean grid to the ocean grid.
+   * If file_ocn_coast_mask was specified, this maps from the coastal ocean to
+     the open ocean; otherwise it maps from the ocean grid onto itself.
+   * By default, the source domain of this map is only cells that appear as
+     destination cells of the nearest neighbor map. Optionally, one can
+     set restrict_smooth_src_to_nn_dest = .false. and generate a map that maps
+     from all ocean cells, even if they are not destination cells of the nearest
+     neighbor map. Enabling this map will not change the resulting map, but will
+     allow the resulting map to be reused if you are generating multiple maps
+     from different runoff grids onto the same ocean grid.
+   * If file_ocn_coast_mask is not specified, be aware that setting
+     restrict_smooth_src_to_nn_dest = .false. will take significantly longer
+     (and generate a much larger file) than leaving it .true.
+3) The product of the previous two maps.
+
+The ncl/ directory contains an NCL script to compute a fourth map (which is
+actually a combination of two existing maps):
+
+4) Map (1) from above for rof grid cells that map into the open ocean, or map
+   (3) for the rof grid cells that map into a marginal sea.
+   * This is only necessary if runoff should be treated differently in the open
+     ocean than in marginal seas, otherwise map (3) from above should be used
+     and this step is not needed.
+
+===================
+MERGE MAPPING FILES
+===================
+
+The run_merge_mapping_files.sh script requires four arguments:
+
+$ ./run_merge_mapping_files --map_in_oo MAP_IN_OO_FNAME     \
+                            --map_in_ms MAP_IN_MS_FNAME     \
+                            --region_mask REGION_MASK_FNAME \
+                            --map_out MAP_OUT_FNAME         \
+                            [-h -v]
+
+  -h,--help     show this message
+  -v,--verbose  echo file names back to screen before running NCL script
+  --map_in_oo MAP_IN_OO_FNAME
+                mapping file containing map to open ocean points
+  --map_in_ms MAP_IN_MS_FNAME
+                mapping file containing map to marginal sea points
+  --region_mask REGION_MASK_FNAME
+                POP region mask file (to specify open ocean vs marginal sea)
+  --map_out MAP_OUT_FNAME
+                output file
+
+A couple of useful notes:
+* MAP_IN_OO_FNAME and MAP_IN_MS_FNAME are typically maps (1) and (3),
+  respectively, from the "WHAT THESE TOOLS PROVIDE" section.
+* REGION_MASK_FNAME is a binary file (written with big endian convention)
+  containing one integer per grid cell -- a positive number represents a region
+  in open ocean, a negative number represents a region in a marginal sea, and a
+  zero is land.
 
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx1v7_nnsm_e1000r300.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx1v7_nnsm_e1000r300.nml
new file mode 100644
index 00000000000..174d39af5f9
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx1v7_nnsm_e1000r300.nml
@@ -0,0 +1,16 @@
+&input_nml
+   gridtype     = 'rtm'
+   file_roff    = '/glade/p/cesm/cseg/inputdata/lnd/clm2/rtmdata/rdirc.05.061026'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx1v7_151008.nc'
+   file_ocn_coastal_mask = '/glade/p/cesm/cseg/mapping/grids/gx1v7_coast_170322.nc'
+   file_nn      = 'map_r05_to_gx1v7_coast_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx1v7_coast_to_gx1v7_sm_e1000r300_170324.nc'
+   file_new     = 'map_r05_to_gx1v7_nnsm_e1000r300_170324.nc'
+   title        = 'runoff map: r05 -> gx1v7, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  300000.0
+   restrict_smooth_src_to_nn_dest = .true.
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx3v7_nnsm_e1000r500.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx3v7_nnsm_e1000r500.nml
new file mode 100644
index 00000000000..d1c72a7382f
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_r05_to_gx3v7_nnsm_e1000r500.nml
@@ -0,0 +1,16 @@
+&input_nml
+   gridtype     = 'rtm'
+   file_roff    = '/glade/p/cesm/cseg/inputdata/lnd/clm2/rtmdata/rdirc.05.061026'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx3v7_120309.nc'
+   file_ocn_coastal_mask = '/glade/p/cesm/cseg/mapping/grids/gx3v7_coast_161214.nc'
+   file_nn      = 'map_r05_to_gx3v7_coast_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx3v7_coast_to_gx3v7_sm_e1000r500_170324.nc'
+   file_new     = 'map_r05_to_gx3v7_nnsm_e1000r500_170324.nc'
+   title        = 'runoff map: r05 -> gx3v7, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  500000.0
+   restrict_smooth_src_to_nn_dest = .true.
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx1v7_nnsm_e1000r300.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx1v7_nnsm_e1000r300.nml
new file mode 100644
index 00000000000..dbbd6a0081f
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx1v7_nnsm_e1000r300.nml
@@ -0,0 +1,16 @@
+&input_nml
+   gridtype     = 'obs'
+   file_roff    = '/glade/p/cesm/cseg/inputdata/lnd/dlnd7/RX1/runoff.daitren.annual.090225.nc'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx1v7_151008.nc'
+   file_ocn_coastal_mask = '/glade/p/cesm/cseg/mapping/grids/gx1v7_coast_170322.nc'
+   file_nn      = 'map_rx1_to_gx1v7_coast_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx1v7_coast_to_gx1v7_sm_e1000r300_170324a.nc'
+   file_new     = 'map_rx1_to_gx1v7_nnsm_e1000r300_170324.nc'
+   title        = 'runoff map: rx1 -> gx1v7, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  300000.0
+   restrict_smooth_src_to_nn_dest = .true.
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx3v7_nnsm_e1000r500.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx3v7_nnsm_e1000r500.nml
new file mode 100644
index 00000000000..9345b5f2199
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_rx1_to_gx3v7_nnsm_e1000r500.nml
@@ -0,0 +1,16 @@
+&input_nml
+   gridtype     = 'obs'
+   file_roff    = '/glade/p/cesm/cseg/inputdata/lnd/dlnd7/RX1/runoff.daitren.annual.090225.nc'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx3v7_120309.nc'
+   file_ocn_coastal_mask = '/glade/p/cesm/cseg/mapping/grids/gx3v7_coast_161214.nc'
+   file_nn      = 'map_rx1_to_gx3v7_coast_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx3v7_coast_to_gx3v7_sm_e1000r500_170324a.nc'
+   file_new     = 'map_rx1_to_gx3v7_nnsm_e1000r500_170324.nc'
+   title        = 'runoff map: rx1 -> gx3v7, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  500000.0
+   restrict_smooth_src_to_nn_dest = .true.
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_ar9v4_nnsm_e1000r300.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_ar9v4_nnsm_e1000r300.nml
new file mode 100644
index 00000000000..2f8fe7a4c21
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_ar9v4_nnsm_e1000r300.nml
@@ -0,0 +1,14 @@
+&input_nml
+   gridtype     = 'scrip'
+   file_roff    = '/glade/p/cesm/cseg/mapping/grids/wr50a_090614.nc'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/ar9v4_100920.nc'
+   file_nn      = 'map_wr50a_to_ar9v4_nearestdtos_170324.nc'
+   file_smooth  = 'map_ar9v4_to_ar9v4_sm_e1000r300_170324.nc'
+   file_new     = 'map_wr50a_to_ar9v4_nnsm_e1000r300_170324.nc'
+   title        = 'runoff map: wr50a -> ar9v4, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  300000.0
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_gx3v7_nnsm_e1000r300.nml b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_gx3v7_nnsm_e1000r300.nml
new file mode 100644
index 00000000000..29a2f9ec33a
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/map_wr50a_to_gx3v7_nnsm_e1000r300.nml
@@ -0,0 +1,14 @@
+&input_nml
+   gridtype     = 'scrip'
+   file_roff    = '/glade/p/cesm/cseg/mapping/grids/wr50a_090614.nc'
+   file_ocn     = '/glade/p/cesm/cseg/mapping/grids/gx3v7_120309.nc'
+   file_nn      = 'map_wr50a_to_gx3v7_nearestdtos_170324.nc'
+   file_smooth  = 'map_gx3v7_to_gx3v7_sm_e1000r300_170324.nc'
+   file_new     = 'map_wr50a_to_gx3v7_nnsm_e1000r300_170324.nc'
+   title        = 'runoff map: wr50a -> gx3v7, nearest neighbor and smoothed'
+   eFold        = 1000000.0
+   rMax         =  300000.0
+   step1 = .true.
+   step2 = .true.
+   step3 = .true.
+/
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/maps/.gitignore b/tools/mapping/gen_mapping_files/runoff_to_ocn/maps/.gitignore
new file mode 100644
index 00000000000..d57580fd328
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/maps/.gitignore
@@ -0,0 +1 @@
+*.nc
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/ncl/merge_mapping_files.ncl b/tools/mapping/gen_mapping_files/runoff_to_ocn/ncl/merge_mapping_files.ncl
new file mode 100644
index 00000000000..1e113bd1a25
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/ncl/merge_mapping_files.ncl
@@ -0,0 +1,210 @@
+;
+; merge_mapping_files.ncl
+;
+; merge two mapping files into one
+;
+; Keith Lindsay, 2017-03-08
+; Mike Levy, 2017-03-23 (updated how arguments are passed to this script)
+;
+;   3 options for running this program:
+;
+;     1) use the ../run_merge_mapping_files.sh wrapper script
+;        $ ../run_merge_mapping_files.sh --map_in_oo MAP_IN_OO_FNAME     \
+;                                        --map_in_ms MAP_IN_MS_FNAME     \
+;                                        --region_mask REGION_MASK_FNAME \
+;                                        --map_out MAP_OUT_FNAME
+;
+;     2) ncl merge_mapping_files.ncl 'MAP_IN_OO_FNAME="file1_name"'   \
+;                                    'MAP_IN_MS_FNAME="file2_name"'   \
+;                                    'REGION_MASK_FNAME="file3_name"' \
+;                                    'MAP_OUT_FNAME="file4_name"'
+;
+;     3) Set following environment variables:
+;        MAP_IN_OO_FNAME (entries into the open ocean)
+;        MAP_IN_MS_FNAME (entries into marginal seas)
+;        REGION_MASK_FNAME (marginal seas designation)
+;        MAP_OUT_FNAME (output file)
+;
+
+begin
+  ; Initialize variables from command line or environment
+  if (.not. isvar("MAP_IN_OO_FNAME")) then
+     MAP_IN_OO_FNAME   = getenv("MAP_IN_OO_FNAME")
+  end if
+
+  if (.not. isvar("MAP_IN_MS_FNAME")) then
+     MAP_IN_MS_FNAME   = getenv("MAP_IN_MS_FNAME")
+  end if
+
+  if (.not. isvar("REGION_MASK_FNAME")) then
+     REGION_MASK_FNAME   = getenv("REGION_MASK_FNAME")
+  end if
+
+  if (.not. isvar("MAP_OUT_FNAME")) then
+     MAP_OUT_FNAME   = getenv("MAP_OUT_FNAME")
+  end if
+
+  ; Make sure all input files exist
+  missing_files = False
+  if (.not. fileexists(MAP_IN_OO_FNAME)) then
+    print((/"ERROR: can not find open ocean map ("+MAP_IN_OO_FNAME+ ")"/))
+    missing_files = True
+  end if
+  if (.not. fileexists(MAP_IN_MS_FNAME)) then
+    print((/"ERROR: can not find marginal sea map ("+MAP_IN_MS_FNAME+ ")"/))
+    missing_files = True
+  end if
+  if (.not. fileexists(REGION_MASK_FNAME)) then
+    print((/"ERROR: can not find region mask ("+REGION_MASK_FNAME+ ")"/))
+    missing_files = True
+  end if
+
+  if (missing_files) then
+    status_exit(1)
+  end if
+
+  map_in_oo_fp      = addfile(MAP_IN_OO_FNAME, "r")
+  map_in_ms_fp      = addfile(MAP_IN_MS_FNAME, "r")
+
+  ; ensure grid definitions in MAP_IN_MS_FNAME agrees with grid definitions in MAP_IN_MS_FNAME
+  print("comparing grid vars in")
+  print("  MAP_IN_OO_FNAME = " + MAP_IN_OO_FNAME)
+  print("  MAP_IN_MS_FNAME = " + MAP_IN_MS_FNAME)
+  grid_varnames = (/ "xc_a", "yc_a", "mask_a", "area_a", "frac_a", "src_grid_dims", \
+                     "xc_b", "yc_b",           "area_b",           "dst_grid_dims" /)
+  diffs_found = False
+  do varind = 0, dimsizes(grid_varnames)-1
+    varname = grid_varnames(varind)
+    field_oo := map_in_oo_fp->$varname$
+    field_ms := map_in_ms_fp->$varname$
+    if (.not. all(ismissing(field_oo) .eq. ismissing(field_ms))) then
+      print("  " + vars1(var_ind) + " _FillValue patterns differ")
+      diffs_found = True
+    end if
+    if (all(field_oo .eq. field_ms)) then
+      print("  " + varname + " values agree")
+    else
+      print("  " + varname + " values differ")
+      if (any(abs(field_oo - field_ms) .gt. 1e-13 * (abs(field_oo)>abs(field_ms)))) then
+        print("    differences appear to be greater than round-off")
+        diffs_found = True
+      else
+        print("    differences appear to be less than round-off")
+      end if
+    end if
+  end do
+  if (.not. diffs_found) then
+    print("grid variables are compatible")
+  else
+    status_exit(1)
+  end if
+
+  print("reading grid info and map from " + MAP_IN_OO_FNAME)
+  xc_a          = map_in_oo_fp->xc_a
+  yc_a          = map_in_oo_fp->yc_a
+  xc_b          = map_in_oo_fp->xc_b
+  yc_b          = map_in_oo_fp->yc_b
+  dst_grid_dims = map_in_oo_fp->dst_grid_dims
+  map_in_oo_S   = map_in_oo_fp->S
+  map_in_oo_col = map_in_oo_fp->col ; 1-based indices
+  map_in_oo_row = map_in_oo_fp->row ; 1-based indices
+  n_s_oo        = dimsizes(map_in_oo_S)
+
+  print("reading map from " + MAP_IN_MS_FNAME)
+  map_in_ms_S   = map_in_ms_fp->S
+  map_in_ms_col = map_in_ms_fp->col
+  map_in_ms_row = map_in_ms_fp->row
+  n_s_ms        = dimsizes(map_in_ms_S)
+
+  ; ensure that maps have same unique col values, i.e., they are mapping from same src points
+  print("checking col value compatibility")
+  map_in_oo_col_unique = get_unique_values(map_in_oo_col)
+  map_in_ms_col_unique = get_unique_values(map_in_ms_col)
+  if (dimsizes(map_in_oo_col_unique) .ne. dimsizes(map_in_ms_col_unique)) then
+    print("number of unique col values in " + MAP_IN_OO_FNAME + " and " + MAP_IN_MS_FNAME + " differ")
+    status_exit(1)
+  end if
+  if (any(map_in_oo_col_unique .ne. map_in_ms_col_unique)) then
+    print("unique col values in " + MAP_IN_OO_FNAME + " and " + MAP_IN_MS_FNAME + " differ")
+    status_exit(1)
+  end if
+
+  ; load POP region mask from REGION_MASK_FNAME
+  print("reading REGION_MASK")
+  setfileoption("bin", "ReadByteOrder", "BigEndian")
+  REGION_MASK = cbinread(REGION_MASK_FNAME, (/ dst_grid_dims(1), dst_grid_dims(0) /), "integer")
+  REGION_MASK_1d = ndtooned(REGION_MASK)
+
+  ; create merged mapping
+
+  ; determine size of merged map
+
+  print("determining map_oo vals where REGION_MASK>0")
+  sign_REGION_MASK_oo = sign_matlab(REGION_MASK_1d(map_in_oo_row-1))
+  ind_vals_oo = ind(sign_REGION_MASK_oo .gt. 0)
+  n_s_subset_oo = dimsizes(ind_vals_oo)
+
+  print("determining map_ms vals where REGION_MASK<0")
+  sign_REGION_MASK_ms = sign_matlab(REGION_MASK_1d(map_in_ms_row-1))
+  ind_vals_ms = ind(sign_REGION_MASK_ms .lt. 0)
+  n_s_subset_ms = dimsizes(ind_vals_ms)
+
+  n_s_out = n_s_subset_oo + n_s_subset_ms
+
+  ; allocate output vars
+
+  map_out_S = new(n_s_out, double)
+  map_out_S!0 = "n_s"
+  delete(map_out_S@_FillValue)
+  map_out_S@long_name = map_in_ms_S@long_name
+
+  map_out_col = new(n_s_out, integer)
+  map_out_col!0 = "n_s"
+  delete(map_out_col@_FillValue)
+  map_out_col@long_name = map_in_ms_col@long_name
+
+  map_out_row = new(n_s_out, integer)
+  map_out_row!0 = "n_s"
+  delete(map_out_row@_FillValue)
+  map_out_row@long_name = map_in_ms_row@long_name
+
+  ; fill output vars
+
+  map_out_S(0:n_s_subset_oo-1)   = (/ map_in_oo_S(ind_vals_oo) /)
+  map_out_col(0:n_s_subset_oo-1) = (/ map_in_oo_col(ind_vals_oo) /)
+  map_out_row(0:n_s_subset_oo-1) = (/ map_in_oo_row(ind_vals_oo) /)
+
+  map_out_S(n_s_subset_oo:n_s_subset_oo+n_s_subset_ms-1)   = (/ map_in_ms_S(ind_vals_ms) /)
+  map_out_col(n_s_subset_oo:n_s_subset_oo+n_s_subset_ms-1) = (/ map_in_ms_col(ind_vals_ms) /)
+  map_out_row(n_s_subset_oo:n_s_subset_oo+n_s_subset_ms-1) = (/ map_in_ms_row(ind_vals_ms) /)
+
+  print("writing merged map, " + MAP_OUT_FNAME)
+  system("rm -f " + MAP_OUT_FNAME)
+  map_out_fp = addfile(MAP_OUT_FNAME, "c")
+
+  map_out_fp@history = "File created: "+systemfunc("date +%Y-%m-%d\ %H:%M:%S")
+  map_out_fp@conventions = "NCAR-CCSM"
+  map_out_fp@open_ocean_mapping_fname = MAP_IN_OO_FNAME
+  map_out_fp@marginal_seas_mapping_fname = MAP_IN_MS_FNAME
+  map_out_fp@region_mask_fname = REGION_MASK_FNAME
+
+  copy_varnames = (/ "xc_a", "yc_a", "xv_a", "yv_a", "mask_a", "area_a", "frac_a", "src_grid_dims", \
+                     "xc_b", "yc_b", "xv_b", "yv_b", "mask_b", "area_b", "frac_b", "dst_grid_dims" /)
+
+  do varind = 0, dimsizes(copy_varnames)-1
+    varname = copy_varnames(varind)
+    map_out_fp->$varname$ = map_in_ms_fp->$varname$
+  end do
+
+  ; read mask from map_ms (for size and metadata)
+  ; set it to 1 where merged mapping maps to
+  mask_b = map_in_ms_fp->mask_b
+  mask_b = 0
+  mask_b(map_out_row-1) = 1
+  map_out_fp->mask_b = (/ mask_b /)
+
+  filedimdef(map_out_fp, "n_s", n_s_out, False)
+  map_out_fp->S   = map_out_S(0:n_s_out-1)
+  map_out_fp->col = map_out_col(0:n_s_out-1)
+  map_out_fp->row = map_out_row(0:n_s_out-1)
+end
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/run_merge_mapping_files.sh b/tools/mapping/gen_mapping_files/runoff_to_ocn/run_merge_mapping_files.sh
new file mode 100755
index 00000000000..8fc5133fe7b
--- /dev/null
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/run_merge_mapping_files.sh
@@ -0,0 +1,130 @@
+#!/bin/bash
+#
+
+################################################################################
+
+main() {
+  parse_args $@
+  check_inputs
+  if [ ! -z $VERBOSE ]; then
+    print_summary
+    echo "Running ncl script..."
+    echo ""
+  fi
+
+  ncl ncl/merge_mapping_files.ncl
+}
+
+################################################################################
+
+usage() {
+  cat << EOF
+$ ./run_merge_mapping_files --map_in_oo MAP_IN_OO_FNAME     \\
+                            --map_in_ms MAP_IN_MS_FNAME     \\
+                            --region_mask REGION_MASK_FNAME \\
+                            --map_out MAP_OUT_FNAME         \\
+                            [-h -v]
+
+  -h,--help     show this message
+  -v,--verbose  echo file names back to screen before running NCL script
+  --map_in_oo MAP_IN_OO_FNAME
+                mapping file containing map to open ocean points
+  --map_in_ms MAP_IN_MS_FNAME
+                mapping file containing map to marginal sea points
+  --region_mask REGION_MASK_FNAME
+                POP region mask file (to specify open ocean vs marginal sea)
+  --map_out MAP_OUT_FNAME
+                output file
+EOF
+}
+
+################################################################################
+
+parse_args() {
+
+  # Clear pre-existing environment variables that will be set
+  export MAP_IN_OO_FNAME=""
+  export MAP_IN_MS_FNAME=""
+  export REGION_MASK_FNAME=""
+  export MAP_OUT_FNAME=""
+
+  while [ $# -gt 0 ]; do
+    case $1 in
+      --map_in_oo )
+        export MAP_IN_OO_FNAME=$2
+        shift
+      ;;
+      --map_in_ms )
+        export MAP_IN_MS_FNAME=$2
+        shift
+      ;;
+      --region_mask )
+        export REGION_MASK_FNAME=$2
+        shift
+      ;;
+      --map_out )
+        export MAP_OUT_FNAME=$2
+        shift
+      ;;
+      -v | --verbose )
+        VERBOSE=TRUE
+      ;;
+      -h | --help )
+        usage
+        exit 0
+      ;;
+      * )
+        echo "ERROR: $1 is not a valid argument"
+        echo ""
+        echo "Run $0 -h for help"
+        exit 1
+    esac
+    shift
+  done
+}
+
+################################################################################
+
+check_inputs() {
+
+  ERROR=""
+
+  if [ -z ${MAP_IN_OO_FNAME} ]; then
+    echo "ERROR: Missing --map_in_oo option"
+    ERROR="TRUE"
+  fi
+
+  if [ -z ${MAP_IN_MS_FNAME} ]; then
+    echo "ERROR: Missing --map_in_ms option"
+    ERROR="TRUE"
+  fi
+
+  if [ -z ${REGION_MASK_FNAME} ]; then
+    echo "ERROR: Missing --region_mask option"
+    ERROR="TRUE"
+  fi
+
+  if [ -z ${MAP_OUT_FNAME} ]; then
+    echo "ERROR: Missing --map_out option"
+    ERROR="TRUE"
+  fi
+
+  if [ ! -z ${ERROR} ]; then
+    echo ""
+    echo "Run $0 -h for help"
+    exit 1
+  fi
+}
+
+################################################################################
+
+print_summary() {
+  echo "Open ocean map: $MAP_IN_OO_FNAME"
+  echo "Marginal seas map: $MAP_IN_MS_FNAME"
+  echo "Region mask: $REGION_MASK_FNAME"
+  echo "Output map: $MAP_OUT_FNAME"
+}
+
+################################################################################
+
+main $@
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/Makefile b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/Makefile
index fa46862ab1d..baf1f6cf754 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/Makefile
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/Makefile
@@ -88,6 +88,7 @@ INCLDIR  += -I$(NETCDF_PATH)/include
 # echo file names, paths, compile flags, etc. used during build
 #-------------------------------------------------------------------------------
 
+.PHONY: db_files
 db_files:
 	@echo " "
 	@echo "* EXEC    := $(EXEC)"
@@ -99,6 +100,8 @@ db_files:
 	@echo "* SRCS    := $(SRCS)"
 	@echo "* OBJS    := $(OBJS)"
 	@echo "* DEPS    := $(DEPS)"
+
+.PHONY: db_flags
 db_flags:
 	@echo " "
 	@echo "* cc      := $(SCC)  $(CFLAGS) $(INCS) $(INCLDIR)"
@@ -129,14 +132,13 @@ else
 	$(SFC)  $(FFLAGS) $(FREEFLAGS) $(CPPDEFS) $(INCS) $(INCLDIR) $*.F90
 endif
 
+.PHONY: clean
 clean:
-	$(RM) -f *.f *.f90 *.d *.mod $(OBJS)
-
-realclean: clean
-	$(RM) -f $(EXEC)
+	$(RM) -f *.f *.f90 *.d *.mod $(OBJS) $(EXEC) $(DEPGEN)
 
-distclean: realclean
-	$(RM) -f $(MACFILE) env_mach_specific Depends* makdep build.csh
+.PHONY: distclean
+distclean: clean
+	../../../../configure --clean
 
 #-------------------------------------------------------------------------------
 # Build & include dependency files
@@ -169,12 +171,10 @@ $(DEPGEN) :
 # the if-tests prevent DEPS files from being created when they're not needed
 ifneq ($(MAKECMDGOALS), db_files)
 ifneq ($(MAKECMDGOALS), db_flags)
-ifneq ($(MAKECMDGOALS), mostlyclean)
 ifneq ($(MAKECMDGOALS), clean)
-ifneq ($(MAKECMDGOALS), realclean)
+ifneq ($(MAKECMDGOALS), distclean)
     -include $(DEPS)
 endif
 endif
 endif
 endif
-endif
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90
index 89cf3e49981..fbcab4d30cc 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: fixroff_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/fixroff_mod.F90 $
-!===============================================================================
-
 MODULE fixroff_mod
 
    use shr_sys_mod
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90
index 6fb4065ff1c..515219ed7cf 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: kind_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/kind_mod.F90 $
-!===============================================================================
-
 MODULE kind_mod
 
    use shr_kind_mod
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90
index f109eb3fff7..e0431901579 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/main.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: main.F90 59443 2014-04-22 22:57:10Z mlevy@ucar.edu $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/main.F90 $
-!===============================================================================
-
 PROGRAM main
 
    use shr_timer_mod
@@ -32,31 +27,36 @@ PROGRAM main
    character(10)     :: tstr            ! wall clock time
 
    !--- namelist vars ---
-   character(180) :: gridtype      ! type of run-off grid
-   character(180) :: file_roff     ! file name: rtm rdirc file
-   character(180) :: file_roff_out ! file name: rtm rdirc file
-   character(180) :: file_ocn      ! file name: ocn scrip grid file
-   character(180) :: file_nn       ! file name: orig matrix, corrected
-   character(180) :: file_new      ! file name: orig matrix, corrected, smoothed, sorted -- done
-   character(180) :: file_smooth   ! file name: smoothing matrix
-   character(180) :: title         ! netCDF title attribute
-   logical        :: step1         ! gen nn
-   logical        :: step2         ! gen smooth
-   logical        :: step3         ! mat mult
-   logical        :: lmake_rSCRIP  ! .true. => convert runoff grid to SCRIP
-
-   namelist / input_nml / &
-      gridtype      &
-   ,  file_roff     &
-   ,  file_roff_out &
-   ,  file_ocn      &
-   ,  file_nn       &
-   ,  file_new      &
-   ,  file_smooth   &
-   ,  title         &
-   ,  eFold         &
-   ,  rMax          &
-   ,  lmake_rSCRIP  &
+   character(180) :: gridtype                        ! type of run-off grid
+   character(180) :: file_roff                       ! file name: rtm rdirc file
+   character(180) :: file_roff_out                   ! file name: rtm rdirc file
+   character(180) :: file_ocn                        ! file name: ocn scrip grid file (all ocean points masked as such)
+   character(180) :: file_ocn_coastal_mask           ! file name: ocn scrip grid file (only coastal points masked as ocean)
+   character(180) :: file_nn                         ! file name: orig matrix, corrected
+   character(180) :: file_new                        ! file name: orig matrix, corrected, smoothed, sorted -- done
+   character(180) :: file_smooth                     ! file name: smoothing matrix
+   character(180) :: title                           ! netCDF title attribute
+   logical        :: restrict_smooth_src_to_nn_dest  ! use step1 dest cells as
+                                                     ! step2 source cells
+   logical        :: step1                           ! gen nn
+   logical        :: step2                           ! gen smooth
+   logical        :: step3                           ! mat mult
+   logical        :: lmake_rSCRIP                    ! .true. => convert runoff grid to SCRIP
+
+   namelist / input_nml /            &
+      gridtype                       &
+   ,  file_roff                      &
+   ,  file_roff_out                  &
+   ,  file_ocn                       &
+   ,  file_ocn_coastal_mask          &
+   ,  file_nn                        &
+   ,  file_new                       &
+   ,  file_smooth                    &
+   ,  title                          &
+   ,  eFold                          &
+   ,  rMax                           &
+   ,  lmake_rSCRIP                   &
+   ,  restrict_smooth_src_to_nn_dest &
    ,  step1, step2, step3
 
    !--- formats ---
@@ -80,8 +80,6 @@ PROGRAM main
 !-------------------------------------------------------------------------------
 
    write(6,F10) "correct/smooth/sort runoff -> ocean map"
-   write(6,F10) "SVN &
-   & $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/main.F90 $"
 
    call shr_timer_init()
 
@@ -89,39 +87,45 @@ PROGRAM main
    write(6,F10) "Step 0:  read input namelist data"
    !----------------------------------------------------------------------------
 
-   gridtype      = 'unset'
-   file_roff     = 'unset'
-   file_ocn      = 'unset'
-   file_nn       = 'unset'
-   file_smooth   = 'unset'
-   file_new      = 'unset'
-   title         = 'unset'
-   eFold         = 1000000.00000 ! smoothing eFold distance in meters
-   rMax          =  500000.00000 ! max smoothing radius in meters
-   step1         = .true.
-   step2         = .true.
-   step3         = .true.
+   gridtype                       = 'unset'
+   file_roff                      = 'unset'
+   file_ocn                       = 'unset'
+   file_ocn_coastal_mask          = 'unset'
+   file_nn                        = 'nn.nc'
+   file_smooth                    = 'smooth.nc'
+   file_new                       = 'nnsm.nc'
+   title                          = 'unset'
+   restrict_smooth_src_to_nn_dest = .true.
+   eFold                          = 1000000._R8 ! smoothing eFold distance in meters
+   rMax                           =  300000._R8 ! max smoothing radius in meters
+   step1                          = .true.
+   step2                          = .true.
+   step3                          = .true.
 
    ! These two variables typically don't appear in namelist
    lmake_rSCRIP  = .false.
    file_roff_out = "runoff.nc"
 
-   open (10,file="runoff_map.nml",status="old",action="read")
-   read (10,nml=input_nml,iostat=rCode)
-   close(10)
+   read (*,nml=input_nml,iostat=rCode)
+   if (trim(file_ocn_coastal_mask) == 'unset') then
+     file_ocn_coastal_mask = file_ocn
+   end if
+
    write(6,F00) "Namelist values..."
-   write(6,F00) "   gridtype       = ",trim(gridtype      )
-   write(6,F00) "   file_roff      = ",trim(file_roff     )
-   write(6,F00) "   file_ocn       = ",trim(file_ocn      )
-   write(6,F00) "   file_nn        = ",trim(file_nn       )
-   write(6,F00) "   file_smooth    = ",trim(file_smooth   )
-   write(6,F00) "   file_new       = ",trim(file_new      )
-   write(6,F00) "   title          = ",trim(title         )
-   write(6,F02) "   eFold distance = ",eFold
-   write(6,F02) "   rMax  distance = ",rMax
-   write(6,F03) "   step1          = ",step1
-   write(6,F03) "   step2          = ",step2
-   write(6,F03) "   step3          = ",step3
+   write(6,F00) "   gridtype                       = ",trim(gridtype      )
+   write(6,F00) "   file_roff                      = ",trim(file_roff     )
+   write(6,F00) "   file_ocn (global)              = ",trim(file_ocn)
+   write(6,F00) "   file_ocn (coast)               = ",trim(file_ocn_coastal_mask)
+   write(6,F00) "   file_nn                        = ",trim(file_nn       )
+   write(6,F00) "   file_smooth                    = ",trim(file_smooth   )
+   write(6,F00) "   file_new                       = ",trim(file_new      )
+   write(6,F00) "   title                          = ",trim(title         )
+   write(6,F02) "   eFold distance                 = ",eFold
+   write(6,F02) "   rMax  distance                 = ",rMax
+   write(6,F03) "   restrict_smooth_src_to_nn_dest = ",restrict_smooth_src_to_nn_dest
+   write(6,F03) "   step1                          = ",step1
+   write(6,F03) "   step2                          = ",step2
+   write(6,F03) "   step3                          = ",step3
 !  if (rCode > 0) then
 !     write(6,F01) 'ERROR: reading input namelist, iostat=',rCode
 !     stop
@@ -147,7 +151,7 @@ PROGRAM main
    !----------------------------------------------------------------------------
    write(6,F10) "Step 1: read grid info & create nearest neighbor map"
    !----------------------------------------------------------------------------
-   call map_gridRead(map_orig , trim(file_roff), trim(file_ocn), gridtype)
+   call map_gridRead(map_orig , trim(file_roff), trim(file_ocn_coastal_mask), gridtype)
 
    call date_and_time(dstr,tstr)
    call map_print(map_orig)
@@ -176,7 +180,7 @@ PROGRAM main
    !----------------------------------------------------------------------------
    write(6,F02) "   eFold distance = ",eFold
    write(6,F02) "   rMax  distance = ",rMax
-   call smooth_init(map_orig,map_smooth)
+   call smooth_init(file_ocn, restrict_smooth_src_to_nn_dest, map_orig,map_smooth)
 !  call restrictSources(map_smooth,trim(file_sources)) ! restrict cells subject to smoothing
    call smooth(map_smooth,eFold,rMax)
    call mapsort_sort(map_smooth)
@@ -204,6 +208,7 @@ PROGRAM main
    !--- create new map datatype to hold result of matrix-matrix multiply ---
    call map_dup(map_orig,map_new)
    map_new%title  = trim(title)
+   map_new%domain_b = trim(map_smooth%domain_b)
    call map_matMatMult(map_orig,map_new,map_smooth) ! mult(A,B,S): B=S*A
    call mapsort_sort(map_new)
    call map_check(map_new)
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/map_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/map_mod.F90
index 38e637bbbef..7cf87a7d846 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/map_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/map_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: map_mod.F90 56089 2013-12-18 00:50:07Z mlevy@ucar.edu $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/map_mod.F90 $
-!===============================================================================
-
 MODULE map_mod
 
    use shr_sys_mod
@@ -385,7 +380,6 @@ SUBROUTINE map_gridRead(map, rfilename, ofilename, gridtype, lmake_rSCRIP)
    map%convention = 'NCAR-CCSM'
    map%history    = 'history'
    map%domain_a   = trim(rfilename)
-   map%domain_b   = trim(ofilename)
 
    if (trim(gridtype) == "rtm") then
       !-------------------------------------------------------------------------
@@ -746,88 +740,7 @@ SUBROUTINE map_gridRead(map, rfilename, ofilename, gridtype, lmake_rSCRIP)
    !----------------------------------------------------------------------------
    write(*,F00) "read destination domain info -- pop grid"
    !----------------------------------------------------------------------------
-   write(6,F00) 'ocn data file',' = ',trim(ofilename)
-   rcode = nf_open(ofilename,NF_NOWRITE,fid)
-
-   rcode = nf_inq_dimid (fid, 'grid_size' , did)
-   rcode = nf_inq_dimlen(fid, did   , map%n_b  )
-   rcode = nf_inq_dimid (fid, 'grid_corners' , did)
-   rcode = nf_inq_dimlen(fid, did   , map%nv_b  )
-   rcode = nf_inq_dimid (fid, 'grid_rank', did)
-   rcode = nf_inq_dimlen(fid, did   , grid_rank)
-   allocate(grid_dims(grid_rank))
-   rcode = nf_inq_varid  (fid, 'grid_dims', vid)
-   rcode = nf_get_var_int(fid, vid   , grid_dims)
-   if (grid_rank.eq.1) then
-     map%ni_b = grid_dims(1)
-     map%nj_b = 1
-   elseif (grid_rank.eq.2) then
-     map%ni_b = grid_dims(1)
-     map%nj_b = grid_dims(2)
-   else
-      deallocate(grid_dims)
-      write(6,*) 'ERROR: grid_rank is ',grid_rank,' in ',trim(ofilename)
-      call shr_sys_abort(subName//"ERROR: ofilename grid_rank")
-   endif
-   deallocate(grid_dims)
-   map%dims_b(1) = map%ni_b
-   map%dims_b(2) = map%nj_b
-
-   allocate(map%  xc_b(         map%n_b)) ! x-coordinates of center
-   allocate(map%  yc_b(         map%n_b)) ! y-coordinates of center
-   allocate(map%  xv_b(map%nv_b,map%n_b)) ! x-coordinates of verticies
-   allocate(map%  yv_b(map%nv_b,map%n_b)) ! y-coordinates of verticies
-   allocate(map%mask_b(         map%n_b)) ! domain mask
-   allocate(map%area_b(         map%n_b)) ! grid cell area
-   allocate(map%frac_b(         map%n_b)) ! grid cell area
-   allocate(map%sn1            (map%n_b))
-   allocate(map%sn2            (map%n_b))
-
-   rcode = nf_inq_varid     (fid,'grid_center_lon'  ,vid)
-   rcode = nf_get_var_double(fid,vid     ,map%xc_b )
-   units = "" ! units needs to be emptied before reading from netCDF file
-   rcode = nf_get_att_text(fid, vid, "units", units)
-   if (trim(units).eq."radians") then
-      map%xc_b = map%xc_b * RADtoDEG
-   end if
-
-   rcode = nf_inq_varid     (fid,'grid_center_lat'  ,vid)
-   rcode = nf_get_var_double(fid,vid     ,map%yc_b )
-   units = "" ! units needs to be emptied before reading from netCDF file
-   rcode = nf_get_att_text(fid, vid, "units", units)
-   if (trim(units).eq."radians") then
-      map%yc_b = map%yc_b * RADtoDEG
-   end if
-
-   rcode = nf_inq_varid     (fid,'grid_corner_lon'  ,vid)
-   rcode = nf_get_var_double(fid,vid     ,map%xv_b )
-   units = "" ! units needs to be emptied before reading from netCDF file
-   rcode = nf_get_att_text(fid, vid, "units", units)
-   if (trim(units).eq."radians") then
-      map%xv_b = map%xv_b * RADtoDEG
-   end if
-
-   rcode = nf_inq_varid     (fid,'grid_corner_lat'  ,vid)
-   rcode = nf_get_var_double(fid,vid     ,map%yv_b )
-   units = "" ! units needs to be emptied before reading from netCDF file
-   rcode = nf_get_att_text(fid, vid, "units", units)
-   if (trim(units).eq."radians") then
-      map%yv_b = map%yv_b * RADtoDEG
-   end if
-
-   rcode = nf_inq_varid     (fid,'grid_imask',vid )
-   rcode = nf_get_var_int   (fid,vid     ,map%mask_b)
-   rcode = nf_inq_varid     (fid,'grid_area',vid )
-   if (rcode.eq.0) then
-      rcode = nf_get_var_double(fid,vid     ,map%area_b)
-   else
-      write(6,*) "ERROR: could not find variable grid_area in destination grid input file!"
-      stop
-   end if
-
-   map%frac_b = map%mask_b * 1.0_r8
-
-   rcode = nf_close(fid)
+   call map_DestGridRead(map, ofilename)
 
    !----------------------------------------------------------------------------
    write(*,F00) "derive info required to compute NN map"
@@ -1474,10 +1387,6 @@ SUBROUTINE map_write(map, filename)
    rcode = nf_put_att_text(fid,NF_GLOBAL,'normalization',len_trim(str),str)
     str  = map%method
    rcode = nf_put_att_text(fid,NF_GLOBAL,'map_method' ,len_trim(str),str)
-    str  = "$SVN"
-   rcode = nf_put_att_text(fid,NF_GLOBAL,'SVN URL'     ,len_trim(str),str)
-    str  = "$Id"
-   rcode = nf_put_att_text(fid,NF_GLOBAL,'SVN Id'      ,len_trim(str),str)
     str  = map%history
    call date_and_time(cdate,ctime) ! f90 intrinsic
     str = 'File created: '//cdate(1:4)//'-'//cdate(5:6)//'-'//cdate(7:8) &
@@ -2593,6 +2502,119 @@ real(r8) FUNCTION map_distance(x0,y0,x1,y1)
 
 END FUNCTION map_distance
 
+!===============================================================================
+
+SUBROUTINE map_DestGridRead(map, filename)
+
+   !--- modules ---
+
+   implicit none
+
+   !--- includes ---
+
+   !--- arguments ---
+   type(sMatrix), intent(inout) :: map       ! sMatrix info to be read in
+   character(*) , intent(in)    :: filename  ! name of data file
+
+   !--- local ---
+   character(strLen)     :: units     ! netCDF attribute name string
+   integer               :: rcode   ! netCDF routine return code
+   integer               :: fid     ! netCDF file      ID
+   integer               :: vid     ! netCDF variable  ID
+   integer               :: did     ! netCDF dimension ID
+   integer               :: grid_rank
+   integer, allocatable, dimension(:) :: grid_dims
+
+   character(*),parameter :: subName = "(map_DestGridRead) "
+   !--- formats ---
+   character(len=*),parameter :: F00 = "('(map_DestGridRead) ',3a)"
+   character(len=*),parameter :: F02 = "('(map_DestGridRead) ',a11,a3,60(a1))"
+
+   write(6,F00) 'ocn data file',' = ',trim(filename)
+   rcode = nf_open(filename,NF_NOWRITE,fid)
+
+   rcode = nf_inq_dimid (fid, 'grid_size' , did)
+   rcode = nf_inq_dimlen(fid, did   , map%n_b  )
+   rcode = nf_inq_dimid (fid, 'grid_corners' , did)
+   rcode = nf_inq_dimlen(fid, did   , map%nv_b  )
+   rcode = nf_inq_dimid (fid, 'grid_rank', did)
+   rcode = nf_inq_dimlen(fid, did   , grid_rank)
+   allocate(grid_dims(grid_rank))
+   rcode = nf_inq_varid  (fid, 'grid_dims', vid)
+   rcode = nf_get_var_int(fid, vid   , grid_dims)
+   if (grid_rank.eq.1) then
+     map%ni_b = grid_dims(1)
+     map%nj_b = 1
+   elseif (grid_rank.eq.2) then
+     map%ni_b = grid_dims(1)
+     map%nj_b = grid_dims(2)
+   else
+      deallocate(grid_dims)
+      write(6,*) 'ERROR: grid_rank is ',grid_rank,' in ',trim(filename)
+      call shr_sys_abort(subName//"ERROR: filename grid_rank")
+   endif
+   deallocate(grid_dims)
+   map%dims_b(1) = map%ni_b
+   map%dims_b(2) = map%nj_b
+
+   allocate(map%  xc_b(         map%n_b)) ! x-coordinates of center
+   allocate(map%  yc_b(         map%n_b)) ! y-coordinates of center
+   allocate(map%  xv_b(map%nv_b,map%n_b)) ! x-coordinates of verticies
+   allocate(map%  yv_b(map%nv_b,map%n_b)) ! y-coordinates of verticies
+   allocate(map%mask_b(         map%n_b)) ! domain mask
+   allocate(map%area_b(         map%n_b)) ! grid cell area
+   allocate(map%frac_b(         map%n_b)) ! grid cell area
+   allocate(map%sn1            (map%n_b))
+   allocate(map%sn2            (map%n_b))
+
+   rcode = nf_inq_varid     (fid,'grid_center_lon'  ,vid)
+   rcode = nf_get_var_double(fid,vid     ,map%xc_b )
+   units = "" ! units needs to be emptied before reading from netCDF file
+   rcode = nf_get_att_text(fid, vid, "units", units)
+   if (trim(units).eq."radians") then
+      map%xc_b = map%xc_b * RADtoDEG
+   end if
+
+   rcode = nf_inq_varid     (fid,'grid_center_lat'  ,vid)
+   rcode = nf_get_var_double(fid,vid     ,map%yc_b )
+   units = "" ! units needs to be emptied before reading from netCDF file
+   rcode = nf_get_att_text(fid, vid, "units", units)
+   if (trim(units).eq."radians") then
+      map%yc_b = map%yc_b * RADtoDEG
+   end if
+
+   rcode = nf_inq_varid     (fid,'grid_corner_lon'  ,vid)
+   rcode = nf_get_var_double(fid,vid     ,map%xv_b )
+   units = "" ! units needs to be emptied before reading from netCDF file
+   rcode = nf_get_att_text(fid, vid, "units", units)
+   if (trim(units).eq."radians") then
+      map%xv_b = map%xv_b * RADtoDEG
+   end if
+
+   rcode = nf_inq_varid     (fid,'grid_corner_lat'  ,vid)
+   rcode = nf_get_var_double(fid,vid     ,map%yv_b )
+   units = "" ! units needs to be emptied before reading from netCDF file
+   rcode = nf_get_att_text(fid, vid, "units", units)
+   if (trim(units).eq."radians") then
+      map%yv_b = map%yv_b * RADtoDEG
+   end if
+
+   rcode = nf_inq_varid     (fid,'grid_imask',vid )
+   rcode = nf_get_var_int   (fid,vid     ,map%mask_b)
+   rcode = nf_inq_varid     (fid,'grid_area',vid )
+   if (rcode.eq.0) then
+      rcode = nf_get_var_double(fid,vid     ,map%area_b)
+   else
+      write(6,*) "ERROR: could not find variable grid_area in destination grid input file!"
+      stop
+   end if
+
+   map%frac_b = map%mask_b * 1.0_r8
+   map%domain_b = trim(filename)
+   rcode = nf_close(fid)
+
+END SUBROUTINE map_DestGridRead
+
 !===============================================================================
 !===============================================================================
 END MODULE map_mod
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90
index 250c8865a5a..df882af8902 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: mapsort_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/mapsort_mod.F90 $
-!===============================================================================
-
 MODULE mapsort_mod
 
    use shr_sys_mod
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90
index 2a2dd64512a..de361d8d747 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: shr_kind_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/shr_kind_mod.F90 $
-!===============================================================================
-
 MODULE shr_kind_mod
 
    !----------------------------------------------------------------------------
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90
index b7aba1066e2..0cd2f896937 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: shr_sys_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/shr_sys_mod.F90 $
-!===============================================================================
-
 MODULE shr_sys_mod
 
    use shr_kind_mod  ! defines real & integer kinds
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90
index ffa9deebaf1..3acf86b8976 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: shr_timer_mod.F90 35698 2012-03-22 23:59:57Z kauff $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/shr_timer_mod.F90 $
-!===============================================================================
-
 module shr_timer_mod
 
    !----------------------------------------------------------------------------
diff --git a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90 b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90
index 03f889db6d8..b7cc1966b61 100644
--- a/tools/mapping/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90
+++ b/tools/mapping/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90
@@ -1,8 +1,3 @@
-!===============================================================================
-! SVN $Id: smooth_mod.F90 56089 2013-12-18 00:50:07Z mlevy@ucar.edu $
-! SVN $URL: https://svn-ccsm-models.cgd.ucar.edu/tools/mapping/trunk_tags/mapping_141106/gen_mapping_files/runoff_to_ocn/src/smooth_mod.F90 $
-!===============================================================================
-
 MODULE smooth_mod
 
 #define _NEW 1
@@ -26,13 +21,26 @@ MODULE smooth_mod
 CONTAINS
 !===============================================================================
 
-SUBROUTINE smooth_init(map_in, map_out)
+SUBROUTINE smooth_init(ofilename, restrict_smooth_src_to_nn_dest,  map_in, map_out)
 
    implicit none
 
    !--- arguments ---
-   type(sMatrix),intent( in)   :: map_in   ! original unsmoothed, matrix
-   type(sMatrix),intent(inout) :: map_out  ! smoothing matrix
+   ! name of ocn scrip grid file
+   character(*), intent(in)    :: ofilename
+
+   ! original unsmoothed, matrix
+   type(sMatrix),intent(in)    :: map_in
+
+   ! map_out%mask_a = pts mapped to map_in?
+   ! This should be set to .true. if you are making a single map, but it may be
+   ! useful to set it to .false. if you plan on mapping several runoff grids to
+   ! the same ocean grid. The nearest-neighbor / smooth map generated in step 3
+   ! will be the same regardless of the value of this variable.
+   logical,      intent(in)    :: restrict_smooth_src_to_nn_dest
+
+   ! smoothing matrix
+   type(sMatrix),intent(inout) :: map_out
 
    !--- local ---
    integer :: i,j,n ! indicies: row, col, sparse matrix
@@ -123,26 +131,19 @@ SUBROUTINE smooth_init(map_in, map_out)
    allocate(map_out%  yv_a(map_in%nv_b,map_in%n_b) )
    allocate(map_out%mask_a(            map_in%n_b) )
    allocate(map_out%area_a(            map_in%n_b) )
-
-   allocate(map_out%  xc_b(            map_in%n_b) )
-   allocate(map_out%  yc_b(            map_in%n_b) )
-   allocate(map_out%  xv_b(map_in%nv_b,map_in%n_b) )
-   allocate(map_out%  yv_b(map_in%nv_b,map_in%n_b) )
-   allocate(map_out%mask_b(            map_in%n_b) )
-   allocate(map_out%area_b(            map_in%n_b) )
-
    allocate(map_out%frac_a(map_in%n_b) )
-   allocate(map_out%frac_b(map_in%n_b) )
 
    allocate(map_out%s  (ns))
    allocate(map_out%row(ns))
    allocate(map_out%col(ns))
-   allocate(map_out%sn1(map_in%n_b) )
-   allocate(map_out%sn2(map_in%n_b) )
 
    !------------------------------------------------
    ! set values
    !------------------------------------------------
+
+   ! map_in maps from runoff -> coastal ocean
+   ! map_out maps from coastal ocean -> global ocean
+   ! So source of map_out = dest of map_in
    map_out%   n_a = map_in%   n_b
    map_out%dims_a = map_in%dims_b
    map_out%  ni_a = map_in%  ni_b
@@ -152,36 +153,30 @@ SUBROUTINE smooth_init(map_in, map_out)
    map_out%  yc_a = map_in%  yc_b
    map_out%  xv_a = map_in%  xv_b
    map_out%  yv_a = map_in%  yv_b
-!  map_out%mask_a = map_in%mask_b ! all active ocn cells
    map_out%area_a = map_in%area_b
+   map_out%domain_a   = map_in%domain_b
 
-   !--- compute minimal src domain mask for smoothing matrix ---
-   jmd_count = 0
-   map_out%mask_a = 0
-   do n=1,map_in%n_s
-      i = map_in%row(n)     ! this ocn cell could get runoff
-      map_out%mask_a(i) = 1 ! this ocn cell's runoff get's smoothed
-      if(map_in%s(n) > 0.0) then
-        jmd_count = jmd_count+1
-      endif
-   end do
-   write(*,*) subName,'number of source points is =  ',jmd_count
-   write(*,*) subName,'map_in%ns                  =  ',map_in%n_s
-
-   map_out%   n_b = map_in%   n_b
-   map_out%dims_b = map_in%dims_b
-   map_out%  ni_b = map_in%  ni_b
-   map_out%  nj_b = map_in%  nj_b
-   map_out%  nv_b = map_in%  nv_b
-   map_out%  xc_b = map_in%  xc_b
-   map_out%  yc_b = map_in%  yc_b
-   map_out%  xv_b = map_in%  xv_b
-   map_out%  yv_b = map_in%  yv_b
-   map_out%mask_b = map_in%mask_b
-   map_out%area_b = map_in%area_b
-
-!  map_out%frac_a = map_in%frac_b
-!  map_out%frac_b = map_in%frac_b
+   if (restrict_smooth_src_to_nn_dest) then
+      !--- compute minimal src domain mask for smoothing matrix ---
+      jmd_count = 0
+      map_out%mask_a = 0
+      do n=1,map_in%n_s
+         i = map_in%row(n)     ! this ocn cell could get runoff
+         map_out%mask_a(i) = 1 ! this ocn cell's runoff get's smoothed
+         if(map_in%s(n) > 0.0) then
+            jmd_count = jmd_count+1
+         endif
+      end do
+      write(*,*) subName,'number of source points is =  ',jmd_count
+      write(*,*) subName,'map_in%ns                  =  ',map_in%n_s
+   else
+      map_out%mask_a = map_in%mask_b ! all active ocn cells
+   end if
+
+   ! destination of map_out must be read in from file
+   call map_DestGridRead(map_out, ofilename)
+
+   ! Overwrite frac_a and frac_b to be 1 globally
    map_out%frac_a = 1.0
    map_out%frac_b = 1.0
 
@@ -189,16 +184,11 @@ SUBROUTINE smooth_init(map_in, map_out)
    map_out%s      = 1.0
    map_out%row    = 1
    map_out%col    = 1
-   map_out%sn1    = map_in%sn1
-   map_out%sn2    = map_in%sn2
 
    map_out%title      = "CCSM conservative smoothing map"
    map_out%normal     = map_in%normal
-   map_out%method     = "created using SVN $Id: smooth_mod.F90 56089 2013-12-18 00:50:07Z mlevy@ucar.edu $"
    map_out%history    = map_in%history
    map_out%convention = map_in%convention
-   map_out%domain_a   = map_in%domain_b
-   map_out%domain_b   = map_in%domain_b
 
 END SUBROUTINE smooth_init
 
@@ -394,9 +384,9 @@ SUBROUTINE smooth(map,efold,rmax)
         wgtsum = 0.0
         nbox = 0
         gdcnt = 0
+        kStart = 1
 
 #ifdef _BREADTH
-        kStart = 2
         i2ind(1) = ic
         j2ind(1) = jc
         imask_JMD = -1000
@@ -408,7 +398,6 @@ SUBROUTINE smooth(map,efold,rmax)
                 rdist,rmax,i2ind,j2ind, strPtr, length)
 
 #else
-        kStart = 1
         !--- recursive function to find dest cells ---
 
         call depth_setDist(ic,jc,ni,nj,map%xc_a,map%yc_a,0.0,imask,rdist,rmax,i2ind,j2ind,length)
diff --git a/tools/mapping/map_field/INSTALL b/tools/mapping/map_field/INSTALL
index 766c76e3fec..a244892d296 100644
--- a/tools/mapping/map_field/INSTALL
+++ b/tools/mapping/map_field/INSTALL
@@ -2,17 +2,16 @@
 HOW TO BUILD
 ============
 
-Prior to building, you must make sure $CIMEROOT is set.
-
 (1) $ cd src
-(2) $ $CIMEROOT/tools/configure macros-format Makefile
-(3) Bash users:
-    $ . .env_mach_specific.sh
-    csh users:
-    $ source .env_mach_specific.csh
-(4) $ gmake
+(2) $ ../../../configure --macros-format Makefile --mpilib mpi-serial
+Bash users:
+(3) $ (. ./.env_mach_specific.sh ; gmake)
+csh users:
+(3) $ (source ./.env_mach_specific.csh ; gmake)
 
-Note: in the second step, replace [machine name] with the machine you are
-building on. Also, some machines have dedicated build nodes, so you might need
-to SSH to another node before the 'gmake' step.
+Note: in the second step, you may need to include "--machine [machine name]",
+where [machine name] is the name of the machine you are building on. In most
+cases configure can figure that out on its own, but if you get an error that is
+the first fix to try. Also, some machines have dedicated build nodes, so you
+might need to SSH to another node before the 'gmake' step.
 
diff --git a/tools/mapping/map_field/src/Makefile b/tools/mapping/map_field/src/Makefile
index bc2dbd56e8e..27b53b6128f 100644
--- a/tools/mapping/map_field/src/Makefile
+++ b/tools/mapping/map_field/src/Makefile
@@ -29,14 +29,26 @@ null :=
 CURDIR = .
 EXENAME = ../map_field
 RM = rm
-MACFILE = Macros.make
+MACROS = Macros.make
 
 default: $(EXENAME)
 
-$(MACFILE):
-	@echo "use the configure script located in the Machines directory to create the Makefile $(MACFILE) file"
+$(MACROS):
+	@echo "use the configure script located in the Machines directory to create the Makefile $(MACROS) file"
+
+# Do not include Macros for clean or distclean targets
+USE_MACROS=TRUE
+ifeq ($(MAKECMDGOALS),clean)
+  USE_MACROS=FALSE
+endif
+ifeq ($(MAKECMDGOALS),distcleanclean)
+  USE_MACROS=FALSE
+endif
+
+ifeq ($(USE_MACROS),TRUE)
+  -include $(MACROS)
+endif
 
--include $(MACFILE)
 # Check for the netcdf library and include directories
 ifdef NETCDF_PATH
   LIB_NETCDF:=$(NETCDF_PATH)/lib
@@ -97,7 +109,7 @@ else
   FPPDEFS := $(CPPDEFS)
 endif
 #Primary Target: build the tool
-all: $(EXENAME) $(MACFILE)
+all: $(EXENAME) $(MACROS)
 
 OBJS := map_field.o
 
@@ -133,7 +145,7 @@ $(EXENAME): $(OBJS)
 clean:
 	$(RM) -f $(OBJS) $(EXENAME) *.mod
 
-veryclean:
-	$(RM) -f $(OBJS) $(EXENAME) *.mod $(MACFILE) .env_mach* env_mach*
+distclean: clean
+	../../../configure --clean
 
 map_field.o : map_field.F90
diff --git a/tools/mapping/map_field/src/map_field.F90 b/tools/mapping/map_field/src/map_field.F90
index a612772d359..3f0a9c132bc 100644
--- a/tools/mapping/map_field/src/map_field.F90
+++ b/tools/mapping/map_field/src/map_field.F90
@@ -469,7 +469,8 @@ subroutine write_file(fid, fout, units, n, ni, nj, &
        if (rcode == 0) then
           call check_ret(nf_put_att_text(fid,NF_GLOBAL,'hostname' ,len_trim(host),host))
        else
-          write(6,*) 'WARNING: could not determine hostname, so that information will not be stored in netCDF attribute. To avoid this warning in the future, set environment variable HOST or HOSTNAME.'
+          write(6,*) 'WARNING: could not determine hostname, so that information will not be &
+               &stored in netCDF attribute. To avoid this warning in the future, set environment variable HOST or HOSTNAME.'
        end if
     end if
 
diff --git a/tools/unit_testing/python/comparable.py b/tools/unit_testing/python/comparable.py
deleted file mode 100644
index 20e52697dba..00000000000
--- a/tools/unit_testing/python/comparable.py
+++ /dev/null
@@ -1,29 +0,0 @@
-"""Portable implementation of comparisons for total ordering.
-
-THIS MODULE IS OBSOLETE AS OF PYTHON 2.7.
-
-This module provides a "Comparable" class for subclassing; it provides the
-various "rich comparison" functions (e.g. __le__) based on the subclass's
-__eq__ and __lt__ methods (so this is a Template Method pattern).
-
-Python 2.7 and Python 3 implement a class decorator for this in the
-standard libraries (functools.total_ordering), and that decorator is much
-more flexible (besides being standard), so this should only be used for
-compatibility with older implementations.
-"""
-
-class Comparable(object):
-
-    """Provides rich comparisons for subclasses that have __eq__ and __lt__."""
-
-    def __le__(self, other):
-        return (self == other) and (self < other)
-
-    def __gt__(self, other):
-        return not (self <= other)
-
-    def __ge__(self, other):
-        return not (self < other)
-
-    def __ne__(self, other):
-        return not (self == other)
diff --git a/tools/unit_testing/python/environment.py b/tools/unit_testing/python/environment.py
deleted file mode 100644
index 62972c6ad57..00000000000
--- a/tools/unit_testing/python/environment.py
+++ /dev/null
@@ -1,263 +0,0 @@
-"""Produce commands for environment module systems.
-
-Public classes:
-EnvSystemInterface - Abstract base class for interfaces.
-NoModuleInterface - Can be constructed, but raises exception if used.
-ModuleInterface - For standard module systems (including Lmod).
-SoftEnvInterface - For SoftEnv, the ANL MCS environment system.
-
-Public routines:
-expand_env - Expand shell variables (like "${FOO}").
-"""
-
-import os
-import re
-import subprocess
-
-__all__ = ("EnvSystemInterface", "NoModuleInterface", "ModuleInterface",
-           "SoftEnvInterface", "expand_env")
-
-class EnvSystemInterface(object):
-
-    """Abstract base class for environment system interfaces.
-
-    This class exists only to document the interface of subclasses, and to
-    provide error messages if an unimplemented function is called by
-    accident.
-
-    Normally, none of these methods should actually be called. All of them
-    raise NotImplementedError.
-
-    Public methods:
-    is_loaded
-    purge
-    load
-    unload
-    purge_str
-    load_str
-    unload_str
-    list
-    """
-
-    @classmethod
-    def _raise_not_implemented(cls, method_name):
-        raise NotImplementedError(
-            cls.__name__ +" does not implement method "+method_name+".")
-
-    def is_loaded(self, modname):
-        """Method not implemented."""
-        self._raise_not_implemented("is_loaded")
-
-    def purge(self):
-        """Method not implemented."""
-        self._raise_not_implemented("purge")
-
-    def list(self):
-        """Method not implemented."""
-        self._raise_not_implemented("list")
-
-    def load(self, modname):
-        """Method not implemented."""
-        self._raise_not_implemented("load")
-
-    def unload(self, modname):
-        """Method not implemented."""
-        self._raise_not_implemented("unload")
-
-    def purge_str(self):
-        """Method not implemented."""
-        self._raise_not_implemented("purge_str")
-
-    def load_str(self, modname):
-        """Method not implemented."""
-        self._raise_not_implemented("load_str")
-
-    def unload_str(self, modname):
-        """Method not implemented."""
-        self._raise_not_implemented("unload_str")
-
-
-class NoModuleInterface(EnvSystemInterface):
-
-    """Module interface for systems with no module system.
-
-    The purpose of this class is to allow code to construct a module
-    interface regardless of system, while still raising an error if the
-    user attempts to actually interact with the non-existent interface.
-
-    As a result, only purge_str is implemented.
-
-    Public methods:
-    purge_str - Returns null command.
-    """
-
-    def purge_str(self):
-        """Returns ":", the null shell command."""
-        return ":"
-
-
-class ModuleInterface(EnvSystemInterface):
-
-    """Module interface for systems with a typical module system.
-
-    Class methods:
-    python_init - Initialize python interface.
-
-    Public methods:
-    is_loaded - Tests if a module is currently loaded.
-    purge - Purge all modules from environment.
-    load - Load a module.
-    unload - Unload a module.
-    list - list currently loaded modules.
-    purge_str - Return command to purge modules.
-    load_str - Return command to load the given module.
-    unload_str  - Return command to unload the given module.
-    """
-
-    # Singleton declaring whether or not we have the module interface
-    # started up yet.
-    _python_initialized = False
-
-    # Magic string to grep for in "module list" output to determine whether
-    # or not a module is present.
-    _not_loaded_string = "None found"
-
-    @classmethod
-    def python_init(cls, filename):
-        """Initialize python to module system interface.
-
-        This must be called before any commands that are intended to
-        modify python's environment. The *_str methods can still be called
-        without this. Only the first call has any effect.
-
-        Arguments:
-        filename - Python file to execute to load the interface.
-        """
-        if not cls._python_initialized:
-            execfile(filename, globals())
-            cls._python_initialized = True
-
-    def is_loaded(self, modname):
-        """"Whether the module is loaded in the current environment."""
-        # This assertion message isn't quite true, but this behavior is
-        # convenient for testing purposes, which is why is_loaded exists.
-        assert self._python_initialized, \
-            "Can't test modules without initializing the python interface!"
-        process = subprocess.Popen("module list "+modname,
-                                   shell=True,
-                                   env=os.environ,
-                                   stderr=subprocess.PIPE)
-        stderr_output = process.communicate()[1]
-        # Handle unexpected error in subprocess.
-        if process.returncode not in (0,1):
-            raise Exception("module list command failed with code: "+
-                            str(process.returncode))
-        # Hack! Assume that process.returncode == 1 means that no modules
-        # are loaded, rather than some other possible error.
-        return process.returncode != 1 and \
-            re.search(self._not_loaded_string, stderr_output) is None
-
-    def purge(self):
-        """"Purge all modules from the current environment."""
-        assert self._python_initialized, \
-            "Can't purge modules without initializing the python interface!"
-        module("purge")
-
-    def list(self):
-        """"List all modules from the current environment."""
-        assert self._python_initialized, \
-            "Can't list modules without initializing the python interface!"
-        module("list")
-
-    def load(self, modname):
-        """"Load a module in the current environment."""
-        assert self._python_initialized, \
-            "Can't load modules without initializing the python interface!"
-        module("load", modname)
-
-    def unload(self, modname):
-        """"Unload a module from the current environment."""
-        assert self._python_initialized, \
-            "Can't unload modules without initializing the python interface!"
-        module("unload", modname)
-
-    def purge_str(self):
-        """Returns the shell command to purge modules as a string."""
-        return "module purge"
-
-    def load_str(self, modname):
-        """Returns the shell command to load the module as a string."""
-        return "module load "+modname
-
-    def unload_str(self, modname):
-        """Returns the shell command to unload the module as a string."""
-        return "module unload "+modname
-
-
-class SoftEnvInterface(EnvSystemInterface):
-
-    """Module interface for systems with SoftEnv.
-
-    Public methods:
-    purge_str - Return command to reset SoftEnv.
-    load_str - Return command to add the given keyword/macro.
-    unload_str  - Return command to delete the given keyword/macro.
-    """
-
-    def purge_str(self):
-        """Returns the shell command to reset SoftEnv as a string.
-
-        Unfortunately, there's not a straightforward way to clear
-        everything, so right now we just run "resoft". If it becomes
-        necessary in the future, this may change to do something else (e.g.
-        resoft using a custom file with only default settings).
-        """
-        return "resoft"
-
-    def load_str(self, modname):
-        """Returns the shell command to add the keyword as a string."""
-        return "soft add "+modname
-
-    def unload_str(self, modname):
-        """Returns the shell command to delete the keyword as a string."""
-        return "soft delete "+modname
-
-
-# Regex that matches an environment variable reference, putting the name in
-# the "name" group of the match.
-_env_re = re.compile(
-    """\$                      # Initial "$"
-       (?P\{)?          # Open brace if present
-       (?P[A-Za-z0-9_]+) # Variable name
-       (?(brace)\})            # Close brace if necessary""", re.X)
-
-
-def expand_env(string, env):
-    """Expand a shell string with given environment.
-
-    Arguments:
-    string - String to expand as if in a shell.
-    env - dict specifying environment variables.
-
-    This is very limited; right now it only handles variable substitution,
-    and only with "${FOO}" or "$FOO" style syntax. The variable name must
-    contain only alphanumeric characters and "_". There is no way to escape
-    a "$".
-
-    Expansion is recursive; the output of this function is its fixed point.
-    """
-    def expand_func(match):
-        """Given an environment variable match, return replacement text."""
-        var_name = match.group("name")
-        if var_name in env:
-            return env[var_name]
-        else:
-            # If there's no match in the environment, return original text.
-            return match.group(0)
-    old_string = ""
-    new_string = string
-    while new_string != old_string:
-        old_string = new_string
-        new_string = _env_re.sub(expand_func, new_string)
-
-    return new_string
diff --git a/tools/unit_testing/python/machine_setup.py b/tools/unit_testing/python/machine_setup.py
deleted file mode 100644
index 1059beca258..00000000000
--- a/tools/unit_testing/python/machine_setup.py
+++ /dev/null
@@ -1,296 +0,0 @@
-"""Functions to set up machine-specific settings.
-
-Currently, this module is not under unit test, because it interacts too
-directly with the OS of the machine it is on. Any changes to address this
-would be welcome (this would likely require the use of mock modules,
-and/or a specific set of machines to test on).
-
-Public classes:
-MachineCompilerSettings - Set up machine/compiler specific environment.
-"""
-
-# Python 3 compatible printing in Python 2.
-from __future__ import print_function
-
-from itertools import chain
-from os import environ
-import os.path
-import platform
-import re
-import subprocess
-from xml.etree.ElementTree import ElementTree
-
-import environment as ENV
-from printer import ScriptPrinter
-from xml_utils import best_match, all_matches
-__all__ = ("MachineCompilerSettings")
-
-
-def get_machine_name():
-    """Returns a canonical version of the machine name."""
-    name = platform.node()
-    # Strip everything after the first ".", and whitespace.
-    name = name.split(".")[0].strip()
-    if re.match("^yslogin[0-9]+", name):
-        name = "yellowstone"
-    elif re.match("^miralac[0-9]+", name):
-        name = "mira"
-    elif re.match("^cetuslac[0-9]+", name):
-        name = "mira"
-    elif re.match("^caldera.*", name) or re.match("^pronghorn.*", name):
-        # Use yellowstone settings for caldera/pronghorn, since the mahcines
-        # files aren't set up explicitly for those machines, and they have the
-        # same configuration as yellowstone.
-        name = "yellowstone"
-    return name
-
-def load_machine_env(compiler):
-    """Add machine environment variables not in config_compilers.xml
-
-    Besides simply setting variables, this may also load some modules.
-    """
-
-    mach = get_machine_name()
-
-    if mach == "yellowstone":
-        mod = ENV.ModuleInterface()
-        mod.python_init("/glade/apps/opt/lmod/lmod/init/env_modules_python.py")
-        mod.purge()
-        mod.load("ncarenv/1.0")
-        mod.load("ncarbinlibs/1.0")
-        if compiler == "intel":
-            mod.load("intel/15.0.1")
-            mod.load("mkl/11.1.2")
-        elif compiler == "pgi":
-            mod.load("pgi/13.9")
-        mod.load("ncarcompilers/1.0")
-        mod.load("cmake/2.8.10.2")
-        mod.load("netcdf/4.3.0")
-
-
-class MachineCompilerSettings(object):
-
-    """Encapsulate machine settings and set environment from them.
-
-    Public methods:
-    __init__ - Discover information about local machine.
-    compiler_xml_to_env - Set a specific variable using config_compilers.
-    set_compiler_env - Set up machine-specific environment.
-    write_cmake_macros - Create CMake "Macros" file.
-    """
-
-    def __init__(self, compiler, compiler_xml_path,
-                 machine=None,
-                 use_env_compiler=False,
-                 mpilib=None, use_openmp=False):
-        """Discover information about the machine and compiler.
-
-        Arguments:
-        compiler - String naming the compiler vendor.
-        compiler_xml_path - Path to config_compilers.xml file.
-        machine - String naming the machine (guessed if not provided)
-        use_env_compiler - Force use of environment variable FC instead of
-                           using the compiler in config_compilers.xml.
-        mpilib - String naming the mpi library if any.
-        use_openmp - Boolean option to use OpenMP compiler flags.
-                     Currently only works for CESM/CESM_DEBUG builds.
-        """
-        if machine is None:
-            machine = get_machine_name()
-        self.machine_dict = {
-            "COMPILER": compiler,
-            "MACH": machine,
-            "OS": platform.system(),
-            "compile_threaded": "true" if use_openmp else "false",
-            }
-        if mpilib:
-            self.machine_dict["MPILIB"] = mpilib
-        else:
-            self.machine_dict["MPILIB"] = 'mpi-serial'
-
-        self.compiler_xml_tree = ElementTree()
-        self.compiler_xml_tree.parse(compiler_xml_path)
-        self.use_env_compiler = use_env_compiler
-        self.mpilib = mpilib
-
-    def compiler_xml_to_env(self, xml_path, var_name):
-        """Look up a config_compilers entry and set a variable from it.
-
-        Arguments:
-        xml_path - Path within the xml file (e.g. "compiler/SFC").
-        var_name - Name of environment variable (e.g. "FC").
-        """
-        match = best_match(self.compiler_xml_tree, xml_path,
-                           self.machine_dict)
-        assert match is not None, "Could not determine "+var_name+ \
-            " from compiler/machines xml file."
-        environ[var_name] = match.text.strip()
-
-    def add_path(self, name, macros_printer):
-        match = best_match(self.compiler_xml_tree, "compiler/"+name+"_PATH",
-                           self.machine_dict)
-        if match is not None:
-            macros_printer.print_header(name + " location.")
-            libpath = match.text
-            _make_env_re = re.compile(
-                """\$\(                    # Initial "$" and brace
-                (?P[A-Za-z0-9_]+) # Variable name
-                \)                      # Close brace""", re.X)
-            libpath = _make_env_re.sub( "$ENV{\g}",libpath)
-            print("libpath = "+libpath)
-            macros_printer.print(
-                "list(APPEND CMAKE_PREFIX_PATH "+libpath+")"
-                )
-
-    def set_compiler_env(self):
-        """Set up the environment for this machine and compiler."""
-        if not self.use_env_compiler:
-            if (self.mpilib == "mpi-serial" or self.mpilib is None):
-                self.compiler_xml_to_env("compiler/SFC", "FC")
-                self.compiler_xml_to_env("compiler/SCC", "CC")
-            else:
-                self.compiler_xml_to_env("compiler/MPIFC", "FC")
-                self.compiler_xml_to_env("compiler/MPICC", "CC")
-        load_machine_env(self.machine_dict["COMPILER"])
-
-    def write_cmake_macros(self, macros_file, model="CESM"):
-        """Write CMake macros file using config_compilers.xml
-
-        Arguments:
-        macros_file - File object to write to.
-        """
-
-        # Print header to file.
-        macros_printer = ScriptPrinter(macros_file)
-        header_lines = [model+
-            " build flags for:",
-            "  Compiler = "+self.machine_dict["COMPILER"],
-            "  Machine = "+self.machine_dict["MACH"],
-            "  OS = "+self.machine_dict["OS"],
-            " MPILIB = "+self.machine_dict["MPILIB"],
-            ]
-
-        for line in header_lines:
-            macros_printer.comment(line)
-
-        macros_printer.print("include(Compilers)")
-        macros_printer.print(
-            "set(CMAKE_Fortran_FLAGS_"+model+" \"\" CACHE STRING \"Flags used by the Fortran compiler during builds.\" FORCE)")
-        macros_printer.print(
-            "set(CMAKE_Fortran_FLAGS_"+model+"_DEBUG \"\" CACHE STRING \"Flags used by the Fortran compiler during DEBUG builds.\" FORCE)")
-
-
-        macros_printer.print(
-            "set(CMAKE_C_FLAGS_"+model+" \"\" CACHE STRING \"Flags used by the C compiler during builds.\" FORCE)")
-        macros_printer.print(
-            "set(CMAKE_C_FLAGS_"+model+"_DEBUG \"\" CACHE STRING \"Flags used by the C compiler during DEBUG builds.\" FORCE)")
-
-        macros_printer.print(
-            "mark_as_advanced(CMAKE_Fortran_FLAGS_"+model+" CMAKE_Fortran_FLAGS_"
-            +model+"_DEBUG)")
-        macros_printer.print(
-            "mark_as_advanced(CMAKE_C_FLAGS_"+model+" CMAKE_C_FLAGS_"+model+"_DEBUG)")
-        macros_printer.print(
-            "set(all_build_types \"None Debug Release RelWithDebInfo MinSizeRel "+model+
-            " "+model+"_DEBUG\")")
-        macros_printer.print(
-            "set(CMAKE_BUILD_TYPE \"${CMAKE_BUILD_TYPE}\" CACHE STRING \"Choose the type of build, options are: ${all_build_types}.\" FORCE)")
-
-
-
-
-        # pFUnit location if it exists.
-        self.add_path("PFUNIT",macros_printer)
-        # NETCDF location if it exists.
-        self.add_path("NETCDF",macros_printer)
-        # PNETCDF location if it exists.
-        self.add_path("PNETCDF", macros_printer)
-        # HDF5 location if it exists.
-        self.add_path("HDF5", macros_printer)
-        # MPI location if it exists.
-        self.add_path("MPI", macros_printer)
-        # PETSc location if it exists.
-        self.add_path("PETSC", macros_printer)
-       # TRILINOS location if it exists.
-        self.add_path("TRILINOS", macros_printer)
-        # ALBANY location if it exists.
-        self.add_path("ALBANY", macros_printer)
-
-
-        # Normal and debug dictionaries for looking things up in
-        # config_compilers.
-        normal_dict = self.machine_dict.copy()
-        normal_dict["DEBUG"] = "FALSE"
-
-        debug_dict = self.machine_dict.copy()
-        debug_dict["DEBUG"] = "TRUE"
-
-        def add_formatted_flags(flags_name, format):
-            """Print CMake flags using macros_printer.
-
-            Arguments:
-            flags_name - Name to search for in config_compilers.
-            format - Function that takes a build type and flag match, and
-            returns the string to print out.
-            """
-
-            paths = ["compiler/"+flags_name, "compiler/ADD_"+flags_name]
-
-            # This creates an iterable over elements in config_compilers
-            # that match in non-debug mode.
-            normal_matches = chain.from_iterable(
-                all_matches(self.compiler_xml_tree, path, normal_dict)
-                for path in paths
-                )
-            for match in normal_matches:
-                macros_printer.print(format(model, match.text))
-
-            # Now the same for debug mode.
-            debug_matches = chain.from_iterable(
-                all_matches(self.compiler_xml_tree, path, debug_dict)
-                for path in paths
-                )
-            for match in debug_matches:
-                macros_printer.print(format(model+"_DEBUG", match.text))
-
-        # Below, we just use a bunch of lambda functions to describe how
-        # the build type and a matching element (e.g. an FFLAGS entry) are
-        # turned into a CMake function call.
-
-        macros_printer.print_header("CPP definitions.")
-        add_formatted_flags(
-            "CPPDEFS",
-            lambda b, m: "add_config_definitions("+b+" "+m+")"
-            )
-        def format_contiguous(build_type, match):
-            comma = "," if self.machine_dict["COMPILER"] != "ibm" else "\\\,"
-            contig_def = "contiguous"+comma if match == "TRUE" else ""
-            return "add_config_definitions("+build_type+\
-                " -DUSE_CONTIGUOUS="+contig_def+")"
-        add_formatted_flags(
-            "HAS_F2008_CONTIGUOUS",
-            format_contiguous
-            )
-
-        macros_printer.print_header("Fortran flags.")
-        add_formatted_flags(
-            "FFLAGS",
-            lambda b, m: "add_flags(CMAKE_Fortran_FLAGS_"+b+" "+m+")"
-            )
-
-        macros_printer.print_header("C flags.")
-        add_formatted_flags(
-            "CFLAGS",
-            lambda b, m: "add_flags(CMAKE_C_FLAGS_"+b+" "+m+")"
-            )
-
-        macros_printer.print_header("Linker flags.")
-        add_formatted_flags(
-            "LDFLAGS",
-            lambda b, m: "add_flags(CMAKE_EXE_LINKER_FLAGS_"+b+" "+m+")"
-            )
-        macros_printer.print_header("External library flags.")
-        add_formatted_flags(
-            "SLIBS",
-            lambda b, m: "add_flags(CMAKE_EXE_LINKER_FLAGS_"+b+" "+m+")"
-            )
diff --git a/tools/unit_testing/python/test_environment.py b/tools/unit_testing/python/test_environment.py
deleted file mode 100755
index a7cbaa73102..00000000000
--- a/tools/unit_testing/python/test_environment.py
+++ /dev/null
@@ -1,237 +0,0 @@
-#!/usr/bin/env python
-"""Unit tests for the environment module.
-
-Public classes:
-TestEnvSys - Test the EnvSystemInterface abstract base class.
-TestNoMod - NoModuleInterface tests.
-TestMod - ModuleInterface tests.
-TestSoft - SoftEnvInterface tests.
-TestExpandEnv - expand_env tests.
-
-To test the actual module system, you must specify these environment
-variables when running the test:
-MODULE_SYSTEM - Type of module system available locally.
-TEST_MODULE - Name of a module which is *not* loaded, to use for testing
-              purposes.
-MODULE_FILE - File that must be executed to load the interface to python
-              (if any).
-"""
-
-from os import environ
-import unittest
-
-from environment import *
-
-__all__ = ("TestEnvSys", "TestNoMod", "TestMod", "TestSoft",
-           "TestExpandEnv")
-
-# Test data gleaned from the environment.
-module_system = "none"
-test_module = "foo"
-module_file = None
-if "MODULE_SYSTEM" in environ:
-    module_system = environ["MODULE_SYSTEM"]
-    assert "TEST_MODULE" in environ, \
-        "MODULE_SYSTEM is set, but TEST_MODULE is not defined."
-    test_module = environ["TEST_MODULE"]
-    if module_system == "module":
-        assert "MODULE_FILE" in environ, \
-            "MODULE_SYSTEM is module, but MODULE_FILE is not defined."
-        module_file = environ["MODULE_FILE"]
-
-
-class TestEnvSys(unittest.TestCase):
-
-    """Tests for the EnvSystemInterface class.
-
-    This tests that each method is NotImplemented. Other test classes in
-    this module inherit from this one in order to detect when a method has
-    been implemented but no test is defined for the new method.
-
-    However, figuring out which class causes the error is a bit of a
-    guessing game. Addressing that issue is difficult in Python 2.6 because
-    of how assertRaises works.
-    """
-
-    test_class = EnvSystemInterface
-
-    def setUp(self):
-        """Set up by creating an instance of the class under test."""
-        self.test_obj = self.test_class()
-
-    def test_is_loaded(self):
-        """is_loaded raises NotImplementedError."""
-        self.assertRaises(NotImplementedError,
-                          self.test_obj.is_loaded, test_module)
-
-    def test_purge(self):
-        """purge raises NotImplementedError."""
-        self.assertRaises(NotImplementedError,
-                          self.test_obj.purge)
-
-    def test_load(self):
-        """load raises NotImplementedError."""
-        self.assertRaises(NotImplementedError,
-                          self.test_obj.load, test_module)
-
-    def test_purge_str(self):
-        """purge_str raises NotImplementedError."""
-        self.assertRaises(NotImplementedError,
-                          self.test_obj.purge_str)
-
-    def test_load_str(self):
-        """load_str raises NotImplementedError."""
-        self.assertRaises(NotImplementedError,
-                          self.test_obj.load_str, test_module)
-
-    def test_unload_str(self):
-        """unload_str raises NotImplementedError."""
-        self.assertRaises(NotImplementedError,
-                          self.test_obj.unload_str, test_module)
-
-
-class TestNoMod(TestEnvSys):
-
-    """Tests for the NoModuleInterface class."""
-
-    test_class = NoModuleInterface
-
-    def test_purge_str(self):
-        """User can call NoModuleInterface.purge_str and get a ":"."""
-        self.assertEqual(self.test_obj.purge_str(), ":")
-
-
-class TestMod(TestEnvSys):
-
-    """Tests for the ModuleInterface class.
-
-    It's not easy to verify that this behaves correctly, so in many cases
-    we do some kind of check for self consistency only. If module_system
-    is not "module", then most of the tests look for an exception.
-    """
-
-    test_class = ModuleInterface
-
-    def setUp(self):
-        """Set up module system if present, then call parent setUp."""
-        if module_system == "module":
-            ModuleInterface.python_init(module_file)
-        super(TestMod, self).setUp()
-
-    def tearDown(self):
-        """Remove the test module from the current environment."""
-        if module_system == "module":
-            self.test_obj.unload(test_module)
-
-    def test_is_loaded(self):
-        """ModuleInterface.is_loaded returns false for unloaded module."""
-        if module_system == "module":
-            self.assertFalse(self.test_obj.is_loaded(test_module))
-        else:
-            self.assertRaises(AssertionError,
-                              self.test_obj.is_loaded, test_module)
-
-    def test_purge(self):
-        """ModuleInterface.purge removes the test module."""
-        if module_system == "module":
-            self.test_obj.load(test_module)
-            self.test_obj.purge()
-            self.assertFalse(self.test_obj.is_loaded(test_module))
-        else:
-            self.assertRaises(AssertionError,
-                              self.test_obj.purge)
-
-    def test_load(self):
-        """ModuleInterface.load loads a module."""
-        if module_system == "module":
-            self.test_obj.load(test_module)
-            self.assertTrue(self.test_obj.is_loaded(test_module))
-        else:
-            self.assertRaises(AssertionError,
-                              self.test_obj.load, test_module)
-
-    def test_unload(self):
-        """ModuleInterface.unload unloads a module."""
-        if module_system == "module":
-            self.test_obj.load(test_module)
-            self.test_obj.unload(test_module)
-            self.assertFalse(self.test_obj.is_loaded(test_module))
-        else:
-            self.assertRaises(AssertionError,
-                              self.test_obj.unload, test_module)
-
-    def test_purge_str(self):
-        """User gets non-null string from ModuleInterface.purge_str."""
-        self.assertNotEqual(self.test_obj.purge_str().strip(),
-                            "")
-
-    def test_load_str(self):
-        """User gets non-null string from ModuleInterface.load_str."""
-        self.assertNotEqual(self.test_obj.load_str(test_module).strip(),
-                            "")
-
-    def test_unload_str(self):
-        """User gets non-null string from ModuleInterface.unload_str."""
-        self.assertNotEqual(self.test_obj.unload_str(test_module).strip(),
-                            "")
-
-
-class TestSoft(TestEnvSys):
-
-    """Tests for the SoftEnvInterface class.
-
-    It's not easy to verify that this behaves correctly without a specific
-    test machine in mind. As with the module tests, we really just verify
-    that no exceptions are raised and the return isn't the null string.
-    """
-
-    test_class = SoftEnvInterface
-
-    def test_purge_str(self):
-        """User gets non-null string from SoftEnvInterface.purge_str."""
-        self.assertNotEqual(self.test_obj.purge_str().strip(), "")
-
-    def test_load_str(self):
-        """User gets non-null string from SoftEnvInterface.load_str."""
-        self.assertNotEqual(self.test_obj.load_str(test_module).strip(),
-                            "")
-
-    def test_unload_str(self):
-        """User gets non-null string from SoftEnvInterface.unload_str."""
-        self.assertNotEqual(self.test_obj.unload_str(test_module).strip(),
-                            "")
-
-
-class TestExpandEnv(unittest.TestCase):
-
-    """Tests for the expand_env function."""
-
-    def test_no_variable(self):
-        """With no variables, expand_env is a no-op."""
-        self.assertEqual(expand_env("foo", {"UNUSED": "not used"}), "foo")
-
-    def test_variable_missing(self):
-        """With variables not present, expand_env is a no-op."""
-        self.assertEqual(expand_env("${NOT_HERE}", {}), "${NOT_HERE}")
-
-    def test_brace_expansion(self):
-        """Test that an expansion works with curly braces."""
-        self.assertEqual(expand_env("${FOO}", {"FOO": "bar"}), "bar")
-
-    def test_bare_expansion(self):
-        """Test that an expansion works with no braces."""
-        self.assertEqual(expand_env("$FOO", {"FOO": "bar"}), "bar")
-
-    def test_recursive_expansion(self):
-        """Test that expansion is done recursively."""
-        self.assertEqual(expand_env("${FOO}",
-                                    {"FOO": "${FOO2}", "FOO2": "bar"}),
-                         "bar")
-
-    def test_brace_closing(self):
-        """Test that braces must be closed for expansion to occur."""
-        self.assertEqual(expand_env("${FOO", {"FOO": "bar"}), "${FOO")
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/tools/unit_testing/python/test_xml_utils.py b/tools/unit_testing/python/test_xml_utils.py
deleted file mode 100755
index f05a6bf72f1..00000000000
--- a/tools/unit_testing/python/test_xml_utils.py
+++ /dev/null
@@ -1,294 +0,0 @@
-#!/usr/bin/env python
-"""Unit tests for the xml_utils module.
-
-Public classes:
-TestBestMatch - best_match tests.
-TestAllMatches - all_matches tests.
-TestElementsToDict - elements_to_dict tests.
-"""
-
-import unittest
-from xml.etree.ElementTree import XML, ElementTree
-
-from xml_utils import best_match, all_matches, elements_to_dict
-
-__all__ = ("TestBestMatch", "TestAllMatches", "TestElementsToDict")
-
-class TestBestMatch(unittest.TestCase):
-
-    """Tests for the best_match function."""
-
-    def setUp(self):
-        """Create an ElementTree object as test data.
-
-        This is a complex set of data, but the tests on it are
-        comparatively simple. Some of the data (such as the empty "foo"
-        tags) are present purely to prove that they are never matched.
-        """
-
-        root = XML("""
-
- 
-  a
-  a
- 
- 
-  generic
-  generic
-  generic
- 
- 
-  b
-  b
- 
- 
-  ab
- 
- 
- 
- 
- 
- 
-  abar
- 
- 
-  bbar
- 
-
-""")
-
-        self.xml_tree = ElementTree(root)
-
-    def test_no_match(self):
-        """best_match returns None when no paths match."""
-
-        self.assertTrue(best_match(self.xml_tree, "invalid")
-                        is None)
-
-    def test_simple_match(self):
-        """best_match can find the only match, when no attributes are used."""
-
-        self.assertEqual(
-            "generic",
-            best_match(self.xml_tree, "foo/data1").text
-            )
-
-    def test_no_match_attr(self):
-        """best_match returns None when there are no attribute matches."""
-
-        self.assertTrue(best_match(self.xml_tree, "bar/data1")
-                        is None)
-        self.assertTrue(best_match(self.xml_tree, "bar/data1", {"a": "1"})
-                        is None)
-
-    def test_match_with_attr(self):
-        """best_match returns the only path match, with matching attribute."""
-
-        self.assertEqual(
-            "abar",
-            best_match(self.xml_tree, "bar/data1", {"a": "2"}).text
-            )
-
-    def test_match_on_attr(self):
-        """best_match returns the only path and attribute match."""
-
-        self.assertEqual(
-            "b",
-            best_match(self.xml_tree, "foo/data2",
-                       {"a": "2", "b": "2"}).text
-            )
-
-    def test_match_most_specific(self):
-        """best_match returns the most specific match for each path."""
-
-        self.assertEqual(
-            "generic",
-            best_match(self.xml_tree, "foo/data1",
-                       {"a": "1", "b": "2"}).text
-            )
-        self.assertEqual(
-            "ab",
-            best_match(self.xml_tree, "foo/data3",
-                       {"a": "1", "b": "2"}).text
-            )
-
-    def test_match_first(self):
-        """best_match returns the first matching entry."""
-
-        self.assertEqual(
-            "abar",
-            best_match(self.xml_tree, "bar/data1",
-                       {"a": "2", "b": "1"}).text
-            )
-
-
-class TestAllMatches(unittest.TestCase):
-
-    """Tests for the all_matches function."""
-
-    def setUp(self):
-        """Create an ElementTree for use as test data."""
-
-        root = XML("""
-
- 
-  the_text
- 
- 
-  first_text
-  second_text
- 
- 
- 
- 
-  first_text
-  second_text
-  second_text
- 
- 
-  first_text
-  second_text
-  third_text
-  fourth_text
-  bad_text
- 
- 
-  the_text
- 
-
-""")
-
-        self.xml_tree = ElementTree(root)
-
-    def test_one_match(self):
-        """all_matches returns one element successfully."""
-        self.assertEqual(
-            [e.text for e in all_matches(self.xml_tree, "test1/data")],
-            ["the_text"],
-            )
-
-    def test_multiple_matches(self):
-        """all_matches returns many elements successfully.
-
-        Elements must be returned in the right order as well.
-        """
-        self.assertEqual(
-            [e.text for e in all_matches(self.xml_tree, "test2/data")],
-            ["first_text", "second_text"],
-            )
-
-    def test_no_matches(self):
-        """all_matches returns no elements if none are found."""
-        self.assertEqual(
-            [e.text for e in all_matches(self.xml_tree, "test3/data")],
-            [],
-            )
-
-    def test_no_attr_matches(self):
-        """all_matches returns no elements if none match attributes."""
-        self.assertEqual(
-            [e.text for e in all_matches(self.xml_tree, "test4/data",
-                                         {"valid": "true"})],
-            [],
-            )
-
-    def test_attr_matches(self):
-        """all_matches returns elements that match attributes."""
-        self.assertEqual(
-            [e.text for e in
-             all_matches(self.xml_tree, "test5/data",
-                         {"valid": "true", "invalid": "false"})],
-            ["first_text", "second_text", "third_text", "fourth_text"],
-            )
-
-    def test_ignore_attribute(self):
-        """all_matches can ignore "extra" attributes."""
-        self.assertEqual(
-            [e.text for e in
-             all_matches(self.xml_tree, "test6/data",
-                         ignore=["ignore"])],
-            ["the_text"]
-            )
-
-
-# In true TDD form, these tests are far longer than the code they apply to!
-# Of course, when they were written, the first version of the code was
-# longer.
-class TestElementsToDict(unittest.TestCase):
-
-    """Tests for the elements_to_dict function."""
-
-    @staticmethod
-    def string_to_elements(xml_str, ignore="key"):
-        """Converts an XML string to a list of its "ENV" elements."""
-        return all_matches(ElementTree(XML(xml_str)), "ENV",
-                           ignore=ignore)
-
-    def test_no_elements(self):
-        """elements_to_dict with no elements produces an empty dict."""
-        self.assertEqual(elements_to_dict([]), {})
-
-    def test_one_element(self):
-        """elements_to_dict can handle one element."""
-        elements = self.string_to_elements("""
-
- bar
-
-""")
-        self.assertEqual(elements_to_dict(elements), {"foo": "bar"})
-
-    def test_multiple_elements(self):
-        """elements_to_dict can handle multiple elements."""
-        elements = self.string_to_elements("""
-
- bar1
- bar2
-
-""")
-        self.assertEqual(elements_to_dict(elements),
-                         {"foo1": "bar1", "foo2": "bar2"})
-
-    def test_key_attr(self):
-        """elements_to_dict uses key_attr as key attribute."""
-        key_attr = "name"
-        xml_str = """
-
- bar
-
-"""
-        elements = self.string_to_elements(xml_str, ignore=key_attr)
-        self.assertEqual(elements_to_dict(elements, key_attr=key_attr),
-                         {"foo": "bar"})
-
-    def test_multiple_key(self):
-        """elements_to_dict lets later values overwrite earlier ones.
-
-        This might not actually be the best behavior, but it's the easiest
-        to implement; this test is really just to prevent *accidentally*
-        changing the design.
-        """
-        elements = self.string_to_elements("""
-
- bar1
- bar2
-
-""")
-        self.assertEqual(elements_to_dict(elements),
-                         {"foo": "bar2"})
-
-    def test_keyless_match(self):
-        """elements_to_dict with a match with no key returns empty dict.
-
-        It might be better to raise an exception; this is a robustness and
-        simplicity vs. correctness tradeoff.
-        """
-        elements = self.string_to_elements("""
-
- bar
-
-""")
-        self.assertEqual(elements_to_dict(elements), {})
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/tools/unit_testing/python/xml_utils.py b/tools/unit_testing/python/xml_utils.py
deleted file mode 100644
index 4089c669afe..00000000000
--- a/tools/unit_testing/python/xml_utils.py
+++ /dev/null
@@ -1,202 +0,0 @@
-"""XML search utilities.
-
-These are based on the standard library xml.etree.ElementTree module.
-However, that module is not explicitly imported at this time, so it is in
-principle possible to pass arguments that imitate the interface of that
-module.
-
-Exported functions:
-best_match - Search for a specific element in an XML tree.
-all_matches - Search for all matching elements in an XML tree.
-elements_to_dict - Create a dict from XML entries with a key attribute.
-"""
-
-from comparable import Comparable
-
-__all__ = ("best_match", "all_matches", "elements_to_dict")
-
-class ElementMatch(Comparable):
-
-    """Class to aid searching for/matching an xml element.
-
-    Public methods:
-    __init__ - Create an ElementMatch.
-    __iadd__ - Add in "quality" of other match.
-    __eq__ - Test if quality is same as another match.
-    __lt__ - Compare quality to another match.
-    __nonzero__/__bool__ - Test for a valid match.
-
-    Public data:
-    element - Element for the match (None if no match). Read-only.
-
-    Note that using __eq__ to test quality interferes with hashable
-    collections, so for now, __hash__ = None.
-    """
-
-    def __init__(self, element=None, quality=0):
-        """Define a element match.
-
-        Arguments:
-        element - The element that was found.
-        quality - Abstract measure of "quality" (specificity) of the match.
-
-        With both options left out, this will return a null ElementMatch.
-        """
-        self._element = element
-        self._quality = quality
-
-    def __iadd__(self, other):
-        """Add the quality from another match to that of this match."""
-        self._quality += other._quality
-        return self
-
-    __hash__ = None
-    """Not implemented, because __eq__ is overridden."""
-
-    def __eq__(self, other):
-        """Check whether two matches have the same quality."""
-        return (not self and not other) or \
-            (self and other and self._quality == other._quality)
-
-    def __lt__(self, other):
-        """Compare matches by their quality; null matches always lose."""
-        return (not self and other) or \
-            (self and other and self._quality < other._quality)
-
-    def __bool__(self):
-        """Test if this is a real match or just the null one."""
-        return self._element is not None
-
-    __nonzero__ = __bool__
-
-    @property
-    def element(self):
-        """Element from a valid match, or None for a null match."""
-        return self._element
-
-# Right now this is treated as private to the module. But it's getting big
-# enough to justify having its own tests, maybe.
-def _element_attribute_match(element, attributes, ignore=[]):
-    """Check an element to see if it matches the given attributes.
-
-    If an element passes the check, give it a "quality" corresponding to
-    the number of matched attributes. Otherwise, return null match.
-
-    The ignore attribute specifies attributes to ignore.
-    """
-    match_quality = 0
-    for key in element.keys():
-        if key in ignore:
-            continue
-        if key in attributes and attributes[key] == element.get(key):
-            match_quality += 1
-        else:
-            return ElementMatch()
-    return ElementMatch(element, match_quality)
-
-def best_match(xml_tree, path, attributes={}):
-    """Find the best match for a path with attributes in an XML tree.
-
-    The return value is the matched element.
-
-    Arguments:
-    xml_tree - A tree from the xml.etree.ElementTree module.
-    path - The path to search for.
-    attributes - A dict containing attributes to match. Not all attributes
-                 must be present on an element to get a match, but any
-                 attributes present must match this input, and the "best"
-                 match has the most matches.
-
-    The attributes argument defaults to an empty dict.
-    """
-
-    # Recursive function to find a match.
-    def find_best_below(element, path):
-        # Done when there's no more of the path to match.
-        if len(path) == 0:
-            return ElementMatch(element)
-
-        # Otherwise, get the beginning part of the path.
-        path_head, slash, path_tail = path.partition("/")
-
-        # Search through subelements that match the next part of the path.
-        # For each one, get the quality of the match based on the number
-        # of matching attributes, then call self recursively to get
-        # best match from subelements.
-        best_match = ElementMatch()
-        for trial_element in element.findall(path_head):
-            local_match = _element_attribute_match(trial_element,
-                                                   attributes)
-            if local_match:
-                new_match = \
-                    find_best_below(trial_element, path_tail)
-                new_match += local_match
-                if best_match < new_match:
-                    best_match = new_match
-
-        return best_match
-
-    # Do the search.
-    element = xml_tree.getroot()
-    match = find_best_below(element, path)
-
-    return match.element
-
-def all_matches(xml_tree, path, attributes={}, ignore=[]):
-    """Find all matches for a path with attributes in an XML tree.
-
-    This is a generator. Each of the returned values will be a matching
-    element.
-
-    Arguments:
-    xml_tree - A tree from the xml.etree.ElementTree module.
-    path - The path to search for.
-    attributes - A dict containing attributes to match. Not all attributes
-                 must be present on an element to get a match, but any
-                 attributes present must match this input.
-    ignore - A list of attributes to ignore when matching. Attributes in
-             this list are ignored regardless of whether or not they are in
-             the attributes dict.
-
-    The attributes argument defaults to an empty dict.
-    """
-
-    # Recursive function to find all matches.
-    def find_matches_below(element, path):
-        # Done when there's no more of the path to match.
-        if len(path) == 0:
-            yield ElementMatch(element)
-            return
-
-        # Otherwise, get the beginning part of the path.
-        path_head, slash, path_tail = path.partition("/")
-
-        # Search through subelements that match the next part of the path.
-        # For each one, call self recursively to get all subelement matches.
-        for trial_element in element.findall(path_head):
-            if _element_attribute_match(trial_element, attributes, ignore):
-                for match in find_matches_below(trial_element, path_tail):
-                    yield match
-
-    element = xml_tree.getroot()
-    for match in find_matches_below(element, path):
-        yield match.element
-
-def elements_to_dict(elements, key_attr="key"):
-    """Uses a key attribute to produce a dict from ElementTree elements.
-
-    For each element, it creates an entry in the returned dict. The key is
-    determined by key_attr, and the value is the text of the element.
-
-    Arguments:
-    elements - An iterable of elements to convert.
-    key_attr - The name of an attribute. Each element must have this
-               attribute to be included in the dict, and the value of this
-               attribute will be used as the key.
-    """
-
-    return dict(
-        (elem.get(key_attr), elem.text)
-        for elem in elements
-        if key_attr in elem.keys()
-        )
diff --git a/utils/perl5lib/Build/Namelist.pm b/utils/perl5lib/Build/Namelist.pm
index 5c222c1fab1..cd3643067d2 100644
--- a/utils/perl5lib/Build/Namelist.pm
+++ b/utils/perl5lib/Build/Namelist.pm
@@ -627,9 +627,10 @@ my $valint = "[+-]?[0-9]+";
 my $valint_repeat = "${valint}\\*$valint";
 
 # Logical data.
-my $vallogical1 = "\\.[Tt][Rr][Uu][Ee]\\.";
-my $vallogical2 = "\\.[Ff][Aa][Ll][Ss][Ee]\\.";
-my $vallogical = "$vallogical1|$vallogical2";
+my $vallogical1 = "\\.?[Tt][Rr][Uu][Ee]\\.?";
+my $vallogical2 = "\\.?[Ff][Aa][Ll][Ss][Ee]\\.?";
+my $vallogical3 = "[FfTt]";
+my $vallogical = "$vallogical1|$vallogical2|$vallogical3";
 my $vallogical_repeat = "${valint}\\*$vallogical1|${valint}\\*$vallogical2";
 
 # Real data.
diff --git a/utils/perl5lib/compilers_translation_tool.pl b/utils/perl5lib/compilers_translation_tool.pl
index 964c734d3bb..15ed6ba02fa 100755
--- a/utils/perl5lib/compilers_translation_tool.pl
+++ b/utils/perl5lib/compilers_translation_tool.pl
@@ -17,7 +17,7 @@
     confirmed by running:
 
     xmllint -noout -schema \
-        \$CIME_ROOT/cime_config/xml_schemas/config_build.xsd OUTPUT_FILE
+        \$CIME_ROOT/config/xml_schemas/config_build.xsd OUTPUT_FILE
 EOF
 }
 
diff --git a/utils/python/cs.status.template b/utils/python/cs.status.template
deleted file mode 100644
index f774bcc03e1..00000000000
--- a/utils/python/cs.status.template
+++ /dev/null
@@ -1,3 +0,0 @@
-#! /bin/bash
-
-/cs_status *./TestStatus