Merge pull request #493 from PCMDI/490_replace_cdms2_with_netcdf4

Replace cdms2 with netCDF4 in PrePARE and Python tests

.circleci/config.yml

@@ -22,7 +22,7 @@ aliases:
        conda config --set always_yes yes --set changeps1 no
        conda update -y -q conda
        conda config --set anaconda_upload no
-       conda create -q -n py$PYTHON_VERSION -c cdat/label/nightly -c conda-forge -c cdat libuuid json-c udunits2 hdf5 libnetcdf=4.6.2 numpy openssl lazy-object-proxy cdms2 python=$PYTHON_VERSION $CONDA_COMPILERS testsrunner
+       conda create -q -n py$PYTHON_VERSION -c cdat/label/nightly -c conda-forge -c cdat libuuid json-c udunits2 hdf5 libnetcdf netcdf4 numpy openssl lazy-object-proxy python=$PYTHON_VERSION $CONDA_COMPILERS testsrunner
 
   - &setup_cmor
     name: setup_cmor
@@ -49,6 +49,9 @@ aliases:
        source activate py$PYTHON_VERSION
        set -e
        make test
+       # run tests again but with cdms2 installed
+       conda install -q -n py$PYTHON_VERSION -c cdat/label/nightly -c conda-forge -c cdat cdms2
+       make test
   - &run_prepare_tests
     name: run_prepare_tests
     command: |

.travis.yml

@@ -23,7 +23,7 @@ before_install:
   - conda update -y -q conda
   - if [[ $TRAVIS_OS_NAME == "linux" ]]; then export CONDA_COMPILERS="gcc_linux-64 gfortran_linux-64"; fi
   - if [[ $TRAVIS_OS_NAME == "osx" ]]; then export CONDA_COMPILERS="clang_osx-64 gfortran_osx-64"; fi
-  - conda create -q -n py${CMOR_PYTHON_VERSION} -c cdat/label/nightly -c conda-forge -c cdat libuuid json-c udunits2 hdf5 libnetcdf=4.6.2 numpy openssl lazy-object-proxy cdms2 python=${CMOR_PYTHON_VERSION} $CONDA_COMPILERS testsrunner
+  - conda create -q -n py${CMOR_PYTHON_VERSION} -c cdat/label/nightly -c conda-forge -c cdat libuuid json-c udunits2 hdf5 libnetcdf netcdf4 numpy openssl lazy-object-proxy python=${CMOR_PYTHON_VERSION} $CONDA_COMPILERS testsrunner
   - source activate py${CMOR_PYTHON_VERSION}
 install:
   - export PREFIX=$(python -c "import sys; print(sys.prefix)")
@@ -36,6 +36,8 @@ install:
   - make install
 
 script:
+  - make test
+  - conda install -q -n py${CMOR_PYTHON_VERSION} -c cdat/label/nightly -c conda-forge -c cdat cdms2
   - make test
   - export PYTHONPATH=Test/:$PYTHONPATH
   - python run_tests.py -v2 -n1 -H Test/test_python_CMIP6_CV*.py

Lib/pywrapper.py

@@ -71,22 +71,23 @@ def time_varying_grid_coordinate(
     if not isinstance(units, six.string_types):
         raise Exception("Error you must pass a string for the variable units")
     if not isinstance(type, six.string_types):
-        raise Exception("error tpye must a a string")
+        raise Exception("error type must be a string")
     type = type.lower()
-    if type == 's':
+    if type == 's' or type == 'u':
         type = 'c'
     if type not in ["c", "d", "f", "l", "i"]:
         raise Exception(
             'error unknown type: "%s", must be one of: "c","d","f","l","i"')
 
-    if not isinstance(grid_id, (int, numpy.int, numpy.int32)):
+    if not isinstance(grid_id, (int, numpy.int, numpy.int32, numpy.int64)):
         raise Exception("error grid_id must be an integer")
 
     grid_id = int(grid_id)
 
     if missing_value is not None:
-        if not isinstance(missing_value, (float, int, numpy.float,
-                                          numpy.float32, numpy.int, numpy.int32)):
+        if not isinstance(missing_value, (float, int, 
+                                          numpy.float, numpy.float32, numpy.float64, 
+                                          numpy.int, numpy.int32, numpy.int64)):
             raise Exception(
                 "error missing_value must be a number, you passed: %s" %
                 type(missing_value))
@@ -242,11 +243,12 @@ def set_grid_mapping(grid_id, mapping_name, parameter_names,
        parameter_units  :: array/list of parameter units  in the same order of parameter_names (ignored if parameter_names is ditcionary)
     """
     if sys.version_info < (3, 0):
-        if not isinstance(grid_id, (numpy.int32, int, long)):
+        if not isinstance(grid_id, (numpy.int32, numpy.int64, int, long)):
             raise Exception("grid_id must be an integer: %s" % type(grid_id))
     else:
-        if not isinstance(grid_id, (numpy.int32, int)):
+        if not isinstance(grid_id, (numpy.int32, numpy.int64, int)):
             raise Exception("grid_id must be an integer: %s" % type(grid_id))
+        grid_id = int(grid_id)
     if not isinstance(mapping_name, six.string_types):
         raise Exception("mapping name must be a string")
 
@@ -424,10 +426,10 @@ def axis(table_entry, units=None, length=None,
         l = len(coord_vals)
         type = coord_vals.dtype.char[0]
 
-        if not type in ['i', 'l', 'f', 'd', 'S']:
+        if not type in ['i', 'l', 'f', 'd', 'S', 'U']:
             raise Exception("error allowed data type are: i,l,f,d or S")
 
-        if type == 'S':
+        if type == 'S' or type == 'U':
             type = 'c'
             cbnds = 0
             for s in coord_vals:
@@ -507,9 +509,9 @@ def variable(table_entry, units, axis_ids, type='f', missing_value=None,
         raise Exception("error axis_ids list/array must be 1D")
 
     if not isinstance(type, six.string_types):
-        raise Exception("error tpye must a a string")
+        raise Exception("error type must be a string")
     type = type.lower()
-    if type == 's':
+    if type == 's' or type == 'u':
         type = 'c'
     if not type in ["c", "d", "f", "l", "i"]:
         raise Exception(
@@ -532,15 +534,17 @@ def variable(table_entry, units, axis_ids, type='f', missing_value=None,
     else:
         comment = str(comment)
 
-    if not isinstance(tolerance, (float, int, numpy.float,
-                                  numpy.float32, numpy.int, numpy.int32)):
+    if not isinstance(tolerance, (float, int, 
+                                  numpy.float, numpy.float32, numpy.float64, 
+                                  numpy.int, numpy.int32, numpy.int64)):
         raise Exception("error tolerance must be a number")
 
     tolerance = float(tolerance)
 
     if missing_value is not None:
-        if not isinstance(missing_value, (float, int, numpy.float,
-                                          numpy.float32, numpy.int, numpy.int32)):
+        if not isinstance(missing_value, (float, int,
+                                          numpy.float, numpy.float32, numpy.float64, 
+                                          numpy.int, numpy.int32, numpy.int64)):
             raise Exception(
                 "error missing_value must be a number, you passed: %s" %
                 repr(missing_value))
@@ -555,7 +559,7 @@ def variable(table_entry, units, axis_ids, type='f', missing_value=None,
 def zfactor(zaxis_id, zfactor_name, units="", axis_ids=None,
             type=None, zfactor_values=None, zfactor_bounds=None):
 
-    if not isinstance(zaxis_id, (int, numpy.int, numpy.int32)):
+    if not isinstance(zaxis_id, (int, numpy.int, numpy.int32, numpy.int64)):
         raise Exception("error zaxis_id must be a number")
     zaxis_id = int(zaxis_id)
 
@@ -578,7 +582,7 @@ def zfactor(zaxis_id, zfactor_name, units="", axis_ids=None,
         axis_ids = numpy.ascontiguousarray(axis_ids)
     elif axis_ids is None:
         pass
-    elif isinstance(axis_ids, (int, numpy.int, numpy.int32)):
+    elif isinstance(axis_ids, (int, numpy.int, numpy.int32, numpy.int64)):
         axis_ids = numpy.array([axis_ids, ])
     elif not isinstance(axis_ids, numpy.ndarray):
         raise Exception(
@@ -632,9 +636,9 @@ def zfactor(zaxis_id, zfactor_name, units="", axis_ids=None,
         type = 'd'
 
     if not isinstance(type, six.string_types):
-        raise Exception("error type must a string")
+        raise Exception("error type must be a string")
     type = type.lower()
-    if type == 's':
+    if type == 's' or type == 'u':
         type = 'c'
     if not type in ["c", "d", "f", "l", "i"]:
         raise Exception(
@@ -683,15 +687,15 @@ def write(var_id, data, ntimes_passed=None, file_suffix="",
     Usage:
     ierr = write(var_id,data,ntimes_passed=None,file_suffix="",time_vals=None,time_bnds=None,store_with=None
     """
-    if not isinstance(var_id, (int, numpy.int, numpy.int32)):
+    if not isinstance(var_id, (int, numpy.int, numpy.int32, numpy.int64)):
         raise Exception("error var_id must be an integer")
     var_id = int(var_id)
 
     if not isinstance(file_suffix, six.string_types):
         raise Exception("Error file_suffix must be a string")
 
     if store_with is not None:
-        if not isinstance(store_with, (int, numpy.int, numpy.int32)):
+        if not isinstance(store_with, (int, numpy.int, numpy.int32, numpy.int64)):
             raise Exception("error store_with must be an integer")
         store_with = int(store_with)
 
@@ -741,7 +745,7 @@ def write(var_id, data, ntimes_passed=None, file_suffix="",
             ntimes_passed = 0
         else:
             ntimes_passed = len(time_vals)
-    if not isinstance(ntimes_passed, (int, numpy.int, numpy.int32)):
+    if not isinstance(ntimes_passed, (int, numpy.int, numpy.int32, numpy.int64)):
         raise Exception("error ntimes_passed must be an integer")
     ntimes_passed = int(ntimes_passed)
 

LibCV/PrePARE/PrePARE.py

@@ -33,8 +33,8 @@
 from uuid import uuid4 as uuid
 
 import numpy
+import netCDF4
 
-from cdms2 import Cdunif
 import cmip6_cv
 
 
@@ -384,7 +384,7 @@ def ControlVocab(self, ncfile, variable=None, print_all=True):
         #  Open file in processing
         #  The file needs to be open before the calling the test.
         # -------------------------------------------------------------------
-        infile = Cdunif.CdunifFile(ncfile, "r")
+        infile = netCDF4.Dataset(ncfile, "r")
         key = '{}_{}'.format(table_id, variable_id)
         variable_cmor_entry = None
         if key in list(out_names_tests.keys()):
@@ -457,15 +457,15 @@ def ControlVocab(self, ncfile, variable=None, print_all=True):
         for attr in ['branch_time_in_child', 'branch_time_in_parent']:
             if attr in list(self.dictGbl.keys()):
                 self.set_double_value(attr)
-                if not isinstance(self.dictGbl[attr], numpy.float64):
+                if not numpy.issubdtype(self.dictGbl[attr], numpy.float64):
                     print(BCOLORS.FAIL)
                     print("=====================================================================================")
                     print("{} is not a double: ".format(attr), type(self.dictGbl[attr]))
                     print("=====================================================================================")
                     print(BCOLORS.ENDC)
                     self.errors += 1
         for attr in ['realization_index', 'initialization_index', 'physics_index', 'forcing_index']:
-            if not isinstance(self.dictGbl[attr], numpy.ndarray):
+            if not numpy.issubdtype(self.dictGbl[attr], numpy.integer):
                 print(BCOLORS.FAIL)
                 print("=====================================================================================")
                 print("{} is not an integer: ".format(attr), type(self.dictGbl[attr]))
@@ -535,7 +535,7 @@ def ControlVocab(self, ncfile, variable=None, print_all=True):
         # -------------------------------------------------------------------
         varid = cmip6_cv.setup_variable(variable_cmor_entry,
                                         self.dictVar['units'],
-                                        self.dictVar['_FillValue'][0],
+                                        self.dictVar['_FillValue'],
                                         startime,
                                         endtime,
                                         startimebnds,

Makefile.in

@@ -124,8 +124,8 @@ backup: clean
 	@tar cfz $$TGZNAME Cmor; \
 	@touch $(TIMESTAMPDIR)/cmor_`$(TIMESTAMP)`_full.time; \
 	@echo "Full backup tar file created : $$TGZNAME")
-test:  cmor test_C @TEST_FORTRAN@ @MAKETESTPYTHON@
-	@echo "All C and Fortran Test passed successfully"
+test:  cmor test_C @TEST_FORTRAN@ test_python
+	@echo "All C, Fortran, and Python tests passed successfully"
 test_C: cmor 
 	rm -f ./ipcc_test_code ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@  Test/ipcc_test_code.c -L@prefix@/lib -I@prefix@/include  -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ @LDFLAGS@  -o ipcc_test_code  @VERB@; ./ipcc_test_code @VERB@
 	rm -f ./test_singletons ; @CC@ @CFLAGS@ @USERCFLAGS@ @CPPFLAGS@  Test/test_singletons.c -L@prefix@/lib -I@prefix@/include  -L. -lcmor @NCCFLAGS@ @NCLDFLAGS@ @UDUNITS2LDFLAGS@ @UDUNITS2FLAGS@ @JSONCLDFLAGS@ @JSONCFLAGS@ @UUIDLDFLAGS@ @UUIDFLAGS@ @LDFLAGS@  -o test_singletons  @VERB@; ./test_singletons @VERB@
@@ -148,10 +148,10 @@ test_a_python:
 	@echo "${OK_COLOR}Testing ${TEST_NAME} ${NO_COLOR}"
 	${PYTHONEXEC} ${TEST_NAME} ${VERB}
 test_python: python
-	env TEST_NAME=Test/test_python_climatology.py make test_a_python
+	# [missing] env TEST_NAME=Test/test_python_climatology.py make test_a_python
 	env TEST_NAME=Test/test_python_history.py make test_a_python
-	env TEST_NAME=Test/test_python_historydefault.py make test_a_python
-	env TEST_NAME=Test/test_python_test_write.py make test_a_python
+	# [missing] env TEST_NAME=Test/test_python_historydefault.py make test_a_python
+	# [missing] env TEST_NAME=Test/test_python_test_write.py make test_a_python
 	env TEST_NAME=Test/test_python_missing_values.py make test_a_python
 	env TEST_NAME=Test/test_python_history.py make test_a_python
 	env TEST_NAME=Test/test_python_sos_psu_units.py make test_a_python
@@ -198,17 +198,15 @@ test_python: python
 	env TEST_NAME=Test/jamie_hybrid_height.py make test_a_python
 	env TEST_NAME=Test/jamie_positive.py make test_a_python
 	env TEST_NAME=Test/test_python_1D_var.py make test_a_python
-	env TEST_NAME=Test/test_python_2Gb_file.py make test_a_python
-	env TEST_NAME=Test/test_python_2Gb_slice.py make test_a_python
 #	env TEST_NAME=Test/test_python_3hr.py make test_a_python
 	env TEST_NAME=Test/test_python_YYYMMDDHH_exp_fmt.py make test_a_python
 #	@env TEST_NAME=Test/test_python_alastair_1.py make test_a_python
 	env TEST_NAME=Test/test_python_cfmip_site_axis_test.py make test_a_python
 #	env TEST_NAME=Test/test_python_fx.py make test_a_python
 	env TEST_NAME=Test/test_python_grid_and_ocn_sigma.py make test_a_python
 	env TEST_NAME=Test/test_python_jamie_site_surface.py make test_a_python
-	env TEST_NAME=Test/test_python_max_variables.py make test_a_python
-	env TEST_NAME=Test/test_python_max_variables_2.py make test_a_python
+#	env TEST_NAME=Test/test_python_max_variables.py make test_a_python
+#	env TEST_NAME=Test/test_python_max_variables_2.py make test_a_python
 	env TEST_NAME=Test/test_python_reverted_lats.py make test_a_python
 	env TEST_NAME=Test/test_lon_gt_360.py make test_a_python
 	env TEST_NAME=Test/test_lon_thro_360.py make test_a_python
@@ -221,6 +219,9 @@ test_python: python
 	env TEST_NAME=Test/test_cmor_half_levels_with_bounds.py make test_a_python
 	env TEST_NAME=Test/test_cmor_half_levels_wrong_generic_level.py make test_a_python
 	env TEST_NAME=Test/test_cmor_double_singleton.py make test_a_python
+test_python_2gb:
+	env TEST_NAME=Test/test_python_2Gb_file.py make test_a_python
+	env TEST_NAME=Test/test_python_2Gb_slice.py make test_a_python
 
 test_case:
 	@echo "${OK_COLOR}Testing: "${TEST_NAME}" with input file: ${INPUT_FILE}${NO_COLOR}"

Src/_cmormodule.c

@@ -170,8 +170,8 @@ static PyObject *PyCMOR_set_variable_attribute(PyObject * self, PyObject * args)
         return NULL;
 
 #if PY_MAJOR_VERSION >= 3
-    if(PyBytes_Check(oValue)) {
-        value = PyBytes_AsString(oValue);
+    if(PyUnicode_Check(oValue)) {
+        value = PyUnicode_AsUTF8(oValue);
 #else
     if(PyString_Check(oValue)) {
         value = PyString_AsString(oValue);
@@ -753,14 +753,14 @@ static PyObject *PyCMOR_grid_mapping(PyObject * self, PyObject * args)
     for (i = 0; i < n; i++) {
         tmp = PyList_GetItem(param_nm_obj, i);
 #if PY_MAJOR_VERSION >= 3
-        strcpy(nms[i], PyBytes_AsString(tmp));
+        strcpy(nms[i], PyUnicode_AsUTF8(tmp));
 #else
         strcpy(nms[i], PyString_AsString(tmp));
 #endif
         //Py_DECREF(tmp); //Not needed get_item does not increase ref
         tmp = PyList_GetItem(param_un_obj, i);
 #if PY_MAJOR_VERSION >= 3
-        strcpy(units[i], PyBytes_AsString(tmp));
+        strcpy(units[i], PyUnicode_AsUTF8(tmp));
 #else
         strcpy(units[i], PyString_AsString(tmp));
 #endif

Test/cmor_speed_and_compression.py

@@ -136,7 +136,7 @@
 ltime = cdtime.reltime(ntimes - 1, 'month since 1980').tocomp()
 #lcmor = os.stat("CMIP6/CMIP/CSIRO-BOM/NICAM/piControl/r1i1p1f1/Amon/tas/gn/v%s/tas_Amon_piControl_NICAM_r1i1p1f1_gn_198001-%i%.2i.nc" % (today,ltime.year,ltime.month))[6]
 lcmor = os.stat(
-    "CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r11i1p1f1/Amon/tas/gr/v%s/tas_Amon_PCMDI-test-1-0_piControl-withism_r11i1p1f1_gr_198001-%i%.2i.nc" %
+    "CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r3i1p1f1/Amon/tas/gn/v%s/tas_Amon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_198001-%i%.2i.nc" %
     (today, ltime.year, ltime.month))[6]
 print('level:', level, "shuffle:", shuffle)
 print('total cmor:', totcmor, mincmor, totcmor / ntimes, maxcmor, lcmor)

Test/cmor_speed_and_compression_01.py

@@ -122,7 +122,7 @@
 ltime = cdtime.reltime(ntimes - 1, 'month since 1980').tocomp()
 #lcmor = os.stat("CMIP6/CMIP/CSIRO-BOM/NICAM/piControl/r1i1p1f1/Amon/tas/gn/v%s/tas_Amon_piControl_NICAM_r1i1p1f1_gn_197901-197912.nc"%(today))[6]
 lcmor = os.stat(
-    "CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r11i1p1f1/Amon/tas/gr/v%s/tas_Amon_PCMDI-test-1-0_piControl-withism_r11i1p1f1_gr_197901-197912.nc" % (today))[6]
+    "CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r3i1p1f1/Amon/tas/gn/v%s/tas_Amon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_197901-197912.nc" % (today))[6]
 print('level:', level, "shuffle:", shuffle)
 print('total cmor:', totcmor, mincmor, totcmor / ntimes, maxcmor, lcmor)
 lcdms = os.stat("Test/crap.nc")[6]

Test/common.py

@@ -111,4 +111,28 @@ def read_cmor_time_lat_lon():
         table_entry="time",
         units="days since 1850",
         length=ntimes)
+    return itim, ilat, ilon
+
+def read_cmor_time1_lat_lon():
+    ilat = cmor.axis(
+        table_entry='latitude',
+        units='degrees_north',
+        length=lat,
+        coord_vals=alats,
+        cell_bounds=bnds_lat)
+    print("ILAT:", ilat)
+    print(lon, alons, bnds_lon)
+    ilon = cmor.axis(
+        table_entry='longitude',
+        coord_vals=alons,
+        units='degrees_east',
+        cell_bounds=bnds_lon)
+
+    print("ILON:", ilon)
+
+
+    itim = cmor.axis(
+        table_entry="time1",
+        units="days since 1850",
+        length=ntimes)
     return itim, ilat, ilon

Test/test_cmor_double_singleton.py

@@ -7,7 +7,7 @@
 pth = os.getcwd()
 common.init_cmor(pth, "CMOR_input_example.json")
 cmor.load_table(os.path.join(pth, 'Tables', 'CMIP6_6hrLev.json'))
-itim, ilat, ilon = common.read_cmor_time_lat_lon()
+itim, ilat, ilon = common.read_cmor_time1_lat_lon()
 
 ilambda = cmor.axis(
     table_entry='lambda550nm',

Test/test_cmor_half_levels.py

@@ -1,8 +1,8 @@
 from __future__ import print_function
-import cdms2
 import cmor
 import numpy
 import os
+import sys
 import common
 
 varin3d = ["MC", ]

Test/test_cmor_half_levels_with_bounds.py

@@ -1,8 +1,8 @@
 from __future__ import print_function
-import cdms2
 import cmor
 import numpy
 import os
+import sys
 
 specs = {"CLOUD": {"convert": [2.0, 0.], "units": "%", "entry": "cl", "positive": ""},
          "U": {"convert": [1., -40.], "units": "m s-1", "entry": "ua", "positive": ""},