diff --git a/conda-env/dev.yml b/conda-env/dev.yml index e46d0f45e..ff48dcc6f 100644 --- a/conda-env/dev.yml +++ b/conda-env/dev.yml @@ -13,7 +13,6 @@ dependencies: - numpy=1.23.5 - cartopy=0.22.0 - matplotlib=3.7.1 - - cdat_info=8.2.1 - cdms2=3.1.5 - genutil=8.2.1 - cdutil=8.2.1 diff --git a/conda-env/readthedocs.yml b/conda-env/readthedocs.yml index cc161e5ad..44321a4ff 100644 --- a/conda-env/readthedocs.yml +++ b/conda-env/readthedocs.yml @@ -12,7 +12,6 @@ dependencies: - numpy=1.23.5 - cartopy=0.21.1 - matplotlib=3.7.1 - - cdat_info=8.2.1 - cdms2=3.1.5 - genutil=8.2.1 - cdutil=8.2.1 diff --git a/doc/jupyter/Demo/Demo_0_download_data.ipynb b/doc/jupyter/Demo/Demo_0_download_data.ipynb index 8b37e4059..ed5c40323 100644 --- a/doc/jupyter/Demo/Demo_0_download_data.ipynb +++ b/doc/jupyter/Demo/Demo_0_download_data.ipynb @@ -11,7 +11,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -36,14 +36,14 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "# This is where you will be downloading the sample_data\n", - "demo_data_directory = \"demo_data\"\n", + "demo_data_directory = \"demo_data_tmp\"\n", "# this line is where your output will be stored\n", - "demo_output_directory = \"demo_output\"" + "demo_output_directory = \"demo_output_tmp\"" ] }, { @@ -55,55 +55,45 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Downloading: 'CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.pr.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.pr.198101-200512.AC.v20200426.nc\n", - "Downloading: 'CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.rlut.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.rlut.198101-200512.AC.v20200426.nc\n", - "Downloading: 'CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.zg.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.zg.198101-200512.AC.v20200426.nc\n", - "Downloading: 'CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.pr.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.pr.198101-200512.AC.v20200426.nc\n", - "Downloading: 'CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.rlut.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.rlut.198101-200512.AC.v20200426.nc\n", - "Downloading: 'CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.zg.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.zg.198101-200512.AC.v20200426.nc\n", - "Downloading: 'CMIP5_demo_clims/cmip6.historical.MCM-UA-1-0.r1i1p1f1.mon.zg.198101-200512.AC.v20201119.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_clims/cmip6.historical.MCM-UA-1-0.r1i1p1f1.mon.zg.198101-200512.AC.v20201119.nc\n", - "Downloading: 'CMIP5_demo_data/psl_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/psl_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", - "Downloading: 'CMIP5_demo_data/sftlf_fx_ACCESS1-0_amip_r0i0p0.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/sftlf_fx_ACCESS1-0_amip_r0i0p0.nc\n", - "Downloading: 'CMIP5_demo_data/cmip5.amip.ACCESS1-0.sftlf.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/cmip5.amip.ACCESS1-0.sftlf.nc\n", - "Downloading: 'CMIP5_demo_data/cmip5.historical.GISS-E2-H.sftlf.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/cmip5.historical.GISS-E2-H.sftlf.nc\n", - "Downloading: 'CMIP5_demo_data/ts_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/ts_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", - "Downloading: 'CMIP5_demo_timeseries/historical/atmos/day/pr/pr_day_GISS-E2-H_historical_r6i1p1_20000101-20051231.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_timeseries/historical/atmos/day/pr/pr_day_GISS-E2-H_historical_r6i1p1_20000101-20051231.nc\n", - "Downloading: 'obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-0/v20210804/rlut_mon_CERES-EBAF-4-0_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-0/v20210804/rlut_mon_CERES-EBAF-4-0_PCMDI_gn.200301-201812.AC.v20210804.nc\n", - "Downloading: 'obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-1/v20210804/rlut_mon_CERES-EBAF-4-1_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-1/v20210804/rlut_mon_CERES-EBAF-4-1_PCMDI_gn.200301-201812.AC.v20210804.nc\n", - "Downloading: 'obs4MIPs_PCMDI_clims/pr/GPCP-2-3/v20210804/pr_mon_GPCP-2-3_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_clims/pr/GPCP-2-3/v20210804/pr_mon_GPCP-2-3_PCMDI_gn.200301-201812.AC.v20210804.nc\n", - "Downloading: 'obs4MIPs_PCMDI_clims/zg/ERA-INT/v20210804/zg_mon_ERA-INT_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_clims/zg/ERA-INT/v20210804/zg_mon_ERA-INT_PCMDI_gn.200301-201812.AC.v20210804.nc\n", - "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/zg/gn/v20210727/zg_mon_ERA-INT_PCMDI_gn_198901-201001.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/zg/gn/v20210727/zg_mon_ERA-INT_PCMDI_gn_198901-201001.nc\n", - "Downloading: 'obs4MIPs_PCMDI_monthly/NASA-LaRC/CERES-EBAF-4-1/mon/rlut/gn/v20210727/rlut_mon_CERES-EBAF-4-1_PCMDI_gn_200301-201812.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/NASA-LaRC/CERES-EBAF-4-1/mon/rlut/gn/v20210727/rlut_mon_CERES-EBAF-4-1_PCMDI_gn_200301-201812.nc\n", - "Downloading: 'obs4MIPs_PCMDI_monthly/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc\n", - "Downloading: 'obs4MIPs_PCMDI_monthly/NOAA-ESRL-PSD/20CR/mon/psl/gn/v20210727/psl_mon_20CR_PCMDI_gn_187101-201212.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/NOAA-ESRL-PSD/20CR/mon/psl/gn/v20210727/psl_mon_20CR_PCMDI_gn_187101-201212.nc\n", - "Downloading: 'obs4MIPs_PCMDI_monthly/MOHC/HadISST-1-1/mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/MOHC/HadISST-1-1/mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc\n", - "Downloading: 'obs4MIPs_PCMDI_daily/NASA-JPL/GPCP-1-3/day/pr/gn/latest/pr_day_GPCP-1-3_PCMDI_gn_19961002-20170101.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_daily/NASA-JPL/GPCP-1-3/day/pr/gn/latest/pr_day_GPCP-1-3_PCMDI_gn_19961002-20170101.nc\n", - "Downloading: 'misc_demo_data/ocn/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/misc_demo_data/ocn/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc\n", - "Downloading: 'misc_demo_data/ocn/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/misc_demo_data/ocn/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc\n", - "Downloading: 'misc_demo_data/ocn/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/misc_demo_data/ocn/ice_conc_nh_ease2-250_cdr-v3p0_198801-202012.nc\n", - "Downloading: 'misc_demo_data/ocn/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/misc_demo_data/ocn/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc\n", - "Downloading: 'misc_demo_data/ocn/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/misc_demo_data/ocn/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc\n", - "Downloading: 'misc_demo_data/ocn/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/misc_demo_data/ocn/ice_conc_sh_ease2-250_cdr-v3p0_198801-202012.nc\n", - "Downloading: 'misc_demo_data/atm/3hr/pr/pr_3hr_IPSL-CM5A-LR_historical_r1i1p1_5x5_1997-1999.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/misc_demo_data/atm/3hr/pr/pr_3hr_IPSL-CM5A-LR_historical_r1i1p1_5x5_1997-1999.nc\n", - "Downloading: 'misc_demo_data/fx/sftlf.GPCP-IP.1x1.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/misc_demo_data/fx/sftlf.GPCP-IP.1x1.nc\n", - "All files downloaded\n" + "Downloading: 'CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.pr.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.pr.198101-200512.AC.v20200426.nc\n", + "Downloading: 'CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.rlut.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.rlut.198101-200512.AC.v20200426.nc\n", + "Downloading: 'CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.zg.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_clims/cmip5.historical.ACCESS1-0.r1i1p1.mon.zg.198101-200512.AC.v20200426.nc\n", + "Downloading: 'CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.pr.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.pr.198101-200512.AC.v20200426.nc\n", + "Downloading: 'CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.rlut.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.rlut.198101-200512.AC.v20200426.nc\n", + "Downloading: 'CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.zg.198101-200512.AC.v20200426.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_clims/cmip5.historical.CanCM4.r1i1p1.mon.zg.198101-200512.AC.v20200426.nc\n", + "Downloading: 'CMIP5_demo_clims/cmip6.historical.MCM-UA-1-0.r1i1p1f1.mon.zg.198101-200512.AC.v20201119.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_clims/cmip6.historical.MCM-UA-1-0.r1i1p1f1.mon.zg.198101-200512.AC.v20201119.nc\n", + "Downloading: 'CMIP5_demo_data/psl_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_data/psl_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/sftlf_fx_ACCESS1-0_amip_r0i0p0.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_data/sftlf_fx_ACCESS1-0_amip_r0i0p0.nc\n", + "Downloading: 'CMIP5_demo_data/cmip5.amip.ACCESS1-0.sftlf.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_data/cmip5.amip.ACCESS1-0.sftlf.nc\n", + "Downloading: 'CMIP5_demo_data/cmip5.historical.GISS-E2-H.sftlf.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_data/cmip5.historical.GISS-E2-H.sftlf.nc\n", + "Downloading: 'CMIP5_demo_data/ts_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_data/ts_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_timeseries/historical/atmos/day/pr/pr_day_GISS-E2-H_historical_r6i1p1_20000101-20051231.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/CMIP5_demo_timeseries/historical/atmos/day/pr/pr_day_GISS-E2-H_historical_r6i1p1_20000101-20051231.nc\n", + "Downloading: 'obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-0/v20210804/rlut_mon_CERES-EBAF-4-0_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-0/v20210804/rlut_mon_CERES-EBAF-4-0_PCMDI_gn.200301-201812.AC.v20210804.nc\n", + "Downloading: 'obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-1/v20210804/rlut_mon_CERES-EBAF-4-1_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/obs4MIPs_PCMDI_clims/rlut/CERES-EBAF-4-1/v20210804/rlut_mon_CERES-EBAF-4-1_PCMDI_gn.200301-201812.AC.v20210804.nc\n", + "Downloading: 'obs4MIPs_PCMDI_clims/pr/GPCP-2-3/v20210804/pr_mon_GPCP-2-3_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/obs4MIPs_PCMDI_clims/pr/GPCP-2-3/v20210804/pr_mon_GPCP-2-3_PCMDI_gn.200301-201812.AC.v20210804.nc\n", + "Downloading: 'obs4MIPs_PCMDI_clims/zg/ERA-INT/v20210804/zg_mon_ERA-INT_PCMDI_gn.200301-201812.AC.v20210804.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/obs4MIPs_PCMDI_clims/zg/ERA-INT/v20210804/zg_mon_ERA-INT_PCMDI_gn.200301-201812.AC.v20210804.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/zg/gn/v20210727/zg_mon_ERA-INT_PCMDI_gn_198901-201001.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/zg/gn/v20210727/zg_mon_ERA-INT_PCMDI_gn_198901-201001.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/NASA-LaRC/CERES-EBAF-4-1/mon/rlut/gn/v20210727/rlut_mon_CERES-EBAF-4-1_PCMDI_gn_200301-201812.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/obs4MIPs_PCMDI_monthly/NASA-LaRC/CERES-EBAF-4-1/mon/rlut/gn/v20210727/rlut_mon_CERES-EBAF-4-1_PCMDI_gn_200301-201812.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/obs4MIPs_PCMDI_monthly/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/NOAA-ESRL-PSD/20CR/mon/psl/gn/v20210727/psl_mon_20CR_PCMDI_gn_187101-201212.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/obs4MIPs_PCMDI_monthly/NOAA-ESRL-PSD/20CR/mon/psl/gn/v20210727/psl_mon_20CR_PCMDI_gn_187101-201212.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/MOHC/HadISST-1-1/mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data_tmp/obs4MIPs_PCMDI_monthly/MOHC/HadISST-1-1/mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc\n" ] } ], "source": [ "# Let's download the files\n", - "import cdat_info\n", + "from pcmdi_metrics.io.base import download_sample_data_files\n", "try:\n", - " cdat_info.download_sample_data_files(\"data_files.txt\", demo_data_directory)\n", + " download_sample_data_files(\"data_files.txt\", demo_data_directory)\n", " print(\"All files downloaded\")\n", - "except:\n", + "except Exception:\n", " print(\"Download failed\")" ] }, @@ -116,33 +106,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Preparing parameter file: basic_annual_cycle_param.py\n", - "Preparing parameter file: basic_diurnal_composite.py\n", - "Preparing parameter file: basic_diurnal_compute_daily_mean.py\n", - "Preparing parameter file: basic_diurnal_fourier.py\n", - "Preparing parameter file: basic_diurnal_fourierAllGrid.py\n", - "Preparing parameter file: basic_diurnal_std_daily_mean.py\n", - "Preparing parameter file: basic_diurnal_std_hourly_mean.py\n", - "Preparing parameter file: basic_enso_param.py\n", - "Preparing parameter file: basic_extremes_param.py\n", - "Preparing parameter file: basic_mjo_param.py\n", - "Preparing parameter file: basic_monsoon_sperber_param.py\n", - "Preparing parameter file: basic_monsoon_wang_param.py\n", - "Preparing parameter file: basic_mov_param.py\n", - "Preparing parameter file: basic_mov_param_sst.py\n", - "Preparing parameter file: basic_param.py\n", - "Preparing parameter file: basic_precip_variability_param.py\n", - "Saving User Choices\n" - ] - } - ], + "outputs": [], "source": [ "from download_sample_data import generate_parameter_files\n", "generate_parameter_files(demo_data_directory, demo_output_directory)" @@ -159,9 +125,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python [conda env:pmp_test] *", "language": "python", - "name": "python3" + "name": "conda-env-pmp_test-py" }, "language_info": { "codemirror_mode": { @@ -173,7 +139,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.10" + "version": "3.10.14" }, "selected_variables": [], "vcdat_file_path": "", diff --git a/doc/jupyter/Demo/Demo_6_ENSO.ipynb b/doc/jupyter/Demo/Demo_6_ENSO.ipynb index 7c5e8d9e7..d0fe37c8d 100644 --- a/doc/jupyter/Demo/Demo_6_ENSO.ipynb +++ b/doc/jupyter/Demo/Demo_6_ENSO.ipynb @@ -89,15 +89,72 @@ "name": "stdout", "output_type": "stream", "text": [ + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-20C/mon/psl/gn/v20210727/psl_mon_ERA-20C_PCMDI_gn_190001-201012.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-20C/mon/psl/gn/v20210727/psl_mon_ERA-20C_PCMDI_gn_190001-201012.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-20C/mon/ts/gn/v20210727/ts_mon_ERA-20C_PCMDI_gn_190001-201012.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-20C/mon/ts/gn/v20210727/ts_mon_ERA-20C_PCMDI_gn_190001-201012.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/NOAA-ESRL-PSD/20CR/mon/psl/gn/v20210727/psl_mon_20CR_PCMDI_gn_187101-201212.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/NOAA-ESRL-PSD/20CR/mon/psl/gn/v20210727/psl_mon_20CR_PCMDI_gn_187101-201212.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/NOAA-ESRL-PSD/20CR/mon/ts/gn/v20210727/ts_mon_20CR_PCMDI_gn_187101-201212.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/NOAA-ESRL-PSD/20CR/mon/ts/gn/v20210727/ts_mon_20CR_PCMDI_gn_187101-201212.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/NOAA-NCEI/CMAP-V1902/mon/pr/gn/v20210727/pr_mon_CMAP-V1902_PCMDI_gn_197901-201901.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/NOAA-NCEI/CMAP-V1902/mon/pr/gn/v20210727/pr_mon_CMAP-V1902_PCMDI_gn_197901-201901.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/MOHC/HadISST-1-1/mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/MOHC/HadISST-1-1/mon/ts/gn/v20210727/ts_mon_HadISST-1-1_PCMDI_gn_187001-201907.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/hfls/gn/v20210727/hfls_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/hfls/gn/v20210727/hfls_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/hfns/gn/v20210727/hfns_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/hfns/gn/v20210727/hfns_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/hfss/gn/v20210727/hfss_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/hfss/gn/v20210727/hfss_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/tas/gn/v20210727/tas_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/tas/gn/v20210727/tas_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/tauu/gn/v20210727/tauu_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/tauu/gn/v20210727/tauu_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/tauv/gn/v20210727/tauv_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/tauv/gn/v20210727/tauv_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/ts/gn/v20210727/ts_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ESSO/TropFlux-1-0/mon/ts/gn/v20210727/ts_mon_TropFlux-1-0_PCMDI_gn_197901-201707.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/hfls/gn/v20210727/hfls_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/hfls/gn/v20210727/hfls_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/hfss/gn/v20210727/hfss_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/hfss/gn/v20210727/hfss_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/hur/gn/v20210727/hur_mon_ERA-INT_PCMDI_gn_198901-201001.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/hur/gn/v20210727/hur_mon_ERA-INT_PCMDI_gn_198901-201001.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/hus/gn/v20210727/hus_mon_ERA-INT_PCMDI_gn_198901-201001.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/hus/gn/v20210727/hus_mon_ERA-INT_PCMDI_gn_198901-201001.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/pr/gn/v20210727/pr_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/pr/gn/v20210727/pr_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/psl/gn/v20210727/psl_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/psl/gn/v20210727/psl_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/rlds/gn/v20210727/rlds_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/rlds/gn/v20210727/rlds_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/rlus/gn/v20210727/rlus_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/rlus/gn/v20210727/rlus_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/rsds/gn/v20210727/rsds_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/rsds/gn/v20210727/rsds_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/rsus/gn/v20210727/rsus_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/rsus/gn/v20210727/rsus_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/sfcWind/gn/v20210727/sfcWind_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/sfcWind/gn/v20210727/sfcWind_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/ta/gn/v20210727/ta_mon_ERA-INT_PCMDI_gn_198901-201001.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/ta/gn/v20210727/ta_mon_ERA-INT_PCMDI_gn_198901-201001.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/tauu/gn/v20210727/tauu_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/tauu/gn/v20210727/tauu_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/tauv/gn/v20210727/tauv_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/tauv/gn/v20210727/tauv_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/ts/gn/v20210727/ts_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/ts/gn/v20210727/ts_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/ua/gn/v20210727/ua_mon_ERA-INT_PCMDI_gn_198901-201001.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/ua/gn/v20210727/ua_mon_ERA-INT_PCMDI_gn_198901-201001.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/uas/gn/v20210727/uas_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/uas/gn/v20210727/uas_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/va/gn/v20210727/va_mon_ERA-INT_PCMDI_gn_198901-201001.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/va/gn/v20210727/va_mon_ERA-INT_PCMDI_gn_198901-201001.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/vas/gn/v20210727/vas_mon_ERA-INT_PCMDI_gn_197901-201903.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/vas/gn/v20210727/vas_mon_ERA-INT_PCMDI_gn_197901-201903.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/zg/gn/v20210727/zg_mon_ERA-INT_PCMDI_gn_198901-201001.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/ECMWF/ERA-INT/mon/zg/gn/v20210727/zg_mon_ERA-INT_PCMDI_gn_198901-201001.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/NOAA-NCEI/GPCP-2-3/mon/pr/gn/v20210727/pr_mon_GPCP-2-3_PCMDI_gn_197901-201907.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/NASA-GSFC/TRMM-3B43v-7/mon/pr/gn/v20210727/pr_mon_TRMM-3B43v-7_PCMDI_gn_199801-201712.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/NASA-GSFC/TRMM-3B43v-7/mon/pr/gn/v20210727/pr_mon_TRMM-3B43v-7_PCMDI_gn_199801-201712.nc\n", + "Downloading: 'obs4MIPs_PCMDI_monthly/CNES/AVISO-1-0/mon/zos/gn/v20210727/zos_mon_AVISO-1-0_PCMDI_gn_199301-201912.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/obs4MIPs_PCMDI_monthly/CNES/AVISO-1-0/mon/zos/gn/v20210727/zos_mon_AVISO-1-0_PCMDI_gn_199301-201912.nc\n", + "Downloading: 'CMIP5_demo_data/psl_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/psl_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/ts_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/ts_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/hfls_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/hfls_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/hfss_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/hfss_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/pr_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/pr_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/rlds_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/rlds_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/rlus_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/rlus_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/rlut_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/rlut_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/rsds_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/rsds_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/rsdt_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/rsdt_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/rsus_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/rsus_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/rsut_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/rsut_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/sftlf_fx_ACCESS1-0_amip_r0i0p0.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/sftlf_fx_ACCESS1-0_amip_r0i0p0.nc\n", + "Downloading: 'CMIP5_demo_data/tauu_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/tauu_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", + "Downloading: 'CMIP5_demo_data/zos_Omon_ACCESS1-0_historical_r1i1p1_185001-200512.nc' from 'https://pcmdiweb.llnl.gov/pss/pmpdata/' in: demo_data/CMIP5_demo_data/zos_Omon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\n", "All files downloaded\n" ] } ], "source": [ "# Let's download the files\n", - "import cdat_info\n", + "from pcmdi_metrics.io.base import download_sample_data_files\n", "try:\n", - " cdat_info.download_sample_data_files(\"enso_data_files.txt\", demo_data_directory)\n", + " download_sample_data_files(\"enso_data_files.txt\", demo_data_directory)\n", " print(\"All files downloaded\")\n", "except:\n", " print(\"Download failed\")" @@ -183,7 +240,7 @@ " [--obs_cmor_path OBS_CMOR_PATH] [-d [DEBUG]]\n", " [--obs_cmor [OBS_CMOR]] [--nc_out [NC_OUT]]\n", "\n", - "optional arguments:\n", + "options:\n", " -h, --help show this help message and exit\n", " --parameters PARAMETERS, -p PARAMETERS\n", " --diags OTHER_PARAMETERS [OTHER_PARAMETERS ...]\n", @@ -222,7 +279,7 @@ " The name of the folder where all runs will be stored.\n", " (default: None)\n", " --case_id CASE_ID version as date, e.g., v20191116 (yyyy-mm-dd)\n", - " (default: v20220401)\n", + " (default: v20240702)\n", " --obs_catalogue OBS_CATALOGUE\n", " obs_catalogue JSON file for CMORized observation,\n", " default is None (default: None)\n", @@ -340,14 +397,14 @@ "debug: False\n", "obs_cmor: True\n", "obs_cmor_path: demo_data/obs4MIPs_PCMDI_monthly\n", - "egg_pth: /Users/ordonez4/Library/Caches/Python-Eggs/pcmdi_metrics-v2.1.0_183_gcf9dc528-py3.9.egg-tmp/share/pmp\n", + "egg_pth: /home/ordonez4/miniconda3/envs/pmp_test/share/pmp\n", "output directory for graphics:demo_output/basicTestEnso/ENSO_perf\n", "output directory for diagnostic_results:demo_output/basicTestEnso/ENSO_perf\n", "output directory for metrics_results:demo_output/basicTestEnso/ENSO_perf\n", "list_variables: ['pr', 'sst', 'taux']\n", "list_obs: ['AVISO-1-0', 'ERA-INT', 'GPCP-2-3', 'HadISST-1-1']\n", "PMPdriver: dict_obs readin end\n", - "Process start: Fri Apr 1 16:45:33 2022\n", + "Process start: Tue Jul 2 14:34:34 2024\n", "models: ['ACCESS1-0']\n", " ----- model: ACCESS1-0 ---------------------\n", "PMPdriver: var loop start for model ACCESS1-0\n", @@ -492,7 +549,28 @@ "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", "\u001b[93m EnsoUvcdatToolsLib AverageMeridional\u001b[0m\n", "\u001b[93m EnsoUvcdatToolsLib AverageMeridional\u001b[0m\n", - "\u001b[94m ComputeCollection: metric = BiasTauxLonRmse\u001b[0m\n", + "\u001b[94m ComputeCollection: metric = BiasTauxLonRmse\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ordonez4/miniconda3/envs/pmp_test/lib/python3.10/site-packages/cdms2/MV2.py:318: Warning: arguments order for compress function has changed\n", + "it is now: MV2.copmress(array,condition), if your code seems to not react or act wrong to a call to compress, please check this\n", + " warnings.warn(\n", + "INFO::2024-07-02 14:42::pcmdi_metrics:: Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_perf/cmip5_historical_ENSO_perf_basicTestEnso_ACCESS1-0_r1i1p1.json\n", + "2024-07-02 14:42:16,724 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_perf/cmip5_historical_ENSO_perf_basicTestEnso_ACCESS1-0_r1i1p1.json\n", + "2024-07-02 14:42:16,724 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_perf/cmip5_historical_ENSO_perf_basicTestEnso_ACCESS1-0_r1i1p1.json\n", + "INFO::2024-07-02 14:42::pcmdi_metrics:: Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_perf/cmip5_historical_ENSO_perf_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n", + "2024-07-02 14:42:29,862 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_perf/cmip5_historical_ENSO_perf_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n", + "2024-07-02 14:42:29,862 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_perf/cmip5_historical_ENSO_perf_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "\u001b[94m ComputeMetric: oneVarRMSmetric, BiasTauxLonRmse = ACCESS1-0_r1i1p1 and ERA-Interim\u001b[0m\n", "\u001b[93m EnsoUvcdatToolsLib AverageHorizontal\u001b[0m\n", "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", @@ -587,18 +665,7 @@ "\u001b[93m EnsoUvcdatToolsLib AverageMeridional\u001b[0m\n", "\u001b[93m EnsoUvcdatToolsLib AverageMeridional\u001b[0m\n", "PMPdriver: model loop end\n", - "Process end: Fri Apr 1 16:52:09 2022\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/ordonez4/miniconda3/envs/pcmdi_metrics/lib/python3.9/site-packages/cdms2/MV2.py:318: Warning: arguments order for compress function has changed\n", - "it is now: MV2.copmress(array,condition), if your code seems to not react or act wrong to a call to compress, please check this\n", - " warnings.warn(\n", - "INFO::2022-04-01 16:52::pcmdi_metrics:: Results saved to a json file: /Users/ordonez4/Documents/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_perf/cmip5_historical_ENSO_perf_basicTestEnso_ACCESS1-0_r1i1p1.json\n", - "INFO::2022-04-01 16:52::pcmdi_metrics:: Results saved to a json file: /Users/ordonez4/Documents/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_perf/cmip5_historical_ENSO_perf_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n" + "Process end: Tue Jul 2 14:42:29 2024\n" ] } ], @@ -648,7 +715,7 @@ " },\n", " \"metric\": {\n", " \"ERA-Interim\": {\n", - " \"value\": 1.107530462384606,\n", + " \"value\": 1.1075304623846065,\n", " \"value_error\": null\n", " },\n", " \"GPCPv2.3\": {\n", @@ -674,11 +741,11 @@ " },\n", " \"metric\": {\n", " \"ERA-Interim\": {\n", - " \"value\": 0.6464917771721342,\n", + " \"value\": 0.6464917771721344,\n", " \"value_error\": null\n", " },\n", " \"GPCPv2.3\": {\n", - " \"value\": 1.4165839641155153,\n", + " \"value\": 1.4165839641155156,\n", " \"value_error\": null\n", " }\n", " }\n", @@ -700,11 +767,11 @@ " },\n", " \"metric\": {\n", " \"ERA-Interim\": {\n", - " \"value\": 0.6404285425003315,\n", + " \"value\": 0.6404287493308193,\n", " \"value_error\": null\n", " },\n", " \"HadISST\": {\n", - " \"value\": 0.49054781781348783,\n", + " \"value\": 0.49054786718298193,\n", " \"value_error\": null\n", " }\n", " }\n", @@ -722,7 +789,7 @@ " },\n", " \"metric\": {\n", " \"ERA-Interim\": {\n", - " \"value\": 5.837312275359632,\n", + " \"value\": 5.8373124987935965,\n", " \"value_error\": null\n", " }\n", " }\n", @@ -782,26 +849,26 @@ " \"EnsoSeasonality\": {\n", " \"diagnostic\": {\n", " \"ACCESS1-0_r1i1p1\": {\n", - " \"value\": 1.65806080684549,\n", - " \"value_error\": 0.26592975680376785\n", + " \"value\": 1.6580607964897247,\n", + " \"value_error\": 0.26592975514284783\n", " },\n", " \"ERA-Interim\": {\n", - " \"value\": 2.0529600453533243,\n", - " \"value_error\": 0.6533381863299897\n", + " \"value\": 2.052960042006758,\n", + " \"value_error\": 0.6533381852649718\n", " },\n", " \"HadISST\": {\n", - " \"value\": 1.6666267718711019,\n", - " \"value_error\": 0.2735312618666441\n", + " \"value\": 1.666626760243124,\n", + " \"value_error\": 0.27353125995822913\n", " }\n", " },\n", " \"metric\": {\n", " \"ERA-Interim\": {\n", - " \"value\": 19.235602728930374,\n", - " \"value_error\": 38.65610570118721\n", + " \"value\": 19.235603101705838,\n", + " \"value_error\": 38.65610552276642\n", " },\n", " \"HadISST\": {\n", - " \"value\": 0.513970204378446,\n", - " \"value_error\": 32.28408174919025\n", + " \"value\": 0.5139701316298112,\n", + " \"value_error\": 32.284081772797805\n", " }\n", " }\n", " },\n", @@ -848,11 +915,11 @@ " },\n", " \"metric\": {\n", " \"ERA-Interim\": {\n", - " \"value\": 0.1640103265768218,\n", + " \"value\": 0.16401033023030476,\n", " \"value_error\": null\n", " },\n", " \"HadISST\": {\n", - " \"value\": 0.14620422999313612,\n", + " \"value\": 0.14620423327573992,\n", " \"value_error\": null\n", " }\n", " }\n", @@ -900,11 +967,11 @@ " },\n", " \"metric\": {\n", " \"ERA-Interim\": {\n", - " \"value\": 0.10604060484676657,\n", + " \"value\": 0.10604060446246283,\n", " \"value_error\": null\n", " },\n", " \"HadISST\": {\n", - " \"value\": 0.07377326633991477,\n", + " \"value\": 0.07377326537861505,\n", " \"value_error\": null\n", " }\n", " }\n", @@ -926,7 +993,7 @@ " },\n", " \"metric\": {\n", " \"ERA-Interim\": {\n", - " \"value\": 1.1594264845137485,\n", + " \"value\": 1.1594264845137487,\n", " \"value_error\": null\n", " },\n", " \"GPCPv2.3\": {\n", @@ -952,11 +1019,11 @@ " },\n", " \"metric\": {\n", " \"ERA-Interim\": {\n", - " \"value\": 1.2638515328921656,\n", + " \"value\": 1.2638515328921658,\n", " \"value_error\": null\n", " },\n", " \"GPCPv2.3\": {\n", - " \"value\": 1.4219913613495796,\n", + " \"value\": 1.4219913613495798,\n", " \"value_error\": null\n", " }\n", " }\n", @@ -978,11 +1045,11 @@ " },\n", " \"metric\": {\n", " \"ERA-Interim\": {\n", - " \"value\": 0.2880146500221596,\n", + " \"value\": 0.28801464882030564,\n", " \"value_error\": null\n", " },\n", " \"HadISST\": {\n", - " \"value\": 0.3080045188066514,\n", + " \"value\": 0.3080045183114524,\n", " \"value_error\": null\n", " }\n", " }\n", @@ -1000,7 +1067,7 @@ " },\n", " \"metric\": {\n", " \"ERA-Interim\": {\n", - " \"value\": 4.234563899898219,\n", + " \"value\": 4.234563898790571,\n", " \"value_error\": null\n", " }\n", " }\n", @@ -1040,6 +1107,15 @@ "id": "6629c5d8", "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ordonez4/miniconda3/envs/pmp_test/lib/python3.10/site-packages/cdms2/MV2.py:318: Warning: arguments order for compress function has changed\n", + "it is now: MV2.copmress(array,condition), if your code seems to not react or act wrong to a call to compress, please check this\n", + " warnings.warn(\n" + ] + }, { "name": "stdout", "output_type": "stream", @@ -1054,14 +1130,14 @@ "debug: False\n", "obs_cmor: True\n", "obs_cmor_path: demo_data/obs4MIPs_PCMDI_monthly\n", - "egg_pth: /Users/ordonez4/Library/Caches/Python-Eggs/pcmdi_metrics-v2.1.0_183_gcf9dc528-py3.9.egg-tmp/share/pmp\n", + "egg_pth: /home/ordonez4/miniconda3/envs/pmp_test/share/pmp\n", "output directory for graphics:demo_output/basicTestEnso/ENSO_tel\n", "output directory for diagnostic_results:demo_output/basicTestEnso/ENSO_tel\n", "output directory for metrics_results:demo_output/basicTestEnso/ENSO_tel\n", "list_variables: ['pr', 'sst']\n", "list_obs: ['AVISO-1-0', 'ERA-INT', 'GPCP-2-3', 'HadISST-1-1']\n", "PMPdriver: dict_obs readin end\n", - "Process start: Fri Apr 1 16:52:10 2022\n", + "Process start: Tue Jul 2 14:42:40 2024\n", "models: ['ACCESS1-0']\n", " ----- model: ACCESS1-0 ---------------------\n", "PMPdriver: var loop start for model ACCESS1-0\n", @@ -1166,7 +1242,7 @@ "\u001b[93m\n", "\u001b[93m\n", " %%%%% ----- %%%%%\n", - " ERROR File /Users/ordonez4/miniconda3/envs/pcmdi_metrics/lib/python3.9/site-packages/EnsoMetrics/EnsoUvcdatToolsLib.py, line 1208, in CheckUnits: units\n", + " ERROR File /home/ordonez4/miniconda3/envs/pmp_test/lib/python3.10/site-packages/EnsoMetrics/EnsoUvcdatToolsLib.py, line 1208, in CheckUnits: units\n", " the file says that temperature (ts) is in K but it seems unlikely ([-1e+30, 304.7203])\n", " %%%%% ----- %%%%%\n", "\u001b[0m\n", @@ -1202,7 +1278,25 @@ "\u001b[94m ComputeMetric: twoVarRMSmetric, EnsoPrMapJja = ACCESS1-0_r1i1p1 and HadISST_ERA-Interim\u001b[0m\n", "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", - "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", + "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO::2024-07-02 14:58::pcmdi_metrics:: Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_tel/cmip5_historical_ENSO_tel_basicTestEnso_ACCESS1-0_r1i1p1.json\n", + "2024-07-02 14:58:27,009 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_tel/cmip5_historical_ENSO_tel_basicTestEnso_ACCESS1-0_r1i1p1.json\n", + "2024-07-02 14:58:27,009 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_tel/cmip5_historical_ENSO_tel_basicTestEnso_ACCESS1-0_r1i1p1.json\n", + "INFO::2024-07-02 14:58::pcmdi_metrics:: Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_tel/cmip5_historical_ENSO_tel_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n", + "2024-07-02 14:58:39,971 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_tel/cmip5_historical_ENSO_tel_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n", + "2024-07-02 14:58:39,971 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_tel/cmip5_historical_ENSO_tel_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "\u001b[94m ComputeMetric: twoVarRMSmetric, EnsoPrMapJja = ACCESS1-0_r1i1p1 and HadISST_GPCPv2.3\u001b[0m\n", "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", @@ -1224,7 +1318,7 @@ "\u001b[93m\n", "\u001b[93m\n", " %%%%% ----- %%%%%\n", - " ERROR File /Users/ordonez4/miniconda3/envs/pcmdi_metrics/lib/python3.9/site-packages/EnsoMetrics/EnsoUvcdatToolsLib.py, line 1208, in CheckUnits: units\n", + " ERROR File /home/ordonez4/miniconda3/envs/pmp_test/lib/python3.10/site-packages/EnsoMetrics/EnsoUvcdatToolsLib.py, line 1208, in CheckUnits: units\n", " the file says that temperature (ts) is in K but it seems unlikely ([-1e+30, 304.7203])\n", " %%%%% ----- %%%%%\n", "\u001b[0m\n", @@ -1246,7 +1340,7 @@ "\u001b[93m\n", "\u001b[93m\n", " %%%%% ----- %%%%%\n", - " ERROR File /Users/ordonez4/miniconda3/envs/pcmdi_metrics/lib/python3.9/site-packages/EnsoMetrics/EnsoUvcdatToolsLib.py, line 1208, in CheckUnits: units\n", + " ERROR File /home/ordonez4/miniconda3/envs/pmp_test/lib/python3.10/site-packages/EnsoMetrics/EnsoUvcdatToolsLib.py, line 1208, in CheckUnits: units\n", " the file says that temperature (ts) is in K but it seems unlikely ([-1e+30, 304.7203])\n", " %%%%% ----- %%%%%\n", "\u001b[0m\n", @@ -1263,18 +1357,7 @@ "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", "PMPdriver: model loop end\n", - "Process end: Fri Apr 1 17:03:43 2022\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/ordonez4/miniconda3/envs/pcmdi_metrics/lib/python3.9/site-packages/cdms2/MV2.py:318: Warning: arguments order for compress function has changed\n", - "it is now: MV2.copmress(array,condition), if your code seems to not react or act wrong to a call to compress, please check this\n", - " warnings.warn(\n", - "INFO::2022-04-01 17:03::pcmdi_metrics:: Results saved to a json file: /Users/ordonez4/Documents/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_tel/cmip5_historical_ENSO_tel_basicTestEnso_ACCESS1-0_r1i1p1.json\n", - "INFO::2022-04-01 17:03::pcmdi_metrics:: Results saved to a json file: /Users/ordonez4/Documents/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_tel/cmip5_historical_ENSO_tel_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n" + "Process end: Tue Jul 2 14:58:39 2024\n" ] } ], @@ -1346,7 +1429,7 @@ "debug: False\n", "obs_cmor: True\n", "obs_cmor_path: demo_data/obs4MIPs_PCMDI_monthly\n", - "egg_pth: /Users/ordonez4/Library/Caches/Python-Eggs/pcmdi_metrics-v2.1.0_183_gcf9dc528-py3.9.egg-tmp/share/pmp\n", + "egg_pth: /home/ordonez4/miniconda3/envs/pmp_test/share/pmp\n", "output directory for graphics:demo_output/basicTestEnso/ENSO_proc\n", "output directory for diagnostic_results:demo_output/basicTestEnso/ENSO_proc\n", "output directory for metrics_results:demo_output/basicTestEnso/ENSO_proc\n", @@ -1356,7 +1439,7 @@ "\u001b[95mObservation dataset GPCP-2-3 is not given for variable thf\u001b[0m\n", "\u001b[95mObservation dataset HadISST-1-1 is not given for variable thf\u001b[0m\n", "PMPdriver: dict_obs readin end\n", - "Process start: Fri Apr 1 17:03:47 2022\n", + "Process start: Tue Jul 2 14:58:50 2024\n", "models: ['ACCESS1-0']\n", " ----- model: ACCESS1-0 ---------------------\n", "PMPdriver: var loop start for model ACCESS1-0\n", @@ -1482,7 +1565,13 @@ " \"demo_data/CMIP5_demo_data/hfss_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\",\n", " \"demo_data/CMIP5_demo_data/rlds_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\",\n", " \"demo_data/CMIP5_demo_data/rlus_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\",\n", - " \"demo_data/CMIP5_demo_data/rsds_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\",\n", + " \"demo_data/CMIP5_demo_data/rsds_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\",\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ " \"demo_data/CMIP5_demo_data/rsus_Amon_ACCESS1-0_historical_r1i1p1_185001-200512.nc\"\n", " ],\n", " \"path + filename_area\": [\n", @@ -1608,7 +1697,22 @@ "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", "\u001b[93m EnsoUvcdatToolsLib AverageMeridional\u001b[0m\n", "\u001b[93m EnsoUvcdatToolsLib AverageMeridional\u001b[0m\n", - "\u001b[94m ComputeCollection: metric = BiasTauxLonRmse\u001b[0m\n", + "\u001b[94m ComputeCollection: metric = BiasTauxLonRmse\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ordonez4/miniconda3/envs/pmp_test/lib/python3.10/site-packages/cdms2/MV2.py:318: Warning: arguments order for compress function has changed\n", + "it is now: MV2.copmress(array,condition), if your code seems to not react or act wrong to a call to compress, please check this\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ "\u001b[94m ComputeMetric: oneVarRMSmetric, BiasTauxLonRmse = ACCESS1-0_r1i1p1 and ERA-Interim\u001b[0m\n", "\u001b[93m EnsoUvcdatToolsLib AverageHorizontal\u001b[0m\n", "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", @@ -1746,23 +1850,30 @@ "\u001b[93m EnsoUvcdatToolsLib AverageMeridional\u001b[0m\n", "\u001b[94m ComputeCollection: metric = EnsoSstSkew\u001b[0m\n", "\u001b[94m ComputeMetric: oneVarmetric = ACCESS1-0_r1i1p1\u001b[0m\n", - "\u001b[94m ComputeMetric: oneVarmetric = ERA-Interim\u001b[0m\n", - "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", - "\u001b[94m ComputeMetric: oneVarmetric = HadISST\u001b[0m\n", - "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", - "PMPdriver: model loop end\n", - "Process end: Fri Apr 1 17:08:20 2022\n" + "\u001b[94m ComputeMetric: oneVarmetric = ERA-Interim\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/Users/ordonez4/miniconda3/envs/pcmdi_metrics/lib/python3.9/site-packages/cdms2/MV2.py:318: Warning: arguments order for compress function has changed\n", - "it is now: MV2.copmress(array,condition), if your code seems to not react or act wrong to a call to compress, please check this\n", - " warnings.warn(\n", - "INFO::2022-04-01 17:08::pcmdi_metrics:: Results saved to a json file: /Users/ordonez4/Documents/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_proc/cmip5_historical_ENSO_proc_basicTestEnso_ACCESS1-0_r1i1p1.json\n", - "INFO::2022-04-01 17:08::pcmdi_metrics:: Results saved to a json file: /Users/ordonez4/Documents/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_proc/cmip5_historical_ENSO_proc_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n" + "INFO::2024-07-02 15:04::pcmdi_metrics:: Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_proc/cmip5_historical_ENSO_proc_basicTestEnso_ACCESS1-0_r1i1p1.json\n", + "2024-07-02 15:04:16,643 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_proc/cmip5_historical_ENSO_proc_basicTestEnso_ACCESS1-0_r1i1p1.json\n", + "2024-07-02 15:04:16,643 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_proc/cmip5_historical_ENSO_proc_basicTestEnso_ACCESS1-0_r1i1p1.json\n", + "INFO::2024-07-02 15:04::pcmdi_metrics:: Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_proc/cmip5_historical_ENSO_proc_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n", + "2024-07-02 15:04:29,807 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_proc/cmip5_historical_ENSO_proc_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n", + "2024-07-02 15:04:29,807 [INFO]: base.py(write:443) >> Results saved to a json file: /home/ordonez4/git/pcmdi_metrics/doc/jupyter/Demo/demo_output/basicTestEnso/ENSO_proc/cmip5_historical_ENSO_proc_basicTestEnso_ACCESS1-0_r1i1p1_diveDown.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", + "\u001b[94m ComputeMetric: oneVarmetric = HadISST\u001b[0m\n", + "\u001b[93m NOTE: Estimated landmask applied\u001b[0m\n", + "PMPdriver: model loop end\n", + "Process end: Tue Jul 2 15:04:29 2024\n" ] } ], @@ -1776,9 +1887,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python [conda env:pmp_test] *", "language": "python", - "name": "python3" + "name": "conda-env-pmp_test-py" }, "language_info": { "codemirror_mode": { @@ -1790,7 +1901,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.10.14" } }, "nbformat": 4, diff --git a/docs/supporting-data.rst b/docs/supporting-data.rst index 36eedaa1f..219383102 100644 --- a/docs/supporting-data.rst +++ b/docs/supporting-data.rst @@ -19,7 +19,7 @@ A location where you want to store the demo data locally can be set: :: After you have set the location for the demo_output you can download it by entering the following: :: - import cdat_info - cdat_info.download_sample_data_files("data_files.txt", demo_data_directory) + from pcmdi_metrics.io.base import download_sample_data_files + download_sample_data_files("data_files.txt", demo_data_directory) The PMP demo data is used for multiple demos. It is ~300MB. The best way to run these demos is via Jupyter notebooks. Running this initial demo for downloading sample data also on-the-fly creates demo parameter files with the user selection of the demo_data_directory. diff --git a/pcmdi_metrics/io/base.py b/pcmdi_metrics/io/base.py index 887b72445..75f4d77c8 100755 --- a/pcmdi_metrics/io/base.py +++ b/pcmdi_metrics/io/base.py @@ -4,16 +4,20 @@ import logging import os import re +import shlex +import sys from collections import OrderedDict from collections.abc import Mapping +from datetime import datetime +from subprocess import PIPE, Popen -import cdat_info import cdms2 import cdp.cdp_io import cdutil import genutil import MV2 import numpy +import requests import xcdat import xcdat as xc @@ -33,6 +37,195 @@ except Exception: basestring = str +CONDA = os.environ.get("CONDA_PYTHON_EXE", "") +if CONDA != "": + CONDA = os.path.join(os.path.dirname(CONDA), "conda") +else: + CONDA = "conda" + + +def download_sample_data_files(files_md5, path): + """Downloads sample data from a list of files""" + if not os.path.exists(files_md5) or os.path.isdir(files_md5): + raise RuntimeError("Invalid file type for list of files: %s" % files_md5) + samples = open(files_md5).readlines() + download_url_root = samples[0].strip() + for sample in samples[1:]: + good_md5, name = sample.split() + local_filename = os.path.join(path, name) + try: + os.makedirs(os.path.dirname(local_filename)) + except BaseException: + pass + attempts = 0 + while attempts < 3: + md5 = hashlib.md5() + if os.path.exists(local_filename): + f = open(local_filename, "rb") + md5.update(f.read()) + if md5.hexdigest() == good_md5: + attempts = 5 + continue + print( + "Downloading: '%s' from '%s' in: %s" + % (name, download_url_root, local_filename) + ) + r = requests.get("%s/%s" % (download_url_root, name), stream=True) + with open(local_filename, "wb") as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter local_filename keep-alive new chunks + f.write(chunk) + md5.update(chunk) + f.close() + if md5.hexdigest() == good_md5: + attempts = 5 + else: + attempts += 1 + return + + +def populate_prov(prov, cmd, pairs, sep=None, index=1, fill_missing=False): + try: + p = Popen(shlex.split(cmd), stdout=PIPE, stderr=PIPE) + except Exception: + return + out, stde = p.communicate() + if stde.decode("utf-8") != "": + return + for strBit in out.decode("utf-8").splitlines(): + for key, value in pairs.items(): + if value in strBit: + prov[key] = strBit.split(sep)[index].strip() + if fill_missing is not False: + for k in pairs: + if k not in prov: + prov[k] = fill_missing + return + + +def generateProvenance(extra_pairs={}, history=True): + """Generates provenance info for PMP + extra_pairs is a dictionary of format: {"name_in_provenance_list" : "python_package"} + """ + prov = OrderedDict() + platform = os.uname() + platfrm = OrderedDict() + platfrm["OS"] = platform[0] + platfrm["Version"] = platform[2] + platfrm["Name"] = platform[1] + prov["platform"] = platfrm + try: + logname = os.getlogin() + except Exception: + try: + import pwd + + logname = pwd.getpwuid(os.getuid())[0] + except Exception: + try: + logname = os.environ.get("LOGNAME", "unknown") + except Exception: + logname = "unknown-loginname" + prov["userId"] = logname + prov["osAccess"] = bool(os.access("/", os.W_OK) * os.access("/", os.R_OK)) + prov["commandLine"] = " ".join(sys.argv) + prov["date"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + prov["conda"] = OrderedDict() + pairs = { + "Platform": "platform ", + "Version": "conda version ", + "IsPrivate": "conda is private ", + "envVersion": "conda-env version ", + "buildVersion": "conda-build version ", + "PythonVersion": "python version ", + "RootEnvironment": "root environment ", + "DefaultEnvironment": "default environment ", + } + populate_prov(prov["conda"], CONDA + " info", pairs, sep=":", index=-1) + pairs = { + "cdp": "cdp ", + "cdat_info": "cdat_info ", + "cdms": "cdms2 ", + "cdtime": "cdtime ", + "cdutil": "cdutil ", + "esmf": "esmf ", + "esmpy": "esmpy ", + "matplotlib": "matplotlib-base ", + "numpy": "numpy ", + "python": "python ", + "scipy": "scipy ", + "xcdat": "xcdat ", + "xarray": "xarray ", + } + # Actual environement used + p = Popen(shlex.split(CONDA + " env export"), stdout=PIPE, stderr=PIPE) + o, e = p.communicate() + prov["conda"]["yaml"] = o.decode("utf-8") + prov["packages"] = OrderedDict() + populate_prov(prov["packages"], CONDA + " list", pairs, fill_missing=None) + populate_prov(prov["packages"], CONDA + " list", extra_pairs, fill_missing=None) + # Trying to capture glxinfo + pairs = { + "vendor": "OpenGL vendor string", + "renderer": "OpenGL renderer string", + "version": "OpenGL version string", + "shading language version": "OpenGL shading language version string", + } + prov["openGL"] = OrderedDict() + populate_prov(prov["openGL"], "glxinfo", pairs, sep=":", index=-1) + prov["openGL"]["GLX"] = {"server": OrderedDict(), "client": OrderedDict()} + pairs = { + "version": "GLX version", + } + populate_prov(prov["openGL"]["GLX"], "glxinfo", pairs, sep=":", index=-1) + pairs = { + "vendor": "server glx vendor string", + "version": "server glx version string", + } + populate_prov(prov["openGL"]["GLX"]["server"], "glxinfo", pairs, sep=":", index=-1) + pairs = { + "vendor": "client glx vendor string", + "version": "client glx version string", + } + populate_prov(prov["openGL"]["GLX"]["client"], "glxinfo", pairs, sep=":", index=-1) + + prov["packages"]["PMP"] = pcmdi_metrics.version.__git_tag_describe__ + prov["packages"][ + "PMPObs" + ] = "See 'References' key below, for detailed obs provenance information." + + # Now the history if requested + if history: + session_history = "" + try: + import IPython + + profile_hist = IPython.core.history.HistoryAccessor() + session = profile_hist.get_last_session_id() + cursor = profile_hist.get_range(session) + for session_id, line, cmd in cursor.fetchall(): + session_history += "{}\n".format(cmd) + if session_history == "": # empty history + # trying to force fallback on readline + raise + except Exception: + # Fallback but does not seem to always work + import readline + + for i in range(readline.get_current_history_length()): + session_history += "{}\n".format(readline.get_history_item(i + 1)) + pass + try: + import __main__ + + with open(__main__.__file__) as f: + script = f.read() + prov["script"] = script + except Exception: + pass + prov["history"] = session_history + return prov + # Convert cdms MVs to json def MV2Json(data, dic={}, struct=None): @@ -82,21 +275,6 @@ def update_dict(d, u): return d -def generateProvenance(): - extra_pairs = { - "matplotlib": "matplotlib ", - "scipy": "scipy", - "xcdat": "xcdat", - "xarray": "xarray", - } - prov = cdat_info.generateProvenance(extra_pairs=extra_pairs) - prov["packages"]["PMP"] = pcmdi_metrics.version.__git_tag_describe__ - prov["packages"][ - "PMPObs" - ] = "See 'References' key below, for detailed obs provenance information." - return prov - - def sort_human(input_list): lst = copy.copy(input_list) @@ -235,13 +413,6 @@ def write( f.close() elif self.type == "nc": - """ - f = cdms2.open(file_name, "w") - f.write(data, *args, **kwargs) - f.metrics_git_sha1 = pcmdi_metrics.__git_sha1__ - f.uvcdat_version = cdat_info.get_version() - f.close() - """ data.to_netcdf(file_name) else: diff --git a/pcmdi_metrics/misc/scripts/get_pmp_data.py b/pcmdi_metrics/misc/scripts/get_pmp_data.py index d3f5d2200..6f9ca35a4 100644 --- a/pcmdi_metrics/misc/scripts/get_pmp_data.py +++ b/pcmdi_metrics/misc/scripts/get_pmp_data.py @@ -4,9 +4,9 @@ import os import tempfile -import cdat_info import requests +from pcmdi_metrics.io.base import download_sample_data_files from pcmdi_metrics.mean_climate.lib.pmp_parser import PMPParser @@ -60,4 +60,4 @@ def download_file(download_url_root, name, local_filename): header = f.readline().strip() version = header.split("_")[-1] pathout = os.path.join(p.output_path, version) - cdat_info.download_sample_data_files(file, path=pathout) + download_sample_data_files(file, pathout)