diff --git a/.github/workflows/binary-applications.yml b/.github/workflows/binary-applications.yml index e355220fd9..33cba3459b 100644 --- a/.github/workflows/binary-applications.yml +++ b/.github/workflows/binary-applications.yml @@ -167,20 +167,6 @@ jobs: - name: Install homebrew dependencies run: | - # There's an issue with the latest versions of homebrew and a - # pending SSL version removal from the MacOS images used by - # github actions that was causing ``brew update`` to fail with - # a nonzero exit code and break our binary builds. - # - # This is the issue we were seeing: - # https://github.com/actions/virtual-environments/issues/1864 - # - # This is the workaround (patched for our use case): - # https://github.com/actions/virtual-environments/issues/1811#issuecomment-708480190 - brew uninstall openssl@1.0.2t - brew uninstall python@2.7.17 - brew untap local/openssl - brew untap local/python2 brew update brew install pandoc diff --git a/HISTORY.rst b/HISTORY.rst index e1b82543d5..6ea127a38c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -32,19 +32,28 @@ .. :changelog: -Unreleased Changes (3.9) ------------------------- -* Annual Water Yield: - * Fixing bug that limited ``rsupply`` result when ``wyield_mn`` or - ``consump_mn`` was 0. +workbench-alpha +--------------- +* General: + * Added ``invest serve`` entry-point to the CLI. This launches a Flask app + and server on the localhost, to support the workbench. + +3.9.1 +----- +* General: + * Include logger name in the logging format. This is helpful for the cython + modules, which can't log module, function, or line number info. + +3.9.0 (2020-12-11) +------------------ * General: * Deprecating GDAL 2 and adding support for GDAL 3. * Adding function in utils.py to handle InVEST coordindate transformations. * Making InVEST compatible with Pygeoprocessing 2.0 by updating: - * ``convolve_2d()`` keyword ``ignore_nodata`` to - ``ignore_nodata_and_edges``. - * ``get_raster_info()`` / ``get_vector_info()`` keyword ``projection`` to - ``projection_wkt``. + * ``convolve_2d()`` keyword ``ignore_nodata`` to + ``ignore_nodata_and_edges``. + * ``get_raster_info()`` / ``get_vector_info()`` keyword ``projection`` + to ``projection_wkt``. * Improve consistency and context for error messages related to raster reclassification across models by using ``utils.reclassify_raster``. * Fixed bug that was causing a TypeError when certain input rasters had an @@ -65,6 +74,13 @@ Unreleased Changes (3.9) * No longer include the HTML docs or HISTORY.rst in the macOS distribution. * Bumped the ``shapely`` requirements to ``>=1.7.1`` to address a library import issue on Mac OS Big Sur. + * Fixing model local documentation links for Windows and Mac binaries. + * The InVEST binary builds now launch on Mac OS 11 "Big Sur". This was + addressed by defining the ``QT_MAC_WANTS_LAYER`` environment variable. + * Fixed the alphabetical ordering of Windows Start Menu shortcuts. +* Annual Water Yield: + * Fixing bug that limited ``rsupply`` result when ``wyield_mn`` or + ``consump_mn`` was 0. * Coastal Blue Carbon * Refactor of Coastal Blue Carbon that implements TaskGraph for task management across the model and fixes a wide range of issues with the model @@ -104,6 +120,17 @@ Unreleased Changes (3.9) 'watersheds' to match the name of the vector file (including the suffix). * Added pour point detection option as an alternative to providing an outlet features vector. +* Finfish + * Fixed a bug where the suffix input was not being used for output paths. +* Forest Carbon Edge Effect + * Fixed a broken link to the local User's Guide + * Fixed bug that was causing overflow errors to appear in the logs when + running with the sample data. + * Mask out nodata areas of the carbon map output. Now there should be no + output data outside of the input LULC rasater area. +* GLOBIO + * Fixing a bug with how the ``msa`` results were masked and operated on + that could cause bad results in the ``msa`` outputs. * Habitat Quality: * Refactor of Habitat Quality that implements TaskGraph * Threat files are now indicated in the Threat Table csv input under @@ -127,6 +154,7 @@ Unreleased Changes (3.9) value. Now, the decay does not ignore those nodata edges causing values on the edges to decay more quickly. The area of study should have adequate boundaries to account for these edge effects. + * Update default half saturation value for sample data to 0.05 from 0.1. * Seasonal Water Yield * Fixed a bug where precip or eto rasters of ``GDT_Float64`` with values greater than 32-bit would overflow to ``-inf``. @@ -136,22 +164,15 @@ Unreleased Changes (3.9) to "333" leading to high export spikes in some pixels. * Fixed an issue where sediment deposition progress logging was not progressing linearly. -* Finfish - * Fixed a bug where the suffix input was not being used for output paths. + * Fixed a task dependency bug that in rare cases could cause failure. * Urban Cooling - * Split energy savings valuation and work productivity valuation into + * Split energy savings valuation and work productivity valuation into separate UI options. * Urban Flood Risk * Changed output field names ``aff.bld`` and ``serv.blt`` to ``aff_bld`` and ``serv_blt`` respectively to fix an issue where ArcGIS would not display properly. -.. -.. -.. - Unreleased Changes - ------------------ - 3.8.9 (2020-09-15) ------------------ * Hydropower diff --git a/Makefile b/Makefile index 11a7441d90..faa6cae1b5 100644 --- a/Makefile +++ b/Makefile @@ -2,15 +2,15 @@ DATA_DIR := data GIT_SAMPLE_DATA_REPO := https://bitbucket.org/natcap/invest-sample-data.git GIT_SAMPLE_DATA_REPO_PATH := $(DATA_DIR)/invest-sample-data -GIT_SAMPLE_DATA_REPO_REV := ae3a596ca875687415095635977e6363c564c26a +GIT_SAMPLE_DATA_REPO_REV := b7a51f189315e08484b5ba997a5c1de88ab7f06d GIT_TEST_DATA_REPO := https://bitbucket.org/natcap/invest-test-data.git GIT_TEST_DATA_REPO_PATH := $(DATA_DIR)/invest-test-data -GIT_TEST_DATA_REPO_REV := 817adf2ffb68a5b5c636e5d8a08c20acd4c8ea81 +GIT_TEST_DATA_REPO_REV := 6fd5fa39cd9d81080caa7581f9acca7b9fadb7c8 GIT_UG_REPO := https://github.com/natcap/invest.users-guide GIT_UG_REPO_PATH := doc/users-guide -GIT_UG_REPO_REV := 57f5c9716709ca2185d8a1607c4ec6f4e7a630d0 +GIT_UG_REPO_REV := bbfa26dc0c9158d13d209c1bc61448a9166708da ENV = env ifeq ($(OS),Windows_NT) @@ -38,6 +38,7 @@ ifeq ($(OS),Windows_NT) .DEFAULT_GOAL := windows_installer RM_DATA_DIR := $(RMDIR) $(DATA_DIR) / := '\' + OSNAME = 'windows' else NULL := /dev/null PROGRAM_CHECK_SCRIPT := ./scripts/check_required_programs.sh @@ -56,6 +57,7 @@ else ifeq ($(shell sh -c 'uname -s 2>/dev/null || echo not'),Darwin) # mac OSX .DEFAULT_GOAL := mac_dmg + OSNAME = 'mac' else .DEFAULT_GOAL := binaries endif @@ -110,6 +112,7 @@ TEST_DATAVALIDATOR := $(PYTHON) -m pytest -vs scripts/invest-autovalidate.py # Target names. INVEST_BINARIES_DIR := $(DIST_DIR)/invest +INVEST_BINARIES_DIR_ZIP := $(OSNAME)_invest_binaries.zip APIDOCS_HTML_DIR := $(DIST_DIR)/apidocs APIDOCS_ZIP_FILE := $(DIST_DIR)/InVEST_$(VERSION)_apidocs.zip USERGUIDE_HTML_DIR := $(DIST_DIR)/userguide @@ -214,19 +217,16 @@ $(GIT_TEST_DATA_REPO_PATH): | $(DATA_DIR) fetch: $(GIT_UG_REPO_PATH) $(GIT_SAMPLE_DATA_REPO_PATH) $(GIT_TEST_DATA_REPO_PATH) -# Python environment management +# Python conda environment management env: - ifeq ($(OS),Windows_NT) - $(PYTHON) -m virtualenv --system-site-packages $(ENV) - $(BASHLIKE_SHELL_COMMAND) "$(ENV_ACTIVATE) && $(PIP) install -r requirements.txt -r requirements-gui.txt" - $(BASHLIKE_SHELL_COMMAND) "$(ENV_ACTIVATE) && $(PIP) install -I -r requirements-dev.txt" - $(BASHLIKE_SHELL_COMMAND) "$(ENV_ACTIVATE) && $(MAKE) install" - else $(PYTHON) ./scripts/convert-requirements-to-conda-yml.py requirements.txt requirements-dev.txt requirements-gui.txt > requirements-all.yml - $(CONDA) create -p $(ENV) -y -c conda-forge + $(CONDA) create -p $(ENV) -y -c conda-forge python=3.8 nomkl $(CONDA) env update -p $(ENV) --file requirements-all.yml - $(BASHLIKE_SHELL_COMMAND) "source activate ./$(ENV) && $(MAKE) install" - endif + @echo "----------------------------" + @echo "To finish the conda env install:" + @echo ">> conda activate ./$(ENV)" + @echo ">> make install" + # compatible with pip>=7.0.0 # REQUIRED: Need to remove natcap.invest.egg-info directory so recent versions @@ -333,13 +333,13 @@ $(WINDOWS_INSTALLER_FILE): $(INVEST_BINARIES_DIR) $(USERGUIDE_ZIP_FILE) build/vc makensis /DVERSION=$(VERSION) /DBINDIR=$(INVEST_BINARIES_DIR) /DARCHITECTURE=$(PYTHON_ARCH) /DFORKNAME=$(INSTALLER_NAME_FORKUSER) /DDATA_LOCATION=$(DATA_BASE_URL) installer\windows\invest_installer.nsi DMG_CONFIG_FILE := installer/darwin/dmgconf.py -mac_dmg: $(MAC_DISK_IMAGE_FILE) +mac_dmg: $(MAC_DISK_IMAGE_FILE) $(MAC_DISK_IMAGE_FILE): $(DIST_DIR) $(MAC_APPLICATION_BUNDLE) $(USERGUIDE_HTML_DIR) dmgbuild -Dinvestdir=$(MAC_APPLICATION_BUNDLE) -s $(DMG_CONFIG_FILE) "InVEST $(VERSION)" $(MAC_DISK_IMAGE_FILE) mac_app: $(MAC_APPLICATION_BUNDLE) -$(MAC_APPLICATION_BUNDLE): $(BUILD_DIR) $(INVEST_BINARIES_DIR) - ./installer/darwin/build_app_bundle.sh $(VERSION) $(INVEST_BINARIES_DIR) $(MAC_APPLICATION_BUNDLE) +$(MAC_APPLICATION_BUNDLE): $(BUILD_DIR) $(INVEST_BINARIES_DIR) $(USERGUIDE_HTML_DIR) + ./installer/darwin/build_app_bundle.sh $(VERSION) $(INVEST_BINARIES_DIR) $(USERGUIDE_HTML_DIR) $(MAC_APPLICATION_BUNDLE) mac_zipfile: $(MAC_BINARIES_ZIP_FILE) $(MAC_BINARIES_ZIP_FILE): $(DIST_DIR) $(MAC_APPLICATION_BUNDLE) $(USERGUIDE_HTML_DIR) @@ -389,6 +389,7 @@ signcode_windows: @echo "Installer was signed with signtool" deploy: + -(cd $(INVEST_BINARIES_DIR) && $(ZIP) -r ../$(INVEST_BINARIES_DIR_ZIP) .) -$(GSUTIL) -m rsync $(DIST_DIR) $(DIST_URL_BASE) -$(GSUTIL) -m rsync -r $(DIST_DIR)/data $(DIST_URL_BASE)/data -$(GSUTIL) -m rsync -r $(DIST_DIR)/userguide $(DIST_URL_BASE)/userguide diff --git a/exe/hooks/rthook.py b/exe/hooks/rthook.py index a7f43fd021..37eefaeab2 100644 --- a/exe/hooks/rthook.py +++ b/exe/hooks/rthook.py @@ -6,3 +6,9 @@ multiprocessing.freeze_support() os.environ['PROJ_LIB'] = os.path.join(sys._MEIPASS, 'proj') + +if platform.system() == 'Darwin': + # This allows Qt 5.13+ to start on Big Sur. + # See https://bugreports.qt.io/browse/QTBUG-87014 + # and https://github.com/natcap/invest/issues/384 + os.environ['QT_MAC_WANTS_LAYER'] = '1' diff --git a/installer/darwin/build_app_bundle.sh b/installer/darwin/build_app_bundle.sh index e016057e35..357378088f 100755 --- a/installer/darwin/build_app_bundle.sh +++ b/installer/darwin/build_app_bundle.sh @@ -5,7 +5,8 @@ # Arguments: # $1 = the version string to use # $2 = the path to the binary dir to package. -# $3 = the path to where the application bundle should be written. +# $3 = the path to the HTML documentation +# $4 = the path to where the application bundle should be written. # remove temp files that can get in the way tempdir=`basename $3` @@ -21,13 +22,19 @@ mkdir -p "$tempdir" # .command extension makes the scripts runnable by the user. # Shell files without the `invest_` prefix will be left alone. new_basename='InVEST' -_APPDIR="$3" +_APPDIR="$4" _MACOSDIR="$_APPDIR/Contents/MacOS" _RESOURCEDIR="$_APPDIR/Contents/Resources" +_INVEST_DIST_DIR="$_MACOSDIR/invest_dist" +_USERGUIDE_HTML_DIR="$_INVEST_DIST_DIR/documentation" CONFIG_DIR="installer/darwin" mkdir -p "${_MACOSDIR}" mkdir -p "${_RESOURCEDIR}" -cp -r "$2" "$_MACOSDIR/invest_dist" + +cp -r "$2" "$_INVEST_DIST_DIR" + +mkdir -p "${_USERGUIDE_HTML_DIR}" +cp -r "$3" "$_USERGUIDE_HTML_DIR" new_command_file="$_MACOSDIR/$new_basename" cp $CONFIG_DIR/invest.icns "$_RESOURCEDIR/invest.icns" diff --git a/installer/darwin/dmgconf.py b/installer/darwin/dmgconf.py index a3784eee42..e8ea5db26f 100644 --- a/installer/darwin/dmgconf.py +++ b/installer/darwin/dmgconf.py @@ -1,5 +1,9 @@ +# Configuration script for DMGBuild +# +# __file__ is not available within this script, so we're just assuming that +# this is being executed relative to the InVEST project root. import os -CWD = os.path.join('installer', 'darwin') +MAC_INSTALLER_DIR = os.path.join('installer', 'darwin') def get_size(start_path = '.'): total_size = 0 @@ -14,7 +18,7 @@ def get_size(start_path = '.'): print('Packaging dirname %s' % defines['investdir']) _invest_dirname = os.path.basename(defines['investdir']) -badge_icon = os.path.join(CWD, 'invest.icns') +badge_icon = os.path.join(MAC_INSTALLER_DIR, 'invest.icns') symlinks = {'Applications': '/Applications'} files = [defines['investdir']] @@ -27,8 +31,13 @@ def get_size(start_path = '.'): # Window Settings window_rect = ((100, 100), (900, 660)) -background = os.path.join(CWD, 'background.png') +background = os.path.join(MAC_INSTALLER_DIR, 'background.png') #background = 'builtin-arrow' default_view = 'icon-view' - +format = 'UDZO' +license = { + # LICENSE.txt assumed to live in the project root. + 'licenses': {'en_US': 'LICENSE.txt'}, + 'default-language': 'en_US', +} diff --git a/installer/windows/invest_installer.nsi b/installer/windows/invest_installer.nsi index 88849c186f..87bee66c12 100644 --- a/installer/windows/invest_installer.nsi +++ b/installer/windows/invest_installer.nsi @@ -337,13 +337,13 @@ Section "InVEST Tools" Section_InVEST_Tools !define COASTALBLUECARBON "${SMPATH}\Coastal Blue Carbon" CreateDirectory "${COASTALBLUECARBON}" - !insertmacro StartMenuLink "${COASTALBLUECARBON}\(1) Coastal Blue Carbon Preprocessor" "cbc_pre" - !insertmacro StartMenuLink "${COASTALBLUECARBON}\(2) Coastal Blue Carbon" "cbc" + !insertmacro StartMenuLink "${COASTALBLUECARBON}\Coastal Blue Carbon (1) Preprocessor" "cbc_pre" + !insertmacro StartMenuLink "${COASTALBLUECARBON}\Coastal Blue Carbon (2)" "cbc" !define FISHERIES "${SMPATH}\Fisheries" CreateDirectory "${FISHERIES}" - !insertmacro StartMenuLink "${FISHERIES}\(1) Fisheries" "fisheries" - !insertmacro StartMenuLink "${FISHERIES}\(2) Fisheries Habitat Scenario Tool" "fisheries_hst" + !insertmacro StartMenuLink "${FISHERIES}\Fisheries" "fisheries" + !insertmacro StartMenuLink "${FISHERIES}\Fisheries Habitat Scenario Tool" "fisheries_hst" ; Write registry keys for convenient uninstallation via add/remove programs. diff --git a/requirements-gui.txt b/requirements-gui.txt index de041e85df..f6a1b6816b 100644 --- a/requirements-gui.txt +++ b/requirements-gui.txt @@ -11,3 +11,4 @@ qtpy>1.3 # pip-only qtawesome # pip-only requests PySide2!=5.15.0 # pip-only +Flask diff --git a/requirements.txt b/requirements.txt index c9c15c4afd..564f867111 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,13 +14,13 @@ GDAL>=3.1.2 Pyro4==4.77 # pip-only -pandas>=1.0 +pandas>=1.0,<1.2.0 numpy>=1.11.0,!=1.16.0 -Rtree>=0.8.2,!=0.9.1 +Rtree>=0.8.2,!=0.9.1,<=0.9.4 # rtree/pyinstaller compatibility Shapely>=1.7.1,<2.0.0 scipy>=0.16.1,<1.5.0 # pip-only pygeoprocessing>=2.1.1 # pip-only -taskgraph[niced_processes]>=0.10.0 +taskgraph[niced_processes]>=0.10.2 # pip-only psutil>=5.6.6 chardet>=3.0.4 xlrd>=1.2.0 diff --git a/src/natcap/invest/cli.py b/src/natcap/invest/cli.py index 77ff338eb1..890381120b 100644 --- a/src/natcap/invest/cli.py +++ b/src/natcap/invest/cli.py @@ -288,7 +288,7 @@ def __call__(self, parser, namespace, values, option_string=None): def main(user_args=None): - """CLI entry point for launching InVEST runs. + """CLI entry point for launching InVEST runs and other useful utilities. This command-line interface supports two methods of launching InVEST models from the command-line: @@ -391,6 +391,12 @@ def main(user_args=None): help=('The model for which the spec should be fetched. Use "invest ' 'list" to list the available models.')) + serve_subparser = subparsers.add_parser( + 'serve', help=('Start the flask app on the localhost.')) + serve_subparser.add_argument( + '--port', type=int, default=56789, + help='Port number for the Flask server') + args = parser.parse_args(user_args) root_logger = logging.getLogger() @@ -569,6 +575,10 @@ def main(user_args=None): parser.exit(app_exitcode, 'App terminated with exit code %s\n' % app_exitcode) + if args.subcommand == 'serve': + import natcap.invest.ui_server + natcap.invest.ui_server.app.run(port=args.port) + parser.exit(0) if __name__ == '__main__': multiprocessing.freeze_support() diff --git a/src/natcap/invest/coastal_vulnerability.py b/src/natcap/invest/coastal_vulnerability.py index de232e769f..f47bf4b7a6 100644 --- a/src/natcap/invest/coastal_vulnerability.py +++ b/src/natcap/invest/coastal_vulnerability.py @@ -1676,7 +1676,7 @@ def zero_negative_values(depth_array, nodata): """Convert negative values to zero for relief.""" result_array = numpy.empty_like(depth_array) if nodata is not None: - valid_mask = depth_array != nodata + valid_mask = ~numpy.isclose(depth_array, nodata) result_array[:] = nodata result_array[valid_mask] = 0 else: @@ -2563,6 +2563,7 @@ def _aggregate_raster_values_in_radius( n_cols = band.XSize geotransform = raster.GetGeoTransform() nodata = band.GetNoDataValue() + LOGGER.debug(f'{base_raster_path} nodata value: {nodata}') # we can assume square pixels at this point because # we already warped input raster and defined square pixels @@ -2644,7 +2645,7 @@ def _aggregate_raster_values_in_radius( xoff=pixel_x, yoff=pixel_y, win_xsize=win_xsize, win_ysize=win_ysize) if nodata is not None: - mask = (array != nodata) & temp_kernel_mask + mask = ~numpy.isclose(array, nodata) & temp_kernel_mask else: mask = kernel_mask if numpy.count_nonzero(mask) > 0: diff --git a/src/natcap/invest/crop_production_regression.py b/src/natcap/invest/crop_production_regression.py index b0fc4b5c92..55c874207b 100644 --- a/src/natcap/invest/crop_production_regression.py +++ b/src/natcap/invest/crop_production_regression.py @@ -286,7 +286,7 @@ def execute(args): missing_lucodes = set(crop_lucodes).difference( set(unique_lucodes)) if len(missing_lucodes) > 0: - LOGGER.warn( + LOGGER.warning( "The following lucodes are in the landcover to crop table but " "aren't in the landcover raster: %s", missing_lucodes) diff --git a/src/natcap/invest/fisheries/fisheries_io.py b/src/natcap/invest/fisheries/fisheries_io.py index c266d97df4..9cf702d95c 100644 --- a/src/natcap/invest/fisheries/fisheries_io.py +++ b/src/natcap/invest/fisheries/fisheries_io.py @@ -79,7 +79,7 @@ def fetch_args(args, create_outputs=True): # Mig Params 'migration_dir': 'path/to/mig_dir', - 'Migration': [numpy.matrix, numpy.matrix, ...] + 'Migration': [numpy.ndarray, numpy.ndarray, ...] }, { ... # additional dictionary doesn't exist when 'do_batch' @@ -397,7 +397,7 @@ def read_migration_tables(args, class_list, region_list): Example Returns:: mig_dict = { - 'Migration': [numpy.matrix, numpy.matrix, ...] + 'Migration': [numpy.ndarray, numpy.ndarray, ...] } Note: @@ -410,29 +410,29 @@ def read_migration_tables(args, class_list, region_list): mig_dict = _parse_migration_tables(args, class_list) # Create indexed list - matrix_list = [None] * len(class_list) + array_list = [None] * len(class_list) # Map numpy.matrices to indices in list for i in range(0, len(class_list)): if class_list[i] in mig_dict.keys(): - matrix_list[i] = mig_dict[class_list[i]] + array_list[i] = mig_dict[class_list[i]] # Fill in rest with identity matrices - for i in range(0, len(matrix_list)): - if matrix_list[i] is None: - matrix_list[i] = numpy.matrix(numpy.identity(len(region_list))) + for i in range(0, len(array_list)): + if array_list[i] is None: + array_list[i] = numpy.array(numpy.identity(len(region_list))) # Check migration regions are equal across matrices - assert all((x.shape == matrix_list[0].shape for x in matrix_list)), ( + assert all((x.shape == array_list[0].shape for x in array_list)), ( "Shape of migration matrices are not equal across lifecycle classes") # Check that all migration vectors approximately sum to one if not all((numpy.allclose(vector.sum(), 1) - for matrix in matrix_list for vector in matrix)): + for array in array_list for vector in array)): LOGGER.warning("Elements in at least one migration matrices source " "vector do not sum to one") - migration_dict['Migration'] = matrix_list + migration_dict['Migration'] = array_list return migration_dict @@ -441,7 +441,7 @@ def _parse_migration_tables(args, class_list): Parses the migration tables given by user Parses all files in the given directory as migration matrices and returns a - dictionary of stages and their corresponding migration numpy matrix. If + dictionary of stages and their corresponding migration numpy array. If extra files are provided that do not match the class names, an exception will be thrown. @@ -454,8 +454,8 @@ def _parse_migration_tables(args, class_list): Example Returns:: mig_dict = { - {'stage1': numpy.matrix}, - {'stage2': numpy.matrix}, + {'stage1': numpy.ndarray}, + {'stage2': numpy.ndarray}, # ... } """ @@ -469,20 +469,20 @@ def _parse_migration_tables(args, class_list): if class_name.lower() in class_list: LOGGER.info('Parsing csv %s for class %s', mig_csv, class_name) - with open(mig_csv, 'rU') as param_file: + with open(mig_csv, 'r') as param_file: csv_reader = csv.reader(param_file) lines = [] for row in csv_reader: lines.append(row) - matrix = [] + float_array = [] for row in range(1, len(lines)): - array = [] - for entry in range(1, len(lines[row])): - array.append(float(lines[row][entry])) - matrix.append(array) + float_row = [] + for col in range(1, len(lines[row])): + float_row.append(float(lines[row][col])) + float_array.append(float_row) - Migration = numpy.matrix(matrix) + Migration = numpy.array(float_array) mig_dict[class_name] = Migration diff --git a/src/natcap/invest/fisheries/fisheries_model.py b/src/natcap/invest/fisheries/fisheries_model.py index 5a490149f7..c1cbec5fe0 100644 --- a/src/natcap/invest/fisheries/fisheries_model.py +++ b/src/natcap/invest/fisheries/fisheries_model.py @@ -329,11 +329,11 @@ def age_based_cycle_func(N_prev): for i in range(1, num_classes): N_next[i] = np.array( - [Migration[i-1].dot(x) for x in N_prev[i-1]])[:, 0, :] * S[i-1] + [Migration[i-1].dot(x) for x in N_prev[i-1]]) * S[i-1] if len(N_prev) > 1: N_next[-1] = N_next[-1] + np.array( - [Migration[-1].dot(x) for x in N_prev[-1]])[:, 0, :] * S[-1] + [Migration[-1].dot(x) for x in N_prev[-1]]) * S[-1] return N_next, spawners @@ -362,9 +362,9 @@ def stage_based_cycle_func(N_prev): N_next[0] = N_next[0] + np.array(Migration[0].dot(N_prev[0][0])) * S[0] for i in range(1, num_classes): G_comp = np.array( - [Migration[i-1].dot(x) for x in N_prev[i-1]])[:, 0, :] * G[i-1] + [Migration[i-1].dot(x) for x in N_prev[i-1]]) * G[i-1] P_comp = np.array( - [Migration[i].dot(x) for x in N_prev[i]])[:, 0, :] * P[i] + [Migration[i].dot(x) for x in N_prev[i]]) * P[i] N_next[i] = G_comp + P_comp return N_next, spawners diff --git a/src/natcap/invest/forest_carbon_edge_effect.py b/src/natcap/invest/forest_carbon_edge_effect.py index 716d9df7ac..64659727a5 100644 --- a/src/natcap/invest/forest_carbon_edge_effect.py +++ b/src/natcap/invest/forest_carbon_edge_effect.py @@ -24,13 +24,13 @@ # grid cells are 100km. Becky says 500km is a good upper bound to search DISTANCE_UPPER_BOUND = 500e3 -# helpful to have a global nodata defined for all the carbon map rasters -CARBON_MAP_NODATA = -9999 +# helpful to have a global nodata defined for the whole model +NODATA_VALUE = -1 ARGS_SPEC = { "model_name": "Forest Carbon Edge Effect Model", "module": __name__, - "userguide_html": "forest_carbon_edge_effect.html", + "userguide_html": "carbon_edge.html", "args_with_spatial_overlap": { "spatial_keys": ["aoi_vector_path", "lulc_raster_path"], }, @@ -378,7 +378,7 @@ def execute(args): func=pygeoprocessing.raster_calculator, args=(carbon_maps_band_list, combine_carbon_maps, output_file_registry['carbon_map'], gdal.GDT_Float32, - CARBON_MAP_NODATA), + NODATA_VALUE), target_path_list=[output_file_registry['carbon_map']], task_name='combine_carbon_maps') @@ -415,10 +415,10 @@ def combine_carbon_maps(*carbon_maps): nodata_mask = numpy.empty(carbon_maps[0].shape, dtype=numpy.bool) nodata_mask[:] = True for carbon_map in carbon_maps: - valid_mask = carbon_map != CARBON_MAP_NODATA + valid_mask = carbon_map != NODATA_VALUE nodata_mask &= ~valid_mask result[valid_mask] += carbon_map[valid_mask] - result[nodata_mask] = CARBON_MAP_NODATA + result[nodata_mask] = NODATA_VALUE return result @@ -550,7 +550,7 @@ def _calculate_lulc_carbon_map( is_tropical_forest = 0 if ignore_tropical_type and is_tropical_forest == 1: # if tropical forest above ground, lookup table is nodata - lucode_to_per_cell_carbon[int(lucode)] = CARBON_MAP_NODATA + lucode_to_per_cell_carbon[int(lucode)] = NODATA_VALUE else: try: lucode_to_per_cell_carbon[int(lucode)] = float( @@ -570,7 +570,7 @@ def _calculate_lulc_carbon_map( utils.reclassify_raster( (lulc_raster_path, 1), lucode_to_per_cell_carbon, - carbon_map_path, gdal.GDT_Float32, CARBON_MAP_NODATA, + carbon_map_path, gdal.GDT_Float32, NODATA_VALUE, reclass_error_details) @@ -607,15 +607,23 @@ def _map_distance_from_tropical_forest_edge( if int(ludata['is_tropical_forest']) == 1] # Make a raster where 1 is non-forest landcover types and 0 is forest - forest_mask_nodata = 255 lulc_nodata = pygeoprocessing.get_raster_info( base_lulc_raster_path)['nodata'] + forest_mask_nodata = 255 def mask_non_forest_op(lulc_array): - """Converts forest lulc codes to 1.""" - non_forest_mask = ~numpy.in1d( - lulc_array.flatten(), forest_codes).reshape(lulc_array.shape) + """Convert forest lulc codes to 0. + Args: + lulc_array (numpy.ndarray): array representing a LULC raster where + each forest LULC code is in `forest_codes`. + Returns: + numpy.ndarray with the same shape as lulc_array. All pixels are + 0 (forest), 1 (non-forest), or 255 (nodata). + """ + non_forest_mask = ~numpy.isin(lulc_array, forest_codes) nodata_mask = lulc_array == lulc_nodata + # where LULC has nodata, set value to nodata value (255) + # where LULC has data, set to 0 if LULC is a forest type, 1 if it's not return numpy.where(nodata_mask, forest_mask_nodata, non_forest_mask) pygeoprocessing.raster_calculator( @@ -623,9 +631,30 @@ def mask_non_forest_op(lulc_array): target_non_forest_mask_path, gdal.GDT_Byte, forest_mask_nodata) # Do the distance transform on non-forest pixels + # This is the distance from each pixel to the nearest pixel with value 1. + # - for forest pixels, this is the distance to the forest edge + # - for non-forest pixels, this is 0 + # - for nodata pixels, distance is calculated but is meaningless pygeoprocessing.distance_transform_edt( (target_non_forest_mask_path, 1), edge_distance_path) + # mask out the meaningless distance pixels so they don't affect the output + lulc_raster = gdal.OpenEx(base_lulc_raster_path) + lulc_band = lulc_raster.GetRasterBand(1) + edge_distance_raster = gdal.OpenEx(edge_distance_path, gdal.GA_Update) + edge_distance_band = edge_distance_raster.GetRasterBand(1) + + for offset_dict in pygeoprocessing.iterblocks((base_lulc_raster_path, 1), offset_only=True): + # where LULC has nodata, overwrite edge distance with nodata value + lulc_block = lulc_band.ReadAsArray(**offset_dict) + distance_block = edge_distance_band.ReadAsArray(**offset_dict) + masked_distance_block = numpy.where( + lulc_block == lulc_nodata, NODATA_VALUE, distance_block) + edge_distance_band.WriteArray( + masked_distance_block, + xoff=offset_dict['xoff'], + yoff=offset_dict['yoff']) + def _build_spatial_index( base_raster_path, local_model_dir, @@ -734,7 +763,11 @@ def _calculate_tropical_forest_edge_carbon_map( None """ - # load spatial indeces from pickle file + # load spatial indices from pickle file + # let d = number of precalculated model cells (2217 for sample data) + # kd_tree.data.shape: (d, 2) + # theta_model_parameters.shape: (d, 3) + # method_model_parameter.shape: (d,) kd_tree, theta_model_parameters, method_model_parameter = pickle.load( open(spatial_index_pickle_path, 'rb')) @@ -742,8 +775,8 @@ def _calculate_tropical_forest_edge_carbon_map( # fill nodata, in case we skip entire memory blocks that are non-forest pygeoprocessing.new_raster_from_base( edge_distance_path, tropical_forest_edge_carbon_map_path, - gdal.GDT_Float32, band_nodata_list=[CARBON_MAP_NODATA], - fill_value_list=[CARBON_MAP_NODATA]) + gdal.GDT_Float32, band_nodata_list=[NODATA_VALUE], + fill_value_list=[NODATA_VALUE]) edge_carbon_raster = gdal.OpenEx( tropical_forest_edge_carbon_map_path, gdal.GA_Update) edge_carbon_band = edge_carbon_raster.GetRasterBand(1) @@ -774,6 +807,7 @@ def _calculate_tropical_forest_edge_carbon_map( last_time = current_time n_cells_processed += ( edge_distance_data['win_xsize'] * edge_distance_data['win_ysize']) + # only forest pixels will have an edge distance > 0 valid_edge_distance_mask = (edge_distance_block > 0) # if no valid forest pixels to calculate, skip to the next block @@ -804,6 +838,8 @@ def _calculate_tropical_forest_edge_carbon_map( row_coords[valid_edge_distance_mask].ravel(), col_coords[valid_edge_distance_mask].ravel())) # note, the 'n_jobs' parameter was introduced in SciPy 0.16.0 + # for each forest point x, for each of its k nearest neighbors + # shape of distances and indexes: (x, k) distances, indexes = kd_tree.query( coord_points, k=n_nearest_model_points, distance_upper_bound=DISTANCE_UPPER_BOUND, n_jobs=-1) @@ -812,51 +848,50 @@ def _calculate_tropical_forest_edge_carbon_map( distances = distances.reshape(distances.shape[0], 1) indexes = indexes.reshape(indexes.shape[0], 1) - # the 3 is for the 3 thetas in the carbon model + # 3 is for the 3 thetas in the carbon model. thetas shape: (x, k, 3) thetas = numpy.zeros((indexes.shape[0], indexes.shape[1], 3)) valid_index_mask = (indexes != kd_tree.n) thetas[valid_index_mask] = theta_model_parameters[ indexes[valid_index_mask]] - # the 3 is for the 3 models (asym, exp, linear) - biomass_model = numpy.zeros( - (indexes.shape[0], indexes.shape[1], 3)) # reshape to an N,nearest_points so we can multiply by thetas valid_edge_distances_km = numpy.repeat( edge_distance_block[valid_edge_distance_mask] * cell_size_km, n_nearest_model_points).reshape(-1, n_nearest_model_points) - # asymptotic model + # For each forest pixel x, for each of its k nearest neighbors, the + # chosen regression method (1, 2, or 3). model_index shape: (x, k) + model_index = numpy.zeros(indexes.shape, dtype=numpy.int8) + model_index[valid_index_mask] = ( + method_model_parameter[indexes[valid_index_mask]]) + + # biomass shape: (x, k) + biomass = numpy.zeros((indexes.shape[0], indexes.shape[1]), + dtype=numpy.float32) + + # mask shapes: (x, k) + mask_1 = model_index == 1 + mask_2 = model_index == 2 + mask_3 = model_index == 3 + + # exponential model # biomass_1 = t1 - t2 * exp(-t3 * edge_dist_km) - biomass_model[:, :, 0] = ( - thetas[:, :, 0] - thetas[:, :, 1] * numpy.exp( - -thetas[:, :, 2] * valid_edge_distances_km) + biomass[mask_1] = ( + thetas[mask_1][:,0] - thetas[mask_1][:,1] * numpy.exp( + -thetas[mask_1][:,2] * valid_edge_distances_km[mask_1]) ) * cell_area_ha # logarithmic model # biomass_2 = t1 + t2 * numpy.log(edge_dist_km) - biomass_model[:, :, 1] = ( - thetas[:, :, 0] + thetas[:, :, 1] * numpy.log( - valid_edge_distances_km)) * cell_area_ha + biomass[mask_2] = ( + thetas[mask_2][:,0] + thetas[mask_2][:,1] * numpy.log( + valid_edge_distances_km[mask_2])) * cell_area_ha # linear regression # biomass_3 = t1 + t2 * edge_dist_km - biomass_model[:, :, 2] = ( - (thetas[:, :, 0] + thetas[:, :, 1] * valid_edge_distances_km) * - cell_area_ha) - - # Collapse the biomass down to the valid models - model_index = numpy.zeros(indexes.shape, dtype=numpy.int8) - model_index[valid_index_mask] = ( - method_model_parameter[indexes[valid_index_mask]] - 1) - - # reduce the axis=1 dimensionality of the model by selecting the - # appropriate value via the model_index array. Got this trick from - # http://stackoverflow.com/questions/18702746/reduce-a-dimension-of-numpy-array-by-selecting - biomass_y, biomass_x = numpy.meshgrid( - numpy.arange(biomass_model.shape[1]), - numpy.arange(biomass_model.shape[0])) - biomass = biomass_model[biomass_x, biomass_y, model_index] + biomass[mask_3] = ( + thetas[mask_3][:,0] + thetas[mask_3][:,1] * + valid_edge_distances_km[mask_3]) * cell_area_ha # reshape the array so that each set of points is in a separate # dimension, here distances are distances to each valid model @@ -876,9 +911,8 @@ def _calculate_tropical_forest_edge_carbon_map( biomass[valid_denom], axis=1) / denom[valid_denom]) # Ensure the result has nodata everywhere the distance was invalid - result = numpy.empty( - edge_distance_block.shape, dtype=numpy.float32) - result[:] = CARBON_MAP_NODATA + result = numpy.full(edge_distance_block.shape, NODATA_VALUE, + dtype=numpy.float32) # convert biomass to carbon in this stage result[valid_edge_distance_mask] = ( average_biomass * biomass_to_carbon_conversion_factor) diff --git a/src/natcap/invest/globio.py b/src/natcap/invest/globio.py index d66b6b9f72..851250bbf9 100644 --- a/src/natcap/invest/globio.py +++ b/src/natcap/invest/globio.py @@ -338,10 +338,8 @@ def execute(args): msa_f_path = os.path.join(output_dir, 'msa_f%s.tif' % file_suffix) calculate_msa_f_task = task_graph.add_task( - func=pygeoprocessing.raster_calculator, - args=([(primary_veg_smooth_path, 1), (primary_veg_mask_nodata, 'raw'), - (msa_f_table, 'raw'), (msa_nodata, 'raw')], - _msa_f_op, msa_f_path, gdal.GDT_Float32, msa_nodata), + func=_msa_f_calculation, + args=(primary_veg_smooth_path, msa_f_table, msa_f_path, msa_nodata), target_path_list=[msa_f_path], dependent_task_list=[smooth_primary_veg_task], task_name='calculate_msa_f') @@ -363,11 +361,10 @@ def execute(args): LOGGER.info('calculate msa_i') msa_i_path = os.path.join(output_dir, 'msa_i%s.tif' % file_suffix) calculate_msa_i_task = task_graph.add_task( - func=pygeoprocessing.raster_calculator, - args=([(globio_lulc_path, 1), (distance_to_infrastructure_path, 1), - (out_pixel_size, 'raw'), (msa_i_primary_table, 'raw'), - (msa_i_other_table, 'raw')], - _msa_i_op, msa_i_path, gdal.GDT_Float32, msa_nodata), + func=_msa_i_calculation, + args=(globio_lulc_path, distance_to_infrastructure_path, + out_pixel_size, msa_i_primary_table, msa_i_other_table, + msa_i_path, msa_nodata), target_path_list=[msa_i_path], dependent_task_list=[distance_to_infrastructure_task], task_name='calculate_msa_i') @@ -392,10 +389,8 @@ def execute(args): msa_path = os.path.join( output_dir, 'msa%s.tif' % file_suffix) calculate_msa_task = task_graph.add_task( - func=pygeoprocessing.raster_calculator, - args=([(msa_f_path, 1), (msa_lu_path, 1), (msa_i_path, 1), - (globio_nodata, 'raw')], - _msa_op, msa_path, gdal.GDT_Float32, msa_nodata), + func=_msa_calculation, + args=(msa_f_path, msa_lu_path, msa_i_path, msa_path, msa_nodata), target_path_list=[msa_path], dependent_task_list=[ calculate_msa_f_task, calculate_msa_i_task, calculate_msa_lu_task], @@ -485,56 +480,75 @@ def _ffqi_op(forest_areas_array, smoothed_forest_areas, forest_areas_nodata): return result -def _msa_f_op( - primary_veg_smooth, primary_veg_mask_nodata, msa_f_table, - msa_nodata): +def _msa_f_calculation( + primary_veg_smooth_path, msa_f_table, msa_f_path, msa_nodata): """Calculate msa fragmentation. Bin ffqi values based on rules defined in msa_parameters.csv. Args: - primary_veg_smooth (array): float values representing ffqi. - primary_veg_mask_nodata (int/float) + primary_veg_smooth (str): path to a raster with float values + representing ffqi. msa_f_table (dict): subset of msa_parameters.csv with fragmentation bins defined. - msa_nodata (int/float) + msa_nodata (int/float): output nodata value Returns: - Array with float values. One component of final MSA score. - + Nothing """ - msa_f = numpy.empty(primary_veg_smooth.shape) - - less_than = msa_f_table.pop('<', None) - greater_than = msa_f_table.pop('>', None) - if greater_than: - msa_f[primary_veg_smooth > greater_than[0]] = ( - greater_than[1]) - for key in reversed(sorted(msa_f_table)): - msa_f[primary_veg_smooth <= key] = msa_f_table[key] - if less_than: - msa_f[primary_veg_smooth < less_than[0]] = ( - less_than[1]) - - if msa_nodata is not None: - nodata_mask = numpy.isclose(primary_veg_smooth, - primary_veg_mask_nodata) - msa_f[nodata_mask] = msa_nodata - - return msa_f - - -def _msa_i_op( - lulc_array, distance_to_infrastructure, out_pixel_size, - msa_i_primary_table, msa_i_other_table): + primary_veg_info = pygeoprocessing.get_raster_info(primary_veg_smooth_path) + primary_veg_mask_nodata = primary_veg_info['nodata'][0] + + msa_f_table_copy = msa_f_table.copy() + less_than = msa_f_table_copy.pop('<', None) + greater_than = msa_f_table_copy.pop('>', None) + + def msa_f_op(primary_veg_smooth): + """Calculate msa fragmentation. + + Bin ffqi values based on rules defined in msa_parameters.csv. + + Args: + primary_veg_smooth (array): float values representing ffqi. + + Returns: + Array with float values. One component of final MSA score. + """ + msa_f = numpy.full_like( + primary_veg_smooth, msa_nodata, dtype=numpy.float32) + + if greater_than: + msa_f[primary_veg_smooth > greater_than[0]] = ( + greater_than[1]) + for key in reversed(sorted(msa_f_table_copy)): + msa_f[primary_veg_smooth <= key] = msa_f_table_copy[key] + if less_than: + msa_f[primary_veg_smooth < less_than[0]] = ( + less_than[1]) + + if msa_nodata is not None: + nodata_mask = numpy.isclose( + primary_veg_smooth, primary_veg_mask_nodata) + msa_f[nodata_mask] = msa_nodata + + return msa_f + + pygeoprocessing.raster_calculator( + [(primary_veg_smooth_path, 1)], msa_f_op, msa_f_path, gdal.GDT_Float32, + msa_nodata) + + +def _msa_i_calculation( + globio_lulc_path, distance_to_infrastructure_path, out_pixel_size, + msa_i_primary_table, msa_i_other_table, msa_i_path, msa_nodata): """Calculate msa infrastructure. Bin distance_to_infrastructure values according to rules defined in msa_parameters.csv. Args: - lulc_array (array): integer values representing globio landcover codes. - distance_to_infrastructure (array): + globio_lulc_path (str): path to raster with globio landcover codes. + distance_to_infrastructure_path (str): path to raster with float values measuring distance from nearest infrastructure present in layers from args['infrastructure_dir']. out_pixel_size (float): from the globio lulc raster info. @@ -544,54 +558,109 @@ def _msa_i_op( msa_i_other_table (dict): subset of msa_parameters.csv with distance to infrastructure bins defined. These bins are applied to areas of not primary veg. + msa_i_path (str): output path for msa infrastructure raster. + msa_nodata (float): output nodata value. Returns: - Array with float values. One component of final MSA score. + Nothing. + """ + lulc_info = pygeoprocessing.get_raster_info(globio_lulc_path) + lulc_nodata = lulc_info['nodata'][0] + + # Create a copy of the dictionary so we don't mutate it outside this scope + msa_i_primary_table_copy = msa_i_primary_table.copy() + msa_i_other_table_copy = msa_i_other_table.copy() + + primary_less_than = msa_i_primary_table_copy.pop('<', None) + primary_greater_than = msa_i_primary_table_copy.pop('>', None) + other_less_than = msa_i_other_table_copy.pop('<', None) + other_greater_than = msa_i_other_table_copy.pop('>', None) + + def msa_i_op(lulc_array, distance_to_infrastructure): + """Calculate msa infrastructure. + + Args: + lulc_array (array): integer values representing globio landcover + codes. + distance_to_infrastructure (array): float values measuring + distance from nearest infrastructure present in layers from + args['infrastructure_dir']. + + Returns: + Array with float values. One component of final MSA score. + """ + distance_to_infrastructure *= out_pixel_size # convert to meters + # Use `full_like` with `msa_nodata` value because we can't know if + # entire range of values will be covered from msa tables + msa_i_primary = numpy.full_like( + lulc_array, msa_nodata, dtype=numpy.float32) + msa_i_other = numpy.full_like( + lulc_array, msa_nodata, dtype=numpy.float32) + nodata_mask = numpy.isclose(lulc_array, lulc_nodata) + + if primary_greater_than: + msa_i_primary[distance_to_infrastructure > primary_greater_than[0]] = ( + primary_greater_than[1]) + for key in reversed(sorted(msa_i_primary_table_copy)): + msa_i_primary[distance_to_infrastructure <= key] = ( + msa_i_primary_table_copy[key]) + if primary_less_than: + msa_i_primary[distance_to_infrastructure < primary_less_than[0]] = ( + primary_less_than[1]) + + if other_greater_than: + msa_i_other[distance_to_infrastructure > other_greater_than[0]] = ( + other_greater_than[1]) + for key in reversed(sorted(msa_i_other_table_copy)): + msa_i_other[distance_to_infrastructure <= key] = ( + msa_i_other_table_copy[key]) + if other_less_than: + msa_i_other[distance_to_infrastructure < other_less_than[0]] = ( + other_less_than[1]) + + # lulc code 1 is primary veg + msa_i = numpy.where(lulc_array == 1, msa_i_primary, msa_i_other) + msa_i[nodata_mask] = msa_nodata + return msa_i + + pygeoprocessing.raster_calculator( + [(globio_lulc_path, 1), (distance_to_infrastructure_path, 1)], + msa_i_op, msa_i_path, gdal.GDT_Float32, msa_nodata) + +def _msa_calculation( + msa_f_path, msa_lu_path, msa_i_path, msa_path, msa_nodata): + """Calculate the MSA which is the product of the sub MSAs. + + Args: + msa_f_path (str): path to the msa_f raster. + msa_lu_path (str): path to the msa_lu raster. + msa_i_path (str): path to the msa_i raster. + msa_path (str): path to the output MSA raster. + msa_nodata (int/float): the output nodata value. + + Returns: + Nothing """ - distance_to_infrastructure *= out_pixel_size # convert to meters - msa_i_primary = numpy.empty(lulc_array.shape) - msa_i_other = numpy.empty(lulc_array.shape) - - primary_less_than = msa_i_primary_table.pop('<', None) - primary_greater_than = msa_i_primary_table.pop('>', None) - if primary_greater_than: - msa_i_primary[distance_to_infrastructure > primary_greater_than[0]] = ( - primary_greater_than[1]) - for key in reversed(sorted(msa_i_primary_table)): - msa_i_primary[distance_to_infrastructure <= key] = ( - msa_i_primary_table[key]) - if primary_less_than: - msa_i_primary[distance_to_infrastructure < primary_less_than[0]] = ( - primary_less_than[1]) - - other_less_than = msa_i_other_table.pop('<', None) - other_greater_than = msa_i_other_table.pop('>', None) - if other_greater_than: - msa_i_other[distance_to_infrastructure > other_greater_than[0]] = ( - other_greater_than[1]) - for key in reversed(sorted(msa_i_other_table)): - msa_i_other[distance_to_infrastructure <= key] = ( - msa_i_other_table[key]) - if other_less_than: - msa_i_other[distance_to_infrastructure < other_less_than[0]] = ( - other_less_than[1]) - - # lulc code 1 is primary veg - msa_i = numpy.where(lulc_array == 1, msa_i_primary, msa_i_other) - return msa_i - - -def _msa_op(msa_f, msa_lu, msa_i, globio_nodata): - """Calculate the MSA which is the product of the sub MSAs.""" - result = numpy.empty_like(msa_f, dtype=numpy.float32) - result[:] = globio_nodata - valid_mask = slice(None) - if globio_nodata is not None: - valid_mask = ~numpy.isclose(msa_f, globio_nodata) - result[valid_mask] = ( - msa_f[valid_mask] * msa_lu[valid_mask] * msa_i[valid_mask]) - return result + msa_f_nodata = pygeoprocessing.get_raster_info(msa_f_path)['nodata'][0] + msa_lu_nodata = pygeoprocessing.get_raster_info(msa_lu_path)['nodata'][0] + msa_i_nodata = pygeoprocessing.get_raster_info(msa_i_path)['nodata'][0] + nodata_array = [msa_f_nodata, msa_lu_nodata, msa_i_nodata] + + def msa_op(msa_f, msa_lu, msa_i): + """Calculate the MSA which is the product of the sub MSAs.""" + result = numpy.full_like(msa_f, msa_nodata, dtype=numpy.float32) + valid_mask = numpy.ones(msa_f.shape, dtype=numpy.bool) + for msa_array, nodata_val in zip([msa_f, msa_lu, msa_i], nodata_array): + if nodata_val is not None: + valid_mask &= ~numpy.isclose(msa_array, nodata_val) + result[valid_mask] = ( + msa_f[valid_mask] * msa_lu[valid_mask] * msa_i[valid_mask]) + return result + + pygeoprocessing.raster_calculator( + [(msa_f_path, 1), (msa_lu_path, 1), (msa_i_path, 1)], + msa_op, msa_path, gdal.GDT_Float32, msa_nodata) def make_gaussian_kernel_path(sigma, kernel_path): diff --git a/src/natcap/invest/habitat_quality.py b/src/natcap/invest/habitat_quality.py index a3a869d688..36d8c8b347 100644 --- a/src/natcap/invest/habitat_quality.py +++ b/src/natcap/invest/habitat_quality.py @@ -180,7 +180,7 @@ "type": "number", "required": True, "about": ( - "A positive floating point value that is defaulted at 0.5. " + "A positive floating point value that is defaulted at 0.05. " "This is the value of the parameter k in equation (4). In " "general, set k to half of the highest grid cell degradation " "value on the landscape. To perform this model calibration " diff --git a/src/natcap/invest/recreation/recmodel_client.py b/src/natcap/invest/recreation/recmodel_client.py index cfd598c757..267bce3f45 100644 --- a/src/natcap/invest/recreation/recmodel_client.py +++ b/src/natcap/invest/recreation/recmodel_client.py @@ -1276,7 +1276,7 @@ def _build_regression( n_features = data_matrix.shape[0] y_factors = data_matrix[:, 0] # useful to have this as a 1-D array coefficients, _, _, _ = numpy.linalg.lstsq( - data_matrix[:, 1:], y_factors) + data_matrix[:, 1:], y_factors, rcond=-1) ssres = numpy.sum(( y_factors - diff --git a/src/natcap/invest/recreation/recmodel_server.py b/src/natcap/invest/recreation/recmodel_server.py index 2fcebaa05e..0062181e52 100644 --- a/src/natcap/invest/recreation/recmodel_server.py +++ b/src/natcap/invest/recreation/recmodel_server.py @@ -13,7 +13,7 @@ import collections import logging import queue -from io import StringIO +from io import BytesIO, StringIO import Pyro4 import numpy @@ -45,6 +45,32 @@ LOGGER = logging.getLogger('natcap.invest.recreation.recmodel_server') +def _numpy_dumps(numpy_array): + """Safely pickle numpy array to string. + Args: + numpy_array (numpy.ndarray): arbitrary numpy array. + Returns: + A string representation of the array that can be loaded using + `numpy_loads. + """ + with BytesIO() as file_stream: + numpy.save(file_stream, numpy_array, allow_pickle=False) + return file_stream.getvalue() + + +def _numpy_loads(queue_string): + """Safely unpickle string to numpy array. + + Args: + queue_string (str): binary string representing a pickled + numpy array. + Returns: + A numpy representation of ``binary_numpy_string``. + """ + with BytesIO(queue_string) as file_stream: + return numpy.load(file_stream) + + def _try_except_wrapper(mesg): """Wrap the function in a try/except to log exception before failing. @@ -490,7 +516,10 @@ def md5hash(user_string): user_day_lng_lat['f1'] = hashes user_day_lng_lat['f2'] = result['lng'] user_day_lng_lat['f3'] = result['lat'] - numpy_array_queue.put(user_day_lng_lat) + # multiprocessing.Queue pickles the array. Pickling isn't perfect and + # it modifies the `datetime64` dtype metadata, causing a warning later. + # To avoid this we dump the array to a string before adding to queue. + numpy_array_queue.put(_numpy_dumps(user_day_lng_lat)) numpy_array_queue.put('STOP') @@ -589,13 +618,15 @@ def _populate_offset_queue(block_offset_size_queue): n_points = 0 while True: - point_array = numpy_array_queue.get() - if (isinstance(point_array, str) and - point_array == 'STOP'): # count 'n cpu' STOPs + payload = numpy_array_queue.get() + # if the item is a 'STOP' sentinel, don't load as an array + if payload == 'STOP': n_parse_processes -= 1 if n_parse_processes == 0: break continue + else: + point_array = _numpy_loads(payload) n_points += len(point_array) ooc_qt.add_points(point_array, 0, point_array.size) diff --git a/src/natcap/invest/reporting/html.py b/src/natcap/invest/reporting/html.py index 5fcb3de46b..c115542e58 100644 --- a/src/natcap/invest/reporting/html.py +++ b/src/natcap/invest/reporting/html.py @@ -217,10 +217,10 @@ def cell_format(data): """Formats the data to put in a table cell.""" if isinstance(data, int): # Add commas to integers. - return locale.format("%d", data, grouping=True) + return locale.format_string("%d", data, grouping=True) elif isinstance(data, float): # Add commas to floats, and round to 2 decimal places. - return locale.format("%.2f", data, grouping=True) + return locale.format_string("%.2f", data, grouping=True) else: return str(data) diff --git a/src/natcap/invest/sdr/sdr.py b/src/natcap/invest/sdr/sdr.py index 983b0a4d62..8662e5b78a 100644 --- a/src/natcap/invest/sdr/sdr.py +++ b/src/natcap/invest/sdr/sdr.py @@ -487,7 +487,7 @@ def execute(args): copy_duplicate_artifact=True, target_path_list=[f_reg['rkls_path']], dependent_task_list=[ - align_task, drainage_raster_path_task[1]], + align_task, drainage_raster_path_task[1], ls_factor_task], task_name='calculate RKLS') usle_task = task_graph.add_task( diff --git a/src/natcap/invest/ui/forest_carbon.py b/src/natcap/invest/ui/forest_carbon.py index b5ffc91744..cd8153f569 100644 --- a/src/natcap/invest/ui/forest_carbon.py +++ b/src/natcap/invest/ui/forest_carbon.py @@ -11,7 +11,7 @@ def __init__(self): label='Forest Carbon Edge Effect Model', target=natcap.invest.forest_carbon_edge_effect.execute, validator=natcap.invest.forest_carbon_edge_effect.validate, - localdoc='forest_carbon_edge_effect.html') + localdoc=natcap.invest.forest_carbon_edge_effect.ARGS_SPEC['userguide_html']) self.lulc_raster_path = inputs.File( args_key='lulc_raster_path', diff --git a/src/natcap/invest/ui/globio.py b/src/natcap/invest/ui/globio.py index b718ec2eb6..0ae83c57a9 100644 --- a/src/natcap/invest/ui/globio.py +++ b/src/natcap/invest/ui/globio.py @@ -11,7 +11,7 @@ def __init__(self): label='GLOBIO', target=natcap.invest.globio.execute, validator=natcap.invest.globio.validate, - localdoc='../documentation/globio.html') + localdoc='globio.html') self.lulc_to_globio_table_path = inputs.File( args_key='lulc_to_globio_table_path', diff --git a/src/natcap/invest/ui/habitat_quality.py b/src/natcap/invest/ui/habitat_quality.py index ed5c8a0590..2a964d1edf 100644 --- a/src/natcap/invest/ui/habitat_quality.py +++ b/src/natcap/invest/ui/habitat_quality.py @@ -11,7 +11,7 @@ def __init__(self): label='Habitat Quality', target=natcap.invest.habitat_quality.execute, validator=natcap.invest.habitat_quality.validate, - localdoc='../documentation/habitat_quality.html') + localdoc='habitat_quality.html') self.current_landcover = inputs.File( args_key='lulc_cur_path', helptext=( @@ -148,7 +148,7 @@ def __init__(self): args_key='half_saturation_constant', helptext=( "A positive floating point value that is defaulted at " - "0.5. This is the value of the parameter k in equation " + "0.05. This is the value of the parameter k in equation " "(4). In general, set k to half of the highest grid " "cell degradation value on the landscape. To perform " "this model calibration the model must be run once in " diff --git a/src/natcap/invest/ui/hra.py b/src/natcap/invest/ui/hra.py index 6dbaff4e37..d84e5b72b6 100644 --- a/src/natcap/invest/ui/hra.py +++ b/src/natcap/invest/ui/hra.py @@ -10,7 +10,7 @@ def __init__(self): label='Habitat Risk Assessment', target=hra.execute, validator=hra.validate, - localdoc='../documentation/habitat_risk_assessment.html') + localdoc='habitat_risk_assessment.html') self.info_table_path = inputs.File( args_key='info_table_path', diff --git a/src/natcap/invest/ui/hydropower.py b/src/natcap/invest/ui/hydropower.py index ee7a6630d0..b1c42cec00 100644 --- a/src/natcap/invest/ui/hydropower.py +++ b/src/natcap/invest/ui/hydropower.py @@ -11,7 +11,7 @@ def __init__(self): label='Hydropower Water Yield', target=hydropower_water_yield.execute, validator=hydropower_water_yield.validate, - localdoc='../documentation/reservoirhydropowerproduction.html') + localdoc='reservoirhydropowerproduction.html') self.precipitation = inputs.File( args_key='precipitation_path', diff --git a/src/natcap/invest/ui/ndr.py b/src/natcap/invest/ui/ndr.py index b996018da7..f21dcc6105 100644 --- a/src/natcap/invest/ui/ndr.py +++ b/src/natcap/invest/ui/ndr.py @@ -11,7 +11,7 @@ def __init__(self): label='Nutrient Delivery Ratio Model (NDR)', target=natcap.invest.ndr.ndr.execute, validator=natcap.invest.ndr.ndr.validate, - localdoc='waterpurification.html') + localdoc='ndr.html') self.dem_path = inputs.File( args_key='dem_path', diff --git a/src/natcap/invest/ui/sdr.py b/src/natcap/invest/ui/sdr.py index b0b5aea6b7..ddc7f2c06b 100644 --- a/src/natcap/invest/ui/sdr.py +++ b/src/natcap/invest/ui/sdr.py @@ -11,7 +11,7 @@ def __init__(self): label='Sediment Delivery Ratio Model (SDR)', target=natcap.invest.sdr.sdr.execute, validator=natcap.invest.sdr.sdr.validate, - localdoc='../documentation/sdr.html') + localdoc='sdr.html') self.dem_path = inputs.File( args_key='dem_path', helptext=( diff --git a/src/natcap/invest/ui/urban_flood_risk_mitigation.py b/src/natcap/invest/ui/urban_flood_risk_mitigation.py index 578cd26a18..5ebea9a926 100644 --- a/src/natcap/invest/ui/urban_flood_risk_mitigation.py +++ b/src/natcap/invest/ui/urban_flood_risk_mitigation.py @@ -11,7 +11,7 @@ def __init__(self): label='UrbanFloodRiskMitigation', target=natcap.invest.urban_flood_risk_mitigation.execute, validator=natcap.invest.urban_flood_risk_mitigation.validate, - localdoc='../documentation/urban_flood_risk_mitigation.html') + localdoc='urban_flood_mitigation.html') self.aoi_watersheds_path = inputs.File( args_key='aoi_watersheds_path', diff --git a/src/natcap/invest/ui_server.py b/src/natcap/invest/ui_server.py new file mode 100644 index 0000000000..689cfbf988 --- /dev/null +++ b/src/natcap/invest/ui_server.py @@ -0,0 +1,207 @@ +"""A Flask app with HTTP endpoints used by the InVEST Workbench.""" +import codecs +import collections +from datetime import datetime +import importlib +import json +import logging +import pprint +import textwrap + +from flask import Flask +from flask import request +import natcap.invest.cli +import natcap.invest.datastack + +logging.basicConfig(level=logging.DEBUG) +LOGGER = logging.getLogger(__name__) + +app = Flask(__name__) + +# Lookup names to pass to `invest run` based on python module names +_UI_META = collections.namedtuple('UIMeta', ['run_name', 'human_name']) +MODULE_MODELRUN_MAP = { + v.pyname: _UI_META( + run_name=k, + human_name=v.humanname) + for k, v in natcap.invest.cli._MODEL_UIS.items()} + + +def shutdown_server(): + """Shutdown the flask server.""" + func = request.environ.get('werkzeug.server.shutdown') + if func is None: + raise RuntimeError('Not running with the Werkzeug Server') + func() + + +@app.route('/ready', methods=['GET']) +def get_is_ready(): + """Returns something simple to confirm the server is open.""" + return 'Flask ready' + + +@app.route('/shutdown', methods=['GET']) +def shutdown(): + """A request to this endpoint shuts down the server.""" + shutdown_server() + return 'Flask server shutting down...' + + +@app.route('/models', methods=['GET']) +def get_invest_models(): + """Gets a list of available InVEST models. + + Returns: + A JSON string + """ + LOGGER.debug('get model list') + return natcap.invest.cli.build_model_list_json() + + +@app.route('/getspec', methods=['POST']) +def get_invest_getspec(): + """Gets the ARGS_SPEC dict from an InVEST model. + + Body (JSON string): "carbon" + + Returns: + A JSON string. + """ + target_model = request.get_json() + target_module = natcap.invest.cli._MODEL_UIS[target_model].pyname + model_module = importlib.import_module(name=target_module) + LOGGER.debug(model_module.__file__) + spec = model_module.ARGS_SPEC + return json.dumps(spec) + + +@app.route('/validate', methods=['POST']) +def get_invest_validate(): + """Gets the return value of an InVEST model's validate function. + + Body (JSON string): + model_module: string (e.g. natcap.invest.carbon) + args: JSON string of InVEST model args keys and values + + Returns: + A JSON string. + """ + payload = request.get_json() + LOGGER.debug(payload) + target_module = payload['model_module'] + args_dict = json.loads(payload['args']) + LOGGER.debug(args_dict) + try: + limit_to = payload['limit_to'] + except KeyError: + limit_to = None + model_module = importlib.import_module(name=target_module) + results = model_module.validate(args_dict, limit_to=limit_to) + LOGGER.debug(results) + return json.dumps(results) + + +@app.route('/post_datastack_file', methods=['POST']) +def post_datastack_file(): + """Extracts InVEST model args from json, logfiles, or datastacks. + + Body (JSON string): path to file + + Returns: + A JSON string. + """ + filepath = request.get_json() + stack_type, stack_info = natcap.invest.datastack.get_datastack_info( + filepath) + run_name, human_name = MODULE_MODELRUN_MAP[stack_info.model_name] + result_dict = { + 'type': stack_type, + 'args': stack_info.args, + 'module_name': stack_info.model_name, + 'model_run_name': run_name, + 'model_human_name': human_name, + 'invest_version': stack_info.invest_version + } + LOGGER.debug(result_dict) + return json.dumps(result_dict) + + +@app.route('/write_parameter_set_file', methods=['POST']) +def write_parameter_set_file(): + """Writes InVEST model args keys and values to a datastack JSON file. + + Body (JSON string): + parameterSetPath: string + moduleName: string(e.g. natcap.invest.carbon) + args: JSON string of InVEST model args keys and values + relativePaths: boolean + + Returns: + A string. + """ + payload = request.get_json() + filepath = payload['parameterSetPath'] + modulename = payload['moduleName'] + args = json.loads(payload['args']) + relative_paths = payload['relativePaths'] + + natcap.invest.datastack.build_parameter_set( + args, modulename, filepath, relative=relative_paths) + return 'parameter set saved' + + +# Borrowed this function from natcap.invest.model because I assume +# that module won't persist if we eventually deprecate the Qt UI. +@app.route('/save_to_python', methods=['POST']) +def save_to_python(): + """Writes a python script with a call to an InVEST model execute function. + + Body (JSON string): + filepath: string + modelname: string (e.g. carbon) + pyname: string (e.g. natcap.invest.carbon) + args_dict: JSON string of InVEST model args keys and values + + Returns: + A string. + """ + payload = request.get_json() + save_filepath = payload['filepath'] + modelname = payload['modelname'] + pyname = payload['pyname'] + args_dict = json.loads(payload['args']) + + script_template = textwrap.dedent("""\ + # coding=UTF-8 + # ----------------------------------------------- + # Generated by InVEST {invest_version} on {today} + # Model: {modelname} + + import {py_model} + + args = {model_args} + + if __name__ == '__main__': + {py_model}.execute(args) + """) + + with codecs.open(save_filepath, 'w', encoding='utf-8') as py_file: + # cast_args = dict((unicode(key), value) for (key, value) + # in args_dict.items()) + args = pprint.pformat(args_dict, indent=4) # 4 spaces + + # Tweak formatting from pprint: + # * Bump parameter inline with starting { to next line + # * add trailing comma to last item item pair + # * add extra space to spacing before first item + args = args.replace('{', '{\n ') + args = args.replace('}', ',\n}') + py_file.write(script_template.format( + invest_version=natcap.invest.cli.__version__, + today=datetime.now().strftime('%c'), + modelname=modelname, + py_model=pyname, + model_args=args)) + + return 'python script saved' diff --git a/src/natcap/invest/utils.py b/src/natcap/invest/utils.py index f3e2855ccb..c1bf6a71f5 100644 --- a/src/natcap/invest/utils.py +++ b/src/natcap/invest/utils.py @@ -20,6 +20,7 @@ LOGGER = logging.getLogger(__name__) LOG_FMT = ( "%(asctime)s " + "(%(name)s) " "%(module)s.%(funcName)s(%(lineno)d) " "%(levelname)s %(message)s") diff --git a/src/natcap/invest/validation.py b/src/natcap/invest/validation.py index 2ced6deed3..c431ca77cd 100644 --- a/src/natcap/invest/validation.py +++ b/src/natcap/invest/validation.py @@ -622,7 +622,7 @@ def wrapper_func(): message_queue.put(func(*args, **kwargs)) thread = threading.Thread(target=wrapper_func) - LOGGER.info(f'Starting file checking thread with timeout={timeout}') + LOGGER.debug(f'Starting file checking thread with timeout={timeout}') thread.start() thread.join(timeout=timeout) @@ -634,7 +634,7 @@ def wrapper_func(): return None else: - LOGGER.info('File checking thread completed.') + LOGGER.debug('File checking thread completed.') # get any warning messages returned from the thread return message_queue.get() diff --git a/tests/test_cli.py b/tests/test_cli.py index e06b3a1c50..bf4d7d5c7f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -328,3 +328,23 @@ def test_validate_fisheries_json(self): # Validation returned successfully, so error code 0 even though there # are warnings. self.assertEqual(exit_cm.exception.code, 0) + + def test_serve(self): + """CLI: serve entry-point exists; flask app can import.""" + from natcap.invest import cli + + with unittest.mock.patch('natcap.invest.ui_server.app.run', + return_value=None) as patched_app: + with self.assertRaises(SystemExit) as exit_cm: + cli.main(['serve']) + self.assertEqual(exit_cm.exception.code, 0) + + def test_serve_port_argument(self): + """CLI: serve entry-point parses port subargument.""" + from natcap.invest import cli + + with unittest.mock.patch('natcap.invest.ui_server.app.run', + return_value=None) as patched_app: + with self.assertRaises(SystemExit) as exit_cm: + cli.main(['serve', '--port', '12345']) + self.assertEqual(exit_cm.exception.code, 0) diff --git a/tests/test_forest_carbon_edge.py b/tests/test_forest_carbon_edge.py index 2aa014c228..6c7980a598 100644 --- a/tests/test_forest_carbon_edge.py +++ b/tests/test_forest_carbon_edge.py @@ -57,6 +57,18 @@ def test_carbon_full(self): args['workspace_dir'], 'aggregated_carbon_stocks.shp'), os.path.join(REGRESSION_DATA, 'agg_results_base.shp')) + expected_carbon_raster = gdal.OpenEx(os.path.join(REGRESSION_DATA, + 'carbon_map.tif')) + expected_carbon_band = expected_carbon_raster.GetRasterBand(1) + expected_carbon_array = expected_carbon_band.ReadAsArray() + actual_carbon_raster = gdal.OpenEx(os.path.join(REGRESSION_DATA, + 'carbon_map.tif')) + actual_carbon_band = actual_carbon_raster.GetRasterBand(1) + actual_carbon_array = actual_carbon_band.ReadAsArray() + self.assertTrue(numpy.allclose(expected_carbon_array, + actual_carbon_array)) + + def test_carbon_dup_output(self): """Forest Carbon Edge: test for existing output overlap.""" from natcap.invest import forest_carbon_edge_effect diff --git a/tests/test_ndr.py b/tests/test_ndr.py index 95970018b8..0098d1848c 100644 --- a/tests/test_ndr.py +++ b/tests/test_ndr.py @@ -258,7 +258,7 @@ def test_validation(self): del args['workspace_dir'] validation_errors = ndr.validate(args) - self.assertEquals(len(validation_errors), 1) + self.assertEqual(len(validation_errors), 1) args = NDRTests.generate_base_args(self.workspace_dir) args['workspace_dir'] = '' diff --git a/tests/test_recreation.py b/tests/test_recreation.py index 4e307d854f..213fcd9131 100644 --- a/tests/test_recreation.py +++ b/tests/test_recreation.py @@ -12,6 +12,8 @@ import logging import json import queue +import multiprocessing +import warnings import Pyro4 import numpy @@ -483,10 +485,34 @@ def test_parse_input_csv(self): recmodel_server._parse_input_csv( block_offset_size_queue, self.resampled_data_path, numpy_array_queue) - val = numpy_array_queue.get() + val = recmodel_server._numpy_loads(numpy_array_queue.get()) # we know what the first date is self.assertEqual(val[0][0], datetime.date(2013, 3, 16)) + def test_numpy_pickling_queue(self): + """Recreation test _numpy_dumps and _numpy_loads""" + from natcap.invest.recreation import recmodel_server + + numpy_array_queue = multiprocessing.Queue() + array = numpy.empty(1, dtype='datetime64,f4') + numpy_array_queue.put(recmodel_server._numpy_dumps(array)) + + out_array = recmodel_server._numpy_loads(numpy_array_queue.get()) + numpy.testing.assert_equal(out_array, array) + # without _numpy_loads, the queue pickles the array imperfectly, + # adding a metadata value to the `datetime64` dtype. + # assert that this doesn't happen. 'f0' is the first subdtype. + self.assertEqual(out_array.dtype['f0'].metadata, None) + + # assert that saving the array does not raise a warning + with warnings.catch_warnings(record=True) as ws: + # cause all warnings to always be triggered + warnings.simplefilter("always") + numpy.save(os.path.join(self.workspace_dir, 'out'), out_array) + # assert that no warning was raised + self.assertTrue(len(ws) == 0) + + @_timeout(30.0) def test_regression_local_server(self): """Recreation base regression test on sample data on local server. diff --git a/tests/test_reporting.py b/tests/test_reporting.py index b1c22698c4..308486ca0f 100644 --- a/tests/test_reporting.py +++ b/tests/test_reporting.py @@ -230,7 +230,7 @@ def test_table_generator_attributes(self): regression_path = os.path.join( REGRESSION_DATA, 'table_strings', 'table_string_attrs.txt') - regression_file = codecs.open(regression_path, 'rU', 'utf-8') + regression_file = codecs.open(regression_path, 'r', 'utf-8') regression_str = regression_file.read() self.assertEqual(result_str, regression_str) @@ -264,7 +264,7 @@ def test_table_generator_no_attributes(self): regression_path = os.path.join( REGRESSION_DATA, 'table_strings', 'table_string_no_attrs.txt') - regression_file = codecs.open(regression_path, 'rU', 'utf-8') + regression_file = codecs.open(regression_path, 'r', 'utf-8') regression_str = regression_file.read() self.assertEqual(result_str, regression_str) @@ -296,7 +296,7 @@ def test_table_generator_no_checkbox(self): regression_path = os.path.join( REGRESSION_DATA, 'table_strings', 'table_string_no_checkbox.txt') - regression_file = codecs.open(regression_path, 'rU', 'utf-8') + regression_file = codecs.open(regression_path, 'r', 'utf-8') regression_str = regression_file.read() self.assertEqual(result_str, regression_str) @@ -330,7 +330,7 @@ def test_table_generator_no_td_classes(self): regression_path = os.path.join( REGRESSION_DATA, 'table_strings', 'table_string_no_td_classes.txt') - regression_file = codecs.open(regression_path, 'rU', 'utf-8') + regression_file = codecs.open(regression_path, 'r', 'utf-8') regression_str = regression_file.read() self.assertEqual(result_str, regression_str) @@ -362,7 +362,7 @@ def test_table_generator_no_col_attrs(self): regression_path = os.path.join( REGRESSION_DATA, 'table_strings', 'table_string_no_col_attrs.txt') - regression_file = codecs.open(regression_path, 'rU', 'utf-8') + regression_file = codecs.open(regression_path, 'r', 'utf-8') regression_str = regression_file.read() self.assertEqual(result_str, regression_str) @@ -394,7 +394,7 @@ def test_table_generator_no_totals(self): regression_path = os.path.join( REGRESSION_DATA, 'table_strings', 'table_string_no_totals.txt') - regression_file = codecs.open(regression_path, 'rU', 'utf-8') + regression_file = codecs.open(regression_path, 'r', 'utf-8') regression_str = regression_file.read() self.assertEqual(result_str, regression_str) diff --git a/tests/test_validation.py b/tests/test_validation.py index f2029821ef..09df5e18bb 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -665,11 +665,11 @@ def test_csv_not_utf_8(self): df.to_csv(target_file, encoding='iso8859_5') # Note that non-UTF8 encodings should pass this check, but aren't - # actually being read correctly. Characters outside the ASCII set may + # actually being read correctly. Characters outside the ASCII set may # be replaced with a replacement character. # UTF16, UTF32, etc. will still raise an error. error_msg = validation.check_csv(target_file) - self.assertEquals(error_msg, None) + self.assertEqual(error_msg, None) def test_excel_missing_fieldnames(self): """Validation: test that we can check missing fieldnames in excel.""" @@ -732,16 +732,16 @@ def test_slow_to_open(self): # define a side effect for the mock that will sleep # for longer than the allowed timeout def delay(*args, **kwargs): - time.sleep(6) + time.sleep(7) return [] # make a copy of the real _VALIDATION_FUNCS and override the CSV function - mock_validation_funcs = {key: val for key, val in validation._VALIDATION_FUNCS.items()} + mock_validation_funcs = validation._VALIDATION_FUNCS.copy() mock_validation_funcs['csv'] = functools.partial(validation.timeout, delay) - # replace the validation.check_csv with the mock function, and try to validate - with unittest.mock.patch('natcap.invest.validation._VALIDATION_FUNCS', mock_validation_funcs): + with unittest.mock.patch('natcap.invest.validation._VALIDATION_FUNCS', + mock_validation_funcs): with warnings.catch_warnings(record=True) as ws: # cause all warnings to always be triggered warnings.simplefilter("always") @@ -1232,8 +1232,7 @@ def test_allow_extra_keys(self): } } message = 'DEBUG:natcap.invest.validation:Provided key b does not exist in ARGS_SPEC' - + with self.assertLogs('natcap.invest.validation', level='DEBUG') as cm: validation.validate(args, spec) self.assertTrue(message in cm.output) - diff --git a/ui_tests/test_ui_inputs.py b/ui_tests/test_ui_inputs.py index cc3a8ed31a..52f7738562 100644 --- a/ui_tests/test_ui_inputs.py +++ b/ui_tests/test_ui_inputs.py @@ -12,7 +12,7 @@ import tempfile import shutil import textwrap -import imp +import importlib import uuid import json @@ -697,7 +697,7 @@ def test_path_selected_cyrillic(self): input_instance.set_value(u'/tmp/fooДЖЩя') input_instance.path_select_button.path_selected.emit( u'/tmp/fooДЖЩя') - self.assertEquals(input_instance.value(), u'/tmp/fooДЖЩя') + self.assertEqual(input_instance.value(), u'/tmp/fooДЖЩя') def test_textfield_drag_n_drop(self): input_instance = self.__class__.create_input(label='text') @@ -2870,7 +2870,8 @@ def test_save_to_python(self): module_name = str(uuid.uuid4()) + 'testscript' try: - module = imp.load_source(module_name, python_file) + spec = importlib.util.spec_from_file_location(module_name, python_file) + module = importlib.util.module_from_spec(spec) self.assertEqual(module.args, model_ui.assemble_args()) finally: del sys.modules[module_name]