From c359d8773b99edd8ebb77c9d09bd43ff9345ab96 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 15 Feb 2024 11:47:04 -0300 Subject: [PATCH 01/27] Change output file format enum values --- rubem/_dynamic_model.py | 4 ++-- rubem/configuration/model_configuration.py | 4 ++-- rubem/configuration/output_format.py | 6 +++--- rubem/configuration/output_variables.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/rubem/_dynamic_model.py b/rubem/_dynamic_model.py index 8830945..ab12cbc 100644 --- a/rubem/_dynamic_model.py +++ b/rubem/_dynamic_model.py @@ -89,7 +89,7 @@ def __stepReport(self): continue # Export TIFF raster series - if self.config.output_variables.file_format is OutputFileFormat.GeoTIFF: + if self.config.output_variables.file_format is OutputFileFormat.GEOTIFF: reportTIFFSeries( self, self.ref, @@ -101,7 +101,7 @@ def __stepReport(self): ) # Export PCRaster map format raster series - if self.config.output_variables.file_format is OutputFileFormat.PCRaster: + if self.config.output_variables.file_format is OutputFileFormat.PCRASTER: self.report(self.outputVarsDict.get(outputVar), outputVar) # Check if we have to export the time series of the selected diff --git a/rubem/configuration/model_configuration.py b/rubem/configuration/model_configuration.py index 1151f29..32cd5a5 100644 --- a/rubem/configuration/model_configuration.py +++ b/rubem/configuration/model_configuration.py @@ -112,9 +112,9 @@ def __init__( rnf=bool(self.__get_setting("GENERATE_FILE", "rnf")), tss=bool(self.__get_setting("GENERATE_FILE", "tss")), output_format=( - OutputFileFormat.PCRaster + OutputFileFormat.PCRASTER if bool(self.__get_setting("RASTER_FILE_FORMAT", "map_raster_series")) - else OutputFileFormat.GeoTIFF + else OutputFileFormat.GEOTIFF ), ) self.raster_series = InputRasterSeries( diff --git a/rubem/configuration/output_format.py b/rubem/configuration/output_format.py index bdfea25..4a8f27a 100644 --- a/rubem/configuration/output_format.py +++ b/rubem/configuration/output_format.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import Enum, auto class OutputFileFormat(Enum): @@ -6,5 +6,5 @@ class OutputFileFormat(Enum): Enum class representing the output file format options. """ - PCRaster = 1 - GeoTIFF = 2 + PCRASTER = auto() + GEOTIFF = auto() diff --git a/rubem/configuration/output_variables.py b/rubem/configuration/output_variables.py index c6e64fe..bd835ba 100644 --- a/rubem/configuration/output_variables.py +++ b/rubem/configuration/output_variables.py @@ -34,7 +34,7 @@ class OutputVariables: :param tss: Enable or disable Create time output time series (TSS). Defaults to `False`. :type tss: bool, optional - :param output_format: The output file format. Defaults to `OutputFileFormat.PCRaster`. + :param output_format: The output file format. Defaults to ``OutputFileFormat.PCRASTER``. :type output_format: OutputFileFormat, optional """ @@ -49,7 +49,7 @@ def __init__( smc: bool = False, rnf: bool = False, tss: bool = False, - output_format: OutputFileFormat = OutputFileFormat.PCRaster, + output_format: OutputFileFormat = OutputFileFormat.PCRASTER, ) -> None: self.logger = logging.getLogger(__name__) self.itp = itp From dfcfb6dbc078ebf529d596cd1a85c27fe27d5457 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 15 Feb 2024 11:48:15 -0300 Subject: [PATCH 02/27] Refactor raster data rules enum --- rubem/configuration/raster_data_rule.py | 13 ------------- rubem/validation/raster_data_rules.py | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 13 deletions(-) delete mode 100644 rubem/configuration/raster_data_rule.py create mode 100644 rubem/validation/raster_data_rules.py diff --git a/rubem/configuration/raster_data_rule.py b/rubem/configuration/raster_data_rule.py deleted file mode 100644 index 1211e05..0000000 --- a/rubem/configuration/raster_data_rule.py +++ /dev/null @@ -1,13 +0,0 @@ -from enum import Enum - - -class RasterDataRule(Enum): - """ - Enumeration class representing different rules for raster data. - """ - - ALLOW_ALL_ZEROES = 1 - ALLOW_ALL_ONES = 2 - FORBID_ALL_ZEROES = 3 - FORBID_ALL_ONES = 4 - FORBID_NO_DATA = 5 diff --git a/rubem/validation/raster_data_rules.py b/rubem/validation/raster_data_rules.py new file mode 100644 index 0000000..348b363 --- /dev/null +++ b/rubem/validation/raster_data_rules.py @@ -0,0 +1,22 @@ +from enum import Flag, auto + + +class RasterDataRules(Flag): + """ + Enumeration class representing different rules for raster data. + """ + + FORBID_ALL_ZEROES = auto() + """ + Raster pixels cannot consist entirely of ``0.0`` values. + """ + + FORBID_ALL_ONES = auto() + """ + Raster pixels cannot consist entirely of ``1.0`` values. + """ + + FORBID_NO_DATA = auto() + """ + None of the raster pixels must contain ``NO_DATA`` values. + """ From 9c3d99b13da7d32a2cad5803faf355b5076e6029 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 15 Feb 2024 11:48:45 -0300 Subject: [PATCH 03/27] Add raster map validation handlers --- rubem/configuration/input_raster_files.py | 51 ++++- rubem/configuration/input_raster_series.py | 48 ++++- rubem/configuration/pcraster_map.py | 179 ------------------ rubem/configuration/raster_map.py | 112 +++++++++++ rubem/validation/handlers/base.py | 35 ++++ rubem/validation/handlers/raster_all_ones.py | 37 ++++ .../validation/handlers/raster_all_zeroes.py | 46 +++++ rubem/validation/handlers/raster_no_data.py | 38 ++++ .../validation/handlers/raster_value_range.py | 35 ++++ rubem/validation/raster_data.py | 37 ++++ tests/unit/configuration/test_pcraster_map.py | 2 +- 11 files changed, 420 insertions(+), 200 deletions(-) delete mode 100644 rubem/configuration/pcraster_map.py create mode 100644 rubem/configuration/raster_map.py create mode 100644 rubem/validation/handlers/base.py create mode 100644 rubem/validation/handlers/raster_all_ones.py create mode 100644 rubem/validation/handlers/raster_all_zeroes.py create mode 100644 rubem/validation/handlers/raster_no_data.py create mode 100644 rubem/validation/handlers/raster_value_range.py create mode 100644 rubem/validation/raster_data.py diff --git a/rubem/configuration/input_raster_files.py b/rubem/configuration/input_raster_files.py index eb68505..4e523b5 100644 --- a/rubem/configuration/input_raster_files.py +++ b/rubem/configuration/input_raster_files.py @@ -2,8 +2,10 @@ import os from typing import Union -from rubem.configuration.pcraster_map import PCRasterMap +from rubem.configuration.raster_map import RasterMap from rubem.configuration.data_ranges_settings import DataRangesSettings +from rubem.validation.raster_data import RasterMapValidator +from rubem.validation.raster_data_rules import RasterDataRules class InputRasterFiles: @@ -67,17 +69,46 @@ def __init__( def __validate_files(self) -> None: files = [ - (self.dem, self.__ranges.rasters["dem"]), - (self.demtif, self.__ranges.rasters["dem"]), - (self.clone, self.__ranges.rasters["clone"]), - (self.ndvi_max, self.__ranges.rasters["ndvi"]), - (self.ndvi_min, self.__ranges.rasters["ndvi"]), - (self.soil, self.__ranges.rasters["soil"]), - (self.sample_locations, self.__ranges.rasters["sample_locations"]), + ( + self.dem, + self.__ranges.rasters["dem"], + RasterDataRules.FORBID_NO_DATA + | RasterDataRules.FORBID_ALL_ZEROES + | RasterDataRules.FORBID_ALL_ONES, + ), + ( + self.demtif, + self.__ranges.rasters["dem"], + RasterDataRules.FORBID_NO_DATA + | RasterDataRules.FORBID_ALL_ZEROES + | RasterDataRules.FORBID_ALL_ONES, + ), + (self.clone, self.__ranges.rasters["clone"], RasterDataRules.FORBID_ALL_ZEROES), + (self.ndvi_max, self.__ranges.rasters["ndvi"], RasterDataRules.FORBID_NO_DATA), + (self.ndvi_min, self.__ranges.rasters["ndvi"], RasterDataRules.FORBID_NO_DATA), + ( + self.soil, + self.__ranges.rasters["soil"], + RasterDataRules.FORBID_NO_DATA | RasterDataRules.FORBID_ALL_ZEROES, + ), + ( + self.sample_locations, + self.__ranges.rasters["sample_locations"], + RasterDataRules.FORBID_ALL_ZEROES | RasterDataRules.FORBID_ALL_ONES, + ), ] - for file, valid_range in files: - _ = PCRasterMap(file, valid_range) + for file, valid_range, rules in files: + raster = RasterMap(file, valid_range, rules) + self.logger.debug(str(raster).replace("\n", ", ")) + + validator = RasterMapValidator() + if not validator.validate(raster): + self.logger.error( + "Raster file '%s' contains invalid data. This may lead to unexpected results.", + file, + ) + raise ValueError(f"Raster file '{file}' contains invalid data.") def __str__(self) -> str: return ( diff --git a/rubem/configuration/input_raster_series.py b/rubem/configuration/input_raster_series.py index be06f56..19fba5a 100644 --- a/rubem/configuration/input_raster_series.py +++ b/rubem/configuration/input_raster_series.py @@ -3,8 +3,10 @@ from typing import Union import re -from rubem.configuration.pcraster_map import PCRasterMap +from rubem.configuration.raster_map import RasterMap from rubem.configuration.data_ranges_settings import DataRangesSettings +from rubem.validation.raster_data import RasterMapValidator +from rubem.validation.raster_data_rules import RasterDataRules RASTER_SERIES_FILENAME_MAX_CHARS = 8 RASTER_SERIES_FILENAME_EXTENSION_NUM_DIGITS = 3 @@ -95,23 +97,40 @@ def __init__( def __validate_directories(self) -> None: directories = [ - (self.__etp_dir_path, self.__etp_filename_prefix, self.__ranges.rasters["etp"]), + ( + self.__etp_dir_path, + self.__etp_filename_prefix, + self.__ranges.rasters["etp"], + RasterDataRules.FORBID_NO_DATA, + ), ( self.__precipitation_dir_path, self.__precipitation_filename_prefix, self.__ranges.rasters["precipitation"], + RasterDataRules.FORBID_NO_DATA, + ), + ( + self.__ndvi_dir_path, + self.__ndvi_filename_prefix, + self.__ranges.rasters["ndvi"], + RasterDataRules.FORBID_NO_DATA, + ), + ( + self.__kp_dir_path, + self.__kp_filename_prefix, + self.__ranges.rasters["kp"], + RasterDataRules.FORBID_NO_DATA, ), - (self.__ndvi_dir_path, self.__ndvi_filename_prefix, self.__ranges.rasters["ndvi"]), - (self.__kp_dir_path, self.__kp_filename_prefix, self.__ranges.rasters["kp"]), ( self.__landuse_dir_path, self.__landuse_filename_prefix, self.__ranges.rasters["landuse"], + RasterDataRules.FORBID_NO_DATA | RasterDataRules.FORBID_ALL_ZEROES, ), ] total_num_files = [] - for directory, prefix, valid_range in directories: + for directory, prefix, valid_range, rules in directories: if not os.path.isdir(directory): raise NotADirectoryError(f"Invalid input data directory: {directory}") @@ -120,7 +139,7 @@ def __validate_directories(self) -> None: self.__validate_raster_series_filenames_prefixes(prefix) total_num_files.append( - self.__validate_files_with_prefix(directory, prefix, valid_range) + self.__validate_files_with_prefix(directory, prefix, valid_range, rules) ) common_total_num_files = set(total_num_files) @@ -130,7 +149,7 @@ def __validate_directories(self) -> None: "This may lead to unexpected results." ) - def __validate_files_with_prefix(self, directory, prefix, valid_range) -> int: + def __validate_files_with_prefix(self, directory, prefix, valid_range, rules) -> int: num_digits = RASTER_SERIES_FILENAME_MAX_CHARS - len(prefix) regex_pattern = rf"^{prefix}[0-9]{{{num_digits}}}\.[0-9]{{{RASTER_SERIES_FILENAME_EXTENSION_NUM_DIGITS}}}$" compiled_pattern = re.compile(regex_pattern, re.IGNORECASE) @@ -139,7 +158,7 @@ def __validate_files_with_prefix(self, directory, prefix, valid_range) -> int: with os.scandir(directory) as it: for entry in it: if entry.is_file() and compiled_pattern.match(entry.name): - self.__validate_raster_file(entry.path, valid_range) + self.__validate_raster_file(entry.path, valid_range, rules) counter += 1 if counter == 0: @@ -159,8 +178,17 @@ def __validate_files_with_prefix(self, directory, prefix, valid_range) -> int: return counter - def __validate_raster_file(self, file, valid_range) -> None: - _ = PCRasterMap(file, valid_range) + def __validate_raster_file(self, file, valid_range, rules) -> None: + raster = RasterMap(file, valid_range, rules) + self.logger.debug(str(raster).replace("\n", ", ")) + + validator = RasterMapValidator() + if not validator.validate(raster): + self.logger.error( + "Raster file '%s' contains invalid data. This may lead to unexpected results.", + file, + ) + raise ValueError(f"Raster file '{file}' contains invalid data.") def __validate_raster_series_filenames_prefixes(self, prefix): num_digits = RASTER_SERIES_FILENAME_MAX_CHARS - len(prefix) diff --git a/rubem/configuration/pcraster_map.py b/rubem/configuration/pcraster_map.py deleted file mode 100644 index 39a100c..0000000 --- a/rubem/configuration/pcraster_map.py +++ /dev/null @@ -1,179 +0,0 @@ -import logging -import re -import os -from typing import Optional, Union - -from osgeo import gdal -import numpy as np - - -class PCRasterMap: - """ - Initialize a PCRasterMap object. - - :param file_path: The path to the raster file. - :type file_path: Union[str, bytes, os.PathLike] - - :param valid_range: The valid range of values for the raster. Defaults to None. - :type valid_range: Optional[dict[str, float]], optional - - :raises FileNotFoundError: If the raster file does not exist. - :raises ValueError: If the raster file is empty or has an invalid extension. - :raises IOError: If the raster file cannot be opened. - """ - - def __init__( - self, - file_path: Union[str, bytes, os.PathLike], - valid_range: Optional[dict[str, float]] = None, - ) -> None: - self.logger = logging.getLogger(__name__) - - self.logger.debug("Opening raster file: %s", file_path) - if not os.path.isfile(file_path): - raise FileNotFoundError(f"Invalid raster file: {file_path}") - - if os.path.getsize(file_path) <= 0: - raise ValueError(f"Empty raster file: {file_path}") - - if not str(file_path).endswith((".map", ".tif")) and not bool( - bool(re.search(r"\.[0-9]{3}$", str(os.path.splitext(file_path)[1]))) - ): - raise ValueError(f"Invalid raster file extension: {file_path}") - - self.raster = gdal.Open(file_path) - if self.raster is None: - raise IOError(f"Unable to open raster file: {file_path}") - - self.logger.debug("Dimensions: %s", self.__check_dimensions(self.raster)) - self.logger.debug("Projection: %s", self.__check_projection(self.raster)) - self.logger.debug("Number of Bands: %s", self.raster.RasterCount) - self.logger.debug("Driver: %s", self.raster.GetDriver().ShortName) - self.logger.debug("Metadata: %s", self.raster.GetMetadata()) - self.logger.debug("GeoTransform: %s", self.raster.GetGeoTransform()) - self.logger.debug("GCPs: %s", self.raster.GetGCPs()) - self.logger.debug("Metadata Domain List: %s", self.raster.GetMetadataDomainList()) - - self.raster_band = [] - for band in range(self.raster.RasterCount): - band += 1 - self.logger.debug("Band: %s", band) - self.raster_band.append(self.raster.GetRasterBand(band)) - if self.raster_band[band - 1] is None: - raise IOError(f"Unable to open band {band} of raster file: {file_path}") - - self.logger.debug("Data Type: %s", self.__check_data_type(self.raster_band[band - 1])) - self.logger.debug( - "NoData Value: %s", self.__check_nodata_value(self.raster_band[band - 1]) - ) - self.logger.debug( - "Statistics: %s", self.__compute_statistics(self.raster_band[band - 1]) - ) - - if valid_range and not self.__check_value_consistency( - self.raster_band[band - 1], valid_range["min"], valid_range["max"] - ): - raise ValueError( - f"Raster file {file_path} has values outside the range [{valid_range['min']}, {valid_range['max']}]" - ) - - def __check_dimensions(self, raster: gdal.Dataset) -> tuple[int, int]: - """ - Check the dimensions of the raster. - - :param raster: The raster object. - :type raster: gdal.Dataset - - :return: The number of columns and rows in the raster. - :rtype: tuple[int, int] - """ - cols = raster.RasterXSize - rows = raster.RasterYSize - return cols, rows - - def __check_data_type(self, band: gdal.Band) -> str: - """ - Check the data type of the raster band. - - :param band: The raster band object. - :type band: gdal.Band - - :return: The data type of the raster band. - :rtype: str - """ - return gdal.GetDataTypeName(band.DataType) - - def __check_nodata_value(self, band: gdal.Band) -> float: - """ - Check the NoData value of the raster band. - - :param band: The raster band object. - :type band: gdal.Band - - :return: The NoData value of the raster band. - :rtype: float - """ - return band.GetNoDataValue() - - def __compute_statistics(self, band: gdal.Band) -> tuple[float, float, float, float]: - """ - Compute the statistics of the raster band. - - :param band: The raster band object. - :type band: gdal.Band - - :return: The minimum, maximum, mean, and standard deviation of the raster band. - :rtype: tuple[float, float, float, float] - """ - return band.GetStatistics(True, True) - - def __check_projection(self, raster: gdal.Dataset) -> str: - """ - Check the projection of the raster. - - :param raster: The raster object. - :type raster: gdal.Dataset - - :return: The projection of the raster. - :rtype: str - """ - projection = raster.GetProjection() - return projection - - def __check_value_consistency( - self, band: gdal.Band, min_value: float, max_value: float - ) -> bool: - """ - Check if the values in the raster band are within the valid range. - - :param band: The raster band object. - :type band: gdal.Band - - :param min_value: The minimum valid value. - :type min_value: float - - :param max_value: The maximum valid value. - :type max_value: float - - :return: True if all values are within the valid range, False otherwise. - :rtype: bool - """ - array = self.__get_array_without_no_data(band) - return ((array >= min_value) & (array <= max_value)).all() - - def __get_array_without_no_data(self, band) -> np.ma.MaskedArray: - """ - Get the array of the raster band without the NoData values. - - :param band: The raster band object. - :type band: gdal.Band - - :return: The array of the raster band without the NoData values. - :rtype: numpy.ma.MaskedArray - """ - band.ComputeStatistics(0) - nodata = band.GetNoDataValue() - array = band.ReadAsArray() - if nodata is not None: - array = np.ma.masked_equal(array, nodata) - return array diff --git a/rubem/configuration/raster_map.py b/rubem/configuration/raster_map.py new file mode 100644 index 0000000..bf87d85 --- /dev/null +++ b/rubem/configuration/raster_map.py @@ -0,0 +1,112 @@ +import logging +import re +import os +from typing import Optional, Union + +from osgeo import gdal + +from rubem.validation.raster_data_rules import RasterDataRules + + +class RasterBand: + """ + Initialize a RasterBand object. + + :param index: The index of the raster band. + :type index: int + + :param band: The raster band object. + :type band: gdal.Band + + :raises ValueError: If the raster band is empty or has an invalid data type. + """ + + def __init__(self, index: int, band: gdal.Band) -> None: + self.logger = logging.getLogger(__name__) + + if band is None: + raise ValueError("Invalid raster band") + + self.index = index + self.band = band + self.band.ComputeStatistics(0) + self.no_data_value = self.band.GetNoDataValue() + self.min, self.max, self.mean, self.std_dev = self.band.GetStatistics(True, True) + self.data_type = gdal.GetDataTypeName(self.band.DataType) + self.data_array = self.band.ReadAsArray() + + def __str__(self) -> str: + return ( + f"Index: {self.index}, " + f"Data Type: {self.data_type}, " + f"NoData Value: {self.no_data_value}, " + f"Statistics: Min: {self.min}, Max: {self.max}, Mean: {self.mean}, Std Dev: {self.std_dev}" + ) + + +class RasterMap: + """ + Initialize a RasterMap object. + + :param file_path: The path to the raster file. + :type file_path: Union[str, bytes, os.PathLike] + + :param valid_range: The valid range of values for the raster. Defaults to None. + :type valid_range: Optional[dict[str, float]], optional + + :raises FileNotFoundError: If the raster file does not exist. + :raises ValueError: If the raster file is empty or has an invalid extension. + :raises IOError: If the raster file cannot be opened. + """ + + def __init__( + self, + file_path: Union[str, bytes, os.PathLike], + valid_range: Optional[dict[str, float]] = None, + rules: Optional[RasterDataRules] = None, + ) -> None: + self.logger = logging.getLogger(__name__) + + self.logger.debug("Opening raster file: %s", file_path) + + self.__validate_file(file_path) + self.__validate_file_extension(file_path) + + self.raster = gdal.OpenEx(file_path, gdal.GA_ReadOnly) + + self.valid_range = valid_range + self.rules = rules + + self.bands = [] + for band_index in range(self.raster.RasterCount): + band_index += 1 + band = RasterBand(band_index, self.raster.GetRasterBand(band_index)) + self.bands.append(band) + + def __validate_file(self, file_path): + if not os.path.isfile(file_path): + raise FileNotFoundError(f"Invalid raster file: {file_path}") + + if os.path.getsize(file_path) <= 0: + raise ValueError(f"Empty raster file: {file_path}") + + def __validate_file_extension(self, file_path): + if not str(file_path).endswith((".map", ".tif")) and not bool( + bool(re.search(r"\.[0-9]{3}$", str(os.path.splitext(file_path)[1]))) + ): + raise ValueError(f"Invalid raster file extension: {file_path}") + + def __str__(self): + if self.raster: + return ( + f"Dimensions: {(self.raster.RasterXSize, self.raster.RasterYSize)}\n" + f"Projection: {self.raster.GetProjection()}\n" + f"Driver: {self.raster.GetDriver().ShortName}\n" + f"Metadata: {self.raster.GetMetadata()}\n" + f"GeoTransform: {self.raster.GetGeoTransform()}\n" + f"GCPs: {self.raster.GetGCPs()}\n" + f"Metadata Domain List: {self.raster.GetMetadataDomainList()}\n" + f"Number of Bands: {self.raster.RasterCount}\n" + f"Bands: {[ str(band) for band in self.bands ]}" + ) + return "No raster file loaded" diff --git a/rubem/validation/handlers/base.py b/rubem/validation/handlers/base.py new file mode 100644 index 0000000..8d6066a --- /dev/null +++ b/rubem/validation/handlers/base.py @@ -0,0 +1,35 @@ +import logging +from abc import ABC, abstractmethod + + +class Handler(ABC): + """Abstract base class for handlers.""" + + @abstractmethod + def set_next(self, handler): + """Set the next handler in the chain.""" + pass + + @abstractmethod + def handle(self, request): + """Handle the request.""" + pass + + +class BaseValidatorHandler(Handler): + """Base class for validator handlers.""" + + def __init__(self): + self.logger = logging.getLogger(__name__) + self.__next: Handler = None + + def set_next(self, handler: Handler) -> Handler: + """Set the next handler in the chain.""" + self.__next = handler + return handler + + def handle(self, request): + """Handle the request by passing it to the next handler in the chain.""" + if self.__next: + return self.__next.handle(request) + return True # Fallback validator: always return True diff --git a/rubem/validation/handlers/raster_all_ones.py b/rubem/validation/handlers/raster_all_ones.py new file mode 100644 index 0000000..be139c1 --- /dev/null +++ b/rubem/validation/handlers/raster_all_ones.py @@ -0,0 +1,37 @@ +import numpy as np + +from rubem.validation.handlers.base import BaseValidatorHandler +from rubem.validation.raster_data_rules import RasterDataRules + + +class AllOnesValidatorHandler(BaseValidatorHandler): + """ + A validator handler that checks if all values in a raster are one. + + This handler checks if the rule ``FORBID_ALL_ONES`` is set for the raster. + If the rule is set and all values in the raster are one, it returns ``False``, + indicating that the validation has failed. Otherwise, it delegates the handling + to the base validator handler. + + :param raster: The raster object to be validated. + :type raster: + + :return: ``True`` if the raster data is valid, ``False`` otherwise. + """ + + def handle(self, raster): + if not raster.rules: + self.logger.info("`FORBID_ALL_ONES` validator skipped because no rules were set.") + return super().handle(raster) + + if not RasterDataRules.FORBID_ALL_ONES in raster.rules: + self.logger.debug("`FORBID_ALL_ONES` validator skipped because the rule was not set.") + return super().handle(raster) + + band_array = raster.bands[0].data_array + all_ones_condition = np.allclose(band_array, 1, atol=1e-8) + + if all_ones_condition: + return False + + return super().handle(raster) diff --git a/rubem/validation/handlers/raster_all_zeroes.py b/rubem/validation/handlers/raster_all_zeroes.py new file mode 100644 index 0000000..82ca1bc --- /dev/null +++ b/rubem/validation/handlers/raster_all_zeroes.py @@ -0,0 +1,46 @@ +import numpy as np + +from rubem.validation.handlers.base import BaseValidatorHandler +from rubem.validation.raster_data_rules import RasterDataRules + + +class AllZeroesValidatorHandler(BaseValidatorHandler): + """ + A validator handler that checks if all values in a raster are zero. + + This handler checks if the rule ``FORBID_ALL_ZEROES`` is set for the raster. + If the rule is set and all values in the raster are zero, it returns ``False``, + indicating that the validation has failed. Otherwise, it delegates the handling + to the base validator handler. + + :param raster: The raster object to be validated. + :type raster: + + :return: ``True`` if the raster data is valid, ``False`` otherwise. + """ + + def handle(self, raster): + """ + Handle the validation for the given raster. + + Args: + raster: The raster object to be validated. + + Returns: + bool: True if the validation passes, False otherwise. + """ + if not raster.rules: + self.logger.info("`FORBID_ALL_ZEROES` validator skipped because no rules were set.") + return super().handle(raster) + + if not RasterDataRules.FORBID_ALL_ZEROES in raster.rules: + self.logger.debug("`FORBID_ALL_ZEROES` validator skipped because the rule was not set.") + return super().handle(raster) + + band_array = raster.bands[0].data_array + zero_condition = np.allclose(band_array, 0, atol=1e-8) + + if zero_condition: + return False + + return super().handle(raster) diff --git a/rubem/validation/handlers/raster_no_data.py b/rubem/validation/handlers/raster_no_data.py new file mode 100644 index 0000000..6c1cf7e --- /dev/null +++ b/rubem/validation/handlers/raster_no_data.py @@ -0,0 +1,38 @@ +import numpy as np + +from rubem.validation.handlers.base import BaseValidatorHandler +from rubem.validation.raster_data_rules import RasterDataRules + + +class NoDataValidatorHandler(BaseValidatorHandler): + """ + A validator handler that checks the presence of ``NO_DATA`` values in a raster. + + This handler checks if the raster data contains ``NO_DATA`` values based on the rules set. + If the rules are not set or the specific rule for forbidding ``NO_DATA`` values is not set, + the validation is skipped. Otherwise, it checks if all the pixels in the raster + have the ``NO_DATA`` value and returns ``False`` if they do. + + :param raster: The raster object to be validated. + :type raster: + + :return: ``True`` if the raster data is valid, ``False`` otherwise. + """ + + def handle(self, raster): + if not raster.rules: + self.logger.info("`FORBID_NO_DATA` validator skipped because no rules were set.") + return super().handle(raster) + + if not RasterDataRules.FORBID_NO_DATA in raster.rules: + self.logger.debug("`FORBID_NO_DATA` validator skipped because the rule was not set.") + return super().handle(raster) + + no_data_value = raster.bands[0].no_data_value + band_array = raster.bands[0].data_array + no_data_condition = np.any(band_array == no_data_value) + + if no_data_condition: + return False + + return super().handle(raster) diff --git a/rubem/validation/handlers/raster_value_range.py b/rubem/validation/handlers/raster_value_range.py new file mode 100644 index 0000000..9c0e7f8 --- /dev/null +++ b/rubem/validation/handlers/raster_value_range.py @@ -0,0 +1,35 @@ +import numpy as np + +from rubem.validation.handlers.base import BaseValidatorHandler + + +class ValueRangeValidatorHandler(BaseValidatorHandler): + """ + A validator handler that checks if the values in a raster are within a valid range. + + :param raster: The raster to be validated. + :type raster: + + :return: ``True`` if all values are within the valid range, ``False`` otherwise. + :rtype: bool + """ + + def handle(self, raster): + if not raster.valid_range: + self.logger.info("`ValueRange` validator skipped because no value range was set.") + return super().handle(raster) + + min_value = raster.valid_range["min"] + max_value = raster.valid_range["max"] + no_data_value = raster.bands[0].no_data_value + band_array = raster.bands[0].data_array + + if no_data_value is not None: + band_array = band_array[band_array != no_data_value] + + has_valid_range_condition = np.all((band_array >= min_value) & (band_array <= max_value)) + + if not has_valid_range_condition: + return False + + return super().handle(raster) diff --git a/rubem/validation/raster_data.py b/rubem/validation/raster_data.py new file mode 100644 index 0000000..0f10f42 --- /dev/null +++ b/rubem/validation/raster_data.py @@ -0,0 +1,37 @@ +import logging + +from rubem.validation.handlers.raster_all_ones import AllOnesValidatorHandler +from rubem.validation.handlers.raster_all_zeroes import AllZeroesValidatorHandler +from rubem.validation.handlers.raster_no_data import NoDataValidatorHandler +from rubem.validation.handlers.raster_value_range import ValueRangeValidatorHandler + + +class RasterMapValidator: + """ + Class to validate raster maps data. + """ + + def __init__(self): + self.logger = logging.getLogger(__name__) + + def validate(self, raster): + """ + Validate the given raster map data. + + :param raster: The raster map to be validated. + :type raster: + + :return: True if the raster map is valid, False otherwise. + :rtype: bool + """ + + value_range = ValueRangeValidatorHandler() + no_data = NoDataValidatorHandler() + all_zeroes = AllZeroesValidatorHandler() + all_ones = AllOnesValidatorHandler() + + value_range.set_next(no_data) + no_data.set_next(all_zeroes) + all_zeroes.set_next(all_ones) + + return value_range.handle(raster) diff --git a/tests/unit/configuration/test_pcraster_map.py b/tests/unit/configuration/test_pcraster_map.py index 837a78b..5e8ccd6 100644 --- a/tests/unit/configuration/test_pcraster_map.py +++ b/tests/unit/configuration/test_pcraster_map.py @@ -1,6 +1,6 @@ import pytest -from rubem.configuration.pcraster_map import PCRasterMap +from rubem.configuration.raster_map import RasterMap class TestPCRasterMap: From d4ad9fc4d05bdcdf6e573f664336561beb769f9a Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Tue, 20 Feb 2024 16:58:21 -0300 Subject: [PATCH 04/27] Update RasterMapValidator and handler methods --- rubem/configuration/input_raster_files.py | 14 +++++---- rubem/configuration/input_raster_series.py | 12 ++++---- rubem/validation/handlers/base.py | 6 ++-- rubem/validation/handlers/raster_all_ones.py | 20 +++++++------ .../validation/handlers/raster_all_zeroes.py | 29 +++++++------------ rubem/validation/handlers/raster_no_data.py | 22 +++++++------- .../validation/handlers/raster_value_range.py | 28 +++++++++++------- rubem/validation/raster_data_rules.py | 8 +++++ ...raster_data.py => raster_map_validator.py} | 3 +- 9 files changed, 80 insertions(+), 62 deletions(-) rename rubem/validation/{raster_data.py => raster_map_validator.py} (93%) diff --git a/rubem/configuration/input_raster_files.py b/rubem/configuration/input_raster_files.py index 4e523b5..358ef06 100644 --- a/rubem/configuration/input_raster_files.py +++ b/rubem/configuration/input_raster_files.py @@ -4,7 +4,7 @@ from rubem.configuration.raster_map import RasterMap from rubem.configuration.data_ranges_settings import DataRangesSettings -from rubem.validation.raster_data import RasterMapValidator +from rubem.validation.raster_map_validator import RasterMapValidator from rubem.validation.raster_data_rules import RasterDataRules @@ -103,12 +103,16 @@ def __validate_files(self) -> None: self.logger.debug(str(raster).replace("\n", ", ")) validator = RasterMapValidator() - if not validator.validate(raster): - self.logger.error( - "Raster file '%s' contains invalid data. This may lead to unexpected results.", + valid, errors = validator.validate(raster) + if not valid: + self.logger.warning( + "Raster file '%s' violated %s. This may lead to unexpected results.", file, + errors, + ) + print( + f"Raster file '{file}' violated {[str(error) for error in errors]} data rule(s)." ) - raise ValueError(f"Raster file '{file}' contains invalid data.") def __str__(self) -> str: return ( diff --git a/rubem/configuration/input_raster_series.py b/rubem/configuration/input_raster_series.py index 19fba5a..0b95a2f 100644 --- a/rubem/configuration/input_raster_series.py +++ b/rubem/configuration/input_raster_series.py @@ -5,7 +5,7 @@ from rubem.configuration.raster_map import RasterMap from rubem.configuration.data_ranges_settings import DataRangesSettings -from rubem.validation.raster_data import RasterMapValidator +from rubem.validation.raster_map_validator import RasterMapValidator from rubem.validation.raster_data_rules import RasterDataRules RASTER_SERIES_FILENAME_MAX_CHARS = 8 @@ -183,12 +183,14 @@ def __validate_raster_file(self, file, valid_range, rules) -> None: self.logger.debug(str(raster).replace("\n", ", ")) validator = RasterMapValidator() - if not validator.validate(raster): - self.logger.error( - "Raster file '%s' contains invalid data. This may lead to unexpected results.", + valid, errors = validator.validate(raster) + if not valid: + self.logger.warning( + "Raster file '%s' violated %s. This may lead to unexpected results.", file, + errors, ) - raise ValueError(f"Raster file '{file}' contains invalid data.") + print(f"Raster file '{file}' violated {[str(error) for error in errors]} data rule(s).") def __validate_raster_series_filenames_prefixes(self, prefix): num_digits = RASTER_SERIES_FILENAME_MAX_CHARS - len(prefix) diff --git a/rubem/validation/handlers/base.py b/rubem/validation/handlers/base.py index 8d6066a..126317b 100644 --- a/rubem/validation/handlers/base.py +++ b/rubem/validation/handlers/base.py @@ -11,7 +11,7 @@ def set_next(self, handler): pass @abstractmethod - def handle(self, request): + def handle(self, request, errors): """Handle the request.""" pass @@ -28,8 +28,8 @@ def set_next(self, handler: Handler) -> Handler: self.__next = handler return handler - def handle(self, request): + def handle(self, request, errors): """Handle the request by passing it to the next handler in the chain.""" if self.__next: - return self.__next.handle(request) + return self.__next.handle(request, errors) return True # Fallback validator: always return True diff --git a/rubem/validation/handlers/raster_all_ones.py b/rubem/validation/handlers/raster_all_ones.py index be139c1..18cdbc1 100644 --- a/rubem/validation/handlers/raster_all_ones.py +++ b/rubem/validation/handlers/raster_all_ones.py @@ -14,24 +14,26 @@ class AllOnesValidatorHandler(BaseValidatorHandler): to the base validator handler. :param raster: The raster object to be validated. - :type raster: + :type raster: RasterMap :return: ``True`` if the raster data is valid, ``False`` otherwise. """ - def handle(self, raster): + def handle(self, raster, errors): if not raster.rules: self.logger.info("`FORBID_ALL_ONES` validator skipped because no rules were set.") - return super().handle(raster) + return super().handle(raster, errors) if not RasterDataRules.FORBID_ALL_ONES in raster.rules: self.logger.debug("`FORBID_ALL_ONES` validator skipped because the rule was not set.") - return super().handle(raster) + return super().handle(raster, errors) - band_array = raster.bands[0].data_array - all_ones_condition = np.allclose(band_array, 1, atol=1e-8) + for band in raster.bands: + band_array = band.data_array + all_ones_condition = np.allclose(band_array, 1, atol=1e-8) - if all_ones_condition: - return False + if all_ones_condition: + errors.append(RasterDataRules.FORBID_ALL_ONES) + return False - return super().handle(raster) + return super().handle(raster, errors) diff --git a/rubem/validation/handlers/raster_all_zeroes.py b/rubem/validation/handlers/raster_all_zeroes.py index 82ca1bc..a8098e4 100644 --- a/rubem/validation/handlers/raster_all_zeroes.py +++ b/rubem/validation/handlers/raster_all_zeroes.py @@ -14,33 +14,26 @@ class AllZeroesValidatorHandler(BaseValidatorHandler): to the base validator handler. :param raster: The raster object to be validated. - :type raster: + :type raster: RasterMap :return: ``True`` if the raster data is valid, ``False`` otherwise. """ - def handle(self, raster): - """ - Handle the validation for the given raster. - - Args: - raster: The raster object to be validated. - - Returns: - bool: True if the validation passes, False otherwise. - """ + def handle(self, raster, errors): if not raster.rules: self.logger.info("`FORBID_ALL_ZEROES` validator skipped because no rules were set.") - return super().handle(raster) + return super().handle(raster, errors) if not RasterDataRules.FORBID_ALL_ZEROES in raster.rules: self.logger.debug("`FORBID_ALL_ZEROES` validator skipped because the rule was not set.") - return super().handle(raster) + return super().handle(raster, errors) - band_array = raster.bands[0].data_array - zero_condition = np.allclose(band_array, 0, atol=1e-8) + for band in raster.bands: + band_array = band.data_array + zero_condition = np.allclose(band_array, 0, atol=1e-8) - if zero_condition: - return False + if zero_condition: + errors.append(RasterDataRules.FORBID_ALL_ZEROES) + return False - return super().handle(raster) + return super().handle(raster, errors) diff --git a/rubem/validation/handlers/raster_no_data.py b/rubem/validation/handlers/raster_no_data.py index 6c1cf7e..390341a 100644 --- a/rubem/validation/handlers/raster_no_data.py +++ b/rubem/validation/handlers/raster_no_data.py @@ -14,25 +14,27 @@ class NoDataValidatorHandler(BaseValidatorHandler): have the ``NO_DATA`` value and returns ``False`` if they do. :param raster: The raster object to be validated. - :type raster: + :type raster: RasterMap :return: ``True`` if the raster data is valid, ``False`` otherwise. """ - def handle(self, raster): + def handle(self, raster, errors): if not raster.rules: self.logger.info("`FORBID_NO_DATA` validator skipped because no rules were set.") - return super().handle(raster) + return super().handle(raster, errors) if not RasterDataRules.FORBID_NO_DATA in raster.rules: self.logger.debug("`FORBID_NO_DATA` validator skipped because the rule was not set.") - return super().handle(raster) + return super().handle(raster, errors) - no_data_value = raster.bands[0].no_data_value - band_array = raster.bands[0].data_array - no_data_condition = np.any(band_array == no_data_value) + for band in raster.bands: + no_data_value = band.no_data_value + band_array = band.data_array + no_data_condition = np.any(band_array == no_data_value) - if no_data_condition: - return False + if no_data_condition: + errors.append(RasterDataRules.FORBID_NO_DATA) + return False - return super().handle(raster) + return super().handle(raster, errors) diff --git a/rubem/validation/handlers/raster_value_range.py b/rubem/validation/handlers/raster_value_range.py index 9c0e7f8..a30ab42 100644 --- a/rubem/validation/handlers/raster_value_range.py +++ b/rubem/validation/handlers/raster_value_range.py @@ -1,6 +1,7 @@ import numpy as np from rubem.validation.handlers.base import BaseValidatorHandler +from rubem.validation.raster_data_rules import RasterDataRules class ValueRangeValidatorHandler(BaseValidatorHandler): @@ -8,28 +9,33 @@ class ValueRangeValidatorHandler(BaseValidatorHandler): A validator handler that checks if the values in a raster are within a valid range. :param raster: The raster to be validated. - :type raster: + :type raster: RasterMap :return: ``True`` if all values are within the valid range, ``False`` otherwise. :rtype: bool """ - def handle(self, raster): + def handle(self, raster, errors): if not raster.valid_range: self.logger.info("`ValueRange` validator skipped because no value range was set.") - return super().handle(raster) + return super().handle(raster, errors) min_value = raster.valid_range["min"] max_value = raster.valid_range["max"] - no_data_value = raster.bands[0].no_data_value - band_array = raster.bands[0].data_array - if no_data_value is not None: - band_array = band_array[band_array != no_data_value] + for band in raster.bands: + no_data_value = band.no_data_value + band_array = band.data_array - has_valid_range_condition = np.all((band_array >= min_value) & (band_array <= max_value)) + if no_data_value is not None: + band_array = band_array[band_array != no_data_value] - if not has_valid_range_condition: - return False + has_valid_range_condition = np.all( + (band_array >= min_value) & (band_array <= max_value) + ) - return super().handle(raster) + if not has_valid_range_condition: + errors.append(RasterDataRules.FORBID_OUT_OF_RANGE) + return False + + return super().handle(raster, errors) diff --git a/rubem/validation/raster_data_rules.py b/rubem/validation/raster_data_rules.py index 348b363..50ef48a 100644 --- a/rubem/validation/raster_data_rules.py +++ b/rubem/validation/raster_data_rules.py @@ -20,3 +20,11 @@ class RasterDataRules(Flag): """ None of the raster pixels must contain ``NO_DATA`` values. """ + + FORBID_OUT_OF_RANGE = auto() + """ + Raster pixels must be within the specified valid range. + """ + + def __str__(self): + return self.name.upper() diff --git a/rubem/validation/raster_data.py b/rubem/validation/raster_map_validator.py similarity index 93% rename from rubem/validation/raster_data.py rename to rubem/validation/raster_map_validator.py index 0f10f42..d151e6f 100644 --- a/rubem/validation/raster_data.py +++ b/rubem/validation/raster_map_validator.py @@ -34,4 +34,5 @@ def validate(self, raster): no_data.set_next(all_zeroes) all_zeroes.set_next(all_ones) - return value_range.handle(raster) + errors = [] + return value_range.handle(raster, errors), errors From f76e5b0fda4c46e6bc2c3e6ae6e2be7cbab60da2 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Tue, 20 Feb 2024 17:27:36 -0300 Subject: [PATCH 05/27] Update input file formats docs --- doc/source/fileformats.rst | 119 ++++++++++++++++++++++--------------- 1 file changed, 71 insertions(+), 48 deletions(-) diff --git a/doc/source/fileformats.rst b/doc/source/fileformats.rst index ceaf18a..f7a9d16 100644 --- a/doc/source/fileformats.rst +++ b/doc/source/fileformats.rst @@ -14,10 +14,12 @@ Mask of Catchment (Clone) raster This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster file through PCRASTER library. - Filetype: PCRaster map format :file:`*.map` raster file. -- Unit: boolean -- Restriction: +- Unit: Boolean +- Valid Range: :math:`[0.0, 1.0]` +- Restrictions: - - ``PCRASTER_VALUESCALE`` = ``VS_boolean`` + - ``PCRASTER_VALUESCALE`` = ``VS_BOOLEAN``; + - Raster pixels cannot consist entirely of ``0.0`` values. - Dimensions: @@ -30,10 +32,13 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster - Filetype: PCRaster map format :file:`*.map` raster file. - Unit: Scalar +- Valid Range: :math:`[-100.0, 10000.0]` - Restrictions: - ``PCRASTER_VALUESCALE`` = ``VS_SCALAR``; - - The file cannot contain a missing value for each pixel that is ``True`` (has a boolean 1 value) on the mask map. + - None of the pixels in the raster must contain ``NO_DATA`` value; + - Raster pixels cannot consist entirely of ``1.0`` values; + - Raster pixels cannot consist entirely of ``0.0`` values. - Dimensions: @@ -46,7 +51,13 @@ Digital Elevation Map (DEM) raster (TIFF) - Filetype: TIFF :file:`*.tif` raster file. - Unit: Scalar -- Restriction: +- Valid Range: :math:`[-100.0, 10000.0]` +- Restrictions: + + - ``PCRASTER_VALUESCALE`` = ``VS_SCALAR``; + - None of the pixels in the raster must contain ``NO_DATA`` value; + - Raster pixels cannot consist entirely of ``1.0`` values; + - Raster pixels cannot consist entirely of ``0.0`` values. - Dimensions: @@ -62,9 +73,10 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series - Filetype: PCRaster map format (:file:`etp00000.001`- :file:`etp99999.999` raster map series). - Unit: mm/month -- Restriction: +- Valid Range: :math:`[0.0, \infty]` +- Restrictions: - - The file cannot contain a missing value for each pixel that is ``True`` (has a boolean 1 value) on the mask map; + - None of the pixels in the raster must contain ``NO_DATA`` value; - Each month of the historical series corresponds to a :raw-html:`ETP` file. - Dimensions: @@ -89,9 +101,10 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series - Filetype: PCRaster map format (:file:`raf00000.001`- :file:`raf99999.999` raster map series). - Unit: mm/month -- Restriction: +- Valid Range: :math:`[0.0, \infty]` +- Restrictions: - - The file cannot contain a missing value for each pixel that is ``True`` (has a boolean 1 value) on the mask map; + - None of the pixels in the raster must contain ``NO_DATA`` value; - Each month of the historical series corresponds to a rainfall file. - Dimensions: @@ -116,9 +129,10 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series - Filetype: PCRaster map format (:file:`ndvi0000.001`- :file:`ndvi9999.999` raster map series). - Unit: Dimensionless -- Restriction: +- Valid Range: :math:`[-1.0, 1.0]` +- Restrictions: - - The file cannot contain a missing value for each pixel that is ``True`` (has a boolean 1 value) on the mask map; + - None of the pixels in the raster must contain ``NO_DATA`` value; - Each month of the historical series corresponds to a NDVI file. - Dimensions: @@ -145,9 +159,10 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series - Filetype: PCRaster map format (:file:`kpc00000.001`- :file:`kpc99999.999` raster map series). - Unit: Dimensionless -- Restriction: +- Valid Range: :math:`[0.0, 1.0]` +- Restrictions: - - The file cannot contain a missing value for each pixel that is ``True`` (has a boolean 1 value) on the mask map; + - None of the pixels in the raster must contain ``NO_DATA`` value; - Each month of the historical series corresponds to a :raw-html:`KP` file. - Dimensions: @@ -172,11 +187,13 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series - Filetype: PCRaster map format (:file:`luc00000.001`- :file:`luc99999.999` raster map series). - Unit: Nominal +- Valid Range: :math:`[0.0, \infty]` - Restrictions: - ``PCRASTER_VALUESCALE`` = ``VS_NOMINAL``; - - The file cannot contain a missing value for each pixel that is ``True`` (has a boolean 1 value) on the mask map; - - A cover file is required for each timestep of the historical series. + - None of the pixels in the raster must contain ``NO_DATA`` value; + - Raster pixels cannot consist entirely of ``0.0`` values; + - A LULC raster file is required for each timestep of the historical series. - Dimensions: @@ -199,10 +216,12 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster - Filetype: PCRaster map format :file:`*.map` raster file. - Unit: Nominal +- Valid Range: :math:`[0.0, \infty]` - Restrictions: - ``PCRASTER_VALUESCALE`` = ``VS_NOMINAL``; - - The file cannot contain a missing value for each pixel that is ``True`` (has a boolean 1 value) on the mask map. + - None of the pixels in the raster must contain ``NO_DATA`` value; + - Raster pixels cannot consist entirely of ``0.0`` values. - Dimensions: @@ -217,9 +236,11 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster - Filetype: PCRaster map format :file:`*.map` raster file. - Unit: Nominal -- Restriction: +- Valid Range: :math:`[0.0, \infty]` +- Restrictions: - - ``PCRASTER_VALUESCALE`` = ``VS_NOMINAL`` + - ``PCRASTER_VALUESCALE`` = ``VS_NOMINAL``; + - Raster pixels cannot consist entirely of ``0.0`` values. - Dimensions: @@ -234,10 +255,11 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster - Filetype: PCRaster map format :file:`*.map` raster file. - Unit: Dimensionless +- Valid Range: :math:`[-1.0, 1.0]` - Restrictions: - ``PCRASTER_VALUESCALE`` = ``VS_SCALAR``; - - The file cannot contain a missing value for each pixel that is ``True`` (has a boolean 1 value) on the mask map. + - None of the pixels in the raster must contain ``NO_DATA`` value. - Dimensions: @@ -252,10 +274,11 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster - Filetype: PCRaster map format :file:`*.map` raster file. - Unit:Dimensionless +- Valid Range: :math:`[-1.0, 1.0]` - Restrictions: - ``PCRASTER_VALUESCALE`` = ``VS_SCALAR``; - - The file cannot contain a missing value for each pixel that is ``True`` (has a boolean 1 value) on the mask map. + - None of the pixels in the raster must contain ``NO_DATA`` value. - Dimensions: @@ -266,9 +289,9 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster Monthly Rainy Days table ^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: rainy days/month -- Restriction: +- Restrictions: - 12 values, one for each month (mean value historic series) @@ -289,9 +312,9 @@ Monthly Rainy Days table Impervious Area Fraction (:raw-html:`ai`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: Dimensionless -- Restriction: +- Restrictions: - :math:`a_i + a_o + a_s + a_v = 1` @@ -312,9 +335,9 @@ Impervious Area Fraction (:raw-html:`ai`) table Open Water Area Fraction (:raw-html:`ao`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: Dimensionless -- Restriction: +- Restrictions: - :math:`a_i + a_o + a_s + a_v = 1` @@ -335,9 +358,9 @@ Open Water Area Fraction (:raw-html:`ao`) table Bare Soil Area Fraction (:raw-html:`as`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: Dimensionless -- Restriction: +- Restrictions: - :math:`a_i + a_o + a_s + a_v = 1` @@ -358,9 +381,9 @@ Bare Soil Area Fraction (:raw-html:`as`) table Vegetated Area Fraction (:raw-html:`av`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: Dimensionless -- Restriction: +- Restrictions: - :math:`a_i + a_o + a_s + a_v = 1` @@ -381,9 +404,9 @@ Vegetated Area Fraction (:raw-html:`av`) table Manning's Roughness Coefficient table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: Dimensionless -- Restriction: +- Restrictions: - One value for each soil class. @@ -404,9 +427,9 @@ Manning's Roughness Coefficient table Bulk Density table ^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: :raw-html:`g/cm3` -- Restriction: +- Restrictions: - One value for each soil class. @@ -427,9 +450,9 @@ Bulk Density table Saturated Hydraulic Conductivity (:raw-html:`KSAT`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: mm/month -- Restriction: +- Restrictions: - One value for each soil class. @@ -450,9 +473,9 @@ Saturated Hydraulic Conductivity (:raw-html:`KSAT`) table Field Capacity (:raw-html:`θFC`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: :raw-html:`θ (cm3/cm3)` -- Restriction: +- Restrictions: - One value for each soil class. @@ -473,9 +496,9 @@ Field Capacity (:raw-html:`θFC`) table Saturated Content (:raw-html:`θSAT`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: :raw-html:`θ (cm3/cm3)` -- Restriction: +- Restrictions: - One value for each soil class. @@ -496,9 +519,9 @@ Saturated Content (:raw-html:`θSAT`) table Wilting Point (:raw-html:`θWP`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: :raw-html:`θ (cm3/cm3)` -- Restriction: +- Restrictions: - One value for each soil class.. @@ -519,10 +542,10 @@ Wilting Point (:raw-html:`θWP`) table Depth Rootzone table ^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: cm -- Restriction: +- Restrictions: - One value for each soil class.. @@ -543,10 +566,10 @@ Depth Rootzone table Minimum Crop Coefficient (:raw-html:`KCMIN`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: Dimensionless -- Restriction: +- Restrictions: - :math:`K_{C_{MAX}} > K_{C_{MIN}}` @@ -568,10 +591,10 @@ Minimum Crop Coefficient (:raw-html:`KCMIN`) table Maximum Crop Coefficient (:raw-html:`KCMAX`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -- Filetype: Text :file:`*txt` or Comma-separated values (CSV) :file:`*.csv` file. +- Filetype: Text :file:`*.txt` or Comma-separated values (CSV) :file:`*.csv` file. - Unit: Dimensionless -- Restriction: +- Restrictions: - :math:`K_{C_{MAX}} > K_{C_{MIN}}` From 7bbdea5b50f358a4f621d113142882539bfcd521 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Tue, 20 Feb 2024 18:07:48 -0300 Subject: [PATCH 06/27] Update fileformats and userguide docs --- doc/source/conf.py | 26 +------ doc/source/fileformats.rst | 150 ++++++++++++++++++------------------- doc/source/userguide.rst | 43 +++++------ 3 files changed, 99 insertions(+), 120 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 5f93627..b9d27bb 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,25 +1,3 @@ -# coding=utf-8 -# RUBEM is a distributed hydrological model to calculate monthly -# flows with changes in land use over time. -# Copyright (C) 2020-2024 LabSid PHA EPUSP - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# Contact: rubem.hydrological@labsid.eng.br - -"""RUBEM help setup""" - # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full @@ -316,3 +294,7 @@ 1, ) ] + +# -- Options for Auto Section Label Extension ---------------------------------- + +autosectionlabel_prefix_document = True diff --git a/doc/source/fileformats.rst b/doc/source/fileformats.rst index f7a9d16..f0d11bf 100644 --- a/doc/source/fileformats.rst +++ b/doc/source/fileformats.rst @@ -31,7 +31,7 @@ Digital Elevation Map (DEM) raster This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster file through PCRASTER library. - Filetype: PCRaster map format :file:`*.map` raster file. -- Unit: Scalar +- Unit: `Meters Above Sea Level (MASL) `_ - Valid Range: :math:`[-100.0, 10000.0]` - Restrictions: @@ -42,15 +42,15 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Digital Elevation Map (DEM) raster (TIFF) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - Filetype: TIFF :file:`*.tif` raster file. -- Unit: Scalar +- Unit: `Meters Above Sea Level (MASL) `_ - Valid Range: :math:`[-100.0, 10000.0]` - Restrictions: @@ -61,9 +61,9 @@ Digital Elevation Map (DEM) raster (TIFF) - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Potential Evapotranspiration (:raw-html:`ETP`) raster series @@ -81,9 +81,9 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. .. note:: @@ -109,9 +109,9 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. .. note:: @@ -137,9 +137,9 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. .. note:: @@ -167,9 +167,9 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. .. note:: @@ -197,9 +197,9 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. .. note:: @@ -225,9 +225,9 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Stations (samples) raster ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -244,9 +244,9 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Maximum NDVI raster ^^^^^^^^^^^^^^^^^^^^ @@ -263,9 +263,9 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Minimum NDVI raster ^^^^^^^^^^^^^^^^^^^^ @@ -282,9 +282,9 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Monthly Rainy Days table ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -619,111 +619,111 @@ Output File Formats Total Interception raster series ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Resulting maps of Total Interception (ITP) [mm] in raster format for all simulation period for each pixel of :ref:`stations map `. +Resulting maps of Total Interception (ITP) [mm] in raster format for all simulation period for each pixel of :ref:`stations map `. - Filetype: PCRaster map format (:file:`itp00000.001`- :file:`itp99999.999` raster map series). - Unit: mm - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Baseflow raster series ^^^^^^^^^^^^^^^^^^^^^^^ -Resulting maps of Baseflow (BFW) [mm] in raster format for all simulation period or for each pixel of :ref:`stations map `. +Resulting maps of Baseflow (BFW) [mm] in raster format for all simulation period or for each pixel of :ref:`stations map `. - Filetype: PCRaster map format (:file:`bfw00000.001`- :file:`bfw99999.999` raster map series). - Unit: mm - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Surface Runoff raster series ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Resulting maps of Surface runoff (SRN) [mm] in raster format for all simulation period or for each pixel of :ref:`stations map `. +Resulting maps of Surface runoff (SRN) [mm] in raster format for all simulation period or for each pixel of :ref:`stations map `. - Filetype: PCRaster map format (:file:`srn00000.001`- :file:`srn99999.999` raster map series). - Unit: mm - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Actual Evapotranspiration raster series ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Resulting maps of Actual Evapotranspiration (ETA) [mm] in raster format for all simulation period or for each pixel of :ref:`stations map `. +Resulting maps of Actual Evapotranspiration (ETA) [mm] in raster format for all simulation period or for each pixel of :ref:`stations map `. - Filetype: PCRaster map format (:file:`eta00000.001`- :file:`eta99999.999` raster map series). - Unit: mm - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Lateral Flow raster series ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Resulting maps of Lateral Flow (LFW) [mm] in raster format for all simulation period for each pixel of :ref:`stations map `. +Resulting maps of Lateral Flow (LFW) [mm] in raster format for all simulation period for each pixel of :ref:`stations map `. - Filetype: PCRaster map format (:file:`lfw00000.001`- :file:`lfw99999.999` raster map series). - Unit: mm - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Recharge raster series ^^^^^^^^^^^^^^^^^^^^^^^ -Resulting maps of Recharge (REC) [mm] in raster format for all simulation period or for each pixel of :ref:`stations map `. +Resulting maps of Recharge (REC) [mm] in raster format for all simulation period or for each pixel of :ref:`stations map `. - Filetype: PCRaster map format (:file:`rec00000.001`- :file:`rec99999.999` raster map series). - Unit: mm - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Soil Moisture Content raster series ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Resulting maps of Soil Moisture Content (SMC) [mm] in raster format for all simulation period or for each pixel of :ref:`stations map `. +Resulting maps of Soil Moisture Content (SMC) [mm] in raster format for all simulation period or for each pixel of :ref:`stations map `. - Filetype: PCRaster map format (:file:`smc00000.001`- :file:`smc99999.999` raster map series). - Unit: mm - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Total Runoff raster series ^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Resulting maps of Total Runoff [:raw-html:`m3s-1`] in raster format for all simulation period for each pixel of :ref:`stations map `. +Resulting maps of Total Runoff [:raw-html:`m3s-1`] in raster format for all simulation period for each pixel of :ref:`stations map `. - Filetype: PCRaster map format (:file:`rnf00000.001`- :file:`rnf99999.999` raster map series). - Unit: :raw-html:`m3s-1` - Dimensions: - - Rows = :ref:`clone rows `; - - Columns = :ref:`clone columns`; - - Cell Size = :ref:`clone cell size`. + - Rows = :ref:`clone rows `; + - Columns = :ref:`clone columns`; + - Cell Size = :ref:`clone cell size`. Total Interception table ^^^^^^^^^^^^^^^^^^^^^^^^^ -Resulting values of Total Interception (ITP) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. +Resulting values of Total Interception (ITP) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. - Filetype: Comma-Separated Values (CSV) :file:`*.csv` - Unit: mm @@ -762,7 +762,7 @@ Resulting values of Total Interception (ITP) [mm] in table format for all simula Baseflow table ^^^^^^^^^^^^^^^ -Resulting maps of Baseflow (BFW) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. +Resulting maps of Baseflow (BFW) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. - Filetype: Comma-Separated Values (CSV) :file:`*.csv` - Unit: mm @@ -801,7 +801,7 @@ Resulting maps of Baseflow (BFW) [mm] in table format for all simulation period Surface Runoff table ^^^^^^^^^^^^^^^^^^^^^ -Resulting maps of Surface runoff (SRN) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. +Resulting maps of Surface runoff (SRN) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. - Filetype: Comma-Separated Values (CSV) :file:`*.csv` - Unit: mm @@ -840,7 +840,7 @@ Resulting maps of Surface runoff (SRN) [mm] in table format for all simulation Actual Evapotranspiration table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Resulting maps of Actual Evapotranspiration (ETA) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. +Resulting maps of Actual Evapotranspiration (ETA) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. - Filetype: Comma-Separated Values (CSV) :file:`*.csv` - Unit: mm @@ -879,7 +879,7 @@ Resulting maps of Actual Evapotranspiration (ETA) [mm] in table format for all s Lateral Flow table ^^^^^^^^^^^^^^^^^^^ -Resulting maps of Lateral Flow (LFW) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. +Resulting maps of Lateral Flow (LFW) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. - Filetype: Comma-Separated Values (CSV) :file:`*.csv` - Unit: mm @@ -918,7 +918,7 @@ Resulting maps of Lateral Flow (LFW) [mm] in table format for all simulation per Recharge table ^^^^^^^^^^^^^^^ -Resulting maps of Recharge (REC) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. +Resulting maps of Recharge (REC) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. - Filetype: Comma-Separated Values (CSV) :file:`*.csv` - Unit: mm @@ -957,7 +957,7 @@ Resulting maps of Recharge (REC) [mm] in table format for all simulation period Soil Moisture Content table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Resulting maps of Soil Moisture Content (SMC) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. +Resulting maps of Soil Moisture Content (SMC) [mm] in table format for all simulation period for each sampling station present in :ref:`stations map `. - Filetype: Comma-Separated Values (CSV) :file:`*.csv` - Unit: mm @@ -996,7 +996,7 @@ Resulting maps of Soil Moisture Content (SMC) [mm] in table format for all simul Total Runoff table ^^^^^^^^^^^^^^^^^^^ -Resulting maps of Total Runoff [:raw-html:`m3s-1`] in table format for all simulation period for each sampling station present in :ref:`stations map `. +Resulting maps of Total Runoff [:raw-html:`m3s-1`] in table format for all simulation period for each sampling station present in :ref:`stations map `. - Filetype: Comma-Separated Values (CSV) :file:`*.csv` - Unit: :raw-html:`m3s-1` diff --git a/doc/source/userguide.rst b/doc/source/userguide.rst index fb60785..90cab7c 100644 --- a/doc/source/userguide.rst +++ b/doc/source/userguide.rst @@ -9,7 +9,7 @@ Inputs To run RUBEM, several sets of input data are required. Check the :doc:`specification ` and :doc:`preprocessing ` of the model's input files. -These input files are listed in a model configuration file that points to the locations of these input files and parameter sets that govern the simulation. Check a model configuration :ref:`template file `. +These input files are listed in a model configuration file that points to the locations of these input files and parameter sets that govern the simulation. Check a model configuration :ref:`template file `. Model General Settings ---------------------- @@ -40,9 +40,9 @@ Mandatory path to the output folder. Must be a valid path to an existing **empty Digital Elevation Map (DEM) ``````````````````````````` -Mandatory path to Digital Elevation Map (DEM) file [masl] in: +Mandatory path to Digital Elevation Map (DEM) file `[masl] `_ in: - * **PCRaster map format** :file:`*.map`: this map contains topographic ground elevation in meters. Must be a valid file path to a PCRaster map format :file:`*.map` file; + * **PCRaster map format** :file:`*.map`: this map contains topographic ground elevation in meters. Must be a valid file path to a PCRaster map format :file:`*.map` file. :ref:`See more. ` .. code-block:: dosini @@ -51,7 +51,7 @@ Mandatory path to Digital Elevation Map (DEM) file [masl] in: ------- - * **TIF format** :file:`*.tif`: this map contains topographic ground elevation in meters. Must be a valid file path to a TIF :file:`*.tif` raster file. + * **TIF format** :file:`*.tif`: this map contains topographic ground elevation in meters. Must be a valid file path to a TIF :file:`*.tif` raster file. :ref:`See more. ` .. code-block:: dosini @@ -61,7 +61,7 @@ Mandatory path to Digital Elevation Map (DEM) file [masl] in: Mask of Catchment (Clone) `````````````````````````` -Mandatory path to Mask of Catchment (Clone) file in PCRaster map format :file:`*.map`. This map defines the basin area to simulate in the model. Must be a valid file path to a PCRaster boolean map format file; +Mandatory path to Mask of Catchment (Clone) file in PCRaster map format :file:`*.map`. This map defines the basin area to simulate in the model. Must be a valid file path to a PCRaster boolean map format file. :ref:`See more. ` .. code-block:: dosini @@ -75,7 +75,7 @@ Gauge Station Location Map Export Results to Station Locations Map ''''''''''''''''''''''''''''''''''''''' -Optional, if enabled, export time series data of selected output variables (comma-separated values :file:`*.csv` files) for each valid pixel in stations maps. A station location map file must be defined; +Optional, if enabled, export time series data of selected output variables (comma-separated values :file:`*.csv` files) for each valid pixel in stations maps. :ref:`A station location map file must be defined. ` .. code-block:: dosini @@ -85,7 +85,7 @@ Optional, if enabled, export time series data of selected output variables (comm Stations Locations (Samples) '''''''''''''''''''''''''''' -Mandatory if ``Export Results to Station Locations`` is enabled. Path to Stations file in PCRaster map format :file:`*.map` and nominal format. This file is a nominal map with unique Ids for cells identified as being a location where time-series output is required. Non-station cells have a value of ``-9999``. Must be a valid path to an existing PCRaster map format :file:`*.map` file. +Mandatory if ``Export Results to Station Locations`` is enabled. Path to Stations file in PCRaster map format :file:`*.map` and nominal format. This file is a nominal map with unique Ids for cells identified as being a location where time-series output is required. Non-station cells have a value of ``-9999``. Must be a valid path to an existing PCRaster map format :file:`*.map` file. :ref:`See more. ` .. code-block:: dosini @@ -108,7 +108,7 @@ Simulation Period Start Date '''''''''' -Mandatory date of the first time step of the simulation scenario (month and year of the start period of simulation scenario); +Mandatory date of the first time step of the simulation scenario (day, month and year of the start period of simulation scenario). Start date must be before the end date. .. code-block:: dosini @@ -118,7 +118,7 @@ Mandatory date of the first time step of the simulation scenario (month and year End Date '''''''' -Mandatory date of the last time step of the simulation scenario (month and year of the last period of simulation scenario). +Mandatory date of the last time step of the simulation scenario (day, month and year of the last period of simulation scenario). End date must be after the start date. .. code-block:: dosini @@ -127,7 +127,7 @@ Mandatory date of the last time step of the simulation scenario (month and year .. note:: - Both dates must be valid and fall within between the time period of the dataset input time scale. The ``end`` date must be greater than the ``start`` date. + Both dates must be valid and fall within between the time period of the dataset input time range. The ``end`` date must be greater than the ``start`` date. Soil Parameters @@ -136,7 +136,7 @@ Soil Parameters Soil Map ```````` -Mandatory path to Soil map in PCRaster map format :file:`*.map` and nominal format. It represents the soil classes of the study area. The number of classes is defined by the user and is related to hydraulic properties. Must be a valid path to an existing PCRaster map format :file:`*.map` file. +Mandatory path to Soil map in PCRaster map format :file:`*.map` and nominal format. It represents the soil classes of the study area. The number of classes is defined by the user and is related to hydraulic properties. Must be a valid path to an existing PCRaster map format :file:`*.map` file. :ref:`See more. ` .. code-block:: dosini @@ -146,7 +146,7 @@ Mandatory path to Soil map in PCRaster map format :file:`*.map` and nominal form Bulk Density ```````````` -Mandatory path to a tabular file with values :raw-html:`[g/cm3]` of Bulk density for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to a tabular file with values :raw-html:`[g/cm3]` of Bulk density for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini @@ -156,8 +156,7 @@ Mandatory path to a tabular file with values :raw-html:`[g/cm3]` of B :raw-html:`Saturated Hydraulic Conductivity (KSAT)` ```````````````````````````````````````````````````````````````````````````````` -Mandatory path to a tabular file with values [mm/month] of saturated hydraulic conductivity for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. - +Mandatory path to a tabular file with values [mm/month] of saturated hydraulic conductivity for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. .. code-block:: dosini [TABLES] @@ -196,7 +195,7 @@ Mandatory path to a tabular file with values :raw-html:`[θ (cm3/cm` .. code-block:: dosini @@ -294,7 +293,7 @@ Land Use Map-series A map-series in PCRaster always starts with the :file:`*.001` extension, corresponding with the start date of your model simulation period. According to `PCRaster documentation `_ the name of each of the files in the series should have eight characters before the dot, and 3 characters after the dot. The name of each map starts with a prefix, and ends with the number of the time step. All characters in between are filled with zeroes. -Mandatory path to a directory containing the land use map-series. The directory containing these files must contain the maps that representing the mean monthly LUC, where each map represents the variable's value at a particular time step. If some \*.00\* file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. +Mandatory path to a directory containing the land use map-series. The directory containing these files must contain the maps that representing the mean monthly LUC, where each map represents the variable's value at a particular time step. If some \*.00\* file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. :ref:`See more. ` .. code-block:: dosini @@ -308,7 +307,7 @@ Mandatory path to a directory containing the land use map-series. The directory Manning's Roughness Coefficient ```````````````````````````````` -Mandatory path to a tabular file with values of Manning's roughness coefficient values for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to a tabular file with values of Manning's roughness coefficient values for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini @@ -351,9 +350,7 @@ NDVI Map-series A map-series in PCRaster always starts with the :file:`*.001` extension, corresponding with the start date of your model simulation period. According to `PCRaster documentation `_ the name of each of the files in the series should have eight characters before the dot, and 3 characters after the dot. The name of each map starts with a prefix, and ends with the number of the time step. All characters in between are filled with zeroes. -Mandatory path to a directory containing the land use map-series. The directory containing these files must contain the maps that representing the mean monthly LUC, where each map represents the variable's value at a particular time step. If some \*.00\* file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. - -Mandatory path to a directory containing the monthly Normalized Difference Vegetation Index (NDVI) map-series format. The directory containing these files must contain the maps representing the mean monthly NDVI, where each map represents the variable's value at a particular time step. If some \*.00\* file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. +Mandatory path to a directory containing the monthly Normalized Difference Vegetation Index (NDVI) map-series format. The directory containing these files must contain the maps representing the mean monthly NDVI, where each map represents the variable's value at a particular time step. If some \*.00\* file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. :ref:`See more. ` .. code-block:: dosini @@ -366,7 +363,7 @@ Mandatory path to a directory containing the monthly Normalized Difference Veget Maximum NDVI Map ''''''''''''''''' -Mandatory path to maximum NDVI file in PCRaster map format :file:`*.map`. This file is a scalar pcraster map with values for each cell, representing the maximum value of NDVI in the historic series available for the cell. Must be a valid path to an existing PCRaster map format :file:`*.map` file. +Mandatory path to maximum NDVI file in PCRaster map format :file:`*.map`. This file is a scalar pcraster map with values for each cell, representing the maximum value of NDVI in the historic series available for the cell. Must be a valid path to an existing PCRaster map format :file:`*.map` file. :ref:`See more. ` .. code-block:: dosini @@ -376,7 +373,7 @@ Mandatory path to maximum NDVI file in PCRaster map format :file:`*.map`. This f Minimum NDVI Map '''''''''''''''' -Mandatory path to minimum NDVI file in PCRaster map format :file:`*.map`. This file is a scalar pcraster map with values for each cell, representing the minimum value of NDVI in the historic series available for the cell. Must be a valid path to an existing PCRaster map format :file:`*.map` file. +Mandatory path to minimum NDVI file in PCRaster map format :file:`*.map`. This file is a scalar pcraster map with values for each cell, representing the minimum value of NDVI in the historic series available for the cell. Must be a valid path to an existing PCRaster map format :file:`*.map` file. :ref:`See more. ` .. code-block:: dosini @@ -433,7 +430,7 @@ Crop Coefficient (K\ :sub:`C`\) :raw-html:`Maximum KC` '''''''''''''''''''''''''''''''''''''''''''''''''''' -Mandatory path to a tabular file with values of maximum crop coefficient for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to a tabular file with values of maximum crop coefficient for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. .. code-block:: dosini From 9bf1565cbf4c42c51757eee915865cb272d20343 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Tue, 20 Feb 2024 18:27:26 -0300 Subject: [PATCH 07/27] Remove VS project files --- RUBEM.pyproj | 236 --------------------------------------------------- RUBEM.sln | 23 ----- 2 files changed, 259 deletions(-) delete mode 100644 RUBEM.pyproj delete mode 100644 RUBEM.sln diff --git a/RUBEM.pyproj b/RUBEM.pyproj deleted file mode 100644 index ab9e7a8..0000000 --- a/RUBEM.pyproj +++ /dev/null @@ -1,236 +0,0 @@ - - - - Debug - 2.0 - {dee732fc-1583-4bc5-9a0e-da9293924f68} - - - - . - . - {888888a0-9f3d-457c-b088-3a5042f75d52} - Standard Python launcher - - - - - - 10.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/RUBEM.sln b/RUBEM.sln deleted file mode 100644 index c7109a2..0000000 --- a/RUBEM.sln +++ /dev/null @@ -1,23 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.6.33712.159 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "RUBEM", "RUBEM.pyproj", "{DEE732FC-1583-4BC5-9A0E-DA9293924F68}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {DEE732FC-1583-4BC5-9A0E-DA9293924F68}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DEE732FC-1583-4BC5-9A0E-DA9293924F68}.Release|Any CPU.ActiveCfg = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {0BCA7672-3D62-4810-A858-5E4543B739F8} - EndGlobalSection -EndGlobal From 968257143e8d7d6f9f4e497557a03cd0401d0cd5 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Wed, 21 Feb 2024 13:30:50 -0300 Subject: [PATCH 08/27] Update userguide and fileformats doc pages --- doc/source/fileformats.rst | 27 +++++++++ doc/source/userguide.rst | 121 ++++++++++++++++++------------------- 2 files changed, 87 insertions(+), 61 deletions(-) diff --git a/doc/source/fileformats.rst b/doc/source/fileformats.rst index f0d11bf..04bae98 100644 --- a/doc/source/fileformats.rst +++ b/doc/source/fileformats.rst @@ -66,6 +66,8 @@ Digital Elevation Map (DEM) raster (TIFF) - Cell Size = :ref:`clone cell size`. +.. _potential-evapotranspiration-raster-series: + Potential Evapotranspiration (:raw-html:`ETP`) raster series ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -94,6 +96,8 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series The format of each individual forcing file should have eight characters before the dot, and 3 characters after the dot. The name of each map starts with a prefix, and ends with the number of the time step. All characters in between are filled with zeroes. `Related PCRaster documentation `__. +.. _rainfall-raster-series: + Rainfall (:raw-html:`PM`) raster series ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -150,6 +154,8 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series The format of each individual forcing file should have eight characters before the dot, and 3 characters after the dot.The name of each map starts with a prefix, and ends with the number of the time step. All characters in between are filled with zeroes. `Related PCRaster documentation `__. +.. _class-a-pan-coefficient-raster-series: + Class A Pan Coefficient (:raw-html:`KP`) raster series ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -193,6 +199,7 @@ These files are the result of pre-processing the TIFF/GeoTIFF raster file series - ``PCRASTER_VALUESCALE`` = ``VS_NOMINAL``; - None of the pixels in the raster must contain ``NO_DATA`` value; - Raster pixels cannot consist entirely of ``0.0`` values; + - LULC map values must adhere strictly to values specified within the land use parameters tables (:ref:`Manning's Roughness Coefficient `, :ref:`Impervious Area Fraction `, :ref:`Open Water Area Fraction `, :ref:`Bare Soil Area Fraction `, :ref:`Vegetated Area Fraction `, :ref:`Max. Crop Coefficient ` and :ref:`Min. Crop Coefficient `), without exceptions; - A LULC raster file is required for each timestep of the historical series. - Dimensions: @@ -221,6 +228,7 @@ This file is the result of pre-processing the corresponding TIFF/GeoTIFF raster - ``PCRASTER_VALUESCALE`` = ``VS_NOMINAL``; - None of the pixels in the raster must contain ``NO_DATA`` value; + - Soil map values must adhere strictly to values specified within the soil parameters tables (:ref:`Bulk Density `, :ref:`Saturated Hydraulic Conductivity `, :ref:`Field Capacity `, :ref:`Wilting Point `, :ref:`Saturated Content ` and :ref:`Depth Rootzone `), without exceptions; - Raster pixels cannot consist entirely of ``0.0`` values. - Dimensions: @@ -309,6 +317,8 @@ Monthly Rainy Days table * - Int <1-12> - Int <1-31> +.. _impervious-area-fraction-table: + Impervious Area Fraction (:raw-html:`ai`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -332,6 +342,8 @@ Impervious Area Fraction (:raw-html:`ai`) table * - Int <1-\*> - Float <\*> +.. _open-water-area-fraction-table: + Open Water Area Fraction (:raw-html:`ao`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -355,6 +367,8 @@ Open Water Area Fraction (:raw-html:`ao`) table * - Int <1-\*> - Float <\*> +.. _bare-soil-area-fraction-table: + Bare Soil Area Fraction (:raw-html:`as`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -378,6 +392,8 @@ Bare Soil Area Fraction (:raw-html:`as`) table * - Int <1-\*> - Float <\*> +.. _vegetated-area-fraction-table: + Vegetated Area Fraction (:raw-html:`av`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -447,6 +463,8 @@ Bulk Density table * - Int <1-\*> - Float <\*> +.. _saturated-hydraulic-conductivity-table: + Saturated Hydraulic Conductivity (:raw-html:`KSAT`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -470,6 +488,8 @@ Saturated Hydraulic Conductivity (:raw-html:`KSAT`) table * - Int <1-\*> - Float <\*> +.. _field-capacity-table: + Field Capacity (:raw-html:`θFC`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -493,6 +513,8 @@ Field Capacity (:raw-html:`θFC`) table * - Int <1-\*> - Float <\*> +.. _saturated-content-table: + Saturated Content (:raw-html:`θSAT`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -516,6 +538,8 @@ Saturated Content (:raw-html:`θSAT`) table * - Int <1-\*> - Float <\*> +.. _wilting-point-table: + Wilting Point (:raw-html:`θWP`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -563,6 +587,8 @@ Depth Rootzone table * - Int <1-\*> - Float <\*> +.. _minimum-crop-coefficient-table: + Minimum Crop Coefficient (:raw-html:`KCMIN`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -587,6 +613,7 @@ Minimum Crop Coefficient (:raw-html:`KCMIN`) table * - Int <1-\*> - Float <\*> +.. _maximum-crop-coefficient-table: Maximum Crop Coefficient (:raw-html:`KCMAX`) table ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/source/userguide.rst b/doc/source/userguide.rst index 90cab7c..2ea3a4c 100644 --- a/doc/source/userguide.rst +++ b/doc/source/userguide.rst @@ -156,7 +156,7 @@ Mandatory path to a tabular file with values :raw-html:`[g/cm3]` of B :raw-html:`Saturated Hydraulic Conductivity (KSAT)` ```````````````````````````````````````````````````````````````````````````````` -Mandatory path to a tabular file with values [mm/month] of saturated hydraulic conductivity for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. +Mandatory path to a tabular file with values [mm/month] of saturated hydraulic conductivity for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini [TABLES] @@ -165,7 +165,7 @@ Mandatory path to a tabular file with values [mm/month] of saturated hydraulic c :raw-html:`Field Capacity (θFC)` ````````````````````````````````````````````````````````````` -Mandatory path to a tabular file with values :raw-html:`[θ (cm3/cm3)]` of field capacity water content (θ) for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to a tabular file with values :raw-html:`[θ (cm3/cm3)]` of field capacity water content (θ) for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini @@ -175,7 +175,7 @@ Mandatory path to a tabular file with values :raw-html:`[θ (cm3/cmWP)` ``````````````````````````````````````````````````````````` -Mandatory path to a tabular file with values :raw-html:`[θ (cm3/cm3)]` of Wilting Point for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to a tabular file with values :raw-html:`[θ (cm3/cm3)]` of Wilting Point for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini @@ -185,7 +185,7 @@ Mandatory path to a tabular file with values :raw-html:`[θ (cm3/cmSAT)` ```````````````````````````````````````````````````````````````` -Mandatory path to a tabular file with values :raw-html:`[θ (cm3/cm3)]` of saturated content for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to a tabular file with values :raw-html:`[θ (cm3/cm3)]` of saturated content for each soil class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini @@ -208,7 +208,7 @@ Initial Soil Conditions Initial Baseflow '''''''''''''''' -Mandatory float value [mm] representing the baseflow (in the cell) at the beginning of the simulation. See :ref:`baseflow-overview-section` for more details. +Mandatory float value [mm] representing the baseflow (in the cell) at the beginning of the simulation. See :ref:`overview:Baseflow` for more details. .. math:: :label: initialbaseflow @@ -237,7 +237,7 @@ Baseflow Threshold Mandatory float value [mm] representing the minimum water store in the saturated zone for generating Baseflow. See :ref:`baseflow-overview-section` for more details. It can be set using the minimum discharge at the gauge station by the relation: .. math:: - :label: initialbaseflow + :label: baseflowthreshold :nowrap: \[BF_{thresh} = \frac{Q_{min} \cdot t}{A \cdot N_{cell}} \cdot 10^{-3}\] @@ -293,7 +293,7 @@ Land Use Map-series A map-series in PCRaster always starts with the :file:`*.001` extension, corresponding with the start date of your model simulation period. According to `PCRaster documentation `_ the name of each of the files in the series should have eight characters before the dot, and 3 characters after the dot. The name of each map starts with a prefix, and ends with the number of the time step. All characters in between are filled with zeroes. -Mandatory path to a directory containing the land use map-series. The directory containing these files must contain the maps that representing the mean monthly LUC, where each map represents the variable's value at a particular time step. If some \*.00\* file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. :ref:`See more. ` +Mandatory path to a directory containing the land use map-series. The directory containing these files must contain the maps that representing the mean monthly LUC, where each map represents the variable's value at a particular time step. If some file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. :ref:`See more. ` .. code-block:: dosini @@ -303,41 +303,6 @@ Mandatory path to a directory containing the land use map-series. The directory [FILENAME_PREFIXES] landuse_prefix = ldu - -Manning's Roughness Coefficient -```````````````````````````````` - -Mandatory path to a tabular file with values of Manning's roughness coefficient values for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` - -.. code-block:: dosini - - [TABLES] - manning = /Dataset/UIGCRB/input/txt/landuse/manning.txt - -:raw-html:`Maximum Leaf Area Index (LAIMAX)` -```````````````````````````````````````````````````````````````````````` - -Mandatory maximum float value [dimensionless quantity] that characterizes plant canopies. It is defined as the one-sided green leaf area per unit ground surface area. - -.. math:: 1 \leq LAI_{MAX} \leq 12 - -.. code-block:: dosini - - [CONSTANTS] - lai_max = 12.0 - -:raw-html:`Impervious Area Interception (II)` -`````````````````````````````````````````````````````````````````````````` - -Mandatory float value [mm] that represents the rainfall interception in impervious areas. - -.. math:: 1 < I_I < 3 - -.. code-block:: dosini - - [CONSTANTS] - i_imp = 2.5 - Normalized Difference Vegetation Index (NDVI) ````````````````````````````````````````````` @@ -350,7 +315,7 @@ NDVI Map-series A map-series in PCRaster always starts with the :file:`*.001` extension, corresponding with the start date of your model simulation period. According to `PCRaster documentation `_ the name of each of the files in the series should have eight characters before the dot, and 3 characters after the dot. The name of each map starts with a prefix, and ends with the number of the time step. All characters in between are filled with zeroes. -Mandatory path to a directory containing the monthly Normalized Difference Vegetation Index (NDVI) map-series format. The directory containing these files must contain the maps representing the mean monthly NDVI, where each map represents the variable's value at a particular time step. If some \*.00\* file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. :ref:`See more. ` +Mandatory path to a directory containing the monthly Normalized Difference Vegetation Index (NDVI) map-series format. The directory containing these files must contain the maps representing the mean monthly NDVI, where each map represents the variable's value at a particular time step. If some file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. :ref:`See more. ` .. code-block:: dosini @@ -380,13 +345,23 @@ Mandatory path to minimum NDVI file in PCRaster map format :file:`*.map`. This f [RASTERS] ndvi_min = /Dataset/UIGCRB/input/maps/ndvi/ndvi_min.map +Manning's Roughness Coefficient +```````````````````````````````` + +Mandatory path to a tabular file with values of Manning's roughness coefficient values for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` + +.. code-block:: dosini + + [TABLES] + manning = /Dataset/UIGCRB/input/txt/landuse/manning.txt + Area Fractions `````````````` Impervious Area Fraction (ai) '''''''''''''''''''''''''''''' -Mandatory path to file with values of fraction of impervious surface area values for each land-use class. This file is a text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv` with values, representing the fraction of impervious surface area for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to file with values of fraction of impervious surface area values for each land-use class. This file is a text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv` with values, representing the fraction of impervious surface area for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini @@ -396,7 +371,7 @@ Mandatory path to file with values of fraction of impervious surface area values Open Water Area Fraction (ao) '''''''''''''''''''''''''''''' -Mandatory path to file with values of fraction of open-water area values for each land-use class. This file is a text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv` with values, representing the fraction of open-water area for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to file with values of fraction of open-water area values for each land-use class. This file is a text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv` with values, representing the fraction of open-water area for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini @@ -406,7 +381,7 @@ Mandatory path to file with values of fraction of open-water area values for eac Bare Soil Area Fraction (as) ''''''''''''''''''''''''''''' -Mandatory path to file with values of fraction of bare soil area values for each land-use class. This file is a text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv` with values, representing the fraction of bare soil area for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to file with values of fraction of bare soil area values for each land-use class. This file is a text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv` with values, representing the fraction of bare soil area for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini @@ -416,7 +391,7 @@ Mandatory path to file with values of fraction of bare soil area values for each Vegetated Area Fraction (av) '''''''''''''''''''''''''''' -Mandatory path to file with values of fraction of vegetated area values for each land-use class. This file is a text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv` with values, representing the fraction of vegetated area for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to file with values of fraction of vegetated area values for each land-use class. This file is a text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv` with values, representing the fraction of vegetated area for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini @@ -430,7 +405,7 @@ Crop Coefficient (K\ :sub:`C`\) :raw-html:`Maximum KC` '''''''''''''''''''''''''''''''''''''''''''''''''''' -Mandatory path to a tabular file with values of maximum crop coefficient for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to a tabular file with values of maximum crop coefficient for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini @@ -440,13 +415,37 @@ Mandatory path to a tabular file with values of maximum crop coefficient for eac :raw-html:`Minimum KC` '''''''''''''''''''''''''''''''''''''''''''''''''''' -Mandatory path to a tabular file with values of minimum crop coefficient for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to a tabular file with values of minimum crop coefficient for each land-use class. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini [TABLES] K_c_min = /Dataset/UIGCRB/input/txt/landuse/kcmin.txt +:raw-html:`Maximum Leaf Area Index (LAIMAX)` +```````````````````````````````````````````````````````````````````````` + +Mandatory maximum float value [dimensionless quantity] that characterizes plant canopies. It is defined as the one-sided green leaf area per unit ground surface area. + +.. math:: 1 \leq LAI_{MAX} \leq 12 + +.. code-block:: dosini + + [CONSTANTS] + lai_max = 12.0 + +:raw-html:`Impervious Area Interception (II)` +`````````````````````````````````````````````````````````````````````````` + +Mandatory float value [mm] that represents the rainfall interception in impervious areas. + +.. math:: 1 < I_I < 3 + +.. code-block:: dosini + + [CONSTANTS] + i_imp = 2.5 + Fraction Photosynthetically Active Radiation (FPAR) ``````````````````````````````````````````````````` @@ -488,7 +487,7 @@ Climate Data Series :raw-html:`Monthly Rainfall (PM)` ```````````````````````````````````````````` -Mandatory path to a directory containing the Monthly Rainfall map-series format [mm/month]. The directory containing these files must contain the maps representing the variable's value at a particular time step the mean monthly :raw-html:`PM`, where each map represents the variable's value at a particular time step. If some \*.00\* file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. +Mandatory path to a directory containing the Monthly Rainfall map-series format [mm/month]. The directory containing these files must contain the maps representing the variable's value at a particular time step the mean monthly :raw-html:`PM`, where each map represents the variable's value at a particular time step. If some file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. :ref:`See more. ` .. code-block:: dosini @@ -501,7 +500,7 @@ Mandatory path to a directory containing the Monthly Rainfall map-series format :raw-html:`Monthly Potential Evapotranspiration (ETP)` `````````````````````````````````````````````````````````````````` -Mandatory path to a directory containing the Monthly Potential Evapotranspiration map-series format [mm/month]. The directory containing these files must contain the maps representing the mean monthly :raw-html:`ETP`, where each map represents the variable's value at a particular time step. If some \*.00\* file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. +Mandatory path to a directory containing the Monthly Potential Evapotranspiration map-series format [mm/month]. The directory containing these files must contain the maps representing the mean monthly :raw-html:`ETP`, where each map represents the variable's value at a particular time step. If some file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. :ref:`See more. ` .. code-block:: dosini @@ -514,7 +513,7 @@ Mandatory path to a directory containing the Monthly Potential Evapotranspiratio :raw-html:`Class A Pan Coefficient (KP)` ```````````````````````````````````````````````````` -Mandatory path to a directory containing the Class A Pan Coefficient map-series format[mm/month]. The directory containing these files must contain the maps representing the mean monthly :raw-html:`KP`, where each map represents the variable's value at a particular time step. If some \*.00\* file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. +Mandatory path to a directory containing the Class A Pan Coefficient map-series format[mm/month]. The directory containing these files must contain the maps representing the mean monthly :raw-html:`KP`, where each map represents the variable's value at a particular time step. If some file is missing, the map of the previous step will be used. Must be a valid path to an existing directory. Note that it is also necessary to indicate the prefix of the filenames of the series. :ref:`See more. ` .. code-block:: dosini @@ -527,7 +526,7 @@ Mandatory path to a directory containing the Class A Pan Coefficient map-series Monthly Rainy Days ``````````````````` -Mandatory path to a tabular file [days/month] with values representing the mean value of rainy days for each month of the simulation period. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. +Mandatory path to a tabular file [days/month] with values representing the mean value of rainy days for each month of the simulation period. Must be a valid path to an existing text file :file:`*.txt` or comma-separated values (CSV) file :file:`*.csv`. :ref:`See more. ` .. code-block:: dosini @@ -698,7 +697,7 @@ Model Output Parameters Total Interception `````````````````` -Optional boolean value. If enabled, this option allows the generation of Total Interception (ITP) [mm] result maps in raster format for each of the time steps included in the simulation period. +Optional boolean value. If enabled, this option allows the generation of Total Interception (ITP) [mm] result maps in raster format for each of the time steps included in the simulation period. :ref:`See more. ` .. code-block:: dosini @@ -708,7 +707,7 @@ Optional boolean value. If enabled, this option allows the generation of Total I Baseflow ```````` -Optional boolean value. If enabled, this option allows the generation of Baseflow (BFW) [mm] result maps in raster format for each of the time steps included in the simulation period. +Optional boolean value. If enabled, this option allows the generation of Baseflow (BFW) [mm] result maps in raster format for each of the time steps included in the simulation period. :ref:`See more. ` .. code-block:: dosini @@ -718,7 +717,7 @@ Optional boolean value. If enabled, this option allows the generation of Basefl Surface Runoff `````````````` -Optional boolean value. If enabled, this option allows the generation of Surface runoff (SRN) [mm] result maps in raster format for each of the time steps included in the simulation period. +Optional boolean value. If enabled, this option allows the generation of Surface runoff (SRN) [mm] result maps in raster format for each of the time steps included in the simulation period. :ref:`See more. ` .. code-block:: dosini @@ -728,7 +727,7 @@ Optional boolean value. If enabled, this option allows the generation of Surfac Actual Evapotranspiration `````````````````````````` -Optional boolean value. If enabled, this option allows the generation of Actual Evapotranspiration (ETA) [mm] result maps in raster format for each of the time steps included in the simulation period. +Optional boolean value. If enabled, this option allows the generation of Actual Evapotranspiration (ETA) [mm] result maps in raster format for each of the time steps included in the simulation period. :ref:`See more. ` .. code-block:: dosini @@ -739,7 +738,7 @@ Optional boolean value. If enabled, this option allows the generation of Actual Lateral Flow ```````````` -Optional boolean value. If enabled, this option allows to generate the resulting maps of Lateral Flow (LFW) [mm] result maps in raster format for each of the time steps included in the simulation period. +Optional boolean value. If enabled, this option allows to generate the resulting maps of Lateral Flow (LFW) [mm] result maps in raster format for each of the time steps included in the simulation period. :ref:`See more. ` .. code-block:: dosini @@ -749,7 +748,7 @@ Optional boolean value. If enabled, this option allows to generate the resultin Recharge ````````` -Optional boolean value. If enabled, this option allows the generation of Recharge (REC) [mm] result maps in raster format for each of the time steps included in the simulation period. +Optional boolean value. If enabled, this option allows the generation of Recharge (REC) [mm] result maps in raster format for each of the time steps included in the simulation period. :ref:`See more. ` .. code-block:: dosini @@ -759,7 +758,7 @@ Optional boolean value. If enabled, this option allows the generation of Recharg Soil Moisture Content `````````````````````` -Optional boolean value. If enabled, this option allows the generation of Soil Moisture Content (SMC) [mm] result maps in raster format for each of the time steps included in the simulation period. +Optional boolean value. If enabled, this option allows the generation of Soil Moisture Content (SMC) [mm] result maps in raster format for each of the time steps included in the simulation period. :ref:`See more. ` .. code-block:: dosini @@ -769,7 +768,7 @@ Optional boolean value. If enabled, this option allows the generation of Soil Mo Total Runoff ```````````` -Optional boolean value. If enabled, this option allows the generation of Total Runoff [:raw-html:`m3s-1`] result maps in raster format for each of the time steps included in the simulation period. +Optional boolean value. If enabled, this option allows the generation of Total Runoff [:raw-html:`m3s-1`] result maps in raster format for each of the time steps included in the simulation period. :ref:`See more. ` .. code-block:: dosini From 32eabb104a74c58a377696dc1eca0587a5b20fb5 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:58:26 -0300 Subject: [PATCH 09/27] Update output file format constants --- .../configuration/test_output_variables.py | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/tests/unit/configuration/test_output_variables.py b/tests/unit/configuration/test_output_variables.py index 91ab4fc..09fe383 100644 --- a/tests/unit/configuration/test_output_variables.py +++ b/tests/unit/configuration/test_output_variables.py @@ -11,10 +11,32 @@ class TestOutputVariables: @pytest.mark.parametrize( "itp, bfw, srn, eta, lfw, rec, smc, rnf, tss, output_format", [ - (True, True, True, True, True, True, True, True, True, OutputFileFormat.PCRaster), - (True, True, True, True, True, True, True, True, True, OutputFileFormat.GeoTIFF), - (False, False, False, False, False, False, False, False, False, OutputFileFormat.PCRaster), - (False, False, False, False, False, False, False, False, False, OutputFileFormat.GeoTIFF), + (True, True, True, True, True, True, True, True, True, OutputFileFormat.PCRASTER), + (True, True, True, True, True, True, True, True, True, OutputFileFormat.GEOTIFF), + ( + False, + False, + False, + False, + False, + False, + False, + False, + False, + OutputFileFormat.PCRASTER, + ), + ( + False, + False, + False, + False, + False, + False, + False, + False, + False, + OutputFileFormat.GEOTIFF, + ), ], ) def test_output_variables_constructor( From 50059d858100a1d0d79d4f6cc9170e03f4e9be2a Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:58:44 -0300 Subject: [PATCH 10/27] Add unit tests for AllOnesValidatorHandler --- tests/unit/validation/test_raster_all_ones.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 tests/unit/validation/test_raster_all_ones.py diff --git a/tests/unit/validation/test_raster_all_ones.py b/tests/unit/validation/test_raster_all_ones.py new file mode 100644 index 0000000..4673509 --- /dev/null +++ b/tests/unit/validation/test_raster_all_ones.py @@ -0,0 +1,84 @@ +from unittest.mock import MagicMock + +import pytest +import numpy as np + +from rubem.validation.handlers.raster_all_ones import AllOnesValidatorHandler +from rubem.validation.raster_data_rules import RasterDataRules +from rubem.configuration.raster_map import RasterMap, RasterBand + + +class TestAllOnesValidatorHandler: + + @pytest.mark.unit + def test_handle_no_rules_set(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + handler = AllOnesValidatorHandler() + raster_mock.rules = None + + # Act + errors = [] + result = handler.handle(raster_mock, errors) + + # Assert + assert result is True + assert not errors + + @pytest.mark.unit + def test_handle_rule_not_set(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = ( + RasterDataRules.FORBID_ALL_ZEROES + | RasterDataRules.FORBID_NO_DATA + | RasterDataRules.FORBID_OUT_OF_RANGE + ) + + # Act + handler = AllOnesValidatorHandler() + errors = [] + result = handler.handle(raster_mock, errors) + + # Assert + assert result is True + assert not errors + + @pytest.mark.unit + def test_handle_all_ones(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = RasterDataRules.FORBID_ALL_ONES + raster_mock.bands = [] + band_mock = MagicMock(spec=RasterBand) + band_mock.data_array = np.ones((10, 10)) + raster_mock.bands.append(band_mock) + + # Act + handler = AllOnesValidatorHandler() + errors = [] + result = handler.handle(raster_mock, errors) + + # Assert + assert result is False + assert len(errors) == 1 + assert errors[0] == RasterDataRules.FORBID_ALL_ONES + + @pytest.mark.unit + def test_handle_not_all_ones(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = RasterDataRules.FORBID_ALL_ONES + raster_mock.bands = [] + band_mock = MagicMock(spec=RasterBand) + band_mock.data_array = np.zeros((10, 10)) + raster_mock.bands.append(band_mock) + + # Act + handler = AllOnesValidatorHandler() + errors = [] + result = handler.handle(raster_mock, errors) + + # Assert + assert result is True + assert not errors From 680a9e9e72313c22ee394b941e1620ce32273230 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:58:50 -0300 Subject: [PATCH 11/27] Add unit tests for AllZeroesValidatorHandler --- .../unit/validation/test_raster_all_zeroes.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 tests/unit/validation/test_raster_all_zeroes.py diff --git a/tests/unit/validation/test_raster_all_zeroes.py b/tests/unit/validation/test_raster_all_zeroes.py new file mode 100644 index 0000000..90b90d7 --- /dev/null +++ b/tests/unit/validation/test_raster_all_zeroes.py @@ -0,0 +1,105 @@ +from unittest.mock import MagicMock + +import numpy as np +import pytest + +from rubem.validation.handlers.raster_all_zeroes import AllZeroesValidatorHandler +from rubem.validation.raster_data_rules import RasterDataRules +from rubem.configuration.raster_map import RasterMap, RasterBand + + +class TestAllZeroesValidatorHandler: + + @pytest.mark.unit + def test_handle_no_rules(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = None + + # Act + errors = [] + handler = AllZeroesValidatorHandler() + result = handler.handle(raster_mock, errors) + + # Assert + assert result is True + assert not errors + + @pytest.mark.unit + def test_handle_rule_not_set(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = ( + RasterDataRules.FORBID_ALL_ONES + | RasterDataRules.FORBID_NO_DATA + | RasterDataRules.FORBID_OUT_OF_RANGE + ) + + # Act + errors = [] + handler = AllZeroesValidatorHandler() + result = handler.handle(raster_mock, errors) + + # Assert + assert result is True + assert not errors + + @pytest.mark.unit + def test_handle_all_zeroes(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = [RasterDataRules.FORBID_ALL_ZEROES] + raster_mock.bands = [] + band_mock = MagicMock(spec=RasterBand) + band_mock.data_array = np.zeros((10, 10)) + raster_mock.bands.append(band_mock) + + # Act + errors = [] + handler = AllZeroesValidatorHandler() + result = handler.handle(raster_mock, errors) + + # Assert + assert result is False + assert errors == [RasterDataRules.FORBID_ALL_ZEROES] + + @pytest.mark.unit + def test_handle_not_all_zeroes(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = [RasterDataRules.FORBID_ALL_ZEROES] + raster_mock.bands = [] + band_mock = MagicMock(spec=RasterBand) + band_mock.data_array = np.ones((10, 10)) + raster_mock.bands.append(band_mock) + + # Act + errors = [] + handler = AllZeroesValidatorHandler() + result = handler.handle(raster_mock, errors) + + # Assert + assert result is True + assert not errors + + @pytest.mark.unit + def test_handle_multiple_bands(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = [RasterDataRules.FORBID_ALL_ZEROES] + raster_mock.bands = [] + band1_mock = MagicMock(spec=RasterBand) + band1_mock.data_array = np.ones((10, 10)) + raster_mock.bands.append(band1_mock) + band2_mock = MagicMock(spec=RasterBand) + band2_mock.data_array = np.zeros((10, 10)) + raster_mock.bands.append(band2_mock) + + # Act + errors = [] + handler = AllZeroesValidatorHandler() + result = handler.handle(raster_mock, errors) + + # Assert + assert result is False + assert errors == [RasterDataRules.FORBID_ALL_ZEROES] From d20313efdd7d4ad071b159c844f8fcc13d48976f Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:58:57 -0300 Subject: [PATCH 12/27] Add unit tests for NoDataValidatorHandler --- tests/unit/validation/test_raster_no_data.py | 86 ++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 tests/unit/validation/test_raster_no_data.py diff --git a/tests/unit/validation/test_raster_no_data.py b/tests/unit/validation/test_raster_no_data.py new file mode 100644 index 0000000..c8b3ad9 --- /dev/null +++ b/tests/unit/validation/test_raster_no_data.py @@ -0,0 +1,86 @@ +from unittest.mock import MagicMock + +import numpy as np +import pytest + +from rubem.validation.handlers.raster_no_data import NoDataValidatorHandler +from rubem.validation.raster_data_rules import RasterDataRules +from rubem.configuration.raster_map import RasterMap, RasterBand + + +class TestNoDataValidatorHandler: + + @pytest.mark.unit + def test_handle_no_rules_set(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = None + + # Act + errors = [] + handler = NoDataValidatorHandler() + result = handler.handle(raster_mock, errors) + + # Assert + assert result is True + assert not errors + + @pytest.mark.unit + def test_handle_no_data_rule_not_set(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = ( + RasterDataRules.FORBID_ALL_ONES + | RasterDataRules.FORBID_ALL_ZEROES + | RasterDataRules.FORBID_OUT_OF_RANGE + ) + + # Act + errors = [] + handler = NoDataValidatorHandler() + result = handler.handle(raster_mock, errors) + + # Assert + assert result is True + assert not errors + + @pytest.mark.unit + def test_handle_no_data_present(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = RasterDataRules.FORBID_NO_DATA + raster_mock.bands = [] + band_mock = MagicMock(spec=RasterBand) + band_mock.no_data_value = -9999 + band_mock.data_array = np.array([[1, 2, 3], [4, 5, -9999], [7, 8, 9]]) + raster_mock.bands.append(band_mock) + + # Act + errors = [] + handler = NoDataValidatorHandler() + result = handler.handle(raster_mock, errors) + + # Assert + assert result is False + assert len(errors) == 1 + assert errors[0] == RasterDataRules.FORBID_NO_DATA + + @pytest.mark.unit + def test_handle_no_data_not_present(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.rules = RasterDataRules.FORBID_NO_DATA + raster_mock.bands = [] + band_mock = MagicMock(spec=RasterBand) + band_mock.no_data_value = -9999 + band_mock.data_array = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) + raster_mock.bands.append(band_mock) + + # Act + errors = [] + handler = NoDataValidatorHandler() + result = handler.handle(raster_mock, errors) + + # Assert + assert result is True + assert not errors From 36f1b1b7569fecfe0ca1e0bd21f65d46dd3c5117 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Wed, 21 Feb 2024 21:59:31 -0300 Subject: [PATCH 13/27] Add unit tests for ValueRangeValidatorHandler --- .../validation/test_raster_value_range.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/unit/validation/test_raster_value_range.py diff --git a/tests/unit/validation/test_raster_value_range.py b/tests/unit/validation/test_raster_value_range.py new file mode 100644 index 0000000..14a7f32 --- /dev/null +++ b/tests/unit/validation/test_raster_value_range.py @@ -0,0 +1,72 @@ +from unittest.mock import MagicMock + +import numpy as np +import pytest + +from rubem.validation.handlers.raster_value_range import ValueRangeValidatorHandler +from rubem.validation.raster_data_rules import RasterDataRules +from rubem.configuration.raster_map import RasterMap, RasterBand + + +class TestValueRangeValidatorHandler: + + @pytest.mark.unit + def test_handle_valid_range(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.valid_range = {"min": 0.0, "max": 100.0} + raster_mock.bands = [] + band_mock = MagicMock(spec=RasterBand) + band_mock.no_data_value = -9999 + band_mock.data_array = np.array([10, 20, 30, 40, 50]) + raster_mock.bands.append(band_mock) + + # Act + handler = ValueRangeValidatorHandler() + errors = [] + result = handler.handle(raster_mock, errors) + + # Assert + assert result is True + assert not errors + + @pytest.mark.unit + def test_handle_invalid_range(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.valid_range = {"min": 0.0, "max": 100.0} + raster_mock.bands = [] + band_mock = MagicMock(spec=RasterBand) + band_mock.no_data_value = -9999 + band_mock.data_array = np.array([10, 20, 30, 110, 120]) + raster_mock.bands.append(band_mock) + + # Act + handler = ValueRangeValidatorHandler() + errors = [] + result = handler.handle(raster_mock, errors) + + # Assert + assert result is False + assert len(errors) == 1 + assert errors[0] == RasterDataRules.FORBID_OUT_OF_RANGE + + @pytest.mark.unit + def test_handle_no_range_set(self): + # Arrange + raster_mock = MagicMock(spec=RasterMap) + raster_mock.valid_range = None + raster_mock.bands = [] + band_mock = MagicMock(spec=RasterBand) + band_mock.no_data_value = -9999 + band_mock.data_array = np.array([10, 20, 30, 40, 50]) + raster_mock.bands.append(band_mock) + + # Act + handler = ValueRangeValidatorHandler() + errors = [] + result = handler.handle(raster_mock, errors) + + # Assert + assert result is True + assert not errors From e947e1d4ae5353c407aa71b9bad13c2846bc4cc9 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Wed, 21 Feb 2024 22:26:48 -0300 Subject: [PATCH 14/27] Fix `ModelConfiguration` unit tests --- .../configuration/test_model_configuration.py | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/unit/configuration/test_model_configuration.py b/tests/unit/configuration/test_model_configuration.py index 51212b1..81b7643 100644 --- a/tests/unit/configuration/test_model_configuration.py +++ b/tests/unit/configuration/test_model_configuration.py @@ -1,6 +1,10 @@ +from unittest.mock import MagicMock + +import numpy as np import pytest from rubem.configuration.model_configuration import ModelConfiguration +from rubem.configuration.raster_map import RasterBand class TestModelConfiguration: @@ -261,7 +265,14 @@ def setup(self, fs): @pytest.mark.unit def test_init_with_dictionary(self, mocker): - with mocker.patch("osgeo.gdal.Open"), mocker.patch("osgeo.gdal.GetDataTypeName"): + band_mock = MagicMock(spec=RasterBand) + band_mock.no_data_value = -9999 + band_mock.data_array = np.ones((3, 3)) + with mocker.patch("osgeo.gdal.OpenEx"), mocker.patch( + "osgeo.gdal.GetDataTypeName" + ), mocker.patch("os.path.getsize", return_value=100), mocker.patch( + "rubem.configuration.raster_map.RasterBand", return_value=band_mock + ): _ = ModelConfiguration(self.valid_config_input) @pytest.mark.unit @@ -281,11 +292,18 @@ def test_init_with_incomplete_dictionary(self): @pytest.mark.unit def test_init_with_ini_file(self, fs, mocker): + band_mock = MagicMock(spec=RasterBand) + band_mock.no_data_value = -9999 + band_mock.data_array = np.ones((3, 3)) fs.create_file( "/test_path/config.ini", contents=self.valid_config_ini_content, ) - with mocker.patch("osgeo.gdal.Open"), mocker.patch("osgeo.gdal.GetDataTypeName"): + with mocker.patch("osgeo.gdal.OpenEx"), mocker.patch( + "osgeo.gdal.GetDataTypeName" + ), mocker.patch("os.path.getsize", return_value=100), mocker.patch( + "rubem.configuration.raster_map.RasterBand", return_value=band_mock + ): _ = ModelConfiguration("/test_path/config.ini") @pytest.mark.unit @@ -316,11 +334,18 @@ def test_init_with_incomplete_ini_file(self, fs): @pytest.mark.unit def test_init_with_json_file(self, fs, mocker): + band_mock = MagicMock(spec=RasterBand) + band_mock.no_data_value = -9999 + band_mock.data_array = np.ones((3, 3)) fs.create_file( "/test_path/config.json", contents=self.valid_config_json_content, ) - with mocker.patch("osgeo.gdal.Open"), mocker.patch("osgeo.gdal.GetDataTypeName"): + with mocker.patch("osgeo.gdal.OpenEx"), mocker.patch( + "osgeo.gdal.GetDataTypeName" + ), mocker.patch("os.path.getsize", return_value=100), mocker.patch( + "rubem.configuration.raster_map.RasterBand", return_value=band_mock + ): _ = ModelConfiguration("/test_path/config.json") @pytest.mark.unit From 9944ef7cee5fbf86cd409a085738b8717aab6c28 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Wed, 21 Feb 2024 22:26:56 -0300 Subject: [PATCH 15/27] Add unit tests for RasterMap class --- tests/unit/configuration/test_raster_map.py | 69 +++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 tests/unit/configuration/test_raster_map.py diff --git a/tests/unit/configuration/test_raster_map.py b/tests/unit/configuration/test_raster_map.py new file mode 100644 index 0000000..6ebb41b --- /dev/null +++ b/tests/unit/configuration/test_raster_map.py @@ -0,0 +1,69 @@ +from unittest.mock import MagicMock + +import pytest + +from rubem.configuration.raster_map import RasterMap, RasterDataRules, RasterBand + + +class TestRasterMap: + + @pytest.mark.unit + def test_init_valid_file_no_range_no_rules(self, mocker): + # Arrange + file_path = "/path/to/raster.tif" + with mocker.patch("osgeo.gdal.OpenEx"), mocker.patch( + "os.path.isfile", return_value=True + ), mocker.patch("os.path.getsize", return_value=100), mocker.patch( + "rubem.configuration.raster_map.RasterBand", return_value=MagicMock(spec=RasterBand) + ): + # Act + raster_map = RasterMap(file_path) + + # Assert + assert raster_map is not None + assert len(raster_map.bands) == 1 + + @pytest.mark.unit + def test_init_valid_file_with_range_with_rules(self, mocker): + # Arrange + file_path = "/path/to/raster.tif" + valid_range = {"min": 0.0, "max": 255.0} + rules = RasterDataRules.FORBID_ALL_ZEROES | RasterDataRules.FORBID_ALL_ONES + with mocker.patch("osgeo.gdal.OpenEx"), mocker.patch( + "os.path.isfile", return_value=True + ), mocker.patch("os.path.getsize", return_value=100), mocker.patch( + "rubem.configuration.raster_map.RasterBand", return_value=MagicMock(spec=RasterBand) + ): + # Act + raster_map = RasterMap(file_path, valid_range, rules) + + # Assert + assert raster_map is not None + assert raster_map.valid_range == valid_range + assert raster_map.rules == rules + assert len(raster_map.bands) == 1 + + @pytest.mark.unit + def test_init_invalid_file(self, mocker): + with mocker.patch("os.path.isfile", return_value=False), pytest.raises(FileNotFoundError): + RasterMap("/path/to/invalid_raster.tif") + + @pytest.mark.unit + def test_init_empty_file(self, mocker): + file_path = "/path/to/empty_raster.tif" + valid_range = {"min": 0.0, "max": 255.0} + rules = RasterDataRules.FORBID_ALL_ZEROES + with mocker.patch("os.path.isfile", return_value=True), mocker.patch( + "os.path.getsize", return_value=0 + ), pytest.raises(ValueError): + RasterMap(file_path, valid_range, rules) + + @pytest.mark.unit + def test_init_invalid_extension(self, mocker): + file_path = "/path/to/invalid_extension.xyz" + valid_range = {"min": 0.0, "max": 255.0} + rules = RasterDataRules.FORBID_ALL_ZEROES + with mocker.patch("os.path.isfile", return_value=True), mocker.patch( + "os.path.getsize", return_value=100 + ), pytest.raises(ValueError): + RasterMap(file_path, valid_range, rules) From efc0bb04a150140e627d9448d79f7b0033105bc9 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Wed, 21 Feb 2024 22:27:01 -0300 Subject: [PATCH 16/27] Add unit tests for RasterBand class --- tests/unit/configuration/test_raster_band.py | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/unit/configuration/test_raster_band.py diff --git a/tests/unit/configuration/test_raster_band.py b/tests/unit/configuration/test_raster_band.py new file mode 100644 index 0000000..8ef9447 --- /dev/null +++ b/tests/unit/configuration/test_raster_band.py @@ -0,0 +1,51 @@ +from unittest.mock import MagicMock + +import pytest + +from rubem.configuration.raster_map import RasterBand + + +class TestRasterBand: + + @pytest.mark.unit + def test_init_valid_band(self): + band_mock = MagicMock() + band_mock.GetNoDataValue.return_value = -9999 + band_mock.GetStatistics.return_value = (0, 255, 127.5, 50) + band_mock.DataType = 1 + band_mock.ReadAsArray.return_value = [[1, 2], [3, 4]] + + raster_band = RasterBand(1, band_mock) + + assert raster_band.index == 1 + assert raster_band.band == band_mock + assert raster_band.no_data_value == -9999 + assert raster_band.min == 0 + assert raster_band.max == 255 + assert raster_band.mean == 127.5 + assert raster_band.std_dev == 50 + assert raster_band.data_type == "Byte" + assert raster_band.data_array == [[1, 2], [3, 4]] + + @pytest.mark.unit + def test_init_invalid_band(self): + with pytest.raises(ValueError): + RasterBand(1, None) + + @pytest.mark.unit + def test_str(self): + band_mock = MagicMock() + band_mock.GetNoDataValue.return_value = -9999 + band_mock.GetStatistics.return_value = (0, 255, 127.5, 50) + band_mock.DataType = 1 + + raster_band = RasterBand(1, band_mock) + + expected_output = ( + "Index: 1, " + "Data Type: Byte, " + "NoData Value: -9999, " + "Statistics: Min: 0, Max: 255, Mean: 127.5, Std Dev: 50" + ) + + assert str(raster_band) == expected_output From 81f7234325f124bc25211bef5a375dbbf8ff3684 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 22 Feb 2024 08:30:34 -0300 Subject: [PATCH 17/27] Add Dependabot configuration for GitHub Actions --- .github/dependabot.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..2d6d188 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +version: 2 +updates: +- package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + day: monday + time: "08:00" + timezone: America/Sao_Paulo + reviewers: + - "soaressgabriel" + labels: + - "dependencies" + - "github_actions" + commit-message: + prefix: "[actions] " + open-pull-requests-limit: 100 From fad1ddf52d5e30fdf7612fe335d21fdc62cadb2b Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 22 Feb 2024 08:46:45 -0300 Subject: [PATCH 18/27] Add unit test workflow using Micromamba --- .github/workflows/build-test-micromamba.yml | 32 +++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/build-test-micromamba.yml diff --git a/.github/workflows/build-test-micromamba.yml b/.github/workflows/build-test-micromamba.yml new file mode 100644 index 0000000..5070661 --- /dev/null +++ b/.github/workflows/build-test-micromamba.yml @@ -0,0 +1,32 @@ +name: Unit tests + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +jobs: + build: + runs-on: self-hosted + strategy: + max-parallel: 5 + + steps: + - uses: actions/checkout@v4 + - uses: mamba-org/setup-micromamba@v1 + with: + environment-file: env-dev.yml + init-shell: bash + cache-environment: true + post-cleanup: "all" + - name: Test with pytest + run: pytest -x -l -ra --junitxml="TestResults-Rubem.xml" + + - name: Upload pytest test results + uses: actions/upload-artifact@v4 + with: + name: pytest-results-rubem + path: "TestResults-Rubem.xml" + retention-days: 1 + if: ${{ always() }} From 98a574b0e5f47a9d3218590b340a55be44bc94e0 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 22 Feb 2024 08:51:30 -0300 Subject: [PATCH 19/27] Update build-test-micromamba.yml --- .github/workflows/build-test-micromamba.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-test-micromamba.yml b/.github/workflows/build-test-micromamba.yml index 5070661..a70c222 100644 --- a/.github/workflows/build-test-micromamba.yml +++ b/.github/workflows/build-test-micromamba.yml @@ -8,7 +8,7 @@ on: jobs: build: - runs-on: self-hosted + runs-on: ubuntu-latest strategy: max-parallel: 5 From 2648a462aea250fe38418b8e9cce58593db5a26d Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 22 Feb 2024 08:53:25 -0300 Subject: [PATCH 20/27] Update environment files --- env-dev.yml | 1 + env-prod.yml | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/env-dev.yml b/env-dev.yml index 5258c0c..5db57a0 100644 --- a/env-dev.yml +++ b/env-dev.yml @@ -1,3 +1,4 @@ +name: rubem-dev channels: - conda-forge - defaults diff --git a/env-prod.yml b/env-prod.yml index cdefcb8..5aee977 100644 --- a/env-prod.yml +++ b/env-prod.yml @@ -1,8 +1,9 @@ +name: rubem-prod channels: - - conda-forge - - defaults +- conda-forge +- defaults dependencies: - - python=3.9 - - gdal=3.* - - pandas=1.* - - pcraster=4.* \ No newline at end of file +- python=3.9 +- gdal=3.* +- pandas=1.* +- pcraster=4.* From 2ff053d4569d8e4d9b022c18c432f28c357aae41 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 22 Feb 2024 08:57:36 -0300 Subject: [PATCH 21/27] Add shell command to run pytest with bash --- .github/workflows/build-test-micromamba.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-test-micromamba.yml b/.github/workflows/build-test-micromamba.yml index a70c222..5e58e19 100644 --- a/.github/workflows/build-test-micromamba.yml +++ b/.github/workflows/build-test-micromamba.yml @@ -22,6 +22,7 @@ jobs: post-cleanup: "all" - name: Test with pytest run: pytest -x -l -ra --junitxml="TestResults-Rubem.xml" + shell: bash -el {0} - name: Upload pytest test results uses: actions/upload-artifact@v4 From e728ba7256db059a2173a189b47a727e413c4458 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 22 Feb 2024 09:29:03 -0300 Subject: [PATCH 22/27] Remove unused test_pcraster_map.py file --- tests/unit/configuration/test_pcraster_map.py | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 tests/unit/configuration/test_pcraster_map.py diff --git a/tests/unit/configuration/test_pcraster_map.py b/tests/unit/configuration/test_pcraster_map.py deleted file mode 100644 index 5e8ccd6..0000000 --- a/tests/unit/configuration/test_pcraster_map.py +++ /dev/null @@ -1,7 +0,0 @@ -import pytest - -from rubem.configuration.raster_map import RasterMap - - -class TestPCRasterMap: - pass From 3831e52ebc1bf2559193da75c689e53aa695d244 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 22 Feb 2024 09:53:55 -0300 Subject: [PATCH 23/27] Refactor output directory validation and error handling --- rubem/configuration/output_data_directory.py | 18 +++++--- .../configuration/test_output_directory.py | 41 ++++++++++--------- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/rubem/configuration/output_data_directory.py b/rubem/configuration/output_data_directory.py index a6495fc..f0887c6 100644 --- a/rubem/configuration/output_data_directory.py +++ b/rubem/configuration/output_data_directory.py @@ -20,13 +20,19 @@ def __init__( self.__validate_directories() - def __validate_directories(self) -> None: - try: - if not os.path.isdir(self.path): + if not os.path.exists(self.path): + self.logger.warning("Output directory does not exist: %s", self.path) + try: + self.logger.info("Creating output directory: %s", self.path) os.makedirs(self.path) - except Exception as e: - self.logger.error("Failed to create output directory: %s", e) - raise + except Exception as e: + self.logger.error("Failed to create output directory: %s", e) + raise + + def __validate_directories(self) -> None: + if os.path.isfile(self.path): + self.logger.error("Output path is not a directory: %s", self.path) + raise NotADirectoryError(f"{self.path} is not a directory") if os.listdir(self.path): self.logger.warning("There is data in the output directory: %s", self.path) diff --git a/tests/unit/configuration/test_output_directory.py b/tests/unit/configuration/test_output_directory.py index 8b748f8..35e732b 100644 --- a/tests/unit/configuration/test_output_directory.py +++ b/tests/unit/configuration/test_output_directory.py @@ -1,3 +1,4 @@ +import re import pytest from rubem.configuration.output_data_directory import OutputDataDirectory @@ -6,35 +7,35 @@ class TestOutputDataDirectory: @pytest.mark.unit - def test_create_directory_when_not_exists(self, mocker): - with mocker.patch("os.path.isdir", return_value=False) as mock_isdir, mocker.patch( - "os.makedirs" - ) as mock_makedirs, pytest.raises(Exception): - _ = OutputDataDirectory("test_path") - mock_isdir.assert_called_once_with("test_path") - mock_makedirs.assert_called_once_with("test_path") + def test_create_directory_when_not_exists_then_create_it(self, mocker): + mock_exists = mocker.patch("os.path.exists", return_value=False) + mock_isdir = mocker.patch("os.path.isfile", return_value=False) + mock_makedirs = mocker.patch("os.makedirs") + _ = OutputDataDirectory("test_path") + mock_exists.assert_called_once_with("test_path") + mock_isdir.assert_called_once_with("test_path") + mock_makedirs.assert_called_once_with("test_path") @pytest.mark.unit def test_directory_exists_and_empty(self, mocker): - with mocker.patch("os.path.isdir", return_value=True), mocker.patch( - "os.listdir", return_value=[] - ): - _ = OutputDataDirectory("test_path") + mocker.patch("os.path.exists", return_value=True) + mocker.patch("os.path.isfile", return_value=False) + mocker.patch("os.listdir", return_value=[]) + _ = OutputDataDirectory("test_path") @pytest.mark.unit def test_directory_exists_and_not_empty(self, mocker): - with mocker.patch("os.path.isdir", return_value=True), mocker.patch( - "os.listdir", return_value=["file1.txt", "file2.txt", "file3.txt"] - ): - _ = OutputDataDirectory("test_path") + mocker.patch("os.path.exists", return_value=True) + mocker.patch("os.path.isfile", return_value=False) + mocker.patch("os.listdir", return_value=["file1.txt", "file2.txt", "file3.txt"]) + _ = OutputDataDirectory("test_path") @pytest.mark.unit def test_error_creating_directory(self, mocker): - with mocker.patch("os.path.isdir", return_value=False), mocker.patch( - "os.makedirs", side_effect=Exception("error") - ) as mock_makedirs, mocker.patch("logging.Logger.error") as mock_error, pytest.raises( - Exception - ): + mocker.patch("os.path.isfile", return_value=False) + mock_makedirs = mocker.patch("os.makedirs", side_effect=Exception("error")) + mock_error = mocker.patch("logging.Logger.error") + with pytest.raises(Exception): _ = OutputDataDirectory("test_path") mock_makedirs.assert_called_once_with("test_path") mock_error.assert_called() From ef3ea0e35c0b8f94751601b5bc28dbc654e9f46f Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 22 Feb 2024 09:54:03 -0300 Subject: [PATCH 24/27] Refactor unit tests for model configuration and raster map classes --- .../configuration/test_model_configuration.py | 33 +++++------ .../configuration/test_output_directory.py | 1 - tests/unit/configuration/test_raster_map.py | 57 ++++++++++--------- 3 files changed, 45 insertions(+), 46 deletions(-) diff --git a/tests/unit/configuration/test_model_configuration.py b/tests/unit/configuration/test_model_configuration.py index 81b7643..dcbea74 100644 --- a/tests/unit/configuration/test_model_configuration.py +++ b/tests/unit/configuration/test_model_configuration.py @@ -268,12 +268,11 @@ def test_init_with_dictionary(self, mocker): band_mock = MagicMock(spec=RasterBand) band_mock.no_data_value = -9999 band_mock.data_array = np.ones((3, 3)) - with mocker.patch("osgeo.gdal.OpenEx"), mocker.patch( - "osgeo.gdal.GetDataTypeName" - ), mocker.patch("os.path.getsize", return_value=100), mocker.patch( - "rubem.configuration.raster_map.RasterBand", return_value=band_mock - ): - _ = ModelConfiguration(self.valid_config_input) + mocker.patch("osgeo.gdal.OpenEx") + mocker.patch("osgeo.gdal.GetDataTypeName") + mocker.patch("os.path.getsize", return_value=100) + mocker.patch("rubem.configuration.raster_map.RasterBand", return_value=band_mock) + _ = ModelConfiguration(self.valid_config_input) @pytest.mark.unit def test_init_with_empty_dictionary(self): @@ -299,12 +298,11 @@ def test_init_with_ini_file(self, fs, mocker): "/test_path/config.ini", contents=self.valid_config_ini_content, ) - with mocker.patch("osgeo.gdal.OpenEx"), mocker.patch( - "osgeo.gdal.GetDataTypeName" - ), mocker.patch("os.path.getsize", return_value=100), mocker.patch( - "rubem.configuration.raster_map.RasterBand", return_value=band_mock - ): - _ = ModelConfiguration("/test_path/config.ini") + mocker.patch("osgeo.gdal.OpenEx") + mocker.patch("osgeo.gdal.GetDataTypeName") + mocker.patch("os.path.getsize", return_value=100) + mocker.patch("rubem.configuration.raster_map.RasterBand", return_value=band_mock) + _ = ModelConfiguration("/test_path/config.ini") @pytest.mark.unit def test_init_with_not_existent_ini_file(self): @@ -341,12 +339,11 @@ def test_init_with_json_file(self, fs, mocker): "/test_path/config.json", contents=self.valid_config_json_content, ) - with mocker.patch("osgeo.gdal.OpenEx"), mocker.patch( - "osgeo.gdal.GetDataTypeName" - ), mocker.patch("os.path.getsize", return_value=100), mocker.patch( - "rubem.configuration.raster_map.RasterBand", return_value=band_mock - ): - _ = ModelConfiguration("/test_path/config.json") + mocker.patch("osgeo.gdal.OpenEx") + mocker.patch("osgeo.gdal.GetDataTypeName") + mocker.patch("os.path.getsize", return_value=100) + mocker.patch("rubem.configuration.raster_map.RasterBand", return_value=band_mock) + _ = ModelConfiguration("/test_path/config.json") @pytest.mark.unit def test_init_with_not_existent_json_file(self): diff --git a/tests/unit/configuration/test_output_directory.py b/tests/unit/configuration/test_output_directory.py index 35e732b..ad535d3 100644 --- a/tests/unit/configuration/test_output_directory.py +++ b/tests/unit/configuration/test_output_directory.py @@ -1,4 +1,3 @@ -import re import pytest from rubem.configuration.output_data_directory import OutputDataDirectory diff --git a/tests/unit/configuration/test_raster_map.py b/tests/unit/configuration/test_raster_map.py index 6ebb41b..ee341cf 100644 --- a/tests/unit/configuration/test_raster_map.py +++ b/tests/unit/configuration/test_raster_map.py @@ -11,17 +11,18 @@ class TestRasterMap: def test_init_valid_file_no_range_no_rules(self, mocker): # Arrange file_path = "/path/to/raster.tif" - with mocker.patch("osgeo.gdal.OpenEx"), mocker.patch( - "os.path.isfile", return_value=True - ), mocker.patch("os.path.getsize", return_value=100), mocker.patch( + mocker.patch("osgeo.gdal.OpenEx") + mocker.patch("os.path.isfile", return_value=True) + mocker.patch("os.path.getsize", return_value=100) + mocker.patch( "rubem.configuration.raster_map.RasterBand", return_value=MagicMock(spec=RasterBand) - ): - # Act - raster_map = RasterMap(file_path) + ) + # Act + raster_map = RasterMap(file_path) - # Assert - assert raster_map is not None - assert len(raster_map.bands) == 1 + # Assert + assert raster_map is not None + assert len(raster_map.bands) == 1 @pytest.mark.unit def test_init_valid_file_with_range_with_rules(self, mocker): @@ -29,23 +30,25 @@ def test_init_valid_file_with_range_with_rules(self, mocker): file_path = "/path/to/raster.tif" valid_range = {"min": 0.0, "max": 255.0} rules = RasterDataRules.FORBID_ALL_ZEROES | RasterDataRules.FORBID_ALL_ONES - with mocker.patch("osgeo.gdal.OpenEx"), mocker.patch( - "os.path.isfile", return_value=True - ), mocker.patch("os.path.getsize", return_value=100), mocker.patch( + mocker.patch("osgeo.gdal.OpenEx") + mocker.patch("os.path.isfile", return_value=True) + mocker.patch("os.path.getsize", return_value=100) + mocker.patch( "rubem.configuration.raster_map.RasterBand", return_value=MagicMock(spec=RasterBand) - ): - # Act - raster_map = RasterMap(file_path, valid_range, rules) + ) + # Act + raster_map = RasterMap(file_path, valid_range, rules) - # Assert - assert raster_map is not None - assert raster_map.valid_range == valid_range - assert raster_map.rules == rules - assert len(raster_map.bands) == 1 + # Assert + assert raster_map is not None + assert raster_map.valid_range == valid_range + assert raster_map.rules == rules + assert len(raster_map.bands) == 1 @pytest.mark.unit def test_init_invalid_file(self, mocker): - with mocker.patch("os.path.isfile", return_value=False), pytest.raises(FileNotFoundError): + mocker.patch("os.path.isfile", return_value=False) + with pytest.raises(FileNotFoundError): RasterMap("/path/to/invalid_raster.tif") @pytest.mark.unit @@ -53,9 +56,9 @@ def test_init_empty_file(self, mocker): file_path = "/path/to/empty_raster.tif" valid_range = {"min": 0.0, "max": 255.0} rules = RasterDataRules.FORBID_ALL_ZEROES - with mocker.patch("os.path.isfile", return_value=True), mocker.patch( - "os.path.getsize", return_value=0 - ), pytest.raises(ValueError): + mocker.patch("os.path.isfile", return_value=True) + mocker.patch("os.path.getsize", return_value=0), + with pytest.raises(ValueError): RasterMap(file_path, valid_range, rules) @pytest.mark.unit @@ -63,7 +66,7 @@ def test_init_invalid_extension(self, mocker): file_path = "/path/to/invalid_extension.xyz" valid_range = {"min": 0.0, "max": 255.0} rules = RasterDataRules.FORBID_ALL_ZEROES - with mocker.patch("os.path.isfile", return_value=True), mocker.patch( - "os.path.getsize", return_value=100 - ), pytest.raises(ValueError): + mocker.patch("os.path.isfile", return_value=True) + mocker.patch("os.path.getsize", return_value=100) + with pytest.raises(ValueError): RasterMap(file_path, valid_range, rules) From 5948dca0f6927777ed2c310753c70d087ede0b74 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 22 Feb 2024 09:56:23 -0300 Subject: [PATCH 25/27] Fix mocker.patch syntax error in test_raster_map.py --- tests/unit/configuration/test_raster_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/configuration/test_raster_map.py b/tests/unit/configuration/test_raster_map.py index ee341cf..ecc49ef 100644 --- a/tests/unit/configuration/test_raster_map.py +++ b/tests/unit/configuration/test_raster_map.py @@ -57,7 +57,7 @@ def test_init_empty_file(self, mocker): valid_range = {"min": 0.0, "max": 255.0} rules = RasterDataRules.FORBID_ALL_ZEROES mocker.patch("os.path.isfile", return_value=True) - mocker.patch("os.path.getsize", return_value=0), + mocker.patch("os.path.getsize", return_value=0) with pytest.raises(ValueError): RasterMap(file_path, valid_range, rules) From 7e76f474ca99e3e4fb9b97ebc65b862f40a26f12 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 22 Feb 2024 10:54:17 -0300 Subject: [PATCH 26/27] Fix `TestOutputDataDirectory` tests --- tests/unit/configuration/test_output_directory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/configuration/test_output_directory.py b/tests/unit/configuration/test_output_directory.py index ad535d3..823bc12 100644 --- a/tests/unit/configuration/test_output_directory.py +++ b/tests/unit/configuration/test_output_directory.py @@ -7,6 +7,7 @@ class TestOutputDataDirectory: @pytest.mark.unit def test_create_directory_when_not_exists_then_create_it(self, mocker): + mocker.patch("os.listdir", return_value=[]) mock_exists = mocker.patch("os.path.exists", return_value=False) mock_isdir = mocker.patch("os.path.isfile", return_value=False) mock_makedirs = mocker.patch("os.makedirs") @@ -31,6 +32,7 @@ def test_directory_exists_and_not_empty(self, mocker): @pytest.mark.unit def test_error_creating_directory(self, mocker): + mocker.patch("os.listdir", return_value=[]) mocker.patch("os.path.isfile", return_value=False) mock_makedirs = mocker.patch("os.makedirs", side_effect=Exception("error")) mock_error = mocker.patch("logging.Logger.error") From a4e8ea05b4e2bcb48565d9d03d00e15e01d49c76 Mon Sep 17 00:00:00 2001 From: Gabriel Soares <70075435+soaressgabriel@users.noreply.github.com> Date: Thu, 22 Feb 2024 10:57:17 -0300 Subject: [PATCH 27/27] Fix unit test for string representation of `OutputDataDirectory` --- tests/unit/configuration/test_output_directory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/configuration/test_output_directory.py b/tests/unit/configuration/test_output_directory.py index 823bc12..eb42851 100644 --- a/tests/unit/configuration/test_output_directory.py +++ b/tests/unit/configuration/test_output_directory.py @@ -42,6 +42,9 @@ def test_error_creating_directory(self, mocker): mock_error.assert_called() @pytest.mark.unit - def test_string_representation(self): + def test_string_representation(self, mocker): + mocker.patch("os.path.exists", return_value=True) + mocker.patch("os.path.isfile", return_value=False) + mocker.patch("os.listdir", return_value=[]) directory = OutputDataDirectory("test_path") assert str(directory) == "test_path"