diff --git a/docs/source/plugins/developing_plugins.rst b/docs/source/plugins/developing_plugins.rst index 80f747436b..9ee8eb1561 100644 --- a/docs/source/plugins/developing_plugins.rst +++ b/docs/source/plugins/developing_plugins.rst @@ -1145,6 +1145,7 @@ on their execution context from within .. code-block:: python :linenos: + import fiftyone as fo import fiftyone.core.storage as fos import fiftyone.core.utils as fou @@ -1168,6 +1169,55 @@ on their execution context from within of their delegated operations from the :ref:`Runs page ` of the Teams App! +For your convenience, all builtin methods of the FiftyOne SDK that support +rendering progress bars provide an optional `progress` method that you can use +trigger calls to +:meth:`set_progress() ` +using the pattern show below: + +.. code-block:: python + :linenos: + + import fiftyone as fo + + def execute(self, ctx): + images_dir = ctx.params["images_dir"] + + # Custom logic that controls how progress is reported + def set_progress(pb): + if pb.complete: + ctx.set_progress(progress=1, label="Operation complete") + else: + ctx.set_progress(progress=pb.progress) + + # Option 1: report progress every five seconds + progress = fo.report_progress(set_progress, dt=5.0) + + # Option 2: report progress at 10 equally-spaced increments + # progress = fo.report_progress(set_progress, n=10) + + ctx.dataset.add_images_dir(images_dir, progress=progress) + +You can also use the builtin +:class:`ProgressHandler ` class to +automatically forward logging messages to +:meth:`set_progress() ` +as `label` values using the pattern shown below: + +.. code-block:: python + :linenos: + + import logging + import fiftyone.operators as foo + import fiftyone.zoo as foz + + def execute(self, ctx): + name = ctx.params["name"] + + # Automatically report all `fiftyone` logging messages + with foo.ProgressHandler(ctx, logger=logging.getLogger("fiftyone")): + foz.load_zoo_dataset(name, persistent=True) + .. _operator-execution: Operator execution diff --git a/fiftyone/__public__.py b/fiftyone/__public__.py index 74020771ec..d40706b6ce 100644 --- a/fiftyone/__public__.py +++ b/fiftyone/__public__.py @@ -213,6 +213,7 @@ disable_progress_bars, pprint, pformat, + report_progress, ProgressBar, ) from .core.view import DatasetView diff --git a/fiftyone/core/collections.py b/fiftyone/core/collections.py index 317849d75d..7320f8b75f 100644 --- a/fiftyone/core/collections.py +++ b/fiftyone/core/collections.py @@ -830,8 +830,9 @@ def iter_samples(self, progress=False, autosave=False, batch_size=None): """Returns an iterator over the samples in the collection. Args: - progress (False): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -855,8 +856,9 @@ def iter_groups( Args: group_slices (None): an optional subset of group slices to load - progress (False): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -2735,6 +2737,7 @@ def compute_metadata( num_workers=None, skip_failures=True, warn_failures=False, + progress=None, ): """Populates the ``metadata`` field of all samples in the collection. @@ -2748,6 +2751,9 @@ def compute_metadata( raising an error if metadata cannot be computed for a sample warn_failures (False): whether to log a warning if metadata cannot be computed for a sample + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ fomt.compute_metadata( self, @@ -2755,6 +2761,7 @@ def compute_metadata( num_workers=num_workers, skip_failures=skip_failures, warn_failures=warn_failures, + progress=progress, ) def apply_model( @@ -2768,6 +2775,7 @@ def apply_model( skip_failures=True, output_dir=None, rel_dir=None, + progress=None, **kwargs, ): """Applies the :class:`FiftyOne model ` or @@ -2816,6 +2824,9 @@ def apply_model( subdirectories in ``output_dir`` that match the shape of the input paths. The path is converted to an absolute path (if necessary) via :func:`fiftyone.core.storage.normalize_path` + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation """ @@ -2830,6 +2841,7 @@ def apply_model( skip_failures=skip_failures, output_dir=output_dir, rel_dir=rel_dir, + progress=progress, **kwargs, ) @@ -2840,6 +2852,7 @@ def compute_embeddings( batch_size=None, num_workers=None, skip_failures=True, + progress=None, **kwargs, ): """Computes embeddings for the samples in the collection using the @@ -2879,6 +2892,9 @@ def compute_embeddings( raising an error if embeddings cannot be generated for a sample. Only applicable to :class:`fiftyone.core.models.Model` instances + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation @@ -2908,6 +2924,7 @@ def compute_embeddings( batch_size=batch_size, num_workers=num_workers, skip_failures=skip_failures, + progress=progress, **kwargs, ) @@ -2922,6 +2939,7 @@ def compute_patch_embeddings( batch_size=None, num_workers=None, skip_failures=True, + progress=None, ): """Computes embeddings for the image patches defined by ``patches_field`` of the samples in the collection using the given @@ -2973,6 +2991,9 @@ def compute_patch_embeddings( applicable for Torch-based models skip_failures (True): whether to gracefully continue without raising an error if embeddings cannot be generated for a sample + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: one of the following: @@ -3003,6 +3024,7 @@ def compute_patch_embeddings( alpha=alpha, handle_missing=handle_missing, skip_failures=skip_failures, + progress=progress, ) def evaluate_regressions( @@ -3012,6 +3034,7 @@ def evaluate_regressions( eval_key=None, missing=None, method=None, + progress=None, **kwargs, ): """Evaluates the regression predictions in this collection with respect @@ -3050,6 +3073,9 @@ def evaluate_regressions( The supported values are ``fo.evaluation_config.regression_backends.keys()`` and the default is ``fo.evaluation_config.regression_default_backend`` + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.regression.RegressionEvaluationConfig` being used @@ -3064,6 +3090,7 @@ def evaluate_regressions( eval_key=eval_key, missing=missing, method=method, + progress=progress, **kwargs, ) @@ -3075,6 +3102,7 @@ def evaluate_classifications( classes=None, missing=None, method=None, + progress=None, **kwargs, ): """Evaluates the classification predictions in this collection with @@ -3122,6 +3150,9 @@ def evaluate_classifications( The supported values are ``fo.evaluation_config.classification_backends.keys()`` and the default is ``fo.evaluation_config.classification_default_backend`` + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.classification.ClassificationEvaluationConfig` being used @@ -3137,6 +3168,7 @@ def evaluate_classifications( classes=classes, missing=missing, method=method, + progress=progress, **kwargs, ) @@ -3153,6 +3185,7 @@ def evaluate_detections( use_boxes=False, classwise=True, dynamic=True, + progress=None, **kwargs, ): """Evaluates the specified predicted detections in this collection with @@ -3245,6 +3278,9 @@ def evaluate_detections( label (True) or allow matches between classes (False) dynamic (True): whether to declare the dynamic object-level attributes that are populated on the dataset's schema + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.detection.DetectionEvaluationConfig` being used @@ -3265,6 +3301,7 @@ def evaluate_detections( use_boxes=use_boxes, classwise=classwise, dynamic=dynamic, + progress=progress, **kwargs, ) @@ -3275,6 +3312,7 @@ def evaluate_segmentations( eval_key=None, mask_targets=None, method=None, + progress=None, **kwargs, ): """Evaluates the specified semantic segmentation masks in this @@ -3328,6 +3366,9 @@ class for the purposes of computing evaluation metrics like The supported values are ``fo.evaluation_config.segmentation_backends.keys()`` and the default is ``fo.evaluation_config.segmentation_default_backend`` + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`fiftyone.utils.eval.segmentation.SegmentationEvaluationConfig` being used @@ -3342,6 +3383,7 @@ class for the purposes of computing evaluation metrics like eval_key=eval_key, mask_targets=mask_targets, method=method, + progress=progress, **kwargs, ) @@ -8155,6 +8197,7 @@ def draw_labels( label_fields=None, overwrite=False, config=None, + progress=None, **kwargs, ): """Renders annotated versions of the media in the collection with the @@ -8184,6 +8227,9 @@ def draw_labels( config (None): an optional :class:`fiftyone.utils.annotations.DrawConfig` configuring how to draw the labels + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments specifying parameters of the default :class:`fiftyone.utils.annotations.DrawConfig` to override @@ -8211,6 +8257,7 @@ def draw_labels( rel_dir=rel_dir, label_fields=label_fields, config=config, + progress=progress, **kwargs, ) @@ -8221,6 +8268,7 @@ def draw_labels( rel_dir=rel_dir, label_fields=label_fields, config=config, + progress=progress, **kwargs, ) @@ -8243,6 +8291,7 @@ def export( label_field=None, frame_labels_field=None, overwrite=False, + progress=None, **kwargs, ): """Exports the samples in the collection to disk. @@ -8416,6 +8465,9 @@ def export( overwrite (False): whether to delete existing directories before performing the export (True) or to merge the export with existing files and directories (False) + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the dataset exporter's constructor. If you are exporting image patches, this can also contain keyword arguments for @@ -8441,6 +8493,7 @@ def export( label_field=label_field, frame_labels_field=frame_labels_field, overwrite=overwrite, + progress=progress, **kwargs, ) @@ -8708,6 +8761,7 @@ def load_annotations( dest_field=None, unexpected="prompt", cleanup=False, + progress=None, **kwargs, ): """Downloads the labels from the given annotation run from the @@ -8735,6 +8789,9 @@ def load_annotations( labels, or ``None`` if there aren't any cleanup (False): whether to delete any informtation regarding this run from the annotation backend after loading the annotations + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: keyword arguments for the run's :meth:`fiftyone.core.annotation.AnnotationMethodConfig.load_credentials` method @@ -8749,6 +8806,7 @@ def load_annotations( dest_field=dest_field, unexpected=unexpected, cleanup=cleanup, + progress=progress, **kwargs, ) @@ -9038,6 +9096,7 @@ def to_dict( include_frames=False, frame_labels_dir=None, pretty_print=False, + progress=None, ): """Returns a JSON dictionary representation of the collection. @@ -9062,6 +9121,9 @@ def to_dict( readable format with newlines and indentations. Only applicable to datasets that contain videos when a ``frame_labels_dir`` is provided + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a JSON dict @@ -9117,7 +9179,7 @@ def to_dict( # Serialize samples samples = [] - for sample in view.iter_samples(progress=True): + for sample in view.iter_samples(progress=progress): sd = sample.to_dict( include_frames=include_frames, include_private=include_private, @@ -10932,6 +10994,7 @@ def _export( label_field=None, frame_labels_field=None, overwrite=False, + progress=None, **kwargs, ): if dataset_type is None and dataset_exporter is None: @@ -11003,6 +11066,7 @@ def _export( dataset_exporter=dataset_exporter, label_field=label_field, frame_labels_field=frame_labels_field, + progress=progress, **kwargs, ) diff --git a/fiftyone/core/dataset.py b/fiftyone/core/dataset.py index 9afc213331..3610da7946 100644 --- a/fiftyone/core/dataset.py +++ b/fiftyone/core/dataset.py @@ -2164,8 +2164,9 @@ def make_label(): sample.ground_truth.label = make_label() Args: - progress (False): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -2178,10 +2179,9 @@ def make_label(): with contextlib.ExitStack() as exit_context: samples = self._iter_samples() - if progress: - pb = fou.ProgressBar(total=len(self)) - exit_context.enter_context(pb) - samples = pb(samples) + pb = fou.ProgressBar(total=self, progress=progress) + exit_context.enter_context(pb) + samples = pb(samples) if autosave: save_context = foc.SaveContext(self, batch_size=batch_size) @@ -2265,8 +2265,9 @@ def make_label(): Args: group_slices (None): an optional subset of group slices to load - progress (False): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -2283,10 +2284,9 @@ def make_label(): with contextlib.ExitStack() as exit_context: groups = self._iter_groups(group_slices=group_slices) - if progress: - pb = fou.ProgressBar(total=len(self)) - exit_context.enter_context(pb) - groups = pb(groups) + pb = fou.ProgressBar(total=self, progress=progress) + exit_context.enter_context(pb) + groups = pb(groups) if autosave: save_context = foc.SaveContext(self, batch_size=batch_size) @@ -2440,6 +2440,7 @@ def add_samples( expand_schema=True, dynamic=False, validate=True, + progress=None, num_samples=None, ): """Adds the given samples to the dataset. @@ -2459,21 +2460,21 @@ def add_samples( document fields that are encountered validate (True): whether to validate that the fields of each sample are compliant with the dataset schema before adding it + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead num_samples (None): the number of samples in ``samples``. If not - provided, this is computed via ``len(samples)``, if possible. - This value is optional and is used only for progress tracking + provided, this is computed (if possible) via ``len(samples)`` + if needed for progress tracking Returns: a list of IDs of the samples in the dataset """ if num_samples is None: - try: - num_samples = len(samples) - except: - pass + num_samples = samples batcher = fou.get_default_batcher( - samples, progress=True, total=num_samples + samples, progress=progress, total=num_samples ) sample_ids = [] @@ -2492,6 +2493,7 @@ def add_collection( include_info=True, overwrite_info=False, new_ids=False, + progress=None, ): """Adds the contents of the given collection to the dataset. @@ -2510,6 +2512,9 @@ def add_collection( information. Only applicable when ``include_info`` is True new_ids (False): whether to generate new sample/frame/group IDs. By default, the IDs of the input collection are retained + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to this dataset @@ -2530,6 +2535,7 @@ def add_collection( insert_new=True, include_info=include_info, overwrite_info=overwrite_info, + progress=progress, ) return self.skip(num_samples).values("id") @@ -2573,15 +2579,13 @@ def _upsert_samples( expand_schema=True, dynamic=False, validate=True, + progress=None, num_samples=None, ): if num_samples is None: - try: - num_samples = len(samples) - except: - pass + num_samples = samples - batcher = fou.get_default_batcher(samples, progress=True) + batcher = fou.get_default_batcher(samples, progress=progress) with batcher: for batch in batcher: @@ -2805,6 +2809,7 @@ def merge_samples( dynamic=False, include_info=True, overwrite_info=False, + progress=None, num_samples=None, ): """Merges the given samples into this dataset. @@ -2897,9 +2902,12 @@ def merge_samples( information. Only applicable when ``samples`` is a :class:`fiftyone.core.collections.SampleCollection` and ``include_info`` is True + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead num_samples (None): the number of samples in ``samples``. If not - provided, this is computed via ``len(samples)``, if possible. - This value is optional and is used only for progress tracking + provided, this is computed (if possible) via ``len(samples)`` + if needed for progress tracking """ if fields is not None: if etau.is_str(fields): @@ -2953,7 +2961,10 @@ def merge_samples( try: tmp.add_samples( - samples, dynamic=dynamic, num_samples=num_samples + samples, + dynamic=dynamic, + progress=progress, + num_samples=num_samples, ) self.merge_samples( @@ -2986,6 +2997,7 @@ def merge_samples( overwrite=overwrite, expand_schema=expand_schema, dynamic=dynamic, + progress=progress, num_samples=num_samples, ) @@ -3962,6 +3974,7 @@ def add_dir( expand_schema=True, dynamic=False, add_info=True, + progress=None, **kwargs, ): """Adds the contents of the given directory to the dataset. @@ -4050,6 +4063,9 @@ def add_dir( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset's ``info`` + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4073,6 +4089,7 @@ def add_dir( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, ) def merge_dir( @@ -4094,6 +4111,7 @@ def merge_dir( expand_schema=True, dynamic=False, add_info=True, + progress=None, **kwargs, ): """Merges the contents of the given directory into the dataset. @@ -4251,6 +4269,9 @@ def merge_dir( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4279,6 +4300,7 @@ def merge_dir( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, ) def add_archive( @@ -4293,6 +4315,7 @@ def add_archive( dynamic=False, add_info=True, cleanup=True, + progress=None, **kwargs, ): """Adds the contents of the given archive to the dataset. @@ -4377,6 +4400,9 @@ def add_archive( add_info (True): whether to add dataset info from the importer (if any) to the dataset's ``info`` cleanup (True): whether to delete the archive after extracting it + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4395,6 +4421,7 @@ def add_archive( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, **kwargs, ) @@ -4418,6 +4445,7 @@ def merge_archive( dynamic=False, add_info=True, cleanup=True, + progress=None, **kwargs, ): """Merges the contents of the given archive into the dataset. @@ -4571,6 +4599,9 @@ def merge_archive( add_info (True): whether to add dataset info from the importer (if any) to the dataset cleanup (True): whether to delete the archive after extracting it + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -4594,6 +4625,7 @@ def merge_archive( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, **kwargs, ) @@ -4605,6 +4637,7 @@ def add_importer( expand_schema=True, dynamic=False, add_info=True, + progress=None, ): """Adds the samples from the given :class:`fiftyone.utils.data.importers.DatasetImporter` to the dataset. @@ -4638,6 +4671,9 @@ def add_importer( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset's ``info`` + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -4650,6 +4686,7 @@ def add_importer( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, ) def merge_importer( @@ -4668,6 +4705,7 @@ def merge_importer( expand_schema=True, dynamic=False, add_info=True, + progress=None, ): """Merges the samples from the given :class:`fiftyone.utils.data.importers.DatasetImporter` into the @@ -4771,6 +4809,9 @@ def merge_importer( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ return foud.merge_samples( self, @@ -4788,9 +4829,16 @@ def merge_importer( expand_schema=expand_schema, dynamic=dynamic, add_info=add_info, + progress=progress, ) - def add_images(self, paths_or_samples, sample_parser=None, tags=None): + def add_images( + self, + paths_or_samples, + sample_parser=None, + tags=None, + progress=None, + ): """Adds the given images to the dataset. This operation does not read the images. @@ -4809,6 +4857,9 @@ def add_images(self, paths_or_samples, sample_parser=None, tags=None): instance to use to parse the samples tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -4817,7 +4868,11 @@ def add_images(self, paths_or_samples, sample_parser=None, tags=None): sample_parser = foud.ImageSampleParser() return foud.add_images( - self, paths_or_samples, sample_parser, tags=tags + self, + paths_or_samples, + sample_parser, + tags=tags, + progress=progress, ) def add_labeled_images( @@ -4828,6 +4883,7 @@ def add_labeled_images( tags=None, expand_schema=True, dynamic=False, + progress=None, ): """Adds the given labeled images to the dataset. @@ -4860,6 +4916,9 @@ def add_labeled_images( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -4872,9 +4931,16 @@ def add_labeled_images( tags=tags, expand_schema=expand_schema, dynamic=dynamic, + progress=progress, ) - def add_images_dir(self, images_dir, tags=None, recursive=True): + def add_images_dir( + self, + images_dir, + tags=None, + recursive=True, + progress=None, + ): """Adds the given directory of images to the dataset. See :class:`fiftyone.types.ImageDirectory` for format details. In @@ -4887,15 +4953,20 @@ def add_images_dir(self, images_dir, tags=None, recursive=True): tags (None): an optional tag or iterable of tags to attach to each sample recursive (True): whether to recursively traverse subdirectories + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset """ image_paths = foud.parse_images_dir(images_dir, recursive=recursive) sample_parser = foud.ImageSampleParser() - return self.add_images(image_paths, sample_parser, tags=tags) + return self.add_images( + image_paths, sample_parser, tags=tags, progress=progress + ) - def add_images_patt(self, images_patt, tags=None): + def add_images_patt(self, images_patt, tags=None, progress=None): """Adds the given glob pattern of images to the dataset. This operation does not read the images. @@ -4905,13 +4976,18 @@ def add_images_patt(self, images_patt, tags=None): ``/path/to/images/*.jpg`` tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset """ image_paths = etau.get_glob_matches(images_patt) sample_parser = foud.ImageSampleParser() - return self.add_images(image_paths, sample_parser, tags=tags) + return self.add_images( + image_paths, sample_parser, tags=tags, progress=progress + ) def ingest_images( self, @@ -4920,6 +4996,7 @@ def ingest_images( tags=None, dataset_dir=None, image_format=None, + progress=None, ): """Ingests the given iterable of images into the dataset. @@ -4943,6 +5020,9 @@ def ingest_images( written. By default, :func:`get_default_dataset_dir` is used image_format (None): the image format to use to write the images to disk. By default, ``fiftyone.config.default_image_ext`` is used + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -4960,7 +5040,9 @@ def ingest_images( image_format=image_format, ) - return self.add_importer(dataset_ingestor, tags=tags) + return self.add_importer( + dataset_ingestor, tags=tags, progress=progress + ) def ingest_labeled_images( self, @@ -4972,6 +5054,7 @@ def ingest_labeled_images( dynamic=False, dataset_dir=None, image_format=None, + progress=None, ): """Ingests the given iterable of labeled image samples into the dataset. @@ -5007,6 +5090,9 @@ def ingest_labeled_images( written. By default, :func:`get_default_dataset_dir` is used image_format (None): the image format to use to write the images to disk. By default, ``fiftyone.config.default_image_ext`` is used + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -5027,9 +5113,16 @@ def ingest_labeled_images( tags=tags, expand_schema=expand_schema, dynamic=dynamic, + progress=progress, ) - def add_videos(self, paths_or_samples, sample_parser=None, tags=None): + def add_videos( + self, + paths_or_samples, + sample_parser=None, + tags=None, + progress=None, + ): """Adds the given videos to the dataset. This operation does not read the videos. @@ -5048,6 +5141,9 @@ def add_videos(self, paths_or_samples, sample_parser=None, tags=None): instance to use to parse the samples tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -5056,7 +5152,7 @@ def add_videos(self, paths_or_samples, sample_parser=None, tags=None): sample_parser = foud.VideoSampleParser() return foud.add_videos( - self, paths_or_samples, sample_parser, tags=tags + self, paths_or_samples, sample_parser, tags=tags, progress=progress ) def add_labeled_videos( @@ -5067,6 +5163,7 @@ def add_labeled_videos( tags=None, expand_schema=True, dynamic=False, + progress=None, ): """Adds the given labeled videos to the dataset. @@ -5101,6 +5198,9 @@ def add_labeled_videos( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -5113,9 +5213,12 @@ def add_labeled_videos( tags=tags, expand_schema=expand_schema, dynamic=dynamic, + progress=progress, ) - def add_videos_dir(self, videos_dir, tags=None, recursive=True): + def add_videos_dir( + self, videos_dir, tags=None, recursive=True, progress=None + ): """Adds the given directory of videos to the dataset. See :class:`fiftyone.types.VideoDirectory` for format details. In @@ -5128,15 +5231,20 @@ def add_videos_dir(self, videos_dir, tags=None, recursive=True): tags (None): an optional tag or iterable of tags to attach to each sample recursive (True): whether to recursively traverse subdirectories + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset """ video_paths = foud.parse_videos_dir(videos_dir, recursive=recursive) sample_parser = foud.VideoSampleParser() - return self.add_videos(video_paths, sample_parser, tags=tags) + return self.add_videos( + video_paths, sample_parser, tags=tags, progress=progress + ) - def add_videos_patt(self, videos_patt, tags=None): + def add_videos_patt(self, videos_patt, tags=None, progress=None): """Adds the given glob pattern of videos to the dataset. This operation does not read/decode the videos. @@ -5146,13 +5254,18 @@ def add_videos_patt(self, videos_patt, tags=None): ``/path/to/videos/*.mp4`` tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset """ video_paths = etau.get_glob_matches(videos_patt) sample_parser = foud.VideoSampleParser() - return self.add_videos(video_paths, sample_parser, tags=tags) + return self.add_videos( + video_paths, sample_parser, tags=tags, progress=progress + ) def ingest_videos( self, @@ -5160,6 +5273,7 @@ def ingest_videos( sample_parser=None, tags=None, dataset_dir=None, + progress=None, ): """Ingests the given iterable of videos into the dataset. @@ -5181,6 +5295,9 @@ def ingest_videos( sample dataset_dir (None): the directory in which the videos will be written. By default, :func:`get_default_dataset_dir` is used + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -5195,7 +5312,9 @@ def ingest_videos( dataset_dir, paths_or_samples, sample_parser ) - return self.add_importer(dataset_ingestor, tags=tags) + return self.add_importer( + dataset_ingestor, tags=tags, progress=progress + ) def ingest_labeled_videos( self, @@ -5205,6 +5324,7 @@ def ingest_labeled_videos( expand_schema=True, dynamic=False, dataset_dir=None, + progress=None, ): """Ingests the given iterable of labeled video samples into the dataset. @@ -5229,6 +5349,9 @@ def ingest_labeled_videos( document fields that are encountered dataset_dir (None): the directory in which the videos will be written. By default, :func:`get_default_dataset_dir` is used + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples in the dataset @@ -5245,6 +5368,7 @@ def ingest_labeled_videos( tags=tags, expand_schema=expand_schema, dynamic=dynamic, + progress=progress, ) @classmethod @@ -5260,6 +5384,7 @@ def from_dir( label_field=None, tags=None, dynamic=False, + progress=None, **kwargs, ): """Creates a :class:`Dataset` from the contents of the given directory. @@ -5348,6 +5473,9 @@ def from_dir( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -5364,6 +5492,7 @@ def from_dir( label_field=label_field, tags=tags, dynamic=dynamic, + progress=progress, **kwargs, ) return dataset @@ -5382,6 +5511,7 @@ def from_archive( tags=None, dynamic=False, cleanup=True, + progress=None, **kwargs, ): """Creates a :class:`Dataset` from the contents of the given archive. @@ -5467,6 +5597,9 @@ def from_archive( dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered cleanup (True): whether to delete the archive after extracting it + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead **kwargs: optional keyword arguments to pass to the constructor of the :class:`fiftyone.utils.data.importers.DatasetImporter` for the specified ``dataset_type`` @@ -5484,6 +5617,7 @@ def from_archive( tags=tags, dynamic=dynamic, cleanup=cleanup, + progress=progress, **kwargs, ) return dataset @@ -5498,6 +5632,7 @@ def from_importer( label_field=None, tags=None, dynamic=False, + progress=None, ): """Creates a :class:`Dataset` by importing the samples in the given :class:`fiftyone.utils.data.importers.DatasetImporter`. @@ -5533,6 +5668,9 @@ def from_importer( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -5543,6 +5681,7 @@ def from_importer( label_field=label_field, tags=tags, dynamic=dynamic, + progress=progress, ) return dataset @@ -5555,6 +5694,7 @@ def from_images( persistent=False, overwrite=False, tags=None, + progress=None, ): """Creates a :class:`Dataset` from the given images. @@ -5581,13 +5721,19 @@ def from_images( the same name tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) dataset.add_images( - paths_or_samples, sample_parser=sample_parser, tags=tags + paths_or_samples, + sample_parser=sample_parser, + tags=tags, + progress=progress, ) return dataset @@ -5602,6 +5748,7 @@ def from_labeled_images( label_field=None, tags=None, dynamic=False, + progress=None, ): """Creates a :class:`Dataset` from the given labeled images. @@ -5637,6 +5784,9 @@ def from_labeled_images( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -5648,6 +5798,7 @@ def from_labeled_images( label_field=label_field, tags=tags, dynamic=dynamic, + progress=progress, ) return dataset @@ -5660,6 +5811,7 @@ def from_images_dir( overwrite=False, tags=None, recursive=True, + progress=None, ): """Creates a :class:`Dataset` from the given directory of images. @@ -5676,12 +5828,17 @@ def from_images_dir( tags (None): an optional tag or iterable of tags to attach to each sample recursive (True): whether to recursively traverse subdirectories + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) - dataset.add_images_dir(images_dir, tags=tags, recursive=recursive) + dataset.add_images_dir( + images_dir, tags=tags, recursive=recursive, progress=progress + ) return dataset @classmethod @@ -5692,6 +5849,7 @@ def from_images_patt( persistent=False, overwrite=False, tags=None, + progress=None, ): """Creates a :class:`Dataset` from the given glob pattern of images. @@ -5708,12 +5866,15 @@ def from_images_patt( the same name tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) - dataset.add_images_patt(images_patt, tags=tags) + dataset.add_images_patt(images_patt, tags=tags, progress=progress) return dataset @classmethod @@ -5725,6 +5886,7 @@ def from_videos( persistent=False, overwrite=False, tags=None, + progress=None, ): """Creates a :class:`Dataset` from the given videos. @@ -5751,13 +5913,19 @@ def from_videos( the same name tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) dataset.add_videos( - paths_or_samples, sample_parser=sample_parser, tags=tags + paths_or_samples, + sample_parser=sample_parser, + tags=tags, + progress=progress, ) return dataset @@ -5772,6 +5940,7 @@ def from_labeled_videos( label_field=None, tags=None, dynamic=False, + progress=None, ): """Creates a :class:`Dataset` from the given labeled videos. @@ -5808,6 +5977,9 @@ def from_labeled_videos( sample dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -5819,6 +5991,7 @@ def from_labeled_videos( label_field=label_field, tags=tags, dynamic=dynamic, + progress=progress, ) return dataset @@ -5831,6 +6004,7 @@ def from_videos_dir( overwrite=False, tags=None, recursive=True, + progress=None, ): """Creates a :class:`Dataset` from the given directory of videos. @@ -5852,7 +6026,9 @@ def from_videos_dir( a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) - dataset.add_videos_dir(videos_dir, tags=tags, recursive=recursive) + dataset.add_videos_dir( + videos_dir, tags=tags, recursive=recursive, progress=progress + ) return dataset @classmethod @@ -5863,6 +6039,7 @@ def from_videos_patt( persistent=False, overwrite=False, tags=None, + progress=None, ): """Creates a :class:`Dataset` from the given glob pattern of videos. @@ -5884,7 +6061,7 @@ def from_videos_patt( a :class:`Dataset` """ dataset = cls(name, persistent=persistent, overwrite=overwrite) - dataset.add_videos_patt(videos_patt, tags=tags) + dataset.add_videos_patt(videos_patt, tags=tags, progress=progress) return dataset @classmethod @@ -5896,6 +6073,7 @@ def from_dict( overwrite=False, rel_dir=None, frame_labels_dir=None, + progress=None, ): """Loads a :class:`Dataset` from a JSON dictionary generated by :meth:`fiftyone.core.collections.SampleCollection.to_dict`. @@ -5920,6 +6098,9 @@ def from_dict( is assumed that the frame labels are included directly in the provided JSON dict. Only applicable to datasets that contain videos + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -5999,7 +6180,10 @@ def parse_sample(sd): _samples = map(parse_sample, samples) dataset.add_samples( - _samples, expand_schema=False, num_samples=len(samples) + _samples, + expand_schema=False, + progress=progress, + num_samples=samples, ) return dataset @@ -6013,6 +6197,7 @@ def from_json( overwrite=False, rel_dir=None, frame_labels_dir=None, + progress=None, ): """Loads a :class:`Dataset` from JSON generated by :func:`fiftyone.core.collections.SampleCollection.write_json` or @@ -6033,6 +6218,9 @@ def from_json( of each sample, if the filepath is not absolute (begins with a path separator). The path is converted to an absolute path (if necessary) via :func:`fiftyone.core.storage.normalize_path` + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`Dataset` @@ -6045,6 +6233,7 @@ def from_json( overwrite=overwrite, rel_dir=rel_dir, frame_labels_dir=frame_labels_dir, + progress=progress, ) def _add_view_stage(self, stage): @@ -7601,6 +7790,7 @@ def _merge_samples_python( overwrite=True, expand_schema=True, dynamic=False, + progress=None, num_samples=None, ): if dataset.media_type == fom.GROUP: @@ -7615,10 +7805,7 @@ def _merge_samples_python( samples = samples.select_group_slices(_allow_mixed=True) if num_samples is None: - try: - num_samples = len(samples) - except: - pass + num_samples = samples if key_fcn is None: id_map = {k: v for k, v in zip(*dst.values([key_field, "_id"]))} @@ -7626,7 +7813,7 @@ def _merge_samples_python( else: id_map = {} logger.info("Indexing dataset...") - for sample in dst.iter_samples(progress=True): + for sample in dst.iter_samples(progress=progress): id_map[key_fcn(sample)] = sample._id _samples = _make_merge_samples_generator( @@ -7648,6 +7835,7 @@ def _merge_samples_python( _samples, expand_schema=expand_schema, dynamic=dynamic, + progress=progress, num_samples=num_samples, ) diff --git a/fiftyone/core/metadata.py b/fiftyone/core/metadata.py index 86737ebd7d..605ef1d59e 100644 --- a/fiftyone/core/metadata.py +++ b/fiftyone/core/metadata.py @@ -235,6 +235,7 @@ def compute_metadata( num_workers=None, skip_failures=True, warn_failures=False, + progress=None, ): """Populates the ``metadata`` field of all samples in the collection. @@ -250,6 +251,9 @@ def compute_metadata( error if metadata cannot be computed for a sample warn_failures (False): whether to log a warning if metadata cannot be computed for a sample + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ num_workers = fou.recommend_thread_pool_workers(num_workers) @@ -259,10 +263,15 @@ def compute_metadata( ) if num_workers <= 1: - _compute_metadata(sample_collection, overwrite=overwrite) + _compute_metadata( + sample_collection, overwrite=overwrite, progress=progress + ) else: _compute_metadata_multi( - sample_collection, num_workers, overwrite=overwrite + sample_collection, + num_workers, + overwrite=overwrite, + progress=progress, ) if skip_failures and not warn_failures: @@ -332,7 +341,9 @@ def get_image_info(f): return width, height, len(img.getbands()) -def _compute_metadata(sample_collection, overwrite=False, batch_size=1000): +def _compute_metadata( + sample_collection, overwrite=False, batch_size=1000, progress=None +): if not overwrite: sample_collection = sample_collection.exists("metadata", False) @@ -351,7 +362,7 @@ def _compute_metadata(sample_collection, overwrite=False, batch_size=1000): values = {} try: - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: for args in pb(inputs): sample_id, metadata = _do_compute_metadata(args) values[sample_id] = metadata @@ -365,7 +376,11 @@ def _compute_metadata(sample_collection, overwrite=False, batch_size=1000): def _compute_metadata_multi( - sample_collection, num_workers, overwrite=False, batch_size=1000 + sample_collection, + num_workers, + overwrite=False, + batch_size=1000, + progress=None, ): if not overwrite: sample_collection = sample_collection.exists("metadata", False) @@ -386,7 +401,7 @@ def _compute_metadata_multi( try: with multiprocessing.dummy.Pool(processes=num_workers) as pool: - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: for sample_id, metadata in pb( pool.imap_unordered(_do_compute_metadata, inputs) ): diff --git a/fiftyone/core/models.py b/fiftyone/core/models.py index 1d99f30954..bd542fe1ac 100644 --- a/fiftyone/core/models.py +++ b/fiftyone/core/models.py @@ -56,6 +56,7 @@ def apply_model( skip_failures=True, output_dir=None, rel_dir=None, + progress=None, **kwargs, ): """Applies the :class:`FiftyOne model ` or @@ -99,6 +100,9 @@ def apply_model( ``output_dir`` that match the shape of the input paths. The path is converted to an absolute path (if necessary) via :func:`fiftyone.core.storage.normalize_path` + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation """ @@ -210,6 +214,7 @@ def apply_model( confidence_thresh, skip_failures, filename_maker, + progress, ) batch_size = _parse_batch_size(batch_size, model, use_data_loader) @@ -226,6 +231,7 @@ def apply_model( batch_size, skip_failures, filename_maker, + progress, ) return _apply_image_model_to_frames_single( @@ -235,6 +241,7 @@ def apply_model( confidence_thresh, skip_failures, filename_maker, + progress, ) if use_data_loader: @@ -247,6 +254,7 @@ def apply_model( num_workers, skip_failures, filename_maker, + progress, ) if batch_size is not None: @@ -258,6 +266,7 @@ def apply_model( batch_size, skip_failures, filename_maker, + progress, ) return _apply_image_model_single( @@ -267,6 +276,7 @@ def apply_model( confidence_thresh, skip_failures, filename_maker, + progress, ) @@ -288,10 +298,11 @@ def _apply_image_model_single( confidence_thresh, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(samples): try: img = foui.read(sample.filepath) @@ -325,11 +336,12 @@ def _apply_image_model_batch( batch_size, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) samples_loader = fou.iter_batches(samples, batch_size) - with fou.ProgressBar(samples) as pb: + with fou.ProgressBar(samples, progress=progress) as pb: for sample_batch in samples_loader: try: imgs = [foui.read(sample.filepath) for sample in sample_batch] @@ -375,6 +387,7 @@ def _apply_image_model_data_loader( num_workers, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) samples_loader = fou.iter_batches(samples, batch_size) @@ -382,7 +395,7 @@ def _apply_image_model_data_loader( samples, model, batch_size, num_workers, skip_failures ) - with fou.ProgressBar(samples) as pb: + with fou.ProgressBar(samples, progress=progress) as pb: for sample_batch, imgs in zip(samples_loader, data_loader): try: if isinstance(imgs, Exception): @@ -427,12 +440,13 @@ def _apply_image_model_to_frames_single( confidence_thresh, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) frame_counts, total_frame_count = _get_frame_counts(samples) is_clips = samples._dataset._is_clips - with fou.ProgressBar(total=total_frame_count) as pb: + with fou.ProgressBar(total=total_frame_count, progress=progress) as pb: for idx, sample in enumerate(samples): if is_clips: frames = etaf.FrameRange(*sample.support) @@ -482,12 +496,13 @@ def _apply_image_model_to_frames_batch( batch_size, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) frame_counts, total_frame_count = _get_frame_counts(samples) is_clips = samples._dataset._is_clips - with fou.ProgressBar(total=total_frame_count) as pb: + with fou.ProgressBar(total=total_frame_count, progress=progress) as pb: for idx, sample in enumerate(samples): if is_clips: frames = etaf.FrameRange(*sample.support) @@ -542,11 +557,12 @@ def _apply_video_model( confidence_thresh, skip_failures, filename_maker, + progress, ): needs_samples = isinstance(model, SamplesMixin) is_clips = samples._dataset._is_clips - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(samples): if is_clips: frames = etaf.FrameRange(*sample.support) @@ -713,6 +729,7 @@ def compute_embeddings( batch_size=None, num_workers=None, skip_failures=True, + progress=None, **kwargs, ): """Computes embeddings for the samples in the collection using the given @@ -749,6 +766,9 @@ def compute_embeddings( skip_failures (True): whether to gracefully continue without raising an error if embeddings cannot be generated for a sample. Only applicable to :class:`Model` instances + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional model-specific keyword arguments passed through to the underlying inference implementation @@ -851,7 +871,7 @@ def compute_embeddings( if samples.media_type == fom.VIDEO and model.media_type == "video": return _compute_video_embeddings( - samples, model, embeddings_field, skip_failures + samples, model, embeddings_field, skip_failures, progress ) batch_size = _parse_batch_size(batch_size, model, use_data_loader) @@ -859,11 +879,16 @@ def compute_embeddings( if samples.media_type == fom.VIDEO and model.media_type == "image": if batch_size is not None: return _compute_frame_embeddings_batch( - samples, model, embeddings_field, batch_size, skip_failures + samples, + model, + embeddings_field, + batch_size, + skip_failures, + progress, ) return _compute_frame_embeddings_single( - samples, model, embeddings_field, skip_failures + samples, model, embeddings_field, skip_failures, progress ) if use_data_loader: @@ -874,27 +899,33 @@ def compute_embeddings( batch_size, num_workers, skip_failures, + progress, ) if batch_size is not None: return _compute_image_embeddings_batch( - samples, model, embeddings_field, batch_size, skip_failures + samples, + model, + embeddings_field, + batch_size, + skip_failures, + progress, ) return _compute_image_embeddings_single( - samples, model, embeddings_field, skip_failures + samples, model, embeddings_field, skip_failures, progress ) def _compute_image_embeddings_single( - samples, model, embeddings_field, skip_failures + samples, model, embeddings_field, skip_failures, progress ): samples = samples.select_fields() embeddings = [] errors = False - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(samples): embedding = None @@ -924,7 +955,7 @@ def _compute_image_embeddings_single( def _compute_image_embeddings_batch( - samples, model, embeddings_field, batch_size, skip_failures + samples, model, embeddings_field, batch_size, skip_failures, progress ): samples = samples.select_fields() samples_loader = fou.iter_batches(samples, batch_size) @@ -932,7 +963,7 @@ def _compute_image_embeddings_batch( embeddings = [] errors = False - with fou.ProgressBar(samples) as pb: + with fou.ProgressBar(samples, progress=progress) as pb: for sample_batch in samples_loader: embeddings_batch = [None] * len(sample_batch) @@ -970,7 +1001,13 @@ def _compute_image_embeddings_batch( def _compute_image_embeddings_data_loader( - samples, model, embeddings_field, batch_size, num_workers, skip_failures + samples, + model, + embeddings_field, + batch_size, + num_workers, + skip_failures, + progress, ): samples = samples.select_fields() samples_loader = fou.iter_batches(samples, batch_size) @@ -981,7 +1018,7 @@ def _compute_image_embeddings_data_loader( embeddings = [] errors = False - with fou.ProgressBar(samples) as pb: + with fou.ProgressBar(samples, progress=progress) as pb: for sample_batch, imgs in zip(samples_loader, data_loader): embeddings_batch = [None] * len(sample_batch) @@ -1021,7 +1058,7 @@ def _compute_image_embeddings_data_loader( def _compute_frame_embeddings_single( - samples, model, embeddings_field, skip_failures + samples, model, embeddings_field, skip_failures, progress ): samples = samples.select_fields() frame_counts, total_frame_count = _get_frame_counts(samples) @@ -1029,7 +1066,7 @@ def _compute_frame_embeddings_single( embeddings_dict = {} - with fou.ProgressBar(total=total_frame_count) as pb: + with fou.ProgressBar(total=total_frame_count, progress=progress) as pb: for idx, sample in enumerate(samples): embeddings = [] @@ -1080,7 +1117,7 @@ def _compute_frame_embeddings_single( def _compute_frame_embeddings_batch( - samples, model, embeddings_field, batch_size, skip_failures + samples, model, embeddings_field, batch_size, skip_failures, progress ): samples = samples.select_fields() frame_counts, total_frame_count = _get_frame_counts(samples) @@ -1088,7 +1125,7 @@ def _compute_frame_embeddings_batch( embeddings_dict = {} - with fou.ProgressBar(total=total_frame_count) as pb: + with fou.ProgressBar(total=total_frame_count, progress=progress) as pb: for idx, sample in enumerate(samples): embeddings = [] @@ -1143,14 +1180,16 @@ def _compute_frame_embeddings_batch( return embeddings_dict -def _compute_video_embeddings(samples, model, embeddings_field, skip_failures): +def _compute_video_embeddings( + samples, model, embeddings_field, skip_failures, progress +): samples = samples.select_fields() is_clips = samples._dataset._is_clips embeddings = [] errors = False - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(samples): if is_clips: frames = etaf.FrameRange(*sample.support) @@ -1196,6 +1235,7 @@ def compute_patch_embeddings( batch_size=None, num_workers=None, skip_failures=True, + progress=None, ): """Computes embeddings for the image patches defined by ``patches_field`` of the samples in the collection using the given :class:`Model`. @@ -1246,6 +1286,9 @@ def compute_patch_embeddings( Only applicable for Torch models skip_failures (True): whether to gracefully continue without raising an error if embeddings cannot be generated for a sample + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: one of the following: @@ -1359,6 +1402,7 @@ def compute_patch_embeddings( handle_missing, batch_size, skip_failures, + progress, ) if use_data_loader: @@ -1373,6 +1417,7 @@ def compute_patch_embeddings( batch_size, num_workers, skip_failures, + progress, ) return _embed_patches( @@ -1385,6 +1430,7 @@ def compute_patch_embeddings( handle_missing, batch_size, skip_failures, + progress, ) @@ -1398,6 +1444,7 @@ def _embed_patches( handle_missing, batch_size, skip_failures, + progress, ): samples = samples.select_fields(patches_field) @@ -1406,7 +1453,7 @@ def _embed_patches( else: embeddings_dict = {} - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(samples): embeddings = None @@ -1494,6 +1541,7 @@ def _embed_patches_data_loader( batch_size, num_workers, skip_failures, + progress, ): samples = samples.select_fields(patches_field) @@ -1513,7 +1561,7 @@ def _embed_patches_data_loader( else: embeddings_dict = {} - with fou.ProgressBar(samples) as pb: + with fou.ProgressBar(samples, progress=progress) as pb: for sample, patches in pb(zip(samples, data_loader)): embeddings = None @@ -1561,6 +1609,7 @@ def _embed_frame_patches( handle_missing, batch_size, skip_failures, + progress, ): _patches_field = samples._FRAMES_PREFIX + patches_field samples = samples.select_fields(_patches_field) @@ -1572,7 +1621,7 @@ def _embed_frame_patches( else: embeddings_dict = {} - with fou.ProgressBar(total=total_frame_count) as pb: + with fou.ProgressBar(total=total_frame_count, progress=progress) as pb: for idx, sample in enumerate(samples): if is_clips: frames = etaf.FrameRange(*sample.support) diff --git a/fiftyone/core/odm/database.py b/fiftyone/core/odm/database.py index 3b24dacaac..38c6d81501 100644 --- a/fiftyone/core/odm/database.py +++ b/fiftyone/core/odm/database.py @@ -631,6 +631,7 @@ def export_collection( key="documents", patt="{idx:06d}-{id}.json", num_docs=None, + progress=None, ): """Exports the collection to disk in JSON format. @@ -647,22 +648,31 @@ def export_collection( to the document's ID num_docs (None): the total number of documents. If omitted, this must be computable via ``len(docs)`` + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if num_docs is None: num_docs = len(docs) if json_dir_or_path.endswith(".json"): - _export_collection_single(docs, json_dir_or_path, key, num_docs) + _export_collection_single( + docs, json_dir_or_path, key, num_docs, progress=progress + ) else: - _export_collection_multi(docs, json_dir_or_path, patt, num_docs) + _export_collection_multi( + docs, json_dir_or_path, patt, num_docs, progress=progress + ) -def _export_collection_single(docs, json_path, key, num_docs): +def _export_collection_single(docs, json_path, key, num_docs, progress=None): etau.ensure_basedir(json_path) with open(json_path, "w") as f: f.write('{"%s": [' % key) - with fou.ProgressBar(total=num_docs, iters_str="docs") as pb: + with fou.ProgressBar( + total=num_docs, iters_str="docs", progress=progress + ) as pb: for idx, doc in pb(enumerate(docs, 1)): f.write(json_util.dumps(doc)) if idx < num_docs: @@ -671,11 +681,13 @@ def _export_collection_single(docs, json_path, key, num_docs): f.write("]}") -def _export_collection_multi(docs, json_dir, patt, num_docs): +def _export_collection_multi(docs, json_dir, patt, num_docs, progress=None): etau.ensure_dir(json_dir) json_patt = os.path.join(json_dir, patt) - with fou.ProgressBar(total=num_docs, iters_str="docs") as pb: + with fou.ProgressBar( + total=num_docs, iters_str="docs", progress=progress + ) as pb: for idx, doc in pb(enumerate(docs, 1)): json_path = json_patt.format(idx=idx, id=str(doc["_id"])) export_document(doc, json_path) @@ -735,7 +747,7 @@ def _import_collection_multi(json_dir): return docs, len(json_paths) -def insert_documents(docs, coll, ordered=False, progress=False, num_docs=None): +def insert_documents(docs, coll, ordered=False, progress=None, num_docs=None): """Inserts documents into a collection. The ``_id`` field of the input documents will be populated if it is not @@ -745,8 +757,9 @@ def insert_documents(docs, coll, ordered=False, progress=False, num_docs=None): docs: an iterable of BSON document dicts coll: a pymongo collection ordered (False): whether the documents must be inserted in order - progress (False): whether to render a progress bar tracking the - insertion + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead num_docs (None): the total number of documents. Only used when ``progress=True``. If omitted, this will be computed via ``len(docs)``, if possible diff --git a/fiftyone/core/storage.py b/fiftyone/core/storage.py index 68ab5322ee..0b05a94d9c 100644 --- a/fiftyone/core/storage.py +++ b/fiftyone/core/storage.py @@ -689,7 +689,7 @@ def copy_file(inpath, outpath): _copy_file(inpath, outpath, cleanup=False) -def copy_files(inpaths, outpaths, skip_failures=False, progress=False): +def copy_files(inpaths, outpaths, skip_failures=False, progress=None): """Copies the files to the given locations. Args: @@ -697,14 +697,15 @@ def copy_files(inpaths, outpaths, skip_failures=False, progress=False): outpaths: a list of output paths skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ _copy_files(inpaths, outpaths, skip_failures, progress) def copy_dir( - indir, outdir, overwrite=True, skip_failures=False, progress=False + indir, outdir, overwrite=True, skip_failures=False, progress=None ): """Copies the input directory to the output directory. @@ -715,8 +716,9 @@ def copy_dir( or merge its contents (False) skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if overwrite and os.path.isdir(outdir): delete_dir(outdir) @@ -741,7 +743,7 @@ def move_file(inpath, outpath): _copy_file(inpath, outpath, cleanup=True) -def move_files(inpaths, outpaths, skip_failures=False, progress=False): +def move_files(inpaths, outpaths, skip_failures=False, progress=None): """Moves the files to the given locations. Args: @@ -749,8 +751,9 @@ def move_files(inpaths, outpaths, skip_failures=False, progress=False): outpaths: a list of output paths skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ tasks = [(i, o, skip_failures) for i, o in zip(inpaths, outpaths)] if tasks: @@ -758,7 +761,7 @@ def move_files(inpaths, outpaths, skip_failures=False, progress=False): def move_dir( - indir, outdir, overwrite=True, skip_failures=False, progress=False + indir, outdir, overwrite=True, skip_failures=False, progress=None ): """Moves the contents of the given directory into the given output directory. @@ -770,8 +773,9 @@ def move_dir( or merge its contents (False) skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if overwrite and os.path.isdir(outdir): delete_dir(outdir) @@ -793,7 +797,7 @@ def delete_file(path): _delete_file(path) -def delete_files(paths, skip_failures=False, progress=False): +def delete_files(paths, skip_failures=False, progress=None): """Deletes the files from the given locations. Any empty directories are also recursively deleted from the resulting @@ -803,8 +807,9 @@ def delete_files(paths, skip_failures=False, progress=False): paths: a list of paths skip_failures (False): whether to gracefully continue without raising an error if an operation fails - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ tasks = [(p, skip_failures) for p in paths] if tasks: @@ -821,15 +826,16 @@ def delete_dir(dirpath): etau.delete_dir(dirpath) -def run(fcn, tasks, num_workers=None, progress=False): +def run(fcn, tasks, num_workers=None, progress=None): """Applies the given function to each element of the given tasks. Args: fcn: a function that accepts a single argument tasks: an iterable of function aguments num_workers (None): a suggested number of threads to use - progress (False): whether to render a progress bar tracking the status - of the operation + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: the list of function outputs @@ -841,7 +847,7 @@ def run(fcn, tasks, num_workers=None, progress=False): except: num_tasks = None - kwargs = dict(total=num_tasks, iters_str="files", quiet=not progress) + kwargs = dict(total=num_tasks, iters_str="files", progress=progress) if num_workers <= 1: with fou.ProgressBar(**kwargs) as pb: @@ -860,7 +866,7 @@ def _copy_files(inpaths, outpaths, skip_failures, progress): _run(_do_copy_file, tasks, progress=progress) -def _run(fcn, tasks, num_workers=None, progress=False): +def _run(fcn, tasks, num_workers=None, progress=None): num_workers = fou.recommend_thread_pool_workers(num_workers) try: @@ -868,7 +874,7 @@ def _run(fcn, tasks, num_workers=None, progress=False): except: num_tasks = None - kwargs = dict(total=num_tasks, iters_str="files", quiet=not progress) + kwargs = dict(total=num_tasks, iters_str="files", progress=progress) if num_workers <= 1: with fou.ProgressBar(**kwargs) as pb: diff --git a/fiftyone/core/utils.py b/fiftyone/core/utils.py index 5a29730c12..92c70f8644 100644 --- a/fiftyone/core/utils.py +++ b/fiftyone/core/utils.py @@ -942,9 +942,22 @@ def __exit__(self, *args): class ProgressBar(etau.ProgressBar): """.. autoclass:: eta.core.utils.ProgressBar""" - def __init__(self, *args, **kwargs): - if "quiet" not in kwargs: - kwargs["quiet"] = not fo.config.show_progress_bars + def __init__(self, total=None, progress=None, quiet=None, **kwargs): + if progress is None: + progress = fo.config.show_progress_bars + + if quiet is not None: + progress = not quiet + + if callable(progress): + callback = progress + progress = False + else: + callback = None + + kwargs["total"] = total + if isinstance(progress, bool): + kwargs["quiet"] = not progress if "iters_str" not in kwargs: kwargs["iters_str"] = "samples" @@ -954,7 +967,117 @@ def __init__(self, *args, **kwargs): if foc.is_notebook_context() and "max_width" not in kwargs: kwargs["max_width"] = 90 - super().__init__(*args, **kwargs) + super().__init__(**kwargs) + + self._progress = progress + self._callback = callback + + def __call__(self, iterable): + # Ensure that `len(iterable)` is not computed unnecessarily + no_len = self._quiet and self._total is None + if no_len: + self._total = -1 + + super().__call__(iterable) + + if no_len: + self._total = None + + return self + + def set_iteration(self, *args, **kwargs): + super().set_iteration(*args, **kwargs) + + if self._callback is not None: + self._callback(self) + + +def report_progress(progress, n=None, dt=None): + """Wraps the provided progress function such that it will only be called + at the specified increments or time intervals. + + Example usage:: + + import fiftyone as fo + import fiftyone.zoo as foz + + def print_progress(pb): + if pb.complete: + print("COMPLETE") + else: + print("PROGRESS: %0.3f" % pb.progress) + + dataset = foz.load_zoo_dataset("cifar10", split="test") + + # Print progress at 10 equally-spaced increments + progress = fo.report_progress(print_progress, n=10) + dataset.compute_metadata(progress=progress) + + # Print progress every 0.5 seconds + progress = fo.report_progress(print_progress, dt=0.5) + dataset.compute_metadata(progress=progress, overwrite=True) + + Args: + progress: a function that accepts a :class:`ProgressBar` as input + n (None): a number of equally-spaced increments to invoke ``progress`` + dt (None): a number of seconds between ``progress`` calls + + Returns: + a function that accepts a :class:`ProgressBar` as input + """ + if n is not None: + return _report_progress_n(progress, n) + + if dt is not None: + return _report_progress_dt(progress, dt) + + return progress + + +def _report_progress_n(progress, n): + def progress_n(pb): + if not hasattr(pb, "_next_idx"): + if pb.has_total and n > 0: + next_iters = [ + int(np.round(i)) + for i in np.linspace(0, pb.total, min(n, pb.total) + 1) + ][1:] + + pb._next_idx = 0 + pb._next_iters = next_iters + else: + pb._next_idx = None + pb._next_iters = None + + if ( + pb._next_idx is not None + and pb.iteration >= pb._next_iters[pb._next_idx] + ): + progress(pb) + + pb._next_idx += 1 + if pb._next_idx >= len(pb._next_iters): + pb._next_idx = None + + return progress_n + + +def _report_progress_dt(progress, dt): + def progress_dt(pb): + if not hasattr(pb, "_next_dt"): + pb._next_dt = dt + + if pb._next_dt is not None and ( + pb.elapsed_time >= pb._next_dt or pb.complete + ): + progress(pb) + + if not pb.complete: + pb._next_dt += dt + else: + pb._next_dt = None + + return progress_dt class Batcher(abc.ABC): @@ -974,6 +1097,9 @@ def __init__( if not isinstance(iterable, foc.SampleCollection): return_views = False + if progress is None: + progress = fo.config.show_progress_bars + self.iterable = iterable self.return_views = return_views self.progress = progress @@ -983,6 +1109,7 @@ def __init__( self._last_batch_size = None self._pb = None self._in_context = False + self._render_progress = bool(progress) # callback function: True self._last_offset = None self._num_samples = None self._manually_applied_backpressure = True @@ -994,7 +1121,7 @@ def __enter__(self): def __exit__(self, *args): self._in_context = False - if self.progress: + if self._render_progress: if self._last_batch_size is not None: self._pb.update(count=self._last_batch_size) @@ -1007,23 +1134,20 @@ def __iter__(self): else: self._iter = iter(self.iterable) - if self.progress: + if self._render_progress: if self._in_context: total = self.total if total is None: - try: - total = len(self.iterable) - except: - pass + total = self.iterable - self._pb = ProgressBar(total=total) + self._pb = ProgressBar(total=total, progress=self.progress) self._pb.__enter__() else: logger.warning( "Batcher must be invoked as a context manager in order to " "print progress" ) - self.progress = False + self._render_progress = False return self @@ -1035,9 +1159,10 @@ def __next__(self): raise ValueError( "Backpressure value not registered for this batcher" ) + self._manually_applied_backpressure = False - if self.progress and self._last_batch_size is not None: + if self._render_progress and self._last_batch_size is not None: self._pb.update(count=self._last_batch_size) batch_size = self._compute_batch_size() @@ -1200,7 +1325,9 @@ class LatencyDynamicBatcher(BaseDynamicBatcher): :class:`fiftyone.core.view.DatasetView`. Only applicable when the iterable is a :class:`fiftyone.core.collections.SampleCollection` progress (False): whether to render a progress bar tracking the - consumption of the batches + consumption of the batches (True/False), use the default value + ``fiftyone.config.show_progress_bars`` (None), or a progress + callback function to invoke instead total (None): the length of ``iterable``. Only applicable when ``progress=True``. If not provided, it is computed via ``len(iterable)``, if possible @@ -1305,7 +1432,9 @@ class BSONSizeDynamicBatcher(BaseDynamicBatcher): :class:`fiftyone.core.view.DatasetView`. Only applicable when the iterable is a :class:`fiftyone.core.collections.SampleCollection` progress (False): whether to render a progress bar tracking the - consumption of the batches + consumption of the batches (True/False), use the default value + ``fiftyone.config.show_progress_bars`` (None), or a progress + callback function to invoke instead total (None): the length of ``iterable``. Only applicable when ``progress=True``. If not provided, it is computed via ``len(iterable)``, if possible @@ -1380,7 +1509,9 @@ class StaticBatcher(Batcher): :class:`fiftyone.core.view.DatasetView`. Only applicable when the iterable is a :class:`fiftyone.core.collections.SampleCollection` progress (False): whether to render a progress bar tracking the - consumption of the batches + consumption of the batches (True/False), use the default value + ``fiftyone.config.show_progress_bars`` (None), or a progress + callback function to invoke instead total (None): the length of ``iterable``. Only applicable when ``progress=True``. If not provided, it is computed via ``len(iterable)``, if possible @@ -1404,7 +1535,7 @@ def _compute_batch_size(self): return self.batch_size -def get_default_batcher(iterable, progress=True, total=None): +def get_default_batcher(iterable, progress=False, total=None): """Returns a :class:`Batcher` over ``iterable`` using defaults from your FiftyOne config. @@ -1413,8 +1544,10 @@ def get_default_batcher(iterable, progress=True, total=None): Args: iterable: an iterable to batch over - progress (True): whether to render a progress bar tracking the - consumption of the batches + progress (False): whether to render a progress bar tracking the + consumption of the batches (True/False), use the default value + ``fiftyone.config.show_progress_bars`` (None), or a progress + callback function to invoke instead total (None): the length of ``iterable``. Only applicable when ``progress=True``. If not provided, it is computed via ``len(iterable)``, if possible diff --git a/fiftyone/core/view.py b/fiftyone/core/view.py index c9da51250b..ca52b9a10f 100644 --- a/fiftyone/core/view.py +++ b/fiftyone/core/view.py @@ -466,8 +466,9 @@ def make_label(): sample.ground_truth.label = make_label() Args: - progress (False): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -480,10 +481,9 @@ def make_label(): with contextlib.ExitStack() as exit_context: samples = self._iter_samples() - if progress: - pb = fou.ProgressBar(total=len(self)) - exit_context.enter_context(pb) - samples = pb(samples) + pb = fou.ProgressBar(total=self, progress=progress) + exit_context.enter_context(pb) + samples = pb(samples) if autosave: save_context = foc.SaveContext(self, batch_size=batch_size) @@ -582,8 +582,9 @@ def make_label(): Args: group_slices (None): an optional subset of group slices to load - progress (False): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead autosave (False): whether to automatically save changes to samples emitted by this iterator batch_size (None): a batch size to use when autosaving samples. Can @@ -605,10 +606,9 @@ def make_label(): with contextlib.ExitStack() as exit_context: groups = self._iter_groups(group_slices=group_slices) - if progress: - pb = fou.ProgressBar(total=len(self)) - exit_context.enter_context(pb) - groups = pb(groups) + pb = fou.ProgressBar(total=self, progress=progress) + exit_context.enter_context(pb) + groups = pb(groups) if autosave: save_context = foc.SaveContext(self, batch_size=batch_size) @@ -679,8 +679,9 @@ def iter_dynamic_groups(self, progress=False): print("%s: %d" % (group_value, len(group))) Args: - progress (False): whether to render a progress bar tracking the - iterator's progress + progress (False): whether to render a progress bar (True/False), + use the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: an iterator that emits :class:`DatasetView` instances, one per @@ -692,10 +693,9 @@ def iter_dynamic_groups(self, progress=False): with contextlib.ExitStack() as context: groups = self._iter_dynamic_groups() - if progress: - pb = fou.ProgressBar(total=len(self)) - context.enter_context(pb) - groups = pb(groups) + pb = fou.ProgressBar(total=self, progress=progress) + context.enter_context(pb) + groups = pb(groups) for group in groups: yield group diff --git a/fiftyone/operators/__init__.py b/fiftyone/operators/__init__.py index d90e6d3c86..6d389bb4ab 100644 --- a/fiftyone/operators/__init__.py +++ b/fiftyone/operators/__init__.py @@ -16,6 +16,7 @@ execute_operator, ExecutionOptions, ) +from .utils import ProgressHandler # This enables Sphinx refs to directly use paths imported here __all__ = [k for k, v in globals().items() if not k.startswith("_")] diff --git a/fiftyone/operators/utils.py b/fiftyone/operators/utils.py new file mode 100644 index 0000000000..cc0e553a72 --- /dev/null +++ b/fiftyone/operators/utils.py @@ -0,0 +1,49 @@ +""" +FiftyOne operator utilities. + +| Copyright 2017-2023, Voxel51, Inc. +| `voxel51.com `_ +| +""" +import logging + + +class ProgressHandler(logging.Handler): + """A logging handler that reports all logging messages issued while the + handler's context manager is active to the provided execution context's + :meth:`set_progress() ` + method. + + Args: + ctx: an :class:`fiftyone.operators.executor.ExecutionContext` + logger (None): a specific ``logging.Logger`` for which to report + records. By default, the root logger is used + level (None): an optional logging level above which to report records. + By default, the logger's effective level is used + """ + + def __init__(self, ctx, logger=None, level=None): + super().__init__() + self.ctx = ctx + self.logger = logger + self.level = level + + def __enter__(self): + if self.logger is None: + self.logger = logging.getLogger() + + if self.level is None: + self.level = self.logger.getEffectiveLevel() + + self.setLevel(self.level) + self.logger.addHandler(self) + + def __exit__(self, *args): + try: + self.logger.removeHandler(self) + except: + pass + + def emit(self, record): + msg = self.format(record) + self.ctx.set_progress(label=msg) diff --git a/fiftyone/utils/annotations.py b/fiftyone/utils/annotations.py index aecac59a17..66c5c9a0e2 100644 --- a/fiftyone/utils/annotations.py +++ b/fiftyone/utils/annotations.py @@ -991,6 +991,7 @@ def load_annotations( dest_field=None, unexpected="prompt", cleanup=False, + progress=None, **kwargs, ): """Downloads the labels from the given annotation run from the annotation @@ -1019,6 +1020,9 @@ def load_annotations( or ``None`` if there aren't any cleanup (False): whether to delete any informtation regarding this run from the annotation backend after loading the annotations + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: keyword arguments for the run's :meth:`fiftyone.core.annotation.AnnotationMethodConfig.load_credentials` method @@ -1068,6 +1072,7 @@ def load_annotations( results, label_field, label_info=label_info, + progress=progress, ) else: _merge_labels( @@ -1079,6 +1084,7 @@ def load_annotations( label_info=label_info, global_attrs=global_attrs, class_attrs=class_attrs, + progress=progress, ) else: # Unexpected labels @@ -1107,10 +1113,21 @@ def load_annotations( ) if anno_type == "scalar": - _merge_scalars(dataset, annos, results, new_field) + _merge_scalars( + dataset, + annos, + results, + new_field, + progress=progress, + ) else: _merge_labels( - dataset, annos, results, new_field, anno_type + dataset, + annos, + results, + new_field, + anno_type, + progress=progress, ) else: if label_field: @@ -1256,7 +1273,9 @@ def _prompt_field(dataset, label_type, label_field, label_schema): return new_field -def _merge_scalars(dataset, anno_dict, results, label_field, label_info=None): +def _merge_scalars( + dataset, anno_dict, results, label_field, label_info=None, progress=None +): if label_info is None: label_info = {} @@ -1290,7 +1309,7 @@ def _merge_scalars(dataset, anno_dict, results, label_field, label_info=None): num_deletions = 0 logger.info("Loading scalars for field '%s'...", label_field) - for sample in view.iter_samples(progress=True): + for sample in view.iter_samples(progress=progress): sample_annos = anno_dict.get(sample.id, None) if is_frame_field: @@ -1355,6 +1374,7 @@ def _merge_labels( label_info=None, global_attrs=None, class_attrs=None, + progress=None, ): if label_info is None: label_info = {} @@ -1472,7 +1492,7 @@ def _merge_labels( # Add/merge labels from the annotation task sample_ids = list(anno_dict.keys()) view = view.select(sample_ids).select_fields(label_field) - for sample in view.iter_samples(progress=True): + for sample in view.iter_samples(progress=progress): sample_id = sample.id sample_annos = anno_dict[sample_id] @@ -2294,7 +2314,13 @@ def __init__(self, d): def draw_labeled_images( - samples, output_dir, rel_dir=None, label_fields=None, config=None, **kwargs + samples, + output_dir, + rel_dir=None, + label_fields=None, + config=None, + progress=None, + **kwargs, ): """Renders annotated versions of the images in the collection with the specified label data overlaid to the given directory. @@ -2319,6 +2345,9 @@ def draw_labeled_images( If omitted, all compatiable fields are rendered config (None): an optional :class:`DrawConfig` configuring how to draw the labels + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments specifying parameters of the default :class:`DrawConfig` to override @@ -2335,7 +2364,7 @@ def draw_labeled_images( output_ext = fo.config.default_image_ext outpaths = [] - for sample in samples.iter_samples(progress=True): + for sample in samples.iter_samples(progress=progress): outpath = filename_maker.get_output_path( sample.filepath, output_ext=output_ext ) @@ -2375,7 +2404,13 @@ def draw_labeled_image( def draw_labeled_videos( - samples, output_dir, rel_dir=None, label_fields=None, config=None, **kwargs + samples, + output_dir, + rel_dir=None, + label_fields=None, + config=None, + progress=None, + **kwargs, ): """Renders annotated versions of the videos in the collection with the specified label data overlaid to the given directory. @@ -2400,6 +2435,9 @@ def draw_labeled_videos( If omitted, all compatiable fields are rendered config (None): an optional :class:`DrawConfig` configuring how to draw the labels + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments specifying parameters of the default :class:`DrawConfig` to override @@ -2419,7 +2457,7 @@ def draw_labeled_videos( num_videos = len(samples) outpaths = [] - for idx, sample in enumerate(samples, 1): + for idx, sample in enumerate(samples.iter_samples(progress=progress), 1): if is_clips: logger.info("Drawing labels for clip %d/%d", idx, num_videos) base, ext = os.path.splitext(sample.filepath) diff --git a/fiftyone/utils/coco.py b/fiftyone/utils/coco.py index a89151d87c..ccaa2cf185 100644 --- a/fiftyone/utils/coco.py +++ b/fiftyone/utils/coco.py @@ -1985,29 +1985,6 @@ def _load_image_ids_json(json_path): return [_id for _id in etas.load_json(json_path)] -def _make_images_list(images_dir): - logger.info("Computing image metadata for '%s'", images_dir) - - image_paths = foud.parse_images_dir(images_dir) - - images = [] - with fou.ProgressBar() as pb: - for idx, image_path in pb(enumerate(image_paths)): - metadata = fom.ImageMetadata.build_for(image_path) - images.append( - { - "id": idx, - "file_name": os.path.basename(image_path), - "height": metadata.height, - "width": metadata.width, - "license": None, - "coco_url": None, - } - ) - - return images - - def _to_labels_map_rev(classes): return {c: i for i, c in enumerate(classes)} diff --git a/fiftyone/utils/csv.py b/fiftyone/utils/csv.py index a9f3839257..a055b1ea40 100644 --- a/fiftyone/utils/csv.py +++ b/fiftyone/utils/csv.py @@ -379,12 +379,12 @@ def setup(self): self._include_media = include_media self._needs_metadata = needs_metadata - def export_samples(self, sample_collection): + def export_samples(self, sample_collection, progress=None): if self._needs_metadata: sample_collection.compute_metadata() idx = self._media_idx - with fou.ProgressBar(total=len(sample_collection)) as pb: + with fou.ProgressBar(total=sample_collection, progress=progress) as pb: for data in pb(zip(*sample_collection.values(self._paths))): data = [_parse_value(d) for d in data] diff --git a/fiftyone/utils/cvat.py b/fiftyone/utils/cvat.py index e2ad1302bf..20fbe2c23b 100644 --- a/fiftyone/utils/cvat.py +++ b/fiftyone/utils/cvat.py @@ -3995,13 +3995,16 @@ def delete_project(self, project_id): if project_name is not None: self._project_id_map.pop(project_name, None) - def delete_projects(self, project_ids): + def delete_projects(self, project_ids, progress=None): """Deletes the given projects from the CVAT server. Args: project_ids: an iterable of project IDs + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for project_id in pb(list(project_ids)): self.delete_project(project_id) @@ -4152,13 +4155,16 @@ def delete_task(self, task_id): if self.task_exists(task_id): self.delete(self.task_url(task_id)) - def delete_tasks(self, task_ids): + def delete_tasks(self, task_ids, progress=None): """Deletes the given tasks from the CVAT server. Args: task_ids: an iterable of task IDs + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for task_id in pb(list(task_ids)): self.delete_task(task_id) diff --git a/fiftyone/utils/data/exporters.py b/fiftyone/utils/data/exporters.py index 5fce0eb30e..86a015389d 100644 --- a/fiftyone/utils/data/exporters.py +++ b/fiftyone/utils/data/exporters.py @@ -57,6 +57,7 @@ def export_samples( dataset_exporter=None, label_field=None, frame_labels_field=None, + progress=None, num_samples=None, **kwargs, ): @@ -192,8 +193,12 @@ def export_samples( or a dictionary mapping field names to output keys describing the frame label fields to export. Only applicable if ``dataset_exporter`` is a :class:`LabeledVideoDatasetExporter` + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead num_samples (None): the number of samples in ``samples``. If omitted, - this is computed (if possible) via ``len(samples)`` + this is computed (if possible) via ``len(samples)`` if needed for + progress tracking **kwargs: optional keyword arguments to pass to the dataset exporter's constructor. If you are exporting image patches, this can also contain keyword arguments for @@ -234,7 +239,7 @@ def export_samples( sample_collection = samples if isinstance(dataset_exporter, BatchDatasetExporter): - _write_batch_dataset(dataset_exporter, samples) + _write_batch_dataset(dataset_exporter, samples, progress=progress) return if isinstance( @@ -252,7 +257,7 @@ def export_samples( **patches_kwargs, ) sample_parser = ImageSampleParser() - num_samples = len(samples) + num_samples = samples else: sample_parser = FiftyOneUnlabeledImageSampleParser( compute_metadata=True @@ -262,7 +267,7 @@ def export_samples( if found_clips and not samples._is_clips: # Export unlabeled video clips samples = samples.to_clips(label_field) - num_samples = len(samples) + num_samples = samples # True for copy/move/symlink, False for manifest/no export _export_media = getattr( @@ -298,7 +303,7 @@ def export_samples( **patches_kwargs, ) sample_parser = ImageClassificationSampleParser() - num_samples = len(samples) + num_samples = samples else: label_fcn = _make_label_coercion_functions( label_field, samples, dataset_exporter @@ -313,7 +318,7 @@ def export_samples( if found_clips and not samples._is_clips: # Export labeled video clips samples = samples.to_clips(label_field) - num_samples = len(samples) + num_samples = samples # True for copy/move/symlink, False for manifest/no export _export_media = getattr( @@ -358,8 +363,9 @@ def export_samples( samples, sample_parser, dataset_exporter, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) @@ -367,8 +373,9 @@ def write_dataset( samples, sample_parser, dataset_exporter, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): """Writes the samples to disk as a dataset in the specified format. @@ -378,20 +385,21 @@ def write_dataset( use to parse the samples dataset_exporter: a :class:`DatasetExporter` to use to write the dataset - num_samples (None): the number of samples in ``samples``. If omitted, - this is computed (if possible) via ``len(samples)`` sample_collection (None): the :class:`fiftyone.core.collections.SampleCollection` from which ``samples`` were extracted. If ``samples`` is itself a :class:`fiftyone.core.collections.SampleCollection`, this parameter defaults to ``samples``. This parameter is optional and is only passed to :meth:`DatasetExporter.log_collection` + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead + num_samples (None): the number of samples in ``samples``. If omitted, + this is computed (if possible) via ``len(samples)`` if needed for + progress tracking """ if num_samples is None: - try: - num_samples = len(samples) - except: - pass + num_samples = samples if sample_collection is None and isinstance(samples, foc.SampleCollection): sample_collection = samples @@ -400,15 +408,17 @@ def write_dataset( _write_generic_sample_dataset( dataset_exporter, samples, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) elif isinstance(dataset_exporter, GroupDatasetExporter): _write_group_dataset( dataset_exporter, samples, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) elif isinstance( dataset_exporter, @@ -418,8 +428,9 @@ def write_dataset( dataset_exporter, samples, sample_parser, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) elif isinstance( dataset_exporter, @@ -429,16 +440,18 @@ def write_dataset( dataset_exporter, samples, sample_parser, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) elif isinstance(dataset_exporter, UnlabeledMediaDatasetExporter): _write_unlabeled_dataset( dataset_exporter, samples, sample_parser, - num_samples=num_samples, sample_collection=sample_collection, + progress=progress, + num_samples=num_samples, ) else: raise ValueError( @@ -803,7 +816,7 @@ def _classification_to_detections(label): ) -def _write_batch_dataset(dataset_exporter, samples): +def _write_batch_dataset(dataset_exporter, samples, progress=None): if not isinstance(samples, foc.SampleCollection): raise ValueError( "%s can only export %s instances" @@ -811,16 +824,17 @@ def _write_batch_dataset(dataset_exporter, samples): ) with dataset_exporter: - dataset_exporter.export_samples(samples) + dataset_exporter.export_samples(samples, progress=progress) def _write_generic_sample_dataset( dataset_exporter, samples, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: with dataset_exporter: if sample_collection is not None: dataset_exporter.log_collection(sample_collection) @@ -838,8 +852,9 @@ def _write_generic_sample_dataset( def _write_group_dataset( dataset_exporter, samples, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): if not isinstance(samples, foc.SampleCollection): raise ValueError( @@ -853,7 +868,7 @@ def _write_group_dataset( % (type(dataset_exporter), samples.media_type) ) - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: with dataset_exporter: if sample_collection is not None: dataset_exporter.log_collection(sample_collection) @@ -866,12 +881,13 @@ def _write_image_dataset( dataset_exporter, samples, sample_parser, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): labeled_images = isinstance(dataset_exporter, LabeledImageDatasetExporter) - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: with dataset_exporter: if sample_collection is not None: dataset_exporter.log_collection(sample_collection) @@ -919,12 +935,13 @@ def _write_video_dataset( dataset_exporter, samples, sample_parser, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): labeled_videos = isinstance(dataset_exporter, LabeledVideoDatasetExporter) - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: with dataset_exporter: if sample_collection is not None: dataset_exporter.log_collection(sample_collection) @@ -967,10 +984,11 @@ def _write_unlabeled_dataset( dataset_exporter, samples, sample_parser, - num_samples=None, sample_collection=None, + progress=None, + num_samples=None, ): - with fou.ProgressBar(total=num_samples) as pb: + with fou.ProgressBar(total=num_samples, progress=progress) as pb: with dataset_exporter: if sample_collection is not None: dataset_exporter.log_collection(sample_collection) @@ -1377,12 +1395,15 @@ def export_sample(self, *args, **kwargs): % type(self) ) - def export_samples(self, sample_collection): + def export_samples(self, sample_collection, progress=None): """Exports the given sample collection. Args: sample_collection: a :class:`fiftyone.core.collections.SampleCollection` + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ raise NotImplementedError("subclass must implement export_samples()") @@ -2052,7 +2073,7 @@ def setup(self): ) self._media_exporter.setup() - def export_samples(self, sample_collection): + def export_samples(self, sample_collection, progress=None): etau.ensure_dir(self.export_dir) if sample_collection.media_type == fomm.GROUP: @@ -2102,6 +2123,7 @@ def _prep_sample(sd): self._samples_path, key="samples", patt=patt, + progress=progress, num_docs=num_samples, ) @@ -2129,6 +2151,7 @@ def _prep_sample(sd): key="frames", patt=patt, num_docs=num_frames, + progress=progress, ) dataset = sample_collection._dataset diff --git a/fiftyone/utils/data/importers.py b/fiftyone/utils/data/importers.py index 999eeb01bf..1ebeb2c603 100644 --- a/fiftyone/utils/data/importers.py +++ b/fiftyone/utils/data/importers.py @@ -57,6 +57,7 @@ def import_samples( expand_schema=True, dynamic=False, add_info=True, + progress=None, ): """Adds the samples from the given :class:`DatasetImporter` to the dataset. @@ -87,6 +88,9 @@ def import_samples( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -114,7 +118,9 @@ def import_samples( ) with dataset_importer: - return dataset_importer.import_samples(dataset, tags=tags) + return dataset_importer.import_samples( + dataset, tags=tags, progress=progress + ) # # Non-batch imports @@ -130,11 +136,6 @@ def import_samples( dynamic, ) - try: - num_samples = len(dataset_importer) - except: - num_samples = None - if isinstance(dataset_importer, GroupDatasetImporter): samples = _generate_group_samples(dataset_importer, parse_sample) else: @@ -144,7 +145,8 @@ def import_samples( samples, expand_schema=expand_schema, dynamic=dynamic, - num_samples=num_samples, + progress=progress, + num_samples=dataset_importer, ) if add_info and dataset_importer.has_dataset_info: @@ -174,6 +176,7 @@ def merge_samples( expand_schema=True, dynamic=False, add_info=True, + progress=None, ): """Merges the samples from the given :class:`DatasetImporter` into the dataset. @@ -265,6 +268,9 @@ def merge_samples( document fields that are encountered add_info (True): whether to add dataset info from the importer (if any) to the dataset + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if etau.is_str(tags): tags = [tags] @@ -282,7 +288,9 @@ def merge_samples( try: with dataset_importer: - dataset_importer.import_samples(tmp, tags=tags) + dataset_importer.import_samples( + tmp, tags=tags, progress=progress + ) dataset.merge_samples( tmp, @@ -317,11 +325,6 @@ def merge_samples( dynamic, ) - try: - num_samples = len(dataset_importer) - except: - num_samples = None - if isinstance(dataset_importer, GroupDatasetImporter): samples = _generate_group_samples(dataset_importer, parse_sample) else: @@ -339,7 +342,8 @@ def merge_samples( overwrite=overwrite, expand_schema=expand_schema, dynamic=dynamic, - num_samples=num_samples, + progress=progress, + num_samples=dataset_importer, ) if add_info and dataset_importer.has_dataset_info: @@ -982,12 +986,15 @@ def __next__(self): def has_dataset_info(self): return False - def import_samples(self, dataset, tags=None): + def import_samples(self, dataset, tags=None, progress=None): """Imports the samples into the given dataset. Args: dataset: a :class:`fiftyone.core.dataset.Dataset` tags (None): an optional list of tags to attach to each sample + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -1764,7 +1771,7 @@ def setup(self): else: self._has_frames = False - def import_samples(self, dataset, tags=None): + def import_samples(self, dataset, tags=None, progress=None): dataset_dict = foo.import_document(self._metadata_path) if len(dataset) > 0 and fomi.needs_migration( @@ -1778,7 +1785,7 @@ def import_samples(self, dataset, tags=None): try: sample_ids = self._import_samples( - tmp_dataset, dataset_dict, tags=tags + tmp_dataset, dataset_dict, tags=tags, progress=progress ) dataset.add_collection(tmp_dataset) finally: @@ -1786,9 +1793,11 @@ def import_samples(self, dataset, tags=None): return sample_ids - return self._import_samples(dataset, dataset_dict, tags=tags) + return self._import_samples( + dataset, dataset_dict, tags=tags, progress=progress + ) - def _import_samples(self, dataset, dataset_dict, tags=None): + def _import_samples(self, dataset, dataset_dict, tags=None, progress=None): name = dataset.name empty_import = not bool(dataset) @@ -1896,7 +1905,7 @@ def _parse_sample(sd): map(_parse_sample, samples), dataset._sample_collection, ordered=self.ordered, - progress=True, + progress=progress, num_docs=num_samples, ) @@ -1924,7 +1933,7 @@ def _parse_frame(fd): map(_parse_frame, frames), dataset._frame_collection, ordered=self.ordered, - progress=True, + progress=progress, num_docs=num_frames, ) @@ -1979,8 +1988,6 @@ def _parse_frame(fd): fomi.migrate_dataset_if_necessary(name) dataset._reload(hard=True) - logger.info("Import complete") - return sample_ids @staticmethod diff --git a/fiftyone/utils/data/parsers.py b/fiftyone/utils/data/parsers.py index ac98d3e47d..220f92de7e 100644 --- a/fiftyone/utils/data/parsers.py +++ b/fiftyone/utils/data/parsers.py @@ -24,7 +24,7 @@ import fiftyone.utils.video as fouv -def add_images(dataset, samples, sample_parser, tags=None): +def add_images(dataset, samples, sample_parser, tags=None, progress=None): """Adds the given images to the dataset. This operation does not read the images. @@ -40,6 +40,9 @@ def add_images(dataset, samples, sample_parser, tags=None): parse the samples tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -76,14 +79,12 @@ def parse_sample(sample): return Sample(filepath=image_path, metadata=metadata, tags=tags) - try: - num_samples = len(samples) - except: - num_samples = None - _samples = map(parse_sample, samples) return dataset.add_samples( - _samples, num_samples=num_samples, expand_schema=False + _samples, + expand_schema=False, + progress=progress, + num_samples=samples, ) @@ -95,6 +96,7 @@ def add_labeled_images( tags=None, expand_schema=True, dynamic=False, + progress=None, ): """Adds the given labeled images to the dataset. @@ -127,6 +129,9 @@ def add_labeled_images( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -191,21 +196,17 @@ def parse_sample(sample): dataset._ensure_label_field(label_field, sample_parser.label_cls) expand_schema = False - try: - num_samples = len(samples) - except: - num_samples = None - _samples = map(parse_sample, samples) return dataset.add_samples( _samples, expand_schema=expand_schema, dynamic=dynamic, - num_samples=num_samples, + progress=progress, + num_samples=samples, ) -def add_videos(dataset, samples, sample_parser, tags=None): +def add_videos(dataset, samples, sample_parser, tags=None, progress=None): """Adds the given videos to the dataset. This operation does not read the videos. @@ -221,6 +222,9 @@ def add_videos(dataset, samples, sample_parser, tags=None): parse the samples tags (None): an optional tag or iterable of tags to attach to each sample + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -251,16 +255,13 @@ def parse_sample(sample): return Sample(filepath=video_path, metadata=metadata, tags=tags) - try: - num_samples = len(samples) - except: - num_samples = None - - _samples = map(parse_sample, samples) - # @todo: skip schema expansion and set media type before adding samples + _samples = map(parse_sample, samples) return dataset.add_samples( - _samples, num_samples=num_samples, expand_schema=True + _samples, + expand_schema=True, + progress=progress, + num_samples=samples, ) @@ -272,6 +273,7 @@ def add_labeled_videos( tags=None, expand_schema=True, dynamic=False, + progress=None, ): """Adds the given labeled videos to the dataset. @@ -303,6 +305,9 @@ def add_labeled_videos( if a sample's schema is not a subset of the dataset schema dynamic (False): whether to declare dynamic attributes of embedded document fields that are encountered + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead Returns: a list of IDs of the samples that were added to the dataset @@ -365,17 +370,13 @@ def parse_sample(sample): return sample - try: - num_samples = len(samples) - except: - num_samples = None - _samples = map(parse_sample, samples) return dataset.add_samples( _samples, expand_schema=expand_schema, dynamic=dynamic, - num_samples=num_samples, + progress=progress, + num_samples=samples, ) diff --git a/fiftyone/utils/eval/activitynet.py b/fiftyone/utils/eval/activitynet.py index dd813b682d..0965352065 100644 --- a/fiftyone/utils/eval/activitynet.py +++ b/fiftyone/utils/eval/activitynet.py @@ -123,7 +123,13 @@ def evaluate(self, sample, eval_key=None): ) def generate_results( - self, samples, matches, eval_key=None, classes=None, missing=None + self, + samples, + matches, + eval_key=None, + classes=None, + missing=None, + progress=None, ): """Generates aggregate evaluation results for the samples. @@ -146,6 +152,9 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched segments are given this label for results purposes + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`DetectionResults` @@ -176,7 +185,7 @@ def generate_results( # IoU sweep logger.info("Performing IoU sweep...") - for sample in _samples.iter_samples(progress=True): + for sample in _samples.iter_samples(progress=progress): # Don't edit user's data during sweep gts = _copy_labels(sample[self.gt_field]) preds = _copy_labels(sample[self.pred_field]) diff --git a/fiftyone/utils/eval/classification.py b/fiftyone/utils/eval/classification.py index c89c689141..5b1d8e360d 100644 --- a/fiftyone/utils/eval/classification.py +++ b/fiftyone/utils/eval/classification.py @@ -35,6 +35,7 @@ def evaluate_classifications( classes=None, missing=None, method=None, + progress=None, **kwargs, ): """Evaluates the classification predictions in the given collection with @@ -81,6 +82,9 @@ def evaluate_classifications( supported values are ``fo.evaluation_config.classification_backends.keys()`` and the default is ``fo.evaluation_config.default_classification_backend`` + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`ClassificationEvaluationConfig` being used @@ -99,7 +103,11 @@ def evaluate_classifications( eval_method.register_samples(samples, eval_key) results = eval_method.evaluate_samples( - samples, eval_key=eval_key, classes=classes, missing=missing + samples, + eval_key=eval_key, + classes=classes, + missing=missing, + progress=progress, ) eval_method.save_run_results(samples, eval_key, results) @@ -148,7 +156,7 @@ def register_samples(self, samples, eval_key): raise NotImplementedError("subclass must implement register_samples()") def evaluate_samples( - self, samples, eval_key=None, classes=None, missing=None + self, samples, eval_key=None, classes=None, missing=None, progress=None ): """Evaluates the predicted classifications in the given samples with respect to the specified ground truth labels. @@ -161,6 +169,9 @@ def evaluate_samples( purposes missing (None): a missing label string. Any None-valued labels are given this label for results purposes + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`ClassificationResults` instance @@ -256,7 +267,7 @@ def register_samples(self, samples, eval_key): dataset.add_sample_field(eval_key, fof.BooleanField) def evaluate_samples( - self, samples, eval_key=None, classes=None, missing=None + self, samples, eval_key=None, classes=None, missing=None, progress=None ): pred_field = self.config.pred_field gt_field = self.config.gt_field @@ -371,7 +382,7 @@ def register_samples(self, samples, eval_key): dataset.add_sample_field(eval_key, fof.BooleanField) def evaluate_samples( - self, samples, eval_key=None, classes=None, missing=None + self, samples, eval_key=None, classes=None, missing=None, progress=None ): if classes is None: raise ValueError( @@ -573,7 +584,7 @@ def register_samples(self, samples, eval_key): dataset.add_sample_field(eval_key, fof.StringField) def evaluate_samples( - self, samples, eval_key=None, classes=None, missing=None + self, samples, eval_key=None, classes=None, missing=None, progress=None ): if classes is None or len(classes) != 2: raise ValueError( diff --git a/fiftyone/utils/eval/coco.py b/fiftyone/utils/eval/coco.py index cde9d5cf23..3efe7838d4 100644 --- a/fiftyone/utils/eval/coco.py +++ b/fiftyone/utils/eval/coco.py @@ -166,7 +166,13 @@ def evaluate(self, sample_or_frame, eval_key=None): return _coco_evaluation_single_iou(gts, preds, eval_key, self.config) def generate_results( - self, samples, matches, eval_key=None, classes=None, missing=None + self, + samples, + matches, + eval_key=None, + classes=None, + missing=None, + progress=None, ): """Generates aggregate evaluation results for the samples. @@ -188,6 +194,9 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched objects are given this label for results purposes + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`DetectionResults` @@ -209,7 +218,9 @@ def generate_results( thresholds, iou_threshs, classes, - ) = _compute_pr_curves(samples, self.config, classes=classes) + ) = _compute_pr_curves( + samples, self.config, classes=classes, progress=progress + ) return COCODetectionResults( samples, @@ -633,7 +644,7 @@ def _compute_matches( return matches -def _compute_pr_curves(samples, config, classes=None): +def _compute_pr_curves(samples, config, classes=None, progress=None): gt_field = config.gt_field pred_field = config.pred_field iou_threshs = config.iou_threshs @@ -650,7 +661,7 @@ def _compute_pr_curves(samples, config, classes=None): _classes = set() logger.info("Performing IoU sweep...") - for sample in samples.iter_samples(progress=True): + for sample in samples.iter_samples(progress=progress): if processing_frames: images = sample.frames.values() else: diff --git a/fiftyone/utils/eval/detection.py b/fiftyone/utils/eval/detection.py index 3b6e0a0be0..5bf8b9081f 100644 --- a/fiftyone/utils/eval/detection.py +++ b/fiftyone/utils/eval/detection.py @@ -40,6 +40,7 @@ def evaluate_detections( use_boxes=False, classwise=True, dynamic=True, + progress=None, **kwargs, ): """Evaluates the predicted detections in the given samples with respect to @@ -131,6 +132,9 @@ def evaluate_detections( label (True) or allow matches between classes (False) dynamic (True): whether to declare the dynamic object-level attributes that are populated on the dataset's schema + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`DetectionEvaluationConfig` being used @@ -182,7 +186,7 @@ def evaluate_detections( matches = [] logger.info("Evaluating detections...") - for sample in _samples.iter_samples(progress=True): + for sample in _samples.iter_samples(progress=progress): if processing_frames: docs = sample.frames.values() else: @@ -211,7 +215,12 @@ def evaluate_detections( sample.save() results = eval_method.generate_results( - samples, matches, eval_key=eval_key, classes=classes, missing=missing + samples, + matches, + eval_key=eval_key, + classes=classes, + missing=missing, + progress=progress, ) eval_method.save_run_results(samples, eval_key, results) @@ -359,7 +368,13 @@ def evaluate(self, doc, eval_key=None): raise NotImplementedError("subclass must implement evaluate()") def generate_results( - self, samples, matches, eval_key=None, classes=None, missing=None + self, + samples, + matches, + eval_key=None, + classes=None, + missing=None, + progress=None, ): """Generates aggregate evaluation results for the samples. @@ -378,6 +393,9 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched objects are given this label for results purposes + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`DetectionResults` diff --git a/fiftyone/utils/eval/openimages.py b/fiftyone/utils/eval/openimages.py index 6eecae7697..d92d371996 100644 --- a/fiftyone/utils/eval/openimages.py +++ b/fiftyone/utils/eval/openimages.py @@ -219,7 +219,13 @@ def evaluate(self, sample_or_frame, eval_key=None): ) def generate_results( - self, samples, matches, eval_key=None, classes=None, missing=None + self, + samples, + matches, + eval_key=None, + classes=None, + missing=None, + progress=None, ): """Generates aggregate evaluation results for the samples. @@ -238,6 +244,9 @@ def generate_results( purposes missing (None): a missing label string. Any unmatched objects are given this label for results purposes + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`OpenImagesDetectionResults` diff --git a/fiftyone/utils/eval/regression.py b/fiftyone/utils/eval/regression.py index 52534282a6..1d1fc17872 100644 --- a/fiftyone/utils/eval/regression.py +++ b/fiftyone/utils/eval/regression.py @@ -36,6 +36,7 @@ def evaluate_regressions( eval_key=None, missing=None, method=None, + progress=None, **kwargs, ): """Evaluates the regression predictions in the given collection with @@ -74,6 +75,9 @@ def evaluate_regressions( supported values are ``fo.evaluation_config.regression_backends.keys()`` and the default is ``fo.evaluation_config.default_regression_backend`` + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`RegressionEvaluationConfig` being used @@ -92,7 +96,7 @@ def evaluate_regressions( eval_method.register_samples(samples, eval_key) results = eval_method.evaluate_samples( - samples, eval_key=eval_key, missing=missing + samples, eval_key=eval_key, missing=missing, progress=progress ) eval_method.save_run_results(samples, eval_key, results) @@ -149,7 +153,9 @@ def register_samples(self, samples, eval_key): else: dataset.add_sample_field(eval_key, fof.FloatField) - def evaluate_samples(self, samples, eval_key=None, missing=None): + def evaluate_samples( + self, samples, eval_key=None, missing=None, progress=None + ): """Evaluates the regression predictions in the given samples with respect to the specified ground truth values. @@ -158,6 +164,9 @@ def evaluate_samples(self, samples, eval_key=None, missing=None): eval_key (None): an evaluation key for this evaluation missing (None): a missing value. Any None-valued regressions are given this value for results purposes + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`RegressionResults` instance @@ -243,7 +252,9 @@ class SimpleEvaluation(RegressionEvaluation): config: a :class:`SimpleEvaluationConfig` """ - def evaluate_samples(self, samples, eval_key=None, missing=None): + def evaluate_samples( + self, samples, eval_key=None, missing=None, progress=None + ): metric = self.config._metric if metric == "squared_error": diff --git a/fiftyone/utils/eval/segmentation.py b/fiftyone/utils/eval/segmentation.py index 2f2c5b8e35..68ff2f8170 100644 --- a/fiftyone/utils/eval/segmentation.py +++ b/fiftyone/utils/eval/segmentation.py @@ -36,6 +36,7 @@ def evaluate_segmentations( eval_key=None, mask_targets=None, method=None, + progress=None, **kwargs, ): """Evaluates the specified semantic segmentation masks in the given @@ -86,6 +87,9 @@ def evaluate_segmentations( supported values are ``fo.evaluation_config.segmentation_backends.keys()`` and the default is ``fo.evaluation_config.default_segmentation_backend`` + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for the constructor of the :class:`SegmentationEvaluationConfig` being used @@ -104,7 +108,10 @@ def evaluate_segmentations( eval_method.register_samples(samples, eval_key) results = eval_method.evaluate_samples( - samples, eval_key=eval_key, mask_targets=mask_targets + samples, + eval_key=eval_key, + mask_targets=mask_targets, + progress=progress, ) eval_method.save_run_results(samples, eval_key, results) @@ -177,7 +184,9 @@ def register_samples(self, samples, eval_key): if processing_frames: dataset.add_frame_field(dice_field, fof.FloatField) - def evaluate_samples(self, samples, eval_key=None, mask_targets=None): + def evaluate_samples( + self, samples, eval_key=None, mask_targets=None, progress=None + ): """Evaluates the predicted segmentation masks in the given samples with respect to the specified ground truth masks. @@ -188,6 +197,9 @@ def evaluate_samples(self, samples, eval_key=None, mask_targets=None): contain a subset of the possible classes if you wish to evaluate a subset of the semantic classes. By default, the observed pixel values are used as labels + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead Returns: a :class:`SegmentationResults` instance @@ -311,7 +323,9 @@ class SimpleEvaluation(SegmentationEvaluation): config: a :class:`SimpleEvaluationConfig` """ - def evaluate_samples(self, samples, eval_key=None, mask_targets=None): + def evaluate_samples( + self, samples, eval_key=None, mask_targets=None, progress=None + ): pred_field = self.config.pred_field gt_field = self.config.gt_field @@ -324,7 +338,9 @@ def evaluate_samples(self, samples, eval_key=None, mask_targets=None): values, classes = zip(*sorted(mask_targets.items())) else: logger.info("Computing possible mask values...") - values, classes = _get_mask_values(samples, pred_field, gt_field) + values, classes = _get_mask_values( + samples, pred_field, gt_field, progress=progress + ) _samples = samples.select_fields([gt_field, pred_field]) pred_field, processing_frames = samples._handle_frame_field(pred_field) @@ -345,7 +361,7 @@ def evaluate_samples(self, samples, eval_key=None, mask_targets=None): dice_field = "%s_dice" % eval_key logger.info("Evaluating segmentations...") - for sample in _samples.iter_samples(progress=True): + for sample in _samples.iter_samples(progress=progress): if processing_frames: images = sample.frames.values() else: @@ -590,7 +606,7 @@ def _compute_accuracy_precision_recall(confusion_matrix, values, average): return metrics["accuracy"], metrics["precision"], metrics["recall"] -def _get_mask_values(samples, pred_field, gt_field): +def _get_mask_values(samples, pred_field, gt_field, progress=None): _samples = samples.select_fields([gt_field, pred_field]) pred_field, processing_frames = samples._handle_frame_field(pred_field) gt_field, _ = samples._handle_frame_field(gt_field) @@ -598,7 +614,7 @@ def _get_mask_values(samples, pred_field, gt_field): values = set() is_rgb = False - for sample in _samples.iter_samples(progress=True): + for sample in _samples.iter_samples(progress=progress): if processing_frames: images = sample.frames.values() else: diff --git a/fiftyone/utils/geojson.py b/fiftyone/utils/geojson.py index 6cf44b31f6..b5e53f2264 100644 --- a/fiftyone/utils/geojson.py +++ b/fiftyone/utils/geojson.py @@ -24,7 +24,11 @@ def load_location_data( - samples, geojson_or_path, location_field=None, skip_missing=True + samples, + geojson_or_path, + location_field=None, + skip_missing=True, + progress=None, ): """Loads geolocation data for the given samples from the given GeoJSON data. @@ -86,6 +90,9 @@ def load_location_data( used, else a new "location" field is created skip_missing (True): whether to skip GeoJSON features with no ``filename`` properties (True) or raise an error (False) + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if location_field is None: try: @@ -137,7 +144,7 @@ def load_location_data( logger.info("Loading location data for %d samples...", len(found_keys)) _samples = samples.select_fields(location_field) - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for key in pb(found_keys): sample_id = lookup[key] geometry = geometries[key] diff --git a/fiftyone/utils/image.py b/fiftyone/utils/image.py index 44c02ce820..7701569067 100644 --- a/fiftyone/utils/image.py +++ b/fiftyone/utils/image.py @@ -59,6 +59,7 @@ def reencode_images( delete_originals=False, num_workers=None, skip_failures=False, + progress=None, ): """Re-encodes the images in the sample collection to the given format. @@ -100,6 +101,9 @@ def reencode_images( num_workers (None): a suggested number of worker processes to use skip_failures (False): whether to gracefully continue without raising an error if an image cannot be re-encoded + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_image_collection(sample_collection) @@ -115,6 +119,7 @@ def reencode_images( delete_originals=delete_originals, num_workers=num_workers, skip_failures=skip_failures, + progress=progress, ) @@ -134,6 +139,7 @@ def transform_images( delete_originals=False, num_workers=None, skip_failures=False, + progress=None, ): """Transforms the images in the sample collection according to the provided parameters. @@ -189,6 +195,9 @@ def transform_images( num_workers (None): a suggested number of worker processes to use skip_failures (False): whether to gracefully continue without raising an error if an image cannot be transformed + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_image_collection(sample_collection) @@ -208,6 +217,7 @@ def transform_images( delete_originals=delete_originals, num_workers=num_workers, skip_failures=skip_failures, + progress=progress, ) @@ -271,6 +281,7 @@ def _transform_images( delete_originals=False, num_workers=None, skip_failures=False, + progress=None, ): ext = _parse_ext(ext) @@ -292,6 +303,7 @@ def _transform_images( update_filepaths=update_filepaths, delete_originals=delete_originals, skip_failures=skip_failures, + progress=progress, ) else: _transform_images_multi( @@ -310,6 +322,7 @@ def _transform_images( update_filepaths=update_filepaths, delete_originals=delete_originals, skip_failures=skip_failures, + progress=progress, ) @@ -328,6 +341,7 @@ def _transform_images_single( update_filepaths=True, delete_originals=False, skip_failures=False, + progress=None, ): if output_field is None: output_field = media_field @@ -336,7 +350,7 @@ def _transform_images_single( view = sample_collection.select_fields(media_field) - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(view): inpath = sample[media_field] @@ -380,6 +394,7 @@ def _transform_images_multi( update_filepaths=True, delete_originals=False, skip_failures=False, + progress=None, ): if output_field is None: output_field = media_field @@ -413,7 +428,7 @@ def _transform_images_multi( outpaths = {} try: - with fou.ProgressBar(inputs) as pb: + with fou.ProgressBar(inputs, progress=progress) as pb: with fou.get_multiprocessing_context().Pool( processes=num_workers ) as pool: diff --git a/fiftyone/utils/iou.py b/fiftyone/utils/iou.py index d75b0e8314..bc6e1e4ef8 100644 --- a/fiftyone/utils/iou.py +++ b/fiftyone/utils/iou.py @@ -149,6 +149,7 @@ def compute_max_ious( other_field=None, iou_attr="max_iou", id_attr=None, + progress=None, **kwargs, ): """Populates an attribute on each label in the given spatial field(s) that @@ -175,6 +176,9 @@ def compute_max_ious( iou_attr ("max_iou"): the label attribute in which to store the max IoU id_attr (None): an optional attribute in which to store the label ID of the maximum overlapping label + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for :func:`compute_ious` """ if other_field is None: @@ -202,7 +206,7 @@ def compute_max_ious( label_ids1 = [] label_ids2 = [] - for sample in view.iter_samples(progress=True): + for sample in view.iter_samples(progress=progress): if is_frame_field: _max_ious1 = [] _max_ious2 = [] @@ -259,7 +263,12 @@ def compute_max_ious( def find_duplicates( - sample_collection, label_field, iou_thresh=0.999, method="simple", **kwargs + sample_collection, + label_field, + iou_thresh=0.999, + method="simple", + progress=None, + **kwargs, ): """Returns IDs of duplicate labels in the given field of the collection, as defined as labels with an IoU greater than a chosen threshold with another @@ -287,6 +296,9 @@ def find_duplicates( labels are duplicates method ("simple"): the duplicate removal method to use. The supported values are ``("simple", "greedy")`` + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional keyword arguments for :func:`compute_ious` Returns: @@ -306,7 +318,7 @@ def find_duplicates( dup_ids = [] - for sample in view.iter_samples(progress=True): + for sample in view.iter_samples(progress=progress): if is_frame_field: for frame in sample.frames.values(): _dup_ids = _find_duplicates( diff --git a/fiftyone/utils/labelbox.py b/fiftyone/utils/labelbox.py index e130d44751..fe0d527998 100644 --- a/fiftyone/utils/labelbox.py +++ b/fiftyone/utils/labelbox.py @@ -397,14 +397,17 @@ def list_datasets(self): datasets = self._client.get_datasets() return [d.uid for d in datasets] - def delete_datasets(self, dataset_ids): + def delete_datasets(self, dataset_ids, progress=None): """Deletes the given datasets from the Labelbox server. Args: dataset_ids: an iterable of dataset IDs + progress (None): whether to render a progress bar (True/False), use + the default value ``fiftyone.config.show_progress_bars`` + (None), or a progress callback function to invoke instead """ logger.info("Deleting datasets...") - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for dataset_id in pb(list(dataset_ids)): dataset = self._client.get_dataset(dataset_id) dataset.delete() @@ -1490,6 +1493,7 @@ def import_from_labelbox( label_prefix=None, download_dir=None, labelbox_id_field="labelbox_id", + progress=None, ): """Imports the labels from the Labelbox project into the FiftyOne dataset. @@ -1546,6 +1550,9 @@ def import_from_labelbox( samples labelbox_id_field ("labelbox_id"): the sample field to lookup/store the IDs of the Labelbox DataRows + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection(dataset, media_type=(fomm.IMAGE, fomm.VIDEO)) is_video = dataset.media_type == fomm.VIDEO @@ -1567,7 +1574,7 @@ def import_from_labelbox( d_list = etas.read_json(json_path) # ref: https://github.com/Labelbox/labelbox/blob/7c79b76310fa867dd38077e83a0852a259564da1/exporters/coco-exporter/coco_exporter.py#L33 - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for d in pb(d_list): labelbox_id = d["DataRow ID"] @@ -1634,6 +1641,7 @@ def export_to_labelbox( labelbox_id_field="labelbox_id", label_field=None, frame_labels_field=None, + progress=None, ): """Exports labels from the FiftyOne samples to Labelbox format. @@ -1690,6 +1698,9 @@ def export_to_labelbox( when constructing the exported frame labels By default, no frame labels are exported + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection( sample_collection, media_type=(fomm.IMAGE, fomm.VIDEO) @@ -1724,7 +1735,7 @@ def export_to_labelbox( etau.ensure_empty_file(ndjson_path) # Export the labels - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(sample_collection): labelbox_id = sample[labelbox_id_field] if labelbox_id is None: @@ -1792,7 +1803,10 @@ def download_labels_from_labelbox(labelbox_project, outpath=None): def upload_media_to_labelbox( - labelbox_dataset, sample_collection, labelbox_id_field="labelbox_id" + labelbox_dataset, + sample_collection, + labelbox_id_field="labelbox_id", + progress=None, ): """Uploads the raw media for the FiftyOne samples to Labelbox. @@ -1806,11 +1820,14 @@ def upload_media_to_labelbox( :class:`fiftyone.core.collections.SampleCollection` labelbox_id_field ("labelbox_id"): the sample field in which to store the IDs of the Labelbox DataRows + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ # @todo use `create_data_rows()` to optimize performance # @todo handle API rate limits # Reference: https://labelbox.com/docs/python-api/data-rows - with fou.ProgressBar() as pb: + with fou.ProgressBar(progress=progress) as pb: for sample in pb(sample_collection): try: has_id = sample[labelbox_id_field] is not None @@ -1859,15 +1876,11 @@ def upload_labels_to_labelbox( else: annos = annos_or_ndjson_path - requests = [] count = 0 for anno_batch in fou.iter_batches(annos, batch_size): count += 1 name = "%s-upload-request-%d" % (labelbox_project.name, count) - request = labelbox_project.upload_annotations(name, anno_batch) - requests.append(request) - - return requests + labelbox_project.upload_annotations(name, anno_batch) def convert_labelbox_export_to_import(inpath, outpath=None, video_outdir=None): diff --git a/fiftyone/utils/labels.py b/fiftyone/utils/labels.py index 1e986c223d..1bb8caa3cd 100644 --- a/fiftyone/utils/labels.py +++ b/fiftyone/utils/labels.py @@ -23,6 +23,7 @@ def objects_to_segmentations( rel_dir=None, overwrite=False, save_mask_targets=False, + progress=None, ): """Converts the instance segmentations or polylines in the specified field of the collection into semantic segmentation masks. @@ -61,6 +62,9 @@ def objects_to_segmentations( if it exists save_mask_targets (False): whether to store the ``mask_targets`` on the dataset + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, @@ -83,7 +87,7 @@ def objects_to_segmentations( output_dir=output_dir, rel_dir=rel_dir, idempotent=False ) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -147,6 +151,7 @@ def export_segmentations( rel_dir=None, update=True, overwrite=False, + progress=None, ): """Exports the segmentations (or heatmaps) stored as in-database arrays in the specified field to images on disk. @@ -170,6 +175,9 @@ def export_segmentations( update (True): whether to delete the arrays from the database overwrite (False): whether to delete ``output_dir`` prior to exporting if it exists + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, in_field, (fol.Segmentation, fol.Heatmap) @@ -185,7 +193,7 @@ def export_segmentations( output_dir=output_dir, rel_dir=rel_dir, idempotent=False ) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -209,7 +217,11 @@ def export_segmentations( def import_segmentations( - sample_collection, in_field, update=True, delete_images=False + sample_collection, + in_field, + update=True, + delete_images=False, + progress=None, ): """Imports the segmentations (or heatmaps) stored on disk in the specified field to in-database arrays. @@ -224,6 +236,9 @@ def import_segmentations( :class:`fiftyone.core.labels.Heatmap` field update (True): whether to delete the image paths from the labels delete_images (False): whether to delete any imported images from disk + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, in_field, (fol.Segmentation, fol.Heatmap) @@ -232,7 +247,7 @@ def import_segmentations( samples = sample_collection.select_fields(in_field) in_field, processing_frames = samples._handle_frame_field(in_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -266,6 +281,7 @@ def transform_segmentations( update=True, update_mask_targets=False, overwrite=False, + progress=None, ): """Transforms the segmentations in the given field according to the provided targets map. @@ -299,6 +315,9 @@ def transform_segmentations( dataset to reflect the transformed targets overwrite (False): whether to delete ``output_dir`` prior to exporting if it exists + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, in_field, fol.Segmentation @@ -315,7 +334,7 @@ def transform_segmentations( output_dir=output_dir, rel_dir=rel_dir, idempotent=False ) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -365,6 +384,7 @@ def segmentations_to_detections( out_field, mask_targets=None, mask_types="stuff", + progress=None, ): """Converts the semantic segmentations masks in the specified field of the collection into :class:`fiftyone.core.labels.Detections` with instance @@ -398,6 +418,9 @@ def segmentations_to_detections( - ``"thing"`` if all classes are thing classes - a dict mapping pixel values (2D masks) or RGB hex strings (3D masks) to ``"stuff"`` or ``"thing"`` for each class + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, @@ -409,7 +432,7 @@ def segmentations_to_detections( in_field, processing_frames = samples._handle_frame_field(in_field) out_field, _ = samples._handle_frame_field(out_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -426,7 +449,12 @@ def segmentations_to_detections( def instances_to_polylines( - sample_collection, in_field, out_field, tolerance=2, filled=True + sample_collection, + in_field, + out_field, + tolerance=2, + filled=True, + progress=None, ): """Converts the instance segmentations in the specified field of the collection into :class:`fiftyone.core.labels.Polylines` instances. @@ -445,6 +473,9 @@ def instances_to_polylines( tolerance (2): a tolerance, in pixels, when generating approximate polylines for each region. Typical values are 1-3 pixels filled (True): whether the polylines should be filled + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, @@ -456,7 +487,7 @@ def instances_to_polylines( in_field, processing_frames = samples._handle_frame_field(in_field) out_field, _ = samples._handle_frame_field(out_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -479,6 +510,7 @@ def segmentations_to_polylines( mask_targets=None, mask_types="stuff", tolerance=2, + progress=None, ): """Converts the semantic segmentations masks in the specified field of the collection into :class:`fiftyone.core.labels.Polylines` instances. @@ -513,6 +545,9 @@ def segmentations_to_polylines( masks) to ``"stuff"`` or ``"thing"`` for each class tolerance (2): a tolerance, in pixels, when generating approximate polylines for each region. Typical values are 1-3 pixels + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, @@ -524,7 +559,7 @@ def segmentations_to_polylines( in_field, processing_frames = samples._handle_frame_field(in_field) out_field, _ = samples._handle_frame_field(out_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -542,7 +577,9 @@ def segmentations_to_polylines( ) -def classification_to_detections(sample_collection, in_field, out_field): +def classification_to_detections( + sample_collection, in_field, out_field, progress=None +): """Converts the :class:`fiftyone.core.labels.Classification` field of the collection into a :class:`fiftyone.core.labels.Detections` field containing a single detection whose bounding box spans the entire image. @@ -554,6 +591,9 @@ def classification_to_detections(sample_collection, in_field, out_field): field out_field: the name of the :class:`fiftyone.core.labels.Detections` field to populate + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, in_field, fol.Classification @@ -563,7 +603,7 @@ def classification_to_detections(sample_collection, in_field, out_field): in_field, processing_frames = samples._handle_frame_field(in_field) out_field, _ = samples._handle_frame_field(out_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: @@ -582,7 +622,9 @@ def classification_to_detections(sample_collection, in_field, out_field): image[out_field] = fol.Detections(detections=[detection]) -def classifications_to_detections(sample_collection, in_field, out_field): +def classifications_to_detections( + sample_collection, in_field, out_field, progress=None +): """Converts the :class:`fiftyone.core.labels.Classifications` field of the collection into a :class:`fiftyone.core.labels.Detections` field containing detections whose bounding boxes span the entire image with one detection @@ -595,6 +637,9 @@ def classifications_to_detections(sample_collection, in_field, out_field): field out_field: the name of the :class:`fiftyone.core.labels.Detections` field to populate + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection_label_fields( sample_collection, in_field, fol.Classifications @@ -604,7 +649,7 @@ def classifications_to_detections(sample_collection, in_field, out_field): in_field, processing_frames = samples._handle_frame_field(in_field) out_field, _ = samples._handle_frame_field(out_field) - for sample in samples.iter_samples(autosave=True, progress=True): + for sample in samples.iter_samples(autosave=True, progress=progress): if processing_frames: images = sample.frames.values() else: diff --git a/fiftyone/utils/scale.py b/fiftyone/utils/scale.py index 16cd8c90a6..d594a0bda5 100644 --- a/fiftyone/utils/scale.py +++ b/fiftyone/utils/scale.py @@ -36,6 +36,7 @@ def import_from_scale( labels_dir_or_json, label_prefix=None, scale_id_field="scale_id", + progress=None, ): """Imports the Scale AI labels into the FiftyOne dataset. @@ -186,6 +187,9 @@ def import_from_scale( that are created, separated by an underscore scale_id_field ("scale_id"): the sample field to use to associate Scale task IDs with FiftyOne samples + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection(dataset, media_type=(fomm.IMAGE, fomm.VIDEO)) is_video = dataset.media_type == fomm.VIDEO @@ -203,7 +207,7 @@ def import_from_scale( else: label_key = lambda k: k - with fou.ProgressBar(total=len(labels)) as pb: + with fou.ProgressBar(total=len(labels), progress=progress) as pb: for task_id, task_labels in pb(labels.items()): if task_id not in id_map: logger.info( @@ -255,6 +259,7 @@ def export_to_scale( video_playback=False, label_field=None, frame_labels_field=None, + progress=None, ): """Exports labels from the FiftyOne samples to Scale AI format. @@ -393,6 +398,9 @@ def export_to_scale( when constructing the exported frame labels By default, no frame labels are exported + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ fov.validate_collection( sample_collection, media_type=(fomm.IMAGE, fomm.VIDEO) @@ -430,7 +438,7 @@ def export_to_scale( # Export the labels labels = {} anno_dict = {} - for sample in sample_collection.iter_samples(progress=True): + for sample in sample_collection.iter_samples(progress=progress): metadata = sample.metadata # Get frame size diff --git a/fiftyone/utils/utils3d.py b/fiftyone/utils/utils3d.py index 17d297d214..501ee77c2e 100644 --- a/fiftyone/utils/utils3d.py +++ b/fiftyone/utils/utils3d.py @@ -446,6 +446,7 @@ def compute_orthographic_projection_images( subsampling_rate=None, projection_normal=None, bounds=None, + progress=None, ): """Computes orthographic projection images for the point clouds in the given collection. @@ -509,6 +510,9 @@ def compute_orthographic_projection_images( to generate each map. Either element of the tuple or any/all of its values can be None, in which case a tight crop of the point cloud along the missing dimension(s) are used + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead """ if in_group_slice is None and samples.media_type == fom.GROUP: in_group_slice = _get_point_cloud_slice(samples) @@ -537,7 +541,7 @@ def compute_orthographic_projection_images( all_metadata = [] - with fou.ProgressBar(total=len(filepaths)) as pb: + with fou.ProgressBar(total=len(filepaths), progress=progress) as pb: for filepath, group in pb(zip(filepaths, groups)): image_path = filename_maker.get_output_path( filepath, output_ext=".png" diff --git a/fiftyone/utils/video.py b/fiftyone/utils/video.py index e3b36e2e49..bbeda4af62 100644 --- a/fiftyone/utils/video.py +++ b/fiftyone/utils/video.py @@ -97,6 +97,7 @@ def reencode_videos( delete_originals=False, skip_failures=False, verbose=False, + progress=None, **kwargs, ): """Re-encodes the videos in the sample collection as H.264 MP4s that can be @@ -152,6 +153,9 @@ def reencode_videos( an error if a video cannot be re-encoded verbose (False): whether to log the ``ffmpeg`` commands that are executed + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: keyword arguments for ``eta.core.video.FFmpeg(**kwargs)`` """ fov.validate_video_collection(sample_collection) @@ -168,6 +172,7 @@ def reencode_videos( delete_originals=delete_originals, skip_failures=skip_failures, verbose=verbose, + progress=progress, **kwargs, ) @@ -190,6 +195,7 @@ def transform_videos( delete_originals=False, skip_failures=False, verbose=False, + progress=None, **kwargs, ): """Transforms the videos in the sample collection according to the provided @@ -258,6 +264,9 @@ def transform_videos( an error if a video cannot be transformed verbose (False): whether to log the ``ffmpeg`` commands that are executed + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: keyword arguments for ``eta.core.video.FFmpeg(**kwargs)`` """ fov.validate_video_collection(sample_collection) @@ -280,6 +289,7 @@ def transform_videos( delete_originals=delete_originals, skip_failures=skip_failures, verbose=verbose, + progress=progress, **kwargs, ) @@ -303,6 +313,7 @@ def sample_videos( delete_originals=False, skip_failures=False, verbose=False, + progress=None, **kwargs, ): """Samples the videos in the sample collection into directories of @@ -404,6 +415,9 @@ def sample_videos( an error if a video cannot be sampled verbose (False): whether to log the ``ffmpeg`` commands that are executed + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: keyword arguments for ``eta.core.video.FFmpeg(**kwargs)`` """ fov.validate_video_collection(sample_collection) @@ -428,6 +442,7 @@ def sample_videos( delete_originals=delete_originals, skip_failures=skip_failures, verbose=verbose, + progress=progress, **kwargs, ) @@ -725,6 +740,7 @@ def _transform_videos( delete_originals=False, skip_failures=False, verbose=False, + progress=None, **kwargs, ): if output_field is None: @@ -744,7 +760,7 @@ def _transform_videos( if frames is None: frames = itertools.repeat(None) - with fou.ProgressBar(total=len(view)) as pb: + with fou.ProgressBar(total=view, progress=progress) as pb: for sample, _frames in pb(zip(view, frames)): inpath = sample[media_field] diff --git a/fiftyone/zoo/datasets/__init__.py b/fiftyone/zoo/datasets/__init__.py index 0a68df2a71..3b00e87ba1 100644 --- a/fiftyone/zoo/datasets/__init__.py +++ b/fiftyone/zoo/datasets/__init__.py @@ -188,6 +188,7 @@ def load_zoo_dataset( persistent=False, overwrite=False, cleanup=True, + progress=None, **kwargs, ): """Loads the dataset of the given name from the FiftyOne Dataset Zoo as @@ -235,6 +236,9 @@ def load_zoo_dataset( dataset is to be downloaded cleanup (True): whether to cleanup any temporary files generated during download + progress (None): whether to render a progress bar (True/False), use the + default value ``fiftyone.config.show_progress_bars`` (None), or a + progress callback function to invoke instead **kwargs: optional arguments to pass to the :class:`fiftyone.utils.data.importers.DatasetImporter` constructor. If ``download_if_necessary == True``, then ``kwargs`` can also @@ -343,14 +347,21 @@ def load_zoo_dataset( dataset_type, dataset_dir=split_dir, **importer_kwargs ) dataset.add_importer( - dataset_importer, label_field=label_field, tags=[split] + dataset_importer, + label_field=label_field, + tags=[split], + progress=progress, ) else: logger.info("Loading '%s'", zoo_dataset.name) dataset_importer, _ = foud.build_dataset_importer( dataset_type, dataset_dir=dataset_dir, **importer_kwargs ) - dataset.add_importer(dataset_importer, label_field=label_field) + dataset.add_importer( + dataset_importer, + label_field=label_field, + progress=progress, + ) if info.classes is not None: dataset.default_classes = info.classes diff --git a/tests/unittests/utils_tests.py b/tests/unittests/utils_tests.py index 17a40dd270..ac7cac3f2a 100644 --- a/tests/unittests/utils_tests.py +++ b/tests/unittests/utils_tests.py @@ -455,6 +455,44 @@ def test_load_dataset_nonexistent(self, mock_get_db_conn): ) +class ProgressBarTests(unittest.TestCase): + def _test_correct_value(self, progress, global_progress, quiet, expected): + with fou.SetAttributes(fo.config, show_progress_bars=global_progress): + with fou.ProgressBar([], progress=progress, quiet=quiet) as pb: + assert pb._progress == expected + + def test_progress_none_uses_global(self): + self._test_correct_value( + progress=None, global_progress=True, quiet=None, expected=True + ) + self._test_correct_value( + progress=None, global_progress=False, quiet=None, expected=False + ) + + def test_progress_overwrites_global(self): + self._test_correct_value( + progress=True, global_progress=True, quiet=None, expected=True + ) + self._test_correct_value( + progress=True, global_progress=False, quiet=None, expected=True + ) + self._test_correct_value( + progress=False, global_progress=True, quiet=None, expected=False + ) + self._test_correct_value( + progress=False, global_progress=False, quiet=None, expected=False + ) + + def test_quiet_overwrites_all(self): + # Careful, we expect here to have progress the opposite value of quiet + self._test_correct_value( + progress=True, global_progress=True, quiet=True, expected=False + ) + self._test_correct_value( + progress=False, global_progress=False, quiet=False, expected=True + ) + + if __name__ == "__main__": fo.config.show_progress_bars = False unittest.main(verbosity=2)