Expose number of jobs to organize

dandi/cli/cmd_organize.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Optional
+
 import click
 
 from .base import dandiset_path_option, devel_debug_option, map_to_click_exceptions
@@ -59,6 +61,13 @@
     ),
 )
 @click.argument("paths", nargs=-1, type=click.Path(exists=True))
+@click.option(
+    "--number-of-jobs",
+    "number_of_jobs",
+    type=int,
+    default=None,
+    help="The number of jobs to use during organization.",
+)
 @devel_debug_option()
 @map_to_click_exceptions
 def organize(
@@ -70,6 +79,7 @@ def organize(
     devel_debug=False,
     update_external_file_paths=False,
     media_files_mode=None,
+    number_of_jobs: Optional[int] = None,
 ):
     """(Re)organize files according to the metadata.
 
@@ -115,4 +125,5 @@ def organize(
         update_external_file_paths=update_external_file_paths,
         media_files_mode=media_files_mode,
         required_fields=required_fields,
+        number_of_jobs=number_of_jobs,
     )

dandi/organize.py

@@ -721,8 +721,10 @@ def organize(
     update_external_file_paths=False,
     media_files_mode=None,
     required_fields=None,
+    number_of_jobs: Optional[int] = None,
 ):
     in_place = False  # If we deduce that we are organizing in-place
+    number_of_jobs = number_of_jobs or -1
 
     # will come handy when dry becomes proper separate option
     def dry_print(msg):
@@ -812,12 +814,12 @@ def _get_metadata(path):
             meta["path"] = path
             return meta
 
-        if not devel_debug:
+        if not devel_debug and number_of_jobs != 1:  # Do not use joblib at all if number_of_jobs=1
             # Note: It is Python (pynwb) intensive, not IO, so ATM there is little
             # to no benefit from Parallel without using multiproc!  But that would
             # complicate progress bar indication... TODO
             metadata = list(
-                Parallel(n_jobs=-1, verbose=10)(
+                Parallel(n_jobs=number_of_jobs, verbose=10)(
                     delayed(_get_metadata)(path) for path in paths
                 )
             )