Histogram refactor

[blowekamp]

No description provided.

[blowekamp]

@philipmac I believe this function implements the minimum requirements: https://github.com/niaid/tomojs-pytools/pull/58/files#diff-0a15d65e391e72614aa4df7a9b0cfbd0718a296bb159b6b4a4c78ee54514e386R28

The CLI interface was removed to prevent duplication of validation and other operations. Old features can always be retrieved from the history and reimplemented in the procedural interface as needed.

[philipmac]

(dev) macmenaminpe@L02147210:~/code/hedwig/dev/image_portal_workflows$ pip install -r requirements.txt --upgrade git+https://github.com/blowekamp/tomojs-pytools.git@histogram_refactor
....

(dev) macmenaminpe@L02147210:~/code/hedwig/dev/image_portal_workflows$ python3
Python 3.8.10 (default, Mar 13 2023, 10:26:41) 
[GCC 9.4.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> from pytools.workflow_functions import visual_min_max
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/macmenaminpe/code/hedwig/dev/lib/python3.8/site-packages/pytools/__init__.py", line 15, in <module>
    from .workflow_functions import visual_min_max
  File "/home/macmenaminpe/code/hedwig/dev/lib/python3.8/site-packages/pytools/workflow_functions.py", line 19, in <module>
    from SimpleITK.utilities.dask import from_sitk
  File "/home/macmenaminpe/code/hedwig/dev/lib/python3.8/site-packages/SimpleITK/utilities/__init__.py", line 41, in <module>
    importlib.util.find_spec("vtk")
AttributeError: module 'importlib' has no attribute 'util'

[philipmac]

>>> importlib.util.find_spec
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
AttributeError: module 'importlib' has no attribute 'util'
>>> importlib
importlib
>>> importlib.
importlib.find_loader(        importlib.invalidate_caches(  importlib.sys                 importlib.warnings
importlib.import_module(      importlib.reload(             importlib.types      

[blowekamp]

The error is addressed in the following PR: SimpleITK/SimpleITKUtilities#17

[blowekamp]

@philipmac I added a pytools.logger as a parent of all loggers to allow for setting the log level and redirected the output to other files or log streams. I also added debug logging of dask.config.global_config to clarify and debug the dask execution environment being used.

Can you please set pytools.logger.level = logging.DEBUG and then execute the workflow and report output for the global_config?

Thank you.

[philipmac]

DEBUG:pytools.workflow_functions:dask.config.global_config: {'temporary-directory': None, 'visualization': {'engine': None}, 'tokenize': {'ensure-deterministic': False}, 'dataframe': {'backend': 'pandas', '
shuffle': {'method': None, 'compression': None}, 'parquet': {'metadata-task-size-local': 512, 'metadata-task-size-remote': 1}, 'convert-string': False}, 'array': {'backend': 'numpy', 'rechunk': {'method': '
tasks'}, 'svg': {'size': 120}, 'slicing': {'split-large-chunks': None}, 'chunk-size': '128MiB', 'rechunk-threshold': 4}, 'optimization': {'annotations': {'fuse': True}, 'fuse': {'active': None, 'ave-width':
 1, 'max-width': None, 'max-height': inf, 'max-depth-new-edges': None, 'subgraphs': None, 'rename-keys': True}}, 'distributed': {'version': 2, 'scheduler': {'allowed-failures': 3, 'bandwidth': 100000000, 'b
locked-handlers': [], 'contact-address': None, 'default-data-size': '1kiB', 'events-cleanup-delay': '1h', 'idle-timeout': None, 'transition-log-length': 100000, 'events-log-length': 100000, 'work-stealing':
 True, 'work-stealing-interval': '100ms', 'worker-saturation': 1.1, 'worker-ttl': '5 minutes', 'pickle': True, 'preload': [], 'preload-argv': [], 'unknown-task-duration': '500ms', 'default-task-durations': 
{'rechunk-split': '1us', 'split-shuffle': '1us'}, 'validate': False, 'dashboard': {'status': {'task-stream-length': 1000}, 'tasks': {'task-stream-length': 100000}, 'tls': {'ca-file': None, 'key': None, 'cert': None}, 'bokeh-application': {'allow_websocket_origin': ['*'], 'keep_alive_milliseconds': 500, 'check_unused_sessions_milliseconds': 500}}, 'locks': {'lease-validation-interval': '10s', 'lease-timeout': '30s'}, 'http': {'routes': ['distributed.http.scheduler.prometheus', 'distributed.http.scheduler.info', 'distributed.http.scheduler.json', 'distributed.http.health', 'distributed.http.proxy', 'distributed.http.statics']}, 'allowed-imports': ['dask', 'distributed'], 'active-memory-manager': {'start': True, 'interval': '2s', 'measure': 'optimistic', 'policies': [{'class': 'distributed.active_memory_manager.ReduceReplicas'}]}}, 'worker': {'blocked-handlers': [], 'multiprocessing-method': 'spawn', 'use-file-locking': True, 'transfer': {'message-bytes-limit': '50MB'}, 'connections': {'outgoing': 50, 'incoming': 10}, 'preload': [], 'preload-argv': [], 'daemon': True, 'validate': False, 'resources': {}, 'lifetime': {'duration': None, 'stagger': '0 seconds', 'restart': False}, 'profile': {'enabled': True, 'interval': '10ms', 'cycle': '1000ms', 'low-level': False}, 'memory': {'recent-to-old-time': '30s', 'rebalance': {'measure': 'optimistic', 'sender-min': 0.3, 'recipient-max': 0.6, 'sender-recipient-gap': 0.1}, 'transfer': 0.1, 'target': 0.6, 'spill': 0.7, 'pause': 0.8, 'terminate': 0.95, 'max-spill': False, 'monitor-interval': '100ms'}, 'http': {'routes': ['distributed.http.worker.prometheus', 'distributed.http.health', 'distributed.http.statics']}}, 'nanny': {'preload': [], 'preload-argv': [], 'environ': {}, 'pre-spawn-environ': {'MALLOC_TRIM_THRESHOLD_': 65536, 'OMP_NUM_THREADS': 1, 'MKL_NUM_THREADS': 1, 'OPENBLAS_NUM_THREADS': 1}}, 'client': {'heartbeat': '5s', 'scheduler-info-interval': '2s', 'security-loader': None, 'preload': [], 'preload-argv': []}, 'deploy': {'lost-worker-timeout': '15s', 'cluster-repr-interval': '500ms'}, 'adaptive': {'interval': '1s', 'target-duration': '5s', 'minimum': 0, 'maximum': inf, 'wait-count': 3}, 'comm': {'retry': {'count': 0, 'delay': {'min': '1s', 'max': '20s'}}, 'compression': 'auto', 'shard': '64MiB', 'offload': '10MiB', 'default-scheme': 'tcp', 'socket-backlog': 2048, 'recent-messages-log-length': 0, 'ucx': {'cuda-copy': None, 'tcp': None, 'nvlink': None, 'infiniband': None, 'rdmacm': None, 'create-cuda-context': None, 'environment': {}}, 'zstd': {'level': 3, 'threads': 0}, 'timeouts': {'connect': '30s', 'tcp': '30s'}, 'require-encryption': None, 'tls': {'ciphers': None, 'min-version': 1.2, 'max-version': None, 'ca-file': None, 'scheduler': {'cert': None, 'key': None}, 'worker': {'key': None, 'cert': None}, 'client': {'key': None, 'cert': None}}, 'tcp': {'backend': 'tornado'}, 'websockets': {'shard': '8MiB'}}, 'diagnostics': {'nvml': True, 'computations': {'max-history': 100, 'nframes': 0, 'ignore-modules': ['distributed', 'dask', 'xarray', 'cudf', 'cuml', 'prefect', 'xgboost']}, 'erred-tasks': {'max-history': 100}}, 'dashboard': {'link': '{scheme}://{host}:{port}/status', 'export-tool': False, 'graph-max-items': 5000, 'prometheus': {'namespace': 'dask'}}, 'admin': {'tick': {'interval': '20ms', 'limit': '3s', 'cycle': '1s'}, 'max-error-length': 10000, 'log-length': 10000, 'log-format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s', 'pdb-on-err': False, 'system-monitor': {'interval': '500ms', 'disk': True, 'host-cpu': False, 'gil': {'enabled': False, 'interval': '1ms'}}, 'event-loop': 'tornado'}, 'rmm': {'pool-size': None}}, 'parent': 33887, 'jobqueue': {'oar': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'project': None, 'walltime': '00:30:00', 'env-extra': None, 'job-script-prologue': [], 'resource-spec': None, 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'scheduler-options': {}}, 'pbs': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'account': None, 'walltime': '00:30:00', 'env-extra': None, 'job-script-prologue': [], 'resource-spec': None, 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'scheduler-options': {}}, 'sge': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'project': None, 'walltime': '00:30:00', 'env-extra': None, 'job-script-prologue': [], 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'resource-spec': None, 'scheduler-options': {}}, 'slurm': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'account': None, 'walltime': '00:30:00', 'env-extra': None, 'job-script-prologue': [], 'job-cpu': None, 'job-mem': None, 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'scheduler-options': {}}, 'moab': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'account': None, 'walltime': '00:30:00', 'env-extra': None, 'job-script-prologue': [], 'resource-spec': None, 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'scheduler-options': {}}, 'lsf': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-extra-args': [], 'shebang': '#!/usr/bin/env bash', 'queue': None, 'project': None, 'walltime': '00:30', 'env-extra': None, 'job-script-prologue': [], 'ncpus': None, 'mem': None, 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'lsf-units': None, 'use-stdin': True, 'scheduler-options': {}}, 'htcondor': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-extra-args': [], 'disk': None, 'env-extra': None, 'job-script-prologue': [], 'job-extra': None, 'job-extra-directives': {}, 'job-directives-skip': [], 'submit-command-extra': [], 'cancel-command-extra': [], 'log-directory': None, 'shebang': '#!/usr/bin/env condor_submit', 'scheduler-options': {}}, 'local': {'name': 'dask-worker', 'cores': None, 'memory': None, 'processes': None, 'interface': None, 'death-timeout': 60, 'local-directory': None, 'shared-temp-directory': None, 'extra': None, 'worker-extra-args': [], 'env-extra': None, 'job-script-prologue': [], 'job-extra': None, 'job-extra-directives': [], 'job-directives-skip': [], 'log-directory': None, 'scheduler-options': {}}}}