diff --git a/.doctrees/array-api.doctree b/.doctrees/array-api.doctree index 85e7f792..9705aae0 100644 Binary files a/.doctrees/array-api.doctree and b/.doctrees/array-api.doctree differ diff --git a/.doctrees/design.doctree b/.doctrees/design.doctree index 9d9726e5..3f312976 100644 Binary files a/.doctrees/design.doctree and b/.doctrees/design.doctree differ diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle index 8c9aac8a..bd1b0102 100644 Binary files a/.doctrees/environment.pickle and b/.doctrees/environment.pickle differ diff --git a/.doctrees/generated/cubed.nanmean.doctree b/.doctrees/generated/cubed.nanmean.doctree index 1837a20a..6d6a4e07 100644 Binary files a/.doctrees/generated/cubed.nanmean.doctree and b/.doctrees/generated/cubed.nanmean.doctree differ diff --git a/.doctrees/generated/cubed.nansum.doctree b/.doctrees/generated/cubed.nansum.doctree index 27cebc2e..8f9ec7bb 100644 Binary files a/.doctrees/generated/cubed.nansum.doctree and b/.doctrees/generated/cubed.nansum.doctree differ diff --git a/_static/documentation_options.js b/_static/documentation_options.js index 35a002f2..577d58e5 100644 --- a/_static/documentation_options.js +++ b/_static/documentation_options.js @@ -1,5 +1,5 @@ const DOCUMENTATION_OPTIONS = { - VERSION: '2065794', + VERSION: '7ca5ae9', LANGUAGE: 'en', COLLAPSE_INDEX: false, BUILDER: 'html', diff --git a/api.html b/api.html index b3b0456c..287cf87d 100644 --- a/api.html +++ b/api.html @@ -36,7 +36,7 @@ - + diff --git a/array-api.html b/array-api.html index f7858fa3..72ba6d5d 100644 --- a/array-api.html +++ b/array-api.html @@ -36,7 +36,7 @@ - + diff --git a/computation.html b/computation.html index 72512606..77d7725f 100644 --- a/computation.html +++ b/computation.html @@ -36,7 +36,7 @@ - + diff --git a/configuration.html b/configuration.html index 0f605ed8..ef0d6b3c 100644 --- a/configuration.html +++ b/configuration.html @@ -36,7 +36,7 @@ - + diff --git a/contributing.html b/contributing.html index 4029615b..7b0c53b9 100644 --- a/contributing.html +++ b/contributing.html @@ -36,7 +36,7 @@ - + diff --git a/design.html b/design.html index 77964f3a..b7b7c8ab 100644 --- a/design.html +++ b/design.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.Array.compute.html b/generated/cubed.Array.compute.html index 161d757d..dacca285 100644 --- a/generated/cubed.Array.compute.html +++ b/generated/cubed.Array.compute.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.Array.html b/generated/cubed.Array.html index ef04af26..8e84a4aa 100644 --- a/generated/cubed.Array.html +++ b/generated/cubed.Array.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.Array.rechunk.html b/generated/cubed.Array.rechunk.html index 909bf3b2..3c4e693e 100644 --- a/generated/cubed.Array.rechunk.html +++ b/generated/cubed.Array.rechunk.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.Array.visualize.html b/generated/cubed.Array.visualize.html index f1d322b9..f2daeb45 100644 --- a/generated/cubed.Array.visualize.html +++ b/generated/cubed.Array.visualize.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.Callback.html b/generated/cubed.Callback.html index c46f3742..e204467b 100644 --- a/generated/cubed.Callback.html +++ b/generated/cubed.Callback.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.Spec.html b/generated/cubed.Spec.html index 35d502b5..4d10c4dd 100644 --- a/generated/cubed.Spec.html +++ b/generated/cubed.Spec.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.TaskEndEvent.html b/generated/cubed.TaskEndEvent.html index 1d348c69..221b5cc8 100644 --- a/generated/cubed.TaskEndEvent.html +++ b/generated/cubed.TaskEndEvent.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.apply_gufunc.html b/generated/cubed.apply_gufunc.html index d2df690a..7ce31112 100644 --- a/generated/cubed.apply_gufunc.html +++ b/generated/cubed.apply_gufunc.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.compute.html b/generated/cubed.compute.html index 51f24df6..1ec8d17a 100644 --- a/generated/cubed.compute.html +++ b/generated/cubed.compute.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.from_array.html b/generated/cubed.from_array.html index 7e90d0f1..b522559f 100644 --- a/generated/cubed.from_array.html +++ b/generated/cubed.from_array.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.from_zarr.html b/generated/cubed.from_zarr.html index 413b42fd..3f986808 100644 --- a/generated/cubed.from_zarr.html +++ b/generated/cubed.from_zarr.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.map_blocks.html b/generated/cubed.map_blocks.html index ee4d8355..b52a6e7b 100644 --- a/generated/cubed.map_blocks.html +++ b/generated/cubed.map_blocks.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.map_overlap.html b/generated/cubed.map_overlap.html index 6145fe4d..fd9e7448 100644 --- a/generated/cubed.map_overlap.html +++ b/generated/cubed.map_overlap.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.measure_reserved_mem.html b/generated/cubed.measure_reserved_mem.html index f547afa6..c102a007 100644 --- a/generated/cubed.measure_reserved_mem.html +++ b/generated/cubed.measure_reserved_mem.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.nanmean.html b/generated/cubed.nanmean.html index 7a212648..6fd3104e 100644 --- a/generated/cubed.nanmean.html +++ b/generated/cubed.nanmean.html @@ -36,7 +36,7 @@ - + @@ -366,7 +366,7 @@

Contents

cubed.nanmean#

-cubed.nanmean(x, /, *, axis=None, keepdims=False, use_new_impl=True, split_every=None)#
+cubed.nanmean(x, /, *, axis=None, keepdims=False, split_every=None)#

Compute the arithmetic mean along the specified axis, ignoring NaNs.

diff --git a/generated/cubed.nansum.html b/generated/cubed.nansum.html index 28c52035..31c6fc61 100644 --- a/generated/cubed.nansum.html +++ b/generated/cubed.nansum.html @@ -36,7 +36,7 @@ - + @@ -366,7 +366,7 @@

Contents

cubed.nansum#

-cubed.nansum(x, /, *, axis=None, dtype=None, keepdims=False, use_new_impl=True, split_every=None)#
+cubed.nansum(x, /, *, axis=None, dtype=None, keepdims=False, split_every=None)#

Return the sum of array elements over a given axis treating NaNs as zero.

diff --git a/generated/cubed.pad.html b/generated/cubed.pad.html index 56b1807e..85340b3f 100644 --- a/generated/cubed.pad.html +++ b/generated/cubed.pad.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.random.random.html b/generated/cubed.random.random.html index 3fe67af8..f4049960 100644 --- a/generated/cubed.random.random.html +++ b/generated/cubed.random.random.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.store.html b/generated/cubed.store.html index e9b6ece8..18400675 100644 --- a/generated/cubed.store.html +++ b/generated/cubed.store.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.to_zarr.html b/generated/cubed.to_zarr.html index cfd2fb09..ef0454fa 100644 --- a/generated/cubed.to_zarr.html +++ b/generated/cubed.to_zarr.html @@ -36,7 +36,7 @@ - + diff --git a/generated/cubed.visualize.html b/generated/cubed.visualize.html index 515ea753..6ecbd43b 100644 --- a/generated/cubed.visualize.html +++ b/generated/cubed.visualize.html @@ -36,7 +36,7 @@ - + diff --git a/genindex.html b/genindex.html index 938533db..c5d64274 100644 --- a/genindex.html +++ b/genindex.html @@ -35,7 +35,7 @@ - + diff --git a/getting-started/demo.html b/getting-started/demo.html index 4b98e9de..5ee0b755 100644 --- a/getting-started/demo.html +++ b/getting-started/demo.html @@ -36,7 +36,7 @@ - + diff --git a/getting-started/index.html b/getting-started/index.html index 83b80617..10a32493 100644 --- a/getting-started/index.html +++ b/getting-started/index.html @@ -36,7 +36,7 @@ - + diff --git a/getting-started/installation.html b/getting-started/installation.html index 124155ac..d698ac57 100644 --- a/getting-started/installation.html +++ b/getting-started/installation.html @@ -36,7 +36,7 @@ - + diff --git a/index.html b/index.html index ec3dd802..52e08fe8 100644 --- a/index.html +++ b/index.html @@ -36,7 +36,7 @@ - + diff --git a/objects.inv b/objects.inv index 1c0487c5..f029b208 100644 Binary files a/objects.inv and b/objects.inv differ diff --git a/operations.html b/operations.html index a8128647..13afa58b 100644 --- a/operations.html +++ b/operations.html @@ -36,7 +36,7 @@ - + diff --git a/related-projects.html b/related-projects.html index 363ae354..27ece9c4 100644 --- a/related-projects.html +++ b/related-projects.html @@ -36,7 +36,7 @@ - + diff --git a/search.html b/search.html index bc94dcd5..db0354c1 100644 --- a/search.html +++ b/search.html @@ -34,7 +34,7 @@ - + diff --git a/searchindex.js b/searchindex.js index 3453c9e0..16db97bf 100644 --- a/searchindex.js +++ b/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {"API Reference": [[0, null]], "Advanced settings": [[37, "advanced-settings"]], "Allowed memory": [[36, "allowed-memory"]], "Array": [[0, "array"]], "Array API": [[5, "array-api"]], "Articles": [[30, "articles"]], "Bounded-memory serverless distributed N-dimensional array processing": [[30, "bounded-memory-serverless-distributed-n-dimensional-array-processing"]], "Callbacks": [[33, "callbacks"]], "Chunk sizes": [[36, "chunk-sizes"]], "Chunk-specific functions": [[0, "chunk-specific-functions"]], "Cloud storage": [[40, "cloud-storage"]], "Computation": [[2, null]], "Conda": [[29, "conda"]], "Configuration": [[3, null]], "Contributing": [[4, null]], "Core operations": [[5, "core-operations"]], "Cubed": [[30, null]], "Dask": [[32, "dask"]], "Debugging optimization": [[37, "debugging-optimization"]], "Deleting intermediate data": [[40, "deleting-intermediate-data"]], "Demo": [[27, null]], "Dependency Tree": [[31, "dependency-tree"]], "Design": [[5, null]], "Development": [[4, "development"]], "Diagnosing Performance": [[39, "diagnosing-performance"]], "Diagnostics": [[33, null]], "Differences between Cubed and the standard": [[1, "differences-between-cubed-and-the-standard"]], "Different Cloud Providers": [[39, "different-cloud-providers"]], "Different Executors": [[39, "different-executors"]], "Documentation": [[30, "documentation"]], "Environment variables": [[3, "environment-variables"]], "Examples in use": [[33, "examples-in-use"]], "Execution": [[2, "execution"]], "Executor options": [[3, "executor-options"]], "Executors": [[34, null]], "For developers": [[30, null]], "For users": [[30, null]], "Getting Started": [[28, null]], "History": [[33, "history"]], "IO": [[0, "io"]], "Installation": [[29, null]], "Local single-machine executors": [[34, "local-single-machine-executors"]], "Map fusion": [[37, "map-fusion"]], "Memory": [[2, "memory"], [36, null]], "Memray": [[33, "memray"]], "Multi-pipeline Calculation": [[39, "multi-pipeline-calculation"]], "Multi-step Calculation": [[39, "multi-step-calculation"]], "Multiple-input fusion": [[37, "multiple-input-fusion"]], "Non-standardised functions": [[0, "non-standardised-functions"]], "Operations": [[31, null]], "Optimization": [[37, null]], "Optional dependencies": [[29, "optional-dependencies"]], "Other Performance Considerations": [[39, "other-performance-considerations"]], "Pip": [[29, "pip"]], "Plan": [[2, "plan"]], "Preface: Types of Scaling": [[39, "preface-types-of-scaling"]], "Previous work": [[32, "previous-work"]], "Primitive operations": [[5, "primitive-operations"]], "Progress bar": [[33, "progress-bar"]], "Projected memory": [[36, "projected-memory"]], "Python Array API": [[1, null]], "Random number generation": [[0, "random-number-generation"]], "Reference": [[3, "reference"]], "Related Projects": [[32, null]], "Reliability": [[38, null]], "Reserved memory": [[36, "reserved-memory"]], "Retries": [[38, "retries"]], "Runtime": [[0, "runtime"], [5, "runtime"]], "Scaling": [[39, null]], "Single-step Calculation": [[39, "single-step-calculation"]], "Spec object": [[3, "spec-object"]], "Spec options": [[3, "spec-options"]], "Specification": [[3, "specification"]], "Specifying an executor": [[34, "specifying-an-executor"]], "Storage": [[5, "storage"], [40, null]], "Stragglers": [[38, "stragglers"]], "Strong consistency": [[38, "strong-consistency"]], "Theoretical vs Practical Scaling of Cubed": [[39, "theoretical-vs-practical-scaling-of-cubed"]], "Timeline": [[33, "timeline"]], "Timeouts": [[38, "timeouts"]], "Tips": [[39, "tips"]], "Total number of input blocks": [[37, "total-number-of-input-blocks"]], "Total number of source arrays": [[37, "total-number-of-source-arrays"]], "User Guide": [[35, null]], "Visualize the computation plan": [[33, "visualize-the-computation-plan"]], "Which cloud service executor should I use?": [[34, "which-cloud-service-executor-should-i-use"]], "Why Cubed?": [[41, null]], "Xarray": [[32, "xarray"]], "YAML file": [[3, "yaml-file"]], "beam": [[3, "beam"]], "blockwise": [[31, "blockwise"]], "coiled": [[3, "coiled"]], "cubed.Array": [[6, null]], "cubed.Array.compute": [[7, null]], "cubed.Array.rechunk": [[8, null]], "cubed.Array.visualize": [[9, null]], "cubed.Callback": [[10, null]], "cubed.Spec": [[11, null]], "cubed.TaskEndEvent": [[12, null]], "cubed.apply_gufunc": [[13, null]], "cubed.compute": [[14, null]], "cubed.from_array": [[15, null]], "cubed.from_zarr": [[16, null]], "cubed.map_blocks": [[17, null]], "cubed.map_overlap": [[18, null]], "cubed.measure_reserved_mem": [[19, null]], "cubed.nanmean": [[20, null]], "cubed.nansum": [[21, null]], "cubed.pad": [[22, null]], "cubed.random.random": [[23, null]], "cubed.store": [[24, null]], "cubed.to_zarr": [[25, null]], "cubed.visualize": [[26, null]], "dask": [[3, "dask"]], "elemwise": [[31, "elemwise"]], "lithops": [[3, "lithops"]], "map_blocks": [[31, "map-blocks"]], "map_direct": [[31, "map-direct"]], "modal": [[3, "modal"]], "processes": [[3, "processes"]], "rechunk": [[31, "rechunk"]], "reduction and arg_reduction": [[31, "reduction-and-arg-reduction"]], "single-threaded": [[3, "single-threaded"]], "threads": [[3, "threads"]]}, "docnames": ["api", "array-api", "computation", "configuration", "contributing", "design", "generated/cubed.Array", "generated/cubed.Array.compute", "generated/cubed.Array.rechunk", "generated/cubed.Array.visualize", "generated/cubed.Callback", "generated/cubed.Spec", "generated/cubed.TaskEndEvent", "generated/cubed.apply_gufunc", "generated/cubed.compute", "generated/cubed.from_array", "generated/cubed.from_zarr", "generated/cubed.map_blocks", "generated/cubed.map_overlap", "generated/cubed.measure_reserved_mem", "generated/cubed.nanmean", "generated/cubed.nansum", "generated/cubed.pad", "generated/cubed.random.random", "generated/cubed.store", "generated/cubed.to_zarr", "generated/cubed.visualize", "getting-started/demo", "getting-started/index", "getting-started/installation", "index", "operations", "related-projects", "user-guide/diagnostics", "user-guide/executors", "user-guide/index", "user-guide/memory", "user-guide/optimization", "user-guide/reliability", "user-guide/scaling", "user-guide/storage", "why-cubed"], "envversion": {"sphinx": 63, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["api.rst", "array-api.md", "computation.md", "configuration.md", "contributing.md", "design.md", "generated/cubed.Array.rst", "generated/cubed.Array.compute.rst", "generated/cubed.Array.rechunk.rst", "generated/cubed.Array.visualize.rst", "generated/cubed.Callback.rst", "generated/cubed.Spec.rst", "generated/cubed.TaskEndEvent.rst", "generated/cubed.apply_gufunc.rst", "generated/cubed.compute.rst", "generated/cubed.from_array.rst", "generated/cubed.from_zarr.rst", "generated/cubed.map_blocks.rst", "generated/cubed.map_overlap.rst", "generated/cubed.measure_reserved_mem.rst", "generated/cubed.nanmean.rst", "generated/cubed.nansum.rst", "generated/cubed.pad.rst", "generated/cubed.random.random.rst", "generated/cubed.store.rst", "generated/cubed.to_zarr.rst", "generated/cubed.visualize.rst", "getting-started/demo.md", "getting-started/index.md", "getting-started/installation.md", "index.md", "operations.md", "related-projects.md", "user-guide/diagnostics.md", "user-guide/executors.md", "user-guide/index.md", "user-guide/memory.md", "user-guide/optimization.md", "user-guide/reliability.md", "user-guide/scaling.md", "user-guide/storage.md", "why-cubed.md"], "indexentries": {"__init__() (cubed.array method)": [[6, "cubed.Array.__init__", false]], "__init__() (cubed.callback method)": [[10, "cubed.Callback.__init__", false]], "__init__() (cubed.spec method)": [[11, "cubed.Spec.__init__", false]], "__init__() (cubed.taskendevent method)": [[12, "cubed.TaskEndEvent.__init__", false]], "apply_gufunc() (in module cubed)": [[13, "cubed.apply_gufunc", false]], "arange() (in module cubed.array_api)": [[1, "cubed.array_api.arange", false]], "array (class in cubed)": [[6, "cubed.Array", false]], "asarray() (in module cubed.array_api)": [[1, "cubed.array_api.asarray", false]], "broadcast_to() (in module cubed.array_api)": [[1, "cubed.array_api.broadcast_to", false]], "callback (class in cubed)": [[10, "cubed.Callback", false]], "compute() (cubed.array method)": [[7, "cubed.Array.compute", false]], "compute() (in module cubed)": [[14, "cubed.compute", false]], "concat() (in module cubed.array_api)": [[1, "cubed.array_api.concat", false]], "empty() (in module cubed.array_api)": [[1, "cubed.array_api.empty", false]], "empty_like() (in module cubed.array_api)": [[1, "cubed.array_api.empty_like", false]], "eye() (in module cubed.array_api)": [[1, "cubed.array_api.eye", false]], "from_array() (in module cubed)": [[15, "cubed.from_array", false]], "from_zarr() (in module cubed)": [[16, "cubed.from_zarr", false]], "full() (in module cubed.array_api)": [[1, "cubed.array_api.full", false]], "full_like() (in module cubed.array_api)": [[1, "cubed.array_api.full_like", false]], "linspace() (in module cubed.array_api)": [[1, "cubed.array_api.linspace", false]], "map_blocks() (in module cubed)": [[17, "cubed.map_blocks", false]], "map_overlap() (in module cubed)": [[18, "cubed.map_overlap", false]], "measure_reserved_mem() (in module cubed)": [[19, "cubed.measure_reserved_mem", false]], "nanmean() (in module cubed)": [[20, "cubed.nanmean", false]], "nansum() (in module cubed)": [[21, "cubed.nansum", false]], "ones() (in module cubed.array_api)": [[1, "cubed.array_api.ones", false]], "ones_like() (in module cubed.array_api)": [[1, "cubed.array_api.ones_like", false]], "pad() (in module cubed)": [[22, "cubed.pad", false]], "random() (in module cubed.random)": [[23, "cubed.random.random", false]], "rechunk() (cubed.array method)": [[8, "cubed.Array.rechunk", false]], "spec (class in cubed)": [[11, "cubed.Spec", false]], "store() (in module cubed)": [[24, "cubed.store", false]], "taskendevent (class in cubed)": [[12, "cubed.TaskEndEvent", false]], "to_zarr() (in module cubed)": [[25, "cubed.to_zarr", false]], "visualize() (cubed.array method)": [[9, "cubed.Array.visualize", false]], "visualize() (in module cubed)": [[26, "cubed.visualize", false]], "zeros() (in module cubed.array_api)": [[1, "cubed.array_api.zeros", false]], "zeros_like() (in module cubed.array_api)": [[1, "cubed.array_api.zeros_like", false]]}, "objects": {"cubed": [[6, 0, 1, "", "Array"], [10, 0, 1, "", "Callback"], [11, 0, 1, "", "Spec"], [12, 0, 1, "", "TaskEndEvent"], [13, 2, 1, "", "apply_gufunc"], [14, 2, 1, "", "compute"], [15, 2, 1, "", "from_array"], [16, 2, 1, "", "from_zarr"], [17, 2, 1, "", "map_blocks"], [18, 2, 1, "", "map_overlap"], [19, 2, 1, "", "measure_reserved_mem"], [20, 2, 1, "", "nanmean"], [21, 2, 1, "", "nansum"], [22, 2, 1, "", "pad"], [24, 2, 1, "", "store"], [25, 2, 1, "", "to_zarr"], [26, 2, 1, "", "visualize"]], "cubed.Array": [[6, 1, 1, "", "__init__"], [7, 1, 1, "", "compute"], [8, 1, 1, "", "rechunk"], [9, 1, 1, "", "visualize"]], "cubed.Callback": [[10, 1, 1, "", "__init__"]], "cubed.Spec": [[11, 1, 1, "", "__init__"]], "cubed.TaskEndEvent": [[12, 1, 1, "", "__init__"]], "cubed.array_api": [[1, 2, 1, "", "arange"], [1, 2, 1, "", "asarray"], [1, 2, 1, "", "broadcast_to"], [1, 2, 1, "", "concat"], [1, 2, 1, "", "empty"], [1, 2, 1, "", "empty_like"], [1, 2, 1, "", "eye"], [1, 2, 1, "", "full"], [1, 2, 1, "", "full_like"], [1, 2, 1, "", "linspace"], [1, 2, 1, "", "ones"], [1, 2, 1, "", "ones_like"], [1, 2, 1, "", "zeros"], [1, 2, 1, "", "zeros_like"]], "cubed.random": [[23, 2, 1, "", "random"]]}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "function", "Python function"]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:function"}, "terms": {"": [3, 5, 7, 30, 31, 32, 33, 34, 36, 37, 39, 40, 41], "0": [1, 11, 23, 29, 33], "00": 33, "001": [33, 37], "002": [33, 37], "003": [33, 37], "004": 37, "005": 37, "1": [1, 11, 12, 23, 27, 33, 34, 37, 40], "10": [4, 27, 37], "100": 33, "1000": 34, "10000": 33, "100kb": 27, "100mb": [3, 36, 39], "11": [3, 33], "12": 1, "180": 3, "2": [3, 11, 27, 29, 33, 34, 37], "20": [34, 36], "2000": 3, "200mb": [3, 33], "2022": 1, "2023": 1, "2gb": [3, 34, 36, 39], "3": [3, 4, 27, 33, 37], "300": 3, "4": [27, 33, 37], "400mb": 33, "438": 1, "5": [27, 33, 37], "5000": [3, 33], "50000": 3, "6": [27, 33, 37], "7": [27, 29, 33, 37], "8": [27, 33, 37], "9": [27, 33, 37], "A": [0, 2, 3, 34, 36, 38, 39, 41], "And": [32, 37], "As": [2, 36], "At": 34, "By": [33, 37, 40], "For": [2, 36, 38, 39, 40], "If": [3, 7, 9, 11, 14, 26, 33, 34, 36, 37, 38, 40], "In": [24, 31, 32, 33, 36, 37, 39, 40], "It": [3, 5, 19, 31, 32, 33, 34], "No": 31, "Not": 31, "On": [37, 40], "One": [2, 3, 39], "Or": [2, 3], "The": [1, 2, 3, 5, 7, 8, 9, 11, 14, 16, 18, 19, 24, 25, 26, 31, 32, 33, 34, 36, 37, 39, 40], "Then": 33, "There": [3, 5, 33, 36, 37, 39], "These": [0, 3, 5, 37], "To": [3, 29, 33, 37, 38, 39, 40], "With": 39, "__init__": [6, 10, 11, 12], "abil": 36, "about": [12, 13, 32, 33], "abov": [33, 34, 37], "abstract": [32, 41], "accept": 1, "access": 31, "account": 34, "accur": 36, "achiev": 39, "act": 38, "activ": 4, "actual": [3, 33, 36], "acycl": [2, 33], "ad": [2, 33, 39], "adapt": 33, "add": [3, 27, 31, 33, 37, 39, 40], "addit": [33, 37], "adjust": 39, "adopt": 41, "advanc": 35, "advantag": [2, 32, 39], "affect": [33, 39], "after": [8, 18, 33, 38, 40], "ag": 40, "again": [37, 38], "aggregr": 31, "ahead": [2, 36], "aim": [28, 36, 39], "algorithm": [5, 31, 36], "all": [3, 5, 31, 33, 34, 39, 41], "alloc": [13, 33, 41], "allow": [31, 35, 37, 38], "allow_rechunk": 13, "allowed_mem": [3, 11, 27, 34, 36, 39], "alon": 33, "along": [20, 31], "alreadi": [7, 14], "also": [2, 3, 27, 29, 34, 38, 39], "although": [31, 34], "alwai": [33, 41], "amazon": [34, 38], "amount": [13, 19, 33, 34, 36, 37, 38, 39, 41], "an": [2, 3, 4, 8, 9, 11, 15, 16, 19, 22, 25, 26, 28, 31, 33, 35, 36, 37, 38, 39, 41], "analys": 36, "ani": [3, 7, 11, 19, 33, 34, 37, 39, 40], "annot": 33, "anoth": [2, 36, 38, 39, 41], "anyth": 3, "apach": [5, 29, 30, 41], "api": [6, 27, 30, 31, 32], "appli": [13, 17, 18, 31, 39], "applic": 5, "apply_gufunc": 30, "approach": [40, 41], "appropri": [3, 40], "ar": [0, 1, 2, 3, 4, 5, 9, 13, 26, 29, 31, 32, 33, 34, 36, 37, 38, 39, 40, 41], "arang": 1, "arbitrari": 11, "arg": [13, 17, 18], "arg_reduct": 30, "argument": [1, 3, 18, 19, 33, 37], "arithmet": 20, "around": [36, 39], "arrai": [2, 3, 10, 13, 14, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 31, 32, 33, 34, 36, 39, 40, 41], "arrang": 31, "array_api": [1, 3, 27, 33, 37], "arrow": 31, "asarrai": [1, 15, 27, 33, 37], "assum": 13, "astyp": 37, "atom": 38, "attempt": [3, 38, 41], "attribut": [5, 6, 11, 12, 31], "auto": [1, 15], "automat": [3, 33, 34, 36, 37, 39, 40], "avail": [3, 11, 27, 36, 38], "averag": 39, "avoid": [31, 41], "aw": [3, 29, 34, 39, 40], "awai": [37, 41], "ax": [13, 31], "axi": [1, 13, 20, 21, 31], "b": [3, 27, 33, 37], "back": [5, 6, 33, 34], "backup": [3, 38, 39], "bag": 32, "base": 39, "baselin": 36, "basi": 36, "basic": [3, 36], "basicconfig": 37, "batch": 3, "batch_siz": 3, "beam": [5, 29, 30, 32, 39, 41], "becaus": [34, 36], "been": [0, 3, 7, 11, 14, 27, 32, 33, 34, 37, 38, 39, 41], "befor": [2, 3, 7, 9, 14, 26, 33, 34, 37, 39, 40], "being": 37, "below": [3, 34, 36, 37], "better": 32, "between": [30, 31, 41], "big": 39, "bin": 33, "block": [5, 17, 18, 31, 33, 39], "blockwis": [2, 30, 32, 37], "blosc": 11, "blue": 5, "bool": [7, 9, 14, 18, 26], "both": [2, 32, 37, 38, 39], "bottom": [5, 31], "bound": [32, 36, 39, 41], "boundari": [18, 31], "box": 33, "branch": 39, "break": 41, "bring": 38, "broadcast": 31, "broadcast_to": 1, "bucket": 40, "budget": 36, "buffer": 33, "build": [2, 34, 36], "built": [2, 5], "bulk": 41, "byte": [11, 13, 19], "c": [3, 27, 29, 33, 37], "calcul": 36, "call": [2, 18, 31, 32, 33, 34, 37, 38], "callabl": [7, 9, 14, 18, 26], "callback": [7, 12, 14, 35], "can": [0, 2, 3, 9, 11, 19, 26, 29, 32, 33, 34, 36, 37, 38, 39, 40, 41], "cancel": 38, "cannot": 39, "carefulli": 41, "carri": 39, "case": [33, 36, 37, 39, 40, 41], "cd": 33, "central": 41, "certain": [38, 40], "challeng": 41, "chang": [3, 8, 31, 32, 33, 37, 39], "characterist": 39, "check": 36, "child": 3, "choic": 33, "choos": 2, "chosen": 5, "chunk": [1, 2, 3, 5, 6, 8, 15, 17, 18, 22, 23, 27, 30, 31, 32, 33, 34, 35, 37, 38, 39, 41], "circumst": 38, "class": [6, 10, 11, 12], "clean": 40, "clear": 40, "clevel": 11, "click": 40, "client": [3, 29, 38], "cloud": [3, 5, 19, 27, 32, 35, 36, 38, 41], "cluster": [34, 39], "clutter": 31, "cname": 11, "code": 3, "coil": [29, 34], "coiled_kwarg": 3, "collect": [24, 33], "colour": 33, "come": [34, 39], "command": 40, "common": [36, 41], "commonli": 37, "compar": 34, "complet": [12, 29, 33, 38, 39], "complex": [36, 39], "compon": 32, "compos": [5, 31], "compress": [11, 33, 36], "compressor": 11, "comput": [3, 5, 9, 10, 11, 16, 19, 20, 24, 25, 26, 27, 30, 32, 34, 35, 36, 37, 38, 39, 40, 41], "compute_arrays_in_parallel": [3, 39], "compute_kwarg": 3, "concat": [1, 31], "concept": 35, "concern": 32, "concurr": [3, 39], "conda": [4, 28], "confid": 36, "configur": [11, 30, 34, 36, 38, 39, 41], "conflict": 29, "conform": 6, "conserv": 36, "consid": [38, 39], "consider": 35, "consist": 35, "consol": 40, "constant": [18, 33], "constraint": 39, "construct": 3, "constructor": 3, "consult": 31, "contain": 36, "continu": 32, "contribut": 30, "control": 41, "conveni": 40, "convent": 27, "convert": [2, 32, 33, 39], "copi": 1, "core": [3, 27, 30, 31, 32, 34, 37], "corearrai": [8, 14, 26], "corner": 33, "correspond": [17, 18, 31, 32], "cost": 40, "could": 39, "coupl": 3, "cov": 29, "cover": [3, 38], "coverag": 1, "cpu": 3, "creat": [0, 4, 15, 19, 31, 32, 33, 37, 40], "creation": [0, 1, 3], "csv": 33, "cube": [0, 2, 3, 4, 5, 27, 28, 29, 31, 32, 33, 34, 35, 36, 37, 38, 40], "cubed_config": 3, "cubed_spec__allowed_mem": 3, "cubed_spec__executor_nam": 3, "cubed_spec__executor_options__runtim": 3, "cubed_spec__executor_options__runtime_memori": 3, "cubed_spec__executor_options__use_backup": 3, "cubed_spec__work_dir": 3, "cumul": 39, "current": [3, 13, 18, 24, 33, 34, 38, 39], "cutdown": 13, "d": [33, 37], "dag": [2, 33], "dai": 40, "dask": [5, 13, 27, 29, 30, 39, 41], "data": [3, 8, 11, 19, 33, 34, 35, 36, 37, 41], "dataflow": [3, 32, 34, 41], "datafram": 32, "dataset": [34, 39], "daunt": 32, "debug": [33, 35], "decompos": 32, "decompress": 36, "dedic": 40, "deeper": 39, "default": [3, 7, 9, 11, 14, 24, 25, 26, 27, 33, 34, 37, 38, 39, 40], "defin": [5, 32, 39], "delai": [32, 39], "deleg": 5, "delet": 35, "deliber": [3, 41], "demo": [28, 30], "depend": [2, 7, 28, 30, 33, 38, 39], "deploi": 3, "depth": [18, 31], "describ": [27, 34, 38], "descript": 3, "design": [3, 30, 39, 41], "desir": 8, "detail": [2, 3, 31, 34, 37], "determin": [33, 38], "develop": 33, "devic": 1, "diagaram": 36, "diagnos": 35, "diagnost": [29, 30, 35, 39], "diagram": 31, "dict": [3, 11, 18], "dictionari": 11, "differ": [3, 29, 30, 31, 37, 38], "difficult": 37, "dill": 29, "dimens": 31, "dimension": 41, "dip": 33, "direct": [2, 33], "directli": [3, 31], "directori": [3, 11, 19, 33, 40], "disabl": 11, "disadvantag": [2, 39], "discuss": [2, 34, 36], "disk": [2, 5, 9, 26, 34], "displai": [9, 26, 33, 39], "disproportion": [38, 39, 41], "distribut": [2, 3, 29, 32, 34, 38, 39, 41], "do": [33, 38, 39, 41], "doc": 3, "document": 3, "doe": [34, 37, 40], "doesn": [3, 9, 26, 34, 36], "don": [2, 3, 33, 34, 40], "done": [33, 39], "donfig": 3, "dot": [9, 26], "doubl": 3, "down": 38, "drop": 31, "drop_axi": 17, "dtype": [1, 2, 5, 17, 18, 21, 31, 36], "due": 29, "duplic": 38, "dure": [10, 36], "e": [4, 37, 39], "each": [2, 3, 18, 31, 33, 36, 38, 39], "eager": [24, 25], "easi": 34, "easier": 32, "echo": 33, "edg": 2, "effect": 37, "effici": [2, 36], "effort": 41, "either": 37, "element": [18, 21, 31, 39], "elementwis": 39, "elemwis": 30, "embarrassingli": 41, "empti": [1, 31], "empty_lik": 1, "enabl": [33, 37, 38, 39], "encapsul": 3, "endpoint": 1, "engin": [2, 41], "enhanc": 39, "enough": 39, "ensur": [33, 36, 40], "enter": 40, "environ": [2, 4, 34, 39], "equal": 39, "equival": [3, 13], "error": 38, "eschew": 41, "especi": [29, 39], "essenti": 27, "estim": 36, "etc": [11, 30], "evalu": 27, "even": [38, 39, 41], "event": [7, 10, 14, 33], "everi": [2, 5, 18, 34, 39], "everyth": 41, "exampl": [2, 3, 5, 11, 27, 30, 31, 34, 36, 37, 38, 39], "exce": [36, 37], "except": [1, 3, 36, 38], "excess": [34, 37], "exclud": 19, "execut": [30, 36, 39], "executor": [5, 7, 11, 14, 19, 24, 25, 27, 29, 30, 33, 35, 36, 38, 40], "executor_nam": [3, 11, 34], "executor_opt": [3, 11], "exist": 39, "expand": 3, "expect": [38, 39], "explain": [33, 37], "explicitli": 2, "export": 3, "expos": 3, "express": 2, "extens": [5, 9, 26], "extent": 41, "extern": 5, "extra": [1, 18], "extra_projected_mem": 13, "ey": 1, "f": 33, "factor": [36, 39], "fail": [3, 38], "fals": [3, 7, 9, 14, 18, 20, 21, 26, 33, 37, 38, 39], "fan": 37, "far": [34, 39], "fashion": 38, "faster": 39, "fault": 38, "featur": [3, 33, 38, 39], "fee": 34, "feed": 39, "few": [1, 5, 38, 39, 41], "fewer": 39, "file": [9, 26, 33, 34, 38, 40], "filenam": [9, 26, 33], "filesystem": 40, "fill_valu": 1, "final": 31, "fine": 32, "first": [31, 33, 36, 37, 39], "firstli": 39, "fit": 34, "five": 5, "fix": 39, "flame": 33, "flamegraph": 33, "flexibl": 32, "float": 23, "float32": 37, "follow": [1, 3, 5, 11, 30, 31, 33, 36, 37, 40], "forg": 29, "form": [11, 40], "format": [9, 26, 33], "formerli": 41, "four": [3, 36], "fourier": 1, "framework": 41, "free": 34, "from": [3, 5, 7, 14, 15, 16, 17, 18, 31, 33, 36, 37, 38, 39, 41], "from_arrai": 0, "from_zarr": 0, "fsspec": [3, 11, 19, 40], "fuctool": 37, "full": [1, 29, 39], "full_lik": 1, "func": [13, 17, 18], "function": [1, 2, 3, 5, 7, 9, 13, 14, 17, 18, 19, 26, 29, 30, 31, 33, 34, 36, 37, 39], "function_end_tstamp": 12, "function_start_tstamp": 12, "functionexecutor": 3, "functool": 37, "fundament": [31, 32], "further": [36, 41], "fuse": [33, 37, 39], "fusion": [2, 7, 9, 14, 26, 35], "futur": 3, "g": 39, "gain": 41, "gb": 11, "gc": 34, "gcf": 39, "gcp": [3, 29], "gcsf": 29, "gener": [5, 13, 30, 31, 33, 36, 39, 41], "get": [4, 30, 34, 35, 37, 39, 40], "github": 4, "given": [3, 19, 21, 36, 39], "global": 38, "go": [5, 33, 41], "good": [3, 11, 36, 39], "googl": [3, 32, 34, 38, 39, 40, 41], "grain": 32, "graph": [2, 7, 9, 14, 26, 32, 33, 37], "graphic": 33, "graphviz": 29, "greater": 36, "green": [5, 31], "group": [16, 25], "guarante": [30, 34, 39, 41], "guid": [19, 28, 30], "ha": [2, 3, 11, 27, 29, 31, 32, 33, 34, 36, 37, 38, 39, 41], "had": 34, "hadoop": 41, "half": 23, "hand": 37, "handi": 3, "handl": [18, 34, 39], "hard": 3, "have": [0, 2, 3, 7, 14, 28, 32, 33, 34, 36, 37, 38, 39], "head": 4, "heavili": [5, 32], "help": [35, 36, 37], "henc": [38, 39], "here": [3, 31, 33, 37, 39, 41], "hidden": [9, 26], "high": [2, 28, 32, 36, 41], "higher": [32, 37], "highest": [34, 36], "hist": 33, "historycallback": 33, "hold": 39, "horizont": [33, 39], "how": [2, 3, 18, 31, 33, 36, 37, 39, 40, 41], "howev": [2, 13, 41], "html": 33, "i": [1, 2, 3, 5, 7, 9, 11, 13, 14, 18, 19, 24, 25, 26, 27, 30, 31, 32, 33, 35, 36, 37, 38, 39, 40, 41], "id": [11, 33], "ideal": [33, 39], "idempot": 38, "ident": 38, "ignor": 20, "illustr": 36, "imag": [3, 9, 26, 33], "immedi": [24, 25, 38], "implement": [1, 5, 19, 24, 27, 30, 31, 32, 41], "implicitli": 2, "import": [3, 9, 26, 27, 33, 34, 36, 37, 39, 40], "importantli": 38, "impos": 37, "improv": [33, 36, 39, 41], "includ": [0, 3, 5, 9, 11, 26, 32], "incur": 40, "independ": 39, "index": 31, "indic": 3, "individu": [3, 37], "infinit": 39, "influenc": 32, "inform": [12, 13, 33, 36, 37], "inherit": 5, "input": [3, 16, 17, 18, 31, 35, 36, 39], "instal": [4, 19, 28, 30, 33], "instanc": [3, 39], "instanti": 3, "instead": 41, "instruct": 40, "int": [11, 18, 19], "integ": 37, "integr": [30, 32], "intend": 34, "interact": 27, "interest": 41, "intermedi": [2, 3, 7, 11, 14, 19, 33, 35, 37, 39], "intern": 33, "interv": 23, "interven": 32, "intro": 30, "introduc": 35, "introduct": 28, "invok": [2, 3], "involv": [2, 4, 33, 41], "io": [5, 30, 37, 38], "ipython": [9, 26], "issu": [1, 33], "iter": 39, "its": [5, 8, 18, 31, 33, 36, 37, 39], "itself": 36, "jinja2": 29, "job": 40, "jpeg": [9, 26], "jpg": [9, 26], "jupyt": 33, "just": [27, 31, 33, 37], "k": 1, "kb": 11, "keepdim": [13, 20, 21], "kei": [3, 38], "keyword": [1, 3, 18, 19], "kill": 3, "kind": 37, "known": [2, 36, 39, 41], "kwarg": [7, 13, 14, 17, 18, 19, 24, 25], "l": 33, "label": 33, "lambda": [34, 39], "larg": [34, 36, 38, 39, 41], "larger": [34, 39], "last": 41, "later": 33, "launch": 38, "layer": [5, 32], "lazi": [2, 27], "lead": [39, 41], "least": 36, "less": 32, "lesser": 41, "let": 5, "level": [2, 28, 32, 37, 41], "librari": [30, 32], "lifecycl": 40, "like": [3, 5, 15, 24, 27, 31, 32, 34, 36, 37, 40, 41], "lilac": 33, "limit": [13, 37, 39], "line": 33, "linear": 39, "linspac": 1, "list": [1, 3, 7, 14, 18, 29], "lithop": [5, 19, 29, 30, 33, 34, 41], "live": 3, "ll": 27, "load": [16, 33], "local": [5, 11, 27, 33, 35, 36, 40], "locat": 3, "log": 37, "logic": 39, "long": [33, 37, 39], "longer": [3, 33, 37, 38, 39], "look": [3, 28, 31, 37], "lot": [40, 41], "lz4": 11, "m": [29, 33], "machin": [27, 35, 36, 39], "made": [3, 41], "magic": 39, "mai": [2, 5, 18, 31, 34, 37, 39, 40], "main": [2, 3], "mainli": 3, "maintain": 39, "major": 41, "make": [2, 32, 36, 39], "manag": [3, 36, 41], "mani": [29, 31, 37], "manipul": 1, "manual": 40, "map": [2, 7, 9, 14, 18, 26, 31, 35, 39], "map_block": [5, 30], "map_direct": [30, 37], "map_overlap": 30, "mapreduc": 41, "mark": [9, 26], "match": [31, 37, 39], "materi": [2, 33], "matplotlib": 29, "matur": [32, 34], "max_tasks_per_child": 3, "max_total_num_input_block": 37, "max_total_source_arrai": 37, "max_work": [3, 39], "maximum": [3, 30, 36, 37, 39], "mb": 11, "mean": [3, 5, 20, 33, 36, 37, 38, 39], "measur": [19, 36, 39], "measure_reserved_mem": [36, 39], "memori": [3, 11, 13, 19, 32, 33, 34, 35, 39, 41], "memrai": 35, "messag": 38, "method": [6, 10, 11, 12, 33, 37], "middl": 31, "might": 39, "minim": 29, "minimum": 39, "minut": 34, "mitig": [3, 38], "mock": 29, "modal": [5, 19, 29, 30, 34, 39], "mode": 22, "model": [2, 5, 41], "monitor": 33, "more": [13, 27, 31, 32, 33, 34, 36, 37, 38, 39], "most": [3, 34, 35, 41], "much": [33, 34, 36, 39], "multi": 27, "multipl": [2, 13, 14, 17, 18, 26, 30, 31, 33, 35], "multiple_inputs_optimize_dag": 37, "must": [3, 19, 24, 37, 38], "n": [3, 41], "n_col": 1, "n_row": 1, "name": [4, 6, 9, 12, 26, 32, 40], "nan": [20, 21], "natur": [2, 32, 36], "nbyte": 37, "necessari": 39, "need": [2, 5, 13, 31, 33, 36, 38, 39, 40], "neg": 37, "neighbor": 18, "neither": 3, "nest": 3, "never": 36, "new": [5, 13, 32, 33, 36, 39], "new_axi": 17, "newer": 32, "next": 39, "node": [2, 33, 41], "non": [3, 11, 30, 36, 39], "none": [1, 3, 7, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 37], "note": [3, 18, 24, 25, 31, 33, 34, 37, 38, 40], "notebook": [9, 26, 33], "noth": [27, 39], "notic": [3, 27, 37], "now": 37, "np": 18, "num": [1, 37], "num_task": 12, "number": [3, 5, 18, 30, 31, 33, 34, 38, 39], "numblock": 31, "numcodec": 11, "numpi": [2, 5, 27, 32], "numpy_groupi": 29, "o": 33, "obj": 1, "object": [2, 10, 15, 19, 24, 27, 40], "observ": 39, "off": [33, 37, 38, 39], "offer": 39, "often": 41, "old": 40, "onc": [14, 33, 38], "one": [0, 2, 3, 18, 31, 33, 37, 38, 39, 40, 41], "ones": [1, 32, 37], "ones_lik": 1, "onli": [2, 3, 7, 9, 14, 18, 26, 31, 32, 33, 34, 37, 38, 39, 41], "op": [33, 37], "open": 23, "oper": [2, 9, 19, 24, 25, 26, 30, 32, 33, 36, 37, 38, 39, 40, 41], "opportun": 2, "opt_fn": 37, "optim": [2, 7, 9, 14, 26, 30, 33, 35, 39, 41], "optimize_funct": [7, 9, 14, 26, 37], "optimize_graph": [7, 9, 14, 26, 33, 37], "option": [7, 9, 11, 14, 16, 19, 24, 25, 26, 28, 33], "optional_depend": 29, "orang": [31, 33], "order": [31, 36, 39], "origin": 38, "other": [3, 5, 35, 37], "otherwis": [7, 9, 14, 26, 37], "out": [33, 34, 35, 39, 40], "outer": 31, "output": [2, 9, 13, 18, 25, 26, 31, 33, 36, 37, 38, 39], "output_dtyp": 13, "output_s": 13, "outsid": 3, "over": [4, 18, 21, 31, 40], "overal": [33, 37, 38, 39], "overhead": 34, "overlap": 18, "overview": 28, "own": 39, "packag": [3, 19, 29, 32], "pad_width": 22, "page": [1, 3, 31, 39], "pair": 2, "panda": 29, "paragraph": 38, "parallel": [2, 3, 32, 39, 41], "paramet": [7, 8, 9, 11, 13, 14, 16, 18, 19, 24, 25, 26, 39], "part": [31, 34, 39], "partial": 37, "partial_reduc": 31, "particular": [3, 34, 36, 37], "pass": [3, 11, 18, 19, 31, 33, 39, 41], "path": [3, 11, 16, 19, 25], "pdf": [9, 26], "peak": 19, "peak_measured_mem": 36, "peak_measured_mem_end": 12, "peak_measured_mem_start": 12, "per": [13, 39], "perfect": 33, "perform": [7, 9, 14, 26, 31, 33, 35, 36, 37, 41], "period": 40, "persist": [39, 40, 41], "phase": 36, "pick": 37, "pink": 31, "pip": [4, 28, 33], "plan": [6, 30, 32, 35, 36, 37, 39], "pleas": 4, "plenti": 36, "plot": 33, "png": [9, 26], "possibl": [3, 28, 36, 37, 39, 41], "potenti": 39, "power": 37, "practic": 35, "pre": 38, "preced": 37, "precis": 36, "predecessor": 37, "prefac": 35, "preserv": 31, "previou": [30, 37, 38], "primit": [2, 30, 31, 32, 37], "print": 27, "problem": [39, 41], "process": [2, 5, 19, 24, 25, 32, 33, 34, 36, 38, 39, 41], "processor": 39, "processpoolexecutor": 3, "produc": [9, 18, 26, 33, 36], "profil": 33, "program": [3, 41], "project": [5, 29, 30, 33, 35], "projected_mem": 36, "properli": 36, "properti": 3, "proportion": 39, "provid": [5, 9, 26, 33, 34, 36, 38, 41], "provis": 39, "public": 5, "purpos": 41, "put": [36, 41], "py": 33, "pydot": 29, "pyproject": 29, "pytest": 29, "python": [0, 3, 4, 5, 6, 13, 19, 24, 25, 27, 29, 30, 32, 33, 34, 36], "pywren": 41, "quickli": [28, 34, 38], "r": 4, "rais": 36, "ran": 36, "random": [3, 5, 30, 33], "rather": [31, 40], "re": 39, "read": [31, 32, 33, 36, 37, 38, 39, 41], "readm": 27, "reason": [3, 32, 36, 37, 39], "receiv": 10, "rechunk": [2, 5, 30, 32, 39, 41], "recommend": 34, "recomput": [7, 14], "red": 5, "reduc": [31, 37, 39], "reduct": [30, 36, 37, 39], "refer": [13, 30, 34, 39], "regard": 31, "region": [34, 40], "regular": 40, "rel": 34, "relat": 30, "reli": 38, "reliabl": [30, 34, 35], "remot": 34, "remov": [37, 40], "render": [9, 26, 33], "report": [19, 36], "repres": [31, 33], "represent": [2, 39], "req": 29, "requir": [2, 4, 19, 33, 34, 36, 39, 41], "reserv": [3, 11, 19, 35], "reserved_mem": [3, 11, 19, 36], "resourc": [11, 27, 39], "restrict": 39, "result": [27, 31, 33, 37], "resum": [7, 14], "retain": 33, "retri": [3, 35], "return": [8, 9, 16, 19, 21, 23, 26, 31], "reus": 3, "rich": [29, 33], "richprogressbar": 33, "right": 33, "rm": 40, "room": 36, "round": [31, 33], "rule": [36, 40], "run": [2, 3, 7, 11, 14, 19, 24, 25, 27, 29, 32, 33, 34, 36, 37, 38, 39, 40, 41], "runner": 32, "runtim": [2, 3, 19, 24, 25, 30, 32, 34, 36, 40, 41], "runtime_memori": 3, "s3": [3, 34, 38, 40], "s3f": 29, "same": [3, 31, 37, 39, 40], "save": [24, 25, 33], "scale": [2, 30, 33, 34, 35, 38, 41], "scenario": 39, "schema": 33, "seaborn": 29, "second": [3, 31, 34], "section": 38, "see": [2, 3, 27, 29, 33, 37, 39], "select": [31, 40], "send": [7, 14], "sent": 31, "separ": [29, 33, 39, 40], "serverless": [2, 32, 39, 41], "servic": [35, 36, 39, 40, 41], "set": [2, 3, 7, 11, 13, 14, 19, 29, 34, 35, 36, 38, 39, 40], "sever": 32, "shape": [1, 5, 8, 18, 31], "share": [3, 18], "short": 40, "should": [11, 18, 35, 36, 37, 39, 40], "show": [9, 26, 31, 33, 34], "show_hidden": [9, 26], "shown": [31, 33, 37], "shuffl": [2, 11, 39, 41], "side": 31, "sign": [34, 39], "signatur": 13, "signifi": 33, "signific": 39, "similar": [13, 32], "similarli": [31, 33], "simpl": [3, 27, 33, 34], "simplest": [31, 37, 39], "simpli": 33, "simultan": 39, "sinc": [2, 3, 31, 32, 33, 34, 36, 37, 38, 41], "singl": [27, 31, 32, 33, 35, 38], "size": [2, 23, 31, 33, 35, 39, 41], "slide": 30, "slow": [2, 37, 38], "slowli": 39, "small": [33, 34, 37], "smaller": 36, "so": [3, 27, 32, 33, 34, 36, 37, 38, 39, 40], "solut": 39, "some": [18, 32, 36, 37, 38, 39], "sometim": [34, 37], "sourc": 24, "space": 40, "spark": 41, "spec": [1, 6, 7, 14, 15, 16, 17, 19, 23, 27, 33, 34, 36, 40], "specif": [11, 30, 39], "specifi": [3, 11, 18, 19, 20, 27, 35, 37, 40], "specul": 38, "speed": [37, 39], "spent": [33, 41], "split_everi": [20, 21], "squar": 33, "squeez": 31, "stage": 37, "standard": [0, 6, 27, 30, 32], "standardis": 30, "start": [1, 27, 30, 34, 37, 39, 41], "startup": [33, 34, 39], "stat": 33, "stateless": [5, 41], "statist": 33, "statu": 1, "step": [1, 33], "stick": 39, "stop": 1, "storag": [2, 6, 11, 16, 25, 30, 35, 38, 39, 41], "storage_opt": 11, "store": [3, 5, 11, 16, 19, 25, 33, 39, 40], "str": [9, 11, 19, 26], "straggler": [3, 33, 35, 39], "straightforward": [2, 34], "string": [16, 25], "strong": [35, 39], "structur": 31, "style": 3, "submit": 3, "subset": [5, 27], "success": 41, "suffici": 39, "suggest": 39, "suit": 33, "sum": [21, 36], "support": [1, 3, 13, 18, 31, 32, 37], "sure": [39, 40], "svg": [9, 26, 33], "system": [5, 19, 38, 39, 40, 41], "t": [2, 3, 7, 9, 14, 26, 32, 33, 34, 36, 37, 40], "tab": 40, "tail": 33, "take": [3, 33, 38, 39], "taken": 38, "target": [24, 41], "task": [2, 3, 5, 11, 12, 13, 19, 32, 33, 34, 36, 37, 38, 39], "task_create_tstamp": 12, "task_result_tstamp": 12, "tb": 11, "tell": 13, "temp": [3, 34, 40], "tempor": 33, "temporari": [3, 40], "ten": 36, "terabyt": 39, "test": [3, 29, 33, 34], "test_mem_util": 33, "than": [3, 13, 31, 32, 36, 37, 38, 39, 40], "thei": [3, 31, 32, 33, 36, 37, 39], "them": [2, 31, 33, 38, 40], "theme": 41, "theoret": 35, "therebi": [39, 41], "therefor": 34, "thi": [2, 3, 5, 7, 8, 9, 11, 13, 18, 19, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41], "thing": 3, "those": 39, "though": [39, 41], "thread": [34, 39], "threadpoolexecutor": 3, "three": [3, 31, 32, 33, 37, 38], "through": [3, 5], "throughput": 39, "thumb": 36, "tightli": [36, 41], "time": [2, 3, 31, 33, 34, 36, 37, 38, 39, 40, 41], "timeline_viz": 33, "timelinevisualizationcallback": 33, "timeout": [3, 35], "tini": [19, 33], "tip": 35, "tmp": 27, "tmpdir": 40, "to_zarr": 33, "togeth": [2, 37], "toler": 38, "toml": 29, "tomwhit": [3, 34, 40], "too": [31, 37], "took": 33, "tool": 33, "top": [5, 31], "total": [3, 11, 38, 39], "tqdm": [29, 33], "tqdmprogressbar": 33, "trace": 33, "track": [1, 33], "traction": 41, "transfer": 34, "transform": 1, "travers": 2, "treat": 21, "tree": 30, "tree_reduc": 31, "trigger": 2, "trim": 18, "trivial": [19, 39], "true": [1, 3, 7, 9, 14, 20, 21, 26, 37, 39], "tune": [39, 41], "tupl": [8, 18], "turn": [3, 31, 33, 37, 38, 39], "twice": 33, "two": [5, 31, 33, 39], "txt": 4, "type": [5, 8, 9, 16, 18, 19, 24, 25, 26, 33, 35], "typic": 40, "ufunc": 13, "under": [3, 33], "underli": 5, "underscor": 3, "understand": [32, 33, 37, 39, 41], "undoubtedli": 41, "unit": [5, 11], "unnecessari": 40, "unoptim": 37, "unpredict": 41, "until": 31, "up": [2, 3, 34, 36, 37, 38, 39, 40], "updat": 38, "upgrad": 39, "upper": 36, "url": [3, 11, 19, 40], "us": [2, 3, 5, 7, 9, 11, 14, 16, 19, 24, 25, 26, 27, 28, 29, 30, 31, 32, 35, 36, 37, 39, 40, 41], "usag": [13, 30, 36, 39, 41], "use_backup": [3, 38, 39], "use_new_impl": [20, 21], "user": [3, 32, 34, 41], "usual": 33, "util": [32, 36], "v": 35, "valu": [3, 11, 18, 31, 36, 37], "vari": [19, 39], "varieti": [3, 32, 33, 34], "variou": 36, "vector": 13, "veri": [2, 4, 33, 34, 39, 41], "versa": 39, "version": [1, 13, 19], "versu": 39, "vertic": [33, 39], "via": [30, 41], "vice": 39, "view": 33, "violat": 39, "visual": [32, 35, 37], "wa": [5, 32, 33], "wai": [3, 31, 33, 38, 39], "wait": 39, "want": [33, 37], "wasn": 32, "we": [3, 27, 31, 33, 34, 36, 37, 39], "weak": 39, "welcom": 4, "well": [5, 32, 33, 36], "what": [32, 33, 37], "whatev": [31, 36], "when": [2, 3, 11, 19, 32, 33, 34, 36, 37, 38, 39, 40], "where": [2, 11, 32, 37, 39], "wherea": [32, 39], "whether": [3, 18], "which": [2, 3, 5, 9, 11, 26, 27, 29, 31, 32, 33, 35, 36, 37, 39, 40, 41], "while": [2, 7, 14, 31, 33, 36, 39], "white": 31, "who": 39, "whole": 38, "why": [30, 37], "widespread": 41, "within": [3, 36], "without": [8, 31, 39], "won": [7, 14], "word": 39, "work": [19, 30, 31, 33, 34, 36, 38, 40, 41], "work_dir": [3, 11, 19, 27, 34, 40], "worker": [3, 11, 33, 34, 36, 39, 41], "would": [33, 36, 37, 39, 40], "write": [2, 9, 24, 26, 33, 34, 36, 38, 39, 40, 41], "written": [2, 33, 37, 38], "x": [1, 15, 20, 21, 22, 24, 25], "xarrai": 30, "xp": [3, 27, 33, 37], "yaml": 34, "year": [32, 41], "yet": [0, 27], "you": [3, 13, 28, 29, 32, 33, 34, 35, 36, 37, 38, 39, 40], "your": [3, 33, 34, 36, 39], "zappi": 32, "zarr": [2, 5, 6, 11, 16, 24, 25, 30, 33, 34, 36, 38, 39, 40, 41], "zarr_compressor": 11, "zarrai": 6, "zero": [1, 21], "zeros_lik": 1}, "titles": ["API Reference", "Python Array API", "Computation", "Configuration", "Contributing", "Design", "cubed.Array", "cubed.Array.compute", "cubed.Array.rechunk", "cubed.Array.visualize", "cubed.Callback", "cubed.Spec", "cubed.TaskEndEvent", "cubed.apply_gufunc", "cubed.compute", "cubed.from_array", "cubed.from_zarr", "cubed.map_blocks", "cubed.map_overlap", "cubed.measure_reserved_mem", "cubed.nanmean", "cubed.nansum", "cubed.pad", "cubed.random.random", "cubed.store", "cubed.to_zarr", "cubed.visualize", "Demo", "Getting Started", "Installation", "Cubed", "Operations", "Related Projects", "Diagnostics", "Executors", "User Guide", "Memory", "Optimization", "Reliability", "Scaling", "Storage", "Why Cubed?"], "titleterms": {"For": 30, "advanc": 37, "allow": 36, "an": 34, "api": [0, 1, 5], "apply_gufunc": 13, "arg_reduct": 31, "arrai": [0, 1, 5, 6, 7, 8, 9, 30, 37], "articl": 30, "bar": 33, "beam": 3, "between": 1, "block": 37, "blockwis": 31, "bound": 30, "calcul": 39, "callback": [10, 33], "chunk": [0, 36], "cloud": [34, 39, 40], "coil": 3, "comput": [2, 7, 14, 33], "conda": 29, "configur": 3, "consider": 39, "consist": 38, "contribut": 4, "core": 5, "cube": [1, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 30, 39, 41], "dask": [3, 32], "data": 40, "debug": 37, "delet": 40, "demo": 27, "depend": [29, 31], "design": 5, "develop": [4, 30], "diagnos": 39, "diagnost": 33, "differ": [1, 39], "dimension": 30, "distribut": 30, "document": 30, "elemwis": 31, "environ": 3, "exampl": 33, "execut": 2, "executor": [3, 34, 39], "file": 3, "from_arrai": 15, "from_zarr": 16, "function": 0, "fusion": 37, "gener": 0, "get": 28, "guid": 35, "histori": 33, "i": 34, "input": 37, "instal": 29, "intermedi": 40, "io": 0, "lithop": 3, "local": 34, "machin": 34, "map": 37, "map_block": [17, 31], "map_direct": 31, "map_overlap": 18, "measure_reserved_mem": 19, "memori": [2, 30, 36], "memrai": 33, "modal": 3, "multi": 39, "multipl": 37, "n": 30, "nanmean": 20, "nansum": 21, "non": 0, "number": [0, 37], "object": 3, "oper": [5, 31], "optim": 37, "option": [3, 29], "other": 39, "pad": 22, "perform": 39, "pip": 29, "pipelin": 39, "plan": [2, 33], "practic": 39, "prefac": 39, "previou": 32, "primit": 5, "process": [3, 30], "progress": 33, "project": [32, 36], "provid": 39, "python": 1, "random": [0, 23], "rechunk": [8, 31], "reduct": 31, "refer": [0, 3], "relat": 32, "reliabl": 38, "reserv": 36, "retri": 38, "runtim": [0, 5], "scale": 39, "serverless": 30, "servic": 34, "set": 37, "should": 34, "singl": [3, 34, 39], "size": 36, "sourc": 37, "spec": [3, 11], "specif": [0, 3], "specifi": 34, "standard": 1, "standardis": 0, "start": 28, "step": 39, "storag": [5, 40], "store": 24, "straggler": 38, "strong": 38, "taskendev": 12, "theoret": 39, "thread": 3, "timelin": 33, "timeout": 38, "tip": 39, "to_zarr": 25, "total": 37, "tree": 31, "type": 39, "us": [33, 34], "user": [30, 35], "v": 39, "variabl": 3, "visual": [9, 26, 33], "which": 34, "why": 41, "work": 32, "xarrai": 32, "yaml": 3}}) \ No newline at end of file +Search.setIndex({"alltitles": {"API Reference": [[0, null]], "Advanced settings": [[37, "advanced-settings"]], "Allowed memory": [[36, "allowed-memory"]], "Array": [[0, "array"]], "Array API": [[5, "array-api"]], "Articles": [[30, "articles"]], "Bounded-memory serverless distributed N-dimensional array processing": [[30, "bounded-memory-serverless-distributed-n-dimensional-array-processing"]], "Callbacks": [[33, "callbacks"]], "Chunk sizes": [[36, "chunk-sizes"]], "Chunk-specific functions": [[0, "chunk-specific-functions"]], "Cloud storage": [[40, "cloud-storage"]], "Computation": [[2, null]], "Conda": [[29, "conda"]], "Configuration": [[3, null]], "Contributing": [[4, null]], "Core operations": [[5, "core-operations"]], "Cubed": [[30, null]], "Dask": [[32, "dask"]], "Debugging optimization": [[37, "debugging-optimization"]], "Deleting intermediate data": [[40, "deleting-intermediate-data"]], "Demo": [[27, null]], "Dependency Tree": [[31, "dependency-tree"]], "Design": [[5, null]], "Development": [[4, "development"]], "Diagnosing Performance": [[39, "diagnosing-performance"]], "Diagnostics": [[33, null]], "Differences between Cubed and the standard": [[1, "differences-between-cubed-and-the-standard"]], "Different Cloud Providers": [[39, "different-cloud-providers"]], "Different Executors": [[39, "different-executors"]], "Documentation": [[30, "documentation"]], "Environment variables": [[3, "environment-variables"]], "Examples in use": [[33, "examples-in-use"]], "Execution": [[2, "execution"]], "Executor options": [[3, "executor-options"]], "Executors": [[34, null]], "For developers": [[30, null]], "For users": [[30, null]], "Getting Started": [[28, null]], "History": [[33, "history"]], "IO": [[0, "io"]], "Installation": [[29, null]], "Local single-machine executors": [[34, "local-single-machine-executors"]], "Map fusion": [[37, "map-fusion"]], "Memory": [[2, "memory"], [36, null]], "Memray": [[33, "memray"]], "Multi-pipeline Calculation": [[39, "multi-pipeline-calculation"]], "Multi-step Calculation": [[39, "multi-step-calculation"]], "Multiple-input fusion": [[37, "multiple-input-fusion"]], "Non-standardised functions": [[0, "non-standardised-functions"]], "Operations": [[31, null]], "Optimization": [[37, null]], "Optional dependencies": [[29, "optional-dependencies"]], "Other Performance Considerations": [[39, "other-performance-considerations"]], "Pip": [[29, "pip"]], "Plan": [[2, "plan"]], "Preface: Types of Scaling": [[39, "preface-types-of-scaling"]], "Previous work": [[32, "previous-work"]], "Primitive operations": [[5, "primitive-operations"]], "Progress bar": [[33, "progress-bar"]], "Projected memory": [[36, "projected-memory"]], "Python Array API": [[1, null]], "Random number generation": [[0, "random-number-generation"]], "Reference": [[3, "reference"]], "Related Projects": [[32, null]], "Reliability": [[38, null]], "Reserved memory": [[36, "reserved-memory"]], "Retries": [[38, "retries"]], "Runtime": [[0, "runtime"], [5, "runtime"]], "Scaling": [[39, null]], "Single-step Calculation": [[39, "single-step-calculation"]], "Spec object": [[3, "spec-object"]], "Spec options": [[3, "spec-options"]], "Specification": [[3, "specification"]], "Specifying an executor": [[34, "specifying-an-executor"]], "Storage": [[5, "storage"], [40, null]], "Stragglers": [[38, "stragglers"]], "Strong consistency": [[38, "strong-consistency"]], "Theoretical vs Practical Scaling of Cubed": [[39, "theoretical-vs-practical-scaling-of-cubed"]], "Timeline": [[33, "timeline"]], "Timeouts": [[38, "timeouts"]], "Tips": [[39, "tips"]], "Total number of input blocks": [[37, "total-number-of-input-blocks"]], "Total number of source arrays": [[37, "total-number-of-source-arrays"]], "User Guide": [[35, null]], "Visualize the computation plan": [[33, "visualize-the-computation-plan"]], "Which cloud service executor should I use?": [[34, "which-cloud-service-executor-should-i-use"]], "Why Cubed?": [[41, null]], "Xarray": [[32, "xarray"]], "YAML file": [[3, "yaml-file"]], "beam": [[3, "beam"]], "blockwise": [[31, "blockwise"]], "coiled": [[3, "coiled"]], "cubed.Array": [[6, null]], "cubed.Array.compute": [[7, null]], "cubed.Array.rechunk": [[8, null]], "cubed.Array.visualize": [[9, null]], "cubed.Callback": [[10, null]], "cubed.Spec": [[11, null]], "cubed.TaskEndEvent": [[12, null]], "cubed.apply_gufunc": [[13, null]], "cubed.compute": [[14, null]], "cubed.from_array": [[15, null]], "cubed.from_zarr": [[16, null]], "cubed.map_blocks": [[17, null]], "cubed.map_overlap": [[18, null]], "cubed.measure_reserved_mem": [[19, null]], "cubed.nanmean": [[20, null]], "cubed.nansum": [[21, null]], "cubed.pad": [[22, null]], "cubed.random.random": [[23, null]], "cubed.store": [[24, null]], "cubed.to_zarr": [[25, null]], "cubed.visualize": [[26, null]], "dask": [[3, "dask"]], "elemwise": [[31, "elemwise"]], "lithops": [[3, "lithops"]], "map_blocks": [[31, "map-blocks"]], "map_direct": [[31, "map-direct"]], "modal": [[3, "modal"]], "processes": [[3, "processes"]], "rechunk": [[31, "rechunk"]], "reduction and arg_reduction": [[31, "reduction-and-arg-reduction"]], "single-threaded": [[3, "single-threaded"]], "threads": [[3, "threads"]]}, "docnames": ["api", "array-api", "computation", "configuration", "contributing", "design", "generated/cubed.Array", "generated/cubed.Array.compute", "generated/cubed.Array.rechunk", "generated/cubed.Array.visualize", "generated/cubed.Callback", "generated/cubed.Spec", "generated/cubed.TaskEndEvent", "generated/cubed.apply_gufunc", "generated/cubed.compute", "generated/cubed.from_array", "generated/cubed.from_zarr", "generated/cubed.map_blocks", "generated/cubed.map_overlap", "generated/cubed.measure_reserved_mem", "generated/cubed.nanmean", "generated/cubed.nansum", "generated/cubed.pad", "generated/cubed.random.random", "generated/cubed.store", "generated/cubed.to_zarr", "generated/cubed.visualize", "getting-started/demo", "getting-started/index", "getting-started/installation", "index", "operations", "related-projects", "user-guide/diagnostics", "user-guide/executors", "user-guide/index", "user-guide/memory", "user-guide/optimization", "user-guide/reliability", "user-guide/scaling", "user-guide/storage", "why-cubed"], "envversion": {"sphinx": 63, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["api.rst", "array-api.md", "computation.md", "configuration.md", "contributing.md", "design.md", "generated/cubed.Array.rst", "generated/cubed.Array.compute.rst", "generated/cubed.Array.rechunk.rst", "generated/cubed.Array.visualize.rst", "generated/cubed.Callback.rst", "generated/cubed.Spec.rst", "generated/cubed.TaskEndEvent.rst", "generated/cubed.apply_gufunc.rst", "generated/cubed.compute.rst", "generated/cubed.from_array.rst", "generated/cubed.from_zarr.rst", "generated/cubed.map_blocks.rst", "generated/cubed.map_overlap.rst", "generated/cubed.measure_reserved_mem.rst", "generated/cubed.nanmean.rst", "generated/cubed.nansum.rst", "generated/cubed.pad.rst", "generated/cubed.random.random.rst", "generated/cubed.store.rst", "generated/cubed.to_zarr.rst", "generated/cubed.visualize.rst", "getting-started/demo.md", "getting-started/index.md", "getting-started/installation.md", "index.md", "operations.md", "related-projects.md", "user-guide/diagnostics.md", "user-guide/executors.md", "user-guide/index.md", "user-guide/memory.md", "user-guide/optimization.md", "user-guide/reliability.md", "user-guide/scaling.md", "user-guide/storage.md", "why-cubed.md"], "indexentries": {"__init__() (cubed.array method)": [[6, "cubed.Array.__init__", false]], "__init__() (cubed.callback method)": [[10, "cubed.Callback.__init__", false]], "__init__() (cubed.spec method)": [[11, "cubed.Spec.__init__", false]], "__init__() (cubed.taskendevent method)": [[12, "cubed.TaskEndEvent.__init__", false]], "apply_gufunc() (in module cubed)": [[13, "cubed.apply_gufunc", false]], "arange() (in module cubed.array_api)": [[1, "cubed.array_api.arange", false]], "array (class in cubed)": [[6, "cubed.Array", false]], "asarray() (in module cubed.array_api)": [[1, "cubed.array_api.asarray", false]], "broadcast_to() (in module cubed.array_api)": [[1, "cubed.array_api.broadcast_to", false]], "callback (class in cubed)": [[10, "cubed.Callback", false]], "compute() (cubed.array method)": [[7, "cubed.Array.compute", false]], "compute() (in module cubed)": [[14, "cubed.compute", false]], "concat() (in module cubed.array_api)": [[1, "cubed.array_api.concat", false]], "empty() (in module cubed.array_api)": [[1, "cubed.array_api.empty", false]], "empty_like() (in module cubed.array_api)": [[1, "cubed.array_api.empty_like", false]], "eye() (in module cubed.array_api)": [[1, "cubed.array_api.eye", false]], "from_array() (in module cubed)": [[15, "cubed.from_array", false]], "from_zarr() (in module cubed)": [[16, "cubed.from_zarr", false]], "full() (in module cubed.array_api)": [[1, "cubed.array_api.full", false]], "full_like() (in module cubed.array_api)": [[1, "cubed.array_api.full_like", false]], "linspace() (in module cubed.array_api)": [[1, "cubed.array_api.linspace", false]], "map_blocks() (in module cubed)": [[17, "cubed.map_blocks", false]], "map_overlap() (in module cubed)": [[18, "cubed.map_overlap", false]], "measure_reserved_mem() (in module cubed)": [[19, "cubed.measure_reserved_mem", false]], "nanmean() (in module cubed)": [[20, "cubed.nanmean", false]], "nansum() (in module cubed)": [[21, "cubed.nansum", false]], "ones() (in module cubed.array_api)": [[1, "cubed.array_api.ones", false]], "ones_like() (in module cubed.array_api)": [[1, "cubed.array_api.ones_like", false]], "pad() (in module cubed)": [[22, "cubed.pad", false]], "random() (in module cubed.random)": [[23, "cubed.random.random", false]], "rechunk() (cubed.array method)": [[8, "cubed.Array.rechunk", false]], "spec (class in cubed)": [[11, "cubed.Spec", false]], "store() (in module cubed)": [[24, "cubed.store", false]], "taskendevent (class in cubed)": [[12, "cubed.TaskEndEvent", false]], "to_zarr() (in module cubed)": [[25, "cubed.to_zarr", false]], "visualize() (cubed.array method)": [[9, "cubed.Array.visualize", false]], "visualize() (in module cubed)": [[26, "cubed.visualize", false]], "zeros() (in module cubed.array_api)": [[1, "cubed.array_api.zeros", false]], "zeros_like() (in module cubed.array_api)": [[1, "cubed.array_api.zeros_like", false]]}, "objects": {"cubed": [[6, 0, 1, "", "Array"], [10, 0, 1, "", "Callback"], [11, 0, 1, "", "Spec"], [12, 0, 1, "", "TaskEndEvent"], [13, 2, 1, "", "apply_gufunc"], [14, 2, 1, "", "compute"], [15, 2, 1, "", "from_array"], [16, 2, 1, "", "from_zarr"], [17, 2, 1, "", "map_blocks"], [18, 2, 1, "", "map_overlap"], [19, 2, 1, "", "measure_reserved_mem"], [20, 2, 1, "", "nanmean"], [21, 2, 1, "", "nansum"], [22, 2, 1, "", "pad"], [24, 2, 1, "", "store"], [25, 2, 1, "", "to_zarr"], [26, 2, 1, "", "visualize"]], "cubed.Array": [[6, 1, 1, "", "__init__"], [7, 1, 1, "", "compute"], [8, 1, 1, "", "rechunk"], [9, 1, 1, "", "visualize"]], "cubed.Callback": [[10, 1, 1, "", "__init__"]], "cubed.Spec": [[11, 1, 1, "", "__init__"]], "cubed.TaskEndEvent": [[12, 1, 1, "", "__init__"]], "cubed.array_api": [[1, 2, 1, "", "arange"], [1, 2, 1, "", "asarray"], [1, 2, 1, "", "broadcast_to"], [1, 2, 1, "", "concat"], [1, 2, 1, "", "empty"], [1, 2, 1, "", "empty_like"], [1, 2, 1, "", "eye"], [1, 2, 1, "", "full"], [1, 2, 1, "", "full_like"], [1, 2, 1, "", "linspace"], [1, 2, 1, "", "ones"], [1, 2, 1, "", "ones_like"], [1, 2, 1, "", "zeros"], [1, 2, 1, "", "zeros_like"]], "cubed.random": [[23, 2, 1, "", "random"]]}, "objnames": {"0": ["py", "class", "Python class"], "1": ["py", "method", "Python method"], "2": ["py", "function", "Python function"]}, "objtypes": {"0": "py:class", "1": "py:method", "2": "py:function"}, "terms": {"": [3, 5, 7, 30, 31, 32, 33, 34, 36, 37, 39, 40, 41], "0": [1, 11, 23, 29, 33], "00": 33, "001": [33, 37], "002": [33, 37], "003": [33, 37], "004": 37, "005": 37, "1": [1, 11, 12, 23, 27, 33, 34, 37, 40], "10": [4, 27, 37], "100": 33, "1000": 34, "10000": 33, "100kb": 27, "100mb": [3, 36, 39], "11": [3, 33], "12": 1, "180": 3, "2": [3, 11, 27, 29, 33, 34, 37], "20": [34, 36], "2000": 3, "200mb": [3, 33], "2022": 1, "2023": 1, "2gb": [3, 34, 36, 39], "3": [3, 4, 27, 33, 37], "300": 3, "4": [27, 33, 37], "400mb": 33, "438": 1, "5": [27, 33, 37], "5000": [3, 33], "50000": 3, "6": [27, 33, 37], "7": [27, 29, 33, 37], "8": [27, 33, 37], "9": [27, 33, 37], "A": [0, 2, 3, 34, 36, 38, 39, 41], "And": [32, 37], "As": [2, 36], "At": 34, "By": [33, 37, 40], "For": [2, 36, 38, 39, 40], "If": [3, 7, 9, 11, 14, 26, 33, 34, 36, 37, 38, 40], "In": [24, 31, 32, 33, 36, 37, 39, 40], "It": [3, 5, 19, 31, 32, 33, 34], "No": 31, "Not": 31, "On": [37, 40], "One": [2, 3, 39], "Or": [2, 3], "The": [1, 2, 3, 5, 7, 8, 9, 11, 14, 16, 18, 19, 24, 25, 26, 31, 32, 33, 34, 36, 37, 39, 40], "Then": 33, "There": [3, 5, 33, 36, 37, 39], "These": [0, 3, 5, 37], "To": [3, 29, 33, 37, 38, 39, 40], "With": 39, "__init__": [6, 10, 11, 12], "abil": 36, "about": [12, 13, 32, 33], "abov": [33, 34, 37], "abstract": [32, 41], "accept": 1, "access": 31, "account": 34, "accur": 36, "achiev": 39, "act": 38, "activ": 4, "actual": [3, 33, 36], "acycl": [2, 33], "ad": [2, 33, 39], "adapt": 33, "add": [3, 27, 31, 33, 37, 39, 40], "addit": [33, 37], "adjust": 39, "adopt": 41, "advanc": 35, "advantag": [2, 32, 39], "affect": [33, 39], "after": [8, 18, 33, 38, 40], "ag": 40, "again": [37, 38], "aggregr": 31, "ahead": [2, 36], "aim": [28, 36, 39], "algorithm": [5, 31, 36], "all": [3, 5, 31, 33, 34, 39, 41], "alloc": [13, 33, 41], "allow": [31, 35, 37, 38], "allow_rechunk": 13, "allowed_mem": [3, 11, 27, 34, 36, 39], "alon": 33, "along": [20, 31], "alreadi": [7, 14], "also": [2, 3, 27, 29, 34, 38, 39], "although": [31, 34], "alwai": [33, 41], "amazon": [34, 38], "amount": [13, 19, 33, 34, 36, 37, 38, 39, 41], "an": [2, 3, 4, 8, 9, 11, 15, 16, 19, 22, 25, 26, 28, 31, 33, 35, 36, 37, 38, 39, 41], "analys": 36, "ani": [3, 7, 11, 19, 33, 34, 37, 39, 40], "annot": 33, "anoth": [2, 36, 38, 39, 41], "anyth": 3, "apach": [5, 29, 30, 41], "api": [6, 27, 30, 31, 32], "appli": [13, 17, 18, 31, 39], "applic": 5, "apply_gufunc": 30, "approach": [40, 41], "appropri": [3, 40], "ar": [0, 1, 2, 3, 4, 5, 9, 13, 26, 29, 31, 32, 33, 34, 36, 37, 38, 39, 40, 41], "arang": 1, "arbitrari": 11, "arg": [13, 17, 18], "arg_reduct": 30, "argument": [1, 3, 18, 19, 33, 37], "arithmet": 20, "around": [36, 39], "arrai": [2, 3, 10, 13, 14, 15, 16, 17, 18, 21, 22, 24, 25, 26, 27, 31, 32, 33, 34, 36, 39, 40, 41], "arrang": 31, "array_api": [1, 3, 27, 33, 37], "arrow": 31, "asarrai": [1, 15, 27, 33, 37], "assum": 13, "astyp": 37, "atom": 38, "attempt": [3, 38, 41], "attribut": [5, 6, 11, 12, 31], "auto": [1, 15], "automat": [3, 33, 34, 36, 37, 39, 40], "avail": [3, 11, 27, 36, 38], "averag": 39, "avoid": [31, 41], "aw": [3, 29, 34, 39, 40], "awai": [37, 41], "ax": [13, 31], "axi": [1, 13, 20, 21, 31], "b": [3, 27, 33, 37], "back": [5, 6, 33, 34], "backup": [3, 38, 39], "bag": 32, "base": 39, "baselin": 36, "basi": 36, "basic": [3, 36], "basicconfig": 37, "batch": 3, "batch_siz": 3, "beam": [5, 29, 30, 32, 39, 41], "becaus": [34, 36], "been": [0, 3, 7, 11, 14, 27, 32, 33, 34, 37, 38, 39, 41], "befor": [2, 3, 7, 9, 14, 26, 33, 34, 37, 39, 40], "being": 37, "below": [3, 34, 36, 37], "better": 32, "between": [30, 31, 41], "big": 39, "bin": 33, "block": [5, 17, 18, 31, 33, 39], "blockwis": [2, 30, 32, 37], "blosc": 11, "blue": 5, "bool": [7, 9, 14, 18, 26], "both": [2, 32, 37, 38, 39], "bottom": [5, 31], "bound": [32, 36, 39, 41], "boundari": [18, 31], "box": 33, "branch": 39, "break": 41, "bring": 38, "broadcast": 31, "broadcast_to": 1, "bucket": 40, "budget": 36, "buffer": 33, "build": [2, 34, 36], "built": [2, 5], "bulk": 41, "byte": [11, 13, 19], "c": [3, 27, 29, 33, 37], "calcul": 36, "call": [2, 18, 31, 32, 33, 34, 37, 38], "callabl": [7, 9, 14, 18, 26], "callback": [7, 12, 14, 35], "can": [0, 2, 3, 9, 11, 19, 26, 29, 32, 33, 34, 36, 37, 38, 39, 40, 41], "cancel": 38, "cannot": 39, "carefulli": 41, "carri": 39, "case": [33, 36, 37, 39, 40, 41], "cd": 33, "central": 41, "certain": [38, 40], "challeng": 41, "chang": [3, 8, 31, 32, 33, 37, 39], "characterist": 39, "check": 36, "child": 3, "choic": 33, "choos": 2, "chosen": 5, "chunk": [1, 2, 3, 5, 6, 8, 15, 17, 18, 22, 23, 27, 30, 31, 32, 33, 34, 35, 37, 38, 39, 41], "circumst": 38, "class": [6, 10, 11, 12], "clean": 40, "clear": 40, "clevel": 11, "click": 40, "client": [3, 29, 38], "cloud": [3, 5, 19, 27, 32, 35, 36, 38, 41], "cluster": [34, 39], "clutter": 31, "cname": 11, "code": 3, "coil": [29, 34], "coiled_kwarg": 3, "collect": [24, 33], "colour": 33, "come": [34, 39], "command": 40, "common": [36, 41], "commonli": 37, "compar": 34, "complet": [12, 29, 33, 38, 39], "complex": [36, 39], "compon": 32, "compos": [5, 31], "compress": [11, 33, 36], "compressor": 11, "comput": [3, 5, 9, 10, 11, 16, 19, 20, 24, 25, 26, 27, 30, 32, 34, 35, 36, 37, 38, 39, 40, 41], "compute_arrays_in_parallel": [3, 39], "compute_kwarg": 3, "concat": [1, 31], "concept": 35, "concern": 32, "concurr": [3, 39], "conda": [4, 28], "confid": 36, "configur": [11, 30, 34, 36, 38, 39, 41], "conflict": 29, "conform": 6, "conserv": 36, "consid": [38, 39], "consider": 35, "consist": 35, "consol": 40, "constant": [18, 33], "constraint": 39, "construct": 3, "constructor": 3, "consult": 31, "contain": 36, "continu": 32, "contribut": 30, "control": 41, "conveni": 40, "convent": 27, "convert": [2, 32, 33, 39], "copi": 1, "core": [3, 27, 30, 31, 32, 34, 37], "corearrai": [8, 14, 26], "corner": 33, "correspond": [17, 18, 31, 32], "cost": 40, "could": 39, "coupl": 3, "cov": 29, "cover": [3, 38], "coverag": 1, "cpu": 3, "creat": [0, 4, 15, 19, 31, 32, 33, 37, 40], "creation": [0, 1, 3], "csv": 33, "cube": [0, 2, 3, 4, 5, 27, 28, 29, 31, 32, 33, 34, 35, 36, 37, 38, 40], "cubed_config": 3, "cubed_spec__allowed_mem": 3, "cubed_spec__executor_nam": 3, "cubed_spec__executor_options__runtim": 3, "cubed_spec__executor_options__runtime_memori": 3, "cubed_spec__executor_options__use_backup": 3, "cubed_spec__work_dir": 3, "cumul": 39, "current": [3, 13, 18, 24, 33, 34, 38, 39], "cutdown": 13, "d": [33, 37], "dag": [2, 33], "dai": 40, "dask": [5, 13, 27, 29, 30, 39, 41], "data": [3, 8, 11, 19, 33, 34, 35, 36, 37, 41], "dataflow": [3, 32, 34, 41], "datafram": 32, "dataset": [34, 39], "daunt": 32, "debug": [33, 35], "decompos": 32, "decompress": 36, "dedic": 40, "deeper": 39, "default": [3, 7, 9, 11, 14, 24, 25, 26, 27, 33, 34, 37, 38, 39, 40], "defin": [5, 32, 39], "delai": [32, 39], "deleg": 5, "delet": 35, "deliber": [3, 41], "demo": [28, 30], "depend": [2, 7, 28, 30, 33, 38, 39], "deploi": 3, "depth": [18, 31], "describ": [27, 34, 38], "descript": 3, "design": [3, 30, 39, 41], "desir": 8, "detail": [2, 3, 31, 34, 37], "determin": [33, 38], "develop": 33, "devic": 1, "diagaram": 36, "diagnos": 35, "diagnost": [29, 30, 35, 39], "diagram": 31, "dict": [3, 11, 18], "dictionari": 11, "differ": [3, 29, 30, 31, 37, 38], "difficult": 37, "dill": 29, "dimens": 31, "dimension": 41, "dip": 33, "direct": [2, 33], "directli": [3, 31], "directori": [3, 11, 19, 33, 40], "disabl": 11, "disadvantag": [2, 39], "discuss": [2, 34, 36], "disk": [2, 5, 9, 26, 34], "displai": [9, 26, 33, 39], "disproportion": [38, 39, 41], "distribut": [2, 3, 29, 32, 34, 38, 39, 41], "do": [33, 38, 39, 41], "doc": 3, "document": 3, "doe": [34, 37, 40], "doesn": [3, 9, 26, 34, 36], "don": [2, 3, 33, 34, 40], "done": [33, 39], "donfig": 3, "dot": [9, 26], "doubl": 3, "down": 38, "drop": 31, "drop_axi": 17, "dtype": [1, 2, 5, 17, 18, 21, 31, 36], "due": 29, "duplic": 38, "dure": [10, 36], "e": [4, 37, 39], "each": [2, 3, 18, 31, 33, 36, 38, 39], "eager": [24, 25], "easi": 34, "easier": 32, "echo": 33, "edg": 2, "effect": 37, "effici": [2, 36], "effort": 41, "either": 37, "element": [18, 21, 31, 39], "elementwis": 39, "elemwis": 30, "embarrassingli": 41, "empti": [1, 31], "empty_lik": 1, "enabl": [33, 37, 38, 39], "encapsul": 3, "endpoint": 1, "engin": [2, 41], "enhanc": 39, "enough": 39, "ensur": [33, 36, 40], "enter": 40, "environ": [2, 4, 34, 39], "equal": 39, "equival": [3, 13], "error": 38, "eschew": 41, "especi": [29, 39], "essenti": 27, "estim": 36, "etc": [11, 30], "evalu": 27, "even": [38, 39, 41], "event": [7, 10, 14, 33], "everi": [2, 5, 18, 34, 39], "everyth": 41, "exampl": [2, 3, 5, 11, 27, 30, 31, 34, 36, 37, 38, 39], "exce": [36, 37], "except": [1, 3, 36, 38], "excess": [34, 37], "exclud": 19, "execut": [30, 36, 39], "executor": [5, 7, 11, 14, 19, 24, 25, 27, 29, 30, 33, 35, 36, 38, 40], "executor_nam": [3, 11, 34], "executor_opt": [3, 11], "exist": 39, "expand": 3, "expect": [38, 39], "explain": [33, 37], "explicitli": 2, "export": 3, "expos": 3, "express": 2, "extens": [5, 9, 26], "extent": 41, "extern": 5, "extra": [1, 18], "extra_projected_mem": 13, "ey": 1, "f": 33, "factor": [36, 39], "fail": [3, 38], "fals": [3, 7, 9, 14, 18, 20, 21, 26, 33, 37, 38, 39], "fan": 37, "far": [34, 39], "fashion": 38, "faster": 39, "fault": 38, "featur": [3, 33, 38, 39], "fee": 34, "feed": 39, "few": [1, 5, 38, 39, 41], "fewer": 39, "file": [9, 26, 33, 34, 38, 40], "filenam": [9, 26, 33], "filesystem": 40, "fill_valu": 1, "final": 31, "fine": 32, "first": [31, 33, 36, 37, 39], "firstli": 39, "fit": 34, "five": 5, "fix": 39, "flame": 33, "flamegraph": 33, "flexibl": 32, "float": 23, "float32": 37, "follow": [1, 3, 5, 11, 30, 31, 33, 36, 37, 40], "forg": 29, "form": [11, 40], "format": [9, 26, 33], "formerli": 41, "four": [3, 36], "fourier": 1, "framework": 41, "free": 34, "from": [3, 5, 7, 14, 15, 16, 17, 18, 31, 33, 36, 37, 38, 39, 41], "from_arrai": 0, "from_zarr": 0, "fsspec": [3, 11, 19, 40], "fuctool": 37, "full": [1, 29, 39], "full_lik": 1, "func": [13, 17, 18], "function": [1, 2, 3, 5, 7, 9, 13, 14, 17, 18, 19, 26, 29, 30, 31, 33, 34, 36, 37, 39], "function_end_tstamp": 12, "function_start_tstamp": 12, "functionexecutor": 3, "functool": 37, "fundament": [31, 32], "further": [36, 41], "fuse": [33, 37, 39], "fusion": [2, 7, 9, 14, 26, 35], "futur": 3, "g": 39, "gain": 41, "gb": 11, "gc": 34, "gcf": 39, "gcp": [3, 29], "gcsf": 29, "gener": [5, 13, 30, 31, 33, 36, 39, 41], "get": [4, 30, 34, 35, 37, 39, 40], "github": 4, "given": [3, 19, 21, 36, 39], "global": 38, "go": [5, 33, 41], "good": [3, 11, 36, 39], "googl": [3, 32, 34, 38, 39, 40, 41], "grain": 32, "graph": [2, 7, 9, 14, 26, 32, 33, 37], "graphic": 33, "graphviz": 29, "greater": 36, "green": [5, 31], "group": [16, 25], "guarante": [30, 34, 39, 41], "guid": [19, 28, 30], "ha": [2, 3, 11, 27, 29, 31, 32, 33, 34, 36, 37, 38, 39, 41], "had": 34, "hadoop": 41, "half": 23, "hand": 37, "handi": 3, "handl": [18, 34, 39], "hard": 3, "have": [0, 2, 3, 7, 14, 28, 32, 33, 34, 36, 37, 38, 39], "head": 4, "heavili": [5, 32], "help": [35, 36, 37], "henc": [38, 39], "here": [3, 31, 33, 37, 39, 41], "hidden": [9, 26], "high": [2, 28, 32, 36, 41], "higher": [32, 37], "highest": [34, 36], "hist": 33, "historycallback": 33, "hold": 39, "horizont": [33, 39], "how": [2, 3, 18, 31, 33, 36, 37, 39, 40, 41], "howev": [2, 13, 41], "html": 33, "i": [1, 2, 3, 5, 7, 9, 11, 13, 14, 18, 19, 24, 25, 26, 27, 30, 31, 32, 33, 35, 36, 37, 38, 39, 40, 41], "id": [11, 33], "ideal": [33, 39], "idempot": 38, "ident": 38, "ignor": 20, "illustr": 36, "imag": [3, 9, 26, 33], "immedi": [24, 25, 38], "implement": [1, 5, 19, 24, 27, 30, 31, 32, 41], "implicitli": 2, "import": [3, 9, 26, 27, 33, 34, 36, 37, 39, 40], "importantli": 38, "impos": 37, "improv": [33, 36, 39, 41], "includ": [0, 3, 5, 9, 11, 26, 32], "incur": 40, "independ": 39, "index": 31, "indic": 3, "individu": [3, 37], "infinit": 39, "influenc": 32, "inform": [12, 13, 33, 36, 37], "inherit": 5, "input": [3, 16, 17, 18, 31, 35, 36, 39], "instal": [4, 19, 28, 30, 33], "instanc": [3, 39], "instanti": 3, "instead": 41, "instruct": 40, "int": [11, 18, 19], "integ": 37, "integr": [30, 32], "intend": 34, "interact": 27, "interest": 41, "intermedi": [2, 3, 7, 11, 14, 19, 33, 35, 37, 39], "intern": 33, "interv": 23, "interven": 32, "intro": 30, "introduc": 35, "introduct": 28, "invok": [2, 3], "involv": [2, 4, 33, 41], "io": [5, 30, 37, 38], "ipython": [9, 26], "issu": [1, 33], "iter": 39, "its": [5, 8, 18, 31, 33, 36, 37, 39], "itself": 36, "jinja2": 29, "job": 40, "jpeg": [9, 26], "jpg": [9, 26], "jupyt": 33, "just": [27, 31, 33, 37], "k": 1, "kb": 11, "keepdim": [13, 20, 21], "kei": [3, 38], "keyword": [1, 3, 18, 19], "kill": 3, "kind": 37, "known": [2, 36, 39, 41], "kwarg": [7, 13, 14, 17, 18, 19, 24, 25], "l": 33, "label": 33, "lambda": [34, 39], "larg": [34, 36, 38, 39, 41], "larger": [34, 39], "last": 41, "later": 33, "launch": 38, "layer": [5, 32], "lazi": [2, 27], "lead": [39, 41], "least": 36, "less": 32, "lesser": 41, "let": 5, "level": [2, 28, 32, 37, 41], "librari": [30, 32], "lifecycl": 40, "like": [3, 5, 15, 24, 27, 31, 32, 34, 36, 37, 40, 41], "lilac": 33, "limit": [13, 37, 39], "line": 33, "linear": 39, "linspac": 1, "list": [1, 3, 7, 14, 18, 29], "lithop": [5, 19, 29, 30, 33, 34, 41], "live": 3, "ll": 27, "load": [16, 33], "local": [5, 11, 27, 33, 35, 36, 40], "locat": 3, "log": 37, "logic": 39, "long": [33, 37, 39], "longer": [3, 33, 37, 38, 39], "look": [3, 28, 31, 37], "lot": [40, 41], "lz4": 11, "m": [29, 33], "machin": [27, 35, 36, 39], "made": [3, 41], "magic": 39, "mai": [2, 5, 18, 31, 34, 37, 39, 40], "main": [2, 3], "mainli": 3, "maintain": 39, "major": 41, "make": [2, 32, 36, 39], "manag": [3, 36, 41], "mani": [29, 31, 37], "manipul": 1, "manual": 40, "map": [2, 7, 9, 14, 18, 26, 31, 35, 39], "map_block": [5, 30], "map_direct": [30, 37], "map_overlap": 30, "mapreduc": 41, "mark": [9, 26], "match": [31, 37, 39], "materi": [2, 33], "matplotlib": 29, "matur": [32, 34], "max_tasks_per_child": 3, "max_total_num_input_block": 37, "max_total_source_arrai": 37, "max_work": [3, 39], "maximum": [3, 30, 36, 37, 39], "mb": 11, "mean": [3, 5, 20, 33, 36, 37, 38, 39], "measur": [19, 36, 39], "measure_reserved_mem": [36, 39], "memori": [3, 11, 13, 19, 32, 33, 34, 35, 39, 41], "memrai": 35, "messag": 38, "method": [6, 10, 11, 12, 33, 37], "middl": 31, "might": 39, "minim": 29, "minimum": 39, "minut": 34, "mitig": [3, 38], "mock": 29, "modal": [5, 19, 29, 30, 34, 39], "mode": 22, "model": [2, 5, 41], "monitor": 33, "more": [13, 27, 31, 32, 33, 34, 36, 37, 38, 39], "most": [3, 34, 35, 41], "much": [33, 34, 36, 39], "multi": 27, "multipl": [2, 13, 14, 17, 18, 26, 30, 31, 33, 35], "multiple_inputs_optimize_dag": 37, "must": [3, 19, 24, 37, 38], "n": [3, 41], "n_col": 1, "n_row": 1, "name": [4, 6, 9, 12, 26, 32, 40], "nan": [20, 21], "natur": [2, 32, 36], "nbyte": 37, "necessari": 39, "need": [2, 5, 13, 31, 33, 36, 38, 39, 40], "neg": 37, "neighbor": 18, "neither": 3, "nest": 3, "never": 36, "new": [5, 13, 32, 33, 36, 39], "new_axi": 17, "newer": 32, "next": 39, "node": [2, 33, 41], "non": [3, 11, 30, 36, 39], "none": [1, 3, 7, 9, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 37], "note": [3, 18, 24, 25, 31, 33, 34, 37, 38, 40], "notebook": [9, 26, 33], "noth": [27, 39], "notic": [3, 27, 37], "now": 37, "np": 18, "num": [1, 37], "num_task": 12, "number": [3, 5, 18, 30, 31, 33, 34, 38, 39], "numblock": 31, "numcodec": 11, "numpi": [2, 5, 27, 32], "numpy_groupi": 29, "o": 33, "obj": 1, "object": [2, 10, 15, 19, 24, 27, 40], "observ": 39, "off": [33, 37, 38, 39], "offer": 39, "often": 41, "old": 40, "onc": [14, 33, 38], "one": [0, 2, 3, 18, 31, 33, 37, 38, 39, 40, 41], "ones": [1, 32, 37], "ones_lik": 1, "onli": [2, 3, 7, 9, 14, 18, 26, 31, 32, 33, 34, 37, 38, 39, 41], "op": [33, 37], "open": 23, "oper": [2, 9, 19, 24, 25, 26, 30, 32, 33, 36, 37, 38, 39, 40, 41], "opportun": 2, "opt_fn": 37, "optim": [2, 7, 9, 14, 26, 30, 33, 35, 39, 41], "optimize_funct": [7, 9, 14, 26, 37], "optimize_graph": [7, 9, 14, 26, 33, 37], "option": [7, 9, 11, 14, 16, 19, 24, 25, 26, 28, 33], "optional_depend": 29, "orang": [31, 33], "order": [31, 36, 39], "origin": 38, "other": [3, 5, 35, 37], "otherwis": [7, 9, 14, 26, 37], "out": [33, 34, 35, 39, 40], "outer": 31, "output": [2, 9, 13, 18, 25, 26, 31, 33, 36, 37, 38, 39], "output_dtyp": 13, "output_s": 13, "outsid": 3, "over": [4, 18, 21, 31, 40], "overal": [33, 37, 38, 39], "overhead": 34, "overlap": 18, "overview": 28, "own": 39, "packag": [3, 19, 29, 32], "pad_width": 22, "page": [1, 3, 31, 39], "pair": 2, "panda": 29, "paragraph": 38, "parallel": [2, 3, 32, 39, 41], "paramet": [7, 8, 9, 11, 13, 14, 16, 18, 19, 24, 25, 26, 39], "part": [31, 34, 39], "partial": 37, "partial_reduc": 31, "particular": [3, 34, 36, 37], "pass": [3, 11, 18, 19, 31, 33, 39, 41], "path": [3, 11, 16, 19, 25], "pdf": [9, 26], "peak": 19, "peak_measured_mem": 36, "peak_measured_mem_end": 12, "peak_measured_mem_start": 12, "per": [13, 39], "perfect": 33, "perform": [7, 9, 14, 26, 31, 33, 35, 36, 37, 41], "period": 40, "persist": [39, 40, 41], "phase": 36, "pick": 37, "pink": 31, "pip": [4, 28, 33], "plan": [6, 30, 32, 35, 36, 37, 39], "pleas": 4, "plenti": 36, "plot": 33, "png": [9, 26], "possibl": [3, 28, 36, 37, 39, 41], "potenti": 39, "power": 37, "practic": 35, "pre": 38, "preced": 37, "precis": 36, "predecessor": 37, "prefac": 35, "preserv": 31, "previou": [30, 37, 38], "primit": [2, 30, 31, 32, 37], "print": 27, "problem": [39, 41], "process": [2, 5, 19, 24, 25, 32, 33, 34, 36, 38, 39, 41], "processor": 39, "processpoolexecutor": 3, "produc": [9, 18, 26, 33, 36], "profil": 33, "program": [3, 41], "project": [5, 29, 30, 33, 35], "projected_mem": 36, "properli": 36, "properti": 3, "proportion": 39, "provid": [5, 9, 26, 33, 34, 36, 38, 41], "provis": 39, "public": 5, "purpos": 41, "put": [36, 41], "py": 33, "pydot": 29, "pyproject": 29, "pytest": 29, "python": [0, 3, 4, 5, 6, 13, 19, 24, 25, 27, 29, 30, 32, 33, 34, 36], "pywren": 41, "quickli": [28, 34, 38], "r": 4, "rais": 36, "ran": 36, "random": [3, 5, 30, 33], "rather": [31, 40], "re": 39, "read": [31, 32, 33, 36, 37, 38, 39, 41], "readm": 27, "reason": [3, 32, 36, 37, 39], "receiv": 10, "rechunk": [2, 5, 30, 32, 39, 41], "recommend": 34, "recomput": [7, 14], "red": 5, "reduc": [31, 37, 39], "reduct": [30, 36, 37, 39], "refer": [13, 30, 34, 39], "regard": 31, "region": [34, 40], "regular": 40, "rel": 34, "relat": 30, "reli": 38, "reliabl": [30, 34, 35], "remot": 34, "remov": [37, 40], "render": [9, 26, 33], "report": [19, 36], "repres": [31, 33], "represent": [2, 39], "req": 29, "requir": [2, 4, 19, 33, 34, 36, 39, 41], "reserv": [3, 11, 19, 35], "reserved_mem": [3, 11, 19, 36], "resourc": [11, 27, 39], "restrict": 39, "result": [27, 31, 33, 37], "resum": [7, 14], "retain": 33, "retri": [3, 35], "return": [8, 9, 16, 19, 21, 23, 26, 31], "reus": 3, "rich": [29, 33], "richprogressbar": 33, "right": 33, "rm": 40, "room": 36, "round": [31, 33], "rule": [36, 40], "run": [2, 3, 7, 11, 14, 19, 24, 25, 27, 29, 32, 33, 34, 36, 37, 38, 39, 40, 41], "runner": 32, "runtim": [2, 3, 19, 24, 25, 30, 32, 34, 36, 40, 41], "runtime_memori": 3, "s3": [3, 34, 38, 40], "s3f": 29, "same": [3, 31, 37, 39, 40], "save": [24, 25, 33], "scale": [2, 30, 33, 34, 35, 38, 41], "scenario": 39, "schema": 33, "seaborn": 29, "second": [3, 31, 34], "section": 38, "see": [2, 3, 27, 29, 33, 37, 39], "select": [31, 40], "send": [7, 14], "sent": 31, "separ": [29, 33, 39, 40], "serverless": [2, 32, 39, 41], "servic": [35, 36, 39, 40, 41], "set": [2, 3, 7, 11, 13, 14, 19, 29, 34, 35, 36, 38, 39, 40], "sever": 32, "shape": [1, 5, 8, 18, 31], "share": [3, 18], "short": 40, "should": [11, 18, 35, 36, 37, 39, 40], "show": [9, 26, 31, 33, 34], "show_hidden": [9, 26], "shown": [31, 33, 37], "shuffl": [2, 11, 39, 41], "side": 31, "sign": [34, 39], "signatur": 13, "signifi": 33, "signific": 39, "similar": [13, 32], "similarli": [31, 33], "simpl": [3, 27, 33, 34], "simplest": [31, 37, 39], "simpli": 33, "simultan": 39, "sinc": [2, 3, 31, 32, 33, 34, 36, 37, 38, 41], "singl": [27, 31, 32, 33, 35, 38], "size": [2, 23, 31, 33, 35, 39, 41], "slide": 30, "slow": [2, 37, 38], "slowli": 39, "small": [33, 34, 37], "smaller": 36, "so": [3, 27, 32, 33, 34, 36, 37, 38, 39, 40], "solut": 39, "some": [18, 32, 36, 37, 38, 39], "sometim": [34, 37], "sourc": 24, "space": 40, "spark": 41, "spec": [1, 6, 7, 14, 15, 16, 17, 19, 23, 27, 33, 34, 36, 40], "specif": [11, 30, 39], "specifi": [3, 11, 18, 19, 20, 27, 35, 37, 40], "specul": 38, "speed": [37, 39], "spent": [33, 41], "split_everi": [20, 21], "squar": 33, "squeez": 31, "stage": 37, "standard": [0, 6, 27, 30, 32], "standardis": 30, "start": [1, 27, 30, 34, 37, 39, 41], "startup": [33, 34, 39], "stat": 33, "stateless": [5, 41], "statist": 33, "statu": 1, "step": [1, 33], "stick": 39, "stop": 1, "storag": [2, 6, 11, 16, 25, 30, 35, 38, 39, 41], "storage_opt": 11, "store": [3, 5, 11, 16, 19, 25, 33, 39, 40], "str": [9, 11, 19, 26], "straggler": [3, 33, 35, 39], "straightforward": [2, 34], "string": [16, 25], "strong": [35, 39], "structur": 31, "style": 3, "submit": 3, "subset": [5, 27], "success": 41, "suffici": 39, "suggest": 39, "suit": 33, "sum": [21, 36], "support": [1, 3, 13, 18, 31, 32, 37], "sure": [39, 40], "svg": [9, 26, 33], "system": [5, 19, 38, 39, 40, 41], "t": [2, 3, 7, 9, 14, 26, 32, 33, 34, 36, 37, 40], "tab": 40, "tail": 33, "take": [3, 33, 38, 39], "taken": 38, "target": [24, 41], "task": [2, 3, 5, 11, 12, 13, 19, 32, 33, 34, 36, 37, 38, 39], "task_create_tstamp": 12, "task_result_tstamp": 12, "tb": 11, "tell": 13, "temp": [3, 34, 40], "tempor": 33, "temporari": [3, 40], "ten": 36, "terabyt": 39, "test": [3, 29, 33, 34], "test_mem_util": 33, "than": [3, 13, 31, 32, 36, 37, 38, 39, 40], "thei": [3, 31, 32, 33, 36, 37, 39], "them": [2, 31, 33, 38, 40], "theme": 41, "theoret": 35, "therebi": [39, 41], "therefor": 34, "thi": [2, 3, 5, 7, 8, 9, 11, 13, 18, 19, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41], "thing": 3, "those": 39, "though": [39, 41], "thread": [34, 39], "threadpoolexecutor": 3, "three": [3, 31, 32, 33, 37, 38], "through": [3, 5], "throughput": 39, "thumb": 36, "tightli": [36, 41], "time": [2, 3, 31, 33, 34, 36, 37, 38, 39, 40, 41], "timeline_viz": 33, "timelinevisualizationcallback": 33, "timeout": [3, 35], "tini": [19, 33], "tip": 35, "tmp": 27, "tmpdir": 40, "to_zarr": 33, "togeth": [2, 37], "toler": 38, "toml": 29, "tomwhit": [3, 34, 40], "too": [31, 37], "took": 33, "tool": 33, "top": [5, 31], "total": [3, 11, 38, 39], "tqdm": [29, 33], "tqdmprogressbar": 33, "trace": 33, "track": [1, 33], "traction": 41, "transfer": 34, "transform": 1, "travers": 2, "treat": 21, "tree": 30, "tree_reduc": 31, "trigger": 2, "trim": 18, "trivial": [19, 39], "true": [1, 3, 7, 9, 14, 26, 37, 39], "tune": [39, 41], "tupl": [8, 18], "turn": [3, 31, 33, 37, 38, 39], "twice": 33, "two": [5, 31, 33, 39], "txt": 4, "type": [5, 8, 9, 16, 18, 19, 24, 25, 26, 33, 35], "typic": 40, "ufunc": 13, "under": [3, 33], "underli": 5, "underscor": 3, "understand": [32, 33, 37, 39, 41], "undoubtedli": 41, "unit": [5, 11], "unnecessari": 40, "unoptim": 37, "unpredict": 41, "until": 31, "up": [2, 3, 34, 36, 37, 38, 39, 40], "updat": 38, "upgrad": 39, "upper": 36, "url": [3, 11, 19, 40], "us": [2, 3, 5, 7, 9, 11, 14, 16, 19, 24, 25, 26, 27, 28, 29, 30, 31, 32, 35, 36, 37, 39, 40, 41], "usag": [13, 30, 36, 39, 41], "use_backup": [3, 38, 39], "user": [3, 32, 34, 41], "usual": 33, "util": [32, 36], "v": 35, "valu": [3, 11, 18, 31, 36, 37], "vari": [19, 39], "varieti": [3, 32, 33, 34], "variou": 36, "vector": 13, "veri": [2, 4, 33, 34, 39, 41], "versa": 39, "version": [1, 13, 19], "versu": 39, "vertic": [33, 39], "via": [30, 41], "vice": 39, "view": 33, "violat": 39, "visual": [32, 35, 37], "wa": [5, 32, 33], "wai": [3, 31, 33, 38, 39], "wait": 39, "want": [33, 37], "wasn": 32, "we": [3, 27, 31, 33, 34, 36, 37, 39], "weak": 39, "welcom": 4, "well": [5, 32, 33, 36], "what": [32, 33, 37], "whatev": [31, 36], "when": [2, 3, 11, 19, 32, 33, 34, 36, 37, 38, 39, 40], "where": [2, 11, 32, 37, 39], "wherea": [32, 39], "whether": [3, 18], "which": [2, 3, 5, 9, 11, 26, 27, 29, 31, 32, 33, 35, 36, 37, 39, 40, 41], "while": [2, 7, 14, 31, 33, 36, 39], "white": 31, "who": 39, "whole": 38, "why": [30, 37], "widespread": 41, "within": [3, 36], "without": [8, 31, 39], "won": [7, 14], "word": 39, "work": [19, 30, 31, 33, 34, 36, 38, 40, 41], "work_dir": [3, 11, 19, 27, 34, 40], "worker": [3, 11, 33, 34, 36, 39, 41], "would": [33, 36, 37, 39, 40], "write": [2, 9, 24, 26, 33, 34, 36, 38, 39, 40, 41], "written": [2, 33, 37, 38], "x": [1, 15, 20, 21, 22, 24, 25], "xarrai": 30, "xp": [3, 27, 33, 37], "yaml": 34, "year": [32, 41], "yet": [0, 27], "you": [3, 13, 28, 29, 32, 33, 34, 35, 36, 37, 38, 39, 40], "your": [3, 33, 34, 36, 39], "zappi": 32, "zarr": [2, 5, 6, 11, 16, 24, 25, 30, 33, 34, 36, 38, 39, 40, 41], "zarr_compressor": 11, "zarrai": 6, "zero": [1, 21], "zeros_lik": 1}, "titles": ["API Reference", "Python Array API", "Computation", "Configuration", "Contributing", "Design", "cubed.Array", "cubed.Array.compute", "cubed.Array.rechunk", "cubed.Array.visualize", "cubed.Callback", "cubed.Spec", "cubed.TaskEndEvent", "cubed.apply_gufunc", "cubed.compute", "cubed.from_array", "cubed.from_zarr", "cubed.map_blocks", "cubed.map_overlap", "cubed.measure_reserved_mem", "cubed.nanmean", "cubed.nansum", "cubed.pad", "cubed.random.random", "cubed.store", "cubed.to_zarr", "cubed.visualize", "Demo", "Getting Started", "Installation", "Cubed", "Operations", "Related Projects", "Diagnostics", "Executors", "User Guide", "Memory", "Optimization", "Reliability", "Scaling", "Storage", "Why Cubed?"], "titleterms": {"For": 30, "advanc": 37, "allow": 36, "an": 34, "api": [0, 1, 5], "apply_gufunc": 13, "arg_reduct": 31, "arrai": [0, 1, 5, 6, 7, 8, 9, 30, 37], "articl": 30, "bar": 33, "beam": 3, "between": 1, "block": 37, "blockwis": 31, "bound": 30, "calcul": 39, "callback": [10, 33], "chunk": [0, 36], "cloud": [34, 39, 40], "coil": 3, "comput": [2, 7, 14, 33], "conda": 29, "configur": 3, "consider": 39, "consist": 38, "contribut": 4, "core": 5, "cube": [1, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 30, 39, 41], "dask": [3, 32], "data": 40, "debug": 37, "delet": 40, "demo": 27, "depend": [29, 31], "design": 5, "develop": [4, 30], "diagnos": 39, "diagnost": 33, "differ": [1, 39], "dimension": 30, "distribut": 30, "document": 30, "elemwis": 31, "environ": 3, "exampl": 33, "execut": 2, "executor": [3, 34, 39], "file": 3, "from_arrai": 15, "from_zarr": 16, "function": 0, "fusion": 37, "gener": 0, "get": 28, "guid": 35, "histori": 33, "i": 34, "input": 37, "instal": 29, "intermedi": 40, "io": 0, "lithop": 3, "local": 34, "machin": 34, "map": 37, "map_block": [17, 31], "map_direct": 31, "map_overlap": 18, "measure_reserved_mem": 19, "memori": [2, 30, 36], "memrai": 33, "modal": 3, "multi": 39, "multipl": 37, "n": 30, "nanmean": 20, "nansum": 21, "non": 0, "number": [0, 37], "object": 3, "oper": [5, 31], "optim": 37, "option": [3, 29], "other": 39, "pad": 22, "perform": 39, "pip": 29, "pipelin": 39, "plan": [2, 33], "practic": 39, "prefac": 39, "previou": 32, "primit": 5, "process": [3, 30], "progress": 33, "project": [32, 36], "provid": 39, "python": 1, "random": [0, 23], "rechunk": [8, 31], "reduct": 31, "refer": [0, 3], "relat": 32, "reliabl": 38, "reserv": 36, "retri": 38, "runtim": [0, 5], "scale": 39, "serverless": 30, "servic": 34, "set": 37, "should": 34, "singl": [3, 34, 39], "size": 36, "sourc": 37, "spec": [3, 11], "specif": [0, 3], "specifi": 34, "standard": 1, "standardis": 0, "start": 28, "step": 39, "storag": [5, 40], "store": 24, "straggler": 38, "strong": 38, "taskendev": 12, "theoret": 39, "thread": 3, "timelin": 33, "timeout": 38, "tip": 39, "to_zarr": 25, "total": 37, "tree": 31, "type": 39, "us": [33, 34], "user": [30, 35], "v": 39, "variabl": 3, "visual": [9, 26, 33], "which": 34, "why": 41, "work": 32, "xarrai": 32, "yaml": 3}}) \ No newline at end of file diff --git a/user-guide/diagnostics.html b/user-guide/diagnostics.html index 03c7a75c..87432b76 100644 --- a/user-guide/diagnostics.html +++ b/user-guide/diagnostics.html @@ -36,7 +36,7 @@ - + diff --git a/user-guide/executors.html b/user-guide/executors.html index 2e88f187..68f29657 100644 --- a/user-guide/executors.html +++ b/user-guide/executors.html @@ -36,7 +36,7 @@ - + diff --git a/user-guide/index.html b/user-guide/index.html index 712b60ea..a3edd671 100644 --- a/user-guide/index.html +++ b/user-guide/index.html @@ -36,7 +36,7 @@ - + diff --git a/user-guide/memory.html b/user-guide/memory.html index 7118e56c..82a0c67a 100644 --- a/user-guide/memory.html +++ b/user-guide/memory.html @@ -36,7 +36,7 @@ - + diff --git a/user-guide/optimization.html b/user-guide/optimization.html index e40779cb..63517eda 100644 --- a/user-guide/optimization.html +++ b/user-guide/optimization.html @@ -36,7 +36,7 @@ - + diff --git a/user-guide/reliability.html b/user-guide/reliability.html index e64c6889..46d3cc33 100644 --- a/user-guide/reliability.html +++ b/user-guide/reliability.html @@ -36,7 +36,7 @@ - + diff --git a/user-guide/scaling.html b/user-guide/scaling.html index f26ffd79..add5a358 100644 --- a/user-guide/scaling.html +++ b/user-guide/scaling.html @@ -36,7 +36,7 @@ - + diff --git a/user-guide/storage.html b/user-guide/storage.html index 161c5018..420bcb13 100644 --- a/user-guide/storage.html +++ b/user-guide/storage.html @@ -36,7 +36,7 @@ - + diff --git a/why-cubed.html b/why-cubed.html index e4d41b27..5cd53f82 100644 --- a/why-cubed.html +++ b/why-cubed.html @@ -36,7 +36,7 @@ - +