-
Notifications
You must be signed in to change notification settings - Fork 19
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add better support for cparams in Proxy and C2Array instances
- Loading branch information
1 parent
ab19f03
commit 1858e08
Showing
9 changed files
with
177 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -28,6 +28,8 @@ Attributes | |
chunks | ||
blocks | ||
dtype | ||
cparams | ||
urlpath | ||
|
||
.. _URLPath: | ||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -45,6 +45,10 @@ Attributes | |
info | ||
schunk | ||
size | ||
cparams | ||
dparams | ||
urlpath | ||
vlmeta | ||
|
||
|
||
.. currentmodule:: blosc2 | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -29,6 +29,10 @@ Attributes | |
:toctree: autofiles/proxy | ||
|
||
shape | ||
chunks | ||
blocks | ||
dtype | ||
cparams | ||
info | ||
fields | ||
vlmeta |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
####################################################################### | ||
# Copyright (c) 2019-present, Blosc Development Team <blosc@blosc.org> | ||
# All rights reserved. | ||
# | ||
# This source code is licensed under a BSD-style license (found in the | ||
# LICENSE file in the root directory of this source tree) | ||
####################################################################### | ||
|
||
# Shows how you can make a proxy of a remote array (served with Caterva2) on disk | ||
# Note that, for running this example, you will need the blosc2-grok package. | ||
|
||
import os | ||
from time import time | ||
|
||
import blosc2 | ||
|
||
urlbase = "https://demo.caterva2.net/" | ||
path = "example/lung-jpeg2000_10x.b2nd" | ||
a = blosc2.C2Array(path, urlbase=urlbase) | ||
blosc2.remove_urlpath("proxy.b2nd") | ||
b = blosc2.Proxy(a, urlpath="proxy.b2nd") | ||
|
||
# Check metadata (note that all should be the same) | ||
print("*** Metadata ***") | ||
print(f"Codec in 'a': {a.cparams.codec}") | ||
print(f"Codec in 'b': {b.cparams.codec}") | ||
print(f"Filters in 'a': {a.cparams.filters}") | ||
print(f"Filters in 'b': {b.cparams.filters}") | ||
|
||
# Check array properties | ||
print("*** Array properties ***") | ||
print(f"Shape in 'a': {a.shape}") | ||
print(f"Shape in 'b': {b.shape}") | ||
print(f"Type in 'a': {a.dtype}") | ||
print(f"Type in 'b': {b.dtype}") | ||
|
||
print("*** Fetching data ***") | ||
t0 = time() | ||
print(f"Data in 'a': {a[0, 0, 0:10]}") | ||
print(f"Time to fetch data in 'a': {time() - t0:.3f}s") | ||
t0 = time() | ||
print(f"Data in 'b': {b[0, 0, 0:10]}") | ||
print(f"Time to fetch data in 'b': {time() - t0:.3f}s") | ||
t0 = time() | ||
print(f"Data in 'b': {b[0, 0, 0:10]}") | ||
print(f"Time to fetch data in 'b' (cached): {time() - t0:.3f}s") | ||
|
||
# Check sizes. Note that the proxy will only have the 'touched' chunks (only 1 in this case) | ||
print("*** Sizes ***") | ||
print(f"Size in 'a': {a.meta['schunk']['cbytes']}") | ||
print(f"Size in 'b': {b.schunk.cbytes}") | ||
# Check sizes on disk | ||
print("*** Disk sizes ***") | ||
print(f"Size 'b' (disk): {os.stat(b.urlpath).st_size}") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
####################################################################### | ||
# Copyright (c) 2019-present, Blosc Development Team <blosc@blosc.org> | ||
# All rights reserved. | ||
# | ||
# This source code is licensed under a BSD-style license (found in the | ||
# LICENSE file in the root directory of this source tree) | ||
####################################################################### | ||
|
||
# Shows how you can make a proxy of a local array on disk. | ||
|
||
import os | ||
|
||
import blosc2 | ||
|
||
cparams = blosc2.CParams( | ||
clevel=5, codec=blosc2.Codec.LZ4, filters=[blosc2.Filter.BITSHUFFLE], filters_meta=[0] | ||
) | ||
cwd = os.getcwd() | ||
a = blosc2.full((128, 128), 1, dtype="float64", urlpath=f"{cwd}/a.b2nd", mode="w", cparams=cparams) | ||
blosc2.remove_urlpath(f"{cwd}/proxy.b2nd") | ||
b = blosc2.Proxy(a, urlpath=f"{cwd}/proxy.b2nd") | ||
|
||
# Check metadata | ||
print("*** Metadata ***") | ||
print(f"Codec in 'a': {a.cparams.codec}") | ||
print(f"Codec in 'b': {b.cparams.codec}") | ||
print(f"Clevel in 'a': {a.cparams.clevel}") | ||
print(f"Clevel in 'b': {b.cparams.clevel}") | ||
print(f"Filters in 'a': {a.cparams.filters}") | ||
print(f"Filters in 'b': {b.cparams.filters}") | ||
|
||
# Check array properties | ||
print("*** Array properties ***") | ||
print(f"Shape in 'a': {a.shape}") | ||
print(f"Shape in 'b': {b.shape}") | ||
print(f"Type in 'a': {a.dtype}") | ||
print(f"Type in 'b': {b.dtype}") | ||
|
||
# Check data | ||
print("*** Fetching data ***") | ||
print(f"Data in 'a': {a[0, 0:10]}") | ||
print(f"Data in 'b': {b[0, 0:10]}") | ||
|
||
# Check sizes. Note that the proxy will only have the 'touched' chunks (only 1 in this case) | ||
print("*** Sizes ***") | ||
print(f"Size in 'a': {a.schunk.cbytes}") | ||
print(f"Size in 'b': {b.schunk.cbytes}") | ||
# Check sizes on disk | ||
print("*** Disk sizes ***") | ||
print(f"Size 'a' (disk): {os.stat(a.urlpath).st_size}") | ||
print(f"Size 'b' (disk): {os.stat(b.urlpath).st_size}") | ||
|
||
# Check vlmeta | ||
print("*** VLmeta ***") | ||
print(f"VLmeta in 'a': {list(a.vlmeta)}") | ||
print(f"VLmeta in 'b': {list(b.vlmeta)}") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters