Skip to content

Commit

Permalink
.eval() -> .compute()
Browse files Browse the repository at this point in the history
  • Loading branch information
FrancescAlted committed Oct 18, 2024
1 parent d271dc7 commit e8c825f
Show file tree
Hide file tree
Showing 23 changed files with 72 additions and 72 deletions.
4 changes: 2 additions & 2 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ Here it is a simple example:
expr = ((a ** 3 + blosc2.sin(c * 2)) < b) & (c > 0)
# Evaluate and get a NDArray as result
out = expr.eval()
out = expr.compute()
print(out.info)
As you can see, the `NDArray` instances are very similar to NumPy arrays, but behind the scenes
Expand All @@ -122,7 +122,7 @@ that you can reach when the operands fit comfortably in-memory:
In this case, performance is a bit far from top-level libraries like Numexpr or Numba, but
it is still pretty nice (and probably using CPUs with more cores than M2 would allow closing the
performance gap even further). One important thing to know is that the memory consumption when
using the `LazyArray.eval()` method is very low, because the output is an `NDArray` object that
using the `LazyArray.compute()` method is very low, because the output is an `NDArray` object that
is compressed and in-memory by default. On its hand `LazyArray.__getitem__()` method returns
an actual NumPy array, so it is not recommended to use it for large datasets, as it will consume
quite a bit of memory (but it can still be convenient for small outputs).
Expand Down
4 changes: 2 additions & 2 deletions bench/lazyarray-expr-small-dask.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@
"source": [
"%%mprof_run 1.lazyexpr::eval-LZ4-1\n",
"# Evaluate and get a NDArray as result\n",
"out = expr.eval()"
"out = expr.compute()"
]
},
{
Expand Down Expand Up @@ -452,7 +452,7 @@
"source": [
"%%mprof_run 2.lazyexpr::eval-nocompr\n",
"# Evaluate and get a NDArray as result\n",
"out4 = expr.eval()"
"out4 = expr.compute()"
]
},
{
Expand Down
4 changes: 2 additions & 2 deletions bench/ndarray/broadcast_expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@
c = a * b
# print(f"Elapsed time (expr): {time() - t0:.6f} s")
t0 = time()
# d = c.eval(cparams=dict(codec=codec, clevel=5), chunks=(chunks, chunks), blocks=(blocks, blocks))
d = c.eval(cparams=dict(codec=codec, clevel=5))
# d = c.compute(cparams=dict(codec=codec, clevel=5), chunks=(chunks, chunks), blocks=(blocks, blocks))
d = c.compute(cparams=dict(codec=codec, clevel=5))
print(f"Elapsed time (eval): {time() - t0:.6f} s")
# print(d[:])
print(f"cratio: {d.schunk.cratio:.2f}x")
Expand Down
6 changes: 3 additions & 3 deletions bench/ndarray/eval_expr_numba.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ def udf_numba(inputs, output, offset):
b2expr = expr.replace("sin", "blosc2.sin").replace("cos", "blosc2.cos")
c = eval(b2expr, b2vardict)
t0 = time()
d = c.eval()
d = c.compute()
print("LazyExpr+eval took %.3f s" % (time() - t0))
# Check
np.testing.assert_allclose(d[:], npres, rtol=rtol, atol=atol)
Expand All @@ -139,7 +139,7 @@ def udf_numba(inputs, output, offset):
# actual benchmark
# eval() uses the udf function as a prefilter
t0 = time()
res = expr_.eval()
res = expr_.compute()
print("LazyUDF+eval took %.3f s" % (time() - t0))
np.testing.assert_allclose(res[...], npres, rtol=rtol, atol=atol)
# getitem uses the same compiled function but as a postfilter
Expand All @@ -152,7 +152,7 @@ def udf_numba(inputs, output, offset):
chunks=chunks, blocks=blocks, cparams=cparams)
# getitem but using chunked evaluation
t0 = time()
res = expr_.eval()
res = expr_.compute()
print("LazyUDF+chunked_eval took %.3f s" % (time() - t0))
np.testing.assert_allclose(res[...], npres, rtol=rtol, atol=atol)
t0 = time()
Expand Down
2 changes: 1 addition & 1 deletion bench/ndarray/eval_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
c = a**2 + b**2 > 2 * a * b + 1
# Evaluate: output is a NDArray
t0 = time()
d = c.eval(cparams=cparams)
d = c.compute(cparams=cparams)
t = time() - t0
print(f"Time to evaluate field expression (eval): {t:.3f} s; {nps.nbytes/2**30/t:.2f} GB/s")

Expand Down
6 changes: 3 additions & 3 deletions bench/ndarray/eval_where.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
# Evaluate: output is a NDArray
t0 = time()
c = a**2 + b**2 > 2 * a * b + 1
d = c.where(0, 1).eval(cparams=cparams)
d = c.where(0, 1).compute(cparams=cparams)
t = time() - t0
print(f"Time to evaluate where expression (eval): {t:.3f} s; {nps.nbytes/2**30/t:.3f} GB/s")

Expand All @@ -69,13 +69,13 @@

# Evaluate and get row values: output is a NDArray
t0 = time()
npd = s[a**2 + b**2 > 2 * a * b + 1].eval(cparams=cparams)
npd = s[a**2 + b**2 > 2 * a * b + 1].compute(cparams=cparams)
t = time() - t0
print(f"Time to get row values (eval): {t:.3f} s; {nps.nbytes/2**30/t:.3f} GB/s")

# Evaluate and get row values: output is a NDArray
t0 = time()
npd = s['a**2 + b**2 > 2 * a * b + 1'].eval(cparams=cparams)
npd = s['a**2 + b**2 > 2 * a * b + 1'].compute(cparams=cparams)
t = time() - t0
print(f"Time to get row values (eval, string): {t:.3f} s; {nps.nbytes/2**30/t:.3f} GB/s")

Expand Down
12 changes: 6 additions & 6 deletions bench/ndarray/lazyarray-expr.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@
"source": [
"%%mprof_run 0.lazyexpr::mmap-warmup\n",
"# Warm memory-map cache\n",
"out1 = expr.eval()"
"out1 = expr.compute()"
]
},
{
Expand All @@ -206,7 +206,7 @@
"source": [
"%%mprof_run 1.lazyexpr::eval\n",
"# Evaluate and get a NDArray as result\n",
"out1 = expr.eval()"
"out1 = expr.compute()"
]
},
{
Expand Down Expand Up @@ -325,7 +325,7 @@
"outputs": [],
"source": [
"# Warm numba jit and compile\n",
"out1 = lzyudf.eval()"
"out1 = lzyudf.compute()"
]
},
{
Expand All @@ -341,7 +341,7 @@
"outputs": [],
"source": [
"#%%mprof_run 2.lazyudf::eval\n",
"#out2 = lzyudf.eval()"
"#out2 = lzyudf.compute()"
]
},
{
Expand Down Expand Up @@ -451,7 +451,7 @@
"source": [
"%%mprof_run 6.lazyexpr::eval-second-time\n",
"# Evaluate and get a NDArray as result\n",
"out6 = expr.eval()"
"out6 = expr.compute()"
]
},
{
Expand All @@ -476,7 +476,7 @@
"source": [
"%%mprof_run 6.lazyexpr::eval-second-time\n",
"# Evaluate and get a NDArray as result\n",
"out6 = expr.eval()"
"out6 = expr.compute()"
]
},
{
Expand Down
2 changes: 1 addition & 1 deletion bench/ndarray/reduce_expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
b2expr = expr.replace("sin", "blosc2.sin").replace("cos", "blosc2.cos")
c = eval(b2expr, b2vardict)
t0 = time()
d = c.eval()
d = c.compute()
d = d.sum(axis=axis) #, dtype=npres.dtype)
print("LazyExpr+eval took %.3f s" % (time() - t0))
# Check
Expand Down
8 changes: 4 additions & 4 deletions doc/getting_started/tutorials/03.lazyarray-expressions.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@
}
],
"source": [
"d = c.eval() # evaluate the expression\n",
"d = c.compute() # evaluate the expression\n",
"print(f\"Class: {type(d)}\")\n",
"print(f\"Compression ratio: {d.schunk.cratio:.2f}x\")"
]
Expand Down Expand Up @@ -148,7 +148,7 @@
"cparams = blosc2.CParams(\n",
" codec=blosc2.Codec.ZSTD, filters=[blosc2.Filter.BITSHUFFLE], clevel=9, filters_meta=[0]\n",
")\n",
"d = c.eval(cparams=cparams)\n",
"d = c.compute(cparams=cparams)\n",
"print(f\"Compression ratio: {d.schunk.cratio:.2f}x\")"
]
},
Expand Down Expand Up @@ -262,7 +262,7 @@
}
],
"source": [
"d2 = c2.eval()\n",
"d2 = c2.compute()\n",
"print(f\"Compression ratio: {d2.schunk.cratio:.2f}x\")"
]
},
Expand Down Expand Up @@ -602,7 +602,7 @@
],
"source": [
"c2 = a + b2\n",
"d2 = c2.eval()\n",
"d2 = c2.compute()\n",
"print(f\"Compression ratio: {d2.schunk.cratio:.2f}x, shape: {d2.shape}\")"
]
},
Expand Down
2 changes: 1 addition & 1 deletion doc/getting_started/tutorials/03.lazyarray-udf.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@
}
],
"source": [
"c = b.eval(urlpath=\"res.b2nd\", mode=\"w\")\n",
"c = b.compute(urlpath=\"res.b2nd\", mode=\"w\")\n",
"print(f\"Class: {type(c)}\")\n",
"print(c.info)"
]
Expand Down
2 changes: 1 addition & 1 deletion examples/ndarray/broadcast_expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
# print(d, d.shape, d.dtype)
# print(d.expression, d.operands)
assert isinstance(d, blosc2.LazyExpr)
e = d.eval()
e = d.compute()
print(e)
assert isinstance(d, blosc2.LazyExpr)
# Check
Expand Down
2 changes: 1 addition & 1 deletion examples/ndarray/c2array_expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,5 @@
np.testing.assert_allclose(c[:], a[:] + b[:])

# Get an NDArray instance instead of a NumPy array
ndarr = c.eval()
ndarr = c.compute()
np.testing.assert_allclose(ndarr[:], a[:] + b[:])
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
# Get a LazyExpr instance
c = a**2 + b**2 + 2 * a * b + 1
# Evaluate: output is a NDArray
d = c.eval()
d = c.compute()
# Check
assert isinstance(d, blosc2.NDArray)
assert np.allclose(d[:], npc)
Expand Down Expand Up @@ -54,7 +54,7 @@
dc = blosc2.open("c.b2nd")

# Evaluate: output is a NDArray
dc2 = dc.eval()
dc2 = dc.compute()
# Check
assert isinstance(dc2, blosc2.NDArray)
assert np.allclose(dc2[:], npc)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
c = a**2 + b**2 > 2 * a * b + 1

# Evaluate: output is a NDArray
d = c.eval()
d = c.compute()
# Check
assert isinstance(d, blosc2.NDArray)
assert np.allclose(d[:], npc)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def func_numba(inputs_tuple, output, offset):

lazyarray = blosc2.lazyudf(func_numba, (npa,), npa.dtype)
print(lazyarray.info)
res = lazyarray.eval()
res = lazyarray.compute()
print(res.info)
np.testing.assert_allclose(res[...], npc)
print("Numba + LazyArray evaluated correctly!")
File renamed without changes.
6 changes: 3 additions & 3 deletions examples/ndarray/general_expressions.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,17 @@

# Get a LazyExpr instance with all NDArray operands
c = blosc2.lazyexpr("a**2 + b**2 + 2 * a * b + 1", {"a": a, "b": b})
d = c.eval()
d = c.compute()
assert np.allclose(d[:], npc)

# A LazyExpr instance with a mix of NDArray and NumPy operands
c = blosc2.lazyexpr("a**2 + b**2 + 2 * a * b + 1", {"a": npa, "b": b})
d = c.eval()
d = c.compute()
assert np.allclose(d[:], npc)

# A LazyExpr instance with a all NumPy operands
c = blosc2.lazyexpr("a**2 + b**2 + 2 * a * b + 1", {"a": npa, "b": npb})
d = c.eval()
d = c.compute()
assert np.allclose(d[:], npc)

# Evaluate partial slices
Expand Down
2 changes: 1 addition & 1 deletion examples/ndarray/reduce_expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
# print(d, d.shape, d.dtype)
# print(d.expression, d.operands)
assert isinstance(d, blosc2.LazyExpr)
e = d.eval()
e = d.compute()
# print(e)
assert isinstance(d, blosc2.LazyExpr)
# Check
Expand Down
24 changes: 12 additions & 12 deletions tests/ndarray/test_c2array_expr.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,11 @@ def test_simple(chunks_blocks, c2sub_context):
sl = slice(10)
expr = a1 + a3
nres = ne.evaluate("na1 + na3")
res = expr.eval(item=sl)
res = expr.compute(item=sl)
np.testing.assert_allclose(res[:], nres[sl])

# All
res = expr.eval()
res = expr.compute()
np.testing.assert_allclose(res[:], nres)


Expand Down Expand Up @@ -98,7 +98,7 @@ def test_ixxx(chunks_blocks, c2sub_context):
expr += 5 # __iadd__
expr /= 7 # __itruediv__
expr **= 2.3 # __ipow__
res = expr.eval()
res = expr.compute()
nres = ne.evaluate("(((na1 ** 3 + na2 ** 2 + na3 ** 3 - na4 + 3) + 5) / 7) ** 2.3")
np.testing.assert_allclose(res[:], nres)

Expand All @@ -111,7 +111,7 @@ def test_complex(c2sub_context):
expr += 2
nres = ne.evaluate("tan(na1) * sin(na2) + (sqrt(na4) * 2) + 2")
# eval
res = expr.eval()
res = expr.compute()
np.testing.assert_allclose(res[:], nres)
# __getitem__
res = expr[:]
Expand Down Expand Up @@ -141,30 +141,30 @@ def test_mix_operands(chunks_blocks, c2sub_context):
expr = a1 + b1
nres = ne.evaluate("na1 + na1")
np.testing.assert_allclose(expr[:], nres)
np.testing.assert_allclose(expr.eval()[:], nres)
np.testing.assert_allclose(expr.compute()[:], nres)

expr = a1 + b3
nres = ne.evaluate("na1 + na3")
np.testing.assert_allclose(expr[:], nres)
np.testing.assert_allclose(expr.eval()[:], nres)
np.testing.assert_allclose(expr.compute()[:], nres)

expr = a1 + b1 + a2 + b3
nres = ne.evaluate("na1 + na1 + na2 + na3")
np.testing.assert_allclose(expr[:], nres)
np.testing.assert_allclose(expr.eval()[:], nres)
np.testing.assert_allclose(expr.compute()[:], nres)

expr = a1 + a2 + b1 + b3
nres = ne.evaluate("na1 + na2 + na1 + na3")
np.testing.assert_allclose(expr[:], nres)
np.testing.assert_allclose(expr.eval()[:], nres)
np.testing.assert_allclose(expr.compute()[:], nres)

# TODO: fix this
# expr = a1 + na1 * b3
# print(type(expr))
# print("expression: ", expr.expression)
# nres = ne.evaluate("na1 + na1 * na3")
# np.testing.assert_allclose(expr[:], nres)
# np.testing.assert_allclose(expr.eval()[:], nres)
# np.testing.assert_allclose(expr.compute()[:], nres)


# Tests related with save method
Expand All @@ -176,7 +176,7 @@ def test_save(c2sub_context):
expr = a1 * a2 + a3 - a4 * 3
nres = ne.evaluate("na1 * na2 + na3 - na4 * 3")

res = expr.eval()
res = expr.compute()
assert res.dtype == np.float64
np.testing.assert_allclose(res[:], nres, rtol=tol, atol=tol)

Expand All @@ -187,7 +187,7 @@ def test_save(c2sub_context):
del op
del expr
expr = blosc2.open(urlpath)
res = expr.eval()
res = expr.compute()
assert res.dtype == np.float64
np.testing.assert_allclose(res[:], nres, rtol=tol, atol=tol)
# Test getitem
Expand Down Expand Up @@ -236,7 +236,7 @@ def test_broadcasting(broadcast_fixture):
expr = expr1 - expr2
assert expr.shape == a1.shape
nres = ne.evaluate("na1 + na2 - (na1 * na2 + 1)")
res = expr.eval()
res = expr.compute()
np.testing.assert_allclose(res[:], nres)
res = expr[:]
np.testing.assert_allclose(res, nres)
Loading

0 comments on commit e8c825f

Please sign in to comment.