From e2fa339dbe775edded04c73d07864c1338e40772 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Wed, 6 Nov 2019 18:48:55 +0000 Subject: [PATCH 1/6] Refactor tests to use absl parametrized --- .../python/ops/bconv2d_ops_test.py | 112 +++++++++--------- .../python/ops/bsign_ops_test.py | 24 +--- 2 files changed, 64 insertions(+), 72 deletions(-) diff --git a/larq_compute_engine/python/ops/bconv2d_ops_test.py b/larq_compute_engine/python/ops/bconv2d_ops_test.py index 2473308f1..d5f32cd4e 100644 --- a/larq_compute_engine/python/ops/bconv2d_ops_test.py +++ b/larq_compute_engine/python/ops/bconv2d_ops_test.py @@ -2,6 +2,7 @@ import numpy as np import tensorflow as tf import itertools +from absl.testing import parameterized try: @@ -16,66 +17,69 @@ from ..utils import eval_op -class BConv2DTest(tf.test.TestCase): - def __test_bconv(self, bconv_op): - data_types = [np.float32, np.float64] - data_formats = ["NHWC"] - in_sizes = [[10, 10], [11, 11], [10, 11]] - filter_sizes = [[3, 3], [4, 5]] - in_channels = [1, 31, 32, 33, 64] - out_channels = [1, 16] - hw_strides = [[1, 1], [2, 2]] - paddings = ["VALID", "SAME"] +def _get_args_list(): + bconv_op = [bconv2d8, bconv2d32, bconv2d64] + data_types = [np.float32, np.float64] + data_formats = ["NHWC"] + in_sizes = [[10, 10], [11, 11], [10, 11]] + filter_sizes = [[3, 3], [4, 5]] + in_channels = [1, 31, 32, 33, 64] + out_channels = [1, 16] + hw_strides = [[1, 1], [2, 2]] + paddings = ["VALID", "SAME"] - args_lists = [ - data_types, - data_formats, - in_sizes, - filter_sizes, - in_channels, - out_channels, - hw_strides, - paddings, - ] - for args in itertools.product(*args_lists): - dtype, data_format, in_size, filter_size, in_channel, out_channel, hw_stride, padding = ( - args - ) - - batch_size = out_channel - h, w = in_size - fh, fw = filter_size - if data_format == "NHWC": - ishape = [batch_size, h, w, in_channel] - strides = [1] + hw_stride + [1] - else: - raise ValueError("Unknown data_format: " + str(data_format)) - fshape = [fh, fw, in_channel, out_channel] - - sample_list = [-1, 1] - inp = np.random.choice(sample_list, np.prod(ishape)).astype(dtype) - inp = np.reshape(inp, ishape) + return ( + bconv_op, + data_types, + data_formats, + in_sizes, + filter_sizes, + in_channels, + out_channels, + hw_strides, + paddings, + ) - filt = np.random.choice(sample_list, np.prod(fshape)).astype(dtype) - filt = np.reshape(filt, fshape) - with self.test_session(): - output = eval_op( - bconv_op(inp, filt, strides, padding, data_format=data_format) - ) - expected = eval_op( - tf.nn.conv2d(inp, filt, strides, padding, data_format=data_format) - ) - self.assertAllClose(output, expected) +class BConv2DTest(tf.test.TestCase, parameterized.TestCase): + @parameterized.parameters(args for args in itertools.product(*_get_args_list())) + def test_bconv( + self, + bconv_op, + dtype, + data_format, + in_size, + filter_size, + in_channel, + out_channel, + hw_stride, + padding, + ): + batch_size = out_channel + h, w = in_size + fh, fw = filter_size + if data_format == "NHWC": + ishape = [batch_size, h, w, in_channel] + strides = [1] + hw_stride + [1] + else: + raise ValueError("Unknown data_format: " + str(data_format)) + fshape = [fh, fw, in_channel, out_channel] - def test_bconv2d8(self): - self.__test_bconv(bconv2d8) + sample_list = [-1, 1] + inp = np.random.choice(sample_list, np.prod(ishape)).astype(dtype) + inp = np.reshape(inp, ishape) - def test_bconv2d32(self): - self.__test_bconv(bconv2d32) + filt = np.random.choice(sample_list, np.prod(fshape)).astype(dtype) + filt = np.reshape(filt, fshape) - def test_bconv2d64(self): - self.__test_bconv(bconv2d64) + with self.test_session(): + output = eval_op( + bconv_op(inp, filt, strides, padding, data_format=data_format) + ) + expected = eval_op( + tf.nn.conv2d(inp, filt, strides, padding, data_format=data_format) + ) + self.assertAllClose(output, expected) if __name__ == "__main__": diff --git a/larq_compute_engine/python/ops/bsign_ops_test.py b/larq_compute_engine/python/ops/bsign_ops_test.py index 32eceadf1..2d14cc011 100644 --- a/larq_compute_engine/python/ops/bsign_ops_test.py +++ b/larq_compute_engine/python/ops/bsign_ops_test.py @@ -1,6 +1,7 @@ """Tests for compute engine ops.""" import numpy as np import tensorflow as tf +from absl.testing import parameterized try: @@ -11,8 +12,9 @@ from compute_engine_ops.python.utils import eval_op -class SignTest(tf.test.TestCase): - def run_test_for_integers(self, dtype): +class SignTest(tf.test.TestCase, parameterized.TestCase): + @parameterized.parameters(np.int8, np.int32, np.int64) + def test_sign_int(self, dtype): with self.test_session(): x = np.array([[2, -5], [-3, 0]]).astype(dtype) expected_output = np.array([[1, -1], [-1, 1]]) @@ -20,27 +22,13 @@ def run_test_for_integers(self, dtype): # Test for +0 and -0 floating points. # We have sign(+0) = 1 and sign(-0) = -1 - def run_test_for_floating(self, dtype): + @parameterized.parameters(np.float32, np.float64) + def test_sign_float(self, dtype): with self.test_session(): x = np.array([[0.1, -5.8], [-3.0, 0.00], [0.0, -0.0]]).astype(dtype) expected_output = np.array([[1, -1], [-1, 1], [1, -1]]) self.assertAllClose(eval_op(bsign(x)), expected_output) - def test_sign_int8(self): - self.run_test_for_integers(np.int8) - - def test_sign_int32(self): - self.run_test_for_integers(np.int32) - - def test_sign_int64(self): - self.run_test_for_integers(np.int64) - - def test_sign_float32(self): - self.run_test_for_floating(np.float32) - - def test_sign_float64(self): - self.run_test_for_floating(np.float64) - if __name__ == "__main__": tf.test.main() From 978ec71ab9a93a6d8fa51a5025f5ff035917ce50 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Wed, 6 Nov 2019 20:39:49 +0000 Subject: [PATCH 2/6] Simplify itertools.product call --- larq_compute_engine/python/ops/bconv2d_ops_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/larq_compute_engine/python/ops/bconv2d_ops_test.py b/larq_compute_engine/python/ops/bconv2d_ops_test.py index d5f32cd4e..5aa3c34a9 100644 --- a/larq_compute_engine/python/ops/bconv2d_ops_test.py +++ b/larq_compute_engine/python/ops/bconv2d_ops_test.py @@ -17,7 +17,7 @@ from ..utils import eval_op -def _get_args_list(): +def _get_test_cases(): bconv_op = [bconv2d8, bconv2d32, bconv2d64] data_types = [np.float32, np.float64] data_formats = ["NHWC"] @@ -42,7 +42,7 @@ def _get_args_list(): class BConv2DTest(tf.test.TestCase, parameterized.TestCase): - @parameterized.parameters(args for args in itertools.product(*_get_args_list())) + @parameterized.parameters(itertools.product(*_get_test_cases())) def test_bconv( self, bconv_op, From b3cedd359fd71e3255528f7c91c3d35931d7400c Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Thu, 7 Nov 2019 18:04:50 +0000 Subject: [PATCH 3/6] Use pytest.parameterized to run tests --- .github/workflows/unittests.yml | 4 +- .../python/ops/bconv2d_ops_test.py | 46 ++++++------------- .../python/ops/bsign_ops_test.py | 16 +++---- larq_compute_engine/python/utils.py | 12 ++++- 4 files changed, 35 insertions(+), 43 deletions(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index c21de60d1..23e0b74aa 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -20,7 +20,7 @@ jobs: - name: Build PIP Package run: bazel-bin/build_pip_pkg artifacts - name: Install PIP Package - run: pip3 install artifacts/*.whl + run: pip3 install pytest artifacts/*.whl - name: Run Python Tests run: bazel test larq_compute_engine:py_tests --python_top=//larq_compute_engine:pyruntime --test_output=errors @@ -41,7 +41,7 @@ jobs: - name: Build PIP Package run: bazel-bin/build_pip_pkg artifacts - name: Install PIP Package - run: pip3 install artifacts/*.whl + run: pip3 install pytest artifacts/*.whl - name: Run Python Tests run: bazel test larq_compute_engine:py_tests --python_top=//larq_compute_engine:pyruntime --test_output=errors diff --git a/larq_compute_engine/python/ops/bconv2d_ops_test.py b/larq_compute_engine/python/ops/bconv2d_ops_test.py index 5aa3c34a9..308371bf6 100644 --- a/larq_compute_engine/python/ops/bconv2d_ops_test.py +++ b/larq_compute_engine/python/ops/bconv2d_ops_test.py @@ -1,8 +1,8 @@ """Tests for compute engine ops.""" import numpy as np import tensorflow as tf -import itertools -from absl.testing import parameterized +import sys +import pytest try: @@ -11,38 +11,22 @@ bconv2d32, bconv2d64, ) - from larq_compute_engine.python.utils import eval_op + from larq_compute_engine.python.utils import eval_op, TestCase except ImportError: from compute_engine_ops import bconv2d8, bconv2d32, bconv2d64 - from ..utils import eval_op + from ..utils import eval_op, TestCase -def _get_test_cases(): - bconv_op = [bconv2d8, bconv2d32, bconv2d64] - data_types = [np.float32, np.float64] - data_formats = ["NHWC"] - in_sizes = [[10, 10], [11, 11], [10, 11]] - filter_sizes = [[3, 3], [4, 5]] - in_channels = [1, 31, 32, 33, 64] - out_channels = [1, 16] - hw_strides = [[1, 1], [2, 2]] - paddings = ["VALID", "SAME"] - - return ( - bconv_op, - data_types, - data_formats, - in_sizes, - filter_sizes, - in_channels, - out_channels, - hw_strides, - paddings, - ) - - -class BConv2DTest(tf.test.TestCase, parameterized.TestCase): - @parameterized.parameters(itertools.product(*_get_test_cases())) +class BConv2DTest(TestCase): + @pytest.mark.parameterize("bconv_op", [bconv2d8, bconv2d32, bconv2d64]) + @pytest.mark.parameterize("data_type", [np.float32, np.float64]) + @pytest.mark.parameterize("data_format", ["NHWC"]) + @pytest.mark.parameterize("in_size", [[10, 10], [11, 11], [10, 11]]) + @pytest.mark.parameterize("filter_size", [[3, 3], [4, 5]]) + @pytest.mark.parameterize("in_channel", [1, 31, 32, 33, 64]) + @pytest.mark.parameterize("out_channel", [1, 16]) + @pytest.mark.parameterize("hw_stride", [[1, 1], [2, 2]]) + @pytest.mark.parameterize("padding", ["VALID", "SAME"]) def test_bconv( self, bconv_op, @@ -83,4 +67,4 @@ def test_bconv( if __name__ == "__main__": - tf.test.main() + sys.exit(pytest.main([__file__])) diff --git a/larq_compute_engine/python/ops/bsign_ops_test.py b/larq_compute_engine/python/ops/bsign_ops_test.py index 2d14cc011..c5b7ba836 100644 --- a/larq_compute_engine/python/ops/bsign_ops_test.py +++ b/larq_compute_engine/python/ops/bsign_ops_test.py @@ -1,19 +1,19 @@ """Tests for compute engine ops.""" import numpy as np -import tensorflow as tf -from absl.testing import parameterized +import sys +import pytest try: from larq_compute_engine.python.ops.compute_engine_ops import bsign - from larq_compute_engine.python.utils import eval_op + from larq_compute_engine.python.utils import eval_op, TestCase except ImportError: from compute_engine_ops import bsign - from compute_engine_ops.python.utils import eval_op + from compute_engine_ops.python.utils import eval_op, TestCase -class SignTest(tf.test.TestCase, parameterized.TestCase): - @parameterized.parameters(np.int8, np.int32, np.int64) +class SignTest(TestCase): + @pytest.mark.parameterize("dtype", [np.int8, np.int32, np.int64]) def test_sign_int(self, dtype): with self.test_session(): x = np.array([[2, -5], [-3, 0]]).astype(dtype) @@ -22,7 +22,7 @@ def test_sign_int(self, dtype): # Test for +0 and -0 floating points. # We have sign(+0) = 1 and sign(-0) = -1 - @parameterized.parameters(np.float32, np.float64) + @pytest.mark.parameterize("dtype", [np.float32, np.float64]) def test_sign_float(self, dtype): with self.test_session(): x = np.array([[0.1, -5.8], [-3.0, 0.00], [0.0, -0.0]]).astype(dtype) @@ -31,4 +31,4 @@ def test_sign_float(self, dtype): if __name__ == "__main__": - tf.test.main() + sys.exit(pytest.main([__file__])) diff --git a/larq_compute_engine/python/utils.py b/larq_compute_engine/python/utils.py index 05618a785..51dbe7ea9 100644 --- a/larq_compute_engine/python/utils.py +++ b/larq_compute_engine/python/utils.py @@ -1,10 +1,11 @@ """Utils for testing compute engine ops.""" -from tensorflow import __version__ +import tensorflow as tf + from distutils.version import LooseVersion def tf_2_or_newer(): - return LooseVersion(__version__) >= LooseVersion("2.0") + return LooseVersion(tf.__version__) >= LooseVersion("2.0") def eval_op(op): @@ -12,3 +13,10 @@ def eval_op(op): return op # op.numpy() also works else: return op.eval() + + +class TestCase(tf.test.TestCase): + """A test case that can be run with pytest parameterized tests""" + + def run(self, **kargs): + pass From e5f1c6eaeab3e2700fa7b772e382e6abf64550f3 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Fri, 8 Nov 2019 17:08:49 +0000 Subject: [PATCH 4/6] Simplify pytest testing --- .../python/ops/bconv2d_ops_test.py | 87 +++++++++---------- .../python/ops/bsign_ops_test.py | 38 ++++---- larq_compute_engine/python/utils.py | 13 +-- 3 files changed, 62 insertions(+), 76 deletions(-) diff --git a/larq_compute_engine/python/ops/bconv2d_ops_test.py b/larq_compute_engine/python/ops/bconv2d_ops_test.py index 308371bf6..5b613af22 100644 --- a/larq_compute_engine/python/ops/bconv2d_ops_test.py +++ b/larq_compute_engine/python/ops/bconv2d_ops_test.py @@ -11,59 +11,54 @@ bconv2d32, bconv2d64, ) - from larq_compute_engine.python.utils import eval_op, TestCase + from larq_compute_engine.python.utils import eval_op except ImportError: from compute_engine_ops import bconv2d8, bconv2d32, bconv2d64 - from ..utils import eval_op, TestCase + from ..utils import eval_op -class BConv2DTest(TestCase): - @pytest.mark.parameterize("bconv_op", [bconv2d8, bconv2d32, bconv2d64]) - @pytest.mark.parameterize("data_type", [np.float32, np.float64]) - @pytest.mark.parameterize("data_format", ["NHWC"]) - @pytest.mark.parameterize("in_size", [[10, 10], [11, 11], [10, 11]]) - @pytest.mark.parameterize("filter_size", [[3, 3], [4, 5]]) - @pytest.mark.parameterize("in_channel", [1, 31, 32, 33, 64]) - @pytest.mark.parameterize("out_channel", [1, 16]) - @pytest.mark.parameterize("hw_stride", [[1, 1], [2, 2]]) - @pytest.mark.parameterize("padding", ["VALID", "SAME"]) - def test_bconv( - self, - bconv_op, - dtype, - data_format, - in_size, - filter_size, - in_channel, - out_channel, - hw_stride, - padding, - ): - batch_size = out_channel - h, w = in_size - fh, fw = filter_size - if data_format == "NHWC": - ishape = [batch_size, h, w, in_channel] - strides = [1] + hw_stride + [1] - else: - raise ValueError("Unknown data_format: " + str(data_format)) - fshape = [fh, fw, in_channel, out_channel] +@pytest.mark.parametrize("bconv_op", [bconv2d8, bconv2d32, bconv2d64]) +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +@pytest.mark.parametrize("data_format", ["NHWC"]) +@pytest.mark.parametrize("in_size", [[10, 10], [11, 11], [10, 11]]) +@pytest.mark.parametrize("filter_size", [[3, 3], [4, 5]]) +@pytest.mark.parametrize("in_channel", [1, 31, 32, 33, 64]) +@pytest.mark.parametrize("out_channel", [1, 16]) +@pytest.mark.parametrize("hw_stride", [[1, 1], [2, 2]]) +@pytest.mark.parametrize("padding", ["VALID", "SAME"]) +def test_bconv( + bconv_op, + dtype, + data_format, + in_size, + filter_size, + in_channel, + out_channel, + hw_stride, + padding, +): + batch_size = out_channel + h, w = in_size + fh, fw = filter_size + if data_format == "NHWC": + ishape = [batch_size, h, w, in_channel] + strides = [1] + hw_stride + [1] + else: + raise ValueError("Unknown data_format: " + str(data_format)) + fshape = [fh, fw, in_channel, out_channel] - sample_list = [-1, 1] - inp = np.random.choice(sample_list, np.prod(ishape)).astype(dtype) - inp = np.reshape(inp, ishape) + sample_list = [-1, 1] + inp = np.random.choice(sample_list, np.prod(ishape)).astype(dtype) + inp = np.reshape(inp, ishape) - filt = np.random.choice(sample_list, np.prod(fshape)).astype(dtype) - filt = np.reshape(filt, fshape) + filt = np.random.choice(sample_list, np.prod(fshape)).astype(dtype) + filt = np.reshape(filt, fshape) - with self.test_session(): - output = eval_op( - bconv_op(inp, filt, strides, padding, data_format=data_format) - ) - expected = eval_op( - tf.nn.conv2d(inp, filt, strides, padding, data_format=data_format) - ) - self.assertAllClose(output, expected) + output = eval_op(bconv_op(inp, filt, strides, padding, data_format=data_format)) + expected = eval_op( + tf.nn.conv2d(inp, filt, strides, padding, data_format=data_format) + ) + np.testing.assert_allclose(output, expected) if __name__ == "__main__": diff --git a/larq_compute_engine/python/ops/bsign_ops_test.py b/larq_compute_engine/python/ops/bsign_ops_test.py index c5b7ba836..dccf42922 100644 --- a/larq_compute_engine/python/ops/bsign_ops_test.py +++ b/larq_compute_engine/python/ops/bsign_ops_test.py @@ -6,28 +6,26 @@ try: from larq_compute_engine.python.ops.compute_engine_ops import bsign - from larq_compute_engine.python.utils import eval_op, TestCase + from larq_compute_engine.python.utils import eval_op except ImportError: from compute_engine_ops import bsign - from compute_engine_ops.python.utils import eval_op, TestCase - - -class SignTest(TestCase): - @pytest.mark.parameterize("dtype", [np.int8, np.int32, np.int64]) - def test_sign_int(self, dtype): - with self.test_session(): - x = np.array([[2, -5], [-3, 0]]).astype(dtype) - expected_output = np.array([[1, -1], [-1, 1]]) - self.assertAllClose(eval_op(bsign(x)), expected_output) - - # Test for +0 and -0 floating points. - # We have sign(+0) = 1 and sign(-0) = -1 - @pytest.mark.parameterize("dtype", [np.float32, np.float64]) - def test_sign_float(self, dtype): - with self.test_session(): - x = np.array([[0.1, -5.8], [-3.0, 0.00], [0.0, -0.0]]).astype(dtype) - expected_output = np.array([[1, -1], [-1, 1], [1, -1]]) - self.assertAllClose(eval_op(bsign(x)), expected_output) + from compute_engine_ops.python.utils import eval_op + + +@pytest.mark.parametrize("dtype", [np.int8, np.int32, np.int64]) +def test_sign_int(dtype): + x = np.array([[2, -5], [-3, 0]]).astype(dtype) + expected_output = np.array([[1, -1], [-1, 1]]) + np.testing.assert_allclose(eval_op(bsign(x)), expected_output) + + +# Test for +0 and -0 floating points. +# We have sign(+0) = 1 and sign(-0) = -1 +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +def test_sign_float(dtype): + x = np.array([[0.1, -5.8], [-3.0, 0.00], [0.0, -0.0]]).astype(dtype) + expected_output = np.array([[1, -1], [-1, 1], [1, -1]]) + np.testing.assert_allclose(eval_op(bsign(x)), expected_output) if __name__ == "__main__": diff --git a/larq_compute_engine/python/utils.py b/larq_compute_engine/python/utils.py index 51dbe7ea9..ec0c84633 100644 --- a/larq_compute_engine/python/utils.py +++ b/larq_compute_engine/python/utils.py @@ -9,14 +9,7 @@ def tf_2_or_newer(): def eval_op(op): - if tf_2_or_newer(): - return op # op.numpy() also works - else: + try: + return op.numpy() + except: return op.eval() - - -class TestCase(tf.test.TestCase): - """A test case that can be run with pytest parameterized tests""" - - def run(self, **kargs): - pass From aea3091066c1a5cb3d8763fa39e70a9834eb1963 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Fri, 8 Nov 2019 17:22:18 +0000 Subject: [PATCH 5/6] Use keras.get_value to make eval op TF 1 compatible --- larq_compute_engine/python/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/larq_compute_engine/python/utils.py b/larq_compute_engine/python/utils.py index ec0c84633..c0eb604bb 100644 --- a/larq_compute_engine/python/utils.py +++ b/larq_compute_engine/python/utils.py @@ -9,7 +9,4 @@ def tf_2_or_newer(): def eval_op(op): - try: - return op.numpy() - except: - return op.eval() + return tf.keras.backend.get_value(op) From 07b4908e96de2a9d4e62d983739a6882b796b68c Mon Sep 17 00:00:00 2001 From: Tom Bannink Date: Mon, 11 Nov 2019 16:18:10 +0100 Subject: [PATCH 6/6] Set python unit test timeouts in bazel --- larq_compute_engine/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/larq_compute_engine/BUILD b/larq_compute_engine/BUILD index c99ccb337..84e396c73 100644 --- a/larq_compute_engine/BUILD +++ b/larq_compute_engine/BUILD @@ -267,6 +267,7 @@ py_test( ":compute_engine_ops_py", ":compute_engine_utils_py", ], + size = "small", python_version = "PY3", srcs_version = "PY3", ) @@ -281,6 +282,7 @@ py_test( ":compute_engine_ops_py", ":compute_engine_utils_py", ], + size = "large", python_version = "PY3", srcs_version = "PY3", )