From 27b7ee44be341989ee75608a7502d80e3d7349d5 Mon Sep 17 00:00:00 2001 From: nmcguire101 Date: Fri, 8 Apr 2022 15:14:21 -0400 Subject: [PATCH 01/11] Added: Improved test tests\ignite\contrib\metrics\regression\test__base.py using pytest parametrize --- .../contrib/metrics/regression/test__base.py | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/tests/ignite/contrib/metrics/regression/test__base.py b/tests/ignite/contrib/metrics/regression/test__base.py index ee5fbe48b48..f3059d9fa60 100644 --- a/tests/ignite/contrib/metrics/regression/test__base.py +++ b/tests/ignite/contrib/metrics/regression/test__base.py @@ -4,7 +4,8 @@ from ignite.contrib.metrics.regression._base import _BaseRegression -def test_base_regression_shapes(): +@pytest.mark.parametrize("OutputShapes", [("NDimensionShape", [1, 1, 1]), ("NDimensionFloat", [1, 1])]) +def test_base_regression_shapes(OutputShapes): class L1(_BaseRegression): def reset(self): self._sum_of_errors = 0.0 @@ -19,21 +20,25 @@ def compute(self): m = L1() - with pytest.raises(ValueError, match=r"Input y_pred should have shape \(N,\) or \(N, 1\)"): - y = torch.rand([1, 1, 1]) - m.update((y, y)) - - with pytest.raises(ValueError, match=r"Input y should have shape \(N,\) or \(N, 1\)"): - y = torch.rand([1, 1, 1]) - m.update((torch.rand(1, 1), y)) + (input_type, input_values) = OutputShapes with pytest.raises(ValueError, match=r"Input data shapes should be the same, but given"): m.update((torch.rand(2), torch.rand(2, 1))) - with pytest.raises(TypeError, match=r"Input y_pred dtype should be float"): - y = torch.tensor([1, 1]) - m.update((y, y)) + if input_type == "NDimensionShape": + with pytest.raises(ValueError, match=r"Input y_pred should have shape \(N,\) or \(N, 1\)"): + y = torch.rand(input_values) + m.update((y, y)) + + with pytest.raises(ValueError, match=r"Input y should have shape \(N,\) or \(N, 1\)"): + y = torch.rand(input_values) + m.update((torch.rand(1, 1), y)) + + elif input_type == "NDimensionFloat": + with pytest.raises(TypeError, match=r"Input y_pred dtype should be float"): + y = torch.tensor(input_values) + m.update((y, y)) - with pytest.raises(TypeError, match=r"Input y dtype should be float"): - y = torch.tensor([1, 1]) - m.update((y.float(), y)) + with pytest.raises(TypeError, match=r"Input y dtype should be float"): + y = torch.tensor(input_values) + m.update((y.float(), y)) From a1977300fca153bc59c8d08623fe446a59f6c80b Mon Sep 17 00:00:00 2001 From: nmcguire101 Date: Sat, 9 Apr 2022 10:29:48 -0400 Subject: [PATCH 02/11] Undid test__base.py changes --- .../contrib/metrics/regression/test__base.py | 33 ++++++++----------- .../metrics/test_root_mean_squared_error.py | 29 ++++++++-------- 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/tests/ignite/contrib/metrics/regression/test__base.py b/tests/ignite/contrib/metrics/regression/test__base.py index f3059d9fa60..ee5fbe48b48 100644 --- a/tests/ignite/contrib/metrics/regression/test__base.py +++ b/tests/ignite/contrib/metrics/regression/test__base.py @@ -4,8 +4,7 @@ from ignite.contrib.metrics.regression._base import _BaseRegression -@pytest.mark.parametrize("OutputShapes", [("NDimensionShape", [1, 1, 1]), ("NDimensionFloat", [1, 1])]) -def test_base_regression_shapes(OutputShapes): +def test_base_regression_shapes(): class L1(_BaseRegression): def reset(self): self._sum_of_errors = 0.0 @@ -20,25 +19,21 @@ def compute(self): m = L1() - (input_type, input_values) = OutputShapes + with pytest.raises(ValueError, match=r"Input y_pred should have shape \(N,\) or \(N, 1\)"): + y = torch.rand([1, 1, 1]) + m.update((y, y)) + + with pytest.raises(ValueError, match=r"Input y should have shape \(N,\) or \(N, 1\)"): + y = torch.rand([1, 1, 1]) + m.update((torch.rand(1, 1), y)) with pytest.raises(ValueError, match=r"Input data shapes should be the same, but given"): m.update((torch.rand(2), torch.rand(2, 1))) - if input_type == "NDimensionShape": - with pytest.raises(ValueError, match=r"Input y_pred should have shape \(N,\) or \(N, 1\)"): - y = torch.rand(input_values) - m.update((y, y)) - - with pytest.raises(ValueError, match=r"Input y should have shape \(N,\) or \(N, 1\)"): - y = torch.rand(input_values) - m.update((torch.rand(1, 1), y)) - - elif input_type == "NDimensionFloat": - with pytest.raises(TypeError, match=r"Input y_pred dtype should be float"): - y = torch.tensor(input_values) - m.update((y, y)) + with pytest.raises(TypeError, match=r"Input y_pred dtype should be float"): + y = torch.tensor([1, 1]) + m.update((y, y)) - with pytest.raises(TypeError, match=r"Input y dtype should be float"): - y = torch.tensor(input_values) - m.update((y.float(), y)) + with pytest.raises(TypeError, match=r"Input y dtype should be float"): + y = torch.tensor([1, 1]) + m.update((y.float(), y)) diff --git a/tests/ignite/metrics/test_root_mean_squared_error.py b/tests/ignite/metrics/test_root_mean_squared_error.py index a92e5908deb..1191f447bfb 100644 --- a/tests/ignite/metrics/test_root_mean_squared_error.py +++ b/tests/ignite/metrics/test_root_mean_squared_error.py @@ -17,7 +17,17 @@ def test_zero_sample(): rmse.compute() -def test_compute(): +@pytest.mark.parametrize( + "test_cases", + [ + (torch.empty(10).uniform_(0, 10), torch.empty(10).uniform_(0, 10), 1), + (torch.empty(10, 1).uniform_(-10, 10), torch.empty(10, 1).uniform_(-10, 10), 1), + # updated batches + (torch.empty(50).uniform_(0, 10), torch.empty(50).uniform_(0, 10), 16), + (torch.empty(50, 1).uniform_(-10, 10), torch.empty(50, 1).uniform_(-10, 10), 16), + ], +) +def test_compute(test_cases): rmse = RootMeanSquaredError() @@ -40,23 +50,10 @@ def _test(y_pred, y, batch_size): assert isinstance(res, float) assert pytest.approx(res) == np_res - def get_test_cases(): - - test_cases = [ - (torch.empty(10).uniform_(0, 10), torch.empty(10).uniform_(0, 10), 1), - (torch.empty(10, 1).uniform_(-10, 10), torch.empty(10, 1).uniform_(-10, 10), 1), - # updated batches - (torch.empty(50).uniform_(0, 10), torch.empty(50).uniform_(0, 10), 16), - (torch.empty(50, 1).uniform_(-10, 10), torch.empty(50, 1).uniform_(-10, 10), 16), - ] - - return test_cases - for _ in range(5): # check multiple random inputs as random exact occurencies are rare - test_cases = get_test_cases() - for y_pred, y, batch_size in test_cases: - _test(y_pred, y, batch_size) + (y_pred, y, batch_size) = test_cases + _test(y_pred, y, batch_size) def _test_distrib_integration(device, tol=1e-6): From 07b55311ee81e430faf5495deb8a46b0bb712cb2 Mon Sep 17 00:00:00 2001 From: nmcguire101 Date: Sat, 9 Apr 2022 14:38:02 -0400 Subject: [PATCH 03/11] Added dummy val to test_root_mean_squared_error.py --- tests/ignite/metrics/test_root_mean_squared_error.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/ignite/metrics/test_root_mean_squared_error.py b/tests/ignite/metrics/test_root_mean_squared_error.py index 1191f447bfb..3e1201919f0 100644 --- a/tests/ignite/metrics/test_root_mean_squared_error.py +++ b/tests/ignite/metrics/test_root_mean_squared_error.py @@ -27,7 +27,8 @@ def test_zero_sample(): (torch.empty(50, 1).uniform_(-10, 10), torch.empty(50, 1).uniform_(-10, 10), 16), ], ) -def test_compute(test_cases): +@pytest.mark.parametrize("dummy_val", [(0, 1, 2, 3, 4)]) +def test_compute(test_cases, dummy_val): rmse = RootMeanSquaredError() @@ -50,10 +51,8 @@ def _test(y_pred, y, batch_size): assert isinstance(res, float) assert pytest.approx(res) == np_res - for _ in range(5): - # check multiple random inputs as random exact occurencies are rare - (y_pred, y, batch_size) = test_cases - _test(y_pred, y, batch_size) + (y_pred, y, batch_size) = test_cases + _test(y_pred, y, batch_size) def _test_distrib_integration(device, tol=1e-6): From b3552ff961dbe8604e9c85cf2ea8563684bff7f6 Mon Sep 17 00:00:00 2001 From: nmcguire101 Date: Mon, 11 Apr 2022 15:27:08 -0400 Subject: [PATCH 04/11] Generates unique values in test_root_mean_squared_error.py --- .../metrics/test_root_mean_squared_error.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/ignite/metrics/test_root_mean_squared_error.py b/tests/ignite/metrics/test_root_mean_squared_error.py index 3e1201919f0..5c8a1ea3fc5 100644 --- a/tests/ignite/metrics/test_root_mean_squared_error.py +++ b/tests/ignite/metrics/test_root_mean_squared_error.py @@ -17,18 +17,19 @@ def test_zero_sample(): rmse.compute() -@pytest.mark.parametrize( - "test_cases", - [ +@pytest.fixture +def generate_tests(): + return [ (torch.empty(10).uniform_(0, 10), torch.empty(10).uniform_(0, 10), 1), (torch.empty(10, 1).uniform_(-10, 10), torch.empty(10, 1).uniform_(-10, 10), 1), # updated batches (torch.empty(50).uniform_(0, 10), torch.empty(50).uniform_(0, 10), 16), (torch.empty(50, 1).uniform_(-10, 10), torch.empty(50, 1).uniform_(-10, 10), 16), - ], -) -@pytest.mark.parametrize("dummy_val", [(0, 1, 2, 3, 4)]) -def test_compute(test_cases, dummy_val): + ] + + +@pytest.mark.parametrize("n_times", range(5)) +def test_compute(n_times, generate_tests): rmse = RootMeanSquaredError() @@ -51,8 +52,9 @@ def _test(y_pred, y, batch_size): assert isinstance(res, float) assert pytest.approx(res) == np_res - (y_pred, y, batch_size) = test_cases - _test(y_pred, y, batch_size) + for test in generate_tests: + (y_pred, y, batch_size) = test + _test(y_pred, y, batch_size) def _test_distrib_integration(device, tol=1e-6): From 7027027aac8944d265b9848c08e1f31c432fad74 Mon Sep 17 00:00:00 2001 From: nmcguire101 Date: Tue, 12 Apr 2022 08:56:12 -0400 Subject: [PATCH 05/11] Removed _test from test_root_mean_squared_error.py --- tests/ignite/metrics/test_root_mean_squared_error.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/ignite/metrics/test_root_mean_squared_error.py b/tests/ignite/metrics/test_root_mean_squared_error.py index 5c8a1ea3fc5..2980a9aa903 100644 --- a/tests/ignite/metrics/test_root_mean_squared_error.py +++ b/tests/ignite/metrics/test_root_mean_squared_error.py @@ -28,12 +28,13 @@ def generate_tests(): ] -@pytest.mark.parametrize("n_times", range(5)) +@pytest.mark.parametrize("n_times", range(3)) def test_compute(n_times, generate_tests): rmse = RootMeanSquaredError() - def _test(y_pred, y, batch_size): + for test in generate_tests: + (y_pred, y, batch_size) = test rmse.reset() if batch_size > 1: n_iters = y.shape[0] // batch_size + 1 @@ -52,10 +53,6 @@ def _test(y_pred, y, batch_size): assert isinstance(res, float) assert pytest.approx(res) == np_res - for test in generate_tests: - (y_pred, y, batch_size) = test - _test(y_pred, y, batch_size) - def _test_distrib_integration(device, tol=1e-6): From 590012a0185f8554790d81052cb6c9e57ecc2104 Mon Sep 17 00:00:00 2001 From: nmcguire101 Date: Wed, 13 Apr 2022 16:22:23 -0400 Subject: [PATCH 06/11] Removed for loop from test_root_mean_squared_error.py --- .../metrics/test_root_mean_squared_error.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/ignite/metrics/test_root_mean_squared_error.py b/tests/ignite/metrics/test_root_mean_squared_error.py index 2980a9aa903..d1ac9c4ffb5 100644 --- a/tests/ignite/metrics/test_root_mean_squared_error.py +++ b/tests/ignite/metrics/test_root_mean_squared_error.py @@ -28,30 +28,30 @@ def generate_tests(): ] +@pytest.mark.parametrize("test_num", range(4)) @pytest.mark.parametrize("n_times", range(3)) -def test_compute(n_times, generate_tests): +def test_compute(test_num, n_times, generate_tests): rmse = RootMeanSquaredError() - for test in generate_tests: - (y_pred, y, batch_size) = test - rmse.reset() - if batch_size > 1: - n_iters = y.shape[0] // batch_size + 1 - for i in range(n_iters): - idx = i * batch_size - rmse.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) - else: - rmse.update((y_pred, y)) - - np_y = y.numpy().ravel() - np_y_pred = y_pred.numpy().ravel() - - np_res = np.sqrt(np.power((np_y - np_y_pred), 2.0).sum() / np_y.shape[0]) - res = rmse.compute() - - assert isinstance(res, float) - assert pytest.approx(res) == np_res + (y_pred, y, batch_size) = generate_tests[test_num] + rmse.reset() + if batch_size > 1: + n_iters = y.shape[0] // batch_size + 1 + for i in range(n_iters): + idx = i * batch_size + rmse.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) + else: + rmse.update((y_pred, y)) + + np_y = y.numpy().ravel() + np_y_pred = y_pred.numpy().ravel() + + np_res = np.sqrt(np.power((np_y - np_y_pred), 2.0).sum() / np_y.shape[0]) + res = rmse.compute() + + assert isinstance(res, float) + assert pytest.approx(res) == np_res def _test_distrib_integration(device, tol=1e-6): From c115bf900a900c26d9f94e12dce241da56541333 Mon Sep 17 00:00:00 2001 From: nmcguire101 Date: Thu, 14 Apr 2022 12:02:14 -0400 Subject: [PATCH 07/11] Cleaned up test_root_mean_squared_error.py --- tests/ignite/metrics/test_root_mean_squared_error.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/ignite/metrics/test_root_mean_squared_error.py b/tests/ignite/metrics/test_root_mean_squared_error.py index d1ac9c4ffb5..25e0a944911 100644 --- a/tests/ignite/metrics/test_root_mean_squared_error.py +++ b/tests/ignite/metrics/test_root_mean_squared_error.py @@ -17,24 +17,23 @@ def test_zero_sample(): rmse.compute() -@pytest.fixture -def generate_tests(): +@pytest.fixture(params=[0, 1, 2, 3]) +def generate_tests(request): return [ (torch.empty(10).uniform_(0, 10), torch.empty(10).uniform_(0, 10), 1), (torch.empty(10, 1).uniform_(-10, 10), torch.empty(10, 1).uniform_(-10, 10), 1), # updated batches (torch.empty(50).uniform_(0, 10), torch.empty(50).uniform_(0, 10), 16), (torch.empty(50, 1).uniform_(-10, 10), torch.empty(50, 1).uniform_(-10, 10), 16), - ] + ][request.param] -@pytest.mark.parametrize("test_num", range(4)) @pytest.mark.parametrize("n_times", range(3)) -def test_compute(test_num, n_times, generate_tests): +def test_compute(n_times, generate_tests): rmse = RootMeanSquaredError() - (y_pred, y, batch_size) = generate_tests[test_num] + (y_pred, y, batch_size) = generate_tests rmse.reset() if batch_size > 1: n_iters = y.shape[0] // batch_size + 1 From 3beaf27f2ce1d38459d641271d426f9ffa69cd13 Mon Sep 17 00:00:00 2001 From: nmcguire101 Date: Sat, 16 Apr 2022 13:47:17 -0400 Subject: [PATCH 08/11] Improved test_accuracy.py --- (N | 0 tests/ignite/metrics/test_accuracy.py | 171 ++++++++++++-------------- 2 files changed, 79 insertions(+), 92 deletions(-) create mode 100644 (N diff --git a/(N b/(N new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ignite/metrics/test_accuracy.py b/tests/ignite/metrics/test_accuracy.py index da6d2a7511f..845d340b068 100644 --- a/tests/ignite/metrics/test_accuracy.py +++ b/tests/ignite/metrics/test_accuracy.py @@ -62,57 +62,50 @@ def test_binary_wrong_inputs(): acc.update((torch.randint(0, 2, size=(10,)).long(), torch.randint(0, 2, size=(10, 5, 6)).long())) -def test_binary_input(): - +@pytest.fixture(params=[item for item in range(11)]) +def test_data_binary(request): + return [ + # Binary accuracy on input of shape (N, 1) or (N, ) + (torch.randint(0, 2, size=(10,)).long(), torch.randint(0, 2, size=(10,)).long(), 1), + (torch.randint(0, 2, size=(10, 1)).long(), torch.randint(0, 2, size=(10, 1)).long(), 1), + # updated batches + (torch.randint(0, 2, size=(50,)).long(), torch.randint(0, 2, size=(50,)).long(), 16), + (torch.randint(0, 2, size=(50, 1)).long(), torch.randint(0, 2, size=(50, 1)).long(), 16), + # Binary accuracy on input of shape (N, L) + (torch.randint(0, 2, size=(10, 5)).long(), torch.randint(0, 2, size=(10, 5)).long(), 1), + (torch.randint(0, 2, size=(10, 8)).long(), torch.randint(0, 2, size=(10, 8)).long(), 1), + # updated batches + (torch.randint(0, 2, size=(50, 5)).long(), torch.randint(0, 2, size=(50, 5)).long(), 16), + (torch.randint(0, 2, size=(50, 8)).long(), torch.randint(0, 2, size=(50, 8)).long(), 16), + # Binary accuracy on input of shape (N, H, W, ...) + (torch.randint(0, 2, size=(4, 1, 12, 10)).long(), torch.randint(0, 2, size=(4, 1, 12, 10)).long(), 1), + (torch.randint(0, 2, size=(15, 1, 20, 10)).long(), torch.randint(0, 2, size=(15, 1, 20, 10)).long(), 1), + # updated batches + (torch.randint(0, 2, size=(50, 1, 12, 10)).long(), torch.randint(0, 2, size=(50, 1, 12, 10)).long(), 16), + (torch.randint(0, 2, size=(50, 1, 20, 10)).long(), torch.randint(0, 2, size=(50, 1, 20, 10)).long(), 16), + ][request.param] + + +@pytest.mark.parametrize("n_times", range(5)) +def test_binary_input(n_times, test_data_binary): acc = Accuracy() - def _test(y_pred, y, batch_size): - acc.reset() - if batch_size > 1: - n_iters = y.shape[0] // batch_size + 1 - for i in range(n_iters): - idx = i * batch_size - acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) - else: - acc.update((y_pred, y)) - - np_y = y.numpy().ravel() - np_y_pred = y_pred.numpy().ravel() - - assert acc._type == "binary" - assert isinstance(acc.compute(), float) - assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute()) - - def get_test_cases(): - - test_cases = [ - # Binary accuracy on input of shape (N, 1) or (N, ) - (torch.randint(0, 2, size=(10,)).long(), torch.randint(0, 2, size=(10,)).long(), 1), - (torch.randint(0, 2, size=(10, 1)).long(), torch.randint(0, 2, size=(10, 1)).long(), 1), - # updated batches - (torch.randint(0, 2, size=(50,)).long(), torch.randint(0, 2, size=(50,)).long(), 16), - (torch.randint(0, 2, size=(50, 1)).long(), torch.randint(0, 2, size=(50, 1)).long(), 16), - # Binary accuracy on input of shape (N, L) - (torch.randint(0, 2, size=(10, 5)).long(), torch.randint(0, 2, size=(10, 5)).long(), 1), - (torch.randint(0, 2, size=(10, 8)).long(), torch.randint(0, 2, size=(10, 8)).long(), 1), - # updated batches - (torch.randint(0, 2, size=(50, 5)).long(), torch.randint(0, 2, size=(50, 5)).long(), 16), - (torch.randint(0, 2, size=(50, 8)).long(), torch.randint(0, 2, size=(50, 8)).long(), 16), - # Binary accuracy on input of shape (N, H, W, ...) - (torch.randint(0, 2, size=(4, 1, 12, 10)).long(), torch.randint(0, 2, size=(4, 1, 12, 10)).long(), 1), - (torch.randint(0, 2, size=(15, 1, 20, 10)).long(), torch.randint(0, 2, size=(15, 1, 20, 10)).long(), 1), - # updated batches - (torch.randint(0, 2, size=(50, 1, 12, 10)).long(), torch.randint(0, 2, size=(50, 1, 12, 10)).long(), 16), - (torch.randint(0, 2, size=(50, 1, 20, 10)).long(), torch.randint(0, 2, size=(50, 1, 20, 10)).long(), 16), - ] + y_pred, y, batch_size = test_data_binary + acc.reset() + if batch_size > 1: + n_iters = y.shape[0] // batch_size + 1 + for i in range(n_iters): + idx = i * batch_size + acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) + else: + acc.update((y_pred, y)) - return test_cases + np_y = y.numpy().ravel() + np_y_pred = y_pred.numpy().ravel() - for _ in range(5): - # check multiple random inputs as random exact occurencies are rare - test_cases = get_test_cases() - for y_pred, y, n_iters in test_cases: - _test(y_pred, y, n_iters) + assert acc._type == "binary" + assert isinstance(acc.compute(), float) + assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute()) def test_multiclass_wrong_inputs(): @@ -208,55 +201,49 @@ def test_multilabel_wrong_inputs(): acc.update((torch.randint(0, 2, size=(10, 1)), torch.randint(0, 2, size=(10, 1)).long())) -def test_multilabel_input(): +@pytest.fixture(params=[item for item in range(12)]) +def test_data(request): + return [ + # Multilabel input data of shape (N, C) and (N, C) + (torch.randint(0, 2, size=(10, 4)).long(), torch.randint(0, 2, size=(10, 4)).long(), 1), + (torch.randint(0, 2, size=(10, 7)).long(), torch.randint(0, 2, size=(10, 7)).long(), 1), + # updated batches + (torch.randint(0, 2, size=(50, 4)).long(), torch.randint(0, 2, size=(50, 4)).long(), 16), + (torch.randint(0, 2, size=(50, 7)).long(), torch.randint(0, 2, size=(50, 7)).long(), 16), + # Multilabel input data of shape (N, H, W) + (torch.randint(0, 2, size=(10, 5, 10)).long(), torch.randint(0, 2, size=(10, 5, 10)).long(), 1), + (torch.randint(0, 2, size=(10, 4, 10)).long(), torch.randint(0, 2, size=(10, 4, 10)).long(), 1), + # updated batches + (torch.randint(0, 2, size=(50, 5, 10)).long(), torch.randint(0, 2, size=(50, 5, 10)).long(), 16), + (torch.randint(0, 2, size=(50, 4, 10)).long(), torch.randint(0, 2, size=(50, 4, 10)).long(), 16), + # Multilabel input data of shape (N, C, H, W, ...) and (N, C, H, W, ...) + (torch.randint(0, 2, size=(4, 5, 12, 10)).long(), torch.randint(0, 2, size=(4, 5, 12, 10)).long(), 1), + (torch.randint(0, 2, size=(4, 10, 12, 8)).long(), torch.randint(0, 2, size=(4, 10, 12, 8)).long(), 1), + # updated batches + (torch.randint(0, 2, size=(50, 5, 12, 10)).long(), torch.randint(0, 2, size=(50, 5, 12, 10)).long(), 16), + (torch.randint(0, 2, size=(50, 10, 12, 8)).long(), torch.randint(0, 2, size=(50, 10, 12, 8)).long(), 16), + ][request.param] + + +@pytest.mark.parametrize("n_times", range(5)) +def test_multilabel_input(n_times, test_data): acc = Accuracy(is_multilabel=True) - def _test(y_pred, y, batch_size): - acc.reset() - if batch_size > 1: - n_iters = y.shape[0] // batch_size + 1 - for i in range(n_iters): - idx = i * batch_size - acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) - else: - acc.update((y_pred, y)) - - np_y_pred = to_numpy_multilabel(y_pred) - np_y = to_numpy_multilabel(y) - - assert acc._type == "multilabel" - assert isinstance(acc.compute(), float) - assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute()) - - def get_test_cases(): + y_pred, y, batch_size = test_data + if batch_size > 1: + n_iters = y.shape[0] // batch_size + 1 + for i in range(n_iters): + idx = i * batch_size + acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) + else: + acc.update((y_pred, y)) - test_cases = [ - # Multilabel input data of shape (N, C) and (N, C) - (torch.randint(0, 2, size=(10, 4)).long(), torch.randint(0, 2, size=(10, 4)).long(), 1), - (torch.randint(0, 2, size=(10, 7)).long(), torch.randint(0, 2, size=(10, 7)).long(), 1), - # updated batches - (torch.randint(0, 2, size=(50, 4)).long(), torch.randint(0, 2, size=(50, 4)).long(), 16), - (torch.randint(0, 2, size=(50, 7)).long(), torch.randint(0, 2, size=(50, 7)).long(), 16), - # Multilabel input data of shape (N, H, W) - (torch.randint(0, 2, size=(10, 5, 10)).long(), torch.randint(0, 2, size=(10, 5, 10)).long(), 1), - (torch.randint(0, 2, size=(10, 4, 10)).long(), torch.randint(0, 2, size=(10, 4, 10)).long(), 1), - # updated batches - (torch.randint(0, 2, size=(50, 5, 10)).long(), torch.randint(0, 2, size=(50, 5, 10)).long(), 16), - (torch.randint(0, 2, size=(50, 4, 10)).long(), torch.randint(0, 2, size=(50, 4, 10)).long(), 16), - # Multilabel input data of shape (N, C, H, W, ...) and (N, C, H, W, ...) - (torch.randint(0, 2, size=(4, 5, 12, 10)).long(), torch.randint(0, 2, size=(4, 5, 12, 10)).long(), 1), - (torch.randint(0, 2, size=(4, 10, 12, 8)).long(), torch.randint(0, 2, size=(4, 10, 12, 8)).long(), 1), - # updated batches - (torch.randint(0, 2, size=(50, 5, 12, 10)).long(), torch.randint(0, 2, size=(50, 5, 12, 10)).long(), 16), - (torch.randint(0, 2, size=(50, 10, 12, 8)).long(), torch.randint(0, 2, size=(50, 10, 12, 8)).long(), 16), - ] - return test_cases + np_y_pred = to_numpy_multilabel(y_pred) + np_y = to_numpy_multilabel(y) - for _ in range(5): - # check multiple random inputs as random exact occurencies are rare - test_cases = get_test_cases() - for y_pred, y, batch_size in test_cases: - _test(y_pred, y, batch_size) + assert acc._type == "multilabel" + assert isinstance(acc.compute(), float) + assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute()) def test_incorrect_type(): From 281a0476385490ffe7975b95dea1a30160ecddb1 Mon Sep 17 00:00:00 2001 From: nmcguire101 Date: Mon, 18 Apr 2022 16:11:26 -0400 Subject: [PATCH 09/11] Improved multiclass test in test_accuracy.py --- tests/ignite/metrics/test_accuracy.py | 93 +++++++++++++-------------- 1 file changed, 44 insertions(+), 49 deletions(-) diff --git a/tests/ignite/metrics/test_accuracy.py b/tests/ignite/metrics/test_accuracy.py index 845d340b068..82b008d399a 100644 --- a/tests/ignite/metrics/test_accuracy.py +++ b/tests/ignite/metrics/test_accuracy.py @@ -62,7 +62,7 @@ def test_binary_wrong_inputs(): acc.update((torch.randint(0, 2, size=(10,)).long(), torch.randint(0, 2, size=(10, 5, 6)).long())) -@pytest.fixture(params=[item for item in range(11)]) +@pytest.fixture(params=[item for item in range(12)]) def test_data_binary(request): return [ # Binary accuracy on input of shape (N, 1) or (N, ) @@ -124,53 +124,48 @@ def test_multiclass_wrong_inputs(): acc.update((torch.rand(10), torch.randint(0, 5, size=(10, 5, 6)).long())) -def test_multiclass_input(): +@pytest.fixture(params=[item for item in range(11)]) +def test_data_multiclass(request): + return [ + # Multiclass input data of shape (N, ) and (N, C) + (torch.rand(10, 4), torch.randint(0, 4, size=(10,)).long(), 1), + (torch.rand(10, 10, 1), torch.randint(0, 18, size=(10, 1)).long(), 1), + (torch.rand(10, 18), torch.randint(0, 18, size=(10,)).long(), 1), + (torch.rand(4, 10), torch.randint(0, 10, size=(4,)).long(), 1), + # 2-classes + (torch.rand(4, 2), torch.randint(0, 2, size=(4,)).long(), 1), + (torch.rand(100, 5), torch.randint(0, 5, size=(100,)).long(), 16), + # Multiclass input data of shape (N, L) and (N, C, L) + (torch.rand(10, 4, 5), torch.randint(0, 4, size=(10, 5)).long(), 1), + (torch.rand(4, 10, 5), torch.randint(0, 10, size=(4, 5)).long(), 1), + (torch.rand(100, 9, 7), torch.randint(0, 9, size=(100, 7)).long(), 16), + # Multiclass input data of shape (N, H, W, ...) and (N, C, H, W, ...) + (torch.rand(4, 5, 12, 10), torch.randint(0, 5, size=(4, 12, 10)).long(), 1), + (torch.rand(100, 3, 8, 8), torch.randint(0, 3, size=(100, 8, 8)).long(), 16), + ][request.param] + + +@pytest.mark.parametrize("n_times", range(5)) +def test_multiclass_input(n_times, test_data_multiclass): acc = Accuracy() - def _test(y_pred, y, batch_size): - acc.reset() - if batch_size > 1: - # Batched Updates - n_iters = y.shape[0] // batch_size + 1 - for i in range(n_iters): - idx = i * batch_size - acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) - else: - acc.update((y_pred, y)) - - np_y_pred = y_pred.numpy().argmax(axis=1).ravel() - np_y = y.numpy().ravel() - - assert acc._type == "multiclass" - assert isinstance(acc.compute(), float) - assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute()) - - def get_test_cases(): - - test_cases = [ - # Multiclass input data of shape (N, ) and (N, C) - (torch.rand(10, 4), torch.randint(0, 4, size=(10,)).long(), 1), - (torch.rand(10, 10, 1), torch.randint(0, 18, size=(10, 1)).long(), 1), - (torch.rand(10, 18), torch.randint(0, 18, size=(10,)).long(), 1), - (torch.rand(4, 10), torch.randint(0, 10, size=(4,)).long(), 1), - # 2-classes - (torch.rand(4, 2), torch.randint(0, 2, size=(4,)).long(), 1), - (torch.rand(100, 5), torch.randint(0, 5, size=(100,)).long(), 16), - # Multiclass input data of shape (N, L) and (N, C, L) - (torch.rand(10, 4, 5), torch.randint(0, 4, size=(10, 5)).long(), 1), - (torch.rand(4, 10, 5), torch.randint(0, 10, size=(4, 5)).long(), 1), - (torch.rand(100, 9, 7), torch.randint(0, 9, size=(100, 7)).long(), 16), - # Multiclass input data of shape (N, H, W, ...) and (N, C, H, W, ...) - (torch.rand(4, 5, 12, 10), torch.randint(0, 5, size=(4, 12, 10)).long(), 1), - (torch.rand(100, 3, 8, 8), torch.randint(0, 3, size=(100, 8, 8)).long(), 16), - ] - return test_cases - - for _ in range(5): - # check multiple random inputs as random exact occurencies are rare - test_cases = get_test_cases() - for y_pred, y, batch_size in test_cases: - _test(y_pred, y, batch_size) + y_pred, y, batch_size = test_data_multiclass + acc.reset() + if batch_size > 1: + # Batched Updates + n_iters = y.shape[0] // batch_size + 1 + for i in range(n_iters): + idx = i * batch_size + acc.update((y_pred[idx : idx + batch_size], y[idx : idx + batch_size])) + else: + acc.update((y_pred, y)) + + np_y_pred = y_pred.numpy().argmax(axis=1).ravel() + np_y = y.numpy().ravel() + + assert acc._type == "multiclass" + assert isinstance(acc.compute(), float) + assert accuracy_score(np_y, np_y_pred) == pytest.approx(acc.compute()) def to_numpy_multilabel(y): @@ -202,7 +197,7 @@ def test_multilabel_wrong_inputs(): @pytest.fixture(params=[item for item in range(12)]) -def test_data(request): +def test_data_multilabel(request): return [ # Multilabel input data of shape (N, C) and (N, C) (torch.randint(0, 2, size=(10, 4)).long(), torch.randint(0, 2, size=(10, 4)).long(), 1), @@ -226,10 +221,10 @@ def test_data(request): @pytest.mark.parametrize("n_times", range(5)) -def test_multilabel_input(n_times, test_data): +def test_multilabel_input(n_times, test_data_multilabel): acc = Accuracy(is_multilabel=True) - y_pred, y, batch_size = test_data + y_pred, y, batch_size = test_data_multilabel if batch_size > 1: n_iters = y.shape[0] // batch_size + 1 for i in range(n_iters): From 6566439cf70658cc603697f2a3a1b96adc76b24c Mon Sep 17 00:00:00 2001 From: nmcguire101 Date: Mon, 18 Apr 2022 21:50:50 -0400 Subject: [PATCH 10/11] Replaced list with range --- tests/ignite/metrics/test_accuracy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ignite/metrics/test_accuracy.py b/tests/ignite/metrics/test_accuracy.py index 82b008d399a..4eb347b2d11 100644 --- a/tests/ignite/metrics/test_accuracy.py +++ b/tests/ignite/metrics/test_accuracy.py @@ -62,7 +62,7 @@ def test_binary_wrong_inputs(): acc.update((torch.randint(0, 2, size=(10,)).long(), torch.randint(0, 2, size=(10, 5, 6)).long())) -@pytest.fixture(params=[item for item in range(12)]) +@pytest.fixture(params=range(12)) def test_data_binary(request): return [ # Binary accuracy on input of shape (N, 1) or (N, ) @@ -124,7 +124,7 @@ def test_multiclass_wrong_inputs(): acc.update((torch.rand(10), torch.randint(0, 5, size=(10, 5, 6)).long())) -@pytest.fixture(params=[item for item in range(11)]) +@pytest.fixture(params=range(11)) def test_data_multiclass(request): return [ # Multiclass input data of shape (N, ) and (N, C) @@ -196,7 +196,7 @@ def test_multilabel_wrong_inputs(): acc.update((torch.randint(0, 2, size=(10, 1)), torch.randint(0, 2, size=(10, 1)).long())) -@pytest.fixture(params=[item for item in range(12)]) +@pytest.fixture(params=range(12)) def test_data_multilabel(request): return [ # Multilabel input data of shape (N, C) and (N, C) From 789e1fbe5c380e5f84433f958efa8a9968d22a02 Mon Sep 17 00:00:00 2001 From: Natalie McGuire Date: Mon, 18 Apr 2022 21:51:59 -0400 Subject: [PATCH 11/11] Delete (N --- (N | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 (N diff --git a/(N b/(N deleted file mode 100644 index e69de29bb2d..00000000000