Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

【Hackathon 5th No.116】paddle.quantile/nanquantile 功能增强 #56461

Closed
wants to merge 29 commits into from
Closed
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
efc2c65
[quantile/nanquantile]易用性提升 No.20
Asthestarsfalll Aug 19, 2023
914b845
update
Asthestarsfalll Aug 19, 2023
3319772
update
Asthestarsfalll Aug 21, 2023
62f7f64
support zero-dim q and add backward test
Asthestarsfalll Aug 22, 2023
4bbaf6f
remove useless code
Asthestarsfalll Aug 22, 2023
8ae9f5f
fix unittest
Asthestarsfalll Aug 31, 2023
e8cdcd4
Merge branch 'PaddlePaddle:develop' into quantile
Asthestarsfalll Aug 31, 2023
1ce4044
fix ci coverage
Asthestarsfalll Sep 1, 2023
84e2aff
fix
Asthestarsfalll Sep 6, 2023
f95ef77
Merge branch 'PaddlePaddle:develop' into quantile
Asthestarsfalll Sep 6, 2023
728a95b
Merge branch 'PaddlePaddle:develop' into quantile
Asthestarsfalll Sep 6, 2023
8eb64da
Merge branch 'PaddlePaddle:develop' into quantile
Asthestarsfalll Sep 7, 2023
b5ffb9c
update docs
Asthestarsfalll Sep 9, 2023
03e36e6
Merge branch 'PaddlePaddle:develop' into quantile
Asthestarsfalll Sep 9, 2023
2d68ad9
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
Asthestarsfalll Nov 8, 2023
75327bd
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
Asthestarsfalll Nov 8, 2023
74b7143
Merge branch 'quantile' of https://github.com/Asthestarsfalll/Paddle …
Asthestarsfalll Nov 8, 2023
edba2f1
update
Asthestarsfalll Nov 8, 2023
6f47fb7
fix codestyle
Asthestarsfalll Dec 6, 2023
d0d42b2
Merge branch 'PaddlePaddle:develop' into quantile
Asthestarsfalll Dec 6, 2023
b4a2591
support interpolation
Asthestarsfalll Dec 6, 2023
34241d1
fix unittest
Asthestarsfalll Dec 6, 2023
6eca5ce
update unittest
Asthestarsfalll Dec 7, 2023
221f09d
add unittest
Asthestarsfalll Dec 7, 2023
38b7cbb
fix
Asthestarsfalll Dec 7, 2023
245fdae
fix
Asthestarsfalll Dec 10, 2023
da6bc4a
Merge branch 'PaddlePaddle:develop' into quantile
Asthestarsfalll Dec 10, 2023
ed0e6d2
Merge branch 'develop' into quantile
Asthestarsfalll Dec 18, 2023
6c6ad3a
Merge branch 'PaddlePaddle:develop' into quantile
Asthestarsfalll Jan 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions python/paddle/tensor/stat.py
Original file line number Diff line number Diff line change
Expand Up @@ -468,8 +468,8 @@ def _compute_quantile(x, q, axis=None, keepdim=False, ignore_nan=False):

Args:
x (Tensor): The input Tensor, it's data type can be float32, float64, int32, int64.
q (int|float|list): The q for calculate quantile, which should be in range [0, 1]. If q is a list,
each q will be calculated and the first dimension of output is same to the number of ``q`` .
q (int|float|list|Tensor): The q for calculate quantile, which should be in range [0, 1]. If q is a list,
a 1-D Tensor or a 0-D Tensor, each q will be calculated and the first dimension of output is same to the number of ``q`` .
axis (int|list, optional): The axis along which to calculate quantile. ``axis`` should be int or list of int.
``axis`` should be in range [-D, D), where D is the dimensions of ``x`` .
If ``axis`` is less than 0, it works the same way as :math:`axis + D`.
Expand Down Expand Up @@ -498,8 +498,15 @@ def _compute_quantile(x, q, axis=None, keepdim=False, ignore_nan=False):
elif isinstance(q, (list, tuple)):
if len(q) <= 0:
raise ValueError("q should not be empty")
elif isinstance(q, paddle.Tensor):
if len(q.shape) > 1:
raise ValueError("q should be a 0-D tensor or a 1-D tensor")
elif len(q.shape) == 0:
q = [q]
else:
raise TypeError("Type of q should be int, float, list or tuple.")
raise TypeError(
"Type of q should be int, float, list or tuple, or tensor"
)

# Validate axis
dims = len(x.shape)
Expand Down Expand Up @@ -603,8 +610,8 @@ def quantile(x, q, axis=None, keepdim=False):

Args:
x (Tensor): The input Tensor, it's data type can be float32, float64, int32, int64.
q (int|float|list): The q for calculate quantile, which should be in range [0, 1]. If q is a list,
each q will be calculated and the first dimension of output is same to the number of ``q`` .
q (int|float|list|Tensor): The q for calculate quantile, which should be in range [0, 1]. If q is a list,
Asthestarsfalll marked this conversation as resolved.
Show resolved Hide resolved
a 1-D Tensor or a 0-D Tensor, each q will be calculated and the first dimension of output is same to the number of ``q`` .
axis (int|list, optional): The axis along which to calculate quantile. ``axis`` should be int or list of int.
``axis`` should be in range [-D, D), where D is the dimensions of ``x`` .
If ``axis`` is less than 0, it works the same way as :math:`axis + D`.
Expand Down Expand Up @@ -671,8 +678,8 @@ def nanquantile(x, q, axis=None, keepdim=False):

Args:
x (Tensor): The input Tensor, it's data type can be float32, float64, int32, int64.
q (int|float|list): The q for calculate quantile, which should be in range [0, 1]. If q is a list,
each q will be calculated and the first dimension of output is same to the number of ``q`` .
q (int|float|list|Tensor): The q for calculate quantile, which should be in range [0, 1]. If q is a list or
a 1-D Tensor, each q will be calculated and the first dimension of output is same to the number of ``q`` .
axis (int|list, optional): The axis along which to calculate quantile. ``axis`` should be int or list of int.
``axis`` should be in range [-D, D), where D is the dimensions of ``x`` .
If ``axis`` is less than 0, it works the same way as :math:`axis + D`.
Expand Down
66 changes: 66 additions & 0 deletions test/legacy_test/test_quantile_and_nanquantile.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,41 @@ def test_nanquantile_all_NaN(self):
paddle_res.numpy(), np_res, rtol=1e-05, equal_nan=True
)

def test_backward(self):
def check_grad(x, q, axis, target_gard, apis=None):
for op, _ in apis or API_list:
x = paddle.to_tensor(x, dtype='float32', stop_gradient=False)
op(x, q, axis).sum().backward()
np.testing.assert_allclose(
x.grad.numpy(),
np.array(target_gard, dtype='float32'),
rtol=1e-05,
equal_nan=True,
)

check_grad([1, 2, 3], 0.5, 0, [0, 1, 0])
check_grad(
[1, 2, 3, 4] * 2, [0.55, 0.7], 0, [0, 0, 0.95, 0, 0, 0.15, 0.9, 0]
)
check_grad(
[[1, 2, 3], [4, 5, 6]],
[0.3, 0.7],
1,
[[0.4, 1.2, 0.4], [0.4, 1.2, 0.4]],
)
# quantile
check_grad(
[1, float('nan'), 3], 0.5, 0, [0, 1, 0], [(paddle.quantile, None)]
)
# nanquantile
check_grad(
[1, float('nan'), 3],
0.5,
0,
[0.5, 0, 0.5],
[(paddle.nanquantile, None)],
)


class TestMuitlpleQ(unittest.TestCase):
"""
Expand Down Expand Up @@ -149,6 +184,24 @@ def test_quantile_multiple_axis_keepdim(self):
)
np.testing.assert_allclose(paddle_res.numpy(), np_res, rtol=1e-05)

def test_quantile_with_tensor_input(self):
Asthestarsfalll marked this conversation as resolved.
Show resolved Hide resolved
x = paddle.to_tensor(self.input_data)
paddle_res = paddle.quantile(
x, q=paddle.to_tensor([0.1, 0.2]), axis=[1, 2], keepdim=True
)
np_res = np.quantile(
self.input_data, q=[0.1, 0.2], axis=[1, 2], keepdims=True
)
np.testing.assert_allclose(paddle_res.numpy(), np_res, rtol=1e-05)

def test_quantile_with_zero_dim_tensor_input(self):
Asthestarsfalll marked this conversation as resolved.
Show resolved Hide resolved
x = paddle.to_tensor(self.input_data)
paddle_res = paddle.quantile(
x, q=paddle.to_tensor(0.1), axis=[1, 2], keepdim=True
)
np_res = np.quantile(self.input_data, q=0.1, axis=[1, 2], keepdims=True)
np.testing.assert_allclose(paddle_res.numpy(), np_res, rtol=1e-05)


class TestError(unittest.TestCase):
"""
Expand Down Expand Up @@ -209,6 +262,19 @@ def test_axis_value_error_2():

self.assertRaises(ValueError, test_axis_value_error_2)

# Test error when q is not a 1-D tensor
def test_tensor_input_1():
paddle_res = paddle.quantile(
self.x, q=paddle.randn((2, 3)), axis=[1, -10]
)

self.assertRaises(ValueError, test_tensor_input_1)

def test_type_q():
paddle_res = paddle.quantile(self.x, q={1}, axis=[1, -10])

self.assertRaises(TypeError, test_type_q)


class TestQuantileRuntime(unittest.TestCase):
"""
Expand Down