From 22022017d9603ff9498cd1d3546dbe2719875f9d Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Wed, 20 Dec 2017 16:48:07 +0800 Subject: [PATCH 1/3] add python wrapper for sequence_pool --- python/paddle/v2/fluid/layers/nn.py | 50 +++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 73f68466da780..59212e8497df8 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -520,9 +520,53 @@ def _get_default_param_initializer(): def sequence_pool(input, pool_type, **kwargs): """ - This function add the operator for sequence pooling. - This is applied on top of the input using pool_type mentioned - in the parameters. + This function add the operator for sequence pooling. + It pools features of all time-steps of each instance, and is applied + on top of the input using pool_type mentioned in the parameters. + + It supports four pool_type: + + - average: :math:`Out[i] = \\frac{\sum_i X_i}{N}` + - sum: :math:`Out[i] = \sum_jX_{ij}` + - sqrt: :math:`Out[i] = \\frac{\sum_jX_{ij}}{\sqrt{len(X_i)}}` + - max: :math:`Out[i] = max(X_i)` + + .. code-block:: text + + x is a 1-level LoDTensor: + x.lod = [[0, 2, 5, 7]] + x.data = [1, 3, 2, 4, 6, 5, 1] + x.dims = [7, 1] + + then output is a Tensor: + out.dim = [3, 1] + with condition len(x.lod[-1]) - 1 == out.dims[0] + + for different pool_type: + average: out.data = [2, 4, 3], where 2=(1+3)/2, 4=(2+4+6)/3, 3=(5+1)/2 + sum : out.data = [4, 12, 6], where 4=1+3, 12=2+4+6, 6=5+1 + sqrt : out.data = [2.82, 6.93, 4.24], where 2.82=(1+3)/sqrt(2), + 6.93=(2+4+6)/sqrt(3), 4.24=(5+1)/sqrt(2) + max : out.data = [3, 6, 5], where 3=max(1,3), 6=max(2,4,6), 5=max(5,1) + + Args: + input(variable): The input variable which is a LoDTensor. + pool_type (string): The pooling type of sequence_pool. + It supports average, sum, sqrt and max. + + Returns: + The sequence pooling variable which is a Tensor. + + Examples: + + .. code-block:: python + + x = fluid.layers.data(name='x', shape=[7, 1], + dtype='float32', lod_level=1) + avg_x = fluid.layers.sequence_pool(input=x, pool_type='average') + sum_x = fluid.layers.sequence_pool(input=x, pool_type='sum') + sqrt_x = fluid.layers.sequence_pool(input=x, pool_type='sqrt') + max_x = fluid.layers.sequence_pool(input=x, pool_type='max') """ helper = LayerHelper('sequence_pool', input=input, **kwargs) dtype = helper.input_dtype() From d7a9bb6e19dd601a554cc157bb741685485cd789 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Thu, 21 Dec 2017 11:31:54 +0800 Subject: [PATCH 2/3] add python wrap for sequence_first/last_step --- python/paddle/v2/fluid/layers/nn.py | 10 +++++++++- .../v2/fluid/tests/book/test_machine_translation.py | 2 +- python/paddle/v2/fluid/tests/test_dyn_rnn.py | 5 ++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index 59212e8497df8..ca073b2914474 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -13,7 +13,7 @@ 'crf_decoding', 'cos_sim', 'cross_entropy', 'square_error_cost', 'accuracy', 'chunk_eval', 'sequence_conv', 'conv2d', 'sequence_pool', 'pool2d', 'batch_norm', 'beam_search_decode', 'conv2d_transpose', 'sequence_expand', - 'lstm_unit', 'reduce_sum' + 'lstm_unit', 'reduce_sum', 'sequence_first_step', 'sequence_last_step' ] @@ -583,6 +583,14 @@ def sequence_pool(input, pool_type, **kwargs): return pool_out +def sequence_first_step(input, **kwargs): + return sequence_pool(input=input, pool_type="first") + + +def sequence_last_step(input, **kwargs): + return sequence_pool(input=input, pool_type="last") + + def pool2d(input, pool_size, pool_type, diff --git a/python/paddle/v2/fluid/tests/book/test_machine_translation.py b/python/paddle/v2/fluid/tests/book/test_machine_translation.py index 80ffc5a544c20..e79864b3977ed 100644 --- a/python/paddle/v2/fluid/tests/book/test_machine_translation.py +++ b/python/paddle/v2/fluid/tests/book/test_machine_translation.py @@ -33,7 +33,7 @@ def encoder_decoder(): fc1 = fluid.layers.fc(input=src_embedding, size=hidden_dim * 4, act='tanh') lstm_hidden0, lstm_0 = layers.dynamic_lstm(input=fc1, size=hidden_dim * 4) - encoder_out = layers.sequence_pool(input=lstm_hidden0, pool_type="last") + encoder_out = layers.sequence_last_step(input=lstm_hidden0) # decoder trg_language_word = layers.data( diff --git a/python/paddle/v2/fluid/tests/test_dyn_rnn.py b/python/paddle/v2/fluid/tests/test_dyn_rnn.py index 034266c26f481..8090c5f47814c 100644 --- a/python/paddle/v2/fluid/tests/test_dyn_rnn.py +++ b/python/paddle/v2/fluid/tests/test_dyn_rnn.py @@ -63,8 +63,7 @@ def test_plain_while_op(self): all_timesteps = fluid.layers.array_to_lod_tensor( x=out, table=rank_table) - last = fluid.layers.sequence_pool( - input=all_timesteps, pool_type='last') + last = fluid.layers.sequence_last_step(input=all_timesteps) logits = fluid.layers.fc(input=last, size=1, act=None) loss = fluid.layers.sigmoid_cross_entropy_with_logits( x=logits, label=label) @@ -101,7 +100,7 @@ def test_train_dyn_rnn(self): rnn.update_memory(mem, out_) rnn.output(out_) - last = fluid.layers.sequence_pool(input=rnn(), pool_type='last') + last = fluid.layers.sequence_last_step(input=rnn()) logits = fluid.layers.fc(input=last, size=1, act=None) label = fluid.layers.data(name='label', shape=[1], dtype='float32') loss = fluid.layers.sigmoid_cross_entropy_with_logits( From f3fc8de1d5a12af4657d2005f1f98a1236784f1c Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 22 Dec 2017 12:55:06 +0800 Subject: [PATCH 3/3] add doc for sequence_first/last_step --- doc/api/v2/fluid/layers.rst | 12 ++++++ python/paddle/v2/fluid/layers/nn.py | 58 +++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/doc/api/v2/fluid/layers.rst b/doc/api/v2/fluid/layers.rst index 842f3b18007a5..fc29795d1241f 100644 --- a/doc/api/v2/fluid/layers.rst +++ b/doc/api/v2/fluid/layers.rst @@ -170,6 +170,18 @@ sequence_pool :noindex: +sequence_first_step +------------------- +.. autofunction:: paddle.v2.fluid.layers.sequence_first_step + :noindex: + + +sequence_last_step +------------------ +.. autofunction:: paddle.v2.fluid.layers.sequence_last_step + :noindex: + + pool2d --------- .. autofunction:: paddle.v2.fluid.layers.pool2d diff --git a/python/paddle/v2/fluid/layers/nn.py b/python/paddle/v2/fluid/layers/nn.py index ca073b2914474..3536a7e3909a8 100644 --- a/python/paddle/v2/fluid/layers/nn.py +++ b/python/paddle/v2/fluid/layers/nn.py @@ -584,10 +584,68 @@ def sequence_pool(input, pool_type, **kwargs): def sequence_first_step(input, **kwargs): + """ + This funciton get the first step of sequence. + + .. code-block:: text + + x is a 1-level LoDTensor: + x.lod = [[0, 2, 5, 7]] + x.data = [1, 3, 2, 4, 6, 5, 1] + x.dims = [7, 1] + + then output is a Tensor: + out.dim = [3, 1] + with condition len(x.lod[-1]) - 1 == out.dims[0] + out.data = [1, 2, 5], where 1=first(1,3), 2=first(2,4,6), 5=first(5,1) + + Args: + input(variable): The input variable which is a LoDTensor. + + Returns: + The sequence's first step variable which is a Tensor. + + Examples: + + .. code-block:: python + + x = fluid.layers.data(name='x', shape=[7, 1], + dtype='float32', lod_level=1) + x_first_step = fluid.layers.sequence_first_step(input=x) + """ return sequence_pool(input=input, pool_type="first") def sequence_last_step(input, **kwargs): + """ + This funciton get the last step of sequence. + + .. code-block:: text + + x is a 1-level LoDTensor: + x.lod = [[0, 2, 5, 7]] + x.data = [1, 3, 2, 4, 6, 5, 1] + x.dims = [7, 1] + + then output is a Tensor: + out.dim = [3, 1] + with condition len(x.lod[-1]) - 1 == out.dims[0] + out.data = [3, 6, 1], where 3=last(1,3), 6=last(2,4,6), 1=last(5,1) + + Args: + input(variable): The input variable which is a LoDTensor. + + Returns: + The sequence's last step variable which is a Tensor. + + Examples: + + .. code-block:: python + + x = fluid.layers.data(name='x', shape=[7, 1], + dtype='float32', lod_level=1) + x_last_step = fluid.layers.sequence_last_step(input=x) + """ return sequence_pool(input=input, pool_type="last")