WIP: Custom encoder/decoder layer sequence.

requirements/requirements.txt

@@ -2,3 +2,4 @@ pyyaml
 mxnet-mkl==1.2.0
 numpy>=1.12
 typing
+parsimonious

sockeye/arguments.py

@@ -637,6 +637,25 @@ def add_model_parameters(params):
                                    'For example: n:drn '
                                    'Default: %(default)s.')
 
+    # Custom sequence encoder or decoder
+    model_params.add_argument('--custom-seq-encoder',
+                              default=None,
+                              help='Specify the layers the custom encoder will consist of.')
+    model_params.add_argument('--custom-seq-decoder',
+                              default=None,
+                              help='Specify the layers the custom decoder will consist of.')
+
+    model_params.add_argument('--custom-seq-num-hidden',
+                              type=int_greater_or_equal(1),
+                              default=1024,
+                              help='Number of hidden units for encoder and decoder. Default: %(default)s.')
+
+    model_params.add_argument('--custom-seq-dropout',
+                              type=float,
+                              default=.0,
+                              help='Dropout used throughout the custom encoder and decoder.'
+                                   'Use "x:x" to specify separate values. Default: %(default)s.')
+
     # LHUC
     # TODO: The convolutional model does not support lhuc yet
     model_params.add_argument('--lhuc',

sockeye/constants.py

@@ -55,14 +55,15 @@
 RNN_WITH_CONV_EMBED_NAME = "rnn-with-conv-embed"
 TRANSFORMER_TYPE = "transformer"
 CONVOLUTION_TYPE = "cnn"
+CUSTOM_SEQ_TYPE = "custom-seq"
 TRANSFORMER_WITH_CONV_EMBED_TYPE = "transformer-with-conv-embed"
 IMAGE_PRETRAIN_TYPE = "image-pretrain-cnn"
 
 # available encoders
-ENCODERS = [RNN_NAME, RNN_WITH_CONV_EMBED_NAME, TRANSFORMER_TYPE, TRANSFORMER_WITH_CONV_EMBED_TYPE, CONVOLUTION_TYPE, IMAGE_PRETRAIN_TYPE]
+ENCODERS = [RNN_NAME, RNN_WITH_CONV_EMBED_NAME, TRANSFORMER_TYPE, TRANSFORMER_WITH_CONV_EMBED_TYPE, CONVOLUTION_TYPE, IMAGE_PRETRAIN_TYPE, CUSTOM_SEQ_TYPE]
 
 # available decoder
-DECODERS = [RNN_NAME, TRANSFORMER_TYPE, CONVOLUTION_TYPE]
+DECODERS = [RNN_NAME, TRANSFORMER_TYPE, CONVOLUTION_TYPE, CUSTOM_SEQ_TYPE]
 
 # rnn types
 LSTM_TYPE = 'lstm'
@@ -148,7 +149,8 @@
 # Swish-1/SiLU (https://arxiv.org/pdf/1710.05941.pdf, https://arxiv.org/pdf/1702.03118.pdf)
 SWISH1 = "swish1"
 TANH = "tanh"
-TRANSFORMER_ACTIVATION_TYPES = [GELU, RELU, SWISH1]
+NO_ACTIVATION = "none"
+TRANSFORMER_ACTIVATION_TYPES = [GELU, RELU, SWISH1, NO_ACTIVATION]
 CNN_ACTIVATION_TYPES = [GLU, RELU, SIGMOID, SOFT_RELU, TANH]
 
 # Convolutional block pad types:
@@ -389,3 +391,6 @@
 DATA_CONFIG = "data.config"
 PREPARED_DATA_VERSION_FILE = "data.version"
 PREPARED_DATA_VERSION = 2
+
+SEQUENCE_LENGTH_MUST_NOT_CHANGE_MSG = "Sequence length may not change within the residual layers."
+