From 3b2a3f7f0fab1615013166b6574bf8f8a03a3b1b Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 31 Aug 2016 13:08:35 +0000 Subject: [PATCH 001/324] Init commit --- .gitignore | 0 README.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/README.md b/README.md new file mode 100644 index 0000000000000..e69de29bb2d1d From 24acf8d5f96809bb13ea4c7f4fd1b1b69e90ce90 Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Thu, 1 Sep 2016 06:04:44 +0000 Subject: [PATCH 002/324] fix cuda-8.0 compile ISSUE=4613907 git-svn-id: https://svn.baidu.com/idl/trunk/paddle@1497 1ad973e4-5ce8-4261-8a94-b56d1f490c56 --- paddle/cuda/include/hl_device_functions.cuh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paddle/cuda/include/hl_device_functions.cuh b/paddle/cuda/include/hl_device_functions.cuh index 408ff35d963c6..27e3f450c5c1c 100644 --- a/paddle/cuda/include/hl_device_functions.cuh +++ b/paddle/cuda/include/hl_device_functions.cuh @@ -36,6 +36,8 @@ static __inline__ __device__ double atomicAdd(double* address, double val) { } // namespace hppl +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 600 using hppl::atomicAdd; +#endif #endif /* HL_DEVICE_FUNCTIONS_CUH_ */ From 8ae1b325d61c2789bdfc9d541e89dd70fb5156c0 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 1 Sep 2016 06:32:51 +0000 Subject: [PATCH 003/324] =?UTF-8?q?=C3=A2=C2=80fix=20bug=20in=20cuda=5Fagg?= =?UTF-8?q?regate=20ISSUE=3D4608831?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: https://svn.baidu.com/idl/trunk/paddle@1498 1ad973e4-5ce8-4261-8a94-b56d1f490c56 --- paddle/cuda/include/hl_cuda.h | 11 ++++++----- paddle/cuda/include/stub/hl_cuda_stub.h | 2 +- paddle/cuda/src/hl_cuda_aggregate.cu | 22 ++++++++++------------ paddle/cuda/src/hl_cuda_device.cc | 5 +++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/paddle/cuda/include/hl_cuda.h b/paddle/cuda/include/hl_cuda.h index ffdf71229abe1..3196db67f61fd 100644 --- a/paddle/cuda/include/hl_cuda.h +++ b/paddle/cuda/include/hl_cuda.h @@ -321,13 +321,14 @@ extern const char* hl_get_device_error_string(size_t err); extern int hl_get_device_last_error(); /** - * @brief hppl query event. + * @brief check cuda event is ready * - * @param[in] event cuda event to query. - * @param[out] isNotReady this work under device has not yet been - * completed, vice versa. + * @param[in] event cuda event to query. + * + * @return true cuda event is ready. + * false cuda event is not ready. */ -extern void hl_cuda_event_query(hl_event_t event, bool& isNotReady); +extern bool hl_cuda_event_is_ready(hl_event_t event); /** * @brief hppl device synchronization. diff --git a/paddle/cuda/include/stub/hl_cuda_stub.h b/paddle/cuda/include/stub/hl_cuda_stub.h index 395101c6f7f08..675ac03b0e188 100644 --- a/paddle/cuda/include/stub/hl_cuda_stub.h +++ b/paddle/cuda/include/stub/hl_cuda_stub.h @@ -89,7 +89,7 @@ inline const char* hl_get_device_error_string() { return NULL; } inline const char* hl_get_device_error_string(size_t err) { return NULL; } -inline void hl_cuda_event_query(hl_event_t event, bool& isNotReady) {} +inline bool hl_cuda_event_is_ready(hl_event_t event) { return true; } inline void hl_device_synchronize() {} diff --git a/paddle/cuda/src/hl_cuda_aggregate.cu b/paddle/cuda/src/hl_cuda_aggregate.cu index c0b84b087b156..4eb775eb7971e 100644 --- a/paddle/cuda/src/hl_cuda_aggregate.cu +++ b/paddle/cuda/src/hl_cuda_aggregate.cu @@ -261,11 +261,7 @@ void hl_vector_sum(real *A_d, real *C_h, int dimM) { struct _hl_event_st hl_event_st = {.cu_event = t_resource.event}; hl_event_t hl_event = &hl_event_st; - - bool isNotReady = false; - do { - hl_cuda_event_query(hl_event, isNotReady); - } while (isNotReady == cudaErrorNotReady); + while (!hl_cuda_event_is_ready(hl_event)) {} KeVectorSum<128><<< grid, threads, 0, STREAM_DEFAULT >>> (A_d, t_resource.gpu_mem, dimM); @@ -275,7 +271,10 @@ void hl_vector_sum(real *A_d, real *C_h, int dimM) { hl_memcpy_async(C_h, t_resource.cpu_mem, sizeof(real), HPPL_STREAM_DEFAULT); hl_stream_record_event(HPPL_STREAM_DEFAULT, hl_event); - CHECK_SYNC("hl_vector_sum failed"); + hl_stream_synchronize(HPPL_STREAM_DEFAULT); + cudaError_t err = (cudaError_t)hl_get_device_last_error(); + CHECK_EQ(cudaSuccess, err) + << "CUDA error: " << hl_get_device_error_string((size_t)err); } template @@ -317,11 +316,7 @@ void hl_vector_abs_sum(real *A_d, real *C_h, int dimM) { struct _hl_event_st hl_event_st = {.cu_event = t_resource.event}; hl_event_t hl_event = &hl_event_st; - - bool isNotReady = false; - do { - hl_cuda_event_query(hl_event, isNotReady); - } while (isNotReady == cudaErrorNotReady); + while (!hl_cuda_event_is_ready(hl_event)) {} KeVectorAbsSum<128><<< grid, threads, 0, STREAM_DEFAULT >>> (A_d, t_resource.gpu_mem, dimM); @@ -331,5 +326,8 @@ void hl_vector_abs_sum(real *A_d, real *C_h, int dimM) { hl_memcpy_async(C_h, t_resource.cpu_mem, sizeof(real), HPPL_STREAM_DEFAULT); hl_stream_record_event(HPPL_STREAM_DEFAULT, hl_event); - CHECK_SYNC("hl_vector_abs_sum failed"); + hl_stream_synchronize(HPPL_STREAM_DEFAULT); + cudaError_t err = (cudaError_t)hl_get_device_last_error(); + CHECK_EQ(cudaSuccess, err) + << "CUDA error: " << hl_get_device_error_string((size_t)err); } diff --git a/paddle/cuda/src/hl_cuda_device.cc b/paddle/cuda/src/hl_cuda_device.cc index 774eef8b894f1..f07538d6ba713 100644 --- a/paddle/cuda/src/hl_cuda_device.cc +++ b/paddle/cuda/src/hl_cuda_device.cc @@ -751,11 +751,12 @@ void hl_set_device_flags_block() { cudaDeviceScheduleBlockingSync)); } -void hl_cuda_event_query(hl_event_t event, bool& isNotReady) { +bool hl_cuda_event_is_ready(hl_event_t event) { cudaError_t err = dynload::cudaEventQuery(event->cu_event); CHECK(cudaSuccess == err || cudaErrorNotReady == err); if (cudaErrorNotReady == err) { - isNotReady = true; + return false; } + return true; } From 9de37056151a97e422c46ffacd969dfb5e75c81e Mon Sep 17 00:00:00 2001 From: yanchong01 Date: Thu, 1 Sep 2016 09:33:22 +0000 Subject: [PATCH 004/324] new paddle fcr for session nn ISSUE=3890409 TESTED=UT git-svn-id: https://svn.baidu.com/idl/trunk/paddle@1499 1ad973e4-5ce8-4261-8a94-b56d1f490c56 From 2c3c027892f755ac44cb64acee0e5d75e47b340e Mon Sep 17 00:00:00 2001 From: wangyanfei01 Date: Thu, 1 Sep 2016 09:54:19 +0000 Subject: [PATCH 005/324] upgrade platform to support ocr specific requirements ISSUE=4618697 git-svn-id: https://svn.baidu.com/idl/trunk/paddle@1500 1ad973e4-5ce8-4261-8a94-b56d1f490c56 From 54d4968d38648d32ce777b5ce1001110d31c64af Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Thu, 1 Sep 2016 10:15:45 +0000 Subject: [PATCH 006/324] Refine code and comments for CRMNormLayer. ISSUE=4612671 git-svn-id: https://svn.baidu.com/idl/trunk/paddle@1501 1ad973e4-5ce8-4261-8a94-b56d1f490c56 --- doc/ui/api/trainer_config_helpers/layers.rst | 6 ---- doc/ui/index.md | 2 +- paddle/gserver/layers/NormProjectionLayer.cpp | 7 ++-- paddle/gserver/layers/NormProjectionLayer.h | 5 +-- paddle/math/Matrix.cpp | 8 ++--- paddle/math/Matrix.h | 30 ++++++++--------- .../paddle/trainer_config_helpers/layers.py | 33 +++---------------- 7 files changed, 28 insertions(+), 63 deletions(-) diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index a09d5e3d4d31c..1583fce981fed 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -82,12 +82,6 @@ img_cmrnorm_layer :members: img_cmrnorm_layer :noindex: -img_rnorm_layer ------------------ -.. automodule:: paddle.trainer_config_helpers.layers - :members: img_rnorm_layer - :noindex: - batch_norm_layer --------------------- .. automodule:: paddle.trainer_config_helpers.layers diff --git a/doc/ui/index.md b/doc/ui/index.md index 829994d56bb65..9c1ba27bdc14f 100644 --- a/doc/ui/index.md +++ b/doc/ui/index.md @@ -7,7 +7,7 @@ ## API Reference -* [Trainer Config Helpers](api/trainer_config_helpers/index.md) +* [Model Config Interface](api/trainer_config_helpers/index.md) ## Command Line Argument diff --git a/paddle/gserver/layers/NormProjectionLayer.cpp b/paddle/gserver/layers/NormProjectionLayer.cpp index f30a3e8df0f87..eab6e904ee998 100644 --- a/paddle/gserver/layers/NormProjectionLayer.cpp +++ b/paddle/gserver/layers/NormProjectionLayer.cpp @@ -46,9 +46,6 @@ bool CMRProjectionNormLayer::init(const LayerMap& layerMap, /* the size of inputs for norm-layer is 1 */ CHECK_EQ(config_.inputs_size(), 1); - auto& inputConfig = config_.inputs(0); - blocked_ = inputConfig.norm_conf().blocked(); - return true; } @@ -69,7 +66,7 @@ void CMRProjectionNormLayer::forward(PassType passType) { denoms_->zeroMem(); outV->crossMapNormalFwd(*input, imgSizeH_, imgSizeW_, *denoms_, channels_, - size_, scale_, pow_, blocked_); + size_, scale_, pow_); } void CMRProjectionNormLayer::backward(const UpdateCallback& callback) { @@ -86,6 +83,6 @@ void CMRProjectionNormLayer::backward(const UpdateCallback& callback) { preOutGrad->crossMapNormalBwd(*localGrad, *denoms_, *preOutV, *localOutV, channels_, imgSizeH_, imgSizeW_, size_, scale_, - pow_, blocked_); + pow_); } } // namespace paddle diff --git a/paddle/gserver/layers/NormProjectionLayer.h b/paddle/gserver/layers/NormProjectionLayer.h index a5e8dce029ae1..728806ea76958 100644 --- a/paddle/gserver/layers/NormProjectionLayer.h +++ b/paddle/gserver/layers/NormProjectionLayer.h @@ -23,15 +23,12 @@ namespace paddle { /** * @brief response normalization across feature maps - * namely normalize in number of size_ channels + * namely normalize in number of size_ channels */ class CMRProjectionNormLayer : public ResponseNormLayer { size_t imgSizeH_, imgSizeW_; size_t outputH_, outputW_; -protected: - bool blocked_; - public: explicit CMRProjectionNormLayer(const LayerConfig& config) : ResponseNormLayer(config) {} diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 7e1840076833f..f3a6503d4a21f 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -943,7 +943,7 @@ void GpuMatrix::avgPoolBackward(Matrix& outGrad, size_t imgSizeH, void GpuMatrix::crossMapNormalFwd(Matrix& input, size_t imgSizeH, size_t imgSizeW, Matrix& denoms, size_t channels, size_t sizeX, float scale, - float pow, bool blocked) { + float pow) { size_t num = input.getHeight(); size_t height = imgSizeH; size_t width = imgSizeW; @@ -960,7 +960,7 @@ void GpuMatrix::crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, Matrix& preOutV, Matrix& localOutV, size_t channels, size_t imgSizeH, size_t imgSizeW, size_t sizeX, float scale, - float pow, bool blocked) { + float pow) { size_t num = preOutV.getHeight(); size_t height = imgSizeH; size_t width = imgSizeW; @@ -1602,7 +1602,7 @@ void CpuMatrix::avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, void CpuMatrix::crossMapNormalFwd(Matrix& input, size_t imgSizeH, size_t imgSizeW, Matrix& denoms, size_t channels, size_t sizeX, float scale, - float pow, bool blocked) { + float pow) { size_t num = input.getHeight(); size_t height = imgSizeH; size_t width = imgSizeW; @@ -1655,7 +1655,7 @@ void CpuMatrix::crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, Matrix& preOutV, Matrix& localOutV, size_t channels, size_t imgSizeH, size_t imgSizeW, size_t size, float scale, - float pow, bool blocked) { + float pow) { LOG(FATAL) << "Not implemented"; CHECK(imgSizeH * imgSizeW * channels == preOutV.getWidth()); diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index f27773d1108b6..cfb30797fcf1b 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -585,7 +585,7 @@ class Matrix : public BaseMatrix { * \f[ * a[i] = \sum_{j=-(N-1)/2}^{(N-1)/2} b_{i+j} * c_{j} * \f] - * + * * b contains M elements, * c contains N elements (N is odd), * b's index arithmetic is computed modulo M, @@ -774,7 +774,7 @@ class Matrix : public BaseMatrix { virtual void crossMapNormalFwd(Matrix& input, size_t imgSizeH, size_t imgSizeW, Matrix& denoms, size_t channels, size_t sizeX, float scale, - float pow, bool blocked) { + float pow) { LOG(FATAL) << "Not implemeted"; } @@ -782,7 +782,7 @@ class Matrix : public BaseMatrix { Matrix& preOutV, Matrix& localOutV, size_t channels, size_t imgSizeH, size_t imgSizeW, size_t size, float scale, - float pow, bool blocked) { + float pow) { LOG(FATAL) << "Not implemeted"; } @@ -883,7 +883,7 @@ class Matrix : public BaseMatrix { * @code * this[i] = -sum(label[i][j]*log(output[i][j]) * + (1-label[i][j])*log(1-output[i][j])) - * @endcode + * @endcode */ virtual void multiBinaryLabelCrossEntropy(Matrix& output, Matrix& label) { LOG(FATAL) << "Not implemented"; @@ -895,7 +895,7 @@ class Matrix : public BaseMatrix { * @code * this[i][j] = -label[i][j]/output[i][j] * + (1-label[i][j])/(1-output[i][j]) - * @endcode + * @endcode */ virtual void multiBinaryLabelCrossEntropyBp(Matrix& output, Matrix& label) { LOG(FATAL) << "Not implemented"; @@ -903,12 +903,12 @@ class Matrix : public BaseMatrix { /** * @brief Calculate the classification error for multi binary labels - * + * * @code * this[i] = sum((output[i][j] >= threshold && label[i][j] == 0) * || (output[i][j] < threshold && label[i][j] == 1)) * / output->getWidth() - * @endcode + * @endcode */ virtual void classificationErrorMulti(Matrix& output, Matrix& label, real threshold) { @@ -1149,12 +1149,12 @@ class GpuMatrix : public Matrix { void crossMapNormalFwd(Matrix& input, size_t imgSizeH, size_t imgSizeW, Matrix& denoms, size_t channels, size_t sizeX, - float scale, float pow, bool blocked); + float scale, float pow); void crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, Matrix& preOutV, Matrix& localOutV, size_t channels, size_t imgSizeH, - size_t imgSizeW, size_t sizeX, float scale, float pow, - bool blocked); + size_t imgSizeW, size_t sizeX, + float scale, float pow); void maxSequenceForward(Matrix& input, const IVector& sequence, IVector& index); @@ -1260,12 +1260,12 @@ class CpuMatrix : public Matrix { void crossMapNormalFwd(Matrix& input, size_t imgSizeH, size_t imgSizeW, Matrix& denoms, size_t channels, size_t sizeX, - float scale, float pow, bool blocked); + float scale, float pow); void crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, Matrix& preOutV, Matrix& localOutV, size_t channels, size_t imgSizeH, - size_t imgSizeW, size_t sizeX, float scale, float pow, - bool blocked); + size_t imgSizeW, size_t sizeX, + float scale, float pow); void maxSequenceForward(Matrix& input, const IVector& sequence, IVector& index); @@ -1307,14 +1307,14 @@ class CpuMatrix : public Matrix { * @code * table.row[ids[i]] += this.row[i] * @endcode - */ + */ virtual void addToRows(Matrix& table, IVector& ids); /** * @code * this[i] = table[i, id[i]] * @endcode - */ + */ virtual void selectElements(Matrix& table, IVector& ids); /** diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index cd00ffefc707f..c23200dfa39b6 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1500,7 +1500,7 @@ def img_pool_layer(input, pool_size, name=None, def __img_norm_layer__(name, input, size, norm_type, scale, power, - num_channels, blocked, layer_attr): + num_channels, blocked=0, layer_attr): if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters @@ -1522,9 +1522,9 @@ def __img_norm_layer__(name, input, size, norm_type, scale, power, @layer_support() def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, name=None, num_channels=None, - blocked=0, layer_attr=None): + layer_attr=None): """ - Convolution cross-map-response-normalize layer. + Response normalization across feature maps. The details please refer to `Alex's paper `_. @@ -1532,7 +1532,7 @@ def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, :type name: None|basestring :param input: layer's input. :type input: LayerOutput - :param size: cross map response size. + :param size: Normalize in number of :math:`size` feature maps. :type size: int :param scale: The hyper-parameter. :type scale: float @@ -1547,30 +1547,7 @@ def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, :rtype: LayerOutput """ return __img_norm_layer__(name, input, size, "cmrnorm-projection", scale, - power, num_channels, blocked, layer_attr) - - -@wrap_name_default("rnorm") -@layer_support() -def img_rnorm_layer(input, size, scale, power, name=None, num_channels=None, - layer_attr=None): - """ - Normalize the input in local region, namely response normalization - across feature maps. - - :param name: The name of this layer. - :rtype name: None|basestring - :param input: The input of this layer. - :param size: - :param scale: - :param power: - :param num_channels: - :param layer_attr: - :return: LayerOutput object. - :rtype: LayerOutput - """ - return __img_norm_layer__(name, input, size, 'rnorm', scale, power, - num_channels, 0, layer_attr) + power, num_channels, 0, layer_attr) @wrap_bias_attr_default() From 6968191ede52ba02da6605ce8e811ff81d08be10 Mon Sep 17 00:00:00 2001 From: luotao02 Date: Thu, 1 Sep 2016 11:17:26 +0000 Subject: [PATCH 007/324] fix bug in img_norm_layer, several dead links and add missing files for semantic_role_labeling ISSUE=4619881 git-svn-id: https://svn.baidu.com/idl/trunk/paddle@1502 1ad973e4-5ce8-4261-8a94-b56d1f490c56 --- demo/recommendation/dataprovider.py | 7 - doc/demo/quick_start/index_en.md | 4 +- doc/demo/rec/ml_regression.rst | 20 +- doc/demo/semantic_role_labeling/index.rst | 7 + .../semantic_role_labeling.md | 183 ++++++++++++++++++ doc/ui/data_provider/index.rst | 4 +- doc/ui/predict/swig_py_paddle_en.rst | 2 +- .../paddle/trainer_config_helpers/layers.py | 2 +- 8 files changed, 205 insertions(+), 24 deletions(-) create mode 100644 doc/demo/semantic_role_labeling/index.rst create mode 100644 doc/demo/semantic_role_labeling/semantic_role_labeling.md diff --git a/demo/recommendation/dataprovider.py b/demo/recommendation/dataprovider.py index 29cfd7224803e..454467f40b44b 100755 --- a/demo/recommendation/dataprovider.py +++ b/demo/recommendation/dataprovider.py @@ -12,15 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - import cPickle as pickle -except ImportError: - import pickle - from paddle.trainer.PyDataProvider2 import * import common_utils # parse - def hook(settings, meta, **kwargs): """ Init hook is invoked before process data. It will set obj.slots and store @@ -47,7 +41,6 @@ def hook(settings, meta, **kwargs): settings.input_types = headers settings.meta = meta - @provider(init_hook=hook, cache=CacheType.CACHE_PASS_IN_MEM) def process(settings, filename): with open(filename, 'r') as f: diff --git a/doc/demo/quick_start/index_en.md b/doc/demo/quick_start/index_en.md index b537d8c8346a4..dc0c6255f32ed 100644 --- a/doc/demo/quick_start/index_en.md +++ b/doc/demo/quick_start/index_en.md @@ -157,9 +157,7 @@ define_py_data_sources2(train_list='data/train.list', obj="process", args={"dictionary": word_dict}) ``` - -You can refer to the following link for more detailed examples -: Python Use Case，The detailed documentation on data format is: PyDataProviderWrapper。 +You can refer to the following link for more detailed examples and data formats: PyDataProvider2. ## Network Architecture You will describe four kinds of network architectures in this section. diff --git a/doc/demo/rec/ml_regression.rst b/doc/demo/rec/ml_regression.rst index 47bcef2d6d59f..472f585e6889e 100644 --- a/doc/demo/rec/ml_regression.rst +++ b/doc/demo/rec/ml_regression.rst @@ -219,9 +219,9 @@ The network structure shows below. The demo's neural network config file "trainer_config.py" show as below. -.. include:: ../../../demo/recommendation/trainer_config.py - :code: python - :literal: +.. literalinclude:: ../../../demo/recommendation/trainer_config.py + :language: python + :lines: 15- In this :code:`trainer_config.py`, we just map each feature type to a feature vector, following shows how to map each feature to a vector shows below. @@ -263,9 +263,9 @@ In these network, we use several api in `trainer_config_helpers Data Provider ''''''''''''' -.. include:: ../../../demo/recommendation/dataprovider.py - :code: python - :literal: +.. literalinclude:: ../../../demo/recommendation/dataprovider.py + :language: python + :lines: 15- The data provider just read the meta.bin and rating file, yield each sample for training. In this :code:`dataprovider.py`, we should set\: @@ -274,7 +274,7 @@ In this :code:`dataprovider.py`, we should set\: * use_seq\: Whether this :code:`dataprovider.py` in sequence mode or not. * process\: Return each sample of data to :code:`paddle`. -The data provider details document see `there <../../ui/DataProvider.html>`_. +The data provider details document see `there <../../ui/data_provider/pydataprovider2.html>`_. Train ````` @@ -283,9 +283,9 @@ After prepare data, config network, writting data provider, now we can run paddl The run.sh is shown as follow: -.. include:: ../../../demo/recommendation/run.sh - :code: bash - :literal: +.. literalinclude:: ../../../demo/recommendation/run.sh + :language: bash + :lines: 16- It just start a paddle training process, write the log to `log.txt`, then print it on screen. diff --git a/doc/demo/semantic_role_labeling/index.rst b/doc/demo/semantic_role_labeling/index.rst new file mode 100644 index 0000000000000..ff3035059bd77 --- /dev/null +++ b/doc/demo/semantic_role_labeling/index.rst @@ -0,0 +1,7 @@ +Semantic Role Labeling Tutorial +=============================== + +.. toctree:: + :maxdepth: 3 + + semantic_role_labeling.md diff --git a/doc/demo/semantic_role_labeling/semantic_role_labeling.md b/doc/demo/semantic_role_labeling/semantic_role_labeling.md new file mode 100644 index 0000000000000..05fbc8278daf2 --- /dev/null +++ b/doc/demo/semantic_role_labeling/semantic_role_labeling.md @@ -0,0 +1,183 @@ +# Semantic Role labeling Tutorial # + +Semantic role labeling (SRL) is a form of shallow semantic parsing whose goal is to discover the predicate-argument structure of each predicate in a given input sentence. SRL is useful as an intermediate step in a wide range of natural language processing tasks, such as information extraction. automatic document categorization and question answering. An instance is as following [1]: + + [ A0 He ] [ AM-MOD would ][ AM-NEG n’t ] [ V accept] [ A1 anything of value ] from [A2 those he was writing about ]. + +- V: verb +- A0: acceptor +- A1: thing accepted +- A2: accepted-from +- A3: Attribute +- AM-MOD: modal +- AM-NEG: negation + +Given the verb "accept", the chunks in sentence would play certain semantic roles. Here, the label scheme is from Penn Proposition Bank. + +To this date, most of the successful SRL systems are built on top of some form of parsing results where pre-defined feature templates over the syntactic structure are used. This tutorial will present an end-to-end system using deep bidirectional long short-term memory (DB-LSTM)[2] for solving the SRL task, which largely outperforms the previous state-of-the-art systems. The system regards SRL task as the sequence labelling problem. + +## Data Description +The relevant paper[2] takes the data set in CoNLL-2005&2012 Shared Task for training and testing. Accordingto data license, the demo adopts the test data set of CoNLL-2005, which can be reached on website. + +To download and process the original data, user just need to execute the following command: + +```bash +cd data +./get_data.sh +``` +Several new files appear in the `data `directory as follows. +```bash +conll05st-release：the test data set of CoNll-2005 shared task +test.wsj.words：the Wall Street Journal data sentences +test.wsj.props: the propositional arguments +src.dict：the dictionary of words in sentences +tgt.dict：the labels dictionary +feature: the extracted features from data set +``` + +## Training +### DB-LSTM +Please refer to the Sentiment Analysis demo to learn more about the long short-term memory unit. + +Unlike Bidirectional-LSTM that used in Sentiment Analysis demo, the DB-LSTM adopts another way to stack LSTM layer. First a standard LSTM processes the sequence in forward direction. The input and output of this LSTM layer are taken by the next LSTM layer as input, processed in reversed direction. These two standard LSTM layers compose a pair of LSTM. Then we stack LSTM layers pair after pair to obtain the deep LSTM model. + +The following figure shows a temporal expanded 2-layer DB-LSTM network. +
+![pic](./network_arch.png) +
+ +### Features +Two input features play an essential role in this pipeline: predicate (pred) and argument (argu). Two other features: predicate context (ctx-p) and region mark (mr) are also adopted. Because a single predicate word can not exactly describe the predicate information, especially when the same words appear more than one times in a sentence. With the predicate context, the ambiguity can be largely eliminated. Similarly, we use region mark mr = 1 to denote the argument position if it locates in the predicate context region, or mr = 0 if does not. These four simple features are all we need for our SRL system. Features of one sample with context size set to 1 is showed as following[2]: +
+![pic](./feature.jpg) +
+ +In this sample, the coresponding labelled sentence is: + +[ A1 A record date ] has [ AM-NEG n't ] been [ V set ] . + +In the demo, we adopt the feature template as above, consists of : `argument`, `predicate`, `ctx-p (p=-1,0,1)`, `mark` and use `B/I/O` scheme to label each argument. These features and labels are stored in `feature` file, and separated by `\t`. + +### Data Provider + +`dataprovider.py` is the python file to wrap data. `hook()` function is to define the data slots for network. The Six features and label are all IndexSlots. +``` +def hook(settings, word_dict, label_dict, **kwargs): + settings.word_dict = word_dict + settings.label_dict = label_dict + #all inputs are integral and sequential type + settings.slots = [ + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(len(word_dict)), + integer_value_sequence(2), + integer_value_sequence(len(label_dict))] +``` +The corresponding data iterator is as following: +``` +@provider(use_seq=True, init_hook=hook) +def process(obj, file_name): + with open(file_name, 'r') as fdata: + for line in fdata: + sentence, predicate, ctx_n1, ctx_0, ctx_p1, mark, label = line.strip().split('\t') + words = sentence.split() + sen_len = len(words) + word_slot = [obj.word_dict.get(w, UNK_IDX) for w in words] + + predicate_slot = [obj.word_dict.get(predicate, UNK_IDX)] * sen_len + ctx_n1_slot = [obj.word_dict.get(ctx_n1, UNK_IDX) ] * sen_len + ctx_0_slot = [obj.word_dict.get(ctx_0, UNK_IDX) ] * sen_len + ctx_p1_slot = [obj.word_dict.get(ctx_p1, UNK_IDX) ] * sen_len + + marks = mark.split() + mark_slot = [int(w) for w in marks] + + label_list = label.split() + label_slot = [obj.label_dict.get(w) for w in label_list] + + yield word_slot, predicate_slot, ctx_n1_slot, ctx_0_slot, ctx_p1_slot, mark_slot, label_slot +``` +The `process`function yield 7 lists which are six features and labels. + +### Neural Network Config +`db_lstm.py` is the neural network config file to load the dictionaries and define the data provider module and network architecture during the training procedure. + +Seven `data_layer` load instances from data provider. Six features are transformed into embedddings respectively, and mixed by `mixed_layer` . Deep bidirectional LSTM layers extract features for the softmax layer. The objective function is cross entropy of labels. + +### Run Training +The script for training is `train.sh`, user just need to execute: +```bash + ./train.sh +``` +The content in `train.sh`: +``` +paddle train \ + --config=./db_lstm.py \ + --save_dir=./output \ + --trainer_count=4 \ + --log_period=10 \ + --num_passes=500 \ + --use_gpu=false \ + --show_parameter_stats_period=10 \ + --test_all_data_in_one_period=1 \ +2>&1 | tee 'train.log' +``` + +- \--config=./db_lstm.py : network config file. +- \--save_di=./output: output path to save models. +- \--trainer_count=4 : set thread number (or GPU count). +- \--log_period=10 : print log every 20 batches. +- \--num_passes=500: set pass number, one pass in PaddlePaddle means training all samples in dataset one time. +- \--use_gpu=false: use CPU to train, set true, if you install GPU version of PaddlePaddle and want to use GPU to train. +- \--show_parameter_stats_period=10: show parameter statistic every 100 batches. +- \--test_all_data_in_one_period=1: test all data in every testing. + + +After training, the models will be saved in directory `output`. + +### Run testing +The script for testing is `test.sh`, user just need to execute: +```bash + ./test.sh +``` +The main part in `tesh.sh` +``` +paddle train \ + --config=./db_lstm.py \ + --model_list=$model_list \ + --job=test \ + --config_args=is_test=1 \ +``` + + - \--config=./db_lstm.py: network config file + - \--model_list=$model_list.list: model list file + - \--job=test: indicate the test job + - \--config_args=is_test=1: flag to indicate test + + +### Run prediction +The script for prediction is `predict.sh`, user just need to execute: +```bash + ./predict.sh + +``` +In `predict.sh`, user should offer the network config file, model path, label file, word dictionary file, feature file +``` +python predict.py + -c $config_file + -w $model_path + -l $label_file + -d $dict_file + -i $input_file +``` + +`predict.py` is the main executable python script, which includes functions: load model, load data, data prediction. The network model will output the probability distribution of labels. In the demo, we take the label with maximum probability as result. User can also implement the beam search or viterbi decoding upon the probability distribution matrix. + +After prediction, the result is saved in `predict.res`. + +## Reference +[1] Martha Palmer, Dan Gildea, and Paul Kingsbury. The Proposition Bank: An Annotated Corpus of Semantic Roles , Computational Linguistics, 31(1), 2005. + +[2] Zhou, Jie, and Wei Xu. "End-to-end learning of semantic role labeling using recurrent neural networks." Proceedings of the Annual Meeting of the Association for Computational Linguistics. 2015. diff --git a/doc/ui/data_provider/index.rst b/doc/ui/data_provider/index.rst index e702b1a7e9239..3db5b57376257 100644 --- a/doc/ui/data_provider/index.rst +++ b/doc/ui/data_provider/index.rst @@ -1,5 +1,5 @@ -PaddlePaddle DataProvider Introduction -================================ +DataProvider Introduction +========================= DataProvider is a module that loads training or testing data into cpu or gpu memory for the following triaining or testing process. diff --git a/doc/ui/predict/swig_py_paddle_en.rst b/doc/ui/predict/swig_py_paddle_en.rst index e22d0bff338d9..9841f124e25a4 100644 --- a/doc/ui/predict/swig_py_paddle_en.rst +++ b/doc/ui/predict/swig_py_paddle_en.rst @@ -28,7 +28,7 @@ python's :code:`help()` function. Let's walk through the above python script: - Note: As swig_paddle can only accept C++ matrices, we offer a utility class DataProviderWraaperConverter that can accept the same input data with PyDataProviderWrapper, for more information please refer to document - of `PyDataProviderWrapper <../py_data_provider_wrapper_api.html>`_. + of `PyDataProvider2 <../data_provider/pydataprovider2.html>`_. * Do the prediction and output the result at line 100, forwardTest is another utility class that directly takes the activations of the output layer. diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index c23200dfa39b6..a01d57269908e 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1500,7 +1500,7 @@ def img_pool_layer(input, pool_size, name=None, def __img_norm_layer__(name, input, size, norm_type, scale, power, - num_channels, blocked=0, layer_attr): + num_channels, blocked, layer_attr): if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters From 0b02a221ff4448aacfbc0733fa991b736628d26d Mon Sep 17 00:00:00 2001 From: wangjiangb Date: Fri, 2 Sep 2016 01:42:12 +0000 Subject: [PATCH 008/324] Fix errors for high version GPU. ISSUE=4622423 git-svn-id: https://svn.baidu.com/idl/trunk/paddle@1503 1ad973e4-5ce8-4261-8a94-b56d1f490c56 --- cmake/flags.cmake | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 351af42ee6f6a..e2fdfb5c46a50 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -8,7 +8,7 @@ include(CheckCXXSymbolExists) # is_c: is C flag or C++ flag, bool type. # src_list: The list name which the flag name will be append to. # flag_name: the flag name for compiler, such as '-Werror' '-Wall' etc -# rest arguments: not used. +# rest arguments: not used. function(safe_set_flag is_c src_list flag_name) string(REPLACE "-" "_" safe_name ${flag_name}) string(REPLACE "=" "_" safe_name ${safe_name}) @@ -44,7 +44,7 @@ CHECK_CXX_SYMBOL_EXISTS(UINT64_MAX "stdint.h" UINT64_MAX_EXISTS) if(NOT UINT64_MAX_EXISTS) set(CMAKE_REQUIRED_DEFINITIONS -D__STDC_LIMIT_MACROS) CHECK_CXX_SYMBOL_EXISTS(UINT64_MAX "stdint.h" UINT64_MAX_EXISTS_HERE) - if(UINT64_MAX_EXISTS_HERE) + if(UINT64_MAX_EXISTS_HERE) set(CMAKE_REQUIRED_DEFINITIONS) add_definitions(-D__STDC_LIMIT_MACROS) else() @@ -75,12 +75,13 @@ endforeach() # So, don't set these flags here. foreach(capability 30 35 50) - list(APPEND __arch_flags "-gencode arch=compute_${capability},code=sm_${capability}") + list(APPEND __arch_flags " -gencode arch=compute_${capability},code=sm_${capability}") endforeach() +message(${__arch_flags}) if (CUDA_VERSION VERSION_GREATER "7.0") - list(APPEND __arch_flags "-gencode arch=compute_52,code=sm_52") + list(APPEND __arch_flags " -gencode arch=compute_52,code=sm_52") endif() -set(CUDA_NVCC_FLAGS ${__arch_flags} ${CUDA_NVCC_FLAGS}) +set(CUDA_NVCC_FLAGS ${__arch_flags} ${CUDA_NVCC_FLAGS}) From c9dc794e19e448e5d5147eae837e0f2356259f21 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Thu, 1 Sep 2016 10:50:48 +0800 Subject: [PATCH 009/324] fixed build issue of double definition of atomicAdd on modern GPUs --- paddle/cuda/include/hl_device_functions.cuh | 4 ++++ 1 file changed, 4 insertions(+) mode change 100644 => 100755 paddle/cuda/include/hl_device_functions.cuh diff --git a/paddle/cuda/include/hl_device_functions.cuh b/paddle/cuda/include/hl_device_functions.cuh old mode 100644 new mode 100755 index 27e3f450c5c1c..2fbc2cfb50221 --- a/paddle/cuda/include/hl_device_functions.cuh +++ b/paddle/cuda/include/hl_device_functions.cuh @@ -16,6 +16,8 @@ limitations under the License. */ #ifndef HL_DEVICE_FUNCTIONS_CUH_ #define HL_DEVICE_FUNCTIONS_CUH_ +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 600 + namespace hppl { static __inline__ __device__ double atomicAdd(double* address, double val) { @@ -40,4 +42,6 @@ static __inline__ __device__ double atomicAdd(double* address, double val) { using hppl::atomicAdd; #endif +#endif + #endif /* HL_DEVICE_FUNCTIONS_CUH_ */ From c6fd55e4d744fa35c3f032d18476b36682dac13d Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 2 Sep 2016 13:47:10 +0800 Subject: [PATCH 010/324] Remove print for cuda gencode Change-Id: I79fd722623430c11363c819bbd3ef10099197dbb --- cmake/flags.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index e2fdfb5c46a50..a0069271252a4 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -78,7 +78,6 @@ foreach(capability 30 35 50) list(APPEND __arch_flags " -gencode arch=compute_${capability},code=sm_${capability}") endforeach() -message(${__arch_flags}) if (CUDA_VERSION VERSION_GREATER "7.0") list(APPEND __arch_flags " -gencode arch=compute_52,code=sm_52") endif() From aa1ef13717853f32203f84148d798a7129a444c8 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Fri, 2 Sep 2016 17:42:59 +0800 Subject: [PATCH 011/324] removed unnecessary lines on environment configuration in image classification demo --- demo/image_classification/preprocess.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/demo/image_classification/preprocess.sh b/demo/image_classification/preprocess.sh index fe89c8f4bb946..dfe3eb95d1ab8 100755 --- a/demo/image_classification/preprocess.sh +++ b/demo/image_classification/preprocess.sh @@ -14,8 +14,6 @@ # limitations under the License. set -e -export PYTHONPATH=$PYTHONPATH:../../ - data_dir=./data/cifar-out python preprocess.py -i $data_dir -s 32 -c 1 From ecdeaad9289109d4ccc1a164b6a7f29248ecb3d2 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 2 Sep 2016 21:17:18 +0800 Subject: [PATCH 012/324] Remove the extra endif by merge error --- paddle/cuda/include/hl_device_functions.cuh | 2 -- 1 file changed, 2 deletions(-) diff --git a/paddle/cuda/include/hl_device_functions.cuh b/paddle/cuda/include/hl_device_functions.cuh index 35358d29210f5..2fbc2cfb50221 100755 --- a/paddle/cuda/include/hl_device_functions.cuh +++ b/paddle/cuda/include/hl_device_functions.cuh @@ -44,6 +44,4 @@ using hppl::atomicAdd; #endif -#endif - #endif /* HL_DEVICE_FUNCTIONS_CUH_ */ From e150be6ecba44749c2eb69cfe893b4b84ce8a586 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Fri, 2 Sep 2016 17:42:59 +0800 Subject: [PATCH 013/324] removed unnecessary lines on environment configuration in image classification demo Change-Id: If3d82f278560fc0f43e8bebeeef0501a4cf0ea7d --- demo/image_classification/preprocess.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/demo/image_classification/preprocess.sh b/demo/image_classification/preprocess.sh index fe89c8f4bb946..dfe3eb95d1ab8 100755 --- a/demo/image_classification/preprocess.sh +++ b/demo/image_classification/preprocess.sh @@ -14,8 +14,6 @@ # limitations under the License. set -e -export PYTHONPATH=$PYTHONPATH:../../ - data_dir=./data/cifar-out python preprocess.py -i $data_dir -s 32 -c 1 From c974a9932533500376c262b3e7f4ca40e4114073 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 2 Sep 2016 21:17:18 +0800 Subject: [PATCH 014/324] Remove the extra endif by merge error Change-Id: Ic35a54a94cbd981defff5b44a063dc680cdaaa7c --- paddle/cuda/include/hl_device_functions.cuh | 2 -- 1 file changed, 2 deletions(-) diff --git a/paddle/cuda/include/hl_device_functions.cuh b/paddle/cuda/include/hl_device_functions.cuh index 35358d29210f5..2fbc2cfb50221 100755 --- a/paddle/cuda/include/hl_device_functions.cuh +++ b/paddle/cuda/include/hl_device_functions.cuh @@ -44,6 +44,4 @@ using hppl::atomicAdd; #endif -#endif - #endif /* HL_DEVICE_FUNCTIONS_CUH_ */ From 9269a5b4cb579941b9eb2951644f6bbafcb168ef Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Sat, 3 Sep 2016 08:58:54 +0800 Subject: [PATCH 015/324] Remove img_rnorm_layer and fix ParameterAttribute bug for some layer wrapper. * Fix ParameterAttribute for img_conv_layer, tensor_layer, crf_layer, crf_decoding_layer. Change-Id: Ic24e4bc1f9d5e03bf35058b49a54c78caf7e97a8 --- demo/sentiment/sentiment_net.py | 2 +- python/paddle/trainer_config_helpers/layers.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/demo/sentiment/sentiment_net.py b/demo/sentiment/sentiment_net.py index f9f784c1f0b20..31e585edcaa11 100644 --- a/demo/sentiment/sentiment_net.py +++ b/demo/sentiment/sentiment_net.py @@ -65,7 +65,7 @@ def bidirectional_lstm_net(input_dim, bi_lstm = bidirectional_lstm(input=emb, size=lstm_dim) dropout = dropout_layer(input=bi_lstm, dropout_rate=0.5) output = fc_layer(input=dropout, size=class_dim, - act_type=SoftmaxActivation()) + act=SoftmaxActivation()) if not is_predict: lbl = data_layer("label", 1) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index a01d57269908e..b7e5f566bb8c3 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -36,7 +36,7 @@ "cos_sim", "hsigmoid", "regression_cost", 'classification_cost', "LayerOutput", 'img_conv_layer', 'img_pool_layer', 'batch_norm_layer', - 'img_cmrnorm_layer', 'img_rnorm_layer', 'addto_layer', + 'img_cmrnorm_layer', 'addto_layer', 'concat_layer', 'lstm_step_layer', 'recurrent_group', 'memory', 'StaticInput', 'expand_layer', 'scaling_layer', 'power_layer', 'interpolation_layer', 'trans_layer', @@ -1419,7 +1419,10 @@ def img_conv_layer(input, filter_size, num_filters, padding_y = padding if param_attr.attr.get('initial_smart') == True: # special initial for conv layers. init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 - param_attr = ParameterAttribute(initial_mean=0.0, initial_std=init_w) + param_attr.attr["initial_mean"] = 0.0 + param_attr.attr["initial_std"] = init_w + param_attr.attr["initial_strategy"] = 0 + param_attr.attr["initial_smart"] = False Layer( name=name, inputs=Input(input.name, conv=Conv( @@ -2724,7 +2727,7 @@ def tensor_layer(input, size, act=None, name=None, type=LayerType.TENSOR_LAYER, active_type=act.name, bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(input[0].name, **param_attr), + inputs=[Input(input[0].name, **param_attr.attr), Input(input[1].name)], **ExtraLayerAttribute.to_kwargs(layer_attr) ) @@ -3067,6 +3070,7 @@ def ctc_layer(input, label, size, name=None, norm_by_times=False): return LayerOutput(name, LayerType.CTC_LAYER, [input, label], size=size) @wrap_name_default() +@wrap_param_attr_default() def crf_layer(input, label, size, weight=None, param_attr=None, name=None): """ A layer for calculating the cost of sequential conditional random @@ -3100,7 +3104,7 @@ def crf_layer(input, label, size, weight=None, param_attr=None, name=None): assert isinstance(label, LayerOutput) assert weight is None or isinstance(weight, LayerOutput) - ipts = [Input(input.name, **param_attr), + ipts = [Input(input.name, **param_attr.attr), Input(label.name)] if weight is not None: ipts.append(Input(weight.name)) @@ -3117,6 +3121,7 @@ def crf_layer(input, label, size, weight=None, param_attr=None, name=None): return LayerOutput(name, LayerType.CRF_LAYER, parents, size=size) @wrap_name_default() +@wrap_param_attr_default() def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): """ A layer for calculating the decoding sequence of sequential conditional @@ -3142,7 +3147,7 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): assert isinstance(input, LayerOutput) assert label is None or isinstance(label, LayerOutput) - ipts = [Input(input.name, **param_attr)] + ipts = [Input(input.name, **param_attr.attr)] if label is not None: ipts.append(Input(label.name)) From c5d1902223eba2631d08da5cb1dd2e24125b9be2 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 2 Sep 2016 17:21:58 +0800 Subject: [PATCH 016/324] Make helper shows the correct output value Change-Id: Ia5358f88129b5c2e2a2653b1d85c168b80d797b3 --- python/paddle/trainer_config_helpers/networks.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/paddle/trainer_config_helpers/networks.py b/python/paddle/trainer_config_helpers/networks.py index 94b5245aba669..93c4261cc48c5 100644 --- a/python/paddle/trainer_config_helpers/networks.py +++ b/python/paddle/trainer_config_helpers/networks.py @@ -1121,12 +1121,13 @@ def __dfs_travel__(layer, logger.info( "".join(["The input order is [", ", ".join(final_inputs), "]"]) ) + + if len(final_outputs) == 0: + final_outputs = map(lambda x: x.name, layers) + logger.info( "".join(["The output order is [", ", ".join(final_outputs), "]" ])) Inputs(*final_inputs) - if len(final_outputs) != 0: - Outputs(*final_outputs) - else: - Outputs(*map(lambda x: x.name, layers)) + Outputs(*final_outputs) From c6950a22943c49cefa5af483e0a93484d4038b26 Mon Sep 17 00:00:00 2001 From: wangmiao1981 Date: Fri, 2 Sep 2016 21:50:02 -0700 Subject: [PATCH 017/324] modify comments --- doc/build/build_from_source.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index 2c32c37e9ddf5..36681727f6430 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -125,6 +125,8 @@ mkdir build cd build # you can add build option here, such as: cmake -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX= .. +# please use sudo make install, if you want +# to install PaddlePaddle into the system make -j `nproc` && make install # PaddlePaddle installation path export PATH=/bin:$PATH From 4866c99550e986a0c6f1faadc24644734186a439 Mon Sep 17 00:00:00 2001 From: liaogang Date: Fri, 2 Sep 2016 19:49:59 +0800 Subject: [PATCH 018/324] add custom CUDA_ARCH for modern Gpus Change-Id: I569a4db567f27c1b9e143086930a9f7079c20f78 --- cmake/flags.cmake | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index a0069271252a4..e05cd52f47338 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -74,13 +74,32 @@ endforeach() # Release/Debug flags set by cmake. Such as -O3 -g -DNDEBUG etc. # So, don't set these flags here. +function(specify_cuda_arch cuda_version cuda_arch) + if(${cuda_version} VERSION_GREATER "8.0") + foreach(capability 60 61 62) + if(${cuda_arch} STREQUAL ${capability}) + list(APPEND __arch_flags " -gencode arch=compute_${cuda_arch},code=sm_${cuda_arch}") + endif() + endforeach() + elseif(${cuda_version} VERSION_GREATER "7.0") + foreach(capability 52 53) + if(${cuda_arch} STREQUAL ${capability}) + list(APPEND __arch_flags " -gencode arch=compute_${cuda_arch},code=sm_${cuda_arch}") + endif() + endforeach() + endif() +endfunction() + +# Common cuda architectures foreach(capability 30 35 50) list(APPEND __arch_flags " -gencode arch=compute_${capability},code=sm_${capability}") endforeach() -if (CUDA_VERSION VERSION_GREATER "7.0") - list(APPEND __arch_flags " -gencode arch=compute_52,code=sm_52") -endif() +# Custom cuda architecture +set(CUDA_ARCH) +if(CUDA_ARCH) + specify_cuda_arch(${CUDA_VERSION} ${CUDA_ARCH}) +endif() set(CUDA_NVCC_FLAGS ${__arch_flags} ${CUDA_NVCC_FLAGS}) From 291318e5a2dd1f0283611d4b915b5a4fe882aeb2 Mon Sep 17 00:00:00 2001 From: liaogang Date: Fri, 2 Sep 2016 19:54:34 +0800 Subject: [PATCH 019/324] delete Bits.h and move its func into utils/Util.h Change-Id: Ib7f7d3c65c9850b8420f9a4799835a7ea03572bd --- doc/source/math/utils/utils.rst | 4 -- .../layers/HierarchicalSigmoidLayer.cpp | 3 +- paddle/math/Bits.h | 53 ------------------- paddle/math/MatrixBitCode.cpp | 2 +- paddle/utils/Util.h | 12 +++++ 5 files changed, 14 insertions(+), 60 deletions(-) delete mode 100644 paddle/math/Bits.h diff --git a/doc/source/math/utils/utils.rst b/doc/source/math/utils/utils.rst index e00dc6229c15e..3df721a47b93b 100644 --- a/doc/source/math/utils/utils.rst +++ b/doc/source/math/utils/utils.rst @@ -1,10 +1,6 @@ Utils ======= -Bits -------- -.. doxygenfile:: paddle/math/Bits.h - Memory Handle -------------- .. doxygenfile:: paddle/math/MemoryHandle.h diff --git a/paddle/gserver/layers/HierarchicalSigmoidLayer.cpp b/paddle/gserver/layers/HierarchicalSigmoidLayer.cpp index fc9832af86793..7091c6aa222e5 100644 --- a/paddle/gserver/layers/HierarchicalSigmoidLayer.cpp +++ b/paddle/gserver/layers/HierarchicalSigmoidLayer.cpp @@ -14,8 +14,7 @@ limitations under the License. */ #include "HierarchicalSigmoidLayer.h" - -#include "paddle/math/Bits.h" +#include "paddle/utils/Util.h" namespace paddle { diff --git a/paddle/math/Bits.h b/paddle/math/Bits.h deleted file mode 100644 index 4114149f6c191..0000000000000 --- a/paddle/math/Bits.h +++ /dev/null @@ -1,53 +0,0 @@ -/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - - -#pragma once - -#include - -namespace paddle { - -/** - * From Facebook folly: - * https://github.com/facebook/folly/blob/master/folly/Bits.h - * - * findLastSet: return the 1-based index of the highest bit set - * - * for x > 0: - * \f[ - * findLastSet(x) = 1 + \floor*{\log_{2}x} - * \f] - */ -template -inline constexpr typename std::enable_if<(std::is_integral::value && - std::is_unsigned::value && - sizeof(T) <= sizeof(unsigned int)), - unsigned int>::type -findLastSet(T x) { - return x ? 8 * sizeof(unsigned int) - __builtin_clz(x) : 0; -} - -template -inline constexpr - typename std::enable_if<(std::is_integral::value && - std::is_unsigned::value && - sizeof(T) > sizeof(unsigned int) && - sizeof(T) <= sizeof(unsigned long)), // NOLINT - unsigned int>::type - findLastSet(T x) { - return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; // NOLINT -} - -} // namespace paddle diff --git a/paddle/math/MatrixBitCode.cpp b/paddle/math/MatrixBitCode.cpp index d179ac1f53355..8497c26e35404 100644 --- a/paddle/math/MatrixBitCode.cpp +++ b/paddle/math/MatrixBitCode.cpp @@ -14,7 +14,7 @@ limitations under the License. */ #include "paddle/utils/Logging.h" -#include "Bits.h" +#include "paddle/utils/Util.h" #include "Matrix.h" #include "hl_gpu.h" diff --git a/paddle/utils/Util.h b/paddle/utils/Util.h index 3729c5c433609..7d43713d5f4c5 100644 --- a/paddle/utils/Util.h +++ b/paddle/utils/Util.h @@ -63,6 +63,18 @@ limitations under the License. */ namespace paddle { +/** + * return the 1-based index of the highest bit set + * + * for x > 0: + * \f[ + * findLastSet(x) = 1 + \floor*{\log_{2}x} + * \f] + */ +inline constexpr size_t findLastSet(size_t x) { + return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; // NOLINT +} + /** * calculate the non-negative remainder of a/b * @param[in] a From 2b795845ad450b74666ee635a81e4c916c6ea2b5 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 2 Sep 2016 17:39:57 +0800 Subject: [PATCH 020/324] fix dead link Change-Id: Id0804d03fc15529a8cf0b13c9e97d628103f5cec --- doc/demo/embedding_model/index.md | 2 +- doc/demo/rec/ml_regression.rst | 4 +- doc_cn/demo/index.rst | 16 ++--- .../trainer_config_helpers/data_sources.py | 72 +++++++++++-------- 4 files changed, 52 insertions(+), 42 deletions(-) diff --git a/doc/demo/embedding_model/index.md b/doc/demo/embedding_model/index.md index 45992ad856e65..06f3ff1f009e4 100644 --- a/doc/demo/embedding_model/index.md +++ b/doc/demo/embedding_model/index.md @@ -93,7 +93,7 @@ where `train.sh` is almost the same as `demo/seqToseq/translation/train.sh`, the - `--init_model_path`: path of the initialization model, here is `data/paraphrase_model` - `--load_missing_parameter_strategy`: operations when model file is missing, here use a normal distibution to initialize the other parameters except for the embedding layer -For users who want to understand the dataset format, model architecture and training procedure in detail, please refer to [Text generation Tutorial](text_generation.md). +For users who want to understand the dataset format, model architecture and training procedure in detail, please refer to [Text generation Tutorial](../text_generation/text_generation.md). ## Optional Function ## ### Embedding Parameters Observation diff --git a/doc/demo/rec/ml_regression.rst b/doc/demo/rec/ml_regression.rst index 472f585e6889e..4917f873a934d 100644 --- a/doc/demo/rec/ml_regression.rst +++ b/doc/demo/rec/ml_regression.rst @@ -291,7 +291,7 @@ It just start a paddle training process, write the log to `log.txt`, then print it on screen. Each command line argument in :code:`run.sh`, please refer to the `command line -arguments `_ page. The short description of these arguments is shown as follow. +arguments <../../ui/index.html#command-line-argument>`_ page. The short description of these arguments is shown as follow. * config\: Tell paddle which file is neural network configuration. * save_dir\: Tell paddle save model into './output' @@ -303,8 +303,6 @@ arguments `_ page. The short description of these arguments is shown as fol * dot_period\: Print a :code:`.` after train :code:`dot_period` batches. * num_passes\: Train at most :code:`num_passes`. - - If training process starts successfully, the output likes follow: .. code-block:: text diff --git a/doc_cn/demo/index.rst b/doc_cn/demo/index.rst index 4c948dadae2d8..df4c0d4ba3474 100644 --- a/doc_cn/demo/index.rst +++ b/doc_cn/demo/index.rst @@ -4,23 +4,23 @@ 图像 '''' -* `图像分类 `_ +* `图像分类 <../..doc/demo/image_classification/index.html>`_ 自然语言处理 '''''''''''' -* `情感分析 `_ -* `文本生成 `_ -* `词性标注 `_ +* `情感分析 <../../doc/demo/sentiment_analysis/index.html>`_ +* `文本生成 <../../doc/demo/text_generation/index.html>`_ +* `词性标注 <../../doc/demo/semantic_role_labeling/index.html>`_ 推荐 '''' -* `MovieLens数据集 `_ -* `MovieLens评分回归 `_ +* `MovieLens数据集 <../../doc/demo/rec/ml_dataset.html>`_ +* `MovieLens评分回归 <../../doc/demo/rec/ml_regression.html>`_ 常用模型 '''''''' -* `ImageNet: ResNet `_ -* `Embedding: Chinese Word `_ +* `ImageNet: ResNet <../../doc/demo/imagenet_model/resnet_model.html>`_ +* `Embedding: Chinese Word <../../doc/demo/embedding_model/index.html>`_ diff --git a/python/paddle/trainer_config_helpers/data_sources.py b/python/paddle/trainer_config_helpers/data_sources.py index 8f3dcb96a931d..8ada3903dc06b 100644 --- a/python/paddle/trainer_config_helpers/data_sources.py +++ b/python/paddle/trainer_config_helpers/data_sources.py @@ -14,10 +14,6 @@ """ Data Sources are helpers to define paddle training data or testing data. -There are several data attributes will be used by paddle: - -- Data ProviderType\: such as Python, Protobuf -- Data File list\: a single file that contains all data file paths """ from paddle.trainer.config_parser import * from .utils import deprecated @@ -27,8 +23,7 @@ except ImportError: import pickle -__all__ = ['define_py_data_sources', - 'define_py_data_sources2'] +__all__ = ['define_py_data_sources2'] def define_py_data_source(file_list, cls, module, @@ -50,11 +45,8 @@ def define_py_data_source(file_list, cls, module, define_py_data_source("train.list", TrainData, "data_provider", "process", args={"dictionary": dict_name}) - The related data provider can refer to - `here `__. - :param data_cls: - :param file_list: file list name. + :param file_list: file list name, which contains all data file paths :type file_list: basestring :param cls: Train or Test Class. :type cls: TrainData or TestData @@ -105,27 +97,10 @@ def py_data2(files, load_data_module, load_data_object, load_data_args, def define_py_data_sources(train_list, test_list, module, obj, args=None, train_async=False, data_cls=PyData): """ - Define python Train/Test data sources in one method. If train/test use - the same Data Provider configuration, module/obj/args contain one argument, - otherwise contain a list or tuple of arguments. For example\: - - .. code-block:: python - - define_py_data_sources("train.list", "test.list", module="data_provider" - obj="process", args={"dictionary": dict_name}) - - Or. - - .. code-block:: python + The annotation is almost the same as define_py_data_sources2, except that + it can specific train_async and data_cls. - define_py_data_sources("train.list", "test.list", module="data_provider" - obj=["process_train", "process_test"], - args=[{"dictionary": dict_train}, {"dictionary": dict_test}]) - - The related data provider can refer to - `here `__. - - :param data_cls: + :param data_cls: :param train_list: Train list name. :type train_list: basestring :param test_list: Test list name. @@ -183,6 +158,43 @@ def __is_splitable__(o): def define_py_data_sources2(train_list, test_list, module, obj, args=None): + """ + Define python Train/Test data sources in one method. If train/test use + the same Data Provider configuration, module/obj/args contain one argument, + otherwise contain a list or tuple of arguments. For example\: + + .. code-block:: python + + define_py_data_sources2(train_list="train.list", + test_list="test.list", + module="data_provider" + # if train/test use different configurations, + # obj=["process_train", "process_test"] + obj="process", + args={"dictionary": dict_name}) + + The related data provider can refer to + `here <../../data_provider/pydataprovider2.html#dataprovider-for-the-sequential-model>`__. + + :param train_list: Train list name. + :type train_list: basestring + :param test_list: Test list name. + :type test_list: basestring + :param module: python module name. If train and test is different, then + pass a tuple or list to this argument. + :type module: basestring or tuple or list + :param obj: python object name. May be a function name if using + PyDataProviderWrapper. If train and test is different, then pass + a tuple or list to this argument. + :type obj: basestring or tuple or list + :param args: The best practice is using dict() to pass arguments into + DataProvider, and use :code:`@init_hook_wrapper` to receive + arguments. If train and test is different, then pass a tuple + or list to this argument. + :type args: string or picklable object or list or tuple. + :return: None + :rtype: None + """ define_py_data_sources(train_list=train_list, test_list=test_list, module=module, From eed8e34393b62c903839791734da5628be949cbe Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 2 Sep 2016 20:47:34 +0800 Subject: [PATCH 021/324] Fix a link typo in previous commit. Change-Id: Ic42414e2e3a81bdce27f399e4944f6e9cb88da5c --- doc_cn/demo/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc_cn/demo/index.rst b/doc_cn/demo/index.rst index df4c0d4ba3474..71f54bc18fbb5 100644 --- a/doc_cn/demo/index.rst +++ b/doc_cn/demo/index.rst @@ -4,7 +4,7 @@ 图像 '''' -* `图像分类 <../..doc/demo/image_classification/index.html>`_ +* `图像分类 <../../doc/demo/image_classification/index.html>`_ 自然语言处理 '''''''''''' From eb26ebf99dc677530e590527d57d5af262df8b98 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Sat, 3 Sep 2016 08:58:54 +0800 Subject: [PATCH 022/324] Remove img_rnorm_layer and fix ParameterAttribute bug for some layer wrapper. * Fix ParameterAttribute for img_conv_layer, tensor_layer, crf_layer, crf_decoding_layer. Change-Id: I88ed5b511fa34cacd8f2d4abf29cee52487fbf68 From 2fe39f8c7abb9344abe1296e6706546b78c7cb16 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 5 Sep 2016 11:13:10 +0800 Subject: [PATCH 023/324] modify flags.cmake to support modern gpu arch Change-Id: Ie237127bb0f537998827ab29ffde2526c21c6dd1 --- cmake/flags.cmake | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index e05cd52f47338..5c386d442e6fd 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -76,7 +76,7 @@ endforeach() function(specify_cuda_arch cuda_version cuda_arch) if(${cuda_version} VERSION_GREATER "8.0") - foreach(capability 60 61 62) + foreach(capability 61 62) if(${cuda_arch} STREQUAL ${capability}) list(APPEND __arch_flags " -gencode arch=compute_${cuda_arch},code=sm_${cuda_arch}") endif() @@ -90,12 +90,17 @@ function(specify_cuda_arch cuda_version cuda_arch) endif() endfunction() -# Common cuda architectures +# Common gpu architectures: Kepler, Maxwell foreach(capability 30 35 50) list(APPEND __arch_flags " -gencode arch=compute_${capability},code=sm_${capability}") endforeach() -# Custom cuda architecture +# Modern gpu architectures: Pascal +if (CUDA_VERSION VERSION_GREATER "8.0") + list(APPEND __arch_flags " -gencode arch=compute_60,code=sm_60") +endif() + +# Custom gpu architecture set(CUDA_ARCH) if(CUDA_ARCH) From 38d735716b52b61e1d067c875148e097807e24aa Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Mon, 5 Sep 2016 15:27:45 +0800 Subject: [PATCH 024/324] fix dead link and some warning, tiny modify on some document Change-Id: I6ba3ec1daeb3d4aee2556959e87bae7c4c1ad668 --- doc/algorithm/rnn/rnn.rst | 6 +-- doc/build/index.rst | 3 ++ doc/ui/data_provider/pydataprovider2.rst | 57 ++++++++++----------- doc_cn/build_and_install/index.rst | 17 ++++-- doc_cn/build_and_install/install/index.rst | 15 ------ doc_cn/ui/data_provider/index.rst | 21 +++----- doc_cn/ui/data_provider/pydataprovider2.rst | 12 ----- 7 files changed, 52 insertions(+), 79 deletions(-) delete mode 100644 doc_cn/build_and_install/install/index.rst diff --git a/doc/algorithm/rnn/rnn.rst b/doc/algorithm/rnn/rnn.rst index a918f02ab160e..9653ddbf37176 100644 --- a/doc/algorithm/rnn/rnn.rst +++ b/doc/algorithm/rnn/rnn.rst @@ -30,7 +30,7 @@ Then at the :code:`process` function, each :code:`yield` function will return th yield src_ids, trg_ids, trg_ids_next -For more details description of how to write a data provider, please refer to :doc:`Python Data Provider <../py_data_provider_wrapper>`. The full data provider file is located at :code:`demo/seqToseq/dataprovider.py`. +For more details description of how to write a data provider, please refer to `PyDataProvider2 <../../ui/data_provider/index.html>`_. The full data provider file is located at :code:`demo/seqToseq/dataprovider.py`. =============================================== Configure Recurrent Neural Network Architecture @@ -106,7 +106,7 @@ We will use the sequence to sequence model with attention as an example to demon In this model, the source sequence :math:`S = \{s_1, \dots, s_T\}` is encoded with a bidirectional gated recurrent neural networks. The hidden states of the bidirectional gated recurrent neural network :math:`H_S = \{H_1, \dots, H_T\}` is called *encoder vector* The decoder is a gated recurrent neural network. When decoding each token :math:`y_t`, the gated recurrent neural network generates a set of weights :math:`W_S^t = \{W_1^t, \dots, W_T^t\}`, which are used to compute a weighted sum of the encoder vector. The weighted sum of the encoder vector is utilized to condition the generation of the token :math:`y_t`. -The encoder part of the model is listed below. It calls :code:`grumemory` to represent gated recurrent neural network. It is the recommended way of using recurrent neural network if the network architecture is simple, because it is faster than :code:`recurrent_group`. We have implemented most of the commonly used recurrent neural network architectures, you can refer to :doc:`Layers <../trainer_config_helpers/layers>` for more details. +The encoder part of the model is listed below. It calls :code:`grumemory` to represent gated recurrent neural network. It is the recommended way of using recurrent neural network if the network architecture is simple, because it is faster than :code:`recurrent_group`. We have implemented most of the commonly used recurrent neural network architectures, you can refer to `Layers <../../ui/api/trainer_config_helpers/layers_index.html>`_ for more details. We also project the encoder vector to :code:`decoder_size` dimensional space, get the first instance of the backward recurrent network, and project it to :code:`decoder_size` dimensional space: @@ -246,6 +246,6 @@ The code is listed below: outputs(beam_gen) -Notice that this generation technique is only useful for decoder like generation process. If you are working on sequence tagging tasks, please refer to :doc:`Semantic Role Labeling Demo <../../../demo/semantic_role_labeling>` for more details. +Notice that this generation technique is only useful for decoder like generation process. If you are working on sequence tagging tasks, please refer to `Semantic Role Labeling Demo <../../demo/semantic_role_labeling/index.html>`_ for more details. The full configuration file is located at :code:`demo/seqToseq/seqToseq_net.py`. diff --git a/doc/build/index.rst b/doc/build/index.rst index 6fefa7990ae46..2b983dceb2777 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -5,6 +5,7 @@ Install PaddlePaddle ---------------------- .. toctree:: + :maxdepth: 1 :glob: install_* @@ -15,6 +16,7 @@ Build from Source If you want to hack and contribute PaddlePaddle source code, following guides can help you\: .. toctree:: + :maxdepth: 1 :glob: build_from_source.md @@ -29,6 +31,7 @@ state and your experience of installation may not be smooth. If you want to pack docker image, the following guide can help you\: .. toctree:: + :maxdepth: 1 :glob: docker_install.md diff --git a/doc/ui/data_provider/pydataprovider2.rst b/doc/ui/data_provider/pydataprovider2.rst index 94472ad0d8cfb..152f8a6df6634 100644 --- a/doc/ui/data_provider/pydataprovider2.rst +++ b/doc/ui/data_provider/pydataprovider2.rst @@ -152,7 +152,6 @@ Please refer to the following section reference for details. Reference --------- -.. _@provider:: @provider +++++++++ @@ -170,31 +169,28 @@ PaddlePaddle from a user defined function. Its parameters are: usefull in sequential model, that defines batch size is counted upon sequence or token. By default, each sample or sequence counts to 1 when calculating batch size. -* cache is a data cache strategy, see `cache`_ +* cache is a data cache strategy, see `cache`_. * Init_hook function is invoked once the data provider is initialized, - see `init_hook`_ + see `init_hook`_. -.. _input_types:: input_types +++++++++++ PaddlePaddle has four data types, and three sequence types. The four data types are: -* dense_vector represents dense float vector. -* sparse_binary_vector sparse binary vector, most of the value is 0, and +* :code:`dense_vector`: dense float vector. +* :code:`sparse_binary_vector`: sparse binary vector, most of the value is 0, and the non zero elements are fixed to 1. -* sparse_float_vector sparse float vector, most of the value is 0, and some - non zero elements that can be any float value. They are given by the user. -* integer represents an integer scalar, that is especially used for label or - word index. +* :code:`sparse_float_vector`: sparse float vector, most of the value is 0, and some + non zero elements can be any float value. They are given by the user. +* :code:`integer`: an integer scalar, that is especially used for label or word index. +The three sequence types are: -The three sequence types are - -* SequenceType.NO_SEQUENCE means the sample is not a sequence -* SequenceType.SEQUENCE means the sample is a sequence -* SequenceType.SUB_SEQUENCE means it is a nested sequence, that each timestep of +* :code:`SequenceType.NO_SEQUENCE` means the sample is not a sequence. +* :code:`SequenceType.SEQUENCE` means the sample is a sequence. +* :code:`SequenceType.SUB_SEQUENCE` means it is a nested sequence, that each timestep of the input sequence is also a sequence. Different input type has a defferenct input format. Their formats are shown @@ -214,36 +210,39 @@ in the above table. where f represents a float value, i represents an integer value. -.. _init_hook:: -.. _settings:: init_hook +++++++++ init_hook is a function that is invoked once the data provoder is initialized. Its parameters lists as follows: -* The first parameter is a settings object, which is the same to :code:'settings' - in :code:`process` method. The object contains several attributes, including: - * settings.input_types the input types. Reference `input_types`_ - * settings.logger a logging object +* The first parameter is a settings object, which is the same to :code:`settings` + in :code:`process` method. The object contains several attributes, including: + + * :code:`settings.input_types`: the input types. Reference `input_types`_. + * :code:`settings.logger`: a logging object. + * The rest parameters are the key word arguments. It is made up of PaddpePaddle pre-defined parameters and user defined parameters. - * PaddlePaddle defines parameters including: - * is_train is a bool parameter that indicates the DataProvider is used in - training or testing - * file_list is the list of all files. + + * PaddlePaddle-defined parameters including: + + * :code:`is_train` is a bool parameter that indicates the DataProvider is used in + training or testing. + * :code:`file_list` is the list of all files. + * User-defined parameters args can be set in training configuration. Note, PaddlePaddle reserves the right to add pre-defined parameter, so please use :code:`**kwargs` in init_hook to ensure compatibility by accepting the parameters which your init_hook does not use. -.. _cache :: cache +++++ -DataProvider provides two simple cache strategy. They are -* CacheType.NO_CACHE means do not cache any data, then data is read at runtime by +DataProvider provides two simple cache strategy. They are: + +* :code:`CacheType.NO_CACHE` means do not cache any data, then data is read at runtime by the user implemented python module every pass. -* CacheType.CACHE_PASS_IN_MEM means the first pass reads data by the user +* :code:`CacheType.CACHE_PASS_IN_MEM` means the first pass reads data by the user implemented python module, and the rest passes will directly read data from memory. diff --git a/doc_cn/build_and_install/index.rst b/doc_cn/build_and_install/index.rst index 67d85eca9bbd0..e9182903c5f62 100644 --- a/doc_cn/build_and_install/index.rst +++ b/doc_cn/build_and_install/index.rst @@ -1,8 +1,15 @@ 编译与安装 ======================== -.. toctree:: - :maxdepth: 1 - - install/index.rst - cmake/index.rst +PaddlePaddle提供数个预编译的二进制来进行安装，包括Docker镜像，ubuntu的deb安装包等。我们推荐使用Docker镜像来部署环境，同时欢迎贡献更多的安装包。 + +Note: The intallation packages are still in pre-release state and your experience of installation may not be smooth. + +注意：目前PaddlePaddle的安装包还处在pre-release的状态，使用起来或许会不是很顺畅。 + +.. toctree:: + :maxdepth: 1 + + install/docker_install.rst + install/ubuntu_install.rst + cmake/index.rst diff --git a/doc_cn/build_and_install/install/index.rst b/doc_cn/build_and_install/install/index.rst deleted file mode 100644 index ce463728c78c9..0000000000000 --- a/doc_cn/build_and_install/install/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -安装PaddlePaddle -========== - -PaddlePaddle提供数个预编译的二进制来进行安装。他们包括Docker镜像，ubuntu的deb安装包等 -。欢迎贡献更多的安装包。我们更推荐使用Docker镜像来部署PaddlePaddle环境。 - -Note: The intallation packages are still in pre-release -state and your experience of installation may not be smooth. - -注意!目前PaddlePaddle的安装包还处在pre-release的状态， -使用起来或许会不是很顺畅。 - -.. toctree:: - docker_install.rst - ubuntu_install.rst diff --git a/doc_cn/ui/data_provider/index.rst b/doc_cn/ui/data_provider/index.rst index 681a131b66389..ec8f8e5dc5b29 100644 --- a/doc_cn/ui/data_provider/index.rst +++ b/doc_cn/ui/data_provider/index.rst @@ -1,24 +1,15 @@ PaddlePaddle的数据提供(DataProvider)介绍 -================================== +======================================== -数据提供(DataProvider，后用DataProvider代替)是PaddlePaddle负责提供数据的模块。其作用是将训练数据 -传入内存或者显存，让神经网络可以进行训练。简单的使用，用户可以使用Python的 -:code:`PyDataProvider` 来自定义传数据的过程。如果有更复杂的使用，或者需要更高的效率， -用户也可以在C++端自定义一个 :code:`DataProvider` 。 +数据提供(DataProvider)是PaddlePaddle负责提供数据的模块。其作用是将训练数据传入内存或者显存，让神经网络可以进行训练。简单的使用，用户可以使用Python的 :code:`PyDataProvider` 来自定义传数据的过程。如果有更复杂的使用，或者需要更高的效率，用户也可以在C++端自定义一个 :code:`DataProvider` 。 -PaddlePaddle需要用户在网络配置(trainer_config.py)中定义使用什么DataProvider，和DataProvider -的一些参数，训练文件列表(train.list)和测试文件列表(test.list)。 +PaddlePaddle需要用户在网络配置(trainer_config.py)中定义使用哪种DataProvider及其参数，训练文件列表(train.list)和测试文件列表(test.list)。 -其中，train.list和test.list均为本地的两个文件(推荐直接放置到训练目录，以相对路径引用)。如果 -test.list不设置，或者设置为None的话，那么在训练过程中，不会执行测试操作。否则，则会根据命令行 -参数指定的测试方式，在训练过程中进行测试，从而防止过拟合。 +其中，train.list和test.list均为本地的两个文件(推荐直接放置到训练目录，以相对路径引用)。如果test.list不设置，或者设置为None，那么在训练过程中，不会执行测试操作。否则，会根据命令行参数指定的测试方式，在训练过程中进行测试，从而防止过拟合。 -一般情况下，train.list和test.list为纯文本文件，其每一行对应这每一个数据文件。数据文件存放在 -本地磁盘中，将文件的绝对路径或相对路径(相对于PaddlePaddle程序运行时的路径)的方式写在train.list和 -test.list中。当然，train.list和test.list也可以放置hdfs文件路径，或者数据库连接地址等等。 -用户在DataProvider中需要实现如何访问其中每一个文件。 +一般情况下，train.list和test.list为纯文本文件，一行对应一个数据文件，数据文件存放在本地磁盘中。将文件的绝对路径或相对路径(相对于PaddlePaddle程序运行时的路径)写在train.list和test.list中。当然，train.list和test.list也可以放置hdfs文件路径，或者数据库连接地址等等。 -DataProvider的具体用法和如何实现一个新的DataProvider，请参考下述文章: +用户在DataProvider中需要实现如何访问其中每一个文件。DataProvider的具体用法和如何实现一个新的DataProvider，请参考下述文章: .. toctree:: diff --git a/doc_cn/ui/data_provider/pydataprovider2.rst b/doc_cn/ui/data_provider/pydataprovider2.rst index 766f583538655..e743e4168821f 100644 --- a/doc_cn/ui/data_provider/pydataprovider2.rst +++ b/doc_cn/ui/data_provider/pydataprovider2.rst @@ -116,8 +116,6 @@ DataProvider创建的时候执行。这个初始化函数具有如下参数: 参考(Reference) --------------- -.. _@provider:: - @provider +++++++++ @@ -134,9 +132,6 @@ DataProvider创建的时候执行。这个初始化函数具有如下参数: * cache 是数据缓存的策略，参考 `cache`_ * init_hook 是初始化时调用的函数，参考 `init_hook`_ - -.. _input_types:: - input_types +++++++++++ @@ -169,16 +164,11 @@ PaddlePaddle的数据包括四种主要类型，和三种序列模式。其中 其中，f代表一个浮点数，i代表一个整数。 -.. _init_hook:: -.. _settings:: - init_hook +++++++++ init_hook可以传入一个函数。这个函数在初始化的时候会被调用。这个函数的参数是: - - * 第一个参数是 settings 对象。这个对象和process的第一个参数一致。具有的属性有 * settings.input_types 设置输入类型。参考 `input_types`_ * settings.logger 一个logging对象 @@ -192,8 +182,6 @@ init_hook可以传入一个函数。这个函数在初始化的时候会被调 注意，PaddlePaddle保留添加参数的权力，所以init_hook尽量使用 :code:`**kwargs` , 来接受不使用的 函数来保证兼容性。 -.. _cache:: - cache +++++ From f15aa8097a2435a4256d52e789122c28f9878b2f Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Mon, 5 Sep 2016 19:57:40 +0800 Subject: [PATCH 025/324] Update installation document and quick start document. * Add wheel into installation package. * Improve quick start doc. Change-Id: I6151f0569fef8b73248ad9aa3452ef5f0c1bb900 --- doc/build/build_from_source.md | 3 ++- doc/demo/quick_start/index_en.md | 4 ++-- doc_cn/demo/quick_start/index.md | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index 36681727f6430..a8de2c42d9a57 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -37,11 +37,12 @@ PaddlePaddle also support some build options, you have to install related librar ```bash # necessary sudo apt-get update -sudo apt-get install -y g++ make cmake build-essential libatlas-base-dev python python-pip libpython-dev m4 libprotobuf-dev protobuf-compiler python-protobuf python-numpy git +sudo apt-get install -y g++ make cmake build-essential libatlas-base-dev python python-pip libpython-dev m4 libprotobuf-dev protobuf-compiler python-protobuf python-numpy git # optional sudo apt-get install libgoogle-glog-dev sudo apt-get install libgflags-dev sudo apt-get install libgtest-dev +pip install wheel pushd /usr/src/gtest cmake . make diff --git a/doc/demo/quick_start/index_en.md b/doc/demo/quick_start/index_en.md index dc0c6255f32ed..179da870e87e4 100644 --- a/doc/demo/quick_start/index_en.md +++ b/doc/demo/quick_start/index_en.md @@ -59,7 +59,7 @@ To build your text classification system, your code will need to perform five st ## Preprocess data into standardized format In this example, you are going to use [Amazon electronic product review dataset](http://jmcauley.ucsd.edu/data/amazon/) to build a bunch of deep neural network models for text classification. Each text in this dataset is a product review. This dataset has two categories: “positive” and “negative”. Positive means the reviewer likes the product, while negative means the reviewer does not like the product. -`demo/quick_start` provides scripts for downloading data and preprocessing data as shown below. The data process takes several minutes (about 3 minutes in our machine). +`demo/quick_start` in the source code provides scripts for downloading data and preprocessing data as shown below. The data process takes several minutes (about 3 minutes in our machine). ```bash cd demo/quick_start @@ -423,7 +423,7 @@ paddle train \ mv rank-00000 result.txt ``` -There are several differences between training and inference network configurations. +User can choose the best model base on the training log instead of model `output/pass-00003`. There are several differences between training and inference network configurations. - You do not need labels during inference. - Outputs need to be specified to the classification probability layer (the output of softmax layer), or the id of maximum probability (`max_id` layer). An example to output the id and probability is given in the code snippet. - batch_size = 1. diff --git a/doc_cn/demo/quick_start/index.md b/doc_cn/demo/quick_start/index.md index b1de49068d0a2..05089267b238e 100644 --- a/doc_cn/demo/quick_start/index.md +++ b/doc_cn/demo/quick_start/index.md @@ -32,7 +32,7 @@ ## 数据格式准备(Data Preparation) 在本问题中，我们使用[Amazon电子产品评论数据](http://jmcauley.ucsd.edu/data/amazon/)， -将评论分为好评(正样本)和差评(负样本)两类。`demo/quick_start`里提供了数据下载脚本 +将评论分为好评(正样本)和差评(负样本)两类。源码的`demo/quick_start`里提供了数据下载脚本 和预处理脚本。 ```bash @@ -144,7 +144,7 @@ PyDataProviderWrapper。 我们将以基本的逻辑回归网络作为起点，并逐渐展示更加深入的功能。更详细的网络配置 连接请参考Layer文档。 -所有配置在`demo/quick_start`目录，首先列举逻辑回归网络。 +所有配置在源码`demo/quick_start`目录，首先列举逻辑回归网络。 ### 逻辑回归模型(Logistic Regression) @@ -407,7 +407,7 @@ paddle train \ mv rank-00000 result.txt ``` -与训练网络配置不同的是：无需label相关的层，指定outputs输出概率层(softmax输出)， +这里以`output/pass-00003`为例进行预测，用户可以根据训练log选择test结果最好的模型来预测。与训练网络配置不同的是：无需label相关的层，指定outputs输出概率层(softmax输出)， 指定batch_size=1，数据传输无需label数据，预测数据指定test_list的位置。 预测结果以文本的形式保存在`result.txt`中，一行为一个样本，格式如下： From d6d85add20861082530d396cacdd849f09085e48 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 5 Sep 2016 20:48:51 +0800 Subject: [PATCH 026/324] fix bug in findLastSet * findLastSet function: size_t could be uint, ulong, ulonglong * add default cuda 7.0 in flags.cmake Change-Id: Ica9d09520e6a1468e5fc97e2c29c1ea7c73dafbb --- cmake/flags.cmake | 12 ++++++------ paddle/utils/Util.h | 6 +++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 5c386d442e6fd..4b99e7f7fb6af 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -81,12 +81,8 @@ function(specify_cuda_arch cuda_version cuda_arch) list(APPEND __arch_flags " -gencode arch=compute_${cuda_arch},code=sm_${cuda_arch}") endif() endforeach() - elseif(${cuda_version} VERSION_GREATER "7.0") - foreach(capability 52 53) - if(${cuda_arch} STREQUAL ${capability}) - list(APPEND __arch_flags " -gencode arch=compute_${cuda_arch},code=sm_${cuda_arch}") - endif() - endforeach() + elseif(${cuda_version} VERSION_GREATER "7.0" and ${cuda_arch} STREQUAL "53") + list(APPEND __arch_flags " -gencode arch=compute_${cuda_arch},code=sm_${cuda_arch}") endif() endfunction() @@ -95,6 +91,10 @@ foreach(capability 30 35 50) list(APPEND __arch_flags " -gencode arch=compute_${capability},code=sm_${capability}") endforeach() +if (CUDA_VERSION VERSION_GREATER "7.0") + list(APPEND __arch_flags " -gencode arch=compute_52,code=sm_52") +endif() + # Modern gpu architectures: Pascal if (CUDA_VERSION VERSION_GREATER "8.0") list(APPEND __arch_flags " -gencode arch=compute_60,code=sm_60") diff --git a/paddle/utils/Util.h b/paddle/utils/Util.h index 7d43713d5f4c5..11a03e141dec5 100644 --- a/paddle/utils/Util.h +++ b/paddle/utils/Util.h @@ -72,7 +72,11 @@ namespace paddle { * \f] */ inline constexpr size_t findLastSet(size_t x) { - return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0; // NOLINT + return std::is_same::value ? + (x ? 8 * sizeof(x) - __builtin_clz(x) : 0) + : (std::is_same::value ? // NOLINT + (x ? 8 * sizeof(x) - __builtin_clzl(x) : 0) + : (x ? 8 * sizeof(x) - __builtin_clzll(x) : 0)); } /** From 5209fccea16dc01ea107f80d37938478e69c5447 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Tue, 6 Sep 2016 13:28:59 +0800 Subject: [PATCH 027/324] Update installation document and quick start. * Add wheel into installation package. * Improve quick start doc. Change-Id: I0254e48fd90b874ed3163654b0fab607c0a4195d --- doc/build/build_from_source.md | 2 +- doc/demo/quick_start/index_en.md | 2 +- doc_cn/demo/quick_start/index.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index a8de2c42d9a57..a191d31318aa6 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -42,7 +42,7 @@ sudo apt-get install -y g++ make cmake build-essential libatlas-base-dev python sudo apt-get install libgoogle-glog-dev sudo apt-get install libgflags-dev sudo apt-get install libgtest-dev -pip install wheel +sudo pip install wheel pushd /usr/src/gtest cmake . make diff --git a/doc/demo/quick_start/index_en.md b/doc/demo/quick_start/index_en.md index 179da870e87e4..ee3fa2a2166f4 100644 --- a/doc/demo/quick_start/index_en.md +++ b/doc/demo/quick_start/index_en.md @@ -59,7 +59,7 @@ To build your text classification system, your code will need to perform five st ## Preprocess data into standardized format In this example, you are going to use [Amazon electronic product review dataset](http://jmcauley.ucsd.edu/data/amazon/) to build a bunch of deep neural network models for text classification. Each text in this dataset is a product review. This dataset has two categories: “positive” and “negative”. Positive means the reviewer likes the product, while negative means the reviewer does not like the product. -`demo/quick_start` in the source code provides scripts for downloading data and preprocessing data as shown below. The data process takes several minutes (about 3 minutes in our machine). +`demo/quick_start` in the [source code](https://github.com/baidu/Paddle) provides scripts for downloading data and preprocessing data as shown below. The data process takes several minutes (about 3 minutes in our machine). ```bash cd demo/quick_start diff --git a/doc_cn/demo/quick_start/index.md b/doc_cn/demo/quick_start/index.md index 05089267b238e..34cd4a840e442 100644 --- a/doc_cn/demo/quick_start/index.md +++ b/doc_cn/demo/quick_start/index.md @@ -32,7 +32,7 @@ ## 数据格式准备(Data Preparation) 在本问题中，我们使用[Amazon电子产品评论数据](http://jmcauley.ucsd.edu/data/amazon/)， -将评论分为好评(正样本)和差评(负样本)两类。源码的`demo/quick_start`里提供了数据下载脚本 +将评论分为好评(正样本)和差评(负样本)两类。[源码](https://github.com/baidu/Paddle)的`demo/quick_start`里提供了数据下载脚本 和预处理脚本。 ```bash @@ -144,7 +144,7 @@ PyDataProviderWrapper。 我们将以基本的逻辑回归网络作为起点，并逐渐展示更加深入的功能。更详细的网络配置 连接请参考Layer文档。 -所有配置在源码`demo/quick_start`目录，首先列举逻辑回归网络。 +所有配置在[源码](https://github.com/baidu/Paddle)`demo/quick_start`目录，首先列举逻辑回归网络。 ### 逻辑回归模型(Logistic Regression) From daaf5a42ec0c058500ff1f8c99a12e3f38f35ef1 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 8 Sep 2016 15:03:45 +0800 Subject: [PATCH 028/324] Add travis support to PaddlePaddle --- .travis.yml | 35 +++++++++++++++++++++++++ CMakeLists.txt | 1 + README.md | 1 + paddle/scripts/travis/before_install.sh | 7 +++++ paddle/scripts/travis/build.sh | 9 +++++++ paddle/scripts/travis/unittest.sh | 6 +++++ paddle/trainer/tests/CMakeLists.txt | 11 ++++---- 7 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 .travis.yml create mode 100755 paddle/scripts/travis/before_install.sh create mode 100755 paddle/scripts/travis/build.sh create mode 100755 paddle/scripts/travis/unittest.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000000..644b8dfb23636 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,35 @@ +language: cpp +cache: ccache +sudo: required +dist: trusty +addons: + apt: + packages: + - gcc-4.8 + - g++-4.8 + - wget + - git + - build-essential + - libatlas-base-dev + - python + - python-pip + - python2.7-dev + - m4 + - libprotobuf-dev + - protobuf-compiler + - python-protobuf + - python-numpy + - python-wheel + - libgoogle-glog-dev + - libgflags-dev + - libgtest-dev +before_install: + - pip install wheel protobuf + - sudo paddle/scripts/travis/before_install.sh +script: + - paddle/scripts/travis/build.sh + - paddle/scripts/travis/unittest.sh +notifications: + email: + on_success: change + on_failure: always diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b83ab256e0c0..0d950e144ffdc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,7 @@ option(WITH_TIMER "Compile PaddlePaddle use timer" OFF) option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ${GTEST_FOUND}) option(WITH_DOC "Compile PaddlePaddle with documentation" OFF) option(WITH_SWIG_PY "Compile PaddlePaddle with py PaddlePaddle prediction api" ${SWIG_FOUND}) +option(ON_TRAVIS "Running test on travis-ci or not." OFF) if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel" diff --git a/README.md b/README.md index 7b948c9e893af..cc2fc68ac3143 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # PaddlePaddle +[![Build Status](https://travis-ci.org/baidu/Paddle.svg?branch=master)](https://travis-ci.org/baidu/Paddle) Welcome to the PaddlePaddle GitHub. diff --git a/paddle/scripts/travis/before_install.sh b/paddle/scripts/travis/before_install.sh new file mode 100755 index 0000000000000..ec2ac1f224076 --- /dev/null +++ b/paddle/scripts/travis/before_install.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e +pushd /usr/src/gtest +cmake . +make +sudo cp *.a /usr/lib +popd diff --git a/paddle/scripts/travis/build.sh b/paddle/scripts/travis/build.sh new file mode 100755 index 0000000000000..638b1bafb4398 --- /dev/null +++ b/paddle/scripts/travis/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +cd `dirname $0` +cd ../../../ +set -e +mkdir build +cd build +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON +make -j `nproc` diff --git a/paddle/scripts/travis/unittest.sh b/paddle/scripts/travis/unittest.sh new file mode 100755 index 0000000000000..a095e308f32ee --- /dev/null +++ b/paddle/scripts/travis/unittest.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e +cd `dirname $0` +cd ../../../build +env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j `nproc`" + diff --git a/paddle/trainer/tests/CMakeLists.txt b/paddle/trainer/tests/CMakeLists.txt index 370f0b4b4113a..aabf44d651200 100644 --- a/paddle/trainer/tests/CMakeLists.txt +++ b/paddle/trainer/tests/CMakeLists.txt @@ -53,12 +53,13 @@ add_test(NAME test_CompareTwoOpts ################# test_CompareSparse ################## add_unittest_without_exec(test_CompareSparse test_CompareSparse.cpp) -add_test(NAME test_CompareSparse - COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ - ./.set_port.sh -p port -n 6 - ${CMAKE_CURRENT_BINARY_DIR}/test_CompareSparse +if(NOT ON_TRAVIS) + add_test(NAME test_CompareSparse + COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ + ./.set_port.sh -p port -n 6 + ${CMAKE_CURRENT_BINARY_DIR}/test_CompareSparse WORKING_DIRECTORY ${PROJ_ROOT}/paddle/) - +endif() ################# test_recurrent_machine_generation ############### add_unittest_without_exec(test_recurrent_machine_generation test_recurrent_machine_generation.cpp) From 1e1a33b574c2dde11c8c112f550b39207673a07e Mon Sep 17 00:00:00 2001 From: Haonan Date: Tue, 6 Sep 2016 11:55:02 -0700 Subject: [PATCH 029/324] Argument concat for subsequence start positions Change-Id: Ia60c008a8c922f66e6b5e2ca3e488fc4625d6506 --- paddle/parameter/Argument.cpp | 46 ++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/paddle/parameter/Argument.cpp b/paddle/parameter/Argument.cpp index 93f86ceccffc5..8610a66452358 100644 --- a/paddle/parameter/Argument.cpp +++ b/paddle/parameter/Argument.cpp @@ -269,6 +269,9 @@ void Argument::concat(const std::vector& args, const std::vector& selectRows, const std::vector& seqStartPos, bool useGpu, hl_stream_t stream, PassType passType) { + CHECK(!subSequenceStartPositions) + << "undefined behavior for subsequence positions"; + size_t batchSize = selectRows.size(); auto copyArg = [batchSize, stream](MatrixPtr& dst, MatrixPtr src, int startRow, int pos, int size, @@ -347,9 +350,11 @@ void Argument::concat(const std::vector& args, bool useGpu, hl_stream_t stream, PassType passType) { int32_t batchSize = 0; int64_t numSequences = 0; + int64_t numSubSequences = 0; for (auto& arg : args) { batchSize += arg.getBatchSize(); numSequences += arg.getNumSequences(); + numSubSequences += arg.getNumSubSequences(); } auto copyArg = [batchSize, stream](MatrixPtr& dst, MatrixPtr src, @@ -393,8 +398,26 @@ void Argument::concat(const std::vector& args, bool useGpu, std::copy(src->begin(), src->end(), dst->begin() + startRow); }; + auto copySequencePos = [] + (ICpuGpuVectorPtr& dstSeq, const ICpuGpuVectorPtr& srcSeq, + int dstNumSequences, int srcNumSequences, + int& startSequences, int startRow) { + if (srcSeq) { + ICpuGpuVector::resizeOrCreate(dstSeq, dstNumSequences + 1, false); + const int* src = srcSeq->getData(false); + int* dest = dstSeq->getMutableData(false); + for (int i = 0; i < srcNumSequences + 1; ++i) { + dest[i + startSequences] = src[i] + startRow; + } + startSequences += srcNumSequences; + } else { + dstSeq.reset(); + } + }; + int startRow = 0; int startSequences = 0; + int startSubSequences = 0; dataId = args[0].dataId; for (auto& arg : args) { CHECK_EQ(arg.dataId, dataId) << "Arguments in concat should have" @@ -403,17 +426,18 @@ void Argument::concat(const std::vector& args, bool useGpu, copyArg(value, arg.value, startRow, useGpu); if (passType != PASS_TEST) copyArg(grad, arg.grad, startRow, useGpu); copyIds(ids, arg.ids, startRow, useGpu); - if (arg.sequenceStartPositions) { - ICpuGpuVector::resizeOrCreate(sequenceStartPositions, - numSequences + 1, - false); - const int* src = arg.sequenceStartPositions->getData(false); - int* dest = sequenceStartPositions->getMutableData(false); - for (int i = 0; i < arg.getNumSequences() + 1; ++i) { - dest[i + startSequences] = src[i] + startRow; - } - startSequences += arg.getNumSequences(); - } + copySequencePos(sequenceStartPositions, + arg.sequenceStartPositions, + numSequences, + arg.getNumSequences(), + startSequences, + startRow); + copySequencePos(subSequenceStartPositions, + arg.subSequenceStartPositions, + numSubSequences, + arg.getNumSubSequences(), + startSubSequences, + startRow); copyStrs(strs, arg.strs, startRow, useGpu); startRow += arg.getBatchSize(); } From 6a873f505f77b3fb183f791358413c27afa5b922 Mon Sep 17 00:00:00 2001 From: Haonan Date: Tue, 6 Sep 2016 11:44:57 -0700 Subject: [PATCH 030/324] fix the layer name error when stacking RecurrentGroups * the last layer in the stack already has all the suffixes --- python/paddle/trainer/config_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 53d8bb98f09e8..b26a63e7f3c1d 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -262,8 +262,8 @@ def SubModelEnd(name = None): def MakeLayerNameInParentSubmodel(name): suffix = "" - for submodel in g_submodel_stack[1:]: - suffix = "@" + submodel.name + suffix + if len(g_submodel_stack) > 1: + suffix = "@" + g_submodel_stack[-1].name return name + suffix def GetLayerBaseName(name): From fbfd24e6d949d60fd415dfd6bb393ca02eb27bcc Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 7 Sep 2016 11:53:40 +0800 Subject: [PATCH 031/324] revert CRFLayer, remove wrong gpu support Change-Id: I636cf13af5becb1168bc9749266b55580c46f6c9 --- paddle/gserver/layers/CRFLayer.cpp | 81 +++++-------------------- paddle/gserver/layers/CRFLayer.h | 5 -- paddle/gserver/tests/test_LayerGrad.cpp | 7 +-- 3 files changed, 17 insertions(+), 76 deletions(-) diff --git a/paddle/gserver/layers/CRFLayer.cpp b/paddle/gserver/layers/CRFLayer.cpp index df8a2b03142b8..fb0a0ddb3d45b 100644 --- a/paddle/gserver/layers/CRFLayer.cpp +++ b/paddle/gserver/layers/CRFLayer.cpp @@ -47,81 +47,40 @@ bool CRFLayer::init(const LayerMap& layerMap, // We don't need sequenceStartPositions because each sample of output_ is // for the cost of one sequence. setNeedSequenceInfo(false); - if (useGpu_) { - tmpCpuInput_.reserve(inputLayers_.size()); - for (size_t i = 0; i < inputLayers_.size(); i++) { - tmpCpuInput_.push_back(Argument()); - } - } + return true; } void CRFLayer::forward(PassType passType) { Layer::forward(passType); - if (useGpu_) { - for (size_t i = 0; i < inputLayers_.size(); i++) { - tmpCpuInput_[i].resizeAndCopyFrom(getInput(i), false, HPPL_STREAM_1); - } - VectorPtr cpuParameterValue; - VectorPtr cpuParameterGradient; - cpuParameterValue = - Vector::create(parameter_->getBuf(PARAMETER_VALUE)->getSize(), false); - cpuParameterValue-> - copyFrom(*parameter_->getBuf(PARAMETER_VALUE), HPPL_STREAM_1); - if (parameter_->getBuf(PARAMETER_GRADIENT)) { - cpuParameterGradient = - Vector::create(parameter_->getBuf(PARAMETER_GRADIENT)->getSize(), - false); - cpuParameterGradient-> - copyFrom(*parameter_->getBuf(PARAMETER_GRADIENT), HPPL_STREAM_1); - } else { - cpuParameterGradient = nullptr; - } - forwardImp(tmpCpuInput_[0], tmpCpuInput_[1], cpuParameterValue, - cpuParameterGradient); - parameter_->getBuf(PARAMETER_VALUE)->copyFrom(*cpuParameterValue, - HPPL_STREAM_1); - if (parameter_->getBuf(PARAMETER_GRADIENT)) { - parameter_->getBuf(PARAMETER_GRADIENT)->copyFrom(*cpuParameterGradient, - HPPL_STREAM_1); - } - } else { - forwardImp(getInput(0), getInput(1), parameter_->getBuf(PARAMETER_VALUE), - parameter_->getBuf(PARAMETER_GRADIENT)); - } -} -void CRFLayer::forwardImp(const Argument&output, - const Argument& label, - VectorPtr parameterValue, - VectorPtr parameterGradient) { + CHECK(!useGpu_) << "GPU is not supported"; + + const Argument& output = getInput(0); + const Argument& label = getInput(1); CHECK(label.sequenceStartPositions); CHECK(label.ids); int batchSize = output.getBatchSize(); size_t numSequences = label.sequenceStartPositions->getSize() - 1; resizeOutput(numSequences, 1); - std::vector out(numSequences); const int* starts = label.sequenceStartPositions->getData(false); CHECK_EQ(starts[numSequences], batchSize); - VectorPtr cpuParameterValue; - VectorPtr cpuParameterGradient; - for (size_t i = 0; i < numSequences; ++i) { if (i >= crfs_.size()) { crfs_.emplace_back(numClasses_, - parameterValue->getData(), - parameterGradient - ? parameterGradient->getData() + parameter_->getBuf(PARAMETER_VALUE)->getData(), + parameter_->getBuf(PARAMETER_GRADIENT) + ? parameter_->getBuf(PARAMETER_GRADIENT)->getData() : nullptr); } - out[i] = crfs_[i].forward( + output_.value->getData()[i] = crfs_[i].forward( output.value->getData() + numClasses_ * starts[i], label.ids->getData() + starts[i], starts[i + 1] - starts[i]); } - output_.value->copyFrom(out.data(), numSequences); + if (weightLayer_) { const MatrixPtr& weight = getInputValue(*weightLayer_); getOutputValue()->dotMul(*getOutputValue(), *weight); @@ -129,22 +88,8 @@ void CRFLayer::forwardImp(const Argument&output, } void CRFLayer::backward(const UpdateCallback &callback) { - (void)callback; - if (useGpu_) { - backwardImp(callback, tmpCpuInput_[0], tmpCpuInput_[1]); - const_cast(getInput(0)). - resizeAndCopyFrom(tmpCpuInput_[0], true, HPPL_STREAM_1); - const_cast(getInput(1)). - resizeAndCopyFrom(tmpCpuInput_[1], true, HPPL_STREAM_1); - - } else { - backwardImp(callback, getInput(0), getInput(1)); - } -} - -void CRFLayer::backwardImp(const UpdateCallback& callback, - const Argument&output, - const Argument& label) { + const Argument& output = getInput(0); + const Argument& label = getInput(1); const int* starts = label.sequenceStartPositions->getData(false); int numSequences = label.sequenceStartPositions->getSize() - 1; @@ -159,9 +104,11 @@ void CRFLayer::backwardImp(const UpdateCallback& callback, grad->mulScalar(weight); } } + if (coeff_ != real(1.0f)) { output.grad->mulScalar(coeff_); } + parameter_->incUpdate(callback); } diff --git a/paddle/gserver/layers/CRFLayer.h b/paddle/gserver/layers/CRFLayer.h index 5facb9b54818c..c6ba8e7c965a3 100644 --- a/paddle/gserver/layers/CRFLayer.h +++ b/paddle/gserver/layers/CRFLayer.h @@ -32,11 +32,7 @@ class CRFLayer : public Layer { explicit CRFLayer(const LayerConfig& config) : Layer(config) {} virtual bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); virtual void forward(PassType passType); - void forwardImp(const Argument&output, const Argument& label, - VectorPtr parameterValue, VectorPtr parameterGradient); virtual void backward(const UpdateCallback& callback); - void backwardImp(const UpdateCallback& callback, const Argument&output, - const Argument& label); protected: size_t numClasses_; @@ -44,7 +40,6 @@ class CRFLayer : public Layer { std::vector crfs_; LayerPtr weightLayer_; // weight for each sequence real coeff_; // weight for the layer - std::vector tmpCpuInput_; }; } // namespace paddle diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 7bb79ff5b702a..5c80eb546cfaf 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -179,10 +179,9 @@ TEST(Layer, CRFLayer) { config.layerConfig.add_inputs(); config.layerConfig.add_inputs(); - for (auto useGpu : {false, true}) { - testLayerGrad(config, "crf", 100, /* trans */ false, /* useGpu */ useGpu, - false /*useWeight*/, 0.03 /*epsilon*/); - } + // Not support GPU now + testLayerGrad(config, "crf", 100, /* trans */ false, /* useGpu */ false, + false /*useWeight*/, 0.03 /*epsilon*/); } TEST(Layer, CTCLayer) { From 721b09eee6f3870cfe7a9ec9b59ef6ba8d42e264 Mon Sep 17 00:00:00 2001 From: liuyuan04 Date: Wed, 7 Sep 2016 13:34:50 +0800 Subject: [PATCH 032/324] Update Jumbo package to 0.8.0b0. Change-Id: I0b8608feab8f6be5094e8981fc5f65cb401ed415 --- CMakeLists.txt | 2 +- paddle/CMakeLists.txt | 3 +++ paddle/{setup.py => setup.py.in} | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) rename paddle/{setup.py => setup.py.in} (93%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d950e144ffdc..007f1f18bb655 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 2.8) project(paddle CXX C) set(PADDLE_MAJOR_VERSION 0) set(PADDLE_MINOR_VERSION 8) -set(PADDLE_PATCH_VERSION 0b) +set(PADDLE_PATCH_VERSION 0b0) set(PADDLE_VERSION ${PADDLE_MAJOR_VERSION}.${PADDLE_MINOR_VERSION}.${PADDLE_PATCH_VERSION}) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") diff --git a/paddle/CMakeLists.txt b/paddle/CMakeLists.txt index c6fa7dc2b16e1..cae0f64400a7e 100644 --- a/paddle/CMakeLists.txt +++ b/paddle/CMakeLists.txt @@ -7,6 +7,9 @@ add_subdirectory(pserver) add_subdirectory(trainer) add_subdirectory(scripts) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in + ${CMAKE_CURRENT_SOURCE_DIR}/setup.py) + if(WITH_PREDICT_SDK) add_subdirectory(predict) endif() diff --git a/paddle/setup.py b/paddle/setup.py.in similarity index 93% rename from paddle/setup.py rename to paddle/setup.py.in index fabe2a6b4c1d5..da86eb795dc58 100644 --- a/paddle/setup.py +++ b/paddle/setup.py.in @@ -35,11 +35,11 @@ pass setup(name="py_paddle", - version="0.8.0b", # TODO(yuyang18): Make this version same as CMake + version="@PADDLE_VERSION@", ext_modules=[ Extension('py_paddle._swig_paddle', # Build SWIG Extension. ['Paddle_wrap.cxx'], - extra_link_args=["-Xlinker", '-start-group'] + + extra_link_args=["-Xlinker", '-start-group'] + extra_links + ["-Xlinker", "-end-group"] ) ], From 7ad55a4e76f334e5b1f86eb45fff5abb74210de8 Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Tue, 6 Sep 2016 23:08:19 -0700 Subject: [PATCH 033/324] Fix ThreadParameterUpdater The reference return type causes ThreadParameterUpdater.cpp:123 seg fault under gcc5.4. Change-Id: I7a1c155892722076a7cb48793b83d5ee525747d1 --- paddle/trainer/ThreadParameterUpdater.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/trainer/ThreadParameterUpdater.h b/paddle/trainer/ThreadParameterUpdater.h index f47d3b08c1677..d8a7a5dd4f12a 100644 --- a/paddle/trainer/ThreadParameterUpdater.h +++ b/paddle/trainer/ThreadParameterUpdater.h @@ -79,7 +79,7 @@ class SgdThreadUpdater : public ParameterUpdater { // The update function for after update operations, such as averager. void threadTraverse(const ParameterOptimizer::TraverseCallback& callback, int tid, size_t numThreads, Parameter* para); - typedef std::function + typedef std::function GetTraverseCallback; void traverse(GetTraverseCallback getTraverseCallback); }; From fdd40e5528576899d5dd744eb772a79c53300d59 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 7 Sep 2016 06:58:45 +0000 Subject: [PATCH 034/324] Fix 32-bit gcc compile warnings. Change-Id: Ibc39ca1d1a27d0d28569e29f41a5647659f8c764 --- paddle/pserver/ParameterClient2.cpp | 6 +++--- paddle/pserver/ParameterServer2.cpp | 2 +- paddle/pserver/SocketChannel.cpp | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/paddle/pserver/ParameterClient2.cpp b/paddle/pserver/ParameterClient2.cpp index 07961cbdcc20c..d0e5352c828d1 100644 --- a/paddle/pserver/ParameterClient2.cpp +++ b/paddle/pserver/ParameterClient2.cpp @@ -278,7 +278,7 @@ void ParameterClient2::prepareSendData( if (sendingPara) { sendJob->parallelInputIovs[serverId].push_back( - {sendMat->getLocalRow(row), sizeof(real) * blockSize}); + {sendMat->getLocalRow(row), sizeof(real) * (size_t) blockSize}); /// detect sparse parameter distribution sparseDistribution_->probeDistribution(serverId, sizeof(real) * blockSize); @@ -302,8 +302,8 @@ void ParameterClient2::prepareSendData( block->set_begin_pos(beginDim); block->set_block_size(endDim - beginDim); if (buf) { - sendJob->parallelInputIovs[serverId].push_back( - {buf + beginDim, sizeof(real) * (endDim - beginDim)}); + sendJob->parallelInputIovs[serverId].push_back({buf + beginDim, + sizeof(real) * ((size_t) (endDim - beginDim))}); } } } diff --git a/paddle/pserver/ParameterServer2.cpp b/paddle/pserver/ParameterServer2.cpp index bb3caeb728d1c..8f72c1988d167 100644 --- a/paddle/pserver/ParameterServer2.cpp +++ b/paddle/pserver/ParameterServer2.cpp @@ -724,7 +724,7 @@ void ParameterServer2::sendBackParameter(const ParameterBlock& block, << " id=" << block.para_id() << " block id=" << block.block_id(); real* valueBuffer = vectors_[parameterType]->getPoint(offset); - outputBuffers->push_back({valueBuffer, block.block_size()}); + outputBuffers->push_back({valueBuffer, (size_t) block.block_size()}); } void ParameterServer2::sendBackParameter(const ParameterBlock& block, diff --git a/paddle/pserver/SocketChannel.cpp b/paddle/pserver/SocketChannel.cpp index ebb4245b9a7df..698473060a4c1 100644 --- a/paddle/pserver/SocketChannel.cpp +++ b/paddle/pserver/SocketChannel.cpp @@ -148,7 +148,8 @@ void SocketChannel::writeMessage(const std::vector& userIovs) { std::vector iovs; iovs.reserve(userIovs.size() + 2); iovs.push_back({&header, sizeof(header)}); - iovs.push_back({&iovLengths[0], sizeof(iovLengths[0]) * header.numIovs}); + iovs.push_back({&iovLengths[0], + sizeof(iovLengths[0]) * (size_t) header.numIovs}); iovs.insert(iovs.end(), userIovs.begin(), userIovs.end()); header.totalLength = 0; From 903d5c7ec04e4074191875abaf58b418219561f2 Mon Sep 17 00:00:00 2001 From: He Date: Wed, 7 Sep 2016 16:01:36 +0800 Subject: [PATCH 035/324] bug fix for hl_matrix_classification_error --- paddle/cuda/src/hl_cuda_matrix.cu | 33 ++++++++++-------------- paddle/math/tests/test_matrixCompare.cpp | 33 +++++++++++++++++++++--- 2 files changed, 43 insertions(+), 23 deletions(-) diff --git a/paddle/cuda/src/hl_cuda_matrix.cu b/paddle/cuda/src/hl_cuda_matrix.cu index 15799919fa137..fc003b7d6377d 100644 --- a/paddle/cuda/src/hl_cuda_matrix.cu +++ b/paddle/cuda/src/hl_cuda_matrix.cu @@ -266,25 +266,21 @@ template __global__ void KeMatrixClassificationError(real* in_A, int* in_B, real* out_C, - int dimM, int dimN) { __shared__ real max_s[blockSize]; __shared__ int max_l[blockSize]; - int cnt = (dimN + blockSize -1) / blockSize; - int tid = threadIdx.x; - int lmt = tid; - int index = 0; - real t; + const int tid = threadIdx.x; + const int rowId = blockIdx.x; max_s[tid] = -1e30f; - for (int ii = 0; ii < cnt && lmt < dimN; ii++) { - index = blockIdx.y*dimN + lmt; - t = in_A[index]; - if (max_s[tid] < t) { - max_s[tid] = t; - max_l[tid] = lmt; + in_A += rowId * dimN; + real tmp; + for (int colId = tid; colId < dimN; colId += blockSize) { + tmp = in_A[colId]; + if (max_s[tid] < tmp) { + max_s[tid] = tmp; + max_l[tid] = colId; } - lmt += blockSize; } __syncthreads(); @@ -300,7 +296,7 @@ __global__ void KeMatrixClassificationError(real* in_A, __syncthreads(); if (tid == 0) { - out_C[blockIdx.y] = (max_l[0] == in_B[blockIdx.y] ? 0 : 1.0f); + out_C[rowId] = (max_l[0] == in_B[rowId] ? 0 : 1.0f); } } @@ -313,12 +309,9 @@ void hl_matrix_classification_error(real* A_d, CHECK_NOTNULL(B_d); CHECK_NOTNULL(C_d); - int blocksX = 1; - int blocksY = dimM; - dim3 threads(1024, 1); - dim3 grid(blocksX, blocksY); - KeMatrixClassificationError<1024><<< grid, threads, 0, STREAM_DEFAULT >>> - (A_d, B_d, C_d, dimM, dimN); + // each sample is calculated by one block + KeMatrixClassificationError<1024><<< dimM, 1024, 0, STREAM_DEFAULT >>> + (A_d, B_d, C_d, dimN); CHECK_SYNC("hl_matrix_classification_error"); } diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index 7caade444b827..fe8eacc2efbc5 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -1697,7 +1697,6 @@ TEST(Matrix, cosSimDerivate) { } } - void testParamReluForward(int height, int width, int w_height, int w_width) { MatrixPtr output = CpuMatrix::create(height, width, false, false); @@ -1736,7 +1735,6 @@ TEST(Matrix, paramReluForward) { } } - void testParamReluBackwardW(int height, int width, int w_height, int w_width) { MatrixPtr oGrad = CpuMatrix::create(height, width, false, false); @@ -1775,7 +1773,6 @@ TEST(Matrix, paramReluBackwardW) { } } - void testParamReluBackwardDiff(int height, int width, int w_height, int w_width) { MatrixPtr oGrad = CpuMatrix::create(height, width, false, false); @@ -1819,6 +1816,36 @@ TEST(Matrix, paramReluBackwardDiff) { } } +void testClassificationError(int numSamples, int dim) { + MatrixPtr cpuError = std::make_shared(numSamples, 1); + MatrixPtr gpuError = std::make_shared(numSamples, 1); + MatrixPtr cpuOutput = std::make_shared(numSamples, dim); + MatrixPtr gpuOutput = std::make_shared(numSamples, dim); + IVectorPtr cpuLabel = std::make_shared(numSamples); + IVectorPtr gpuLabel = std::make_shared(numSamples); + + cpuOutput->randomizeUniform(); + cpuLabel->rand(dim); + gpuOutput->copyFrom(*cpuOutput); + gpuLabel->copyFrom(*cpuLabel); + + cpuError->classificationError(cpuOutput, cpuLabel); + gpuError->classificationError(gpuOutput, gpuLabel); + + MatrixPtr check = std::make_shared(numSamples, 1); + check->copyFrom(*gpuError); + MatrixCheckEqual(*cpuError, *check); +} + +TEST(Matrix, classificationError) { + for (auto numSamples : {1, 10, 100, 1000, 70000}) { + for (auto dim : {1, 10, 100, 1000}) { + VLOG(3) << " numSamples=" << numSamples << " dim=" << dim; + testClassificationError(numSamples, dim); + } + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); From 5547a7601b5fda56a8c7edda45ae91ec4e51cacc Mon Sep 17 00:00:00 2001 From: liuyuan04 Date: Thu, 8 Sep 2016 11:10:23 +0800 Subject: [PATCH 036/324] Refine doc of Python Prediction API, replace DataProviderWrapperConverter with DataProviderConverter. --- doc/ui/predict/predict_sample.py | 8 ++++---- doc/ui/predict/swig_py_paddle_en.rst | 30 ++++++++++++++++++---------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/doc/ui/predict/predict_sample.py b/doc/ui/predict/predict_sample.py index ac16b2b48b9f7..d55d2c730dece 100644 --- a/doc/ui/predict/predict_sample.py +++ b/doc/ui/predict/predict_sample.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from py_paddle import swig_paddle, DataProviderWrapperConverter -from paddle.trainer.PyDataProviderWrapper import DenseSlot +from py_paddle import swig_paddle, DataProviderConverter +from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config TEST_DATA = [[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -89,12 +89,12 @@ def main(): - conf = parse_config("./mnist_model/trainer_config.conf.norm", "") + conf = parse_config("./mnist_model/trainer_config.py", "") print conf.data_config.load_data_args network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) # For code hint. network.loadParameters("./mnist_model/") - converter = DataProviderWrapperConverter(False, [DenseSlot(784)]) + converter = DataProviderConverter([dense_vector(784)]) inArg = converter(TEST_DATA) print network.forwardTest(inArg) diff --git a/doc/ui/predict/swig_py_paddle_en.rst b/doc/ui/predict/swig_py_paddle_en.rst index 9841f124e25a4..b743fc4569146 100644 --- a/doc/ui/predict/swig_py_paddle_en.rst +++ b/doc/ui/predict/swig_py_paddle_en.rst @@ -10,27 +10,35 @@ SWIG. The main steps of predict values in python are: * Predict Here is a sample python script that shows the typical prediction process for the -MNIST classification problem. +MNIST classification problem. A complete sample code could be found at +:code:`src_root/doc/ui/predict/predict_sample.py`. .. literalinclude:: ./predict_sample.py :language: python - :linenos: + :lines: 15-18,90-100,101-104 The module that does the most of the job is py_paddle.swig_paddle, it's generated by SWIG and has complete documents, for more details you can use python's :code:`help()` function. Let's walk through the above python script: -* At the beginning, initialize PaddlePaddle with command line arguments(line 90). -* Parse the configuration file that is used in training(line 93). -* Create a neural network at line 95 according the parsed configuration, then - load the trained parameters from model at line 97. -* A utility class for data transformation is created at line 98. +* At the beginning, use :code:`swig_paddle.initPaddle()` to initialize + PaddlePaddle with command line arguments, for more about command line arguments + see `Command Line Arguments <../cmd_argument/detail_introduction.html>`_. +* Parse the configuration file that is used in training with :code:`parse_config()`. + Because data to predict with always have no label, and output of prediction work + normally is the output layer rather than the cost layer, so you should modify + the configuration file accordingly before using it in the prediction work. +* Create a neural network with + :code:`swig_paddle.GradientMachine.createFromConfigproto()`, which takes the + parsed configuration :code:`conf.model_config` as argument. Then load the + trained parameters from the model with :code:`network.loadParameters()`. +* Create a data converter object of utility class :code:`DataProviderConverter`. - Note: As swig_paddle can only accept C++ matrices, we offer a utility - class DataProviderWraaperConverter that can accept the same input data with - PyDataProviderWrapper, for more information please refer to document + class DataProviderConverter that can accept the same input data with + PyDataProvider2, for more information please refer to document of `PyDataProvider2 <../data_provider/pydataprovider2.html>`_. -* Do the prediction and output the result at line 100, forwardTest is another - utility class that directly takes the activations of the output layer. +* Do the prediction with :code:`forwardTest()`, which takes the converted + input data and outputs the activations of the output layer. Here is a typical output: From d6d91223b5303d0e7dbe10abdde832d904e5bb6c Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 8 Sep 2016 15:26:58 +0800 Subject: [PATCH 037/324] fix docker tag mistake Change-Id: Ia84860cdc25945ececba84fc9807495c9e5f047b --- doc/build/docker_install.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/build/docker_install.md b/doc/build/docker_install.md index 997a755956bf2..3cd9d1730a22b 100644 --- a/doc/build/docker_install.md +++ b/doc/build/docker_install.md @@ -8,12 +8,12 @@ Docker is a tool designed to make it easier to create, deploy, and run applicati ### PaddlePaddle Docker images There are six Docker images: -- paddledev/paddle:latest-cpu: PaddlePaddle CPU binary image. -- paddledev/paddle:latest-gpu: PaddlePaddle GPU binary image. -- paddledev/paddle:latest-cpu-devel: PaddlePaddle CPU binary image plus source code. -- paddledev/paddle:latest-gpu-devel: PaddlePaddle GPU binary image plus source code. -- paddledev/paddle:latest-cpu-demo: PaddlePaddle CPU binary image plus source code and demo -- paddledev/paddle:latest-gpu-demo: PaddlePaddle GPU binary image plus source code and demo +- paddledev/paddle:cpu-latest: PaddlePaddle CPU binary image. +- paddledev/paddle:gpu-latest: PaddlePaddle GPU binary image. +- paddledev/paddle:cpu-devel-latest: PaddlePaddle CPU binary image plus source code. +- paddledev/paddle:gpu-devel-latest: PaddlePaddle GPU binary image plus source code. +- paddledev/paddle:cpu-demo-latest: PaddlePaddle CPU binary image plus source code and demo +- paddledev/paddle:gpu-demo-latest: PaddlePaddle GPU binary image plus source code and demo Tags with latest will be replaced by a released version. @@ -23,7 +23,7 @@ You have to install Docker in your machine which has linux kernel version 3.10+ You can use ```docker pull ```to download images first, or just launch a container with ```docker run```: ```bash -docker run -it paddledev/paddle:lastest-cpu +docker run -it paddledev/paddle:cpu-latest ``` If you want to launch container with GPU support, you need to set some environment variables at the same time: @@ -31,7 +31,7 @@ If you want to launch container with GPU support, you need to set some environme ```bash export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}" export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') -docker run -it paddledev/paddle:latest-gpu +docker run -it paddledev/paddle:gpu-latest ``` ### Notice From dbaabc94fb0b21b7bf91132eab5de954143d870b Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Wed, 7 Sep 2016 15:48:36 +0800 Subject: [PATCH 038/324] fix unitest of test_RecurrentGradientMachine, and some tiny doc update Change-Id: I028e402c964ca4f4431cbf8153bea4379dd4df70 --- doc/demo/imagenet_model/resnet_model.md | 2 +- doc/demo/rec/ml_regression.rst | 2 +- paddle/gserver/tests/sequenceGen.py | 30 +++++++++++-------- .../gserver/tests/sequence_layer_group.conf | 10 +++---- .../tests/sequence_nest_layer_group.conf | 10 +++---- 5 files changed, 30 insertions(+), 24 deletions(-) diff --git a/doc/demo/imagenet_model/resnet_model.md b/doc/demo/imagenet_model/resnet_model.md index 2e5c7f324434d..5403ab9f17d23 100644 --- a/doc/demo/imagenet_model/resnet_model.md +++ b/doc/demo/imagenet_model/resnet_model.md @@ -165,7 +165,7 @@ We provide both C++ and Python interfaces to extract features. The following exa ### C++ Interface -First, specify image data list in `define_py_data_sources` in the config, see example `demo/model_zoo/resnet/resnet.py`. +First, specify image data list in `define_py_data_sources2` in the config, see example `demo/model_zoo/resnet/resnet.py`. ``` train_list = 'train.list' if not is_test else None diff --git a/doc/demo/rec/ml_regression.rst b/doc/demo/rec/ml_regression.rst index 4917f873a934d..0c14e4f5bb7f8 100644 --- a/doc/demo/rec/ml_regression.rst +++ b/doc/demo/rec/ml_regression.rst @@ -257,7 +257,7 @@ In these network, we use several api in `trainer_config_helpers * Text Convolution Pooling Layer, `text_conv_pool <../../ui/api/trainer_config_helpers/networks.html #trainer_config_helpers.networks.text_conv_pool>`_ -* Declare Python Data Sources, `define_py_data_sources +* Declare Python Data Sources, `define_py_data_sources2 <../../ui/api/trainer_config_helpers/data_sources.html>`_ Data Provider diff --git a/paddle/gserver/tests/sequenceGen.py b/paddle/gserver/tests/sequenceGen.py index dd2b90dd4986c..e4727e472d446 100644 --- a/paddle/gserver/tests/sequenceGen.py +++ b/paddle/gserver/tests/sequenceGen.py @@ -18,27 +18,33 @@ import os import sys -from paddle.trainer.PyDataProviderWrapper import * +from paddle.trainer.PyDataProvider2 import * -@init_hook_wrapper -def hook(obj, dict_file, **kwargs): - obj.word_dict = dict_file - obj.slots = [IndexSlot(len(obj.word_dict)), IndexSlot(3)] - obj.logger.info('dict len : %d' % (len(obj.word_dict))) +def hook(settings, dict_file, **kwargs): + settings.word_dict = dict_file + settings.input_types = [integer_value_sequence(len(settings.word_dict)), + integer_value_sequence(3)] + settings.logger.info('dict len : %d' % (len(settings.word_dict))) -@provider(use_seq=True, init_hook=hook) -def process(obj, file_name): +@provider(init_hook=hook) +def process(settings, file_name): with open(file_name, 'r') as fdata: for line in fdata: label, comment = line.strip().split('\t') label = int(''.join(label.split())) words = comment.split() - word_slot = [obj.word_dict[w] for w in words if w in obj.word_dict] + word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] yield word_slot, [label] ## for hierarchical sequence network -@provider(use_seq=True, init_hook=hook) -def process2(obj, file_name): +def hook2(settings, dict_file, **kwargs): + settings.word_dict = dict_file + settings.input_types = [integer_value_sub_sequence(len(settings.word_dict)), + integer_value_sub_sequence(3)] + settings.logger.info('dict len : %d' % (len(settings.word_dict))) + +@provider(init_hook=hook2) +def process2(settings, file_name): with open(file_name) as fdata: label_list = [] word_slot_list = [] @@ -47,7 +53,7 @@ def process2(obj, file_name): label,comment = line.strip().split('\t') label = int(''.join(label.split())) words = comment.split() - word_slot = [obj.word_dict[w] for w in words if w in obj.word_dict] + word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] label_list.append([label]) word_slot_list.append(word_slot) else: diff --git a/paddle/gserver/tests/sequence_layer_group.conf b/paddle/gserver/tests/sequence_layer_group.conf index 9ad2b3762845f..ac031b31280df 100644 --- a/paddle/gserver/tests/sequence_layer_group.conf +++ b/paddle/gserver/tests/sequence_layer_group.conf @@ -21,11 +21,11 @@ dict_file = dict() for line_count, line in enumerate(open(dict_path, "r")): dict_file[line.strip()] = line_count -define_py_data_sources(train_list='gserver/tests/Sequence/train.list', - test_list=None, - module='sequenceGen', - obj='process', - args={"dict_file":dict_file}) +define_py_data_sources2(train_list='gserver/tests/Sequence/train.list', + test_list=None, + module='sequenceGen', + obj='process', + args={"dict_file":dict_file}) settings(batch_size=5) ######################## network configure ################################ diff --git a/paddle/gserver/tests/sequence_nest_layer_group.conf b/paddle/gserver/tests/sequence_nest_layer_group.conf index 8c3a08f16cd1c..38c60b657b969 100644 --- a/paddle/gserver/tests/sequence_nest_layer_group.conf +++ b/paddle/gserver/tests/sequence_nest_layer_group.conf @@ -21,11 +21,11 @@ dict_file = dict() for line_count, line in enumerate(open(dict_path, "r")): dict_file[line.strip()] = line_count -define_py_data_sources(train_list='gserver/tests/Sequence/train.list.nest', - test_list=None, - module='sequenceGen', - obj='process2', - args={"dict_file":dict_file}) +define_py_data_sources2(train_list='gserver/tests/Sequence/train.list.nest', + test_list=None, + module='sequenceGen', + obj='process2', + args={"dict_file":dict_file}) settings(batch_size=2) ######################## network configure ################################ From 7cf8e0c9e6a619cc27c678cd2624c74717ad51ce Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 6 Sep 2016 20:09:52 +0800 Subject: [PATCH 039/324] Lazy install Paddle wheels * install wheels when invoke paddle script if current python don't have paddle packages, or installed a previous version. * Also add `make install` to travis --- .travis.yml | 1 + paddle/scripts/submit_local.sh.in | 34 +++++++++++++++++++++++++++ paddle/scripts/travis/build.sh | 6 +---- paddle/scripts/travis/common.sh | 5 ++++ paddle/scripts/travis/make_install.sh | 5 ++++ paddle/scripts/travis/unittest.sh | 3 +-- python/CMakeLists.txt | 4 ---- 7 files changed, 47 insertions(+), 11 deletions(-) create mode 100755 paddle/scripts/travis/common.sh create mode 100755 paddle/scripts/travis/make_install.sh diff --git a/.travis.yml b/.travis.yml index 644b8dfb23636..a78853e15b158 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,7 @@ before_install: script: - paddle/scripts/travis/build.sh - paddle/scripts/travis/unittest.sh + - paddle/scripts/travis/make_install.sh notifications: email: on_success: change diff --git a/paddle/scripts/submit_local.sh.in b/paddle/scripts/submit_local.sh.in index eed2d315932c5..6d2cab6b16bab 100644 --- a/paddle/scripts/submit_local.sh.in +++ b/paddle/scripts/submit_local.sh.in @@ -43,6 +43,40 @@ fi export PYTHONPATH=${PWD}:${PYTHONPATH} + +# Check python lib installed or not. +pip --help > /dev/null +if [ $? -ne 0 ]; then + echo "pip should be installed to run paddle." + exit 1 +fi + +INSTALLED_VERSION=`pip freeze 2>/dev/null | grep '^paddle' | sed 's/.*==//g'` + +if [ -z ${INSTALLED_VERSION} ]; then + INSTALLED_VERSION="0.0.0" # not installed +fi +cat < Date: Wed, 7 Sep 2016 22:06:27 +0800 Subject: [PATCH 040/324] Fix bugs in CustomStackTrace. * Make layer stack trace shows ThreadId, Forward or Backward. Change-Id: Iba1477adb8c9115c3a67ff2959bb5c878ca706c7 --- paddle/.gitignore | 1 + .../gradientmachines/NeuralNetwork.cpp | 1 + paddle/utils/CustomStackTrace.cpp | 35 ++++ paddle/utils/CustomStackTrace.h | 164 ++++++++++++++---- paddle/utils/Util.cpp | 8 +- 5 files changed, 166 insertions(+), 43 deletions(-) diff --git a/paddle/.gitignore b/paddle/.gitignore index b89bd9d94633a..f921eef14156a 100644 --- a/paddle/.gitignore +++ b/paddle/.gitignore @@ -40,3 +40,4 @@ HPPL_ERROR_LOG unittest.list proto dist +setup.py diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.cpp b/paddle/gserver/gradientmachines/NeuralNetwork.cpp index fca52828957a2..eb1522a178d48 100644 --- a/paddle/gserver/gradientmachines/NeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/NeuralNetwork.cpp @@ -277,6 +277,7 @@ void NeuralNetwork::getState(MachineState& machineState) { } void NeuralNetwork::backward(const UpdateCallback& callback) { + gLayerStackTrace.pop(""); // tell layer trace is during backward. FOR_EACH_R(layer, layers_) { REGISTER_TIMER_INFO("BackwardTimer", (*layer)->getName().c_str()); if ((*layer)->needGradient()) { diff --git a/paddle/utils/CustomStackTrace.cpp b/paddle/utils/CustomStackTrace.cpp index 50d7f5402f586..232a478ecd93a 100644 --- a/paddle/utils/CustomStackTrace.cpp +++ b/paddle/utils/CustomStackTrace.cpp @@ -14,9 +14,44 @@ limitations under the License. */ #include "CustomStackTrace.h" +#include "CommandLineParser.h" +#include + +P_DEFINE_bool(layer_stack_error_only_current_thread, + true, + "Dump current thread or whole process layer stack when signal error " + "occurred. true means only dump current thread layer stack"); namespace paddle { CustomStackTrace gLayerStackTrace; +static std::mutex gLayerStackTraceMtx; +void installLayerStackTracer() { + logging::installFailureWriter([](const char* data, int sz) { + std::lock_guard guard(gLayerStackTraceMtx); + if (!gLayerStackTrace.empty()) { + size_t curTid = -1UL; + std::hash hasher; + gLayerStackTrace.dump([&curTid, &hasher](std::thread::id tid, + bool* isForwarding, + const std::string& layerName) { + if (curTid != hasher(tid)) { + if (curTid != -1UL) { + std::cerr << std::endl; + } + curTid = hasher(tid); + std::cerr << "Thread [" << tid << "] "; + if (isForwarding) { + std::cerr << (*isForwarding ? "Forwarding ": "Backwarding "); + } + } + std::cerr << layerName << ", "; + }, FLAGS_layer_stack_error_only_current_thread); + std::cerr << std::endl; + } + std::cerr.write(data, sz); + }); +} + } // namespace paddle diff --git a/paddle/utils/CustomStackTrace.h b/paddle/utils/CustomStackTrace.h index e1b2d2d8e5ee6..774c4db2b9be4 100644 --- a/paddle/utils/CustomStackTrace.h +++ b/paddle/utils/CustomStackTrace.h @@ -15,6 +15,9 @@ limitations under the License. */ #pragma once #include +#include +#include +#include #include "ThreadLocal.h" @@ -29,25 +32,18 @@ namespace paddle { * @code{.cpp} * * paddle::CustomStackTrace stack; - * PASS_TEST=0; * for (auto& layer : layers){ * stack.push(layer->getName()); - * layer->forward(passType); + * layer->forward(); * } - * for (auto& layer : layers){ + * + * stack.pop(""); // mark under pop stage. + * + * for (auto it = layers.rbegin(); it != layers.rend(); ++it){ + * auto& layer = *it; * layer->backward(passType); * stack.pop(layer->getName()); * } - * - * if(passType == PASS_TEST) { - * stack.clear(); - * } - * else { - * stack.dump([](const std::string& layername){ - * LOG(INFO) << "LayerName: " << layername; - * }) - * } - * * * @endcode */ @@ -55,45 +51,141 @@ template class CustomStackTrace{ public: /** - * @brief Pop out an item from the top of the stack. For safety the item - * will be poped should equal to ip. + * @brief Pop out an item from the top of the stack if item == top. + * Else, just set status to popping. */ - void pop(const T& ip) { - auto& p = *logstack_; - CHECK_EQ(ip, p.top()); - p.pop(); + void pop(const T& item) { + pushing() = false; + auto& s = this->stack(); + if (item == s.top()) { + s.pop(); + } } + /** - * @brief Empty the stack by sequence from top to button. - * @param[in] callback A function deal with each item while dumping. - * It must have and only have a in parameter which is the stack item. + * @brief clear current thread stack. */ - template - void dump(Callback callback) { - auto& p = *logstack_; - while (!p.empty()) { - callback(p.top()); - p.pop(); + void clear() { + auto& s = stack(); + while (!s.empty()) { + s.pop(); } } + /** - * @brief Only empty the stack. + * @brief return true if all thread's stack is empty. + * @return true if empty */ - void clear() { - dump([](const T& ip){}); + bool empty() const { + std::lock_guard g(this->mtx_); + for (auto p : this->stackBuffers_) { + std::stack& s = *p.second; + if (!s.empty()) { + return false; + } + } + return true; + } + + + /** + * @brief DumpCallback Type. It will be invoked many times by dump method. + * + * The first parameter is stack thread id. + * The second parameter is the last action of stack is push or not. + * The third parameter is the item in stack. + */ + typedef std::function DumpCallback; + + /** + * Dump all thread stack, and all stack will be cleared. + */ + void dump(const DumpCallback& callback, bool onlyCurrentThread = false) { + std::lock_guard g(this->mtx_); + for (auto p : this->stackBuffers_) { + std::thread::id tid = p.first; + if (onlyCurrentThread && tid != std::this_thread::get_id()) { + continue; + } + std::stack& s = *p.second; + bool* isPush = nullptr; + auto it = this->pushingBuffers_.find(tid); + if (it != this->pushingBuffers_.end()) { + isPush = it->second; + } + + while (!s.empty()) { + callback(tid, isPush, s.top()); + s.pop(); + } + } } + /** - * @brief Push item ip to the top of the stack. + * @brief Push item to current thread stack. */ - void push(const T& ip) { - auto& p = *logstack_; - p.push(ip); + void push(const T& item) { + pushing() = true; + auto& p = this->stack(); + p.push(item); } private: - ThreadLocalD > logstack_; + /** + * Get thread local attribute, and save them into a map (threadId => TYPE*) + * + * @tparam TYPE thread local attribute type. + * @param threadLocal Thread Local object. + * @param buffers a map from threadId to TYPE* + */ + template + inline TYPE& getThreadLocal( + ThreadLocal& threadLocal, + std::unordered_map& buffers) { + TYPE* retv = threadLocal.get(false); + if (retv) { + return *retv; + } else { + std::lock_guard guard(this->mtx_); + retv = threadLocal.get(); + auto id = std::this_thread::get_id(); + buffers.insert({id, retv}); + return *retv; + } + } + + /** + * @brief Get thread local stack reference. + */ + std::stack& stack() { + return this->getThreadLocal(this->logStack_, + this->stackBuffers_); + } + + /** + * @brief Get thread local pushing flag. + */ + bool& pushing() { + return this->getThreadLocal(this->isPushing_, + this->pushingBuffers_); + } + +private: + mutable std::mutex mtx_; + + std::unordered_map* > stackBuffers_; + std::unordered_map pushingBuffers_; + ThreadLocal isPushing_; + ThreadLocal > logStack_; }; extern CustomStackTrace gLayerStackTrace; +/** + * @brief Install a failure handler to print layer stack when error. + */ +extern void installLayerStackTracer(); + } // namespace paddle diff --git a/paddle/utils/Util.cpp b/paddle/utils/Util.cpp index 1c1d75dc5bed9..d8c3376fb18c4 100644 --- a/paddle/utils/Util.cpp +++ b/paddle/utils/Util.cpp @@ -129,13 +129,7 @@ void runInitFunctions() { void initMain(int argc, char** argv) { initializeLogging(argc, argv); - logging::installFailureWriter([](const char* data, int sz) { - std::cerr << "Current Layer forward/backward stack is " << std::endl; - gLayerStackTrace.dump([](const std::string& layername){ - std::cerr << "LayerName: " << layername << std::endl; - }); - std::cerr.write(data, sz); - }); + installLayerStackTracer(); std::string line; for (int i = 0; i < argc; ++i) { line += argv[i]; From 4a880f03c3e6881ca04c408b32921c33482f44b7 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 9 Sep 2016 05:26:02 +0000 Subject: [PATCH 041/324] Fix bug in NO_AVX when using lstm * libpaddle_cuda add wrong -mavx to some cpp files. --- paddle/.gitignore | 1 + paddle/cuda/CMakeLists.txt | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/paddle/.gitignore b/paddle/.gitignore index b89bd9d94633a..f921eef14156a 100644 --- a/paddle/.gitignore +++ b/paddle/.gitignore @@ -40,3 +40,4 @@ HPPL_ERROR_LOG unittest.list proto dist +setup.py diff --git a/paddle/cuda/CMakeLists.txt b/paddle/cuda/CMakeLists.txt index 2ccbf311bf8b9..e03a9a1baa004 100644 --- a/paddle/cuda/CMakeLists.txt +++ b/paddle/cuda/CMakeLists.txt @@ -1,8 +1,11 @@ +set(AVX_SOURCES + src/hl_math.cc + src/hl_avx_functions.cc +) set(CUDA_SOURCES src/hl_time.cc - src/hl_math.cc src/hl_cpu_functions.cc - src/hl_avx_functions.cc) + ${AVX_SOURCES}) set(CUDA_CXX_WITH_GPU_SOURCES src/hl_cuda_cublas.cc @@ -12,7 +15,7 @@ set(CUDA_CXX_WITH_GPU_SOURCES set_source_files_properties(${CUDA_CXX_WITH_GPU_SOURCES} PROPERTIES COMPILE_FLAGS "-D__NVCC__") -set_source_files_properties(${CUDA_SOURCES} +set_source_files_properties(${AVX_SOURCES} PROPERTIES COMPILE_FLAGS "-mavx") set(CUDA_DSO_SOURCES @@ -73,4 +76,3 @@ endif() add_style_check_target(paddle_cuda ${CUDA_SOURCES}) add_style_check_target(paddle_cuda ${CUDA_HEADERS}) -# add_style_check_target(hppl ${HPPL_CU_SOURCES}) # TODO(yuyang18): Format hppl style From 30bb3830becf2619c722946ac4e77d44fd8dbd76 Mon Sep 17 00:00:00 2001 From: comeonfox Date: Fri, 9 Sep 2016 14:03:51 +0800 Subject: [PATCH 042/324] Replace DataProviderWrapperConverter with DataProviderConverter in demos (#50) * Replace DataProviderWrapperConverter with DataProviderConverter in all demo prediction scripts. * Minor refinement of swig_py_paddle.rst to keep consistency with the English version. --- demo/image_classification/prediction.py | 9 +++---- demo/model_zoo/resnet/classify.py | 10 +++----- demo/semantic_role_labeling/predict.py | 22 ++++++++++------ demo/sentiment/predict.py | 10 ++++---- doc_cn/ui/predict/swig_py_paddle.rst | 34 +++++++++++++++---------- 5 files changed, 48 insertions(+), 37 deletions(-) diff --git a/demo/image_classification/prediction.py b/demo/image_classification/prediction.py index 6d585ee094ef8..5d9e932658673 100755 --- a/demo/image_classification/prediction.py +++ b/demo/image_classification/prediction.py @@ -20,9 +20,8 @@ import paddle.utils.image_util as image_util -from py_paddle import swig_paddle, util -from py_paddle import DataProviderWrapperConverter -from paddle.trainer.PyDataProviderWrapper import DenseSlot +from py_paddle import swig_paddle, DataProviderConverter +from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config logging.basicConfig(format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') @@ -75,8 +74,8 @@ def __init__(self, self.network.loadParameters(self.model_dir) data_size = 3 * self.crop_dims[0] * self.crop_dims[1] - slots = [DenseSlot(data_size)] - self.converter = util.DataProviderWrapperConverter(False, slots) + slots = [dense_vector(data_size)] + self.converter = DataProviderConverter(slots) def get_data(self, img_path): """ diff --git a/demo/model_zoo/resnet/classify.py b/demo/model_zoo/resnet/classify.py index fbc30d30e6224..06d471722f805 100755 --- a/demo/model_zoo/resnet/classify.py +++ b/demo/model_zoo/resnet/classify.py @@ -22,9 +22,8 @@ import paddle.utils.image_util as image_util -from py_paddle import swig_paddle, util -from py_paddle import DataProviderWrapperConverter -from paddle.trainer.PyDataProviderWrapper import DenseSlot +from py_paddle import swig_paddle, DataProviderConverter +from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config logging.basicConfig(format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') @@ -85,9 +84,8 @@ def __init__(self, train_conf, model_dir=None, self.network.loadParameters(self.model_dir) data_size = 3 * self.crop_dims[0] * self.crop_dims[1] - slots = [DenseSlot(data_size)] - is_sequence = False - self.converter = util.DataProviderWrapperConverter(is_sequence, slots) + slots = [dense_vector(data_size)] + self.converter = DataProviderConverter(slots) def get_data(self, img_path): """ diff --git a/demo/semantic_role_labeling/predict.py b/demo/semantic_role_labeling/predict.py index 5250ec6dc6855..9a27112828e44 100644 --- a/demo/semantic_role_labeling/predict.py +++ b/demo/semantic_role_labeling/predict.py @@ -15,12 +15,12 @@ import os import numpy as np from optparse import OptionParser -from py_paddle import swig_paddle, util, DataProviderWrapperConverter -from paddle.trainer.PyDataProviderWrapper import IndexSlot +from py_paddle import swig_paddle, DataProviderConverter +from paddle.trainer.PyDataProvider2 import integer_value_sequence from paddle.trainer.config_parser import parse_config """ Usage: run following command to show help message. - python predict.py -h + python predict.py -h """ UNK_IDX = 0 @@ -43,16 +43,22 @@ def __init__(self, train_conf, dict_file, model_dir, label_file): conf = parse_config( train_conf, - 'dict_len=' + str(len_dict) + + 'dict_len=' + str(len_dict) + ',label_len=' + str(len_label) + ',is_predict=True') self.network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) self.network.loadParameters(model_dir) - slots = [IndexSlot(len_dict), IndexSlot(len_dict), IndexSlot(len_dict), - IndexSlot(len_dict), IndexSlot(len_dict), IndexSlot(2)] - self.converter = util.DataProviderWrapperConverter(True, slots) + slots = [ + integer_value_sequence(len_dict), + integer_value_sequence(len_dict), + integer_value_sequence(len_dict), + integer_value_sequence(len_dict), + integer_value_sequence(len_dict), + integer_value_sequence(2) + ] + self.converter = DataProviderConverter(slots) def load_dict_label(self, dict_file, label_file): """ @@ -109,7 +115,7 @@ def predict(self, data_file): def option_parser(): - usage = ("python predict.py -c config -w model_dir " + usage = ("python predict.py -c config -w model_dir " "-d word dictionary -l label_file -i input_file") parser = OptionParser(usage="usage: %s [options]" % usage) parser.add_option( diff --git a/demo/sentiment/predict.py b/demo/sentiment/predict.py index 4ece6bb06d9e3..c61628d34db4a 100755 --- a/demo/sentiment/predict.py +++ b/demo/sentiment/predict.py @@ -15,13 +15,13 @@ import os import numpy as np from optparse import OptionParser -from py_paddle import swig_paddle, util, DataProviderWrapperConverter -from paddle.trainer.PyDataProviderWrapper import IndexSlot +from py_paddle import swig_paddle, DataProviderConverter +from paddle.trainer.PyDataProvider2 import integer_value_sequence from paddle.trainer.config_parser import parse_config """ Usage: run following command to show help message. - python predict.py -h + python predict.py -h """ class SentimentPrediction(): @@ -46,8 +46,8 @@ def __init__(self, train_conf, dict_file, model_dir=None, label_file = None): conf = parse_config(train_conf, "is_predict=1") self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) self.network.loadParameters(self.model_dir) - slots = [IndexSlot(self.dict_dim)] - self.converter = util.DataProviderWrapperConverter(True, slots) + slots = [integer_value_sequence(self.dict_dim)] + self.converter = DataProviderConverter(slots) def load_dict(self): """ diff --git a/doc_cn/ui/predict/swig_py_paddle.rst b/doc_cn/ui/predict/swig_py_paddle.rst index 284c60686d311..012ac4ff6e66a 100644 --- a/doc_cn/ui/predict/swig_py_paddle.rst +++ b/doc_cn/ui/predict/swig_py_paddle.rst @@ -9,22 +9,30 @@ PaddlePaddle目前使用Swig对其常用的预测接口进行了封装，使在P * 准备数据 * 预测 -典型的预测代码如下，使用mnist手写识别作为样例。 +典型的预测代码如下，使用mnist手写识别作为样例, 完整代码见 +:code:`src_root/doc/ui/predict/predict_sample.py` 。 .. literalinclude:: ../../../doc/ui/predict/predict_sample.py :language: python - :linenos: - -主要的软件包为py_paddle.swig_paddle，这个软件包文档相对完善。可以使用python的 :code:`help()` 函数查询文档。主要步骤为: - -* 在程序开始阶段，使用命令行参数初始化PaddlePaddle -* 在98行载入PaddlePaddle的训练文件。读取config -* 在100行创建神经网络，并在83行载入参数。 -* 103行创建一个从工具类，用来转换数据。 + :lines: 15-18,90-100,101-104 + +主要的软件包为py_paddle.swig_paddle，这个软件包文档相对完善。可以使用python的 +:code:`help()` 函数查询文档。主要步骤为: + +* 在程序开始阶段，使用 :code:`swig_paddle.initPaddle()` 传入命令行参数初始化 + PaddlePaddle。详细的命令行参数请参考 + `命令行参数 <../cmd_argument/detail_introduction.html>`_ 。 +* 接下来使用 :code:`parse_config()` 解析训练时的配置文件。这里要注意预测数据通常 + 不包含label, 而且预测网络通常直接输出最后一层的结果而不是像训练时一样以cost + layer作为输出，所以用于预测的配置文件要做相应的修改。 +* 使用 :code:`swig_paddle.GradientMachine.createFromConfigproto()` 根据上一步解 + 析好的配置创建神经网络。 +* 创建一个 :code:`DataProviderConverter` 对象converter。 - swig_paddle接受的原始数据是C++的Matrix，也就是直接写内存的float数组。 - - 这个接口并不用户友好。所以，我们提供了一个工具类DataProviderWrapperConverter. - - 这个工具类接收和PyDataProviderWrapper一样的输入数据，请参考PyDataProviderWrapper的文档。 -* 在第105行执行预测。forwardTest是一个工具类，直接提取出神经网络Output层的输出结果。典型的输出结果为\: + 这个接口并不用户友好。所以，我们提供了一个工具类DataProviderConverter。 + 这个工具类接收和PyDataProvider2一样的输入数据，详情请参考 + `PyDataProvider2文档 <../../../doc/ui/data_provider/pydataprovider2.html>`_ 。 +* 最后使用 :code:`forwardTest()` 直接提取出神经网络Output层的输出结果。典型的输出结果为\: .. code-block:: text @@ -37,4 +45,4 @@ PaddlePaddle目前使用Swig对其常用的预测接口进行了封装，使在P 2.70634608e-08, 3.48565123e-08, 5.25639710e-09, 4.48684503e-08]], dtype=float32)}] -其中，value即为softmax层的输出。由于数据是两个，所以输出的value。 +其中，value即为softmax层的输出。由于数据是两条，所以输出的value包含两个向量 。 From bb9470b3cdc6906000c75de11ff388ac023805c8 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 9 Sep 2016 14:44:23 +0800 Subject: [PATCH 043/324] Add unittest to CustomStackTrace. --- paddle/utils/tests/CMakeLists.txt | 10 ++ paddle/utils/tests/test_CustomStackTrace.cpp | 95 +++++++++++++++++++ .../tests/test_CustomStackTracePrint.cpp | 29 ++++++ .../utils/tests/test_CustomStackTracePrint.sh | 15 +++ 4 files changed, 149 insertions(+) create mode 100644 paddle/utils/tests/test_CustomStackTrace.cpp create mode 100644 paddle/utils/tests/test_CustomStackTracePrint.cpp create mode 100755 paddle/utils/tests/test_CustomStackTracePrint.sh diff --git a/paddle/utils/tests/CMakeLists.txt b/paddle/utils/tests/CMakeLists.txt index 147ee3f6d6d86..be59a785ecf36 100644 --- a/paddle/utils/tests/CMakeLists.txt +++ b/paddle/utils/tests/CMakeLists.txt @@ -2,3 +2,13 @@ add_simple_unittest(test_CommandLineParser) add_simple_unittest(test_Logging) add_simple_unittest(test_Thread) add_simple_unittest(test_StringUtils) +add_simple_unittest(test_CustomStackTrace) + +add_executable( + test_CustomStackTracePrint + test_CustomStackTracePrint.cpp +) +link_paddle_exe(test_CustomStackTracePrint) +add_test(NAME test_CustomStackTracePrint + COMMAND ${PROJ_ROOT}/paddle/utils/tests/test_CustomStackTracePrint.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/paddle/utils/tests/test_CustomStackTrace.cpp b/paddle/utils/tests/test_CustomStackTrace.cpp new file mode 100644 index 0000000000000..26ca4c678a650 --- /dev/null +++ b/paddle/utils/tests/test_CustomStackTrace.cpp @@ -0,0 +1,95 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include + +#include "paddle/utils/CustomStackTrace.h" +#include "paddle/utils/CommandLineParser.h" +#include "paddle/utils/Util.h" +#include "paddle/utils/Locks.h" + +P_DEFINE_int32(test_thread_num, 10, "testing thread number"); + +void testNormalImpl(const std::function&, + size_t, size_t, + paddle::ThreadBarrier&, + paddle::ThreadBarrier&)>& callback) { + paddle::CustomStackTrace tracer; + paddle::ThreadBarrier doneBarrier(FLAGS_test_thread_num + 1); + paddle::ThreadBarrier startBarrier(FLAGS_test_thread_num + 1); + constexpr size_t countDown = 10; + constexpr size_t layerSize = 1000; + std::vector> threads; + threads.reserve(FLAGS_test_thread_num); + + for (int32_t i=0; i < FLAGS_test_thread_num; ++i) { + threads.emplace_back(new std::thread([&tracer, &countDown, &layerSize, + &startBarrier, &doneBarrier, + &callback]{ + callback(tracer, countDown, layerSize, startBarrier, doneBarrier); + })); + } + size_t cntDown = countDown; + while (cntDown-- > 0) { + startBarrier.wait(); + doneBarrier.wait(); + ASSERT_TRUE(tracer.empty()); + } + + for (auto& thread : threads) { + thread->join(); + } +} + + +TEST(CustomStackTrace, normalTrain) { + testNormalImpl([](paddle::CustomStackTrace& tracer, + size_t countDown, size_t layerSize, + paddle::ThreadBarrier& start, paddle::ThreadBarrier& finish){ + while (countDown-- > 0) { + start.wait(); + for (size_t i=0; i < layerSize; ++i) { + tracer.push("layer_" + std::to_string(i)); + } + tracer.pop(""); + for (size_t i=0; i < layerSize; ++i) { + tracer.pop("layer_" + std::to_string(layerSize - 1 - i)); + } + finish.wait(); + } + }); +} + +TEST(CustomStackTrace, normalTest) { + testNormalImpl([] (paddle::CustomStackTrace& tracer, + size_t countDown, size_t layerSize, + paddle::ThreadBarrier& start, paddle::ThreadBarrier& finish){ + while (countDown-- > 0) { + start.wait(); + for (size_t i=0; i < layerSize; ++i) { + tracer.push("layer_" + std::to_string(i)); + } + tracer.clear(); // in forward test, tracer will clear after forward. + finish.wait(); + } + }); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + paddle::initMain(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/paddle/utils/tests/test_CustomStackTracePrint.cpp b/paddle/utils/tests/test_CustomStackTracePrint.cpp new file mode 100644 index 0000000000000..c19c98614e6a7 --- /dev/null +++ b/paddle/utils/tests/test_CustomStackTracePrint.cpp @@ -0,0 +1,29 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/utils/Util.h" +#include "paddle/utils/CustomStackTrace.h" + +int main(int argc, char** argv) { + paddle::initMain(argc, argv); + + for (size_t i=0; i < 1000; ++i) { + paddle::gLayerStackTrace.push("layer_" + std::to_string(i)); + if (i == 998) { + throw "Unhandle exception"; + } + } + + return 0; +} diff --git a/paddle/utils/tests/test_CustomStackTracePrint.sh b/paddle/utils/tests/test_CustomStackTracePrint.sh new file mode 100755 index 0000000000000..b5543485f365a --- /dev/null +++ b/paddle/utils/tests/test_CustomStackTracePrint.sh @@ -0,0 +1,15 @@ +#!/bin/bash +echo "Test Custom Stack Trace print correct result when fail" +./test_CustomStackTracePrint >customStackTraceLog 2>&1 +if [ $? -eq 0 ]; then + exit 1 +else + set -e + TEXT="" + for ((i=0; i<=998; i++)) + do + TEXT="layer_$i, "$TEXT + done + TEXT="Forwarding "$TEXT + grep -q "$TEXT" customStackTraceLog +fi From b384af58bec85702b33dc0080102e24d464503cf Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 7 Sep 2016 22:06:27 +0800 Subject: [PATCH 044/324] Fix bugs in CustomStackTrace. * Make layer stack trace shows ThreadId, Forward or Backward. Change-Id: Iba1477adb8c9115c3a67ff2959bb5c878ca706c7 --- paddle/.gitignore | 1 + .../gradientmachines/NeuralNetwork.cpp | 1 + paddle/utils/CustomStackTrace.cpp | 35 ++++ paddle/utils/CustomStackTrace.h | 164 ++++++++++++++---- paddle/utils/Util.cpp | 8 +- 5 files changed, 166 insertions(+), 43 deletions(-) diff --git a/paddle/.gitignore b/paddle/.gitignore index b89bd9d94633a..f921eef14156a 100644 --- a/paddle/.gitignore +++ b/paddle/.gitignore @@ -40,3 +40,4 @@ HPPL_ERROR_LOG unittest.list proto dist +setup.py diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.cpp b/paddle/gserver/gradientmachines/NeuralNetwork.cpp index fca52828957a2..eb1522a178d48 100644 --- a/paddle/gserver/gradientmachines/NeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/NeuralNetwork.cpp @@ -277,6 +277,7 @@ void NeuralNetwork::getState(MachineState& machineState) { } void NeuralNetwork::backward(const UpdateCallback& callback) { + gLayerStackTrace.pop(""); // tell layer trace is during backward. FOR_EACH_R(layer, layers_) { REGISTER_TIMER_INFO("BackwardTimer", (*layer)->getName().c_str()); if ((*layer)->needGradient()) { diff --git a/paddle/utils/CustomStackTrace.cpp b/paddle/utils/CustomStackTrace.cpp index 50d7f5402f586..232a478ecd93a 100644 --- a/paddle/utils/CustomStackTrace.cpp +++ b/paddle/utils/CustomStackTrace.cpp @@ -14,9 +14,44 @@ limitations under the License. */ #include "CustomStackTrace.h" +#include "CommandLineParser.h" +#include + +P_DEFINE_bool(layer_stack_error_only_current_thread, + true, + "Dump current thread or whole process layer stack when signal error " + "occurred. true means only dump current thread layer stack"); namespace paddle { CustomStackTrace gLayerStackTrace; +static std::mutex gLayerStackTraceMtx; +void installLayerStackTracer() { + logging::installFailureWriter([](const char* data, int sz) { + std::lock_guard guard(gLayerStackTraceMtx); + if (!gLayerStackTrace.empty()) { + size_t curTid = -1UL; + std::hash hasher; + gLayerStackTrace.dump([&curTid, &hasher](std::thread::id tid, + bool* isForwarding, + const std::string& layerName) { + if (curTid != hasher(tid)) { + if (curTid != -1UL) { + std::cerr << std::endl; + } + curTid = hasher(tid); + std::cerr << "Thread [" << tid << "] "; + if (isForwarding) { + std::cerr << (*isForwarding ? "Forwarding ": "Backwarding "); + } + } + std::cerr << layerName << ", "; + }, FLAGS_layer_stack_error_only_current_thread); + std::cerr << std::endl; + } + std::cerr.write(data, sz); + }); +} + } // namespace paddle diff --git a/paddle/utils/CustomStackTrace.h b/paddle/utils/CustomStackTrace.h index e1b2d2d8e5ee6..774c4db2b9be4 100644 --- a/paddle/utils/CustomStackTrace.h +++ b/paddle/utils/CustomStackTrace.h @@ -15,6 +15,9 @@ limitations under the License. */ #pragma once #include +#include +#include +#include #include "ThreadLocal.h" @@ -29,25 +32,18 @@ namespace paddle { * @code{.cpp} * * paddle::CustomStackTrace stack; - * PASS_TEST=0; * for (auto& layer : layers){ * stack.push(layer->getName()); - * layer->forward(passType); + * layer->forward(); * } - * for (auto& layer : layers){ + * + * stack.pop(""); // mark under pop stage. + * + * for (auto it = layers.rbegin(); it != layers.rend(); ++it){ + * auto& layer = *it; * layer->backward(passType); * stack.pop(layer->getName()); * } - * - * if(passType == PASS_TEST) { - * stack.clear(); - * } - * else { - * stack.dump([](const std::string& layername){ - * LOG(INFO) << "LayerName: " << layername; - * }) - * } - * * * @endcode */ @@ -55,45 +51,141 @@ template class CustomStackTrace{ public: /** - * @brief Pop out an item from the top of the stack. For safety the item - * will be poped should equal to ip. + * @brief Pop out an item from the top of the stack if item == top. + * Else, just set status to popping. */ - void pop(const T& ip) { - auto& p = *logstack_; - CHECK_EQ(ip, p.top()); - p.pop(); + void pop(const T& item) { + pushing() = false; + auto& s = this->stack(); + if (item == s.top()) { + s.pop(); + } } + /** - * @brief Empty the stack by sequence from top to button. - * @param[in] callback A function deal with each item while dumping. - * It must have and only have a in parameter which is the stack item. + * @brief clear current thread stack. */ - template - void dump(Callback callback) { - auto& p = *logstack_; - while (!p.empty()) { - callback(p.top()); - p.pop(); + void clear() { + auto& s = stack(); + while (!s.empty()) { + s.pop(); } } + /** - * @brief Only empty the stack. + * @brief return true if all thread's stack is empty. + * @return true if empty */ - void clear() { - dump([](const T& ip){}); + bool empty() const { + std::lock_guard g(this->mtx_); + for (auto p : this->stackBuffers_) { + std::stack& s = *p.second; + if (!s.empty()) { + return false; + } + } + return true; + } + + + /** + * @brief DumpCallback Type. It will be invoked many times by dump method. + * + * The first parameter is stack thread id. + * The second parameter is the last action of stack is push or not. + * The third parameter is the item in stack. + */ + typedef std::function DumpCallback; + + /** + * Dump all thread stack, and all stack will be cleared. + */ + void dump(const DumpCallback& callback, bool onlyCurrentThread = false) { + std::lock_guard g(this->mtx_); + for (auto p : this->stackBuffers_) { + std::thread::id tid = p.first; + if (onlyCurrentThread && tid != std::this_thread::get_id()) { + continue; + } + std::stack& s = *p.second; + bool* isPush = nullptr; + auto it = this->pushingBuffers_.find(tid); + if (it != this->pushingBuffers_.end()) { + isPush = it->second; + } + + while (!s.empty()) { + callback(tid, isPush, s.top()); + s.pop(); + } + } } + /** - * @brief Push item ip to the top of the stack. + * @brief Push item to current thread stack. */ - void push(const T& ip) { - auto& p = *logstack_; - p.push(ip); + void push(const T& item) { + pushing() = true; + auto& p = this->stack(); + p.push(item); } private: - ThreadLocalD > logstack_; + /** + * Get thread local attribute, and save them into a map (threadId => TYPE*) + * + * @tparam TYPE thread local attribute type. + * @param threadLocal Thread Local object. + * @param buffers a map from threadId to TYPE* + */ + template + inline TYPE& getThreadLocal( + ThreadLocal& threadLocal, + std::unordered_map& buffers) { + TYPE* retv = threadLocal.get(false); + if (retv) { + return *retv; + } else { + std::lock_guard guard(this->mtx_); + retv = threadLocal.get(); + auto id = std::this_thread::get_id(); + buffers.insert({id, retv}); + return *retv; + } + } + + /** + * @brief Get thread local stack reference. + */ + std::stack& stack() { + return this->getThreadLocal(this->logStack_, + this->stackBuffers_); + } + + /** + * @brief Get thread local pushing flag. + */ + bool& pushing() { + return this->getThreadLocal(this->isPushing_, + this->pushingBuffers_); + } + +private: + mutable std::mutex mtx_; + + std::unordered_map* > stackBuffers_; + std::unordered_map pushingBuffers_; + ThreadLocal isPushing_; + ThreadLocal > logStack_; }; extern CustomStackTrace gLayerStackTrace; +/** + * @brief Install a failure handler to print layer stack when error. + */ +extern void installLayerStackTracer(); + } // namespace paddle diff --git a/paddle/utils/Util.cpp b/paddle/utils/Util.cpp index 1c1d75dc5bed9..d8c3376fb18c4 100644 --- a/paddle/utils/Util.cpp +++ b/paddle/utils/Util.cpp @@ -129,13 +129,7 @@ void runInitFunctions() { void initMain(int argc, char** argv) { initializeLogging(argc, argv); - logging::installFailureWriter([](const char* data, int sz) { - std::cerr << "Current Layer forward/backward stack is " << std::endl; - gLayerStackTrace.dump([](const std::string& layername){ - std::cerr << "LayerName: " << layername << std::endl; - }); - std::cerr.write(data, sz); - }); + installLayerStackTracer(); std::string line; for (int i = 0; i < argc; ++i) { line += argv[i]; From 49a92da24cc44d6a8cc22f148dc7e6746d034edb Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 9 Sep 2016 14:44:23 +0800 Subject: [PATCH 045/324] Add unittest to CustomStackTrace. --- paddle/utils/tests/CMakeLists.txt | 10 ++ paddle/utils/tests/test_CustomStackTrace.cpp | 95 +++++++++++++++++++ .../tests/test_CustomStackTracePrint.cpp | 29 ++++++ .../utils/tests/test_CustomStackTracePrint.sh | 15 +++ 4 files changed, 149 insertions(+) create mode 100644 paddle/utils/tests/test_CustomStackTrace.cpp create mode 100644 paddle/utils/tests/test_CustomStackTracePrint.cpp create mode 100755 paddle/utils/tests/test_CustomStackTracePrint.sh diff --git a/paddle/utils/tests/CMakeLists.txt b/paddle/utils/tests/CMakeLists.txt index 147ee3f6d6d86..be59a785ecf36 100644 --- a/paddle/utils/tests/CMakeLists.txt +++ b/paddle/utils/tests/CMakeLists.txt @@ -2,3 +2,13 @@ add_simple_unittest(test_CommandLineParser) add_simple_unittest(test_Logging) add_simple_unittest(test_Thread) add_simple_unittest(test_StringUtils) +add_simple_unittest(test_CustomStackTrace) + +add_executable( + test_CustomStackTracePrint + test_CustomStackTracePrint.cpp +) +link_paddle_exe(test_CustomStackTracePrint) +add_test(NAME test_CustomStackTracePrint + COMMAND ${PROJ_ROOT}/paddle/utils/tests/test_CustomStackTracePrint.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/paddle/utils/tests/test_CustomStackTrace.cpp b/paddle/utils/tests/test_CustomStackTrace.cpp new file mode 100644 index 0000000000000..26ca4c678a650 --- /dev/null +++ b/paddle/utils/tests/test_CustomStackTrace.cpp @@ -0,0 +1,95 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include + +#include "paddle/utils/CustomStackTrace.h" +#include "paddle/utils/CommandLineParser.h" +#include "paddle/utils/Util.h" +#include "paddle/utils/Locks.h" + +P_DEFINE_int32(test_thread_num, 10, "testing thread number"); + +void testNormalImpl(const std::function&, + size_t, size_t, + paddle::ThreadBarrier&, + paddle::ThreadBarrier&)>& callback) { + paddle::CustomStackTrace tracer; + paddle::ThreadBarrier doneBarrier(FLAGS_test_thread_num + 1); + paddle::ThreadBarrier startBarrier(FLAGS_test_thread_num + 1); + constexpr size_t countDown = 10; + constexpr size_t layerSize = 1000; + std::vector> threads; + threads.reserve(FLAGS_test_thread_num); + + for (int32_t i=0; i < FLAGS_test_thread_num; ++i) { + threads.emplace_back(new std::thread([&tracer, &countDown, &layerSize, + &startBarrier, &doneBarrier, + &callback]{ + callback(tracer, countDown, layerSize, startBarrier, doneBarrier); + })); + } + size_t cntDown = countDown; + while (cntDown-- > 0) { + startBarrier.wait(); + doneBarrier.wait(); + ASSERT_TRUE(tracer.empty()); + } + + for (auto& thread : threads) { + thread->join(); + } +} + + +TEST(CustomStackTrace, normalTrain) { + testNormalImpl([](paddle::CustomStackTrace& tracer, + size_t countDown, size_t layerSize, + paddle::ThreadBarrier& start, paddle::ThreadBarrier& finish){ + while (countDown-- > 0) { + start.wait(); + for (size_t i=0; i < layerSize; ++i) { + tracer.push("layer_" + std::to_string(i)); + } + tracer.pop(""); + for (size_t i=0; i < layerSize; ++i) { + tracer.pop("layer_" + std::to_string(layerSize - 1 - i)); + } + finish.wait(); + } + }); +} + +TEST(CustomStackTrace, normalTest) { + testNormalImpl([] (paddle::CustomStackTrace& tracer, + size_t countDown, size_t layerSize, + paddle::ThreadBarrier& start, paddle::ThreadBarrier& finish){ + while (countDown-- > 0) { + start.wait(); + for (size_t i=0; i < layerSize; ++i) { + tracer.push("layer_" + std::to_string(i)); + } + tracer.clear(); // in forward test, tracer will clear after forward. + finish.wait(); + } + }); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + paddle::initMain(argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/paddle/utils/tests/test_CustomStackTracePrint.cpp b/paddle/utils/tests/test_CustomStackTracePrint.cpp new file mode 100644 index 0000000000000..c19c98614e6a7 --- /dev/null +++ b/paddle/utils/tests/test_CustomStackTracePrint.cpp @@ -0,0 +1,29 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/utils/Util.h" +#include "paddle/utils/CustomStackTrace.h" + +int main(int argc, char** argv) { + paddle::initMain(argc, argv); + + for (size_t i=0; i < 1000; ++i) { + paddle::gLayerStackTrace.push("layer_" + std::to_string(i)); + if (i == 998) { + throw "Unhandle exception"; + } + } + + return 0; +} diff --git a/paddle/utils/tests/test_CustomStackTracePrint.sh b/paddle/utils/tests/test_CustomStackTracePrint.sh new file mode 100755 index 0000000000000..b5543485f365a --- /dev/null +++ b/paddle/utils/tests/test_CustomStackTracePrint.sh @@ -0,0 +1,15 @@ +#!/bin/bash +echo "Test Custom Stack Trace print correct result when fail" +./test_CustomStackTracePrint >customStackTraceLog 2>&1 +if [ $? -eq 0 ]; then + exit 1 +else + set -e + TEXT="" + for ((i=0; i<=998; i++)) + do + TEXT="layer_$i, "$TEXT + done + TEXT="Forwarding "$TEXT + grep -q "$TEXT" customStackTraceLog +fi From b159a36e3f8598e9b161554f58ea5243cf8610ba Mon Sep 17 00:00:00 2001 From: gangliao Date: Fri, 9 Sep 2016 15:48:26 +0800 Subject: [PATCH 046/324] fix redundant macro in hl_device_functions.cuh --- paddle/cuda/include/hl_device_functions.cuh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/paddle/cuda/include/hl_device_functions.cuh b/paddle/cuda/include/hl_device_functions.cuh index 2fbc2cfb50221..27e3f450c5c1c 100755 --- a/paddle/cuda/include/hl_device_functions.cuh +++ b/paddle/cuda/include/hl_device_functions.cuh @@ -16,8 +16,6 @@ limitations under the License. */ #ifndef HL_DEVICE_FUNCTIONS_CUH_ #define HL_DEVICE_FUNCTIONS_CUH_ -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 600 - namespace hppl { static __inline__ __device__ double atomicAdd(double* address, double val) { @@ -42,6 +40,4 @@ static __inline__ __device__ double atomicAdd(double* address, double val) { using hppl::atomicAdd; #endif -#endif - #endif /* HL_DEVICE_FUNCTIONS_CUH_ */ From 33767364d1e01c9067dd553abd7eb94bae5b7d1e Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 9 Sep 2016 16:20:34 +0800 Subject: [PATCH 047/324] reorganize the chinese doc catalog (#53) Change-Id: I2a72eb0a296a8cf297e2a53029d2602d85f40296 --- doc_cn/algorithm/rnn/rnn.rst | 7 ------- doc_cn/cluster/index.rst | 4 ---- doc_cn/demo/embedding_model/index.md | 1 - doc_cn/demo/image_classification/index.rst | 4 ---- doc_cn/demo/imagenet_model/index.md | 2 -- doc_cn/demo/semantic_role_labeling/index.md | 2 -- doc_cn/demo/sentiment_analysis/index.md | 2 -- doc_cn/demo/text_generation/index.rst | 3 --- doc_cn/dev/new_layer/index.rst | 4 ---- doc_cn/index.md | 19 ------------------- doc_cn/index.rst | 19 +++++++++++++++++++ doc_cn/ui/model.rst | 4 ---- 12 files changed, 19 insertions(+), 52 deletions(-) delete mode 100644 doc_cn/algorithm/rnn/rnn.rst delete mode 100644 doc_cn/cluster/index.rst delete mode 100644 doc_cn/demo/embedding_model/index.md delete mode 100644 doc_cn/demo/image_classification/index.rst delete mode 100644 doc_cn/demo/imagenet_model/index.md delete mode 100644 doc_cn/demo/semantic_role_labeling/index.md delete mode 100644 doc_cn/demo/sentiment_analysis/index.md delete mode 100644 doc_cn/demo/text_generation/index.rst delete mode 100644 doc_cn/dev/new_layer/index.rst delete mode 100644 doc_cn/index.md create mode 100644 doc_cn/index.rst delete mode 100644 doc_cn/ui/model.rst diff --git a/doc_cn/algorithm/rnn/rnn.rst b/doc_cn/algorithm/rnn/rnn.rst deleted file mode 100644 index f073ac4e20eeb..0000000000000 --- a/doc_cn/algorithm/rnn/rnn.rst +++ /dev/null @@ -1,7 +0,0 @@ -RNN 配置 -======== - -.. toctree:: - :maxdepth: 3 - -* `RNN配置 <../../../doc/algorithm/rnn/rnn.html>`_ diff --git a/doc_cn/cluster/index.rst b/doc_cn/cluster/index.rst deleted file mode 100644 index 16c1f0e37bce0..0000000000000 --- a/doc_cn/cluster/index.rst +++ /dev/null @@ -1,4 +0,0 @@ -集群训练 -======== - -参见 `集群训练 <../../doc/cluster/index.html>`_ diff --git a/doc_cn/demo/embedding_model/index.md b/doc_cn/demo/embedding_model/index.md deleted file mode 100644 index 5894a4de5a14a..0000000000000 --- a/doc_cn/demo/embedding_model/index.md +++ /dev/null @@ -1 +0,0 @@ -# Embedding Demo diff --git a/doc_cn/demo/image_classification/index.rst b/doc_cn/demo/image_classification/index.rst deleted file mode 100644 index 98cbdc29b9d4a..0000000000000 --- a/doc_cn/demo/image_classification/index.rst +++ /dev/null @@ -1,4 +0,0 @@ -图片分类教程 -============ - -TBD diff --git a/doc_cn/demo/imagenet_model/index.md b/doc_cn/demo/imagenet_model/index.md deleted file mode 100644 index b54b28401ce16..0000000000000 --- a/doc_cn/demo/imagenet_model/index.md +++ /dev/null @@ -1,2 +0,0 @@ -# Resnet - TBD diff --git a/doc_cn/demo/semantic_role_labeling/index.md b/doc_cn/demo/semantic_role_labeling/index.md deleted file mode 100644 index a1594577bb511..0000000000000 --- a/doc_cn/demo/semantic_role_labeling/index.md +++ /dev/null @@ -1,2 +0,0 @@ -# 语义标注 -TBD diff --git a/doc_cn/demo/sentiment_analysis/index.md b/doc_cn/demo/sentiment_analysis/index.md deleted file mode 100644 index d95f2803a43ed..0000000000000 --- a/doc_cn/demo/sentiment_analysis/index.md +++ /dev/null @@ -1,2 +0,0 @@ -# 情感分析 -TBD diff --git a/doc_cn/demo/text_generation/index.rst b/doc_cn/demo/text_generation/index.rst deleted file mode 100644 index 147b77646536c..0000000000000 --- a/doc_cn/demo/text_generation/index.rst +++ /dev/null @@ -1,3 +0,0 @@ -文本生成 -======== -TBD diff --git a/doc_cn/dev/new_layer/index.rst b/doc_cn/dev/new_layer/index.rst deleted file mode 100644 index aafeceff5b969..0000000000000 --- a/doc_cn/dev/new_layer/index.rst +++ /dev/null @@ -1,4 +0,0 @@ -新写Layer -========= - -* `新写Layer <../../../doc/dev/new_layer/index.html>`_ diff --git a/doc_cn/index.md b/doc_cn/index.md deleted file mode 100644 index f21f60e146c1b..0000000000000 --- a/doc_cn/index.md +++ /dev/null @@ -1,19 +0,0 @@ -PaddlePaddle文档 -================ - -使用指南 --------- -* [快速入门](demo/quick_start/index.md) -* [编译与安装](build_and_install/index.rst) -* [用户接口](ui/index.rst) -* [使用示例](demo/index.rst) -* [模型配置](ui/model.rst) -* [集群训练](cluster/index.rst) - -开发指南 --------- -* [新写Layer](dev/new_layer/index.rst) - -算法教程 --------- -* [RNN配置](algorithm/rnn/rnn.rst) diff --git a/doc_cn/index.rst b/doc_cn/index.rst new file mode 100644 index 0000000000000..5f06463899f6b --- /dev/null +++ b/doc_cn/index.rst @@ -0,0 +1,19 @@ +PaddlePaddle文档 +================ + +使用指南 +-------- +* `快速入门 `_ +* `编译与安装 `_ +* `用户接口 `_ +* `使用示例 `_ +* `模型配置 <../doc/ui/api/trainer_config_helpers/index.html>`_ +* `集群训练 <../doc/cluster/index.html>`_ + +开发指南 +-------- +* `新写Layer <../doc/dev/new_layer/index.html>`_ + +算法教程 +-------- +* `RNN配置 <../doc/algorithm/rnn/rnn.html>`_ diff --git a/doc_cn/ui/model.rst b/doc_cn/ui/model.rst deleted file mode 100644 index 7a81236d6f547..0000000000000 --- a/doc_cn/ui/model.rst +++ /dev/null @@ -1,4 +0,0 @@ -模型配置 -======== - -* `Model Config Interface <../../doc/ui/api/trainer_config_helpers/index.html>`_ From b76682c5e3a82aedf64ddbecb9747f6c525794e5 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 9 Sep 2016 16:25:00 +0800 Subject: [PATCH 048/324] Exit when pip install failed --- paddle/scripts/submit_local.sh.in | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/scripts/submit_local.sh.in b/paddle/scripts/submit_local.sh.in index 6d2cab6b16bab..4cf5f41f195df 100644 --- a/paddle/scripts/submit_local.sh.in +++ b/paddle/scripts/submit_local.sh.in @@ -73,6 +73,7 @@ if [ $? -eq 1 ]; then # Older version installed, or not installed at all echo "pip install wheels failed. " echo "Please use 'sudo paddle' at the first time you use PaddlePaddle" echo "PaddlePaddle will install some python dependencies automatically." + exit 1 fi echo "Python dependencies are installed." fi From 0c8aeffb9acd1d727d4f1c4bb18d4c6ce3346891 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 9 Sep 2016 17:01:20 +0800 Subject: [PATCH 049/324] Fix a PyDataProvider2 bug when use calc_batch_size * Need PyGuard when set args for calc_batch_size --- paddle/gserver/dataproviders/PyDataProvider2.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/paddle/gserver/dataproviders/PyDataProvider2.cpp b/paddle/gserver/dataproviders/PyDataProvider2.cpp index f7886c4e014d7..8e51752dc29ee 100644 --- a/paddle/gserver/dataproviders/PyDataProvider2.cpp +++ b/paddle/gserver/dataproviders/PyDataProvider2.cpp @@ -340,6 +340,7 @@ class PyDataProvider2 : public DataProvider { size_t additionalBatchSize = 1; if (calcBatchSize_) { + PyGuard guard; py::CallableHelper calcBatchSize(this->calcBatchSize_); calcBatchSize.setArgsSize(1); calcBatchSize.getArgs().set(0, data); @@ -513,6 +514,7 @@ class PyDataProvider2 : public DataProvider { } { if (calcBatchSize_) { // custom calc batch size. + PyGuard guard; Py_INCREF(data.back().get()); py::CallableHelper calcBatchSize(calcBatchSize_); calcBatchSize.setArgsSize(1); @@ -575,6 +577,11 @@ class PyDataProvider2 : public DataProvider { scanners[i]->finishFill(inArgs[i]); } + { + PyGuard g; + cache_->drop(&data); + } + DBG << "Reading CPU Batch Done."; if (useGpu_) { @@ -592,10 +599,6 @@ class PyDataProvider2 : public DataProvider { *batch = cpuBatch; } - { - PyGuard g; - cache_->drop(&data); - } return bsize; } }; From 445d1e820122ece4e0a3e8343b886319ee79cf66 Mon Sep 17 00:00:00 2001 From: liaogang Date: Fri, 9 Sep 2016 19:46:48 +0800 Subject: [PATCH 050/324] Support MAC OS for PaddlePaddle --- cmake/cudnn.cmake | 2 +- cmake/flags.cmake | 4 +- cmake/util.cmake | 41 +++++-- paddle/api/Util.cpp | 1 + .../gserver/dataproviders/PyDataProvider.cpp | 2 + .../gradientmachines/NeuralNetwork.cpp | 12 +- paddle/math/Allocator.h | 13 ++- paddle/math/MathFunctions.h | 2 + paddle/math/Storage.cpp | 4 +- paddle/pserver/LightNetwork.cpp | 29 ++++- paddle/pserver/SocketChannel.cpp | 12 +- paddle/trainer/Trainer.cpp | 1 + paddle/trainer/TrainerMain.cpp | 1 + paddle/utils/Excepts.cpp | 54 +++++++++ paddle/utils/Excepts.h | 26 +++++ paddle/utils/Locks.cpp | 106 ++++++++++++++++++ paddle/utils/Locks.h | 69 ++++++++++++ paddle/utils/PythonUtil.h | 20 +++- 18 files changed, 361 insertions(+), 38 deletions(-) create mode 100644 paddle/utils/Excepts.cpp create mode 100644 paddle/utils/Excepts.h create mode 100644 paddle/utils/Locks.cpp diff --git a/cmake/cudnn.cmake b/cmake/cudnn.cmake index e2ff923a22923..e5b59be19369d 100644 --- a/cmake/cudnn.cmake +++ b/cmake/cudnn.cmake @@ -15,7 +15,7 @@ list(APPEND CUDNN_CHECK_LIBRARY_DIRS $ENV{CUDNN_ROOT}/lib64 $ENV{CUDNN_ROOT}/lib /usr/lib) -find_library(CUDNN_LIBRARY NAMES libcudnn.so # libcudnn_static.a +find_library(CUDNN_LIBRARY NAMES libcudnn.so libcudnn.dylib # libcudnn_static.a PATHS ${CUDNN_CHECK_LIBRARY_DIRS} ${CUDNN_INCLUDE_DIR} ${__libpath_hist} NO_DEFAULT_PATH DOC "Path to cuDNN library.") diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 4b99e7f7fb6af..c95d4063105e7 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -58,8 +58,8 @@ set(COMMON_FLAGS -fPIC -fno-omit-frame-pointer -Wall - -Wextra - -Werror +# -Wextra +# -Werror -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter diff --git a/cmake/util.cmake b/cmake/util.cmake index e0e372fed0b04..bad44c7e9dda6 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -1,4 +1,21 @@ # Some common routine for paddle compile. +if(APPLE) + # ------------------------------------------------------- + # OSX + # ------------------------------------------------------- + set(GROUP_START "-Wl,-force_load") + set(GROUP_END "") + + set(ARCHIVE_START "-Wl,-force_load") + set(ARCHIVE_END "") +else() + set(GROUP_START "-Wl,--start-group") + set(GROUP_END "-Wl,--end-group") + + set(ARCHIVE_START "-Wl,--whole-archive") + set(ARCHIVE_END "-Wl,--no-whole-archive") +endif() + # target_circle_link_libraries @@ -7,10 +24,18 @@ # First Argument: target name want to be linked with libraries # Rest Arguments: libraries which link together. function(target_circle_link_libraries TARGET_NAME) - target_link_libraries(${TARGET_NAME} - -Wl,--start-group - ${ARGN} - -Wl,--end-group) + if(APPLE) + foreach(f ${ARGN}) + list(APPEND OSX_LIBRARIES "-Wl,-force_load" "${f}") + endforeach(f) + target_link_libraries(${TARGET_NAME} + ${OSX_LIBRARIES}) + else() + target_link_libraries(${TARGET_NAME} + ${GROUP_START} + ${ARGN} + ${GROUP_END}) + endif() endfunction() # compile_cu_as_cpp @@ -41,20 +66,18 @@ function(link_paddle_exe TARGET_NAME) if(PADDLE_WITH_INTERNAL) set(INTERAL_LIBS paddle_internal_gserver paddle_internal_parameter) target_circle_link_libraries(${TARGET_NAME} - -Wl,--whole-archive paddle_internal_gserver paddle_internal_owlqn - -Wl,--no-whole-archive paddle_internal_parameter) else() set(INTERAL_LIBS "") endif() target_circle_link_libraries(${TARGET_NAME} - -Wl,--whole-archive +# ${ARCHIVE_START} paddle_gserver ${METRIC_LIBS} - -Wl,--no-whole-archive +# ${ARCHIVE_END} paddle_pserver paddle_trainer_lib paddle_network @@ -69,7 +92,7 @@ function(link_paddle_exe TARGET_NAME) ${CBLAS_LIBS} ${CMAKE_DL_LIBS} ${INTERAL_LIBS} - -lz) + ) if(WITH_PYTHON) target_link_libraries(${TARGET_NAME} diff --git a/paddle/api/Util.cpp b/paddle/api/Util.cpp index 4e655c324a1ed..8a6741078f2f1 100644 --- a/paddle/api/Util.cpp +++ b/paddle/api/Util.cpp @@ -17,6 +17,7 @@ limitations under the License. */ #include "paddle/utils/Util.h" #include "paddle/utils/PythonUtil.h" #include "paddle/utils/Flags.h" +#include "paddle/utils/Excepts.h" #include "paddle/parameter/Parameter.h" #include diff --git a/paddle/gserver/dataproviders/PyDataProvider.cpp b/paddle/gserver/dataproviders/PyDataProvider.cpp index aeefd16063df8..cc3e09a3c2ecb 100644 --- a/paddle/gserver/dataproviders/PyDataProvider.cpp +++ b/paddle/gserver/dataproviders/PyDataProvider.cpp @@ -17,6 +17,8 @@ limitations under the License. */ #include "paddle/utils/PythonUtil.h" #include #include "paddle/utils/Util.h" +#include "paddle/utils/Excepts.h" + namespace paddle { diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.cpp b/paddle/gserver/gradientmachines/NeuralNetwork.cpp index fca52828957a2..f9da812027dc3 100644 --- a/paddle/gserver/gradientmachines/NeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/NeuralNetwork.cpp @@ -384,17 +384,17 @@ void NeuralNetwork::setOutputGrad(const std::vector& args) { } } -extern NeuralNetwork* newCustomNeuralNetwork( - const std::string& name, NeuralNetwork* network) __attribute__((weak)); +// extern NeuralNetwork* newCustomNeuralNetwork( +// const std::string& name, NeuralNetwork* network) __attribute__((weak)); NeuralNetwork* NeuralNetwork::newNeuralNetwork( const std::string& name, NeuralNetwork* rootNetwork) { - if (newCustomNeuralNetwork) { - return newCustomNeuralNetwork(name, rootNetwork); - } else { +// if (newCustomNeuralNetwork) { +// return newCustomNeuralNetwork(name, rootNetwork); +// } else { return new NeuralNetwork(name, rootNetwork); - } +// } } } // namespace paddle diff --git a/paddle/math/Allocator.h b/paddle/math/Allocator.h index 36166236e9eff..7d277b1c10d2b 100644 --- a/paddle/math/Allocator.h +++ b/paddle/math/Allocator.h @@ -16,7 +16,7 @@ limitations under the License. */ #pragma once #include -#include +#include #include "hl_gpu.h" #include "paddle/utils/Logging.h" @@ -48,9 +48,14 @@ class CpuAllocator : public Allocator { * @return Pointer to the allocated memory */ virtual void* alloc(size_t size) { - void* ptr = memalign(32ul, size); - CHECK(ptr) << "Fail to allocate CPU memory: size=" << size; - return ptr; + #if defined(__APPLE__) || defined(__OSX__) + return malloc(size); + #else + void* ptr; + posix_memalign(&ptr, 32ul, size); + CHECK(ptr) << "Fail to allocate CPU memory: size=" << size; + return ptr; + #endif } /** diff --git a/paddle/math/MathFunctions.h b/paddle/math/MathFunctions.h index fe486c741d6f5..43075977dc9ce 100644 --- a/paddle/math/MathFunctions.h +++ b/paddle/math/MathFunctions.h @@ -23,6 +23,8 @@ extern "C" { } #endif +#include + namespace paddle { template diff --git a/paddle/math/Storage.cpp b/paddle/math/Storage.cpp index 9a879a964ec6d..2bd3db2341638 100644 --- a/paddle/math/Storage.cpp +++ b/paddle/math/Storage.cpp @@ -25,8 +25,8 @@ namespace paddle { // Initialization StorageEngine singleton. // Other modules may rely on storage management, // so StorageEngine need to be initialized before other modules. -static InitFunction __init_storage_engine( - StorageEngine::singleton, std::numeric_limits::max()); +// static InitFunction __init_storage_engine( +// StorageEngine::singleton, std::numeric_limits::max()); StorageEngine::StorageEngine() : cpuAllocator_(nullptr) { } diff --git a/paddle/pserver/LightNetwork.cpp b/paddle/pserver/LightNetwork.cpp index fb427832fad64..c42d2dbe4bbf5 100644 --- a/paddle/pserver/LightNetwork.cpp +++ b/paddle/pserver/LightNetwork.cpp @@ -24,7 +24,12 @@ limitations under the License. */ #include #include #include + +#if defined(__OSX__) || defined(__APPLE__) +#include +#else #include +#endif #include "LightNetwork.h" #include "paddle/utils/Util.h" @@ -92,10 +97,12 @@ void setOption(int sockfd) { CHECK_GE( setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &optval, sizeof(optval)), 0); +#ifdef TCP_QUICKACK optval = 1; CHECK_GE( setsockopt(sockfd, IPPROTO_TCP, TCP_QUICKACK, &optval, sizeof(optval)), 0); +#endif } int reuse = 1; CHECK_GE(setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)), @@ -340,17 +347,27 @@ void SocketWorker::run() { */ void SocketClient::TcpClient(const std::string &serverAddr, int serverPort) { struct sockaddr_in serv_addr; - struct hostent hostinfo, *server; - char buf[1024]; // temp for gethostbyname_r + struct hostent *server; + int errRet; // temp for gethostbyname_r /// Create a socket point int sockfd = socket(AF_INET, SOCK_STREAM, 0); PCHECK(sockfd >= 0) << "ERROR opening socket"; - CHECK_EQ(0, gethostbyname_r(serverAddr.c_str(), &hostinfo, buf, sizeof(buf), - &server, &errRet)) - << "ERROR, no such host: " << serverAddr << " ret = " << errRet; - CHECK(server) << "gethostbyname_r err"; + +#if defined(__OSX__) || defined(__APPLE__) + server = getipnodebyname(serverAddr.c_str(), AF_INET, AI_DEFAULT, &errRet); + CHECK_NE(HOST_NOT_FOUND, errRet) + << "ERROR, no such host: " << serverAddr << " ret = " << errRet; + CHECK(server) << "getipnodebyname error!"; +#else + struct hostent hostinfo; + char buf[1024]; // temp for gethostbyname_r + CHECK_EQ(0, gethostbyname_r(serverAddr.c_str(), &hostinfo, buf, sizeof(buf), + &server, &errRet)) + << "ERROR, no such host: " << serverAddr << " ret = " << errRet; + CHECK(server) << "gethostbyname_r error!"; +#endif bzero((char *)&serv_addr, sizeof(serv_addr)); serv_addr.sin_family = AF_INET; diff --git a/paddle/pserver/SocketChannel.cpp b/paddle/pserver/SocketChannel.cpp index 698473060a4c1..b9d542a296ddd 100644 --- a/paddle/pserver/SocketChannel.cpp +++ b/paddle/pserver/SocketChannel.cpp @@ -27,6 +27,15 @@ limitations under the License. */ namespace paddle { +/** + * UIO_MAXIOV is documented in writev(2), but only + * declares it on osx/ios if defined(KERNEL) + */ +#ifndef UIO_MAXIOV +#define UIO_MAXIOV 512 +#endif + + SocketChannel::~SocketChannel() { if (tcpRdma_ == F_TCP) close(tcpSocket_); @@ -148,8 +157,7 @@ void SocketChannel::writeMessage(const std::vector& userIovs) { std::vector iovs; iovs.reserve(userIovs.size() + 2); iovs.push_back({&header, sizeof(header)}); - iovs.push_back({&iovLengths[0], - sizeof(iovLengths[0]) * (size_t) header.numIovs}); + iovs.push_back({&iovLengths[0], sizeof(iovLengths[0]) * header.numIovs}); iovs.insert(iovs.end(), userIovs.begin(), userIovs.end()); header.totalLength = 0; diff --git a/paddle/trainer/Trainer.cpp b/paddle/trainer/Trainer.cpp index 2890f5b5d7ad9..84d2ee1e73a54 100644 --- a/paddle/trainer/Trainer.cpp +++ b/paddle/trainer/Trainer.cpp @@ -28,6 +28,7 @@ limitations under the License. */ #include "paddle/utils/PythonUtil.h" #include "paddle/utils/Stat.h" #include "paddle/utils/Util.h" +#include "paddle/utils/Excepts.h" #include "paddle/utils/GlobalConstants.h" #include "paddle/gserver/gradientmachines/NeuralNetwork.h" diff --git a/paddle/trainer/TrainerMain.cpp b/paddle/trainer/TrainerMain.cpp index dd30b2c8a5b45..94266639f94ad 100644 --- a/paddle/trainer/TrainerMain.cpp +++ b/paddle/trainer/TrainerMain.cpp @@ -16,6 +16,7 @@ limitations under the License. */ #include #include "paddle/utils/PythonUtil.h" #include "paddle/utils/StringUtil.h" +#include "paddle/utils/Excepts.h" #include "paddle/pserver/ParameterServer2.h" #include "ParamUtil.h" diff --git a/paddle/utils/Excepts.cpp b/paddle/utils/Excepts.cpp new file mode 100644 index 0000000000000..9123508fc78d0 --- /dev/null +++ b/paddle/utils/Excepts.cpp @@ -0,0 +1,54 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "Excepts.h" + +#if defined(__APPLE__) || defined(__OSX__) + +#include + +int fegetexcept(void) { + static fenv_t fenv; + return fegetenv(&fenv) ? -1 : (fenv.__control & FE_ALL_EXCEPT); +} + +int feenableexcept(unsigned int excepts) { + static fenv_t fenv; + unsigned int new_excepts = excepts & FE_ALL_EXCEPT, old_excepts; + + if ( fegetenv (&fenv) ) return -1; + old_excepts = fenv.__control & FE_ALL_EXCEPT; + + // unmask + fenv.__control &= ~new_excepts; + fenv.__mxcsr &= ~(new_excepts << 7); + + return ( fesetenv (&fenv) ? -1 : old_excepts ); +} + +int fedisableexcept(unsigned int excepts) { + static fenv_t fenv; + unsigned int new_excepts = excepts & FE_ALL_EXCEPT, old_excepts; + + if ( fegetenv (&fenv) ) return -1; + old_excepts = fenv.__control & FE_ALL_EXCEPT; + + // mask + fenv.__control |= new_excepts; + fenv.__mxcsr |= new_excepts << 7; + + return ( fesetenv (&fenv) ? -1 : old_excepts ); +} + +#endif diff --git a/paddle/utils/Excepts.h b/paddle/utils/Excepts.h new file mode 100644 index 0000000000000..a84a2d33a6a3d --- /dev/null +++ b/paddle/utils/Excepts.h @@ -0,0 +1,26 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifndef EXCEPTS_H_ +#define EXCEPTS_H_ + +#if defined(__APPLE__) || defined(__OSX__) + +int fegetexcept(void); +int feenableexcept(unsigned int excepts); +int fedisableexcept(unsigned int excepts); + +#endif + +#endif // EXCEPTS_H_ diff --git a/paddle/utils/Locks.cpp b/paddle/utils/Locks.cpp new file mode 100644 index 0000000000000..c2f58cf5764ef --- /dev/null +++ b/paddle/utils/Locks.cpp @@ -0,0 +1,106 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#ifdef __APPLE__ +#include +#endif + +#ifdef __APPLE__ +#ifndef PTHREAD_BARRIER_H_ +#define PTHREAD_BARRIER_H_ + +#include +#include + +typedef int pthread_barrierattr_t; +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond; + int count; + int tripCount; +} pthread_barrier_t; + +int pthread_barrier_init(pthread_barrier_t *barrier, + const pthread_barrierattr_t *attr, unsigned int count) { + if (count == 0) { + errno = EINVAL; + return -1; + } + if (pthread_mutex_init(&barrier->mutex, 0) < 0) { + return -1; + } + if (pthread_cond_init(&barrier->cond, 0) < 0) { + pthread_mutex_destroy(&barrier->mutex); + return -1; + } + barrier->tripCount = count; + barrier->count = 0; + + return 0; +} + +int pthread_barrier_destroy(pthread_barrier_t *barrier) { + pthread_cond_destroy(&barrier->cond); + pthread_mutex_destroy(&barrier->mutex); + return 0; +} + +int pthread_barrier_wait(pthread_barrier_t *barrier) { + pthread_mutex_lock(&barrier->mutex); + ++(barrier->count); + if (barrier->count >= barrier->tripCount) { + barrier->count = 0; + pthread_cond_broadcast(&barrier->cond); + pthread_mutex_unlock(&barrier->mutex); + return 1; + } else { + pthread_cond_wait(&barrier->cond, &(barrier->mutex)); + pthread_mutex_unlock(&barrier->mutex); + return 0; + } +} + +#endif // PTHREAD_BARRIER_H_ + +typedef int pthread_spinlock_t; + +int pthread_spin_init(pthread_spinlock_t *lock, int pshared) { + __asm__ __volatile__("" ::: "memory"); + *lock = 0; + return 0; +} + +int pthread_spin_destroy(pthread_spinlock_t *lock) { + return 0; +} + +int pthread_spin_lock(pthread_spinlock_t *lock) { + while (1) { + int i; + for (i=0; i < 10000; i++) { + if (__sync_bool_compare_and_swap(lock, 0, 1)) { + return 0; + } + } + sched_yield(); + } +} + +int pthread_spin_unlock(pthread_spinlock_t *lock) { + __asm__ __volatile__("" ::: "memory"); + *lock = 0; + return 0; +} + +#endif // __APPLE__ diff --git a/paddle/utils/Locks.h b/paddle/utils/Locks.h index 085aca508dbbe..e7b0b77081f36 100644 --- a/paddle/utils/Locks.h +++ b/paddle/utils/Locks.h @@ -23,6 +23,50 @@ limitations under the License. */ #include #include +#ifdef __APPLE__ +#include +#endif + +#ifdef __APPLE__ +#ifndef PTHREAD_BARRIER_H_ +#define PTHREAD_BARRIER_H_ + +#include +#include + +typedef int pthread_barrierattr_t; +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond; + int count; + int tripCount; +} pthread_barrier_t; + + +extern int pthread_barrier_init(pthread_barrier_t *barrier, + const pthread_barrierattr_t *attr, + unsigned int count); + +extern int pthread_barrier_destroy(pthread_barrier_t *barrier); + +extern int pthread_barrier_wait(pthread_barrier_t *barrier); + +#endif // PTHREAD_BARRIER_H_ + +typedef int pthread_spinlock_t; + +extern int pthread_spin_init(pthread_spinlock_t *lock, int pshared); + +extern int pthread_spin_destroy(pthread_spinlock_t *lock); + +extern int pthread_spin_lock(pthread_spinlock_t *lock); + +extern int pthread_spin_unlock(pthread_spinlock_t *lock); + +#endif + + + namespace paddle { /** @@ -117,6 +161,29 @@ class SpinLock { /** * A simple wapper of semaphore which can only be shared in the same process. */ + +#ifdef __APPLE__ + +class Semaphore { +public: + explicit Semaphore(int initValue = 0) { + sem_ = dispatch_semaphore_create(initValue); + } + + ~Semaphore() { dispatch_release(sem_); } + bool timeWait(struct timespec* ts) { + dispatch_time_t m = dispatch_walltime(ts, 0); + return (0 == dispatch_semaphore_wait(sem_, m)); + } + void wait() { dispatch_semaphore_wait(sem_, DISPATCH_TIME_FOREVER); } + void post() { dispatch_semaphore_signal(sem_);} + +protected: + dispatch_semaphore_t sem_; +}; + +#else + class Semaphore { public: /** @@ -153,6 +220,8 @@ class Semaphore { sem_t sem_; }; +#endif + static_assert(sizeof(SpinLock) == 64, "Wrong padding"); /** diff --git a/paddle/utils/PythonUtil.h b/paddle/utils/PythonUtil.h index 4467fd784ec4e..397229d803df9 100644 --- a/paddle/utils/PythonUtil.h +++ b/paddle/utils/PythonUtil.h @@ -18,6 +18,12 @@ limitations under the License. */ #ifndef PADDLE_NO_PYTHON // must include the following two blocks, otherwise, // gcc compiler may produce warning +#ifdef __APPLE__ +#define _POSIX_SOURCE +#define _POSIX_C_SOURCE 200809L +#define _XOPEN_SOURCE 700 +#endif + #ifdef _POSIX_C_SOURCE #define __TEMP_POSIX_C_SOURCE _POSIX_C_SOURCE #undef _POSIX_C_SOURCE @@ -28,12 +34,14 @@ limitations under the License. */ #endif #include #include -#ifndef _POSIX_C_SOURCE -#warning "no _POSIX_C_SOURCE defined in Python.h" -#endif -#ifndef _XOPEN_SOURCE -#warning "no _XOPEN_SOURCE defined in Python.h" -#endif + +// #ifndef _POSIX_C_SOURCE +// #warning "no _POSIX_C_SOURCE defined in Python.h" +// #endif +// #ifndef _XOPEN_SOURCE +// #warning "no _XOPEN_SOURCE defined in Python.h" +// #endif + #endif #include "paddle/utils/Util.h" From ad6cadeb9a5206ea9ea21679267a14687a278f80 Mon Sep 17 00:00:00 2001 From: liaogang Date: Fri, 9 Sep 2016 19:50:55 +0800 Subject: [PATCH 051/324] replace linux/tcp by netinet/tcp --- paddle/pserver/LightNetwork.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/paddle/pserver/LightNetwork.cpp b/paddle/pserver/LightNetwork.cpp index c42d2dbe4bbf5..5dc04ee6c2ef1 100644 --- a/paddle/pserver/LightNetwork.cpp +++ b/paddle/pserver/LightNetwork.cpp @@ -17,6 +17,7 @@ limitations under the License. */ #include #include #include +#include #include #include @@ -25,12 +26,6 @@ limitations under the License. */ #include #include -#if defined(__OSX__) || defined(__APPLE__) -#include -#else -#include -#endif - #include "LightNetwork.h" #include "paddle/utils/Util.h" #include "paddle/utils/StringUtil.h" From 3f5ce64cab69a98c524d3d481b11e97516b4eb69 Mon Sep 17 00:00:00 2001 From: liaogang Date: Fri, 9 Sep 2016 19:50:55 +0800 Subject: [PATCH 052/324] fix StorageEngine::singleton and std::function type unmatched. --- paddle/math/Storage.cpp | 4 ++-- paddle/pserver/LightNetwork.cpp | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/paddle/math/Storage.cpp b/paddle/math/Storage.cpp index 2bd3db2341638..0403c3521cf54 100644 --- a/paddle/math/Storage.cpp +++ b/paddle/math/Storage.cpp @@ -25,8 +25,8 @@ namespace paddle { // Initialization StorageEngine singleton. // Other modules may rely on storage management, // so StorageEngine need to be initialized before other modules. -// static InitFunction __init_storage_engine( -// StorageEngine::singleton, std::numeric_limits::max()); +static InitFunction __init_storage_engine([](){StorageEngine::singleton();}, + std::numeric_limits::max()); StorageEngine::StorageEngine() : cpuAllocator_(nullptr) { } diff --git a/paddle/pserver/LightNetwork.cpp b/paddle/pserver/LightNetwork.cpp index c42d2dbe4bbf5..5dc04ee6c2ef1 100644 --- a/paddle/pserver/LightNetwork.cpp +++ b/paddle/pserver/LightNetwork.cpp @@ -17,6 +17,7 @@ limitations under the License. */ #include #include #include +#include #include #include @@ -25,12 +26,6 @@ limitations under the License. */ #include #include -#if defined(__OSX__) || defined(__APPLE__) -#include -#else -#include -#endif - #include "LightNetwork.h" #include "paddle/utils/Util.h" #include "paddle/utils/StringUtil.h" From a3941cbc1f2b1087af7a5869b516ab29be3aaa36 Mon Sep 17 00:00:00 2001 From: liaogang Date: Fri, 9 Sep 2016 21:22:40 +0800 Subject: [PATCH 053/324] remove weak attribute for internal FPGA --- paddle/gserver/gradientmachines/NeuralNetwork.cpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.cpp b/paddle/gserver/gradientmachines/NeuralNetwork.cpp index f9da812027dc3..903922204343e 100644 --- a/paddle/gserver/gradientmachines/NeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/NeuralNetwork.cpp @@ -384,17 +384,10 @@ void NeuralNetwork::setOutputGrad(const std::vector& args) { } } -// extern NeuralNetwork* newCustomNeuralNetwork( -// const std::string& name, NeuralNetwork* network) __attribute__((weak)); - NeuralNetwork* NeuralNetwork::newNeuralNetwork( const std::string& name, NeuralNetwork* rootNetwork) { -// if (newCustomNeuralNetwork) { -// return newCustomNeuralNetwork(name, rootNetwork); -// } else { return new NeuralNetwork(name, rootNetwork); -// } } } // namespace paddle From 38166e29d7e1e3e682d90b107bead6599b9fbb73 Mon Sep 17 00:00:00 2001 From: liaogang Date: Sat, 10 Sep 2016 07:46:53 +0800 Subject: [PATCH 054/324] Add default openblas path on MAC OS --- cmake/cblas.cmake | 10 ++++++---- cmake/util.cmake | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/cmake/cblas.cmake b/cmake/cblas.cmake index 617bd7ea7162b..5568f927572f5 100644 --- a/cmake/cblas.cmake +++ b/cmake/cblas.cmake @@ -44,8 +44,8 @@ set(ATLAS_LIB_SEARCH_PATHS /usr/lib /usr/lib/blas/atlas /usr/lib/atlas - /usr/lib/atlas-base # special for ubuntu 14.04. - ) + /usr/lib/atlas-base) # special for ubuntu 14.04. + find_path(ATLAS_INC_DIR NAMES cblas.h PATHS ${ATLAS_INCLUDE_SEARCH_PATHS}) find_library(ATLAS_CBLAS_LIB NAMES cblas libcblas.so.3 @@ -65,12 +65,14 @@ set(OPENBLAS_ROOT $ENV{OPENBLAS_ROOT} CACHE PATH "Folder contains Openblas") set(OPENBLAS_INCLUDE_SEARCH_PATHS ${OPENBLAS_ROOT}/include /usr/include - /usr/include/openblas) + /usr/include/openblas + /usr/local/opt/openblas/include) set(OPENBLAS_LIB_SEARCH_PATHS ${OPENBLAS_ROOT}/lib /usr/lib /usr/lib/blas/openblas - /usr/lib/openblas) + /usr/lib/openblas + /usr/local/opt/openblas/lib) find_path(OPENBLAS_INC_DIR NAMES cblas.h PATHS ${OPENBLAS_INCLUDE_SEARCH_PATHS}) diff --git a/cmake/util.cmake b/cmake/util.cmake index bad44c7e9dda6..f3227d27c53c2 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -1,12 +1,9 @@ -# Some common routine for paddle compile. +# MAC OS does not contain start-up and whole-archive args if(APPLE) - # ------------------------------------------------------- - # OSX - # ------------------------------------------------------- - set(GROUP_START "-Wl,-force_load") + set(GROUP_START "") set(GROUP_END "") - set(ARCHIVE_START "-Wl,-force_load") + set(ARCHIVE_START "") set(ARCHIVE_END "") else() set(GROUP_START "-Wl,--start-group") @@ -17,6 +14,7 @@ else() endif() +# Some common routine for paddle compile. # target_circle_link_libraries # Link libraries to target which has circle dependencies. @@ -25,15 +23,16 @@ endif() # Rest Arguments: libraries which link together. function(target_circle_link_libraries TARGET_NAME) if(APPLE) - foreach(f ${ARGN}) - list(APPEND OSX_LIBRARIES "-Wl,-force_load" "${f}") - endforeach(f) + foreach(arg ${ARGN}) + list(APPEND OSX_LIBRARIES "-Wl,-force_load" "${arg}") + endforeach() target_link_libraries(${TARGET_NAME} - ${OSX_LIBRARIES}) + ${OSX_LIBRARIES} -lz) else() target_link_libraries(${TARGET_NAME} ${GROUP_START} ${ARGN} + -lz ${GROUP_END}) endif() endfunction() @@ -66,18 +65,20 @@ function(link_paddle_exe TARGET_NAME) if(PADDLE_WITH_INTERNAL) set(INTERAL_LIBS paddle_internal_gserver paddle_internal_parameter) target_circle_link_libraries(${TARGET_NAME} + ${ARCHIVE_START} paddle_internal_gserver paddle_internal_owlqn + ${ARCHIVE_END} paddle_internal_parameter) else() set(INTERAL_LIBS "") endif() target_circle_link_libraries(${TARGET_NAME} -# ${ARCHIVE_START} + ${ARCHIVE_START} paddle_gserver ${METRIC_LIBS} -# ${ARCHIVE_END} + ${ARCHIVE_END} paddle_pserver paddle_trainer_lib paddle_network @@ -91,8 +92,7 @@ function(link_paddle_exe TARGET_NAME) ${CMAKE_THREAD_LIBS_INIT} ${CBLAS_LIBS} ${CMAKE_DL_LIBS} - ${INTERAL_LIBS} - ) + ${INTERAL_LIBS}) if(WITH_PYTHON) target_link_libraries(${TARGET_NAME} From 58f74e2ca17fd6b417355e2686ce00f3ff898fbd Mon Sep 17 00:00:00 2001 From: liaogang Date: Sat, 10 Sep 2016 09:05:57 +0800 Subject: [PATCH 055/324] Add main entry for unit test files and replace memalign by posix_memalign --- paddle/math/Allocator.h | 4 ---- paddle/math/tests/test_SIMDFunctions.cpp | 6 ++++-- paddle/math/tests/test_perturbation.cpp | 5 +++++ paddle/parameter/tests/test_common.cpp | 10 ++++++---- paddle/utils/tests/test_StringUtils.cpp | 5 +++++ 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/paddle/math/Allocator.h b/paddle/math/Allocator.h index 7d277b1c10d2b..ca8eadbc1aa42 100644 --- a/paddle/math/Allocator.h +++ b/paddle/math/Allocator.h @@ -48,14 +48,10 @@ class CpuAllocator : public Allocator { * @return Pointer to the allocated memory */ virtual void* alloc(size_t size) { - #if defined(__APPLE__) || defined(__OSX__) - return malloc(size); - #else void* ptr; posix_memalign(&ptr, 32ul, size); CHECK(ptr) << "Fail to allocate CPU memory: size=" << size; return ptr; - #endif } /** diff --git a/paddle/math/tests/test_SIMDFunctions.cpp b/paddle/math/tests/test_SIMDFunctions.cpp index 631d0516cf409..bae5d8c684d89 100644 --- a/paddle/math/tests/test_SIMDFunctions.cpp +++ b/paddle/math/tests/test_SIMDFunctions.cpp @@ -24,7 +24,7 @@ limitations under the License. */ #include #include -#include +#include #include static constexpr size_t VECTOR_LEN = 3072; @@ -37,7 +37,9 @@ static std::mt19937 RandomEngine(time(0)); inline static std::unique_ptr NewVector(size_t len = VECTOR_LEN, size_t align = ALIGN) { - return std::unique_ptr((float*)memalign(align, len * sizeof(float))); + float* ptr; + posix_memalign((void**)&ptr, align, len * sizeof(float)); + return std::unique_ptr(ptr); } inline static std::unique_ptr NewRandomVector(size_t len = VECTOR_LEN, diff --git a/paddle/math/tests/test_perturbation.cpp b/paddle/math/tests/test_perturbation.cpp index 4fa9bc72013da..51e346fef91bf 100644 --- a/paddle/math/tests/test_perturbation.cpp +++ b/paddle/math/tests/test_perturbation.cpp @@ -249,4 +249,9 @@ TEST_F(PerturbationTest, scale_test) { } } +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + #endif diff --git a/paddle/parameter/tests/test_common.cpp b/paddle/parameter/tests/test_common.cpp index 3db96ccf941e3..4f92aec1d9671 100644 --- a/paddle/parameter/tests/test_common.cpp +++ b/paddle/parameter/tests/test_common.cpp @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include +#include #include #include @@ -124,9 +124,11 @@ void CommonTest::test_sgdUpadate(real* gradientBuffer, real* valueBuffer, TEST_F(CommonTest, sgdUpdate) { const size_t alignHeader[] = {0, 2, 3, 5, 7, 8}; for (auto& size : sizeVec_) { - real* gradientBuffer = (real*)memalign(32, sizeof(real) * size); - real* valueBuffer = (real*)memalign(32, sizeof(real) * size); - real* momentumBuffer = (real*)memalign(32, sizeof(real) * size); + real *gradientBuffer, *valueBuffer, *momentumBuffer; + posix_memalign((void**)&gradientBuffer, 32, sizeof(real) * size); + posix_memalign((void**)&valueBuffer, 32, sizeof(real) * size); + posix_memalign((void**)&momentumBuffer, 32, sizeof(real) * size); + for (size_t i = 0; i < size; i++) { gradientBuffer[i] = 1.0; valueBuffer[i] = 2.0; diff --git a/paddle/utils/tests/test_StringUtils.cpp b/paddle/utils/tests/test_StringUtils.cpp index b8636709e9b42..95290005ae983 100644 --- a/paddle/utils/tests/test_StringUtils.cpp +++ b/paddle/utils/tests/test_StringUtils.cpp @@ -22,3 +22,8 @@ TEST(StringUtil, to) { ASSERT_DEATH(paddle::str::to("12.45x23"), ".*"); ASSERT_DEATH(paddle::str::to(""), ".*"); } + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 87f96f873aaea36387c3914fdb6f85d6ff86af06 Mon Sep 17 00:00:00 2001 From: liaogang Date: Sat, 10 Sep 2016 10:00:53 +0800 Subject: [PATCH 056/324] fix dynamic load PaddlePaddle for Mac OS --- paddle/cuda/src/hl_dso_loader.cc | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/paddle/cuda/src/hl_dso_loader.cc b/paddle/cuda/src/hl_dso_loader.cc index 3558b163b5ae0..eee9984e07326 100644 --- a/paddle/cuda/src/hl_dso_loader.cc +++ b/paddle/cuda/src/hl_dso_loader.cc @@ -69,23 +69,40 @@ static inline void GetDsoHandleWithSearchPath( CHECK(nullptr != *dso_handle) << "For Gpu version of PaddlePaddle, it couldn't find CUDA library: " - << dlPath.c_str() << " Please make sure you already specify its path." - << "Note: for training data on Cpu using Gpu version of PaddlePaddle," - << "you must specify libcudart.so via LD_LIBRARY_PATH."; + << dlPath.c_str() << ". Please make sure you already specify its path. " + << "Note: for training data on Cpu using Gpu version of PaddlePaddle, " + << "you must specify libcudart via export LD_LIBRARY_PATH for Linux or " + << "export DYLD_LIBRARY_PATH for MAC OS."; } void GetCublasDsoHandle(void** dso_handle) { +#if defined(__APPLE__) || defined(__OSX__) + GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcublas.dylib", dso_handle); +#else GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcublas.so", dso_handle); +#endif } void GetCudnnDsoHandle(void** dso_handle) { +#if defined(__APPLE__) || defined(__OSX__) + GetDsoHandleWithSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", dso_handle); +#else GetDsoHandleWithSearchPath(FLAGS_cudnn_dir, "libcudnn.so", dso_handle); +#endif } void GetCudartDsoHandle(void** dso_handle) { +#if defined(__APPLE__) || defined(__OSX__) + GetDsoHandleWithSearchPath("", "libcudart.dylib", dso_handle); +#else GetDsoHandleWithSearchPath("", "libcudart.so", dso_handle); +#endif } void GetCurandDsoHandle(void** dso_handle) { +#if defined(__APPLE__) || defined(__OSX__) + GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcurand.dylib", dso_handle); +#else GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcurand.so", dso_handle); +#endif } From b664ca0321f139ad33577066d52a1d4f5e868a28 Mon Sep 17 00:00:00 2001 From: liaogang Date: Sat, 10 Sep 2016 15:36:13 +0800 Subject: [PATCH 057/324] auto-tuning SND/REV buff size on MAC OS --- paddle/pserver/LightNetwork.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paddle/pserver/LightNetwork.cpp b/paddle/pserver/LightNetwork.cpp index 5dc04ee6c2ef1..ff2875fc702ff 100644 --- a/paddle/pserver/LightNetwork.cpp +++ b/paddle/pserver/LightNetwork.cpp @@ -79,6 +79,7 @@ std::string getIpAddr(std::string &device) { * @note adjust some default sock option for better performance */ void setOption(int sockfd) { +#if !defined(__APPLE__) && !defined(__OSX__) int sendSize = FLAGS_sock_send_buf_size; int recvSize = FLAGS_sock_recv_buf_size; CHECK_GE( @@ -87,6 +88,8 @@ void setOption(int sockfd) { CHECK_GE( setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF, &sendSize, sizeof(sendSize)), 0); +#endif + if (FLAGS_small_messages) { int optval = 1; CHECK_GE( From 70cceb0cdd1335fbba38a45d82e1333961a2659a Mon Sep 17 00:00:00 2001 From: liaogang Date: Sun, 11 Sep 2016 14:13:59 +0800 Subject: [PATCH 058/324] fix compile paddle swig bug on MAC OS --- paddle/api/paddle_ld_flags.py | 17 +++++++++++++++-- paddle/setup.py.in | 23 +++++++++++++++++++---- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/paddle/api/paddle_ld_flags.py b/paddle/api/paddle_ld_flags.py index 21b4ca1dd6171..bc1afc5898e82 100644 --- a/paddle/api/paddle_ld_flags.py +++ b/paddle/api/paddle_ld_flags.py @@ -15,6 +15,19 @@ try: from paddle_api_config import * import os.path + import platform + + system = platform.system().lower() + is_osx = (system == 'darwin') + is_win = (system == 'windows') + is_lin = (system == 'linux') + + if is_lin: + whole_start = "-Wl,--whole-archive" + whole_end = "-Wl,--no-whole-archive" + elif is_osx: + whole_start = "" + whole_end = "" LIB_DIRS = ["math", 'utils', 'parameter', "gserver", "api", "cuda", "pserver", "trainer"] PARENT_LIB_DIRS = ['proto'] @@ -56,9 +69,9 @@ def parent_dir_str(self): def libs_str(self): libs = [ - "-Wl,--whole-archive", + whole_start, "-lpaddle_gserver", - "-Wl,--no-whole-archive", + whole_end, "-lpaddle_pserver", "-lpaddle_trainer_lib", "-lpaddle_network", diff --git a/paddle/setup.py.in b/paddle/setup.py.in index da86eb795dc58..02ea9067431c6 100644 --- a/paddle/setup.py.in +++ b/paddle/setup.py.in @@ -17,6 +17,14 @@ from setuptools import setup, Extension import numpy as np import api.paddle_ld_flags +import platform + +system = platform.system().lower() + +is_osx = (system == 'darwin') +is_win = (system == 'windows') +is_lin = (system == 'linux') + # The extra links will passed from COMAKE # because generate paddle LDFLAGS is too complicated to do in setup.py @@ -34,17 +42,24 @@ try: except: pass +if is_lin == True: + extra_links = ["-Xlinker", '-start-group'] + extra_links + ["-Xlinker", "-end-group"] +elif is_osx == True: + extra_links = ["-Wl,-all_load"] + extra_links + +include_dirs = [np.get_include(), "../"] # include numpy and paddle. + setup(name="py_paddle", version="@PADDLE_VERSION@", ext_modules=[ Extension('py_paddle._swig_paddle', # Build SWIG Extension. - ['Paddle_wrap.cxx'], - extra_link_args=["-Xlinker", '-start-group'] + - extra_links + ["-Xlinker", "-end-group"] + ['Paddle_wrap.cxx'], + include_dirs = include_dirs, + extra_link_args = extra_links ) ], packages=['py_paddle'], - include_dirs = [np.get_include(), "../"], # include numpy and paddle. + include_dirs = include_dirs, install_requires = [ 'numpy>=1.8.0', # The numpy is required. 'protobuf>=2.4.1' # The paddle protobuf version From dcd87fd68971d85b7faa90c2dfee6b6534ec2315 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Mon, 12 Sep 2016 09:58:35 +0800 Subject: [PATCH 059/324] fix CUDNN_VERSION for backward of CudnnBatchNormLayer (#61) --- paddle/gserver/layers/CudnnBatchNormLayer.cpp | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/paddle/gserver/layers/CudnnBatchNormLayer.cpp b/paddle/gserver/layers/CudnnBatchNormLayer.cpp index e1762e8d360de..cef8772fc254f 100644 --- a/paddle/gserver/layers/CudnnBatchNormLayer.cpp +++ b/paddle/gserver/layers/CudnnBatchNormLayer.cpp @@ -114,27 +114,30 @@ void CudnnBatchNormLayer::backward(const UpdateCallback& callback) { } else { create(tmpBiasGrad_, 1, channels_, &betaGrad); } -#if CUDNN_VERSION < 5000 + // because of the different api of cudnn v4 and v5. - if (weight_->getWGrad()) { - create(tmpWGrad_, 1, channels_, &gammaGrad); - } - if (biases_ && biases_->getWGrad()) { - create(tmpBiasGrad_, 1, channels_, &betaGrad); + if (hl_get_cudnn_lib_version() < 5000) { + if (weight_->getWGrad()) { + create(tmpWGrad_, 1, channels_, &gammaGrad); + } + if (biases_ && biases_->getWGrad()) { + create(tmpBiasGrad_, 1, channels_, &betaGrad); + } } -#endif + hl_batch_norm_backward(ioDesc_, input, ioDesc_, outGrad, ioDesc_, inGrad, bnParamDesc_, gamma, gammaGrad, betaGrad, EPS, savedMean, savedInvVar); -#if CUDNN_VERSION < 5000 // because of the different api of cudnn v4 and v5. - if (weight_->getWGrad() && biases_->getWGrad()) { - weight_->getWGrad()->add(*tmpWGrad_); - biases_->getWGrad()->add(*tmpBiasGrad_); + if (hl_get_cudnn_lib_version() < 5000) { + if (weight_->getWGrad() && biases_->getWGrad()) { + weight_->getWGrad()->add(*tmpWGrad_); + biases_->getWGrad()->add(*tmpBiasGrad_); + } } -#endif + { REGISTER_TIMER_INFO("WeightUpdate", getName().c_str()); biases_->getParameterPtr()->incUpdate(callback); From c7ece60e2d3715a1b37973fd503eea0848a795d5 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 12 Sep 2016 19:43:12 +0800 Subject: [PATCH 060/324] add gettid syscall for MAC OS --- paddle/utils/Stat.cpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/paddle/utils/Stat.cpp b/paddle/utils/Stat.cpp index 14aae6909d404..aab5446a98c0b 100644 --- a/paddle/utils/Stat.cpp +++ b/paddle/utils/Stat.cpp @@ -23,10 +23,14 @@ namespace paddle { // return the thread id used by glog pid_t getTID() { -#ifndef __NR_gettid -#define __NR_gettid 224 -#endif - pid_t tid = syscall(__NR_gettid); + #if defined(__APPLE__) || defined(__OSX__) + pid_t tid = syscall(SYS_thread_selfid); + #elif defined(__LINUX__) + #ifndef __NR_gettid + #define __NR_gettid 224 + #endif + pid_t tid = syscall(__NR_gettid); + #endif CHECK_NE(tid, -1); return tid; } From 8d0214193e79c3b68e06ab460acb5a37c705aea8 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 12 Sep 2016 19:45:32 +0800 Subject: [PATCH 061/324] fix unit test bug when only one gpu --- paddle/trainer/tests/test_Trainer.cpp | 8 ++++++-- paddle/trainer/tests/test_TrainerOnePass.cpp | 6 +++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/paddle/trainer/tests/test_Trainer.cpp b/paddle/trainer/tests/test_Trainer.cpp index 8ca9be71de9ac..2044279c2151f 100644 --- a/paddle/trainer/tests/test_Trainer.cpp +++ b/paddle/trainer/tests/test_Trainer.cpp @@ -62,7 +62,11 @@ TEST(checkGradient, multiGpu) { } } -TEST(checkGradient, parallel) { checkGradientTest(configFile4, true, true); } +TEST(checkGradient, parallel) { + if (hl_get_device_count() >= 2) { + checkGradientTest(configFile4, true, true); + } +} TEST(checkGradient, multiParallel) { FLAGS_allow_only_one_model_on_one_gpu = false; @@ -90,7 +94,7 @@ TEST(checkGradient, multi) { TEST(checkGradient, hsigmoid) { checkGradientTest(configFile2, false, false); } TEST(checkGradient, chunk) { - EXPECT_EQ(0, system("python2 trainer/tests/gen_proto_data.py")); + EXPECT_EQ(0, system("python trainer/tests/gen_proto_data.py")); checkGradientTest(configFile3, false, false); #ifndef PADDLE_ONLY_CPU checkGradientTest(configFile3, true, true); diff --git a/paddle/trainer/tests/test_TrainerOnePass.cpp b/paddle/trainer/tests/test_TrainerOnePass.cpp index 6d8b8e0ca5c98..4554b94485f99 100644 --- a/paddle/trainer/tests/test_TrainerOnePass.cpp +++ b/paddle/trainer/tests/test_TrainerOnePass.cpp @@ -82,7 +82,11 @@ TEST(trainerOnePass, gpu2) { trainerOnePassTest(configFile1, true, false, 2); } TEST(trainerOnePass, gpu4) { trainerOnePassTest(configFile1, true, false, 4); } -TEST(trainerOnePass, parallel) { trainerOnePassTest(configFile2, true, true); } +TEST(trainerOnePass, parallel) { + if (hl_get_device_count() >= 2) { + trainerOnePassTest(configFile2, true, true); + } +} #endif // 2. test average_window. From d3eef0c9af6dec65a51b042f0bb1dbbdb030caaa Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 12 Sep 2016 21:31:54 +0800 Subject: [PATCH 062/324] reduce data_layer size of unit test to avoid cuda out of memory on MAC OS --- paddle/gserver/tests/concat_table_a.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/gserver/tests/concat_table_a.conf b/paddle/gserver/tests/concat_table_a.conf index 2e3c518883e20..a8ff70f883318 100644 --- a/paddle/gserver/tests/concat_table_a.conf +++ b/paddle/gserver/tests/concat_table_a.conf @@ -16,9 +16,9 @@ from paddle.trainer_config_helpers import * -settings(batch_size=1000) +settings(batch_size=300) -data = data_layer(name ="input", size=100000) +data = data_layer(name ="input", size=10000) # emb1 is equal to emb2, note that bias_attr=false # and act=LinearActivation() in default. From eb5cf64b5e7f1b12a6d48057b630067dfd73851c Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 12 Sep 2016 21:32:38 +0800 Subject: [PATCH 063/324] reduce data_layer size of unit test to avoid cuda out of memory on MAC OS --- paddle/gserver/tests/concat_table_b.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/gserver/tests/concat_table_b.conf b/paddle/gserver/tests/concat_table_b.conf index 6da24a5fbc55c..95d7c10f7b0cd 100644 --- a/paddle/gserver/tests/concat_table_b.conf +++ b/paddle/gserver/tests/concat_table_b.conf @@ -16,9 +16,9 @@ from paddle.trainer_config_helpers import * -settings(batch_size=1000) +settings(batch_size=300) -data = data_layer(name ="input", size=100000) +data = data_layer(name ="input", size=10000) proj1 = table_projection(input=data, size=128) From 0ad0e4ac4831c12187b6a2f0005742e84b85df8d Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 12 Sep 2016 21:50:25 +0800 Subject: [PATCH 064/324] Add Wall Werror --- cmake/flags.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index c95d4063105e7..4b99e7f7fb6af 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -58,8 +58,8 @@ set(COMMON_FLAGS -fPIC -fno-omit-frame-pointer -Wall -# -Wextra -# -Werror + -Wextra + -Werror -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter From 738b6894bce2b84c87431f1f84edbde7be4f4f24 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 12 Sep 2016 22:13:25 +0800 Subject: [PATCH 065/324] Make whole-archive to less library * Also make use cmake find to zlib. * circle link in osx, use reverse link all libs instead. But maybe osx just don't care circle link. --- CMakeLists.txt | 1 + cmake/util.cmake | 70 +++++++++++++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 007f1f18bb655..99c6c0d373052 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,6 +14,7 @@ find_package(CUDA QUIET) find_package(Protobuf REQUIRED) find_package(PythonLibs 2.7 REQUIRED) find_package(PythonInterp 2.7 REQUIRED) +find_package(ZLIB REQUIRED) find_package(NumPy) find_package(Threads REQUIRED) find_package(Glog) diff --git a/cmake/util.cmake b/cmake/util.cmake index f3227d27c53c2..a91e1d5643014 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -1,19 +1,3 @@ -# MAC OS does not contain start-up and whole-archive args -if(APPLE) - set(GROUP_START "") - set(GROUP_END "") - - set(ARCHIVE_START "") - set(ARCHIVE_END "") -else() - set(GROUP_START "-Wl,--start-group") - set(GROUP_END "-Wl,--end-group") - - set(ARCHIVE_START "-Wl,--whole-archive") - set(ARCHIVE_END "-Wl,--no-whole-archive") -endif() - - # Some common routine for paddle compile. # target_circle_link_libraries @@ -23,17 +7,46 @@ endif() # Rest Arguments: libraries which link together. function(target_circle_link_libraries TARGET_NAME) if(APPLE) + set(LIBS) + set(inArchive OFF) + set(libsInArgn) + foreach(arg ${ARGN}) - list(APPEND OSX_LIBRARIES "-Wl,-force_load" "${arg}") + if(${arg} STREQUAL "ARCHIVE_START") + set(inArchive ON) + elseif(${arg} STREQUAL "ARCHIVE_END") + set(inArchive OFF) + else() + if(inArchive) + list(APPEND LIBS "-Wl,-force_load") + endif() + list(APPEND LIBS ${arg}) + list(APPEND libsInArgn ${arg}) + endif() endforeach() + + list(REVERSE libsInArgn) target_link_libraries(${TARGET_NAME} - ${OSX_LIBRARIES} -lz) - else() + ${LIBS} + ${libsInArgn}) + + else() # LINUX + set(LIBS) + + foreach(arg ${ARGN}) + if(${arg} STREQUAL "ARCHIVE_START") + list(APPEND LIBS "-Wl,--whole-archive") + elseif(${arg} STREQUAL "ARCHIVE_END") + list(APPEND LIBS "-Wl,--no-whole-archive") + else() + list(APPEND LIBS ${arg}) + endif() + endforeach() + target_link_libraries(${TARGET_NAME} - ${GROUP_START} - ${ARGN} - -lz - ${GROUP_END}) + "-Wl,--start-group" + ${LIBS} + "-Wl,--end-group") endif() endfunction() @@ -65,20 +78,20 @@ function(link_paddle_exe TARGET_NAME) if(PADDLE_WITH_INTERNAL) set(INTERAL_LIBS paddle_internal_gserver paddle_internal_parameter) target_circle_link_libraries(${TARGET_NAME} - ${ARCHIVE_START} + ARCHIVE_START paddle_internal_gserver paddle_internal_owlqn - ${ARCHIVE_END} + ARCHIVE_END paddle_internal_parameter) else() set(INTERAL_LIBS "") endif() target_circle_link_libraries(${TARGET_NAME} - ${ARCHIVE_START} + ARCHIVE_START paddle_gserver ${METRIC_LIBS} - ${ARCHIVE_END} + ARCHIVE_END paddle_pserver paddle_trainer_lib paddle_network @@ -92,7 +105,8 @@ function(link_paddle_exe TARGET_NAME) ${CMAKE_THREAD_LIBS_INIT} ${CBLAS_LIBS} ${CMAKE_DL_LIBS} - ${INTERAL_LIBS}) + ${INTERAL_LIBS} + ${ZLIB_LIBRARIES}) if(WITH_PYTHON) target_link_libraries(${TARGET_NAME} From db13fddcc4882cd5505e1c628770131b47cd70e1 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 13 Sep 2016 10:27:37 +0800 Subject: [PATCH 066/324] Clean locks.h, Add arch Locks.cpp --- .gitignore | 1 + paddle/utils/CMakeLists.txt | 14 +++- paddle/utils/Locks.cpp | 106 ------------------------- paddle/utils/Locks.h | 126 ++++++++---------------------- paddle/utils/arch/linux/Locks.cpp | 85 ++++++++++++++++++++ paddle/utils/arch/osx/Locks.cpp | 112 ++++++++++++++++++++++++++ 6 files changed, 239 insertions(+), 205 deletions(-) delete mode 100644 paddle/utils/Locks.cpp create mode 100644 paddle/utils/arch/linux/Locks.cpp create mode 100644 paddle/utils/arch/osx/Locks.cpp diff --git a/.gitignore b/.gitignore index 00368ede67d3d..801c76325c92e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.DS_Store build/ +*.user diff --git a/paddle/utils/CMakeLists.txt b/paddle/utils/CMakeLists.txt index 3c08f1e3055f8..0557b01e36f07 100644 --- a/paddle/utils/CMakeLists.txt +++ b/paddle/utils/CMakeLists.txt @@ -2,12 +2,18 @@ file(GLOB UTIL_HEADERS . *.h) file(GLOB UTIL_SOURCES . *.cpp) - +if(APPLE) + file(GLOB UTIL_ARCH_SOURCES . arch/osx/*.cpp) +else() + file(GLOB UTIL_ARCH_SOURCES . arch/linux/*.cpp) +endif() add_library(paddle_utils STATIC - ${UTIL_SOURCES}) + ${UTIL_SOURCES} + ${UTIL_ARCH_SOURCES}) add_style_check_target(paddle_utils ${UTIL_HEADERS}) -add_style_check_target(paddle_utils ${UTIL_SOURCES}) +add_style_check_target(paddle_utils ${UTIL_SOURCES} + ${UTIL_ARCH_SOURCES}) add_dependencies(paddle_utils gen_proto_cpp) if(WITH_TESTING) add_subdirectory(tests) -endif() \ No newline at end of file +endif() diff --git a/paddle/utils/Locks.cpp b/paddle/utils/Locks.cpp deleted file mode 100644 index c2f58cf5764ef..0000000000000 --- a/paddle/utils/Locks.cpp +++ /dev/null @@ -1,106 +0,0 @@ -/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#ifdef __APPLE__ -#include -#endif - -#ifdef __APPLE__ -#ifndef PTHREAD_BARRIER_H_ -#define PTHREAD_BARRIER_H_ - -#include -#include - -typedef int pthread_barrierattr_t; -typedef struct { - pthread_mutex_t mutex; - pthread_cond_t cond; - int count; - int tripCount; -} pthread_barrier_t; - -int pthread_barrier_init(pthread_barrier_t *barrier, - const pthread_barrierattr_t *attr, unsigned int count) { - if (count == 0) { - errno = EINVAL; - return -1; - } - if (pthread_mutex_init(&barrier->mutex, 0) < 0) { - return -1; - } - if (pthread_cond_init(&barrier->cond, 0) < 0) { - pthread_mutex_destroy(&barrier->mutex); - return -1; - } - barrier->tripCount = count; - barrier->count = 0; - - return 0; -} - -int pthread_barrier_destroy(pthread_barrier_t *barrier) { - pthread_cond_destroy(&barrier->cond); - pthread_mutex_destroy(&barrier->mutex); - return 0; -} - -int pthread_barrier_wait(pthread_barrier_t *barrier) { - pthread_mutex_lock(&barrier->mutex); - ++(barrier->count); - if (barrier->count >= barrier->tripCount) { - barrier->count = 0; - pthread_cond_broadcast(&barrier->cond); - pthread_mutex_unlock(&barrier->mutex); - return 1; - } else { - pthread_cond_wait(&barrier->cond, &(barrier->mutex)); - pthread_mutex_unlock(&barrier->mutex); - return 0; - } -} - -#endif // PTHREAD_BARRIER_H_ - -typedef int pthread_spinlock_t; - -int pthread_spin_init(pthread_spinlock_t *lock, int pshared) { - __asm__ __volatile__("" ::: "memory"); - *lock = 0; - return 0; -} - -int pthread_spin_destroy(pthread_spinlock_t *lock) { - return 0; -} - -int pthread_spin_lock(pthread_spinlock_t *lock) { - while (1) { - int i; - for (i=0; i < 10000; i++) { - if (__sync_bool_compare_and_swap(lock, 0, 1)) { - return 0; - } - } - sched_yield(); - } -} - -int pthread_spin_unlock(pthread_spinlock_t *lock) { - __asm__ __volatile__("" ::: "memory"); - *lock = 0; - return 0; -} - -#endif // __APPLE__ diff --git a/paddle/utils/Locks.h b/paddle/utils/Locks.h index e7b0b77081f36..1fc0363d34597 100644 --- a/paddle/utils/Locks.h +++ b/paddle/utils/Locks.h @@ -16,56 +16,11 @@ limitations under the License. */ #pragma once #include -#include #include -#include - #include #include -#ifdef __APPLE__ -#include -#endif - -#ifdef __APPLE__ -#ifndef PTHREAD_BARRIER_H_ -#define PTHREAD_BARRIER_H_ - -#include -#include - -typedef int pthread_barrierattr_t; -typedef struct { - pthread_mutex_t mutex; - pthread_cond_t cond; - int count; - int tripCount; -} pthread_barrier_t; - - -extern int pthread_barrier_init(pthread_barrier_t *barrier, - const pthread_barrierattr_t *attr, - unsigned int count); - -extern int pthread_barrier_destroy(pthread_barrier_t *barrier); - -extern int pthread_barrier_wait(pthread_barrier_t *barrier); - -#endif // PTHREAD_BARRIER_H_ - -typedef int pthread_spinlock_t; - -extern int pthread_spin_init(pthread_spinlock_t *lock, int pshared); - -extern int pthread_spin_destroy(pthread_spinlock_t *lock); - -extern int pthread_spin_lock(pthread_spinlock_t *lock); - -extern int pthread_spin_unlock(pthread_spinlock_t *lock); - -#endif - - +#include "DisableCopy.h" namespace paddle { @@ -142,58 +97,44 @@ class ReadLockGuard { * which means it will keep trying to lock until lock on successfully. * The SpinLock disable copy. */ +class SpinLockPrivate; class SpinLock { public: - SpinLock() { pthread_spin_init(&lock_, 0); } - ~SpinLock() { pthread_spin_destroy(&lock_); } - SpinLock(const SpinLock&) = delete; - SpinLock& operator=(const SpinLock&) = delete; + DISABLE_COPY(SpinLock); + SpinLock(); + ~SpinLock(); // std::mutext interface - void lock() { pthread_spin_lock(&lock_); } - void unlock() { pthread_spin_unlock(&lock_); } + void lock(); + void unlock(); -protected: - pthread_spinlock_t lock_; - char padding_[64 - sizeof(pthread_spinlock_t)]; +private: + SpinLockPrivate* m; }; /** * A simple wapper of semaphore which can only be shared in the same process. */ - -#ifdef __APPLE__ - +class SemaphorePrivate; class Semaphore { public: - explicit Semaphore(int initValue = 0) { - sem_ = dispatch_semaphore_create(initValue); - } - - ~Semaphore() { dispatch_release(sem_); } - bool timeWait(struct timespec* ts) { - dispatch_time_t m = dispatch_walltime(ts, 0); - return (0 == dispatch_semaphore_wait(sem_, m)); - } - void wait() { dispatch_semaphore_wait(sem_, DISPATCH_TIME_FOREVER); } - void post() { dispatch_semaphore_signal(sem_);} - -protected: - dispatch_semaphore_t sem_; -}; + //! Disable copy & assign + Semaphore(const Semaphore& other) = delete; + Semaphore& operator= (const Semaphore&& other) = delete; -#else + //! Enable move. + Semaphore(Semaphore&& other): m(std::move(other.m)) { + } -class Semaphore { public: /** * @brief Construct Function. * @param[in] initValue the initial value of the * semaphore, default 0. */ - explicit Semaphore(int initValue = 0) { sem_init(&sem_, 0, initValue); } + explicit Semaphore(int initValue = 0); - ~Semaphore() { sem_destroy(&sem_); } + ~Semaphore(); /** * @brief The same as wait(), except if the decrement can not @@ -203,43 +144,38 @@ class Semaphore { * @return ture if the decrement proceeds before ts, * else return false. */ - bool timeWait(struct timespec* ts) { return (0 == sem_timedwait(&sem_, ts)); } + bool timeWait(struct timespec* ts); /** * @brief decrement the semaphore. If the semaphore's value is 0, then call blocks. */ - void wait() { sem_wait(&sem_); } + void wait(); /** * @brief increment the semaphore. If the semaphore's value * greater than 0, wake up a thread blocked in wait(). */ - void post() { sem_post(&sem_); } + void post(); -protected: - sem_t sem_; +private: + SemaphorePrivate* m; }; -#endif - -static_assert(sizeof(SpinLock) == 64, "Wrong padding"); - /** * A simple wrapper of thread barrier. * The ThreadBarrier disable copy. */ +class ThreadBarrierPrivate; class ThreadBarrier { public: + DISABLE_COPY(ThreadBarrier); + /** * @brief Construct Function. Initialize the barrier should * wait for count threads in wait(). */ - explicit ThreadBarrier(int count) { - pthread_barrier_init(&barrier_, NULL, count); - } - ~ThreadBarrier() { pthread_barrier_destroy(&barrier_); } - ThreadBarrier(const ThreadBarrier&) = delete; - ThreadBarrier& operator=(const ThreadBarrier&) = delete; + explicit ThreadBarrier(int count); + ~ThreadBarrier(); /** * @brief . @@ -247,10 +183,10 @@ class ThreadBarrier { * then wake up all the count - 1 threads and continue run together. * Else block the thread until waked by other thread . */ - void wait() { pthread_barrier_wait(&barrier_); } + void wait(); -protected: - pthread_barrier_t barrier_; +private: + ThreadBarrierPrivate* m; }; /** diff --git a/paddle/utils/arch/linux/Locks.cpp b/paddle/utils/arch/linux/Locks.cpp new file mode 100644 index 0000000000000..939a04dc0fc2c --- /dev/null +++ b/paddle/utils/arch/linux/Locks.cpp @@ -0,0 +1,85 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/util/Locks.h" +#include +#include + +namespace paddle { +class SemaphorePrivate { +public: + sem_t sem; +}; + +Semaphore::Semaphore(int initValue): m(new SemaphorePrivate()) { + sem_init(&m->sem, 0, initValue); +} + +Semaphore::~Semaphore() { + sem_destroy(&m->sem); +} + +bool Semaphore::timeWait(struct timespec* ts) { + return (0 == sem_timedwait(&m->sem, ts)); +} + +void Semaphore::wait() { + sem_wait(&m->sem); +} + +void Semaphore::post() { + sem_post(&m->sem); +} + + +class SpinLockPrivate { +public: + inline SpinLockPrivate() { pthread_spin_init(&lock_, 0); } + inline ~SpinLockPrivate() { pthread_spin_destroy(&lock_); } + pthread_spinlock_t lock_; + char padding_[64 - sizeof(pthread_spinlock_t)]; +}; + +SpinLock::SpinLock():m(new SpinLockPrivate()) {} + + +SpinLock::~SpinLock() { delete m; } + +void SpinLock::lock() { + pthread_spin_lock(&m->lock_); +} + +void SpinLock::unlock() { + pthread_spin_unlock(&m->lock_); +} + +class ThreadBarrierPrivate { +public: + pthread_barrier_t barrier_; +}; + +ThreadBarrier::ThreadBarrier(int count): m(new ThreadBarrierPrivate()) { + pthread_barrier_init(&m->barrier_, nullptr, count); +} + +ThreadBarrier::~ThreadBarrier() { + pthread_barrier_destroy(&m->barrier_); + delete m; +} + +void ThreadBarrier::wait() { + pthread_barrier_wait(&m->barrier_); +} + +} // namespace paddle \ No newline at end of file diff --git a/paddle/utils/arch/osx/Locks.cpp b/paddle/utils/arch/osx/Locks.cpp new file mode 100644 index 0000000000000..5e0411624fd60 --- /dev/null +++ b/paddle/utils/arch/osx/Locks.cpp @@ -0,0 +1,112 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/utils/Locks.h" +#include "paddle/utils/Logging.h" +#include +#include +namespace paddle { +class SemaphorePrivate { +public: + ~SemaphorePrivate() { + dispatch_release(sem); + } + + dispatch_semaphore_t sem; +}; + +Semaphore::Semaphore(int initValue): m(new SemaphorePrivate()) { + m->sem = dispatch_semaphore_create(initValue); +} + +Semaphore::~Semaphore() { + delete m; +} + +bool Semaphore::timeWait(timespec *ts) { + dispatch_time_t tm = dispatch_walltime(ts, 0); + return (0 == dispatch_semaphore_wait(m->sem, tm)); +} + +void Semaphore::wait() { + dispatch_semaphore_wait(m->sem, DISPATCH_TIME_FOREVER); +} + +void Semaphore::post() { + dispatch_semaphore_signal(m->sem); +} + +class SpinLockPrivate { +public: + SpinLockPrivate(): lock_(0) {} + + OSSpinLock lock_; + char padding_[64 - sizeof(OSSpinLock)]; // Padding to cache line size +}; + +SpinLock::SpinLock(): m(new SpinLockPrivate()) {} +SpinLock::~SpinLock() { delete m; } + +void SpinLock::lock() { + OSSpinLockLock(&m->lock_); +} + +void SpinLock::unlock() { + OSSpinLockUnlock(&m->lock_); +} + + +class ThreadBarrierPrivate { +public: + pthread_mutex_t mutex; + pthread_cond_t cond; + int count; + int tripCount; + + inline explicit ThreadBarrierPrivate(int cnt):count(0), tripCount(cnt) { + CHECK_NE(cnt, 0); + CHECK_GE(pthread_mutex_init(&mutex, 0), 0); + CHECK_GE(pthread_cond_init(&cond, 0), 0); + } + + inline ~ThreadBarrierPrivate() { + pthread_cond_destroy(&cond); + pthread_mutex_destroy(&mutex); + } + + /** + * @brief wait + * @return true if the last wait + */ + inline bool wait() { + pthread_mutex_lock(&mutex); + ++count; + if (count > tripCount) { + count = 0; + pthread_cond_broadcast(&cond); + pthread_mutex_unlock(&mutex); + return true; + } else { + pthread_cond_wait(&cond, &mutex); + pthread_mutex_unlock(&mutex); + return false; + } + } +}; + +ThreadBarrier::ThreadBarrier(int count): m(new ThreadBarrierPrivate(count)) {} +ThreadBarrier::~ThreadBarrier() { delete m; } +void ThreadBarrier::wait() { m->wait(); } + +} // namespace paddle From a238b11f835653a86341d26738c95035aebd271d Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 13 Sep 2016 11:14:58 +0800 Subject: [PATCH 067/324] Fix linux compile --- cmake/util.cmake | 5 +++-- paddle/math/Allocator.h | 2 +- paddle/math/tests/test_SIMDFunctions.cpp | 2 +- paddle/parameter/tests/test_common.cpp | 8 +++++--- paddle/utils/Stat.cpp | 2 +- paddle/utils/arch/linux/Locks.cpp | 6 +++--- 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/cmake/util.cmake b/cmake/util.cmake index a91e1d5643014..5f2f4a075cc57 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -104,9 +104,10 @@ function(link_paddle_exe TARGET_NAME) ${PROTOBUF_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${CBLAS_LIBS} - ${CMAKE_DL_LIBS} ${INTERAL_LIBS} - ${ZLIB_LIBRARIES}) + ${ZLIB_LIBRARIES} + ${CMAKE_DL_LIBS} + ) if(WITH_PYTHON) target_link_libraries(${TARGET_NAME} diff --git a/paddle/math/Allocator.h b/paddle/math/Allocator.h index ca8eadbc1aa42..f7aa60380f23e 100644 --- a/paddle/math/Allocator.h +++ b/paddle/math/Allocator.h @@ -49,7 +49,7 @@ class CpuAllocator : public Allocator { */ virtual void* alloc(size_t size) { void* ptr; - posix_memalign(&ptr, 32ul, size); + CHECK_EQ(posix_memalign(&ptr, 32ul, size), 0); CHECK(ptr) << "Fail to allocate CPU memory: size=" << size; return ptr; } diff --git a/paddle/math/tests/test_SIMDFunctions.cpp b/paddle/math/tests/test_SIMDFunctions.cpp index bae5d8c684d89..491b0cda7b9e1 100644 --- a/paddle/math/tests/test_SIMDFunctions.cpp +++ b/paddle/math/tests/test_SIMDFunctions.cpp @@ -38,7 +38,7 @@ static std::mt19937 RandomEngine(time(0)); inline static std::unique_ptr NewVector(size_t len = VECTOR_LEN, size_t align = ALIGN) { float* ptr; - posix_memalign((void**)&ptr, align, len * sizeof(float)); + CHECK_EQ(posix_memalign((void**)&ptr, align, len * sizeof(float)), 0); return std::unique_ptr(ptr); } diff --git a/paddle/parameter/tests/test_common.cpp b/paddle/parameter/tests/test_common.cpp index 4f92aec1d9671..1a22abf7cf801 100644 --- a/paddle/parameter/tests/test_common.cpp +++ b/paddle/parameter/tests/test_common.cpp @@ -125,9 +125,11 @@ TEST_F(CommonTest, sgdUpdate) { const size_t alignHeader[] = {0, 2, 3, 5, 7, 8}; for (auto& size : sizeVec_) { real *gradientBuffer, *valueBuffer, *momentumBuffer; - posix_memalign((void**)&gradientBuffer, 32, sizeof(real) * size); - posix_memalign((void**)&valueBuffer, 32, sizeof(real) * size); - posix_memalign((void**)&momentumBuffer, 32, sizeof(real) * size); + CHECK_EQ(posix_memalign((void**)&gradientBuffer, 32, sizeof(real) * size), + 0); + CHECK_EQ(posix_memalign((void**)&valueBuffer, 32, sizeof(real) * size), 0); + CHECK_EQ(posix_memalign((void**)&momentumBuffer, 32, sizeof(real) * size), + 0); for (size_t i = 0; i < size; i++) { gradientBuffer[i] = 1.0; diff --git a/paddle/utils/Stat.cpp b/paddle/utils/Stat.cpp index aab5446a98c0b..ff6e8ade2cd48 100644 --- a/paddle/utils/Stat.cpp +++ b/paddle/utils/Stat.cpp @@ -25,7 +25,7 @@ namespace paddle { pid_t getTID() { #if defined(__APPLE__) || defined(__OSX__) pid_t tid = syscall(SYS_thread_selfid); - #elif defined(__LINUX__) + #else #ifndef __NR_gettid #define __NR_gettid 224 #endif diff --git a/paddle/utils/arch/linux/Locks.cpp b/paddle/utils/arch/linux/Locks.cpp index 939a04dc0fc2c..347ae64c26dfd 100644 --- a/paddle/utils/arch/linux/Locks.cpp +++ b/paddle/utils/arch/linux/Locks.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "paddle/util/Locks.h" +#include "paddle/utils/Locks.h" #include #include @@ -35,7 +35,7 @@ bool Semaphore::timeWait(struct timespec* ts) { } void Semaphore::wait() { - sem_wait(&m->sem); + sem_wait(&m->sem); } void Semaphore::post() { @@ -82,4 +82,4 @@ void ThreadBarrier::wait() { pthread_barrier_wait(&m->barrier_); } -} // namespace paddle \ No newline at end of file +} // namespace paddle From 55a1a7588be825072299ec198a00970df98639d8 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 13 Sep 2016 14:23:38 +0800 Subject: [PATCH 068/324] Disable a unittest will use large memory by grep --- paddle/math/tests/test_perturbation.cpp | 5 ----- paddle/utils/tests/CMakeLists.txt | 8 +++++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/paddle/math/tests/test_perturbation.cpp b/paddle/math/tests/test_perturbation.cpp index 51e346fef91bf..4fa9bc72013da 100644 --- a/paddle/math/tests/test_perturbation.cpp +++ b/paddle/math/tests/test_perturbation.cpp @@ -249,9 +249,4 @@ TEST_F(PerturbationTest, scale_test) { } } -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - #endif diff --git a/paddle/utils/tests/CMakeLists.txt b/paddle/utils/tests/CMakeLists.txt index be59a785ecf36..5b31cd393dd1f 100644 --- a/paddle/utils/tests/CMakeLists.txt +++ b/paddle/utils/tests/CMakeLists.txt @@ -9,6 +9,8 @@ add_executable( test_CustomStackTracePrint.cpp ) link_paddle_exe(test_CustomStackTracePrint) -add_test(NAME test_CustomStackTracePrint - COMMAND ${PROJ_ROOT}/paddle/utils/tests/test_CustomStackTracePrint.sh - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) +if(NOT APPLE) + add_test(NAME test_CustomStackTracePrint + COMMAND ${PROJ_ROOT}/paddle/utils/tests/test_CustomStackTracePrint.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) +endif() From d6cc5203badafd0c18da9d7fe620a9bfc9c00a3e Mon Sep 17 00:00:00 2001 From: emailweixu Date: Tue, 13 Sep 2016 02:41:30 -0700 Subject: [PATCH 069/324] Fix LayerOutput.parents for some layers and mixed_layer for Operator (#69) --- python/paddle/trainer_config_helpers/layers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index b7e5f566bb8c3..bbd8891ce6952 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -171,6 +171,8 @@ def __init__(self, name, layer_type, parents=None, activation=None, assert LayerType.is_layer_type(layer_type) self.name = name self.layer_type = layer_type + if parents is not None and type(parents) != list: + parents = [parents] self.parents = [] if parents is None else parents self.activation = activation self.num_filters = num_filters @@ -512,7 +514,7 @@ def __add__(self, other): :rtype: MixedLayerType """ if not self.finalized: - assert isinstance(other, Projection) + assert isinstance(other, Projection) or isinstance(other, Operator) self.inputs.append(other) self.parents.append(other.origin) return self From 7dbc092c875b67cbfc91941cf240500c083ea459 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Wed, 14 Sep 2016 01:53:50 +0800 Subject: [PATCH 070/324] fix cudnn version number for batch norm. (#71) * fix CUDNN_VERSION for backward of CudnnBatchNormLayer * fix cudnn version number for BatchNorm --- paddle/cuda/src/hl_cuda_cudnn.cc | 16 +++++++--------- paddle/gserver/layers/CudnnBatchNormLayer.cpp | 18 ------------------ python/paddle/trainer/config_parser.py | 2 +- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/paddle/cuda/src/hl_cuda_cudnn.cc b/paddle/cuda/src/hl_cuda_cudnn.cc index 19c94b2453981..c2dce1977bdf5 100644 --- a/paddle/cuda/src/hl_cuda_cudnn.cc +++ b/paddle/cuda/src/hl_cuda_cudnn.cc @@ -150,7 +150,7 @@ CUDNN_DNN_ROUTINE_EACH_AFTER_R3(DYNAMIC_LOAD_CUDNN_WRAP) // APIs available after R4: -#if CUDNN_VERSION >= 4000 +#if CUDNN_VERSION >= 4007 #define CUDNN_DNN_ROUTINE_EACH_AFTER_R4(__macro) \ __macro(cudnnBatchNormalizationForwardTraining) \ __macro(cudnnBatchNormalizationForwardInference) \ @@ -999,7 +999,7 @@ void hl_batch_norm_forward_training(hl_tensor_descriptor inputDesc, double epsilon, real *savedMean, real *savedVar) { -#if CUDNN_VERSION >= 4000 +#if CUDNN_VERSION >= 4007 if ((NULL != runningMean && NULL == runningInvVar) || (NULL == runningMean && NULL != runningInvVar)) { LOG(FATAL) << "runningMean and runningInvVar can be NULL " @@ -1024,7 +1024,7 @@ void hl_batch_norm_forward_training(hl_tensor_descriptor inputDesc, CHECK_SYNC("hl_batch_norm_forward_training failed"); #else - LOG(FATAL) << "CudnnBatchNorm requires cudnn version >= 4000. " + LOG(FATAL) << "CudnnBatchNorm requires cudnn version >= 4007. " << "But cudnn lib version is " << g_cudnn_lib_version; #endif } @@ -1039,7 +1039,7 @@ void hl_batch_norm_forward_inference(hl_tensor_descriptor inputDesc, real *estimatedMean, real *estimatedInvVar, double epsilon) { -#if CUDNN_VERSION >= 4000 +#if CUDNN_VERSION >= 4007 cudnnTensorDescriptor_t xDesc = GET_TENSOR_DESCRIPTOR(inputDesc); cudnnTensorDescriptor_t yDesc = GET_TENSOR_DESCRIPTOR(outputDesc); cudnnTensorDescriptor_t bnDesc = GET_TENSOR_DESCRIPTOR(bnParamDesc); @@ -1053,7 +1053,7 @@ void hl_batch_norm_forward_inference(hl_tensor_descriptor inputDesc, CHECK_SYNC("hl_batch_norm_forward_inference failed"); #else - LOG(FATAL) << "CudnnBatchNorm requires cudnn version >= 4000. " + LOG(FATAL) << "CudnnBatchNorm requires cudnn version >= 4007. " << "But cudnn lib version is " << g_cudnn_lib_version; #endif } @@ -1071,7 +1071,7 @@ void hl_batch_norm_backward(hl_tensor_descriptor inputDesc, double epsilon, real *savedMean, real *savedInvVar) { -#if CUDNN_VERSION >= 4000 +#if CUDNN_VERSION >= 4007 if ((NULL != savedMean && NULL == savedInvVar) || (NULL == savedMean && NULL != savedInvVar)) { LOG(FATAL) << "savedMean and savedVar can be NULL " @@ -1087,16 +1087,14 @@ void hl_batch_norm_backward(hl_tensor_descriptor inputDesc, cudnnBatchNormMode_t mode = CUDNN_BATCHNORM_SPATIAL; CHECK_CUDNN(dynload::cudnnBatchNormalizationBackward( t_resource.cudnn_handle, mode, &alpha, &beta, -#if CUDNN_VERSION >= 5000 &alpha, &beta, -#endif xDesc, input, dyDesc, outGrad, dxDesc, inGrad, bnDesc, scale, scaleGrad, biasGrad, epsilon, savedMean, savedInvVar)); CHECK_SYNC("hl_batch_norm_backward failed"); #else - LOG(FATAL) << "CudnnBatchNorm requires cudnn version >= 4000. " + LOG(FATAL) << "CudnnBatchNorm requires cudnn version >= 4007. " << "But cudnn lib version is " << g_cudnn_lib_version; #endif } diff --git a/paddle/gserver/layers/CudnnBatchNormLayer.cpp b/paddle/gserver/layers/CudnnBatchNormLayer.cpp index cef8772fc254f..3c6d13b0bf92e 100644 --- a/paddle/gserver/layers/CudnnBatchNormLayer.cpp +++ b/paddle/gserver/layers/CudnnBatchNormLayer.cpp @@ -115,29 +115,11 @@ void CudnnBatchNormLayer::backward(const UpdateCallback& callback) { create(tmpBiasGrad_, 1, channels_, &betaGrad); } - // because of the different api of cudnn v4 and v5. - if (hl_get_cudnn_lib_version() < 5000) { - if (weight_->getWGrad()) { - create(tmpWGrad_, 1, channels_, &gammaGrad); - } - if (biases_ && biases_->getWGrad()) { - create(tmpBiasGrad_, 1, channels_, &betaGrad); - } - } - hl_batch_norm_backward(ioDesc_, input, ioDesc_, outGrad, ioDesc_, inGrad, bnParamDesc_, gamma, gammaGrad, betaGrad, EPS, savedMean, savedInvVar); - // because of the different api of cudnn v4 and v5. - if (hl_get_cudnn_lib_version() < 5000) { - if (weight_->getWGrad() && biases_->getWGrad()) { - weight_->getWGrad()->add(*tmpWGrad_); - biases_->getWGrad()->add(*tmpBiasGrad_); - } - } - { REGISTER_TIMER_INFO("WeightUpdate", getName().c_str()); biases_->getParameterPtr()->incUpdate(callback); diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index b26a63e7f3c1d..3656d9e7d8242 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1614,7 +1614,7 @@ def __init__( # Also based on cudnn version. use_cudnn = use_gpu and batch_norm_type != "batch_norm" and \ ((not parallel_nn) or self.config.device > -1) and \ - cudnn_version >= 4000 + cudnn_version >= 4007 self.layer_type = "cudnn_batch_norm" if use_cudnn else "batch_norm" super(BatchNormLayer, self).__init__(name, self.layer_type, 0, active_type=active_type, From 1df0c7b1495c6cf2c13b7e8d3b2fc965fdf08748 Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 14 Sep 2016 09:46:58 +0800 Subject: [PATCH 071/324] avoid link failed on unit test for mac os --- paddle/math/tests/test_CpuGpuVector.cpp | 6 ++++++ paddle/math/tests/test_matrixCompare.cpp | 5 +++++ paddle/math/tests/test_perturbation.cpp | 6 ++++++ paddle/math/tests/test_sparseMatrixCompare.cpp | 6 ++++++ 4 files changed, 23 insertions(+) diff --git a/paddle/math/tests/test_CpuGpuVector.cpp b/paddle/math/tests/test_CpuGpuVector.cpp index 7b50b020cda93..61b424e3c6647 100644 --- a/paddle/math/tests/test_CpuGpuVector.cpp +++ b/paddle/math/tests/test_CpuGpuVector.cpp @@ -84,4 +84,10 @@ int main(int argc, char** argv) { return ret; } +#else + +int main(int argc, char const* argv[]) { + return 0; +} + #endif diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index fe8eacc2efbc5..ac50e7b7499d8 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -1851,5 +1851,10 @@ int main(int argc, char** argv) { initMain(argc, argv); return RUN_ALL_TESTS(); } +#else + +int main(int argc, char const* argv[]) { + return 0; +} #endif diff --git a/paddle/math/tests/test_perturbation.cpp b/paddle/math/tests/test_perturbation.cpp index 51e346fef91bf..050f2ca9ced80 100644 --- a/paddle/math/tests/test_perturbation.cpp +++ b/paddle/math/tests/test_perturbation.cpp @@ -254,4 +254,10 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } +#else + +int main(int argc, char const* argv[]) { + return 0; +} + #endif diff --git a/paddle/math/tests/test_sparseMatrixCompare.cpp b/paddle/math/tests/test_sparseMatrixCompare.cpp index 6048dd8112229..b3467e4982e24 100644 --- a/paddle/math/tests/test_sparseMatrixCompare.cpp +++ b/paddle/math/tests/test_sparseMatrixCompare.cpp @@ -178,4 +178,10 @@ int main(int argc, char** argv) { return ret; } +#else + +int main(int argc, char const* argv[]) { + return 0; +} + #endif From 7481429f99f175c0749ea84712f78dafbaf03c6e Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Tue, 13 Sep 2016 17:57:33 -0700 Subject: [PATCH 072/324] Change cos_sim to use CosSimLayer layer when size=1 and rename convex_comb_layer to linear_comb_layer Also add an unittest for layers.py, currently for a few things. More need to be added later. --- doc/ui/api/trainer_config_helpers/layers.rst | 18 ++-- .../gserver/layers/ConvexCombinationLayer.cpp | 16 ++-- paddle/gserver/layers/CosSimLayer.cpp | 4 +- paddle/gserver/layers/CosSimLayer.h | 4 +- python/CMakeLists.txt | 2 + python/paddle/trainer/config_parser.py | 8 ++ .../paddle/trainer_config_helpers/layers.py | 94 ++++++++++++------- .../tests/CMakeLists.txt | 5 + .../tests/layers_test.py | 19 ++++ .../tests/layers_test_config.py | 43 +++++++++ 10 files changed, 158 insertions(+), 55 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/CMakeLists.txt create mode 100644 python/paddle/trainer_config_helpers/tests/layers_test.py create mode 100644 python/paddle/trainer_config_helpers/tests/layers_test_config.py diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index 1583fce981fed..f902d1c995bc5 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -245,10 +245,10 @@ addto_layer :members: addto_layer :noindex: -convex_comb_layer +linear_comb_layer ----------------- .. automodule:: paddle.trainer_config_helpers.layers - :members: convex_comb_layer + :members: linear_comb_layer :noindex: interpolation_layer @@ -280,7 +280,13 @@ tensor_layer .. automodule:: paddle.trainer_config_helpers.layers :members: tensor_layer :noindex: - + +cos_sim +------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: cos_sim + :noindex: + trans_layer ------------ .. automodule:: paddle.trainer_config_helpers.layers @@ -341,12 +347,6 @@ rank_cost :members: rank_cost :noindex: -cos_sim -------- -.. automodule:: paddle.trainer_config_helpers.layers - :members: cos_sim - :noindex: - crf_layer ----------------- .. automodule:: paddle.trainer_config_helpers.layers diff --git a/paddle/gserver/layers/ConvexCombinationLayer.cpp b/paddle/gserver/layers/ConvexCombinationLayer.cpp index e092b2e390f37..a81cf939af671 100644 --- a/paddle/gserver/layers/ConvexCombinationLayer.cpp +++ b/paddle/gserver/layers/ConvexCombinationLayer.cpp @@ -21,18 +21,20 @@ limitations under the License. */ namespace paddle { /** - * @brief A layer for convex weighted average of vectors, + * @brief A layer for weighted sum of vectors, * which is used in NEURAL MACHINE TRANSLATION BY JOINTLY LEARNING TO ALIGN AND * TRANSLATE - * - Input: the first input contains the convex weights (batchSize x weightDim), - * and the shape of second input is (batchSize x (weightdim*dataDim)). - * - Output: the shape of output is (batchSize x dataDim). + * - Input: the the size of the first input is weightDim, + * and the size of the second input is weightdim * dataDim. + * - Output: the sizeof the output is dataDim * \f[ - * out[i][j] = \sum_{j}(in0(i, j) * in1(i,j + i * dataDim)), - * i = 0,1,...,(batchSize-1); j = 0, 1,...,(dataDim-1) + * out(j) = \sum_{i}(in0(i) * in1(i,j + i * dataDim)), + * i = 0,1,...,(weightDim-1); j = 0, 1,...,(dataDim-1) * \f] + * Note that the above computation is for one sample. Multiple samples are + * processed in one batch. * - * The config file api is convex_comb_layer. + * The config file api is linear_comb_layer. */ class ConvexCombinationLayer : public Layer { protected: diff --git a/paddle/gserver/layers/CosSimLayer.cpp b/paddle/gserver/layers/CosSimLayer.cpp index b10bd1d886ecf..05a70aeff5e8f 100644 --- a/paddle/gserver/layers/CosSimLayer.cpp +++ b/paddle/gserver/layers/CosSimLayer.cpp @@ -48,7 +48,7 @@ void CosSimLayer::forward(PassType passType) { REGISTER_TIMER_INFO("CosFwAtvTimer", getName().c_str()); MatrixPtr prevOut1 = getInputValue(0); MatrixPtr prevOut2 = getInputValue(1); - outV->cosSim(*prevOut1, *prevOut2, kCosSimScale_); + outV->cosSim(*prevOut1, *prevOut2, config_.cos_scale()); } } @@ -59,7 +59,7 @@ void CosSimLayer::backward(const UpdateCallback& callback) { outG->cosSimDerivative(*this->getOutputValue(), *getInputValue(0), *getInputValue(1), *getInputGrad(0), - *getInputGrad(1), kCosSimScale_); + *getInputGrad(1), config_.cos_scale()); } } diff --git a/paddle/gserver/layers/CosSimLayer.h b/paddle/gserver/layers/CosSimLayer.h index 9b0e53335b250..65eb807ab2e6f 100644 --- a/paddle/gserver/layers/CosSimLayer.h +++ b/paddle/gserver/layers/CosSimLayer.h @@ -36,7 +36,7 @@ namespace paddle { class CosSimLayer : public Layer { public: explicit CosSimLayer(const LayerConfig& config) - : Layer(config), kCosSimScale_(5.0f) {} + : Layer(config) {} ~CosSimLayer() {} @@ -44,8 +44,6 @@ class CosSimLayer : public Layer { void forward(PassType passType); void backward(const UpdateCallback& callback = nullptr); - - const real kCosSimScale_; }; } // namespace paddle diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 68cc402470410..fd9a003bb018c 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -22,6 +22,8 @@ find_python_module(pip REQUIRED) find_python_module(wheel REQUIRED) find_python_module(google.protobuf REQUIRED) +add_subdirectory(paddle/trainer_config_helpers/tests) + install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dist/ DESTINATION opt/paddle/share/wheels ) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 3656d9e7d8242..5b60cf8410e0c 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2264,6 +2264,9 @@ def __init__( name, 'convex_comb', size, inputs=inputs, device=device) config_assert(len(self.inputs) == 2, 'ConvexCombinationLayer must have 2 inputs') + config_assert( + size * self.get_input_layer(0).size == self.get_input_layer(1).size, + 'Wrong input size for ConvexCombinationLayer') self.set_layer_size(size) @config_layer('interpolation') @@ -2313,6 +2316,9 @@ def __init__( self.config.cos_scale = cos_scale config_assert(len(self.inputs) == 2, 'CosSimVecMatLayer must have 2 inputs') + config_assert( + size * self.get_input_layer(0).size == self.get_input_layer(1).size, + 'Wrong input size for CosSimVecMatLayer') @config_layer('sampling_id') class SamplingIdLayer(LayerBase): @@ -2361,6 +2367,7 @@ def __init__( self, name, inputs, + cos_scale=5, device=None): super(CosSimLayer, self).__init__( name, 'cos', 1, inputs=inputs, device=device) @@ -2368,6 +2375,7 @@ def __init__( config_assert( self.get_input_layer(0).size == self.get_input_layer(1).size, 'inputs of CosSimLayer must have same dim') + self.config.cos_scale = cos_scale @config_layer('tensor') diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index bbd8891ce6952..f3f0077f9798f 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -47,6 +47,7 @@ 'BaseGeneratedInput', 'conv_operator', 'conv_shift_layer', 'tensor_layer', 'selective_fc_layer', 'sampling_id_layer', 'slope_intercept_layer', 'trans_full_matrix_projection', + 'linear_comb_layer', 'convex_comb_layer', 'ctc_layer', 'crf_layer', 'crf_decoding_layer', 'cross_entropy_with_selfnorm', 'cross_entropy', 'multi_binary_label_cross_entropy', @@ -70,7 +71,8 @@ class LayerType(object): POOLING_AVG = 'average' FC_LAYER = "fc" COST = 'cost' - COSINE_SIM = 'cos_vm' + COSINE_SIM_VEC = 'cos_vm' + COSINE_SIM = 'cos' HSIGMOID = 'hsigmoid' CONV_LAYER = "conv" POOL_LAYER = "pool" @@ -102,7 +104,7 @@ class LayerType(object): SEL_FC_LAYER = "selective_fc" SAMPLING_ID_LAYER = "sampling_id" SLOPE_INTERCEPT_LAYER = "slope_intercept" - CONVEX_COMBINATION_LAYER = "convex_comb" + LINEAR_COMBINATION_LAYER = "convex_comb" BLOCK_EXPAND = "blockexpand" CTC_LAYER = "ctc" @@ -1171,13 +1173,16 @@ def power_layer(input, weight, name=None, layer_attr=None): @layer_support() def scaling_layer(input, weight, name=None, layer_attr=None): """ - A layer for each row of a matrix, multiplying with a element of a vector. + A layer for multiplying input vector by weight scalar. .. math:: - y.row[i] = w[i] * x.row[i] + y = w x - where :math:`x` is (batchSize x dataDim) input, :math:`w` is - (batchSize x 1) weight vector, and :math:`y` is (batchSize x dataDim) output. + where :math:`x` is size=dataDim input, :math:`w` is size=1 weight, + and :math:`y` is size=dataDim output. + + Note that the above computation is for one sample. Multiple samples are + processed in one batch. The example usage is: @@ -1251,11 +1256,14 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): .. math:: similarity = cos(\\theta) = {\\mathbf{a} \\cdot \\mathbf{b} - \\over \\|\\mathbf{b}\\| \\|\\mathbf{b}\\|} + \\over \\|\\mathbf{a}\\| \\|\\mathbf{b}\\|} + + The size of a is M, size of b is M*N, + Similarity will be calculated N times by step M. The output size is + N. The scale will be multiplied to similarity. - And the input dimension is :math:`a \in R^M`, :math:`b \in R^{MN}`. The - similarity will be calculated N times by step M. The output dimension is - :math:`R^N`. The scale will be multiplied to similarity. + Note that the above computation is for one sample. Multiple samples are + processed in one batch. :param name: layer name :type name: basestring @@ -1272,14 +1280,23 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - Layer( - name=name, - type=LayerType.COSINE_SIM, - size=size, - cos_scale=scale, - inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + if size == 1: + Layer( + name=name, + type=LayerType.COSINE_SIM, + cos_scale=scale, + inputs=[a.name, b.name], + **ExtraLayerAttribute.to_kwargs(layer_attr) + ) + else: + Layer( + name=name, + type=LayerType.COSINE_SIM_VEC, + size=size, + cos_scale=scale, + inputs=[a.name, b.name], + **ExtraLayerAttribute.to_kwargs(layer_attr) + ) return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b]) @wrap_name_default() @@ -2911,29 +2928,37 @@ def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0): @wrap_name_default() -def convex_comb_layer(input, size, name=None): +def linear_comb_layer(weights, vectors, size, name=None): """ - A layer for convex weighted average of vectors takes two inputs. - - Input: a vector containing the convex weights (batchSize x weightdim), - and a matrix in a vector form (batchSize x (weightdim * datadim)). - - Output: a vector (batchSize * datadim). + A layer for weighted sum of vectors takes two inputs. + - Input: size of weights is M + size of vectors is M*N + - Output: a vector of size=N .. math:: - y[i][j] = \sum_{j}(x_{1}(i, j) * x_{2}(i,j + i * dataDim)), + z(i) = \sum_{j=0}^{M-1} x(j) y(i+Nj) + where :math:`0 \le i \le N-1` + + Or in the matrix notation: + + .. math:: - i = 0,1,...,(batchSize-1); j = 0, 1,...,(dataDim-1) + z = x^T Y In this formular: - - :math:`x_{1}`: the first input. - - :math:`x_{2}`: the second input. - - :math:`y`: the output. + - :math:`x`: weights + - :math:`y`: vectors. + - :math:`z`: the output. + + Note that the above computation is for one sample. Multiple samples are + processed in one batch. The simple usage is: .. code-block:: python - convex_comb = convex_comb_layer(input=inputs, + linear_comb = linear_comb_layer(weighs=weight, vectors=vectors, size=elem_dim) :param input: The input layers. @@ -2946,15 +2971,16 @@ def convex_comb_layer(input, size, name=None): :rtype: LayerOutput """ - assert isinstance(input, list) or isinstance(input, tuple) - assert len(input) == 2 Layer( name=name, - type=LayerType.CONVEX_COMBINATION_LAYER, + type=LayerType.LINEAR_COMBINATION_LAYER, size=size, - inputs=[Input(input[0].name), Input(input[1].name)], + inputs=[Input(weights.name), Input(vectors.name)], ) - return LayerOutput(name, LayerType.CONVEX_COMBINATION_LAYER, input, size=size) + return LayerOutput(name, LayerType.LINEAR_COMBINATION_LAYER, + [weights, vectors], size=size) + +convex_comb_layer = linear_comb_layer @wrap_name_default() def block_expand_layer(input, diff --git a/python/paddle/trainer_config_helpers/tests/CMakeLists.txt b/python/paddle/trainer_config_helpers/tests/CMakeLists.txt new file mode 100644 index 0000000000000..611fb855a8c9a --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/CMakeLists.txt @@ -0,0 +1,5 @@ +#################### test_config_parser ######################### +add_test(NAME layers_test + COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ + python ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/layers_test.py + WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) diff --git a/python/paddle/trainer_config_helpers/tests/layers_test.py b/python/paddle/trainer_config_helpers/tests/layers_test.py new file mode 100644 index 0000000000000..3b55667354750 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/layers_test.py @@ -0,0 +1,19 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer.config_parser import parse_config_and_serialize + +if __name__ == '__main__': + parse_config_and_serialize( + 'trainer_config_helpers/tests/layers_test_config.py', '') diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py new file mode 100644 index 0000000000000..ec171fc6013f4 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -0,0 +1,43 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +num_classes = 5 + +x = data_layer(name="input1", size=3) +y = data_layer(name="input2", size=5) + +x1 = fc_layer(input=x, size=5) +y1 = fc_layer(input=y, size=5) +y2 = fc_layer(input=y, size=15) + +cos1 = cos_sim(a=x1, b=y1) +cos3 = cos_sim(a=x1, b=y2, size=3) + +linear_comb = linear_comb_layer(weights=x1, vectors=y2, size=3) + +out = fc_layer(input=[cos1, cos3, linear_comb], + size=num_classes, + act=SoftmaxActivation()) + +outputs(classification_cost(out, data_layer(name="label", size=num_classes))) + +settings( + batch_size=10, + learning_rate=2e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(8e-4), + gradient_clipping_threshold=25 +) From 42a117913563b1f107093402857d7849e2caed47 Mon Sep 17 00:00:00 2001 From: liuyuan04 Date: Fri, 9 Sep 2016 17:48:23 +0800 Subject: [PATCH 073/324] Fix internal documents. --- doc/build/index.rst | 1 + doc/cluster/index.rst | 1 + doc_cn/build_and_install/index.rst | 4 ++++ doc_cn/cluster/index.rst | 11 +++++++++++ doc_cn/index.rst | 2 +- 5 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 doc_cn/cluster/index.rst diff --git a/doc/build/index.rst b/doc/build/index.rst index 2b983dceb2777..d6d0d19e110fc 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -9,6 +9,7 @@ Install PaddlePaddle :glob: install_* + internal/install_from_jumbo.md Build from Source ----------------- diff --git a/doc/cluster/index.rst b/doc/cluster/index.rst index cf1ea97715402..9062f85f98d29 100644 --- a/doc/cluster/index.rst +++ b/doc/cluster/index.rst @@ -5,3 +5,4 @@ Cluster Train :glob: opensource/cluster_train.md + internal/index.md diff --git a/doc_cn/build_and_install/index.rst b/doc_cn/build_and_install/index.rst index e9182903c5f62..e21fc98c63dcd 100644 --- a/doc_cn/build_and_install/index.rst +++ b/doc_cn/build_and_install/index.rst @@ -9,7 +9,11 @@ Note: The intallation packages are still in pre-release state and your experienc .. toctree:: :maxdepth: 1 + :glob: + 源码下载(对内) <../build/internal/download_paddle_source_zh_cn.rst> + 使用Jumbo安装(对内) <../build/internal/install_from_jumbo.rst> + 从源码编译安装(对内) <../build/internal/build_from_source_zh_cn.rst> install/docker_install.rst install/ubuntu_install.rst cmake/index.rst diff --git a/doc_cn/cluster/index.rst b/doc_cn/cluster/index.rst new file mode 100644 index 0000000000000..25313a9635bbf --- /dev/null +++ b/doc_cn/cluster/index.rst @@ -0,0 +1,11 @@ +集群训练 +======== + +* `集群训练 <../../doc/cluster/index.html>`_ + +.. toctree:: + :maxdepth: 2 + :glob: + + 集群训练(对内) + diff --git a/doc_cn/index.rst b/doc_cn/index.rst index 5f06463899f6b..6cf5588b5b34f 100644 --- a/doc_cn/index.rst +++ b/doc_cn/index.rst @@ -8,7 +8,7 @@ PaddlePaddle文档 * `用户接口 `_ * `使用示例 `_ * `模型配置 <../doc/ui/api/trainer_config_helpers/index.html>`_ -* `集群训练 <../doc/cluster/index.html>`_ +* `集群训练 `_ 开发指南 -------- From 688eeefab3f23bc709c367bb51684b1a36b49f65 Mon Sep 17 00:00:00 2001 From: Haonan Date: Tue, 6 Sep 2016 16:31:30 -0700 Subject: [PATCH 074/324] fixed issues with synchronizing streams when copy from gpu to cpu * by default, synchronize default_stream after resizeAndCopyFrom * add sync in some places after resizeAndCopyFrom using other streams --- .../gserver/evaluators/CTCErrorEvaluator.cpp | 1 - .../gradientmachines/MultiGradientMachine.cpp | 6 +---- paddle/gserver/layers/CTCLayer.cpp | 1 + paddle/gserver/layers/CostLayer.cpp | 1 + paddle/gserver/layers/SamplingIdLayer.cpp | 1 + paddle/gserver/tests/LayerGradUtil.cpp | 2 -- paddle/gserver/tests/test_RecurrentLayer.cpp | 1 - paddle/math/Matrix.cpp | 2 ++ paddle/math/Vector.cpp | 1 + paddle/parameter/Argument.cpp | 27 +++++++++++-------- paddle/parameter/Argument.h | 11 +++++--- 11 files changed, 30 insertions(+), 24 deletions(-) diff --git a/paddle/gserver/evaluators/CTCErrorEvaluator.cpp b/paddle/gserver/evaluators/CTCErrorEvaluator.cpp index d0b1c0447d23d..a0c68fc9c29c4 100644 --- a/paddle/gserver/evaluators/CTCErrorEvaluator.cpp +++ b/paddle/gserver/evaluators/CTCErrorEvaluator.cpp @@ -196,7 +196,6 @@ class CTCErrorEvaluator : public Evaluator { Argument output, label; output.resizeAndCopyFrom(arguments[0], false); label.resizeAndCopyFrom(arguments[1], false); - hl_stream_synchronize(HPPL_STREAM_DEFAULT); CHECK(label.sequenceStartPositions); CHECK(label.ids); size_t numSequences = label.sequenceStartPositions->getSize() - 1; diff --git a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp index 787ce703a08ae..74a743145da8b 100644 --- a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp @@ -878,11 +878,7 @@ void TrainerThread::copyOutputGrad() { outArgs_.resize(outputGradArgs.size()); for (size_t i = 0; i < outputGradArgs.size(); i++) { outArgs_[i].resizeAndCopyFrom(outputGradArgs[i], startSeq, copySize, - multiMachine_->useGpu(), - HPPL_STREAM_DEFAULT); - } - if (multiMachine_->useGpu()) { - hl_stream_synchronize(HPPL_STREAM_DEFAULT); + multiMachine_->useGpu()); } gradientMachine_->setOutputGrad(outArgs_); } diff --git a/paddle/gserver/layers/CTCLayer.cpp b/paddle/gserver/layers/CTCLayer.cpp index db1450694ecf7..aa9c0d8a4b68f 100644 --- a/paddle/gserver/layers/CTCLayer.cpp +++ b/paddle/gserver/layers/CTCLayer.cpp @@ -51,6 +51,7 @@ void CTCLayer::forward(PassType passType) { for (size_t i = 0; i < inputLayers_.size(); i++) { tmpCpuInput_[i].resizeAndCopyFrom(getInput(i), false, HPPL_STREAM_1); } + hl_stream_synchronize(HPPL_STREAM_1); forwardImp(tmpCpuInput_[0], tmpCpuInput_[1]); } else { forwardImp(getInput(0), getInput(1)); diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index f353afabb3b71..b778f3e9b064f 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -511,6 +511,7 @@ void HuberTwoClass::forwardImp(Matrix &output, Argument &label, for (size_t i = 0; i < inputLayers_.size(); i++) { tmpCpuInput_[i].resizeAndCopyFrom(getInput(i), false, HPPL_STREAM_1); } + hl_stream_synchronize(HPPL_STREAM_1); } forwardImpIn(output, label, cost); } diff --git a/paddle/gserver/layers/SamplingIdLayer.cpp b/paddle/gserver/layers/SamplingIdLayer.cpp index 41c1461967ae1..cbc85b946f079 100644 --- a/paddle/gserver/layers/SamplingIdLayer.cpp +++ b/paddle/gserver/layers/SamplingIdLayer.cpp @@ -54,6 +54,7 @@ class SamplingIdLayer : public Layer { for (size_t i = 0; i < inputLayers_.size(); i++) { tmpCpuInput_[i].resizeAndCopyFrom(getInput(i), false, HPPL_STREAM_1); } + hl_stream_synchronize(HPPL_STREAM_1); forwardImp(tmpCpuInput_[0]); } else { forwardImp(getInput(0)); diff --git a/paddle/gserver/tests/LayerGradUtil.cpp b/paddle/gserver/tests/LayerGradUtil.cpp index f72011ae16cb3..552a6c5b41c7f 100644 --- a/paddle/gserver/tests/LayerGradUtil.cpp +++ b/paddle/gserver/tests/LayerGradUtil.cpp @@ -92,7 +92,6 @@ void testState(LayerPtr testLayer, vector& dataLayers, testLayer->forward(PASS_TEST); Argument out; out.resizeAndCopyFrom(testLayer->getOutput(), /* useGpu= */ false); - hl_stream_synchronize(HPPL_STREAM_DEFAULT); if (batchOut.value) { size_t dim = batchOut.value->getWidth(); ASSERT_TRUE((bool)out.value); @@ -220,7 +219,6 @@ void testBatchState(LayerPtr testLayer, vector& dataLayers, testLayer->forward(PASS_TEST); Argument out; out.resizeAndCopyFrom(testLayer->getOutput(), /* useGpu= */ false); - hl_stream_synchronize(HPPL_STREAM_DEFAULT); if (batchOut.value) { size_t dim = batchOut.value->getWidth(); ASSERT_TRUE((bool)out.value); diff --git a/paddle/gserver/tests/test_RecurrentLayer.cpp b/paddle/gserver/tests/test_RecurrentLayer.cpp index 2cea190b85949..9b933b153d158 100644 --- a/paddle/gserver/tests/test_RecurrentLayer.cpp +++ b/paddle/gserver/tests/test_RecurrentLayer.cpp @@ -299,7 +299,6 @@ void checkRecurrentLayer(LayerConfig layerConfig, size_t batchSize, Argument& cpuInput = testCpu.dataLayer_->getOutput(); Argument& gpuInput = testGpu.dataLayer_->getOutput(); gpuInput.resizeAndCopyFrom(cpuInput, true); - hl_stream_synchronize(HPPL_STREAM_DEFAULT); const VectorPtr& cpuVec = testCpu.para_->getBuf(PARAMETER_VALUE); const VectorPtr& gpuVec = testGpu.para_->getBuf(PARAMETER_VALUE); diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index f3a6503d4a21f..1b7f9ac5dac16 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -146,6 +146,7 @@ void Matrix::resizeOrCreate(MatrixPtr& matrix, size_t height, size_t width, if (!matrix) { matrix = Matrix::create(height, width, trans, useGpu); } else { + CHECK_EQ(matrix->useGpu(), useGpu); matrix->resize(height, width); } } @@ -161,6 +162,7 @@ void Matrix::resizeOrCreateSparseMatrix(MatrixPtr& matrix, size_t height, } else { CHECK(dynamic_cast(matrix.get()) || dynamic_cast(matrix.get())); + CHECK_EQ(matrix->useGpu(), useGpu); matrix->resize(height, width, nnz, valueType, format); } } diff --git a/paddle/math/Vector.cpp b/paddle/math/Vector.cpp index b1a459b86aa4f..7553ea25e09d2 100644 --- a/paddle/math/Vector.cpp +++ b/paddle/math/Vector.cpp @@ -800,6 +800,7 @@ void CpuGpuVectorT::resizeOrCreate(size_t size, bool useGpu) { } else if ((!useGpu) && (!cpuVectorT_)) { cpuVectorT_ = VectorT::create(size, false); } else { + CHECK((useGpu && gpuVectorT_) || (!useGpu && cpuVectorT_)); this->resize(size, useGpu); } } diff --git a/paddle/parameter/Argument.cpp b/paddle/parameter/Argument.cpp index 8610a66452358..0e4d676c899ae 100644 --- a/paddle/parameter/Argument.cpp +++ b/paddle/parameter/Argument.cpp @@ -22,11 +22,8 @@ namespace paddle { static void resizeAndCopy(MatrixPtr& dest, const MatrixPtr& src, bool useGpu, hl_stream_t stream) { if (src) { - if (!dest) { - dest = src->clone(0, 0, useGpu); - } else { - dest->resize(src->getHeight(), src->getWidth()); - } + Matrix::resizeOrCreate(dest, src->getHeight(), + src->getWidth(), false, useGpu); dest->copyFrom(*src, stream); } else { dest.reset(); @@ -60,14 +57,9 @@ static void resizeAndCopy(MatrixPtr& dest, const MatrixPtr& src, hl_stream_t stream = HPPL_STREAM_DEFAULT) { if (src) { CHECK_LE((size_t)startRow + copySize, src->getHeight()); - int height = copySize; int width = src->getWidth(); - if (!dest) { - dest = src->clone(height, width, useGpu); - } else { - dest->resize(height, width); - } + Matrix::resizeOrCreate(dest, height, width, false, useGpu); MatrixPtr submat = src->subMatrix(startRow, copySize); if (dynamic_cast(dest.get())) { // copy a subMatrix of CpuSparseMatrix to GpuSparseMatrix. @@ -182,6 +174,11 @@ static void resizeAndCopy(SVectorPtr& dest, const SVectorPtr& src, } } +void Argument::resizeAndCopyFrom(const Argument& src, bool useGpu) { + resizeAndCopyFrom(src, useGpu, HPPL_STREAM_DEFAULT); + hl_stream_synchronize(HPPL_STREAM_DEFAULT); +} + void Argument::resizeAndCopyFrom(const Argument& src, bool useGpu, hl_stream_t stream) { dataId = src.dataId; @@ -199,6 +196,14 @@ void Argument::resizeAndCopyFrom(const Argument& src, bool useGpu, resizeAndCopy(strs, src.strs, useGpu, stream); } +int32_t Argument::resizeAndCopyFrom(const Argument& src, int32_t startSeq, + int32_t copySize, bool useGpu) { + int32_t size = resizeAndCopyFrom(src, startSeq, copySize, useGpu, + HPPL_STREAM_DEFAULT); + hl_stream_synchronize(HPPL_STREAM_DEFAULT); + return size; +} + int32_t Argument::resizeAndCopyFrom(const Argument& src, int32_t startSeq, int32_t copySize, bool useGpu, hl_stream_t stream) { diff --git a/paddle/parameter/Argument.h b/paddle/parameter/Argument.h index c444ebaf12930..34ffeba7b5307 100644 --- a/paddle/parameter/Argument.h +++ b/paddle/parameter/Argument.h @@ -205,11 +205,14 @@ struct Argument { * return value: how many samples are copied */ int32_t resizeAndCopyFrom(const Argument& src, int32_t startSeq, - int32_t copySize, bool useGpu = FLAGS_use_gpu, - hl_stream_t stream = HPPL_STREAM_DEFAULT); + int32_t copySize, bool useGpu, hl_stream_t stream); - void resizeAndCopyFrom(const Argument& src, bool useGpu = FLAGS_use_gpu, - hl_stream_t stream = HPPL_STREAM_DEFAULT); + int32_t resizeAndCopyFrom(const Argument& src, int32_t startSeq, + int32_t copySize, bool useGpu = FLAGS_use_gpu); + + void resizeAndCopyFrom(const Argument& src, bool useGpu, hl_stream_t stream); + + void resizeAndCopyFrom(const Argument& src, bool useGpu = FLAGS_use_gpu); /* @brief Concatenate several arguments into one and put the result into it. From 0f91ea7ebbdfb50567ed5408562ca09eadec24ad Mon Sep 17 00:00:00 2001 From: Haonan Date: Wed, 7 Sep 2016 11:20:36 -0700 Subject: [PATCH 075/324] use HPPL_STREAM_DEFAULT for layer computation Change-Id: Id66da7b7f5bf9ec80cc19b347e4fb822a5a6f197 --- paddle/gserver/evaluators/CTCErrorEvaluator.cpp | 5 +++-- .../gradientmachines/MultiGradientMachine.cpp | 6 +++++- paddle/gserver/layers/CTCLayer.cpp | 9 +++++---- paddle/gserver/layers/ConvOperator.cpp | 2 +- paddle/gserver/layers/CostLayer.cpp | 5 +++-- paddle/gserver/layers/SamplingIdLayer.cpp | 5 +++-- paddle/parameter/Argument.cpp | 15 ++++++++++++--- paddle/parameter/Argument.h | 12 ++++++++++++ 8 files changed, 44 insertions(+), 15 deletions(-) diff --git a/paddle/gserver/evaluators/CTCErrorEvaluator.cpp b/paddle/gserver/evaluators/CTCErrorEvaluator.cpp index a0c68fc9c29c4..cd4ed19c2ca45 100644 --- a/paddle/gserver/evaluators/CTCErrorEvaluator.cpp +++ b/paddle/gserver/evaluators/CTCErrorEvaluator.cpp @@ -194,8 +194,9 @@ class CTCErrorEvaluator : public Evaluator { virtual real evalImp(std::vector& arguments) { CHECK_EQ(arguments.size(), (size_t)2); Argument output, label; - output.resizeAndCopyFrom(arguments[0], false); - label.resizeAndCopyFrom(arguments[1], false); + output.resizeAndCopyFrom(arguments[0], false, HPPL_STREAM_DEFAULT); + label.resizeAndCopyFrom(arguments[1], false, HPPL_STREAM_DEFAULT); + hl_stream_synchronize(HPPL_STREAM_DEFAULT); CHECK(label.sequenceStartPositions); CHECK(label.ids); size_t numSequences = label.sequenceStartPositions->getSize() - 1; diff --git a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp index 74a743145da8b..787ce703a08ae 100644 --- a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp @@ -878,7 +878,11 @@ void TrainerThread::copyOutputGrad() { outArgs_.resize(outputGradArgs.size()); for (size_t i = 0; i < outputGradArgs.size(); i++) { outArgs_[i].resizeAndCopyFrom(outputGradArgs[i], startSeq, copySize, - multiMachine_->useGpu()); + multiMachine_->useGpu(), + HPPL_STREAM_DEFAULT); + } + if (multiMachine_->useGpu()) { + hl_stream_synchronize(HPPL_STREAM_DEFAULT); } gradientMachine_->setOutputGrad(outArgs_); } diff --git a/paddle/gserver/layers/CTCLayer.cpp b/paddle/gserver/layers/CTCLayer.cpp index aa9c0d8a4b68f..6b9ffc5c749fb 100644 --- a/paddle/gserver/layers/CTCLayer.cpp +++ b/paddle/gserver/layers/CTCLayer.cpp @@ -49,9 +49,10 @@ void CTCLayer::forward(PassType passType) { Layer::forward(passType); if (useGpu_) { for (size_t i = 0; i < inputLayers_.size(); i++) { - tmpCpuInput_[i].resizeAndCopyFrom(getInput(i), false, HPPL_STREAM_1); + tmpCpuInput_[i].resizeAndCopyFrom( + getInput(i), false, HPPL_STREAM_DEFAULT); } - hl_stream_synchronize(HPPL_STREAM_1); + hl_stream_synchronize(HPPL_STREAM_DEFAULT); forwardImp(tmpCpuInput_[0], tmpCpuInput_[1]); } else { forwardImp(getInput(0), getInput(1)); @@ -93,9 +94,9 @@ void CTCLayer::backward(const UpdateCallback &callback) { if (useGpu_) { backwardImp(callback, tmpCpuInput_[0], tmpCpuInput_[1]); const_cast(getInput(0)). - resizeAndCopyFrom(tmpCpuInput_[0], true, HPPL_STREAM_1); + resizeAndCopyFrom(tmpCpuInput_[0], true, HPPL_STREAM_DEFAULT); const_cast(getInput(1)). - resizeAndCopyFrom(tmpCpuInput_[1], true, HPPL_STREAM_1); + resizeAndCopyFrom(tmpCpuInput_[1], true, HPPL_STREAM_DEFAULT); } else { backwardImp(callback, getInput(0), getInput(1)); } diff --git a/paddle/gserver/layers/ConvOperator.cpp b/paddle/gserver/layers/ConvOperator.cpp index d08c422764e56..8c72c1778451d 100644 --- a/paddle/gserver/layers/ConvOperator.cpp +++ b/paddle/gserver/layers/ConvOperator.cpp @@ -248,7 +248,7 @@ void ConvOperator::forward() { CHECK_EQ(ins_[1]->value->getHeight(), batchSize); checkFilterSize(ins_[1]->value); Matrix::resizeOrCreate(out_->value, batchSize, - outputH_ * outputW_ * numFilters_); + outputH_ * outputW_ * numFilters_, false, useGpu_); { AsyncGpuBlock block; for (size_t batchId = 0; batchId < batchSize; ++batchId) { diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index b778f3e9b064f..0f99aee03200c 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -509,9 +509,10 @@ void HuberTwoClass::forwardImp(Matrix &output, Argument &label, Matrix &cost) { if (useGpu_) { for (size_t i = 0; i < inputLayers_.size(); i++) { - tmpCpuInput_[i].resizeAndCopyFrom(getInput(i), false, HPPL_STREAM_1); + tmpCpuInput_[i].resizeAndCopyFrom( + getInput(i), false, HPPL_STREAM_DEFAULT); } - hl_stream_synchronize(HPPL_STREAM_1); + hl_stream_synchronize(HPPL_STREAM_DEFAULT); } forwardImpIn(output, label, cost); } diff --git a/paddle/gserver/layers/SamplingIdLayer.cpp b/paddle/gserver/layers/SamplingIdLayer.cpp index cbc85b946f079..b39c9948b5311 100644 --- a/paddle/gserver/layers/SamplingIdLayer.cpp +++ b/paddle/gserver/layers/SamplingIdLayer.cpp @@ -52,9 +52,10 @@ class SamplingIdLayer : public Layer { Layer::forward(passType); if (useGpu_) { for (size_t i = 0; i < inputLayers_.size(); i++) { - tmpCpuInput_[i].resizeAndCopyFrom(getInput(i), false, HPPL_STREAM_1); + tmpCpuInput_[i].resizeAndCopyFrom( + getInput(i), false, HPPL_STREAM_DEFAULT); } - hl_stream_synchronize(HPPL_STREAM_1); + hl_stream_synchronize(HPPL_STREAM_DEFAULT); forwardImp(tmpCpuInput_[0]); } else { forwardImp(getInput(0)); diff --git a/paddle/parameter/Argument.cpp b/paddle/parameter/Argument.cpp index 0e4d676c899ae..a81c72aacbed3 100644 --- a/paddle/parameter/Argument.cpp +++ b/paddle/parameter/Argument.cpp @@ -22,8 +22,12 @@ namespace paddle { static void resizeAndCopy(MatrixPtr& dest, const MatrixPtr& src, bool useGpu, hl_stream_t stream) { if (src) { - Matrix::resizeOrCreate(dest, src->getHeight(), - src->getWidth(), false, useGpu); + if (!dest) { + dest = src->clone(0, 0, useGpu); + } else { + CHECK_EQ(dest->useGpu(), useGpu); + dest->resize(src->getHeight(), src->getWidth()); + } dest->copyFrom(*src, stream); } else { dest.reset(); @@ -59,7 +63,12 @@ static void resizeAndCopy(MatrixPtr& dest, const MatrixPtr& src, CHECK_LE((size_t)startRow + copySize, src->getHeight()); int height = copySize; int width = src->getWidth(); - Matrix::resizeOrCreate(dest, height, width, false, useGpu); + if (!dest) { + dest = src->clone(height, width, useGpu); + } else { + CHECK_EQ(dest->useGpu(), useGpu); + dest->resize(height, width); + } MatrixPtr submat = src->subMatrix(startRow, copySize); if (dynamic_cast(dest.get())) { // copy a subMatrix of CpuSparseMatrix to GpuSparseMatrix. diff --git a/paddle/parameter/Argument.h b/paddle/parameter/Argument.h index 34ffeba7b5307..3cab87c700225 100644 --- a/paddle/parameter/Argument.h +++ b/paddle/parameter/Argument.h @@ -203,15 +203,27 @@ struct Argument { * startSeq: the sample id of start * copySize: how many samples need to copy * return value: how many samples are copied + * Note that when specifying the stream explicitly in this case, + * synchronize should also be called somewhere after this function */ int32_t resizeAndCopyFrom(const Argument& src, int32_t startSeq, int32_t copySize, bool useGpu, hl_stream_t stream); + /* + * same with the above function, except that the stream is + * HPPL_STREAM_DEFAULT and synchronize is automatically called + * inside it + */ int32_t resizeAndCopyFrom(const Argument& src, int32_t startSeq, int32_t copySize, bool useGpu = FLAGS_use_gpu); void resizeAndCopyFrom(const Argument& src, bool useGpu, hl_stream_t stream); + /* + * same with the above function, except that the stream is + * HPPL_STREAM_DEFAULT and synchronize is automatically called + * inside it + */ void resizeAndCopyFrom(const Argument& src, bool useGpu = FLAGS_use_gpu); /* From 699d5f26388d01666692565ec8e8f4599c993208 Mon Sep 17 00:00:00 2001 From: zhangruiqing01 Date: Mon, 12 Sep 2016 15:06:18 +0800 Subject: [PATCH 076/324] modify RecurrentGradientMachine to support unequal length inputs * modify RecurrentGradientMachine to support hasSubSeq sequence inlinks with the same number of sentence but different number of tokens for each sentence Change-Id: Ic71f00a4bb346b4fa93e650dfb4b1a0d8d2338b0 --- .../RecurrentGradientMachine.cpp | 262 +++++++++++------- .../RecurrentGradientMachine.h | 43 ++- proto/ModelConfig.proto.m4 | 3 + python/paddle/trainer/config_parser.py | 15 +- 4 files changed, 211 insertions(+), 112 deletions(-) diff --git a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp index 7bc5fe51813c9..e000bb2e5d6e9 100644 --- a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/utils/Stat.h" #include "paddle/utils/Util.h" #include "paddle/utils/Flags.h" @@ -291,6 +290,8 @@ void RecurrentGradientMachine::init( if (subModelConfig->evaluator_names_size() > 0) { evaluator_.reset(frames_[0]->makeEvaluator()); } + + targetInfoInlinkId_ = subModelConfig->target_inlinkid(); } void RecurrentGradientMachine::resizeOrCreateFrames(int numFrames) { @@ -325,7 +326,7 @@ void RecurrentGradientMachine::resizeOrCreateFrames(int numFrames) { for (int i = frames_.size(); i < numFrames; ++i) { std::unique_ptr frame( - NeuralNetwork::newNeuralNetwork(subModelName_)); + NeuralNetwork::newNeuralNetwork(subModelName_)); frame->init(config_, subParamInitCb); for (auto& inFrameLine : inFrameLines_) { @@ -382,6 +383,16 @@ void RecurrentGradientMachine::forward(const std::vector& inArgs, size_t numSequences = input.getNumSequences(); const int* starts = input.sequenceStartPositions->getData(false); bool hasSubseq = input.hasSubseq(); + + // In case of !hasSubseq or targetInfoInlinkId_ == -1, all inlinks share the + // same inframe info + bool shareInlinkInfo = !hasSubseq || targetInfoInlinkId_ == -1; + + // Defaultly, share info with the first inlink + if (shareInlinkInfo) { + targetInfoInlinkId_ = 0; + } + // check hasSubseq in both config and input are the same CHECK_EQ(hasSubseq, inFrameLines_[0].hasSubseq); @@ -394,9 +405,17 @@ void RecurrentGradientMachine::forward(const std::vector& inArgs, CHECK_EQ((size_t)input1.getNumSequences(), numSequences); // check all inputs should have same hasSubseq flag CHECK_EQ(input.hasSubseq(), inFrameLines_[0].hasSubseq); - CHECK_EQ(input1.getBatchSize(), batchSize); - CHECK(std::equal(starts, starts + numSequences + 1, - input1.sequenceStartPositions->getData(false))); + + // if shareInlinkInfo, checks: + // 1. all inlinks have same number of total tokens + // 2. all inlinks have same number of tokens for each sentence of each + // sample. If hasSubseq, one sample has multiple sentence, else, one + // sample is one sentence + if (shareInlinkInfo) { + CHECK_EQ(input1.getBatchSize(), batchSize); + CHECK(std::equal(starts, starts + numSequences + 1, + input1.sequenceStartPositions->getData(false))); + } } if (hasSubseq) { @@ -408,19 +427,44 @@ void RecurrentGradientMachine::forward(const std::vector& inArgs, for (size_t i = 1; i < inFrameLines_.size(); ++i) { const Argument& input1 = inFrameLines_[i].inLayer->getOutput(); CHECK_EQ((size_t)input1.getNumSubSequences(), numSubSequences); - CHECK(std::equal(subStarts, subStarts + numSubSequences + 1, - input1.subSequenceStartPositions->getData(false))); + if (shareInlinkInfo) { + CHECK(std::equal(subStarts, subStarts + numSubSequences + 1, + input1.subSequenceStartPositions->getData(false))); + } } } seqLengthAndStart_.clear(); - input.getSeqLengthAndStart(&seqLengthAndStart_, &maxSequenceLength_); + info_.clear(); + info_.resize(inFrameLines_.size()); + seqLengthAndStart_.resize(inFrameLines_.size()); + + { + AsyncGpuBlock asyncGpuBlock; + // if shareInlinkInfo, only calculate info of the first inlink + // else, calculate info for each inlink + if (shareInlinkInfo) { + input.getSeqLengthAndStart(&seqLengthAndStart_[0], &maxSequenceLength_); + createInFrameInfo(0, input, passType); + } else { + for (size_t i = 0; i < inFrameLines_.size(); i++) { + const Argument& input1 = inFrameLines_[i].inLayer->getOutput(); + input1.getSeqLengthAndStart(&seqLengthAndStart_[i], + &maxSequenceLength_); + createInFrameInfo(i, input1, passType); + } + } + + // inFrameLine select rows in real layer one time + for (size_t i = 0; i < inFrameLines_.size(); i++) { + int curInlinkId = shareInlinkInfo ? 0 : i; + selectRowsOneTime(inFrameLines_[i].inLayer, info_[curInlinkId].allIds, + &(inFrameLines_[i].outArg), passType); + } + } resizeOrCreateFrames(maxSequenceLength_); resizeBootFrame(numSequences); - AsyncGpuBlock asyncGpuBlock; - createInFrameInfo(input, passType); - for (auto& memoryFrameLine : memoryFrameLines_) { if (memoryFrameLine.rootAgent) { auto scatterAgent = @@ -443,23 +487,29 @@ void RecurrentGradientMachine::forward(const std::vector& inArgs, auto gatherAgent = dynamic_cast(outFrameLine.agentLayer.get()); CHECK_NOTNULL(gatherAgent); - gatherAgent->copyIdAndSequenceInfo(input, info_.allIds, info_.idIndex); + gatherAgent->copyIdAndSequenceInfo(input, info_[targetInfoInlinkId_].allIds, + info_[targetInfoInlinkId_].idIndex); } for (int i = 0; i < maxSequenceLength_; ++i) { - int idSize = info_.idIndex[i + 1] - info_.idIndex[i]; - + int idSize = 0; // connect in_links - for (auto& inFrameLine : inFrameLines_) { + for (size_t j = 0; j < inFrameLines_.size(); ++j) { + // idSize denotes the sum number of tokens in each length i + idSize = info_[j].idIndex[i + 1] - info_[j].idIndex[i]; + InFrameLine inFrameLine = inFrameLines_[j]; auto scatterAgent = dynamic_cast(inFrameLine.agents[i].get()); scatterAgent->setRealLayerAndOutput(inFrameLine.inLayer, - inFrameLine.outArg, info_.allIds, - info_.idIndex[i], idSize); + inFrameLine.outArg, info_[j].allIds, + info_[j].idIndex[i], idSize); if (hasSubseq) { - int size = info_.seqStartPosIndex[i + 1] - info_.seqStartPosIndex[i]; - scatterAgent->setSequenceStartPositions( - info_.sequenceStartPositions, info_.seqStartPosIndex[i], size); + // size: the length of subsequence + int size = + info_[j].seqStartPosIndex[i + 1] - info_[j].seqStartPosIndex[i]; + scatterAgent->setSequenceStartPositions(info_[j].sequenceStartPositions, + info_[j].seqStartPosIndex[i], + size); } } @@ -471,6 +521,10 @@ void RecurrentGradientMachine::forward(const std::vector& inArgs, } // connect memory links + // Adopt info_[0].idIndex because seq which has_subseq=True + // doesn't support Memory with !hasSubseq bootlayer; + // And inlinks that !hasSubSeq must have same inlink length. + idSize = info_[0].idIndex[i + 1] - info_[0].idIndex[i]; for (auto& memoryFrameLine : memoryFrameLines_) { NeuralNetwork::connect( memoryFrameLine.agents[i], @@ -560,62 +614,68 @@ void RecurrentGradientMachine::removeBeamSearchStatisticsCallbacks() { * If hasSubseq, will also create scattered sequenceStartPositions infomation * for all realLayer of inFrameLines one time. */ -void RecurrentGradientMachine::createInFrameInfo(const Argument& input, + +void RecurrentGradientMachine::createInFrameInfo(int inlinks_id, + const Argument& input, PassType passType) { bool hasSubseq = input.hasSubseq(); + // numSequences: # samples(sequences) in a batch size_t numSequences = input.getNumSequences(); std::vector allIds; - info_.idIndex.clear(); - info_.idIndex.push_back(0); // first idIndex = 0 - if (hasSubseq) { // for sequenceScatterAgentLayer + Info* inlink_info = &info_[inlinks_id]; + inlink_info->idIndex.clear(); + inlink_info->idIndex.push_back(0); // first idIndex = 0 + if (hasSubseq) { // for sequenceScatterAgentLayer + // numSubSequences : all sentences within all samples(batch) size_t numSubSequences = input.getNumSubSequences(); std::vector sequenceStartPositions; - info_.seqStartPosIndex.clear(); - info_.seqStartPosIndex.push_back(0); // first seqStartPosIndex = 0 + inlink_info->seqStartPosIndex.clear(); + inlink_info->seqStartPosIndex.push_back(0); // first seqStartPosIndex = 0 + // maxSequenceLength_: max number of sentences(subseq) in allsamples for (int i = 0; i < maxSequenceLength_; ++i) { - sequenceStartPositions.push_back(0); // first element = 0 - for (size_t j = 0; j < numSubSequences; ++j) { - if (std::get<3>(seqLengthAndStart_[j]) == i) { - int subSeqStart = std::get<1>(seqLengthAndStart_[j]); - int subSeqLength = std::get<0>(seqLengthAndStart_[j]); + sequenceStartPositions.push_back(0); // first element = 0 + for (size_t j = 0; j < numSubSequences; ++j) { // for each sentence + // seqLengthAndStart_[inlinks_id][j]: + // a 4-tuple including + if (std::get<3>(seqLengthAndStart_[inlinks_id][j]) == i) { + // subseqstart: the cpuSubSequenceStartPositions of this subseq + int subSeqStart = std::get<1>(seqLengthAndStart_[inlinks_id][j]); + int subSeqLength = std::get<0>(seqLengthAndStart_[inlinks_id][j]); for (int k = subSeqStart; k < subSeqStart + subSeqLength; ++k) { allIds.push_back(k); } sequenceStartPositions.push_back(sequenceStartPositions.back() + - subSeqLength); + subSeqLength); } } - info_.idIndex.push_back(allIds.size()); - info_.seqStartPosIndex.push_back(sequenceStartPositions.size()); + inlink_info->idIndex.push_back(allIds.size()); + inlink_info->seqStartPosIndex.push_back(sequenceStartPositions.size()); } // inFrameLine create sequenceStartPositions one time CHECK_EQ(sequenceStartPositions.size(), maxSequenceLength_ + numSubSequences); - CHECK_EQ(info_.seqStartPosIndex.size(), + CHECK_EQ(inlink_info->seqStartPosIndex.size(), static_cast(maxSequenceLength_ + 1)); - createSeqPos(sequenceStartPositions, &info_.sequenceStartPositions); + createSeqPos(sequenceStartPositions, &inlink_info->sequenceStartPositions); } else { // for scatterAgentLayer for (int i = 0; i < maxSequenceLength_; ++i) { for (size_t j = 0; j < numSequences; ++j) { - int seqLength = std::get<0>(seqLengthAndStart_[j]); + int seqLength = std::get<0>(seqLengthAndStart_[inlinks_id][j]); if (i >= seqLength) { break; } - int seqStart = std::get<1>(seqLengthAndStart_[j]); + int seqStart = std::get<1>(seqLengthAndStart_[inlinks_id][j]); allIds.push_back(reversed_ ? (seqStart + seqLength - 1 - i) : (seqStart + i)); } - info_.idIndex.push_back(allIds.size()); + inlink_info->idIndex.push_back(allIds.size()); } } + // copy and check scatterId - copyScattedId(allIds, &info_.allIds, input.getBatchSize()); - CHECK_EQ(info_.idIndex.size(), static_cast(maxSequenceLength_ + 1)); - // inFrameLine select rows in real layer one time - for (auto& inFrameLine : inFrameLines_) { - selectRowsOneTime(inFrameLine.inLayer, info_.allIds, &inFrameLine.outArg, - passType); - } + copyScattedId(allIds, &inlink_info->allIds, input.getBatchSize()); + CHECK_EQ(inlink_info->idIndex.size(), + static_cast(maxSequenceLength_ + 1)); } /* like createInFrameInfo, but for all realLayer of memoryFrameLines*/ @@ -633,19 +693,20 @@ void RecurrentGradientMachine::createMemoryFrameInfo( sequenceStartPositions.push_back(0); // first element = 0 const int* starts = input.sequenceStartPositions->getData(false); for (size_t i = 0; i < numSequences; ++i) { - int seqId = std::get<2>(seqLengthAndStart_[i]); + // memory info adopt info of inlinks[0] + int seqId = std::get<2>(seqLengthAndStart_[0][i]); for (int k = starts[seqId]; k < starts[seqId + 1]; ++k) { allIds.push_back(k); } sequenceStartPositions.push_back(sequenceStartPositions.back() + - starts[seqId + 1] - starts[seqId]); + starts[seqId + 1] - starts[seqId]); } createSeqPos(sequenceStartPositions, &(*memoryFrameLine).sequenceStartPositions); } else { // for scatterAgentLayer for (size_t i = 0; i < numSequences; ++i) { - allIds.push_back(std::get<2>(seqLengthAndStart_[i])); + allIds.push_back(std::get<2>(seqLengthAndStart_[0][i])); } } // copy and check scatterId @@ -699,18 +760,19 @@ size_t RecurrentGradientMachine::getGenBatchSize() { for (auto& memoryFrameLine : memoryFrameLines_) { if (!memoryFrameLine.rootLayer) continue; Argument& bootArg = memoryFrameLine.rootLayer->getOutput(); - size_t batchSize = memoryFrameLine.is_sequence ? - bootArg.getNumSequences() : bootArg.getBatchSize(); + size_t batchSize = memoryFrameLine.is_sequence ? bootArg.getNumSequences() + : bootArg.getBatchSize(); if (numSequences) { CHECK_EQ(numSequences, batchSize); } else { numSequences = batchSize; } } - CHECK(numSequences) << "Fail to get batch size in generation. " - "At least one of the Memory layer MUST have a layer that is NOT in " - "the layer group to boot it, and this boot layer is used to " - "decide batch_size in generation process."; + CHECK(numSequences) + << "Fail to get batch size in generation. " + "At least one of the Memory layer MUST have a layer that is NOT in " + "the layer group to boot it, and this boot layer is used to " + "decide batch_size in generation process."; return numSequences; } @@ -732,7 +794,9 @@ void RecurrentGradientMachine::generateSequence() { // connect boot frame memory links std::vector ids(numSequences); - for (size_t i = 0; i < numSequences; ++i) { ids[i] = i; } + for (size_t i = 0; i < numSequences; ++i) { + ids[i] = i; + } for (auto& memoryFrameLine : memoryFrameLines_) { if (memoryFrameLine.rootAgent) { auto scatterAgent = @@ -756,7 +820,8 @@ void RecurrentGradientMachine::generateSequence() { // init outArg size_t resultNum = generator_.config.num_results_per_sample(); - IVector::resizeOrCreate(generator_.outArg.ids, + IVector::resizeOrCreate( + generator_.outArg.ids, generator_.config.max_num_frames() * numSequences * resultNum, false); if (resultNum > 1) { CHECK_LE(resultNum, static_cast(generator_.config.beam_size())); @@ -847,7 +912,9 @@ void RecurrentGradientMachine::oneWaySearch(size_t batchSize) { // path.seqId = -1 indicates end of generation // of an input sequence finalPaths[seqIds_[j]].seqId = -1; - } else { scatterIds.push_back(j); } + } else { + scatterIds.push_back(j); + } } } @@ -856,13 +923,12 @@ void RecurrentGradientMachine::oneWaySearch(size_t batchSize) { starts[0] = 0; generator_.ids.clear(); for (size_t i = 0; i < batchSize; ++i) { - generator_.ids.insert(generator_.ids.end(), - finalPaths[i].ids.begin(), + generator_.ids.insert(generator_.ids.end(), finalPaths[i].ids.begin(), finalPaths[i].ids.end()); starts[i + 1] = generator_.ids.size(); batchMachineIdVec_.insert(batchMachineIdVec_.end(), - finalPaths[i].machineIdVec.begin(), - finalPaths[i].machineIdVec.end()); + finalPaths[i].machineIdVec.begin(), + finalPaths[i].machineIdVec.end()); } } @@ -920,9 +986,9 @@ void RecurrentGradientMachine::forwardFrame(int machineCur) { } } -void RecurrentGradientMachine::singlePathExpand( - Path& curPath, size_t curPathId, std::vector& newPaths, - size_t expandWidth) { +void RecurrentGradientMachine::singlePathExpand(Path& curPath, size_t curPathId, + std::vector& newPaths, + size_t expandWidth) { int calc_id = gDiyProbStart ? gDiyProbStart(curPath.ids.size(), curPath.ids.data()) : 0; @@ -946,19 +1012,20 @@ void RecurrentGradientMachine::singlePathExpand( if (id == -1) break; real newLogProb = generator_.config.log_prob() ? std::log(prob) : prob; - Path newPath(curPath, id, newLogProb, - curPathId /*machineId*/, k /*topIndex*/); + Path newPath(curPath, id, newLogProb, curPathId /*machineId*/, + k /*topIndex*/); if (this->beamSearchCtrlCallbacks_) { if (beamSearchCtrlCallbacks_->stopDetermineCandidates( - newPath.seqId, newPath.ids, newPath.probHistory)) return; + newPath.seqId, newPath.ids, newPath.probHistory)) + return; } // outFrameLines_.size() > 1UL if (dataArgsSize_) { newPath.machineIdVec = curPath.machineIdVec; newPath.machineIdVec.push_back(curPathId); } - bool atEos = eosVec[index] == 1U || - newPath.ids.size() >= (size_t)maxSequenceLength_; + bool atEos = + eosVec[index] == 1U || newPath.ids.size() >= (size_t)maxSequenceLength_; // adjustNewPath newPath.adjustProb(calc_id, atEos); if (this->beamSearchCtrlCallbacks_) { @@ -966,16 +1033,18 @@ void RecurrentGradientMachine::singlePathExpand( newPath.seqId, newPath.ids, newPath.probHistory, &newPath.logProb); } if (!newPath.isDropable()) { - atEos ? finalPaths_[curPath.seqId].push_back(newPath) : - newPaths.push_back(newPath); + atEos ? finalPaths_[curPath.seqId].push_back(newPath) + : newPaths.push_back(newPath); } } // for expandWidth - if (gDiyProbStop) { gDiyProbStop(calc_id); } + if (gDiyProbStop) { + gDiyProbStop(calc_id); + } } -void RecurrentGradientMachine::beamExpand( - std::vector& paths, std::vector& newPaths) { +void RecurrentGradientMachine::beamExpand(std::vector& paths, + std::vector& newPaths) { size_t candidatePathCount = paths.size(); // idVec.size() could be larger than candidatePathCount * beam, // so user can drop some node customly. @@ -988,7 +1057,7 @@ void RecurrentGradientMachine::beamExpand( int curSeqId = 0; for (size_t j = 0; j <= candidatePathCount; j++) { // expansions of a single sequence are all processed - curSeqId = (j < candidatePathCount? paths[j].seqId : curSeqId + 1); + curSeqId = (j < candidatePathCount ? paths[j].seqId : curSeqId + 1); if (prevSeqId != -1 && curSeqId != prevSeqId) { totalExpandCount += beamShrink(newPaths, prevSeqId, totalExpandCount); } @@ -1000,11 +1069,14 @@ void RecurrentGradientMachine::beamExpand( } // Drop extra nodes to beam size. -size_t RecurrentGradientMachine::beamShrink( - std::vector& newPaths, size_t seqId, size_t totalExpandCount) { - size_t minNewPathSize = std::min(getBeamSize(), - newPaths.size() - totalExpandCount); - if (!minNewPathSize) { return 0; } +size_t RecurrentGradientMachine::beamShrink(std::vector& newPaths, + size_t seqId, + size_t totalExpandCount) { + size_t minNewPathSize = + std::min(getBeamSize(), newPaths.size() - totalExpandCount); + if (!minNewPathSize) { + return 0; + } std::nth_element(newPaths.begin() + totalExpandCount, newPaths.begin() + totalExpandCount + minNewPathSize, newPaths.end(), Path::greaterPath); @@ -1017,11 +1089,8 @@ size_t RecurrentGradientMachine::beamShrink( // Remove the already formed paths that are relatively short finalPaths_[seqId].erase( - std::remove_if(finalPaths_[seqId].begin(), - finalPaths_[seqId].end(), - [&](Path& p) { - return p.logProb < minPathLogProb; - }), + std::remove_if(finalPaths_[seqId].begin(), finalPaths_[seqId].end(), + [&](Path& p) { return p.logProb < minPathLogProb; }), finalPaths_[seqId].end()); for (auto p : finalPaths_[seqId]) { if (minFinalPathLogProb_[seqId] > p.logProb) { @@ -1030,7 +1099,7 @@ size_t RecurrentGradientMachine::beamShrink( } if (finalPaths_[seqId].size() >= getBeamSize() && - minFinalPathLogProb_[seqId] >= maxPathLogProb) { + minFinalPathLogProb_[seqId] >= maxPathLogProb) { newPaths.resize(totalExpandCount); return 0; } @@ -1067,7 +1136,8 @@ void RecurrentGradientMachine::fillGenOutputs() { // in beam search, here only reserved the top 1 generated result // for out_links that are not the generated word indices. batchMachineIdVec_.insert(batchMachineIdVec_.end(), - path.machineIdVec.begin(), path.machineIdVec.end()); + path.machineIdVec.begin(), + path.machineIdVec.end()); } } starts[i + 1] = generator_.ids.size(); @@ -1091,21 +1161,21 @@ void RecurrentGradientMachine::copyDataOutlinkFrame(size_t machineCur) { void RecurrentGradientMachine::createDataOutlink( std::vector& machineIdVec) { - size_t seqNum = getBeamSize() > 1UL ? - finalPaths_.size() : finalPaths_[0].size(); + size_t seqNum = + getBeamSize() > 1UL ? finalPaths_.size() : finalPaths_[0].size(); std::vector starts(seqNum + 1, 0); for (size_t i = 0; i < seqNum; ++i) { - size_t seqLen = getBeamSize() > 1UL ? finalPaths_[i][0].ids.size() : - finalPaths_[0][i].ids.size(); + size_t seqLen = getBeamSize() > 1UL ? finalPaths_[i][0].ids.size() + : finalPaths_[0][i].ids.size(); starts[i + 1] = starts[i] + seqLen; } for (size_t i = 0; i < dataArgsSize_; i++) { - dataArgs_[i].concat(dataArgsFrame_[i], machineIdVec, - starts, useGpu_, HPPL_STREAM_1, PASS_TEST); + dataArgs_[i].concat(dataArgsFrame_[i], machineIdVec, starts, useGpu_, + HPPL_STREAM_1, PASS_TEST); - auto dataAgent = dynamic_cast( - outFrameLines_[i + 1].agentLayer.get()); + auto dataAgent = + dynamic_cast(outFrameLines_[i + 1].agentLayer.get()); CHECK_NOTNULL(dataAgent); dataAgent->setData(dataArgs_[i]); } diff --git a/paddle/gserver/gradientmachines/RecurrentGradientMachine.h b/paddle/gserver/gradientmachines/RecurrentGradientMachine.h index cc49d13952323..4ca545b504f73 100644 --- a/paddle/gserver/gradientmachines/RecurrentGradientMachine.h +++ b/paddle/gserver/gradientmachines/RecurrentGradientMachine.h @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once #include "GradientMachine.h" @@ -101,7 +100,7 @@ class RecurrentGradientMachine : public NeuralNetwork { * Return true if this prefix or candidate is expected to be dropped. */ typedef std::function&, - const std::vector&)> DropCallback; + const std::vector&)> DropCallback; /** * @brief NormOrDropNodeCallback @@ -117,7 +116,7 @@ class RecurrentGradientMachine : public NeuralNetwork { * The fourth parameter is the probability of the whole path. */ typedef std::function&, - std::vector&, real*)> NormOrDropNodeCallback; + std::vector&, real*)> NormOrDropNodeCallback; /** * @brief Register beam search control callbacks. Used for prediction. @@ -192,7 +191,7 @@ class RecurrentGradientMachine : public NeuralNetwork { int machineId; // index of sample in frame int topIndex; // index of MaxIdLayer output in one sample - int seqId; // index of sequence in batch generation + int seqId; // index of sequence in batch generation std::vector machineIdVec; /** @@ -206,7 +205,10 @@ class RecurrentGradientMachine : public NeuralNetwork { /** * @brief Path default ctor, first logProb is 0. */ - Path() { logProb = 0; seqId = 0; } + Path() { + logProb = 0; + seqId = 0; + } explicit Path(size_t seqId) : seqId(seqId) { logProb = 0; } /** @@ -319,21 +321,33 @@ class RecurrentGradientMachine : public NeuralNetwork { }; std::vector memoryFrameLines_; - // All inFrameLines and outFrameLines have the same element as follows. + // Each inFrameLines(inlinks) has its own info(elements) below, + // and all outFrameLines(outlinks) share the info with one inFrameLine, + // which is assigned by targetInfoInlinkId_. struct Info { IVectorPtr allIds; // scattered id of realLayer std::vector idIndex; // index of allIds ICpuGpuVectorPtr - sequenceStartPositions; // scattered sequenceStartPositions + sequenceStartPositions; // scattered sequenceStartPositions std::vector seqStartPosIndex; // index of sequenceStartPositions }; - Info info_; + std::vector info_; - // if no subSeq, tuple of (seqLength, seqStart, seqIndex, seqIndex) - // else, tuple of (subSeqLength, subSeqStart, seqIndex, subSeqIndex) - std::vector> seqLengthAndStart_; + // each inlinks has a "std::vector>" denotes + // its sequence info: + // if hasSubSeq, tuple of (subSeqLength, subSeqStart, seqIndex, subSeqIndex) + // else, tuple of (seqLength, seqStart, seqIndex, seqIndex) + std::vector>> seqLengthAndStart_; - void createInFrameInfo(const Argument& input, PassType passType); + // the id of inlink which share info with outlinks + int targetInfoInlinkId_; + + /* create scattered id infomation for all realLayer of inFrameLines one time. + * If hasSubseq, will also create scattered sequenceStartPositions infomation + * for all realLayer of inFrameLines one time. + */ + void createInFrameInfo(int inlinks_id, const Argument& input, + PassType passType); void createMemoryFrameInfo(MemoryFrameLine* memoryFrameLine, PassType passType); @@ -363,6 +377,9 @@ class RecurrentGradientMachine : public NeuralNetwork { NeuralNetwork* rootNetwork_; bool reversed_; + + // if hasSubseq: max number of sentences(subseq)in batchsize samples + // else: max number of tokens in batchsize samples(sentences) int maxSequenceLength_; bool useGpu_; bool stopBeamSearch_; @@ -415,7 +432,7 @@ class RecurrentGradientMachine : public NeuralNetwork { * @param machineIdVec : select a row of output matrix in each frame * that the generation process expanded. */ - void createDataOutlink(std::vector & machineIdVec); + void createDataOutlink(std::vector& machineIdVec); /* * @brief used in beam search, connect previous frame to form recurrent link diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index d04620d363c14..a2b243a7869ea 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -452,6 +452,9 @@ message SubModelConfig { repeated LinkConfig out_links = 10; optional GeneratorConfig generator = 11; + + // the id of inlink which share info with outlinks, used in recurrent layer group + optional int32 target_inlinkid = 12; } message ModelConfig { diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index b26a63e7f3c1d..aed317df67bd0 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -303,7 +303,8 @@ def MakeLayerNameInSubmodel(name, submodel_name = None): @config_func def RecurrentLayerGroupWithoutOutLinksBegin(name, in_links, - seq_reversed=False): + seq_reversed=False, + target_inlinkname=""): global g_current_submodel config_assert(g_config.model_config.type == "recurrent_nn", "RecurrentLayerGroup should be used only in recurrent_nn") @@ -311,14 +312,19 @@ def RecurrentLayerGroupWithoutOutLinksBegin(name, SubModelBegin(name) g_current_submodel.is_recurrent_layer_group = True g_current_submodel.reversed = seq_reversed + g_current_submodel.target_inlinkid = -1 in_links_count = 0 - for link in in_links: + for linkid, link in enumerate(in_links): if isinstance(link, basestring): name = link has_subseq = False else: name = link.link_name has_subseq = link.has_subseq + # assign target_inlinkid according to target_inlinkname + if target_inlinkname == name: + g_current_submodel.target_inlinkid = linkid + if in_links_count == 0: in_links_has_subseq = has_subseq else: @@ -331,6 +337,7 @@ def RecurrentLayerGroupWithoutOutLinksBegin(name, SequenceScatterAgentLayer(name=name, size=layer.size) else: ScatterAgentLayer(name=name, size=layer.size) + pair = g_current_submodel.in_links.add() pair.layer_name = layer_name pair.link_name = MakeLayerNameInSubmodel(name) @@ -362,10 +369,12 @@ def RecurrentLayerGroupBegin(name, in_links, out_links, generator=None, + target_inlinkname="", seq_reversed=False): RecurrentLayerGroupWithoutOutLinksBegin(name, in_links, - seq_reversed) + seq_reversed, + target_inlinkname) for link in out_links: RecurrentLayerGroupSetOutLink(link) From 9a9de9240dc3fa03fcb8c0982b26d1b1a2ed01f2 Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Fri, 9 Sep 2016 10:51:54 -0700 Subject: [PATCH 077/324] Correctly handle memory in RecurrentGradientMachine for hirarchical RNN Change-Id: I8e0a8ea6fc2760652d9c76440a539c90860062d3 --- .../RecurrentGradientMachine.cpp | 11 ++- .../RecurrentGradientMachine.h | 4 + paddle/gserver/tests/Sequence/dummy.list | 1 + paddle/gserver/tests/rnn_data_provider.py | 35 +++++++++ paddle/gserver/tests/sequenceGen.py | 3 - paddle/gserver/tests/sequence_nest_rnn.conf | 75 +++++++++++++++++++ paddle/gserver/tests/sequence_rnn.conf | 57 ++++++++++++++ .../tests/test_RecurrentGradientMachine.cpp | 21 +++++- paddle/parameter/Argument.h | 9 +++ 9 files changed, 207 insertions(+), 9 deletions(-) create mode 100644 paddle/gserver/tests/Sequence/dummy.list create mode 100644 paddle/gserver/tests/rnn_data_provider.py create mode 100644 paddle/gserver/tests/sequence_nest_rnn.conf create mode 100644 paddle/gserver/tests/sequence_rnn.conf diff --git a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp index e000bb2e5d6e9..96b0e19880b68 100644 --- a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp @@ -519,7 +519,6 @@ void RecurrentGradientMachine::forward(const std::vector& inArgs, dynamic_cast(outFrameLine.agentLayer.get()); gatherAgent->addRealLayer(outFrameLine.frames[i]); } - // connect memory links // Adopt info_[0].idIndex because seq which has_subseq=True // doesn't support Memory with !hasSubseq bootlayer; @@ -529,7 +528,7 @@ void RecurrentGradientMachine::forward(const std::vector& inArgs, NeuralNetwork::connect( memoryFrameLine.agents[i], i == 0 ? memoryFrameLine.bootLayer : memoryFrameLine.frames[i - 1], - idSize /*height of agent*/); + numSeqs_[i] /*height of agent*/); } } @@ -622,6 +621,8 @@ void RecurrentGradientMachine::createInFrameInfo(int inlinks_id, // numSequences: # samples(sequences) in a batch size_t numSequences = input.getNumSequences(); std::vector allIds; + + numSeqs_.clear(); Info* inlink_info = &info_[inlinks_id]; inlink_info->idIndex.clear(); inlink_info->idIndex.push_back(0); // first idIndex = 0 @@ -634,10 +635,12 @@ void RecurrentGradientMachine::createInFrameInfo(int inlinks_id, // maxSequenceLength_: max number of sentences(subseq) in allsamples for (int i = 0; i < maxSequenceLength_; ++i) { sequenceStartPositions.push_back(0); // first element = 0 + int numSeqs = 0; for (size_t j = 0; j < numSubSequences; ++j) { // for each sentence // seqLengthAndStart_[inlinks_id][j]: // a 4-tuple including if (std::get<3>(seqLengthAndStart_[inlinks_id][j]) == i) { + ++numSeqs; // subseqstart: the cpuSubSequenceStartPositions of this subseq int subSeqStart = std::get<1>(seqLengthAndStart_[inlinks_id][j]); int subSeqLength = std::get<0>(seqLengthAndStart_[inlinks_id][j]); @@ -650,6 +653,7 @@ void RecurrentGradientMachine::createInFrameInfo(int inlinks_id, } inlink_info->idIndex.push_back(allIds.size()); inlink_info->seqStartPosIndex.push_back(sequenceStartPositions.size()); + numSeqs_.push_back(numSeqs); } // inFrameLine create sequenceStartPositions one time CHECK_EQ(sequenceStartPositions.size(), @@ -659,16 +663,19 @@ void RecurrentGradientMachine::createInFrameInfo(int inlinks_id, createSeqPos(sequenceStartPositions, &inlink_info->sequenceStartPositions); } else { // for scatterAgentLayer for (int i = 0; i < maxSequenceLength_; ++i) { + int numSeqs = 0; for (size_t j = 0; j < numSequences; ++j) { int seqLength = std::get<0>(seqLengthAndStart_[inlinks_id][j]); if (i >= seqLength) { break; } + ++numSeqs; int seqStart = std::get<1>(seqLengthAndStart_[inlinks_id][j]); allIds.push_back(reversed_ ? (seqStart + seqLength - 1 - i) : (seqStart + i)); } inlink_info->idIndex.push_back(allIds.size()); + numSeqs_.push_back(numSeqs); } } diff --git a/paddle/gserver/gradientmachines/RecurrentGradientMachine.h b/paddle/gserver/gradientmachines/RecurrentGradientMachine.h index 4ca545b504f73..d9901ad81d60a 100644 --- a/paddle/gserver/gradientmachines/RecurrentGradientMachine.h +++ b/paddle/gserver/gradientmachines/RecurrentGradientMachine.h @@ -333,6 +333,10 @@ class RecurrentGradientMachine : public NeuralNetwork { }; std::vector info_; + // numSeqs_[i] is the number sequences which is longer than i (for sequence + // data) or has more than i subsequences (for subsequence data) + std::vector numSeqs_; + // each inlinks has a "std::vector>" denotes // its sequence info: // if hasSubSeq, tuple of (subSeqLength, subSeqStart, seqIndex, subSeqIndex) diff --git a/paddle/gserver/tests/Sequence/dummy.list b/paddle/gserver/tests/Sequence/dummy.list new file mode 100644 index 0000000000000..0e52665e11298 --- /dev/null +++ b/paddle/gserver/tests/Sequence/dummy.list @@ -0,0 +1 @@ +dummy_file_no_use diff --git a/paddle/gserver/tests/rnn_data_provider.py b/paddle/gserver/tests/rnn_data_provider.py new file mode 100644 index 0000000000000..85a83554c5c30 --- /dev/null +++ b/paddle/gserver/tests/rnn_data_provider.py @@ -0,0 +1,35 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer.PyDataProvider2 import * + +data = [ + [[[1, 3, 2], [4, 5, 2]], 0], + [[[0, 2], [2, 5], [0, 1, 2]], 1], +] + +@provider(input_types=[integer_value_sub_sequence(10), + integer_value(2)]) +def process_subseq(settings, file_name): + for d in data: + yield d + +@provider(input_types=[integer_value_sequence(10), + integer_value(2)]) +def process_seq(settings, file_name): + for d in data: + seq = [] + for subseq in d[0]: + seq += subseq + yield seq, d[1] diff --git a/paddle/gserver/tests/sequenceGen.py b/paddle/gserver/tests/sequenceGen.py index e4727e472d446..cb83d79d78cc6 100644 --- a/paddle/gserver/tests/sequenceGen.py +++ b/paddle/gserver/tests/sequenceGen.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -#coding=utf-8 - # Copyright (c) 2016 Baidu, Inc. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/paddle/gserver/tests/sequence_nest_rnn.conf b/paddle/gserver/tests/sequence_nest_rnn.conf new file mode 100644 index 0000000000000..03eef7a217588 --- /dev/null +++ b/paddle/gserver/tests/sequence_nest_rnn.conf @@ -0,0 +1,75 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +######################## data source ################################ +define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', + test_list=None, + module='rnn_data_provider', + obj='process_subseq') + + +settings(batch_size=2, learning_rate=0.01) +######################## network configure ################################ +dict_dim = 10 +word_dim = 8 +hidden_dim = 8 +label_dim = 3 + +data = data_layer(name="word", size=dict_dim) + +emb = embedding_layer(input=data, size=word_dim) + +# This hierachical RNN is designed to be equivalent to the simple RNN in +# sequence_rnn.conf + +def outer_step(x): + outer_mem = memory(name="outer_rnn_state", size=hidden_dim) + def inner_step(y): + inner_mem = memory(name="inner_rnn_state", + size=hidden_dim, + boot_layer=outer_mem) + return fc_layer(input=[y, inner_mem], + size=hidden_dim, + act=TanhActivation(), + bias_attr=True, + name="inner_rnn_state") + + inner_rnn_output = recurrent_group( + step=inner_step, + input=x) + last = last_seq(input=inner_rnn_output, name="outer_rnn_state") + + # "return last" should also work. But currently RecurrentGradientMachine + # does not handle it correctly. Current implementation requires that + # all the out links are from sequences. However, it does not report error + # when the out links are not sequences. + return inner_rnn_output + +out = recurrent_group( + step=outer_step, + input=SubsequenceInput(emb)) + +value_printer_evaluator(input=out) + +rep = last_seq(input=out) +prob = fc_layer(size=label_dim, + input=rep, + act=SoftmaxActivation(), + bias_attr=True) + +outputs(classification_cost(input=prob, + label=data_layer(name="label", size=label_dim))) diff --git a/paddle/gserver/tests/sequence_rnn.conf b/paddle/gserver/tests/sequence_rnn.conf new file mode 100644 index 0000000000000..73e7a5935f6c3 --- /dev/null +++ b/paddle/gserver/tests/sequence_rnn.conf @@ -0,0 +1,57 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +######################## data source ################################ +define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', + test_list=None, + module='rnn_data_provider', + obj='process_seq') + + +settings(batch_size=2, learning_rate=0.01) +######################## network configure ################################ +dict_dim = 10 +word_dim = 8 +hidden_dim = 8 +label_dim = 3 + +data = data_layer(name="word", size=dict_dim) + +emb = embedding_layer(input=data, size=word_dim) + +def step(y): + mem = memory(name="rnn_state", size=hidden_dim) + return fc_layer(input=[y, mem], + size=hidden_dim, + act=TanhActivation(), + bias_attr=True, + name="rnn_state") + +out = recurrent_group( + step=step, + input=emb) + +value_printer_evaluator(input=out) + +rep = last_seq(input=out) +prob = fc_layer(size=label_dim, + input=rep, + act=SoftmaxActivation(), + bias_attr=True) + +outputs(classification_cost(input=prob, + label=data_layer(name="label", size=label_dim))) diff --git a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp index 35d6ee7f4a402..f6989e9a6463a 100644 --- a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp +++ b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp @@ -21,6 +21,8 @@ limitations under the License. */ #include #include +P_DECLARE_int32(seed); + using namespace paddle; // NOLINT using namespace std; // NOLINT class TrainerForTest : public paddle::Trainer { @@ -68,7 +70,9 @@ void CalCost(const string& conf, const string& dir, real* cost, CpuVector vecMomentum(dim); // vecW needs to be assigned, otherwise the variable is an uncertain value. - vecW.zeroMem(); + + *ThreadLocalRand::getSeed() = FLAGS_seed; + vecW.randnorm(0, 0.1); trainer.startTrain(); for (int i = 0; i < num_passes; ++i) { @@ -88,15 +92,13 @@ void CalCost(const string& conf, const string& dir, real* cost, rmDir(dir.c_str()); } -TEST(RecurrentGradientMachine, HasSubSequence) { +void test(const string& conf1, const string& conf2) { int num_passes = 5; real* cost1 = new real[num_passes]; - const string conf1 = "gserver/tests/sequence_layer_group.conf"; const string dir1 = "gserver/tests/t1"; CalCost(conf1, dir1, cost1, num_passes); real* cost2 = new real[num_passes]; - const string conf2 = "gserver/tests/sequence_nest_layer_group.conf"; const string dir2 = "gserver/tests/t2"; CalCost(conf2, dir2, cost2, num_passes); @@ -109,6 +111,17 @@ TEST(RecurrentGradientMachine, HasSubSequence) { delete[] cost2; } +TEST(RecurrentGradientMachine, HasSubSequence) { + test("gserver/tests/sequence_layer_group.conf", + "gserver/tests/sequence_nest_layer_group.conf"); +} + +TEST(RecurrentGradientMachine, rnn) { + test("gserver/tests/sequence_rnn.conf", + "gserver/tests/sequence_nest_rnn.conf"); +} + + int main(int argc, char** argv) { if (paddle::version::isWithPyDataProvider()) { if (!paddle::version::isWithGpu()) { diff --git a/paddle/parameter/Argument.h b/paddle/parameter/Argument.h index 3cab87c700225..5474a05b84101 100644 --- a/paddle/parameter/Argument.h +++ b/paddle/parameter/Argument.h @@ -255,6 +255,15 @@ struct Argument { /* Get Sequence Length, startPositions and max Length according to input + 1. For sequence data: + Each tuple is (seq_length, seq_start, seq_id, seq_id) + The tuples are sorted according to seq_length or subseq_length + *maxSequenceLength is the maximal sequence length + + 2. For subsequence data: + Each tuple is (subseq_length, subseq_start, seq_id, subseq_id) + The tuples are not sorted. They are in the original order. + *maxSequenceLenth is the maximal number of subsequences in each sequence. */ void getSeqLengthAndStart( std::vector>* seqLengthAndStart, From d8366a67fa91586f93d737a0af78422074914829 Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 14 Sep 2016 11:30:30 +0800 Subject: [PATCH 078/324] fix bug on thread barrier --- paddle/math/tests/test_perturbation.cpp | 6 ++++++ paddle/utils/arch/osx/Locks.cpp | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/paddle/math/tests/test_perturbation.cpp b/paddle/math/tests/test_perturbation.cpp index 51e346fef91bf..050f2ca9ced80 100644 --- a/paddle/math/tests/test_perturbation.cpp +++ b/paddle/math/tests/test_perturbation.cpp @@ -254,4 +254,10 @@ int main(int argc, char** argv) { return RUN_ALL_TESTS(); } +#else + +int main(int argc, char const* argv[]) { + return 0; +} + #endif diff --git a/paddle/utils/arch/osx/Locks.cpp b/paddle/utils/arch/osx/Locks.cpp index 5e0411624fd60..8fe482ddddd3c 100644 --- a/paddle/utils/arch/osx/Locks.cpp +++ b/paddle/utils/arch/osx/Locks.cpp @@ -49,7 +49,7 @@ void Semaphore::post() { class SpinLockPrivate { public: - SpinLockPrivate(): lock_(0) {} + SpinLockPrivate(): lock_(OS_SPINLOCK_INIT) {} OSSpinLock lock_; char padding_[64 - sizeof(OSSpinLock)]; // Padding to cache line size @@ -92,7 +92,7 @@ class ThreadBarrierPrivate { inline bool wait() { pthread_mutex_lock(&mutex); ++count; - if (count > tripCount) { + if (count >= tripCount) { count = 0; pthread_cond_broadcast(&cond); pthread_mutex_unlock(&mutex); From 0ba302f7d7680de974690567c003f07154b81d4b Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 14 Sep 2016 14:51:15 +0800 Subject: [PATCH 079/324] fix bug on paddle api when WITH_DOUBLE --- paddle/api/Matrix.cpp | 32 +++++++++++++++--------------- paddle/api/PaddleAPI.h | 45 +++++++++++++++++++++--------------------- paddle/api/Util.cpp | 4 ++-- paddle/api/Vector.cpp | 30 ++++++++++++++-------------- 4 files changed, 56 insertions(+), 55 deletions(-) diff --git a/paddle/api/Matrix.cpp b/paddle/api/Matrix.cpp index 6a79f83495a56..c40a47f3accf9 100644 --- a/paddle/api/Matrix.cpp +++ b/paddle/api/Matrix.cpp @@ -44,7 +44,7 @@ Matrix* Matrix::createZero(size_t height, size_t width, bool useGpu) { return m; } -Matrix* Matrix::createDense(const std::vector& data, size_t height, +Matrix* Matrix::createDense(const std::vector& data, size_t height, size_t width, bool useGpu) { auto m = new Matrix(); m->m->mat = paddle::Matrix::create(height, width, useGpu); @@ -52,7 +52,7 @@ Matrix* Matrix::createDense(const std::vector& data, size_t height, return m; } -Matrix* Matrix::createCpuDenseFromNumpy(float* data, int dim1, int dim2, +Matrix* Matrix::createCpuDenseFromNumpy(real* data, int dim1, int dim2, bool copy) { auto m = new Matrix(); if (copy) { @@ -64,7 +64,7 @@ Matrix* Matrix::createCpuDenseFromNumpy(float* data, int dim1, int dim2, return m; } -Matrix* Matrix::createGpuDenseFromNumpy(float* data, int dim1, int dim2) { +Matrix* Matrix::createGpuDenseFromNumpy(real* data, int dim1, int dim2) { auto m = new Matrix(); m->m->mat = paddle::Matrix::create(dim1, dim2, false, true); m->m->mat->copyFrom(data, dim1 * dim2); @@ -86,7 +86,7 @@ size_t Matrix::getHeight() const { return m->mat->getHeight(); } size_t Matrix::getWidth() const { return m->mat->getWidth(); } -float Matrix::get(size_t x, size_t y) const throw(RangeError) { +real Matrix::get(size_t x, size_t y) const throw(RangeError) { if (x > this->getWidth() || y > this->getHeight()) { RangeError e; throw e; @@ -94,8 +94,8 @@ float Matrix::get(size_t x, size_t y) const throw(RangeError) { return m->mat->getElement(x, y); } -void Matrix::set(size_t x, size_t y, float val) throw(RangeError, - UnsupportError) { +void Matrix::set(size_t x, size_t y, real val) throw(RangeError, + UnsupportError) { if (x > this->getWidth() || y > this->getHeight()) { RangeError e; throw e; @@ -193,10 +193,10 @@ FloatArray Matrix::getData() const { auto rawMat = m->mat.get(); if (dynamic_cast(rawMat->getMemoryHandle().get())) { // is gpu. then copy data - float* data = rawMat->getData(); + real* data = rawMat->getData(); size_t len = rawMat->getElementCnt(); - float* cpuData = new float[len]; - hl_memcpy_device2host(cpuData, data, len * sizeof(float)); + real* cpuData = new real[len]; + hl_memcpy_device2host(cpuData, data, len * sizeof(real)); FloatArray ret_val(cpuData, len); ret_val.needFree = true; return ret_val; @@ -208,7 +208,7 @@ FloatArray Matrix::getData() const { void Matrix::sparseCopyFrom( const std::vector& rows, const std::vector& cols, - const std::vector& vals) throw(UnsupportError) { + const std::vector& vals) throw(UnsupportError) { auto cpuSparseMat = std::dynamic_pointer_cast(m->mat); if (cpuSparseMat != nullptr) { @@ -217,7 +217,7 @@ void Matrix::sparseCopyFrom( // <<" ValSize = "<copyFrom(const_cast&>(rows), const_cast&>(cols), - const_cast&>(vals)); + const_cast&>(vals)); } else { UnsupportError e; throw e; @@ -226,7 +226,7 @@ void Matrix::sparseCopyFrom( void* Matrix::getSharedPtr() const { return &m->mat; } -void Matrix::toNumpyMatInplace(float** view_data, int* dim1, +void Matrix::toNumpyMatInplace(real** view_data, int* dim1, int* dim2) throw(UnsupportError) { auto cpuMat = std::dynamic_pointer_cast(m->mat); if (cpuMat) { @@ -237,9 +237,9 @@ void Matrix::toNumpyMatInplace(float** view_data, int* dim1, throw UnsupportError(); } } -void Matrix::copyToNumpyMat(float** view_m_data, int* dim1, +void Matrix::copyToNumpyMat(real** view_m_data, int* dim1, int* dim2) throw(UnsupportError) { - static_assert(sizeof(paddle::real) == sizeof(float), + static_assert(sizeof(paddle::real) == sizeof(real), "Currently PaddleAPI only support for single " "precision version of paddle."); if (this->isSparse()) { @@ -247,7 +247,7 @@ void Matrix::copyToNumpyMat(float** view_m_data, int* dim1, } else { *dim1 = m->mat->getHeight(); *dim2 = m->mat->getWidth(); - *view_m_data = new float[(*dim1) * (*dim2)]; + *view_m_data = new real[(*dim1) * (*dim2)]; if (auto cpuMat = dynamic_cast(m->mat.get())) { auto src = cpuMat->getData(); auto dest = *view_m_data; @@ -264,7 +264,7 @@ void Matrix::copyToNumpyMat(float** view_m_data, int* dim1, } } -void Matrix::copyFromNumpyMat(float* data, int dim1, +void Matrix::copyFromNumpyMat(real* data, int dim1, int dim2) throw(UnsupportError, RangeError) { if (isSparse()) { throw UnsupportError(); diff --git a/paddle/api/PaddleAPI.h b/paddle/api/PaddleAPI.h index 79487c4cf4d41..69f3240a77974 100644 --- a/paddle/api/PaddleAPI.h +++ b/paddle/api/PaddleAPI.h @@ -20,6 +20,7 @@ limitations under the License. */ #include #include #include "paddle/utils/GlobalConstants.h" +#include "paddle/utils/TypeDefs.h" /// Import PaddlePaddle's enumeration into global namespace. using namespace paddle::enumeration_wrapper; // NOLINT @@ -55,10 +56,10 @@ class UnsupportError {}; /// This type will map to python's list of float. struct FloatArray { - const float* buf; + const real* buf; const size_t length; bool needFree; // true if the buf is dynamic alloced. - FloatArray(const float* b, const size_t l); + FloatArray(const real* b, const size_t l); }; /// This type will map to python's list of int @@ -71,11 +72,11 @@ struct IntArray { /// This type will map to python's list of (int, float) struct IntWithFloatArray { - const float* valBuf; + const real* valBuf; const int* idxBuf; const size_t length; bool needFree; - IntWithFloatArray(const float* v, const int* i, size_t l, bool f = false); + IntWithFloatArray(const real* v, const int* i, size_t l, bool f = false); }; enum SparseValueType { SPARSE_NON_VALUE = 0, SPARSE_VALUE = 1 }; @@ -121,7 +122,7 @@ class Matrix { * @param data list of float should be passed in python. * @note the value will be copy into a new matrix. */ - static Matrix* createDense(const std::vector& data, size_t height, + static Matrix* createDense(const std::vector& data, size_t height, size_t width, bool useGpu = false); /** @@ -133,11 +134,11 @@ class Matrix { * @param copy true if copy into a new matrix, false will create * matrix inplace. */ - static Matrix* createCpuDenseFromNumpy(float* data, int dim1, int dim2, + static Matrix* createCpuDenseFromNumpy(real* data, int dim1, int dim2, bool copy = false); /// Create Gpu Dense Matrix from numpy matrix, dtype=float32 - static Matrix* createGpuDenseFromNumpy(float* data, int dim1, int dim2); + static Matrix* createGpuDenseFromNumpy(real* data, int dim1, int dim2); /** * Cast to numpy matrix. @@ -153,15 +154,15 @@ class Matrix { * numpy_mat = m.toNumpyMat() * @endcode */ - void toNumpyMatInplace(float** view_data, int* dim1, + void toNumpyMatInplace(real** view_data, int* dim1, int* dim2) throw(UnsupportError); /// Copy To numpy mat. - void copyToNumpyMat(float** view_m_data, int* dim1, + void copyToNumpyMat(real** view_m_data, int* dim1, int* dim2) throw(UnsupportError); /// Copy From Numpy Mat - void copyFromNumpyMat(float* data, int dim1, int dim2) throw(UnsupportError, + void copyFromNumpyMat(real* data, int dim1, int dim2) throw(UnsupportError, RangeError); /// return true if this matrix is sparse. @@ -180,9 +181,9 @@ class Matrix { size_t getWidth() const; - float get(size_t x, size_t y) const throw(RangeError); + real get(size_t x, size_t y) const throw(RangeError); - void set(size_t x, size_t y, float val) throw(RangeError, UnsupportError); + void set(size_t x, size_t y, real val) throw(RangeError, UnsupportError); /// return type is list of float FloatArray getData() const; @@ -194,8 +195,8 @@ class Matrix { */ void sparseCopyFrom(const std::vector& rows, const std::vector& cols, - const std::vector& values = - std::vector()) throw(UnsupportError); + const std::vector& values = + std::vector()) throw(UnsupportError); bool isGpu() const; @@ -227,33 +228,33 @@ class Vector { * * It will create a new vector, and copy data into it. */ - static Vector* create(const std::vector& data, bool useGpu = false); + static Vector* create(const std::vector& data, bool useGpu = false); /** * Create Cpu Vector from numpy array, which dtype=float32 * * If copy is false, it will create vector inplace. */ - static Vector* createCpuVectorFromNumpy(float* data, int dim, + static Vector* createCpuVectorFromNumpy(real* data, int dim, bool copy = false); /// Create Gpu Vector from numpy array, which dtype=float32 - static Vector* createGpuVectorFromNumpy(float* data, int dim); + static Vector* createGpuVectorFromNumpy(real* data, int dim); /// Cast to numpy array inplace. - void toNumpyArrayInplace(float** view_data, int* dim1) throw(UnsupportError); + void toNumpyArrayInplace(real** view_data, int* dim1) throw(UnsupportError); /// Copy to numpy array. - void copyToNumpyArray(float** view_m_data, int* dim1); + void copyToNumpyArray(real** view_m_data, int* dim1); /// Copy from numpy array. - void copyFromNumpyArray(float* data, int dim); + void copyFromNumpyArray(real* data, int dim); /// __getitem__ in python - float get(const size_t idx) const throw(RangeError, UnsupportError); + real get(const size_t idx) const throw(RangeError, UnsupportError); /// __setitem__ in python - void set(const size_t idx, float val) throw(RangeError, UnsupportError); + void set(const size_t idx, real val) throw(RangeError, UnsupportError); /// Return is GPU vector or not. bool isGpu() const; diff --git a/paddle/api/Util.cpp b/paddle/api/Util.cpp index 8a6741078f2f1..fe89a62cd3908 100644 --- a/paddle/api/Util.cpp +++ b/paddle/api/Util.cpp @@ -31,13 +31,13 @@ void initPaddle(int argc, char** argv) { feenableexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW); } -FloatArray::FloatArray(const float* b, const size_t l) +FloatArray::FloatArray(const real* b, const size_t l) : buf(b), length(l), needFree(false) {} IntArray::IntArray(const int* b, const size_t l, bool f) : buf(b), length(l), needFree(f) {} -IntWithFloatArray::IntWithFloatArray(const float* v, const int* i, size_t l, +IntWithFloatArray::IntWithFloatArray(const real* v, const int* i, size_t l, bool f) : valBuf(v), idxBuf(i), length(l), needFree(f) {} diff --git a/paddle/api/Vector.cpp b/paddle/api/Vector.cpp index 1affc1a5fefb8..b61eb7934b781 100644 --- a/paddle/api/Vector.cpp +++ b/paddle/api/Vector.cpp @@ -140,7 +140,7 @@ struct VectorPrivate { paddle::VectorPtr vec; void safeAccessData(const size_t idx, - const std::function& func) const + const std::function& func) const throw(RangeError, UnsupportError) { auto cpuVec = std::dynamic_pointer_cast(vec); if (cpuVec != nullptr) { @@ -170,7 +170,7 @@ Vector* Vector::createZero(size_t sz, bool useGpu) { return retVec; } -Vector* Vector::create(const std::vector& data, bool useGpu) { +Vector* Vector::create(const std::vector& data, bool useGpu) { auto retVec = new Vector(); retVec->m->vec = paddle::Vector::create(data.size(), useGpu); retVec->m->vec->copyFrom(data.data(), data.size()); @@ -188,7 +188,7 @@ Vector* Vector::createByPaddleVectorPtr(void* ptr) { } } -Vector* Vector::createCpuVectorFromNumpy(float* data, int dim, bool copy) { +Vector* Vector::createCpuVectorFromNumpy(real* data, int dim, bool copy) { CHECK_GT(dim, 0); auto retVec = new Vector(); if (copy) { @@ -200,7 +200,7 @@ Vector* Vector::createCpuVectorFromNumpy(float* data, int dim, bool copy) { return retVec; } -Vector* Vector::createGpuVectorFromNumpy(float* data, int dim) { +Vector* Vector::createGpuVectorFromNumpy(real* data, int dim) { CHECK_GT(dim, 0); auto retVec = new Vector(); retVec->m->vec = paddle::Vector::create((size_t)dim, true); @@ -208,7 +208,7 @@ Vector* Vector::createGpuVectorFromNumpy(float* data, int dim) { return retVec; } -void Vector::toNumpyArrayInplace(float** view_data, +void Vector::toNumpyArrayInplace(real** view_data, int* dim1) throw(UnsupportError) { auto v = std::dynamic_pointer_cast(m->vec); if (v != nullptr) { @@ -219,20 +219,20 @@ void Vector::toNumpyArrayInplace(float** view_data, } } -void Vector::copyToNumpyArray(float** view_m_data, int* dim1) { +void Vector::copyToNumpyArray(real** view_m_data, int* dim1) { *dim1 = m->vec->getSize(); - *view_m_data = new float[*dim1]; + *view_m_data = new real[*dim1]; if (auto cpuVec = dynamic_cast(m->vec.get())) { - std::memcpy(*view_m_data, cpuVec->getData(), sizeof(float) * (*dim1)); + std::memcpy(*view_m_data, cpuVec->getData(), sizeof(real) * (*dim1)); } else if (auto gpuVec = dynamic_cast(m->vec.get())) { hl_memcpy_device2host(*view_m_data, gpuVec->getData(), - sizeof(float) * (*dim1)); + sizeof(real) * (*dim1)); } else { LOG(INFO) << "Unexpected situation"; } } -void Vector::copyFromNumpyArray(float* data, int dim) { +void Vector::copyFromNumpyArray(real* data, int dim) { m->vec->resize(dim); m->vec->copyFrom(data, dim); } @@ -241,15 +241,15 @@ bool Vector::isGpu() const { return std::dynamic_pointer_cast(m->vec) != nullptr; } -float Vector::get(const size_t idx) const throw(RangeError, UnsupportError) { - float r; - m->safeAccessData(idx, [&](float& o) { r = o; }); +real Vector::get(const size_t idx) const throw(RangeError, UnsupportError) { + real r; + m->safeAccessData(idx, [&](real& o) { r = o; }); return r; } -void Vector::set(const size_t idx, float val) throw(RangeError, +void Vector::set(const size_t idx, real val) throw(RangeError, UnsupportError) { - m->safeAccessData(idx, [&](float& o) { o = val; }); + m->safeAccessData(idx, [&](real& o) { o = val; }); } size_t Vector::getSize() const { return m->vec->getSize(); } From 8a379fce1651640f36bbc979653c62aeedcc1c9c Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 14 Sep 2016 14:52:33 +0800 Subject: [PATCH 080/324] add sse instructions support for double on MAC OS --- paddle/cuda/include/hl_matrix_type.cuh | 2 ++ paddle/cuda/include/hl_sse_matrix_kernel.cuh | 3 +++ 2 files changed, 5 insertions(+) diff --git a/paddle/cuda/include/hl_matrix_type.cuh b/paddle/cuda/include/hl_matrix_type.cuh index 85b60cc313fa7..6917f36290141 100644 --- a/paddle/cuda/include/hl_matrix_type.cuh +++ b/paddle/cuda/include/hl_matrix_type.cuh @@ -27,6 +27,8 @@ typedef float4 vecType; typedef double2 vecType; #endif #else +#include +#include #include #ifndef HPPL_TYPE_DOUBLE typedef __m128 vecType; diff --git a/paddle/cuda/include/hl_sse_matrix_kernel.cuh b/paddle/cuda/include/hl_sse_matrix_kernel.cuh index d774150c21e61..c90d49e4adeb5 100644 --- a/paddle/cuda/include/hl_sse_matrix_kernel.cuh +++ b/paddle/cuda/include/hl_sse_matrix_kernel.cuh @@ -25,6 +25,9 @@ limitations under the License. */ #define VECTOR_LEN 4 #define VECTOR_SET _mm_set_ps1 #else +#if defined(__APPLE__) || defined(__OSX__) +#define _mm_set_pd1 _mm_set1_pd +#endif /* number of double in vector */ #define VECTOR_LEN 2 #define VECTOR_SET _mm_set_pd1 From 1f8c21978d750c3e541dd91cd43cdb232b4ee36e Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 14 Sep 2016 14:54:04 +0800 Subject: [PATCH 081/324] replace fabsf() using std::abs --- paddle/math/tests/test_matrixUtil.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/paddle/math/tests/test_matrixUtil.h b/paddle/math/tests/test_matrixUtil.h index 1310e509877a0..fa682164aa864 100644 --- a/paddle/math/tests/test_matrixUtil.h +++ b/paddle/math/tests/test_matrixUtil.h @@ -124,8 +124,8 @@ void checkSMatrixErr(const CpuSparseMatrixPtr& a, if (a->getValueType() == FLOAT_VALUE) { real aVal = a->getValue()[r]; real bVal = b->getValue()[r]; - if (fabs(aVal - bVal) > err) { - if ((fabsf(aVal - bVal) / fabsf(aVal)) > (err / 10.0f)) { + if (std::abs(aVal - bVal) > err) { + if ((std::abs(aVal - bVal) / std::abs(aVal)) > (err / 10.0f)) { LOG(INFO) << "a=" << aVal << "\t" << "b=" << bVal; count++; } @@ -141,8 +141,8 @@ void checkSMatrixErr(const CpuSparseMatrixPtr& a, if (a->getValueType() == FLOAT_VALUE) { real aVal = a->getValue()[r]; real bVal = b->getValue()[r]; - if (fabs(aVal - bVal) > err) { - if ((fabsf(aVal - bVal) / fabsf(aVal)) > (err / 10.0f)) { + if (std::abs(aVal - bVal) > err) { + if ((std::abs(aVal - bVal) / std::abs(aVal)) > (err / 10.0f)) { count++; } } @@ -173,8 +173,8 @@ void checkMatrixErr(const Matrix& matrix1, const Matrix& matrix2) { for (int j = 0; j < width; j++) { real a = data1[i * width + j]; real b = data2[i * width + j]; - if (fabs(a - b) > err) { - if ((fabsf(a - b) / fabsf(a)) > (err / 10.0f)) { + if (std::abs(a - b) > err) { + if ((std::abs(a - b) / std::abs(a)) > (err / 10.0f)) { count++; } } From 9f3cbed2f2b915a02f4db7dae42746b39543c29e Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Wed, 14 Sep 2016 15:20:47 +0800 Subject: [PATCH 082/324] Add more details for CTC layer, fix CTC evalutor and add their interface test (#74) * Add some comments for CTC layer and fix CTC evalutors, also add interface test --- doc/build/contribute_to_paddle.md | 4 ++-- .../paddle/trainer_config_helpers/evaluators.py | 10 +++++++--- python/paddle/trainer_config_helpers/layers.py | 15 +++++++++++++-- .../tests/layers_test_config.py | 9 +++++++++ 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/doc/build/contribute_to_paddle.md b/doc/build/contribute_to_paddle.md index b3d5fa7c9ff5f..10d5d86311333 100644 --- a/doc/build/contribute_to_paddle.md +++ b/doc/build/contribute_to_paddle.md @@ -25,7 +25,7 @@ repo or just head straight to the command line: ```shell # Clone your fork to your local machine -git clone git@github.com:USERNAME/paddle.git +git clone git@github.com:USERNAME/Paddle.git ``` Then you can start to develop. @@ -52,7 +52,7 @@ To do this, you'll need to add a remote at first: # see the current configured remote repository git remote -v # add upstream repository -git remote add upstream https://github.com/paddle/paddle.git +git remote add upstream https://github.com/baidu/Paddle.git # verify the new upstream git remote -v ``` diff --git a/python/paddle/trainer_config_helpers/evaluators.py b/python/paddle/trainer_config_helpers/evaluators.py index 956bedadd75e5..179a3a053a961 100644 --- a/python/paddle/trainer_config_helpers/evaluators.py +++ b/python/paddle/trainer_config_helpers/evaluators.py @@ -94,7 +94,7 @@ def evaluator_base( Batch=200 samples=20000 AvgCost=0.679655 CurrentCost=0.662179 Eval: classification_error_evaluator=0.4486 CurrentEval: ErrorRate=0.3964 - + :param input: Input layers, a object of LayerOutput or a list of LayerOutput. :type input: list|LayerOutput @@ -296,6 +296,7 @@ def precision_recall_evaluator( @wrap_name_default() def ctc_error_evaluator( input, + label, name=None, ): """ @@ -305,16 +306,19 @@ def ctc_error_evaluator( .. code-block:: python - eval = ctc_error_evaluator(input) + eval = ctc_error_evaluator(input=input, label=lbl) :param name: Evaluator name. :type name: None|basestring :param input: Input Layer. :type input: LayerOutput + :param label: input label, which is a data_layer. + :type label: LayerOutput """ evaluator_base(name=name, type="ctc_edit_distance", - input=input) + input=input, + label=label) @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index f3f0077f9798f..47eadf5d04e50 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -2944,7 +2944,7 @@ def linear_comb_layer(weights, vectors, size, name=None): .. math:: - z = x^T Y + z = x^\mathrm{T} Y In this formular: - :math:`x`: weights @@ -3064,6 +3064,17 @@ def ctc_layer(input, label, size, name=None, norm_by_times=False): classication task. That is, for sequence labeling problems where the alignment between the inputs and the target labels is unknown. + More details can be found by referring to `Connectionist Temporal + Classification: Labelling Unsegmented Sequence Data with Recurrent + Neural Networks `_ + + Note: + Considering the 'blank' label needed by CTC, you need to use + (num_classes + 1) as the input size. num_classes is the category number. + And the 'blank' is the last category index. So the size of 'input' layer, such as + fc_layer with softmax activation, should be num_classes + 1. The size of ctc_layer + should also be num_classes + 1. + The simple usage: .. code-block:: python @@ -3077,7 +3088,7 @@ def ctc_layer(input, label, size, name=None, norm_by_times=False): :type input: LayerOutput :param label: The data layer of label with variable length. :type label: LayerOutput - :param size: category numbers. + :param size: category numbers + 1. :type size: int :param name: The name of this layer, which can not specify. :type name: string|None diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py index ec171fc6013f4..d479fb263fb66 100644 --- a/python/paddle/trainer_config_helpers/tests/layers_test_config.py +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -34,6 +34,15 @@ outputs(classification_cost(out, data_layer(name="label", size=num_classes))) +# for ctc +tmp = fc_layer(input=x1, + size=num_classes + 1, + act=SoftmaxActivation()) +ctc = ctc_layer(input=tmp, + label=y, + size=num_classes + 1) +ctc_eval = ctc_error_evaluator(input=ctc, label=y) + settings( batch_size=10, learning_rate=2e-3, From df0c7cd9c06706e0c0851a42c0e1f2cdb8f656fb Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 14 Sep 2016 19:50:29 +0800 Subject: [PATCH 083/324] Add documentation ci scripts (#66) * Add documentation build to ci test * Push master branch doc changes to gh-pages branch --- .travis.yml | 10 +-- .../travis/{build.sh => build_and_test.sh} | 4 +- paddle/scripts/travis/deploy_key.enc | Bin 0 -> 1680 bytes paddle/scripts/travis/docs.sh | 63 ++++++++++++++++++ paddle/scripts/travis/main.sh | 11 +++ paddle/scripts/travis/make_install.sh | 5 -- paddle/scripts/travis/unittest.sh | 5 -- 7 files changed, 83 insertions(+), 15 deletions(-) rename paddle/scripts/travis/{build.sh => build_and_test.sh} (60%) create mode 100644 paddle/scripts/travis/deploy_key.enc create mode 100755 paddle/scripts/travis/docs.sh create mode 100755 paddle/scripts/travis/main.sh delete mode 100755 paddle/scripts/travis/make_install.sh delete mode 100755 paddle/scripts/travis/unittest.sh diff --git a/.travis.yml b/.travis.yml index a78853e15b158..d3dae9efd416b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: cpp cache: ccache sudo: required dist: trusty +env: + - JOB=DOCS + - JOB=BUILD_AND_TEST addons: apt: packages: @@ -16,6 +19,7 @@ addons: - python2.7-dev - m4 - libprotobuf-dev + - doxygen - protobuf-compiler - python-protobuf - python-numpy @@ -24,12 +28,10 @@ addons: - libgflags-dev - libgtest-dev before_install: - - pip install wheel protobuf + - pip install wheel protobuf sphinx breathe recommonmark - sudo paddle/scripts/travis/before_install.sh script: - - paddle/scripts/travis/build.sh - - paddle/scripts/travis/unittest.sh - - paddle/scripts/travis/make_install.sh + - paddle/scripts/travis/main.sh notifications: email: on_success: change diff --git a/paddle/scripts/travis/build.sh b/paddle/scripts/travis/build_and_test.sh similarity index 60% rename from paddle/scripts/travis/build.sh rename to paddle/scripts/travis/build_and_test.sh index a644f2a4164f8..3ea633be32702 100755 --- a/paddle/scripts/travis/build.sh +++ b/paddle/scripts/travis/build_and_test.sh @@ -1,5 +1,7 @@ #!/bin/bash -cd `dirname $0` source ./common.sh cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON make -j `nproc` +env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j `nproc`" +sudo make install +sudo paddle version diff --git a/paddle/scripts/travis/deploy_key.enc b/paddle/scripts/travis/deploy_key.enc new file mode 100644 index 0000000000000000000000000000000000000000..b0aa45c5ac626c735735fd8541a43bf8b099d0a0 GIT binary patch literal 1680 zcmV;B25yyUqY#1NIzV3W)fPPOg)SfJ)J5*WYWTdF5%54;p8(|fd*x$z<@A&DU_zt zThpN*QPO0+!aSC9<;RB^hjX1A&oEM;0$@rtHQxz@;%~(tgcuxer~cO>Qpn{Rn+7-b zFj(wXoxNk^UF1c0!vIFZ%8{#{AG_w!BV#y$ZpoM>TFS0UP~c_yjiN3}YrEy7m1CJd zkvb$OEPjLWhf}TiyV@DQg?3LZZc|bm1NXkyQ{C~ZrC_`<4yG7&`!KS7)!9Zr9|%U5 zWJ&Ik^}`*I``C9U#xJnkSv;Qh*R4v*jh-IJ#>QA#(Y}~4Za(4C)v8aSHfE@7l%F1C ziB@x}o(c?7Ra>u*BH8;?vm0Cka2KbkAXIw%=KV=b&!rTm~ASFMe=UDdw1p(u!uJ4a@|vRxK|%8o+wC z5EsiEVeC*W18KR9Z?B#^`#>CEsyH#~*Abv!>3W_SMUsQYJ{DU4ZrbwZnt=3@Qi=ltR*wP5nMAWWUaz5S zuaVr=+dI^jDm~2q~dgVCS5982tJrp;Sw#h!_k)my1UinflqM@WoGq3P6t!(6FM>| zegx!9TWw!?g+?75zZ+uhYNW1^|L+G9=6V*ajG9!&!Nok(g4EqEu>VO+tIC~RtHj(* zySCm`tj~1B-sh_go(`tvh4V3kC+=G1@EtLz+?x!I0H~W#sm%`;t1*tO9sh#YEHM!D zs4IgrmN-Q=4l8jIn{;sRCjX9=d@k^ejA;}KnwMZNZ65EWJee2ZX5P~ydjn&301(Zf$LY$mK- zylTPQS?#_=%{m$w*bL6`=E*CDqrlT9xk#qgIV~!N`jOC7vWzW3!zOL`Nwh9Dru{bB zI(07C%WZ6Y?MUEb#<&go$NjhUE%NlB25zp)m&;C$o~P~RpTm&0`(T_p7_@q!ID)80 zDi~n+Snf(b-|f4M4$eoC{A#vE7Z2(jz+X(3r2?@@qJPl)K%~MXx;)zd(AxIF7Kj4X z3hb3dyJ^bJoIg26DY7uisD$bklWozv73{rRdMZ64d`*eV_yNl>8d)bF>4)t*e5}m$ z*@7neNIV;I96z!?sTov23(FjGdR{C4Qz<-c=J;Z6FVMWETH4^S(wRyli5x3pF+Gnw ztp--c`pbgb5etvS&U(ZZ6~nS{UqQj>;7W87BZ3DXi;&U|Eunk$nYL`Yup25`OVkem zqP?Gd&9q!B8b-}dJt#y)#)Wb|=Q~N3(1yj9`4fY5KpAfr6oGix5Mu11S~|Kok<&E! z1`ToGlgP(SX#;M8H|aWrakI09}o>3vSb z;Fi1>16#t{5zB&7)AU)u9Ok`PWz!OW)&JepUHU#oa|26mI7GT95Dt!N3mX- z+{G+b#Lhz!yKuah&i@(8YwD8uJ;)+~G6C3oqdTE>tVyx^R ae@DhPn_m4OxNmJBUw&oN)8D0viSvK*KT_QQ literal 0 HcmV?d00001 diff --git a/paddle/scripts/travis/docs.sh b/paddle/scripts/travis/docs.sh new file mode 100755 index 0000000000000..c2a4809d75b97 --- /dev/null +++ b/paddle/scripts/travis/docs.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Add set -e, cd to directory. +source ./common.sh + +# Compile Documentation only. +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=ON +make paddle_docs paddle_docs_cn + +# Parse Github URL +REPO=`git config remote.origin.url` +SSH_REPO=${REPO/https:\/\/github.com\//git@github.com:} +SHA=`git rev-parse --verify HEAD` + +# Documentation branch name +# gh-pages branch is used for PaddlePaddle.org. The English version of +# documentation in `doc` directory, and the chinese version in `doc_cn` +# directory. +TARGET_BRANCH="gh-pages" + +# Only deploy master branch to build latest documentation. +SOURCE_BRANCH="master" + +# If is not a Github pull request, and in master branch. +if [ "$TRAVIS_PULL_REQUEST" != "false" -o "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then + exit 0 +fi + +# Clone the repo to output directory +git clone $REPO output +cd output + +# checkout github page branch +git checkout $TARGET_BRANCH || git checkout --orphan $TARGET_BRANCH + +# remove old docs. mv new docs. +rm -rf doc doc_cn +mv ../doc_cn/html doc_cn +mv ../doc/html doc + +# Check is there anything changed. +set +e +git diff --exit-code >/dev/null +if [ $? -eq 0 ]; then + echo "No changes to the output on this push; exiting." + exit 0 +fi +set -e + +# Commit +git add . +git config user.name "Travis CI" +git config user.email "paddle-dev@baidu.com" +git commit -m "Deploy to GitHub Pages: ${SHA}" + +# Set ssh private key +openssl aes-256-cbc -K $SSL_KEY -iv $SSL_IV -in ../../paddle/scripts/travis/deploy_key.enc -out deploy_key -d +chmod 600 deploy_key +eval `ssh-agent -s` +ssh-add deploy_key + +# Push +git push $SSH_REPO $TARGET_BRANCH diff --git a/paddle/scripts/travis/main.sh b/paddle/scripts/travis/main.sh new file mode 100755 index 0000000000000..c49d4546c24ac --- /dev/null +++ b/paddle/scripts/travis/main.sh @@ -0,0 +1,11 @@ +#!/bin/bash +cd `dirname $0` + +if [ ${JOB} == "BUILD_AND_TEST" ]; then + ./build_and_test.sh +elif [ ${JOB} == "DOCS" ]; then + ./docs.sh +else + echo Unknown job ${JOB} + exit 1 +fi diff --git a/paddle/scripts/travis/make_install.sh b/paddle/scripts/travis/make_install.sh deleted file mode 100755 index 08b2a648bb97d..0000000000000 --- a/paddle/scripts/travis/make_install.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -cd `dirname $0` -source ./common.sh -sudo make install -sudo paddle version diff --git a/paddle/scripts/travis/unittest.sh b/paddle/scripts/travis/unittest.sh deleted file mode 100755 index 45e8c85c1028e..0000000000000 --- a/paddle/scripts/travis/unittest.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -cd `dirname $0` -source ./common.sh -env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j `nproc`" - From 7e79c911b4d60333a10fa4ab9771eb3cd5e52a63 Mon Sep 17 00:00:00 2001 From: emailweixu Date: Wed, 14 Sep 2016 09:59:47 -0700 Subject: [PATCH 084/324] Temporarily disable automatic doc deployment to restore successful build status. travis DOCS job on master failed due to permisstion issue: https://travis-ci.org/baidu/Paddle/jobs/159851334 @reyoung, please resolve the permission issue. --- paddle/scripts/travis/docs.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paddle/scripts/travis/docs.sh b/paddle/scripts/travis/docs.sh index c2a4809d75b97..432277a01567f 100755 --- a/paddle/scripts/travis/docs.sh +++ b/paddle/scripts/travis/docs.sh @@ -59,5 +59,8 @@ chmod 600 deploy_key eval `ssh-agent -s` ssh-add deploy_key +# Temporarily disable automatic doc deployment due to permission issue. +exit 0 + # Push git push $SSH_REPO $TARGET_BRANCH From f392452308ee7d02617671c09971e4c744ac2379 Mon Sep 17 00:00:00 2001 From: emailweixu Date: Wed, 14 Sep 2016 10:12:46 -0700 Subject: [PATCH 085/324] Update layers_test_config.py --- .../paddle/trainer_config_helpers/tests/layers_test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py index d479fb263fb66..763993231f72f 100644 --- a/python/paddle/trainer_config_helpers/tests/layers_test_config.py +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -41,7 +41,7 @@ ctc = ctc_layer(input=tmp, label=y, size=num_classes + 1) -ctc_eval = ctc_error_evaluator(input=ctc, label=y) +ctc_eval = ctc_error_evaluator(input=tmp, label=y) settings( batch_size=10, From 363b60616c533de05bb9f5305aee43d0f8fa9b14 Mon Sep 17 00:00:00 2001 From: emailweixu Date: Wed, 14 Sep 2016 10:19:48 -0700 Subject: [PATCH 086/324] Update comment for ctc_error_evaluator --- python/paddle/trainer_config_helpers/evaluators.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/paddle/trainer_config_helpers/evaluators.py b/python/paddle/trainer_config_helpers/evaluators.py index 179a3a053a961..985fae9f955c9 100644 --- a/python/paddle/trainer_config_helpers/evaluators.py +++ b/python/paddle/trainer_config_helpers/evaluators.py @@ -310,9 +310,10 @@ def ctc_error_evaluator( :param name: Evaluator name. :type name: None|basestring - :param input: Input Layer. + :param input: Input Layer. Should be the same as the input for ctc_layer. :type input: LayerOutput - :param label: input label, which is a data_layer. + :param label: input label, which is a data_layer. Should be the same as the + label for ctc_layer :type label: LayerOutput """ evaluator_base(name=name, From 703cce35a871c78a5e3f1c33abf7fe788e2bbda5 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 15 Sep 2016 10:05:18 +0800 Subject: [PATCH 087/324] Enable docs update. (#78) Just forget to add deploy_key to this repository. --- paddle/scripts/travis/docs.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/paddle/scripts/travis/docs.sh b/paddle/scripts/travis/docs.sh index 432277a01567f..c2a4809d75b97 100755 --- a/paddle/scripts/travis/docs.sh +++ b/paddle/scripts/travis/docs.sh @@ -59,8 +59,5 @@ chmod 600 deploy_key eval `ssh-agent -s` ssh-add deploy_key -# Temporarily disable automatic doc deployment due to permission issue. -exit 0 - # Push git push $SSH_REPO $TARGET_BRANCH From df28da76c5d1459f924ff2ef6bc5870474924b1d Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Fri, 16 Sep 2016 02:44:45 +0800 Subject: [PATCH 088/324] try to fix bug for CTCErrorEvaluator.cpp when batch_size > 1 (#82) * try to fix bug for ctc_error_evaluator --- .../gserver/evaluators/CTCErrorEvaluator.cpp | 5 ++++- paddle/gserver/tests/test_Evaluator.cpp | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/paddle/gserver/evaluators/CTCErrorEvaluator.cpp b/paddle/gserver/evaluators/CTCErrorEvaluator.cpp index cd4ed19c2ca45..e397c71c877dc 100644 --- a/paddle/gserver/evaluators/CTCErrorEvaluator.cpp +++ b/paddle/gserver/evaluators/CTCErrorEvaluator.cpp @@ -207,7 +207,7 @@ class CTCErrorEvaluator : public Evaluator { real err = 0; err = editDistance( output.value->getData() + output.value->getWidth() * outputStarts[i], - output.value->getHeight(), output.value->getWidth(), + outputStarts[i+1] - outputStarts[i], output.value->getWidth(), label.ids->getData() + labelStarts[i], labelStarts[i + 1] - labelStarts[i]); @@ -224,6 +224,9 @@ class CTCErrorEvaluator : public Evaluator { for (const std::string& name : config_.input_layers()) { arguments.push_back(nn.getLayer(name)->getOutput()); } + } + + virtual void updateSamplesNum(const std::vector& arguments) { numSequences_ += arguments[1].getNumSequences(); } diff --git a/paddle/gserver/tests/test_Evaluator.cpp b/paddle/gserver/tests/test_Evaluator.cpp index 8e857781468fe..3a591a316b8ba 100644 --- a/paddle/gserver/tests/test_Evaluator.cpp +++ b/paddle/gserver/tests/test_Evaluator.cpp @@ -87,18 +87,31 @@ void testEvaluator(TestConfig testConf, string testEvaluatorName, return; } + ICpuGpuVectorPtr sequenceStartPositions; + if (testConf.inputDefs[i].inputType == INPUT_SEQUENCE_DATA || + testConf.inputDefs[i].inputType == INPUT_SEQUENCE_LABEL) { + if (!sequenceStartPositions) { + generateSequenceStartPositions(batchSize, sequenceStartPositions); + } + data.sequenceStartPositions = sequenceStartPositions; + } + arguments.push_back(data); } Evaluator* testEvaluator = Evaluator::create(testConf.evaluatorConfig); double totalScore = 0.0; + testEvaluator->start(); totalScore += testEvaluator->evalImp(arguments); testEvaluator->updateSamplesNum(arguments); + testEvaluator->finish(); LOG(INFO) << *testEvaluator; double totalScore2 = 0.0; if (testConf.testAccumulate) { + testEvaluator->start(); totalScore2 += testEvaluator->evalImp(arguments); + testEvaluator->finish(); EXPECT_LE(fabs(totalScore - totalScore2), 1.0e-5); } } @@ -202,6 +215,15 @@ TEST(Evaluator, precision_recall) { false); } +TEST(Evaluator, ctc_error_evaluator) { + TestConfig config; + config.evaluatorConfig.set_type("ctc_edit_distance"); + + config.inputDefs.push_back({INPUT_SEQUENCE_DATA, "output", 32}); + config.inputDefs.push_back({INPUT_SEQUENCE_LABEL, "label", 1}); + testEvaluatorAll(config, "ctc_error_evaluator", 100); +} + int main(int argc, char** argv) { initMain(argc, argv); FLAGS_thread_local_rand_use_global_seed = true; From a9d327bd878396da499753d74a391b3348f2515d Mon Sep 17 00:00:00 2001 From: Haonan Date: Thu, 15 Sep 2016 21:43:07 -0700 Subject: [PATCH 089/324] add wrapper for out_prod_layer (#83) --- .../paddle/trainer_config_helpers/layers.py | 36 ++++++++++++++++++- .../tests/layers_test_config.py | 4 ++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 47eadf5d04e50..7b8f55abdc9a8 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -52,7 +52,7 @@ 'cross_entropy_with_selfnorm', 'cross_entropy', 'multi_binary_label_cross_entropy', 'rank_cost', 'lambda_cost', 'huber_cost', - 'block_expand_layer', + 'block_expand_layer', 'out_prod_layer', ] @@ -93,6 +93,7 @@ class LayerType(object): POWER_LAYER = 'power' SCALING_LAYER = 'scaling' TRANS_LAYER = 'trans' + OUT_PROD_LAYER = 'out_prod' MEMORY = 'memory' MAXID_LAYER = 'maxid' @@ -2345,6 +2346,39 @@ def maxid_layer(input, name=None, layer_attr=None): layer_type=LayerType.MAXID_LAYER, parents=[input]) +@wrap_name_default() +def out_prod_layer(input1, input2, name=None, layer_attr=None): + """ + A layer for computing the outer product of two vectors + The result is a matrix of size(input1) x size(input2) + + The example usage is: + + .. code-block:: python + + out_prod = out_prod_layer(input1=vec1, input2=vec2) + + :param name: Layer name. + :type name: basestring + :param input1: The first input layer name. + :type input: LayerOutput + :param input2: The second input layer name. + :type input2: LayerOutput + :param layer_attr: extra layer attributes. + :type layer_attr: ExtraLayerAttribute. + :return: LayerOutput object. + :rtype: LayerOutput + """ + + assert isinstance(input1, LayerOutput) + assert isinstance(input2, LayerOutput) + Layer(name=name, + type="out_prod", + inputs=[input1.name, input2.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput(name=name, + layer_type=LayerType.OUT_PROD_LAYER, + parents=[input1,input2]) @wrap_name_default() def eos_layer(input, eos_id, name=None, layer_attr=None): diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py index 763993231f72f..a858582dc449b 100644 --- a/python/paddle/trainer_config_helpers/tests/layers_test_config.py +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -19,6 +19,8 @@ x = data_layer(name="input1", size=3) y = data_layer(name="input2", size=5) +z = out_prod_layer(input1=x, input2=y) + x1 = fc_layer(input=x, size=5) y1 = fc_layer(input=y, size=5) y2 = fc_layer(input=y, size=15) @@ -28,7 +30,7 @@ linear_comb = linear_comb_layer(weights=x1, vectors=y2, size=3) -out = fc_layer(input=[cos1, cos3, linear_comb], +out = fc_layer(input=[cos1, cos3, linear_comb, z], size=num_classes, act=SoftmaxActivation()) From 05a97ab50e1df310fd39d552831922e463fa5e63 Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Wed, 14 Sep 2016 16:14:32 -0700 Subject: [PATCH 090/324] Further fix the memory for Hierarchical RNN Sequences should be sorted according to the number of subsequences they have. --- paddle/cuda/src/hl_cuda_matrix.cu | 1 + .../RecurrentGradientMachine.cpp | 115 +++++++++--------- .../RecurrentGradientMachine.h | 6 +- paddle/gserver/layers/PrintLayer.cpp | 58 +++++++++ paddle/gserver/tests/sequence_nest_rnn.conf | 7 +- paddle/gserver/tests/sequence_rnn.conf | 6 +- .../tests/test_RecurrentGradientMachine.cpp | 13 +- paddle/parameter/Argument.cpp | 65 ++++------ paddle/parameter/Argument.h | 36 +++--- python/paddle/trainer/config_parser.py | 8 ++ .../paddle/trainer_config_helpers/layers.py | 17 ++- .../tests/layers_test_config.py | 2 + 12 files changed, 206 insertions(+), 128 deletions(-) create mode 100644 paddle/gserver/layers/PrintLayer.cpp diff --git a/paddle/cuda/src/hl_cuda_matrix.cu b/paddle/cuda/src/hl_cuda_matrix.cu index fc003b7d6377d..ecc44944e4fa1 100644 --- a/paddle/cuda/src/hl_cuda_matrix.cu +++ b/paddle/cuda/src/hl_cuda_matrix.cu @@ -19,6 +19,7 @@ limitations under the License. */ #include "hl_matrix_apply.cuh" #include "hl_sequence.h" #include "paddle/utils/Logging.h" +#include "hl_device_functions.cuh" DEFINE_MATRIX_UNARY_OP(Zero, a = 0); DEFINE_MATRIX_TERNARY_PARAMETER_OP(_add, TWO_PARAMETER, c = p1*a + p2*b); diff --git a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp index 96b0e19880b68..bf7aa1c8d89ae 100644 --- a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp @@ -434,23 +434,25 @@ void RecurrentGradientMachine::forward(const std::vector& inArgs, } } - seqLengthAndStart_.clear(); info_.clear(); info_.resize(inFrameLines_.size()); - seqLengthAndStart_.resize(inFrameLines_.size()); + + seqInfos_.clear(); + seqInfos_.resize(inFrameLines_.size()); { AsyncGpuBlock asyncGpuBlock; // if shareInlinkInfo, only calculate info of the first inlink // else, calculate info for each inlink if (shareInlinkInfo) { - input.getSeqLengthAndStart(&seqLengthAndStart_[0], &maxSequenceLength_); + input.getSeqInfo(&seqInfos_[0]); + maxSequenceLength_ = seqInfos_[0][0].topLevelLength; createInFrameInfo(0, input, passType); } else { for (size_t i = 0; i < inFrameLines_.size(); i++) { const Argument& input1 = inFrameLines_[i].inLayer->getOutput(); - input1.getSeqLengthAndStart(&seqLengthAndStart_[i], - &maxSequenceLength_); + input1.getSeqInfo(&seqInfos_[i]); + maxSequenceLength_ = seqInfos_[i][0].topLevelLength; createInFrameInfo(i, input1, passType); } } @@ -614,7 +616,7 @@ void RecurrentGradientMachine::removeBeamSearchStatisticsCallbacks() { * for all realLayer of inFrameLines one time. */ -void RecurrentGradientMachine::createInFrameInfo(int inlinks_id, +void RecurrentGradientMachine::createInFrameInfo(int inlinkId, const Argument& input, PassType passType) { bool hasSubseq = input.hasSubseq(); @@ -622,66 +624,67 @@ void RecurrentGradientMachine::createInFrameInfo(int inlinks_id, size_t numSequences = input.getNumSequences(); std::vector allIds; + auto& seqInfo = seqInfos_[inlinkId]; + numSeqs_.clear(); - Info* inlink_info = &info_[inlinks_id]; - inlink_info->idIndex.clear(); - inlink_info->idIndex.push_back(0); // first idIndex = 0 + Info* inlinkInfo = &info_[inlinkId]; + inlinkInfo->idIndex.clear(); + inlinkInfo->idIndex.push_back(0); // first idIndex = 0 + + std::vector sequenceStartPositions; + const int* subSequenceStartPositions = nullptr; + if (hasSubseq) { // for sequenceScatterAgentLayer - // numSubSequences : all sentences within all samples(batch) - size_t numSubSequences = input.getNumSubSequences(); - std::vector sequenceStartPositions; - inlink_info->seqStartPosIndex.clear(); - inlink_info->seqStartPosIndex.push_back(0); // first seqStartPosIndex = 0 - // maxSequenceLength_: max number of sentences(subseq) in allsamples - for (int i = 0; i < maxSequenceLength_; ++i) { + subSequenceStartPositions = + input.subSequenceStartPositions->getData(false); + inlinkInfo->seqStartPosIndex.clear(); + inlinkInfo->seqStartPosIndex.push_back(0); // first seqStartPosIndex = 0 + } + // maxSequenceLength_: max topLevelLength in allsamples + for (int i = 0; i < maxSequenceLength_; ++i) { + if (hasSubseq) { sequenceStartPositions.push_back(0); // first element = 0 - int numSeqs = 0; - for (size_t j = 0; j < numSubSequences; ++j) { // for each sentence - // seqLengthAndStart_[inlinks_id][j]: - // a 4-tuple including - if (std::get<3>(seqLengthAndStart_[inlinks_id][j]) == i) { - ++numSeqs; - // subseqstart: the cpuSubSequenceStartPositions of this subseq - int subSeqStart = std::get<1>(seqLengthAndStart_[inlinks_id][j]); - int subSeqLength = std::get<0>(seqLengthAndStart_[inlinks_id][j]); - for (int k = subSeqStart; k < subSeqStart + subSeqLength; ++k) { - allIds.push_back(k); - } - sequenceStartPositions.push_back(sequenceStartPositions.back() + - subSeqLength); - } - } - inlink_info->idIndex.push_back(allIds.size()); - inlink_info->seqStartPosIndex.push_back(sequenceStartPositions.size()); - numSeqs_.push_back(numSeqs); } - // inFrameLine create sequenceStartPositions one time - CHECK_EQ(sequenceStartPositions.size(), - maxSequenceLength_ + numSubSequences); - CHECK_EQ(inlink_info->seqStartPosIndex.size(), - static_cast(maxSequenceLength_ + 1)); - createSeqPos(sequenceStartPositions, &inlink_info->sequenceStartPositions); - } else { // for scatterAgentLayer - for (int i = 0; i < maxSequenceLength_; ++i) { - int numSeqs = 0; - for (size_t j = 0; j < numSequences; ++j) { - int seqLength = std::get<0>(seqLengthAndStart_[inlinks_id][j]); - if (i >= seqLength) { - break; + int numSeqs = 0; + for (size_t j = 0; j < numSequences; ++j) { + int seqLength = seqInfo[j].topLevelLength; + if (i >= seqLength) { + break; + } + ++numSeqs; + if (hasSubseq) { + int subSeqStart = subSequenceStartPositions[seqInfo[j].subSeqStart + i]; + int subSeqEnd = + subSequenceStartPositions[seqInfo[j].subSeqStart + i + 1]; + for (int k = subSeqStart; k < subSeqEnd; ++k) { + allIds.push_back(k); } - ++numSeqs; - int seqStart = std::get<1>(seqLengthAndStart_[inlinks_id][j]); + sequenceStartPositions.push_back(sequenceStartPositions.back() + + subSeqEnd - subSeqStart); + } else { + int seqStart = seqInfo[j].seqStart; allIds.push_back(reversed_ ? (seqStart + seqLength - 1 - i) : (seqStart + i)); } - inlink_info->idIndex.push_back(allIds.size()); - numSeqs_.push_back(numSeqs); } + inlinkInfo->idIndex.push_back(allIds.size()); + numSeqs_.push_back(numSeqs); + if (hasSubseq) { + inlinkInfo->seqStartPosIndex.push_back(sequenceStartPositions.size()); + } + } + if (hasSubseq) { + // inFrameLine create sequenceStartPositions one time + CHECK_EQ(sequenceStartPositions.size(), + maxSequenceLength_ + input.getNumSubSequences()); + CHECK_EQ(inlinkInfo->seqStartPosIndex.size(), + static_cast(maxSequenceLength_ + 1)); + createSeqPos(sequenceStartPositions, &inlinkInfo->sequenceStartPositions); } // copy and check scatterId - copyScattedId(allIds, &inlink_info->allIds, input.getBatchSize()); - CHECK_EQ(inlink_info->idIndex.size(), + copyScattedId(allIds, &inlinkInfo->allIds, input.getBatchSize()); + CHECK_EQ(inlinkInfo->idIndex.size(), static_cast(maxSequenceLength_ + 1)); } @@ -701,7 +704,7 @@ void RecurrentGradientMachine::createMemoryFrameInfo( const int* starts = input.sequenceStartPositions->getData(false); for (size_t i = 0; i < numSequences; ++i) { // memory info adopt info of inlinks[0] - int seqId = std::get<2>(seqLengthAndStart_[0][i]); + int seqId = seqInfos_[0][i].seqId; for (int k = starts[seqId]; k < starts[seqId + 1]; ++k) { allIds.push_back(k); } @@ -713,7 +716,7 @@ void RecurrentGradientMachine::createMemoryFrameInfo( } else { // for scatterAgentLayer for (size_t i = 0; i < numSequences; ++i) { - allIds.push_back(std::get<2>(seqLengthAndStart_[0][i])); + allIds.push_back(seqInfos_[0][i].seqId); } } // copy and check scatterId diff --git a/paddle/gserver/gradientmachines/RecurrentGradientMachine.h b/paddle/gserver/gradientmachines/RecurrentGradientMachine.h index d9901ad81d60a..6328213793ed6 100644 --- a/paddle/gserver/gradientmachines/RecurrentGradientMachine.h +++ b/paddle/gserver/gradientmachines/RecurrentGradientMachine.h @@ -337,11 +337,7 @@ class RecurrentGradientMachine : public NeuralNetwork { // data) or has more than i subsequences (for subsequence data) std::vector numSeqs_; - // each inlinks has a "std::vector>" denotes - // its sequence info: - // if hasSubSeq, tuple of (subSeqLength, subSeqStart, seqIndex, subSeqIndex) - // else, tuple of (seqLength, seqStart, seqIndex, seqIndex) - std::vector>> seqLengthAndStart_; + std::vector> seqInfos_; // the id of inlink which share info with outlinks int targetInfoInlinkId_; diff --git a/paddle/gserver/layers/PrintLayer.cpp b/paddle/gserver/layers/PrintLayer.cpp new file mode 100644 index 0000000000000..68fee69f44d0c --- /dev/null +++ b/paddle/gserver/layers/PrintLayer.cpp @@ -0,0 +1,58 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "Layer.h" + +namespace paddle { + +class PrintLayer : public Layer { +public: + explicit PrintLayer(const LayerConfig& config) + : Layer(config) {} + void forward(PassType passType); + void backward(const UpdateCallback& callback) {} +}; + +void PrintLayer::forward(PassType passType) { + Layer::forward(passType); + for (size_t i = 0; i != inputLayers_.size(); ++i) { + const auto& argu = getInput(i); + const std::string& name = inputLayers_[i]->getName(); + if (argu.value) { + std::ostringstream os; + argu.value->print(os); + LOG(INFO) << "layer=" << name << " value matrix:\n" << os.str(); + } + if (argu.ids) { + std::ostringstream os; + argu.ids->print(os, argu.ids->getSize()); + LOG(INFO) << "layer=" << name << " ids vector:\n" << os.str(); + } + if (auto startPos = argu.sequenceStartPositions) { + std::ostringstream os; + startPos->getVector(false)->print(os, startPos->getSize()); + LOG(INFO) << "layer=" << name << " sequence pos vector:\n" << os.str(); + } + if (auto subStartPos = argu.subSequenceStartPositions) { + std::ostringstream os; + subStartPos->getVector(false)->print(os, subStartPos->getSize()); + LOG(INFO) << "layer=" << name << " sub-sequence pos vector:\n" + << os.str(); + } + } +} + +REGISTER_LAYER(print, PrintLayer); + +} // namespace paddle diff --git a/paddle/gserver/tests/sequence_nest_rnn.conf b/paddle/gserver/tests/sequence_nest_rnn.conf index 03eef7a217588..62b8c5d072d7b 100644 --- a/paddle/gserver/tests/sequence_nest_rnn.conf +++ b/paddle/gserver/tests/sequence_nest_rnn.conf @@ -42,14 +42,16 @@ def outer_step(x): inner_mem = memory(name="inner_rnn_state", size=hidden_dim, boot_layer=outer_mem) - return fc_layer(input=[y, inner_mem], + out = fc_layer(input=[y, inner_mem], size=hidden_dim, act=TanhActivation(), bias_attr=True, name="inner_rnn_state") + return out inner_rnn_output = recurrent_group( step=inner_step, + name="inner", input=x) last = last_seq(input=inner_rnn_output, name="outer_rnn_state") @@ -60,11 +62,10 @@ def outer_step(x): return inner_rnn_output out = recurrent_group( + name="outer", step=outer_step, input=SubsequenceInput(emb)) -value_printer_evaluator(input=out) - rep = last_seq(input=out) prob = fc_layer(size=label_dim, input=rep, diff --git a/paddle/gserver/tests/sequence_rnn.conf b/paddle/gserver/tests/sequence_rnn.conf index 73e7a5935f6c3..3294c2c3fc431 100644 --- a/paddle/gserver/tests/sequence_rnn.conf +++ b/paddle/gserver/tests/sequence_rnn.conf @@ -35,18 +35,18 @@ emb = embedding_layer(input=data, size=word_dim) def step(y): mem = memory(name="rnn_state", size=hidden_dim) - return fc_layer(input=[y, mem], + out = fc_layer(input=[y, mem], size=hidden_dim, act=TanhActivation(), bias_attr=True, name="rnn_state") + return out out = recurrent_group( + name="rnn", step=step, input=emb) -value_printer_evaluator(input=out) - rep = last_seq(input=out) prob = fc_layer(size=label_dim, input=rep, diff --git a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp index f6989e9a6463a..b73fdd18abf35 100644 --- a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp +++ b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp @@ -92,7 +92,7 @@ void CalCost(const string& conf, const string& dir, real* cost, rmDir(dir.c_str()); } -void test(const string& conf1, const string& conf2) { +void test(const string& conf1, const string& conf2, double eps) { int num_passes = 5; real* cost1 = new real[num_passes]; const string dir1 = "gserver/tests/t1"; @@ -104,8 +104,9 @@ void test(const string& conf1, const string& conf2) { for (int i = 0; i < num_passes; i++) { LOG(INFO) << "num_passes: " << i << ", cost1=" << cost1[i] - << ", cost2=" << cost2[i]; - ASSERT_NEAR(cost1[i], cost2[i], 1e-3); + << ", cost2=" << cost2[i] + << ", diff=" << std::abs(cost1[i] - cost2[i]); + ASSERT_NEAR(cost1[i], cost2[i], eps); } delete[] cost1; delete[] cost2; @@ -113,12 +114,14 @@ void test(const string& conf1, const string& conf2) { TEST(RecurrentGradientMachine, HasSubSequence) { test("gserver/tests/sequence_layer_group.conf", - "gserver/tests/sequence_nest_layer_group.conf"); + "gserver/tests/sequence_nest_layer_group.conf", + 1e-5); } TEST(RecurrentGradientMachine, rnn) { test("gserver/tests/sequence_rnn.conf", - "gserver/tests/sequence_nest_rnn.conf"); + "gserver/tests/sequence_nest_rnn.conf", + 0); } diff --git a/paddle/parameter/Argument.cpp b/paddle/parameter/Argument.cpp index a81c72aacbed3..0ca56b29b39b3 100644 --- a/paddle/parameter/Argument.cpp +++ b/paddle/parameter/Argument.cpp @@ -477,51 +477,34 @@ void Argument::splitByDataId(const std::vector& argus, } } -void Argument::getSeqLengthAndStart( - std::vector>* seqLengthAndStart, - int* maxSequenceLength) const { +void Argument::getSeqInfo(std::vector* seqInfo) const { const int* starts = sequenceStartPositions->getData(false); - if (hasSubseq()) { - size_t numSubSequences = getNumSubSequences(); - (*seqLengthAndStart).reserve(numSubSequences); - const int* subStarts = subSequenceStartPositions->getData(false); - int seqIndex = 0; - int subSeqIndex = 0; - *maxSequenceLength = 0; - for (size_t i = 0; i < numSubSequences; ++i) { - if (subStarts[i] == starts[seqIndex]) { - subSeqIndex = 0; - (*seqLengthAndStart) - .push_back(std::make_tuple( - subStarts[i + 1] - subStarts[i], (int)subStarts[i], - (int)seqIndex, (int)subSeqIndex)); - ++subSeqIndex; - ++seqIndex; - } else if (subStarts[i] < starts[seqIndex]) { - (*seqLengthAndStart) - .push_back(std::make_tuple( - subStarts[i + 1] - subStarts[i], (int)subStarts[i], - (int)seqIndex - 1, (int)subSeqIndex)); - ++subSeqIndex; + const int* subStarts = hasSubseq() + ? subSequenceStartPositions->getData(false) : nullptr; + size_t numSequences = getNumSequences(); + seqInfo->reserve(numSequences); + int subSeqEnd = 0; + for (size_t i = 0; i < numSequences; ++i) { + SeqInfo info; + info.seqStart = starts[i]; + info.subLevelLength = starts[i + 1] - starts[i]; + info.seqId = i; + if (hasSubseq()) { + info.subSeqStart = subSeqEnd; + while (subStarts[subSeqEnd] < starts[i + 1]) { + ++subSeqEnd; } - // maxSequenceLength_ = 1 + max(subSeqIndex) in each Seq. - if (*maxSequenceLength < std::get<3>((*seqLengthAndStart)[i])) - *maxSequenceLength = std::get<3>((*seqLengthAndStart)[i]); - } - *maxSequenceLength += 1; - } else { - size_t numSequences = getNumSequences(); - (*seqLengthAndStart).reserve(numSequences); - for (size_t i = 0; i < numSequences; ++i) { - (*seqLengthAndStart) - .push_back(std::make_tuple( - starts[i + 1] - starts[i], (int)starts[i], (int)i, (int)i)); + info.topLevelLength = subSeqEnd - info.subSeqStart; + } else { + info.topLevelLength = info.subLevelLength; + info.subSeqStart = 0; // not used } - std::sort((*seqLengthAndStart).begin(), (*seqLengthAndStart).end(), - std::greater>()); - - *maxSequenceLength = std::get<0>((*seqLengthAndStart)[0]); + seqInfo->push_back(info); } + std::sort(seqInfo->begin(), seqInfo->end(), + [](const SeqInfo& a, const SeqInfo& b) { + return a.topLevelLength > b.topLevelLength; + }); } void Argument::checkSubset() const { diff --git a/paddle/parameter/Argument.h b/paddle/parameter/Argument.h index 5474a05b84101..81cd117fc45cf 100644 --- a/paddle/parameter/Argument.h +++ b/paddle/parameter/Argument.h @@ -253,21 +253,29 @@ struct Argument { static void splitByDataId(const std::vector& argus, std::vector>* arguGroups); + struct SeqInfo { + // Equal to sequence length for sequence data + // Equal to number of subsequences for subsequence data + int topLevelLength; + + int seqStart; + int seqId; + + // Equal to topLevelLength for sequence data + // Equal to sum of the length of subsequences for subsequence data + int subLevelLength; + + // Only used for subsequence data, start position of this sequence + // is subSequenceStartPositions, i.e. + // subSequenceStartPositions[subSeqStart] == seqStart + int subSeqStart; + }; /* - Get Sequence Length, startPositions and max Length according to input - 1. For sequence data: - Each tuple is (seq_length, seq_start, seq_id, seq_id) - The tuples are sorted according to seq_length or subseq_length - *maxSequenceLength is the maximal sequence length - - 2. For subsequence data: - Each tuple is (subseq_length, subseq_start, seq_id, subseq_id) - The tuples are not sorted. They are in the original order. - *maxSequenceLenth is the maximal number of subsequences in each sequence. - */ - void getSeqLengthAndStart( - std::vector>* seqLengthAndStart, - int* maxSequenceLength) const; + Get SeqInfo for each sequence of this argument + Elements in *seqInfo are sorted by topLevelLength in descending order + */ + void getSeqInfo(std::vector* segInfo) const; + /* Check Whether sequenceStartPositions is subset of subSequenceStartPositions. diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 7a6053940178d..f2f67f9bd66a4 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1408,6 +1408,14 @@ def __init__( input_index, psize, dims, sparse, format) self.create_bias_parameter(bias, self.config.size) +@config_layer('print') +class PrintLayer(LayerBase): + def __init__( + self, + name, + inputs): + super(PrintLayer, self).__init__(name, 'print', 0, inputs) + @config_layer('data') class DataLayer(LayerBase): def __init__( diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 7b8f55abdc9a8..dfcabfdb8430b 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -52,7 +52,7 @@ 'cross_entropy_with_selfnorm', 'cross_entropy', 'multi_binary_label_cross_entropy', 'rank_cost', 'lambda_cost', 'huber_cost', - 'block_expand_layer', 'out_prod_layer', + 'block_expand_layer', 'out_prod_layer', 'print_layer' ] @@ -108,6 +108,8 @@ class LayerType(object): LINEAR_COMBINATION_LAYER = "convex_comb" BLOCK_EXPAND = "blockexpand" + PRINT_LAYER = "print" + CTC_LAYER = "ctc" CRF_LAYER = "crf" CRF_DECODING_LAYER = "crf_decoding" @@ -729,6 +731,19 @@ def __idx_to_input__(i): return LayerOutput(name, LayerType.FC_LAYER, input, activation=act, size=size) +@wrap_name_default("print") +def print_layer(input, name=None): + """ + Print the output value of input layers. This layer is useful for debugging. + """ + assert isinstance(input, list) + + Layer( + name=name, + type=LayerType.PRINT_LAYER, + inputs=[l.name for l in input], + ) + return LayerOutput(name, LayerType.PRINT_LAYER, input) @wrap_name_default("seq_pooling") @wrap_bias_attr_default(has_bias=False) diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py index a858582dc449b..39c85c788eeca 100644 --- a/python/paddle/trainer_config_helpers/tests/layers_test_config.py +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -34,6 +34,8 @@ size=num_classes, act=SoftmaxActivation()) +print_layer(input=[out]) + outputs(classification_cost(out, data_layer(name="label", size=num_classes))) # for ctc From 9d12ca95e3ce0a490e8d8d52976f00b24c0a9142 Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Wed, 14 Sep 2016 16:14:32 -0700 Subject: [PATCH 091/324] Further fix the memory for Hierarchical RNN Sequences should be sorted according to the number of subsequences they have. --- .../paddle/trainer_config_helpers/layers.py | 32 +++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index dfcabfdb8430b..bda0b4f5d60e8 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -21,7 +21,6 @@ from .poolings import MaxPooling, AvgPooling, BasePoolingType from .attrs import * from .default_decorators import * - try: import cPickle as pickle except ImportError: @@ -204,6 +203,25 @@ def __str__(self): DROPOUT = 'drop_rate' +def check_input(input): + """ + Check input is a LayerOutput or list of LayerOutput or tuple of LayerOutput + if is a LayerOutput, + + :param input: The input layer. Could be a list/tuple of input layer. + :type input: LayerOutput|list|tuple + :return: list of LayerOutput + :rtype: list of LayerOutput + """ + + if isinstance(input, LayerOutput): + return [LayerOutput] + assert isinstance(input, list) + for inp in input: + assert isinstance(inp, LayerOutput) + return list(input) + + def layer_support(*attrs): def decorator(method): @functools.wraps(method) @@ -731,19 +749,27 @@ def __idx_to_input__(i): return LayerOutput(name, LayerType.FC_LAYER, input, activation=act, size=size) + @wrap_name_default("print") def print_layer(input, name=None): """ Print the output value of input layers. This layer is useful for debugging. + + :param name: The Layer Name. + :type name: basestring + :param input: The input layer. Could be a list/tuple of input layer. + :type input: LayerOutput|list|tuple + :return: No return """ - assert isinstance(input, list) + check_input(input) Layer( name=name, type=LayerType.PRINT_LAYER, inputs=[l.name for l in input], ) - return LayerOutput(name, LayerType.PRINT_LAYER, input) + LayerOutput(name, LayerType.PRINT_LAYER, input) + @wrap_name_default("seq_pooling") @wrap_bias_attr_default(has_bias=False) From 48eb5ff01f035ceb56f37136623aaa3d30d2fb2e Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Fri, 16 Sep 2016 14:57:02 -0700 Subject: [PATCH 092/324] Fix signed/unsigned comparison for gcc 4.9 --- paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp index bf7aa1c8d89ae..bee82faa5fca8 100644 --- a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp @@ -676,7 +676,8 @@ void RecurrentGradientMachine::createInFrameInfo(int inlinkId, if (hasSubseq) { // inFrameLine create sequenceStartPositions one time CHECK_EQ(sequenceStartPositions.size(), - maxSequenceLength_ + input.getNumSubSequences()); + static_cast(maxSequenceLength_ + + input.getNumSubSequences())); CHECK_EQ(inlinkInfo->seqStartPosIndex.size(), static_cast(maxSequenceLength_ + 1)); createSeqPos(sequenceStartPositions, &inlinkInfo->sequenceStartPositions); From 75beeaf743ae717a91aab5f2a6c5cd43f771b557 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 19 Sep 2016 13:37:31 +0800 Subject: [PATCH 093/324] Fix unit test stack trace bug on MAC OS --- paddle/utils/tests/test_CustomStackTrace.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/utils/tests/test_CustomStackTrace.cpp b/paddle/utils/tests/test_CustomStackTrace.cpp index 26ca4c678a650..3e665021471cb 100644 --- a/paddle/utils/tests/test_CustomStackTrace.cpp +++ b/paddle/utils/tests/test_CustomStackTrace.cpp @@ -45,6 +45,7 @@ void testNormalImpl(const std::function 0) { startBarrier.wait(); + sleep(1); doneBarrier.wait(); ASSERT_TRUE(tracer.empty()); } From 4e37b226f4bc79709d81c2d00cabaac02876d53b Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 19 Sep 2016 13:40:44 +0800 Subject: [PATCH 094/324] Revise member variable in private barrier class --- paddle/utils/arch/osx/Locks.cpp | 35 +++++++++++++++++---------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/paddle/utils/arch/osx/Locks.cpp b/paddle/utils/arch/osx/Locks.cpp index 8fe482ddddd3c..47e44e9d7c114 100644 --- a/paddle/utils/arch/osx/Locks.cpp +++ b/paddle/utils/arch/osx/Locks.cpp @@ -17,6 +17,7 @@ limitations under the License. */ #include #include namespace paddle { + class SemaphorePrivate { public: ~SemaphorePrivate() { @@ -69,20 +70,20 @@ void SpinLock::unlock() { class ThreadBarrierPrivate { public: - pthread_mutex_t mutex; - pthread_cond_t cond; - int count; - int tripCount; + pthread_mutex_t mutex_; + pthread_cond_t cond_; + int count_; + int tripCount_; - inline explicit ThreadBarrierPrivate(int cnt):count(0), tripCount(cnt) { + inline explicit ThreadBarrierPrivate(int cnt):count_(0), tripCount_(cnt) { CHECK_NE(cnt, 0); - CHECK_GE(pthread_mutex_init(&mutex, 0), 0); - CHECK_GE(pthread_cond_init(&cond, 0), 0); + CHECK_GE(pthread_mutex_init(&mutex_, 0), 0); + CHECK_GE(pthread_cond_init(&cond_, 0), 0); } inline ~ThreadBarrierPrivate() { - pthread_cond_destroy(&cond); - pthread_mutex_destroy(&mutex); + pthread_cond_destroy(&cond_); + pthread_mutex_destroy(&mutex_); } /** @@ -90,16 +91,16 @@ class ThreadBarrierPrivate { * @return true if the last wait */ inline bool wait() { - pthread_mutex_lock(&mutex); - ++count; - if (count >= tripCount) { - count = 0; - pthread_cond_broadcast(&cond); - pthread_mutex_unlock(&mutex); + pthread_mutex_lock(&mutex_); + ++count_; + if (count_ >= tripCount_) { + count_ = 0; + pthread_cond_broadcast(&cond_); + pthread_mutex_unlock(&mutex_); return true; } else { - pthread_cond_wait(&cond, &mutex); - pthread_mutex_unlock(&mutex); + pthread_cond_wait(&cond_, &mutex_); + pthread_mutex_unlock(&mutex_); return false; } } From 494f5a836b8d0714b57d3f711715e4442334906a Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 19 Sep 2016 17:13:44 +0800 Subject: [PATCH 095/324] Add findZLIB in cmake * Also fix a bug, we should link -ldl at last --- CMakeLists.txt | 3 ++- cmake/util.cmake | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 007f1f18bb655..1bb73f8b98fbd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,7 +14,8 @@ find_package(CUDA QUIET) find_package(Protobuf REQUIRED) find_package(PythonLibs 2.7 REQUIRED) find_package(PythonInterp 2.7 REQUIRED) -find_package(NumPy) +find_package(ZLIB REQUIRED) +find_package(NumPy REQUIRED) find_package(Threads REQUIRED) find_package(Glog) find_package(Gflags QUIET) diff --git a/cmake/util.cmake b/cmake/util.cmake index e0e372fed0b04..5b56304656e38 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -67,9 +67,9 @@ function(link_paddle_exe TARGET_NAME) ${PROTOBUF_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${CBLAS_LIBS} - ${CMAKE_DL_LIBS} + ${ZLIB_LIBRARIES} ${INTERAL_LIBS} - -lz) + ${CMAKE_DL_LIBS}) if(WITH_PYTHON) target_link_libraries(${TARGET_NAME} From 2d13462a2ccb358cc0b09ddb1b12ef23a68e9742 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 19 Sep 2016 17:14:11 +0800 Subject: [PATCH 096/324] Fix incompatible on CUDA atomicAdd operation --- paddle/cuda/include/hl_device_functions.cuh | 49 ++++++++++++--------- paddle/cuda/include/hl_gpu_lstm.cuh | 6 +-- paddle/cuda/src/hl_cuda_lstm.cu | 6 +-- paddle/cuda/src/hl_cuda_matrix.cu | 4 +- paddle/cuda/src/hl_cuda_sequence.cu | 2 +- paddle/cuda/src/hl_cuda_sparse.cuh | 10 ++--- paddle/cuda/src/hl_table_apply.cu | 2 +- 7 files changed, 44 insertions(+), 35 deletions(-) diff --git a/paddle/cuda/include/hl_device_functions.cuh b/paddle/cuda/include/hl_device_functions.cuh index 27e3f450c5c1c..88d950d6c1713 100755 --- a/paddle/cuda/include/hl_device_functions.cuh +++ b/paddle/cuda/include/hl_device_functions.cuh @@ -16,28 +16,37 @@ limitations under the License. */ #ifndef HL_DEVICE_FUNCTIONS_CUH_ #define HL_DEVICE_FUNCTIONS_CUH_ -namespace hppl { - -static __inline__ __device__ double atomicAdd(double* address, double val) { - // NOLINTNEXTLINE - unsigned long long int* address_as_ull = (unsigned long long int*)address; - unsigned long long int old = *address_as_ull, assumed; // NOLINT - - do { - assumed = old; - old = atomicCAS(address_as_ull, - assumed, - __double_as_longlong(val + - __longlong_as_double(assumed))); - } while (assumed != old); - - return __longlong_as_double(old); -} +namespace paddle { + +template +inline __device__ T paddleAtomicAdd(T* address, T val); -} // namespace hppl +template <> +inline __device__ float paddleAtomicAdd(float* address, float val) { + return atomicAdd(address, val); +} -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ < 600 -using hppl::atomicAdd; +template <> +inline __device__ double paddleAtomicAdd(double* address, double val) { +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 600 + return atomicAdd(address, val); +#else + // NOLINTNEXTLINE + unsigned long long int* address_as_ull = (unsigned long long int*)address; + unsigned long long int old = *address_as_ull, assumed; // NOLINT + + do { + assumed = old; + old = atomicCAS(address_as_ull, + assumed, + __double_as_longlong(val + + __longlong_as_double(assumed))); + } while (assumed != old); + + return __longlong_as_double(old); #endif +} +} // namespace paddle + #endif /* HL_DEVICE_FUNCTIONS_CUH_ */ diff --git a/paddle/cuda/include/hl_gpu_lstm.cuh b/paddle/cuda/include/hl_gpu_lstm.cuh index 2ca33f2b13a1f..07806e11c18a2 100644 --- a/paddle/cuda/include/hl_gpu_lstm.cuh +++ b/paddle/cuda/include/hl_gpu_lstm.cuh @@ -192,10 +192,10 @@ __global__ void KeLstmBackward(Op op, if (isBatch) { if (value.prevStateValue) { - if (grad.checkIgGrad) atomicAdd(grad.checkIgGrad+frameIdx, rCheckIGrad); - if (grad.checkFgGrad) atomicAdd(grad.checkFgGrad+frameIdx, rCheckFGrad); + if (grad.checkIgGrad) paddle::paddleAtomicAdd(grad.checkIgGrad+frameIdx, rCheckIGrad); + if (grad.checkFgGrad) paddle::paddleAtomicAdd(grad.checkFgGrad+frameIdx, rCheckFGrad); } - if (grad.checkOgGrad) atomicAdd(grad.checkOgGrad+frameIdx, rCheckOGrad); + if (grad.checkOgGrad) paddle::paddleAtomicAdd(grad.checkOgGrad+frameIdx, rCheckOGrad); } else { if (value.prevStateValue) { if (grad.checkIgGrad) grad.checkIgGrad[frameIdx] += rCheckIGrad; diff --git a/paddle/cuda/src/hl_cuda_lstm.cu b/paddle/cuda/src/hl_cuda_lstm.cu index 64699c9f6d450..cf009620bf69d 100644 --- a/paddle/cuda/src/hl_cuda_lstm.cu +++ b/paddle/cuda/src/hl_cuda_lstm.cu @@ -564,11 +564,11 @@ __global__ void KeLstmBackward(real *gateValue, /* TODO: Temporary save & merger in another kernel */ if (frameIdy == 1) { - if (checkIgGrad) atomicAdd(checkIgGrad+frameIdx, rCheckGrad); + if (checkIgGrad) paddle::paddleAtomicAdd(checkIgGrad+frameIdx, rCheckGrad); } else if (frameIdy == 2) { - if (checkFgGrad) atomicAdd(checkFgGrad+frameIdx, rCheckGrad); + if (checkFgGrad) paddle::paddleAtomicAdd(checkFgGrad+frameIdx, rCheckGrad); } else if (frameIdy == 3) { - if (checkOgGrad) atomicAdd(checkOgGrad+frameIdx, rCheckGrad); + if (checkOgGrad) paddle::paddleAtomicAdd(checkOgGrad+frameIdx, rCheckGrad); } } diff --git a/paddle/cuda/src/hl_cuda_matrix.cu b/paddle/cuda/src/hl_cuda_matrix.cu index ecc44944e4fa1..38e4f16217c2a 100644 --- a/paddle/cuda/src/hl_cuda_matrix.cu +++ b/paddle/cuda/src/hl_cuda_matrix.cu @@ -623,7 +623,7 @@ __global__ void KeCosSimDerivative(real* grad, prevGradY[index] += scale * grad[ty] * prevOutX[index] * reciprocal; } else { - atomicAdd(prevGradY + index, + paddle::paddleAtomicAdd(prevGradY + index, scale * grad[ty] * prevOutX[index] * reciprocal); } } @@ -640,7 +640,7 @@ __global__ void KeCosSimDerivative(real* grad, (prevOutX[index] * reciprocalXY - prevOutY[index] * reciprocalSquareSumY); } else { - atomicAdd(prevGradY + index, output[ty] * grad[ty] * + paddle::paddleAtomicAdd(prevGradY + index, output[ty] * grad[ty] * (prevOutX[index] * reciprocalXY - prevOutY[index] * reciprocalSquareSumY)); } diff --git a/paddle/cuda/src/hl_cuda_sequence.cu b/paddle/cuda/src/hl_cuda_sequence.cu index f88a2682fd060..e028880156e5b 100644 --- a/paddle/cuda/src/hl_cuda_sequence.cu +++ b/paddle/cuda/src/hl_cuda_sequence.cu @@ -362,7 +362,7 @@ __global__ void KeMatrixAddRows(real* output, if (AddRow == 0) { outputData[i] += tableData[i]; } else { - atomicAdd(&tableData[i], outputData[i]); + paddle::paddleAtomicAdd(&tableData[i], outputData[i]); } } } diff --git a/paddle/cuda/src/hl_cuda_sparse.cuh b/paddle/cuda/src/hl_cuda_sparse.cuh index becb6c66492c1..db5c9ce979885 100644 --- a/paddle/cuda/src/hl_cuda_sparse.cuh +++ b/paddle/cuda/src/hl_cuda_sparse.cuh @@ -280,7 +280,7 @@ __global__ void KeSMatrixCscMulDense(real *C_d, if (index_n_t < dimN) { real tmp; tmp = alpha*a_r*b_r[n]; - atomicAdd(C_d_r, tmp); + paddle::paddleAtomicAdd(C_d_r, tmp); C_d_r += CU_CSC_MUL_DENSE_THREAD_X; index_n_t += CU_CSC_MUL_DENSE_THREAD_X; } @@ -328,7 +328,7 @@ __global__ void KeSMatrixCscMulDense(real *C_d, if (index_n_t < dimN) { real tmp; tmp = alpha*a_r*b_r[n]; - atomicAdd(C_d_r, tmp); + paddle::paddleAtomicAdd(C_d_r, tmp); C_d_r += CU_CSC_MUL_DENSE_THREAD_X; index_n_t += CU_CSC_MUL_DENSE_THREAD_X; } @@ -629,7 +629,7 @@ __global__ void KeSMatrixDenseMulCsr(real *C_d, for (int n=0; n < CU_DM_CSR_N; n++) { if (index_m_t++ < dimM) { tmp = alpha * b_r * a_r[n]; - atomicAdd(C_d_r, tmp); + paddle::paddleAtomicAdd(C_d_r, tmp); C_d_r += dimN; } } @@ -660,7 +660,7 @@ __global__ void KeSMatrixDenseMulCsr(real *C_d, for (int n=0; n < CU_DM_CSR_N; n++) { if (index_m_t++ < dimM) { tmp = alpha * b_r * a_r[n]; - atomicAdd(C_d_r, tmp); + paddle::paddleAtomicAdd(C_d_r, tmp); C_d_r += dimN; } } @@ -912,7 +912,7 @@ __global__ void KeSMatrixCsrColumnSum(real* a_val, real* csr_val, for (int idx = gid; idx < dimNNZ; idx += gridDim.x * blockDim.x) { int colIdx = csr_col[idx]; real val = csr_val[idx]; - atomicAdd(a_val + colIdx, val); + paddle::paddleAtomicAdd(a_val + colIdx, val); } } diff --git a/paddle/cuda/src/hl_table_apply.cu b/paddle/cuda/src/hl_table_apply.cu index 05335c5f835fc..52ee4610edf67 100644 --- a/paddle/cuda/src/hl_table_apply.cu +++ b/paddle/cuda/src/hl_table_apply.cu @@ -35,7 +35,7 @@ __global__ void KeMatrixAddRows(real* output, int ldo, real *tab = table + tableId * ldt; for (int i = idx; i < dim; i += blockDimX) { if (AddRow) { - atomicAdd(&tab[i], out[i]); + paddle::paddleAtomicAdd(&tab[i], out[i]); } else { out[i] += tab[i]; } From 90b9cba7fe2b516172f76e56ef45a5263ca432c1 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 20 Sep 2016 00:51:14 +0800 Subject: [PATCH 097/324] Add min_pool_size, Add default value of should_shuffle (#70) * min_pool_size would be infinite by default. * add unittest for min_pool_size * Fix bug in can_over_batch_size * add unittest for can_over_batch_size * Add DEFINE_PROVIDER_EX * Add default value of should_shuffle * When training, the default value of should_shuffle is True. * When testing, the default value of should_shuffle is False. * User a set a provider should_shuffle or not by pass it to `@provider` * should_shuffle can handle a list of value, not just boolean * Add input order mapping by using name * Add unittest * Add check to check input format. * Default is close for speed reason. * User could stop train when check error, or continue train without this train sample. * use deque instead of vector in generators pool, make erase generator faster. * Add chinese/english documentation * Make should shuffle = false in unittest * Add python files to depends. --- CMakeLists.txt | 2 +- doc/ui/data_provider/pydataprovider2.rst | 37 ++-- doc_cn/ui/data_provider/mnist_config.py | 2 + .../ui/data_provider/mnist_provider.dict.py | 25 +++ doc_cn/ui/data_provider/pydataprovider2.rst | 71 ++++++- paddle/gserver/dataproviders/DataProvider.cpp | 10 +- paddle/gserver/dataproviders/DataProvider.h | 39 +++- .../dataproviders/MultiDataProvider.cpp | 10 +- .../gserver/dataproviders/MultiDataProvider.h | 4 +- .../gserver/dataproviders/PyDataProvider2.cpp | 103 +++++++--- paddle/gserver/tests/rnn_data_provider.py | 8 +- paddle/gserver/tests/sequenceGen.py | 18 +- paddle/gserver/tests/test_PyDataProvider2.cpp | 118 +++++++++++ paddle/gserver/tests/test_PyDataProvider2.py | 45 ++++- paddle/trainer/Trainer.cpp | 4 +- paddle/utils/PythonUtil.h | 22 ++- python/CMakeLists.txt | 10 +- python/paddle/trainer/PyDataProvider2.py | 183 +++++++++++++++++- 18 files changed, 631 insertions(+), 80 deletions(-) create mode 100644 doc_cn/ui/data_provider/mnist_provider.dict.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 1bb73f8b98fbd..92c866da8fc7c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 2.8) project(paddle CXX C) set(PADDLE_MAJOR_VERSION 0) set(PADDLE_MINOR_VERSION 8) -set(PADDLE_PATCH_VERSION 0b0) +set(PADDLE_PATCH_VERSION 0b1) set(PADDLE_VERSION ${PADDLE_MAJOR_VERSION}.${PADDLE_MINOR_VERSION}.${PADDLE_PATCH_VERSION}) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") diff --git a/doc/ui/data_provider/pydataprovider2.rst b/doc/ui/data_provider/pydataprovider2.rst index 152f8a6df6634..e105d3be30870 100644 --- a/doc/ui/data_provider/pydataprovider2.rst +++ b/doc/ui/data_provider/pydataprovider2.rst @@ -24,7 +24,7 @@ A small part of the original data as an example is shown as below: .. literalinclude:: ../../../doc_cn/ui/data_provider/mnist_train.txt -Each line of the data contains two parts, separated by ';'. The first part is +Each line of the data contains two parts, separated by :code:`;`. The first part is label of an image. The second part contains 28x28 pixel float values. Just write path of the above data into train.list. It looks like this: @@ -74,7 +74,20 @@ you can take this as an example. .. literalinclude:: ../../../doc_cn/ui/data_provider/mnist_config.py -Here we specify training data by 'train.list', and no testing data is specified. +Here we specify training data by :code:`train.list`, and no testing data is specified. +The method which actually provide data is :code:`process`. + +User also can use another style to provide data, which defines the +:code:`data_layer`'s name explicitly when `yield`. For example, +the :code:`dataprovider` is shown as below. + +.. literalinclude:: ../../../doc_cn/ui/data_provider/mnist_provider.dict.py + :linenos: + +If user did't give the :code:`data_layer`'s name, PaddlePaddle will use +the order of :code:`data_layer` definition roughly to determine which feature to +which :code:`data_layer`. This order may be not correct, so TO DEFINE THE +:code:`data_layer`'s NAMES EXPLICITLY IS THE RECOMMANDED WAY TO PROVIDER DATA. Now, this simple example of using PyDataProvider is finished. The only thing that the user should know is how to generte **one sample** from @@ -93,7 +106,7 @@ DataProvider for the sequential model ------------------------------------- A sequence model takes sequences as its input. A sequence is made up of several timesteps. The so-called timestep, is not necessary to have something to do -with 'time'. It can also be explained to that the order of data are taken into +with time. It can also be explained to that the order of data are taken into consideration into model design and training. For example, the sentence can be interpreted as a kind of sequence data in NLP tasks. @@ -155,23 +168,7 @@ Reference @provider +++++++++ -'@provider' is a Python `Decorator`_, it can construct a PyDataProvider in -PaddlePaddle from a user defined function. Its parameters are: - -* `input_types`_ defines format of the data input. -* should_shuffle defines whether to shuffle data or not. By default, it is set - true during training, and false during testing. -* pool_size is the memory pool size (in sample number) in DataProvider. - -1 means no limit. -* can_over_batch_size defines whether PaddlePaddle can store little more - samples than pool_size. It is better to set True to avoid some deadlocks. -* calc_batch_size is a function define how to calculate batch size. This is - usefull in sequential model, that defines batch size is counted upon sequence - or token. By default, each sample or sequence counts to 1 when calculating - batch size. -* cache is a data cache strategy, see `cache`_. -* Init_hook function is invoked once the data provider is initialized, - see `init_hook`_. +.. autofunction:: paddle.trainer.PyDataProvider2.provider input_types +++++++++++ diff --git a/doc_cn/ui/data_provider/mnist_config.py b/doc_cn/ui/data_provider/mnist_config.py index 0f9094cd2776f..7ba344338c374 100644 --- a/doc_cn/ui/data_provider/mnist_config.py +++ b/doc_cn/ui/data_provider/mnist_config.py @@ -4,3 +4,5 @@ test_list=None, module='mnist_provider', obj='process') +img = data_layer(name='pixel', size=784) +label = data_layer(name='label', size=10) diff --git a/doc_cn/ui/data_provider/mnist_provider.dict.py b/doc_cn/ui/data_provider/mnist_provider.dict.py new file mode 100644 index 0000000000000..4eab5b1fd3b50 --- /dev/null +++ b/doc_cn/ui/data_provider/mnist_provider.dict.py @@ -0,0 +1,25 @@ +from paddle.trainer.PyDataProvider2 import * + + +# Define a py data provider +@provider(input_types=[ + dense_vector(28 * 28), + integer_value(10) +]) +def process(settings, filename): # settings is not used currently. + f = open(filename, 'r') # open one of training file + + for line in f: # read each line + label, pixel = line.split(';') + + # get features and label + pixels_str = pixel.split(' ') + + pixels_float = [] + for each_pixel_str in pixels_str: + pixels_float.append(float(each_pixel_str)) + + # give data to paddle. + yield { "pixel": pixels_float, 'label': int(label) } + + f.close() # close file diff --git a/doc_cn/ui/data_provider/pydataprovider2.rst b/doc_cn/ui/data_provider/pydataprovider2.rst index e743e4168821f..9e1d8c531f5ba 100644 --- a/doc_cn/ui/data_provider/pydataprovider2.rst +++ b/doc_cn/ui/data_provider/pydataprovider2.rst @@ -56,6 +56,14 @@ process函数调用多次 :code:`yield` 即可。 :code:`yield` 是Python的一 这里说明了训练数据是 'train.list'，而没有测试数据。引用的DataProvider是 'mnist_provider' 这个模块中的 'process' 函数。 +同时，根据模型配置文件中 :code:`data_layer` 的名字，用户也可以显式指定返回的数据对应关系。例如: + +.. literalinclude:: mnist_provider.dict.py + :linenos: + +如果用户不指定返回数据的对应关系，那么PaddlePaddle会粗略的根据layer的声明顺序， +来确定对应关系。这个对应关系可能不正确。所以推荐使用显式指定返回值和数据对应关系。 + 至此，简单的PyDataProvider样例就说明完毕了。对于用户来说，讲数据发送给PaddlePaddle，仅仅需要 知道如何从 **一个文件** 里面读取 **一条** 样本。而PaddlePaddle进程帮助用户做了 @@ -119,11 +127,13 @@ DataProvider创建的时候执行。这个初始化函数具有如下参数: @provider +++++++++ -'@provider'是一个Python的 `Decorator`_ ，他可以将某一个函数标记成一个PyDataProvider。它包含的参数有: +:code:`@provider` 是一个Python的 `Decorator`_ ，他可以将某一个函数标记成一个PyDataProvider。它包含的参数有: * `input_types`_ 是数据输入格式。具体有哪些格式，参考 `input_types`_ 。 * should_shuffle 是个DataProvider是不是要做shuffle，如果不设置的话，训练的时候默认shuffle， - 测试的时候默认不shuffle + 测试的时候默认不shuffle。 +* min_pool_size 是设置DataProvider在内存中最小暂存的数据条数。这个也是PaddlePaddle所能够保证的shuffle粒度。 + 设置成-1的话，会预先读取全部数据到内存中。 * pool_size 是设置DataProvider在内存中暂存的数据条数。设置成-1的话，即不在乎内存暂存多少条数据。 * can_over_batch_size 表示是否允许Paddle暂存略微多余pool_size的数据。这样做可以避免很多死锁问题。 一般推荐设置成True @@ -131,6 +141,11 @@ DataProvider创建的时候执行。这个初始化函数具有如下参数: 是一个batch size，但是有时为了计算均衡性，可以将一条数据设置成多个batch size * cache 是数据缓存的策略，参考 `cache`_ * init_hook 是初始化时调用的函数，参考 `init_hook`_ +* use_dynamic_order 如果是true的话，可以返回一个dict，key是data_layer的名字，value是特征值。同时，也可以 + 返回一个list或者tuple。如果是false的话，只能够返回list或者tuple +* check 设置成true的话，会根据input_types检查数据的合法性。 +* check_fail_continue 如果设置成true的话，即使在check中数据不合法，也会扔到这条数据，继续训练。 如果 + check是false的话，没有作用。 input_types +++++++++++ @@ -190,3 +205,55 @@ DataProvider提供了两种简单的Cache策略。他们是 * CacheType.NO_CACHE 不缓存任何数据，每次都会从python端读取数据 * CacheType.CACHE_PASS_IN_MEM 第一个pass会从python端读取数据，剩下的pass会直接从内存里 读取数据。 + + +注意事项 +-------- + +可能的内存泄露问题 +++++++++++++++++++ + +PaddlePaddle将train.list中的每一行，都传递给process函数，从而生成多个generator。 +即如果train.list中，有100个训练文件，即会生成100个generator。这个本身不是一个很 +严重的问题。 + +但是，如果在训练时，每一条训练数据都是一个文件，并且，训练数据非常多的情况下，就 +会生成多个generator。每个generator在没有调用的时候，是几乎不占内存的。但是，当调 +用过一次的时候，generator便会存下当前的上下文(Context)。而这个Context可能会非常 +大。并且，generator至少调用两次才会知道是否停止。所以，即使在process里面只会有一 +个yield，也需要两次随机选择到同样的generator的时候，才会释放该段内存。 + +.. code-block:: python + + def func(): + yield 0 + + f = func() # 创建generator + tmp = next(f) # 调用一次，返回0 + tmp = next(f) # 调用第二次的时候，才会Stop Iteration + +而如果按顺序调用这些generator就不会出现这个问题。 + +所以最佳实践推荐不要将每一个样本都放入train.list。而是将样本的地址放入另一个文本 +文件，train.list写入那个文本文件的地址。 或者在python generator的上下文中尽量留 +下非常少的变量引用。例如 + +.. code-block:: python + + def real_process(fn): + # ... read from fn + return result # 当函数返回的时候，python可以解除掉内部变量的引用。 + + def process(fn): + yield real_process(fn) + +这个问题是PyDataProvider读数据时候的逻辑问题，基本上不能整体修正。 + + +内存不够用的情况 +++++++++++++++++ + +PyDataProvider2会尽量使用内存。所以如果对于内存比较小的机器，推荐设置 +:code:`pool_size` 变量，而这个变量推荐大于训练的batch size，并且在内存足够 +的情况下越大越好。 + diff --git a/paddle/gserver/dataproviders/DataProvider.cpp b/paddle/gserver/dataproviders/DataProvider.cpp index ba05b70fe9a3d..c3b4769f7612b 100644 --- a/paddle/gserver/dataproviders/DataProvider.cpp +++ b/paddle/gserver/dataproviders/DataProvider.cpp @@ -149,9 +149,13 @@ void DoubleBuffer::startAsyncLoad() { taskReadySem_.post(); } -ClassRegistrar DataProvider::registrar_; -DataProvider* DataProvider::create(const DataConfig& config, bool useGpu) { - return registrar_.createByType(config.type(), config, useGpu); +ClassRegistrar +DataProvider::registrar_; + +DataProvider* DataProvider::create(const DataConfig& config, + const ModelConfig& modelConfig, + bool useGpu) { + return registrar_.createByType(config.type(), config, modelConfig, useGpu); } REGISTER_DATA_PROVIDER(simple, SimpleDataProvider); diff --git a/paddle/gserver/dataproviders/DataProvider.h b/paddle/gserver/dataproviders/DataProvider.h index aab5d93fcaa1e..534491d70d546 100644 --- a/paddle/gserver/dataproviders/DataProvider.h +++ b/paddle/gserver/dataproviders/DataProvider.h @@ -39,15 +39,30 @@ limitations under the License. */ #include "paddle/parameter/Argument.h" namespace paddle { - /** * @def REGISTER_DATA_PROVIDER - * @brief Macro for registering a data provider + * @brief Macro for registering a data provider. The class type should contain + * a consturctor with parameter (DataConfig, bool). */ -#define REGISTER_DATA_PROVIDER(__type_name, __class_name) \ - static InitFunction __reg_type_##__type_name([]() { \ - DataProvider::registrar_.registerClass<__class_name>(#__type_name); \ - }) +#define REGISTER_DATA_PROVIDER(__type_name, __class_name)\ + static InitFunction __reg_type_##__type_name([]() {\ + DataProvider::registrar_.registerClass(\ + #__type_name, \ + [](DataConfig conf, ModelConfig, bool useGpu) -> DataProvider* { \ + DataProvider* dp = new __class_name (conf, useGpu);\ + return dp;\ + });\ +}) + +/** + * @def REGISTER_DATA_PROVIDER_EX + * @brief Macro for registering a data provider, which contains a constructor + * with parameter (DataConfig, ModelConfig, bool). + */ +#define REGISTER_DATA_PROVIDER_EX(__type_name, __class_name) \ + static InitFunction __reg_type_##__type_name([] { \ + DataProvider::registrar_.registerClass<__class_name>(#__type_name); \ +}) class DataBatch; class BufferBatch; @@ -285,10 +300,18 @@ class DoubleBuffer { */ class DataProvider { public: - static ClassRegistrar registrar_; + static ClassRegistrar registrar_; static DataProvider* create(const DataConfig& config, + const ModelConfig& modelConfig, bool useGpu = FLAGS_use_gpu); + /** + * @brief create only used for unittest. + */ + inline static DataProvider* create(const DataConfig &config, bool useGpu) { + return create(config, ModelConfig(), useGpu); + } + DataProvider(const DataConfig& config, bool useGpu) : config_(config), skipShuffle_(false), @@ -336,13 +359,13 @@ class DataProvider { * @note return -1 to indicate unlimited number of samples. */ virtual int64_t getSize() = 0; + /** * @brief Get next batch training samples internally * @param[in] size size of training samples to get * @param[out] batch a batch of training samples * @return actual size of obtained training samples */ - virtual int64_t getNextBatchInternal(int64_t size, DataBatch* batch) = 0; protected: diff --git a/paddle/gserver/dataproviders/MultiDataProvider.cpp b/paddle/gserver/dataproviders/MultiDataProvider.cpp index c3d14a7069bd3..8e4f53978a045 100644 --- a/paddle/gserver/dataproviders/MultiDataProvider.cpp +++ b/paddle/gserver/dataproviders/MultiDataProvider.cpp @@ -22,7 +22,9 @@ namespace paddle { using namespace std; -MultiDataProvider::MultiDataProvider(const DataConfig& config, bool useGpu) +MultiDataProvider::MultiDataProvider(const DataConfig& config, + const ModelConfig& modelConfig, + bool useGpu) : DataProvider(config, useGpu) { bool atLeastOneMainDataFlag = false; totalDataRatio_ = 0; @@ -58,7 +60,9 @@ MultiDataProvider::MultiDataProvider(const DataConfig& config, bool useGpu) subConfig.set_async_load_data(false); } subDataProviders_[i] = - std::unique_ptr(DataProvider::create(subConfig, useGpu_)); + std::unique_ptr(DataProvider::create(subConfig, + modelConfig, + useGpu_)); } } @@ -116,6 +120,6 @@ int64_t MultiDataProvider::getNextBatchInternal(int64_t size, return batch->getSize(); } -REGISTER_DATA_PROVIDER(multi, MultiDataProvider); +REGISTER_DATA_PROVIDER_EX(multi, MultiDataProvider); } // namespace paddle diff --git a/paddle/gserver/dataproviders/MultiDataProvider.h b/paddle/gserver/dataproviders/MultiDataProvider.h index 714421286376b..b498ba6516c43 100644 --- a/paddle/gserver/dataproviders/MultiDataProvider.h +++ b/paddle/gserver/dataproviders/MultiDataProvider.h @@ -24,7 +24,9 @@ class MultiDataProvider : public DataProvider { std::vector> subDataProviders_; public: - MultiDataProvider(const DataConfig& config, bool useGpu); + MultiDataProvider(const DataConfig& config, + const ModelConfig& modelConfig, + bool useGpu); ~MultiDataProvider() {} virtual void reset(); virtual void shuffle(); diff --git a/paddle/gserver/dataproviders/PyDataProvider2.cpp b/paddle/gserver/dataproviders/PyDataProvider2.cpp index 8e51752dc29ee..0b41f6a02aecc 100644 --- a/paddle/gserver/dataproviders/PyDataProvider2.cpp +++ b/paddle/gserver/dataproviders/PyDataProvider2.cpp @@ -24,6 +24,27 @@ limitations under the License. */ namespace paddle { +namespace unittest { + +static std::unique_ptr> + OnPoolFilled; + +namespace pydp2 { + +void setOnPoolFilledHook(const std::function& callback) { + OnPoolFilled.reset(new std::function()); + *OnPoolFilled = callback; +} + +void clearOnPoolFilledHook() { + OnPoolFilled.reset(); +} + +} // namespace pydp2 +} // namespace unittest + + + /** * Slot type */ @@ -179,6 +200,7 @@ class PyDataProvider2 : public DataProvider { * Ctor */ PyDataProvider2(const DataConfig& config, + const ModelConfig& modelConfig, bool useGpu) :DataProvider(config, useGpu), callingContextCreated_(2) { auto& args = config.load_data_args(); @@ -192,6 +214,12 @@ class PyDataProvider2 : public DataProvider { py::DictHelper kwargsDict(kwargs); kwargsDict.setBool("is_train", !config.for_test()); + std::vector inputs; + inputs.reserve(modelConfig.input_layer_names().size()); + std::copy(modelConfig.input_layer_names().begin(), + modelConfig.input_layer_names().end(), + std::back_inserter(inputs)); + kwargsDict.setStringList("input_order", inputs); // kwargs is keyword arguemts to create object. this->createPyDataObj(config.load_data_module(), @@ -199,7 +227,7 @@ class PyDataProvider2 : public DataProvider { config.files(), std::move(kwargs)); DBG << "Instance " << instance_.get() << " loaded."; - this->readPyFields(); + this->readPyFields(config.for_test()); DBG << "Py Field Done"; } @@ -253,14 +281,28 @@ class PyDataProvider2 : public DataProvider { CHECK_PY(instance_) << "Cannot Create instance"; } - void readPyFields() { + void readPyFields(bool testing) { py::ObjectHelper self(this->instance_); - this->skipShuffle_ = !self.getBoolAttr("should_shuffle"); bool ok; + + this->skipShuffle_ = !self.getBoolAttr("should_shuffle", + &ok /*isBoolType*/); + if (!ok) { + this->skipShuffle_ = testing; // shuffle when is training, skip shuffle + // when is testing. + } + DBG << "Provider Skip Shuffle " << this->skipShuffle_; + this->poolSize_ = self.getIntAttr("pool_size", &ok); if (!ok) { this->poolSize_ = -1UL; } + this->minPoolSize_ = self.getIntAttr("min_pool_size", &ok); + if (!ok) { + this->minPoolSize_ = -1UL; + } + this->minPoolSize_ = std::min(this->poolSize_, this->minPoolSize_); + this->canOverBatchSize_ = self.getBoolAttr("can_over_batch_size"); calcBatchSize_.reset(self.getAttr("calc_batch_size")); @@ -307,7 +349,6 @@ class PyDataProvider2 : public DataProvider { } void loadThread() { - callingContexts_.reserve(fileLists_.size()); DBG << "Creating context"; for (auto& filename : fileLists_) { PyGuard g; @@ -332,7 +373,14 @@ class PyDataProvider2 : public DataProvider { bool atEnd; data = py::iterNext(callingContexts_[cid], &atEnd); if (atEnd || data == nullptr) { - callingContexts_.erase(callingContexts_.begin() + cid); + if (cid != 0) { + std::swap(callingContexts_[cid], callingContexts_[0]); + cid = 0; + } + { + PyGuard g; + callingContexts_.pop_front(); + } this->pullCV_.notify_all(); continue; } @@ -354,11 +402,7 @@ class PyDataProvider2 : public DataProvider { if (this->loadThread_){ // wait poolActualSize < poolSize; std::unique_lock l(mtx_); pushCV_.wait(l, [this, additionalBatchSize] { - if (this->canOverBatchSize_) { - return this->poolActualSize_ < poolSize_; - } else { - return this->poolActualSize_ + additionalBatchSize < poolSize_; - } + return this->poolActualSize_ < poolSize_; }); } @@ -402,7 +446,7 @@ class PyDataProvider2 : public DataProvider { private: std::unique_ptr loadThread_; std::atomic exit_; - std::vector callingContexts_; + std::deque callingContexts_; std::deque dataPool_; size_t poolActualSize_; std::condition_variable pushCV_; @@ -413,6 +457,7 @@ class PyDataProvider2 : public DataProvider { PyObjectPtr instance_; size_t poolSize_; + size_t minPoolSize_; bool canOverBatchSize_; PyObjectPtr calcBatchSize_; PyObjectPtr generator_; @@ -478,8 +523,13 @@ class PyDataProvider2 : public DataProvider { // data pool ready. std::unique_lock l(mtx_); pullCV_.wait(l, [this, &size] { - return this->poolActualSize_ >= size || callingContexts_.empty(); + return this->poolActualSize_ >= std::max(size, this->minPoolSize_) + || callingContexts_.empty(); }); + + if (unittest::OnPoolFilled) { + (*unittest::OnPoolFilled)(this->poolActualSize_); + } } std::deque data; size_t bsize = 0; @@ -495,7 +545,8 @@ class PyDataProvider2 : public DataProvider { std::deque& pool = *poolPtr; while (bsize < size && !pool.empty()) { - { // move data from pool to data + { + // move data from pool to data std::lock_guard guard(mtx_); if (skipShuffle_) { size_t i = 0; @@ -505,14 +556,13 @@ class PyDataProvider2 : public DataProvider { } else { // when shuffle, use swap to drop only last pool element. size_t i = ThreadLocalRand::rand() % pool.size(); CHECK(pool[i] != nullptr); - if (i != pool.size() - 1) { - std::swap(pool[i], pool.back()); + if (i != 0) { + std::swap(pool[i], pool.front()); } - data.emplace_back(std::move(pool.back())); - pool.pop_back(); + data.emplace_back(std::move(pool.front())); + pool.pop_front(); } - } - { + if (calcBatchSize_) { // custom calc batch size. PyGuard guard; Py_INCREF(data.back().get()); @@ -521,8 +571,17 @@ class PyDataProvider2 : public DataProvider { calcBatchSize.getArgs().set(0, data.back()); PyObjectPtr customBatchSize(calcBatchSize()); bool ok; - bsize += py::castInt(customBatchSize.get(), &ok); + size_t tmp = py::castInt(customBatchSize.get(), &ok); CHECK(ok) << "calc_batch_size must return int"; + + if (bsize + tmp > size && !canOverBatchSize_) { + // Put data back. + pool.push_front(std::move(data.back())); + data.pop_back(); + break; + } else { + bsize += tmp; + } } else { bsize += 1; } @@ -598,7 +657,6 @@ class PyDataProvider2 : public DataProvider { } else { *batch = cpuBatch; } - return bsize; } }; @@ -606,7 +664,8 @@ class PyDataProvider2 : public DataProvider { std::unordered_set PyDataProvider2::gModuleClsPtrs_; PyObjectPtr PyDataProvider2::zeroTuple_(PyTuple_New(0)); -REGISTER_DATA_PROVIDER(py2, PyDataProvider2); +REGISTER_DATA_PROVIDER_EX(py2, PyDataProvider2); + /** * Scanner for dense slot. diff --git a/paddle/gserver/tests/rnn_data_provider.py b/paddle/gserver/tests/rnn_data_provider.py index 85a83554c5c30..347d5891b906b 100644 --- a/paddle/gserver/tests/rnn_data_provider.py +++ b/paddle/gserver/tests/rnn_data_provider.py @@ -19,14 +19,18 @@ [[[0, 2], [2, 5], [0, 1, 2]], 1], ] + @provider(input_types=[integer_value_sub_sequence(10), - integer_value(2)]) + integer_value(2)], + should_shuffle=False) def process_subseq(settings, file_name): for d in data: yield d + @provider(input_types=[integer_value_sequence(10), - integer_value(2)]) + integer_value(2)], + should_shuffle=False) def process_seq(settings, file_name): for d in data: seq = [] diff --git a/paddle/gserver/tests/sequenceGen.py b/paddle/gserver/tests/sequenceGen.py index cb83d79d78cc6..cbed1f15fc415 100644 --- a/paddle/gserver/tests/sequenceGen.py +++ b/paddle/gserver/tests/sequenceGen.py @@ -17,22 +17,26 @@ from paddle.trainer.PyDataProvider2 import * + def hook(settings, dict_file, **kwargs): settings.word_dict = dict_file - settings.input_types = [integer_value_sequence(len(settings.word_dict)), + settings.input_types = [integer_value_sequence(len(settings.word_dict)), integer_value_sequence(3)] settings.logger.info('dict len : %d' % (len(settings.word_dict))) -@provider(init_hook=hook) + +@provider(init_hook=hook, should_shuffle=False) def process(settings, file_name): with open(file_name, 'r') as fdata: for line in fdata: label, comment = line.strip().split('\t') label = int(''.join(label.split())) words = comment.split() - word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] + word_slot = [settings.word_dict[w] for w in words if + w in settings.word_dict] yield word_slot, [label] + ## for hierarchical sequence network def hook2(settings, dict_file, **kwargs): settings.word_dict = dict_file @@ -40,17 +44,19 @@ def hook2(settings, dict_file, **kwargs): integer_value_sub_sequence(3)] settings.logger.info('dict len : %d' % (len(settings.word_dict))) -@provider(init_hook=hook2) + +@provider(init_hook=hook2, should_shuffle=False) def process2(settings, file_name): with open(file_name) as fdata: label_list = [] word_slot_list = [] for line in fdata: if (len(line)) > 1: - label,comment = line.strip().split('\t') + label, comment = line.strip().split('\t') label = int(''.join(label.split())) words = comment.split() - word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] + word_slot = [settings.word_dict[w] for w in words if + w in settings.word_dict] label_list.append([label]) word_slot_list.append(word_slot) else: diff --git a/paddle/gserver/tests/test_PyDataProvider2.cpp b/paddle/gserver/tests/test_PyDataProvider2.cpp index 824295eb6e9f2..c5fe31b29187f 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.cpp +++ b/paddle/gserver/tests/test_PyDataProvider2.cpp @@ -20,6 +20,18 @@ limitations under the License. */ #include "paddle/gserver/dataproviders/DataProvider.h" P_DEFINE_string(train_list, "unittest.list", "file list for unittest"); + +namespace paddle { +namespace unittest { +namespace pydp2 { +extern void setOnPoolFilledHook(const std::function& func); +extern void clearOnPoolFilledHook(); + +} // namespace pydp2 +} // namespace unittest +} // namespace paddle + + const paddle::real epsilon = 1e-5; static inline int64_t readDataBatch( @@ -235,6 +247,112 @@ TEST(PyDataProvider2, index_sub_seq) { } } +TEST(PyDataProvider2, min_pool_size) { + paddle::DataConfig config; + config.set_type("py2"); + config.set_files(FLAGS_train_list.c_str()); + config.set_load_data_module("test_PyDataProvider2"); + config.set_load_data_object("test_min_pool_size"); + config.set_load_data_args(""); + size_t totalData = 1 << 14; + constexpr size_t batchSize = 100; + constexpr size_t minPoolSize = 1000; + paddle::DataBatch batch; + std::unique_ptr provider( + paddle::DataProvider::create(config, false)); + provider->reset(); + + paddle::unittest::pydp2::setOnPoolFilledHook([&](size_t poolSize) { + if (totalData > batchSize) { + CHECK_GE(poolSize, std::min(totalData-batchSize, minPoolSize)); + } + }); + while (true) { + size_t realBatchSize = provider->getNextBatchInternal(batchSize, &batch); + if (realBatchSize) { + totalData -= realBatchSize; + } else { + break; + } + } + paddle::unittest::pydp2::clearOnPoolFilledHook(); +} + +TEST(PyDataProvider2, can_over_batch_size) { + paddle::DataConfig config; + config.set_type("py2"); + config.set_files(FLAGS_train_list.c_str()); + config.set_load_data_module("test_PyDataProvider2"); + config.set_load_data_object("test_can_over_batch_size"); + config.set_load_data_args(""); + paddle::DataBatch batch; + std::unique_ptr provider( + paddle::DataProvider::create(config, false)); + provider->reset(); + constexpr size_t batchSize = 100; + while (true) { + size_t realBatchSize = provider->getNextBatchInternal(batchSize, &batch); + if (realBatchSize) { + CHECK_LE(realBatchSize, batchSize); + } else { + break; + } + } +} + +TEST(PyDataProvider2, input_order) { + paddle::DataConfig config; + config.set_type("py2"); + config.set_files(FLAGS_train_list.c_str()); + config.set_load_data_module("test_PyDataProvider2"); + config.set_load_data_object("test_input_order"); + config.set_load_data_args(""); + + paddle::ModelConfig modelConfig; + *modelConfig.add_input_layer_names() = "input1"; + *modelConfig.add_input_layer_names() = "input2"; + paddle::DataBatch batch; + std::unique_ptr provider( + paddle::DataProvider::create(config, modelConfig, false)); + provider->reset(); + constexpr size_t batchSize = 100; + while (true) { + size_t realBatchSize = provider->getNextBatchInternal(batchSize, &batch); + if (!realBatchSize) { + break; + } + ASSERT_EQ(batch.getStreams().size(), 2); + for (size_t i = 0; i < realBatchSize; ++i) { + ASSERT_EQ(batch.getStream(0).ids->getData()[i], 0); + ASSERT_EQ(batch.getStream(1).ids->getData()[i], 1); + } + } +} + +TEST(PyDataProvider2, test_check) { + paddle::DataConfig config; + config.set_type("py2"); + config.set_files(FLAGS_train_list.c_str()); + config.set_load_data_module("test_PyDataProvider2"); + config.set_load_data_object("test_check"); + config.set_load_data_args(""); + paddle::DataBatch batch; + std::unique_ptr provider( + paddle::DataProvider::create(config, false)); + provider->reset(); + while (true) { + size_t realBatchSize = provider->getNextBatchInternal(100, &batch); + if (!realBatchSize) { + break; + } else { + auto& ivec = batch.getStream(0).ids; + for (size_t i=0; i < ivec->getSize(); ++i) { + CHECK_LT(ivec->getData()[i], 10); + } + } + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); paddle::initMain(argc, argv); diff --git a/paddle/gserver/tests/test_PyDataProvider2.py b/paddle/gserver/tests/test_PyDataProvider2.py index a88c48cb4e295..145fe85cff7d8 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.py +++ b/paddle/gserver/tests/test_PyDataProvider2.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import random + from paddle.trainer.PyDataProvider2 import * @@ -39,7 +41,8 @@ def test_init_hook(setting, filename): @provider( - input_types=[sparse_binary_vector(30000, seq_type=SequenceType.NO_SEQUENCE)]) + input_types=[ + sparse_binary_vector(30000, seq_type=SequenceType.NO_SEQUENCE)]) def test_sparse_non_value_no_seq(setting, filename): for i in xrange(200): yield [(i + 1) * (j + 1) for j in xrange(10)] @@ -66,3 +69,43 @@ def gen_sub_seq(l): for i in xrange(200): yield list(gen_sub_seq(i)) + + +@provider(input_types=[index_slot(100)], min_pool_size=1000) +def test_min_pool_size(setting, filename): + for _ in xrange(1 << 14): + yield random.randint(0, 100 - 1) + + +@provider(input_types=[index_slot(100, seq_type=SequenceType.SEQUENCE)], + can_over_batch_size=False, + calc_batch_size=lambda x: len(x[0])) +def test_can_over_batch_size(setting, filename): + for _ in xrange(1 << 10): + seq_len = random.randint(0, 99) + yield [random.randint(0, 100 - 1) for _ in xrange(seq_len)] + + +@provider(input_types=[index_slot(10), index_slot(10)]) +def test_input_order(setting, filename): + for _ in xrange(1000): + yield { + 'input1': 0, + 'input2': 1 + } + + +@provider(input_types=[index_slot(10)], + check=True, + check_fail_continue=True, + should_shuffle="123") # also test should shuffle +def test_check(settings, filename): + yield_good_value = False + + while not yield_good_value: + for _ in xrange(10000): + i = random.randint(0, 100) + if i < 10: + yield_good_value = True + yield i + diff --git a/paddle/trainer/Trainer.cpp b/paddle/trainer/Trainer.cpp index 2890f5b5d7ad9..c0e5ec3bd6bd8 100644 --- a/paddle/trainer/Trainer.cpp +++ b/paddle/trainer/Trainer.cpp @@ -193,7 +193,7 @@ void Trainer::init(const std::shared_ptr &config, dataProvider_ = dataProvider; if (!dataProvider_ && config_->hasDataConfig()) { - dataProvider_.reset(DataProvider::create(*config_, gpuData)); + dataProvider_.reset(DataProvider::create(*config_, *config_, gpuData)); } if (dataProvider_) { evaluator_.reset(trainerInternal_.getGradientMachine()->makeEvaluator()); @@ -211,7 +211,7 @@ void Trainer::init(const std::shared_ptr &config, testDataProvider_ = testDataProvider; if (!testDataProvider_ && config_->hasTestDataConfig()) { testDataProvider_.reset( - DataProvider::create(config_->getTestDataConfig(), gpuData)); + DataProvider::create(config_->getTestDataConfig(), *config_, gpuData)); } if (testDataProvider_) { tester_.reset(new Tester(config_, createTesterConfig(), diff --git a/paddle/utils/PythonUtil.h b/paddle/utils/PythonUtil.h index 4467fd784ec4e..928db486fa9e9 100644 --- a/paddle/utils/PythonUtil.h +++ b/paddle/utils/PythonUtil.h @@ -175,10 +175,21 @@ class ObjectHelper { /** * Get bool attribute. * @param field + * @param [out] isBoolType return true if attribute is bool type. If the + * attribute is not bool type, then an implicit + * conversion will happens, and will return the + * conversion result. + * + * Such as, if the attribute is 1, then the return + * value of function will be true, but the isBoolType + * will return false. * @return */ - bool getBoolAttr(const std::string& field) const { + bool getBoolAttr(const std::string& field, bool* isBoolType = nullptr) const { PyObjectPtr tmp(getAttr(field)); + if (isBoolType) { + *isBoolType = PyBool_Check(tmp.get()); + } return PyObject_IsTrue(tmp.get()); } @@ -258,6 +269,15 @@ class DictHelper { this->set(key, PyBool_FromLong(b)); } + void setStringList(const std::string& key, + const std::vector& items) { + auto * list = PyList_New(items.size()); + for (size_t i=0; i < items.size(); ++i) { + PyList_SetItem(list, i, PyString_FromString(items[i].c_str())); + } + this->set(key, list); + } + private: inline void checkDict() { CHECK(PyDict_Check(this->dict_)); diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index fd9a003bb018c..dce0b90952436 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -1,6 +1,14 @@ set(OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/build") +file(GLOB TRAINER_PY_FILES . ./paddle/trainer/*.py) +file(GLOB HELPERS_PY_FILES . ./paddle/trainer_config_helpers/*.py) +file(GLOB UTILS_PY_FILES . ./paddle/utils/*.py) + +set(PY_FILES paddle/__init__.py + ${TRAINER_PY_FILES} + ${HELPERS_PY_FILES} + ${UTILS_PY_FILES}) set(PADDLE_INTERNAL_PACKAGE "") if (PADDLE_WITH_INTERNAL) @@ -13,7 +21,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp COMMAND ${PYTHON_EXECUTABLE} setup.py bdist_wheel COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT_DIR}/.timestamp - DEPENDS gen_proto_py) + DEPENDS gen_proto_py ${PY_FILES}) add_custom_target(paddle_python ALL DEPENDS ${OUTPUT_DIR}/.timestamp) diff --git a/python/paddle/trainer/PyDataProvider2.py b/python/paddle/trainer/PyDataProvider2.py index c4f61473933d0..34f5dd41b7e68 100644 --- a/python/paddle/trainer/PyDataProvider2.py +++ b/python/paddle/trainer/PyDataProvider2.py @@ -14,6 +14,13 @@ import cPickle import logging +import collections +import functools +import itertools + +logging.basicConfig( + format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" + " %(message)s") class SequenceType(object): @@ -68,30 +75,39 @@ def index_slot(dim, seq_type=SequenceType.NO_SEQUENCE): sparse_vector = sparse_value_slot integer_value = index_slot + def dense_vector_sequence(dim): return dense_vector(dim, seq_type=SequenceType.SEQUENCE) + def dense_vector_sub_sequence(dim): return dense_vector(dim, seq_type=SequenceType.SUB_SEQUENCE) + def sparse_binary_vector_sequence(dim): return sparse_binary_vector(dim, seq_type=SequenceType.SEQUENCE) + def sparse_binary_vector_sub_sequence(dim): return sparse_binary_vector(dim, seq_type=SequenceType.SUB_SEQUENCE) + def sparse_vector_sequence(dim): return sparse_vector(dim, seq_type=SequenceType.SEQUENCE) + def sparse_vector_sub_sequence(dim): return sparse_vector(dim, seq_type=SequenceType.SUB_SEQUENCE) + def integer_value_sequence(dim): return integer_value(dim, seq_type=SequenceType.SEQUENCE) + def integer_value_sub_sequence(dim): return integer_value(dim, seq_type=SequenceType.SUB_SEQUENCE) + def integer_sequence(dim): return index_slot(dim, seq_type=SequenceType.SEQUENCE) @@ -102,13 +118,97 @@ def __init__(self, generator): def __call__(self, obj, filename): for item in self.generator(obj, filename): - yield [item] + if isinstance(item, dict): + yield item + else: + yield [item] -def provider(input_types=None, should_shuffle=True, pool_size=-1, +class InputOrderWrapper(object): + def __init__(self, generator, input_order): + self.generator = generator + self.input_order = input_order + + def __call__(self, obj, filename): + for item in self.generator(obj, filename): + if isinstance(item, dict): + yield [item.get(input_name, None) for input_name in + self.input_order] + else: + yield item + + +class CheckWrapper(object): + def __init__(self, generator, input_types, check_fail_continue, logger): + self.generator = generator + self.input_types = input_types + self.check_fail_continue = check_fail_continue + self.logger = logger + + def __call__(self, obj, filename): + for items in self.generator(obj, filename): + try: + assert len(items) == len(self.input_types) + assert len(filter(lambda x: x is None, items)) == 0 + for item, input_type in itertools.izip(items, self.input_types): + callback = functools.partial(CheckWrapper.loop_callback, + input_type) + + for _ in xrange(input_type.seq_type): + callback = functools.partial(CheckWrapper.loop_check, + callback) + callback(item) + + yield items + except AssertionError as e: + self.logger.warning( + "Item (%s) is not fit the input type with error %s" + % (repr(item), repr(e))) + + if self.check_fail_continue: + continue + else: + raise + + @staticmethod + def loop_callback(input_type, each): + assert isinstance(input_type, InputType) + if input_type.type == DataType.Dense: + assert isinstance(each, collections.Sequence) + for d in each: + assert isinstance(d, float) + assert len(each, input_type.dim) + elif input_type.type == DataType.Index: + assert isinstance(each, int) + assert each < input_type.dim + elif input_type.type == DataType.SparseNonValue \ + or input_type.type == DataType.SparseValue: + assert isinstance(each, collections.Sequence) + sparse_id = set() + for k in each: + if input_type.type == DataType.SparseValue: + k, v = k + assert isinstance(v, float) + assert isinstance(k, int) + assert k < input_type.dim + sparse_id.add(k) + assert len(sparse_id) == len(each) + else: + raise RuntimeError("Not support input type") + + @staticmethod + def loop_check(callback, item): + for each in item: + callback(each) + + +def provider(input_types=None, should_shuffle=None, pool_size=-1, + min_pool_size=-1, can_over_batch_size=True, calc_batch_size=None, cache=CacheType.NO_CACHE, + check=False, check_fail_continue=False, + use_dynamic_order=True, init_hook=None, **kwargs): """ Provider decorator. Use it to make a function into PyDataProvider2 object. @@ -130,30 +230,63 @@ def process(settings, file_name): :param input_types: Specify the input types, can also be set in init_hook. It is a list of InputType object. For example, input_types= \ [dense_vector(9), integer_value(2)]. - :param should_shuffle: True if data should shuffle. + :type input_types: list|tuple + + :param should_shuffle: True if data should shuffle. Pass None means shuffle + when is training and not to shuffle when is testing. :type should_shuffle: bool + :param pool_size: Max number of sample in data pool. :type pool_size: int + + :param min_pool_size: Set minimal sample in data pool. The PaddlePaddle will + random pick sample in pool. So the min_pool_size + effect the randomize of data. + :type min_pool_size: int + :param can_over_batch_size: True if paddle can return a mini-batch larger than batch size in settings. It is useful when custom calculate one sample's batch_size. It is very danger to set it to false and use calc_batch_size together. Default is false. + :type can_over_batch_size: bool + :param calc_batch_size: a method to calculate each sample's batch size. Default each sample's batch size is 1. But to you can customize each sample's batch size. + :type calc_batch_size: callable + :param cache: Cache strategy of Data Provider. Default is CacheType.NO_CACHE + :type cache: int :param init_hook: Initialize hook. Useful when data provider need load some external data like dictionary. The parameter is (settings, file_list, \*\*kwargs). - - settings\: Is the global settings. User can set - settings.input_types here. - - file_list\: All file names for passed to data provider. - - kwargs: Other keyword arguments passed from + - settings. It is the global settings object. User can set + settings.input_types here. + - file_list. All file names for passed to data provider. + - is_train. Is this data provider used for training or not. + - kwargs. Other keyword arguments passed from trainer_config's args parameter. + :type init_hook: callable + + :param check: Check the yield data format is as same as input_types. Enable + this will make data provide process slow but it is very useful + for debug. Default is disabled. + :type check: bool + + :param check_fail_continue: Continue train or not when check failed. Just + drop the wrong format data when it is True. Has + no effect when check set to False. + :type check_fail_continue: bool + + :param use_dynamic_order: Allow provider to yield a dictionary object, whose + key is a input data layer name, and value is the + feature value. The tuples are still allowed when + use_dynmaic_order is True. + :type use_dynamic_order: bool """ def __wrapper__(generator): @@ -168,12 +301,38 @@ def __init__(self, file_list, **kwargs): self.slots = kwargs['slots'] self.slots = input_types self.should_shuffle = should_shuffle + + true_table = [1, 't', 'true', 'on'] + false_table = [0, 'f', 'false', 'off'] + if not isinstance(self.should_shuffle, bool) and \ + self.should_shuffle is not None: + + if isinstance(self.should_shuffle, basestring): + self.should_shuffle = self.should_shuffle.lower() + + if self.should_shuffle in true_table: + self.should_shuffle = True + elif self.should_shuffle in false_table: + self.should_shuffle = False + else: + self.logger.warning( + "Could not recognize should_shuffle (%s), " + "just use default value of should_shuffle." + " Please set should_shuffle to bool value or " + "something in %s" % ( + repr(self.should_shuffle), + repr(true_table + false_table))) + self.should_shuffle = None + self.pool_size = pool_size self.can_over_batch_size = can_over_batch_size self.calc_batch_size = calc_batch_size self.file_list = file_list self.generator = generator self.cache = cache + self.min_pool_size = min_pool_size + self.input_order = kwargs['input_order'] + self.check = check if init_hook is not None: init_hook(self, file_list=file_list, **kwargs) if self.input_types is not None: @@ -184,6 +343,15 @@ def __init__(self, file_list, **kwargs): if len(self.slots) == 1: self.generator = SingleSlotWrapper(self.generator) + if use_dynamic_order: + self.generator = InputOrderWrapper(self.generator, + self.input_order) + if self.check: + self.generator = CheckWrapper(self.generator, + self.slots, + check_fail_continue, + self.logger) + return DataProvider return __wrapper__ @@ -196,3 +364,4 @@ def deserialize_args(args): :return: """ return cPickle.loads(args) + From 159dd8331cfbbeff80f98ad0373da44e2cad28e5 Mon Sep 17 00:00:00 2001 From: Haichao-Zhang Date: Mon, 19 Sep 2016 15:57:00 -0700 Subject: [PATCH 098/324] split dotmul_projection and dotmul_operator (#87) * split dotmul_projection and dotmul_operator * bug fix in outputsize checking for mixed layer --- python/paddle/trainer/config_parser.py | 19 +++-- .../paddle/trainer_config_helpers/layers.py | 74 +++++++++---------- .../tests/layers_test_config.py | 5 +- 3 files changed, 55 insertions(+), 43 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index f2f67f9bd66a4..c5709208d4f1b 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2430,7 +2430,6 @@ def __init__( config_assert(inputs, 'inputs cannot be empty') super(MixedLayer, self).__init__( name, 'mixed', size, inputs=inputs, **xargs) - operator_input_index = [] for operator in self.operators: operator_conf = operator.operator_conf @@ -2445,21 +2444,31 @@ def __init__( input_layer = self.get_input_layer(input_index) operator_conf.input_sizes.append(input_layer.size) operator_input_index.append(input_index) - if self.config.size == 0: + if self.config.size == 0: size = operator.calc_output_size(operator_conf.input_sizes) if size != 0: self.set_layer_size(size) - + else: + size = operator.calc_output_size(operator_conf.input_sizes) + if size != 0: + config_assert(size == self.config.size, + "different inputs have different size: %s vs. %s" % + (size, self.config.size)) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) input = self.inputs[input_index] if input_index not in operator_input_index: config_assert(isinstance(input, Projection), "input should be projection or operation") - if self.config.size == 0 and isinstance(input, Projection): + if self.config.size == 0 and isinstance(input, Projection): size = input.calc_output_size(input_layer) if size != 0: self.set_layer_size(size) - + elif isinstance(input, Projection): + sz = input.calc_output_size(input_layer) + if sz != 0: + config_assert(sz == self.config.size, + "different inputs have different size: %s vs. %s" % + (sz, self.config.size)) config_assert(size != 0, "size is not set") for input_index in xrange(len(self.inputs)): diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index bda0b4f5d60e8..fab7e6e091863 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -28,7 +28,7 @@ import copy __all__ = ["full_matrix_projection", "AggregateLevel", "ExpandLevel", - "identity_projection", "dotmul_projection", + "identity_projection", "dotmul_projection", "dotmul_operator", "table_projection", "mixed_layer", "data_layer", "embedding_layer", "fc_layer", "grumemory", "pooling_layer", "lstmemory", "last_seq", "first_seq", @@ -389,7 +389,7 @@ def identity_projection(input, offset=None): @wrap_param_attr_default() def dotmul_projection(input, param_attr=None, scale=1): """ - 1. DotMulProjection if input is a layer. + DotMulProjection with a layer as input. It performs element-wise multiplication with weight. .. math:: @@ -403,48 +403,45 @@ def dotmul_projection(input, param_attr=None, scale=1): proj = dotmul_projection(input=layer) - 2. DotMulOperator if input is a list or tuple. - It takes two inputs, performs element-wise multiplication: - - .. math:: - out.row[i] += scale * (in1.row[i] .* in2.row[i]) - - where :math:`.*` means element-wise multiplication, and - scale is a config scalar, its default value is one. - - The example usage is: - - .. code-block:: python - - op = dotmul_projection(input=[layer1, layer2], - scale=2.0) - :param input: Input layer. - :type input: LayerOutput|list|tuple + :type input: LayerOutput :param param_attr: Parameter config, None if use default. :type param_attr: ParameterAttribute :param scale: config scalar, default value is one. :type scale: float - :return: A DotMulProjection or DotMulOperator Object. - :rtype: DotMulProjection or DotMulOperator + :return: A DotMulProjection Object. + :rtype: DotMulProjection """ - if isinstance(input, LayerOutput): - proj = DotMulProjection(input_layer_name=input.name, + proj = DotMulProjection(input_layer_name=input.name, size=input.size, **param_attr.attr) - proj.origin = input - proj.origin.projection = "dot_mul" - return proj - else: - assert isinstance(input, list) or isinstance(input, tuple) - assert len(input) == 2 - assert param_attr is None - op = DotMulOperator(input_layer_name=[x.name for x in input], - scale=scale) - op.origin = input - op.origin.operator = "dot_mul" - return op + proj.origin = input + return proj +def dotmul_operator(x, y, scale=1): + """ + DotMulOperator takes two inputs and performs element-wise multiplication: + .. math:: + out.row[i] += scale * (in1.row[i] .* in2.row[i]) + where :math:`.*` means element-wise multiplication, and + scale is a config scalar, its default value is one. + The example usage is: + .. code-block:: python + op = dotmul_operator(x, y, + scale=1) + :param input: Input layer + :type input: LayerOutput + :param scale: config scalar, default value is one. + :type scale: float + :return: A DotMulOperator Object. + :rtype: DotMulOperator + """ + assert isinstance(x, LayerOutput) + assert isinstance(y, LayerOutput) + op = DotMulOperator(input_layer_names=[x.name, y.name], + scale=scale) + op.origin = [x, y] + return op @wrap_bias_attr_default(['padding_attr']) def context_projection(input, context_len, context_start=None, @@ -539,7 +536,10 @@ def __add__(self, other): if not self.finalized: assert isinstance(other, Projection) or isinstance(other, Operator) self.inputs.append(other) - self.parents.append(other.origin) + if isinstance(other, Projection): + self.parents.append(other.origin) + else: + self.parents.extend(other.origin) return self else: raise MixedLayerType.AddToSealedMixedLayerException() @@ -565,7 +565,7 @@ def __exit__(self, *args, **kwargs): @wrap_act_default(act=LinearActivation()) @wrap_bias_attr_default(has_bias=False) @layer_support(ERROR_CLIPPING, DROPOUT) -def mixed_layer(size, input=None, name=None, act=None, bias_attr=False, +def mixed_layer(size=0, input=None, name=None, act=None, bias_attr=False, layer_attr=None): """ Mixed Layer. A mixed layer will add all inputs together, then activate. diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py index 39c85c788eeca..27b22ecb701c5 100644 --- a/python/paddle/trainer_config_helpers/tests/layers_test_config.py +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -38,8 +38,11 @@ outputs(classification_cost(out, data_layer(name="label", size=num_classes))) +dotmul = mixed_layer(input=[dotmul_operator(x=x1, y=y1), + dotmul_projection(input=y1)]) + # for ctc -tmp = fc_layer(input=x1, +tmp = fc_layer(input=[x1, dotmul], size=num_classes + 1, act=SoftmaxActivation()) ctc = ctc_layer(input=tmp, From 425e5b0b07cbb1358562037d39ca291fab30fbf5 Mon Sep 17 00:00:00 2001 From: emailweixu Date: Mon, 19 Sep 2016 19:20:50 -0700 Subject: [PATCH 099/324] Instructions for update pull request (#84) --- doc/build/contribute_to_paddle.md | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/doc/build/contribute_to_paddle.md b/doc/build/contribute_to_paddle.md index 10d5d86311333..06fcff6172075 100644 --- a/doc/build/contribute_to_paddle.md +++ b/doc/build/contribute_to_paddle.md @@ -25,9 +25,12 @@ repo or just head straight to the command line: ```shell # Clone your fork to your local machine -git clone git@github.com:USERNAME/Paddle.git +git clone https://github.com/USERNAME/Paddle.git +``` +Then you can start to develop by making a local developement branch +```shell +git checkout -b MY_COOL_STUFF_BRANCH origin/master ``` -Then you can start to develop. ## Commit @@ -45,7 +48,7 @@ are the details if any. ## Keeping Fork Up to Date -Before pull your request, you shold sync you code from the latest PaddlePaddle. +Before pull your request, you should sync your code from the latest PaddlePaddle. To do this, you'll need to add a remote at first: ```shell @@ -60,8 +63,7 @@ git remote -v Update your fork with the latest upstream changes: ```shell -git fetch upstream -git pull upstream master +git pull --rebase upstream HEAD ``` If there are no unique commits locally, git will simply perform a fast-forward. @@ -74,10 +76,26 @@ Now, your local master branch is up-to-date with everything modified upstream. ```shell # push to your repository in Github -git push origin master +git push origin HEAD ``` ## Pull Request Go to the page for your fork on GitHub, select your development branch, and click the **pull request button**. + +## Update your pull request with the lastest version + +During the code review, your pull request may become stale because new commits in +baidu/Paddle. GitHub allows autmotic update if there is no conflict. You can do this +by clicking the "Update Branch" button in your pull request page. However, in the case +of conflict, you need to do the update manually. You need to do the following on +your local repository: +```shell +git checkout MY_COOL_STUFF_BRANCH +git pull --rebase upstream HEAD +# You may need to resolve the conflict according to the git prompt. +# Make and test your code. +git push -f origin HEAD +``` +Now your Pull Request is updated with the latest version. From d2e1b46f2411dbc715dc43e6d299a70c27593197 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Tue, 20 Sep 2016 11:48:16 +0800 Subject: [PATCH 100/324] update beam_search and seqToseq config, and add ExpActivation api --- demo/seqToseq/seqToseq_net.py | 22 ++- .../trainer_config_helpers/activations.rst | 7 + .../trainer/tests/sample_trainer_rnn_gen.conf | 127 ++++++------------ .../trainer_config_helpers/activations.py | 11 +- 4 files changed, 68 insertions(+), 99 deletions(-) diff --git a/demo/seqToseq/seqToseq_net.py b/demo/seqToseq/seqToseq_net.py index 479a64fa00d5f..a9c0dd4af600c 100644 --- a/demo/seqToseq/seqToseq_net.py +++ b/demo/seqToseq/seqToseq_net.py @@ -128,12 +128,16 @@ def gru_decoder_with_attention(enc_vec, enc_proj, current_word): return out decoder_group_name = "decoder_group" + group_inputs=[StaticInput(input=encoded_vector,is_seq=True), + StaticInput(input=encoded_proj,is_seq=True)] + if not is_generating: trg_embedding = embedding_layer( input=data_layer(name='target_language_word', size=target_dict_dim), size=word_vector_dim, param_attr=ParamAttr(name='_target_language_embedding')) + group_inputs.append(trg_embedding) # For decoder equipped with attention mechanism, in training, # target embeding (the groudtruth) is the data input, @@ -142,22 +146,13 @@ def gru_decoder_with_attention(enc_vec, enc_proj, current_word): # for the recurrent_group. decoder = recurrent_group(name=decoder_group_name, step=gru_decoder_with_attention, - input=[ - StaticInput(input=encoded_vector, - is_seq=True), - StaticInput(input=encoded_proj, - is_seq=True), trg_embedding - ]) + input=group_inputs) lbl = data_layer(name='target_language_next_word', size=target_dict_dim) - cost = classification_cost(input=decoder, label=lbl, ) + cost = classification_cost(input=decoder, label=lbl) outputs(cost) else: - gen_inputs = [StaticInput(input=encoded_vector, - is_seq=True), - StaticInput(input=encoded_proj, - is_seq=True), ] # In generation, the decoder predicts a next target word based on # the encoded source sequence and the last generated target word. @@ -171,10 +166,11 @@ def gru_decoder_with_attention(enc_vec, enc_proj, current_word): size=target_dict_dim, embedding_name='_target_language_embedding', embedding_size=word_vector_dim) - gen_inputs.append(trg_embedding) + group_inputs.append(trg_embedding) + beam_gen = beam_search(name=decoder_group_name, step=gru_decoder_with_attention, - input=gen_inputs, + input=group_inputs, id_input=data_layer(name="sent_id", size=1), dict_file=trg_dict_path, diff --git a/doc/ui/api/trainer_config_helpers/activations.rst b/doc/ui/api/trainer_config_helpers/activations.rst index 294f6e4d3127e..c4e14ed779efb 100644 --- a/doc/ui/api/trainer_config_helpers/activations.rst +++ b/doc/ui/api/trainer_config_helpers/activations.rst @@ -12,6 +12,13 @@ AbsActivation :members: AbsActivation :noindex: +ExpActivation +=============== + +.. automodule:: paddle.trainer_config_helpers.activations + :members: ExpActivation + :noindex: + IdentityActivation ================== diff --git a/paddle/trainer/tests/sample_trainer_rnn_gen.conf b/paddle/trainer/tests/sample_trainer_rnn_gen.conf index 5b65310e7649c..abb6e9b179326 100644 --- a/paddle/trainer/tests/sample_trainer_rnn_gen.conf +++ b/paddle/trainer/tests/sample_trainer_rnn_gen.conf @@ -13,96 +13,53 @@ # See the License for the specific language governing permissions and # limitations under the License. -#Todo(luotao02) This config is only used for unitest. It is out of date now, and will be updated later. -import math +from paddle.trainer_config_helpers import * -beam_search = get_config_arg('beam_search', bool, False) - -model_type("recurrent_nn") - -Settings(learning_rate=0, batch_size=15, algorithm='sgd') - -Inputs("sent_id", "dummy_data_input") -Outputs("predict_word") +settings(batch_size=15, learning_rate=0) num_words = 5 +beam_flag = get_config_arg('beam_search', bool, False) -DataLayer(name="sent_id", size=1, ) +sent_id = data_layer(name="sent_id", size=1) # This layer has no actual use, but only to decide batch_size in generation. # When generating, at least one Memory in RecurrentLayer MUST have a boot layer. -DataLayer(name="dummy_data_input", size=2, ) - -if beam_search: - RecurrentLayerGroupBegin("decoding_layer_group", - in_links=[], - out_links=["predict_word"], - generator=Generator(max_num_frames=10, - beam_size=2, - num_results_per_sample=2, )) -else: - RecurrentLayerGroupBegin("decoding_layer_group", - in_links=[], - out_links=["predict_word"], - generator=Generator(max_num_frames=10, )) -dummy_memory = Memory(name="dummy_memory", - size=2, - boot_layer="dummy_data_input") -MixedLayer(name="dummy_memory", - size=2, - bias=False, - inputs=[IdentityProjection(dummy_memory)], ) -state_memory = Memory(name="state", - size=num_words, - #boot_bias=True, - #boot_bias_active_type = "tanh", - ) - -predict_word_memory = Memory(name="predict_word", - size=num_words, - boot_with_const_id=0, ) - -MixedLayer( - name = "word_embedding", - size = num_words, # word embedding dim is the same as num_words in this test. - bias = False, - inputs = TableProjection(predict_word_memory, - initial_std=1, - learning_rate=0, - parameter_name="wordvec")) - -Layer( # simplified RNN for testing - name="state", - type="mixed", - size=num_words, - bias=False, - inputs=[FullMatrixProjection("word_embedding", - parameter_name="transtable")]) - -Layer(name="output", - type="mixed", - size=num_words, - active_type="exponential", - bias=False, - inputs=TransposedFullMatrixProjection("state", - initial_std=1, - learning_rate=0, - parameter_name="wordvec"), ) - -Layer(name="predict_word", type="maxid", inputs=["output"], ) - -Layer(name="eos_check", - type="eos_id", - eos_id=num_words - 1, - inputs=["predict_word"], ) -RecurrentLayerGroupEnd("decoding_layer_group") - -Evaluator(name="answer_printer", - type="seq_text_printer", - dict_file="./trainer/tests/test_gen_dict.txt", - result_file="./trainer/tests/dump_text.test", - inputs=[ - "sent_id", - "predict_word", - ], ) +dummy_data = data_layer(name="dummy_data_input", size=2) + +gen_inputs = [StaticInput(input=dummy_data, size=2), + GeneratedInput(size=num_words, + embedding_name="wordvec", + embedding_size=num_words)] + +def step(dummy_memory, predict_word): + + # simplified RNN for testing + with mixed_layer(size=num_words) as layer: + layer += full_matrix_projection(input=predict_word, + param_attr=ParamAttr(name="transtable")) + + with mixed_layer(size=num_words, act=ExpActivation()) as out: + out += trans_full_matrix_projection(input=layer, + param_attr=ParamAttr(name="wordvec")) + + return out + +beam_gen = beam_search(name="rnn_gen", + step=step, + input=gen_inputs, + id_input=sent_id, + dict_file="./trainer/tests/test_gen_dict.txt", + result_file="./trainer/tests/dump_text.test", + bos_id=0, + eos_id=num_words-1, + beam_size=2 if beam_flag else 1, + num_results_per_sample=2 if beam_flag else 1, + max_length=10) + +#outputs(beam_gen) +# In this config, as dummy_data_input doesn't work on beam_gen (we can find dummy_memory +# is read-only memory, and isn't used by other layers of step), we show the Inputs and Outputs +# as follows. Note that "__beam_search_predict__" is the default output name of beam_search. +Inputs("sent_id","dummy_data_input") +Outputs("__beam_search_predict__") diff --git a/python/paddle/trainer_config_helpers/activations.py b/python/paddle/trainer_config_helpers/activations.py index 24defb06a6d66..85534675199e7 100644 --- a/python/paddle/trainer_config_helpers/activations.py +++ b/python/paddle/trainer_config_helpers/activations.py @@ -14,7 +14,7 @@ __all__ = ["TanhActivation", "SigmoidActivation", "SoftmaxActivation", "IdentityActivation", "LinearActivation", - 'SequenceSoftmaxActivation', + 'SequenceSoftmaxActivation', 'ExpActivation', "ReluActivation", "BReluActivation", "SoftReluActivation", "STanhActivation", "AbsActivation", "SquareActivation", "BaseActivation"] @@ -185,3 +185,12 @@ class SquareActivation(BaseActivation): """ def __init__(self): BaseActivation.__init__(self, 'square', False) + +class ExpActivation(BaseActivation): + """ + Exponential Activation. + + .. math:: + f(z) = e^z. + """ + def __init__(self): BaseActivation.__init__(self, 'exponential', False) From 75cbf5ea9636ef74ec2682fd221db9a6ce5ec719 Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 20 Sep 2016 12:02:33 +0800 Subject: [PATCH 101/324] add gitignore for VS CODE on MAC OS --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 801c76325c92e..7e21ba0b750df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ *.DS_Store build/ *.user + +.vscode +.idea \ No newline at end of file From 2c5a6ac09575d66669e207fbc366fd981902cca7 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 20 Sep 2016 15:03:57 +0800 Subject: [PATCH 102/324] Optional fields to shrink generated proto size (#93) * remove unnecessary field set in ParameterConfig, Evaluators, etc --- paddle/gserver/layers/CRFLayer.cpp | 2 +- paddle/gserver/layers/CostLayer.cpp | 6 +- paddle/trainer/tests/.gitignore | 1 + proto/ModelConfig.proto.m4 | 2 +- proto/ParameterConfig.proto.m4 | 8 +- python/paddle/trainer/config_parser.py | 106 ++++++++++++------ .../trainer_config_helpers/evaluators.py | 37 +++--- .../trainer_config_helpers/optimizers.py | 2 +- 8 files changed, 99 insertions(+), 65 deletions(-) diff --git a/paddle/gserver/layers/CRFLayer.cpp b/paddle/gserver/layers/CRFLayer.cpp index fb0a0ddb3d45b..c1dcad2b5f2a8 100644 --- a/paddle/gserver/layers/CRFLayer.cpp +++ b/paddle/gserver/layers/CRFLayer.cpp @@ -31,7 +31,7 @@ bool CRFLayer::init(const LayerMap& layerMap, } // coeff only affect bp, keep consistent with CostLayer - coeff_ = config_.has_coeff() ? config_.coeff() : real(1.0); + coeff_ = config_.coeff(); if (inputLayers_.size() == 3) { weightLayer_ = inputLayers_[2]; } diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index 0f99aee03200c..14ff8510f7b19 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -26,11 +26,7 @@ namespace paddle { bool CostLayer::init(const LayerMap& layerMap, const ParameterMap& parameterMap) { bool ret = Layer::init(layerMap, parameterMap); - if (config_.has_coeff()) { - coeff_ = config_.coeff(); // coeff only affact bp - } else { - coeff_ = real(1.0); - } + coeff_ = config_.coeff(); if (!ret) return ret; CHECK_GE(inputLayers_.size(), 2UL); CHECK_LE(inputLayers_.size(), 3UL); diff --git a/paddle/trainer/tests/.gitignore b/paddle/trainer/tests/.gitignore index 79f701203671c..aedb0ef22e023 100644 --- a/paddle/trainer/tests/.gitignore +++ b/paddle/trainer/tests/.gitignore @@ -1,2 +1,3 @@ dump_text.test test_pydata_provider_wrapper.json +*proto.bin diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index a2b243a7869ea..b32f8b1ee9072 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -299,7 +299,7 @@ sinclude(`ModelConfigLayer.proto.m4') optional bool norm_by_times = 25; // for CostLayers - optional real coeff = 26; + optional real coeff = 26 [default = 1.0]; // for AverageLayer // can be set to: 'average', 'sum' or 'squarerootn' diff --git a/proto/ParameterConfig.proto.m4 b/proto/ParameterConfig.proto.m4 index 222e070089116..e8d512445e502 100644 --- a/proto/ParameterConfig.proto.m4 +++ b/proto/ParameterConfig.proto.m4 @@ -31,8 +31,8 @@ message ParameterUpdaterHookConfig { message ParameterConfig { required string name = 1; required uint64 size = 2; - required real learning_rate = 3; - required real momentum = 4; + optional real learning_rate = 3 [default = 1.0]; + optional real momentum = 4 [default = 0.0]; optional real initial_mean = 5 [default = 0.0]; optional real initial_std = 6 [default = 0.01]; // use L2-regularization if decay_rate set and decay_rate_l1 not set @@ -54,8 +54,8 @@ message ParameterConfig { optional int32 num_batches_regularization = 13 [default = 1]; // if is_sparse is true, para is sparse, else para is dense optional bool is_sparse = 14[default = false]; - // if para is sparse, format should be "csc" or "csr" - optional string format = 15[default = "csr"]; + // if para is sparse, format should be "csc" or "csr", empty means is not sparse + optional string format = 15 [default = ""]; // sparse remote update or not optional bool sparse_remote_update = 16 [default = false]; // gradient clipping threshold, no clipping by default diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index c5709208d4f1b..4ce01e005ae3c 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -114,15 +114,15 @@ # Initialize global variables. We use this function so that we can # call parse_config() multiple times def init_config_environment( - g_default_momentum = 0., - g_default_decay_rate = 0., + g_default_momentum = None, + g_default_decay_rate = None, g_default_initial_mean = 0., g_default_initial_std = 0.01, - g_default_num_batches_regularization = 1, + g_default_num_batches_regularization = None, g_default_initial_strategy = 0, g_default_initial_smart = False, - g_default_gradient_clipping_threshold = 0., - g_default_device = -1, + g_default_gradient_clipping_threshold = None, + g_default_device = None, g_default_update_hooks = None, g_default_compact_func = None, @@ -1099,12 +1099,12 @@ def Evaluator( inputs, chunk_scheme = None, num_chunk_types = None, - classification_threshold = 0.5, - positive_label = -1, - dict_file = "", - result_file = "", - num_results = 1, - delimited = True, + classification_threshold = None, + positive_label = None, + dict_file = None, + result_file = None, + num_results = None, + delimited = None, ): evaluator = g_config.model_config.evaluators.add() evaluator.type = type @@ -1120,12 +1120,19 @@ def Evaluator( evaluator.num_chunk_types = num_chunk_types g_current_submodel.evaluator_names.append(evaluator.name) - evaluator.classification_threshold = classification_threshold - evaluator.positive_label = positive_label - evaluator.dict_file = dict_file - evaluator.result_file = result_file - evaluator.num_results = num_results - evaluator.delimited = delimited + if classification_threshold is not None: + evaluator.classification_threshold = classification_threshold + if positive_label is not None: + evaluator.positive_label = positive_label + if dict_file is not None: + evaluator.dict_file = dict_file + + if result_file is not None: + evaluator.result_file = result_file + if num_results is not None: + evaluator.num_results = num_results + if delimited is not None: + evaluator.delimited = delimited class LayerBase(object): def __init__( @@ -1137,7 +1144,7 @@ def __init__( device=None, active_type="", drop_rate=0., - coeff=1.): + coeff=None): config_assert('@' not in name, "layer name: %s contain special character @" % name) global g_current_submodel @@ -1155,10 +1162,12 @@ def __init__( self.inputs = [self.inputs] self.config = g_config.model_config.layers.add() + assert isinstance(self.config, LayerConfig) self.config.name = name self.config.type = type self.config.active_type = active_type - self.config.coeff = coeff + if coeff is not None: + self.config.coeff = float(coeff) if size != 0: self.config.size = size if drop_rate != 0: @@ -1166,7 +1175,7 @@ def __init__( if device is not None: self.config.device = device - else: + elif g_default_device is not None: self.config.device = g_default_device for input_index in xrange(len(self.inputs)): @@ -1236,10 +1245,12 @@ def create_bias_parameter( if bias.parameter_name is None: bias.parameter_name = gen_bias_parameter_name(self.config.name) if bias.parameter_name not in g_parameter_map: + assert isinstance(self.config, LayerConfig) + Parameter( bias.parameter_name, size, - self.config.device, + self.config.device if self.config.HasField('device') else None, dims, bias.learning_rate, bias.momentum, @@ -1265,7 +1276,7 @@ def create_input_parameter( input_index, size, dims=None, - sparse = False, + sparse = None, format = "csr"): if dims is None: # TODO(yuyang18): print warning and callstack here! @@ -1293,7 +1304,7 @@ def create_input_parameter( Parameter( input_config.parameter_name, size, - self.config.device, + self.config.device if self.config.HasField("device") else None, dims, input_config.learning_rate, input_config.momentum, @@ -1353,6 +1364,8 @@ def __init__( if sparse: psize = self.inputs[input_index].nnz + else: + sparse = None self.create_input_parameter(input_index, psize, dims, sparse, format) self.create_bias_parameter(bias, self.config.size) @@ -2836,27 +2849,44 @@ def Parameter( para = g_config.model_config.parameters.add() para.name = name para.size = size - para.device = device - para.dims.extend(dims); - para.learning_rate = default(learning_rate, 1.) - para.momentum = default(momentum, g_default_momentum) + if device is not None: + para.device = int(device) + para.dims.extend(dims) + + if learning_rate is not None: + para.learning_rate = float(learning_rate) + + momentum = default(momentum, g_default_momentum) + if momentum is not None: + para.momentum = float(momentum) + config_assert(not momentum or not decay_rate_l1, "momentum and decay_rate_l1 cannot both be non-zero") - para.decay_rate = default(decay_rate, g_default_decay_rate) + + decay_rate = default(decay_rate, g_default_decay_rate) + if decay_rate is not None: + para.decay_rate = decay_rate + if decay_rate_l1 is not None: para.decay_rate_l1 = decay_rate_l1 para.initial_std = default(initial_std, g_default_initial_std) para.initial_mean = default(initial_mean, g_default_initial_mean) - para.num_batches_regularization = default( + + num_batches_regularization = default( num_batches_regularization, g_default_num_batches_regularization) + if num_batches_regularization is not None: + para.num_batches_regularization = int(num_batches_regularization) + if sparse_remote_update is not None: para.sparse_remote_update = sparse_remote_update if sparse_remote_update: g_config.opt_config.use_sparse_remote_updater = True if sparse_update is not None: para.sparse_update = sparse_update - para.gradient_clipping_threshold = default( - gradient_clipping_threshold, g_default_gradient_clipping_threshold); + gradient_clipping_threshold = default( + gradient_clipping_threshold, g_default_gradient_clipping_threshold) + if gradient_clipping_threshold is not None: + para.gradient_clipping_threshold = gradient_clipping_threshold para.initial_strategy = default(initial_strategy, g_default_initial_strategy) para.initial_smart = default(initial_smart, g_default_initial_smart) if para.initial_smart: @@ -2869,15 +2899,19 @@ def Parameter( para.initial_std = 1. / math.sqrt(para.size) if g_default_compact_func is not None: sparse, format, need_compact = g_default_compact_func(para.name) - para.is_sparse = default(sparse, False) - para.format = default(format, "") - para.need_compact = default(need_compact, False) + + if sparse is not None: + para.is_sparse = sparse + if format is not None: + para.format = format + if need_compact is not None: + para.need_compact = need_compact if is_static is not None: para.is_static = is_static config_assert(not para.sparse_remote_update or not para.is_static, "sparse_remote_update and is_static cannot both be true") - - para.is_shared = default(is_shared, False) + if is_shared is not None: + para.is_shared = is_shared update_hooks = default(update_hooks, g_default_update_hooks) diff --git a/python/paddle/trainer_config_helpers/evaluators.py b/python/paddle/trainer_config_helpers/evaluators.py index 985fae9f955c9..7a00d0b7ec57a 100644 --- a/python/paddle/trainer_config_helpers/evaluators.py +++ b/python/paddle/trainer_config_helpers/evaluators.py @@ -65,12 +65,12 @@ def evaluator_base( name=None, chunk_scheme=None, num_chunk_types=None, - classification_threshold=0.5, - positive_label=-1, - dict_file="", - result_file="", - num_results=1, - delimited=True): + classification_threshold=None, + positive_label=None, + dict_file=None, + result_file=None, + num_results=None, + delimited=None): """ Evaluator will evaluate the network status while training/testing. @@ -105,9 +105,10 @@ def evaluator_base( :type weight: LayerOutput. """ # inputs type assertions. - assert isinstance(classification_threshold, float) - assert isinstance(positive_label, int) - assert isinstance(num_results, int) + assert classification_threshold is None or isinstance( + classification_threshold, float) + assert positive_label is None or isinstance(positive_label, int) + assert num_results is None or isinstance(num_results, int) if not isinstance(input, list): input = [input] @@ -136,7 +137,7 @@ def classification_error_evaluator( label, name=None, weight=None, - threshold=0.5): + threshold=None): """ Classification Error Evaluator. It will print error rate for classification. @@ -253,7 +254,7 @@ def pnpair_evaluator( def precision_recall_evaluator( input, label, - positive_label=-1, + positive_label=None, weight=None, name=None, ): @@ -494,7 +495,7 @@ def gradient_printer_evaluator( @wrap_name_default() def maxid_printer_evaluator( input, - num_results=1, + num_results=None, name=None, ): """ @@ -518,13 +519,14 @@ def maxid_printer_evaluator( """ evaluator_base(name=name, type="max_id_printer", - input=input) + input=input, + num_results=num_results) @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def maxframe_printer_evaluator( input, - num_results=1, + num_results=None, name=None, ): """ @@ -556,9 +558,9 @@ def maxframe_printer_evaluator( @wrap_name_default() def seqtext_printer_evaluator( input, - dict_file="", - result_file="", - delimited=True, + result_file, + dict_file=None, + delimited=None, name=None, ): """ @@ -616,6 +618,7 @@ def seqtext_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ + assert isinstance(result_file, basestring) evaluator_base(name=name, type="seq_text_printer", input=input, diff --git a/python/paddle/trainer_config_helpers/optimizers.py b/python/paddle/trainer_config_helpers/optimizers.py index ed676ac2152a6..af85f745f63e5 100644 --- a/python/paddle/trainer_config_helpers/optimizers.py +++ b/python/paddle/trainer_config_helpers/optimizers.py @@ -79,7 +79,7 @@ def to_setting_kwargs(self): 'learning_method': 'momentum' } - def __init__(self, momentum=1e-3): + def __init__(self, momentum=None): self.momentum = momentum From 2daa05c0b5a14d9b02267a7fff1d48f409f0cb50 Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 20 Sep 2016 16:09:05 +0800 Subject: [PATCH 103/324] add build on MAC OSX docs --- doc/build/build_from_source.md | 142 +++++++++++++++++++++++++++++++-- 1 file changed, 136 insertions(+), 6 deletions(-) diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index a191d31318aa6..c71ff260f8d0a 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -1,7 +1,11 @@ Build and Install ================= -## Requirement +* [1. Requirement](#Requirement) +* [2. Build on Ubuntu](#ubuntu) +* [3. Build on Mac OS X](#mac) + +## Requirement ### Dependents @@ -28,7 +32,7 @@ PaddlePaddle also support some build options, you have to install related librar - **WITH_STYLE_CHECK**: Style check for source code -## Building on Ubuntu14.04 +## Building on Ubuntu14.04 ### Install Dependencies @@ -44,7 +48,7 @@ sudo apt-get install libgflags-dev sudo apt-get install libgtest-dev sudo pip install wheel pushd /usr/src/gtest -cmake . +cmake .. make sudo cp *.a /usr/lib popd @@ -102,19 +106,19 @@ Here are some examples of cmake command with different options: **only cpu** ```bash -cmake -DWITH_GPU=OFF -DWITH_DOC=OFF +cmake -DWITH_GPU=OFF -DWITH_DOC=OFF .. ``` **gpu** ```bash -cmake -DWITH_GPU=ON -DWITH_DOC=OFF +cmake -DWITH_GPU=ON -DWITH_DOC=OFF .. ``` **gpu with doc and swig** ```bash -cmake -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON +cmake -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON .. ``` Finally, you can download source code and build: @@ -139,3 +143,129 @@ And if you set WITH_SWIG_PY=ON, you have to install related python predict api a ```bash pip install /opt/paddle/share/wheels/*.whl ``` +## Building on Mac OS X + +### Prerequisites +This guide is based on Mac OS X 10.11 (El Capitan). Note that if you are running an up to date version of OS X, +you will already have Python 2.7.10 and Numpy 1.8 installed. + +The best option is to use the package manager homebrew to handle installations and upgrades for you. +To install homebrew, first open a terminal window (you can find Terminal in the Utilities folder in Applications), and issue the command: + +```bash +# install brew +/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +# install pip +easy_install pip +``` + +### Install Dependencies + +- **CPU Dependencies** + +```bash +# Install fundamental dependents +brew install glog gflags cmake protobuf openblas + +# Install google test on Mac OS X +# Download gtest 1.7.0 +wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz +tar -xvf googletest-release-1.7.0.tar.gz && cd googletest-release-1.7.0 +# Build gtest +mkdir build && cmake .. +make +# Install gtest library +sudo cp -r ../include/gtest /usr/local/include/ +sudo cp lib*.a /usr/local/lib +``` + + +- **GPU Dependencies(optional)** + +If you need to build GPU version, the first thing you need is a machine that has NVIDIA GPU and CUDA installed. +And you also need to install cuDNN. + +You can download CUDA toolkit and cuDNN from nvidia website: + +```bash +https://developer.nvidia.com/cuda-downloads +https://developer.nvidia.com/cudnn +``` +You can copy cuDNN files into the CUDA toolkit directory, for instance: + +```bash +sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C /usr/local +sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* +``` +Then you need to set DYLD\_LIBRARY\_PATH, CUDA\_HOME and PATH environment variables in ~/.bashrc. + +```bash +export DYLD_LIBRARY_PATH=/usr/local/cuda/lib:$DYLD_LIBRARY_PATH +export PATH=/usr/local/cuda/bin:$PATH +``` +- **Python Dependencies(optional)** + +If you want to compile PaddlePaddle with python predict API, you need to add -DWITH_SWIG_PY=ON in cmake command and install these first: + +```bash +brew install swig +``` + +- **Doc Dependencies(optional)** + +If you want to compile PaddlePaddle with doc, you need to add -DWITH_DOC=ON in cmake command and install these first: + +```bash +pip install 'sphinx>=1.4.0' +pip install sphinx_rtd_theme breathe recommonmark +brew install doxygen +``` + +### Build and Install + +CMake can find dependent libraries in system default paths firstly. +After installing some optional libraries, corresponding build option will be on automatically (for instance, glog, gtest and gflags). +If not found, you have to set following variables manually via CMake command (CUDNN_ROOT, ATLAS_ROOT, MKL_ROOT, OPENBLAS_ROOT). + +Here are some examples of CMake command with different options: + +**only cpu** + +```bash +cmake -DWITH_GPU=OFF -DWITH_DOC=OFF .. +``` + +**gpu** + +```bash +cmake -DWITH_GPU=ON -DWITH_DOC=OFF .. +``` + +**gpu with doc and swig** + +```bash +cmake -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON .. +``` + +Finally, you can download source code and build: + +```bash +git clone https://github.com/baidu/Paddle paddle +cd paddle +mkdir build +cd build +# you can add build option here, such as: +cmake -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX= .. +# please use sudo make install, if you want +# to install PaddlePaddle into the system +make -j `nproc` && make install +# PaddlePaddle installation path +export PATH=/bin:$PATH +``` +**Note** + +And if you set WITH_SWIG_PY=ON, you have to install related python predict api at the same time: + +```bash +sudo pip install /opt/paddle/share/wheels/*.whl +``` \ No newline at end of file From 536bf7d87f22ba2dbc84288a35341af523c9169d Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 20 Sep 2016 20:23:58 +0800 Subject: [PATCH 104/324] fix poolAllocator unordered_map bug on Mac OS X * using map to replace unordered_map on Mac --- paddle/math/PoolAllocator.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/paddle/math/PoolAllocator.h b/paddle/math/PoolAllocator.h index 22af0eb893753..a50deece3a853 100644 --- a/paddle/math/PoolAllocator.h +++ b/paddle/math/PoolAllocator.h @@ -19,6 +19,7 @@ limitations under the License. */ #include #include #include +#include #include "Allocator.h" namespace paddle { @@ -52,7 +53,11 @@ class PoolAllocator { void printAll(); std::unique_ptr allocator_; std::mutex mutex_; +#if defined(__APPLE__) || defined(__OSX__) + std::map> pool_; +#else std::unordered_map> pool_; +#endif size_t sizeLimit_; size_t poolMemorySize_; std::string name_; From 8b1c76af3c0ffdcc548bbf952f00f6b0a5186871 Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 20 Sep 2016 20:31:01 +0800 Subject: [PATCH 105/324] comment LOG(info) in createPythonClass * it makes unit test failed. --- paddle/utils/PythonUtil.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/utils/PythonUtil.cpp b/paddle/utils/PythonUtil.cpp index 78c3a80674f9c..9ee7a29aad0b6 100644 --- a/paddle/utils/PythonUtil.cpp +++ b/paddle/utils/PythonUtil.cpp @@ -144,12 +144,12 @@ PyObjectPtr createPythonClass( const std::map& kwargs) { PyGuard guard; PyObjectPtr pyModule(PyImport_ImportModule(moduleName.c_str())); - LOG(INFO) << "createPythonClass moduleName.c_str:" << moduleName.c_str(); + // LOG(INFO) << "createPythonClass moduleName.c_str:" << moduleName.c_str(); CHECK_PY(pyModule) << "Import module " << moduleName << " failed."; PyObjectPtr pyDict(PyModule_GetDict(pyModule.get())); CHECK_PY(pyDict) << "Get Dict failed."; PyObjectPtr pyClass(PyDict_GetItemString(pyDict.get(), className.c_str())); - LOG(INFO) << "createPythonClass className.c_str():" << className.c_str(); + // LOG(INFO) << "createPythonClass className.c_str():" << className.c_str(); CHECK_PY(pyClass) << "Import class " << className << " failed."; PyObjectPtr argsObjectList(PyTuple_New(args.size())); for (size_t i = 0; i < args.size(); ++i) { From 7ff8e76229325b6f5dcb11dce83fff30493ea6bf Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 20 Sep 2016 20:51:02 +0800 Subject: [PATCH 106/324] Shrink batch size on unit test for Mac OS X --- paddle/gserver/tests/test_LayerGrad.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 5c80eb546cfaf..3150c31e4900c 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -50,7 +50,7 @@ TEST(Operator, dot_mul) { TEST(Projection, context) { for (auto contextStart : {-5, -3, -1, 0, 3}) { for (auto contextLength : {1, 2, 5, 7}) { - for (auto batchSize : {1, 2, 5, 20, 100}) { + for (auto batchSize : {1, 2, 5, 20, 50}) { for (auto trainablePadding : {false, true}) { LOG(INFO) << " contextStart=" << contextStart << " contextLength=" << contextLength From aaed5cfccc6460b32c7884ac1a07391bb7b3d869 Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 20 Sep 2016 22:26:45 +0800 Subject: [PATCH 107/324] revert real into float for swig API --- paddle/api/Matrix.cpp | 34 ++++++++++++++++---------------- paddle/api/PaddleAPI.h | 44 +++++++++++++++++++++--------------------- paddle/api/Util.cpp | 4 ++-- paddle/api/Vector.cpp | 30 ++++++++++++++-------------- 4 files changed, 56 insertions(+), 56 deletions(-) diff --git a/paddle/api/Matrix.cpp b/paddle/api/Matrix.cpp index c40a47f3accf9..9ae3716fa862c 100644 --- a/paddle/api/Matrix.cpp +++ b/paddle/api/Matrix.cpp @@ -44,7 +44,7 @@ Matrix* Matrix::createZero(size_t height, size_t width, bool useGpu) { return m; } -Matrix* Matrix::createDense(const std::vector& data, size_t height, +Matrix* Matrix::createDense(const std::vector& data, size_t height, size_t width, bool useGpu) { auto m = new Matrix(); m->m->mat = paddle::Matrix::create(height, width, useGpu); @@ -52,7 +52,7 @@ Matrix* Matrix::createDense(const std::vector& data, size_t height, return m; } -Matrix* Matrix::createCpuDenseFromNumpy(real* data, int dim1, int dim2, +Matrix* Matrix::createCpuDenseFromNumpy(float* data, int dim1, int dim2, bool copy) { auto m = new Matrix(); if (copy) { @@ -64,7 +64,7 @@ Matrix* Matrix::createCpuDenseFromNumpy(real* data, int dim1, int dim2, return m; } -Matrix* Matrix::createGpuDenseFromNumpy(real* data, int dim1, int dim2) { +Matrix* Matrix::createGpuDenseFromNumpy(float* data, int dim1, int dim2) { auto m = new Matrix(); m->m->mat = paddle::Matrix::create(dim1, dim2, false, true); m->m->mat->copyFrom(data, dim1 * dim2); @@ -86,7 +86,7 @@ size_t Matrix::getHeight() const { return m->mat->getHeight(); } size_t Matrix::getWidth() const { return m->mat->getWidth(); } -real Matrix::get(size_t x, size_t y) const throw(RangeError) { +float Matrix::get(size_t x, size_t y) const throw(RangeError) { if (x > this->getWidth() || y > this->getHeight()) { RangeError e; throw e; @@ -94,7 +94,7 @@ real Matrix::get(size_t x, size_t y) const throw(RangeError) { return m->mat->getElement(x, y); } -void Matrix::set(size_t x, size_t y, real val) throw(RangeError, +void Matrix::set(size_t x, size_t y, float val) throw(RangeError, UnsupportError) { if (x > this->getWidth() || y > this->getHeight()) { RangeError e; @@ -193,10 +193,10 @@ FloatArray Matrix::getData() const { auto rawMat = m->mat.get(); if (dynamic_cast(rawMat->getMemoryHandle().get())) { // is gpu. then copy data - real* data = rawMat->getData(); + float* data = rawMat->getData(); size_t len = rawMat->getElementCnt(); - real* cpuData = new real[len]; - hl_memcpy_device2host(cpuData, data, len * sizeof(real)); + float* cpuData = new float[len]; + hl_memcpy_device2host(cpuData, data, len * sizeof(float)); FloatArray ret_val(cpuData, len); ret_val.needFree = true; return ret_val; @@ -208,7 +208,7 @@ FloatArray Matrix::getData() const { void Matrix::sparseCopyFrom( const std::vector& rows, const std::vector& cols, - const std::vector& vals) throw(UnsupportError) { + const std::vector& vals) throw(UnsupportError) { auto cpuSparseMat = std::dynamic_pointer_cast(m->mat); if (cpuSparseMat != nullptr) { @@ -217,7 +217,7 @@ void Matrix::sparseCopyFrom( // <<" ValSize = "<copyFrom(const_cast&>(rows), const_cast&>(cols), - const_cast&>(vals)); + const_cast&>(vals)); } else { UnsupportError e; throw e; @@ -226,7 +226,7 @@ void Matrix::sparseCopyFrom( void* Matrix::getSharedPtr() const { return &m->mat; } -void Matrix::toNumpyMatInplace(real** view_data, int* dim1, +void Matrix::toNumpyMatInplace(float** view_data, int* dim1, int* dim2) throw(UnsupportError) { auto cpuMat = std::dynamic_pointer_cast(m->mat); if (cpuMat) { @@ -237,9 +237,9 @@ void Matrix::toNumpyMatInplace(real** view_data, int* dim1, throw UnsupportError(); } } -void Matrix::copyToNumpyMat(real** view_m_data, int* dim1, +void Matrix::copyToNumpyMat(float** view_m_data, int* dim1, int* dim2) throw(UnsupportError) { - static_assert(sizeof(paddle::real) == sizeof(real), + static_assert(sizeof(float) == sizeof(float), "Currently PaddleAPI only support for single " "precision version of paddle."); if (this->isSparse()) { @@ -247,16 +247,16 @@ void Matrix::copyToNumpyMat(real** view_m_data, int* dim1, } else { *dim1 = m->mat->getHeight(); *dim2 = m->mat->getWidth(); - *view_m_data = new real[(*dim1) * (*dim2)]; + *view_m_data = new float[(*dim1) * (*dim2)]; if (auto cpuMat = dynamic_cast(m->mat.get())) { auto src = cpuMat->getData(); auto dest = *view_m_data; - std::memcpy(dest, src, sizeof(paddle::real) * (*dim1) * (*dim2)); + std::memcpy(dest, src, sizeof(float) * (*dim1) * (*dim2)); } else if (auto gpuMat = dynamic_cast(m->mat.get())) { auto src = gpuMat->getData(); auto dest = *view_m_data; hl_memcpy_device2host(dest, src, - sizeof(paddle::real) * (*dim1) * (*dim2)); + sizeof(float) * (*dim1) * (*dim2)); } else { LOG(WARNING) << "Unexpected Situation"; throw UnsupportError(); @@ -264,7 +264,7 @@ void Matrix::copyToNumpyMat(real** view_m_data, int* dim1, } } -void Matrix::copyFromNumpyMat(real* data, int dim1, +void Matrix::copyFromNumpyMat(float* data, int dim1, int dim2) throw(UnsupportError, RangeError) { if (isSparse()) { throw UnsupportError(); diff --git a/paddle/api/PaddleAPI.h b/paddle/api/PaddleAPI.h index 69f3240a77974..b3140617af188 100644 --- a/paddle/api/PaddleAPI.h +++ b/paddle/api/PaddleAPI.h @@ -56,10 +56,10 @@ class UnsupportError {}; /// This type will map to python's list of float. struct FloatArray { - const real* buf; + const float* buf; const size_t length; bool needFree; // true if the buf is dynamic alloced. - FloatArray(const real* b, const size_t l); + FloatArray(const float* b, const size_t l); }; /// This type will map to python's list of int @@ -72,11 +72,11 @@ struct IntArray { /// This type will map to python's list of (int, float) struct IntWithFloatArray { - const real* valBuf; + const float* valBuf; const int* idxBuf; const size_t length; bool needFree; - IntWithFloatArray(const real* v, const int* i, size_t l, bool f = false); + IntWithFloatArray(const float* v, const int* i, size_t l, bool f = false); }; enum SparseValueType { SPARSE_NON_VALUE = 0, SPARSE_VALUE = 1 }; @@ -122,7 +122,7 @@ class Matrix { * @param data list of float should be passed in python. * @note the value will be copy into a new matrix. */ - static Matrix* createDense(const std::vector& data, size_t height, + static Matrix* createDense(const std::vector& data, size_t height, size_t width, bool useGpu = false); /** @@ -134,11 +134,11 @@ class Matrix { * @param copy true if copy into a new matrix, false will create * matrix inplace. */ - static Matrix* createCpuDenseFromNumpy(real* data, int dim1, int dim2, + static Matrix* createCpuDenseFromNumpy(float* data, int dim1, int dim2, bool copy = false); /// Create Gpu Dense Matrix from numpy matrix, dtype=float32 - static Matrix* createGpuDenseFromNumpy(real* data, int dim1, int dim2); + static Matrix* createGpuDenseFromNumpy(float* data, int dim1, int dim2); /** * Cast to numpy matrix. @@ -154,15 +154,15 @@ class Matrix { * numpy_mat = m.toNumpyMat() * @endcode */ - void toNumpyMatInplace(real** view_data, int* dim1, + void toNumpyMatInplace(float** view_data, int* dim1, int* dim2) throw(UnsupportError); /// Copy To numpy mat. - void copyToNumpyMat(real** view_m_data, int* dim1, + void copyToNumpyMat(float** view_m_data, int* dim1, int* dim2) throw(UnsupportError); /// Copy From Numpy Mat - void copyFromNumpyMat(real* data, int dim1, int dim2) throw(UnsupportError, + void copyFromNumpyMat(float* data, int dim1, int dim2) throw(UnsupportError, RangeError); /// return true if this matrix is sparse. @@ -181,9 +181,9 @@ class Matrix { size_t getWidth() const; - real get(size_t x, size_t y) const throw(RangeError); + float get(size_t x, size_t y) const throw(RangeError); - void set(size_t x, size_t y, real val) throw(RangeError, UnsupportError); + void set(size_t x, size_t y, float val) throw(RangeError, UnsupportError); /// return type is list of float FloatArray getData() const; @@ -195,8 +195,8 @@ class Matrix { */ void sparseCopyFrom(const std::vector& rows, const std::vector& cols, - const std::vector& values = - std::vector()) throw(UnsupportError); + const std::vector& values = + std::vector()) throw(UnsupportError); bool isGpu() const; @@ -228,33 +228,33 @@ class Vector { * * It will create a new vector, and copy data into it. */ - static Vector* create(const std::vector& data, bool useGpu = false); + static Vector* create(const std::vector& data, bool useGpu = false); /** * Create Cpu Vector from numpy array, which dtype=float32 * * If copy is false, it will create vector inplace. */ - static Vector* createCpuVectorFromNumpy(real* data, int dim, + static Vector* createCpuVectorFromNumpy(float* data, int dim, bool copy = false); /// Create Gpu Vector from numpy array, which dtype=float32 - static Vector* createGpuVectorFromNumpy(real* data, int dim); + static Vector* createGpuVectorFromNumpy(float* data, int dim); /// Cast to numpy array inplace. - void toNumpyArrayInplace(real** view_data, int* dim1) throw(UnsupportError); + void toNumpyArrayInplace(float** view_data, int* dim1) throw(UnsupportError); /// Copy to numpy array. - void copyToNumpyArray(real** view_m_data, int* dim1); + void copyToNumpyArray(float** view_m_data, int* dim1); /// Copy from numpy array. - void copyFromNumpyArray(real* data, int dim); + void copyFromNumpyArray(float* data, int dim); /// __getitem__ in python - real get(const size_t idx) const throw(RangeError, UnsupportError); + float get(const size_t idx) const throw(RangeError, UnsupportError); /// __setitem__ in python - void set(const size_t idx, real val) throw(RangeError, UnsupportError); + void set(const size_t idx, float val) throw(RangeError, UnsupportError); /// Return is GPU vector or not. bool isGpu() const; diff --git a/paddle/api/Util.cpp b/paddle/api/Util.cpp index fe89a62cd3908..8a6741078f2f1 100644 --- a/paddle/api/Util.cpp +++ b/paddle/api/Util.cpp @@ -31,13 +31,13 @@ void initPaddle(int argc, char** argv) { feenableexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW); } -FloatArray::FloatArray(const real* b, const size_t l) +FloatArray::FloatArray(const float* b, const size_t l) : buf(b), length(l), needFree(false) {} IntArray::IntArray(const int* b, const size_t l, bool f) : buf(b), length(l), needFree(f) {} -IntWithFloatArray::IntWithFloatArray(const real* v, const int* i, size_t l, +IntWithFloatArray::IntWithFloatArray(const float* v, const int* i, size_t l, bool f) : valBuf(v), idxBuf(i), length(l), needFree(f) {} diff --git a/paddle/api/Vector.cpp b/paddle/api/Vector.cpp index b61eb7934b781..1affc1a5fefb8 100644 --- a/paddle/api/Vector.cpp +++ b/paddle/api/Vector.cpp @@ -140,7 +140,7 @@ struct VectorPrivate { paddle::VectorPtr vec; void safeAccessData(const size_t idx, - const std::function& func) const + const std::function& func) const throw(RangeError, UnsupportError) { auto cpuVec = std::dynamic_pointer_cast(vec); if (cpuVec != nullptr) { @@ -170,7 +170,7 @@ Vector* Vector::createZero(size_t sz, bool useGpu) { return retVec; } -Vector* Vector::create(const std::vector& data, bool useGpu) { +Vector* Vector::create(const std::vector& data, bool useGpu) { auto retVec = new Vector(); retVec->m->vec = paddle::Vector::create(data.size(), useGpu); retVec->m->vec->copyFrom(data.data(), data.size()); @@ -188,7 +188,7 @@ Vector* Vector::createByPaddleVectorPtr(void* ptr) { } } -Vector* Vector::createCpuVectorFromNumpy(real* data, int dim, bool copy) { +Vector* Vector::createCpuVectorFromNumpy(float* data, int dim, bool copy) { CHECK_GT(dim, 0); auto retVec = new Vector(); if (copy) { @@ -200,7 +200,7 @@ Vector* Vector::createCpuVectorFromNumpy(real* data, int dim, bool copy) { return retVec; } -Vector* Vector::createGpuVectorFromNumpy(real* data, int dim) { +Vector* Vector::createGpuVectorFromNumpy(float* data, int dim) { CHECK_GT(dim, 0); auto retVec = new Vector(); retVec->m->vec = paddle::Vector::create((size_t)dim, true); @@ -208,7 +208,7 @@ Vector* Vector::createGpuVectorFromNumpy(real* data, int dim) { return retVec; } -void Vector::toNumpyArrayInplace(real** view_data, +void Vector::toNumpyArrayInplace(float** view_data, int* dim1) throw(UnsupportError) { auto v = std::dynamic_pointer_cast(m->vec); if (v != nullptr) { @@ -219,20 +219,20 @@ void Vector::toNumpyArrayInplace(real** view_data, } } -void Vector::copyToNumpyArray(real** view_m_data, int* dim1) { +void Vector::copyToNumpyArray(float** view_m_data, int* dim1) { *dim1 = m->vec->getSize(); - *view_m_data = new real[*dim1]; + *view_m_data = new float[*dim1]; if (auto cpuVec = dynamic_cast(m->vec.get())) { - std::memcpy(*view_m_data, cpuVec->getData(), sizeof(real) * (*dim1)); + std::memcpy(*view_m_data, cpuVec->getData(), sizeof(float) * (*dim1)); } else if (auto gpuVec = dynamic_cast(m->vec.get())) { hl_memcpy_device2host(*view_m_data, gpuVec->getData(), - sizeof(real) * (*dim1)); + sizeof(float) * (*dim1)); } else { LOG(INFO) << "Unexpected situation"; } } -void Vector::copyFromNumpyArray(real* data, int dim) { +void Vector::copyFromNumpyArray(float* data, int dim) { m->vec->resize(dim); m->vec->copyFrom(data, dim); } @@ -241,15 +241,15 @@ bool Vector::isGpu() const { return std::dynamic_pointer_cast(m->vec) != nullptr; } -real Vector::get(const size_t idx) const throw(RangeError, UnsupportError) { - real r; - m->safeAccessData(idx, [&](real& o) { r = o; }); +float Vector::get(const size_t idx) const throw(RangeError, UnsupportError) { + float r; + m->safeAccessData(idx, [&](float& o) { r = o; }); return r; } -void Vector::set(const size_t idx, real val) throw(RangeError, +void Vector::set(const size_t idx, float val) throw(RangeError, UnsupportError) { - m->safeAccessData(idx, [&](real& o) { o = val; }); + m->safeAccessData(idx, [&](float& o) { o = val; }); } size_t Vector::getSize() const { return m->vec->getSize(); } From b130ba73286dfcf60dc800eb8c8625d294acd20f Mon Sep 17 00:00:00 2001 From: Haonan Date: Tue, 20 Sep 2016 22:45:33 -0700 Subject: [PATCH 108/324] fixed bugs in conv_operator; add calc_output_size (#101) --- python/paddle/trainer/config_parser.py | 8 +++++--- python/paddle/trainer_config_helpers/layers.py | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 4ce01e005ae3c..79b85214228f3 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -636,7 +636,6 @@ def __init__( input_layer_names, ): self.add_keys(locals()) - self.operator_conf = OperatorConfig() self.operator_conf.type = self.type @@ -686,12 +685,15 @@ def __init__( if num_filters is not None: self.operator_conf.num_filters = num_filters - parse_conv(conv_conf, input_layer_names[0], self.operator_conf.conv_conf, True) + parse_conv(conv_conf, + MakeLayerNameInSubmodel(input_layer_names[0]), + self.operator_conf.conv_conf) self.operator_conf.output_size = (self.operator_conf.conv_conf.output_x ** 2) * num_filters config_assert(len(input_layer_names) == 2, "Conv is binary operator") - + def calc_output_size(self, input_sizes): + return self.operator_conf.output_size # please refer to the comments in proto/ModelConfig.proto diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index fab7e6e091863..a5bacaf079278 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -2668,7 +2668,7 @@ def __add_evaluator__(e): return LayerOutput(name, LayerType.COST, parents=[input, label]) def conv_operator(input, filter_size, num_filters, - num_channel=None, stride=1, padding=0, + num_channel=None, stride=1, padding=0, groups=1, filter_size_y=None, stride_y=None, padding_y=None): """ Different from img_conv_layer, conv_op is an Operator, which can be used @@ -2715,7 +2715,7 @@ def conv_operator(input, filter_size, num_filters, stride_y = stride if padding_y is None: padding_y = padding - op = ConvOperator(input_layer_name=[x.name for x in input], + op = ConvOperator(input_layer_names=[x.name for x in input], num_filters = num_filter, conv_conf=Conv(filter_size=filter_size, padding=padding, @@ -2723,7 +2723,8 @@ def conv_operator(input, filter_size, num_filters, channels=num_channel, filter_size_y=filter_size_y, padding_y=padding_y, - stride_y=stride_y)) + stride_y=stride_y, + groups=groups)) op.origin = input op.origin.operator = "conv_op" return op From b8d26ff4d7ceb2a7c78741e98992024fe9254ac2 Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 21 Sep 2016 17:10:15 +0800 Subject: [PATCH 109/324] fix float except bugs --- paddle/gserver/dataproviders/PyDataProvider.cpp | 3 +-- paddle/math/PoolAllocator.h | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/paddle/gserver/dataproviders/PyDataProvider.cpp b/paddle/gserver/dataproviders/PyDataProvider.cpp index cc3e09a3c2ecb..1332c0ab635b6 100644 --- a/paddle/gserver/dataproviders/PyDataProvider.cpp +++ b/paddle/gserver/dataproviders/PyDataProvider.cpp @@ -46,7 +46,6 @@ PyDataProvider::PyDataProvider(const DataConfig& config, bool useGpu, } void PyDataProvider::loadData(const std::vector& fileList) { - int feFlag = fegetexcept(); VLOG(1) << "module:" << pyModuleName_ << " class:" << pyClassName_; classInstance_ = createPythonClass(pyModuleName_, pyClassName_, fileList, pyUserArgs_); @@ -57,7 +56,7 @@ void PyDataProvider::loadData(const std::vector& fileList) { std::string headerInfo = std::string(PyString_AsString(obj.get()), PyString_Size(obj.get())); parseHeaderData(headerInfo); - feenableexcept(feFlag); + feenableexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW); } void PyDataProvider::parseHeaderData(const std::string& headerData) { diff --git a/paddle/math/PoolAllocator.h b/paddle/math/PoolAllocator.h index a50deece3a853..aca8ffb0ab42e 100644 --- a/paddle/math/PoolAllocator.h +++ b/paddle/math/PoolAllocator.h @@ -53,11 +53,7 @@ class PoolAllocator { void printAll(); std::unique_ptr allocator_; std::mutex mutex_; -#if defined(__APPLE__) || defined(__OSX__) - std::map> pool_; -#else std::unordered_map> pool_; -#endif size_t sizeLimit_; size_t poolMemorySize_; std::string name_; From 98bc889cb5d97215c62300bf30da45165d34b175 Mon Sep 17 00:00:00 2001 From: Haonan Date: Wed, 21 Sep 2016 14:00:57 -0700 Subject: [PATCH 110/324] split the input list of conv_operator into two inputs: image and filter (#104) --- python/paddle/trainer_config_helpers/layers.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index a5bacaf079278..9963b38134046 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -2667,7 +2667,7 @@ def __add_evaluator__(e): return LayerOutput(name, LayerType.COST, parents=[input, label]) -def conv_operator(input, filter_size, num_filters, +def conv_operator(img, filter, filter_size, num_filters, num_channel=None, stride=1, padding=0, groups=1, filter_size_y=None, stride_y=None, padding_y=None): """ @@ -2680,13 +2680,16 @@ def conv_operator(input, filter_size, num_filters, .. code-block:: python - op = conv_operator(input=[layer1, layer2], + op = conv_operator(img=input1, + filter=input2, filter_size=3.0, num_filters=64, num_channels=64) - :param input: Input layer. - :type input: LayerOutput|list|tuple + :param img: input image + :type img: LayerOutput + :param filter: input filter + :type filter: LayerOutput :param filter_size: The x dimension of a filter kernel. :type filter_size: int :param filter_size_y: The y dimension of a filter kernel. Since @@ -2708,14 +2711,13 @@ def conv_operator(input, filter_size, num_filters, :return: A ConvOperator Object. :rtype: ConvOperator """ - assert isinstance(input, list) or isinstance(input, tuple) if filter_size_y is None: filter_size_y = filter_size if stride_y is None: stride_y = stride if padding_y is None: padding_y = padding - op = ConvOperator(input_layer_names=[x.name for x in input], + op = ConvOperator(input_layer_names=[img.name, filter.name], num_filters = num_filter, conv_conf=Conv(filter_size=filter_size, padding=padding, @@ -2725,7 +2727,7 @@ def conv_operator(input, filter_size, num_filters, padding_y=padding_y, stride_y=stride_y, groups=groups)) - op.origin = input + op.origin = [img, filter] op.origin.operator = "conv_op" return op From 8e957df4b2f74cdf6587f4877c26de6d71894d6d Mon Sep 17 00:00:00 2001 From: luotao1 Date: Thu, 22 Sep 2016 12:31:31 +0800 Subject: [PATCH 111/324] fix bug in dotmul_operator's api and anotation (#99) * fix bug in dotmul_operator's api and anotation * update rnn document * remove redundant info of projection and operator in layers.py --- doc/algorithm/rnn/rnn.rst | 23 +++++-------- doc/ui/api/trainer_config_helpers/layers.rst | 6 ++++ python/paddle/trainer/config_parser.py | 8 ++--- .../paddle/trainer_config_helpers/layers.py | 33 +++++++++---------- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/doc/algorithm/rnn/rnn.rst b/doc/algorithm/rnn/rnn.rst index 9653ddbf37176..4753db450b744 100644 --- a/doc/algorithm/rnn/rnn.rst +++ b/doc/algorithm/rnn/rnn.rst @@ -142,12 +142,15 @@ We also project the encoder vector to :code:`decoder_size` dimensional space, ge The decoder uses :code:`recurrent_group` to define the recurrent neural network. The step and output functions are defined in :code:`gru_decoder_with_attention`: .. code-block:: python - + group_inputs=[StaticInput(input=encoded_vector,is_seq=True), + StaticInput(input=encoded_proj,is_seq=True)] trg_embedding = embedding_layer( input=data_layer(name='target_language_word', size=target_dict_dim), size=word_vector_dim, param_attr=ParamAttr(name='_target_language_embedding')) + group_inputs.append(trg_embedding) + # For decoder equipped with attention mechanism, in training, # target embedding (the groudtruth) is the data input, # while encoded source sequence is accessed to as an unbounded memory. @@ -156,13 +159,7 @@ The decoder uses :code:`recurrent_group` to define the recurrent neural network. # All sequence inputs should have the same length. decoder = recurrent_group(name=decoder_group_name, step=gru_decoder_with_attention, - input=[ - StaticInput(input=encoded_vector, - is_seq=True), - StaticInput(input=encoded_proj, - is_seq=True), - trg_embedding - ]) + input=group_inputs) The implementation of the step function is listed as below. First, it defines the **memory** of the decoder network. Then it defines attention, gated recurrent unit step function, and the output function: @@ -217,10 +214,8 @@ The code is listed below: .. code-block:: python - gen_inputs = [StaticInput(input=encoded_vector, - is_seq=True), - StaticInput(input=encoded_proj, - is_seq=True), ] + group_inputs=[StaticInput(input=encoded_vector,is_seq=True), + StaticInput(input=encoded_proj,is_seq=True)] # In generation, decoder predicts a next target word based on # the encoded source sequence and the last generated target word. # The encoded source sequence (encoder's output) must be specified by @@ -231,10 +226,10 @@ The code is listed below: size=target_dict_dim, embedding_name='_target_language_embedding', embedding_size=word_vector_dim) - gen_inputs.append(trg_embedding) + group_inputs.append(trg_embedding) beam_gen = beam_search(name=decoder_group_name, step=gru_decoder_with_attention, - input=gen_inputs, + input=group_inputs, id_input=data_layer(name="sent_id", size=1), dict_file=trg_dict_path, diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index f902d1c995bc5..c1d7a7ce81530 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -169,6 +169,12 @@ dotmul_projection :members: dotmul_projection :noindex: +dotmul_operator +--------------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: dotmul_operator + :noindex: + full_matrix_projection ---------------------- .. automodule:: paddle.trainer_config_helpers.layers diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 79b85214228f3..a57e9065c6f98 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2464,11 +2464,11 @@ def __init__( if size != 0: self.set_layer_size(size) else: - size = operator.calc_output_size(operator_conf.input_sizes) - if size != 0: - config_assert(size == self.config.size, + sz = operator.calc_output_size(operator_conf.input_sizes) + if sz != 0: + config_assert(sz == self.config.size, "different inputs have different size: %s vs. %s" % - (size, self.config.size)) + (sz, self.config.size)) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) input = self.inputs[input_index] diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 9963b38134046..8b7cabf2fad50 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -286,7 +286,6 @@ def full_matrix_projection(input, size=0, param_attr=None): size=size, **param_attr.attr) proj.origin = input - proj.origin.projection = "matrix" return proj @@ -333,7 +332,6 @@ def table_projection(input, size=0, param_attr=None): size=size, **param_attr.attr) proj.origin = input - proj.origin.projection = "table" return proj @@ -377,17 +375,15 @@ def identity_projection(input, offset=None): if offset is None: proj = IdentityProjection(input_layer_name=input.name) proj.origin = input - proj.origin.projection = 'identity' else: proj = IdentityOffsetProjection(input_layer_name=input.name, offset=offset) proj.origin = input - proj.origin.projection = 'identity_offset' return proj @wrap_param_attr_default() -def dotmul_projection(input, param_attr=None, scale=1): +def dotmul_projection(input, param_attr=None): """ DotMulProjection with a layer as input. It performs element-wise multiplication with weight. @@ -407,30 +403,35 @@ def dotmul_projection(input, param_attr=None, scale=1): :type input: LayerOutput :param param_attr: Parameter config, None if use default. :type param_attr: ParameterAttribute - :param scale: config scalar, default value is one. - :type scale: float :return: A DotMulProjection Object. :rtype: DotMulProjection """ proj = DotMulProjection(input_layer_name=input.name, - size=input.size, - **param_attr.attr) - proj.origin = input + size=input.size, + **param_attr.attr) + proj.origin = input return proj def dotmul_operator(x, y, scale=1): """ DotMulOperator takes two inputs and performs element-wise multiplication: + .. math:: - out.row[i] += scale * (in1.row[i] .* in2.row[i]) + out.row[i] += scale * (x.row[i] .* y.row[i]) + where :math:`.*` means element-wise multiplication, and scale is a config scalar, its default value is one. + The example usage is: + .. code-block:: python - op = dotmul_operator(x, y, - scale=1) - :param input: Input layer - :type input: LayerOutput + + op = dotmul_operator(x=layer1, y=layer2, scale=0.5) + + :param x: Input layer1 + :type x: LayerOutput + :param y: Input layer2 + :type y: LayerOutput :param scale: config scalar, default value is one. :type scale: float :return: A DotMulOperator Object. @@ -487,7 +488,6 @@ def context_projection(input, context_len, context_start=None, trainable_padding=trainable, **extra_dict) proj.origin = input - proj.origin.projection = 'context' return proj @@ -2728,7 +2728,6 @@ def conv_operator(img, filter, filter_size, num_filters, stride_y=stride_y, groups=groups)) op.origin = [img, filter] - op.origin.operator = "conv_op" return op From 7eb29f264b07e7739dafec03a94a47a88faf7588 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 23 Sep 2016 12:53:19 +0800 Subject: [PATCH 112/324] Try to fix MultinomialSampler (#102) * Also refine unittest to multiple iteration to prevent luckily random number. --- paddle/gserver/layers/MultinomialSampler.cpp | 5 +- .../gserver/tests/test_MultinomialSampler.cpp | 62 ++++++++++--------- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/paddle/gserver/layers/MultinomialSampler.cpp b/paddle/gserver/layers/MultinomialSampler.cpp index 710772c0cf476..518dc0c60cbdc 100644 --- a/paddle/gserver/layers/MultinomialSampler.cpp +++ b/paddle/gserver/layers/MultinomialSampler.cpp @@ -19,7 +19,7 @@ namespace paddle { MultinomialSampler::MultinomialSampler(const real* prob, int size) : rand_(0.0, size) { - intervals_.reserve(size + 1); + intervals_.resize(size + 1); double sum = 0; for (int i = 0; i < size; ++i) { sum += prob[i]; @@ -50,12 +50,13 @@ MultinomialSampler::MultinomialSampler(const real* prob, int size) int bigPos = nextBigPos(0); auto fillIntervals = [&]() { - while (bigPos < size && smallPos < size) { + while (bigPos < size) { while (intervals_[bigPos].thresh > 1 && smallPos < size) { intervals_[smallPos].otherId = bigPos; intervals_[bigPos].thresh -= 1 - intervals_[smallPos].thresh; smallPos = nextSmallPos(smallPos + 1); } + if (smallPos >= size) break; bigPos = nextBigPos(bigPos + 1); // If intervals_[bigPos].thresh < 1, it becomes a small interval } diff --git a/paddle/gserver/tests/test_MultinomialSampler.cpp b/paddle/gserver/tests/test_MultinomialSampler.cpp index 39a90958331f6..73b4d0b8b7110 100644 --- a/paddle/gserver/tests/test_MultinomialSampler.cpp +++ b/paddle/gserver/tests/test_MultinomialSampler.cpp @@ -41,39 +41,42 @@ class MultinomialSamplerTester : public MultinomialSampler { TEST(MultinomialSampler, gen) { int numGrids = 1024 * 1024; int size = 1024 * 4; - default_random_engine reng; - uniform_int_distribution rand(1, numGrids / size * 1.8); - vector prob; - int sum = 0; - for (int i = 0; i < size; ++i) { - prob.push_back(rand(reng)); - sum += prob.back(); - } - CHECK_LE(sum, numGrids); - prob.back() += numGrids - sum; - vector counts(size); - MultinomialSamplerTester sampler(&prob[0], size); - counts.assign(size, 0); - { - double s = (double)size / (double)numGrids; - REGISTER_TIMER("MultinomialSampler"); - for (double i = 0; i < numGrids; ++i) { - int ret = sampler.testGen([i, s]() { return s * i; }); - if (ret < 0 || ret >= size) { - EXPECT_GE(ret, 0); - EXPECT_LT(ret, size); - break; + for (size_t iter=0; iter < 256; ++iter) { + uniform_int_distribution rand(1, numGrids / size * 1.8); + vector prob; + int sum = 0; + for (int i = 0; i < size; ++i) { + prob.push_back(rand(reng)); + sum += prob.back(); + } + + CHECK_LE(sum, numGrids); + prob.back() += numGrids - sum; + + vector counts(size); + MultinomialSamplerTester sampler(&prob[0], size); + counts.assign(size, 0); + { + double s = (double)size / (double)numGrids; + REGISTER_TIMER("MultinomialSampler"); + for (double i = 0; i < numGrids; ++i) { + int ret = sampler.testGen([i, s]() { return s * i; }); + if (ret < 0 || ret >= size) { + EXPECT_GE(ret, 0); + EXPECT_LT(ret, size); + break; + } + ++counts[ret]; } - ++counts[ret]; } - } - for (int i = 0; i < size; ++i) { - if (prob[i] != counts[i]) { - EXPECT_EQ(prob[i], counts[i]); - LOG(INFO) << "i=" << i; - break; + for (int i = 0; i < size; ++i) { + if (prob[i] != counts[i]) { + EXPECT_EQ(prob[i], counts[i]); + LOG(INFO) << iter; + break; + } } } } @@ -135,6 +138,7 @@ void benchmarkRandom() { LOG(INFO) << "sum1=" << sum1; } + int main(int argc, char** argv) { initMain(argc, argv); testing::InitGoogleTest(&argc, argv); From 95da095d108dadb6755b378d3970021cf35c4bcd Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Fri, 23 Sep 2016 15:58:22 +0800 Subject: [PATCH 113/324] fix cudnn conv bug which occurs in image classfication demo in GTX GPU --- paddle/gserver/layers/CudnnConvLayer.cpp | 8 ++++++++ paddle/gserver/layers/CudnnConvLayer.h | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/paddle/gserver/layers/CudnnConvLayer.cpp b/paddle/gserver/layers/CudnnConvLayer.cpp index a74e6ba38dfc6..a395a7223e394 100644 --- a/paddle/gserver/layers/CudnnConvLayer.cpp +++ b/paddle/gserver/layers/CudnnConvLayer.cpp @@ -85,6 +85,7 @@ bool CudnnConvLayer::init(const LayerMap &layerMap, biasOffset_ = numFilters_ / groups_[0]; } + batchNum_ = 0; isSelectAlgo_ = false; return true; } @@ -132,6 +133,9 @@ void CudnnConvLayer::reshape(int batchSize) { getOutput().setFrameHeight(outputH_); getOutput().setFrameWidth(outputW_); + isSelectAlgo_ = (batchSize == batchNum_); + batchNum_ = batchSize; + size_t maxWorkSpace = 0; for (size_t i = 0; i < inputLayers_.size(); i++) { CHECK_EQ(inputLayers_[i]->getOutput().value->getWidth(), @@ -160,6 +164,10 @@ void CudnnConvLayer::reshape(int batchSize) { maxWorkSpace = std::max(fwdLimitBytes_[i], bwdDataLimitBytes_[i]); maxWorkSpace = std::max(maxWorkSpace, bwdFilterLimitBytes_[i]); + + VLOG(3) << getName() << " Fwd / BwdData / BwdFilter algo: " << fwdAlgo_[i] + << " / " << bwdDataAlgo_[i] + << " / " << bwdFilterAlgo_[i]; } } diff --git a/paddle/gserver/layers/CudnnConvLayer.h b/paddle/gserver/layers/CudnnConvLayer.h index 2c72ba885ed10..a6dadba10daa4 100644 --- a/paddle/gserver/layers/CudnnConvLayer.h +++ b/paddle/gserver/layers/CudnnConvLayer.h @@ -87,6 +87,10 @@ class CudnnConvLayer : public ConvBaseLayer { /// Is or not select conv algorihtm. bool isSelectAlgo_; + /// batchNum is used to record batch size. If the batch size is changed, + /// the selection algorithm will be called. + int batchNum_; + public: explicit CudnnConvLayer(const LayerConfig& config) : ConvBaseLayer(config) {} From c1c07bbef65deb2ca91554c3110b15b35a71dd5e Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Fri, 23 Sep 2016 17:22:46 +0800 Subject: [PATCH 114/324] Update CudnnConvLayer.cpp --- paddle/gserver/layers/CudnnConvLayer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paddle/gserver/layers/CudnnConvLayer.cpp b/paddle/gserver/layers/CudnnConvLayer.cpp index a395a7223e394..0f932f960f6ba 100644 --- a/paddle/gserver/layers/CudnnConvLayer.cpp +++ b/paddle/gserver/layers/CudnnConvLayer.cpp @@ -133,6 +133,8 @@ void CudnnConvLayer::reshape(int batchSize) { getOutput().setFrameHeight(outputH_); getOutput().setFrameWidth(outputW_); + // if the batchSize remains the same, set isSelectAlgo_ true. + // Otherwise, set isSelectAlgo_ false and select algo again. isSelectAlgo_ = (batchSize == batchNum_); batchNum_ = batchSize; From 32b55573292b40452f99616db40f2f8c8d7809da Mon Sep 17 00:00:00 2001 From: liaogang Date: Fri, 23 Sep 2016 21:43:53 +0800 Subject: [PATCH 115/324] Add thread Barrier unit test --- paddle/utils/tests/CMakeLists.txt | 1 + paddle/utils/tests/test_ThreadBarrier.cpp | 68 +++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 paddle/utils/tests/test_ThreadBarrier.cpp diff --git a/paddle/utils/tests/CMakeLists.txt b/paddle/utils/tests/CMakeLists.txt index 5b31cd393dd1f..51f1889392845 100644 --- a/paddle/utils/tests/CMakeLists.txt +++ b/paddle/utils/tests/CMakeLists.txt @@ -3,6 +3,7 @@ add_simple_unittest(test_Logging) add_simple_unittest(test_Thread) add_simple_unittest(test_StringUtils) add_simple_unittest(test_CustomStackTrace) +add_simple_unittest(test_ThreadBarrier) add_executable( test_CustomStackTracePrint diff --git a/paddle/utils/tests/test_ThreadBarrier.cpp b/paddle/utils/tests/test_ThreadBarrier.cpp new file mode 100644 index 0000000000000..241cdda7bd1c9 --- /dev/null +++ b/paddle/utils/tests/test_ThreadBarrier.cpp @@ -0,0 +1,68 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include +#include "paddle/utils/Logging.h" +#include "paddle/utils/CommandLineParser.h" +#include "paddle/utils/Util.h" +#include "paddle/utils/Locks.h" + +P_DEFINE_int32(test_thread_num, 100, "testing thread number"); + +void testNormalImpl(size_t thread_num, + const std::function&, + paddle::ThreadBarrier&)>& callback) { + std::mutex mutex; + std::set tids; + paddle::ThreadBarrier barrier(thread_num); + + std::vector threads; + threads.reserve(thread_num); + for (int32_t i = 0; i < thread_num; ++i) { + threads.emplace_back([&thread_num, &mutex, + &tids, &barrier, &callback]{ + callback(thread_num, mutex, tids, barrier); + }); + } + + for (auto& thread : threads) { + thread.join(); + } +} + +TEST(ThreadBarrier, normalTest) { + for (auto &thread_num : {10, 30, 50 , 100 , 300, 1000}) { + testNormalImpl(thread_num, + [](size_t thread_num, std::mutex& mutex, + std::set& tids, + paddle::ThreadBarrier& barrier){ + { + std::lock_guard guard(mutex); + tids.insert(std::this_thread::get_id()); + } + barrier.wait(); + // Check whether all threads reach this point or not + CHECK_EQ(tids.size(), thread_num); + }); + } +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + paddle::initMain(argc, argv); + return RUN_ALL_TESTS(); +} From 77ed98d1a86f408db1f4043bd9d6fc14c48295e9 Mon Sep 17 00:00:00 2001 From: Zrachel Date: Sat, 24 Sep 2016 00:29:52 +0800 Subject: [PATCH 116/324] fix bugs under kSgdSparseCpuTraining mode (#100) Local training with "sparse_update = True" parameter triggers kSgdSparseCpuTraining mode, fix bugs under it. --- paddle/trainer/ThreadParameterUpdater.cpp | 2 +- paddle/trainer/TrainerInternal.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/trainer/ThreadParameterUpdater.cpp b/paddle/trainer/ThreadParameterUpdater.cpp index 65d827787ee78..91f7f4d29df93 100644 --- a/paddle/trainer/ThreadParameterUpdater.cpp +++ b/paddle/trainer/ThreadParameterUpdater.cpp @@ -141,7 +141,7 @@ void SgdThreadUpdater::traverse(GetTraverseCallback getTraverseCallback) { } else if (hasCpuPara) { getGlobalSyncThreadPool()->exec(cpuTraverse); } else if (hasGpuPara) { - cpuTraverse(0, 0); + gpuTraverse(0, 0); } } diff --git a/paddle/trainer/TrainerInternal.cpp b/paddle/trainer/TrainerInternal.cpp index 76b6b9bc3ee38..6029a4b2c1d0a 100644 --- a/paddle/trainer/TrainerInternal.cpp +++ b/paddle/trainer/TrainerInternal.cpp @@ -101,6 +101,7 @@ void TrainerInternal::trainOneBatch(int64_t batchId, // it //! to ParameterHook. auto& grad = para->getBuf(PARAMETER_GRADIENT); + SetDevice device(para->getDeviceId()); paraStats[para->getID()].avgAbsGrad = grad->getAbsSum() / para->getSize(); paraStats[para->getID()].maxAbsGrad = grad->getAbsMax(); } From 332194c88132cdef60f6f2a14bb96d5f693ab279 Mon Sep 17 00:00:00 2001 From: Haichao-Zhang Date: Fri, 23 Sep 2016 18:15:37 -0700 Subject: [PATCH 117/324] add type compatible check for ParamAttr (#113) * add type compatible check for ParamAttr --- python/paddle/trainer_config_helpers/attrs.py | 52 ++++++++++++++++--- .../tests/layers_test_config.py | 14 ++++- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/python/paddle/trainer_config_helpers/attrs.py b/python/paddle/trainer_config_helpers/attrs.py index 7b0a398d19172..2b5b451edded5 100644 --- a/python/paddle/trainer_config_helpers/attrs.py +++ b/python/paddle/trainer_config_helpers/attrs.py @@ -17,6 +17,42 @@ 'ExtraLayerAttribute'] +def convert_and_compare(x, Type): + """ + Convert x to be the same type as Type and then convert back to + check whether there is a loss of information + :param x: object to be checked + :param Type: target type to check x over + + """ + return type(x)(Type(x))==x + +def is_compatible_with(x, Type): + """ + Check if x has a type compatible with Type + :param x: object to be checked + :param Type: target type to check x over + + """ + if type(x) == Type: + return True + try: + if float == Type or int == Type: + # avoid those types that can be converted to float/int but not very + # meaningful and could potentially lead to error + # i.e., str and bool typed value should not be used for initializing float/int variable + if not isinstance(x, str) and not isinstance(x, bool): + return convert_and_compare(x, Type) + elif bool == Type: + # should not use string type to initialize bool variable + if not isinstance(x, str): + return convert_and_compare(x, Type) + else: + return False + except: + return False + + class ParameterAttribute(object): """ Parameter Attributes object. To fine-tuning network training process, user @@ -65,14 +101,18 @@ def __init__(self, name=None, is_static=False, initial_std=None, elif initial_std is None and initial_mean is None and initial_max \ is None and initial_min is None: self.attr = {'initial_smart': True} - elif isinstance(initial_std, float) or isinstance(initial_mean, float): + elif is_compatible_with(initial_std, float) or \ + is_compatible_with(initial_mean, float): self.attr = dict() if initial_std is not None: self.attr['initial_std'] = initial_std if initial_mean is not None: self.attr['initial_mean'] = initial_mean self.attr['initial_strategy'] = 0 # Gauss Random - elif isinstance(initial_max, float) and isinstance(initial_min, float): + elif is_compatible_with(initial_max, float) and \ + is_compatible_with(initial_min, float): + initial_max = initial_max + initial_min = initial_min assert initial_min < initial_max initial_mean = (initial_max + initial_min) / 2 initial_std = initial_mean - initial_min @@ -83,16 +123,16 @@ def __init__(self, name=None, is_static=False, initial_std=None, else: raise RuntimeError("Unexpected branch.") - if not is_static and isinstance(l1_rate, float): + if not is_static and is_compatible_with(l1_rate, float): self.attr['decay_rate_l1'] = l1_rate - if not is_static and isinstance(l2_rate, float): + if not is_static and is_compatible_with(l2_rate, float): self.attr['decay_rate'] = l2_rate - if not is_static and isinstance(learning_rate, float): + if not is_static and is_compatible_with(learning_rate, float): self.attr['learning_rate'] = learning_rate - if not is_static and isinstance(momentum, float): + if not is_static and is_compatible_with(momentum, float): self.attr['momentum'] = momentum if name is not None: diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py index 27b22ecb701c5..b9eaf2fce7572 100644 --- a/python/paddle/trainer_config_helpers/tests/layers_test_config.py +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -39,10 +39,20 @@ outputs(classification_cost(out, data_layer(name="label", size=num_classes))) dotmul = mixed_layer(input=[dotmul_operator(x=x1, y=y1), - dotmul_projection(input=y1)]) + dotmul_projection(input=y1)]) + +proj_with_attr_init = mixed_layer(input=full_matrix_projection(input=y1, + param_attr=ParamAttr(learning_rate = 0, + initial_mean = 0, + initial_std = 0)), + bias_attr = ParamAttr(initial_mean=0, initial_std=0, learning_rate=0), + act = LinearActivation(), + size = 5, + name='proj_with_attr_init') + # for ctc -tmp = fc_layer(input=[x1, dotmul], +tmp = fc_layer(input=[x1, dotmul, proj_with_attr_init], size=num_classes + 1, act=SoftmaxActivation()) ctc = ctc_layer(input=tmp, From 2289c141c2722c10d3dc4146d58ba2384fb5c770 Mon Sep 17 00:00:00 2001 From: liuyuan Date: Mon, 26 Sep 2016 12:42:17 +0800 Subject: [PATCH 118/324] Refine comment for CRF related headers. (#117) --- paddle/gserver/layers/CRFLayer.h | 2 +- paddle/gserver/layers/LinearChainCRF.h | 48 +++++++++++++------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/paddle/gserver/layers/CRFLayer.h b/paddle/gserver/layers/CRFLayer.h index c6ba8e7c965a3..58902a0d3b7e4 100644 --- a/paddle/gserver/layers/CRFLayer.h +++ b/paddle/gserver/layers/CRFLayer.h @@ -25,7 +25,7 @@ namespace paddle { /** * A layer for calculating the cost of sequential conditional random field * model. - * See LinearChainCRF.h for the detail of the CRF formulation. + * See class LinearChainCRF for the detail of the CRF formulation. */ class CRFLayer : public Layer { public: diff --git a/paddle/gserver/layers/LinearChainCRF.h b/paddle/gserver/layers/LinearChainCRF.h index 3bde1aa415ce9..c33c83b25987e 100644 --- a/paddle/gserver/layers/LinearChainCRF.h +++ b/paddle/gserver/layers/LinearChainCRF.h @@ -21,39 +21,39 @@ namespace paddle { class LinearChainCRF { public: - /* - The size of para and grad must be (numClasses + 2) * numClasses. - The first numClasses values of para are for starting weights (a). - The next numClasses values of para are for ending weights (b), - The remaning values are for transition weights (w). - - The probability of a state sequence s of length L is defined as: - P(s) = (1/Z) exp(a_{s_1} + b_{s_L} - + \sum_{l=1}^L x_{s_l} - + \sum_{l=2}^L w_{s_{l-1},s_l}) - where Z is a normalization value so that the sum of P(s) over all possible - sequences is 1, and x is the input feature to the CRF. + /** + * The size of para and grad must be \f$(numClasses + 2) * numClasses\f$. + * The first numClasses values of para are for starting weights (\f$a\f$). + * The next numClasses values of para are for ending weights (\f$b\f$), + * The remaning values are for transition weights (\f$w\f$). + * + * The probability of a state sequence s of length \f$L\f$ is defined as: + * \f$P(s) = (1/Z) exp(a_{s_1} + b_{s_L} + * + \sum_{l=1}^L x_{s_l} + * + \sum_{l=2}^L w_{s_{l-1},s_l})\f$ + * where \f$Z\f$ is a normalization value so that the sum of \f$P(s)\f$ over all possible + * sequences is \f$1\f$, and \f$x\f$ is the input feature to the CRF. */ LinearChainCRF(int numClasses, real* para, real* grad); - /* - Calculate the negative log likelihood of s given x. - The size of x must be length * numClasses. Each consecutive numClasses - values are the features for one time step. + /** + * Calculate the negative log likelihood of s given x. + * The size of x must be length * numClasses. Each consecutive numClasses + * values are the features for one time step. */ real forward(real* x, int* s, int length); - /* - Calculate the gradient with respect to x, a, b, and w. - The gradient of x will be stored in dx. - backward() can only be called after a corresponding call to forward() with - the same x, s and length. - NOTE: The gradient is added to dx and grad (provided at constructor). + /** + * Calculate the gradient with respect to x, a, b, and w. + * The gradient of x will be stored in dx. + * backward() can only be called after a corresponding call to forward() with + * the same x, s and length. + * @note The gradient is added to dx and grad (provided at constructor). */ void backward(real* x, real* dx, int* s, int length); - /* - Find the most probable sequence given x. The result will be stored in s. + /** + * Find the most probable sequence given x. The result will be stored in s. */ void decode(real* x, int* s, int length); From 1d4bc47805252b7b8859cc6d65bbad508aa95028 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 26 Sep 2016 17:27:43 +0800 Subject: [PATCH 119/324] support gettid() on MAC OS X --- paddle/cuda/src/hl_cuda_device.cc | 13 ++++++++++++- paddle/utils/Thread.h | 13 ++++++++++++- paddle/utils/ThreadLocal.h | 10 +++++++++- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/paddle/cuda/src/hl_cuda_device.cc b/paddle/cuda/src/hl_cuda_device.cc index f07538d6ba713..acd8e2fe6afb4 100644 --- a/paddle/cuda/src/hl_cuda_device.cc +++ b/paddle/cuda/src/hl_cuda_device.cc @@ -209,7 +209,18 @@ __thread cudaStream_t default_stream = 0; __thread bool g_sync_flag = true; bool hl_start_flag = false; -#define gettid() syscall(SYS_gettid) +inline pid_t gettid() { +#if defined(__APPLE__) || defined(__OSX__) + pid_t tid = syscall(SYS_thread_selfid); +#else + #ifndef __NR_gettid + #define __NR_gettid 224 + #endif + pid_t tid = syscall(__NR_gettid); +#endif + CHECK_NE(tid, -1); + return tid; +} void hl_init(int device) { CHECK(hl_start_flag) diff --git a/paddle/utils/Thread.h b/paddle/utils/Thread.h index 3e1d95ab1fcde..f1352e75d73a0 100644 --- a/paddle/utils/Thread.h +++ b/paddle/utils/Thread.h @@ -18,7 +18,18 @@ limitations under the License. */ #include #include -inline pid_t gettid() { return syscall(SYS_gettid); } +inline pid_t gettid() { +#if defined(__APPLE__) || defined(__OSX__) + pid_t tid = syscall(SYS_thread_selfid); +#else + #ifndef __NR_gettid + #define __NR_gettid 224 + #endif + pid_t tid = syscall(__NR_gettid); +#endif + CHECK_NE(tid, -1); + return tid; +} #include "Queue.h" #include "ThreadLocal.h" diff --git a/paddle/utils/ThreadLocal.h b/paddle/utils/ThreadLocal.h index e782868f69a5d..686a1a99a4aa0 100644 --- a/paddle/utils/ThreadLocal.h +++ b/paddle/utils/ThreadLocal.h @@ -156,7 +156,15 @@ class ThreadLocalD { static void dataDestructor(void* p) { delete (T*)p; } void updateMap(T* p) { - pid_t tid = syscall(SYS_gettid); +#if defined(__APPLE__) || defined(__OSX__) + pid_t tid = syscall(SYS_thread_selfid); +#else + #ifndef __NR_gettid + #define __NR_gettid 224 + #endif + pid_t tid = syscall(__NR_gettid); +#endif + CHECK_NE(tid, -1); std::lock_guard guard(mutex_); auto ret = threadMap_.insert(std::make_pair(tid, p)); if (!ret.second) { From ffc341675dd8c8057a105fb8ebfe5a91b43ce6ca Mon Sep 17 00:00:00 2001 From: luotao1 Date: Tue, 27 Sep 2016 13:20:49 +0800 Subject: [PATCH 120/324] Add parallel_nn api and unittest (#110) * Add `device` parameter to ExtraAttr in trainer_config_helpers. * add unittest for it. --- .../tests/sample_trainer_config_parallel.conf | 151 +++++------------- python/paddle/trainer_config_helpers/attrs.py | 13 +- .../paddle/trainer_config_helpers/layers.py | 14 +- 3 files changed, 65 insertions(+), 113 deletions(-) diff --git a/paddle/trainer/tests/sample_trainer_config_parallel.conf b/paddle/trainer/tests/sample_trainer_config_parallel.conf index 3563fede1c182..e35a1f26dad2f 100644 --- a/paddle/trainer/tests/sample_trainer_config_parallel.conf +++ b/paddle/trainer/tests/sample_trainer_config_parallel.conf @@ -13,137 +13,74 @@ # See the License for the specific language governing permissions and # limitations under the License. -#Todo(luotao02) This config is only used for unitest. It is out of date now, and will be updated later. +from paddle.trainer_config_helpers import * -TrainData( - SimpleData( - files = "trainer/tests/sample_filelist.txt", - feat_dim = 3, - context_len = 0, - buffer_capacity = 1000000, - ) -) +TrainData(SimpleData( + files = "trainer/tests/sample_filelist.txt", + feat_dim = 3, + context_len = 0, + buffer_capacity = 1000000)) -TestData( - SimpleData( - files = "trainer/tests/sample_filelist.txt", - feat_dim = 3, - context_len = 0, - buffer_capacity = 1000000, - ) -) +TestData(SimpleData( + files = "trainer/tests/sample_filelist.txt", + feat_dim = 3, + context_len = 0, + buffer_capacity = 1000000)) -Settings( - algorithm = "sgd", - num_batches_per_send_parameter = 1, - num_batches_per_get_parameter = 1, - batch_size = 100, - learning_rate = 0.001, - learning_rate_decay_a = 1e-5, - learning_rate_decay_b = 0.5, -) +settings(batch_size = 100) -default_initial_std(0.2) # Output layer, label layer, cost layer, preferably set to the same environment. output_device = 0 -model_type("nn") - # Input Layer does not need to specify the device number. -Layer( - name = "input", - type = "data", - size = 3, -) +data = data_layer(name='input', size=3) # Calculate in the CPU. -Layer( - name = "layer1_1", - type = "fc", - size = 5, - active_type = "sigmoid", - device = -1, - inputs = "input", -) +fc1 = fc_layer(input=data, size=5, + bias_attr=True, + layer_attr=ExtraAttr(device=-1), + act=SigmoidActivation()) # Calculate in the GPU 0. -Layer( - name = "layer2_1", - type = "fc", - size = 10, - active_type = "sigmoid", - device = 0, - inputs = "layer1_1", -) +fc2 = fc_layer(input=fc1, size=10, + bias_attr=True, + layer_attr=ExtraAttr(device=0), + act=SigmoidActivation()) # Calculate in the GPU 1. -Layer( - name = "layer2_2", - type = "fc", - size = 10, - active_type = "sigmoid", - device = 1, - inputs = "layer1_1", -) +fc3 = fc_layer(input=fc1, size=10, + bias_attr=True, + layer_attr=ExtraAttr(device=1), + act=SigmoidActivation()) # Calculate in the GPU 0. -Layer( - name = "layer3_1", - type = "fc", - size = 10, - device = 0, - active_type = "sigmoid", - inputs = ["layer2_1", "layer2_2"], -) +fc4 = fc_layer(input=[fc2,fc3], size=10, + bias_attr=True, + layer_attr=ExtraAttr(device=0), + act=SigmoidActivation()) # Calculate in the GPU 1. -Layer( - name = "layer3_2", - type = "fc", - size = 10, - device = 1, - active_type = "sigmoid", - inputs = ["layer2_1", "layer2_2"], -) - +fc5 = fc_layer(input=[fc2,fc3], size=10, + bias_attr=True, + layer_attr=ExtraAttr(device=1), + act=SigmoidActivation()) -Layer( - name = "output", - type = "fc", - size = 10, - device = output_device, - active_type = "sigmoid", - inputs = ["layer3_1", "layer3_2"], -) +output = fc_layer(input=[fc4,fc5], size=10, + bias_attr=True, + layer_attr=ExtraAttr(device=output_device), + act=SoftmaxActivation()) if get_config_arg('with_cost', bool, True): # This is for training the neural network. # We need to have another data layer for label # and a layer for calculating cost - Layer( - name = "label", - type = "data", - device = output_device, - size = 1, - ) - - Layer( - name = "cost", - type = "multi-class-cross-entropy", - device = output_device, - inputs = ["output", "label"], - ) - - Evaluator( - name = "error", - type = "classification_error", - inputs = ["output", "label"]) - - Inputs("input", "label") - Outputs("cost") - + lbl = data_layer(name='label', size=1, + layer_attr=ExtraAttr(device=output_device)) + + outputs(classification_cost(input=output, + label=lbl, + layer_attr=ExtraAttr(device=output_device))) else: # This is for prediction where we don't have label # and don't need to calculate cost - Inputs("input") - Outputs("output") + outputs(output) diff --git a/python/paddle/trainer_config_helpers/attrs.py b/python/paddle/trainer_config_helpers/attrs.py index 2b5b451edded5..d263441247332 100644 --- a/python/paddle/trainer_config_helpers/attrs.py +++ b/python/paddle/trainer_config_helpers/attrs.py @@ -174,12 +174,16 @@ class ExtraLayerAttribute(object): The dropout rate is the zero rate of this mask. The details of what dropout is please refer to `here `_ + JMLRdropout.pdf>`_. :type drop_rate: float - + :param device: device ID of layer. device=-1, use CPU. device>0, use GPU. + The details allocation in parallel_nn please refer to `here + `_. + :type device: int """ - def __init__(self, error_clipping_threshold=None, drop_rate=None): + def __init__(self, error_clipping_threshold=None, drop_rate=None, device=None): self.attr = dict() if isinstance(error_clipping_threshold, float): assert error_clipping_threshold > 0 @@ -189,6 +193,9 @@ def __init__(self, error_clipping_threshold=None, drop_rate=None): assert drop_rate > 0 self.attr["drop_rate"] = drop_rate + if isinstance(device, int): + self.attr["device"] = device + def check(self, layer_name): for key in self.attr: if not hasattr(self, 'can_%s' % key) or \ diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 8b7cabf2fad50..76b0db546b18b 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -201,6 +201,7 @@ def __str__(self): ERROR_CLIPPING = 'error_clipping_threshold' DROPOUT = 'drop_rate' +DEVICE = 'device' def check_input(input): @@ -223,10 +224,12 @@ def check_input(input): def layer_support(*attrs): + attrs_list = list(attrs) + attrs_list.append(DEVICE) def decorator(method): @functools.wraps(method) def wrapper(*args, **kwargs): - for attr in attrs: + for attr in attrs_list: for each in args: if isinstance(each, ExtraLayerAttribute): setattr(each, '_'.join(['can', attr]), True) @@ -2625,9 +2628,11 @@ def regression_cost(input, label, cost='square_error', name=None): @wrap_name_default("cost") +@layer_support() def classification_cost(input, label, name=None, cost="multi-class-cross-entropy", - evaluator=classification_error_evaluator): + evaluator=classification_error_evaluator, + layer_attr=None): """ classification cost Layer. @@ -2640,13 +2645,16 @@ def classification_cost(input, label, name=None, :param cost: cost method. :type cost: basestring :param evaluator: Evaluator method. + :param layer_attr: layer's extra attribute. + :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput """ assert input.layer_type != LayerType.DATA assert isinstance(input.activation, SoftmaxActivation) assert label.layer_type == LayerType.DATA - Layer(name=name, type=cost, inputs=[Input(input.name), Input(label.name)]) + Layer(name=name, type=cost, inputs=[Input(input.name), Input(label.name)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) def __add_evaluator__(e): assert callable(e) From b15a4783cbaa48ed5d7f73639c9afe0568d34972 Mon Sep 17 00:00:00 2001 From: emailweixu Date: Mon, 26 Sep 2016 22:55:49 -0700 Subject: [PATCH 121/324] =?UTF-8?q?Correctly=20handling=20multiple=20input?= =?UTF-8?q?s=20and=20integer=20inputs=20for=20recurrent=5Fg=E2=80=A6=20(#1?= =?UTF-8?q?14)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Correctly handling multiple inputs and integer inputs for recurrent_group * Fix ScatterAgentLayer for generation * Revert sequence_(nest)_rnn.conf --- paddle/cuda/src/hl_cuda_cublas.cc | 4 +- .../RecurrentGradientMachine.cpp | 41 ++++++---- paddle/gserver/layers/AgentLayer.cpp | 26 +++---- paddle/gserver/tests/CMakeLists.txt | 1 - .../tests/sequence_nest_rnn_multi_input.conf | 77 +++++++++++++++++++ .../tests/sequence_rnn_multi_input.conf | 58 ++++++++++++++ .../tests/test_RecurrentGradientMachine.cpp | 29 +++++-- paddle/parameter/Argument.cpp | 13 +++- paddle/parameter/Argument.h | 4 +- .../paddle/trainer_config_helpers/layers.py | 4 +- 10 files changed, 210 insertions(+), 47 deletions(-) create mode 100644 paddle/gserver/tests/sequence_nest_rnn_multi_input.conf create mode 100644 paddle/gserver/tests/sequence_rnn_multi_input.conf diff --git a/paddle/cuda/src/hl_cuda_cublas.cc b/paddle/cuda/src/hl_cuda_cublas.cc index 445279fa01034..dc109487ded20 100644 --- a/paddle/cuda/src/hl_cuda_cublas.cc +++ b/paddle/cuda/src/hl_cuda_cublas.cc @@ -217,7 +217,7 @@ void hl_matrix_mul(real *A_d, hl_trans_op_t transa, } else { LOG(FATAL) << "parameter transa error!"; } - CHECK_EQ(stat, CUBLAS_STATUS_SUCCESS); + CHECK_EQ(stat, CUBLAS_STATUS_SUCCESS) << hl_cublas_get_error_string(stat); CHECK_SYNC("hl_matrix_mul failed"); } @@ -266,7 +266,7 @@ void hl_matrix_mul_vector(real *A_d, hl_trans_op_t trans, LOG(FATAL) << "parameter transa error!"; } - CHECK_EQ(stat, CUBLAS_STATUS_SUCCESS); + CHECK_EQ(stat, CUBLAS_STATUS_SUCCESS) << hl_cublas_get_error_string(stat); CHECK_SYNC("hl_matrix_mul_vector"); } diff --git a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp index bee82faa5fca8..fc38bca3c403b 100644 --- a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp @@ -497,20 +497,21 @@ void RecurrentGradientMachine::forward(const std::vector& inArgs, int idSize = 0; // connect in_links for (size_t j = 0; j < inFrameLines_.size(); ++j) { + Info& info = info_[shareInlinkInfo ? 0 : j]; // idSize denotes the sum number of tokens in each length i - idSize = info_[j].idIndex[i + 1] - info_[j].idIndex[i]; + idSize = info.idIndex[i + 1] - info.idIndex[i]; InFrameLine inFrameLine = inFrameLines_[j]; auto scatterAgent = dynamic_cast(inFrameLine.agents[i].get()); scatterAgent->setRealLayerAndOutput(inFrameLine.inLayer, - inFrameLine.outArg, info_[j].allIds, - info_[j].idIndex[i], idSize); + inFrameLine.outArg, info.allIds, + info.idIndex[i], idSize); if (hasSubseq) { // size: the length of subsequence int size = - info_[j].seqStartPosIndex[i + 1] - info_[j].seqStartPosIndex[i]; - scatterAgent->setSequenceStartPositions(info_[j].sequenceStartPositions, - info_[j].seqStartPosIndex[i], + info.seqStartPosIndex[i + 1] - info.seqStartPosIndex[i]; + scatterAgent->setSequenceStartPositions(info.sequenceStartPositions, + info.seqStartPosIndex[i], size); } } @@ -744,16 +745,24 @@ void RecurrentGradientMachine::selectRowsOneTime(LayerPtr layer, const IVectorPtr& allIds, Argument* arg, PassType passType) { - const MatrixPtr& realV = layer->getOutputValue(); - int height = realV->getHeight(); - int width = realV->getWidth(); - Matrix::resizeOrCreate(arg->value, height, width, /* trans */ false, useGpu_); - arg->value->zeroMem(); - arg->value->selectRows(*realV, *allIds); - if (passType != PASS_TEST) { - Matrix::resizeOrCreate(arg->grad, height, width, /* trans */ false, - useGpu_); - arg->grad->zeroMem(); + Argument& src = layer->getOutput(); + if (src.value) { + const MatrixPtr& realV = src.value; + int height = realV->getHeight(); + int width = realV->getWidth(); + Matrix::resizeOrCreate( + arg->value, height, width, /* trans */ false, useGpu_); + arg->value->zeroMem(); + arg->value->selectRows(*realV, *allIds); + if (passType != PASS_TEST) { + Matrix::resizeOrCreate(arg->grad, height, width, /* trans */ false, + useGpu_); + arg->grad->zeroMem(); + } + } + if (src.ids) { + IVector::resizeOrCreate(arg->ids, src.ids->getSize(), useGpu_); + arg->ids->selectFrom(*src.ids, *allIds); } } diff --git a/paddle/gserver/layers/AgentLayer.cpp b/paddle/gserver/layers/AgentLayer.cpp index c1bef18ed38af..056e9568852ac 100644 --- a/paddle/gserver/layers/AgentLayer.cpp +++ b/paddle/gserver/layers/AgentLayer.cpp @@ -139,15 +139,16 @@ void ScatterAgentLayer::forward(PassType passType) { Layer::forward(passType); CHECK_EQ(realLayer_->getDeviceId(), this->getDeviceId()); - if (realLayer_->getOutput().ids) { // ids scatter - IVector::resizeOrCreate(output_.ids, ids_->getSize(), useGpu_); - output_.ids->selectFrom(*realLayer_->getOutput().ids, *ids_); - } else { // value scatter - int width = this->getSize(); - if (realOutArg_.value) { - output_.subArgFrom(realOutArg_, /* offset */ idIndex_ * width, idSize_, - width, useGpu_); - } else { // used in generation + int width = this->getSize(); + if (realOutArg_.value || realOutArg_.ids) { + output_.subArgFrom(realOutArg_, /* offset */ idIndex_, idSize_, + width, useGpu_); + } else { // used in generation + if (realLayer_->getOutput().ids) { + IVector::resizeOrCreate(output_.ids, ids_->getSize(), useGpu_); + output_.ids->selectFrom(*realLayer_->getOutput().ids, *ids_); + } + if (realLayer_->getOutput().value) { int height = ids_->getSize(); resetOutput(height, width); @@ -213,18 +214,17 @@ void SequenceGatherAgentLayer::forward(PassType passType) { void SequenceScatterAgentLayer::forward(PassType passType) { Layer::forward(passType); CHECK_EQ(realLayer_->getDeviceId(), this->getDeviceId()); - CHECK(!realLayer_->getOutput().ids) << "Not supported"; const Argument& input = realLayer_->getOutput(); - CHECK_EQ(input.value->getWidth(), this->getSize()); + CHECK_EQ(realLayer_->getSize(), this->getSize()); int width = this->getSize(); AsyncGpuBlock asyncGpuBlock; REGISTER_TIMER_INFO("SequenceAgentLayerForward", getName().c_str()); - if (realOutArg_.value) { + if (realOutArg_.value || realOutArg_.ids) { CHECK(realOutArg_.sequenceStartPositions); - output_.subArgFrom(realOutArg_, /* offset */ idIndex_ * width, idSize_, + output_.subArgFrom(realOutArg_, /* offset */ idIndex_, idSize_, width, useGpu_, /* trans */ false, /* seqFlag */ true, /* seqStart */ seqStartPosIndex_, /* seqSize */ numSequences_); diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index 129f10fac114d..ff2abf7697317 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -56,7 +56,6 @@ add_test(NAME test_RecurrentGradientMachine COMMAND .set_python_path.sh -d ${PROJ_ROOT}/python:${PROJ_ROOT}/paddle/gserver/tests ${CMAKE_CURRENT_BINARY_DIR}/test_RecurrentGradientMachine - --use_gpu=false WORKING_DIRECTORY ${PROJ_ROOT}/paddle) add_unittest_without_exec(test_NetworkCompare diff --git a/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf b/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf new file mode 100644 index 0000000000000..e01b3f8e7aa5c --- /dev/null +++ b/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf @@ -0,0 +1,77 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +######################## data source ################################ +define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', + test_list=None, + module='rnn_data_provider', + obj='process_subseq') + + +settings(batch_size=2, learning_rate=0.01) +######################## network configure ################################ +dict_dim = 10 +word_dim = 8 +hidden_dim = 8 +label_dim = 3 + +data = data_layer(name="word", size=dict_dim) + +emb = embedding_layer(input=data, size=word_dim) + +# This hierachical RNN is designed to be equivalent to the simple RNN in +# sequence_rnn.conf + +def outer_step(wid, x): + outer_mem = memory(name="outer_rnn_state", size=hidden_dim) + def inner_step(y, wid): + z = embedding_layer(input=wid, size=word_dim) + inner_mem = memory(name="inner_rnn_state", + size=hidden_dim, + boot_layer=outer_mem) + out = fc_layer(input=[y, z, inner_mem], + size=hidden_dim, + act=TanhActivation(), + bias_attr=True, + name="inner_rnn_state") + return out + + inner_rnn_output = recurrent_group( + step=inner_step, + name="inner", + input=[x, wid]) + last = last_seq(input=inner_rnn_output, name="outer_rnn_state") + + # "return last" should also work. But currently RecurrentGradientMachine + # does not handle it correctly. Current implementation requires that + # all the out links are from sequences. However, it does not report error + # when the out links are not sequences. + return inner_rnn_output + +out = recurrent_group( + name="outer", + step=outer_step, + input=[SubsequenceInput(data), SubsequenceInput(emb)]) + +rep = last_seq(input=out) +prob = fc_layer(size=label_dim, + input=rep, + act=SoftmaxActivation(), + bias_attr=True) + +outputs(classification_cost(input=prob, + label=data_layer(name="label", size=label_dim))) diff --git a/paddle/gserver/tests/sequence_rnn_multi_input.conf b/paddle/gserver/tests/sequence_rnn_multi_input.conf new file mode 100644 index 0000000000000..968621cab59be --- /dev/null +++ b/paddle/gserver/tests/sequence_rnn_multi_input.conf @@ -0,0 +1,58 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +######################## data source ################################ +define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', + test_list=None, + module='rnn_data_provider', + obj='process_seq') + + +settings(batch_size=2, learning_rate=0.01) +######################## network configure ################################ +dict_dim = 10 +word_dim = 8 +hidden_dim = 8 +label_dim = 3 + +data = data_layer(name="word", size=dict_dim) + +emb = embedding_layer(input=data, size=word_dim) + +def step(y, wid): + z = embedding_layer(input=wid, size=word_dim) + mem = memory(name="rnn_state", size=hidden_dim) + out = fc_layer(input=[y, z, mem], + size=hidden_dim, + act=TanhActivation(), + bias_attr=True, + name="rnn_state") + return out + +out = recurrent_group( + name="rnn", + step=step, + input=[emb, data]) + +rep = last_seq(input=out) +prob = fc_layer(size=label_dim, + input=rep, + act=SoftmaxActivation(), + bias_attr=True) + +outputs(classification_cost(input=prob, + label=data_layer(name="label", size=label_dim))) diff --git a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp index b73fdd18abf35..550df0a31844e 100644 --- a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp +++ b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp @@ -92,7 +92,11 @@ void CalCost(const string& conf, const string& dir, real* cost, rmDir(dir.c_str()); } -void test(const string& conf1, const string& conf2, double eps) { +void test(const string& conf1, const string& conf2, double eps, bool useGpu) { + if (!paddle::version::isWithGpu() && useGpu) { + return; + } + FLAGS_use_gpu = useGpu; int num_passes = 5; real* cost1 = new real[num_passes]; const string dir1 = "gserver/tests/t1"; @@ -113,17 +117,28 @@ void test(const string& conf1, const string& conf2, double eps) { } TEST(RecurrentGradientMachine, HasSubSequence) { - test("gserver/tests/sequence_layer_group.conf", - "gserver/tests/sequence_nest_layer_group.conf", - 1e-5); + for (bool useGpu : {false, true}) { + test("gserver/tests/sequence_layer_group.conf", + "gserver/tests/sequence_nest_layer_group.conf", + 1e-5, useGpu); + } } TEST(RecurrentGradientMachine, rnn) { - test("gserver/tests/sequence_rnn.conf", - "gserver/tests/sequence_nest_rnn.conf", - 0); + for (bool useGpu : {false, true}) { + test("gserver/tests/sequence_rnn.conf", + "gserver/tests/sequence_nest_rnn.conf", + 1e-6, useGpu); + } } +TEST(RecurrentGradientMachine, rnn_multi_input) { + for (bool useGpu : {false, true}) { + test("gserver/tests/sequence_rnn_multi_input.conf", + "gserver/tests/sequence_nest_rnn_multi_input.conf", + 1e-6, useGpu); + } +} int main(int argc, char** argv) { if (paddle::version::isWithPyDataProvider()) { diff --git a/paddle/parameter/Argument.cpp b/paddle/parameter/Argument.cpp index 0ca56b29b39b3..42c74661d2b2c 100644 --- a/paddle/parameter/Argument.cpp +++ b/paddle/parameter/Argument.cpp @@ -554,11 +554,16 @@ void Argument::degradeSequence(const Argument& input, bool useGpu) { void Argument::subArgFrom(const Argument& input, size_t offset, size_t height, size_t width, bool useGpu, bool trans, bool seqFlag, size_t seqStart, size_t seqSize) { - value = Matrix::create(input.value->getData() + offset, height, width, trans, - useGpu); + if (input.value) { + value = Matrix::create(input.value->getData() + offset * width, + height, width, trans, useGpu); + } + if (input.ids) { + ids = IVector::create(input.ids->getData() + offset, height, useGpu); + } if (input.grad) { - grad = Matrix::create(input.grad->getData() + offset, height, width, trans, - useGpu); + grad = Matrix::create(input.grad->getData() + offset * width, + height, width, trans, useGpu); } if (seqFlag) { sequenceStartPositions = std::make_shared( diff --git a/paddle/parameter/Argument.h b/paddle/parameter/Argument.h index 81cd117fc45cf..81ff9029bc4c8 100644 --- a/paddle/parameter/Argument.h +++ b/paddle/parameter/Argument.h @@ -177,11 +177,11 @@ struct Argument { } /** - * @brief (value, grad, sequenceStartPositions) of output are subset of + * @brief (value, ids, grad, sequenceStartPositions) of output are subset of * input. Note that, output share the same memory of input. * * @param input[in] input - * @param offset[in] offset of input.value + * @param offset[in] offset in terms of rows * @param height[in] height of output.value * @param width[in] width of output.value * @param useGpu[in] diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 76b0db546b18b..dabf8e2953054 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -216,7 +216,7 @@ def check_input(input): """ if isinstance(input, LayerOutput): - return [LayerOutput] + return [input] assert isinstance(input, list) for inp in input: assert isinstance(inp, LayerOutput) @@ -764,7 +764,7 @@ def print_layer(input, name=None): :type input: LayerOutput|list|tuple :return: No return """ - check_input(input) + input = check_input(input) Layer( name=name, From 942d2b6222fe5c3502305d78c963e1b4d2c8c6b2 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Tue, 27 Sep 2016 15:35:53 +0800 Subject: [PATCH 122/324] fix dead link in quick start (#123) --- doc_cn/demo/quick_start/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc_cn/demo/quick_start/index.md b/doc_cn/demo/quick_start/index.md index 34cd4a840e442..aa6b66ca8c024 100644 --- a/doc_cn/demo/quick_start/index.md +++ b/doc_cn/demo/quick_start/index.md @@ -4,7 +4,7 @@ ## 安装(Install) -首先请参考安装教程安装PaddlePaddle。 +首先请参考安装教程安装PaddlePaddle。 ## 使用概述(Overview) From 1c56e0dc9e105bb2190adba4e6bde3e0bf40c4b5 Mon Sep 17 00:00:00 2001 From: Haonan Date: Tue, 27 Sep 2016 22:38:09 -0700 Subject: [PATCH 123/324] fixed the name issue for conv_operator and added a test case (#131) --- python/paddle/trainer_config_helpers/layers.py | 6 +++--- .../tests/layers_test_config.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index dabf8e2953054..dc5acb19b35e9 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -2704,8 +2704,8 @@ def conv_operator(img, filter, filter_size, num_filters, PaddlePaddle now supports rectangular filters, the filter's shape can be (filter_size, filter_size_y). :type filter_size_y: int - :param num_filter: channel of output data. - :type num_filter: int + :param num_filters: channel of output data. + :type num_filters: int :param num_channel: channel of input data. :type num_channel: int :param stride: The x dimension of the stride. @@ -2726,7 +2726,7 @@ def conv_operator(img, filter, filter_size, num_filters, if padding_y is None: padding_y = padding op = ConvOperator(input_layer_names=[img.name, filter.name], - num_filters = num_filter, + num_filters = num_filters, conv_conf=Conv(filter_size=filter_size, padding=padding, stride=stride, diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py index b9eaf2fce7572..cd368d6b12cbd 100644 --- a/python/paddle/trainer_config_helpers/tests/layers_test_config.py +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -23,6 +23,15 @@ x1 = fc_layer(input=x, size=5) y1 = fc_layer(input=y, size=5) + +z1 = mixed_layer(act=LinearActivation(), + input=[conv_operator(img=x1, + filter=y1, + filter_size=1, + num_filters=5, + num_channel=5, + stride=1)]) + y2 = fc_layer(input=y, size=15) cos1 = cos_sim(a=x1, b=y1) @@ -30,7 +39,7 @@ linear_comb = linear_comb_layer(weights=x1, vectors=y2, size=3) -out = fc_layer(input=[cos1, cos3, linear_comb, z], +out = fc_layer(input=[cos1, cos3, linear_comb, z, z1], size=num_classes, act=SoftmaxActivation()) From d130d181463dd085f4932328e9dfa1f494fe4ca5 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 28 Sep 2016 14:37:42 +0800 Subject: [PATCH 124/324] Complete unittest for trainer_config_helpers. (#108) * Fix lots of trainer_config_helpers bug, and complete unittest for `layers.py` --- python/paddle/trainer/config_parser.py | 12 +- .../trainer_config_helpers/activations.py | 9 +- .../paddle/trainer_config_helpers/layers.py | 635 +++++++++++------- .../paddle/trainer_config_helpers/networks.py | 9 +- .../paddle/trainer_config_helpers/poolings.py | 7 +- .../tests/CMakeLists.txt | 5 + .../tests/configs/.gitignore | 1 + .../tests/configs/check.md5 | 17 + .../tests/configs/generate_protostr.sh | 18 + .../tests/configs/img_layers.py | 20 + .../tests/configs/last_first_seq.py | 26 + .../tests/configs/layer_activations.py | 21 + .../tests/configs/projections.py | 47 ++ .../tests/configs/run_tests.sh | 5 + .../tests/configs/simple_rnn_layers.py | 36 + .../tests/configs/test_cost_layers.py | 26 + .../tests/configs/test_expand_layer.py | 14 + .../tests/configs/test_fc.py | 20 + .../tests/configs/test_grumemory_layer.py | 11 + .../tests/configs/test_hsigmoid.py | 11 + .../tests/configs/test_lstmemory_layer.py | 11 + .../tests/configs/test_ntm_layers.py | 23 + .../tests/configs/test_print_layer.py | 12 + .../tests/configs/test_rnn_group.py | 35 + .../tests/configs/test_sequence_pooling.py | 30 + .../tests/configs/unused_layers.py | 14 + .../tests/configs/util_layers.py | 15 + .../tests/layers_test_config.py | 2 +- 28 files changed, 844 insertions(+), 248 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/.gitignore create mode 100644 python/paddle/trainer_config_helpers/tests/configs/check.md5 create mode 100755 python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh create mode 100644 python/paddle/trainer_config_helpers/tests/configs/img_layers.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/layer_activations.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/projections.py create mode 100755 python/paddle/trainer_config_helpers/tests/configs/run_tests.sh create mode 100644 python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_fc.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/unused_layers.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/util_layers.py diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index a57e9065c6f98..1f55298f24f07 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1279,7 +1279,7 @@ def create_input_parameter( size, dims=None, sparse = None, - format = "csr"): + format = None): if dims is None: # TODO(yuyang18): print warning and callstack here! dims = list() @@ -2074,7 +2074,7 @@ def __init__( active_type='linear', device=None, bias=False, - output_max_index=False): + output_max_index=None): super(MaxLayer, self).__init__(name, 'max', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'MaxLayer must have 1 input') self.config.trans_type = trans_type @@ -2083,7 +2083,8 @@ def __init__( input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) - self.config.output_max_index=output_max_index + if output_max_index is not None: + self.config.output_max_index = output_max_index @config_layer('maxid') @@ -2440,7 +2441,7 @@ def __init__( inputs, size=0, bias=True, - error_clipping_threshold=0.0, + error_clipping_threshold=None, **xargs): config_assert(inputs, 'inputs cannot be empty') super(MixedLayer, self).__init__( @@ -2510,7 +2511,8 @@ def __init__( self.create_bias_parameter(bias, self.config.size) - self.config.error_clipping_threshold = error_clipping_threshold + if error_clipping_threshold is not None: + self.config.error_clipping_threshold = error_clipping_threshold # like MixedLayer, but no bias parameter @config_func diff --git a/python/paddle/trainer_config_helpers/activations.py b/python/paddle/trainer_config_helpers/activations.py index 85534675199e7..292014519374e 100644 --- a/python/paddle/trainer_config_helpers/activations.py +++ b/python/paddle/trainer_config_helpers/activations.py @@ -15,8 +15,10 @@ __all__ = ["TanhActivation", "SigmoidActivation", "SoftmaxActivation", "IdentityActivation", "LinearActivation", 'SequenceSoftmaxActivation', 'ExpActivation', - "ReluActivation", "BReluActivation", "SoftReluActivation", "STanhActivation", - "AbsActivation", "SquareActivation", "BaseActivation"] + "ReluActivation", "BReluActivation", "SoftReluActivation", + "STanhActivation", + "AbsActivation", "SquareActivation", + "BaseActivation"] class BaseActivation(object): @@ -36,6 +38,9 @@ def __init__(self, name, support_hppl): self.name = name self.support_hppl = support_hppl + def __repr__(self): + return self.name + class TanhActivation(BaseActivation): """ diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index dc5acb19b35e9..b28dd02b70946 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -13,6 +13,7 @@ # limitations under the License. import functools +import collections from paddle.trainer.config_parser import * from .activations import LinearActivation, SigmoidActivation, TanhActivation, \ @@ -21,6 +22,7 @@ from .poolings import MaxPooling, AvgPooling, BasePoolingType from .attrs import * from .default_decorators import * + try: import cPickle as pickle except ImportError: @@ -51,7 +53,8 @@ 'cross_entropy_with_selfnorm', 'cross_entropy', 'multi_binary_label_cross_entropy', 'rank_cost', 'lambda_cost', 'huber_cost', - 'block_expand_layer', 'out_prod_layer', 'print_layer' + # 'block_expand_layer', # TODO(yuyang18): this layer is not correct + 'out_prod_layer', 'print_layer' ] @@ -165,11 +168,12 @@ class LayerOutput(object): :param activation: Layer Activation. :type activation: BaseActivation. :param parents: Layer's parents. - :type parents: list|tuple + :type parents: list|tuple|collection.Sequence """ def __init__(self, name, layer_type, parents=None, activation=None, - num_filters=None, img_norm_type=None, size=None, outputs=None): + num_filters=None, img_norm_type=None, size=None, outputs=None, + reverse=None): assert isinstance(name, basestring) assert isinstance(layer_type, basestring) assert LayerType.is_layer_type(layer_type) @@ -185,6 +189,7 @@ def __init__(self, name, layer_type, parents=None, activation=None, if outputs is None: outputs = ['default'] self.outputs = outputs + self.reverse = reverse def __repr__(self): """ @@ -204,25 +209,6 @@ def __str__(self): DEVICE = 'device' -def check_input(input): - """ - Check input is a LayerOutput or list of LayerOutput or tuple of LayerOutput - if is a LayerOutput, - - :param input: The input layer. Could be a list/tuple of input layer. - :type input: LayerOutput|list|tuple - :return: list of LayerOutput - :rtype: list of LayerOutput - """ - - if isinstance(input, LayerOutput): - return [input] - assert isinstance(input, list) - for inp in input: - assert isinstance(inp, LayerOutput) - return list(input) - - def layer_support(*attrs): attrs_list = list(attrs) attrs_list.append(DEVICE) @@ -292,6 +278,43 @@ def full_matrix_projection(input, size=0, param_attr=None): return proj +@wrap_param_attr_default() +def trans_full_matrix_projection(input, size=0, param_attr=None): + """ + Different from full_matrix_projection, this projection performs matrix + multiplication, using transpose of weight. + + .. math:: + out.row[i] += in.row[i] * w^\mathrm{T} + + :math:`w^\mathrm{T}` means transpose of weight. + The simply usage is: + + .. code-block:: python + + proj = trans_full_matrix_projection(input=layer, + size=100, + param_attr=ParamAttr( + name='_proj', + initial_mean=0.0, + initial_std=0.01)) + + :param input: input layer + :type input: LayerOutput + :param size: The parameter size. Means the width of parameter. + :type size: int + :param param_attr: Parameter config, None if use default. + :type param_attr: ParameterAttribute + :return: A TransposedFullMatrixProjection Object. + :rtype: TransposedFullMatrixProjection + """ + proj = TransposedFullMatrixProjection(input_layer_name=input.name, + size=size, + **param_attr.attr) + proj.origin = input + return proj + + @wrap_param_attr_default() def table_projection(input, size=0, param_attr=None): """ @@ -369,7 +392,7 @@ def identity_projection(input, offset=None): Note that both of two projections should not have any parameter. :param input: Input Layer. - :type input: LayerOutput. + :type input: LayerOutput :param offset: Offset, None if use default. :type offset: int :return: A IdentityProjection or IdentityOffsetProjection Object @@ -412,10 +435,11 @@ def dotmul_projection(input, param_attr=None): proj = DotMulProjection(input_layer_name=input.name, size=input.size, **param_attr.attr) - proj.origin = input + proj.origin = input return proj -def dotmul_operator(x, y, scale=1): + +def dotmul_operator(a=None, b=None, scale=1, **kwargs): """ DotMulOperator takes two inputs and performs element-wise multiplication: @@ -431,22 +455,31 @@ def dotmul_operator(x, y, scale=1): op = dotmul_operator(x=layer1, y=layer2, scale=0.5) - :param x: Input layer1 - :type x: LayerOutput - :param y: Input layer2 - :type y: LayerOutput + :param a: Input layer1 + :type a: LayerOutput + :param b: Input layer2 + :type b: LayerOutput :param scale: config scalar, default value is one. :type scale: float :return: A DotMulOperator Object. :rtype: DotMulOperator """ - assert isinstance(x, LayerOutput) - assert isinstance(y, LayerOutput) - op = DotMulOperator(input_layer_names=[x.name, y.name], + if 'x' in kwargs or 'y' in kwargs: + logger.warning('x and y arguments for dotmul_operator is deprecated. ' + 'Please use a and b as parameter.') + a = kwargs.get('x', a) # For Backward capacity. + b = kwargs.get('y', b) + assert isinstance(a, LayerOutput) + assert isinstance(b, LayerOutput) + if a.size is not None and b.size is not None: + assert a.size == b.size + + op = DotMulOperator(input_layer_names=[a.name, b.name], scale=scale) - op.origin = [x, y] + op.origin = [a, b] return op + @wrap_bias_attr_default(['padding_attr']) def context_projection(input, context_len, context_start=None, padding_attr=False): @@ -615,7 +648,7 @@ def mixed_layer(size=0, input=None, name=None, act=None, bias_attr=False, else: with mixed_layer(name=name, size=size, act=act, bias_attr=bias_attr, layer_attr=layer_attr) as m: - if isinstance(input, list) or isinstance(input, tuple): + if isinstance(input, collections.Sequence): for each in input: m += each else: @@ -725,23 +758,19 @@ def fc_layer(input, size, act=None, name=None, """ if isinstance(input, LayerOutput): input = [input] - assert not isinstance(param_attr, list) + assert not isinstance(param_attr, collections.Sequence) param_attr = [param_attr] else: - if isinstance(param_attr, list) or isinstance(param_attr, tuple): + if isinstance(param_attr, collections.Sequence): assert len(input) == len(param_attr) else: param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))] - assert isinstance(input, list) - - def __idx_to_input__(i): - attr = param_attr[i] - assert isinstance(attr, ParameterAttribute) - return Input(input[i].name, **attr.attr) + assert isinstance(input, collections.Sequence) Layer( - inputs=map(__idx_to_input__, range(len(input))), + inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( + input, param_attr)], name=name, type=LayerType.FC_LAYER, size=size, @@ -762,16 +791,20 @@ def print_layer(input, name=None): :type name: basestring :param input: The input layer. Could be a list/tuple of input layer. :type input: LayerOutput|list|tuple - :return: No return + :return: LayerOutput """ - input = check_input(input) + if isinstance(input, LayerOutput): + input = [input] + assert isinstance(input, collections.Sequence) # list or tuple + for each in input: + assert isinstance(each, LayerOutput) Layer( name=name, type=LayerType.PRINT_LAYER, inputs=[l.name for l in input], ) - LayerOutput(name, LayerType.PRINT_LAYER, input) + # this layer don't return anything, can not be input of other layer. @wrap_name_default("seq_pooling") @@ -810,8 +843,13 @@ def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, :rtype: LayerType """ extra_dict = dict() + # noinspection PyUnresolvedReferences if isinstance(pooling_type, AvgPooling): extra_dict['average_strategy'] = pooling_type.strategy + elif isinstance(pooling_type, MaxPooling) and \ + pooling_type.output_max_index is not None: + assert isinstance(pooling_type.output_max_index, bool) + extra_dict['output_max_index'] = pooling_type.output_max_index extra_dict.update(ExtraLayerAttribute.to_kwargs(layer_attr)) Layer( @@ -835,7 +873,7 @@ def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, @wrap_name_default("lstmemory") @layer_support(DROPOUT) def lstmemory(input, name=None, reverse=False, act=None, - gate_act=None, + gate_act=None, size=None, state_act=None, bias_attr=None, param_attr=None, layer_attr=None): """ @@ -900,6 +938,16 @@ def lstmemory(input, name=None, reverse=False, act=None, assert gate_act.support_hppl assert state_act.support_hppl assert act.support_hppl + assert input.size is not None and input.size % 4 == 0 + if size is not None: + if input.size / 4 == size: + plog = logger.warning + else: + plog = logger.fatal + + plog("NOTE: The lstmemory layer[%s]'s size is set by previous input " + "layer. The lstm size should be equal with input layer size/4. The" + " size which is set explicitly will be ignored." % name) Layer(name=name, type=LayerType.LSTMEMORY, @@ -911,8 +959,9 @@ def lstmemory(input, name=None, reverse=False, act=None, inputs=[Input(input.name, **param_attr.attr)], **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.LSTMEMORY, [input], - size=input.size / 4 if input.size is not None else None) + return LayerOutput(name, LayerType.LSTMEMORY, [input], size=input.size / 4, + reverse=reverse) + @wrap_bias_attr_default() @wrap_param_attr_default() @@ -922,7 +971,7 @@ def lstmemory(input, name=None, reverse=False, act=None, @wrap_name_default("gru") @layer_support(DROPOUT) def grumemory(input, name=None, reverse=False, act=None, - gate_act=None, + gate_act=None, size=None, bias_attr=None, param_attr=None, layer_attr=None): """ @@ -980,7 +1029,7 @@ def grumemory(input, name=None, reverse=False, act=None, :type name: None|basestring :param input: input layer. :type input: LayerOutput. - :param reverse: Wether sequence process is reversed or not. + :param reverse: Whether sequence process is reversed or not. :type reverse: bool :param act: activation type, TanhActivation by default. This activation affects the :math:`{\\tilde{h_t}}`. @@ -996,12 +1045,23 @@ def grumemory(input, name=None, reverse=False, act=None, :type param_attr: ParameterAttribute|None|False :param layer_attr: Extra Layer attribute :type layer_attr: ExtraLayerAttribute|None + :param size: Stub parameter of size, but actually not used. If set this size + will get a warning. + :type size: None :return: LayerOutput object. :rtype: LayerOutput """ - assert act.support_hppl assert gate_act.support_hppl + assert input.size is not None and input.size % 3 == 0 + if size is not None: + if input.size / 3 == size: + plog = logger.warning + else: + plog = logger.fatal + plog("NOTE: the gru memory layer's size is set by previous input layer," + " and should be input size / 3. Set size explicitly will be " + "ignored.") Layer(name=name, type=LayerType.GRUMEMORY, @@ -1013,8 +1073,9 @@ def grumemory(input, name=None, reverse=False, act=None, **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.GRUMEMORY, [input], - size=input.size / 3 if input.size is not None else None) + return LayerOutput(name, LayerType.GRUMEMORY, [input], size=input.size / 3, + reverse=reverse) + @wrap_name_default() @layer_support() @@ -1033,6 +1094,12 @@ def last_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, :return: LayerOutput object. :rtype: LayerOutput """ + if input.reverse is not None and input.reverse: + logger.warning("You are getting the last instance of a sequence that" + " is a output of a REVERSED layer. There is no time" + " series information at all. Maybe you want to use" + " first_seq instead.") + Layer( name=name, type=LayerType.SEQUENCE_LAST_INSTANCE, @@ -1061,6 +1128,13 @@ def first_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, :return: LayerOutput object. :rtype: LayerOutput """ + + if input.reverse is not None and not input.reverse: + logger.warning('You are getting the first instance for a time series,' + ' and it is a normal recurrent layer output. There is no' + ' time series information at all. Maybe you want to use' + ' last_seq instead.') + Layer( name=name, type=LayerType.SEQUENCE_FIRST_INSTANCE, @@ -1076,6 +1150,7 @@ class ExpandLevel(object): FROM_TIMESTEP = AggregateLevel.EACH_TIMESTEP FROM_SEQUENCE = AggregateLevel.EACH_SEQUENCE + @wrap_name_default() @layer_support() def expand_layer(input, expand_as, @@ -1126,7 +1201,6 @@ def expand_layer(input, expand_as, parents=[input, expand_as]) - @wrap_name_default() @layer_support() def interpolation_layer(input, weight, name=None, layer_attr=None): @@ -1158,10 +1232,15 @@ def interpolation_layer(input, weight, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - assert isinstance(input, list) or isinstance(input, tuple) + assert isinstance(input, collections.Sequence) assert len(input) == 2 - assert input[0].size == input[1].size - assert weight.size == 1 + assert isinstance(input[0], LayerOutput) and isinstance(input[1], + LayerOutput) + if input[0].size is not None and input[1].size is not None: + assert input[0].size == input[1].size + assert isinstance(weight, LayerOutput) + if weight.size is not None: + assert weight.size == 1 Layer( name=name, type=LayerType.INTERPOLATION_LAYER, @@ -1203,11 +1282,13 @@ def power_layer(input, weight, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - assert weight.size == 1 + assert isinstance(input, LayerOutput) and isinstance(weight, LayerOutput) + if weight.size is not None: + assert weight.size == 1 Layer( name=name, type=LayerType.POWER_LAYER, - inputs=[input.name, weight.name], + inputs=[weight.name, input.name], **ExtraAttr.to_kwargs(layer_attr) ) return LayerOutput(name, LayerType.POWER_LAYER, @@ -1246,7 +1327,9 @@ def scaling_layer(input, weight, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - assert weight.size == 1 + assert isinstance(weight, LayerOutput) and isinstance(input, LayerOutput) + if weight.size is not None: + assert weight.size == 1 Layer( name=name, type=LayerType.SCALING_LAYER, @@ -1325,6 +1408,7 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ + assert isinstance(a, LayerOutput) and isinstance(b, LayerOutput) if size == 1: Layer( name=name, @@ -1334,6 +1418,8 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): **ExtraLayerAttribute.to_kwargs(layer_attr) ) else: + if a.size is not None and b.size is not None: + assert size == b.size / a.size Layer( name=name, type=LayerType.COSINE_SIM_VEC, @@ -1344,11 +1430,13 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): ) return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b]) + @wrap_name_default() @wrap_bias_attr_default(has_bias=True) +@wrap_param_attr_default() @layer_support() def hsigmoid(input, label, num_classes, name=None, bias_attr=None, - layer_attr=None): + param_attr=None, layer_attr=None): """ Organize the classes into a binary tree. At each node, a sigmoid function is used to calculate the probability of belonging to the right branch. @@ -1382,15 +1470,23 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, """ if isinstance(input, LayerOutput): input = [input] - assert isinstance(input, list) or isinstance(input, tuple) + if not isinstance(param_attr, collections.Sequence): + param_attr = [param_attr] + else: + if not isinstance(param_attr, collections.Sequence): + param_attr = [param_attr] * len(input) + else: + assert len(param_attr) == len(input) + + assert isinstance(input, collections.Sequence) assert isinstance(label, LayerOutput) assert label.layer_type == LayerType.DATA ipts_for_layer = [] parents = [] - for each_input in input: + for each_input, each_param_attr in zip(input, param_attr): assert isinstance(each_input, LayerOutput) - ipts_for_layer.append(each_input.name) + ipts_for_layer.append(Input(each_input.name, **each_param_attr.attr)) parents.append(each_input) ipts_for_layer.append(label.name) parents.append(label) @@ -1405,6 +1501,7 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, ) return LayerOutput(name, LayerType.HSIGMOID, parents=parents) + @wrap_name_default("conv") @wrap_param_attr_default() @wrap_bias_attr_default() @@ -1438,23 +1535,26 @@ def img_conv_layer(input, filter_size, num_filters, :type name: basestring :param input: Layer Input. :type input: LayerOutput - :param filter_size: The x dimension of a filter kernel. - :type filter_size: int + :param filter_size: The x dimension of a filter kernel. Or input a tuple for + two image dimension. + :type filter_size: int|tuple|list :param filter_size_y: The y dimension of a filter kernel. Since PaddlePaddle currently supports rectangular filters, the filter's shape will be (filter_size, filter_size_y). - :type filter_size_y: int + :type filter_size_y: int|None :param num_filters: Each filter group's number of filter :param act: Activation type. Default is tanh :type act: BaseActivation :param groups: Group size of filters. :type groups: int - :param stride: The x dimension of the stride. - :type stride: int + :param stride: The x dimension of the stride. Or input a tuple for two image + dimension. + :type stride: int|tuple|list :param stride_y: The y dimension of the stride. :type stride_y: int - :param padding: The x dimension of the padding. - :type padding: int + :param padding: The x dimension of the padding. Or input a tuple for two + image dimension + :type padding: int|tuple|list :param padding_y: The y dimension of the padding. :type padding_y: int :param bias_attr: Convolution bias attribute. None means default bias. @@ -1475,13 +1575,30 @@ def img_conv_layer(input, filter_size, num_filters, if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters + if filter_size_y is None: - filter_size_y = filter_size + if isinstance(filter_size, collections.Sequence): + assert len(filter_size) == 2 + filter_size, filter_size_y = filter_size + else: + filter_size_y = filter_size + if stride_y is None: - stride_y = stride + if isinstance(stride, collections.Sequence): + assert len(stride) == 2 + stride, stride_y = stride + else: + stride_y = stride + if padding_y is None: - padding_y = padding - if param_attr.attr.get('initial_smart') == True: # special initial for conv layers. + if isinstance(padding, collections.Sequence): + assert len(padding) == 2 + padding, padding_y = padding + else: + padding_y = padding + + if param_attr.attr.get('initial_smart'): + # special initial for conv layers. init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 param_attr.attr["initial_mean"] = 0.0 param_attr.attr["initial_std"] = init_w @@ -1492,8 +1609,9 @@ def img_conv_layer(input, filter_size, num_filters, inputs=Input(input.name, conv=Conv( filter_size=filter_size, padding=padding, stride=stride, channels=num_channels, groups=groups, - filter_size_y=filter_size_y, padding_y=padding_y, stride_y=stride_y), - **param_attr.attr), + filter_size_y=filter_size_y, padding_y=padding_y, + stride_y=stride_y), + **param_attr.attr), active_type=act.name, num_filters=num_filters, bias=ParamAttr.to_bias(bias_attr), @@ -1553,7 +1671,7 @@ def img_pool_layer(input, pool_size, name=None, type=LayerType.POOL_LAYER, inputs=[Input(input.name, pool=Pool( - pool_type=pool_type.name + '-projection', + pool_type=''.join([pool_type.name, '-projection']), channels=num_channels, size_x=pool_size, start=start, @@ -1607,7 +1725,6 @@ def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, :type power: float :param num_channels: input layer's filers number or channels. If num_channels is None, it will be set automatically. - :param blocked: namely normalize in number of blocked feature maps. :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. @@ -1660,7 +1777,7 @@ def batch_norm_layer(input, act=None, name=None, num_channels=None, batch_norm for CPU. Otherwise, select batch norm type based on the specified type. If you use cudnn_batch_norm, we suggested you use latest version, such as v5.1. - :type type: None|string, None or "batch_norm" or "cudnn_batch_norm" + :type batch_norm_type: None|string, None or "batch_norm" or "cudnn_batch_norm" :param act: Activation Type. Better be relu. Because batch normalization will normalize input near zero. :type act: BaseActivation @@ -1821,7 +1938,7 @@ def addto_layer(input, act=None, name=None, bias_attr=None, if isinstance(input, LayerOutput): input = [input] - assert isinstance(input, list) or isinstance(input, tuple) + assert isinstance(input, collections.Sequence) ipts_for_layer = [] for each_input in input: assert isinstance(each_input, LayerOutput) @@ -1835,7 +1952,7 @@ def addto_layer(input, act=None, name=None, bias_attr=None, active_type=act.name, **ExtraLayerAttribute.to_kwargs(layer_attr) ) - assert isinstance(input, list) or isinstance(input, tuple) + return LayerOutput(name, LayerType.ADDTO_LAYER, parents=input, activation=act, num_filters=num_filters) @@ -1851,7 +1968,7 @@ def concat_layer(input, act=None, name=None, layer_attr=None): :param name: Layer name. :type name: basestring :param input: input layers or projections - :type input: list|tuple + :type input: list|tuple|collection.Sequence :param act: Activation type. :type act: BaseActivation :param layer_attr: Extra Layer Attribute. @@ -1865,10 +1982,10 @@ def concat_layer(input, act=None, name=None, layer_attr=None): elif isinstance(input, Projection): input = [input] else: - assert isinstance(input, list) or isinstance(input, tuple) + assert isinstance(input, collections.Sequence) def __is_type__(o, tp): - if not isinstance(o, list) and not isinstance(o, tuple): + if not isinstance(o, collections.Sequence): if o == tp: return True elif len(o.__bases__) == 0: @@ -2150,28 +2267,51 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): @wrap_param_attr_default() @layer_support() def recurrent_layer(input, act=None, bias_attr=None, - param_attr=None, name=None, layer_attr=None): + param_attr=None, name=None, reverse=False, layer_attr=None): """ - TODO(yuyang18): Add docs + Simple recurrent unit layer. It is just a fully connect layer through both + time and neural network. - :param input: - :param size: - :param act: - :param bias_attr: - :param param_attr: - :param name: - :param layer_attr: + For each sequence [start, end] it performs the following computation\: + + .. math:: + + out_{i} = act(in_{i}) \\ \\ \\text{for} \\ i = start \\\\ + out_{i} = act(in_{i} + out_{i-1} * W) \\ \\ \\text{for} \\ start < i <= end + + If reversed is true, the order is reversed\: + + .. math:: + + out_{i} = act(in_{i}) \\ \\ \\text{for} \\ i = end \\\\ + out_{i} = act(in_{i} + out_{i+1} * W) \\ \\ \\text{for} \\ start <= i < end + + + :param input: Input Layer + :type input: LayerOutput + :param act: activation. + :type act: BaseActivation + :param bias_attr: bias attribute. + :type bias_attr: ParameterAttribute + :param param_attr: parameter attribute. + :type param_attr: ParameterAttribute + :param name: name of the layer + :type name: basestring + :param layer_attr: Layer Attribute. + :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. + :rtype: LayerOutput """ Layer(name=name, type=LayerType.RECURRENT_LAYER, inputs=Input(input.name, **param_attr.attr), active_type=act.name, - size=input.size, bias=ParamAttr.to_bias(bias_attr), + reversed=reverse, **ExtraAttr.to_kwargs(layer_attr)) return LayerOutput(name=name, layer_type=LayerType.RECURRENT_LAYER, - parents=[input], size=input.size, activation=act) + parents=[input], size=input.size, activation=act, + reverse=reverse) class StaticInput(object): @@ -2179,6 +2319,7 @@ class StaticInput(object): StaticInput is only used in recurrent_group which defines a read-only memory that can be a sequence or non-sequence. """ + def __init__(self, input, is_seq=False, size=None): assert isinstance(input, LayerOutput) self.input = input @@ -2198,6 +2339,7 @@ class SubsequenceInput(object): input = SubsequenceInput(layer) """ + def __init__(self, input): assert isinstance(input, LayerOutput) assert input.size is not None @@ -2270,7 +2412,7 @@ def is_single_input(x): if is_single_input(input): input = [input] - assert isinstance(input, list) or isinstance(input, tuple) + assert isinstance(input, collections.Sequence) def is_in_links(x): return isinstance(x, LayerOutput) or isinstance(x, SubsequenceInput) @@ -2314,6 +2456,7 @@ def map_in_links(x): for ot in layer_outs: assert isinstance(ot, LayerOutput) + ot.reverse = reverse if contains_sub_seq[0]: RecurrentLayerGroupSetOutLink(Link(ot.name, has_subseq=True)) else: @@ -2326,6 +2469,7 @@ def map_in_links(x): else: return layer_outs + class BaseGeneratedInput(object): def __init__(self): self.bos_id = None @@ -2354,6 +2498,7 @@ def before_real_step(self): return trg_emb def __init__(self, size, embedding_name, embedding_size): + super(GeneratedInput, self).__init__() self.size = size self.embedding_name = embedding_name self.embedding_size = embedding_size @@ -2390,6 +2535,7 @@ def maxid_layer(input, name=None, layer_attr=None): layer_type=LayerType.MAXID_LAYER, parents=[input]) + @wrap_name_default() def out_prod_layer(input1, input2, name=None, layer_attr=None): """ @@ -2422,7 +2568,8 @@ def out_prod_layer(input1, input2, name=None, layer_attr=None): **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name=name, layer_type=LayerType.OUT_PROD_LAYER, - parents=[input1,input2]) + parents=[input1, input2]) + @wrap_name_default() def eos_layer(input, eos_id, name=None, layer_attr=None): @@ -2475,14 +2622,14 @@ def beam_search(step, input, bos_id, eos_id, beam_size, def rnn_step(input): last_time_step_output = memory(name='rnn', size=512) - with mixed_layer(size=512) as simple_rnn: + with mixed_layer(size=512, name='rnn') as simple_rnn: simple_rnn += full_matrix_projection(input) simple_rnn += last_time_step_output return simple_rnn beam_gen = beam_search(name="decoder", step=rnn_step, - input=[StaticInput("encoder_last")], + input=[StaticInput(encoder_last)], bos_id=0, eos_id=1, beam_size=5, @@ -2496,18 +2643,18 @@ def rnn_step(input): :param name: Name of the recurrent unit that generates sequences. :type name: base string :param step: A callable function that defines the calculation in a time - step, and it is appled to sequences with arbitrary length by + step, and it is applied to sequences with arbitrary length by sharing a same set of weights. You can refer to the first parameter of recurrent_group, or demo/seqToseq/seqToseq_net.py for more details. :type step: callable :param input: Input data for the recurrent unit - :type input: StaticInput|GeneratedInput + :type input: list :param bos_id: Index of the start symbol in the dictionary. The start symbol is a special token for NLP task, which indicates the beginning of a sequence. In the generation task, the start - symbol is ensential, since it is used to initialize the RNN + symbol is essential, since it is used to initialize the RNN internal state. :type bos_id: int :param eos_id: Index of the end symbol in the dictionary. The end symbol is @@ -2516,6 +2663,8 @@ def rnn_step(input): symbol is generated, or a pre-defined max iteration number is exceeded. :type eos_id: int + :param max_length: Max generated sequence length. + :type max_length: int :param beam_size: Beam search for sequence generation is an iterative search algorithm. To maintain tractability, every iteration only only stores a predetermined number, called the beam_size, @@ -2556,8 +2705,8 @@ def rnn_step(input): real_input = [] for i, each_input in enumerate(input): # print type(each_input) - assert isinstance(each_input, StaticInput) or isinstance(each_input, - BaseGeneratedInput) + assert isinstance(each_input, StaticInput) or isinstance( + each_input, BaseGeneratedInput) if isinstance(each_input, BaseGeneratedInput): assert generated_input_index == -1 generated_input_index = i @@ -2667,7 +2816,7 @@ def __add_evaluator__(e): e(name=e.__name__, input=input, label=label) - if not isinstance(evaluator, list) and not isinstance(evaluator, tuple): + if not isinstance(evaluator, collections.Sequence): evaluator = [evaluator] for each_evaluator in evaluator: @@ -2675,8 +2824,9 @@ def __add_evaluator__(e): return LayerOutput(name, LayerType.COST, parents=[input, label]) + def conv_operator(img, filter, filter_size, num_filters, - num_channel=None, stride=1, padding=0, groups=1, + num_channel=None, stride=1, padding=0, filter_size_y=None, stride_y=None, padding_y=None): """ Different from img_conv_layer, conv_op is an Operator, which can be used @@ -2690,7 +2840,7 @@ def conv_operator(img, filter, filter_size, num_filters, op = conv_operator(img=input1, filter=input2, - filter_size=3.0, + filter_size=3, num_filters=64, num_channels=64) @@ -2725,8 +2875,16 @@ def conv_operator(img, filter, filter_size, num_filters, stride_y = stride if padding_y is None: padding_y = padding + + if num_channel is None: + num_channel = img.num_filters + + assert isinstance(filter, LayerOutput) + if filter.size is not None: + filter.size = filter_size * filter_size_y * num_filters * num_channel + op = ConvOperator(input_layer_names=[img.name, filter.name], - num_filters = num_filters, + num_filters=num_filters, conv_conf=Conv(filter_size=filter_size, padding=padding, stride=stride, @@ -2734,13 +2892,13 @@ def conv_operator(img, filter, filter_size, num_filters, filter_size_y=filter_size_y, padding_y=padding_y, stride_y=stride_y, - groups=groups)) + groups=1)) op.origin = [img, filter] return op @wrap_name_default() -def conv_shift_layer(input, name=None): +def conv_shift_layer(a, b, name=None): """ This layer performs cyclic convolution for two input. For example: - a[in]: contains M elements. @@ -2752,68 +2910,77 @@ def conv_shift_layer(input, name=None): c[i] = \sum_{j=-(N-1)/2}^{(N-1)/2}a_{i+j} * b_{j} In this formular: - - a's index is computed modulo M. - - b's index is computed modulo N. + - a's index is computed modulo M. When it is negative, then get item from + the right side (which is the end of array) to the left. + - b's index is computed modulo N. When it is negative, then get item from + the right size (which is the end of array) to the left. The example usage is: .. code-block:: python - conv_shift = conv_shif_layer(input=[layer1, layer2]) + conv_shift = conv_shift_layer(input=[layer1, layer2]) :param name: layer name :type name: basestring - :param input: Input layer. - :type input: LayerOutput|list|tuple. + :param a: Input layer a. + :type a: LayerOutput + :param b: input layer b + :type b: LayerOutput :return: LayerOutput object. :rtype: LayerOutput """ - assert isinstance(input, list) or isinstance(input, tuple) + assert isinstance(a, LayerOutput) and isinstance(b, LayerOutput) + assert b.size is None or b.size % 2 == 1 # size of b must be odd. Layer( name=name, type=LayerType.CONV_SHIFT_LAYER, - inputs=[x.name for x in input], + inputs=[a.name, b.name], ) - return LayerOutput(name, LayerType.CONV_SHIFT_LAYER, parents=input) + return LayerOutput(name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], + size=a.size) @wrap_name_default() @wrap_param_attr_default() @wrap_bias_attr_default() +@wrap_act_default(act=LinearActivation()) @layer_support(ERROR_CLIPPING, DROPOUT) -def tensor_layer(input, size, act=None, name=None, +def tensor_layer(a, b, size, act=None, name=None, param_attr=None, bias_attr=None, layer_attr=None): """ This layer performs tensor operation for two input. For example, each sample: .. math:: - y_{i} = x_{1} * W_{i} * {x_{2}^\mathrm{T}}, i=0,1,...,K-1 + y_{i} = a * W_{i} * {b^\mathrm{T}}, i=0,1,...,K-1 In this formular: - - :math:`x_{1}`: the first input contains M elements. - - :math:`x_{2}`: the second input contains N elements. + - :math:`a`: the first input contains M elements. + - :math:`b`: the second input contains N elements. - :math:`y_{i}`: the i-th element of y. - :math:`W_{i}`: the i-th learned weight, shape if [M, N] - - :math:`{x_{2}}^\mathrm{T}`: the transpose of :math:`x_{2}`. + - :math:`b^\mathrm{T}`: the transpose of :math:`b_{2}`. The simple usage is: .. code-block:: python - tensor = tensor_layer(input=[layer1, layer2]) + tensor = tensor_layer(a=layer1, b=layer2, size=1000) :param name: layer name :type name: basestring - :param input: Input layer. - :type input: LayerOutput|list|tuple. + :param a: Input layer a. + :type a: LayerOutput + :param b: input layer b. + :type b: LayerOutput :param size: the layer dimension. :type size: int. :param act: Activation Type. Default is tanh. :type act: BaseActivation :param param_attr: The Parameter Attribute. - :type param_attr: ParameterAttribute|list + :type param_attr: ParameterAttribute :param bias_attr: The Bias Attribute. If no bias, then pass False or something not type of ParameterAttribute. None will get a default Bias. @@ -2823,65 +2990,26 @@ def tensor_layer(input, size, act=None, name=None, :return: LayerOutput object. :rtype: LayerOutput """ - assert isinstance(input, list) or isinstance(input, tuple) - assert len(input) == 2 + assert isinstance(a, LayerOutput) and isinstance(b, LayerOutput) Layer( name=name, size=size, type=LayerType.TENSOR_LAYER, active_type=act.name, bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(input[0].name, **param_attr.attr), - Input(input[1].name)], + inputs=[Input(a.name, **param_attr.attr), + Input(b.name)], **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.TENSOR_LAYER, parents=input, + return LayerOutput(name, LayerType.TENSOR_LAYER, parents=[a, b], activation=act, size=size) -@wrap_param_attr_default() -def trans_full_matrix_projection(input, size=0, param_attr=None): - """ - Different from full_matrix_projection, this projection performs matrix - multiplication, using transpose of weight. - - .. math:: - out.row[i] += in.row[i] * w^\mathrm{T} - - :math:`w^\mathrm{T}` means transpose of weight. - The simply usage is: - - .. code-block:: python - - proj = trans_full_matrix_projection(input=layer, - size=100, - param_attr=ParamAttr( - name='_proj', - initial_mean=0.0, - initial_std=0.01)) - - :param input: input layer - :type input: LayerOutput - :param size: The parameter size. Means the width of parameter. - :type size: int - :param param_attr: Parameter config, None if use default. - :type param_attr: ParameterAttribute - :return: A TransposedFullMatrixProjection Object. - :rtype: TransposedFullMatrixProjection - """ - proj = TransposedFullMatrixProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) - proj.origin = input - proj.origin.projection = "trans_matrix" - return proj - - @wrap_name_default() @wrap_param_attr_default() @wrap_bias_attr_default() @wrap_act_default() -def selective_fc_layer(input, size, act=None, name=None, +def selective_fc_layer(input, select, size, act=None, name=None, pass_generation=False, has_selected_colums=True, mul_ratio=0.02, @@ -2896,12 +3024,15 @@ def selective_fc_layer(input, size, act=None, name=None, .. code-block:: python - sel_fc = selective_fc_layer(input=input, 128, act=TanhActivation()) + sel_fc = selective_fc_layer(input=input, size=128, act=TanhActivation()) :param name: The Layer Name. :type name: basestring :param input: The input layer. :type input: LayerOutput|list|tuple + :param select: The select layer. The output of select layer should be a + sparse binary matrix, and treat as the mask of selective fc. + :type select: LayerOutput :param size: The layer dimension. :type size: int :param act: Activation Type. Default is tanh. @@ -2919,33 +3050,33 @@ def selective_fc_layer(input, size, act=None, name=None, """ if isinstance(input, LayerOutput): input = [input] - assert not isinstance(param_attr, list) + assert not isinstance(param_attr, collections.Sequence) param_attr = [param_attr] else: - if isinstance(param_attr, list) or isinstance(param_attr, tuple): + if isinstance(param_attr, collections.Sequence): assert len(input) == len(param_attr) else: param_attr = [copy.deepcopy(param_attr) for _ in range(len(input))] - assert isinstance(input, list) - - def __idx_to_input__(i): - attr = param_attr[i] - assert isinstance(attr, ParameterAttribute) - return Input(input[i].name, **attr.attr) - + assert isinstance(input, collections.Sequence) + assert isinstance(select, LayerOutput) + if select.size is not None: + assert select.size == size Layer( - inputs=map(__idx_to_input__, range(len(input))), + inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( + input, param_attr)] + [select.name], name=name, type=LayerType.SEL_FC_LAYER, size=size, + bias=ParameterAttribute.to_bias(bias_attr), active_type=act.name, selective_fc_pass_generation=pass_generation, has_selected_colums=has_selected_colums, selective_fc_full_mul_ratio=mul_ratio, **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.SEL_FC_LAYER, input, activation=act, + return LayerOutput(name, LayerType.SEL_FC_LAYER, list(input) + [select], + activation=act, size=size) @@ -3013,7 +3144,7 @@ def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0): @wrap_name_default() -def linear_comb_layer(weights, vectors, size, name=None): +def linear_comb_layer(weights, vectors, size=None, name=None): """ A layer for weighted sum of vectors takes two inputs. - Input: size of weights is M @@ -3043,11 +3174,13 @@ def linear_comb_layer(weights, vectors, size, name=None): .. code-block:: python - linear_comb = linear_comb_layer(weighs=weight, vectors=vectors, + linear_comb = linear_comb_layer(weights=weight, vectors=vectors, size=elem_dim) - :param input: The input layers. - :type input: LayerOutput + :param weights: The weight layer. + :type weights: LayerOutput + :param vectors: The vector layer. + :type vectors: LayerOutput :param size: the dimension of this layer. :type size: int :param name: The Layer Name. @@ -3055,7 +3188,13 @@ def linear_comb_layer(weights, vectors, size, name=None): :return: LayerOutput object. :rtype: LayerOutput """ - + assert isinstance(weights, LayerOutput) and isinstance(vectors, LayerOutput) + if vectors.size is not None and weights.size is not None: + assert vectors.size % weights.size == 0 + if size is None: + size = vectors.size / weights.size + else: + assert size == vectors.size / weights.size Layer( name=name, type=LayerType.LINEAR_COMBINATION_LAYER, @@ -3065,8 +3204,10 @@ def linear_comb_layer(weights, vectors, size, name=None): return LayerOutput(name, LayerType.LINEAR_COMBINATION_LAYER, [weights, vectors], size=size) + convex_comb_layer = linear_comb_layer + @wrap_name_default() def block_expand_layer(input, channel=0, @@ -3128,22 +3269,22 @@ def block_expand_layer(input, """ Layer(name=name, input=Input(input.name, - block_expand=BlockExpand(channel=channel, + block_expand=BlockExpand(channels=channel, block_x=block_x, block_y=block_y, stride_x=stride_x, stride_y=stride_y, padding_x=padding_x, padding_y=padding_y) - ), + ), type=LayerType.BLOCK_EXPAND, - ) + ) + + return LayerOutput(name, LayerType.BLOCK_EXPAND, parents=[input]) - return LayerOutput(name, LayerType.BLOCK_EXPAND, - parents=[input], size=size) @wrap_name_default() -def ctc_layer(input, label, size, name=None, norm_by_times=False): +def ctc_layer(input, label, size=None, name=None, norm_by_times=False): """ Connectionist Temporal Classification (CTC) is designed for temporal classication task. That is, for sequence labeling problems where the @@ -3151,7 +3292,8 @@ def ctc_layer(input, label, size, name=None, norm_by_times=False): More details can be found by referring to `Connectionist Temporal Classification: Labelling Unsegmented Sequence Data with Recurrent - Neural Networks `_ + Neural Networks `_ Note: Considering the 'blank' label needed by CTC, you need to use @@ -3169,14 +3311,14 @@ def ctc_layer(input, label, size, name=None, norm_by_times=False): size=9055, norm_by_times=True) - :param input: The input layers. + :param input: The input layer. :type input: LayerOutput :param label: The data layer of label with variable length. :type label: LayerOutput :param size: category numbers + 1. :type size: int - :param name: The name of this layer, which can not specify. - :type name: string|None + :param name: The name of this layer + :type name: basestring|None :param norm_by_times: Whether to normalization by times. False by default. :type norm_by_times: bool :return: LayerOutput object. @@ -3184,18 +3326,24 @@ def ctc_layer(input, label, size, name=None, norm_by_times=False): """ assert isinstance(input, LayerOutput) assert isinstance(label, LayerOutput) + if label.size is not None: + if size is not None: + assert size == label.size + 1 + else: + size = label.size + 1 Layer( - name = name, - type = LayerType.CTC_LAYER, - size = size, - norm_by_times = norm_by_times, - inputs = [input.name, label.name] + name=name, + type=LayerType.CTC_LAYER, + size=size, + norm_by_times=norm_by_times, + inputs=[input.name, label.name] ) return LayerOutput(name, LayerType.CTC_LAYER, [input, label], size=size) + @wrap_name_default() @wrap_param_attr_default() -def crf_layer(input, label, size, weight=None, param_attr=None, name=None): +def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None): """ A layer for calculating the cost of sequential conditional random field model. @@ -3211,7 +3359,7 @@ def crf_layer(input, label, size, weight=None, param_attr=None, name=None): :param input: The first input layer is the feature. :type input: LayerOutput :param label: The second input layer is label. - :type input: LayerOutput + :type label: LayerOutput :param size: The category number. :type size: int :param weight: The third layer is "weight" of each sample, which is an @@ -3227,6 +3375,12 @@ def crf_layer(input, label, size, weight=None, param_attr=None, name=None): assert isinstance(input, LayerOutput) assert isinstance(label, LayerOutput) assert weight is None or isinstance(weight, LayerOutput) + if input.size is not None and label.size is not None: + assert input.size == label.size + if size is None: + size = input.size + else: + assert size == input.size ipts = [Input(input.name, **param_attr.attr), Input(label.name)] @@ -3234,16 +3388,17 @@ def crf_layer(input, label, size, weight=None, param_attr=None, name=None): ipts.append(Input(weight.name)) Layer( - name = name, - type = LayerType.CRF_LAYER, - size = size, - inputs = ipts, + name=name, + type=LayerType.CRF_LAYER, + size=size, + inputs=ipts, ) parents = [input, label] if weight is not None: parents.append(weight) return LayerOutput(name, LayerType.CRF_LAYER, parents, size=size) + @wrap_name_default() @wrap_param_attr_default() def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): @@ -3276,24 +3431,28 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): ipts.append(Input(label.name)) Layer( - name = name, - type = LayerType.CRF_DECODING_LAYER, - size = size, - inputs = ipts, + name=name, + type=LayerType.CRF_DECODING_LAYER, + size=size, + inputs=ipts, ) parents = [input] if label is not None: parents.append(label) return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=size) + """ following are cost Layers. """ + + @wrap_name_default() -def rank_cost(left, right, lable, weight=None, name=None, coeff=1.0): +def rank_cost(left, right, label, weight=None, name=None, coeff=1.0): """ A cost Layer for learning to rank using gradient descent. Details can refer - to `papers `_. + to `papers `_. This layer contains at least three inputs. The weight is an optional argument, which affects the cost. @@ -3350,12 +3509,13 @@ def rank_cost(left, right, lable, weight=None, name=None, coeff=1.0): type=LayerType.RANK_COST, inputs=ipts, coeff=coeff, - ) + ) return LayerOutput(name, LayerType.RANK_COST, parents=parents) + @wrap_name_default() -def lambda_cost(input, score, NDCG_num=5, max_sort_size=-1, coeff=1.0): +def lambda_cost(input, score, name, NDCG_num=5, max_sort_size=-1): """ lambdaCost for lambdaRank LTR approach. @@ -3368,9 +3528,7 @@ def lambda_cost(input, score, NDCG_num=5, max_sort_size=-1, coeff=1.0): NDCG_num=8, max_sort_size=-1) - :param input: The 1st input. Samples of the same query should be loaded - as sequence. User should provided socres for each sample. - The score should be the 2nd input of this layer. + :param input: Samples of the same query should be loaded as sequence. :type input: LayerOutput :param score: The 2nd input. Score of each sample. :type input: LayerOutput @@ -3388,21 +3546,22 @@ def lambda_cost(input, score, NDCG_num=5, max_sort_size=-1, coeff=1.0): :type max_sort_size: int :param name: The name of this layers. It is not necessary. :type name: None|basestring - :param coeff: The coefficient affects the gradient in the backward. - :type coeff: float :return: LayerOutput object. :rtype: LayerOutput """ + assert isinstance(input, LayerOutput) and isinstance(score, LayerOutput) + if score.size is not None: + assert score.size == 1 Layer(name=name, type=LayerType.LAMBDA_COST, inputs=[input.name, score.name], NDCG_num=NDCG_num, - max_sort_size=max_sort_size, - coeff=coeff, - ) + max_sort_size=max_sort_size + ) return LayerOutput(name, LayerType.LAMBDA_COST, parents=[input, score]) + @wrap_name_default() def cross_entropy(input, label, name=None, coeff=1.0): """ @@ -3430,9 +3589,10 @@ def cross_entropy(input, label, name=None, coeff=1.0): type=LayerType.CROSS_ENTROPY, inputs=[input.name, label.name], coeff=coeff, - ) + ) return LayerOutput(name, LayerType.CROSS_ENTROPY, parents=[input, label]) + @wrap_name_default() def cross_entropy_with_selfnorm(input, label, name=None, coeff=1.0, softmax_selfnorm_alpha=0.1): @@ -3463,12 +3623,13 @@ def cross_entropy_with_selfnorm(input, label, name=None, coeff=1.0, inputs=[input.name, label.name], coeff=coeff, softmax_selfnorm_alpha=softmax_selfnorm_alpha, - ) + ) return LayerOutput(name, LayerType.CROSS_ENTROPY_WITH_SELFNORM, parents=[input, label]) + @wrap_name_default() def huber_cost(input, label, name=None, coeff=1.0): """ @@ -3482,8 +3643,6 @@ def huber_cost(input, label, name=None, coeff=1.0): :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param type: The type of cost. - :type type: basestring. :param name: The name of this layers. It is not necessary. :type name: None|basestring. :param coeff: The coefficient affects the gradient in the backward. @@ -3491,14 +3650,17 @@ def huber_cost(input, label, name=None, coeff=1.0): :return: LayerOutput object. :rtype: LayerOutput. """ - + assert isinstance(input, LayerOutput) + if input.size is not None: + assert input.size == 1 Layer(name=name, type=LayerType.HUBER, inputs=[input.name, label.name], coeff=coeff, - ) + ) return LayerOutput(name, LayerType.HUBER, parents=[input, label]) + @wrap_name_default() def multi_binary_label_cross_entropy(input, label, name=None, coeff=1.0): """ @@ -3522,15 +3684,16 @@ def multi_binary_label_cross_entropy(input, label, name=None, coeff=1.0): :rtype: LayerOutput """ - if not isinstance(input.act, SigmoidActivation): + if input.activation is None or \ + not isinstance(input.activation, SigmoidActivation): logger.log(logging.WARN, "%s is not recommend for batch normalization's activation, " - "maybe the relu is better" % act.name) + "maybe the relu is better" % repr(input.activation)) Layer(name=name, type=LayerType.MULTI_BIN_LABEL_CROSS_ENTROPY, inputs=[input.name, label.name], coeff=coeff, - ) + ) return LayerOutput(name, LayerType.MULTI_BIN_LABEL_CROSS_ENTROPY, parents=[input, label]) diff --git a/python/paddle/trainer_config_helpers/networks.py b/python/paddle/trainer_config_helpers/networks.py index 93c4261cc48c5..e59e93acbe33a 100644 --- a/python/paddle/trainer_config_helpers/networks.py +++ b/python/paddle/trainer_config_helpers/networks.py @@ -616,7 +616,7 @@ def lstmemory_group(input, size=None, name=None, cell states, or hidden states in every time step are accessible to for the user. This is especially useful in attention model. If you do not need to access to the internal states of the lstm, but merely use its outputs, - it is recommanded to use the lstmemory, which is relatively faster than + it is recommended to use the lstmemory, which is relatively faster than lstmemory_group. NOTE: In PaddlePaddle's implementation, the following input-to-hidden @@ -1052,7 +1052,7 @@ def dropout_layer(input, dropout_rate, name=None): layer_attr=ExtraAttr(drop_rate=dropout_rate)) -def outputs(layers): +def outputs(layers, *args): """ Declare the end of network. Currently it will only calculate the input/output order of network. It will calculate the predict network or @@ -1089,9 +1089,12 @@ def __dfs_travel__(layer, if isinstance(layers, LayerOutput): layers = [layers] + if len(args) != 0: + layers.extend(args) + assert len(layers) > 0 if len(layers) != 1: - logger.warning("EndOfNetwork routine try to calculate network's" + logger.warning("`outputs` routine try to calculate network's" " inputs and outputs order. It might not work well." "Please see follow log carefully.") inputs = [] diff --git a/python/paddle/trainer_config_helpers/poolings.py b/python/paddle/trainer_config_helpers/poolings.py index 5e06d82005841..d627daab0c496 100644 --- a/python/paddle/trainer_config_helpers/poolings.py +++ b/python/paddle/trainer_config_helpers/poolings.py @@ -47,9 +47,14 @@ class MaxPooling(BasePoolingType): .. math:: max(samples\\_of\\_a\\_sequence) + + :param output_max_index: True if output sequence max index instead of max + value. None means use default value in proto. + :type output_max_index: bool|None """ - def __init__(self): + def __init__(self, output_max_index=None): BasePoolingType.__init__(self, "max") + self.output_max_index = output_max_index class AvgPooling(BasePoolingType): diff --git a/python/paddle/trainer_config_helpers/tests/CMakeLists.txt b/python/paddle/trainer_config_helpers/tests/CMakeLists.txt index 611fb855a8c9a..cf52b06bfea06 100644 --- a/python/paddle/trainer_config_helpers/tests/CMakeLists.txt +++ b/python/paddle/trainer_config_helpers/tests/CMakeLists.txt @@ -3,3 +3,8 @@ add_test(NAME layers_test COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ python ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/layers_test.py WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) + +add_test(NAME test_layerHelpers + COMMAND + ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh +) diff --git a/python/paddle/trainer_config_helpers/tests/configs/.gitignore b/python/paddle/trainer_config_helpers/tests/configs/.gitignore new file mode 100644 index 0000000000000..52378fe7a4865 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/.gitignore @@ -0,0 +1 @@ +*protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 new file mode 100644 index 0000000000000..29928b6f7b423 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/check.md5 @@ -0,0 +1,17 @@ +7e6919d17562516e9a1d9a88de1fb3b9 img_layers.protostr +a5d9259ff1fd7ca23d0ef090052cb1f2 last_first_seq.protostr +9c038249ec8ff719753a746cdb04c026 layer_activations.protostr +5913f87b39cee3b2701fa158270aca26 projections.protostr +6b39e34beea8dfb782bee9bd3dea9eb5 simple_rnn_layers.protostr +0fc1409600f1a3301da994ab9d28b0bf test_cost_layers.protostr +144bc6d3a509de74115fa623741797ed test_expand_layer.protostr +2378518bdb71e8c6e888b1842923df58 test_fc.protostr +8bb44e1e5072d0c261572307e7672bda test_grumemory_layer.protostr +1f3510672dce7a9ed25317fc58579ac7 test_hsigmoid.protostr +d350bd91a0dc13e854b1364c3d9339c6 test_lstmemory_layer.protostr +251a948ba41c1071afcd3d9cf9c233f7 test_ntm_layers.protostr +e6ff04e70aea27c7b06d808cc49c9497 test_print_layer.protostr +2a75dd33b640c49a8821c2da6e574577 test_rnn_group.protostr +67d6fde3afb54f389d0ce4ff14726fe1 test_sequence_pooling.protostr +f586a548ef4350ba1ed47a81859a64cb unused_layers.protostr +8122477f4f65244580cec09edc590041 util_layers.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh new file mode 100755 index 0000000000000..fc2acbd41ed90 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +cd `dirname $0` +export PYTHONPATH=$PWD/../../../../ + +configs=(test_fc layer_activations projections test_print_layer +test_sequence_pooling test_lstmemory_layer test_grumemory_layer +last_first_seq test_expand_layer test_ntm_layers test_hsigmoid +img_layers util_layers simple_rnn_layers unused_layers test_cost_layers +test_rnn_group) + + +for conf in ${configs[*]} +do + echo "Generating " $conf + python -m paddle.utils.dump_config $conf.py > $conf.protostr +done diff --git a/python/paddle/trainer_config_helpers/tests/configs/img_layers.py b/python/paddle/trainer_config_helpers/tests/configs/img_layers.py new file mode 100644 index 0000000000000..6c8ba8be846e5 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/img_layers.py @@ -0,0 +1,20 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-3, + batch_size=1000 +) + +img = data_layer(name='image', size=256*256) + +img_conv = img_conv_layer(input=img, num_channels=1, num_filters=64, + filter_size=(32, 64), padding=(1, 0), stride=(1, 1), + act=LinearActivation()) +img_bn = batch_norm_layer(input=img_conv, act=ReluActivation()) + +img_norm = img_cmrnorm_layer(input=img_bn, size=32) + +img_pool = img_pool_layer(input=img_conv, pool_size=32, pool_type=MaxPooling()) + + +outputs(img_pool, img_norm) \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py new file mode 100644 index 0000000000000..d54a1c49fd3fd --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.py @@ -0,0 +1,26 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +din = data_layer(name='data', size=30) + +seq_op = [ + first_seq, + last_seq +] + +agg_level = [ + AggregateLevel.EACH_SEQUENCE, + AggregateLevel.EACH_TIMESTEP +] + +opts = [] + +for op in seq_op: + for al in agg_level: + opts.append(op(input=din, agg_level=al)) + +outputs(opts) \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py b/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py new file mode 100644 index 0000000000000..ba10dc78e1e3b --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/layer_activations.py @@ -0,0 +1,21 @@ +''' +Test all activations. +''' + +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +din = data_layer(name='input', size=100) + +acts = [ + TanhActivation, SigmoidActivation, SoftmaxActivation, IdentityActivation, + LinearActivation, ExpActivation, ReluActivation, BReluActivation, + SoftReluActivation, STanhActivation, AbsActivation, SquareActivation] + +outputs( + [fc_layer(input=din, size=100, act=act(), name="layer_%d" % i) for i, act in + enumerate(acts)]) diff --git a/python/paddle/trainer_config_helpers/tests/configs/projections.py b/python/paddle/trainer_config_helpers/tests/configs/projections.py new file mode 100644 index 0000000000000..4066c5bc6e0f0 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/projections.py @@ -0,0 +1,47 @@ +''' +Test mixed layer, projections and operators. +''' +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-4 +) + +din = data_layer(name='test', size=100) + +din = embedding_layer(input=din, size=256) + +with mixed_layer(size=100) as m1: + m1 += full_matrix_projection(input=din) + +with mixed_layer(size=100) as m2: + m2 += table_projection(input=m1) + +with mixed_layer(size=100) as m3: + m3 += identity_projection(input=m2) + +with mixed_layer(size=100) as m4: + m4 += dotmul_projection(input=m3) + +with mixed_layer() as m5: + m5 += context_projection(input=m4, context_len=3) + +with mixed_layer() as m6: + m6 += dotmul_operator(a=m3, b=m4) + +img = data_layer(name='img', size=32*32) +flt = data_layer(name='filter', size=3*3*1*64) + +with mixed_layer() as m7: + m7 += conv_operator(img=img, filter=flt, num_filters=64, + num_channel=1, filter_size=3) + +end = mixed_layer(input=[full_matrix_projection(input=m5), + trans_full_matrix_projection(input=m6), + full_matrix_projection(input=m7)], + size=100, + layer_attr=ExtraAttr(drop_rate=0.5, + error_clipping_threshold=40)) + +outputs(end) diff --git a/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh b/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh new file mode 100755 index 0000000000000..78114ce32b019 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cd `dirname $0` +set -e +./generate_protostr.sh +md5sum -c check.md5 diff --git a/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py b/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py new file mode 100644 index 0000000000000..87c2a85cf92dd --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.py @@ -0,0 +1,36 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-4 +) + +din = data_layer(name='data', size=200) + +hidden = fc_layer(input=din, size=200, act=SigmoidActivation()) + +rnn = recurrent_layer(input=hidden, act=SigmoidActivation()) + +rnn2 = recurrent_layer(input=hidden, act=SigmoidActivation(), reverse=True) + +lstm1_param = fc_layer(input=hidden, size=200*4, act=LinearActivation(), + bias_attr=False) + +lstm1 = lstmemory(input=lstm1_param, act=SigmoidActivation()) + +lstm2_param = fc_layer(input=hidden, size=200*4, act=LinearActivation(), + bias_attr=False) + +lstm2 = lstmemory(input=lstm2_param, act=SigmoidActivation(), reverse=True) + +gru1_param = fc_layer(input=hidden, size=200*3, act=LinearActivation(), + bias_attr=False) +gru1 = grumemory(input=gru1_param, act=SigmoidActivation()) + +gru2_param = fc_layer(input=hidden, size=200*3, act=LinearActivation(), + bias_attr=False) +gru2 = grumemory(input=gru2_param, act=SigmoidActivation(), reverse=True) + +outputs(last_seq(input=rnn), first_seq(input=rnn2), + last_seq(input=lstm1), first_seq(input=lstm2), + last_seq(input=gru1), first_seq(gru2)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py new file mode 100644 index 0000000000000..64b45f4ded10b --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py @@ -0,0 +1,26 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +seq_in = data_layer(name='input', size=200) +labels = data_layer(name='labels', size=5000) + +probs = data_layer(name='probs', size=10) +xe_label = data_layer(name='xe-label', size=10) + +outputs(ctc_layer(input=seq_in, label=labels), + crf_layer(input=fc_layer(input=seq_in, size=4), + label=data_layer(name='crf_label', size=4)), + rank_cost(left=data_layer(name='left', size=1), + right=data_layer(name='right', size=1), + label=data_layer(name='label', size=1)), + lambda_cost(input=data_layer(name='list_feature', size=100), + score=data_layer(name='list_scores', size=1)), + cross_entropy(input=probs, label=xe_label), + cross_entropy_with_selfnorm(input=probs, label=xe_label), + huber_cost(input=data_layer(name='huber_probs', size=1), + label=data_layer(name='huber_label', size=1)), + multi_binary_label_cross_entropy(input=probs, label=xe_label)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py new file mode 100644 index 0000000000000..d9c841ab277e1 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.py @@ -0,0 +1,14 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +din = data_layer(name='data', size=30) +data_seq = data_layer(name='data_seq', size=30) + +outputs(expand_layer(input=din, expand_as=data_seq, + expand_level=ExpandLevel.FROM_SEQUENCE), + expand_layer(input=din, expand_as=data_seq, + expand_level=ExpandLevel.FROM_TIMESTEP)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_fc.py b/python/paddle/trainer_config_helpers/tests/configs/test_fc.py new file mode 100644 index 0000000000000..a6d033f291d2c --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_fc.py @@ -0,0 +1,20 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +din = data_layer(name='data', size=100) + +trans = trans_layer(input=din) + +hidden = fc_layer(input=trans, size=100, + bias_attr=False) + +mask = data_layer(name='mask', size=100) + +hidden_sel = selective_fc_layer(input=din, select=mask, size=100, + act=SigmoidActivation()) + +outputs(hidden, hidden_sel) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py new file mode 100644 index 0000000000000..8d9fd9df5179c --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.py @@ -0,0 +1,11 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-4 +) + +din = data_layer(name='data', size=120) + +outputs(grumemory(input=din, size=40, reverse=True, gate_act=TanhActivation(), + act=SigmoidActivation())) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py b/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py new file mode 100644 index 0000000000000..46069074ded56 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.py @@ -0,0 +1,11 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +din = data_layer(name='data', size=100) +label = data_layer(name='label', size=10) + +outputs(hsigmoid(input=din, label=label, num_classes=10)) \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py new file mode 100644 index 0000000000000..56304addb17b2 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.py @@ -0,0 +1,11 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +din = data_layer(name='data', size=128) + +outputs(lstmemory(input=din, reverse=True, gate_act=TanhActivation(), + act=TanhActivation(), size=32)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py new file mode 100644 index 0000000000000..4d8e1fdc6b598 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.py @@ -0,0 +1,23 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +weight = data_layer(name='w', size=1) +a = data_layer(name='a', size=100) +b = data_layer(name='b', size=100) +c = data_layer(name='c', size=200) +d = data_layer(name='d', size=31) + +outputs(interpolation_layer(input=[a, b], weight=weight), + power_layer(input=a, weight=weight), + scaling_layer(input=a, weight=weight), + cos_sim(a=a, b=b), + cos_sim(a=a, b=c, size=2), + sum_to_one_norm_layer(input=a), + conv_shift_layer(a=a, b=d), + tensor_layer(a=a, b=b, size=1000), + slope_intercept_layer(input=a, slope=0.7, intercept=0.9), + linear_comb_layer(weights=b, vectors=c)) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py new file mode 100644 index 0000000000000..f6b2661c7b9e8 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.py @@ -0,0 +1,12 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +din = data_layer(name='input', size=100) + +print_layer(input=din) + +outputs(din) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py new file mode 100644 index 0000000000000..53f5c5d2499f9 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.py @@ -0,0 +1,35 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +seq = data_layer(name='seq_input', size=100) +sub_seq = data_layer(name='sub_seq_input', size=100) +lbl = data_layer(name='label', size=1) + + +def generate_rnn_simple(name): + def rnn_simple(s): + m = memory(name=name, size=200) + fc = fc_layer(input=[s, m], size=200, name=name) + return fc + + return rnn_simple + + +with mixed_layer() as lstm_param: # test lstm unit, rnn group + lstm_param += full_matrix_projection(input=seq, size=100 * 4) + +with mixed_layer() as gru_param: + gru_param += full_matrix_projection(input=seq, size=100 * 3) + +outputs(last_seq(input=recurrent_group(step=generate_rnn_simple('rnn_forward'), + input=seq)), + first_seq(input=recurrent_group(step=generate_rnn_simple('rnn_back'), + input=seq, reverse=True)), + last_seq(input=recurrent_group(step=generate_rnn_simple( + 'rnn_subseq_forward'), input=SubsequenceInput(input=sub_seq))), + last_seq(input=lstmemory_group(input=lstm_param, size=100)), + last_seq(input=gru_group(input=gru_param, size=100))) diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py new file mode 100644 index 0000000000000..2e24164b5578c --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.py @@ -0,0 +1,30 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +din = data_layer(name='dat_in', size=100) + +POOL_TYPE = [ + MaxPooling, + AvgPooling, + SumPooling +] + +AGG_LEVEL = [ + AggregateLevel.EACH_SEQUENCE, + AggregateLevel.EACH_TIMESTEP +] + +opts = [] + +for pt in POOL_TYPE: + for al in AGG_LEVEL: + opts.append(pooling_layer(input=din, agg_level=al, pooling_type=pt())) + +opts.append(pooling_layer(input=din, + pooling_type=MaxPooling(output_max_index=True))) + +outputs(opts) diff --git a/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py b/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py new file mode 100644 index 0000000000000..a6a3d09a4315a --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/unused_layers.py @@ -0,0 +1,14 @@ +from paddle.trainer_config_helpers import * +settings( + batch_size=1000, + learning_rate=1e-4 +) + +probs = data_layer(name='probs', size=100) + +outputs( + sampling_id_layer(input=probs), # It seems not support training + + # It seems this layer is not correct, and should be rewrite. + # block_expand_layer(input=probs, channel=1, block_x=1, block_y=3), +) \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/configs/util_layers.py b/python/paddle/trainer_config_helpers/tests/configs/util_layers.py new file mode 100644 index 0000000000000..aadb3f3f5e799 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/util_layers.py @@ -0,0 +1,15 @@ +from paddle.trainer_config_helpers import * + +settings(learning_rate=1e-4, batch_size=1000) + +a = data_layer(name='a', size=10) +b = data_layer(name='b', size=10) + +result = addto_layer(input=[a, b]) +concat1 = concat_layer(input=[a, b]) +concat2 = concat_layer(input=[ + identity_projection(input=a), + identity_projection(input=b) +]) + +outputs(result, concat1, concat2) \ No newline at end of file diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py index cd368d6b12cbd..faaab9107d8fb 100644 --- a/python/paddle/trainer_config_helpers/tests/layers_test_config.py +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -47,7 +47,7 @@ outputs(classification_cost(out, data_layer(name="label", size=num_classes))) -dotmul = mixed_layer(input=[dotmul_operator(x=x1, y=y1), +dotmul = mixed_layer(input=[dotmul_operator(a=x1, b=x1), dotmul_projection(input=y1)]) proj_with_attr_init = mixed_layer(input=full_matrix_projection(input=y1, From a8df411192f9831a2c99916dbd88db409dbcf01b Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 28 Sep 2016 15:56:29 +0800 Subject: [PATCH 125/324] Replace random_shuffle using shuffle. * reduce trainer count for unit test on MAC OSX --- paddle/gserver/dataproviders/DataProviderGroup.h | 3 ++- paddle/gserver/dataproviders/ProtoDataProvider.cpp | 3 ++- paddle/math/Matrix.cpp | 3 ++- paddle/trainer/tests/test_CompareSparse.cpp | 6 +++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/paddle/gserver/dataproviders/DataProviderGroup.h b/paddle/gserver/dataproviders/DataProviderGroup.h index decbde6c91758..0689f90f3e7dd 100644 --- a/paddle/gserver/dataproviders/DataProviderGroup.h +++ b/paddle/gserver/dataproviders/DataProviderGroup.h @@ -65,7 +65,8 @@ void DataProviderGroup::reset() { provider_ = nullptr; // shuffle file list - std::random_shuffle(fileList_.begin(), fileList_.end()); + std::shuffle(fileList_.begin(), fileList_.end(), + ThreadLocalRandomEngine::get()); startLoader(); DataProvider::reset(); diff --git a/paddle/gserver/dataproviders/ProtoDataProvider.cpp b/paddle/gserver/dataproviders/ProtoDataProvider.cpp index b0c14c85b2d81..344644755f240 100644 --- a/paddle/gserver/dataproviders/ProtoDataProvider.cpp +++ b/paddle/gserver/dataproviders/ProtoDataProvider.cpp @@ -374,7 +374,8 @@ void ProtoDataProvider::reset() { } void ProtoDataProvider::shuffle() { - std::random_shuffle(shuffledSequenceIds_.begin(), shuffledSequenceIds_.end()); + std::shuffle(shuffledSequenceIds_.begin(), shuffledSequenceIds_.end(), + ThreadLocalRandomEngine::get()); } /* diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 1b7f9ac5dac16..e351bede724ac 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -2514,7 +2514,8 @@ void SharedCpuMatrix::mul(CpuSparseMatrix* a, CpuMatrix* b, real scaleAB, for (int k = 0; k < blockNum_; ++k) { blockSeq.push_back(k); } - std::random_shuffle(blockSeq.begin(), blockSeq.end()); + std::shuffle(blockSeq.begin(), blockSeq.end(), + ThreadLocalRandomEngine::get()); } std::vector& localBufRows = *localBufRows_; int* cols = a->getCols(); diff --git a/paddle/trainer/tests/test_CompareSparse.cpp b/paddle/trainer/tests/test_CompareSparse.cpp index 3070682c0a2ef..ff37d7b364840 100644 --- a/paddle/trainer/tests/test_CompareSparse.cpp +++ b/paddle/trainer/tests/test_CompareSparse.cpp @@ -146,12 +146,12 @@ TEST(compareSparse, remote_cpu) { TEST(compareSparse, cpu10_local_vs_remote) { FLAGS_local = 1; // disable remote sparse update in parameter config std::vector localParameters = - trainerOnePassTest(configFile1, true, 10); + trainerOnePassTest(configFile1, true, 2); FLAGS_local = 0; // will enable remote sparse update FLAGS_ports_num_for_sparse = 5; std::vector remoteParameters = - trainerOnePassTest(configFile1, true, 10); + trainerOnePassTest(configFile1, true, 2); compareValue(localParameters, remoteParameters); } @@ -174,7 +174,7 @@ TEST(compareSparse, multiGradientMachine) { FLAGS_parallel_nn = useGpu; LOG(INFO) << " local=" << local << " useGpu=" << useGpu; - int trainerCount = useGpu ? numGpu : 10; + int trainerCount = useGpu ? numGpu : 2; std::vector parameters = trainerOnePassTest(configFile1, true, trainerCount, useGpu); compareValue(getDenseParameters(), parameters, eps); From 34f1b8802790607f45fdc6b733af23d5d0ef4510 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 28 Sep 2016 16:56:12 +0800 Subject: [PATCH 126/324] Add analystic scripts to documentation --- doc/conf.py.in | 4 ++-- doc/templates/layout.html | 16 ++++++++++++++++ doc_cn/conf.py.in | 4 +--- 3 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 doc/templates/layout.html diff --git a/doc/conf.py.in b/doc/conf.py.in index 8515042747ec3..6c221f598b805 100644 --- a/doc/conf.py.in +++ b/doc/conf.py.in @@ -23,6 +23,8 @@ AutoStructify = transform.AutoStructify # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, '@PROJ_ROOT@/python') +templates_path = ["@PROJ_ROOT@/doc/templates"] + # -- Doxygen Settings breathe_projects = { 'paddle': '@PADDLE_DOXYGEN_OUTPUT@/xml' @@ -66,8 +68,6 @@ extensions = [ autodoc_member_order = 'bysource' -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: diff --git a/doc/templates/layout.html b/doc/templates/layout.html new file mode 100644 index 0000000000000..47329c2a928d0 --- /dev/null +++ b/doc/templates/layout.html @@ -0,0 +1,16 @@ +{# layout.html #} +{# Import the theme's layout. #} +{% extends "!layout.html" %} + + +{%- block extrahead %} + +{% endblock %} diff --git a/doc_cn/conf.py.in b/doc_cn/conf.py.in index e1c63cf9f1125..391f7981eab80 100644 --- a/doc_cn/conf.py.in +++ b/doc_cn/conf.py.in @@ -22,6 +22,7 @@ AutoStructify = transform.AutoStructify # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, '@PROJ_ROOT@/python') +templates_path = ["@PROJ_ROOT@/doc/templates"] # -- General configuration ------------------------------------------------ @@ -51,9 +52,6 @@ table_styling_embed_css = True autodoc_member_order = 'bysource' -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] From 0072ef50bfa5574ad4d084968b8fbbb0380549c6 Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 28 Sep 2016 18:00:39 +0800 Subject: [PATCH 127/324] Fix compile check type failed in linux --- paddle/gserver/tests/test_PyDataProvider2.cpp | 2 +- paddle/utils/tests/test_ThreadBarrier.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/gserver/tests/test_PyDataProvider2.cpp b/paddle/gserver/tests/test_PyDataProvider2.cpp index c5fe31b29187f..e75e53ab7f431 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.cpp +++ b/paddle/gserver/tests/test_PyDataProvider2.cpp @@ -321,7 +321,7 @@ TEST(PyDataProvider2, input_order) { if (!realBatchSize) { break; } - ASSERT_EQ(batch.getStreams().size(), 2); + ASSERT_EQ(batch.getStreams().size(), (size_t)2); for (size_t i = 0; i < realBatchSize; ++i) { ASSERT_EQ(batch.getStream(0).ids->getData()[i], 0); ASSERT_EQ(batch.getStream(1).ids->getData()[i], 1); diff --git a/paddle/utils/tests/test_ThreadBarrier.cpp b/paddle/utils/tests/test_ThreadBarrier.cpp index 241cdda7bd1c9..90bd6c21bc8e5 100644 --- a/paddle/utils/tests/test_ThreadBarrier.cpp +++ b/paddle/utils/tests/test_ThreadBarrier.cpp @@ -32,7 +32,7 @@ void testNormalImpl(size_t thread_num, std::vector threads; threads.reserve(thread_num); - for (int32_t i = 0; i < thread_num; ++i) { + for (size_t i = 0; i < thread_num; ++i) { threads.emplace_back([&thread_num, &mutex, &tids, &barrier, &callback]{ callback(thread_num, mutex, tids, barrier); From eaedef89d0e65a378f1253509c014d4f7d6a197e Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 28 Sep 2016 18:01:25 +0800 Subject: [PATCH 128/324] Update build doc --- doc/build/build_from_source.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index c71ff260f8d0a..6f9e03f2c28f1 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -48,7 +48,7 @@ sudo apt-get install libgflags-dev sudo apt-get install libgtest-dev sudo pip install wheel pushd /usr/src/gtest -cmake .. +cmake . make sudo cp *.a /usr/lib popd @@ -106,19 +106,19 @@ Here are some examples of cmake command with different options: **only cpu** ```bash -cmake -DWITH_GPU=OFF -DWITH_DOC=OFF .. +cmake -DWITH_GPU=OFF -DWITH_DOC=OFF ``` **gpu** ```bash -cmake -DWITH_GPU=ON -DWITH_DOC=OFF .. +cmake -DWITH_GPU=ON -DWITH_DOC=OFF ``` **gpu with doc and swig** ```bash -cmake -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON .. +cmake -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON ``` Finally, you can download source code and build: @@ -232,19 +232,19 @@ Here are some examples of CMake command with different options: **only cpu** ```bash -cmake -DWITH_GPU=OFF -DWITH_DOC=OFF .. +cmake -DWITH_GPU=OFF -DWITH_DOC=OFF ``` **gpu** ```bash -cmake -DWITH_GPU=ON -DWITH_DOC=OFF .. +cmake -DWITH_GPU=ON -DWITH_DOC=OFF ``` **gpu with doc and swig** ```bash -cmake -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON .. +cmake -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON ``` Finally, you can download source code and build: From 4bc4a44ed98bba70664abf0fbdc922415fafa211 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 29 Sep 2016 00:43:22 +0800 Subject: [PATCH 129/324] Check if paddle is installed when unittest. (#134) --- paddle/.set_python_path.sh | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/paddle/.set_python_path.sh b/paddle/.set_python_path.sh index afde3e51db45d..f7019b27f8f02 100755 --- a/paddle/.set_python_path.sh +++ b/paddle/.set_python_path.sh @@ -22,15 +22,21 @@ # It same as PYTHONPATH=${YOUR_PYTHON_PATH}:$PYTHONPATH {exec...} # -PYPATH="" -set -x -while getopts "d:" opt; do - case $opt in - d) - PYPATH=$OPTARG - ;; - esac -done -shift $(($OPTIND - 1)) -export PYTHONPATH=$PYPATH -$@ +if ! python -c "import paddle" >/dev/null 2>/dev/null; then + PYPATH="" + set -x + while getopts "d:" opt; do + case $opt in + d) + PYPATH=$OPTARG + ;; + esac + done + shift $(($OPTIND - 1)) + export PYTHONPATH=$PYPATH + $@ +else + echo "paddle package is already in your PYTHONPATH. But unittest need a clean environment." + echo "Please uninstall paddle package before start unittest. Try to 'pip uninstall paddle'" + exit 1 +fi From baaaa0b09df723abbe6450c255a21f1a3b783d4a Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 29 Sep 2016 00:51:05 +0800 Subject: [PATCH 130/324] Add NOAVX Docker image scripts. (#129) * Add noavx to docker * Not to use directory to split Docker image. --- .../docker/{cpu/Dockerfile => Dockerfile.cpu} | 1 + .../Dockerfile => Dockerfile.cpu-demo} | 1 + .../Dockerfile => Dockerfile.cpu-devel} | 1 + paddle/scripts/docker/Dockerfile.cpu-noavx | 11 ++++ .../scripts/docker/Dockerfile.cpu-noavx-demo | 11 ++++ .../scripts/docker/Dockerfile.cpu-noavx-devel | 11 ++++ .../docker/{gpu/Dockerfile => Dockerfile.gpu} | 1 + .../Dockerfile => Dockerfile.gpu-demo} | 1 + .../Dockerfile => Dockerfile.gpu-devel} | 1 + paddle/scripts/docker/Dockerfile.gpu-noavx | 11 ++++ .../scripts/docker/Dockerfile.gpu-noavx-demo | 11 ++++ .../scripts/docker/Dockerfile.gpu-noavx-devel | 11 ++++ paddle/scripts/docker/Dockerfile.m4 | 1 + paddle/scripts/docker/build.sh | 2 +- paddle/scripts/docker/cpu-demo/build.sh | 48 ---------------- paddle/scripts/docker/cpu-devel/build.sh | 48 ---------------- paddle/scripts/docker/cpu/build.sh | 48 ---------------- paddle/scripts/docker/generate.sh | 57 ++++++++++++++----- paddle/scripts/docker/gpu-demo/build.sh | 48 ---------------- paddle/scripts/docker/gpu-devel/build.sh | 48 ---------------- paddle/scripts/docker/gpu/build.sh | 48 ---------------- 21 files changed, 116 insertions(+), 304 deletions(-) rename paddle/scripts/docker/{cpu/Dockerfile => Dockerfile.cpu} (94%) rename paddle/scripts/docker/{cpu-demo/Dockerfile => Dockerfile.cpu-demo} (93%) rename paddle/scripts/docker/{cpu-devel/Dockerfile => Dockerfile.cpu-devel} (94%) create mode 100644 paddle/scripts/docker/Dockerfile.cpu-noavx create mode 100644 paddle/scripts/docker/Dockerfile.cpu-noavx-demo create mode 100644 paddle/scripts/docker/Dockerfile.cpu-noavx-devel rename paddle/scripts/docker/{gpu/Dockerfile => Dockerfile.gpu} (94%) rename paddle/scripts/docker/{gpu-demo/Dockerfile => Dockerfile.gpu-demo} (94%) rename paddle/scripts/docker/{gpu-devel/Dockerfile => Dockerfile.gpu-devel} (94%) create mode 100644 paddle/scripts/docker/Dockerfile.gpu-noavx create mode 100644 paddle/scripts/docker/Dockerfile.gpu-noavx-demo create mode 100644 paddle/scripts/docker/Dockerfile.gpu-noavx-devel delete mode 100644 paddle/scripts/docker/cpu-demo/build.sh delete mode 100644 paddle/scripts/docker/cpu-devel/build.sh delete mode 100644 paddle/scripts/docker/cpu/build.sh delete mode 100644 paddle/scripts/docker/gpu-demo/build.sh delete mode 100644 paddle/scripts/docker/gpu-devel/build.sh delete mode 100644 paddle/scripts/docker/gpu/build.sh diff --git a/paddle/scripts/docker/cpu/Dockerfile b/paddle/scripts/docker/Dockerfile.cpu similarity index 94% rename from paddle/scripts/docker/cpu/Dockerfile rename to paddle/scripts/docker/Dockerfile.cpu index 119154200a135..3aa8cb1a3a869 100644 --- a/paddle/scripts/docker/cpu/Dockerfile +++ b/paddle/scripts/docker/Dockerfile.cpu @@ -7,4 +7,5 @@ ENV WITH_DEMO=OFF ENV PIP_INSTALL_ARGS "" ENV PIP_GENERAL_ARGS "" ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=ON RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/cpu-demo/Dockerfile b/paddle/scripts/docker/Dockerfile.cpu-demo similarity index 93% rename from paddle/scripts/docker/cpu-demo/Dockerfile rename to paddle/scripts/docker/Dockerfile.cpu-demo index b2291203829d2..22c0b9e701bfc 100644 --- a/paddle/scripts/docker/cpu-demo/Dockerfile +++ b/paddle/scripts/docker/Dockerfile.cpu-demo @@ -7,4 +7,5 @@ ENV WITH_DEMO=ON ENV PIP_INSTALL_ARGS "" ENV PIP_GENERAL_ARGS "" ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=ON RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/cpu-devel/Dockerfile b/paddle/scripts/docker/Dockerfile.cpu-devel similarity index 94% rename from paddle/scripts/docker/cpu-devel/Dockerfile rename to paddle/scripts/docker/Dockerfile.cpu-devel index 1bfa202d0c574..b40f3c0a30ba3 100644 --- a/paddle/scripts/docker/cpu-devel/Dockerfile +++ b/paddle/scripts/docker/Dockerfile.cpu-devel @@ -7,4 +7,5 @@ ENV WITH_DEMO=OFF ENV PIP_INSTALL_ARGS "" ENV PIP_GENERAL_ARGS "" ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=ON RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx b/paddle/scripts/docker/Dockerfile.cpu-noavx new file mode 100644 index 0000000000000..5cb5ac7dc4e68 --- /dev/null +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx @@ -0,0 +1,11 @@ +FROM ubuntu:14.04 +MAINTAINER PaddlePaddle Dev Team +COPY build.sh /root/ +ENV WITH_GPU=OFF +ENV IS_DEVEL=OFF +ENV WITH_DEMO=OFF +ENV PIP_INSTALL_ARGS "" +ENV PIP_GENERAL_ARGS "" +ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=OFF +RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx-demo b/paddle/scripts/docker/Dockerfile.cpu-noavx-demo new file mode 100644 index 0000000000000..bec401960efb2 --- /dev/null +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx-demo @@ -0,0 +1,11 @@ +FROM ubuntu:14.04 +MAINTAINER PaddlePaddle Dev Team +COPY build.sh /root/ +ENV WITH_GPU=OFF +ENV IS_DEVEL=ON +ENV WITH_DEMO=ON +ENV PIP_INSTALL_ARGS "" +ENV PIP_GENERAL_ARGS "" +ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=OFF +RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx-devel b/paddle/scripts/docker/Dockerfile.cpu-noavx-devel new file mode 100644 index 0000000000000..b7c3eaed97aa5 --- /dev/null +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx-devel @@ -0,0 +1,11 @@ +FROM ubuntu:14.04 +MAINTAINER PaddlePaddle Dev Team +COPY build.sh /root/ +ENV WITH_GPU=OFF +ENV IS_DEVEL=ON +ENV WITH_DEMO=OFF +ENV PIP_INSTALL_ARGS "" +ENV PIP_GENERAL_ARGS "" +ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=OFF +RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/gpu/Dockerfile b/paddle/scripts/docker/Dockerfile.gpu similarity index 94% rename from paddle/scripts/docker/gpu/Dockerfile rename to paddle/scripts/docker/Dockerfile.gpu index 62d6f1f98769b..b7f5b6d93df50 100644 --- a/paddle/scripts/docker/gpu/Dockerfile +++ b/paddle/scripts/docker/Dockerfile.gpu @@ -7,4 +7,5 @@ ENV WITH_DEMO=OFF ENV PIP_INSTALL_ARGS "" ENV PIP_GENERAL_ARGS "" ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=ON RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/gpu-demo/Dockerfile b/paddle/scripts/docker/Dockerfile.gpu-demo similarity index 94% rename from paddle/scripts/docker/gpu-demo/Dockerfile rename to paddle/scripts/docker/Dockerfile.gpu-demo index f3b8cd568db60..2d1411de09f2a 100644 --- a/paddle/scripts/docker/gpu-demo/Dockerfile +++ b/paddle/scripts/docker/Dockerfile.gpu-demo @@ -7,4 +7,5 @@ ENV WITH_DEMO=ON ENV PIP_INSTALL_ARGS "" ENV PIP_GENERAL_ARGS "" ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=ON RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/gpu-devel/Dockerfile b/paddle/scripts/docker/Dockerfile.gpu-devel similarity index 94% rename from paddle/scripts/docker/gpu-devel/Dockerfile rename to paddle/scripts/docker/Dockerfile.gpu-devel index 2e600f34d03ed..eb13f4304fa06 100644 --- a/paddle/scripts/docker/gpu-devel/Dockerfile +++ b/paddle/scripts/docker/Dockerfile.gpu-devel @@ -7,4 +7,5 @@ ENV WITH_DEMO=OFF ENV PIP_INSTALL_ARGS "" ENV PIP_GENERAL_ARGS "" ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=ON RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx b/paddle/scripts/docker/Dockerfile.gpu-noavx new file mode 100644 index 0000000000000..0944b0e152af3 --- /dev/null +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx @@ -0,0 +1,11 @@ +FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 +MAINTAINER PaddlePaddle Dev Team +COPY build.sh /root/ +ENV WITH_GPU=ON +ENV IS_DEVEL=OFF +ENV WITH_DEMO=OFF +ENV PIP_INSTALL_ARGS "" +ENV PIP_GENERAL_ARGS "" +ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=OFF +RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx-demo b/paddle/scripts/docker/Dockerfile.gpu-noavx-demo new file mode 100644 index 0000000000000..2da2a55d696a3 --- /dev/null +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx-demo @@ -0,0 +1,11 @@ +FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 +MAINTAINER PaddlePaddle Dev Team +COPY build.sh /root/ +ENV WITH_GPU=ON +ENV IS_DEVEL=ON +ENV WITH_DEMO=ON +ENV PIP_INSTALL_ARGS "" +ENV PIP_GENERAL_ARGS "" +ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=OFF +RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx-devel b/paddle/scripts/docker/Dockerfile.gpu-noavx-devel new file mode 100644 index 0000000000000..9f551462f206a --- /dev/null +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx-devel @@ -0,0 +1,11 @@ +FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 +MAINTAINER PaddlePaddle Dev Team +COPY build.sh /root/ +ENV WITH_GPU=ON +ENV IS_DEVEL=ON +ENV WITH_DEMO=OFF +ENV PIP_INSTALL_ARGS "" +ENV PIP_GENERAL_ARGS "" +ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=OFF +RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.m4 b/paddle/scripts/docker/Dockerfile.m4 index 89a1147103ce3..129d21b36abd9 100644 --- a/paddle/scripts/docker/Dockerfile.m4 +++ b/paddle/scripts/docker/Dockerfile.m4 @@ -7,4 +7,5 @@ ENV WITH_DEMO=PADDLE_WITH_DEMO ENV PIP_INSTALL_ARGS "" ENV PIP_GENERAL_ARGS "" ENV USE_UBUNTU_MIRROR OFF +ENV WITH_AVX=PADDLE_WITH_AVX RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 1f74e1f1af22a..33689e736cda7 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -26,7 +26,7 @@ cd paddle mkdir build cd build cmake .. -DWITH_DOC=OFF -DWITH_GPU=${WITH_GPU} -DWITH_SWIG_PY=ON\ - -DCUDNN_ROOT=/usr/ -DWITH_STYLE_CHECK=OFF + -DCUDNN_ROOT=/usr/ -DWITH_STYLE_CHECK=OFF -DWITH_AVX=${WITH_AVX} make -j `nproc` # because durning make install, there are several warning, so set +e, do not cause abort make install diff --git a/paddle/scripts/docker/cpu-demo/build.sh b/paddle/scripts/docker/cpu-demo/build.sh deleted file mode 100644 index 1f74e1f1af22a..0000000000000 --- a/paddle/scripts/docker/cpu-demo/build.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -function abort(){ - echo "An error occurred. Exiting..." 1>&2 - exit 1 -} - -trap 'abort' 0 -set -e -if [ ${USE_UBUNTU_MIRROR} == "ON" ]; then - sed -i 's#http://archive\.ubuntu\.com/ubuntu/#mirror://mirrors\.ubuntu\.com/mirrors\.txt#g'\ - /etc/apt/sources.list -fi -apt-get update -apt-get install -y cmake libprotobuf-dev protobuf-compiler git \ - libgoogle-glog-dev libgflags-dev libatlas-dev libatlas3-base g++ m4 python-pip\ - python-protobuf python-numpy python-dev swig - -if [ ${WITH_GPU} == 'ON' ]; then - ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so -fi - -cd ~ -git clone https://github.com/baidu/Paddle.git paddle -cd paddle -mkdir build -cd build -cmake .. -DWITH_DOC=OFF -DWITH_GPU=${WITH_GPU} -DWITH_SWIG_PY=ON\ - -DCUDNN_ROOT=/usr/ -DWITH_STYLE_CHECK=OFF -make -j `nproc` -# because durning make install, there are several warning, so set +e, do not cause abort -make install -echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile -pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} /usr/local/opt/paddle/share/wheels/*.whl -paddle version # print version after build - -if [ ${WITH_DEMO} == "ON" ]; then - apt-get install -y wget unzip perl python-matplotlib tar xz-utils bzip2 gzip coreutils\ - sed grep graphviz libjpeg-dev zlib1g-dev - pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} BeautifulSoup docopt \ - PyYAML pillow -fi -if [ ${IS_DEVEL} == "OFF" ]; then # clean build packages. - cd ~ - rm -rf paddle -fi -apt-get clean -y -trap : 0 diff --git a/paddle/scripts/docker/cpu-devel/build.sh b/paddle/scripts/docker/cpu-devel/build.sh deleted file mode 100644 index 1f74e1f1af22a..0000000000000 --- a/paddle/scripts/docker/cpu-devel/build.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -function abort(){ - echo "An error occurred. Exiting..." 1>&2 - exit 1 -} - -trap 'abort' 0 -set -e -if [ ${USE_UBUNTU_MIRROR} == "ON" ]; then - sed -i 's#http://archive\.ubuntu\.com/ubuntu/#mirror://mirrors\.ubuntu\.com/mirrors\.txt#g'\ - /etc/apt/sources.list -fi -apt-get update -apt-get install -y cmake libprotobuf-dev protobuf-compiler git \ - libgoogle-glog-dev libgflags-dev libatlas-dev libatlas3-base g++ m4 python-pip\ - python-protobuf python-numpy python-dev swig - -if [ ${WITH_GPU} == 'ON' ]; then - ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so -fi - -cd ~ -git clone https://github.com/baidu/Paddle.git paddle -cd paddle -mkdir build -cd build -cmake .. -DWITH_DOC=OFF -DWITH_GPU=${WITH_GPU} -DWITH_SWIG_PY=ON\ - -DCUDNN_ROOT=/usr/ -DWITH_STYLE_CHECK=OFF -make -j `nproc` -# because durning make install, there are several warning, so set +e, do not cause abort -make install -echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile -pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} /usr/local/opt/paddle/share/wheels/*.whl -paddle version # print version after build - -if [ ${WITH_DEMO} == "ON" ]; then - apt-get install -y wget unzip perl python-matplotlib tar xz-utils bzip2 gzip coreutils\ - sed grep graphviz libjpeg-dev zlib1g-dev - pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} BeautifulSoup docopt \ - PyYAML pillow -fi -if [ ${IS_DEVEL} == "OFF" ]; then # clean build packages. - cd ~ - rm -rf paddle -fi -apt-get clean -y -trap : 0 diff --git a/paddle/scripts/docker/cpu/build.sh b/paddle/scripts/docker/cpu/build.sh deleted file mode 100644 index 1f74e1f1af22a..0000000000000 --- a/paddle/scripts/docker/cpu/build.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -function abort(){ - echo "An error occurred. Exiting..." 1>&2 - exit 1 -} - -trap 'abort' 0 -set -e -if [ ${USE_UBUNTU_MIRROR} == "ON" ]; then - sed -i 's#http://archive\.ubuntu\.com/ubuntu/#mirror://mirrors\.ubuntu\.com/mirrors\.txt#g'\ - /etc/apt/sources.list -fi -apt-get update -apt-get install -y cmake libprotobuf-dev protobuf-compiler git \ - libgoogle-glog-dev libgflags-dev libatlas-dev libatlas3-base g++ m4 python-pip\ - python-protobuf python-numpy python-dev swig - -if [ ${WITH_GPU} == 'ON' ]; then - ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so -fi - -cd ~ -git clone https://github.com/baidu/Paddle.git paddle -cd paddle -mkdir build -cd build -cmake .. -DWITH_DOC=OFF -DWITH_GPU=${WITH_GPU} -DWITH_SWIG_PY=ON\ - -DCUDNN_ROOT=/usr/ -DWITH_STYLE_CHECK=OFF -make -j `nproc` -# because durning make install, there are several warning, so set +e, do not cause abort -make install -echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile -pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} /usr/local/opt/paddle/share/wheels/*.whl -paddle version # print version after build - -if [ ${WITH_DEMO} == "ON" ]; then - apt-get install -y wget unzip perl python-matplotlib tar xz-utils bzip2 gzip coreutils\ - sed grep graphviz libjpeg-dev zlib1g-dev - pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} BeautifulSoup docopt \ - PyYAML pillow -fi -if [ ${IS_DEVEL} == "OFF" ]; then # clean build packages. - cd ~ - rm -rf paddle -fi -apt-get clean -y -trap : 0 diff --git a/paddle/scripts/docker/generate.sh b/paddle/scripts/docker/generate.sh index 009c4a8a56558..8a50aefd34955 100644 --- a/paddle/scripts/docker/generate.sh +++ b/paddle/scripts/docker/generate.sh @@ -2,33 +2,60 @@ set -e cd `dirname $0` m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=OFF -DPADDLE_WITH_DEMO=OFF \ - -DPADDLE_BASE_IMAGE=ubuntu:14.04\ - Dockerfile.m4 > cpu/Dockerfile -cp build.sh cpu/ + -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=ON\ + Dockerfile.m4 > Dockerfile.cpu + +m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=OFF -DPADDLE_WITH_DEMO=OFF \ + -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=OFF\ + Dockerfile.m4 > Dockerfile.cpu-noavx m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=OFF \ - -DPADDLE_BASE_IMAGE=ubuntu:14.04\ - Dockerfile.m4 > cpu-devel/Dockerfile -cp build.sh cpu-devel/ + -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=OFF\ + Dockerfile.m4 > Dockerfile.cpu-noavx-devel + +m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=OFF \ + -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=ON\ + Dockerfile.m4 > Dockerfile.cpu-devel + m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=ON \ - -DPADDLE_BASE_IMAGE=ubuntu:14.04\ - Dockerfile.m4 > cpu-demo/Dockerfile -cp build.sh cpu-demo/ + -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=ON\ + Dockerfile.m4 > Dockerfile.cpu-demo + +m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=ON \ + -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=OFF\ + Dockerfile.m4 > Dockerfile.cpu-noavx-demo + + +m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=OFF -DPADDLE_WITH_DEMO=OFF \ + -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ + -DPADDLE_WITH_AVX=ON \ + Dockerfile.m4 > Dockerfile.gpu m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=OFF -DPADDLE_WITH_DEMO=OFF \ -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ - Dockerfile.m4 > gpu/Dockerfile -cp build.sh gpu/ + -DPADDLE_WITH_AVX=OFF \ + Dockerfile.m4 > Dockerfile.gpu-noavx + m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=OFF \ -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ - Dockerfile.m4 > gpu-devel/Dockerfile -cp build.sh gpu-devel/ + -DPADDLE_WITH_AVX=ON \ + Dockerfile.m4 > Dockerfile.gpu-devel + +m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=OFF \ + -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ + -DPADDLE_WITH_AVX=OFF \ + Dockerfile.m4 > Dockerfile.gpu-noavx-devel m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=ON \ -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ - Dockerfile.m4 > gpu-demo/Dockerfile -cp build.sh gpu-demo/ + -DPADDLE_WITH_AVX=ON \ + Dockerfile.m4 > Dockerfile.gpu-demo + +m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=ON \ + -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ + -DPADDLE_WITH_AVX=OFF \ + Dockerfile.m4 > Dockerfile.gpu-noavx-demo diff --git a/paddle/scripts/docker/gpu-demo/build.sh b/paddle/scripts/docker/gpu-demo/build.sh deleted file mode 100644 index 1f74e1f1af22a..0000000000000 --- a/paddle/scripts/docker/gpu-demo/build.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -function abort(){ - echo "An error occurred. Exiting..." 1>&2 - exit 1 -} - -trap 'abort' 0 -set -e -if [ ${USE_UBUNTU_MIRROR} == "ON" ]; then - sed -i 's#http://archive\.ubuntu\.com/ubuntu/#mirror://mirrors\.ubuntu\.com/mirrors\.txt#g'\ - /etc/apt/sources.list -fi -apt-get update -apt-get install -y cmake libprotobuf-dev protobuf-compiler git \ - libgoogle-glog-dev libgflags-dev libatlas-dev libatlas3-base g++ m4 python-pip\ - python-protobuf python-numpy python-dev swig - -if [ ${WITH_GPU} == 'ON' ]; then - ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so -fi - -cd ~ -git clone https://github.com/baidu/Paddle.git paddle -cd paddle -mkdir build -cd build -cmake .. -DWITH_DOC=OFF -DWITH_GPU=${WITH_GPU} -DWITH_SWIG_PY=ON\ - -DCUDNN_ROOT=/usr/ -DWITH_STYLE_CHECK=OFF -make -j `nproc` -# because durning make install, there are several warning, so set +e, do not cause abort -make install -echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile -pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} /usr/local/opt/paddle/share/wheels/*.whl -paddle version # print version after build - -if [ ${WITH_DEMO} == "ON" ]; then - apt-get install -y wget unzip perl python-matplotlib tar xz-utils bzip2 gzip coreutils\ - sed grep graphviz libjpeg-dev zlib1g-dev - pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} BeautifulSoup docopt \ - PyYAML pillow -fi -if [ ${IS_DEVEL} == "OFF" ]; then # clean build packages. - cd ~ - rm -rf paddle -fi -apt-get clean -y -trap : 0 diff --git a/paddle/scripts/docker/gpu-devel/build.sh b/paddle/scripts/docker/gpu-devel/build.sh deleted file mode 100644 index 1f74e1f1af22a..0000000000000 --- a/paddle/scripts/docker/gpu-devel/build.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -function abort(){ - echo "An error occurred. Exiting..." 1>&2 - exit 1 -} - -trap 'abort' 0 -set -e -if [ ${USE_UBUNTU_MIRROR} == "ON" ]; then - sed -i 's#http://archive\.ubuntu\.com/ubuntu/#mirror://mirrors\.ubuntu\.com/mirrors\.txt#g'\ - /etc/apt/sources.list -fi -apt-get update -apt-get install -y cmake libprotobuf-dev protobuf-compiler git \ - libgoogle-glog-dev libgflags-dev libatlas-dev libatlas3-base g++ m4 python-pip\ - python-protobuf python-numpy python-dev swig - -if [ ${WITH_GPU} == 'ON' ]; then - ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so -fi - -cd ~ -git clone https://github.com/baidu/Paddle.git paddle -cd paddle -mkdir build -cd build -cmake .. -DWITH_DOC=OFF -DWITH_GPU=${WITH_GPU} -DWITH_SWIG_PY=ON\ - -DCUDNN_ROOT=/usr/ -DWITH_STYLE_CHECK=OFF -make -j `nproc` -# because durning make install, there are several warning, so set +e, do not cause abort -make install -echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile -pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} /usr/local/opt/paddle/share/wheels/*.whl -paddle version # print version after build - -if [ ${WITH_DEMO} == "ON" ]; then - apt-get install -y wget unzip perl python-matplotlib tar xz-utils bzip2 gzip coreutils\ - sed grep graphviz libjpeg-dev zlib1g-dev - pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} BeautifulSoup docopt \ - PyYAML pillow -fi -if [ ${IS_DEVEL} == "OFF" ]; then # clean build packages. - cd ~ - rm -rf paddle -fi -apt-get clean -y -trap : 0 diff --git a/paddle/scripts/docker/gpu/build.sh b/paddle/scripts/docker/gpu/build.sh deleted file mode 100644 index 1f74e1f1af22a..0000000000000 --- a/paddle/scripts/docker/gpu/build.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -function abort(){ - echo "An error occurred. Exiting..." 1>&2 - exit 1 -} - -trap 'abort' 0 -set -e -if [ ${USE_UBUNTU_MIRROR} == "ON" ]; then - sed -i 's#http://archive\.ubuntu\.com/ubuntu/#mirror://mirrors\.ubuntu\.com/mirrors\.txt#g'\ - /etc/apt/sources.list -fi -apt-get update -apt-get install -y cmake libprotobuf-dev protobuf-compiler git \ - libgoogle-glog-dev libgflags-dev libatlas-dev libatlas3-base g++ m4 python-pip\ - python-protobuf python-numpy python-dev swig - -if [ ${WITH_GPU} == 'ON' ]; then - ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so -fi - -cd ~ -git clone https://github.com/baidu/Paddle.git paddle -cd paddle -mkdir build -cd build -cmake .. -DWITH_DOC=OFF -DWITH_GPU=${WITH_GPU} -DWITH_SWIG_PY=ON\ - -DCUDNN_ROOT=/usr/ -DWITH_STYLE_CHECK=OFF -make -j `nproc` -# because durning make install, there are several warning, so set +e, do not cause abort -make install -echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile -pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} /usr/local/opt/paddle/share/wheels/*.whl -paddle version # print version after build - -if [ ${WITH_DEMO} == "ON" ]; then - apt-get install -y wget unzip perl python-matplotlib tar xz-utils bzip2 gzip coreutils\ - sed grep graphviz libjpeg-dev zlib1g-dev - pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} BeautifulSoup docopt \ - PyYAML pillow -fi -if [ ${IS_DEVEL} == "OFF" ]; then # clean build packages. - cd ~ - rm -rf paddle -fi -apt-get clean -y -trap : 0 From 4615c5172c82b44c60a6b6839f323559bf93448d Mon Sep 17 00:00:00 2001 From: luotao1 Date: Thu, 29 Sep 2016 10:06:27 +0800 Subject: [PATCH 131/324] beam search api and unitest in hierarchical rnn (#122) --- demo/seqToseq/seqToseq_net.py | 11 +-- doc/algorithm/rnn/rnn.rst | 20 +++-- .../tests/rnn_gen_test_model_dir/r1.test.nest | 16 ++++ .../tests/sample_trainer_nest_rnn_gen.conf | 73 +++++++++++++++++++ .../trainer/tests/sample_trainer_rnn_gen.conf | 7 +- .../test_recurrent_machine_generation.cpp | 62 +++++++++++----- .../trainer_config_helpers/evaluators.py | 33 +++++++-- .../paddle/trainer_config_helpers/layers.py | 36 +-------- 8 files changed, 184 insertions(+), 74 deletions(-) create mode 100644 paddle/trainer/tests/rnn_gen_test_model_dir/r1.test.nest create mode 100644 paddle/trainer/tests/sample_trainer_nest_rnn_gen.conf diff --git a/demo/seqToseq/seqToseq_net.py b/demo/seqToseq/seqToseq_net.py index a9c0dd4af600c..2b0c3f34648b0 100644 --- a/demo/seqToseq/seqToseq_net.py +++ b/demo/seqToseq/seqToseq_net.py @@ -171,12 +171,13 @@ def gru_decoder_with_attention(enc_vec, enc_proj, current_word): beam_gen = beam_search(name=decoder_group_name, step=gru_decoder_with_attention, input=group_inputs, - id_input=data_layer(name="sent_id", - size=1), - dict_file=trg_dict_path, bos_id=0, eos_id=1, beam_size=beam_size, - max_length=max_length, - result_file=gen_trans_file) + max_length=max_length) + + seqtext_printer_evaluator(input=beam_gen, + id_input=data_layer(name="sent_id", size=1), + dict_file=trg_dict_path, + result_file=gen_trans_file) outputs(beam_gen) diff --git a/doc/algorithm/rnn/rnn.rst b/doc/algorithm/rnn/rnn.rst index 4753db450b744..0ab75a130686f 100644 --- a/doc/algorithm/rnn/rnn.rst +++ b/doc/algorithm/rnn/rnn.rst @@ -202,14 +202,17 @@ After training the model, we can use it to generate sequences. A common practice * use :code:`GeneratedInput` for trg_embedding. :code:`GeneratedInput` computes the embedding of the generated token at the last time step for the input at the current time step. * use :code:`beam_search` function. This function needs to set: - - :code:`id_input`: the integer ID of the data, used to identify the corresponding output in the generated files. - - :code:`dict_file`: the dictionary file for converting word id to word. - :code:`bos_id`: the start token. Every sentence starts with the start token. - :code:`eos_id`: the end token. Every sentence ends with the end token. - :code:`beam_size`: the beam size used in beam search. - :code:`max_length`: the maximum length of the generated sentences. - - :code:`result_file`: the path of the generation result file. +* use :code:`seqtext_printer_evaluator` to print text according to index matrix and dictionary. This function needs to set: + + - :code:`id_input`: the integer ID of the data, used to identify the corresponding output in the generated files. + - :code:`dict_file`: the dictionary file for converting word id to word. + - :code:`result_file`: the path of the generation result file. + The code is listed below: .. code-block:: python @@ -230,14 +233,15 @@ The code is listed below: beam_gen = beam_search(name=decoder_group_name, step=gru_decoder_with_attention, input=group_inputs, - id_input=data_layer(name="sent_id", - size=1), - dict_file=trg_dict_path, bos_id=0, # Beginnning token. eos_id=1, # End of sentence token. beam_size=beam_size, - max_length=max_length, - result_file=gen_trans_file) + max_length=max_length) + + seqtext_printer_evaluator(input=beam_gen, + id_input=data_layer(name="sent_id", size=1), + dict_file=trg_dict_path, + result_file=gen_trans_file) outputs(beam_gen) diff --git a/paddle/trainer/tests/rnn_gen_test_model_dir/r1.test.nest b/paddle/trainer/tests/rnn_gen_test_model_dir/r1.test.nest new file mode 100644 index 0000000000000..02c7f142a34d8 --- /dev/null +++ b/paddle/trainer/tests/rnn_gen_test_model_dir/r1.test.nest @@ -0,0 +1,16 @@ +0 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + 1 2 3 4 + diff --git a/paddle/trainer/tests/sample_trainer_nest_rnn_gen.conf b/paddle/trainer/tests/sample_trainer_nest_rnn_gen.conf new file mode 100644 index 0000000000000..613fd325e10fb --- /dev/null +++ b/paddle/trainer/tests/sample_trainer_nest_rnn_gen.conf @@ -0,0 +1,73 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from paddle.trainer_config_helpers import * + +settings(batch_size=15, learning_rate=0) + +num_words = 5 +beam_flag = get_config_arg('beam_search', bool, False) + +sent_id = data_layer(name="sent_id", size=1) + +# This layer has no actual use, but only to decide batch_size in generation. +# When generating, at least one Memory in RecurrentLayer MUST have a boot layer. +dummy_data = data_layer(name="dummy_data_input", size=2) + +def outer_step(dummy_data): + + gen_inputs = [StaticInput(input=dummy_data, size=2, is_seq=True), + GeneratedInput(size=num_words, + embedding_name="wordvec", + embedding_size=num_words)] + + def inner_step(dummy_memory, predict_word): + + # simplified RNN for testing + with mixed_layer(size=num_words) as layer: + layer += full_matrix_projection(input=predict_word, + param_attr=ParamAttr(name="transtable")) + + with mixed_layer(size=num_words, act=ExpActivation()) as out: + out += trans_full_matrix_projection(input=layer, + param_attr=ParamAttr(name="wordvec")) + + return out + + beam_gen = beam_search(name="rnn_gen", + step=inner_step, + input=gen_inputs, + bos_id=0, + eos_id=num_words-1, + beam_size=2 if beam_flag else 1, + num_results_per_sample=2 if beam_flag else 1, + max_length=10) + return beam_gen + +beam_gen_concat = recurrent_group(name="rnn_gen_concat", + step=outer_step, + input=[SubsequenceInput(dummy_data)]) + +seqtext_printer_evaluator(input=beam_gen_concat, + id_input=sent_id, + dict_file="./trainer/tests/test_gen_dict.txt", + result_file="./trainer/tests/dump_text.test") +#outputs(beam_gen_concat) +# In this config, as dummy_data_input doesn't work on beam_gen (we can find dummy_memory +# is read-only memory, and isn't used by other layers of step), we show the Inputs and Outputs +# as follows. Note that "__beam_search_predict__" is the default output name of beam_search. +Inputs("sent_id","dummy_data_input") +Outputs("__beam_search_predict__") diff --git a/paddle/trainer/tests/sample_trainer_rnn_gen.conf b/paddle/trainer/tests/sample_trainer_rnn_gen.conf index abb6e9b179326..ec1c12cc896fb 100644 --- a/paddle/trainer/tests/sample_trainer_rnn_gen.conf +++ b/paddle/trainer/tests/sample_trainer_rnn_gen.conf @@ -48,15 +48,16 @@ def step(dummy_memory, predict_word): beam_gen = beam_search(name="rnn_gen", step=step, input=gen_inputs, - id_input=sent_id, - dict_file="./trainer/tests/test_gen_dict.txt", - result_file="./trainer/tests/dump_text.test", bos_id=0, eos_id=num_words-1, beam_size=2 if beam_flag else 1, num_results_per_sample=2 if beam_flag else 1, max_length=10) +seqtext_printer_evaluator(input=beam_gen, + id_input=sent_id, + dict_file="./trainer/tests/test_gen_dict.txt", + result_file="./trainer/tests/dump_text.test") #outputs(beam_gen) # In this config, as dummy_data_input doesn't work on beam_gen (we can find dummy_memory # is read-only memory, and isn't used by other layers of step), we show the Inputs and Outputs diff --git a/paddle/trainer/tests/test_recurrent_machine_generation.cpp b/paddle/trainer/tests/test_recurrent_machine_generation.cpp index cf52c568e5868..fcee318d16e00 100644 --- a/paddle/trainer/tests/test_recurrent_machine_generation.cpp +++ b/paddle/trainer/tests/test_recurrent_machine_generation.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include #include @@ -24,6 +23,8 @@ using namespace paddle; // NOLINT using namespace std; // NOLINT static const string& CONFIG_FILE = "trainer/tests/sample_trainer_rnn_gen.conf"; +static const string& NEST_CONFIG_FILE = + "trainer/tests/sample_trainer_nest_rnn_gen.conf"; static const string& OUTPUT_DIR = "trainer/tests/dump_text.test"; static string modelDir = "trainer/tests/rnn_gen_test_model_dir/t1"; // NOLINT static string expectFile = // NOLINT @@ -50,32 +51,52 @@ void checkOutput(const string& expRetFile) { } } -void prepareInArgs(vector& inArgs, - const size_t batchSize, bool useGpu) { +void prepareInArgs(vector& inArgs, const size_t batchSize, + bool useGpu, bool hasSubseq) { inArgs.clear(); // sentence id Argument sentId; sentId.value = nullptr; - IVector::resizeOrCreate(sentId.ids, batchSize, useGpu); - for (size_t i = 0; i < batchSize; ++i) sentId.ids->setElement(i, i); + if (hasSubseq) { + // as there is only one sequence, there is only one label. + IVector::resizeOrCreate(sentId.ids, 1, useGpu); + sentId.ids->setElement(0, 0); + } else { + // as there is batchSize word, there is batchSize label. + IVector::resizeOrCreate(sentId.ids, batchSize, useGpu); + for (size_t i = 0; i < batchSize; ++i) sentId.ids->setElement(i, i); + } inArgs.emplace_back(sentId); // a dummy layer to decide batch size Argument dummyInput; dummyInput.value = Matrix::create(batchSize, 2, false, useGpu); dummyInput.value->randomizeUniform(); + if (hasSubseq) { + // generate one sequence with batchSize subsequence, + // and each subsequence has only one word. + dummyInput.sequenceStartPositions = ICpuGpuVector::create(2, false); + int* buf = dummyInput.sequenceStartPositions->getMutableData(false); + dummyInput.subSequenceStartPositions = + ICpuGpuVector::create(batchSize + 1, false); + int* subBuf = dummyInput.subSequenceStartPositions->getMutableData(false); + buf[0] = 0; + buf[1] = batchSize; + for (size_t i = 0; i < batchSize + 1; i++) subBuf[i] = i; + } inArgs.emplace_back(dummyInput); } -void testGeneration(bool useGpu, const string& expRetFile) { +void testGeneration(const string& configFile, bool useGpu, bool hasSubseq, + const string& expRetFile) { FLAGS_use_gpu = useGpu; - auto config = std::make_shared(CONFIG_FILE); + auto config = std::make_shared(configFile); unique_ptr gradientMachine(GradientMachine::create(*config)); gradientMachine->loadParameters(modelDir); vector inArgs(2); const size_t batchSize = 15; - prepareInArgs(inArgs, batchSize, useGpu); + prepareInArgs(inArgs, batchSize, useGpu, hasSubseq); vector outArgs; unique_ptr testEvaluator(gradientMachine->makeEvaluator()); testEvaluator->start(); @@ -93,16 +114,21 @@ TEST(RecurrentGradientMachine, test_generation) { #else const auto useGpuConfs = {true, false}; #endif - FLAGS_config_args = "beam_search=0"; // no beam search - string expectRetFileNoBeam = expectFile + ".nobeam"; - for (auto useGpu : useGpuConfs) { - testGeneration(useGpu, expectRetFileNoBeam); - } - FLAGS_config_args = "beam_search=1"; // no beam search - string expectRetFileBeam = expectFile + ".beam"; - for (auto useGpu : useGpuConfs) { - testGeneration(useGpu, expectRetFileBeam); - } + auto testGen = [&](const string& configFile, bool hasSubseq, + const string& expRetFile, bool beam_search) { + FLAGS_config_args = beam_search ? "beam_search=1" : "beam_search=0"; + for (auto useGpu : useGpuConfs) { + testGeneration(configFile, useGpu, hasSubseq, expRetFile); + } + }; + testGen(CONFIG_FILE, false, expectFile + ".nobeam", false); // no beam search + testGen(CONFIG_FILE, false, expectFile + ".beam", true); // beam search + // In hierarchical RNN, beam search and one way search are only in inner-RNN, + // outer-RNN will concat the generated inner-results (first for beam search) + // from inner-RNN. Thus, they have the same outer-results. + testGen(NEST_CONFIG_FILE, true, expectFile + ".nest", + false); // no beam search + testGen(NEST_CONFIG_FILE, true, expectFile + ".nest", true); // beam search } #endif diff --git a/python/paddle/trainer_config_helpers/evaluators.py b/python/paddle/trainer_config_helpers/evaluators.py index 7a00d0b7ec57a..ded124a5c8ca4 100644 --- a/python/paddle/trainer_config_helpers/evaluators.py +++ b/python/paddle/trainer_config_helpers/evaluators.py @@ -559,6 +559,7 @@ def maxframe_printer_evaluator( def seqtext_printer_evaluator( input, result_file, + id_input=None, dict_file=None, delimited=None, name=None, @@ -567,11 +568,10 @@ def seqtext_printer_evaluator( Sequence text printer will print text according to index matrix and a dictionary. There can be multiple input to this layer: - 1. If there is only one input, the input must be a matrix containing + 1. If there is no id_input, the input must be a matrix containing the sequence of indices; - 2. If there are more than one input, the first input should be ids, - and are interpreted as sample ids. + 2. If there is id_input, it should be ids, and interpreted as sample ids. The output format will be: @@ -602,26 +602,43 @@ def seqtext_printer_evaluator( .. code-block:: python - eval = seqtext_printer_evaluator(input, + eval = seqtext_printer_evaluator(input=maxid_layer, + id_input=sample_id, dict_file=dict_file, result_file=result_file) :param input: Input Layer name. :type input: LayerOutput|list - :param dict_file: The input dictionary which contains a list of tokens. - :type dict_file: basestring - :param result_file: The file is to save the results. + :param result_file: Path of the file to store the generated results. :type result_file: basestring + :param id_input: Index of the input sequence, and the specified index will + be prited in the gereated results. This an optional + parameter. + :type id_input: LayerOutput + :param dict_file: Path of dictionary. This is an optional parameter. + Every line is a word in the dictionary with + (line number - 1) as the word index. + If this parameter is set to None, or to an empty string, + only word index are printed in the generated results. + :type dict_file: basestring :param delimited: Whether to use space to separate output tokens. Default is True. No space is added if set to False. :type delimited: bool :param name: Evaluator name. :type name: None|basestring + :return: The seq_text_printer that prints the generated sequence to a file. + :rtype: evaluator """ assert isinstance(result_file, basestring) + if id_input is None: + inputs = [input] + else: + inputs = [id_input, input] + input.parents.append(id_input) + evaluator_base(name=name, type="seq_text_printer", - input=input, + input=inputs, dict_file=dict_file, result_file=result_file, delimited=delimited) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index b28dd02b70946..c355dc042ac18 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -2608,7 +2608,6 @@ def eos_layer(input, eos_id, name=None, layer_attr=None): @wrap_name_default() def beam_search(step, input, bos_id, eos_id, beam_size, - result_file, dict_file="", id_input=None, max_length=500, name=None, num_results_per_sample=None): """ @@ -2632,8 +2631,7 @@ def rnn_step(input): input=[StaticInput(encoder_last)], bos_id=0, eos_id=1, - beam_size=5, - result_file="./generated_sequences.txt") + beam_size=5) Please see the following demo for more details: @@ -2671,24 +2669,12 @@ def rnn_step(input): of the most promising next words. The greater the beam size, the fewer candidate words are pruned. :type beam_size: int - :param result_file: Path of the file to store the generated results. - :type result_file: basestring - :param dict_file: Path of dictionary. This is an optional parameter. - Every line is a word in the dictionary with - (line number - 1) as the word index. - If this parameter is set to None, or to an empty string, - only word index are printed in the generated results. - :type dict_file: basestring :param num_results_per_sample: Number of the generated results per input sequence. This number must always be less than beam size. :type num_results_per_sample: int - :param id_input: Index of the input sequence, and the specified index will - be prited in the gereated results. This an optional - parameter. - :type id_input: LayerOutput - :return: The seq_text_printer that prints the generated sequence to a file. - :rtype: evaluator + :return: The generated word index. + :rtype: LayerOutput """ if num_results_per_sample is None: @@ -2704,7 +2690,6 @@ def rnn_step(input): real_input = [] for i, each_input in enumerate(input): - # print type(each_input) assert isinstance(each_input, StaticInput) or isinstance( each_input, BaseGeneratedInput) if isinstance(each_input, BaseGeneratedInput): @@ -2740,20 +2725,7 @@ def __real_step__(*args): tmp = recurrent_group(step=__real_step__, input=real_input, reverse=False, name=name) - - if id_input is None: - inputs = [tmp.name] - else: - assert isinstance(id_input, LayerOutput) - inputs = [id_input.name, tmp.name] - tmp.parents.append(id_input) - - Evaluator(name='target_printer', - type='seq_text_printer', - dict_file=dict_file, - result_file=result_file, - inputs=inputs - ) + return tmp From 6decbdf4f53cf1733b2ffc8cfd29120010f4264e Mon Sep 17 00:00:00 2001 From: emailweixu Date: Wed, 28 Sep 2016 20:35:11 -0700 Subject: [PATCH 132/324] Fix potential dead lock in PyDataProvider2 (#140) This bug occasionally causes dead lock in test_RecurrentGradientMachine In general, conditional_variable::notify should be used together with mutex for changing condition. --- paddle/gserver/dataproviders/PyDataProvider2.cpp | 13 ++++++++----- paddle/utils/Logging.h | 2 +- paddle/utils/Util.h | 11 +++++++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/paddle/gserver/dataproviders/PyDataProvider2.cpp b/paddle/gserver/dataproviders/PyDataProvider2.cpp index 0b41f6a02aecc..2f9a1223c6e45 100644 --- a/paddle/gserver/dataproviders/PyDataProvider2.cpp +++ b/paddle/gserver/dataproviders/PyDataProvider2.cpp @@ -377,9 +377,15 @@ class PyDataProvider2 : public DataProvider { std::swap(callingContexts_[cid], callingContexts_[0]); cid = 0; } + + PyObjectPtr front; + { + std::unique_lock l(mtx_); + front = pop_get_front(callingContexts_); + } { PyGuard g; - callingContexts_.pop_front(); + front.reset(); } this->pullCV_.notify_all(); continue; @@ -411,10 +417,7 @@ class PyDataProvider2 : public DataProvider { poolActualSize_ += additionalBatchSize; dataPool_.emplace_back(data); } - - { - pullCV_.notify_all(); - } + pullCV_.notify_all(); } DBG << "load thread end"; } diff --git a/paddle/utils/Logging.h b/paddle/utils/Logging.h index 7fdfa3240c1de..b3f439804686f 100644 --- a/paddle/utils/Logging.h +++ b/paddle/utils/Logging.h @@ -191,7 +191,7 @@ void installFailureWriter(void(*callback)(const char*, int)); } #endif // PADDLE_USE_GLOG -#ifndef NDEBUG +#ifdef NDEBUG #define DEBUG_LEVEL 5 #define DBG VLOG(DEBUG_LEVEL) #else diff --git a/paddle/utils/Util.h b/paddle/utils/Util.h index 11a03e141dec5..57839f2e21573 100644 --- a/paddle/utils/Util.h +++ b/paddle/utils/Util.h @@ -112,6 +112,17 @@ static bool contains(const Container& container, const T& val) { return std::find(container.begin(), container.end(), val) != container.end(); } +/** + * pop and get the front element of a container + */ +template +typename Container::value_type pop_get_front(Container& c) { + typename Container::value_type v; + swap(v, c.front()); + c.pop_front(); + return v; +} + #define ARRAYSIZE(a) (sizeof(a) / sizeof(*(a))) /** From 1fc4352754b37f298353b0d0d4b0ddd28b937b01 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Thu, 29 Sep 2016 13:08:51 +0800 Subject: [PATCH 133/324] refine sparse momentum api and unittest (#126) * refine sparse momentum api and unittest * fix unittests bug --- doc/algorithm/rnn/rnn.rst | 1 + .../api/trainer_config_helpers/optimizers.rst | 6 +++ paddle/trainer/tests/CMakeLists.txt | 2 +- paddle/trainer/tests/mnist.list | 1 + paddle/trainer/tests/mnist_bin_part | Bin 0 -> 3861383 bytes .../tests/sample_trainer_config_opt_a.conf | 47 ++++++++---------- .../tests/sample_trainer_config_opt_b.conf | 47 ++++++++---------- .../trainer_config_helpers/optimizers.py | 33 ++++++++++-- 8 files changed, 82 insertions(+), 55 deletions(-) create mode 100644 paddle/trainer/tests/mnist.list create mode 100644 paddle/trainer/tests/mnist_bin_part diff --git a/doc/algorithm/rnn/rnn.rst b/doc/algorithm/rnn/rnn.rst index 0ab75a130686f..343f55a20e464 100644 --- a/doc/algorithm/rnn/rnn.rst +++ b/doc/algorithm/rnn/rnn.rst @@ -142,6 +142,7 @@ We also project the encoder vector to :code:`decoder_size` dimensional space, ge The decoder uses :code:`recurrent_group` to define the recurrent neural network. The step and output functions are defined in :code:`gru_decoder_with_attention`: .. code-block:: python + group_inputs=[StaticInput(input=encoded_vector,is_seq=True), StaticInput(input=encoded_proj,is_seq=True)] trg_embedding = embedding_layer( diff --git a/doc/ui/api/trainer_config_helpers/optimizers.rst b/doc/ui/api/trainer_config_helpers/optimizers.rst index 3c683914f4d14..b487fec64c4eb 100644 --- a/doc/ui/api/trainer_config_helpers/optimizers.rst +++ b/doc/ui/api/trainer_config_helpers/optimizers.rst @@ -4,6 +4,12 @@ BaseSGDOptimizer :members: BaseSGDOptimizer :noindex: +MomentumOptimizer +================= +.. automodule:: paddle.trainer_config_helpers.optimizers + :members: MomentumOptimizer + :noindex: + AdamOptimizer ============= .. automodule:: paddle.trainer_config_helpers.optimizers diff --git a/paddle/trainer/tests/CMakeLists.txt b/paddle/trainer/tests/CMakeLists.txt index aabf44d651200..60c129f4e2386 100644 --- a/paddle/trainer/tests/CMakeLists.txt +++ b/paddle/trainer/tests/CMakeLists.txt @@ -47,7 +47,7 @@ add_test(NAME test_CompareTwoOpts COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ ${CMAKE_CURRENT_BINARY_DIR}/test_CompareTwoOpts --config_file_a=trainer/tests/sample_trainer_config_opt_a.conf --config_file_b=trainer/tests/sample_trainer_config_opt_b.conf - --num_passes=1 --need_high_accuracy=1 + --num_passes=1 --need_high_accuracy=0 WORKING_DIRECTORY ${PROJ_ROOT}/paddle/) ################# test_CompareSparse ################## diff --git a/paddle/trainer/tests/mnist.list b/paddle/trainer/tests/mnist.list new file mode 100644 index 0000000000000..703e87753d5a4 --- /dev/null +++ b/paddle/trainer/tests/mnist.list @@ -0,0 +1 @@ +trainer/tests/mnist_bin_part diff --git a/paddle/trainer/tests/mnist_bin_part b/paddle/trainer/tests/mnist_bin_part new file mode 100644 index 0000000000000000000000000000000000000000..08b93a0ebb5698bdafbc36c3c757918a50bab621 GIT binary patch literal 3861383 zcmeF)54?9tc@XwDnzR99j9N9-)WrYRq}BMZ*z9|6s?kQ9)>KoqZQ90aTU*m6QEP2Y zb%ob-yP>%3uE?%5ECdsXxGaQhR%Br%KnVh&xIqdVia;q@2n{r$%WJ}?^PT&9e(!VV z&N*k!@7~?K@9w#u&$)Acb7tn5d1lU>=l}e|r+?n(U;3ik{<)`r?&tsfFM9e9fAKH* zp)Y>=LtlJp``a+EVc>?xz_;9U&t-#n`CGpB=x3;E(^8$DjP< zw;g}#V}JJebAIM8A3t{Wf2hCj`d|O02A9?S4kKju?Z-2+XbNr(3`qjs`-}x_`f&=FFe)OZ4^+CDs{d@n) zYIu*l``t$$`|zKx^&frozbZJpdi2fr{kFO`@|fIz`1KFn^o}?Gfm-%6KlK-m;k_+0 z_5140moC5Vb+0@6)c^A>$F8@(y!WQpFjSQhaQ?N zn_+76zL$T+@pt}D|9q_rFE4x1ubITlZ7=!tN4Gxr7au?TPyYBG)-#{;oFQ1?)^%23 zfa7}jfB$!nU-9Z!AN}k<{OX&2`ks4kdhQEeaP+o!zVqnb*S_}X2fzP!9DmN$|9)Tg zeUE(O@g2{8_Ao5)?s)bW9LIbGOsO><>pbVXH&@U8>z{Z0%9p;hJFnWE`}^^q{K?DK z#dm(^FF1bFKl-=!{lE0t|FMq8d+@r4zWVs?yZ*&vWA#qXwYTQkw|w$jFF*aK|H6^$ zhd=zMkKXy}-+A=sZ@#n&XKUbXWj73L7%5*S)cK*!sqxkPP95)I zyMEWL%}bZO554sk<9y-weczF*9oNY_+v(I`*5Y`Z`C2=!scy`TH})Iv$W;A9^QXoO z*2g~ZfrD7HL;JGT+c0pWV_;(?ZuA-4`u`Ni07>(ozxn3<^%H;f=9?}GB|m-7mt8I- zLzM=<`5*op$F5KP#D9IlUKCBhKmI@d{o^N8kVaFFtzi3%;bL?tkL1Ub+d%>|PmKk=H-)jAK&ukuet%;ifyD^M%Jh z{*&KYrROv^#yC<}8PZ=Q^wU1`*y1<~j3Z;AwK+tq5Pf>9G2>( zfA`DloZNZKEsOn&dTsi$_q^vaZ9$vg{=WBt2QJeJkX(QHXC52IVcQN;@-v_Qi;M80 z-h*%Wy~mG!^rHul??o^EwMAp_1;g-ezw^$cAAS43ef*01e%or8FTVTkqqo29>yCf& zp|4(rpV!}i|Ir8kum7yTyl)uhgKv1l(T{)Pzd641%9Y_dfm7NB_2Bltd;itr7rpq! zRr?apbNZ7HJ#_hjSAOO3hu`-fp3&OnawGRN6UO-PKmMb~@MM26H~;L>M~__C74Iso z&%F;kWA(VE`tSp{+*0?D`(FO?quX!)B~|l6`$FqvZyDWBrrO`~Hw--0FtD)_Pqi7} z#``3~07<0JZB9wjFKUQBwIrZ(5@C`<#xW=54uc&1ZzZ}*zxVZ4fJM5-c-pj;^iKUY z-Vw0IvnR=41lRO0&sT5$V?FlpLIbr3uNj{{$CcsDX`tZeA~Y}XIX9lr z*cjKEJ%{F|U7t(8jmJJ$wLZ&8JX6BFO}$sf;TcU@%#_closX48HxXf9S+0D3NhUmVDj+ z_kT~7h_WM}YCGqD+ihQT`QH0~ca;d{cG~+OGur(4{kqrQbbPhg>*uz6_tgj{_4(0{ ze{+#S{?0Sk5B|XKI{wV3KYfIJ2^eiULKYT0R;lXUf9qEq^I7TnLqtE>i}RnzaA;fJ z^uvGfT3!^MNy~f{UyT2N1ieeP@>3v9CcO?=AN$CEe*Dp&`YA>oW!~RdS&{l|22MPB z`CHy{Sy@v1$dl4!ef*I}j^H@v&-ws|zetq5`)~fT+K)QU$ec;5;Zz6SU4Q#m?Z*Qi zpNe)JzVan6IeP48zP*;aa^-VR^>+#+c-nj0JHP%ow0-(4&o$)_aoauj)OyO$){RQJ z1D|ku$8EP&kpO?1*T3dpKjvf8j!D0N>|O7wJ~8TCx%pBV_c|>bslU1;^4gtMF(Pi#D~w%;i4u&UaqE`iV~*z3=`1@qSLd zlQgfn+~_^6Ex#dS1k;|AFMchaxx72@PHQpReytf$|*?0fCS)Za_ zyGwcXyHMa_yW-;GvQ%$-zG2|J7}!{e^I~wTzhU4B!GKRGvH>X*mrw2}nKIf*Zl#Pl zxU0q|Hm8gY9{uQ_sVHZ(R=RHI2pe?M=Joc9= znMacEs^stvKJw~q@A@N!r6`6sB@ao=tv32HVt7)GaCYs{Oqi>T7n%Shrz^Au&-#w< zsK2ps++j)g!Q_6O>-O%Et;*^rvbk4gDG&P-V_Yg5@VHtd|CY-4p09z&Ek62YuCQOd zHmCAP@^63MFW$@L#OAJ6Af?#$Lt)MdOkS3aIGE;H{@&-ydg(%uon zq9_+>!Cws*5SlObth36JM4^Qgi)5{4l3F`L3otxlXk@*^f2_yoAC~Rv02|-y!h>&q z^ATgJXlpKIND24$re0g4dFIsIxZvr(-~asQpTYGgjFEEKcJmqR z=}4We%!YyM83P+DaXrt@*0(1S2A=k_KXb%R|M46W67t91@s6V!@n|Q3CDHrt+ixEt zftLD4>W%CUzlk8VTQUGC{@mw3|6rJ;=d=h6iQ2ZOephl}X#nXt7KpY5DJ&9B+b%Nd zvCn?CN@z*@Tp~T4Dwo=uyCsPyT|QN}kS_dANMEs2#YX%TzL02t`B^`IUx(kt9E^B| zjLcf3fGMihFZhXXJ_RG{NI&}6U)qzf>LB&<^WHrl36XyS23z|gy*`4cDHazwclRB? z^u*_;z(K;^`Zs>jF)dXOM$UWfTi?3xqdDV8lApKt)@#Ng_Hm8R!vnAQt^2W61p1~(_vsdVS?|=VMUawe3YhMJUUirDa3=doIwuisr z2##qY43kt?@XLF?=Qm#aMDOgbeL@9uT+`lMS?o%6%Ps9P7-2NP z)|h=bLlAk@kYuirfoF_uw22hZ-rF927+HVHP>3}Dyzm}+)vJyk{?UK0;_X8p`cpL; z(5vqGvMyH}lQMM%Z36ay6V!4;dPfs1D@lkb&(JNLdVHntVmFtA}@5(676F^QV(!G?j0zyMMM$&1W#JzlPNq^8=g zw#?XwbbaUl@(a7<$syy5vE=sMgL@kGA+2AlqtBp{Cx$^W)@yaw+-a+7 zKN7zDmT&H&E%&GU6Hmrc?V10Kx4I@uey@d-F`kTVbp9%C3XFC<>Yw1M2k9j51jY%x zJnpF)C+j$Hf07!{5}ar+aCy>B+nwCyeZ&g)BC`tGW0vdm-r4mj4WIT*pk6zn_~`8|lXBM(e?7@6NG zSUF#xwdc}@_R3|q`wasd2G(O>Vmh85*8SOI|A<=gsJ>obp#u2@~&gD7k>q?O{SLt{;elB2q=*Rw} z>PE`|r%0hsk=N%)nWYrB2M_FUfs1|u6srQLbub0?2+i#8astbxZ+11y>x5_~rP zqpq=na+2Z}-;|;8F_L08&$xy`R`hu^ty1rTOYYGlb-0wnMfG@{OY5 zu_s15+VU>Np2Y>+Mf>NOpv&fLEvy#y(|-2bkGvBM&uICqaqYXXOW+Hn-0o*>JBR+Z z4)|ikb>8tD+K;W?QwIYZEAiBs>usb@I0k%r+v`vl)n4btgZDXR_~FV%1IQA;k>U&) zl#CeOJA+KsCRFP?FSUek7z+kGY1KjfaMD)z)S?M3}M1jbfp!@wK{HdbN|JKM7j0|#S(#7Sh7vb9(8 zRA4}oK-vR&huS!|9pdIkYUh)<|77LQSH5p6ulW6(mw!c#xDs~p0qe1%K|p%bjJRsF zp@6xf{V2*+R#YQ?3jsrl#6bF!E=O4@%@5yz>Ry>2EZk!!M9isD?p=Smt0Rp`Dm|NP zf}Q`|^ZblWfA|N)m`El?Y|7=P?vZUZn$T2%ly9Ee=v&<6k>SQXLRu7Dp13!sc6+yc zfGP{O!yj@JS&E$IdWuBDUi9il9*Lrg}{Qtydw?z6q);fn28xBY|3o11HJ~&I*)gHX|7ZgDD^x4=cn!cW3Ys4j^M+Vz`gZ6RgWJBt%snk5n0N+$zKB-x-_o* zTtp_C5HQUFu}Z!KZH+mqYoe6<(I5H4!+6iI;@h)=!dYDAzxc-25;R~GsA z((nCE$D5`}-HRqS+`v|F!@zZjfsK{84ku;n(v6ORU?s2<`6?3J3;kpu1781{*X*Om z1lbX(&!j+=hvJj8j23{6zl&{kv^|v~XW0Rhf{((*CW-D663Dh~(2ym zr4bo{2WKI@ZQ=M$LjLx*yyZ+=nU&brTL36W)cMZKCXz{{j| z)T{6p>c!(~G*fU8bt8W&?ZHDf*9cNml~Vqha|P0~-fa(_zB*E{S3dIYec%}Y#c7E4i&WlU>>R;?asxD4V{iON!?c4$+v4sjVnj3-j>BhB8a zh6aClhr*S-)pSPtv}sXB|3)-UBj6cu(NNH)<% zNJO8J*OdsAq}S~3{XUQT8>wWRLhYW$;YHu|t84sspY?159QRI4MrO^Ct1VjD+OoHI zMnJUVKHD(+1jn;ps{RxSl;M$1GLY!Ck@J?|DT`dpZTIex_PO2OJxM7;3WgE+B1NsI zHy2pyai#3R$ay5-Js9$kY{-_L+iA4%tXprbtVpyG`IBj9UiiwKkViK!8sum<`#wy< zi3bflWYg7^#uTgIiL$jLLm8o1Khc$uw7i4+`ds0c>f|+V4j99Xo)KVPtBdk(yIZBop2r2H-Bq~rD!JS>)aYC= z#9~w!b$jXD`%dGsb|1xtFe;t$o0x(}3o?iMrS(JeVGp_Wxxcsu`i-$>c=i<9H6o{r z)J`xG3j?Q|hGu4|@7wbY18Xp_u@Y;rwUym4FoXfpe5AvbqQK5qPue}yV5bmL{S~i% z^^s^Otz@x&s*0UANgvnI(a(GbOwD@X_kG_{7=~z@Z^VmU{A;V<0vFM^!j*PSnxds8XcWBd)3e3%PrxNQxth69X$oEAf0T#cIK8H71%XxO*0` z;7uJ8`w=!PH_f(M{hZgkN9INZEM(By6QQHcD`nk|z+*G79F{fe9+K{*@e7stLzrtv z2QCzye2l~n*ZRpqy9c2K5d_tm<}-mcTXQZ9TX>p@6F2F3hyf2y`I&HiUlXRmY2Mx+ z8n1asCr@j+0*k0%jVmxx9{For_Jt{*hqe#=paQ0T@_llI+Z=1a01n#-H#DRpEK&~Coa{KJF1s~0M#U6Nwl$nBwRK!X|n{26lav5v+?2`Ca z!>!#IYq_b$7V>FrkVM+>dH?pm_BmA&+1kQa1F9W763&HYyN_w80+y84p;wSAY~Yk{ z$DOy_(k<7koAWBkZ4DF+%A65UeYP9hv;X?%9W*vJ^i~Z9UegUZorf{#x?zYGH5V#OfAa z(Lk0X_Fh-V2#(*#UaYXP>h}6Hmq#2Bq7Cb-XINy7lBB~QSR4UA|+Kx zTlpNfX$1CkI`wJ*y9(ft+y&Vt7%Y+z3)5bFDUcn?j-E>p{UWVI>Tzi`4sDI}`;O;) z;eOoF)=~;qxOpOH*p@H3v__le6UkG_tH_*4k;o*jA#rCkvereIR66E$0?a-etRVZO zYwfY&g{EnAd@-(=uYE@bZ-QJ_UzmW!YJvBXT6>G-Qvqu!0-s7p>W8LJ{}>R$Pik`V zRs^dGtJH%_@6Tl6^nfdK#lvo#<|6MOajzdp*7&`juTszZxpv-l!F!IK^j^5U!?Esq zsqr~8VeNWO!##EDbb0J&L<7s~*!b3layZra_G81qhJnW&0~;&xxTAOj@YKTqQiV+~ z5+q+FvKNVsd%ov;D)|wgV4qzQH%BhHM-q^{9SW0FkU@`&Uj_*ui6IRLX)qFv{|3n( z;}h^m`)X94l|Yg6L8|LJDd?4A7(lZ#)zpJ;cta)a*_cn2>E)Bg1}g@ZW26M2?Ul`W zH6m6iFNkB#pSu@)G)N<`Tx_@J>cq2vSL;RQOv*xh+AjLJ7%Ph&|Ox5}*;8pF@ zLSY7C+~KV?>q4}nIlqrya|}GpGXuKb^V4}Z*C4D&^^=pYPd{DyMN5EHt?$ZG;qshe zQdxU9d4CUqS2ZPN4n~cQqD$b7h7gE4*btg0F5%dM$RA19jt?yM{npoupZB~~u=HmH zp1HDDx`Yj|-!N`E#88t4$DTw($9Ey)#POaCkg)mn9V=r3YwdoL+Y~IRA*$ASsXk zOzxk1?^r5wNjL?pO#dB4!nLIk>ICF z0FkDUw)U^7zF8aQ%XR0K&#juSx$;x(@N=<;n0G1P99m4S>?aca?>cfkog?1rS}f&G zj=_6qvTswcBA>P_!YowQp|nYB0A~RMTS|+de`~`c1e`W4`{NPqmf+Z#eRoeI(U#wS zZy2~yF|e@`H|or7{k{${Km#DCA7dGc{J2P?4*Bf%hUgPS@z1Ur$sLOMHhtY|Zz{Id zUw5w06|$QFO566-Z=dE&`uF-jmB$C88rBpegJ_@hlJP-%OkFnQS!)au@T*OEf;DxX zM3gs&(XYrGO}SemCtrN`-A4>wNDG$&^qvPEjTClbw13L2fybDFKMbj~K%fkqu88`y z;%^PS@cRk;zx}O$Xcenm-LDJjsQ!riwQ&V3DUy4XHDkOM$2($mM~oVsOq}7l7u|?f zXO0a)@`itlI-|y0rpd91Nck2n7O3^NU|8gAi5gC!z)=S&u8kE_e(4kM{Q4szh!!TJ zJU$w5<$M`piMEX6nyFLeHCR_x zuIF}aV|~2_&wP*7q1iFN*XoXq4N|YwiTk;6g42%cJpBxqk%B!1V+|Z5%YVH-2h6GT z%F6l9@Z76E0(X3_S_sp>4_^<{ZEjCC3@l+_VJ7X4h*(CRVyJ#x7KMP2xth${hC$ktxkj!P<^7fu2ug#0!BKx#9 z@>9TQajHEyo9WI)g6TA07Z>8VNPTnpQ6r}xd)K?p908^s+7QxR((9|m(8`tjOe$M# zkav9~MIABbszy|4T%`A0%SVjQ;7|a`8tcBaF=F4}&dVA@tQfDe2?Do~H=P!8w5$c= znl3Z_#QaA9*(x2LE?S?E=IR@x9Gafdbw<0!FfOmP8k%4MUb|kbZFADX^_yFJT*?Iw zrPfOg5wHM{aZT=Hv_V5*Ux|yBB%bwB#(@i<_K%$$-)kf1wd*QB)3*^{kN{{dZol)+ zVg!2Ukuzaxw5arsXglE1vIuZ(9Y>u*T(1MR)LtC>NTf6&?!Du=>~_ClV8g%|1~yh= z3{P9B4FfY6V65VE#OBz=b=rb}6 zsYU9ByomD7)EG4$Lq=e=)Mb3aH-jO-Qv7=EqU_k?^kc>tr5}s9t*)NCag7EAjqytL z@7g%uvb>IVEFSzgo+`IBa(}h3P=VW_vq5(7u(l)Z_sa0Kh%xq_pUyjBUC!g5!55*j zEv4_L+dlgXi>_~Q(y|X{OP|eQ*A+`Y5%==T2bOpDRVmFUp$o{I*8lcYU>QVEeIi z0hg#mRIa$RDO3f&2KUPGo4D(RU%Glf3;Yze@3KJC+O2+OrOkk!_iK13CUEvd)ID#? zSk$`n4Pscu)%{ArA$IA!tk`+`w$*=fVqjw>o}BZ#jp_--fKQ=-mUCAjMHWfVrA|j= zCM3PjlTXrA>eE|ou$LN5&qX?^pSj|ecxbvx12fZql;?*Oz9};KvzFcA2keo$+|l#At)2fG^t8Z) zpEfd5ucOv`9$&A_T;AIH?fB-}?mYu9ydR9`w(23|N0IN?r+v)?x!}aJGhE#ZXc@sZ4tNdyL}I%-%BYR9=dNEXMUf+ zYR4VFNAbQe%uD*(?w7saOYLmWHw>)Bz{X0f#n@JU!$8FVNw%mXt}N+sFORgHp+UYz zB-uUmf<+tQbL1ig<`($A_rJeHGx~~}7teCaaAz${(PQo^(zx7?dy>aCulJkL!YEAR zs_vA#arN5Ec{C%W6~soHbOdsGQy zmQNIS+sKQwvSYh>uWruqU?Mda`p39wO_ZW&P9W77-_(iKQT8O=8E+MMb5irubxwW> zx|_olh)C2ccRKF`o<3Gm`t-xeU&zPmZoR{$`Xic!>XyH&Lt@_=X7&Jr<^2vkGB~3t zoO2D2cCbmpP()q$an6U+p`%Pr!K&}|u8$=5G_ASJ(!I!F87A1YwxT&{E7oclTSecb zhn~kH3KjS9jNfRN@fsskeTY2cqJcec)F@hzs`=dEE85Sg_e}JyCA^J3*XQTH;7g9I zS6A%G;4l_D@0gbQA;bYq+~W;rsUO?(Eg0;!1~v>Fhyl?#*it)=h|vtAm%IPwFDu6( zefr2L#TOL>R`d;u>Kco^NE=DA^2K$~Z{#6`dLgwB{!}>1C;~MiJ83@{N-(^5+Ry&X zQN&%ZiuzqYkOYEe#%FmHojyS^CP6v{GM%dHvqVW>eK>Y-N$S@GRAhh#PugJgFa;~; z3xH|t$k&SV5>aX-B9Asc^r1gh>#RA>E?|?+*7Fxif8iId;@MOxUcEjtZp*CgRQWdF zv5GOT49wc+?Qbk{#Yuq)NBm4i@fvN_W9~}*5lJ)B63H8`vUM&Cdw_I{_LfpXN7bt& z&r;#?9NAU9HT#K?PwOjd71^A+78B6#mdK2lGf4w%NxM0za zvevmB?1wdgU1y}ZG?&#dt-#VZhEln$HTNRkf$6>MX+*4jY%jbU+o2KLa-aBjXv>~S zEJHMb?QmMS$W#^Y>h@o zSU)V1eeFDLZ*?~ev>4d55-moy-%kMykd`8S*N`D=(86w*RGE9Qs~X1?TUnnBzb`5{ zA999UHOX7p;r3E&!PUl2cXE!x zSI?4r*c@zQz>==@%c9YY`X}KMzUd51@4sXCNYA6Jlz3xQ`xtm`-Pqss!+)>_y|uRp zjA}fUT}=Cn_lC6nkiFX6Yd811cc1r{SKN2sxk!2}nR&3jvG+7aElgwdY~(zZz};x- z+!br}l`nZo4I1u#XT&*?kz!mQv5BPAqaZu-FoxC6cbTeWa7me;8W) z=#Tv2VG;_d-nN&22Ru=t*b}?L?ULk6*=l$-HJ~QnICA@OPqNPl!bKVh=ea8PLK02o zNx1W~se8v&F*G2N#Ex(SMHV5!mp-}18+}8XPr=CfRa;Z~_sF~7JybsGngf~%@UR@# zZ9JPxuM`pnnY~7uYR!^bt{t0o>3kFGY_3o3Gt%FH#Ta6AY}NRy;A0KEO1>5eGk93; zIo7T1XJyOEyH(e8`>^vH%N|1+2L#N8AESgRIy!(z{x`uF4`6!dX-~ZyVGJnWnx`?0I_F6lZDK9!`&Zi5 ze!jfRYu>NXzR-|)k7?UzFpbAIdvxCzh0`lHr!DW5f2!tfti)4w=AW|T=0eMWWUT~m z1YwK}wC}SK(T=LYP?6#}nYtPbmF*TeJJQ}<8LSA&F^QM$yp??Aya!(Sm5ZcF4;F(E zpT)v19an>$0;>m)L^B7dM5eDj*_olkJ{EiPiDJ}(e&%&BRsMATHH}6+*6sAe z?k_=Z_3_O=^!pa^7Y)!P1qrJz$$JqijJ@x>_jnYZcP921>k6#w{}O$gOn2}YM$H|` zW^FKN3RuQv-q(_id5^Fsd7pFNURREumuDZyYoqL|N2BX1x2MjTZ$4zd!=!iEB-n~1T(fSPJ=k}%ly!`SvN>+X=7o zjg>eMaa*+;5d)+cNRWIDf93}V%$A;1(G~W@A*qy7xDPhfEyg$s8AietQm)*7d+)O~ zYHY-Hmu+%fPPio@Y}?Gg`H7IgRs)`$4e+QGH*C9edsFw=Cbr8#QaC3dm+c}eWs*!* z@?_V?T$f~>8!$hY^4Y!YB7sZQ%jv-1x(XiSn+zjdE}eKi!b)P3a{b(w-aVt3D34U) zhS@9E%VR844a;`88DYzuj}h=j0*_|6+>)a%a^30K`FZa?!(#~3W4(IyNG~hfxEot_ zz3&xl@;*^q#j*d<&kH17{C`N*4>iuQFGioGf|a%gpZ1+x#>wi<^bG&xr*Qv+m#_Id?O@tRTr^~# z`q-bXeKIbvM%UI{C!BJvy%% zdyjqgvpd$}12?tg4WErpv)wGL!_*uZORz_|G9!ZM7krG`a5Jo6C=+!fMY3a+dZZ#U z8A}-C4y{`5%hWyAv-+!)c?md?inP_M$Cg~Ef+#0`e@jaC>dDLOMAVm>uO1H7!BRz8 zb1>pAt)4$)h`KrT-kp><`yXirU@E&!g)~NosY;82)41{`2WhqOG)SKHV z+e@($ncSsOxte^ch*|hU=!3BYELukfoV_-e@-n+*jMo1@d-TyG7xFug{{#oi!J6k_ z(q>j|TY=TK-TsC{--x;PMxF@=QT@sqnDKsr3!iJ}YYsM;Jr+XVsPX4q``fb(19KSI zScy68Y|ox#7$BiU#>C|_6E^-AIaCo?_AZg-vk~^VJ|U!V>y0zc_KtEq$1dmY>i3~T zlf=X28q$Nmktj!kTwNkJ3fJNg{eU7FIck;3R_96Zca}mvGjqKOfA?+&7%+5gZn(3-?ZK0GUYA0h^GRZp8!&th1p+1Q_FNF$Y!5B(&_AR^4a`{j4qgpJ zTa3rr_K*S5pZ3|vZ{$(+1u4>T^JQ+3JoTK0rUAYq$P@5J{12My^=E5G?Ni??CokKK zubDrRjCFCP^jzzNbX!^rp1|7SkuIs5)2VQE^FgL7K8+?Tm~ zS~6|M6*m1yP?so=Y8Oue~Be>V6(fMk}7xR4{L#$1^Dn743)#s){;d12|6 zZ%FJ%d4IrmnV*C4HZx}Zw%&XD<4m>f(UTkl8!Pc7pWtnHfdM49NDyq^Nuym{m**r_ zQTLFv+7c(SAU-im@!{9P;(A#P9d?n70#2k$B;+VO_QP>DL$oaw51vGtRLpOm%h5Kc z%5jO0qw7f7v@1FY_#yc&-uZ|qrVh7xtX7nn zOX2Wt8mnDDrFH%~l^EAyGjdt?uYb*Jx_#?3QMD3neecqXpZC0@d|9L6lNIGf7Q;SH z8D90^aR+C_6m3kU@xFU*{bO4WJ##^zU`C2lu;d3k##>WHa=r0P<@wIoqg=EO*gK-C z$cywM45_C2u;p(Uc%m_|u@X--#y5CBM;P$gqlC`a8U_7mw~-%Q+^VZ!`Bdf4m{{US z|IxOV%8|aZh0QS9dy>5ox7ImWjDHwQA?Mrnd%qovG}qqTvoa@_D|y-A4jd#T+kCzN zne62bI@jjZGpq^Q?RE_Bc=I2q#$s7=7?MS0Mf z(FXE00*hu?3}{+uqM)KJSvop@*h@i)8SrH|3{$MRUV!CFiZ^pR0*~=rK4BX?hE-fm zug(~xRO5~W&wMYPMFEdhQymPcYQ|+J4zZOAj4ZIVdgU?p2#o0Eea}F7wpwovKnd!v| z+whx@oIw)fV`M6S>Jlh+dIV!nhztalb**)~AA_)!+Ay$;fsK_|#?e;dNr3@m59y`t zo$o-t*4p7NAp|+h7$!a`ZOX`Hsg056+#%UyXX$ebYyW$s!HF`)f;WOe}LPLAy} zNBOGK^(d55?10n1qwJjfa;jEu;ULdJj0^ck+t`-J(r1Y@ob<}I_2PGp`=90tJQ}_p z`3F{>Gn$FD5svaYkFn99Mt>s3`sHyka2a96qJE?iur^>xvCx7)fB*BpxZ3$+wKHj~ z%%SHYAgAxR?Y5JE&kbG~#9<4yA9#R6d-Qq9V|%-!6!SRqJ$Tl}+W87R_EKxr{$a18 zU5yJ{D1Xg5?6tq;Z5X)TFtD)_*V~M2{dt@)Kq_n-cda3p)^x`w_!X~ybv39UO^Z(! zX`xS5kCa=}zOu9{-1|r*#d!}Hy_5tN>9H0@l*bMg_G=ObcfYWC5BT!1ZPCVPdS!R| zakLXD0~Q7X=8TjZ*;YRNrzHH(`i}27Lvl^Q<;Ow-%*`@jNaL)6s}mCL%EIhOpwS?X z$fs75!@y=l@@c%Mv=~zZ+U3(Wggv|QFb<&|gRh0E^)YQhtf!hTRAL~Q@h7AKvh5^} z{Rf;kQQoy_sVkN~>W>9Tz9!XX7f#yWlDV{gQ{4`OIWR|}IK|PD#z|`w?=yCWMgaV& zb?5vMBtBSNc}L!R?=^!<^H>eqN?mN^5-E+XjlO|55~?8MsJf$tn_EGF8Ot$1@;gK!V9l+g z-1gKxF-L|8UsD%Wc&eQ>`HUC2_}g#tK2Yb%$?%0O^jvVCPbda_Qjj@Z;~xGef4oKr;u^a2sb1Sj(JYYHV$#RwBP4Bp`^2!B zU)ye5o`%H<*kMl}ut=C0Vq^oL`mAkW@Hf)>N+I*Z#@|@j71IQS4X~yvd7M9)gY7GZ z)H(pa zN=sAud~2=7oQ}*NTm9BZ#`qYbyyJ~uQ_r#pElY+I_ZeO#;o`wWT77;pKFU3<%Mv*n z&54>7;~UW`A&>HW^MAm)M|*SWgKv1lQC?Rgk&A*n_dYR_Dk6v>60DMIJHH|0<2F1} zH?unCYi>O8jJ>@%&+B+px=#BL__tpzoqquq_iJ?m9D@fMU!&5KSZ1~)&7=${bOKbC9eND+In~) z82Cg9I7ymK8YPY2j885Ez^)PY^?OpaL_4XU-juRozo^I>B*F1%id6e9N6XdESNVLd zaW$R7h2W3jQYs3HBjfT3#YkCs`|(aSw2)7N>(t~to+&9lwk^snG~0cwekH}qc(~Lu z%$b{;8uSn;WjF-q8$d9AklwadpW4?l% z1+(HP8l6c66$Y!mD>LQVM-_*=_h}R_86R`=+ro-t0_&O{oCkYL#eFHi=DpKL@0Yp} z38wUhJou50~;%G zJlkWsFY>$5Z3XtYWN7=^fAc z!r^xUdD+gRaq5S)?}~S`CAsUnQ^q%PDt8`hycc<#wXNnoY@d5CdwxE-?Wy}pMkf{` z&ufgMO%H_?eFzbcy~Ks3fJ6HUoJT%r88u(1z#$f6PB8jY-lC-7*HfZ!p; zmQIgHLNO{~C{v?u6{c9tTSWU!K=o4XF;6GY2i5;dX5FDnw&hI za_=4FS2JRWmDjh(P3-vXRr!K9r?9*D_5>cJ*7TUqNeuIv?@4ZFmC@2jI_f*n z!sm`XYP6O6f%S9-2RvaHmM-Vx2>d%2SU9)8-ESD!FmMe68!K@Q7u(&F5(8{kk%?2+ zs9WkNa@b)bA)H<-x{-UIN2#kGLp9piL^)&O=fVEfCsMm$VPRT$q=%IZo@i?%Tf`XS zaY}d|ON^q>U&XHQm z=lR1Dj~F;{tN!-C_Bs1?9qp9-I#1j@V_Z__R(KZWqMxIuPfx84^ObQ37K0sQ+FxnC z#Jnmu_eW|SeTWv{dyu%vC7i~e2SwQ2-soyJo9Il%pUu7w`vCvdFv!IParM#e` zFmj#0bzj2{q+VVFH+-MlsScw~L#h z#DLFn7&ql~$rxh|7@vtBd)K?FM7bqf;?s!49*I;l2e%ljR91x)G}@g@k*(s#GnSZZ zxAzRJHmG30LS`^7iK{iNsV4;Y8lA}N!Sbmu0ZMDMtCrDkrWR*nj`k5u@bA9ye5O^Y??fX|? zl2$LliRU?D)X00`q~w;>>_zs9_H*i8kHEUxc*h&xI6K!TbA1(z>r-oxN`o>I<*N&{ zWWq_iXXUaqbXZf@MH}2^INSY(fhP(B8!PcdVS0n}bA$oXQ=bpg!F(Y@Mo)%a>iTnE z@Fhp$&kK%76VUq<%JVrJC4Z)13ZNKAip`N9(FdQ*skY|w8OtEQGHJ?K=BV~qtb z9(-Jny!X9FK@LmWMW3-g;P79Vi~k=i8*(N!IeCj!s4lP#DKoML z;39(SR&Bu~l#>TY`pkxjk|Ve!SR--xjXmZM`?~<{5)w}!R*{eIo6xEQk30Jgcoh5PCvU;0V|vQV@> zcH|)=E_0{O968x*uWIv3T`VUJ&(#^@D9=dH!Qzio&*|tjI+Lf3qIle|wR`XX-IK>5 z*10OMz-x`yWIBp_+_AM$Rr0ZM6i?4z_^f}4)Kqh|o6jN0E_-}T=*ipSj4!*2(y z$Ujd?G?L$8dz;_1VSLH-fwu6x|XmID&Qf_DMeHlaDbG%z|O@$^{ z+xAxe#=*eGO58Yewe@s01{eWQR=3wh#=J%viB0ZgR<5T#@*Wv^5lVAzmo2N(gXdF) zBtL{KBysh5MaqQsgMkh^@qA^Y9q?SnxfZv`^r+AGgYi+%^^xmpPm-#iJwJfnVnI?= z_xKXaohHKi=#`5+uDc|g;lB|595HqgUp?lk){jO9Qrd*dBJJ=x!k%RV8EoBJ(((*$ z7y9Ys@xX`FQEXGkjrE$!rr~3K zj$5Ou7%v~2B{I-^6cT)_UTS*s)_QyJ4ZrskUIQjFcW#`{cv04x)6OK2<^hA7U`Ya2 zqzsT~82JtbZ10TyF4lJO2O1h%1`Bt5kU|^~@T9Sy5&K|mxS+>#+N(8ea)#+%_>J~m z4tN1WdZLN>f}!(3S{Utb{!;c3-v{pM8J;sG>_c;gm2G=#erp$7Lu2l&p*8R48aP|o z4Fel1v8{n8F$NfI_Ud^uNiEf@?8s7u@|w%)(BHlmig%+$^AZ5}XGK?{Y)h{PK8@;_!*q_n)qtS?Ha9|!X&LERX{zv;* z&%W5r`5H3ao@Y-(b|aTZ;Sq0yWPXiy>48^%WyNLb23SH-}X#eZ|q6zZ&GdN-|l^< z>T!d;hF0d;0|cIC>=@sooHER-#*-`C-HP8>7}!{e8*9F{-mby`O#t$dkpt4_p;x`? zhWSYRMGjkmyN_Elw5e{J{1EK4=RVA$E#2S*QIM>TzRDiQZ3!M%lNfO1gUmn>;LF z^ietr@6nHbbgyYs?`fZTY+!7cE|LCccx_~==$`e+e*sy|KP8iom2@qB4N-P3ZQ)gM zNMjsF6VQ9#h%~(aYy7ZUc*d%2;VKw+!J9RdaS-=h%nOM=G_DwP51#fI7iFzjz{qJm z*CiSnOtHo^IriMS{TYjj4Lb>-{6IZ6b>tnL>IT-a1{*J9%VTg)+gOhdovM6bzHlbq zS?^L@Sp88A@qu8sYS$?SHdf*~otUj#7l8rxskh($OO8ptFQUD+4P-qn0Ew{6C&Kpa zQxbVn%KGytv3+JqEqkPP`{{_^BEazBKz6kKUi#f59irW(bnTfEd|6se3m9<5?Yehm za$^l{%|T1nGYs-c=Tyb@W zs}YUPq>NXx&HM1!VAN>neOc#on|AzC562@<4+K6GQP_FtOh;t za3XE(>jm0eBKH`zRsW(wW@mU@#wWn+v#m}3E>RDQj@-qP=gZn4>7@l^#2Dox<)bsV zmiGY54e?6UJ2bsQ^D_8nFA^NNvSexgM_|*kp3K4e{jB1)?1v+8lmV+6a1D;DS4@yp zVxy63#9*=qW6RqxaJ^t)V_=p@|@|d*t2ku2=bdT^=Df)`DlmQ>0K; zBFH9M#G$IFE~F{DW)id-^<;q5nS+gt&=wMfflc^7P-rvK0s}nKFk~_5zVe5#5TwbZ zkVuq~Q!*jR}hZN|3#&R_ufOftwOn&hs%*dK@c z;7l`%kGQWPD}6$%JvE!^&`9j=&DIp<#&jmf+>*cmJ(lu}HpTSU%UM zBzMJk!KaszSkHgKCs-_YWz?&rbI}6jHs|h1i*tXqG2)`ir8ojhN|6b3aQ4siG0hJb z*U|M*eNJ}uk@i)hyKlIEQ~f0pydDnwHBWMLeWcD@*^_72B=@V2EGrFEx6Cre{Tf&T znD9Azig#`jyqJ#~MJGS(-6wcN1id12c^O_c1e+`DS%PKL^xB1fj!)cEEI;AvEFK0Sm7EmPSm(^RdG;yU%|63dq$5vIBlN!Yxl>PpE{Pt zk73PuH7%_C+x|$LL?ds1j4|7%?Oj+~&vTuNx+h*L+PiwqoeqCd@V0iY4-9Oq#Pu-` zTQAOq0TNqEY!_)-EZN_3WpB$M&yd_B67+1qliNW; zkr;pConL>Z?}8l$Ns;SdAD6~=wMaI(J?3M%1l3(KT4Dd(Mk}HbS%MeOg9Xvw9E38S z_fkGRd3=vIyw)G4CFy#7j651Z+KJKev9ZnuM_fQjAwUc zKW8}R9gFKz_uV(!SI0F**^$Up8J2rw>t{aw7x(YhlCc^82IEtQ@l)I0$nShSp?7#{ z-@tZ94T;7H82OoUZXW@(<= zy1m~ON4x&pKXW(=T0dI0=+8yKt6?#A@U%+=^qEh8`skxS^;7!*;_`h9&!cULu^K}* zHmzgXogNn?oYDHs1V5A1VGB9EC*4)*AQKE`6ktg)+1-B%fhi zWh=GiQOuLnpCS<#+FE)H7Uh480U2HcP^OBsK=C|fA9R~FK8zuH!E`SI9w}Ol=(C$2 zEK*OxO2UZ!F}}6hoWa0JinE-m?@hdS@Yrkj$~Of z8d9D_7dssM;aq;|eVl{IZ8~5@ zcna(fa@StP?=0%Ohr=k}Bm1N5bb6vuB^lRV>q3vM$Ske-wWqCNqcdzGkScQ6WzwjS z{4;C|)-c8+wU!RiFmXjAGQSzAmMX zdT?vMjAwJ{U9`PrT*NVJ_HKcSvoxOOW{wrqAJN2OtyL}GT*o@kPVW&ncYU+Wtv$%v zsIbaDbg0O{9)Wd&Q*GIQ?Z@4ju9R4T4R?NaA3A+9wk zuSgkHQ&6^dTfB=qvp&q#U-RtV2cB_+gpc-nspE>)wAm}m*nlww>1Q2Bk*Lu^Fv#J; z=R!7B_(clVYhxtO=M9O%@IhNrK>!)5wDoH&{vBT1XCuFn+58s*XUJ3R##G&B-SYYS zvQuSx`7{iS-x$h(g`Ht^F?SL*mvl#FDPL{UdkuH;MB%~a#h4fzgzJ2#_&)1|wD~L5 zSdHP{{=vjek$_efvuXxPACW60_9Mn-n&e8ZkI0b;5T~-8+#6%^4pqakQzhnql)0|( z7)Z^RjHf3oruP^5yCyC^95CWp<0f9Udj%g{px5q6js$}nh9%Mar4*Z_j)2EYW%F}u znRW&{MpB)h&E10o9=wc_u+}8<+Ip_WM{@w0XAF4CK3m7==)WZc8{>NY1J9`CN!Od^ zhJIpk;(P~Mi7A-K@PLJF8GUDZoe-?rI)huJwY7lMj_btr`j6G;#0xz`&R)B7d0=88 z!0cTjgNgVx>K8T#E1toTa2e)OD2ll3$vrVs)Z5Z0Ck8fF;>kI$H_)iWvtJv*XLNYT zD_b=McRk^DCnc)Vnw?GOsGH#uOt2`qs}u<74pt{c3W?aU)ZW~4(N;*fLgJKD67wqc z3!Bw*NZ0g%afm**<8jOsvceI7ShYgNR2E9PstuVNKzfFB4QX0@4r4rE#3wja+210| zeL9ujXHMLhpF5uOg=fYQ$ST7&Xn~iq@VhypKIPD?|k4xI&xH*^%eaa!AEqzeJ_!BBNEd* zf9gbgb17K$JdQN)qZ$J99a_YZ^`#%RZLre3uh~o0k9jbkaBK|PMs~x22v6S6cE7QX zN7t72{MEiS{k2zlAK*BTBi_qM*TOo1SdNQxl%D--w;#rMr&9aP8XN|koZ8=)#uNKz zu6Le$^A>w$q;WLdA3C#PT@`dMsYGODUJ=ezMRojh4sfHGCd24EKUf z(vJ0{l68DLY2yey<=L-03nbt$N~1Mk;Kt6m2Pg0%YE9I0gjeKGrHtO9e5CcPiKz-r zen^ZN{*6d4V||A~pWmZKLo{chO&f8YKI!Vs)eorpbd&O>#{orG}h0d|yayg3+IC)aA8+{Kw23|`u2?Qib@gA%Z;t3ye~6PN98EdhY1 zHAG<14%D?(_JQ*lY1YVO=CH8uQ}4iDqq=ggTO9eF_d@4|WbF<3<~F_ySivdnMX&8% z9*v49Q}G^hzee8s)>Q3S8Ix!Bp&<8r?X1b82?(uez|E<^#v||b8kksek>czD@JBO2 zW6Q8}YF^j0KUKEO9}U;y`!Q92%fEOGY^=n^<8*6r2m>ddrVms@oS_N_6tdOv5n!Xv zPSn;adW8r%w);7QmQiqwPej;pg;7M_!t*_HeAE^>;KLJ}YZqmCHkQe4Pu=@m*?ObB zxfCpc0;k#<&94%`IY8wZ+%q^EZF8>N2s}8LYh%r`5qP3}t!Z=O-7UlykC(#ZJO zor@~N$cxX-{21pUF2Uo*j*RDD6UHi2@I(xnhj!^wgff_FgWE0=vHC*nNW-bJBd+Ca z*{ABS$!CK<^1j;x*R(rTmRs){eC=->CqJh2GFCwk4;>#H z@xUS0b4gnu;! zzOo&=@?ctLqX9g<<48R9oX|WXO>d*62~OJUSDObG4G>t|j7Qj)lQym$X{au?y=}Yw zonfi7CZbaGQ9XVkb0HPLb<(&wR=(CLk@(ga&R|_?kla$Aahp&9c|44$7zXm!aP8k~ z{aXnu+NC{?{(`ChaYazrc)nxHs-ftbaRn@?ZY2Ax*cEOm>AMT z9Q!IgFQhKGanFACqQLq5<+hRY+#8)G5%5L~EJpgH-|_iF%4s)ThvBzsV#-)A0-kYI z$yaF)8PL-dAeoHuJn(!{b3jZI{HdU@mHaI82<`~JdS$?hdEr`GNZamOAPKC-Wo3Mh zI_$Nzlvhc--MZs;>6oLbI@m@1S}HeHUw$0tr=2BfJUGg&iB6SaSKp38I>p*YHX+@5 z#uRlv3m%Zphe(2vG1WD>v{AK0(%sdv2*e5du{ES*?*H69xA0gav^PWsd?=7;@^f8m zjl4=W=LduLL=xEOfG5aiRHgBxfr#ItFoC0N5s6H$r6}J^XqbcU^;>u@+@c+07Scd8QQZ-a%{=kj*J^E0;e+eHK zaZ6l;g=a|so^LtU2!2Oc^o!`j*2X*rHdbODKikvC4FkvlZoP3mUVsF1^yx3rNcNv+ z+a3cFkwY%R6-}fQ(q^upNT}Wx8(XeODLyNQj2F)02&^EXNJ>d+!Tv(pb_`DW+ZTIr3p+>1s%Gi6J z^Qytu4i=aB{A}bNX-b<$yD;(&25E`i!`6+!$YtV)w7K=0%ac~aJ%gB>zjV((%e+@( zmJ-Z#seC*~7FC!0VssNmGVyFKt^P)39;$Y&kXv(Q+h;M~-k+*DYYU{`jAe)jyx-Q@ zA*Tx4Q;?dqjly<%7a3-`fbxFD?|eiZ_T$h!Q@i{n7-gEzxXe%7OZ|rbHCh&JPyJa3 zca<|70$&xz?tMXu(-;4pI4vs{E5o>-k7hEzVgF(xTi zrgwftfQ}=%)*A9SLMf9L3{gf%2!;e$f_nT^bYdZ8L!6B?tixh>aRz_AW~?bJ)~(O| z#X}^i^4UlX+^fm;5qsn6!ID?CBn9sy@JKnc(V*yP$R|=>k;Q0LM3mu=GzGJg^#$ID z|4?O(N?GedEc;kvb?$fh@k~E@W2_pNQa0-L%0ynUal0U?6=yVkL=XYTIH$%&^Wq(K zn)8_P%FV^7=9|wPpH`zll|Ii&%Ng?6y zOtz~=mNSk(iq4S*Bk+(f<&#+9Am_mCZP`_lTlz9;OWuR6uT?uy+TkZLCrM*)W$cH- z6a9k*37#nEm+(Z%eJDIqU?c_hr=B^|8+?<;JsCWjq2|%-&l*{%uzCW+3QRd$I6Y+d zPJ($gu>k+o9ZI5V#F_3+yeKz$wBFrJForN=1uFWT{ zx2~|^qZ$`&Ft&>T%viHgvN^&JIBTNq%hGRic=zA@W%V5bdkLmJlX2RK_~M&}=B@1m z(M#Z#QCj;veq&>>eS#DVGZFcRq}sACn-fQt_S$fqsJ&#=uc+5X&STVM-y}ZJc4+SS zwmk5P-&*%M-@Tr}VQy#5eT~*sLYNo_HW=&(s!Q8ACk9#rCzti@^Zwg+z{MJ2F@1k? zx$XXm#lXf&JhAxRpzSe$B=tGV*OWW&zS4m=q`5vQ{qyo^j>1!HDK+hfQG6bl&lqFF1^LrbxfG z+@bi)YoppUpG9@e$>sKv*@?1|jRb@f7rzB*taEFl$BuZfbGg975Zjpch^Pl1Snb+~ zJ_-h1ahK|1=SCtw#aIGXw3E-DVWe2ygNvk@w3**hltaD=;vDZs^l2(}u0w_u?KrVK z&iD~jcB)@QKvQi`s#+VX-D&%m5%Z|YH%0v98$FxJp!z;oOmSVgLQ8GCjS@Uovq zvA`(=zvT1nqs z4U)^M5s2~Z(v!IQw+%_RN&885UDZ$~wM7d+b3p4uQ$<^n`x5ugotPX%dVn-P!;U9- z(O#^V_}!+XnxUn3qTRW2qP5X7q+v|nit@FzPl4IOtA1d`wH=!_Wk@y!w2i(MW6EiU zXH5|UO|`d_uYcZO;Rggb?p@XjmQcB?9TCD3jP3b`f$J3m8!K_W&dig(U$#`F1^2qR zkaeS{(T*aBWC%$Uff@6s#69WMMZpvgK7XECSn+!#@a5e2e1;faP*`Vpl;v!+N!Y6P zBa`9RmWCXiMeuZRiwMM@XD9Gq#ijL{gqa{wiR~2ol_EJ z=gPI!tK_}3EuxY4EZSfcH)R0xnNNRuA5m?tTno$ZfEQ88z!Ns17j;N^u)+?WrfaI4 zvb{fhZHy_>o_0rFVjl*fQEuEH;~M0=_M@Hlq;qhDuE1JH8OAxiXmQtC?%xGI%|phM z_c*TV7m+j}v`B}33)Hi&t&QxL7vtEc1s-i7e-mtKzxV5M87FE_1S;=&T;65*L_zCX z+)8g4Scid)l~{+bt@MRq0BK;m_ldHFBQY+l@zfBKM&zMzLyj%oXN0jtDtfM4_Oa~y zu|=(-K4AKU6nF1)!_)@*^k^3e2bV}tq*i8|4)t3IWH^ta2DU7u4S3)p&$4euB?)%V zM9$j_z^ukd zf$xXEVKpnkZn%<3r43q;B{DO!9fz!!N?H!iKXyAvbL{RRY+10-*8Nf>p*wE-qCJZv z`cus5nly~*Kf=k4b-`^>C%JCtH^}EK$CHH}Qe6%q&zRKDr zSmy0elDmb+rr!JIvQDkV%;tb43SaK?ntO({nGMQgjeG0JJ->{4PRbepow8F}>41kN zW=J_2rLF_wR2EGLKZ$7HyHNRS}E<<)2{-CFB1u@Saen)@v( zj3`&R?PxFZM8tXU&2O&l`YdLcbN2Er+=!N=-B<_pez!KJ&N`?kCCB=yb=#KD{wB?( ziE77!jCkqy{-$HG->0O7_3VVahR!yBDo0!9I$%iycZ?lQ74H043oinmW_To&=4dJ4 zDuW)j@=>=&^6B&`XZ7IKHIoZX+zWRQu%sjFv#=c%e|c?8G~da-b{yNN)~uc;o#uNq z=xq(GTGnHXulBpsqc$r74M)8ZoczRl{zQ%QAM1){Er)w(VVwG%`ZzKNG=bO`Wx=CW zYR~aF)qeYXvEd5v=x_`dYk)zqFd2aijC zeun~2((0R0f_7K!V1oxy+NSoLIe4CVujBln49|UBu{Qk1P9J(MPV@y#dy2J|+s^Q; zjioRh(cWlkuBr=t8XK5=qVjntqwQ_wE)D}5D{*nS+*(?N0j_`H;wK$FZFm$28*OfP zNXH;?^C!iYE}t*Kd3`2)2C|VzhSR$zks2LGE_byUHj(0HSjIqW(7QtVYpjB`=C(%e z!OHz-?~J^is$0(^$e-@1f}Q*x4(ln_owAX2C_HN;<1og!1dntyOQ@qCb1CBjpSpk% zDVA@_z$w~ef1Ncakw2GumjXtl#`WME-Y_|D(O$+UG9dD&Qd0LY#0pr_QB6^XHxg~? z3W;Q1U%Pwc1bh0ae#)oGgY)Pgt<+q3&q7+=!t~BP{ipxJZ2fFJH~zj~V|OylrL!$a z-f4E#*##ROGMt$j-zR1wNP(5tWZ7G0@sM#$Bjw5lO!}M`ehk=ci zn8VKY>~X??NEuEUrmz%6!zax}xxMs>9_^AqgQPF)MKk%wZdSyTu%GpbS?ki*_JIT~ z$G5MNl4A4ArBpCJ<=Fv;^lRh{gBC{ohP#NaLgGzIGi69o72D-gEpT4cUy~v!;5jWfJr?@WN$1lS)sSOXc%(hBb~*T(iHFIk+i|7uUd}<*}yb<~R2(?(2J2#z)++Mx4W~pAlC+(K*_j zO9P%YIae+}^KSA{I>e6$KD|TF{_CGN1do=7mcTay>yrC1dhb1AaJICEIp4wFz^Lo=3b?MMR^6nJ7nVO-!fn+I5*DsqdftN9YQMcoR*m z&IZ%`&<+|a7$U-1`||nPt!{Lya3L7jScwZkbIR4VTqk}->!8stkWk0`V|#jTrOM?{0Ls^BTDfdHOd6n(NX9EK=9 zSvfE6ea;RUC!;THQM40j`RrFI6Y`Tki;G>hs4G#%Cp`Qo=Ef7xkglYuQD4g5*r#$z znoHW}r0b|N8h{oKeSV`J(vrbeNYSHDy;NWJcrIFnUYVT7^%+&ujLjxWS(o!Gexv^op)txhV zjPj8RbR6)oHGOA%y4&-4+xRLtwSsS!R$cd@(A=Q){ zXu4aZ!YcFCu0vz(83Dxnb{kvBWGoirlF}NGc25}(dLNY=@sDUhTi>B2Fds;wrT3U7 z5*`jQ>Zpcj)^_Rd$op%atrgPfL!`j=sd&$p#sxPq&vnmfku~t!I)s~wqk;1A-c@bJ zK>0I8;Ihoy2yU#aSid9X&%i1f=FNQcp5;9A1{b6Aoa^TM*e9mm>nQ8phlizoC(2Ky z<{yr#W_6;FcgDDu%1u;1m2FMd$bn@a@trZwJXZaP_j8Tm;wd1VA2H4tN1Iyzhpv$h zxJ$-qA2U|Qwy_SKALF`6SUvb#-aH02R$?AM+tbGd10pee{E^+SW(gnkuJ_d+-0UX6)*Wj*lNe;3U#Y zVw`RMY)*j{bw=RPbX7_C?){CrG>mMh&jM(2R>I5cJzzz;)Zg-s?n!Ra+It^(Mwk6Q z-NU}iwSLUOs~SM_Rt(0fkx)4X@pQZWU&1i(ckq7kuZXcpi^ih6KJD8v2m1BaN4{|g zR@(#bdp)i&&1Jg}^AoXGM!y&>jj(bptoZF*7uu|7&t8+s&)B!)Ircj09-3C)GIO(= zpPDwmeVZ-7srY1%Hj~L30to0WS(J#jk*$arqSHPl?au zMZt2W#MEiISe!lFa{%O$DH2f)2Dy`{F#@10=eNON%_SgW9b5f;t_R4C6G_khJNgYK zf0sL+{p>TJcb~HJz{{sXb4grsTXl9o^lNl<*cbIk2g8ONJF!G!T0_V>pXD0(b`T&p z%;+Oh`$}QEri{g+Nd`lVztm%37WqploBC`|KN^&=;<>;QacGUP5(5=b@o&$>Swho~w@>bF|u&8#Gs9T2QQmeu=2W=y0U{ zwhVY+wfXsXgo}tZ64mqCnxv=6+wzt}S#iqT&&w&cI!iZe~XN_K% z7JLbR;M>N(#&yxK0=DlmW4!pirPn70Hdf;LoR=qgudoufc@b=aEVziHfil`oNYZbu zq*y3lV@sF%+%pVf^uf0x*FEGu2N&(Z%dp3TL!Rvk&5bRcO>eZh1P_UP=qIcmZ&vUX zu%zIT;)#5OtRQ_Oea&U~Tb%9ve2yp_)r~sA^QlLkI`<&g&MW}9hrkPITJ(DkPv?P$ zm3Y~Ue$A;*JraM8l{gQ)%u2-A=H4&sAL$!q89J>Y@9P;EY0em?^jx81{@OK_-F>6X zREo67`vxgF+x_YCUwd>yDxdgTV0*awPL*%-Yowtxww^0=)Im-$wg@~**RX)v$*c=f zdWyWow%qr9zvUDx{s(LQaWXt&ff!3p`Ly*!6-#V)Zo@rRA_qim*S~cLUv<6`&f&4L zAuIRk2;UI%K{FAogb)Z3I63xDq&RlpySFwnmGMP80Z$~Z%t}Oll&kPnJn(NXLTPXn za3b}7UMb>)^ZFBrh#0|nqwO|5ZxG!U72bDyyNiDuc=ngbJ2c8x3upU%J`8NE#QE^I z)n1JOQe+W-T0;zZQ?}fzf$S>$z%94zNu*G`Mruo9XWWvghr%KytnSAc^0~8p%l$Uju@2Rr!(pyhU7$<(wMTOy&jK&?Q=c21xUa#Bckjc}nF(zZ)$gd( zMK+H}{TV*+M_~ZuXvem_rQcXU<9pFPU$$7+x)9z$c~asv^A`OGc#dboE<}Etdf(WM z2kz?`9c`h3-vLiMb73JIw-~V27!+)I8MwlFdPiIge&p*5mgSJMetJC;q9WkMeipx_#(zjOu^udWb+0eD zim_x^=BtV_rYawpk2a|5Ww&q8>iT`2wQ*oqwrU*=Y^+2FDO=X#iGk2ah;vOk6Bk8v z8}#$ge%M~LSIM^m!}i*{NWZFG^MQaw{-Gs6x<&gTaf;t<%2?&laZ=2;?JdDm{t$`1 zPfsh6)z%2SQIayy4u?5lZXP3eG%!vO4k==OzQ2#g7SgN5u#HA#U$0pkj9|i^SYXAT zgtXROfD?~vfa3;6z-HW|KT%f7cp#piN^_x!`>Gx41U=SHPt)-F*SzMGc55xXEYUty zx4`RZ8W?=E<@UcP#uV9Ygd^)Bu3oe%MnG+pDI}vhG@{Y&R7yj&?^~Kb+N2&Uja}@O zr!~3lo_iMI=~r8yl-wBwZ68V2gN<(MjGjT#gIC>iTVUlkch?q9{GMV5+GZ;Ik?^QK z4I~4(T(;F*1RN=8w0X_tr|xMTumv^3RDw%hG7}MO%^f z@a`F-!qKtiG4qcI-?@NsDlKb1|7OH!T!+94xcqahv+6$IjMez8dCy(FeIVLdn_^*z z=8VTUeAm3oBk$K*SX=pZ7}!{eb@#>kDi^m&* z7xkobFyfhY*shH!`}e35=^^l}`P@eme3I8_b1rS`&Xx6y3vM2pwXswuKlg4%Uq|ZJ z(|XH%nX~#{=f)Q9k@=Y=99f^)4VQn*oMBT2Pm=&vjLrKuBI$O_R@9MVS#rBm_X0G- zNz@&U!iAis366TwDHxH@Ff7i`P@X?i!R65U(#F~~EwF9v*KT}EzADkrNbB*4yEOwx zC+zg;y(`L|Vsp#)h}1itji%l;`)1%2Oe}(N=L>6AgU%KlDhsEFXt4Cl-oB6aw!>4v z&$x>`r_*DzM#t1ECyJVHSzdS1e&o>_e*0hhoHN?;UV9^X_M6b^1gscedB2YtUyZc0 zm3e$Hu(1-44_>!cyBMHs4rQ;jka3U8>+i%*$5Vv?i|x&2`<8zZN>TRNq`G(9_C?k2 zB4G4V5;?~Qb&)cWiJXtP=Rfk$L;J0}*M2V#OmWpGnm?0WkAm9aDYxg$2F4TZRSmV8 z^0xF*_x0@S0Upf`_Ca))Go33LB(bgSvwEYfSt|`3kz}zguCeBeUrpd7j@Ih+@pRs4 zX+1;l8XXYDC&tH?eMHin*NgXT4VgzOiqr_0HI2?pq#53b zu~FMh7)I@3wgsBsG(15rVF$c}O_AHZTyoV1adMil146Mq>aeb zsWzwb3)y@5*1z$K_U%QxBk9NySpk<@GnPevbK8z>V*Aau9huc$NPpu~2U#+5 zPMuZH=T*Q@Wa@}QytqwP5>ed|pO7l~-OY(3@dSX3agU}E@deD~IwuA$KINm*`nJt{ z^jQ0>{r%WyKUUW$FJvP2emjoKFK`(cBWe6}TQP+pH z@c59xr~bY4dwWRzZ1^^Oj|qsIt$-(qMD#5yV3K z2QxSPKftSO-AuXOBSviMI;OMtyqD)2qYlmEn(@JXXtb+lqt|#RPdE(lTtYgGxH+Fh zcl({S4&Rqfvy)j^g#4~!&+UiRb-l2ezEM$`KN(RCe-Zt|2Il8cZYiaac&r2*wy(rk zBejRicj-&yV^_~>=)@)8J1bNQqtU6$*(ZmLYiZou^LY$xti(Kiwx<`00g^qRyK1Pj z+j4C;Bu4VrL{^#W7*dTQSD?I*2K7?WTv~VKxpv!U+(+XGq+a{Iyio~dhQ**rP|%+J zHbX5WLB6t7n*?x8q!l*AJ^TJoe)8KY$|c9uay!+K zBTGy3!-;#Jx)xSpvxdl^%Koi@7eQYG9+G4(s!`kJI{p>#yhi~`>WH&7YnuTs(_QzDW5JPhStHdf*Qq-|9W#Q}e)d%0!Psh~r1XbK8GEq!3G`fL z84A!G1YDm&_OiV;M)Qhvr`o4$Q`-zZuro9UNDZHo7vFvN-Y|pP^r6w4ki9i%ZGi=z zEqh9Pvp*)Qlv^9!$k>f5!(wPa`uNDZ-`&Mml_>A}bqKsF0pHa(cQj&Gi~I8wDRUlp z3=G1c!?;GYZ|8xRNyE{oK8M0%Oc6$e+<-@q5UVUe!9U6E)SBwO6IJ?M>a8)TBBf+( zguJTm!@Y(l@&ewdL6tQWZlTqHYogt$?0{$OkG!{hd}4oD!&U|SfLGt|&Y3MyPO5B3 z>WM=RvA53#Rak-=$twfd-7|pOpDthKo3y`EY$z)mR1wFt>y4)TuD|`O_WQIoxrXKM zbpo827i^?;;Sv~@#NIR79DAzoN3V@FdG?eZs}y~Z64k9~bEzz`A76-I@d6gDhduec zIGloyZL$9NSNZM`eNC0!@}D>iY^=l+hwG;}Bnoj`4Cw^B@hU|XU7}p>eOi1nMht1@ zQ0Y@anM)!R>T#hLaPID|F^ z%(df0ZjXR=0^N%>Q^U_}7U zD=T3|o5;5jc-Ek~hmR_yjs{ky>JGsxpLnp4zBRJTN;pwm(8v*ZKC$3& zS>41-KkIog`N6_HqeOjVXgI>6X^sd03rdqIa?k>ZGmmL8%^_{^5=^kFOK~Yb5?$;J zu##aOBVwWN45L_kj=uASii5);Nqq+Q6}L!~B20|r1jB22X3!pr&MsShJ-R}kwp6(rT?pA9w^Oj@rd z5GV!kC`{73DajUG_DIvKMbcL7dkdb8^`(FJ%ZH5*sv>_^FXKsI!#Vd#pOJuxM32C& zOXI9k;vKyE|Nif(5^DGzu}SB)8GRfz;z7Eo!w{>Lgf8`!%XCQlM&Z@t#R|N&WLEUW zI?`X!V9NdCx{1KNhr)ZsefRBg%I7HhF_qFJd6y4`cM`?yL~Jnv4-Vi$fYc>pyI!A> zY_Vn-TlUJddG;=K%ssd7_W9iJ9GG*|ps^n9SKxt(Uki$eoko)r*Kox!$^j%RcqHNOv_r6L@Go-aAZBW7~n`- zILL5Xv$f$l@=%DeV&BkrSN{G*o^&ekZ*Fhoo^czt$-5Hp+}pSAD4H&j8Qqs_)p}ln zO=Imn<{rQ1IL)PYx95jpU}Gf?!`oJCH3mp#D!Eg%5KiAj#;tB-u!2up1Q@;l!OK-D zjFdd&voHcOY=3R#%2V8v)UY2d+K1H=tvQ$s4Sb&3vm3bE+RE3sbkD(wXGk}=Y$bCf zs0=YkYnRA@Xlp4YwZalj89r2d+cL)Ud~S0eNSV3kPm%hLTJmZ3NzC;Qxv%bmMH)L7 zw2*d;25rO$YN-#{Ixh2b<+vw$FQ)K4SGRoz-pIT#F1(1q&)CV4_nD8+c?}DV)t)tC zTW)+K9t{gk)QHP^ zt~YfbJ`*&dy)v|$C*P>v)5G$v6iuuBv(-WqWmzku5$Hyt)4USL`1bt&?7a=VT}N5( zf2ydo$h8+0E22~_coDp!yok{3T@)4daxDs1P*GI8;rr8-D{+X(GZg z^36u9BoI1(_XumT?3(kE&n={D=lKYjD3OngT}_SvJI%0U*paFdUyYmCIboMFW{0?H zVC+(0bR-5B?i0gj20BvfdH))p+}I$Gd34zs@wF5D*<*K?$UW?qW5>xc%2#!*V>hqK z=N#<#)1$jgu<>PLisUqKpL6ETJnLY~=U@Fi(0-utDPp&MTzotIIvG#JRnWZy%&o{j z!l$FhZk&?InCHYkS#OloWC3Rf5M2247n|onyHp$QG8RbNQsW;4Eh25jVCX-~vDgB; z=MmY$d*M0JzSt3De&W^~Hi$T%bUV%>f;gI0%|$G7?L?zd)wtlkzH5^1uAq%x#sG)4 z<~OC+>ehURV&50l2)3oi`tuvA9&7JA#X45E&2NFW7x6eIijB?>gSPzMs;{EWh&2|J z->Aa^|0I69)x8h4K$D4Rc-QT(k9rmwXf9dro`G6idON7XhJz28=?6zqel~IKwFB9; zW9CU4Xt8b6WLswKx^-FD3*4#sMxmMFUH1#`1~m9aFs~-GboThOAkKp=@3&v{8t>s?fwRK`1Vo|m-ue*Uh(^idFOTfR;NY4M2WO`T5p92VB6m_Ea({-s2IT~6t+9X z&M(Aq{hYY=MOk_}{}bmQW1r7%WmRO9!U?$>e7cAe?)O`D+{9*{Pd<@E@?ywW(Tg$W z2dmtXN*xBI(mLEgK^pXjiCEWOQJUok`rvE8vMg{-NFd;uO!>0zf_ zWiu}#kw*{zBG`iv_wXFoaD0q>YHp)2{!#Qu9&}~95_V=DgugV_^ z4FWsAiV52qvHl!+^r~F9B@&f~NuAsX*yc{P$kjN+Zdz|1$eq;2eb zsmq!7drSUR<%KMrdo)QE?VDM5wnVp-LH>;i4yfOvnOgJ zz-}zcc-R4U5GZ_hWB=^cA;_O1*n4)ZQY@Zka_yKfb%h@I5Gs~EoM(u!>1g-7&gWW%W z-a{olgXZ%tZhVd&#yNFGLL$L1FB!Aj;X<<`5rs++fu;r{W!I~u>A)@C4MY)LFIeAL z?TNVHx;e6@yvqX}=bW(K;wJ2#fgbNhPP4*G7_6zSb<7yUcO#Z%_6HyuH8)l=FZN-svk!Ex5YG^8 zsSexCw}juJu%1pk&F@4seL9hx8f}}3G47jM{aXn75SME=R{ajU->+<2`O%tKFEFG) zi@7njD)BKOC8romVOYdk@^?qS8+>zk=ef?i0Szh6Xp?qzG{AXu^%{Y$c@OAY=d83a znv0Djr~Sp$ymNdP!(x6DG3cdlW1>WQOs&Tu1lV}4>i?NzT;Leax2WH|>#P^XH2%E!(_J=hL{PtdY~1&}dBv~^%#(LU=NY?>dGm^0 ze|bN>_qPm>DO`KTF*;7hJ$0}A2n_W(YVrLke`L&m&sdMzzbkKkvYr+6bp8BsGKScB zhR;=y8)K~m_h6jN)Bhb5_7gl;*e;HT(VLQ!H9foe!b}-wHiv9Qi%PUu8YP zNZr-_?`8PAmAS@uUD`7IjzCN2rFCfs4Q}e+i=w(gm-a{l1Wc4@0Gd5g7XfTE*{$bu zBA*n;p6W5XXecM^_LDfth81BOP-likfaHu^lP zh9nYi-S4>M7P9-{%-)X@y zJ7*Z3;J3rN6U}Md?XF_KT=2-iW!@p4Wp^>g<{EopWgA#~)`R!PX|AGwU#jhiO_ilN zB5*C`{ioE)IY2Zm#u`gc{c6yH_`kOS;7U%gujZYJrN{fi_|O(qvGaLk!INk{v6%2~ z@lC87sR-Kq2Jw@r6POnxyr#bg(JDA>NN1Nf_q=PO@sjn9%WkfH5E?*zoa%-pV9mj_ zPE;%~8~KAHMlv)@MQ@jJt1- z8bk4SGWI(Z($aInMEFLx{N3PhE%f=l$nVB&GZ5f5mFqmed49Y2O{uXYp;gen5cUKg zK^>pf(}#eG64QtEJfwszI7Tr=rH#Q)2q&K^3|9Dj6_wR4f)1ZlShVo@*mLHD&946m zEuyE_F{;65p-Kb@#{eyI3TQuCV`PJmiC++n^uE^oiDJI7ILn_F4?mRRj5{wf55_|5 zAnKoYrAZy@$}A#epB{6HVV6n42kWALs-VX_%l=#|u84d(&Z3}@+BU3DO->F3=8Y3W zr`-LU#EWMqUJZK0-8&pqa1J}7^J_wn?grkYp#SjHr?=>mua9OY{eGI#gXNBW&zjN; zv%c50pGT{q@2R9<^aql2%x_AKGFeAh(K^NC`L~lQ?w1VFotWx za*Nxx>qpmxns=A)oqtV_W7n-{qKG1}_=r#1XI{;`m~-skh9Y`;oxiDI2d4OEDNX+N zF(-HcZJWAg4!nPqn7;$5>05teV&k}7 zpA~kT*cnwDO+ICM?Avf+m)ex=VGO+>er`8h?N0kOe}2#e5VgXmr|p`scFaq5-aT5_ zxccd*a98A<;@drgMcthsFx2r<$M{;HfzbJiOI+DP1@}a;-=7<_V%QNf=7KTn%Pw4% zUxkufd(~B0)?>z7XV+J=WAj{Wv>QoX*q zly$C%39y=eoA8)-Mcaq>V$ORaMGs%b4NhfZOj~-4iSMPXNB{SQC|mT~OhHd^@PD5& zSND7MUGjaAqt5%O{f9MuvvtPb3)TplXj+&PgIqRp}1Nv?#eP~HQ z@1~s>JjeQhSy)oetAMm!?jvBLguMzQM8N$i6(0$9Eog$2E+SFo??QtLhrA-TG&U zzR?u0({tuir#dKfvHj(o&$K@VSlO}|@kMm~C@uUMD#8jEQ4AyeIhXA$=FwX=F(C#@ z%nMq1ZTxG9onl1QH;^NONR0TQeJGBMIo8+#gA9L7>+FP!DCTIzYFrp6?3<*Ia?deN z%~hgvD60|C^&Q|HEoHU}emE}rjSxOS+to3o8X)LS$SL8!2lUB%$hMM( zQGgZgeTNpx_q0ls3TcGrh?6R|_-o2`7NG`Im~w%raS_ET#sa=(L}~Cj)nlz| zbe-Fj=o|DE&V@g{L<(^VWr;7Jf*u$$FKGGNr7(q7n_W?zB-vnj53+GPny}XVi3nL6 z)-`LV9n`3?efMe67p{s$5n_BTN)0m7R9G(Wy~FUOD&uS$dg349(W|0G#3_v=Q#bmP zXlg3njSw%1+Qc)i@UDjUi_Xy!5Idi>(sv&}l0fSfmo>*J2ylt^ywT6VwR4h-|gHP!d<5&i1Cw5DG@ z);hd~7{&V5^;y8+5&Y?wT8!ZwJ$w_C7PZpDKo%{~#yxxD)@yi*5*h#vmaaL+6n^V{ zUm`i@T3rz{hn!t91PT!_twbR-^ZABjd*stTKBC*+RRVqMO`*_ZP%M;eTx3`WpuP>A5*>?J&q5IRj(G;b=wd3R1Zsk z&4&9bzf$)-p|>W_@Z7kEWAB)&)&H8(Q(St_SJRnlQ4fFr{jX08s&Vw7Gwxy7jMmip z6+I7~-o4(q7++D_Da?wG!ZCAB327(}b$_3El*{^8`YGsywCP*diR19TozE`>Oq7^k z>m4y$Y#lK?L0iBl59OT z0+2KAl=X6S(7q6LXfiUi1V&)e5>AK7LK#SrNjHA%d^35kF z4#t~gl@2vZ>fIj^_oPDF5+6p06Dk}=Bk-wBL#pwlEjfxa*JnP7a$ z_k?^X6y~m)3dY3RVrVpe@x_H}Ta0Ay4q*X&V?kY$yjl!bt2&IhkGni;mq&wZXpqQ> zQ&+1KLE4(kU<6E*n8AxP{X@f^pHCj8ZmO}^(c(({iN%W-sExmVl3GZ=Akt?U`S7iKROUk$xkP>W%V_n!E!t_6?CWyD)pJsl^tp9Hl0vDJ+(B|GCDF7yS94_l3q7JmyUL=NMc z&P!>=xPkjnDVwd&*Qs~j{AzZwjCim8f=>{JX?m^5_r>qWjtzmP2$(3*6uCM19PAo94wO+u zT2Rn%3l4Q@aWwp+ysv?AD^7j>gKB^Vb`wTjvmT}Rwfw`ktS{aasA+K1SUJ6hOR zBT5zZTXbF0hV}7@$q%CRgzYNowBS0T3-1L5JMn^M+2x*%eb1<*t{9IPr{4|MjBpEC z$2kXl%hbKKPQ)*|V$Ik^XU*|rfHlh@F{W9Jansef{XA25ZvL}HVF3vAiG4k^d@&9- zHdahx4j)|fwHc0RZ_#e}b7Ss^Q)tx?Y5cL_8jcw0x34orKN zceee`J3HW}zXzWtnkIhB&}plo4r#+1@D22f{szLfc0NB4Fi~QDtaHRH`K+-!Nwgu@ zLUtHAM^PW~c&#R6_{124Kk->{EEC84X;C+_TAk)JhR&&{pYFNl`m{Li)9cZ!pA|ZX zoD1UUvOfoXhx1ECk3SLO&OHtE1V`}d7v2rnVKh2uuh`KSRYZd^aR&llQ)WXZU?-`flWn$6?mlvHkq1HDtaXeU1@H z>%JR#wDg+%jZ+$q9&_fm#rGv89i<0Pz;{BwFZ}+vR_D8+eOWy>5HL|RhR z?(DJIovf?8u1lUjnb-vDpA(elW%0!8_8+!K*{igu*}XxzjID0iw(U95Y&60QpIXV{ z?C3#PS%S7S^mz%S+3%bF-RE3HH0GC@o@hK=ziJqvSzwobQP?r`#b>2QWS2J4R&?I%HWScS=@Bqe?_mrhsp#CC zF<40vbp@X%r4>eIVokXRo=w<#9rsi{wZj`m_o9ve4wXeFn(8nA2zz-AxXSp6k)QYO zty}XvnLApOW;@$G`fhhOiu5*DLF-`X1j6e7p2L zS3DU9?+~=~9U{PWb&u6897gA>!5Z%i*y+2$yo%nT>Yj^R$alo`&0PG#?=bJTQQTAf z9e4$wR``91@5x%ub_7h6nC&Z=Wc(QC@cF}~iq9^}XU=2oeCrGXHb4{J%byeer1T4R z@~PN+=EP36@Aj&6{2H2VUBqBh7A<~eHV<-Ws6hm~v#^JL&Wko|4SyQZ9O)cTWZ!z* zZ9{(oN&BUW?{KF*cD>j|R8a_e7_VUTQ1raIgo}-NQKN$E?B)@-U+2ZKWcn^>&t;{D z5mZUDpal$>i?dweXqh$M@{|% z^@)iMU@AFB`n14A@&#yz6}9hXzZ$=gihuHra$=soAB-_J1tTIiJ~#TT<`F0(>CcVd z*rH1=DSYq!elR>D{DIQKz_Q9$h3Eau?|eJ_KFf}M*KR}t5!cEd=-*SrEyl@jyvoOg z#}MxoN4^EIw0b0`#7t;nFqKCW(E>jZwwioe;MyO0RE58vu7TEw4H$-E45r0564Q2k zi=v2ljaXy8A#6ElKUH;4#67!}i!y#4xXEY1y1n0{(#M^iHucuan_SKz-x{<@Z1ecV zZM}5x-N<>wF3n*COq7_ziyJ>RF$i#xn_zj_ZHD!15{)H>BDo|`qKB?<;}H>RV*1xM zM8;r8#<#tNC4g<@6U%N&k?phg5mWOggk4C)2kF-tc)-FDc?E?pnh4lA?6k4zFSEyw zp@-%{|3pw@+<+3BHT0h+t~mGcp_q(}=HfrkF)88v2qFCGD`L6*apBS&_EmLG^ctmw z2!uU$**hrr`hPa-(xbh;2e`)~I#-Ra4Lvcy>5o;`rHYul-;|zeSo9qs5>5oU+Vo)U zaeEfSHuug`*KrT;t?j~|)-GazVpP+|Q0G=h%co%hu!oNzCt2;qy}nNqRxLJq5bTtS z2C!Q(^k^gEN}rzO>WFxdbs)bAVjy83>THYn^kTIQvcv7h$Zxc6R3N@l`nyA8>%NDj z*z9l)UmD6nSo8=y!X~Qadj)nUbb-nY$Lw`P7lN{ib6`&TE#K?O;GnUb;qk4zcbEQ2E1NevoRSe4Th+diK-hnH-jp8E zt;A;__%ns_e9Oj{c2U@i3vzIuv~Oi|D7xooGxq1eE|fll+bart|Gpe+gUA|I54Jw+ zXUoQook>p=!tInuOMW}iaz)ainMCL5yVKGSY#2M1h(jR@;b;G*E$qnEn>Iz`iIyN_ zEznH3KeL_C)fE@_FsQHuO06+BpZ#ULXyq{U$?HP?)Ccd%<*UxgZ5uw%?~C1R{SE7S zU|%1$JwFEvrSbpAn-aUL76l<*st}0sx z#hnk`3FpUn**f6cgss2!?bpl;S{OYIby#r$QB1`azt6m5=)l&CrmUjhvfE%*rX7Gb zzi+;uu~E{>=y4C4q*&DmacG2j7u8*!?@Uotx~jg*?&D9%-!Xz~6SeBZ{tJhK`j$O! z_vRJ?CQ8h$^=vpB>_)}roj*^`ew9z2*t9n6FYht>gU!GWSfl2s0S+~Gv8il2#n#yE zk}?nd6IN8WooCT}pzY63&x=^CJj}vt(9-qM`4GW_J~he@^AY>_{``B-ggsTC(@)+y z<*kv-sXyN)>*&u*=T|mv?&#{Zecy(R!=A|fw?d!NQB07tmdGJ1J-+6VL`LPS0 z_ul2v;ac80*O{Nc4*Gi&qniSPvP&QQE^Dn{&hM&hydfTp&51Pw8oYv`j)m)>^D7%~ zIKQ&@IHViCXS$AMc=B#(J8J18>^SEf5nbn#SBK*c=cVhO({=8pa2=rOkDKp?KlZ}* zxxMFKELy+3`RaVM=DS+p}-jmQqJrzCXwQ1X{vaz#kowp;Iik=^MmR;wf z9c~S*iguxi{}BEygcb1LMF|XB0KIA^dM@n+jY=*PW|l#+fB*N40auwXh|Uc*r>~gKdADSI{dqwX|D(KiT*KJI z|DBYjl`HBamL9a!-wdq~;$V~a(x*qzT=$OTidgtaDCVSeD;j^*b)R0IW|nUa2DYl5 zM61vy&rlgXzAL`J6nM6o8*}&R@muHj#5;p#QVg{E?-G6@7zNcaM$5yN?`4>W2R0rJ zi>?=%Y7C(Ao^Nsu@qqj;n$}PJs@%K~y;##ctz8p3_RQ=;z(k4JwO%6_Ew<1@;@`l@#>&(+6wl)8JEwJ*h;F>eH89_Z9?Uqlg5> zXIp7ua|?d@XM#^F1~uv%BKaU1&}2QZ;Z-{1jlkBq=+ljONLzd*bS!nqu6C}lN(+15 zDv<r?7g8V<__ZedirZlpZ&~A`vI=8J}KN8ocm%;>Gh`6}?kod(I{Lj~gR% z^aJ+hwGFRfm?6KP@J7iw{!NM}Dd0+)4&9PSI?nhb6orQxdKYLbK`-6>5N!#<9D@&r ze;Cg2tgx6~pH+u^BXo|`xDZ#-lxRAxtr)mf5eV*Mq#}`c-1ADe%v~Sj77YD(rfIfj zcCT@(R(fpl@SWsufM2k=7g}y?ZT$-HzY6E6^uroMp%g4QI0}33VH?t7`vl|0U|AZU z;<}FE6Cw+LZ;6n|yURI$7gB85v0Xs84+>%|z7iZ`#E1bZ>#O(c`m!~ixIK8<5HL~V zv_XHm3G?Z?d)Kb)&oZAkj7zAAOU-9JmZhtj3oN(qufQ(o!3TaxVz2A&2{rDiZ9jAD zQo=sdA9lXD)M{{B+=w}6M^x$7jsD}$6*dFbOK7?I@HzGG>oI>C)mR7{Y;bQfPc?vm zUX?}#+f|(-wWN^K0Y%oT`CzBH>RsT^2|Il{&sdv#v|pEBd1YtKUwmt3@K7h>l(^2j z`N6iuxcPy-j|(=&byz(DgKA`%GD8=ISbkS-d`ffTn)7P0zx1O!(4x`PjtlCx^oU%f z!8oUbo}U>whMpS0HNg`fDnGgs+b9JvW|u=e*#c=nzTCr-fa<{HFX)OODm2 z1XjIB41o^6E&p!@bAxW(fXqZ{Qe4owBfd`deXsBbM3dytN89q7t4oV%Gj?CIN`Gjh z>CcZ^rgR8Hf*W`n?Tds!aPH*VXHk91ZGkk=Uga$({g9?m(qFncDorr{%>DOwxYuM0 zE0304oB0TsC^7TbXhb9A(?zgPiMfp!(f?G#)^R;0S?tAS$GRW$8d|bD)IaB#nAGh0 zwxx@Erf5FAzVqnGd|e>e(Dk(rec8FonF~8Wu;H>p9L(9zBT$zXyL0|r*i~l-Q+Ln6 z4;OF=#@ZiOlPFc99dgeLE%$DKr~FAS7%(E6vRq|`I|YxANu(_gr>fm%jq1XC1B+JF zZ{Bs-2r=Z-_rnv66ld(zn z8|7~hA_`khw2gVM!hSgS=g0cBs;wjR``A(%GwhprD|6*oU8f< zFc+dT71_RHf4}~3Um{VB{P|0Irm)lC9IXtIZ!kJh5eYUnCUO{ATzz_e^qGK6c1k8^ zT^nk417rVu%PzF{{z_!4tmmSGsR%i&S5d2?>sc5CN0*(`qITJJZ+u0&y;<9w1KtzG zCu)8z>G|syLyvEij!!g!Iddqwq|u-Bc_P#mwbkoB{){z76QR$l5f;ypw;<+O3{rG_ z)bbnD4ohmj*JRdKJlS5v>ECS3aX+li&ZdOFJ?c=k*hFEzE447idx+N zh8}DozcF2R5v{Ap==}!s<2$IdFvyJkR!I6gj~+%XF@`e!x1`QHEn=oeFZx;U88l_| zBk!JKmQ!Ou$otHiDK>fEp`1tErG;)A<(%%JfBCJccc%Y+DGCV?eqV6O&hLBpD*gg| zF9>&_W7TmZdZCTy_u?1TCTP1lc}sKzh(U-)(A0jEe%ZCE?~wMNSA!o0Vf_8}X%*39 zyAAD9USD;tA;OSH!lxO|8Eb3yI9({);>M1p&g)Y8%+pt>$U%|4(aKA z{7;(9Q{A?;AFZ(^CIu6D1Bd~y*R_U4zz7R%O)Op3D-S~=g*c^u)(?Hc&M7Uwj*W!E zyB5A1^h4y9ytUyPYv1UWI8MJv4#X5f?q%h8_kZ z{#bd(t8B2*%-|~NKgV&*{QPYQ-g)v&#Kf?d=skRQyb?o?Z&WzWDm#5dReX&6=V0~l zn<`oZy`Hyzyz_bQraqImezCs?STjTqf2U4J{&^|P zFy_(uNnO=p#eK=V>)wl^V3?%M^6lj-x~sg64K|vPvIs;Pz*Kf+D|^gufYty*7It#W zSYUim)M%mU`@*}Rw#DL_oaSlcJ$5DiRCJ=@>$B=ax+)jx`_ffE`irA3W#4lFPQqd~rBzQ~L?j~J+PLky|%Q}G8YxoGr) z1`FCz&@AgcjB@I30ZFKd3VPaqSU1_(<~PTh@xEec#&Jx6R-0hxY{&48_OZ5OLtri= zV4}oaUR|4;Az%m?0wY6U+_9P1%MY-9eWR7S*W!%-azu&-^oSNFd0<9{jCE`X7y^dC z{6XM(XPoirbDs05bI#gz&it9Q&D0Pu1Pp=si2#N@X#TKUr>J{=W~}5`b9TDxphF0U zr=EPWgB^WXaNGB7wh=LM&npPxyTEwG#hyO;ZnR;I#)mNwWY#ZeFF2<@>xuVQYeM?t zBo_`YxPHO75p5%7IyA-=(AaH5zz{G5PAdW?N}N{sjkqCT2p9sRMgXH6q84c(0@k9% z&y)g-jL`=MhZqBDA$&pAxG)MM^DP<$E%fC_5);uOh@RtL!`2`BcnqVe=xM4r#0M>i zKzgh-*RbnnOhh!oUkLvU^4nC=@b7~Wxi0*o*9b6+Uxb#_DLwsh*s&pC2p9r0837X| zX7cLT*bD(fzz}GS04|VQHf|g$_=ymJSL*)#I>hJ~8+!VXXigX#H7GH;EP8E<`^V3U zHWZE-hkq@+S2a&<-xM3ae~CooU$A3Czz{G5<`4oVO3b0fw5b>ZhJYb3hyc6gxUZvS zz(t=zc3(Z#I=r^%lCQ5cazc|p08PITiDtl0!ZSwlucJZm&mTPaE0y$cpH$%p53~eORMD{ehJYbp2+U>#Oq7_-t7l_31PlQ~AVc8befM=R?Ai8r zKiVbb8OM&*?IK?I1s2*r=NN;WtfSb2zE*+Yu4P3u1wot5H~esy{0sIkmAd43t7KBKJH(LmZI?GPc(_{s;f zXYjX($(g}$PVB$1mY8$*u3a4>VO@UZm9?I$n7Z9I1PlQ~U`8WgqQs0|9vhq?U zBZt8kwD6_qX(?h*ZQq968)n?p9!KlJ5HJL0E&?V>%-prHkr@Jpz`RF*49&+syzu-g zAY9;4jG0=M|2TTIbNHWsy}N1KtEyT196t#$W(s^FqOVm<@aD}wI3&8L;naXOBu@+C zU1NSLF8#(zk%=f+XMXiVE`aWEK@>alsZ$*(!Tse|JhzL_iaz)8_3IlOkX1lpw+#V9 zzz~=b2$(1_BbLDiVF(xkhJcU23&t0|7QcYC6X$l_2%OpnLFv z2Rc_@e|_P2eLTq9K&m;!9mYy=B0(3UqOw9@)+Z(=Dn&7$mK_@chJYbp2*e{`qC`Aa z*3u9#1Pp-@BY`>NU_k9u&mY}c% zWzFl~zqV{*qEei=^iRI7o0myj#hh$KWx@;AQ! zH@~`43xXB}5y5x&e(KBb7mQ8h!RATBG{~mH11s6#n5~7kw+v_)S^o+>8RLn zV*h?w8*)_WHE<=ORJazLYr7ac^PabBhJYbp2+RfqOq7@nt7Ky|1Pp;Wj==E`kDY&V z%a#Qc+=Pinq=n$3O{f2{T^9`aPasyT-|%9|H=*o&$nJmc^KYwU?R7k`(R&XZkaF9yr6S9TB|@zH47hdpNq7y^dC=|I3l ziPM3)(KG}M0Ye}|KzFz+LR?yLyA?|c~WW6)C8Pu@Lz67IVF^&P|lv>y~S9ok=E?ZgeDE9Ld+U&DQwtlbYB zI@BS52HKUvKHWSptYV&s0R(l`G;?SQ{C@cMfHkHFYRF9 zBz_5wZ6`SFfzQ6Fu-_tj+4Ukq+4Y6FedEi3rUUK4z5CucRMuT_g`~kF6`o(RZVL|3 zWN4W@aiVkY-9Ojm{IZQN6*2FLiF1v`FTl<&8v=&F{6fG)iTSnOHd{l$5SUR25bRa` zBN*7;18?q-i=b{kg7+?4wgC6yusF?tq-}fZ$tOD)?I0@PBXRKlw@P6rT-kX<3=E(q z7mii3$@o?1b7eKV`DORHx9F0ukI=d-EYw2m*wGOmj}QIv8C|p@%z5XwZ3Ef|G#ccw zU_3JRinjnw2L2GNO<0JoRq?po-n;+8?!LYM4b8~FSYn>FHim#9U1Wc5eEh}Z? zGz1KRIfTHnYtHLn$V7n!)-2On!}fEpj{7xs_87|K6-0+BmY(|4)@yVX33}K=kB!izZdJDt6BhFa$<{fQb^LK-Ky*1PlQ~pb!D<-|<_} zqOQF71L!rse%JmBE42g=+qh@Xf)6hG+Af7%c&?qAoa{V!@K-W68n&3ArnEV5?^_10 zE6aTA?%kEF@1ZaKjs%j`XUHdU=wE+ZjE(eKY|VHWYx%!?1E*LTUpn!7BCg?Q5#z>A zmT*M8rlpPuSUzvNYX}$uhQQ25z(k3ey*@TNL%L*E#08 zE2NEMS06DZkA{DZG;{JS&%1sI(u%(V+7AM-QqUA|9iyt%fAt-5tifC}!zver2^VL- z@4RFD6>h!l$Aq=txaSSEFe0Gqq%+#Woq6}pZkK-YXxKGFzz{G541xLxm?%*nGkd}i zFa!*N2m}Zox?$~ii%T+Y$&`;SUp^45G@_g6CVB4Eu=^+f?L87S72AIz8AW$k(U1s6 zXeWHl0QCHG#0lu%@tL3P?r=^G1XT_5Ea1W&qbcx*L&PZhI(G12hbT>G9f+_2tIrTw9|cMrQl9*Yo%yLSKN$ix@0)4;*h7(0FS zlNVTq@4Q8`YleU!kQ@OMC6YsG9U20LfFS?^6W6||ORj_C_rJA+TXa}Hv3PN|`#*T^ z>;<@E4_;fAwJk<>n1%LPYJ3UwSt2j#JyN+T8?M)c71qWDHGUO-`|!AkUf5303&;#z zCieyzwb7!WWgxc&$E1tX#%OmoZ`qO+0=9Dg@oTQh+GyK4HNFUFTnN@%=eag*C!I!R z-I_JiMz76S?O8*>5SV`mm?$y-*4$=n2p9q*N8r87&X!bjWrQ$p!u{IqZg-Y1e~#o~ zAQuB=+uy&o`_Mi2bl$i4nNchxhDjJ2?f%q{7ZzpN{aSbHwMMfM<91CHD8HY|V>dRU z8EFx}ioV8xXUlNn9!6J-7C$pHk~-rnACzMZy40YG8gT*xsL)pd<1uIpE$1&%MK8Rs z25I5-qI0+YF2&&g?c%3`aaK{gRM*Kq&ir|oQuS&57y^cXArOOri4rj=SsO#Z5HJL4 zA%K1Uky9U-Mz(HY_pyKfqul6Cc@dq8uEj-a?jY+wU2*(2F* zJ3H!`10<~TW;y-IraF$u`aI3S*Ilj7z|M!_~I8Ev-gY>O$w>*N=Oae zA`q(vPQ|bZH|nY~lm}y|*R&VTeCkw3=Z8Cd+TMqILo0Fll~;C1Q8;|pt{4J_fFUqC z1Wc3|9k$l9Az%p1TLiGx*Mb`?$JC<(#05)ruElwUdb=KyT~NzNh242H6HN;~ z2l!h&wrf`uU3@{D??H!DY)xtM+{xqb?k-w#R`jm@{;jWY21je85FHDj3%`8s^Ka{3 zzUrKTYtRVCILcRqH6$+u#!X6hl=QT3v|!u*?nk>}rf@ z0*1gmM*#O<*n0x7Vtg`cIAEvMu!-!-xOB%9L;~8?9V5nBYu9~GMpKM=aGws0t6-)5 zYxomz4HxKfXHbRsR0o@tix+oxY(HOI%hgp}*>!4s74GZt}#|}L{!n^zC%@-#z(4-tfKP$%b0EP}i#o}a+a%aQ1LSpF2MDv*PQ47>G|qc^~n2S?fFQ_=)a+96j2hY`)=#C6jl257yEU zFa!*N=|{jsiRs7OC>jEWz^q2#qsy0fnzN_*Q%5{eH)(P;Ed8@HyPLMXO2i0UcZtlC z|784CBKzER`|Ta_I1qWs*Al>nvx~m^uf9WA@uSC%RkFyp-gaAuRClMIe!7DjHqoi_ zT%oz9Zr0FmnnI1-5&oDNFJcyM*h~NP8-%S__vgB9wVy}ZyM4}-1Lc8I&H|FMrC$U>Fq!qeZ-6XCI?7M4Wb7gJG=%>v8kPe@KeV!DXBd zhS~TOZ4L%md++&q322M)*t_;$*ifU=co_DeAz%m?0<{n@QKA-H*4Pj*1m-USH?BG- z8QYkLB4WhK@$n9Y-G10&jDeP4@!ZxL3FyJPOHKyY_l_N%F4?cu=*X|3nK!<&YiLw3 z%G!SCos!!CLnF~31c6w6I(j^L;zT8`M^1g9v*s_pwYz2GOS?ypU6ea+>2LD?bPwM5 zKe`jwoYzIOgBFCbAyP2D*bL*igX{RR+V%a&gTE}A8?-kM-E~)|zE4X&imVIkjfk{n z#Wzb3;k^gmT!~A5Z*f`2fB)KUSa3zo*kngXk7MXz zY$S$K2Y;oz`;)KB3WiMDkJfj5`lkm{@xkIF8u+DZcH)xmXlb8*jqk!uH@>{P>xLUT zSzg+1;_tFkEuf+hoJkR)AfJ*l3w6gPXzJ%%Z{BvlZr&p#%?8D=Kl_ z@X_y;Ie`fot??s4D{|;dzas_z3xdF^+jUqYs-b0wb(g0duGYuze{1E~^6{~&hJYbp z2+R)zOq7@(>uj?$1Pp;z2$0?R>g%s}yWU^u?AyCHIeUr5g9t%rB$izMpSvgj?LAq# zI7uVdYV>1c80C=qPE+T>if8HQ5HY;;biK^g^$Eg$?>%s!Lj7y^bs zh5-2w2*^r#;LxFrP0bqAJb<;w2nVAb(wUJ%f`UO3?$EHvY9u5^K{fl=<|{6}R1BzK zr?tPT`g5OuTcs96`zOK>uFQ-Z_S+wK8J$NTd8Y)3#eV}4NBb+PVW%-HyXmGMDW8Wk zTR%xCt#s(fARQz>{c81o15zbDir?efj-D4m|a_YDC05X&jf;H>^knUn)dja zP{8!#*Id&fHv{g{739VW$Jrl0eXse;Z_5hW1SfhlUMGAMFs4%6IM%VJT>r{>TLVMD z5HJL0I07b0%s*S7R)xL48aEYm zk$ahOqnV)K>WuaSQM8R$}v(mvxqY8a9~}a}+X0H(4RXkVQ+*>K^*n-_GXY z)7iWKLSdDcuUy$--b&-6%a>=zdabAy8ZLk?F54J{`TZaG?3=nfc3mK>{>sIRTcxX0 zI*4*U4UDOF-troW;)Li_G|sx$$saO6$BBCwtC6X@ZeKC&$=FS_EBs_|@g_o6%yUiK z;EQ7#ou=s7bB4goMZiRfnY%VNGDE-+IPC}!pq0Wv7F}|QxL)Ia-DY6qQNWd3Gh0)W zMvibbiR6Uwkr*IF5F{NN{t>X~>o>f(s|68(9em%AN&79lRzVNZ2RHG` ze)54U)?m7@)`(|Bt*SFGSnXaugu&f=rSf^ z0TU%=!YbGp3;{!61|smj#m^M?;6+Q8bgui@4|M&KVz}>l{P(k+?d8j#)BVzk-y7KF zAK1xnND#jSSoTdf-`v@E=g-LQz29%qaUF6P#$5U*U)QCG3wB?VLf`5x9nv8U+vJH8 z9W7P#&;$@TR&#D-S_oew!Lbutn5T}1EZ|jr=iMi=R5)hN5%&-)F?`cPHuJnb>Re-9 z2M@g2X;8jP%9&4{>YUiWf5D_n%~z+N*tTQ?ckjeV`i>ogwKD_^fw_Tzi4t>T9c>nd zfFY0^0aB-pFC6QJh3hGZ`TdKr4#qi!r5;5cb8&ygw9rn(P}A}E+86idk3ri~>lkD3 z?6rxb0X^+MW*+%%@@Z+mj6MIkW>zFa**eV4_4iEUhO)zz~@A z2>ijqu})d3n>E=71f!l)Pd`0S*j>k{$uIsj@-Og=-%r_bZ|n=*92R!)k?{NF*za5K zze|s8x&?zXxewK@Wcl zg4G^9c9F~vH*#E{39joG+BvKXclYsqpnV0o>^A+Sh(MKfXa|OXAz%oM7y%O{MhviZ zZwMFyvjqVn{}5zU$BO~Te;hq3>?FZcwGf3!+q9{1(_K0K3~}WqY7*|&YWE-3ZRqeU zw*46A5V0rhoAbjDy}gT(6>RLjJ$s^AUw>RgiNckd;IgMC|IZMen>YVp7nk5?9lwZx zvxV%p-=}{}X0-6f*n7{NxJDnl@Sq(#7dcJawK%VB8tGG@(1`?CN2o z5eq1Fv=WFhjEz2G=Ihw_&X()Hzj7{l^YgF00wzk#jCHV47y^dCv?8!EyWy`_~YY*zI2P7vGwdn{Ir0=3Y#cC9-;J zpRqN&^Yn>3GV|gX(L`8xd_O`Xq`dQ3Vg>DRktYc5!3Tax<^u~a`6b9P5frd0=%)5% z)@bvVEiRbvwaNOdJqUZi5HJJ`fzyb9i4vy~d?RfL7y{FY0Cwop8(?#ojmE&}-~$gh zmiGZ?6aS&4>?gbD_yYLigMUnfsLy@=ZCQcs4$pVqa!ZGFaj_}m*sUiz6k-VFv48)g zj6NcSPrvfK!OOsSKXd>6S;1%rzXq-q(aY{Ppl6R50)~JgU)nBMj8;+9pjNNe_6B=p#}hP;lRCb$qL3V z_$yGzv|X2u-+tYiH61c0-|^|6mRu4c?eLtu59_Xcepc4}<+pXw%=mrA9;3}!yKbG( zXAGL0J$hnd;(RU6-=MSMqu(n5T?y(cg^b}&oZ9d=FPs*G zBLe@fS@F#x9{OV)-(GGYn!8>nxO?U5s1D#uMyUn>AFI%9+f2PebsqS#A#)vA<bLrA&iFr%_1ogq_pQ!Pjh(qqhT(IsEsbANZNJ-#n>o^RR~F@`8vh>}hFu;&Z`L%r-DP%I%V`_G_>(cPs zk6-*XJwFLg6XI(I!oaEcH4vo=OfbMAf{=3+mvY^aBT;NV?TC(qHsbOB_Xk5-0H1!@ zF@{h6Tw;##(ZE=WAi6%*Wpw;|F*J(WuvP@|)q>kK$GDsmJb2>T7j@APAT}YkO=m=< zqA}ZbL%v+l=SY zM;>wM@BXjMv#16K#zuMc3txy58Op|lb_0z^Hb0$bPE9-5Yoc3eUDkKiJ?5+Z==v7T zJG@@ich!BcL#xI7sd<-j*IRL@>M+55u+VuX=&rI07i~;j+ZQc4Yh=Hp%0XBIL%9dnYJP}9z(zoh(rK);iJcnbqI2L z{QkFAvcECi;*yPxJ-f}=)pJalJaIy@UW;bK?KC@A<4X3viXJ=L_wIY6?C8VVlNL?; z&#kLEboZBE@mz@*wHH=<2+^GpJt9@0~h?(zs{P!1Nh=uP&1bhQc5Fe^& z`1i>iJ5m2D)8bV?vU{m45z zL{vg6aq{@PGdfC7>J|jiNZ{gq-N$~QyYJ4Q>7M+z_vF)XBEdly!1RGHez6ft46BQX zh2O-$*ga82uPYryx7ev6df|%z%T2+JTl))Zg0T{*3E!CUi!YYQU9og%N0cixE@8iV zj`)gZhC&XDHX4xLxQ26(KnptSulv5P7!%c*Kt)@5hgt9Y-`ZU}ajv-GBR;35qr>1g zislV%i(a$Z5HJMh1_CBZ%#C%lSr`KI3<0%~ryd3SgD=4ycl?AHw!qTq4z<>);6{wV zcK{lrvbBeXjytA?rED@k^<8-I3-73&Pk+8W_O;q)1DEx@btsw-5u?;6qsjZh+Lx|J zuwG4Q3}W09CCcX0p2if#GT#n-A5zkqqH0HmfFUp!5in6=F0QK0$q<;+2wb#n2)h9W+90^!$#`{A?E%nEnu*!am=(_?bi3!ZrivZVuYF)@Y{Itp0Y1#Dzh_ zK>L;_yu*se6BFkyp!h~=ExWhJ4FN;I5EvN(CQ6J9TI<*lFa$<}faXjf0P42Ce{Gkn z$)lk(Hg;-qvV$=QyTyJ8-MA`~EgB;z z{0dUg!4PSD;aHbB>0DsPm;TwAvfGVoaSA$&m)&(|Se4PicVo?8d~26ru_ES5ONGD@gd1ts* z=Y1&~H(A3GV_45fm@1p)(L>B52MOY4Wd}7I@Ealr20k>DMVDOCxoFGFlGCX< zT>rIiv<6M0MCtPnKK!d09i{i!zyFaFeQs~HCfff{?Y+|&Zs9r(ZbR)7UBK4`trW!> zCXS3wqTyOMhJYbZ0|65yYJg=;4FN;obR$3n7a8d@ zvOffX#dw7R>nl6p-tIT4>aqc~IKXwBbZuq9Su@%#=#Y;ARuQ-EtUqr)u$1h6<6fTK zuSpMsFfZXgtU_64Osin;knkk|I$1r$$`i9igQ5l?XLDM67Bb z(6)G{2k*O2G>N>6AKP`o2a#xg)f|+Gl-u;zQ-N z3WCs4YPf^jNS8CI*&j8UdbsTnj??4M96Y(^nb8$tn!^HOS8u~$i7X) zC8A$x#3+0*xJE(4fbT&CjM?oQSDhmUK!*?i=FmKG??wxgHBNZo-ep{%C7PDbd6aot z{Gp@uBiF>J=H}C~V?)3Y7##v8N{kL$>)8-61d<{E3rN;xa!F8dDc0k>9q6XSVF6(a z!{DQvZ}{QNo*ivM)Bg2242sYM_&fQqlBBC63KF~a)C3@H6Qv55-En6mqdU>q`@^0( zSMFR^7q+&Eu@lBXtdHd5==)`mN-lMd0<;R`{aCl=c?0XQ_rL+kP_Fo)WGMgM*uXm)V%M^r%^+J5Jqx$EOU)8<&;=e_$c6zyJ{zG9w5`-A~nlXoko ze`{k17y_pU0TU%o5AH_Q5HJJ;0rh zBoq-5zX~^=mfc4^zHiSS$qm9>qaH}m=;PO1;|#Fcg#}134C}%WFa%B`0wzkFM(~Za zAz%m)w3MKoewp+lRD3b=ckYfBO<~?XF83d-uDypFdC}4V_Da2W^6~hQ?t!Cj28{uW#MGyXBn6 z##k?jwB$4*>s^pu@^C1fu*P-z?p?cD)AX^z^&0VkpsIO3FNhNu@%YcAI>yBvLp%yT z2%4Q+=ADX2Nk2{tCB7s(Z+XqMh%Sh0Xf&FN+a*)6TZVuk&;kJyC0c-Ij~W7oKpO;L z>oDY?;PWu*v{k!5_2UCO)#6X!A{V7fnHFL};8NIi-4WNCtjx}Rd?f58F6gj$7+_&s zq%>i3@ku~~KtY=jj<@YOVj;OQprv%R{@mx^*2SO(4bWq|b}i6?2-%h{#%=73hwH1f z{rZl7_$lX}J=vkOrHhR}+91`&s1`!$`PvvUE^A9LcpfbAb0{i7b2~Kz3;{!6BnX%& zF%npOxW4o!_~H-T6wqX$NACtPgT6&f}QuJ-uRNMP4K4ieoeEjjDW5HJJ`fkFgK zlqiJ8ZWscFKr;kzziswp{8Q{ilaB%S;Y0uW+anVV*u7?_9b+BXMJ@3oEpLhDV%)hz zlZ)Zx@potaD;*4h2&7rwOZi>gl%YXJX+IBx(!gH=)*fGt$%e*dMB_nJfvv@WN&A-C zvA6`)z4TAM&ed0Sn&e18<3Y3}@?+rEjfgZr!>tWs;+hrT>>_NfY=swSl4>vPpL5*g z*G`<t5HL~V^x$q(4S_j~z;s1uVMlz? zk|iBdxjCEpq~Zkb;1~n*g#7IrbXq76P4`drkhRUq*|Ubg{6@e;iTS<$HhV)LF#?VykzMx0-R5>Tt^rORt2zo>bR{}WXQ8+i z>-`k<6mB|~UJY&TrJz9{^o2`w53fO5sp;g==+RCynsu|A`jkJ7Q)5HJL$69E$?rW0+WWeCh^1lY;f^l_t{ zC_Cq*kHeKW+ci!^@WxfE7M%Ij4|T|RO@yJPA9;57=&_5Y=(ru@C71u_Mj8Tth~i_g zY*`1xCX9733d-iZ?p}8 zIfVd5JFr!hqV#7eCcJjtx(>MzuxD2`QjCvMbl&d+kpi*krW;@0^$lHMxyj#D#(QJ|hZa6T&`#BN*bm~C2CUui(eLHmPP#@RJ<*CWFK6I49sXP3hmQ+e zD#K&3D~7<_MZiRfxx2PDGecmS5pXQA#MmDtJ3?&vyKcW-T&`i~h*Crl&{0ekKZB)z zc4n4Q8Cn#{U66?2MO$9BfZc6G30OL%K}}Fqv?6U;F5H~4!$C|F_AHg4XsMaoN_ zI@M@RV%E*t7y^dC+(y7eiMhSLHakOLnh?MxcInb*bzzCuUHSZxu=Y%aoppBp!U#U8 zr+M}D*GuFkv=E^?F@ZNz&_j$MauV*md6AWvC&513rsN`(*`iCnzI*V2UmBVV8kccr z1VxU5Hgw1jAui7+Pj(O)Ld?(_5y7d6Teo7(ZZFT{UJWZB(m)g=&@U<5+K3U*BWpH> zVM8<=tIzH;@A|WjU({Gs(K$`k3x{Z}zWN1jH~k$6X-rY$cS>2m;l)nlaeIfkAf`{X zup>jj5HJMd5HL|94kv462p9sT2yD9f)e;DF_a|THTq?&#!v09)9LE-un}A(h-60l3 zn$pqLzxkfWe}4fPsFg-o*YC0T3Fd?UfYM-h8nzo_B-rD1Yo6CybeVKfjbQf)Vv1h_ zv=Ecn{XTl^n5-8DL#zMlJ6h8MI}CK7fi^L5-U7Rspruo3A1=@&$ z^?S^J;(d#s*~=#FB1h?5v|hgaIk5Y!cl0r0di2n5NKtK@(m>oJFf8knKf(rAy>)>O zL2eO|3FezB@D351Ke~K*2Qhl!e&Q2+hsflfVr>!+&AKxL<{SbhO3b+> zwFwyl(~ST@O))5e715G<4EPJc7OYryc6aB^ubyHZ9gEU4q)AJW{0C~7lUpYuhu_D( zJ#Xl$4Y|@nBLSvb+LA-O8+Y0V#1y5WHJPH@Xe7e1L5Ca<+CMc$VTddkDYZ4$fen>> z9B$kgGr%5(&3y8=_&GZg!4m$Ri75}XMFmRcXkLG-S%Aw>GN#%LWqril1G7a-&JwrUu>bI!-S?P2^V(n#hFCPPa}RskXC=|@;6K5R zI5oZ@P4a@UKJ1+1CLE5Ra}1KuAhh)@V0|zMD_S2!i2sY(YBJZ|aAGAlT3C3r6Yun};Y5sJrSRc+Gy6djwbo?>mB9GYW zwgg5>$ERt>hJYb3dIU_A7(K>D!4R0^2;g?B0WyiwgPnQmDPur|VFdQ$%HF{O37hH+ zCsNVdzx5Rz(!u#`meTR-HuMNIic2qsS(4$pKM%iuJw|Lrr}@HpR_!D2>HEdaGg=OqFrO3ce;w@DiUsStoIy!^`N zyZHLgcadOT*1=bTLPVfcH2i)LSwa>vJNFy5IEJ;r*8cEAZx?@ptp7gyiY@!c*Z-H+ zEET>VXbcX0>34?MVvIr9$){ih)nYnhh)2C#1wCy}I3L!FEY59c@GP#ozDHuo=wX^Gn zfFaNh0TU(K;b%`90`ndL$Bq#2bEu30m^?aqEhRnev#d@*udM&@UK@I0UsdNbnO4=< z!}~LNF8z{k6{}s_oO~?8(1Pp=cM!-ag=|83Jsq!VAdmmORX4t zjCdxvY~%Kffe&orh{=sjTzhR6@O6~*$h$#w9zW1$nNblDmz<$VBO?={4#8%Jw*C_A z^Dn-&yC$c=a`TMdBYjy6_F)$2Tmv06R(<2J(z=2mW5O6KoL6`bjqqA{j!!}bxiP;V zd_gck$}9Myd{h=F{s{gFRHGec{= z>xLV$Y|-M{?&hS=p&5y#$2&Ob0+#OFwylG!HNG*5RS_;kb;5sQ@BRz3H7g(V4}qQTz8wbA&`T>ky9UN9WjQO|NMCzWxRNH-T&k0(GEdD4}A7b!oJ<{ z4?mUcPcg<@L`@=64_S>5{mXC3uDY^`)Ub{@5Z1cHPBj5V{jAEe6Yl!gT4^%hTAlYB zlhzFP;k6Uz4&<(&UuY;DYP2#0Sta;vteYzBaD~SB2yK$@avjnjqc;XktQGF&bq(Zj z>)!Cu@6FhC#U`xp-S@^Wk%!tw{y_{2G17CiW@OgpJwXg;vNprdOlfZvI}E>YgT{4u zxP7}Th%TgZ(}LEgEoSXTWKSCchQMq@z(k4JxN0_LLm(Ofc95Ozwv_0H8aCXrabw5N zLfx>N(#NnthqYoiR&4dXfSsie*0>1`(sYq!8dfeej^UgXay7afXbM|Mq$4eP#z;yN zws`4Bp4~-!8t5eQgy=%}B4~fqB0hMy={;C{-j&F%vu5}Pz?L5P$6w4?Z)Lfq_6(fZ zVP?md-QTQ($OFU`7bUE4O>`d6#F&hIpCuhC`^@rcM}N&I-+xc&ag zD}iW6iaInask6uKgBJd&cZI^ zNI|WNPH(O$D2u}}&k*u!JwvvPdA zvt#@Dy&dAdy*)eYtN-ddy0|q@CNQ8OPXPAV;%?0Dcb^V5JKp4bNHHh+U_NSi=F`Dv zg7k&>g0!K97KFSX7>Dit)Q@LITF}DP`M|wz5x1NzWvBh;J9Mh z*}UtOeho2mz2nnAJ+LNnQDQpLHd=~!^lN61&$pa zG@?mM5%*+RMbgwkN9RSIGLjYI9&WzSQ5s6`*4uszH+ClqrirkFf5P%Bo}1B8da%kw z@_En1L@O3wS76?_g}?F6}3} z4t_(>bACC=<`a&CfWf?jh~nWrJ2wOjfwTyiD3KOV>&+0DK?q=v4x31%nNbc4J8^=l zV%wdubt57OTZL~yM#r%zay$Zhu-@X%tu$OsX6n><-Y{`)2Ullypmko*Auubs9#Tz> z-F)b<0!&_R4n;$xCCKeiii@ke__G_y7t)^ocLF7mA%MTYQzjGY<+GZFz4C1&KZ*q{u7 z6bKNT{)3CYwo6ITUr`?z0ujwdf_p04UDUY6b=aHLSHGZ3jDKYvaT|p#Y0>S_qZouu zo;)e;%1VctU3EmJp$A%ASaiuHoqc=Wkj)1=u;6GuFtTaU^}wUJ(H=c^OoG1pb7I{P zF$NkpJi$1IK&j-fPy-z`3Sx&C(X3(D*<%>`pxMBtpNLTYTsX!Z|Dk*CiH<;(bsQWZ z?hplvK%F5Cg405u-xV(U!J~~%Lzi`lrsdv?@r%C(m;2VZg@cj^WDYOp(#tI|S|dpJ zj*0e$K*ix$?A#DA1ZFn^CQ8ii^|R3%0woAwe1Y*y%AM_y96NDDe|hA=UzW6Tuw4=m zG;EmQoWQ2g3RycjPtd=1G3j=pVc`xQcymVU@yFjQcJn9pU)Z5&+Y~jLqi{}(mYkJT zJR)c-q9wT?)OZG(?A+dU`|U}Cjp9E*2JLW8N00umF42KfV8%}K`s=>0OWAkl&kW(m zj=M%AYC{j!7t!UxFT7)j2J<7r7eR1SL?i+;($j^w={oJ*zdxB)0^=rMBv+bf77=aP z9%zHx@UuO`GmNN?eEG{AEu%8N#K4X3Uh1esNM!8rRp9z_R&17iFIYG|?Oo z4XNn4y4MkvF{DDP;?v?dr7sI2Ecq!|`;ZRvf)*N|y5lcufBC9&2IwG?qctLZB&i0A zo~wJEjM}Wd&WUXHPBfEj=ym#v)p$y7lm7PsUnl%r>d>j%R@Teq;~>AtC~{$DwQ@tHjZ(;)th92{RR^1HLAr6Qu+Ap;~7roB=fopI0Z0tFj57C0< zz6U)+8t4+Zw~fYt-wmQ8JLpAofsXhaI733jsMtA#?fAX2Mv74xJ;aLe`MT#AoB6bS zdKfFR1%a4^c7k`PZvR8=y?0va${IPB|2DV{(dl16E3o#e?{fY*Q-yC;QG2^?2p9s* z5in7rId=A>A#fTIK-oM>}w|4N48F#rI=$m4OBne)6>%tH)1V)a4i4r4+*a#Q`O%PCx0(D9Ql)|xi zyM|~O*v%_!o8U&C&jZ))-nDBa^JRybop4xBe^-=aY^n)fI*Ndihz6wP(txE(o0^DF zsoapmCAo3css$o0I66K}L&s&2p46HdpQ;E7p zHg`k>=%Iz-_myH@v=1~Z8~)~nT_Te~hp0`sbrT9e+oY`NA5o7Wx~v!OyD9q*`@)|I z(Kftp=Z1hGkRAaOCDLPRJsJWf2w=!?`Knb?N}9F4g^TNsT^9_dplaRga0jG-!YveA zdsq_6Dd+Bpz4dVG=2u`DVb$*V^iOAWpatDgxRtZ>8RmOHbCH_WA_~!>#m~%k@D)Q0 zj#9JUu)_yF`=$Z>I45XuTOWyVWyw_#(mHtG|46P8K@7X~w%a=77g6l+T^JvKMk_h^~Zqky?ud9de5(c76@Ec{nt^w)uu1 z7O~u?!7-vUsp{BfY1()Gb5njZ&_N@I@!hDla+WWDj^wx@A{yWMQTTxP^CbH@qHL;^ zkAB~FYzWjrz(k2UXjxlBpfv)xOybr{Y28^x2c$Z~o*dSmXigQ)Yv0Cq!Wr)LQj)>S zIpY#oIs$a22$mWS2=eOtUjuBey4uhT28^wA# zR=77O=(vJAaVi&bor~6J!~FR$uTjv_-xC?TeR|{0=Rn7js)l9L-;=CUf4!iEwk#DL z`hk}CX!O^Lxo!C9_a>hcw7^j2)t?V@TfgDOBcpZTbHC7g&-?TU+b{~hFaP_;oX8V{ zrfL+lz!7@*IcedV{WApS0|F*W%!gIBInEFSU^m#sr=(^*!YRVy-*n^4d+e3_lRYZq z9C8GZ4vbwtSf7C%$8bHbd6XqGzY0&h!OczOXbf8BQD&o+u-Ftivb#jQ}|*) zhk23Df;CEQxW(@&6ZiV>9OtR+=>6XlqPeaWJNo|IQyfE!cTd*{T3|^eFTyOOpyBty zcS31VW28t>TE9)wW4=lKwSv7z^O3awpffQfbu=*`95pRS^EZTOX81(~L$OmsU|JC{ zQDR!*He&M|fl=AN=b?(Yg8le(Xp-oviu$wz;;n<5aWPm5y=oH9C&y(it?t-E(8WwDqOIV>EM0$^|UN*9y%{r$JBWx7%K)x zN<(WziBY(Y^L@ab89GG4B18b+sZ`i0_J}7$2NTxb(L$^}kvMC-(xS$XO0mNQp70g^ z{nW?2bI>EW^T{n+M$XzJ2I32n-Y6_JGS=UGK)^(a`LN11$J2}ef1a_UrHK9W2*$&% zGj6Vr{rexupKIOCf6Hp3qee?r*7=FcTNv;MtN9E(wpqXj+e zjz>Ym?^AZm`}9^UJA0(`*xe%wud*Jaj4;EzWWD-x!<8420Zj-ch4$*B%a?c1Y>YdN z5v-`rE3YQo_OXKpJ3Qm#Uv&%{=)@{<5k_Rq>$~h4&man5q!hzaB9fQ&hZc^gM0v$$ zE9nE8RrB-36rLfgI{qH452efqh|I^6Cr&t#;7Jib<9?q#b`?DvH3llvk-m)xGHQ?e=||pKi|uOyjs$mg zN_%$VQ?R#1V?ZPlmug3g9nviuH`baP>~~S$dDn@L4{L})0I521CElKRy}wM^MNEz;BUx`b8q9IZYbc zw3El*-Rq;A+!zKq>#zGh@iD^BNDHD<#iQA6(&}ZJr|jzJ4%or~pEWDKIqH3BGZKQ+ z@~#nL0gYXozG9v|dhD3Ep-;zmkMWQ%Wbc6k$+c-QbG0^xz-&XnM2Xq9Vm96pAwU4e z5p_M#Eo?Iy3s|3l8zVTuAOb@cu}@~#ctrn*h(gLN0<}Vm-P=(an!v&k6jSGfZSpAA zj=-~s0l0=j3l=oBYd6dBzQxb%Acn60+m~c>f<+sJkqJ?i$Wy@jD1O`i?nj-?`u?P> zAQ~ME;GiR{zjGhv-5_OKU9qp|m1W0%(IuC3)~x<^$=t2;!R3`q*uEH5)JN=f$y?#v z)0GB>Z_tI+Oc@ObF@^Dq?}B^hbWeT{8^=@7|u+-E)U{%M()(zAN@AYh`z>{%-t z=|~WOWqiT-!q<+Z^M0rN2_xVpS!`kLF*X?$J^nO{SQHq8jDp_sl`A_K#`s1j9K)KV zqz7w5&_fJrloo%oNfkyaxs*{%FxJ>|{r6{dphqO9)ImSNhpfz@D=ONm)Oj+%Z}Zlz znemfixZ1_@zii`6ldcu-AHE7oLu-tWQX2uT93OWXs4pC#1%8M~scFF;9zAwZMvM24 zG<>7Ts7$n>%n*zB&(Te-<%0#qUj#oH#3EfMLM9LyE9LK(z7Y461~nokMrf%65$m|c zxX#gm78(PDn7F{Qs@{CaatO{-{5T%Rsly}71dvB7tObs7bo>BO(M z9>eqMjUgR=MRa<<<+OCB3%?>d%&B)T-Mo4|4X+dLU+UkKj=DF89!==<&h>6_-F7^& zbc#n{cMO5qhk%I^|37>89&}rB9fbXww?qXOCcy?wY+gNgzcECxn!f zGIoeH)mUqSEB%pto;`* z9X7}bLo9(LfsFf{k5(H*14yVehn#RCD7dn0s;zd)G7$dO6)reeLr?&5E;3g!ESPLIqtVc<=DB&t&q5&65yB02R9(nn{c#I1j3qH8k+{5CNAN}ZUNo%e9prkz3e&RdLxk6#i>$N{y6xvB$B`vm zVn9|v6Ckj%@l1{2le~2;Ttb)+xGH?CO4fwmTDVwmJgZ4JC5woJ#B0WzWPfvQ2-i18 zzr1&e(mHtj)n5^Gy9E3e=9=rEg}t3`6j(}uO_W$lv911nL;;^t&Lo#IvL~s0Dj)x@ zUw_#1LH_%^cZpKI`(&-MTpQ2NJpT9#X#n)iFZK+0#Ot?y8$#W?U>`c8{hNTL2;z)clV67hxeSIB> z)p_B=&tFwM%60YepZ@8{R%RXe*$*V|ZRJLRjRNaZU=tvI`Fm3ny>CgqFb?uZZu`4x*&#XoH6P8Rw#(GBr-3o4vgJTQr{#6m#-uKS0 z>1OX}u#e!3zzk)|PCHQRyWgvx__Ys~JvdH= zU*pPSq6>$g{Js3@=bhB|gtwI&1vUz76u1crY@)3{N1 zVhb@DW|p*FQrB3p1=y@$L-*R*B`}uMrI9D1m3ZPcrOvB z2dk}g(LN&|_>n&vW4zQ}AZ=G=DihyjzoPcucD_;IrYNw95;sNZ4NnIJF7*i;Le0OK zhFL<#V7-Sbt#qw8emsEfNxxm?N}#WS@j1h`B`)EHLd&mJT;T%e5jopAIMHao{*C|o z#X`#Y#pf1R*b$oV6^1Yf!rHJ4HX340tq(C2gO$&`_>RB#_ktzP`C2=XZrwd=1MLzQ z(QM5`x{2%iUil@vEf@=~TpM-|&iiYY;$-Z_kz*2P!Y_YK90`jRC1XHxZREX6loH$K zxa@IK9NsQ6E^#Fhmb_mZ-WO}Lrkp&*5QIaMby0qvEo+d&+S>FZjwkIn!m;?RjiZLg z0W-!^#^_yD-@C-|PL1>K2W|q_CHlX?yuT>0i4ynMeBR@uDb|ScS-F|ytoJynw(jpK zaCLzstwY4ME_caOcL^qk>YQ>-|BvA&{I)y=`mI~w`$as9xP^}2u0Q?wmcp<5uxmSy zYxiF0??1S`u!pga<4V})!Edo1<1+`Bei!QhrE#>Q+|D-&Y!tZvD6okV_urh}qa&hS zdGq)Ej*Dp&Te5I$q8XC~h*q<`zt*|%MVdH1;_w}QYayuMFm|&H9B}OM2^FrYt#T+U zm0go-T(p42e|(iN)Vh;sm|D{sAD{KQ-WtbZTZF&Xc#P}2|K`6^*sgW8r?Q8&ThzfT z3(Oj0sbPHT{a<^#-iiz!=O6mWM@o{|^&F4jG_DpG&9FYyeOlvKsCY|5yKJ+JtO37< zVVlC%AxP5OwH(B1T&!TW!K>GyV~$;Qy)}+y@v7&G^OjKM_r3F-x7XfHIcQjoE3BDi z`@}WI(ZYD9Y!z42ki#PDPFruS^LTFe?_6{B*rUJVVz@ZM)eeg?%&omG<1t-|sJkEA z5B%UCdJt`o@2`9I$SZz%I4W{hyu|jP>-Khowuutk8n{a+;MJH0X*=^?vRih;4P0`m z?|+xT#{uZEZ~Nj0WyiTRKB1Zn>^YuG;)5%~Q45Qmb4Jsg$4kO#d+5$jy&X@U^S54e zcV6poXuf0}@_XasU%uMfv&aE_=xYs!l8DjT!Tw2LBh}V zt~9PSap@nYrUQ;@d}}{naJBD4=hpnC)?xj=Fh;ERC7v&1^C#s5Jdc*FTWhbu9fsro zQX1sCIKH!;-HjC3M2Wj`R&LSw@`b#q&hcZtgl6rg`oH1XC~zqX$n*cwZ~yk&GWOWR z!cUUcHSiTS_yNg?{e&yCvON8v|KyJ4_;!1sj5HXz zjGk8|rodRuiS64s{@4jx_Mw{I!j*|plJ}~A>TcVdJz3f$q_y2&CI+@IjX9Gc>ysb( z6T7V;3!ZoVgWvz)Uw_@}ZhN*yhOy`}+8Gxy=BGaRb>&(3Q2dCCc1_(~z4;>GUZ1rHh!SDcR}8RZa11g=Ojc!*Bl z{kMN}^xZt#mDw*71RE0zRU{}LgmCMYvD(7!;cxlES$seckBJLOuPpVKRcX$0jQtnq zvm_=(b~&Ot(UN#X%Rhn5tod0XmO*()cS7gZ|55Yo~snt zM2Y9>%x~k~C~%+vE&pr&#&76icMgrtIH0-z#~=J)x7I#U(V)xYUs%c5?bwdTJxN-z zn7v6>B_t*)U|DItwXJ^6H)s=nwsoxF@WgjM!t3WdC6YskK^{&Bh zE`-Lrv`dY0loGI>e62yyoYH)=@YKKW{qBEtCz4dO8XQYCKH1a2Xux-$5=;Kk(_er4 z8(;F0+r*)_z4L3L4~}zxgzN*&@|e}28{tI7?|#j1jCo6Z`ZZDg>{}eIVlD3Jd1-v{ zd{|HWb@j}rK6TOY!B>xugBd5lvk5l>fH)6F&L7eAlIuk`p7+mxHpJt6-l;1_y0_lt z>iN3~e4NwJ)bA$naq_%Lda}w<9eC$led9M?e*V^53HLmtO_aFjr*Ipm@;;;TM^=5UJ!RU7j`Qyj1k zcZUy%e~AxW!~hJ}ynZ$DNhEsx?)57#{>#5G9Jdl*IBs39HhP7H22PGXL$NuMJ9BJG zD@*iGR{lMeC~?F~5tftNl2``LOkyHv--mvzTiM6D^qg~W(??=seeY7o9Iy^F zxfGHLuK9%Ts;Drk%NIK|Z{EK>X$UUXi4E{ZffFdOi4rFeZELbo;O?XVqx|1VVqG{?U?q}G!}hXntc{a~msXqBobey-2vaU`1V8Y<{X4Ut_$49UKI2$2zT@pU z-uA9{-452Nz0sUY?#idTxjc9(e3@fW>zn?nC!UC~?;bHirXS8bSll^ZdR8Q3h`5o` zBRf#bJ|vX4H)*BVkN3Un-;Q|~2O%7KaB%yJSX*o3Z+_2jiS@$S%KS~WY45=qyTuo= zfOC`>lFvT#%x!&^cmC1hsFA-&uDcf>u*N+!Z1Lr}uxV^NoE*?XGhf6LGz`bIeIGao zdJc%hty5RRT2kiv6Z2q>hGKhO_)WhctP)~YJQGXcpIR3yPXx5)=GNM&INEkQG3U?3 zj`QukOPya`{k%wgx^|M}ss6s^JAd7zQ{+_JQ||db+?2!LDgE2p+*=gbM2UNAzP8>z z=P2;fSH3c~;G(s^^x&~g2d!Vx#`S1%SrXDIX^A#8#!_&k0bDudRK*9J;|g{y+t5I} zHr4KkW(~fhd;1gP`P^qdbJ+eVaVgP8DIB*;M47>dUa|@$j)XsSE+Oque~+&T*dVKY~w*9Klt@*^+;1CZVOH<+)!)IrA;o$n1d-KpTYlk&nVl}Xq?Pc-FkNxRu;}cRXlz+$ph-d-d zO=KG?{X4AFa9EpJE9c%$aRobn6OK6fuHe9SZr|tHv+fWR#c5p9s+;}bP-q>qc;>ZB zmK@Po2QP^uVS85-pO-qkX8TjhB@UXWz~9=8Q(zM%#%Z(N*(k6Q1!!pHocHV2pr44F z6H7RG?6t3jot*tl(T2{&DU5X|eA6pUujH3wbr07In}yw4>gW>fv8@%po=;DC^<{ zk+o}2PhwNAW*IzU3U(S!b1pt-2#;&ld}@@j*qmf~Giuc^g%{`COJe)Fx4!inPBb*G ztOp+_vKr2k;}EY4j0|xG@9-N5r|*q58tMo3oEXb-DaVi{yhJ|Z;n3!~Yv z5JFb-XI~4ViaK@Z^&0pR?^t=i<1_9M%gP$M33|d8ma3K33wU#XKl$|2@hqOB5}U)( z-oIozfEL$weun~^C~=1t+tEgW3sHb02i6CB(5r>buF&Gr%yES2+4+{AKCQj{`dBua z*g1dp#Dz`99)9|#{`=UCxQDfT*E=2zjjcTdXq&M&yH8|79WIGHSZ!8?B^htD;eRio zD#%!9tj{o(J6DCn(g4yj-?=(}1Ya$V!6#N=u4N`66~8i9H}4ZU^)cfXXxYqP&pgO- z^PV664`R!?gf~}83X2srKJ0!8p?(m{KiAHE)(d@IDjK8bs~-RIV1T*o;Ksu81D_ZUFqZt4tU3=w97t)e9J6Y957?0$9&tos zWx5>F209}?=*@5bJv$l|c;=`5#TqI5oQNI47D}%`^z?2T06BX}Y>>9#uOQvUm!PMn20v|JZapD9^#n`*zEfMFt!6;urto5MOAy zg`K|YJO1S``eXakFC7lQ#OL_apZssTKKGn4>g~&-zeVfs8n%Hr#CQ6$pZ|QYkk}i} zK*SY$7zEx@XG$BM<4P-mHnFEc5zVk|5~)73-;05BLy9Lw!5*R*);yh+_=Lkw^{c*@jw8{}vR<&jy&@e^&YX)=sbP*C z6Q9FUR@yu&M0ziVSVu|ov9_+KQ_Y+&y*0%`65;WMmaI^m{6ZAl)3#WaX6GGlXc=XRMESCDw%5+)CS-LFtjlzY~8 zFZ;{65KfLt){Xhh9}CRbdH_G?zsDk&{j|o3$kZvE(5$~V{H@>gAj=2V-gvWamc%z= zr+)egL(hAcC}T(aLPvDauS?)an6_M`CNQV01#RkuthYu<1oTW8m*?+P3v^Fk!2Sz= z@e4x4<DJ&W$D^VC>i4xVR^xG> zsFUN1auCCvi*I=CYj^(n`|%K2gmC9L>+w0xab@Uvva@;p4&>pZc3g6-XRJayz8Gtc zuYgO3tR>^iap<3%@zz;AKiSLndX6W{nhx1!u+09JQf41(S_yX~?c6tN=}@mr<11ou z7ke0%pn@y=xwJIjy|{E8osQ^Vjd$)kE5;IBNni7R$~Zj>31ey5&Rfo3TV9(tdlX#w zuYL~E<(&6sPgjqg!~MGMX?ISYcD~Pn)zh|K*Ric(Pou5QMu8a$Y@);rfwtQl1D zu(0QWxCI6cUb3|L~j+wnntgI_=n!##3!u+zD$phW766e5g*2c{dls?S`)AI+?<^8&`>KDeEbo zhhTSi=S#*0f6`<(hM^wquI<*+4##di?$*zEi{U%^ei&m}+f!>a>6-YIuC?dvj4C8iP31&Vs`iAy(efO5)gk764x&>ZhfAyzsj(JC;>gIyOfa6`d|IIbc`Pi+6 zx!UL7kNMcYyU&N#2LUZknZrKlTdn5`1vXLQxiZt+IG+UyaBO<`TfXo?zo*{##@if- zdb`1iNhl=aza08KTU`m)q&>`^(AFND>OA&sUmUR&`q|S*`TF@w%jCmRC7d$~?sW2) zg27RV!F`G?Y}=<3M1iriWx}ArezX|&^Sr0z=*0mc@yUwBfh1@<3*ey# zIUVsspRHO|_>uqPKR)1l;fsFJVIHe0gNL~@@Y5go55t<%ix)nYlH#|1o4>uWMJV0GD0?oP zOUohp4qdaxIExP5yTmp4ZX#P$_MIc=&=!w*e=9c%oI-(3lsJWITbqpnT@;Xyyx0G{ z1t^goA7;O)uxYHjEJ5mF=dl*qE_Uo~0gs;F$y6k}5)o#oDo33$c1qZ!aO$CU?X`4J zNmy-62R6wzVChVf_1aRHc#nU0i!Ty(?mLBs5hh!)xhv&);AUv=*TKk`S*wq~~?w3W!3u_uYUCpl${Pg}bK zx46W-v=#x|)Fm#Z{tOK1=#ur&iXt_PJRXXtXz@keBk02?o}+|2RC?B^3)39<* z?xxw;`f{I80Bf4sSuE-$wNAx<_yt0uvC8=qrpd}sLV2&TIXxs6rYxyu{_5P|tQLt! zDboxs9+uIkX-S4EzRyGxVXFxf3ts%R53B+ko7*+*Kp0JcJ zd&w_3`rKf1Xr8l9#1-t}vV!FGo+PdIu*mwf_Rss=XRpl#y3)+6=Mz{vwPeO3_hgNl z8Q!hTq^16cfB1*5;mY<0dY1XhYsw6TCW1_{TiGT;;|nhMrgjA$!dKD{?2@`h`_GT| zHE}6r^}&9|_$IbZJK!tpBGz%Cdn8jWxQM17|LmXZ!Uv~0@GP;|2!FQr>b0rP0pFe$ z`sZg}tr>Ne{1KVVB?}lh(in(0vAZ=fDbp$>9@*B2AgAD}`$e?0W?2|noUB>TbkffH zv_HofyI;dP=QuLZp<%N=&w)ABmN*sr$J`J;eTl@#WsGAPx~BSj%01$V^|I6|cM2}= zG!|Lm@~5<4+UA*(Hskegdeh?PY$=|t{xk(PQDT}t+ue-q}nymA8o+OEXo`?E0KzV||9^ zvie|WWO9-S6N@9OP2G>VW0{$>>-D?FlB(~=sfu&g9FDxNpP7`m=aa`G5ZlrfT%5p` z;DVDgh<*l)?8HDxwBo!KTm0?E*6VDP;ACt0i4XtrV0ow3KFz1Kgq`7&uk3N8N<3K- z&vPE0`Shm;=inFr)PHwdpR!(-r{J?KUbe6O$*BrBg;__$4C5h+5#2L;6uK^TItuoA zKNr|~j!Zo%y%Qvf^hjwhd#x#k5u*!Y*f?#bU>H25#?`sCIyeRi=bKA$~%OGrSOi3XQy zhu})~P~P{tAMG(sX882)*~4HFi?O$@H+GJl<@lo=wyf3o*6oNXnXEC{9`xuPAC}<2 zHWkfl%^KEt9GfFJzz;{W_x$18k?mxE-qs&FkG&yEeK2Ih8LHzNr|l*v!;ww^kBafO_XoGEn1s{JVbrgF|d9!#}5{OmGTU?Vs&gAQIE~ zi%H3GjFlV+;A_PytRAg$_J_z84tNcqU~@}*&NR)f!PCTN*KBONN=3xIoz7e6WUku& zB(SCUz)q0_?DqHM-~GKTn+Ho>;>i*|ZFU$?K1`S@}&;B>&@21{Mv#}aeY2}gud&nG^# zw{@ij$5?Z)z@cHScVeXpr8Q4nS#QWcy%%Nn9llm8jvcLWCBAdUm-HN>Q#jC+vDY|< z>h+FIJ@~-DDrCR0(sHgx2A*=y{lkw~R>JI;;Hc}`ow=nGZFOEFBGIts7~#cQ)G(GD z*Z4Tv5ev-1gsCKs4#i@mHO|;VJgysE;5Zx*_w>B9cHaBV{N{RD;*fUA z(7jU=moidUoC2t`O^a`-<*F0Sp<1Ye_V1UmeNqmUS2(aIPtvof*-n0s^>!ukQGYEF zPkZ6l^*r|>dY5&w)Dm99={=^VwEv1RifQX@_iWVLM2TmE>Ko2`lmaxXw%qZX(h{yU zJYjiQP`|ZydL8SVP{#DPr@3xJWwn!0hO^4hkCkQ&_R#h4m*0=AjuEKqPHAd~w9j?h z_L%6Gv>e)JBiQNk$;~>l8|8Lqt`Vzx9F!6-hnj>H4mWvYU$U^|eP!-wEmS_QFJWtH zdeTm};glRpj3;61lijX4l;yQ0_^{V3N!fO2ecE3l?`N3~PR^I$L$<4jZXNJ(`~ja> zW|wfs*fdt8#MPD@y3)$9pN7f8a~|;m`*j+9E@Ni#YhjO_3o)KV(%cNoQE}WGgO}IA zWvR?xu-~PhM4l9>8qlMs!egx&mZF5Obj{CNZnqlS?M%IUH!XQC>Q0E2VhzjR)HZD_ zplLy24Xfc?>X=9$Y;mE*rEs)xS?bWfy?&{4_`K^b1;2&+-gmz{p4s;Lc7AVDU=t&kA?LlGFjCPM~>~rEw%+K+xDHv9V~RA!C3#r;#jA4Xdgjw?Q&Wp+PIzA z@w~voMvsLB(oy=|RkXGZUk59p;y=R!U!LOam8h(3xD!vv@ z&$%_74sAX4&#hzbJ;Ot4hjU>Mwf8)j@80Kq2*#3Y&lbL$2sK9s<0F!_BKHy+Evd`7 za>+M-3G7qqZlc5~RNLCz2Nb{(TzVMd4&L#WuL`yR`+P}!90O!up+%J;mW@W0^q|!Vq0s?{)_+eFTB|N8CN9bguMJ=DWjcw`; zok}7ajuNGx+RTXwd{fad;Gckx7N_nbN0p&|xn^96tHoDX$SyX&#W&^%QpObtIqSaB z;5o-W_G}mUlTSY#T6gXIA{)@a;12UuVkEL|l>75u=L43P>@UY7Tk5sw%x{RBWgKv1 zePy0YCD67TJg?e|m#FJl7ufp5mGH64w61xsE@JL@-awfFCwVs)#vQ0Z5g9yt-eI3GSjnjBahC8&5PQ|Am*1P`Izq#kB#b^G<#r?T27x}Tko(f}Yw^88PqQE9fJX@6C@IK2Fu*KGrJrK_6?qe_eW#IrsJIe`! z=IEpU(1(a${o$r zMQqGIXJ|Cht}E-;9^Y8fQ)`3PJ@J{Z%oYVFXDYOe9i_7q6J=zKD)l+ySPxlsY;DxI zh>W!U@co6q_yt{Tq1YJp{dTP04x{9^d0m-nBm1LFF`g(xw5acm9fKom9KDPcel*K9 zP_K;|#t#2(tx4uu-iBh~)(qU)~t&KCC z=PoT4>9=|FZti}UoH^KhUk}CGoe6rdd08tDaDGz`R@g3HoC2FDad8rF2<|=#h%3N0 zV->KCV@VySdd>krJcCfhtKxJp=JU(1Ka*7^8xkzZ6!7LAT-c0kuarQtsTLZ)*ejvj zuj|HfTY2Ps2p8N!uG=P$J^CxIIGl#~;!Q*?s>$|S?t?jFG zY-HyBPx@f5WB2K%L1;>weg)kZjEz=4@hEA4Yp1CiDkVx4LCJw5p}6Q5W_SGb8r@LCu7i{09@ zV^!&0pf7NGo_Z}j6^O-$z7fQ$sjVd!1Kk>I6u3bOY@);s5_>~+w@|<*H$!SK-6z^a zBFoZG65nvU2>1GV=Q#2+IOGVCb=uNaR1qRvKA%fI)vPPY0xik;P9vusF>EN=W!t|z zmuxgM04>=m*7kQ;11w>}nkzZL@$1)5PHDu?xi)z}Y0}$AmKjb-!M+rH!Qv&bjxx8U zvs3U?by{o=MY-q5srVuUxX{!5W%eExm1HZ;sl>i4aB$==tX3zC&zOX~qig796^!@3=RKoh%ox7N$kR^J z$a$T7)Ear|w}0s*KCECZZuIWPoam$detcnp$s0{+J|j74MGFp9Q&yS=M4Z<6#&+_? z0tzF~`O30)&T72HVs(JS$>!+~eJFOf_KqnyZgbAEp6Y(VM=ZC4dkw4SShC|_lPfx9 z|LankiZ_4X?})xdR=9^BUW_aIWBn6py@L~nchshw35^x%8#DeVSkHbpVpDR5N!D<#b?HaR3hpq7Hxnhap%X}(6{mEDY z>}>k8OMrZBZ4~DsqLnr^zEAw*PcC9hTT!rH2e~#P@h?RCzZd^6|3Zi`?)N4u&bIvZ z&3UeeTnEn0%~NlD|;d}+B`4fp_{#3yO^kjiGD z$yt{Y-&sKJlg67Uai2`&HjEw$$T7_sU}~r89ymR9I9Fh&f?X@S@%FSiwGIs}XA+^1 zeBhTlQ!T~I=O5d{2U&)pbZl7)r>DLx#e}WXs1eJ=xb(C;wa(k$`hsE^`E=Q3_yds{ zXf1<0r@HoY>T^fyYprmq7>jLKIu;sn$wDPvt=k$4JhZ(n3tZP*din(JS^>-99>`rDrj+?@I1j z!K%sfKyZ*n{%$SvVdP}F%$1OaEICwAtHw0 zhVT&`4rAGC&-1)xm52Jf)3neBc{ty%Fzd(D79da#y>1bEua_p18l!n5k^VuC+ z2XTF$3^wbWvo^dhu+~HdKh9iAdt{U`#Lr$M_4vlpJSDaOw-_q?e zr^YdMug2GF`5S}Pa~(cwg(~YNECNE@&7Wtxr+??xVZnpXZr10*>S@o(G-9=i(_c^h zbL;pnM4FLv0o|wVH&Nm~o!D(yEd`2U{`SM)@`c4p@FzpVcWSP18a#&0F|8fEw9QYx z?Sb$!rge^AdChvgxBZ$s$J%I5TV~6>md_iLaCly>0!ebK~;4OIS3?Eem14 zguGAVOp`8mzH8cYVwu8G-wOkA)n71VX-YqMtYC)fx+e<{Etl8|=N#0QSVYWA?Q;)? zA1%F4ZX}o8U*ohRjz)jfF*c%T-T$6rY(ubQd$@_SEv-lJ^EyYm9{59b+HKkrW|p+| zwD+I<$e)O@xqgbjeT|!zF5%CW{I<`HQSuymp0N{S_{`bMibYV}y07OPb4p_gAigAC zS;1H!r{YD3us6<17x9)Ip~~>u)6%Cvl2FX}3J= zhtyZIrY#ka`Rs`?>w`XAidkrn&AB zaV7iy!hsM={?SC1tfL&-Hmim?cFb8SHQ2^5l$e$Och>Q#XE;CAVMVBE_IoUUeC$fT zUni}@%-A@_uGjclCq5#cE&e2Z%{~cn9fQ?#-P+5!NqoZb-(ChQ<4)qq63v&0nnQgE zcD>MxIElrr#{&=XQv5`Q_+n|!aeB}ej2r{86V8wP(?1oCgz3Nk{Ga)T*xpgSb2YPQ zWLa2Cm^DPmw?*p8zquyG0 z5>6e1Lw!NbmG{;xsae40bZ1=NYnK#5BUY#2$$6u;TXQ@uEUbK3ruR5M^s$d!gpM^{ zPQ>9zS)5?Yb2^#;yKeJbINn5w=fW&M=f{ZU;d4uyI25|LHi3jMrr3hzv4vXkqh3uj zTKAWqbZY6bYN<)&^DNJPU3ZP++A86bC;?_>dcjWh;_w-~P?v_l|vqua|`)O6L6cgu>REb&e2;uh-Ed zHDf|W?W{gFw!EH}E`8)MM62#td`Q-rDpPwL+xAUW+~1>-%r%;HVRac2RfoW7V@taw zjv=Ha(Z-oFvC3N`JlY0skik=VJR&(taBB{6a|^ZD~0$*}yPwnPio zNDg|`4?}gu#_T6|V4CEn>toC;Hj5N?_l8(y`7&ha=pyTMfS0I_{`a-wD>F&R7F( zXwd5BE8=VY&EaEtILg+tB0d^=8tj%{8B!a=9Kd%@*u3mXvN^E|BwpIVLlWTzA2DJL zb_pw#_>>2+q`|jj8@zfMw8y>pmh4;7;%Lw7ttRc?va>YZ*~-rQWlelCf7LW~3`c!! z%`jS4I*xrc{1zWke2NWkX*PEr+1VY(r5`utL}tCfCzZew=g=WK_FS`8tQk1x_PyzW*Hg!P-h21b)}^{H z#8r-?^|j~H@g_<mN(SB?wYGtoSVE4{LV`oI{QzHO{riIE9aP zeQkUZHdxUy$EK7EzXu;fxLo)>>;W;V#*xQ6N10n)2JP5MxAe~^bk5nV-kvIP*bH5uG*|`bsLr`RuWoe;$baPCq8}1>_m0jq2qcF1=gqtC*40&I`)7XAb+&RH4Y)#9P>D-T@?*`eUCWRL(|lp*KONlnPSB4)Q%})Kg37Xb$iB$ z)e39Zz#30H2Qjdj5Ugpvd%K;Yn<|5+90XYxSiM-5%G!?K7@h^^88d>@cFx%fSow%a#vr=^ZtyrrEiVr&hdG8##+OFH9*_rIUZ|$-t~IiH-XPM z(Q{Xx*Y~;ByYZa<49vlA?_cf5b`x}Lao>&Sn<#NN&dS!OGbkV|D#B*VD~!?c3{YK7 z!(n5uIlJbZ(*>cw60IN~=hO#=Y9;?*4t z>fXo;fXuW5;+pEa}`FoC?JfN)&%O1bb7-2%BBRd|tR^mb(}Tqs z8%%ZGCqa_SY^xL!TvfStC}gr0r+ViapmaP+e8h;U{QR6hj3?`J=;`$nuQU2&`@({b z&U+++t(-@+S+ZNQdBWz^&Kkv;X^@u0nRJXXz8vEcd_rJlgNg0wN?-Y253vyMd*|2O z7FUoo5K^A&WeL7q2$`HW9I5i+zAO%GWxTr(X7Ngq1*qK11i)B=rpfH73_JRsA^#c_ zaU!zf7#BLoLS~PU1eW7liqBSM>43(!{>w_ZvG)cRKIvzV2jWrUQYOB+_d?yqv3Fm> zHNJ4hD`SoE7L~_Xll#E#AoZsb_{dpDYn96O<=3%#ID45%KG$9 zd+}d+-h+?4;+H=-l#y4&I69(-wYDrB!712o-QtKx+~*W{8icLY z-A;i`l(^exXzO1m1>^+xS;tCD`6&CGdDRPZlUKYG{82Tb$eC=?D|rl`&m(QVI{D)V ze*|pi1S?i+ji3JD*A23>*c;*9i6_fo6K8PLw?oX?_m|%|-{ty#9!IEhHLg?eWf-)! za82zVN*k=6PZD-M!okb=lJUiSci=L{#HWw^m{Zd)L^EkHS0=t-*(NGmKiDbbqan3E zlb-+V6Q3BQ{}8xj1v5eEP#xDIG`Qe`gP57(hqc-hPm*wnE?~@U@m1r?acMinJ|>R3 zwClO{60uH&H%C^m{rJ?!@|(vRR?l(HUyM<#MM(^-9r_ZR#2bh~seSt4TKbGVfP5sVs-)~OzPS|55wweVc;mwt@OpU?nz0Ruh!f*NoWlsL1__6j7G4i|? zH{4r0Jh-TxiVEqK2a8d=K0o=JA83*I^#Ki{rJMtGg1)?g@un#`@TN* zwbzoAU~7D1qBLB=dRBBdHr5>)_=MF{*7#~RP+NVcK^Eqo^V^mop7kKw*CDbdjC5?1 z>55ZDihX(CF7`ytaG=_c)fg9`yH#5`6Yfxn-Oeq>i21qBh-SU-KmGFFLX%{@YFrYo z8mHaSv4r(FWO&QEr|dSbkMJ4?=R209^3EAu=b>KM(=;qLb!RScOiQ27A=OOp^L|cO zya&E|!{gfGR%*@puW7T6CfeaR$uoMIA}M1GPajyWFGW1*p8 zP@Z#6s@urhFHd~pqdg=pOyT2%5soG0b*kM_qSG4qz%gxT;u^R;cnNE&jZd#KGVTo4 zynYcrY(no2vwZupjfqc&pt*f8YOL_h^^dPF8o%zz-@V%tq>KqZqJ*8jYj{(~o&jxG z7$TOSsr>Lue`#FL`Az!4B@SSzmJ+^tt3u9&#I=YQDsw|bPnI*v{ds7>o4)m)*3X@O z*FQMSk7p$II@qs|Q!NDg#6n{ZtZDOD|6EH}C1W@Ar#J|9&sp!d+j=#=x%JhyJ>@)} z@BREyHPp^rr@-rKv!6dS4fRG>qth$-Zg`jHn)2+Q4y~^x%S__qoO)ib->Vg#hq-?2aNOMd-Sqfw8sCoIJ9^I1evSL} z_|||o2ji;xUd`2An#*QOeXz?oAhjYMG2djHRR?(3|N_F|#AzT^Wn{?0e9dG%n$+k7H9(?vA2+PNQ zOq?n>L?yl@KEFNq#DFAh&kSc74pFXg3Rse*B=OP44aL-OHsH($wwMgrs?dj}z_sv+ zuL(ADT<@}M_vgBL=|f~{im?@(85@DV{Ow)ub-YWscCik4r7f*B>!V-La6c~hBZgz& zh*My;nG5etJXzNG5=L8Em-R08ECyMp-XS_+PL)D@r5?!kb!iPs1<*Q^*;HzHi6rbC?-0-u~2A4`z{B zbMV0)?VT$ail>s%d2Px9B#CSKH!tanPcDDfG=jh8Z~TVXm30UPhYJZ#)eUEll5kM{ zmK7Lko7eO;aVdu~4)G=SdgcqBb>Yinj#UY7uEZv1yD@wS#0+ns~Y z8j?9IVZjGi%jRPBvrZ4b_9#eW=y#p{Iu45%uox+8R^6HVz1ygZkO+#Rr77Cfpt# zDhr8TC4OzdUpo~)l5}5!4-tWU1sG?~;e>a8i-2SfJeiJ3ursxLmhAALiS2m1dRK2(lrlAS1 zk|~F_rDv#E{7B~5Q+c2cZAXvho%Z#r7)g( zHBE3P-c)~yWuP8&;)~GH`bKq(^|qdJY|+1>U9YNiA11yrda`~HgKJzgTj_(}jc8v{ zm@KP?WRI=(PiV4zLf{gwpsr(dJ;%yreS-6pEGtF7+d_`wSttcCGkB@7GI>kH%~)d9RFLZNlO)(Wgb1$zb6W{>vgj%)`=W zKZv$hMZ}e3->=N^3uLB>j8Xf(N`~2EwX5eZ;nyWxV<`e68`d75Hbkj*Jni`uK1o_> z*4s9DE?F#xEV!P0+XIu<%WNaNPMrlR$F_3}d3@CQPkSj6a?SS6pLRqdtX}=YCb>8D z$00HMw*=o5%&B|flvrpfwZKiNjS}Bb3WKSBo^p?sQj%|DlOgaF7`<(LBXeJgn8V(N zC)UeEyR0rZ;lP#n?CWu2mlrjI&vR}Y%S8?Ev&x$&ai2}-y+0fp*|`L(YmG8AoQ3_c z`ictB!RXV&&}ut-pTQxVl-ez18Q> zc5HCjrMA~1mGEBm#1psEUq3bJcJ$5rSP65K&SAtR?6@^q*US1-C2`aIgU5#-K66y} z*;uf*uE|uxSw|w(DN$|=XPNid8t;3*>$@V;5%w}1gQ^BJ`F#8z{j0w!`lC;O;V*tc zv7Ah^#}$Y;#EOWmAr@hsSw4o?p}+Y(zvX}@>8%WFQ`uUwm!)A{K}(soDk? zD8{<;Xi;l$=JKo&2{_cjYwmk?_2x9hF^c~@OQx;8Y8r6DEAPh#v7JNPpNXMhEtJon z154d@=va&kYYT04Bum~`;&?=SB`R{7oO_n7QSY@O9Q8F8EBj-hr8d}F+a+47*1NUT zBgWJ)=Z?`)+=uua)>C+hyAu1(wLRs&xe2vMQH>baITpB6@NaE43OpMW*hGnEgX$a3 z0SXir_yO7`?C8MlRWFIJ5ITv4@+n&tcms^12{tMuDyyBMVXM_#f)5Kw~2d#A|G+d*on{|eaBOy??xhodOhof$P)A{Nq||!SToY1!C}p| zm~$d*SY!%5mch2)?fJ`J^rGXKlhTeQim2QhFOm54yM^DLvwRVeiBx_>!q(xhZPT77 zzIb=HH!pBr)CUiQGhWB|)906z`ebQnFC5LUNDi%!Fq5HWvK3rw@o4MIf@M8r|CDW=`SYD~ z(^T9^XwZvR8!nl^QiSPY&#vL$_jhck>#_yi?VR5jSJ86Ak9BcwZC`^qbzuM>c$~Ln zl1ln<-l%u6&-GF}d`+P_+-vX=SFqqYNAM9(TlPLIL1lc4@$EypIbLNJwV0QJ$FqOv zS96fhAJMS6HF=6{&9NxqvTYm|dg^`X*^zRA8enYhhY%BW>|c^UPEkGK-C#ZW_kUmX z@8duH(~F+Lp?>sSi@7WG@NMX+e{LP$udrT~Ik~9}Kso=mV!R2QaKL4)-Neq<=EVEK zK0xn%&wFlU-JH7)?&bEIC~+^(-hI{g6eX5^5|) z`;mY8r;eO0LOVVOaN_W7|3o;~g}nMXMvF0cQ`fPdIWBOB zdTq=_;&VTLAAaBa7UL7POY=*+%)tk(YG*po!O_hnyb89ZJ>a^m)@|S+-i*bb&ssa?wldt)W zi=VxKop6iFBGeZ9b1`AYmo!Xt&N*&<_X+qoG+CGF_$9VQSiX1inAJd{SmXX@pZLUq zzS)=Fk~)R;qbELwW0%ltPHnyLPpwCsww5`OoeHD3-Q%x*b*Splvd5Ct3wNwut}XA` zfAsX%U!0ba4X}(>*U2ZPzbHflZXS9Bntq-4tjwd)*BeR^)iXN0zpUUwX^>F9An8 zM+@INZX^t!uOqE&(Z(=h=l$N&CzjJfCW+*KJ7l%tdtPf99b->>@u=sOUu#~$5MCY* zI~5@8afnFeevK3BQB|8fB>M1KuG`Zf4QWKLWTt9xImcQK)gMCqxzBv&V3pt_&dM;< zmJz;AbZhtK8h0yl4B=7-JE$F4&wT1rgY$sZq(x8oXev1;|L(ljW3drBoAvq95B144 zTHUNC27m((mI*!b`k#N{&mC=_a6>fM&j)9-yxx`}uI=m9j)W1ju)p4eFGSR$C8jwK z$CCz;_eWSJ_L_Yf*ZR5SgBhVia{@cR$+ z1gE|?c)Z7FpO0E3ONX?a8|91t%JZ(l%S4n}ZxBa)agR@Z??PgMUs`Yy&JZqOmg4jN z3NfkROW1Yk-jqdiDL#D^pT@dc<8kaAHJ84EMXhcJUeBL*8qufzxi;0-vuoZ@!8ZkW z?4CFrDh)9siF2&(n%8B;G9SJtYr>s^<^6h7yv0;Mo(uOjQR2BU%Xj@4Y4ou8@d_`R zg{9ic-T^Xra(+lZ=43CT>;xi-FFL(8~m=ZBo^mf|Zk+|jCt3$1^*_0R7fEp3xnk1LPE!L8g^Cup8G zM#_5rYI+hS9+fq2Ker>zZ$;LvtfQRy6PlhdYkW1GrZn@BMQH)PTz7ER_%fT<^ZxnI zPR2{b9PIH8?Hke#x5h_grpoanHnWDz!Km@|Wc#wEVA8WLJ&QfbUYFvdhLPwPE+Qz^ zk6%&y96e_(UGjV>KJCOE2q&+%NKANhC2MlhrH7t#@aFDQ+ps3H@J;Q-GzVwNeJZe< z!nLGd_ha2nl(-*f^`09O%_z3=(ne*UWzIh|GG3#~OW|bkXnmgHw6k!}1PLczVl|0R zR)J3w%-AcDrB7NJ-z+|_Tc5!QkL>AOsS`T~7kq`)5BOkkT(H~t90)ALj-~s2*JqYE zp?FJr46BFrVHINHuRd|34*;hI;*QncH< zoo{d5!Zw>nqFj#^#Q3m)z3WBaaxW0#Qur_z{6krRhQ=E4K7~GoJ(#71_MBycMJ>ld z%RL86_6EuIIVL8weUy4Yrl;L{DIwHlUD^%1tv7Wp?B1NPlugxh@2V0}t;w-%@}}Sv zYs&+~E-?t;nTvKl!F4Zo2`@B6M=Wz&ZwgO+4_(8GRp971M;k1ra9q!~YjEU`Gg96D zlw*!z_8{?&GDitp2_N=5@RakOXM0H>B<^IX6(i&q&aQi_0hXN8<`Q`4)Ri$yY|1$> zmbA4kT!h;%X>>+i?=9>33=mt>dx!#?C~*%>*8SU6eyqK0%qdi$$tS8{DSWE2*y0T$ zgHSOp_hb<2#S2fck>yj!Kc1!?3+oqJ#d3WK8#RYFJ`V+^PjS10aj>ELdvkCK`&@jp z|F6hSGv;W7rAI@!u(_=-c+jyEU*u~qb8ukq13=XDl} zWKTIS<@Kqu2VcZ%?B_2t5V24st2mTgbqr@}?WdBeib2Vo`PQgsj`N6PBFqw&J5wc0 ztrvYw=Q23*N90_Z1e8;Zbu5mAMP#b4nL#V@^<-@tqH83;-uGK*vrI=~*1V71Bcl3o zgh^aV&%r4Pm|1lA?5BfU!ptmH&c!Ln#>HV%>H^Pz>*9^h`9^R;vo`7hsC1HH<#LJr;=}SRBpLf12~iP#r$*2VdKQi{pU(6Nc(e zU90BfBw1gh1IAUzwyn0zjm8OIWJD(qXEurXYlw18=Q?h%Ua8y zjzbC0x&sCcB5c@rs__lYZztMAwRBv``AIQq=)9fZ1CRv=o`D>nvK~)+&M{_jHms&E z-W^2Ve8v(VYn68hrA>bgFD5AAi3KyR#F_mOgHo@BQNSnM zAX20V(a!G);JTV{x(l_^e!WT$@Iy~MwfG&{g{-Q?>gp#Jr@FiyZWK6~0-GptGU>L4 z>rp@`C2jtC5XT{16$b&DE?S^AGs}3Z<#+6l5#gD|Ng`PeA-u~W^|c6gqGCtkyd=J~ zD&*LdA|Uc>L905l&?_ zo;)TR4#Awhc4sH=*z;pw+1fB)H4W2pmiVy5t@e2u*R5NuW!YX#i~2&M>vgD(-Ope9 z?mu`t6NqLTbKkpX0@!GhW4J2sD6D$M2qnzgdS~AUEO_2i#+GvvRw;Puzj*(Zb{x4_ zY7-WgmQ}2^jIes9l_2Y4_hU`ffJct8uAW&=Oxk;2y!YMjK8O%Q%~+vYQ7&)~RUWx! zjyNy%B+~}MGZ`Xk=_9AFr52`WTBsZjS z8r$w^O^OI__35N}ijdz<;1XzRyMzrQ9`P52%a7Kt6J7XUYZEaE9gTc~I0q!IrTF;H z_u`Y(QK)8(>r{LkrKX0*$5$fS7T2lxk}Vv=i3P54h0pv6^Oy7-%Z1O;Cqt9r!=hrf zh-is_iRC5njjb11q!KRn537m>kv(QVJ`OHpbZM_AK9;sRG~c*9%b9GMrnlbfqQsuX#Ll_Hn-zjf={Nj1HH^o$pTfM8&`|#9DI?!09(JGi=(p4S53ou zE>5K<80PK>k8zfDm}{fPlgEkAyA1vFdQZvWk~jp0j9b?GGY{RGWStXV9U9()tFB|M zaNw-3jUBInZwz*Oo%$z}h5HtHueNp8JojwM+?I2`_&m-7du<;D+tr(yps2@RmPR)r{L3fbCNLXd9go6URMem zea)+W&EeV-d>kweyL9i@LCu0kvdz*DeKD@IVAa-xnw_m_)*ctEX^d}@q|&2ujeytPI4=##D@i~!#g9H>Ym26wjItI-&kVK?3)rRl6bO=U6A*c zh!d$*cE3*0s>)J)wAL9b0A8I*MmTql9gQFTYrpzvJ2z}4jc@jIihbR8KZ<(J(<&uq zCs>DqC+1UpMDwSw8S<@8&w2qjoqLEfnG{)ig6ucej0LJecWo|uo`-t=^4V_J(3pn2 zZmVxS_^|akj(_~Y55^ea&1-!4$9BDrj93Lv*6qP(j4%z`}NejUUDoPXN7U=CtO2uUOBETBMEhhlkv_e@59^A z%!pXguA*Jia7>kuc;gT zi4qqe>IQNN1+bB}mBAuJ7~qn|3+rOV<5laGxYsl;1hJ@khDUl8dL>4F`$Z6rD`R08 z!jG@+&R9$K8L3Ll$txcarg{!OFgPIWG&~O@ojh%)aKF77_ne}^1 zaINcrRT`lcQ6@FluID};-kHvc)~yPo7oU38x_*0C)DC&ps%JYz38G=L*RjL)5+qe! zEPRArm%e-cj0c-Iv{sY;nfNE0k<6IA5_@Kdq`eXt z?hhUAIIs@VaS46=>TSj%Nn^RD4%l^8pt{~1Dr}gP0&?fWfj&xNUWReYM{iuNvphUn z_AJPHl5#GI&p;P&@|SSxq_#)Ch+Y-5J z7CiFx=gkFWB+lI98w;Zrj$HT47~|PkfG4ZQeC_cWLta;!8(E=@yRMfsJtaK`u_5O_ zOR9m~r8uv#PUjlp)FmcI+a+-)#z17gD(}Qh{H^cpScqZ7_FA*fQlj2|`^&}bFTB(} z>5vDnjDDQAPMJ=LqY?+=-VNsDAb(7CAWb2iKUE|7>Ih07s@=o2TL;V%99! zSijJcm;#?{Pp8t5^OX3DDY|griY!EhmhgorJ=U>_>%_mDi=hOh?uQdxHD_7Bb|{-b zada#8t|rD zs<7ed^rv~TP{gk2+E5#`A=wRzqf@j=e9HQqMW@D7H;io^_x0y23U#~MiUSuMcGI>N z>v}`SJB{`-3#)K+;sSAVOk5bk<(l`|>)-UI+mX0?tkS^srE@+2V!O0aU^NPCqQq*X z+yI?M0S*B&l`y=HY2!|VeB26akF}^XvtS2o({x<({4<}1)HM5a!b!8{ICeEXAH@*_ zKCAhumJpLh|Ixqts}9$WDxY~G*1)G4KEBB16aN4$SniVByN%eAZshnfY|j>VIbP!7 z5?DQTYkYjdGroWibdQgH501J{;={(IcG@~wsxg$rx5trSr}cg4AiBhS)*b3NPO+TG zHMhq=j?bJC373ZJT4O)@Lw_XJ9JV_46B+0uC$uZpxgv7p`e^rB8EAIx61F+p*ICf) zTVej~4xH?&dNk0vH-f&Yvt;f#KqkIAIdwYoI)zqIRF zs9AaMUfOWK6i)(PiIZS4t65eKg`R=z4-v8Fon5!dw&u~k5t2$156484XNt8_OpJOQ zU_(W7ZN`o-P%n5#utph|~`Y|j5N_8W|(+@ri3qR?^m8Gq@=CVv>-KmnLBec4u66YZ7?$B=E`n3i=`#o@q z!R8vLEKBKUU-xq@e5@Np1}r^i6pl?bo;;@UUt@gWaqQzjGR3ls9m(~ZeYjM~G0FP8 z6#N#hcS~3w+Zz`+7vgSgxrQ%*;s_**g0&upE*KV&j(j*DvWKZCh&JvXo5;YrX`Iy4WBNJ$|y_)LP4gRqs?>YXBo8 zHY;_MjPpnF97WqX6rPvb%bj+#gu9kxzQ##I-77Yk_lzY8cBr%mgCj!SRdE_=NrKP4 z@JBCoLlazAo;Z%oH>FKFZ{)mO715||mKtVGVrU|(iu$p=TU&YLJaqB&NVFMo~8GY%(>yO^)>u?*eWjZfPeC)QS`t+e{( zm{=#MEHccTHEW5ppXVXtjXcku3%^|JiO2iER)hK8VIfG1tM};8NXD*FJA_rXS?h37 z8F;RTLdf%Z_a2CM>llK8>)tc0DMa-8H|0zlmaKgrStKtV67Bm`oUabg|2eU310)!{ zWIi({%$gglwDLu&1?q;{(zP;*4cxOtflZWnmI%M0MFCoqOTVU*(lkL>Zl4A_yAEkg zv+f$DPgU)JfSsArW)9)Ro^e|7n-W8^9(E%Va_&EaKAl5whpu4<>+sR3>`%ec7aYQF z(J?YNm2+90=2$(r-~4xe*AdGiBhQqzAv9cjeA){SPIkXe8tdNBU$TYC9z|A>p^E!! znR%=Xe+EpoH~;)%(ZU zsdkt$uh@1!4u05FA}2av!?E2m9btP}?>_v)KRig2Df}GJQe02^D%*8j-}7DzHNq6& ziF@|7kS;+2*n~TUZ>{$>Yak1g#I)wi8c7v{et0*ymZHqW5Kg{Ds~vl|6nu3G4x-u; z@oY&4ywi=3_*T|yIOci(T0cwR^wwn!&G)0KpvmMVRvmql*ktY7#*t z)3pw*eLq`qQZcY*MTWaF#<6V&d&kyWFC84${`O4a_YU<=5mz)-Z>@VoVXB3mR1j<7 z+~C}yz$QxEp~ZG|;}i&IgyP42X~aH>d~UH=K2b}ZIhOR#r-YV1KUGSsjL*vwIK6cv z#C!j#^f{yfhx5^rRJ0eyl6o>U#n=jKjWzfArF~r@vhXGEbpTm18c3v=$AMR`c&FxPIozfqv2z$Qwx6xq&ikOE#~YiTWHSD9in z`34&h5@Bki&6HSD2de)>7K=l%1aovd%%DOz#BVuh+ji=NmT?E+s} zYnkkLs9$;Q(f!^W*~e@v?q}*9k?qSme1P>zyQ_g%nyjb90;cnn`l;m>hF;g3JAU|E zzA*5i4a*J7Mv7`OUSX$Q=lpauu7Z>ENYA+HK6poQc&TBr2838zthM!Z$Hd1>jO^U; zQZ}@xSZ=dk>Te4_&xtl1@MLu=eSCoBE=5=GzEd2XtU;DPR=fHqJKGT5a_lYOXs zl7%U9-nleB4kPJka_RBW+My+Uw4vtklx^u++RF+x6l)RNxD_0nr}$J`FX#H*rgI9% zgSH>xG*#FtT0&wB=Zj}1oKD*I?K$>`rWu`56sqgQc5MY813h-Ghn=t6x5wcOR^Y%_ z?^BTV(iv@st0Yc6^DjbS3qI>^D4R-+Z>dG1?HaAN&|PEOIW)C3Zr#F;xzAd5ZZ;+C zFFN@RsR20r#5Qr|{u&v15?`!^W@uRA471jsrOt#6xHwIGt-~Lu7~8|4BXOUoZ_CCZ z7)!31KcaPryv4#{eyAqW@sOjdHFPDZZ&9Co>iu7PHfv2~@sa=MzmD&W@%Tw$L2Z(K zM+es_C7}@YlL=q>2^~stoj@qDpu#J(Q@-2hcDtI>IP>X%lHLuy=yp-SSZ(7 z==g1)i6)vy2yS6*vAHzK36mIug%f5^{IgAC-aEA!&iD=@ANI&LbcriPCv?X8vS{?e9;+9Y&O%S384;9KhR=WE(vCQf25YTR4xsQ+ z32;hl`vxEJ<&>C{7FN$WhtLo&dMd9_N2spxuK=nG1PjZM6}GGK!b;vfoXJjcf?J3x zs&mxrv8J8^d#H^>zQnbmd)u{}p};0e+zg#JEJqZer4`C5*K3d4`tAAE)z3R_Uo1r&r6L1VTe+NP>_Uqt&j*?u zU9z6Du9C=nRG~TH$iYdhZc`sO)vuk}xf)9?dlqf4P~&TrnhUIDHI6*4!?Bwthf^^w zL^_UT#B=QRT)65k-1Bu3BMg25Pu`M!jo4SW zwLZt#72&jXUB^U#AyGAPwB_5r|NCzX)ve)BM_7+_Y5_4VwS!sKC;ARqpjtfjx%X{^ zs233-;V6j=G8(}@wKl^VRRioiM#prb415#e&~?8ezq6ohMsd?YVt%#MJn)nAOjV?eh{}qy=dkuYB&gNr%DS=kJ^@urhmQjq#>LL*G#E z98QDh^nGjFPk~L8=%>|I^DGMRiIx~~$oKv%z)j=A@`uqlSb@T#(V$<-pv#dY{EZ79 zpK`BgRtTE(F;S!GZuhB}lK2E*JIgN(#u8Eri!A47YeTa$w4YdsU?U1HmZS7hHiu&g zE@`8#iq@UA!Y|>?mEk0{9~U;U{?v}O6|x^rL;H66I###7PU9NB+WU6mQ0n_b$Fzqn zF8h=W)orhZ)nSk8s?er&bg1jotjBhPSp8~T97~3_m#f=4#-@bEzO)60HnOeboL@Gl zk^`$);gVIyr+BDcdyNVp`$=SL@8@}4FY7DOki{_T&6m5q7sf0&v7K3AgIPeZ!}V{; zdOY+4i(UGeyR7GRYskDNt>K~3j`)^6EqCHmw&*PP_@4Z`zc(zIteIj<>h-EWSTb=6 z?k7D@!53oszP|~lE}gyC7llWNdq?2H^Y%_(OGmfdY=39)>RGsbK zdKB11iS=l?LApi(ntUO1;VXUZ?)n$_ruwYYJgpD?I>gkNpYn;j4sH61uyX7QtH5;4 zb?zoUq2=p9oQHDQyJ5cSDm79gC`5V{fHOd_%~(EK<*}+_ddBXZa-H!XYh7EJSnBxo zr#5s=?IiXu)(p0IAA0OP;JK6nAog4+Rv~dQr;KN)4c0#W`Z=fxSvEG~(jUi5GH?yy zXs?+kV`1F_-}qR&uuDYmb7M4GeD%5YnwiGdN?TFGA3FY2iCHjC&&<&MOI))?cg}H_ zfPNh&3)?9o*>y<$SHwyTIh6H7SXR@wn1kN3R;Sm7Z_Q_Yt|uJTtkt=C+x>MZu!$1u z(sP4$L;+!5Lfd*Ysz(h^IN_rm4n*aXj+GhH$R=o7Dai~^(UQ;{#dcc-7Z$>AEF%zn zr9Pu})T3bx?P+#ori6;moV>eHWCWK$C6Nn@8|87h15MF{D>zd!c$*ALPo!e&RtRO9WzmpdM7 z*Y2uqTrvKlNlv(|b=Y~<$=J)K>PHX0+R~7rxk=9mm960E+fIbKbxYFE+|@Vx(t|I> zgKXouz()Jk^95J!_{2FX+bFFg>nZy>R)+IOHHI2SeH?p1R6tsMJx*$xbF7PXGJ#Wg zd$9l=_5D+hQ^qQUFK8F$B-}oIr!l8M|AeC$&6ULtHb{@dsEW9m!8AW z=C4L9Q#A*d#^;?E@pOBfm&S(z&y?7uOew+naGomX*C6V9&X-v|&&T^*g|LYd&()c~ z=(sue_~ib?|NUQdhXpbu3%};SyQFhO0tqP@Y_Tgn4pibD!soxVS?U<1Z5oH$tS3YL z#yW8n$Tmyx5p}T1+P?j*FL*FQABT|auc2XVdsgaeBi@2D%S4l-TSBpY7QXKd zE_gWx)nUwIF!Fj2KCuq|{D0E~RH= z?J6vg7>&e{C6oVW5?PnpjHhnIT$K5Pnc9^G18;Kp7J<=)cq zfs4HcS5_$EGj&YtHBzp*UXKed>mw2|mwQY0BGJ}ggyHXD=Q>c*mJf+so{g+ma35&A zgEs%4v!IapgyB7G9gV4e9(&m@i@MZ9r#c%^g;25hal)MHB`b)&4JF;4YIn*#@ApUt zQt(|h=@?FdH`j*4kFk0F)<_S~i|;1JrZ2|FS!^f^;9Os~`{z+$6D7{0-`0AN0$%yF zdVW5&gOygin2t>}Ni@K%Z}W<%ZZV)n_}^ZuLmM0p9GyrQUnPdcR`oOqU*$b6<7-)o zDHy~a+mC6-=X$EHd&Hg$!_-C?=$!bHFpK8zx|TCW8>iP=5^72;z_B&m-_ zzY#onxyC0HP@>+vH&$YYBU4a=O9%muR?#J+9H*<&dFr+<#Wdw zMK&cdQn_1nJ+25bEtZP<+R#q!Q~0#S%6r$qQ^U$*aZ}=?uurma<#m0Eq@P0wnrk_} zY&%wB^@;M?yvDa=OSQ4Jop@O&IHlEM1$x(3AmUc`W3KdG=7bhWa|=y9v(EE3^loDz z$^IyxIUxx&RT^LYAuCJUXWI}aJ}R-bW9(XdIX>&uyRE&y_POyz{9CcUt_^z#9xGZc z_7BzBuB}gjO_W%lrW?dt{F8@X)0ZF(&0%iabP4!1ObJkl5e)QwtWVu~`El+XFv7VVTv zyUe%0#Ybz~lWB}L8;dvQBsSDt<7+Jg@C8dd)S$ZdvTy$8+h6dre?Afd_gXGF5(XPu z!NJz|&{Go*G5RKhR0U%6&|2n1mJVOuP2LxC+xR6w|>bmAh+KLyx=obZ@Ss?vn{4r+lP3u|=9I>%(d&y((oI1hr zC2@gmZ7s2SUb;UU{Ck%In<#Pb&fQ(ybB0@=1KNlW{O|vM*Jt?D;SgJB(tUDhcjVI_ zg5^_-{aK>bYT+2y8(#NY15bS7N*F0kep}CZYAiWI4Bel*#xcof9Uj`|sZX}N{_5rL z>I7qXh!TZan`XokJ*VPttvvZMnPr+v`=X$Z;=GgMYrR2N_XC1Cy-*1mS@9O6-wjFFM zO`Ln?h~mU0?@dWy$KEZ#M0`eBm#NV{h2<58U@pMWZ_hFdksqr_9xI9ciAzaDDkA)1 z)}F;d(%Lpo1O6Jmb7A`GN^qR_IeHS&CP+DXivz6n*#6r0D`%(_amHegCJ~)*lUQgD zf(|FNKcxwJ_BiUE0()!o+@ZiGN<4Sw`UXad)y4+hghc{d5Wd0XvnHf#8JkzOcq*E0 z2_3QeJ&rDQSSw8@?OZQOgAlYx#!bV8`_uPYr`TBc;^OK zCG5Njd_>#q+fCp*b^eCv$Juyng(2lQHW6x<3o1Jsm!{T#< z3yVVa+{|G4-fK%vo#{N1eN-;N7YU!M5PT{U7DlXd{S(75=E$`K zpD~D6V%4boyySQ(J~-=l%@SP1wNth;HE$pP>7Tyc^NYO*_H#ZLQ)MjY{?*UJP2yuo zZbRzN9m`Pr9AA8shAOOl&3g2FP%*9ydIL8>flZW{AkKDcDFtMsz#7;#i*vwI_I?RI z;mg<_+A3_Kocc>bjF;e2mlhl@8eEA|m$qM8-@XnUczW2ngzfc-p zuwISrtZlono}FA-#~v*X9CQ6LXIP{$3j}SczR$ry3!367O*R-Dg}`fb_$T~cd}VD# zXgclr)bn85FZjluO_rFR@#!B6mb%#M>~C8pzO=Zs_r)I2#J1Om&iSo#;=^i>(N(r8 zeHseW4JWx0=DKt%H_w(eK{RPuB7H(L&qMAW&N1q9pZQFvE3CVv>^9X;Zmr33SJ=PU zI;)Y_iHvGgALvPLc0HgzeEwmwl}$ce43x1Dbkn4`cZO3YE{ zIe(v};Sxk(kf$}QVf#d4)$40(9rGox^JxoL#Cr9FBcIX-_`PCC+vBu|2_BWU6Xz0PYvfY zC&ZI6_BQ(xNlS~hCdWv;>t&y0UgCG|hb04JDvUk+-3x1IT@u43=iK4_{3x5A{Z#Td zC&p!8g+UL+O>ii~E@I6dd-PWvo^8%C+R9kequKTzvd*;O<38jwFGe8aro^e#Y#?i0 zsvd|_^;m|Ev9oYt*W1_(=i&>DNgdgihB9Ay4zc{=9)@t#*X)wc^3lE@&@{G-EOik1jkl)qrlxrflZXS8)xON>Ju$<{x03IFnn`- zDL+|`E{TseHFDMO@$JqpgK@`V@5Xfp=Vfq#x})74d^@@>!-s{^ZcaQH8rxD@)-dMU z?lio2#)h7AeQZab>kOTyVkdT5-yN>w_0mEdKd;9&*U!k-Q^o^V_Hnmf#^JYjl0I|K zPhAfUPQ_REx$R4y*EC2?`YH6xKJRcQ^yAW4vQIS)cJrL;=D5w+S<+-z4%f;$aJFm7 zSm)-vhI)V%@22|B?l{MP%9_6p&6OQJ+CJ7aR43C%5#+hn<(*Io%z{u zGeCz^NzokbUW>C$&Z@v@o_@2D!1OF=(~Vz#b#>b=wTZ9C?|K^I_+9%muzn;ir7SJu zEynNAbTRyoCF(z!3;Y@%&1$Q~9cX%V zN&j;GX`Umw?@^7C0JbEyt&>6~&Qo@3<^+@NcOlNl$i%_aNpZ8TvS?Hn&0Re<;py#m zjWpLZ>4_s)ojpxCx%65kas=T>v~S;zjRu-lIB`8%9I%A6=J}2_Nl5c0i&y%V8(SS} zY+N-insw_l(jx5pH)o}3dth6&>}$Pa_@_#O`BLV83F2(>BC%byho39*~Hj(st-==8c7pY#v zi6vb)F2yd-dswb>>}vOMkWnt%EOl&(t#pySG57wlM$|YQ zTQlDM&kQ7ZWtq$Ry&1dezSQ`vaa-NRoYs}9<8$$OM|t-dPtW_^vzl9Uo-)3w>)nU? z#Tt84__{l`Ra~6{n<%k5N$(ecv?vVRewT8RK`X3J?EYL{^|@`)Ri{t;DYOnh-{QaQ zvyu3er@)$PgY68bjFs@Al}|^6yf(2}det7>V0Eenk|Yk{?fkd+u+>BOYIgYmbmRa#M6?PP zg$B5c(eLeVzkO{!Y)6;(mF6#Yg&u)DbUES<_VX1m_Qyl>flaN)C(|1CbqMa*HNN}_ zBk^Hr2Qwwj%Z^{Bk$)q%~(K>p0JqL)+I);pE&_x6k#G z*AC}}1?TYAzF*qsa|)Mx;ze?6!kEMA`EyQWZ}%SxtI&SV=d7bc_`QcXQE@Wl)Rqp1 zVtj^TZR~fE<8S*i1pnM?`Vd=h78;)mtGB&aqNQSBFYK+}-9v#*l(>85;x6im*RanC z-`pieV0#kKxVIXnxuMlrRw5bRT!V5yLT&f2{T^T41c|+qPd`2{`}RFnf=GCppP5dU z3(L>4Q{$omlxQ&fGhXgkPoLl(EkBLxd;ihDdxQ&K7M`9k%w&anV^xGg=eWcXh&@Ss z<72At?8euNFTxSac;uKbKFPO>hCT3&Sxjn7yW@rU-teylE$ue0JvPqxh%VP*U%#@1 zo;19v=@t)SuLrR&@b%Is`dR9WVVToX3LOWV_8SXnhhOK`X570zoNHHvt!m>pC(0R9 z)~!o^`PTrCTsRjW?f#NoaK&rJ9D0g-W+E z7725zjQ8Gt4u`A^#J3d75~t*`p;4a#JaUf}GuOGjOVCw(i=X{I+AR}}>r`CXg(B`< zqgRflF7@XW8YZ4B+bsiY9C^%A)+V3L_NSDrdqf29`udmax87dx6!^Vuh)%5LYtXS5 z=T`6Tpui?d+#U09m-WIYm(zfd%yV*h_Y8}{o`xv2Ger}-BvEG%{I*VL#|j>-UjEuT zYn{KW?C^;_BkK56XBh8V@M}0Uq(liU6HTq3&%Qkp>UurLoSWwA5hc*?(c*OROmP?} z*+60rg<;OMt?&EvgmX->w$#_#V_Hf0Xh*~Gq-YuY#fTlWU$vSmOWJp{@c-epI`%}lZFvOy1@A>Ho`8>{$vrJ3*yt30c zT*Q8Be5!B68fzuPZiR&Rh_*G%9NU}&7MgR0NjfU6tsyqKhTk6N7HqN8e0y_$PSadx zFyqLPTQ-Ix&`7(X%i-u%)+NW7R&8LrG@@9BoM$=*q0;DRzJ-4;b<*VQgFaYo*}tfB zSj%#r;6uxBDyur}xslPz{UM7`RrTnQ!;iVB=O&JexW*zQBim7f>Vz6W-KEx58gc9V z(ejSUU(entRAE#Gobsw}T=a11wt^S!%m#d;z(NXaqQpXiJvXZJC8ueVLB=b1O$|y- zLD~yG=`_vtCq{fhU2lzJEN*@#Yg~@k#)o}sS(?PjfBcyNU)@v3i1nadO!#d%rBOWv zr|dB;jy%U&F7Zd7!w(lHg@oUhu~)<#kOlX1YdUzaN}L>OoJ(jan+tZ+Z_If?QqqJ^ z49qNFu5p@Kvdej4Ef0Ow_m(w#;<3jN0U*-tsNLdQSW1GBViJoYO(EH zeCWe6nDFb;{I%+tbMc`OkAWHwYde*WzPMli$!1osv9NB{6*`Ws59`dk$+wjCbgCIg zwN%}mf^Q1;cJJ<@z$QxEUGs4l^oI5{Kb2m+r|hD-WEeDmX^2&ZGsF@A3+iI;BH?Bk zSA>$6GCykzz;so`2HI^mK3{uB3FA8jpMIN{HVm@vNN7;Yc(C(!FZBwC`xJcUK!{$1 zLf1(4$8y-`xN@gdIoVGcAM01ZDxZ91i_W<9#V6DKWUo&hA0PHyZ?wuSzFd!3(+JPr zS2%Ti;p1KUt^aL|3k`3Mt6le}j1O&sr7r6X&Fb}H4aHoQz8H6#PyW>LW&c}8tXz*n z<3`ea4wtM?u^_qbYSuE=1R9rd6I*EV+j-4)OYm9e`cLFac-BvfEy}!}W&8eY=QZmM zmC?5{j@lBy!jbE>UXR({9LGA|kH7bQ&GMY%fKR`rY{>bHXFQQKrG93;UVQo7!D)Wt z8?%2e4n2o8N={DJvwHQs*LB+Ch*c>x(O2uHUT^K%Z`(Tey!6z|wmClWJektNye)x| zuzJcIpYIr&8fPzUdf>NptTWGnc@Rf;Nn9=N?fmYiz$QxE{d07~J@krTji7a+Ik_Zy zaSK2C-hVGxL`HgA+Dqc(paKuJLg}}}e|iZmXm4nv&c(<1fo6Noap+2@dKp*zsu05^ zN!^=jG)s5frs$*?v$KheSYEj|Jl3ynCr6ZAnY$C zP@GgiLjm(?^O1fC3KilbP|y1&L@H2KZ9u6SXw+6zMFP^IQvN_{R88ZD)Ui>BCqwWI zCYf;(6N_ZV7&9@c9Z#IZBxW3fG1Si(^J*M0Wg&pzkwbDw*k z_v<{;dGu|(+rH_y?Bgoq8#$G^?#?@&dG&1pLEBGxUB^9k zBadU^c8l#jhMvR+?WTgwCSI(BdJNOTJ+=6QU<(e}&1d`Qb79Mkjooz`QPk!2ed4oL zL+j+4&0|5`^7+Xu*3&XUWewOMvtv$Y(OhHa;zPeqh6bQCG??&Id`9iQ4zsw zuHv?gjeQ<3M~n#9+JSai<>{~c6+xF#+w~`&_}m>G(UTT+ScE7@gO;V;T<};%g?4$s z&ib`M4g(kdL^+1JR2TvMWW(RqqpfE6wh8!oxjq=%(e675J=Y96we5S)Nl(8a>@p)G zmpL$`x#L9ZoxUUtPey)&-tAcGgxRM0;deKU=T=Y8*s-@kTzM0#Ut z^ZdrCjcQ!pAC8*ja^2Q%pTifc;M2Y>n|k(M6qGoxCGNoHA^SQ7|CITO&#|JXggXU` zC~W7`Zq!trHGieRCX`IB~;3JBR+W7bADR6L&qrUIqi!&|iN;CIt_LJ4hv^_JUZ7vKz*7a_Q zHQQF2c(zm8(sOQnTy;|uo$c~$hzS&X+1T=Im+2HfB7a2oYUb~~-}#-B^xs2P9nOuJVYrpuYXGsID8IKRaVcT;H(j>wTPQ`u432EeEG(qcM*>=8~`6J?-cK&C4Pdh zf8)O$QP8sg#J;@Wr3K$6%G#@k=vED1{yEKd z`6Nap4|oK=tb?YCc(g=Y#f8=Yn>txTh94 zb8f2XQYd(0vLA4!?RAI20+_69i6{$pDk_bQ~3!1=X#bBl}1^^4Dc$b>mef*JCr?9=M7)|ilKFByhJR0#QOEd-{v`! zpdOq1Lq>@;j8S~vBMhc;u4|k`HEU@sve|}Z_wEI=?u&az{}G2noZZeH!z-@k#DE&_ zOF#6*2Wgb!)6Tn>>c{UsuXa4%X&K!~=m_t5b844h?%R^%uwz>Haq8YKllLZ8Jlh_V z7V#=XFL$pBu!<5(sP#zH#WK;FU?qp`ijSlWwl`0yrA4u+zC@F^ z6fG7X9I0%Ek0V{r7)J%zt{NqT8#A9n#OwNHPS|% zc6_5zYclBUj-41p7$Olis%S2k7&;Jpay`d-8^l+SZ;1idb@7RGWOr|{o42lu&(2d)jWI>8-uFzv<&DZ< zBj$`4hvitNQjwB~pZZVbWL{l-iy~5C$wxm%q)X>)3a9V1cT>(M_z^8_J730iDqM9R zL~WxMFh5drb?!f(Mm*z$@);|xeP6L-3D`I1 z7*8?2pZv&|#!1@7;J&4Sai0=j7<2i=7wu_1E_i9}h=D#~p>b|IbHp=z`VS@>>xfvg zg=2l@*do%=5%Ak4i~y_)_`s$;REI%D*xZ*q`R284JNHc7VJwxqMQvSl0~Zc!(=J)| z0c&cVwr!@K?J_Xx0Izo4t?P%c$#yL2x!|+IGis~?-reF_ignSJamCy@7$O6ymlt1z zLz@1&&+f4;+27bfX)9FE^>0m^(VdS9#I2f6VXRmeR`(2Z^8J$i^IltORFe4kJxtAc z8bKwVOwpCGXw|KQw}~%y>YCc#n~sx9qe*xK67@acV*wvlwP`l{2%mP|R~f9-d&9MN z(t8o3Yu}yCPRoAYETU~;j!)n1e*10st?kY*D*4Lb;+~x_i5z}|OW)CU^CO6EfhkSn zs+=QfFCeNNpJUIjU|dwgM_4=jO}j-iEeux(>NdkHGHL@%aTG%XUFzfv>n;FF6^#2X`uu z%QRYopJ#)d(>6CEteR^OyOoj%>ZSib=|L3 zSc%P?6h?-thG|-M(AH8t>zDvCc7G7av~)u(((=I?X=O zVt?@a|8R`W_|t`bPGrosb7|tMLj?G3*vZikYj$(|<>nlUDUo;%M_qfl);mo z2*16AQN+{N@Nc^AZyh}1gwyA#%ULV9@~Yj#1@{U6VqBHWrsKk#(407@d=QT9LI$Kv z_)k3fm=7qrgK{UJpkJW#x409;%_l$?2?-MTsK6e9?ZGCFw)qrF>L1) zZLnv1hESyqRgAd%F45lR@MIZ7v+QqAT*-Pw&a9Vdt{2Y?$EXvs{#2T6>>&0u%Jtev zJXrGBNiMz)nI|P~OyS9VyAE*1StFTQQtBkM_S?#}AtF@kN5GE)9-_AQTLq$fo9o=J z3eCNsug>JiHI)57W^MEk5|48SCt)Xz^O;L|JFXtA(LCapV<*iy9PquT9Oo&IDAtOh zqi_$^yyx$p@ZEa|?^=F;Q(zS(?(g}z;odPS!4ifG;SJ#4;3;;h1?L9$`H)^j z5YP=S_%_ey!IyM|C$DqTH0dba$9~#0^*CzuICh=P+Grd`Z`1#+Gn4>2#m6UX5#x9#fDK&L*9zb?a8rJn5^dVH5Pj=Q_s|D^9+dK@mT>fh7; zI5xkpi=g@bnmO)wciBfcH*hy&0x$7g)^c|_=403BKD_y-nidz^anH9nMd@aYH-&G5 zXa4--qen-z-3ZNQX}{UsXiK-xTYQXL%%i}Pv;CU@V|}G2&jx+3 zTKHvb@zLCxM}}3sx^sE7))^4ed_+LnP6PN_E#@)Cg$%&-)rCe=Huuqm?fBFkjT+Q; zJq#VT_&)gk-#_^~ir;jFPe9Wa*2weT`(mcS?u7B}*lNR_nPyjt;Pabe5O$@oWW=t- z_IH=_H@Bu+_;ZN7W|v{A?kVya%H@s?gTr%&aydnxR~XAFu!<7P zY4(t|z~XLPSRcRh-gkX|1o%6Jo$(a8_QaO$EB+Yw$SL@3e!2_tTd+avlfs5|)Ci%4 zH}}jYHdx)V!4CeDuY282L_0?WrwH6z#uGs|OUKTQFW9AmFC(5r5XJ2ro@0E$R&Vj$ zEt~4;-j3}#hiI1cG_EJU{#OP2VXefPotyY0%Mu?Jhfj#Ld5k-`0Fgw=n2(vhi1xy_ zC0Q$)S}v6{O}a&Cm?N;JQ3eBbJ9jSCrs{r51kgnHZip ziG=D{Kca~7c}YC{+5FDKFtgyij^T{p&(;cG5UP(jqou%vi6b z+^}fvoYZ;_wyY%w4PP92S3VOWpNKQx);ZPldtZ1VSa|l@E}ty+wmsL^Q{ZH)3I}bI z&u@N`dyzX%i7#DQ?SR04aSbirT))E2wZMPot#6GvOMDT@XRhIqWx?VUd=dSnz{tSC zq2)Rf=3KMHmpc$SzH{O8+rw62DL(y~&qTn@WUJ=-IZ`&|UKz(v7MjA7*Tpr~i{Q;P zom@X#9E4&kWFcfolJEG%%k z=+ekc4te1GcifqOm;&dP9! z(B(Qt)HZJZAik?aK39rLNs~2QDey2*U=<}EhFKgMgU<}sdd_8W2+)B_@CXveFvROW z+!7D8xlVD!-!FLJa!nbLTn9(pHu2Gn`t)8mvQG`Wy{EZiSYTXnI%`|u#Lp>u!FLlI z?lnGI(i0Eo3b>{Ch}R;{U}v!Wc{u_t+`q42*w0vk5Brve0J)ayGXCr?-pk%ZbS#@?x) zSv9_Mf5YIsNqp8-9-I2iTmM~#iMvQzIsQ|ocFpa&Sue3ee@fsJuo!u@yKNIy*X#$7 z0;?$T08ZL*UFPO>6YgYsSG$y*8)uD+f5*A;)$Vy0*U;vU%wjvXkv4T5?e)<$LKpSw z@zt;n-M4++z&g~bu%cUc$0;L{9)uPb7)$Yu)!$#s+1F$6ZQAxmv!A!=*CkzQoclGe zKDBKR(X>a$#5b~DHt_fRy6MB!zRlqprOgzK-Z)!YxV)Y>ujsw(?nC`5A8k_u%JvrnpPH;&f2u~>CBFOwjI_zR zaMgH6+9tlsc9}DsZr~;^^)~FtCg1b?U-tK6fQ}$#-!4Cc8+@0~^Y}J+v>E-|z*qYs z`gr*~0wMc5Qg>75683^M4oH`Cc-1qCvl`dsoQ?If#kj%Kr>U~~cjkG68naK@2S3?2^rvB745!EmTG2$^c?c)=W(l-qZn3O&{4GQWH}w}u#O ztQp+RxsH~x#nD{ZM#@zu6Da3%v8p zknu?7#HEmzIEXDvOe`gUwr#x{4uiKv1t$ZM4pfg@Bf}Jy5ltP5xk7bPVv5W z{tf>eS43@V#>yRAnMi6PKJE2wVl1QC{8kub)mLp4%YQ=ID}MefZr8{5jly5+trWN) zD6onW_roj<^}**x^dZ-|BFSV&1c>8GnXlBat96AX+Fn^do+R*cSHea zwL9dO5iY$Qe6RSqPnXkMHz(N6^;cQlXQ%hh)$ifMx?)M@+SO(JDUv2j@nMD8GuQAA z-SgWa#<;%jYkukE_=E(=dXW~l+7K-cA!y3AH9qdoG1jL1%isPVJ;psh>m@A&9X%#8 zT*8USN)0ZL+}5K%+kPcJ=_h~mPcEis;_Dr!$Il{J+^kpQb1&jlr<;>m3CAhrS&4o5 zQ{_F(4Vy!jeW-d}v>zj0^u^P6lF8qKkHT#s8iCO#>4bY#6GK5c&ZLm%pn(e8(6I|-ji z?d+2{dt^I@%KZKh&6g926HX7GprUz_w|dxIXI_A7Ov#2T8#!YF?WgcjMeKIOwz%5g zbge#!_sxL;PT{kQ7SYNI9{m9~$5n$&u_pUmm-X&ze1ejG(-TkJ_U@?PRa3aYi-=g; zK8-{+s__v8Gtyl4SA@ORjlS-8oO`jwHxlf##+Nvx0(4Uc9C9m>TEOd%tnlP2gJEx3&I* zNr6?Acp&DDcI2tQ{_F0z;wZ8+_WF%)esc+e*BD%!37>fC8MTJymi90{HD|!b@F9YH zmd}G@oqTS->%}iP3qG&ujEv&=HJ>AaEO&9?PdvV0oyxed%Rb9kxt&2y$zO^N`#Tre zBUyWFl!!q2Z0=tE=tsM(>{K5F^b{1>=bHw$tp`Wa?a7PJx}EXn;i8FO`Z=uG;NJ0G zBH_duEbyZo{mLbR)>1aX5mgCGN|b5)E7DHQTC0a1WgZu?Ff@%B!6$J>4F84|BAV2! zAiDd_w{$=BYM?e|zjCy5`Z7*{@6;6Ij;D z@EG%43hm$UhR?aOCaw88kg6za=DI%OjP`x>m_hG#xQJh^h)mq8#WFiijPTHhp&^_p zN}BK3ovex1xzG1t9mUEH03)^9V6JyWDHdInp?XK8gJLf$`&ZWVZS!D` zT&a1Q4D2I3UjJAe_vgarv+Q$=Rim+Y@AI@|BYSkzY`n1;*b;1^&x)~~OMCCL4?b2I ztAYK%7A#?*M#tu}<-}-Wpy0oGx^`$EM}aJ;=k!=(V;EsPf{1#CdjXi?s)^5fw7cVX zcTqm^BRd3BNCaSQ4LGtcHqLw*lgo*}S++FAQd3T@z}KKKV8`N$AnhL=QhZT*IcPc1sXwKP&m<0N!v@M8Vg zb;={z+P3YpcW#cY4ZfOnlh%if^s@d`>U3p>o{6s&DTuLD1~K-sF6aDxrf}5x=m8c{ z#OdJ9PRMei9-}wqjH|xp+kX45EbA;!9}j$Cc+@ne9?NmtJ!p!LdQ5oMn4J$Q2|nRhxkF<^N8*Ja+e=zsc(947(ZdeQI~L|#yRZx1YNit3E}d(5WLfTj>~Pe$ zkAykZjAkFp+r>kL}^UL44uj`&`EZ=6J9=;l8+CFSTa#nnT)aPTP%^Shw%yC-tc zwr9Q3l;)bY91K_ed$Nm^ArZV&7)&y#;m4sa@rK`-%lwb*kk|<_ro6PZ6C;F3?t>V z5)L-QCO7RG4U~0~<2wgF8g}r*m!Nba*t_zKwd;MJv!BPNIbZ2c>$d=ZJ|7Ers$I?( z;{nl$f|V)rH|L5v1#2W9KI4fwEJiL^5n{(NwA5dbcIM_i@#Px9E*?9!)NY4g$i!gT zz(43{nW1+rm1&f7*iD(Q}Y22W3m*b_@U6^ zUlbLI^-07Y5h08>4DHO@_A)L@-w4t7s+K73z@uw3pl z8djDuK&^2F&eE5;^_=~xQ!uC=TKMqzeNn&ojs3Qt`}CFfzx-o<>+ni{iA&d;E$~A1!~Bm38E|xd7d*9H0JrPco3RTb#6`?rGo4z7@i9sIx?5 z(y!PNEA6!PuJV3NTv+^eS1ZS27sam7)bA+kwJB9j!q0W!`K1s2v9tI9`d!%Jv0jK< zc2bBmMLfA8vb23&pH~W;iUO-BaVkol1r7s_VB4yV;d7lSfn93tki80zWi<{FBtMI6 zslDR!({v7eL?lKdaEO4y2*K<9T;oeEoJd$ci*}a&<){9qz;_*Ek|~-p$VvLK%TFUU zV_q69X9`E1ALk@&*-;zpX_ymn=j)0my07u&IHl)uFHFCN8W&A4V~!IIX%CE`ur)6C zg@swf5+*7 zy>(D)mDQs~T>`(Zd$;VqaCuIA@msDdlzZb3AD1mXX_%WT!;g0S5|^|WY?TpcT}PCl zV^sz%zK?(UFYfg{-EHB+7Q^M9sC&##q@96Ul~WTYPkyG!f6x2g7yW?4`-PF&F-Iwa zpF0zAeo=Y06DaCd<|nS~3!Fj_7-y&zvS1*scZlsmgc0a=7bM0t;b+1E!1#|jzA?w4 zzkXj*hQsIZWw2Z2`w0it6>*}7VzbBM4UlcEvQl6X1y)gF5xG=Oms79Oa|QOKW#ZCm z!&1!c{Oreh4~u&|-vF+j_6g2&;EO0R1$K_9PJ1q*^SwV@I)|_40(%(Ze2po5KG$vJ zo_R=PrBnFgsI@T=i)uZp_~y>p=n;^y5jNh%xS?kwt=(t;%wO2m%lgtKnpn2cc8BWU zQpCjXbZjbW+K4M_f{SD6n%}Tfn6o@nn%ZDHo4WeBWPOg}>(xt|EWszpaD7$^7@`U} zBf~myftNdtiK9-5pi9KTsR(5?OzWb4Zts`8&!&FIHOxc!s1$Op52Zpf5HoA|SMJL< zRi3kBF19`ybsfG~OQJ{FB_(3jFzmehuJL8_edVu5B|^ni?^a}+diUhe7?s-SNO z+nt)tL+#l_6Xn`X|7?eq5UD1g>r@+Y;R9#U{hZV-f8d9zb`CQ5Eq&;nwL!OaBLB zH&s4%l_Er9evoM>_HO2)~cE@Wjs%c(Q0D9alDGkKUrhLfRSBW=S7+xQZ_U!-^IzSiSu0 zmom8dBD8JW=S`c(@1SK{=6Tca$GiCI`U$7b>)q$rdS#n26|FDL?}5Kjd|+*G?gKPl zG%~)NpB_!K)m*=`&L-W=cjm#9ZF4?yoKtHlY2NF1!p$`6f|ct{yLLX|R43_ye)?1A zCCo#qws2_Q;%VCko-JOuN7rQwXX&$k&CTahcx(NY0xJa`YznNR#DhJ5YwuSI3{Zd( z+lcWPtt1<7@!SW%PF8WuQ`iYl*4Pwa)zZe7kwINu;7L0UBGuIS?L7^&h%zJiXo=Np z{Va@|-M2EAvk6)}`OI!xoiJ|6%&Bw&AA`cgC8fol!f7tlaaP0>nq7VeAC@nJj3!QW zV63xix4q$G!1!YnpP;;8F$~F_fZAA+k&p5aUrHLTuocp;{pQ~quCcWItv^RqZFkF{ zG{?YThJRPuiDTQjj@u2&c?MTB8vtnRW0#@Ovh4UAre$dZ-oN@+|8vkoR5Uwp@H3Lh zlad*t8GrUG0_&E(aQhBAmYt0`BbVLawl^7oHsMJHF|F&NHJA6gPTtQ2lC5Fnec9c4 zOrTWl1yJQ5)Qf6V*HhQ}7ZQZ~eZm?|8RXf?JapV@)>HQ!#-XGAUF)qBSSc`1fmM{4 zr_WltQs5z=0M?+695Oc~ELvfYZueX`#hP~{*Axo?2m4=9eJV@f!BF%^{ z*uy%KRvIs5zZt9af~nT{(0&5VE<5qHQDH6`yYn7C23~wR^aGUhv^K`=GECJ2m z_=qbuHZ$?@?`ZoZkY}*dW$eBF3oD_$$X0FJ-gDw}tXRAL?v`jv^{K-xJ+YJ-fO^FL zWP^X>noEqx{$Qy^#}Um)-?9DNSRGR*Qe0kq50shJJJHPElsJfjDytSSyT`0xJdX9R*fV;@;7I#k^8rkOEjGZ2b}q=OE}S z)o63EPkGIqDqJ_F`4u<4Wy|Y2?L8JO42udrY>db!8PK+^+dk6*J2wMcHyXJn-F%G= zi&w+M;tOm$w|gTX>H**zB8EtkOxyg7Em$cn&szKT<+wAW2eJ<*{?um-liDSozd&HE) zWoN)WcDr}(0bF4eRO+b5mAHl9KHHh0sZSi4+WoLrzU$qui8AxcuuGmx^e*Gd`sQ$M zUDbGri`JRnlJOFmtQU1egt9}CaN87Ztm}HM)v%Afw?juPi_^8Y^rJ=~QXdO*ZMRZj zrNF~OfmM`vc;<2)$wNy4+G8%3#~KtYB@B%;HpMv(OWPh(KiO(Ay9gY2HJ+@ zSw5VujOx>=wwLu(iC_dDZGX*D#%b>*Ul~IcHt};BVSzN&-2>ivcxsEU4z_A+#0-XA zc3i&i2mVOF5)72a8*7>dUv>wK@4fH({G;$;vx!G+_XXC~MdQ*hhFVE`ec*fZ!5{q3 zBai5@ajWhOs>*uT^S?9dnUxFCzct{bm7=Ni}9z3_AYlUL(wvI*xCm8Y$ zxj8ckw3h#u$8RnA&GL`&yGuIhD<6fuPM!FHVWUW6#MULMhZA>W1-DXQrND_Ou!<5V zBIF8YrNDV8K>JN2%@|}U20IySsl&)*uyGin6vK+N&+awj%8D&)MU61PNu!Kj!zK^g z!l;QT&^sZRfer%_!7dqk6hp2(8sw?oPB3pkUwO3afV~B$XA8~kmzYo+l+^g_proM{ zt@9NGO-A+&om#^r+U_*QnZoBp>MR?<7w)!YY%MKGZY zXH0fKu;}u!(AHg?Jy4!$_mf2YQqo9BKgQVKjbRgeb}VxQkS%Z%7jXpq7vA$&EW@wD`&z*+g>M;&NQr42r!XVq^XutPMK=g7UVk zc8-teDbTSrb07mC2{%sFvgx36pa zl>#dTHWXMziFGBc6u7S`P&BONv;Fv;h?a8$J-rBbX#qnYr?nnp88&^}wAr*D>u8QG zBK&mlUBbPDSNCtK-DU0lyGwYRzSgDteW^=s-MT0=aV5TS+Fa7)DlM}Q(nEh)_i(+7 z=;waDdfc1){qn0%$KzXyYmV-{eu1w|jVI^jqBHgyMvd!GUF~`};L>)}e)>0(i#93Otf9>X_RM_C#sC44C(Erx%ayw4n@Cpe?{ z{1&W1G<0-dy(`G)ViUz^vl9Yt?=O!vbbMT|d^FF_q~DhSvvxXaLpd<~Hp@|m@mo^o zy1|^mgd>977COgo`y*c(F6QW$QKaCCC{`Mr;LrNweve-DJiBL1Sj4Kc8eW}X2rGz{ z0*@pGR#D=Soc49l4+aI;F^jx3!n)>&7zk{Q7H)@;OcoqSi31TgzkIeCMzz(k>|odS z_v&Eh^0WW^JAd~Oo2?zU*=)y+R$l&LmZ+{ z&Ys||EYN5IGPXR%FzaU7RBA0d9?|^g(RA=(Iqif5t6lJAAFQhsIrzsU9_bVfrf^#? z=Fsngrd~a}gE6nA4>fG}*$4OfT6QRQW1qq?nos2CKUL#Qe8v@LyNm)`SdVMAcOX2q zIKB|fT79L!N`W&|U==0KOyCv#N`Yw##Ib18#aBw~GB{=TZlif+WPnY~Ys$5h5yTas zoeV54(Sk-4+aUT+YF}xrGd{mESJdmk89Jf4w8NSuEU6Qh(-riGKlGu!GUeFC4!T<- z_|TBHv&JV_CwOsWUJ;4vvN?Qd$KLBp=F#@I@Dd+QIjwG6PP?A%>vUv%Tz3;*7^Zyq zOYS`Ry4MA3X$}MScHXjH?+gLKQ;ASD{@y(eZo&yS;Hr{!8XEEjUu&RaPQvzHy#s$W zz9*je+$;6ttc(VqogpFt`Z>mpOLO-50J>6qDIPI4tm|X*Yj^b>%m)8nehr{=Uym|AqVc<`+V^hjb1X!q@*8k5=&IrBO9r%MoY-Dj*27WfF;cQF z_h?d$Nxq$@C2ENpe(&DdI8+_+-l%ZdHG1!N{(;?YORw#9@`MMl>qsPY%%0uW>%d#v zt`xXmDX@wX_v_58{d&kLKnp&ijpt9mz@|qIHq;jb20jcTh#;-8P!A^7Ni?8j!QO{a^WIcZf4$_8-C*8t4M!e|_yQyuw;0 z%uM5ahEg|nVYZo}H7}T5G~ckp3T{t(}vTiVU!;B)2x(ZgKD z3DHxPoBTYfS+t&}X!4OC$FtA8dZT$Sbg7T3V<^(DaoTKbu0AIFYL{XXK~rG6*QJ7^7;IVFNw|90Nk{^!m@1G@WIixL;jZ`;m9hgM$ZW!w+M%$cWKE5%$99l*<7rBOv_+9v?;vrsRBt4`FZvyDsr z8XuM@4Ud$4^v8d^J3esYyoFLHcNS{cz55m)TW^LiIX0U7xw$3Ys5^oWJBzhdA8RO> zsQF4<(z)aO%pdktJ;pt~@zi-llML#*V>c@;zP?P^fD?nU=i1HHtMPG}ugBv4Z-48* zAFR9kdQR(h#syEd9W@%%zidM+NMoMxcPaR|{_o@Y+Q+cJ$>M8E%ot^d!?(gX%I-jE zvwKI_{c;uvG2+>`yd~Vg^~LTWu^%2j9mX|^OTW!upakvMv7b_(XWDO{Cr-c}#P{Mm-VtX#TomoABfLg-FPkiFf?8cGf$&@Q_izlCn zo9gA+7MXu2wMLz2A3IQG7s^=ebmUnt#FSc8FfQ)U#$|Vp>+&HSBjw&Vc9@KT_#zAg z(E$x}T!(hkM)1^i(3^;z^vx6vPc^O@pL;uL#8XC2F1tFX9#@NVeO@W>Fj8O@B_76E zUB`5vQb2?jT6mhpb9a%(5i7Jam^cAKECxd8Fc@!2&Am+b=cgWvy$_t;NIrX|c)9Lm$CmB-O| zIq;b@0^Sz3QX8z8V4hh$cX{M1ucLH*`-@)?N6Teju=&|;D#hOMX-Rm}dQH?mBG}x! zTjMOOe(WxkKENj$*N6y-mCgQ1zvDZ8U(r@?T#^OWjoq6qO#MpnB!cd4$E;l&40vjM zwLg$`Q8ea369!IsincN5`28lVD+XejhYhaR&=jss{-sX_yMB4bC&+H%>7~XbUmr(L zd9>S|!ZpDx zXuf4n7PL6FYS^q}*$OLrEDRBcoX0R?0EU)3mgES8!BTGf%U1tT;M}n1FZD;xT)=`E zM|R6|2HM3w>Gzyp$EO749H^J@qi&%)pOWw^*^Vpocj>Od{tZ?!=B>0F2?daJZ&SGW zDrCMn+Q+nc%JNYk3AD=oNzsxkEod=s+wn5;X#@S%>vGiS4qWZ{@66;nw%umC5mBI`Vs&f;*S*d>JC) zZoO{jRiNs*cGoTQzCq90@8-HN*Y+y~9wiE_qQs*#+v`~GAq8ke7ynxOc<9 z1fO-nkge_yqpM`md*iv9cd1Fo{CXq)tX3FTOM2^;Tz@< z6Zz20k%+}CnEga9-3x-JWVSxp5Z0iMkQ_7=W zTi84^cmb=Y)mAlOsS}#mroG2!zWKKX3_cv4;wEdu)`#yvQ{wT{ckk{Z@$`bIn$HaW zIf5^CUC^o%m>0X5EVl7+CuZQHAJ#s%{WK=ypOJ=2%BTL~Y>fg7n_b(pgO?h6xbm`d z!6iF(4Yv6VhpdAs9Cf}ir0u>?2@D2_bZ>v|cLWY%7k?tgVGQwVe-vXUZr1&*VNcy# z6AX=v!5RyHjB;-*40P&t#3DvPe1-J2E?aWXfXI9k-CcZFGTPV@IBVUN0xJa`T?(wC z#G^a+>&S1E0ygjMNYMP98!@)fPDC0ch>$}&Jc3srvB4u}7}W6K6!j;yr%tJ@%X{|> zH?V8jhM3Z;HNoo%I4^)kP$-`n-by1le*->XySWsq5{VUOjeg;vM84qRyKw(iqd zDcR$;ynU{BY`$Ea&?0D` zxApmsv?5^`Q&sT5^_xo&DN-c9DUmDdZ_vzfxeOz*P#YqQq62tT`(MPD}xYO|{GLiNWuI zdfes8Y2C8hfx*4A~pN_92cB>O3C>6B%k`1r?f2m4;?WS-P+Q;GyrcrxD_uiMVuch^V3 zH!^RDk83d&eGbo5nK^s!3ok4>zXD!osMyi5>rumY>UsTK-+QNA_xDZI;YKf%0iPwm zGor74)EAL)i7R>1a!pqXtQ5Fe3ap~U&C+|t_DE8ImLHoy;u(v+Mf9F+du_-1yJ1JOpQCiidd8LG9I_X$#uwvEK!Yi| zE}>7|zhK82Ja#$e=2z6E&`NK$+fDN;k9Gfg;|n|da{QdWOwDhyv%U5$!aSus=P!)9 zs!F*lljT*>?pB+*@?FJp(~d9tx~<@MWv3zMr`Lb<%yUnAfpcqId0+Q0$7kMRH*GsF z*?;h}oiyjS&9CdU_Za|K|INKI=DeA+n78dZRvz`YK$1RnUJJkXoc(XptXr=`G|n}v z{?t0kw!PGEwH}A{*v^MBsOP;>r(^Gx9@aH)rNByob5URwCC)|J74Axbpg;+5o49NJpF5TnpS~e5B0AtPqsKjg2MW=IUk1*D&=qn%GEQhn5F9A+3zVa&% z{tj}CH1)N+c)}<8FebuUGcLk{xA@H$Uyg(|ntCr7r(+c1v$G_iZQb6x;G)pcyUSep zt=02g*In;d3ak`(G%2u(5|8G*ucN*(3NToqJX=;)`K)B)I4V zh+EoXfkyC!!B45n7>dzP4bxbieE?P%AJv7`ebV&hkN%0tK4ta9C-P0^Wl9W!%Qa{C zmiR^tWw^^?r=v|GDk&Be3#}~J?W(N3MH_W~_(LB$xL$tjWB+yZzaAgjwZ<~?xa+sL z3Ol;n;aM_&#%I3D+}{qPszO7?d|~8P?)Bf8i6`nwx#vd^-%?I27%||=`OA5#Q||j* zs*C-7ncICsuezT5xzojKns6Uy#Fb;x);T1^loajti6LCB`xY02A>u&|_t1UpIQ-xL zZ5T*Ro!{V{^)E`Dy!=+YQ$~%&4)tLzUn#Is;K8QADoQ-q^SAc?zM+7KKXKCY4fKds zkgKOpdmFs;9OyHS{-JXKM3mv~{q$e^)mPZ(Ucay+L@n*=^4Mi4tW+G&RTnh(FTCfm zU}uA+YRcM2QG&P?W6g@e#E!KsZ0BQ++ydiJKv^s?KZ6l`5jCs8Y+bwaJPR-mv0{R# zW)z=xJngf{Ld3}Wm@gXm)L>&lYk!Tly=M`8Vr->vz4AH_%P#nA=Dp(Qzaoxq+qFM1 zK7A3ORek3pB%G-fKH`#c?!alNRCZbJJom!y4LGm;s;}BxM>Q8SwYTJ4a@vqb&8gSL_)v5DDQF7w4wdmXkyJ&Z}k(sPfW%~b* z|JfgpfW0Z65gkMcBQ7dmayEjsG9^BM52qa_wD7JG12@ck-pUiE-Ye{4zw!@*Wt8Dz+5QT7RX$!%Bfwlz3QYb{*FZQQ(3R#SQ$?`UmS( zw4j@G45@bdsEu^2)va5bx<%tk>l)Ws%VXN<<0YOAz73oW&K&!uUfYlCU#4yO)#r<# zL)Oo9Q?KqzOXsA2_H_g2D*QI5ZI^IsTnRJV&Xr}IskFs+v77fC>n6?o937+W-nf$XQ?UE_moR&Hrt*%#)$8*V{5t;_T*i9Q z{Vg0G@1~!9YR%=gbDv{Kc+#x1>DwH%k+Q2}ISt%C?hWh>PUFaN^=q%WD+L}73ap~U z!!eKRDDDXbiUzBzkvvCdXsEyLYk&E$C~%!V(1t$$&fi^*Mtj%M<13%ga48mEfBXI4^F?>es}$(AoIRuYPQdjV^A#@b`Xy34*xN5dj6g740s}X zami)yBeEE6GM@#}#hhco_kc!QQ) z7Ru;z?s|!o#CXgp*6B+kIve96?%x-MZr(rE$SCYyJ!fS&ja}JIzR!AnS@Tv3tQ2@K zDX@wX59YkBy}suZz-lp+8D+Vy+u>ku3mdcK*emSnF-8Gj`88h?r!He_KKp0>!Yo^z z<9hZjzc~y-7-(Q;M{Tz67O^Gc^gC`V;W3!Og1elGo%xwE*r{2#(epaO51`NydtSq> z*}|To6MJ=rCJ||-wC7fveVIzF4@P0eR~xwy3!SX3{|v6?Slnqm`Xc%)t0+e17MmL^+L|!ISk4AA`Q0k<-sS^UP~L{h7~1 z--w-a##cEWYZ83o1H&!w{RW9k*51=!_bWCuEq*GF8T9)mLJ=);F0>gj;DXaR3+W=S z9nlu9>`!atq>c++q{e%%prJl0}iW?1g7=*{rwj^rI09C(@B{#I%uQow(jdFS0jEu#5sM(p1<+Uzy7ueU<_8|)fYGZ7FQgnx2@;o z>sr)m>*h0<`fm>FW8sJw=_{Z-wfNc*rJcEvZI!Fr6BiRtrgaT{|`mG5qj1>587I-j$CUwmZ=mS^sS{w z9Ot%$U(?~Ie)4~eUAxiu$rRpRzW0^)4YA0M5+AC&TYr&W{WWi;z)FGpg9584aevIi z+KUH@0<@b;So-S>j>v+g_y-^P$Q9P_I&gb!u_4$)qQ^1fPQs+Y9oemU;_H7^7>m%F zXIm+)J(i+pgjCnddqGu$4bF^-l%dMA-|!o+u(eJI#x61HnrOP0%^f`0tG(Q-545&+ z18R|zX7v-F_%p$BGt3zY;z^8(U4^SC_HrVCX&DE%RmX^NCFM%3uW57?r$Xg_eEpTm`PY+N6<94^Nv%JRn+Y=WGi* zA?i5Mm;n?v)NizH_m-!hj=;>yM_KKJ0S7nv#FhEP2mRn)52xP{F^>2ozp&|70<31+ zV^Z^@|6sd_g?aBBlM#_@DPD$TL{d8iL_q6IG^x4Ww52%K`YQ!i3Ou+JSVf5kckb4n z-xCVZk`*>P+((xf8)?IMfbILYzxV~iVgdVT+G`hW=%GPlTZ3IL8pUkqG~Usml;QHY z)ka37nnMVya;z&ma$3?6gB4B1wAW8s`dZV;HeH#kLUHAm5lN2oN@0_M?oRSmKcW>? ztxa_A zmcFYmXmR=}WL}N0yI!_!t+U)&n8cF>+cx@XmjMp=7^OKi-R_M%5vz#$`sBTm##VMJ zM&1#Lt~ZwVfA<&eXd~KK_TfAJ?(d1+ryRGoMDQJ~fg3xd+PRFc73;mbHi4Ee*Oj{Z zBx=_HioGh{w@!L@Id5mLaQk34%{w9MIlE(MZS;Vxc`F4TQ3|Z0#3MTK>#&cd0Je{o zm1fQ*ZvK*^sFvVm@Q^X&X&!x6oxxCcDY~D<9&DQ0p4)F*m)HOY!wU5oZZLAFcMNDJ z5BXoD;Ytg?eNJPXiy7N?Y?3k0v9)1@RO+_TPE;nc32V8X82cLAHLA1?j`JxJC|^tD&GFwf4-+n)VR zF^@#V-cHW$jFg42r>e&eUZJTmeA5$8?2S?2T619zrrIyfx9gibNY+mJw4iT_#c9O0 z6!u(w0kgf2az4{A@Yv%9=Wu12cLUW7bq}LQ{$bwfS-z$#1&*b_DoPwnxwYLRN&$A) z?|J|G_dapwB85*UP2|QMauWq-2`=riTZJ7N`dobDbI%3qzJ%4P`;*#Hk$FZwQBQs2 ztHM<~+SP!{{YGs0w`PrFMCHj7hiT*+5LX|uDCMkx9xF|3N?2DLZgSY7XPqLF9A;+< zKCD4Y3&-dtg4iioKRGgG3r z9RdbqnTOsjEG#NkH0zA47pywCMA^J@m%$neU!>A+;*=BZ(>Trv)#@iMn*G`pzU|xS zGyfbglg4&SKKbK+YIj7QGUGa>J2c-Y@4VgG*{%(STytuG(J}Sv*)e3GW*5fpmFrx% zdidA8l>!es1y)hwL7&03|2Is5I5w?(-a;F9X58$aos#X7JsOqeI$eq#g@TK%{glzo zb>O#c8Mqi*>;Bx~c39f7WzQSdcg_bR?E1Iti-P9=^=O-kJ~BRZ;X!j}Z^EKS^J-WI);0Png$9!OZz49riuTyHeocqQELj zJY4g+j^-!|{QP%+!E2^s`yT@k8&EW$q3PXs zS>hI_6NTffi-KzeU#ycYYmTxva1%x zLDdo@2^Y4 zDZJDTjO`*a9OJAK(APCpf6ZSh@Ssv)6(t_jnOpmPA_}nMX1J0DAtwSm2`04A#SWOU zz+|PSfRRer%s2TXk67kQd)&h@0hslV|Xy7WqX3@_@oy?dYWU>^#+@YQH42~3-Pf5TV5q6;ta(Ns6huRJnN zNq9udx`GHtH0)VUbZDakIchD8dP;vcEO`y#>b(d%0k4RZ+)3*ic61U~rutOlbd+6| zwP#b*urBX6fnUQf;xhA=u@OrWR-M{?z|zs=zx?4Zi8?g+?Ji0MZg-n&r8S6i76u$x zQ)MAKl;;}n{(U*`Vc^e>fOd8vvbx> znt$yXzuEOx($OUhJD6Av`bbuF8Ag>4B86(?6=hKU$m>H`bi$W z%LGRL(SP};?}#SG532W`ISH~y6bMy|_m_)-D>4voGcN-Do{%=&)YHH*LLn5`zN?ux zWpH@HF07#LF$Gpp;vP@HivC_xz^f12!#?&V`VuTw(Zv1ZfA+_B?8y?m?6-Tn6sZYq zS^Tswui2pyd^GlqMX(mwr@9Rr?eF`6KN4E#I>M4jHEGxq$7$7#4&2jHV)Gr{R@jX_ zmgzgbs-TV@2!K>+}B1?AOG}U3|#sTU%|E47aDzK zM7Q>gffh&o9DV`?9&sSoo^=o!?2-?cU5R!LIigG%W0@M>+&#GbaP+f|5Qa6CmlvAW zv<#R9m-5=^X%3gNPyNsr2Th0$2@e~F)wZ+cj0Z6FsfNXn%-ELV%iYCv`wtiTEgYf+ zO+A?H_ymiFk3RvCvF^*iO-+nq$viQ?x(~%`O5Q}s#n8a z?^g=k&lFfiiTinW*1p{U1+Z*bx+&w6Q+HuwfB$%#6uSgAJN>!%{8O+JUwodYDmD3b zCe(K>LQ{0`anq*wB$W0CD{m`=|4FI8Nk?1Vy0w|J#5IPmZqu7{dHT^CyE)j5YrL;X z%T3#>W6QAzy({00x}VBM=RWam=6xUE2F4}5>N@qjIiFy^<@8gouO2OHnDzLwk4ekX zv1R}3{$<<5C9UDt^sW0aJ)+B!99vB%xHtGWbll*&bxT_3!?koh=NJ;Ep2#ys>!;3F z7HhTOm7jEwKmM0%VI$UN|9f*e4s)(%;@hOgw!t~q=J+x+Nt&7uG@SY!9*@uhl>!e61y)hwL7AzwpJ$+e*FMemO}+N*NYG}s8sWL2>DyHQ^xf6AH?A}= zp#h`~{NlJ|8>55eTe)lFKGl;Z7Vj<@mVi*edcN9vpDm2 zTFflirurH7-CHducA@QW(`55l_3xRtzV&i^Reo=L8poV%>y6J}VVO#s2pU>o^yWo* zJHKtcY`+=A%>Sl;y?N=azqZ~GEu$No@#Xr;`D)X8eD3S{9a)#V5pTzbX6<^bpX>Rl z=WW++2XOwT?x(gnF6*scf7oF(Z0irN%k0L)xQwAYj;4M+?gV)#^}Dvdw3XI#A8dX# z?t1*k;5yWob{*z?)qOeyV`+Ik{#s;Q3TLgqQs7ahz$!{S%CrB-kKL!s=5`*hHA)aC z^QJM!XdCsyzP@~#MSmG>=V&E1*h`MJOW5J2S`1GZG8~GvuML}SM{2RvZqEuxn1|9} zd&?L{j<@5Zf?aU@RSTCPlfaN|e)}AetqP10UjjDeu6FJUtMVA5kzP>G8Xq5k40bx= zg39*-JJuR7gaKFSvy-5mj9lZ$dv@>)UeJeu&4}%~h*(6j7GJRPTYRk#1$@?jG31K5 zfN$iaYwfU>{2pq53nw7|i+}OoN52Ivbp}c?x(b6e21?ddo+$zsmw9vr@9^LL?hOBm z8WwmiehTL2!~f*JjOSf`2~?YXIgIPpEp$jRh_O!?s1^9uih1L|kg(g75!{qp{}gV< zb=C_!{E-r7o=|R{xV7_R$}%anLHw4c08ixm9q_Ml#z4Yjc-Fc*GXlLRJ|FyNo_O-f z+jq;+i4!0XW(&m3=OwSC@4<#^g_Qy;1@2o4tfItyJ2h*^&QAd+7YlybJ7-`DJCOn%Slc=K?-WNyM2W1cf5lFDM^CJ9YJEl9 znPWGxy=>FDi{`ke(!|HMycUb%bmY~*rhy|$SbF1Mv`BlG9CYJ7gTZ1>&Q7dv<2Tf%Nr zZlKYpL40=0z|Y{a?j0-0#yx}55(kMEg3A)u^Rh-VAgnbt;#!j+crOx`iQsDlgz$!}IU-Pl{=Ef)>U?~>u<&S>!sOTq4I{M_rXGJ6!tb(hJ=CcIW zp}O3Jxmdg6+=eaAU^5RjKC=9|cYS`uz9(W}$&GP@jZAddFIwcHh2LW{?2-u7h^@%F zZCXUSz{l0NEt`6d0H39gEgNq===&6{XoW?dQD@X$*y+6)h01PL_6fU3y9}1yvCPj@ zIgvoQv4vQYg)=E5jc8MLqf)zU7bK3lE3j%qvIIi}a5A@c5n#0HXV&SZWv*}2Xlo}j zS=M>V(Ss$=RkQX)Q6ddrnNQ1~ zCmHu(_wpimXc>d!;6%3)=vBXo4>@<>M_x(w1AQHLc1UFF;mbK9f+T$7xhOjVb~x-D z`CaAJ21V|{oXZnPH|Zm_;}$OCbB(T1c2UORx98$EPi(ygxV7m@ft3RH2?bVB;y#&( zwHr500h%tF>5H%Y$}4Q#srw>`V`?^iuFw4mWi&#Im|9V+{wc>K>I&9VR3LB}t_TQ7 zTTE+NYv8Yo7aK?03@)RH8eVGsM>N{mP9!15Q=%dlxdhrrtI^~Y|}+rH)V zb~K3lrXu4uV~fa0Od1E7llOA>&2&692tXjBp&&CDrPkr*wMti?C z;xZf=px3$}vdcG4c%<-VojDx6GHct8!=a-B(NR5OXb;Ysw^CrGz`duyDoWh@Iaqse zPbffp&Mh)n#~bLuMPW18koyGJ8r*ZD&Lh%jXGaW<^o8v?Hp11~)!V_5&(Nf&5ycut zoch9!)J8T_V{|MadSG8%wFrWJp?ys;0z21=V0K_=inIL`o?U)nfwC?!tJd_hea|wi zl5w1LGUxJO+t=y-vl6nFhx8>dEQa5P^ALAOkZ<&{A?N{>W zU-wJG7%Fxs%J>*=iFx0J-)1)W9Ql_x>a=e5!SDaWyZ#7_j1{iSXYVU~48>wMYda^2 zFWuCQ3oZsq%7_Aa`t~f2sxZz-Y2bt9)NJo6_rwLebj0fH1M&8dUr~i-cT3Fr@c;E+ z?cjjtcgA3>KK0$2NLZfjF#oFVfu7i84 z6};WX#K#h`$?h53U=?Xk8AUiTc@#eumH`<37kUE(C2fZF>ukMjd<2&R)Fz(ecMg z)di-Q`7O*iReJ(o)@2B!PiuN^~9nt>gxd2$GY@ex^ z{P59y_vo4Rq;^hT_}?iu^^+`R1?!Nn$?(N1X8+s4&1H~4l(rYhdu5{JOt zWt1CQd-VC(%U^cK`6I?jWWU?&NYrf@w;k$}`n2@zE)wG~bC&F8oU~q^%b6T|{YrfL z$B<}b{_H5!epHT3#M;_@%k|@ThE|T;OB{7t=pOwWITp+~QpDre!Fa4U?|f7o?`J2l z#-**@DZf9ougm)Pj|V>Nv)`{Z((`=~`a$37_m4I8R54_;e(f6c;qT}E&2Rq?*X0&*fno8h_%+WS6^P(m2d0Q@WoM6Q-3%(F3@t;_%d3Re$bB6%DZpdGt8*lwdK8M(X;9{j@Yxi@ps@;D+@;TDtU(g2HWXzLw>`aBR(;n=r4=pWoJZ%bYJ%2>* z@Q>Nf!{~8m-2HnrO?(XHu($9TbNcj%vXphG6_)r-wCgkB5Pj|zaV(FFlg~hp|4sdF z7#&+ux3|+m9LaZ@-@BcO>_?q$b_!oT{2qhTT<~K_IAHK6%Kk`+-Zc%?Rqt4J5Vgid z`Q6#knSw!GN>-|cBl>!?ItfIua5*{K7V9$O2rvhxAa;R{G3vT($(OxcL z7nb%T+;y>Pg(a&EWtPGjsn2#g$Ax`OmSUvs(mJ&pb8lSO-H}L1qj<4d*b?JwcTHl= z6dEf>gDIHm(TtDSZb#n>%od(C8Z<7NUTkRkoJ`>~@8MH1T_H-||J`30{co%7^0(vT zN72^Vd*0UD%j-PQREIdQEh7%-r+niJaizfexu5;nd3=N83)tKF0W+Va=F<+G`J?sj zboVl=^eLjLUFLM}a{h9hQtLF|FH(kb^6gmkfNW^}_?;1d5bMGYjZ1weT(`eJEH#?< zelK>`V(g{7c6aafYc$W=O@AHqu+yfWu8a(3_n;LAM&bAB!DAkcvzOXs?zOv4p7n%w z^+i-W=X?D+{axFx6u6%#u!<7*(`>ALxh4f@lo&j4dprb+7TR^F-dwrJIzDwp8#xPy zVF0`7h{`hIHadp~J3*6&rDRYr#kPPEffLIoI6lK$Rj#8W`uW`B0-x~v}wyeg7#btog)=6#nRC!xJ zUA@~isoxp>XzKUK@Q{dMt&JHQ73|yjsqxtjOByL}&!kaqmy<|?~PJ z&7r=;&fO93=g9$26fhVoV&CnhzAY2r*6fu6_nZQ&C~?oHVC}%^D8LYhw(N8ery+gf z;~x)~;WnV?bc^FlYIgG6_-yoZz0Obi65HfsM#!EyaS0GP>=JtnveJf4K78;0o-ev` zegw9IOXNw7Tn$B1xM`hi_?jjSOws14gD`?DP% z?K|7|JOd-=R62r>%d5JRc(P!NuZEA^rL8CS%I}F9niFps=UHb&L6NQ!>{Ke$02gR* z85apOo36WHWk14Ts{(@cRyWI~BXj_Q7{qqIh!lI`kp4S|1uSguCx84;?JfaolQlQ4 zfB52yC-D#CXN8`29&-Iii!Vyh&wH@v3sjNfs9Wli;tR67?#}qy9ED(Qv{GQDz{x1E ziV`Oy>0_&v12r$+wu{yRrLGDqEMM(UR;U zjL?&>5Z2W$4oxd|JK4TCf1&tl^!nN6(HuWj+<4pk_F2THbkXMKUu(wU>m9Y$!VuMm z7Jo!kLIbpLW(v)D?c;>K9>Ir3b|pvZ5(VMo%G|nK6Vnc*7sIAEM}fgWZ$);xE(l)!J2RZqVKs``?8k znibQR?P!jg)!!`+a(k zQeF6t@b3|N;?%gZKMWmz_(LDME%bujx%xbvth1&M7X?;P;^CUlM|3o__OyVlUGX`{ zfkg}Z=%N|CIH@+d-&;l`sSTt}ZEInL^Rv{JF{}!9uxJtWb?TJrx?MzTDPyDg>>0UW z?-@Zb4yfC?&(n`n6I%y&ZR_SUb|UMyMlgBewemP2IvZ_MqDQ@h((7;2TR{&#Lu>^=eoUq_3G4l_52mB@{VO;j28wwh0S5WQn#t^gWW29vIg4z*7fo}jgIuK z#+Mp$a6}o)`fXYVg{|AS_42u{IxH4d1qJBTsd4MdoIghh{!-z4V{$evfGB_tgeQCHi7vvwX@3s*7{mKlBLK{_Qhy zm)k#>et(Rb@=Bw&r1#wXUctGAuWh!clHtEYOiLrEx;*dw-b79t5%m+VcUZQU`VG`U znXMt>HBm#k@d~}*-5|14!Wm7yuhEBb>ARl)os++_QFx>Eu4_Cyif662Qs91|z$!}I z53{iL;Tjb1d8ehC(gIxr^p&Qxyv{M8?~WdAnob%{Yr~wwMXPQXhL)8v&Y^V>F0`fJo3?c?uHMsj+JUR- z2bZ}|V zR~j$OAUcGrb@`0?l(xnP3&9<%UTdSVBfSPRZBwivm*3Qq)~tFDj=a>K)4ul3y%0w} zw6Defz1?lnk2(M*d+jManVYw}oepu;=xBfB_$4q;_oC z|D2ELQ_(A(i{EZ^H|)yMVu|Z@uRmcl z*3jF|in&gjXMY&!oq`o^cPkQpi?2P(vDg36Jnz)pvB=+>xrW>PqwgrJCH25H&pFr9 zbKzRjueI(-l%Sa~B0E^QO|IXRs%q`OMz< zjEy!v@s&B;D^Hl=V^R7Pw9U+kbce67p3WestA$p43+EF*@#TDA_R6#FPyN4tZa0sa zH>$96G{|8PLua#UaE z_jk9r%+Zy8PHaaE7^iucjBp0EMEvGnqCe0CkYGx(Z-!+*e8Hrm+v zi{J7Ve&Vrn;8;mMKRwvJJnx{r-OU1%p_;QR>iX&tPx*Z0#r?gmGkPE44z_(at`^rQ zoY8tzK;`{@+k7PrN87ITRth|*6j()x2X*GweisU0BWUEx#`IC2nWaR;{$WFj3QOA_ zQ$IfM1s{9mW7=!ubN?0p`!BdNVrQJ^E*y)`TwpQKC|!N$`pu9^-6?# zD zVJrm3RGqq6&L6w_Hh5;ZpB6Y^5fyuG{|sl8yV{6EHJ-e`;44&UN))I?FCriVr(`>c z3oR>%>o{T)~ji$96T&`mP@gxwNJswk$63P44i81{6GF@e?0Kj zN8-Wv+`B%1SC7av;^JE4I#pS*&^Gv;CH^*-u9=kus;_AVl%cGJpuXAbEZNz3~wj)zt=j7$QM=79GZzja9idF|`h{ z%jzt3nb!tB<4XFXQJ!%!<)WVWMvR<{IgN+lqP=e;wlOp!nknzm2tH*?jGUZ7Vtl>b z95{&0%G+b_7)YhK)SfHS>s#h&=Qls)|HX)kFN<-0{HK3<=VqPrPvtRU)R_22(l~RW z(9JlB8@Yq&`IJUZbzi)<)^jFmimdl51;!|_iV|aVT5H{x6!@<4&E;@*dj&_H1AWaE4B?wy(A#x9L@{uZW$xZa<5(nK|Pf} zOa^D=%H9)s1g6!WQ{giv1~*{#{9R(_X#n4d);{USu+B~awy4&2-YvU$p$$HO&s>b` zf+RjEQNS3(Xl7e*v7KASK*m{i3id=yJ7v~5TubTgv~l=ouhYmT0x_4o3|x$j_uEj8 zH5KSr8zRmz?!vK)rhPHCDQ6x;Bsa!Cb~r+8DrM;Fv}$-<{6+k(wj^1U5D5cLVV}8| zyH7US+Re4faED=`otlx|kEu=&_lQc_ck8v@&9r`m38>f96s*yFb5r0s%ViJOX#2I^ zN`aLE_nrc)C~@!SVC})t6nNXWd|t#Tx4Hj~&po&Ixn{TAV-=4^RvP-mhW4>&%V-XZ zCNEec;*5=PMgdsW>)@!{*7)G1rDlIC(#WnKg>9+Z)c1^Bmh`8_N1U+n9xPYEo%rT_ zAZVs(iMddY(7wjUHk#JHb~R6&^_c4W)P$?Y(4sEu-PicE;a-YG+u_M^_8OPG8eg=T zsF&F-J~s7NaKSn8w-g`S`7Zn%*EUU`nilO+eCaD;oO2g{fmUm6eWb61w%G5sKkYNt zIwCza<7vkXUTts%PKYs8yAcCC?W{|pP~xyN;VgyN0owW&z~}q{N6TsB?~QN%^*i4T z;`tAL@AnSB7h;!f+gH8<8;HeNvCvUJ;FF4;<=uk~=W~#KB)(*ukrX@c-Qr6BC+)mD z_%diFAc9oVIHc-7|uBy8GJW!Nozy8LvWUs zJI^9~DvDo)kO96Y&hS|ZXRW_d;9gT;6(#QV46OJY3e=zDrs&EeEm~?MuM{50@XG$W zulc2S1om`I(Qe>b69gOS`6kM3c5oiw1V=7oL#ylTa!mCld@TxmqKl9e=xBbf`yQCW zm%F>+7gBJ+S08OS)$Z7Qzq6nEZR25vmPqlMA)hF+4J|^dOE1$x@=0DmLSVf6P zd;agk(Th?t5^MidU3a>wPt2*{&A}n|I8`>ZUUL;sRCc!*B&3b}i9pVRA==Rz*6h`6 zgKHLs>uk{+_9FJQpt)4`4AY2|<^>xs$`W>PN~{bYl7f#mzJ=dDzyG_x@D7*UjOvr+ zL;}WD5oM{svKFw^Szf1M?_SEe@YeagdvX3-d{{Ls4IIQ8hBnw++U%ac&k}sWwim;j z+6N--*}v}x{>Ys__slb)q0fj{Z0y^9JEMixe@qj*@f%8f;2Rs`zxsV)p4rp$->7NN zMcKIJxBcinM-bmaH~6fx=U(`|LC0QXGiPd`af)!(PZ(R3eoW16i!)u&FN#rEoTXCW zp(7Egt()^|JT&y$#2kyjHu%S-xZaT?9{1Knf-44HL05)zL@PTry&%q6zn2n=wJqy# zX!kb6^+E@0jX_%v=Ak^Sb*`C1b%{-OczUN_A8NanuM~KcDX@wXkMiu_mt!YN&_3EJ zU_^B8PQu~yme;xQ`TTzDXEFP^Ukj(u`HsaENPeEeP%>KNuP(z zpGrgfx&8dr?xj=pd-=~k^Xgz{Mp&neiSE-K=O^?49>`;;BI?wLb~N#YCi?)~xng2O zoKZ1Xsg#MfgO%vvW4}ymJ;JWT7wpLad<+}Z1GlXjd8Lf2?%aAH(q;Vp7DoG=_YjPUfCEAd2?U3TK5&E92hZ-Ol?aKLiEeZELRzAtaM(H0w!W+(?+!je#AX3sv5&5-PgF98Q}Czq8BhrpES!Rrkrmcj zeMU!o!0NuZhb#O~rhpFQPo4E+r-flBJpAG8W-Kv$9YB4hir>85h^ZaPD-G6+l>#dT zb`)4ei5)4{n|n_I8_|w;qCq@|jcz-_PyEDB?AW=R&>q$JL|XU=HFv9!U~wiPCxea6v$WEZ0b5vTfYSuVyNb=ll~8sgM0t#kFWELi3(zR~%^ z@`k}zX*VJcBrXxMQe>eaZdvE7*QWIvBlgtg472!TI6|&v&C;b8`!;%R3ft^BXQRw_ zxczRCSqFAC;$(Q@TRB&v)+%p|RUVHuf<h~1iPAF%bSJi|D&p zT-mhW%sCu++4&fq|GEz&_eneS69gHqk#@({u_F=2vhaQUx%IZ9<3vDhVo;pA81yGIPnuP_OJ3T#4tuFn}c2lW{KG^I$@w~1l zFEzcfqf!?(_oMjoTn}318Uz>k6lJhfr^3e%A>C(_EzDFtQ3$|UqGhy_cD z=2YJZa#-9VMV1Ne_EI1V>W7A^dMjKgI%HF+Kua|cUe6-Cq zY yj$P@i*B=^ic|?ol?#Y+yAlE~#1LNq4-EbcJ$CW(Ck}162Rb?miB;r=xH@jn8 z#p~11iOT1|RpWi~;LWFN1X8abTn8SW54a2XRHCXwykS71sioJP$L#2j$11@gT?G2sq!wiY;XIS zCFbdO*nd~`d$)OV@d!RF3+;TmvL-zEuoojyyab88WPuXGL|S0>q6OvnXgw!d?y@7u zZJ`Bv>@ufr4&cLP9twhb3VhZ|^tbfqFaM=qG}~`%vAXT4&AlPgO8fn^Zn4BLb3%Kr z94p5^$hytQJ=7Crecu*PRo$tRuxq^FMBmEsg2V@9c15s>H1SV7@ws~}Be>??kHnn(G^WIFi27yx_M3xkMGiXQXcLo`{yaJo<)))=lpk z7FlwTC1oI%XDJAHdraWz3fy|TQedUPeL#U#l(-KjVeJA6a3jPP(5juABD6SI51(J! z)Hws1slG6d67Ug=lB~v5UHPE_FZkfZ*3H#z%V^Vstz7bHi$Bem+HK{&y*qL$a}BFX^n@d zRNnR8_qjHnc=E~EDK!D3I$eC z;tEODjC(@?E|P44eXeh!2R>)CaN$~c0|+A?{r2%ymGozSG5-_%OBJ#rLJ z+pTd5yJtr@raNyBA0v|#qs;H<*$e!Oz|RZ);1Zzp4PX6=pl2&WSvw*`X|vR5hv;`v zb_UHgG2Bi~?w+8JXimmy_e_`qG&HX4->AW4OV_FA*l{UylfSW?FXK8juIvN)nM3LY zJF?VSxxTW_nywUBDR7tqt0-}pMr)0;Q$QfsVAal!X@kIVP*@o@&({rTc&N+u4v5dB zPydL<_YmBvax60qEIaD>^cNN@T^Og@TqmChk=jWtxiOCJNL&Z6wtxGZzd7)QtL7lE zLZ8QaN7dae0Xx|>W2xQuz$NQiM=R>ptMeGPS!?Z_J1!2buuQ#OChRp)Nu7}$2{;8j z1z+!RuFQ^0%S!Wc5Vc7;x8}P4bN3-~l>W7@`oyu++}pWpVQ@(|uJ8N%e`s$v zqpepzix!psI8HClG9p*(_!M|xK2`iF`1sgGz{Y!yahKhn)_5;r^wNlaRp2o?%Alph zgq#yc490F+p&7PYcmVia`m;1NL+s3ywy|qgI%LlE8$9CA|Ign2$KJAC2VuWZMDs%; z4K2t72@(}T3z1q1RHfv7A5sxYf2566P$?ipRf<{(i7MicRz=m5{Z(qcASZTAlxHHa z-C$zsI*tiWNNvXo#xlV+R&dhPd5O}HT5e*jHUgB~g4O+5=d5$rci)+F_M9{4`|iDO zzS4YW&Yr#3+I#IiGwb&zI{kk?|MQ13<<{Xy(%z}@u|9xb4`tSCzdl%2`m$-5^QD>i zIcem)k0Y3T*5)CN-SJIf0jbAW%dOT%fsF!7DX@tWODVS1zXAoYQa=4x0R0$9Y!1J8 z$rU+hI3)4w9WWkicxD0LX$~WtKo@+Ew z-n4U(aY{SdQC6cA8ESPb@85j=>+i4wnG?R&J9VA=0(reI+Cl5L&3 zKUKy7Cw%W)HXp30Iw=MbOO9nz$=#cSb({h|QO2(x9qoUTw}}!m`!t$9m#k{u%QH8F?P92W&KUH1>#SonpO`)KnR^dv3p% z*q;L{+r~${n;TCpkG&2WEjZC(5%rNcf;A;wNL6w8tIKK{|KW3`#=(?rt;A)@MI+jcIKiY7ke?c1P#jHk8Xpfj}- zci*P|Hb)7+p2aEk)-uloF+)9;Cr)D(*2s0m_r<^bOE*H_)A4K<_Wj*Aob+r@m-9sw z?cXmWxYTL%xv}M*xg&}x^G=lPwvJc7?7%H%C+n%Z)*m&Qc$duB1W%1eSqskwXQRL? zMS)F}c%>-6;Vl%vBG5WI=GcK!{o>1TYHFWEpA!jJ`%un{5p)2IDcLkQ`8#)2R{v>l zjBq<3F;qC0nrB4xq8Z^Tz@EzLEyJ>pjO_psq9>Ip|? zIr)*7|9E5{%CQku?*O0pku0Y+@ht7@zb`?&$(u4 zFfYc5wt=fyD*9NPb8z%+JmVSLVLWhq?!-c!QW^BHMt_?#c=KrMbXc?-#j(}fDDY~h zz$Qw(+E>6UK7PKy5%$&+Rc09894og*XNfH|(GpVHvdSlI#5X)~W6{Mr^bREM`_zBw zKl(GXtV1TqoN{V@qw+m zgXQjzWVNQkU+>o3{fZ=@4PROZvQCN#v7KBNPMm`+Z0$#^1hzlKKG8~6@zI(HcDBJe z)$dU{=xXHuZ!od7wd=vAwQ}@4<)9MISo^va#?&<@8FT#-QDy%}jJ4i!t(;;@g}<+7NmBQGhg&G>C?`~!e+f!Xw>05 zHP3J-Y-}hxa%xh>a>lB{(qr8aqx!!39s5b_`!*6EmH3Dx$#N@8hde0|^xay|#Gh~z zY0O1hAKf2Q$vp1lBsL_+ld$KUq0ocloOj~X`1`V4FS$M{u3FbPeG6D-Jg=@}|ByjE z35!KcDhKP+dCo$X^;yR=q&>^F^G1P<0;egki4v!2wAHvO1#AcAvniX+wc{l?w$NPq zJjc<(6@?0ibIlb-`I~QF_j&3Q}_F_mw)`nZ)^`H+YxrW z;_mDAzd6xx0K*>i_eOq)k9u?Df-~5@6OCx&qkm2FdyZC8$aan|LQ_xT`|1NddVGyt zYzZ%qJEIWi;3I-YV%QV-vTx3`^;sRu!UBLr%DoZtUY?0PL>_HNV?ioDi9Qn+wmI)5 zTblRsEW9@BjE1>peP`AO^KU$?0eLUB*h?h)v$;x3#T&olHABu+nkG!UJF{+Z4AVZ# zooU!BJ4g(`lDaoa$LwyxO|e75>Wq)bd(Qe`?R?;ee>irR&wH$032Wq>?chVpcfReb zqTj5G?H#b&3#Z|;{)p^Z-&t7roTJN8Ww zHPN#e$UuUXcb1IO=l@zbh1zDq%glVjfnk(YJEvbVp>Wbke9rItz;ky#pU2?pGq%D3 z*0H+vUhmso@6BCf&1zN{+H-75@25T0K3Ly2wjAeFoxUu>R%@HK4rrBy=lU#TC-UgS zOqb%*KC=KF`q=PgLR&11@?49reZw2>(0VDpUlKlix2ZdG>7uirdbpYeXqEp-=$vP z2fv9JryK(VXMsaxQKp}D9qhuQf96KQ)WlW$fQ22x)nHH0`EG5XA-wf{L$wsvympUO zM4h^ZVhl(AdB@faZ?)g9Kk$){9B7^B9R2%C^odsMbN!N}s)7SiMKyX%*J! zwU{eTx(*`f5*m%-*y?Q*I7NX?lsHA9?f%LXc-XXALfl^d)E^2BP8!cg9dMR*$7ex^ zq)&%TM!9{I%q&$?ycF(eeVTsagU?Ez7<3fI(t43J@}BU~o@SWoeT*+0H`>@}OIfs* z(q?J>#21-+mNuTLFKbvhzf9cE@bgoST@y_wi>!a)yw>|v=ytOFBUYVB1YKdUgKd7$ zt4|Lg9z&dlXP*|oYkJnt#5d(2l?j-agj%2KL}slb0*ilPdDQ0iwXeOQ^~V0LmBIll z8zNF$>nwOQ&(_6v|Fz#1>z{LAjdwZK_x^sLs5Y#^ZSEq*XJ45UnOFufDVJ)!iLFTJBA zvlWxYIc^lj(t1)Gu(Y5fC%S9I)6_mS@iHumQlogj>b>py$)dm}N<3Nf@eK8bhKB>n zrJtWuB=NcR%1JFQKij3+q*FNRecIgAIyuWWQZ=Nt?i$y(oD3QsS#dZGab8*j?i37c z3AT%9K-)F?sh|lHwmFyOX$qH2KQi(3aRgI!Ynjic3^O?g=)yUv)|t9aixlns)O}?{ zp+0+4YV#AHv(NV&j4T)8t?u9d(1#}L5^b@b5fhS)+F8aL?PUK+oHhArse0$qI^@GY z^}HDq_7FUw*%{}fJVI~bYx5lrHSKx?-!2dTH%7bK5peEg$Y~nobo2-|r?HM{5kYT# zB4$?@lh>_|x=By(7jtS2DJQO{1(36k^#vdHs>b1Jt^$s@DbWgX|-fg62UNYLHkS{Zz6>|2vKJr_9!XR$50S4ozS?L(3fkDsYR z#Dyj7{%kKDb&f+>b{FkgU0(W2zwvnvu+9!^pm)4n2KjXDQ}H6FA#61^3Or7MO_X?? zCR@ox6rgQn^q$*#>=+e6jE`C=Zu!2p&nrvE2uoS<`y~FufB0YSSeXW+-kZ9{ zrbY70rX{q+-Bg`pWz7lX5Z*hY-Nq* z^)P+ohPL0=!k3O9i7)mOS%6PwC9F3mjfA1!bASE#u=U1< z{dLA}&b3t@doQ$lH1EgiJ-WAZbU}Ms;0(!)kmd#lyMN5mxD;O|nnlNO5^OkF2fX9r z`E9>nY9a2N;~w>KcZL1TWU`}sb$A|fZ8#JE(vn2WooiJFy58~;@dmApcB9row8}II zM9j<_m19A#%z{M(BXVH7(Ixw0u1D5N!j$e{Ps7QzAxxjMQ>8;*Q*n$ucZqBC%o@Wg zCxt^E91%y}DWf>HdK(3vWeRMf#IroRugKW=C@D-RWVuJNycs-W6;<_)w&h-)OE0yOP>Nhz!A zv-XoOe_b0acbUy7Lx_P1^~9bYy|`S^94^gqW>SKWwS%SMVOj_dH6J`k)*9=b4m2^= z1^9rohk~`@IHnp&eEKCxF%~<^SPRzn%}>Wr_-yU@wzq!eVXXw~Uudz$C8w&KW8zAR zr^;fTu7RY$AdY4#fW+&}0{V@w|DrqZfzs@v%liI<3w~nOML>@<_MT&9&Es0>Mvio0 zyY-AxU=t;t(TRQKhK1#conr5)3&El^L->7&5{3ky*Zv3J|Fwgf)XNCL3UZ(L;!}2y z9BdPn`_PydO>tPI(nb+(u%~`o1852FXyV!_I9u69fsF#w6xc+GX}WBsiz&d!ZR@oW z?df9R6AjXN14}n?d-bAlY-;RBY>Blx^eKQZM4tHjvhbvHOs|Qb>e*Bowr+}To2siU z_i;EE-&^1D)z8y7+deE=<~dx-bG=MC(^$LUIOd!hZ0RP{8w)kU$|SplMeZ|ph4|3m zfF-y7T))Mhs57Fm$8rhBRiAZ(<~WaJP2GFn-}=3?_#z8atE1jrpK7^hLFSmKpY6Ky zF?`Y?Buy*aW7jmRJx(zalHXcW9N6Ym8Js7v+N0ykek2=AEu9DXyNAe;4ff{@ zD_)AP19SMvij#AR1^=`E?|*x60OXL9w3KZ1(CESoR+xNN#qqRp22P>#IqyxRo)r_% zK`-_##Uxg=<}e#!=J2sVT_c_#99FxvE3|mZdF`Ar^=+sVLNxoUzcuK{8E}cCSPyUX zw|AF!T<;(^+J39IQDBM!n+U}B>)oUprf_D}3o-R7+V61EH9B{3cq9`;|xoN#)m zH`+B#G$%LsM#65%1gsp%ZuTshY(W{eyNl2G(1Dffx^?&&yG^&gO@)`g z`~B~~!3M`VXl#t+vDuIoKSO`rr(w@~$Ii#_-C1`HAKV;cgznlK0UihEl@al5*Nl=Cq8U?j-wYzCt5ErD)=~}B^@Oey$Wl) z$Hys^h+=G>xobHQS!+qZplcI>h$_noHP(Xok@#4l#IRU1W389h^6bKQ+hPXOv2Hn( z@fJiY@4*pu$~e@m=F?7rO_X@rXYlI%XMC>OuWKTtDdG6RG2$BGbyIz}B^!qTp^Q`a z*DjMKNyuh6F0BoG0v@YIIDv7@I`K#2=+k$}XPxF3z7er3oKafc+yX4S=QD*xOATn2 zPn~mz=LlJys!+-zKmG(hW7g+W4n(w}Ldw;(OM3x5dCBoc-G0YsBGCvwVesm4RvFR6 zs_Uppa?#S3Zm00EC}kYQ&^`z;;@y^G72nT)_OsDv_{{GlVOrr&{pgP#>QPn9)vM)L zcW^V#0bW_nlsV=^1(poH_UhPie$;x#(xDA!DZ7&A+$TO(pFUfh_^l9kT07$AG3T8+ zHrGO|f9&I5e;5xuEFQ9|MYg2&J5vLw%Mu~ognHg7y%SUqf9ki`9OCDv{`dbZ#JbeN zv$%lEL5%}bZkMQx$a@a1Dg0SxoyuZFyoAF+3=n%Fvr*Pt`^+-X-pOIXVc=O^CtNh9&MXjz zM&*L2a2y0Tzd#l5$Q18uO@9>4iqg1N+BI{DQmzNQK z-Ehg+HmVxQj7l`M$alHl2-@Xuu;zp}_>VKREP7($)Pv z))!-PBrchvYCm%9b7xjOPD>RRQHI#WB4NxSUQhG~9h;~;IB&Zi=$nJMBK?4UKYFo_ zHB1OJCp}`KgSy~ve8+1R;hG!sR=!bSqrmzU*hGo-Y5EL7eB0aKZd;^#d4>P-KmDg+ z$j6GxCUb2v<~D9VqjSF1YxaSLGe6nK{NvBT$)hP#lQQz~fuO|)VXrA&* zh(tq`TORd3>X*QsYirxA^l5dTtG8x3TEZ1M>emE311s@O#Td*qd4S~$_IIwrk@BVZ zh(O_>(rD7Nwuy!#jNNdjLOYk@%N=O3FjM&I{oX28vl@-k6DwxEv9CFvdd~Y%$98As zFV`n$zhuwL?Z+uY3QJaG)Q9{2x-{#Oc zkNSHluEZSb~o~ zezIP1y+_K?hh-&j9l;lwjrQzggemu(?tb4k))P$~*jJ7xVk=}2%5q`IU9-baG#IIu z4I%|Y(yX2L&MuF})yiJ{mM@#c$GIg$t9|>C>{P66-~G;i{SN#C4NE_!-c`AMUDnS8 zt~r*uvsj&)r6}9iGiRXc`e3DG*Y&`W&S z@VW8Km9x-s!pcK`BZg@ovyia+v&?y{FmrI1mdki%d_5k&;i6Jl3iDKb8OZuMW(7T! zEcq1NWA|B`SX%PvV{MML-R^A^cv2~_i4srh%)Fxg;=k&+uh*OmLlVh;@jw4(p~=Cj zZ~(a$5rp?K>eGk|Da1;pk$n^=jT5Jbx#Xceoa#{J^LUK4>)Y}Z*5{x5$v<|;J|a~3 z7k=*N+I|YRjScvJ?l_NVPeU3w_N>;NKYowP9txqgZtaM58TwpVX~{UJwE7aJ-eCoL z;N5RDtgKV@;jj3x7_{bH^9M`uRTh}#t7;8L@R{p7iAehnb-y|7B|U=0EcYuu@fTB? zS8+<|2On@s`>D&^U0L0J`>BeL^>3@kAX? zC?oG0b(5aGm8y=fio955&Naf9v+XF$J&I3=GAAmQ1lEF#NfGMJN$%L58igzm^10K` zIzEo3F=zW{xmsv)OwM+SU-BLcmU#~|@hJ<-O)DoN3T5{@o$fPiW<^7vu$na`o;8+Vc5ogJmdi-Ir+K*8dG%6GLI8#=K(7xTdzo~$1cS&Vn zeG`K|@R5()q|@fPaF4a4HW1z~#^u^pm)eiy6KdIZ{pzH^CQ7_I=l#T}Y2+LupZ61> zPTuyK%}+KhTh;QNY}i%+W2$e9&3}?e~dNa zG>%o?-=n3|&tsW)=D=7l$r6O0`CP?)ISqYoy%zcTPt@;kSA2oDuh9K&tZT6KtY5O$ z)Z4Vd*vra-*88rP;){%6ZNBU7&IvPOF!pO9%lh+De6*?NS_Y*Gi})n|$h}zi^=7$- zFYENN2Bq5JGGMhagg^bE)RuKWb65wlgLt`DFLSFmF${A_g=_u9*Kg^Udv90+A?DRa zpnyDj$zr0N-Mqy`gsP)Cfe~xNGT2&EVMxRrj&b|-6S$g_RqWJ! z6vN${M3uUGyK#B`V;$VB8auCx6EaR#xpVi{@%df8nV*eVI?Kuu&F}+=i>KGxxjV8;NzH5lro8vP%^v&HO^@B~@*8@NHH`w2wbIze<4qWP;_=|XxJv{rCRA(((<^!BoXUg#k z_$Byq3qElBF!bDQIdM*D$!UI<&@;Eg>+#Vh*M1W#r!04My+prG(6egdZ-s2PKU(bG z+Bdf)Ue!*AW_kfW;tFdWwmWHQyV0ndZy|2?Br2# zaUP@XKQ+8cj|$s08rJt!e?NcIx=&drr*Mnasm@;sL&ms=FFbgTvs7l;%6s(;*HX&J*s-lJIH<@-ZMY&y}vV3KDg%SIrYw&8rQjHED6`Nyr0{zt=&d}b11Nh66a9u znQuc=MEjK6BwcgB@p)*UFZdi&TB>9CwfC7nR;S*>#<4`u)?x$rAuq`iqRp{Hpezu< zM(&aNT=-P_N$YQnk=>?tYUcWbO$P8eo%0hG%4j>aIW4kqUHJNa8>Z0FO3Q*HTs9nm zCMscxz?;Lzy5oMZnG@}6QEZ#n>K@xa1gkiLkAqBwJ#{TSmWXp~e+Jo)j^QJY5JN}C z2TnMJOm(J}p_Tp|!>1`0X9OQs`xrh6e&KS2CV!@1qxepZuj1pFH-*l#@xk|h?a=sS zBeRdg9Q$Q$dY1K{C^MDK2ph?1XM~=zG8tn&yS4w`7pxP+Bw~YlSkhX@^*PT2r^Oyl zu1glD#5ZMW#4_v4i0I;ZCsLtReSfmNc<*yO^lqhA0+Uw1?^kRI+1^i8Oz!K{-_k5_ z)Pj?G!@=@EbV?ZBoqlIlwQNr;K=a5-mxTl8H@xd>Ziv#>M$Bi!A3M>PwnUiTqLt!) zWOXZbM=x2|rE!>AKjw}VDaTCAVx7BILULAE@Wt79KJvSF&6p4tEoocO9PJ~e#VGN% zdQTk%Hc{fKo3d+nSSFaAU-t`lG`H8nTQzD}!l_s9nz*qDsqON~m4I+f_&qE;8mERY z^7i)?7XCIr$JlS+I1l+W(lYy$E<|yt4wujR2y4Pm8(kyE{{h1I1r{;1^9@Gglekq3 z!IV$4&!DU<$DC-6VGnI;!==x=3_lMZL_9|Oz01%4!X3+)%vNZ6F22ax)nH?zu$cPB z;*x8z-+LkU_GtAl#fJ?|I$-5a;hVF_B#fyuxT!_2e(i~mKYWc#!P;ah0#?Z>n8(V3 z^=-7tovpKOCi5jyWk|5Klh3@J%BNBbWc#^u;uG#%>tx727R|fMI{Nv~eeOWxU`K0% z!Rrh$uPiTXs`YOEux8G=D?2Nccfy>QbqX!@4c0c-py!Td5*}K5=^fC}vBo=$8iC_d zJ_oU8+IXmVPWAPi`^JD?$vPXG-@o*QFDzbrBlvT@NLEZ#Hcwdja9SJzu&_?zlMTzb zY+IPIFAG5pRi}<&RU`^>M6*ZCP{Xa#GfRO@lz3*R_S}KdoMY+o5UO@AtdVv!-1&)& zgndnuG1BhbI{boDe1XqvKEij;1vt2hXF^03$iFT|l@Bfh>X=Ui)hS7Gl|1uU+fcLxW>w59sq54x*4KVG z6P;>!!F`(Uif@E9ya3$;4jw{(6X&(@WnVKR(#MTnGlppIiCl7Vt9Nm4liUWyHe9{AW3wHR}_`>0=;Zlc~)x)0po#Fsi&^WVB6(y!bN~~eBmFSS|PMzV?KS%8I z(HcZ3bkoLWxpUoel9Nt~xbB7Z>=dH?ecPI8V@*e_O)KYYsEi7N0|&ZdO<9rP!UnsK zCdRmg-S)oKHnV-jwtjndILmsqg!&GSKckM5hvDyG=UT2Rh=hf0l#U?rICGFWhh_Cm zK^L(c=(5`a@`cc%r-dT>W)7{BI+uBG_}p+&F^JWP^d7D6tY@R|ieWtaibN#z5r@K3 z1f`C!{~s(A)%OG-qNT`Ql7b6{<4o-GP&qQtW`pBIisxHBVtJ+Aq> zpaJ%IsmHZm^GIVXoKz~lH4OI0$lkO@&xJO1KTe-g{;J2?xW}(s!bKL7>Kt;ao%@_d z>KJ>!<0%-ztZC4+8)YRE4wf}D!f0RF%B<6$t+DK5WIHMq_+a}iJErD`Z?SAJ+1|M{ zpS%bAnJBgk`CFC6;!SKj^ZmH_oX3MaP;C$G`2vbbo070vr27M{AM?W^?3XiR-R5{(oTfS5qps<|)%NZnj%MFj z1`;;0R$uaWo|nWDa4gDp#0aSq5|^_qYI(mudvDr;FVzjT-P|=XNP=nalU`KOCPV*w zTz*f9z;k$(l*hJ%`!xd($0zUSkx=<1c(Sfo7tg!H+BnCdFkx-yjRJEN*hGmr%DgJe zX@El$cIDAyJ$g8qNbf&G9E6Aje6`bm8%`|yDSU7G z_FsRP3vDv~1B(m4G7dXEyOD5xA|sqU$D221&XM3Y>$bDxsbCXJl1|QZ;j`jKu)v49 zGkCJN?&)D1965-UBly6u=}0lu2S>2=@qO)OZk-+{ehvLDXj2j6x4@BF?0@J^kjF<;JtX*xa}4F2Bcz))I`3P{tQB3C9{ZVVKHHsh7T3h;_89Pq0*+d_zA7%_1;@ci zxl!h^)^Y859~D~?JqN!C!QaCro|s4XBUY+8ZbiIAhZh@s?z>ui0_QYwGGRr_&g{=b z6Yn=kcv+WjU;Emd`tIwh5C2R{&v1jS;zog|m;#$9@f1(qx?O+R{Kfe@Gjcm5WyRuS z57x!BgNF5`$@Ix`95h4iGyIc3{nJ;$NmQck_nN1b!y>@TdElDXm!x0a>tTeQ$#&ES{mlsmg>*17HV;geiHa|GFHU^U99BF7tU7)ZL6^iS2?yrHq3zq>@)ZYqrTC&8%a~Rs z>jK-WoTXr?vkVq9Em@VG*$3|}tS7M#j(Ut_6|3}eU9(=r_J*wt>>NX`N$)VpjEShk z0jUnoVSg(=Ynm9%naeoLQLo-101Kj{T^%U4d ziKl)F*BSt=vriT_Dp`!RU{Ar3rG)m*vENns4l6R%=8`f#p0vAI3ghI2Q=KkGU=u;W z={48H66I3~C=aYa8)s_ZMsTSUe*3MjSlDc{o12{mdJbM>Kx*yAwAnkz#>UtlbAo-@ z{~1@XGDrH)0Ykeb_`;c?wdu3sBtBN7xpjCBzFcqpF3{p~jL%kFIp+R+4nCFvpZ0!D zq&RXcp>U)+STKLqF~9wOJ+o?A@8-3MVzZxop}Xd9blWjv9mlXp23m-f;obHd2K`Y3yBt7jkA0kPEa}9i+aaKBXV|}}Ads)!fOW-N4CbI!e^))jFEg2s; z#1D8J=yPzyYHML?w|8y)g(_wtJ+&i#b1q~ZfGeLj+jtUDX2tW?pQZB$U;ZVY{GIDB z9FDSZe~x!;!$-A6gz9de^%H*TGIXoY1xHfWN+M_`YCQ|VL?g6BQz6pUvRD-JjgB1f zCYoGaB!gIf@8EGr06b!)J{Nh=ZNEMO*LQyB;ahSp%&py%Pk~L8c=G3Hr5-Z$hUTdm z;%7e*E8(gM7_n(fSj=;}5U~Ue4_q7(nls5&Utf`tV~?-;xzkX3O*L-NOU3eb!^%u%wA6&lMl- za)xRsPDvW6Vm&f?kK z2$8yje`vceKQ=zw5MrkeZBF1b$JIFs`yiM2Y*Q_bY@AJ9uBH`O|Zrc;h{3 zBW!(xJ?I@Qt}$Mk$OtdJ0G|-oQGaHdNMqwf;)eGV>ohtKUyC5}Qv>#gBpndq%p3D#O*ku%4^;^vKV_Uw9| zth-V?ecA8@J2`##0YCJ_u0z7-mZdH0Ih$YSrOz++l~uAAq5Hmx!@9!?xA{fi-2bAlzs7HBsn@;uCGi|d^4p70 zYv9;mY!rCbD6okV&)SUM84qWGrscSy`M7iAN;r3r@rgyK4ij)GcVLld zhmZAr^xnf?{KZ2e%}hj!hT+dYfRB}EN#afZ%W16=3lu*0Z9E6@$aQ1MLZdt}2(-p7 zPlc#XRF5KTY{X3L$AgBy@R>gy8c(p`wqLv2RoN=gNp6aC-I~sin?D&pW^I z|N5~x%%q?(&vU>RlncjCmU_w}a%WvFfP1WJ#CSED5jpz(5w3xchB?2gPH`0V zn6q$Qife1XQQ(>s*hGnIQuh^yjRoN-5udv=dgJrX-*L2QX4E-1zcnLvAAWiacl52X zDNEW^c<}igkFu%oVYPa{`ld0ydEM*3D6+Uz`2BSn>DBM=P5p*Xu6ery=lzGxaVm1m zCwrJTCeBOnVF!gHMwN0DtoF6><(f^r_kmAB()2lB6Cdjpkt8*@92m^MdBNhko?@@jpTD|y zQ_!PM$dPMS4fB*SAygNBV>^STZr65Eo&pipkamB`yCT=3~7q(PueVTjxlfc5lSaBVz%cp zL8GNPl%XoioV^2tx7WF-_fK83Tu-gTT|(U#9$Pc9KhAz#or1mOK1;9hvE&fJwIw!~ zWAPFrPK8VGshbXt=2FsabFB~JQz`xiYMW!%oMX~wUvka9Za=n5 zw}DDcw($waulP7J)kL@I^cIv_Kd)~a3y4LT>YF_oz;W1SD%)_N%W(${_jP1V$vSiA z#xev;!ilNY!7g%;!mc>-9-rmd5^Z0e_xb);YfaYg&l%1sBU8Os<6LTJd1#UMrB(+V_W6AYyT+JW4 z;oBWcTAx>UH?I4`mt)(F^}IeWYqcBO8u*e{bFg;h>(H{Uy*YfR`txAHU+w!GzMQ*V z-^PHeJN@h=onDw^r)q!3FvT7chv&7nskAhP9qd};!Gm*Xxr6ig`btKNB4HL%yy5%y1QoT{iS2u;e0fA71k8~ zyEJ?u*zz5WCI27wzrw3)V2Ar|ym)PL}hR|vh4ixqrOk!Iv?(I%k2Ao z;F|8iR=QE(!Gledc&b(bZQ*Zj;eXfeG%YiS9ft4wux`XQR(q}3Tqx+8W2W)Tobwz_ z?t`Z3*g_MR4k4F}iI(rSefsR>P@NJIrUZ*Q#?Tb!jU#7y<2j`C{@0pBV$1y;iDQ@o z6S@9dod|=Ss?e2LW9ShXs=9kMlhj-&an9oiK3e#6z@x$M&^?I#6!znr!zadp)u3vs zdpP-9=AG2OPGw1{`1pZmyLyg{R{K8H@~P`3_-N8+V4Ms&&vy2Wm>VC>Y&svQGlws& zbR+bUeEk}>9U;qv{*0`b#3x=U$Lt(2G&v{xGkj^HXNjf7faGuY!eOiQ$5xCZ@3XXSp`pn=_^G=q&5caE)f8?6>#ZoYie-QTY8Q=W1&Hyn|z_!j`laon(Dt#bP0ws=rnli@NX0656bVdxNu4 zAStkk5=n`t;w-ZUfAstCJ@k4C2Z$E-_Qk*aOOvd}nnWW)WT@L4zvDHB&*_>tDlDI? z#OGY^UE`Y1>V4vi#Gd=G#rqiFebaL(K3Q(Ce2!pq7fy2SOM#Xx?qFcOH0@+z@xph0 z0nX~*{NlHK*=c+YSFSHvfHDl3lZPa@*s*E}^I_Jbq^#&PXF{7f~SFOr62W zWV=)MmD#~sjK%bT?w0m+t1!`v$fN}tOI%v-yOw!r$r|&H?P2&$l=*z;FopHDo+N~J z2*a+)lJAaW`WID%IDy2;LC!Ngr3RUXu)LI$k;L7Oxh!hSp>8cU3TzbUDX@tWJw=|_ z-*TL%Mha_jZEFFIO1}1;`|YUKCe*WzPdH?yr#%gp(2%eBODz$iZHG>^^9jXTRD4T5 z{qoAwl4_4_ZpQPYnz{%{ZFLi$ZReK6pYXxXw$72){m3A}ul0vm?L9nnh#heE{S`-0 zbJ+K2iMW|%DseVZ{^T4a9N~@G&N-|R`s*NIUKedpjZcJGx6zrPMk*C`?`xKHfrapZ4f zTH9jay=%L_#F1z3t7=JGFocs;D^8-TaP}qd-jfKn3L6CmD6okV14Mc6+^UmK;9D{kb}QIWa&R2g^;~f7sYY zP9s12fB!eLuo7Qnpvi_O&Y^v8xNHk$>#M9z6Px|Wvi_`3L^PUq^V#2@`knZCrz+cF z!RPZo)h^3qNaCp0Yl`!Je`b*Yw>s4%orn{O=ERJcf@*z!^xLT}G|199o-UnP_cf^x=4-;Gk^?qzgxW~5} z|kn`3aNaE8@ z2C7`!eoN?!?e6ix<-S8)!4hjV@gdtzo$X0t%q%h3QOP~)8X4K~z4jLyJ%uOBz$OYi zDjxUW`9J@wi@!lVKI0`Ogz9Kdv&3D`M8p)0YyVGOhuFM_D|=Zw6Uyi$jw0VZa9M}4 zgn33r*8UW3RuyqSlD6mbSL<81x z#`uI;VkwetH9!9~@Pz|kS@YG|Vr_g_;I_ruVQj`&#X+KKj@$L6{^XIUxr~h_HenJA zD~q!1&sSn< zRxW{gtgW3sIh*9CiU#ah{Y&pPcHjm}k{TqLN-l*t(jGoqve0agG`O%%j_;>F@rkR_ zGrO7ima?MRe(5>!#X4Qu_>B6Ak2A!n=2#!>Gk8(lA%O$kdz z+eri2_c2uw{dQyO(N@X^#IbE^`?DUF@L9M#NBzD0o!gtin`L?C2x8tMP9qDCoYCQ! zqZ{WhtoxMp6n(=Y)bNQ9IfCYJi)sTrEc>CbT%^UUcW`0>%Ka=jMUB3za|<@2PH$BZ zJCjai9Mr&9KX$yPEW@Z%?rIYLBeB7=1x+ZDL^9mq4W^R}t8 z`hHF#+81K8!|AcG_6ghE$Rb4)rK;eRguag0@qW;eu*Yvri;dwFY@PXjx?Zx2dk=H_ zy-{GJz%&ImQDT}dPf#h1qE81O=GVUA4U?a?wYp1;U`XdH%ZHsdig9)=_&pq1SgHen z5YD=-RbPMUZ{fmxMpIMsusJ7H7}!H&oMY!|-H~gWdY{NX#vt4Jq>4Qm5hrNRz+#M_ zVo6x4!cnHN5=61Y(VvZPL|mfbX3h2`@wvMkbM~J?C|J#}l?R$S? z{~N0k;j}049UQV|;AAm;$NH^avcK5zY}cPXt6s(|Ge{=u)u%+G1{VXPSp?k3L zSy;{-$8zW3WAq$dFPw>320VvT4tec=rgsYa;7aEBAi!{bBXl#bCVkPGZwYCDH2j+Z5Km*K1Bg z=t&IKmV;P?`J-bKU()$5lQ!GlF+~=k-o8udIVCDrJoTEm3n#jYrHN;^U02bIsGrGu ziF3XSVfAYDmWJxX9U`_dHQMMeajX|F+ zgZMV*$axCic7LNlPk~L8=qd8V{f?xb?Uj_T{A+G_KV_uF&P{1m*CG^Of1fI^+{U8poP&?{ zHN(my<5a^XUMI!eI|FM!s1EeUu_WA7D5-Y^)(Ae%99UvyG4>u&`)$7h_Z_{aeWLoI zh!arEPwQjSGpvgHeoxT}%b#%z#!3J9ZtGmr&?uv& z3`;bmH0^!|eJ_CDH^s&YSxhZH_#6nr$zjiy^ljGo%~?&7#TZZG$@3ZrW?+PPk$Cbv zQ+KMEBY`-C2=$+;(3MrKzID`FYr5)peqwoZ&L#D{b93egc1>t>wwuZXGu0HUK9z|| zXelQ*&jDwgUi*s8s$tTR8EgtcS7z?fBM**T5BJG=v&A3u9hSCvfMe> ztanNKM(kovnnNWg&H8RjxO3-lz-)cgPhya4gpNC6Zo3D+qpmBs!nth@_(ZulB&xTh zo3ruM-+h_zd~?o8XaeNPBHv21%nWw$d9J)CiAkP^JcNWtywFLgrz^0$STBxi^9iSW zv(M4T(`VH2d7^=<4hcW-y}xsar91=Op$Yaeog;{$bK$Co z2Ia#39z@vD-22(IX5p%S&v}39%;6_OecSv;)}m60W7Dy%w_=QYCp@95rRnUYEL1u7 z#EGmZegb>-lJo{g{!*(RFpkvATj|WfS)=ZmL7V{tD|3v6S%OcJVr;MXhUY8$;7p^O zM#h({;M97yeiC1@;T4zb*ww)Na| z7|UmH^m&-VC+Rm!ZQ{x^4Y)974nXE0%bmp_WM7V-ch;z+r{~anIAO62gw-tPr#lk^ zB#IqJ@vToNBw>-zf03Q(zM%p2!*WO2t|+)H-PKeg2MxaGpC5pL&iQX`prX%5@Hw z;9S5n?TiO(nMhbDWL3eAC<|6*rro*4jL)=ORbwZ+3VfmcIv4OvJI({W!$FSu8u+HV z5hBw5Gs*%pRcFmIV@6L$Bw@S|IOmtQy!0Dpo!$8VW3kr2Ct+x0$Z35K-}ITjhxKQL z@rCo&o>rkboxz9Y7t2GlnK2eKRqqsslVT}}3|{-S5Ps_AUcL0r$;7E64jSngrhj!y zuijCo;;7e^4kP&L8mo?96<;Jrtw_ps;v03MoWjR}3!9j0-FYq*hS1g}>$Bp^q@>vS zip%wsRg)z~)~TiVQfv?w9tp?y$xMl)EFK}oG%JK`V*2MG0!FC)eofuBY)*yDvzKV+ z{g>lOQH!N09d|P!bJlT=koEodi1;z;s792wTX%)E6D2--VC{t=%YIht#B~!DIjJpi zutFQ-t2p2OmallMzsAaOnDxcG(jhg-Ea7HumHs8({-Tv1VQV5>cF%V8^H=dTc6t(@v;-OM z9oq1_-FWPmcT0|;?lmBTOw~GvCjYV44<6L^0cWC4EsAVG`|;HbSy=b%hw~i1x~1I+ zzSsr2IV-A*9>h*#tJ}M|$Dt8>SNrOEDL!rby)dlFPz&tG9sO?C%6kW@iZ9q_08jj} zM9Y{|=PPkkXCZ5uV=9(7L2|ND?jVk5`N$c)(y|2~u_eS58O_RAC238^wc4L!*}UfP z8*5l3npGml!BK}+zdozPT)2YOZ(|b|HkH|K4$rA_VnaOh`*9}jJYORfh%j_cbHeqB z!PbZ8iM8iGwvwxs%Tx5(?r#)0MuAO~I7Xe-@6kwbKJa>y#U!;ct0US2jIE*euSO0X z`KrDazREU@=8V6jFBRV>KJo__eKya*8ydkqJ(q@!o`WxQ##=Av;^UaZ*LY;D96p(ND%%zzpxuNSUpO}H=Rfdv zb^4!T1+ssYt;0HHxw9mYeOtddIM#Yob!wUC!5AIs-|s$rS?-+MyVbU?GdXD1?eF)$ z`OjznXzSB@f>w1G=C#I$t<5Bz@S1aLqo=2L&s3b(QhFLMO>b(@;S_^X;~|ux*VN9> z=ir0W)^_P|3U<;s&vW>y7?_Sr;EnvT+R8R$9*M5z%sKG0Of+`4!tp*~QE=bUi^;X_ zv1?(&?Yk5rUpl^mC!-&yp=?{fWkiapcS6=3J-7Ay9H+k+W1B}=&!j$(HOknM-m;>N z!aP+^{3GYugq`Q?1G-lW`YCuL_q{8LC>gH@r{CV!eQePPysgegft~`JDA7}7_&1HD z&-Hz<%sQ2yFIt)V(7p%0_6hOYxppQe`k4XX;#h)*5jn`)VzipSgsuu~)tc$)28Z~V^vKGy#n-?1dM z+4p*`&r|fd-NgRp{6u^~ZLq)2@g0jfsLhOv1tZpWn+InwlqdUuZJAr|>T+}%-QoV* z-tcPzM~;Idp}7jH&i?tz@z#F2HqU(fBd>pdbLKZ#IgY?x>V-J7A8#o>TxKZs?p<>z zp1sKa<$EG&W{u`v1jgNu53TEb$Gc`0%cC3Ff8$H1BX?`LzIcyVs~k?O;iQLon|n?k zVR>}rTaxo;eGy}7zjNGyv$6d3w|O32BZ)ETPi5ixiL6`JNE>fHht@-mC(rqOCY;f8 z#F6aZe>u)P=le;_Ql$gy!zlczdb_^vzGKE|T-HafovAjLmU+(;$*dVp$=AenXR8^HED#?uk{x`^QXg)_HK>!e+k?5 zS#jJ4TYum9;*-`s(?0+AF~0N-Uz?sn^!XP{j9SC<&&Ahh5x&>5YxS#&c23UpTo+Z- zO5>m9OXx}Sn(g|W^%^??>+B&SHKiw zA*)T|6StJ@PMvcOu!GGRCE#(i32V&0AIV;GB;}tUyOZz7Ow&I)HhoEabsf}oUdQK} z-+L_!tHh~6?eE97miu+mM4vtV_bl+f+}Lux#C4XpH8kfSH;2B?xo?ZHChd&DI*=?F zT;c?p6M5U(3|#h^nS(u5UcV{eS30?_+jF54(?XIQQ%3Zz$QwZHAC1!nyEF6tRo!=EnE9+q~>O%%_Vhq+Qk>{ z4PRu;<_h{Rl-~H}0eIKD}IX<759q!Sdo>#AGyzc{FwJ4p7PoL{(Y4YbRe&#GS zq)YLohPql-!s^nVFY`0y?;SYy_m|>}j6$s~4RvmR2S@gQN+S1L=+&Mnuu*U1W#dTI}XSk{*h6X(27fQ9Gm%BOJiz|DPHs^(S zffZH&Gt-Wzm>t0<6UY& z>T!(xW}Pq=j+;v@NF(sKIvWLEDGF?&#A3<|&7#qpOFFn1;E4t_!0~!*zN6RNzO8nH zbBEWwJlppX{slL`O|SJ)=Y!hcbFO_-gl4XhacC4jdct|mHcz8CYQ31pM$;)4tJYiV zI$8XYxP>)v%KT%^EX61Nz&Tsnor5o&9a`p@sRXjt0d4(UUnE&wIzC%*Y140i_pqgS z5HIE4oWn)NN`!T0V1j6kWz`W9i`}eESYY7d^F9@$)kMYTe)8o%9@_5^KWgKd>)iJs zGW~RQGERgQwP(|sUw_0q=+CjNSrsIU1`Rfv_46a^o%bVntOVkNWWi%?ab%3a%Pm}a zj@A{IYjwdB(p>MEx5zlx?in|cA+iLu-|Az5C07;8FclwCVbyDG9iMyo<%4xW+(NMW zt=uz}A>?Sm8biZ=u5Ud&@X1yr8LNzF;**qlFTzp#cj>ifOQK%zBiq~lO#E6Za*g&y zqF|m1EH{$TdS8sT-Rf-=c$z7&i4w!JC*zXWQfzrN-0J*_c6RkCyN__<>?QU?R+Rf1 zqwqqRU4*%=nT;dIGj)#r1e(UHI-5+@TdPbsCheDSQEb6laF4-BXNj7@=UDwq?_mwF zG>&swjir5v#HMW?zvugY_Z`-2DV)*zu`_hT$7y1;P1cie*7jl_|HOZMXANL?u!&gF z8e*F5Xn$in$1E5;%j!eRD??OjVX>Lv@88O?zZ`%nIBX=RDs1Y~Im~d``mXw(_q-?i zpdZAEKI}iW-0oLLx!R|xYZ-r1^PeIQF($HGHCWbQvu+;7Cc6`HjtCDImS5(aDcDnG zHQd*jhz3}9upD4#I~j4pQEz`w81(3#CkZ3*aa1!$5%Rl-NA$8+!S{UM_XW%9w?42S z*hT%EcWC1Ct;rpyy^}L*82G+>&S;D6@AnbOzK@)1Mk_F_IqD-jp?kI*#6204j_va@ zf=iunifnj-PEG8M@s2h=qn>9^D7P^*2hGFUI0MwyWTU`SLxD|{xHDA@y739TQ+mB0 z$3AJ;Gq3Zv{Ka3lm~}a)$Bh5^$(YJua}Lb8HlO(LABeHRmpEfPuek;nmdl!vxbhs? zPA-Hy*PPZH`;@qxeO~73uTd_8%3Ao6{XEt$e*G06RxQGdk2UJP1@Ci`vdRU_@g(hmWNQo0u@+{QM`s;V=e!LhM-4-Lt5kEH-oF%5oOcevKPn zjxF&y=eVWRYvTixIGBBj_=){*n*OYxhm)U{`fNYdjH6Q$q8L+_IQ z{yDg^FWxT|o@c5$pRu?|#!7^8cpjFHD{m~8AWHgn?LF;b2!eGrBj9r-b)W{w@c-SmvAyTw@u~)=&wL@v* z)3>Cj_k?(-TmyD@KQ!nr;K+I-PDkjN^{D|QX0Byd1@GRjs-5)wkH*v;X2DW+1j4ZxUvk}%9*Pl z%hYe|?T>x@>vwGXo-XDt_vyga=iV9FpFG1Sg#8>RUk-hoN;#`d^{a-^CqCQnEsYtP zf<0B{S>cFOkCFMBg0+=x6nM2#;0Y8ZUaeEHjeDcO(?Efy^}MkQIu=GK@iXSowz10 z+1Z_F4lW$&^N*g&u%$BAa8zL}kfn-fuylJYR;8pHJXmiIOxeZVF7_`(AE;7{pSZz6 zY0A+m+s&O>jEswBxjGkRec7FikL45Xh%Djgk`-1w6Nxe=1WpM~ucZ}c%`%RhPtV3R z{P2y5WtAwHxJE38=a3Z}-Pr2DL$$-=at&PG8*lv1?~MH$Bz$F6V~Ibf&&S$$zJ;?B zi{K%?Tl2@-oO@6Itv%Vx#7&&zaJB?DN5A_}^DXJqR`=COfv@-#zv}P4=GXr1*L=x` zUi0cCz&7fQ0viQJDZola!yA78&6_bvVVI8YSEa1#UT#>nkrjk=k`1pVOx39u( zaL}P4O<#0zCDp-+c*6PW6CeKYD)=~5&|cf!ww)(;vlTL(&M>sxw7!mOSxw=y{TdO1 zV+`jGF+H^1$L44B9u`=6?q*Dj82cnN z_K*c+O^q-1oW`0TJ%4c`11G1{GXI^w`|nTUqfNzvO*zuc(E&?}^}`<4j5+GFykLK6 zGP6CJ6D4F+!n$%O!G`DkQ)eQ}w|(osG+96K5g%AUXf-nx0Xzx$U8MtH5h5LwJAdgfR-PYI2-qxoDPa~!A6=0$wZH-Gb&?r`km6R_FB z=ZV^D4dacGG0Y^|9<|heWB3i>z!}=s+KwMJBHW7F?{HM+bm!* zDsiSFPMS0ELM&(02BR6Tj^W9BlCHAOflWNIrdb(T#@xF^%uF2hY~6YGvdlB=`MQ>M z!u#PG@8RtKcHSs3N`Xz37^T!!Z==AgjsiZtKJERHNg~%Gh|eKyeE4RUaMEB2+wWZl zpr5(9loibW!bJ^ z_y3LGbzo=apm4@vg_yfPS}rZbNNk(TLu+Id8_h0m{0Y&zKP8-M%9&#JX83SXzjqED8xz%#Ct~*cxJ>YYuy(^D9|& zhR}1Uvz=jDj7H-Ith*6MC}Jb=SPYNscGdCj7`@NE$GUheC(LvEw6)tP@T^f_6D6Lt z8QsRSQQ&L}5FI#GU`K>GI?nN(AMF6`VAZy|mKYUV!^uV{>WBlzD83Ab#eNAntp^qj zeykL2U$Ro;uph4(TcS0XQ>-?>>W_c)>ki|`_K7#RL_<#tN-UWZxv;11zx?Ar9^ui( zI}-Bz_{bhFVbx)Qc>K<;B;pksJ}mHDoh9YiUo0t6rmd03@TTJ$@o)*;xw@=aH0eYJ z79yH^nr(-9Lrac-EIr10$voCR+&0l9!=K@pJJb>5Km$)qyyUr&xlD?P$R4=I`GFQg zWM2vMXYYi*A9LGD*0G$r7B3;TM>edg$tMo5NXp>#@bjNP9H-!&x^GTBqib6yPT}3| zZxndeD6okV&)SS`6kyPA8u3*e6FC0-qo4RA;UrQ&@z)}?aN}fE_>Z$3TniVt zoJlxaSH3tSi<^*fr;` zIXI)`L<#%j{OterUmfVniABtdFwslJMxrREoR?ZQHi>#!S<(P7uY2*u8=^HFtO8;@jE|EH_O|w0>`lyLEnIkwMnq|j zU-phT)*M^CBpLD)EckAls4{i0mYLUgzwh4;OF}JQdi}9aed=b!35-a5AMcr^{om?u z6xb**O@U36n5N5Cx>4XMqW~@Xl;aVfXWG6?cD%+mVb5%RM=Oa<>&X-QHZ)Cf?l5bs zv%b#AZ+`c*tJs`8uxK@5EOsP!{Kb07NJL{g0>9RgxHGf;5FfDB;T+O%r}mt0JRJJ+ z!REknZpIW&RN@1`i9R<+Dvo2{{6GDRQN|L(p(X1Or_L1xtp!O)Ik{j( zJY!4GW!;EEaB&!$vH*OrnU@le^KD~Y&&6d7XcpnfEh|rsDbL28u{F+cUn86rOwRxd zR*tPX(LC0l?AIk{7LM3Dw9yVO;-%PvYm7@wM1+3taW&EM>@|35%=Sv*h-XYhOuJvp zwsfl9IroWv5);E^old>K&Vf7ChN#KH2^UMvHL_AAUfTr{aamjKlHSLX5drP1lJze= z+xutd(D*_VrC+!sN?!_lYyWgpU=t;t?)lq#zfqu3V9LQ`ZNj|sCw}5N+nxFRW4o`l zl_TG2jxsdPKF4`*+*o@|_7~vn6gb)^nO{0f^oUwvQ43k|59^-l2+uwfBpc8{)Y%9@{=mnlo`6+ts#I6^fkFDj-8Bo{er#|tC z=s)Wb=de6(n|Si+&wS>_p+77~UDLCEaunm_CleI7a|e;add-9Q_HBRRFPxoutE~&7 zdmLgzY^BH&mSQ?9iLC}>--XgI&m~KY!x}ql(4`Sar(IU8F9bbVCSUjZFM6K1!6M>F z(KmKdFKw@NNepEH+F03ZQFw#9QD8X*Hc?_Z&9)XB1)eYp2T@93ZISn@A7eLVJ6bD#qW0~!vHMt0e(4|o=tpltqq=8< z?OjmoUUE$nYYzrYfQ2q(_w5@)oBElZRTW)?s( z7$L5;^+7AD6+pW#&cGp zO%Dg1J*z3ac1?V;@6g}|tJ!d1r6i3MtDvkVbX<$B*jtW94jN~0W9zwwj}Q4;c#gq| zxPgWbD|)ORv2E$LcvJ8=v7jew0?j(s)xkl@8nTugITuqfM#_kuSZ}b1WGR03Rs``U z>Tq5=MT5jEd>MQFLqGU~H$so?!@&v8xNPC(%sRp;QVVm%2J@+2D|5^g$*`D`Y*u#77F|vwI7M9I@W8{}EBTub=$u$QgA`*wDU<>!aXqw@(CqfV3{99QweKh9cJL(v)p)GDB=8zU$xbkhY7+{w*1Gxph>tw+OOjP%B3kxNQ(z?P>ps73aVW$TTh__ww8ZLT`#yaItKU~a z{0zFeZw$iKIaTRf?2D4bS#V}iWbNpi*mpyiJbGC>#?7MRd2d$1$Lja(m;AQpNs33r z>3-ZxV4bSVG6k3UVXZh-|C0N$+jXO%xfEw|39ON}_NFmEtV<*Hw>ldIo^}dsqQuia zgIoV!cxgzVocvg%dZb2~mB7I{CGTGg4#=z(*uH6;mRl7CniE;(IQc zHzEiziB%vEjylBLJZx_WACZE!Bg^~qh=?z17)Pdh5NC6JEIFT>O^VZ^_zcN?g7TOYFW0OHwtVNc(N(5i4srt{A|73C~$%TSYt^l zuN4NJ=xLZ~50ka0+0J`;=3}kSZd~ z|37)GFOO=@Ux|g_SQR@Yw`@o9~oHb*gIeQvO6^R#2UC{ zB4Q1(#oJ|2A8##oh|?21L^11+V;W23L8txv-!187@qDHdA`)ZL zY^SC@X(EJQQrUCyWIOm`ZMIN-tXwAX%`)qE%^k4AVQN3V91E896Ce45k=XZ=Fy$PR zGlwdfq*yI-{+zARkA3o!7txKi1l=6oC7dLUA;;rv{dz8~$J(!@*Aj`=XQ7TI@o~ue z{3pNRqA@+{*Dhs4zDIH$*cx-qb&&Y1apJl8_dFA;9hZjrC;aWaQQ%3Xz$QvOsWY?n zYoowP3ec#hbCd1UPBu8B0=60ZO)DRoy*-P+W_TnE1J>N}@~8jkL4*hvAT)_hOF9vb z$=T&S?|N4YWqwU$1{xWjU3@d|du1^sYyH3v|8UR~t6TBOX7j~uABaf6xgY=N?!K~X zB_4Z6V96w*O*WGw2Uuc4(%}j#l5O%F+US7+iYbt(sGTSC^s~C?N5ai>ifhKm$;+G! zpgUPb3_;?kXKX%)ptaa6mMm;s?h8_JaZXz6`LK73OkBAi2|P5>SYgkHXXR3s8&Tnj zGzTraRgQ&op*RUNWr>X4s87zo=?YX3_MS;R&cqZVjAz1c<2nuLcnzYtc$&<5MO>9; z!EFV2-tnv_MQ#o*uvN@G`ecTN6`yA z#a{OL@{`4>+DfcAXNA0nJs!oQ9<8J>%xus3A)JDmqf?d%+r?(l?qf%@9NSHkz62i; z25Uo%fj2bF4M2uKgUzWV<42OUSD*Fg+&LyXRBKh$)aQ7d;p3RmSh`h22(-aE<~|)2 zmuutaG`EC?$Lg}2$*|Pp;dsR1t2(tDgLCSh7zhq=Q*;z^JrWaf3J3Lo?FxnNR`Ic&0co63sI7KKo+V_iT6WnS)mD z9Phd6*?ZPdVc+_-hQ)KFpWY>x(s2aOR%fHYeNtc(CGL~j8?KE4t5JX!Pl%*r4ff?& z$A}Ui|B?UTzy@G58k=~rmNIerjNiWYSB4*W_=FdBK3R{oG_Y_AAx*gTT(y{M;**_+ zMj5M)J>;}RY?6pF@vn)`7_srab{u;;g^#$g3s?TajZj+GH4_tf=ln_fn)vKlfISSW z!NmMxE6;IqBF66(@&C%&mOwh?O{?D@VAOUK2DWeRqd!9k?apDLCa8?o>dKB48T zQn>}(lm&^XO$1vGppEIapiyMuCk2(-hc5iD|lQr5gpFa0NZh<=9t#dk!(e>OwJsEYWAaZaSJz2_}Dzw0f(^}qeDJ_I5~Zs ziznOhI|oaM_>}d8&p38!tzD>T-|gT%6^4n8j~&F@xZ%w{UgIp>`F;P}?+ZH4h0@aQ zf8xV`AohWnW5GuHMihCeB`HQ{>@YghJUh^XC}jud%m}7l_|cC5pQw|WqQDWdPuz7K z_&aU5GMR^(MQE7b9H=CH<&dLog`aq2J0hk^13+AKy!|b2xecKM6>~c&IvmlP)9N1HB&~qTW2fhfw zJ_o?UHi_?8Gi)lc2D^8x-pIXU_^{eoa$=+%$AwCdz&uu`;$yj^nGb7Bb70GLRp57Se`!K zgfpLKaVgxn`b2r^56?I;+Vk%nW8VzxfaB&|+qKHQ+r+pKXV-#x3XU}rq57vPZud6| zJY^KvM2V+t!nW>i6xdO~=T?%>JbW6jMH|{_EIpPtv#tmgzLbTT>Mu4Ua5Pq_KV$u; z+MH8{ei84`!ewdbEB!d*z&)qAUy2WVCe8#6Wl^fQz@i=Jw1Q3S?=8i(gST53b1YeC zU;{$=zaLm+B}NsiLL+Ic!{d!_0f40;&i?2e>@u# z>lH4^P2bW&aU*-yG?>{^KJ~^ToVS1e{I)(n`;Y(e&D=cC0`6`YhY)1Xv~}1&cjAhA z@umYY^fSkj?-6k{`fu*7L;c|JGz4s*QqBwdvkIOo+6|zJDmjkP`dht?0?!HsHc{eP znaOP&uQCb<&7{qH@r_@;8hbrOLQWmDhw=GutQd{!RK2OPV}9JVKJ78Ta=!I6g&bY5 z`LxwEq_ytUH3t$7S))4*a}dcaPVnq99{tJZXvq2OPvKCuOPKku9oRZ7%3R%A&Y|bF zCF`Xjmkr82S)9;Rn0c-18au_gE49Nv_VKTe?ZF;ikc?AkD;B!$_G^nlES7#aSY)vE zeRyi$zmea3(7}YyAvtD-U=ywEti5LVY2p)Sknshc525pGhIbpgtVW+InU$;=4ndd9 zPft^{VNKyImGfi%?DMe3{3JdJdf)z*uQ<@mGw|^9pTD_n>lI!Zm$L2}=S+3rY^|kJ z=CSAD61*&H)}J&DH725on9q3!Og(>>;5*iq)s6Guu{ztmjRH?21vXLQX`HpK&#!U{ z(BAnJI%c zD{bCOEvywAgAHz)*VR9X;KGr4;8Qz_&8N{50=gD4Xb)C#@KD_1I{^dvGYU3tOh>2JOUxfiL%TkUh&m*Eqg)?$3v1U(0B7}LIYHyDDsyk?h zZI?GyuP>8G*4*^CI4`kyJZ#n?Y@$>Mwv2PgMpUz&p#w3zhu#0?=#wH*(k{=L^jG z^&#t(>|yS&X?dCJ%T~ToV57h~6xc+Gb?CT3*(h*N6cA#Kh2p%CnU3D`*MIw}2^4cE z@~#VutSm)d+c9=6#}O=^9QNq|BYf5F%N$2+EtNQh>1N+9Iinw#*!$b&$iWhl@UV?o zY)NVLV~sJ|Z^Kq~9H|mdekaRy61p2RzoN@r-7Nnj4E)3nH&~N{<5`+ZMj*O~3Z}Kijfz&A}Kg z7jE6FCg3nPZLBwMzP-J9-HTrm;~Ir|yxuNDmHja|!LhQk_$AGVforY_Ru0els3q}u z-?zIP1)dHHY@)=|F;81BHwx@2;Ir>@$jA9wL}86!6_8CO5_C4Uk`~`@mIKLK*Tntt zkA5`RzFnfyFWg{r`7q1elrar&|MqXcgA>b7gKL{EVh1*eHuzM(jLlYXG`m=ezW!hP z>}Q{oc$Q`uyP=M~1c)@)Yiy73Xj*h(T_vWK-<%TsB$e4>>PS*Z8i#Q6nOF}u!#~+{v2&R+$gZ802U8hja_nftg}tkBND?Q z`KxW=(gCHiOjut|9cyU8>^7Sk>xyIQnsdUVUY>WA7LJ7f5B`t;JTfD(L|}9IvgbWc zK9`0`vZN4K_=I!b=^d10ut|K*jcvU_S9@*LgU|b=dxx;FeD;_AyF2(Q?*3Z8&!h5# z2WYtj7u?wYrTDP<@BFL3_0IN*l?}&w6J0n6^~Y!eRWV8E|JHYWb?nC8)2aSu30XRZ zS|1x<9kLR(%x3z_I#$QW`Kh*Fdi}u<{rG)3J;~*xz0DR(Es$=D;w&&jGI0~N>1n0t7(vAqtVeCFsF-!Wj z)x9nSHc{fb^xdFu6j+G@vI_~>t;d=<*t45H1spcgP|s432geb+TaU2Vqo2uOkM=ku z@wByd_m^IO!0z*04ywE@yfAy*2E#L^(;+y<%QVdk4j=~mC_VG)~Yq) zf2Lq>WluK+Hc{f~p1-a4uL=q@HZyj#O~)o@Y#JF&mTdFg z%^6}2#*%W*E%kJ(Q+35xa184YKGf)%eTl3~SN5ysNR{|FtW5QH ztuk6dF(B}X&3JeP_PX}<4lO_X*{hBXUhHAg@|x@$@l*T=s*750)tL0CBYO zkBHycYu2!OSoeu%$$3Py#$q3X=@}r3O&z_3&-z}u+wlEgn3z^Z9~#I-dU8 zySjg`^mRslUprQAqz_x2jRGjJi4xm7*eGxn3ixDl76@mZt6toTsqW=qdb~Gh4)wv=Cn`xXe~d@f@oUe(e2_>jewhuHkCqTV2~L<5#W~ zXY(lbWi8e|$I4n@-EfqH18Zk4^Zr`rTn9As@Z{KtWNYC7$2`6NM}FicLcJT^h`W(U zwOz9mw7I%CIj|TK8vYPt%UFbW+j~Qti1AskYm7}ihCM2B+e_AycYXBhVmE5=(H1`M zv`g^GYGiI4V!On#eCWOJeGDeiLLV-*aPUPa;na zGU0I1KJ`BL*xyU71o}@)Cqb!Cet!OgMQhsE`dc54cJ#zens01OvVMVMKNexeL_b!F z2)}OSoX9FjteZKtEg)@iy*GJXSTi``V>QC^)6qn|c4_^S;(PJciVEe7R$#&jXZZSyhrlk>qjz}VoWP+&zlciz^PAqZ3R_smp4YE? z@x|R5X}E%IYY9s8-r8re7_{2P#V3EQ@L+JvC6OfiZ2c2c;KCNJnao%G47gZq7CAV> z83qlT(^lZx#?V?(zBU9apzgoTA$~WgOBHpZ5`GJnM)P z;-!|6ss30m9ID`qFzr^!^YNZ{z3cG&OtrhDES%xsYmR;+F*}#w(l+A0_G@;{Z71jY z6?-f-d>lb#0h_D0Mmfi{P$M;b#OyU-9)lqpR<6%$C1O8@fAroaN{mu!tG7{LhyvIf znOtbM`Q{EaT3Ctpm55?fxYp{ zzSj7r2w=~JR|211pr`t?cG-9Twci%w`+VEgdu`ZL@c7}Q=e6-+BRJ)7Sb>jthP{|N zrutdSJiDA?uwg7#Q%Aw&HFD(va&-5wZIaIm=F|2Kb6WO2&-SYG|Zg4Uzo z%5fUkg9j{hMDhu@Z zyEf)6+`Is=-wz1_I{+# z6*h5(l|qO;ttHW4I)RaKsly3wL?rt7N5AfQPFBr$40|XJ3k|dEejq zy{-R$a*|&B?cw9!U&CL8_F^6LNWc739g+Em1%QZ~2ew@}cc=`SCLUTwM!S|6Jvq?Z!G!i=}t^ zHJbEG{A=Rd!MeLH;Bn}8o?Tz^ciYxOKTm7Hp6=J9F~NOZTst}$-?i~fk98}3k}0r> z5>N8%Y<+vmD8ONbR(-9kDs#gZIwvHMMS`~Se|X)$a29Jj2S3YcBl(PH>_BQpIgPZ> z$w}vzgfnuc(H4f7I9wZ}a7$&W$sd6=Qins15O0UfBa$X+_uc=?@3<;W`@YGFV|)>A z+TYy&7A8)6Jachjqa*PHYBi69*`XVp;DZMEMlZBll&} z%5mtsEgHdGnyl2C8nL*TXMOjsT8g6%L)$lYd$ZL0+x13)S33nZQR3CU0=Dt5NCDY@ zuw=q#=bW6{r}=B$PuBGu^v&1!CfMJ7W*ML7cmCDi`WWk{US?*BgpOw*lh~5zAd7^= zf>;3fumu$yKXq{dnQa1_Rqu~nCtDraO`2~xF@}f%ABQJfGTC#2Prb6l#gBVw$TGHb zmn;8;@By#=vn?F>eC~6{ZCcZ}{pH`(a_xWk=A4)fYz@aSnYa2(SMcRnoYmu(-q)*r zJarA9gqu_KDn40=F5N9%e2~8O?3%TXt;U`jmw8)51eGyrDsxgeQ8hf~PMk!aook8~ zBV|24u?T2rTyTwrRgYlW)v5U465E1)Vkpp1?9w>2(=`(x(OnEtu0`diTQ93Y^dUls z=?F)#MnB@^UwQYtSJ}3$?@hcH8trmS=1VGvzTO(YiH{M3)o{Q=EwKjNIT+qgIWGIv zTw8@@Oo`8Y&Gl=oa?f}sR9|CDzqR_c!Fikln<()(O}3Ja0%uTwri>ptzv_C}gWwEs zvrTA3f9gkn^acy))!rX#e7?Qgb+7-Tv*`Kaw|rU9NNRy&A~Dvg?r<$uKw@7BP3v)D>Y4+}gPS+M{`HgMxhz9s zUUu<3?|F>wN&k92g>UM<`I6)^EE{G05Hat5ewI9}AZCEie6s)pFe5-u|jxCaO9wZA?UY;*DqMlCfyZ zIa4?}5wFfmt^<~`wX76U8vNMDzy8r0+dpfG%UOo6OUI-=>z-%6t_ANe?=h(stUFG9 zYv2OIyR41_e3t69;P-G?{=6H!r)=|ii6yg#z5U)OFiL?g@i1g3jIqJMgOsXGi;XBqnpM5*+-WNXXGmY$h;WPGY z(39Uf%UOTe^3NJ3cCIylEHJSK_iTKu(fxI@jLJNVlD!g#>Vefewg&opv42E?KZh@I z=Q-<%OKDC$A!78fr+(9%vy}D65B=Z|-Wb!|J0S5T{-`&FU_lx20UT-s?Dy31u}^+7 z>ae^m06ke%mMi1<(0ku|l-1|Bx`wly^~xGD*_kU9$|!gx&>Pr|0tX6gqQrp|+r>tK zb0~mSs0Q)8zEtZu)fYa#!Xyud`Nm@BHOGt8;!m|5Df7961AFVpaM{GspT!YiMMhvx z)e+j;!#wr-o4@%>&;AtG?fhuR?IDpzXh{n_^@CHAlt`V!wfi==bNN?S*;68SSrD=|N6PwZg?57bnuN9=pjjgvSb)I94PJCES2b*K!yBkLjj{u=MsAcSQFNxkp<|Iv1!Zcfd;!CVo!wk zVRynYso@gNjTV>UJJmk%*~02n-AnHa4b6Cm+ek2aDa7OL`E8pw+Uu#kAda_w@Ic%E z3jimdQ+3@J9|7Mx-}Y5!(UOHq``AyR?ek!-hkCr{U0?G&v4L+me3=@ct>Yo~U7Pyn z86pP#+)w_o*=In1qym78^N-(lpGMm|$AEJStLC?U*Kdj4rfq{~H*&{ZTn}9Js4PA4 zzm8?ewdXieI!No#4zWeKW#b&aH8OA2K61)~m!q2X)x`1G?$dMV`2X3vAE5oRyCCdu zD*u92!H$X)Nkq_TK^c`&5Pt8ETE>AoW0}fSMQW|nc5KyZv7KQmQfbKi5Hd(ZiP&-tEn@BN5J_AkA_JF3Hy3 z!Ie;nEoem)5;Tf<@w`i8f^1Nbl=Dgo_!js?qRGKJ3j>Q1XBaiKVz05XC4A?Cz+%yy zx5kH#fsX-)^aA5&Ov+1kX;yPolyiM8C^H`d+F#ei~ zTL9CbF?zC92aDtvduls@-VSuoH}mcH6r2qGN_;dy!V+NdWBru(<6JogpJzRXG%#;* z)czO;eviom@xyoCnWHw9ea-z%c*|?`_-LBUuSjEK#0jzfOV~9&#!hI>=rFXFQ{Qua z_|V8R=@=61b?E$Ad-0A|dS?%9yVhGNuu|YS1=g*^aT={PRtj`bKyHT|&@cr`6)ZIl zkz_im&${%~``>@sC!xfSbLS<=lRbGBqSTnHwOkHZdeZWg<$CYHreqc-Nk&QT=itQh zaqIqp@B8elv^sr#CvoN$OfpJqF-0o@2bL9^HpG5CNk?;}Avizyp%0CoQwCIH4bB8k zg|CGjcpojx>-=5dBY7^bFLl20ueoipJScU)J)h>Bu~mJq^>5oG(S`U+%z@1iiX^`8 zNNjP=%1&!2e+KOky4J^#K6tNi-L?ki9z*bjZ&z_T*LD_XaGW@5%1|kRaY&?A?mF6RtgNGz`B(fMyIvbN`Z<3DR^QI zwD-yPe%qpKiXe>)eXyojcmYyLN9U%Cs)n=snIjA(JU-jbeg3nnV7exjK7*Lw_-EfV z%9V5nyyHYL(TCrDCFhWc+q44QT|@dSws_oYQFa+*851Kb<7ETiDuT?h{xT*Lpq1ek zcC@a~PG964!E7IV(|WADeb2Wte9&}O?O^9&SmHJMf=Hj!yL>j{4JFLHb z3Y`Cfk9KLSZ_7D^lan_;`YoqJw~}a!2FJD^CHy(({FfNf4H0e3!4qxan>&92Ye;$7 z_&lHT=?v-HTBj`q)~!TadalrJ846%KecEXwNYjP{>YPd(QA_wHJp7YipWHj&k&Q$c zAdtFFh1u86xZq;Q5Wz&T**>%E$8Dkd+D^Gn`+=3LSr@vzv-XyF(y$wq94@%bKJ);7%CT;^J7y%ugZc_fm-$9AgdZj(hJt6`w!G`L2KV zpDg~FBC#%WTue>jR*&V-Z#YG)F;0@_zAWj7V2r(IOA&MxpmJ>cq4fnRW}GGpPvDc4 z9eXOG)3cxX%xUR9eGD#Nq}yNm`8mgih+Fj$+OxoQsLX5Y7{c23AURA#j7L6H^LflUVh&k;ajaD47^$57fN^6jdbJsx&>D?0d>Wt>cV6XRA z3ak{k`6;k&C2syXUPsuR0zTO!u0G=qN@F(o%@HMlq&#qtu8Lr@ld>kQ1PN$MVci7 zy|+%(9a1tzJCt#r){b^}2&}cvN`a{qSho^WDYv$}1u202WccINOk(Z4K76`rHfbuU z_$$;=Rl0ujHJBPrrR3Ndwx58Ju`SVK<^ws*8TgCGMC%=-MOCv?=L0jii zK4N3BS3|UT_r3PZvH>HXkvODO#G01r9O3JFF8MOVbz9G4b1MBLHeT2Rz8B8RBU0tL z$Ka5!MtR>kzYsWc>c?XUxA%!b7>m`_bKZvp>rmJMZLza_66*G2e+Rx-+;PWjSVM3# z7~A-`45_`=St+nmppOFUR-%tG>-C$E0;KHhmf5H;IXtXzlCmw}kDf_7O{#st@FC#mdb07)1m~0naL3wj zpGUuCQ8zwr+C%#t^}?@$K^wRA%$64O!hp*=j4@njEIf-eCR1GGV;-XP4xv>MjqZU5 zAKdFde0bgioep&S$C%etGS|H^2FmggtPHv?O9o zS*O)G8|Kh}1imF$L4E@Q!ZPXzJW<-_v>Fd-o}jmkV*-9%4sn_7>W|1x?50I4BF)3f z3bt9pH(zm$pF|zastb14HCxz#(^X=pvA&`a#r?3Owxu$+wh}{A9v;G%*ukCA9X2iS zVGps|g5SoqTy_npx`v)4%vd(#Ku^(!__pxxKx??l=WY+j_E{W?uS8FZO$5P!(RK){ z-3zN|7#HvT;UUuj;sER(=9;MsNj$cdaT*(a!=V+ptK zoLl)3(4Duriub`RuY2~(b%7?G0FPeWY$s&6w_b=Ox(;+Lqe=zKe-wOGf$<*K|2pJB zRW1LFtr}N+Vuh|cEw5!wT$*QF>6UQ&U|l9Y5z|6ku|#-)a*WXx$@0ELUv5>H>sI1co%vgC+&)KKqz@^&TM@-}g+$NipLDhr+%7l*YWggN;lNV( z7=@4?!h_`rV=5BVu71tA_R06Y_lih0=*Qg^t17M4QuLHz*SHv4VBz)s5I*xeXM6&r zo>c*S17EpKvW?4j+-ZL2;OlDpisWNs%=*EV*?RTcVJpzpX3n*{&d&BUCh*u2FyI;D z0zDRYPQ{hhPhLaG<(Xuf=3*?IWA$k&-uT16FVhV!!EQy9I@WIJeQdk=Ve}IQTb@5| z+;i7q-3M#`NB{93%rrAU#`DZG|6_js>bi4(+dh`tS7`;Yp$xC)!YMh}wb!F5 z;FC-$DXkRNbTojG4MYbaIdm)$&{TJ9k9I6m5d(j!RP@jg;Uck*F^M1(=R>o}9)e%r zGcNEMZprn46%BkXSv8+~tRS|DbT;xVVC~_f4QV0iq7^$!Y&$%TIVx~(gPQhZeq(bZ&jEuK+K#M+DS%|$4e;4w-u;>v9*&{#=G5~%V*8iaVs`b57WZxMe)pw}&H5Bz zx2qrPwUq)lDh1Z9#Em+0YrjWQAbkd}v~agPhL$nF3+K#Cs>n#6;Fq|@1Pd)MN znU)Nyrus+JeMoCA2iX18$Now-NE4(Ko2Xu5)v}+4`GfONeE1q}*Kfu<3>ip!<%qbj zeb_e2nVS>NoQ|&x(1>e8v-gT@LxRc$7JlQ!!sW52EV(049<0jGgG89UeLNdV+ASIb znd6gx%&he!FA3HkozYulD26!jh_b}cajp@icNGn)KHIw3rg4!M8Ol9ILJUb5BpoEk zk()vqm=iuc`p;*mFIRy)C{r~%e78hVLM!ub9t6#1HVSeFgXsnJ#HKAD%=Mtgs*Hu(0h)lk~}-lX6Qnzn&?2uk2v za;=p3WBzJ#^2_z#Am&1)@ac8wyV}K+%2%7<$3@2alpNy^>j?y$1_pm$=7GH z^?H_{h4k!|@WIZsOYq~FpF0+K0+#1H&wqknKV#?pm`*n>0}ecg`mWxl?mT?PHst-J zFZGzGUu2njkG(JDDa;d%Q{ zfAb!ExM(VrG+krdT+NAXLw2jK`dyA8&bgPHKhKJCtX3;D5dp6(Y3MwEv@8Ounx8b! z7v?o+G^C_8OEC71_dfJBr=NQ8!7Jzr25k=wEBEXnFz41`K$#63=K>noo&n`$8Ey^i z(pqh$z-$VvTZ!2uTmfAF6kyy$X^%}>BKRKo?1&j|2ODkyS2 zAN&j8%YjAfn*)B^nhOByz}iM&*uFB}F(MsB9RaT82N|-(&zb8}fAx9pA!zHdA=-v9 zRuU9j_=dj2bAa7u*yWvr9`a|XZ@bS(&T$Oil>0O$wue*NPHBTCOMH)}^kr?cQeX}R z)~&=G>aA^W5DH+eMV=4@bcrsr$Hlsk;yJLzK7-sC=i;-C5F_1I#Wn&9ku?K}4dhlT z=xk(i^~s%!FB>zZzUx2RXxj&_oh1vOU4zs8Lw3$X`(Ya)0u?FbwXI~bZrc)H4ZG|2 zbAgXEda3nUkJ0$pk=NIk`h6?lT#M^da=$0aIdWX>*bsaP> zS(K38HTC__rn#SU+XTQPWf?46+pa@?kjOuA->biLxBcK&&-cal;s~1EZQCr{m^k={ z#JMs@-bzMry9VpYN`aLEgDJ3XB?gmft$5>7fbu_%xk~y&RG>8=o%C6ZINtd)e5lcg zd$cOXp%jUVG!d@4909VkDg%&u95J5GvJ=Yji^h>z2>u?}iK?`1(vNJ@U5xWM)y510*>n@De1BtD11njPQ@Rjp86(88}9KxkeJwDf#{Hw?3cX_nLjdq@m zgY!I*WsE-8&)bmSW1_fqy+;m>$J*(Ij0MWUi zgb*o~e*KweCQI^yPWl~3T$45}XH!lSmT1@npHxj!slnlpy|;^fux6xAQ)kZjthKw& zJ}ae7`;vV8@*|IQu0Q?}k!EmcM5H0a%yc>tL73=Js4?mnUmk-6I-c#>?f5vAFrOySr!Uz8tc*p8` z7c;VY;Gu_hW6;Ljz-P>5KaRnhb00oyS$%K)y*vfht;F&)UNJnU6yVn9SgI2~;rs` zekp6ydj&pj;AhE&-+SjBk9^bBfAxR=*Xi0{$j4*v{?B&X@S>|1PGtzM5A3FQjf3H& zd4tb8hxkAWI|ngCoGbZiT|iJ>;P(JldDw>8MuV+nB=w8$d*7+RtX1>9pF@H*bHVps zsSlzF!1$P47CR4Li~Ka;tbZ#7t~Uy-TZ!v!M%Ml`ra+eB|Hi*P6+eCHS2=gVh#Vv1 z=}uwacuc@?O(s?MiQj$dfBA15{QOSs2KIv8GF&CTIU10zzR=+C2MGy1T=Fr*`1rNUE6Ibq%898rCTMXS93ceZTjr`BODq-CZxo zhwUcG4!RFvjcv8CVqxsUJs&+woBA4`+!OkfS;5o?#$-SCy4T6QaA8YV4=w1#X9u1x zcmEQf$XIOLjn$TzhG2}p4VPG&0a4QGzQpemU!J3ci(S45R#B&ADOF==CD4N%f11am zv>V$8&ju}BM0YYa+7H^2v9ONS$AS}&1pUkvt;8pu{IktzNNUZ%C~)2P+Aqsn622bC z`Z@MKzq2>|=pWeOBCQ{@{)Xb?^CYAF?mdy_iY7N1YklKT7-Q;*{1x=GK8SeAXMPN< zV|C3HjV*lMFP^t!ZKmG$?e_lno%4?BdcRDC-Pev*cxgVN5BH6Hecejj$g{WheE4ph=QE>>NXmsqhZ9^GOs)l*_VLdX^j}oFtWp+D*Aff``q9 z3m%3CY^a@cHLYRl+Z01Gc#H{K8lOc*DL%njF`venzFHaIwd9^nnkdbcI+;DKJ96MR zTu=PMpWdTgX_uY1PHB9f`ov$$srWwqYrnS3v~ApDgTxAeNg|7O*fK^OC0h~rz;-{@ zkl(gdV*;}pwuoXr!&ur)xzECs_~0nlP_S|b$T#X1TJsly$6@>CS@dj(c&2}&0QOWC z7Ql99=WmnNi?nf1Kl|M|SSo|Po~ENN8_(am{Won4m@3+BwiPEQ`r^Ga3UEhdp%$L_ z7k_4_>9jWZHsk=%3Ag(l%|>7H%;wx4#k)lMJ(;}$&DzIt7pDZ(%rTO)$8N z?HBqI!`RM7nvpu;iTm{m^BX&w4F(E!CPzh?1GJ|tx&^MBNjVukVF8xFCpe-2teNdh?2MT6Bt>2L1p4Gw ziFNMjdhg}LvRsh`_w7IRQ(2-X2SC8dJ`L$p&0vHf1w8ls{oj%2)!dSh^#ZPLz=Oq= z+ab0uj8RGh4fu@DVP12v%Y-Mcjd9s!V;}95wjvDH%5IhTqAr#d3q<w=-l{d zP#kyu(C^%{-;Z`RK5Q3SYdZ#4z&ljRjGp}ow)8ZEoax9+p;sn>$*4enst#R^{0+QS-MHt1=4LTCg|&DJnz;` zU!9iMpZWO5v-X0&VoSJvu!IN*u|Z#*TkZO~mAKVr{FWLoC9fc`aV+TqM){g!%BRmr z7=DoAEJbjis`zXLzNLH#Ui$Jc%wr-g%r3jzd=q1(^FwIi6Q5l?6CF2F=97~cA1U(? z7-Q;06e%pzwQVrkz{lt(yT6VBc%&{ncJo%I6AXN=w=^OlRR;U1_y3ny%lP$6;S(qn zTYWJX$GNL=tMSoRMJ|=nFbw(2@%ii@nUbToX;4DC&FHK)9x3rL5Iajg3|+@jM5zM9 zGVY`CY-4c+V|K`m!vW=iAVNre`mDQTCGnlWnnxL*=-gn?Luv4B}{H1P4tL z$P4XiIDQ+KEgwS|+uN=>v1sl=J5rRy-P1696$aL}knh0u8voJy%RC%p*OHp9m zO59S@eX9+YBsQ^cr#_DY)P(`dQgjf>gd&}!P-$z?8kbKxDRah=KJ#UnQ`I#>2oHEZ z!!aH#F9Qgyn(?{Uij{#c412f^Km9AevSVM&VT@DY)J8+h5@7J^s8R66JhsHw*Y9$C z)@js>^&7uim0Z5VxFq~**rxB+Z!1JwIOX_A+_7~rpUTGQPRjX#>d1=g*^%`l7W7|lZT6O6$Tb*&G%K(rTtGu!oU@r8j72|vkZ_@$6`PtpF+=5QIO zK?(ycY%jVpunI#z>nO&roNACqiVe1)uQ6WvIFi^hq>wuN>n@I+ZZ@8}m=o~YBh%o@Rj2d*hB_L%;dpR^^*^LiOtRvIg7 z$k^l_8PnIXb-g!e54^8!{TRTGg)yW)8ZxlUQI*nj=X+#*4rx2KPB~xUjy)E}(E6UI z9PB*Jmv_~achpc^>P6U#sqe!vxYW(sGS6Akl8 zq4D|5V0ngWWJn%^rS)m5KkeYuFk-BziaB{!X{6!r7J?+4jwL*{3 z9t^>Gs7{QJW(oV9X};AGpkqysL$HS2i}A%ejdF;s`w;k1$Fpv|mg6yx*{3KfH7c#S zE-e9?wzA3hT+pW0v62`u73)wTZ!v` zj@BMdra+Pcax%ap4N18(n@O6!cN}Rr;)1NM*21Tbzai0#CMkH z=w96q*GCNFLe-jfewX9ez&{^rdG7+SBNs|soAbT~8+_^`eAq7IftQrSB>*)=XJ|9<=lK8rp*S5dhj?7tpWFI9Dc(Q?KX8z zX8oP_dw*;#VAmb1*HSsYmiWNx!{uIUT;(|G^Om?a^S(b%u@+9!A6(^c*QWlRRW$Uw z73{iCVApi3>$Lh^mNK@r`;wUuimNvVVJO`MDFB-e%C zB0RMUD$4cP+>rze9vUg-xY9b8uY>VXNt3DPLJx4EW@Hf=*tzh+?{K}c8gZBPEMpa0KufYEGJl`^-t!Vz!; z%w^a&l?da2|gT%(~rN%Dr07D9| zSVv}I7K*9O5XLJna0NaATU(BgFM{Bcwg)Ay*7&eR*uCecjO1X8up%omyZi6?-sfG# zYM>|fCU2j%gA{E>JH|tgJ^C$I)jc+T@y8jV#dzQvE*dz` z_T0(vLS75~$i5skfRVpzd_fCiG8ZGk9j%_$$Ie|oa4B1)AnQUdiki905@BUJ__}n} zoqLV36WHU`}ZPV~Oc8|%fR6j-+sM@h6+xG^a}L3vgvnKugiWyZ2mhMp^8 z&y~$>@%0%Uz-O$juo+K{>zsa@adzQzEsh}yVlZnw#tfjjbFLynZ{*;m3G8Iy&JJ*!sU2<3516w<-7U zE%R$Lw)3$rg9X&H1LJOubu*sL*fw}uV>pLb;_8~y^O~*BHsiV+TpI(`%UB<+sPkOG zS+^3;73EjF3sZnp`(q#eOII1jJn-Oyi%1*77+~kJxHXbtU?Dqft09tdky>*T4&$3Q z{>WEcr2rP8Dqp+yzojh2XHCZW%*X%g4v#1&(ql1HF&@TZ(T5f;x}>-Pj~1ksbdj!c zQ&cCXcbiF6Q8~c6J7w(di^ZNJcr?Z*gK%HhTjC?BmIt73Z2Hg=U*DL@>&NhkIA!g& zGPLT#$6zKIot(-cBAuEwkTpKKAJ%WR=C3;SjwuhC z7ty%_Cbm4zpH1%oMptoUqr?(m$GPWCIDmU!_vP7`WCttJr|+CpT6T1r#14=LFVS%H z!7Z&ouEslRb>zn1;oArrOhXtVTZ!vq9@btoqW~in#!SEd%rjYe&KTe7<`s`c@T$Jmw*Z1N#dvxQI>MX7?Cb1nTjZCurdQ|8sbKiUJ zxt#3Xa|%Fk!vM-QJEuOY)^J`|2Vl8%tWUO};E1;`&@A>kZ-+Z@)d6_BP~U#UXi!8T z_}~g-AiqUhDh<>a#LYFDz}|a4%pF!#$w-VLniPoHKPz3c&6tY|E$({BOLoRUaM?D5 zah`XZVAuS@VqUxGH{_bW3jUl^6>jHAw&=hGhSZ10=3)r0Urv4{Prd*Bd5xGS@3SQg zj`RgC?=E;UuCx}t(}whOY#r~=EdD6yw^YDcqLy-PTLZ1OqLmP#?R~1-HJ&w)laE9?CA5C7w91@yRw_eyY_Y%_d2&VR!w;-UsZi(zYhr z@wusKK!W*>l%yo;94gIs$wI@&&w_n(i5m=d!L{wUaSg?l`+=oRILd=|LEQNqbas1> zfj|zzThzt&52ay@IU6-3oLFg6aV(pA+(AjZGh6TembH)H#)Z8PI73S;`paJSo3CPL zYBzXWh8TBd?YCni30{J|>caO6|MJg`rs0sWW9Q3tXneN3+=+*@A6my2oA{%CZI% zqe}!dr2Sgw=App4mAHB4avce2S+ltAedH83+ZM$^IxY5k{Eo7(?Ud_SupMi(WsxY! zGyAeF^`lihd(!%@ycIrC!7?dHnx~AXs|_i9(5@^aet(R&%k|TIw2Wn*wH0Z6#;Y%N ze;IX=>bEfN3fM6|+Ml?lM3iQ#k`-e06y~uG{i+ z6m7A0E{qOs|$(Qij3v1`-xVH0B_qXr2dAcCN7jQQYsR+knHJBPz7hqLN11OW*5Q(ozDl}yzrs0{h9#^a=kQz> zt|9$d>#P)LL4kEE(SnLAjO%~`q*x4&hOh{8yZgo`pS&Vn8&-n-KI4KV6{1P{nj`zb z7Y7!n`~=S(Q{A4DJZ#DXr{5&{b=_9K8Q|15=2#n&`__?aqR(iEWiRc|jdzM2c)%P| z@_!*!4%+hTVP}j6UB_GlJ@lhFf)5K{mi4*cf^&{t(;`%nRLbZb{sUkQJs3m{p&dN5 zJ*DqZiKAy}nXlr9eLg)~!T8ZPq(C6a}z^wi4Kp_dN0Ki+wf($rNx8>xs<` zi6M5u7NaH680|3R2|RTvUqB0+y8)yz=oZQY|uwYzx^vDKG+qr~y zX*Wp$VN2s9^K83sTjO}cKl;k6*gV^(G4pk(ZuS}4j*;Y^(NWMi%AC3V5RifQJoAhx z1Mb0cAk4CwEde|KY`B^7;WFlm5VF`OWxNKPkBqzsaeV9} zA4%&j<>zVq$EVCU=rY7wjk~^rTPZM!0_#>{64};9^C{qUJ0Hq%2#g=F6F#%z=pSz} zy@rS@+!cMc7*b?5aXPqzEp3~)*p-Gyt)z!+Y>g!jEMKrF`SYKo+N7CW{tk9>z)h~q zrG0G9+@7V}EQ5x6J{Oq^p{28{4lMrkh@dd%IhkHEOfk#QB zb(Z|Y=CH-)N<<^!4+2l%fR6@E9t}s|T0J(#Ye~an{`sHmws(bJ3%qam(LaziBKmU- z-m&{qLB<%O4B9bJoI|fJ+~qnzNAF`e9CNosUHE7=6YP9=&veyUudNig3<|7UiOZn! zilc)9Y<#i99k-faB0=Xycp0&%xt!({a>=-1$&8>RUR)tb2^lDGwQK1<2tH}(KYrw! zGAkPfK%}2+i+!3~*iOe}mszG6Qeib;`uBcYj((J*hNLmJNShY;uy&7O3zt#9(G&=8n z=xeg3!*jv)G_#Ir?Sv7}9z}qG1)pn!zFsoNaQMI@Eg_ z?(s9<-Qrj@E6;b!cV>4l)~hQ8Zhi`^TZx;0j<3@Z%Bo9|P1?0o7qKmZOkz{wlaI9^ zC2oatqs)FH$0*0Tl%IX)v>|M}=itM#%ht*rI!9J1KK&)WBrUv3f;k75wxp9K+^%!d z!LkB8TtJ#b?CNL$6B>j+2%46;AL?_)Lu0^WsTpk2-UyZ)M~+(7 z^X$j7qRd;UBM3!w6CBk+ds4Qaa()EeiFZA?-o=sQK`93Z4H5B6)`+D#O82tiR4Q%R z!v7^{wgv@;7WmnguZ)mZ_L1*Vj5`~36*u{V(PY2pz3& zu;EL^5*N(WCl8)bqLXZsz)tIZcPoY{_r3Z{ljQTeHWI}GA1PyZtMOMU&qepa#%IZ8 z!V_hTuO&VPJORHhTjI0kY+r5hN<4y#=3D`!<68rg7UL5@v&19NWv~0jwE{N z?qp+!#G-PsWS_evK86oD`p}l;X1K8=J`&jY#K$Lc34Ca1T$%MN?3zWIX^oHcHSDw* zIhEsUIe)jm_{FCW|K!(a_@s}tIc&IX9fFp9_~iC@&70nok7MJ<)-!U8c<DJ&vUxZKc)}Q;Ohlz zb@rkcWl3rH_ZSmg{6`WSH``bz_o=`~3sbHi&m&DqQ;kf4FUF&k-(yJhq@i?VY!(y;{4R1NyzM))eEwS6{(FvSI?DAx zOcKwZwb5d}vcYbe8+aVqMZTaCe6D-n6ql!;CRpajJB-0wowGuY7x!D?1Bb|=7EixjLFhSDnj84g!m~Jay$o%~ zASUP$Z9ErC`K5f|qaV%q?)m$_BR}sCzWL3kY0JN%adexq{mvX=EO02Z4?+^1w0f_RrVAL%sHwz5J7V1kAUky| z(T;XikZA|Eb$Eo5klVtuYFjn;IiLlg?JzexO>V+t9DGu+_4;E>bH01hk;*GX<6s;N zYrX%CM}XV8IcS1r=jFyg@q-`w3sRw;UK?pZcMfOR+5K<*x>H9JqqAWs*IF|vux=%0 z5^!xkmIByOTZy48&sg|neHw$a^vUzOWj{Vu5?R9RIv3ECWGzRCO1R*|wzQOT>ATMa zNhS6VjyMDpl|8nrZ&R-MB$l5-c=(%0)#aR)XA0cDHvDF6S-5hukLJ#u)<>Q@w)RWr z2`T&JbIgvrytd@|SbUzFq^+4e&sv{iA0K_DFZ!`~>avDt=5@I9ME2-3Ro#VW@_ghBdj zEsq++wlYnt`M}PT*ykf{?*j+C44phb43nm^r|@8b85joaC`F`V80NV1uDfQa{B``s%#vpZ}*#BhZz774Zr!U;6Sd%}{hAl-K3R+|mLda|DH}I!ZE?fycL{uqwIbbE;EA#|K3kU@1a>Qx z3OrG^!iNnatqWU!@UfTJu)w99i;pJ6y2#mp6RU@{uw9b&4{Wq05et^XR>L+;8^@R} zqPp2|s-PW~k1cwPgY=M;G{(s89!ng*jX&!4l_ag$%p5r>;mUNv#wJZtz3uz1mweu7 zHqI&P1+FOj@Wgf6kEp9;Ach4ENodqllD?L8(UsvG138+TEFCTzbev~nsVyFM+I>M! z%~dwa+l~!hYba=7Ys~kg1I6e|8&>xs+t_+5Y7NoU<;X&bb|o!i&5rJTPXp3)QuDxJ zuHu|Ok#|8))6tjZ;4%LJUs>X7Wk9K8o+0ZXa9MLXKyNWVTIJD$=xAVEd0V{o^&-(f zmw0d1O9U(KXg=zawZWjzxNQAwTe)`cfNaM%#2{?f=D>-d>On*9`4_4-#z(sXAD8uW z8lGeIZRaC0=AC@r`r;Ymte5?r=l5#BXd77`Tx_{sxl9VITZzl0_KGV}fO58_wEhrL z8L&{sw@B}G2v6LTMb)P_ODE&vLM4MEcD~q|oEC2(JiubtLmR@c0h_;+R*6eKT*k%F zN>EJW>~bTXI%JYw{g?WU&j(j;xWHo=*BW28tKUZVl0T!&ry_a|d|aicj;C&yjh)hX z`lQsP*|s@#yQO~Hx-A`spzOPOW$a;rM$kq58lo*I`-A0-`NGzatc$qh z+VemC?1jGXv_acB9*QqXSF@2y;9@Km5+o^kr2*^;wGjAXJW6agpNYP$RTrKi*BGs^i!X6Wu~ySy2&`juu-e$~z-7JD?DT1a zj=}4?ABIk$9SmGbbKKQt>NQ&!Yr4eYyFkk9-PQ_UmTD(Ir0gG&v=V`jhJ^u~{xcG5 zh41bpm50N-lEEv9Ijx&{x|X&|49QxKkFgj{XL_IP-XAe1L<$j#pw;*Sc3=5D-}}5g z{>#GWo8f(GTin;D_4@J@Sho_((|EL2 z$CiqZNsNoTdf;+D@EISY0LDV$9$wC^xEDv zMjYxx&G@eN!?3Mr837M&tf%KaYdO|qyg7y7Onk9jfy-7% z%E7+A=FIjp@nvmQ(q?k&KO|<5evkHPEq_#ZLeNgOy|JgZ+kA`0=w&Ath3){`Y*5;YA|*X)E3Lg zz`-lq0e?$;T|LdwVJ;CL4Z&rOPvo7LM{M??Pi7Z=#$JmkPsYg5YHa&4_1U5GuL*iC zjgJj?;8F_8Ii|m3>t>tr;yIj)k3UKF6-sMR=48Je;r4!Pd|2gZqdYbq>#n3%{I&(f z#@jlS_vK)~LSebw$9k9dkKsE@+xBd9Ry@yT`@kR94*kxo<1vKAeZE2Qj^<|Xpd}Eh z!n!1;py6_EoN)UsEW?m!Xa%Z4r==E&Z_m>2(LtSBE3Fi`fhe$UC2pX}y79Y8V!=p6 zX6U6xBO5fsn#UgfmMj_aDI{fUEwR8Vu&?*oi_dM8K7I0Lv>2l;7wK|n0|JL@B*L!4 zM+4IWA1OMRq-%^)WK1R%#a44A4wj`=G#7oA1-Fac%4yQl{AHiyzi@Mg=vcq+1olfwF49=10kZ>jGn;dY(N)e_HhyjQwPYo_b^g=<|%iff|6 zd7sdB7)N=y=AOg&%X{THt-=m(0eh)a-xD--EQMo*yHemrr@*?ExY1|uhU!1t-Z%d6 z@4M<~9aW_!1lL-2=<(U%YFXE5^_wJX`}yDDqdjP?bno{mZ3S5f5<-IBA+*(PF!R!ie zkhj$79{c+iz6vyR)`O$VcO-mkY=^VvV9KD1Hl<6os8|Ag(o!&JvHgu468gh7-hGFF zpq>|^fCAjnQ*h8Svb{UEPUR*D;~CeYo^ywFWgEpc`xpM@pUcOPEgVsZh(p2<&=lBa z+P*LbHTDQ_4IfR9>)h9^E9d-O_^nuLB8D9u;E@WkUd&$@7}ia^Zy6od*QfmUKHY>M zxGlZk?ie*!Q}HpdVzAcY`8(E!@PBEMZ{!#*_xY|7LD3$?J9S9AIdvGR;VFFc=fGLp zPN%@Sm6%SuwdH6E&=!c;A?nFek3GpbLno1Pe1gMQVu?r|+*7%!Mlil8eFoY6jwaBN z2C^e#8^n6gcFf$olP`$Clt-FQzKykrx#Fg{G`_N<*)>0n_hoN21A=i+%l-}U`+Y^4ml&v(Z<^*j%2F~64g#+Y}9gX8kX7xT9%qZi%fh8ErThF{%`E%24=Y6JhajnsR$ zrqrMFx4{Rt#_dpd>Uz@Fv7zC`am0P)$&A1KwYZOG?ik8$^^GIu zW-6UZ9Q)_{V}VB>+T+<^1vlt*Hgit)$FcreDRBK#VBJbwzjJew^^D6ksgr|JdCB34 z=tCsu9&YFWBfjZPy3(!$bXw$WKWK6QMpsrso*H%!56XV5I%Swf7hSL(VO&-Wcb(^ z6I-!MSH-}G??AY6lL+_WBXuwD@91$1VY*K;NJ?Kz>S(geUjz{aO8?HVJKH&q`Sp9q z_`vA36wJ1GLWhzKx64=x8*fKk`=P9Dl zmGebgNe9X_Iyp8lU*CIebse(Gro-8h3dGcS#gj zw-T2`=oJk&WdS40823w11s)!Q}mz)NR#zr}EBKUl?4Ahuj z-#Or>@xX)ia_w3Qh?-fNBxz%a6n&dovhC(K!0wq*hw;?x!~Jjlx`Zc5_L9y0Twf4D zPvKz~p60E$|2cbX8v9&dkRN~LE3fp~Uzaq?sk;)s%ZlW}Hr?E2e~Pjr3!MkP>=wNc zjuI?<*35gJc%pZX%ldQAtxN0bZf^9&-?j9-*k-xsC~nOqFhQoY$FAV_MTZLZ-H+6XUlL&VpXN`Fip2PdJWmpb5 zOr-bY*E2-K(cAHXH}rjJ@!Y_!tkrH&3and+TXfw1g zNN<*spm3oJsjd<}<$o+XL0PZoS!AG_L2y=K00jQ;dl=NcSr zygAIDo#&$qKy}TSV!oNRUp|3fojOb%}DQ`q=J{lH#_Yks-@XkJC&JJfdUJw9#a^OUy_ zX-x3iLX3rXsJ`t)q-z7;`_uF0dqm@c#)sOExfh~{c%Gz@)4ugwff*q{1xO}If?5Sl>_SK>E@BGSm-Ifr%2PTq zzY~kmiUn`c^)+KDr z`^M%vdt+H-ThJAY88*12<=7I3v{lgupH}Qs3w(Vr9{=f|zEpfUd+>H{8IZ7t@9W## z>(*n~@}ud}>FGokD%rNPq&>rGI%H(GuEQ`;JuF*937az3*WUBRd(R}8v3B~(m;Sxq zHu}B5pM&Nr-Y)@r2)8S-~jb*W;7E zk>(Mnv#^B1g)n^v&-1(V4FyK86dRuTPQoqz=yqZ z`H6Dw(^1xIasGpE{;E?34mB=zzb#!9qp#ssTH|AcP#XnBn<*uq55^lL%5d32*wRFH zUy@5K+!WYdZLnG8n8Jt$`{FuVYOGvW+bP$A4LeKhG`dA@1?+b8qvd*e`0ZD+;ak0J zf9F5>W4m#e_^|MIot>S=A;vye={Tm0SsT*&7~kFh$G@MSduur7cd6J&&nYRtqXj;! zzO;s6s22TD(zb1)2u;PbX;+S~gdgV&^=ubg;DWIa-)2kpnZ$Pga+7Do7DSAGMl3Ow zDEUq$7P$!?&mH3&?~{NZrJ%Q@_@$xa)Y-}*B;ZM!m4L5|=fU+)Jo#svJ@0%>N<2~v z%5~!KZ4x?T>R7tk$?szf@ZB!P?sbl8$3F7!~jxK+o3run)8jSoP?4 zwx!?Fw>3V-619e7Nqn9;TR}F|42x(UXqV90Vf^aE*yonkI(o+1?<#3Iya)8_EN$WD zZbhX-i}l}y&$U=r&TZ#HL#o88j?FW}rI|6P=Z#_C1-wTz6VlWry6@K97`n%L*<{py zeM&`l$0?Bo`*;S27y7bY%lY~kMBBQ7j|9JNZtk$dj&~$AO*s!a&raRo$Ztkq;a}E; zuWl~;cxXzey!VJjXaR{vz^dUd@!Rtzn$!}oSC}gWx+t)2CAz4xUTa7JY$2_LpoQGe zoXd#W5XBA@3AfL9{i$0j(nq|Yje?*84EiqH5&$ib3XyF>dq zwO-JZ4gM0^v$|j5pK*5;y{WF(>bGqM%|H&nFcaug?|*+{mrrw4BW!yd?Dm75^$xY| z^&Xvx?nLw=nw$EFZFRgyJrA@5T%jG>IcPBA+8j4&TP7l($SfYuU*NAxTget8A{s(l zmHLpWe4DL2-@BIl8|Kn_ZNF0BW~RWpmAIK__c&t)70lGpg4TIV+i}CkZ^JeHV|-|f7~`LdPo$T# zv(MVC&lsmtUIF;HrE9FcuG`tP@BKqTL_W@s!P^B^Kmxh@$4p;wG+EX=xo zJZIKP%tNe8W8lh*PH30cZ4F@b@7r?D5Az>K*E|?IR(>@2szHl+s6ehiR|>2Y=%B#5 zmFOVKt$9hrh$#x(vDAVC8{ulj8F5K)!%zI}lSQ;W zjMLbC8&{0)fd?O){i*L-J8&5bTnwwkxOYC*wX+L$eND74DX{_<_f9DX{jTfI{ViTU zcU7=Og9)t_EKbOyY-h>!}ZXM+$539a$Tq6lutoi;tOi3Qe!RVDX2HVQ3xN+^)J2aM~Po z-EV!(yM=+?W%%;7?)wUFrNH$-fpsf!J6}{mE#+LZIx5+C+c{sc$K_LcZrMdRulJIOD@xS(fZ#TRFv z%JB()+t;^N*GqgP#kO#^0ePFB3?IuaK?|HEK8839VuHR=w!jlGN_^m#TQOTz!9}C) z)bjD4{^o21l(%N-IgD{AZNY4hxXkmbnu^0Vo}ngfNY_>mE`C(Dol|jWXU)Iu-S6I8 z_vLuX^Qq6fEv+qyccpDY3DY?Zm1jF`vv=Pj*S!;_jE&Z&#_{0&e<+JWyvIUJhjw?T z_o^)%dNbmjg3mgL1C3t(%2(!;rBh(7ZB`1*rNFwCm`lN%qJ4UOexs-zGLLc-TXIlx z1(bm>mr8ZxQ{uB2_O}cL9HfS9V_S~Vn8a5vjS&K$lFMK#d?c^sn0&%S3GrD8oRJb~ zE>2R|G61fv(lbv#oySrJkZdtN5>K(%V+_ia|6xbBmiQid%UiO2d*G5|fz-U*M(3z5 z4c3cDP2ocW-@UK<@+7%G*^MjaNT1x#>A|)atzVH}%56^6?<)Bvg&`Gx9RY5Gg{kWY zJ!!UV=NUqU@zq&!Rj%vL`HimLFEy(bzjGSGIS8t@d}_*f47%ntd}-SvsPIsFR$x1S z(ir7uBH@VjRF9I&~Qdb#`{ZMbF{SJ^AEm)--K3MsI3WotsPSSvDj|u7h3^OQYRv9a3EMWJIylHX_DQ;)1Vrw%fp?%s~JX4tN}!-(`*Y z=h#vaU20tLxjr954dHw|=VRJjBUP(w&;1?aGH=#m4w$;7=MrnvdMMXK;NwO+cPvBN zZ^m?6&aqI}M_>JLv>JPiuUwmfC)Ta9q)FYcF~2u!WHUbNb3@ntHRW0>e6fDYvBbH1 zt#EbGEbzEDcRbpo*XEhtr&oz*&boj@Uk}a8K0U|G{KdL`V}WC{E>BK2bc^v`gfZp{ z+QfRx<4o_)?s;j0Z)o45Pci19?WWcUbpR}l?QQx_~vSFu-HSRcOF=OM7rXU2^ERYk*=={bCbQ&zNTV5bN_pEYXjy!=(K%Gsv9Lj+`wXRIy*UwxgS6AglZ z*j-w-^MPGFW8frEulHg$4%LCHs&i4)b;kaNbE>vwEUvZw^+ADk zD{+0y!*kFJO8$ti?e)EcLcSc9tj6&QRbKnlr!KtK%KCH8#h%ap1jQ$6Nqp?h%dssv zzOxtoJ6E}1*7!sU#lVRW(c$Bt= zL*4R+^aYD)JC^-;68vECN!5uj+#dpGNF9;PB1+V5Ov$*)R>OB=2#g_hvWP25>7}9E z;n}VYYi0=H4Y4x=@_}xaf8gp4QWj9>I3g*f0WH-NQoI&3u@a^!#$tB|pyZ6@XC5BU%woK@GS@;;Lb>Z{;Sko{2{?EPo znm4`abdI4;IYum6iBIZ5`3|tDB09zT&1_d%-(}me=lY*>J8YlF47+$9yMB+w!G|QY z5&9hck7uIqH&^c=?(`(b=x-Y3?eNfV32kb?Qg-1JZoxf4bZ@3HjDAt-w0$FsnH$66c{4)t!xwxo?DW%-qtK7H7krR*pf14!Kz?v>oI@_G7Hj8(`VA~rlyQ^lA_&iwZJ ze&;{=V;Lv6=vMBQ0zJl z8!LWohSWQ4N51imPd<6YkZ)_VQ7@Id)LIC_?sbe z*61XBw#&i`z(LcqCzhEAzi!4*nbwjPtmQVW#7ip;irl=q!u#v{e79#Goz(*4i@Mv`y<@g-OI7l;n zuGB&Ixmz@!Lxv2FJ{#MZU#v~^W63fCd@k*qVGN|Y=={8Y`?s#X_o1(8#?tiBhfOxt zS}@kWI<2nT29l^V_Oay$*VfybEsq>As{vA5ZR?W6TaE^XRJJj(o}>Kqv)?^y9Wn6X z(wdD}I@m^Aq`(#BoZ+EQ*Ej;UQV>_qKP~K(b!*G>1K;=A+21TjysE~9W{k!}c=DW< z5sAi(%~t-dv$KN%VMQ`)kPufB=betLl zw#3V)q8tZ}j=;n5T`23eIJb?s^UmL%4K-~e`rg$oaIG+IEDEe!i5qLaX7m=@PV(c( z%JTH7^SO20o#I^Yxf0I6<8?#Q9L5x((5{u)n^agc5N(@E_f^C0Ql7gu);+N99ixWK zg7{n{cyUnn+d5$b#SqVIcDm24u7)w@x6e+tGf((P24D7fo}UdD>Q8uP4HG_6uhf@W z?4fzdxDK%Fj3ThT*rnDs#(ne&`*4PGeRW!1j{u)oQPMr``##OlO zmiTBvhp?yE%OUFu4P#s}Ux5#O4r$nijHRx_r8@9L8Te==>iScEd&lK~u*LTXo_wTh zw7zrS+xjL=BmaeqQuZsbeYI!4-5<20n&cS@g5LHrL0EHibc3d~*UP zd~#-3quDYb;iLT8{7yG1UfiOsDxKgycfjQWBAtE{Anr7*DZ@EmN71@X8YFRa4h*_518aFUrCH zVysH9(4{0a`b{e1vk=*t;bdebtM8H$8=qF6z;dkjzW(*iKJQ&~B9amMBo^L(VWA2`R9BCOGt z@C7)AFYXgh&>GR)j=@#e4R}LK8c9Z3IlZ0~d(~kJTH}b{S&TFk(ZG7%+aelu9hdRF z9@x#b+Dd`ziUR9a;<}oSwKFRPRtgNKfY{}vU}IRd;n2@(%8JYVRa6vGI9t@QcE}6@ zXmdms;v3>%v=!&dxSzL)g(MSBlD_jE9X^BWP*{MFB!;BaR`@JQ7325r{$VuvZcZ<^ zl_F#D7~!A~cW2^~GF6=WaKusm7NqtVIzBqa75%&Y#h-JP2%&7Z$zYte*%kfD-+YdS zY~3zsR!c<1IfEja$*u!yj1EQh{?kO87+dadf6YdB&Nb7v>y3c+gV~Oczr_&x_X8Ss zC!SYs$-`RAtCikESFFc%%Q26RYwH`20$=>R&;Hpj`tASxi$4F6FS_wMzxH^gz)FG3 zq`*?{`Q;cG$Cc@*`|d!&mfmL$ky6@-yj~+a8B&_utKM4y=d%mecQyq5pI&Ys-}aw>AaVt;DT8 z|Lf>i3Jjot&oP4-vGW;k^)+E6p_{Ve5PA=69Sh63+R(j1Qw=c&5 zDK!Z0%l)HG8uk60UO)Ktu@hTuMfxYk&gG_ ziRNy~p^eEjEgSWm)2669By55-pP%}~U)x(l#_VA9)q|q*vK#@OH>!(V z<*f^cuoFtgUrTEQt8T16R|>2YxDF|>ZY8e6Nm;wJQsB9#fGq)|n)@IA+I(Q&u)y3G zTlY@dJ?v(%LuDzNREZDE_4c3oktROM?__D7>PeL>q=H zC3}uN&MawEI8w5^)>m*vK8fmrTVg-4o2471s5a%;oeO3>G(r}GA8iCAEz@8HTU+N> z0hgPxpuX8HIWUCo{UdxpND+NwY)~&?4 z3RViN6sRb`?Nd|}TM6#g)$Kt)1$>l+^;l(fa^Tfu+s-~*gph21f2?739}6 z@@B_*&e^c-8~CEekgk_FMd*>|r4If~BK|W!_a}FCheZQBx9`vF_qI>5jSBV?3;fgv zzT+zTI|R<$-AiooP;G%WF_tpA4?5Xy+m5#P#$BR8{lX&Jnt=C@vClL_(@|`ue}SE)(a~I zRtij}z`B)~OuDt{;UeNugO1Uew$qV|2_Z{N1dJJW|%TGB&WJ(?WGQ|1xg@^B3emyF~#Rcz8X-u3!Qfm?+F z>sI1cndx<$wXr**Iz2zw&z5IUf_5VM%=9U<^b;e)~&4 zZ-X-#${aGv!Fok{!kqr93rTV9gHL_uZY+WlV`bguyAor>s&z@Nzw*=1USX#n^|~bD z1*&|SwGUfbpiA60myv-NJVsB9o#J3x7&!|7748tl0RCWC~Kl-HP?-tdpU@&Z2iZQq{yzz0(5 z&+~)P)eu&`?!Vv1@O8DBbIq0qefq{X2Xp2$ATL0#vz`B*Vx#n{n%}RlS0wm2t3}}ibvrYwg#6p#9Sz^2D47&ob z9t-4%o#jt(2#7i!q^R2dW?0hY)*J9iW;2Yq&{O)n#}WAl zNO`f@B;0&9HX4qN8#vZojQtXvpsr}bCcV6V>^!HyO5>@0Bedgd664SA*afg0pfl0r zY=DL!nn@wYmHFAve5RLmdh9*#$?TjhRmoZh8#^ZDS=N4}J{n{6p{wM+Z=5t&?vvb` zTU%9I$*wkC*K+!_1gF+Y)OD?8ZuKPi0)jI1g1=AeRJZq=)+I^puV&3jSB2`DZ7CdC z^RZpO{6VT}y+6%2Haq*Q^wv7Rfd;gb+Sr1>@khSmC|%B5pPa<{%6=N*N>1t{zp@{quD zEo{*th9%g?E?BMydsw;|chwzoO%`u<)zWqK)qlF1GVvlm}PZx&^HSO^JGAC4nU7wnW*UzQScYB-&Pq z&lZDgZysC9_FQjk%DUybb@k4NA3kL;l*f}`h;-$!RpE!C53wa==%s!(C`}@#y+u)3~*j?#~0*4t|GO}g{wWI zo5vn~bWd83l`nH1&?>Y@uZ1PXQjrMvvC%JoKId= zv&t`~t6kqUz9rTn+YFxeN~h zi?s`BbzlA3uml_P{7*l-%P5PTWFy~Y|6jwzc42P?&t!AX&A4aSC>xL3UE7ai{mKSk|KH!-3#=P$tP}VcgB_YbfrlA74P|A+ zNG+P{B4FhxPiejxi80ojVswQ5G*N%^wv(gl&~ssnI_Dkcy&#*rkObwpmpsqm z0hUCt!d@wGol#)jN?d0XvUX>sz&r}bhRbGqY^L3*SkE)qrpQZ)mDyzD9S;rLz&L7? zefPGb>@^H<8|P4*Ia_aHw>PZ&roY;6B=N&=Zj3;*J^W`Z=z-01u8bCee^V!}c9%o( z!L>O~PD)$>HL@fhGGb9z+szzA%kDC0wkf;o=+B`sZZtB(*G#O9K5Rj&4Sn}(1up&R zy1!q0^J_!L!*d;VO1cKEjejUE7>CdvymLCloF^KlXMPN>z*qKrGwSaD`WEYWXqy_+ z;k8oNUVpC?SSfI$P+;9k+$b}(_H(7cQ3?cmhlLr+ULOUzt$qv^r{M}7UgM%US4c3lf=yKAn;;u}7`u72`o z_(+m&U!-$)?by~GA75Yl&Gn1xsH^_w*&9~3t9@LHF&=uB0~cD1o9n=`Uz*3h?NIaw zu9n-8@_h;RW__J+&o*mozn^n|S$|!44$*op&b9qYfu$+1ZY7qc@CtvWz~U5O^urBq zjA6snPUT27zw}F+=rsS&e{=QDyY4z=EEGrBtxtttwv%@tN8l+8gt+Twc5rJj#Mr59 zbLgDQFQc5=7>Zv)WI|>nQyyWU!)PJgr5Sn*@n7H;%#aC-#5hOa7@sld2?MA1yzjqY z@Ue@k6!_GA%e^nm#w_ZJX64$Uznr^ZODhIS47T`xXwQ&};S=K@a~1fx@CHm7va{dG z7Dim}DNF4tu{MmS*r6k~$l`JvnsC5%|t+8o2D9YaDs9SnxFVeBXP zFr%>&Zr^zrsA)SaJ>UkCb$S2W-gY{Kh9z$JVool@WjDr$KK!oAzASMb_#XbJ->_Tb zOW+BZIa*hmyDkI6sME5H=kJ(NYAc+cKZc!C109d;hwa=U5BF;ceq}wANcTxPX*R=3X&&lwljSve25q=N}BTkx8G$;~vg_jB>d?O;48`hw-Ptwtgd6ag(x7R3fIJsKlRip3697vagcb+U2rS|a7ixG8e5 zU^__pNIyx%;vg9#`Nk>;;#$HybWW?F4OeFDeH?;_y3UQZ&y7)V%7@>6Wy3xlLQ>p| z=szUq3=JHcya@k$mIT)ti=t^hC4Dx|`g+&zl*`}>7gd3ABlxPoo!r!o>wmuEj-#v> zxI^g~v@8SJk~Rw%npplpxZsoXLV4(qcE$Sog@5_yu1Y-@bwzAiA_%5+Y&~!j4qQqR zy!4~&LwSGdIn4zZ{W=43tQF7PP}oD zE^*H2AxE`JQfI8$5;#hjwpYfMH4o*Dh0bls;H%RJE`iRxGl&!Pwlqvz0vDLJ*v3`| z?Oh6=?GSwOnJk5)gloI&-F4`hE9=iW_lz?deCq>Wz4$YypBrPN+tPOL{g*;Zk@CF% zFT;4(_yn}RtW50iiG*l#1+Y%+{-bl)TERC3 zox~21y!%~Br=#wWl5P8IU-c!K-No{Hl?O{r3nI%d_B|U?Fsj%dSYP*t;D|c54W&Os zz=~2X2X(Q#*=VQ0r;gy40qa;vgMmH)yQ@4awm&}8B(gtz=bhPTNd3HJNp|ALaG^gI zK5)=Z(A=0ihZJt^kpN?@N_aVCTQSbJzw2E`@$C#-w(UrEOPJZ%WqZ$@mN=*2vsH|- zDQkSRFWHb~i?a-#TjT57R+at7mW7W=7d~}uHKR|Zp;(P?7(J5_*rVUFgC|I9*&pY& z=Uw=+?fQ1yo{4OfRX6VVUE*VOS68uxjsU~9qx_U;ooMu;AD-K~UEgmh^t6-FyU(~x z-j0AR=q>-7_${+NtwA6B`a0hyw0Ypc2YcJl3dR;O@I<-P*2DIRFHhi6dWU}M!3X!A z%X&O>e$ybDLkG8V=_u#GTidP_crGchZY7>e>aVz03XG-z2_pZ5I1Wi`)D_2_!3mcA z7&5TrTzD{LJC5U>Z~eNxPb|rNVnaJ&Nv~CYj*sEXhC*AGYp-2(EwkhmFvb<7n=y8% zs&e5c$>bqCaWBQ7--9K4ZFU#l_6rRXTh~d05^1@W2-saE7uK>}Y%`*6>55&}tIy^1 zh&HlM%Yor=9@iNJ)!C6VYgf>GXr}U#oVvCmiBw}pJAui>*T#k>1Cx(&TT(^D4 zzGnlVv{fv(PJz#sHM`YE;{#=hua+{CmIlnelJ;zz3+~pBBaJ1$D~i4`IFg~f3)dzO z$d{%Rn8^=l#2+7$fY)UtDdnMYGGtQ5F*N5$-4)M(bdYcO(Ld0cmNDvIOBwIud@WZB z+zb?0w-PtQEUsg?1t@^Ez&cAG<{*t1ZHI4x-z5orrl)=m15TDQZ`qgl9G7M3LTsDD z_8&_;HLGkb6)bT%x9x~*!z7nEDo(mCI?~g>@+*7ek1_O>42Ht(8f+{-X>XQ}Z^w+y zDizPP6WqTUc5O8YgK>rdRgCFv?|ye~%Rnm|CM2Bjg$AbVcV_F|PX=5=?Wy;oFr~J9 zUjnAA*7^ZoS4QUSo}BKhV@{*pp^~n5@#lY~G2H@;`o_A}t- z?bo)SrDlrHgzZboCL1SXz#Qy%+eTl;7=@JEyAq%N*jktPz^)It#sp0bpA2wsfANcV z^B3cXlQzQr@}Ql{eT|9(%kudv*Gt*l7Syx%{NMPur`F{&AOCoch8FW#);)C2DAD!+ zjo=;POL7R_v3vRbxSf-%)j8H+=Kbu>zr{C1gfj;nO^+=dT6zYTrX{1+SldgF4P8Qn z^AkV*N3YVB&!N?jwhTgPG~-({q|I9A@2o{jkbfNjoa?tk*;+a ziCjz4FE-av_t-%BT==y6%*Wxg5$935mOm*4KyAH-qmHt;-FYgwYJ8S9MJ{_&c(l_Sapo){Rn<6|# zUN;@@99yzYA70(No(_$(>z?4fjP1O0z&5XpJL6hO%}=A2u5!4yRRgb&f1oa3j8wQ$#(v>$RoQDnnGfzL=J4U%Z z%tto(I)dYvoxJBT#@Gjno3A>KzB+L|Z*$^DN9mdO9P;%kwLWFltc^*yJX_`UspoQ5 zcn9$J;FnRtA(75emlD1){t_jO`aCpLjI_Wmw?p0s=xA)mLQK?f5B(;6_FmZYq1tXO zTpRMu_(lJhR@Q(yZz*E5~x09f0u6sRb$ZY3&$tUp%@EJFc)2&BWL zT?`U@F6UrkB*VafJG8Qf#buaOFH7ALJCHw>b)^Bnu?eJ|40{3=V;jaVb%c`a@|kd{ zHx(b&Iq=kF9i(&)KI|Cwgru{^6TdSnwWX15MClGXW_)Of4Gg$S1}}9=GcX0GMc2|k zUj5pr=FrrYw*1=CWHE%mu0HnYqdNnI=npo$H15LMWS8PYNQ!%TEhjtyuZ(hpZegSl z_si0l?|uF6AN4HM_-a>wY%i&Axiy8;9E5RM(fxy`jB)Z-U^~9I{@5SBdacLSUGy!k z%xEVx0Uw)#z&maVxLneQF;PdgW|wMwW7E!!?a0lJPdE(kM{~%SDx%GerPG-D@jTq~ z_kYKz=V%O$vhJ;dZ?2JWNwY)e-c`gsm$6n3^<&Pxbt^H4dTZO20)r_)O3lc@YlBPk zSeE7(B5aJU3VU!zZAYD5L5~4BaQ}@@K6!=I6RXP&I7?+KJMGeDoAmwwxP!MCPuWtG z@EBvf=1p(f9g&tx9Z^umu)Za7QhWwrB`)V&C!?NEH1;6s|@_ zvD7pyqPavcSoh!fBVQq(z%FmYF?3jWbjonjb~SKFmnLJjEn!@%OHsMH+*1$XBss3( z(E3D#F_D|F^mQFHAB#&ra>}^vy2H}wbFGPB8=ocS+qGe<6xqfPjiu{eiSJO5TVtW6 znHnS*Zo!B7@C2=B&bs>4>KZ9{;8IQvIy|&LqF@C+#*Is7F)lNn_i1v5_I*e_>oLZc ze8v`?JN*e<-t~;(hV*e>ovj#*Z^kA3#rTK-XlR#48Sg3X|K6iZ<0D@5&N!wGU%DSH z;ICVW7F1ketQ6>_fX_4b8*A5FX~Yfiu(B`vyT2vFtX(-bY_611w@qdZzz7`r4ptoe zA&;+q|_) zbFZE82D=u0+0btvN=ft0`TqUxU$<@Cio0)~1k5gc(U)`j_W;>m-oOZ&bl2Ui_e9S$ z=DG%YpW_1O@ZS)8QKySuaqk@Ge0(5m*89chY%0F4zLz}>y1~1_vAH`IU(C(1capUl zV+@?ubo8U^c`5rF=dr%ZKF7VfEODOCSFE+P#s|Dd1K-ehgMLMu66cWq>i>D$dNhZB z8>?ezKhL2Ze%8DxY2P-We?6k zMF7*5gdnY!BuB}|q6I#RUfHjWA>h+W2#CrSo*|X%+z3;jMvakVk{<)* zt_VhVbFogw0f9$JN}mF>?#3c2P)ICFc(f~kH9Il zVr8Z9M`Dl`_&hJ#k-U%bTi^#kqX*8tuYY~#It<+W6GAIE)uvz8YPe{|1hWpA483jntLaImN zxTK_obPIf?0fgi*e$N>cDUop;g0&f2w-kB|ZgAj2FGnBFkgWFIAC7aQ*#IZ-)-dDV zhNi?$lGY*gu3>lm4)~?iGbE_S;(n}?P2DaiW3OG5U>e^>;=a-P)Z-nC zVZZ)GP164Tw!dPHw1wIaE#|AgM%z1=j&G+CXgiKRXx5%N?7QFkI(lws+8S~{YeoMrbM9KwYu!q;q~;21rND_;`Rt}k`$rlBuOxXP z#!Bsjq%;t0?YXqF!+rRlenU=GC0!cH8%b+KHX(^7U3Z-gzUWT@XK2aBUHiU+OCi_p zfJXu!b*H?3`Hn>UuJWwXK9F>WzXV!DKZceiK6d+ZHjsD+J?oTR;L1;+Z?%Ma2+k5; zxEZtm4SZcWBf^Jc3O%K23p$>?=tWuLoF&##1-pgKk0knz!s?eG$WsliTY*c?E>V%Q~piSzQpI5D}4Z> zj*{*C_k8d3M)|r#C%EMYtU7Ou+!?ei?|9^!@^<^4yMOmpQL~O|76Nux8CpI#`?7(1 z-s2t91!K&$(yt(|y)lhP)}<8+S}K>uM^jsBBbUa990KMV+a_c}cQEUbWII@0NQ$88 zWsEQ0BYggjMOQo6@qssKmyB;{deDJjd#mx~%6X!m{>ulogS`Nr;~VQ%;yIpx75&Xb z0ZMyrnq{ORu9FNa7+JI&t5jlA>0tjDS_EvkMV#@`Jj?IA%#%Qoe?IQ{jygO3qYSuf7Bi|>r_!BwU$1Q)(n{NQixq}j_~`9*_* zq(=XSl%5BMR%ln&H@DaVYpT3H_x$sJ^?%9RuWYcheZI@}JH{9;`eHrGoe;n4(iY0| zW;|#f<5Jf2uiMtY^E)ZgcL+T~|k*STB9=|B;{uRflhEdeuRIpMy&4 zR^mAz`ik;qr2so-8i2m&4+3(w8hZ{Bxn%?1@Q=Q7?{!N7EE-T|jZ?}p{uFaw$8eE$ z1$&S6D77nWs53jV?T_;*5taCs$ef&6KCpuQOXnrNpkY=9r+ZO9t`8~sWyqkauMT#{ z<@mbtA;1^)l|%4Eovg{&(uYQehJd|uz!PLN>P9&g9|>0knl-Kn+M8LO?Rep>SuE+_ zqCaECCm%_kgXFd@MPBq@S!NnGpZtc<_AF@<%K7C}RJJ>G4qxtHqNgB=WnG#cu;TvS zc((L+Ei^S{+pcr)L?a7$H9l?BiNM5N_@sZMN$7%KU!x5R7--ssFWN{ISFRyaP2(Dh zD;tU=9P#|{`xuflCGd|ev!<+Ae^a!3V{p;}2Q8ILN5m@MAD4kIZ^?@JtGCc&Xf>p+ zG^9DgSfXX#rVVK{vyOKejrY(sJrmv9d?E$bt;9sat<6>n>{EbIkPQmU~1-VXa!8wOy3?>hUe z(=2UGpD@xe#t`xb2sCOd0AG%)J`BaKmQ2492veFg1|EWAssJCjl+En^xoNxZ^TMqa z3zUqLMs3riMFaDx4}3@3ru@Y#rQt^4D}4;$^JgBl8>3}1j&N_yxhXcF@_&X}IX6g> zrq_ylF6ms|jgNs_(d^)rFpxQ-0&0ebXLYXa(Zg$eN^WU%0}?99UaL7Z09V8)1y&8Sknb z>xCG>c(HWWWzcdAi|5(S?b7)A@X1}_nU8u=qVd?=0=7BYvw80e&gz|lb>u=Xv;z8? z^C>p#_rC7SGrl3|5`)(DI*sc?%2*3ogEXYUj5_G#S+!ouY~C|K9&E~>&D?q4AFSBu zX5dR}{8I59%3$AUXc-Lc8+u+AK1gi`?>aj>Z8?trpS}AJx^FuR!hTg$f*q?^Eg}kb z)c#QGOl>=&FZIfumF15G~ZeUTeXMv`osVD z561RtweN7&esOlH&*!egzrKv6;>qi|IkIO**7+wt_TL}HM??sJ={>$5`q3|Xl65Cx zIOpCAQ~kxd)w)x4Y8gvd)|cIgCF(1ldFB8k_8eIdN7AyCFe>b+>nm1;{n#3-9Q3I8 zu$@!wvTQBBh+(hpZLfZ+!+57+@)tSSNV;A-Q!~kWFf8M{t~RwXTCw!ZGW6&ogT%#)qDIX zu!$0n-<)qFz7q=klArif4}9jZ#x(U)Y=%$isfiS`(+~Gnhkp0uUK^}FM-YxlG#_&s z&V*afLdR)WSx;ie=k~e1!G^Va+mO}J@@yYTI@@zv%Pdc;754z9+zvZ%{#y0 zvk$nZvJ3_5wQsY8$20AFx+UIBqS?fo1Isyu?}k&s5riyMJ>6(6X`Zubge>m+-{Uh6 zp|NkzW9Q~@Wu9M`(1&xd;8aa{edRey}fyrYk}6+6XFByeCzk2?G!#*ZDMZD#b^epu$rJ} z>nBzi&OTA_6h5MU;zFCzWWH70nR6hi_&7E?5*Mq>T;g4Q@Bdo*h7fmS4x8xOKX=c0 zD}SGfC(ozs#O?jKgbz;QP0Vqly}H)0ck9FU?(=-jU&Ng>7(}t9c7FFPCejCRC_Yu53*gulT)p8}gGaXt+<2pa_&1%!5r zm1qX_JALi3K(GS%TwxutjZ0ygr%n#-U05F)9!?Fi?9d`|WIAUj>#6==p|L@BD86di z?&=(1yvOiixu@VxmC@F*bjWzpJKU%bAJ$RM{i*iKh)JWn*`~wF_x->ROv1y~n#ZNx zYV|o-u(=MEa`6wwh6OCehf7*YS-qA-Jz{W(C5Kk?7uYXaQgMNa{~)Y!m}{rHbRutgN> zgh5H%>Rtj-&KV5Zml$l-niD7ZO@o^nVPy{YSB)*#zfMRC!u!A~e6NU~SsQuE!-gvoX(YLD+%0ahv$446xHT9`0{A z6-hKJLzZjvxyJHXVJ1Ffs18rH&Lyw!8(*qk?qhtfd*v$+zX$KcKK|*FbOb9BVefwn z_YuPoX%+VPh=!LQPld+;G_A~@@6uztYj~R| zan}^y;6LINkTHed_tG%2E^qs$Z)#SHYe$Tw`g6}Xjxc8XL`}U`Wrz~a7)~wibuZ+0 zs@@#@S`HU0LB3k2_F0I~$^B>LP_JcR8r|(xh83Se2V7Xh_kZZ~0;kt&?GMX?@MJ8S zJsRHrj(3EAIJ}WTs8*i1ZY)ucIA*5hO4=neT_{gIEnH0e5tq!cE;XD1vvu-CN7bw7ovkn+%mw?4<` zyTG2hT*5cnZz%|NwEw4zzaRK_NfV!H^!%1vbK!(5%L>zw18 z%et0XtNL}CFq%aq#x@iuF@O(C%9`dk(#Jv&t6BAkZKDoO$Hr$IoDW$zSOhZTm42kt zA&Beb&?pD50;Dt_sO_X?{&(PMt7bpd2bFnpfeAVChx8w82ks
_^IcEBWLrFeZ;CaQ6gBC zh!NPso5Gh)X4`(tIH z{k9feud&X7m9Y{@qa}P5_o>%Jip1r7*Z69B0QgSXesRrM>-c01J15)K2woOAzhf$` z+-h^A-8pqwv8LXQ=fK?BUX}uzC~;YOZqObZ3eaYAOyOMMOoJ!f(&74&21`y-X=z9n zu|Kb2+vg@ppM&LX4q5T}-JkPwRnNU<6inLhUHdr)CaoYjz9p76+hd1El37Yn zYn|kHgknY}s69KSd}%u?>p16qR2)v{DC0?dIbP=_e11Dg2T40~KIYEq2kc}ITg5fw z19vxfxxVKx?aFKaqMxle=Ao?5soz0DHP<(OxA|b^T*p`U{0OH(IQQe8OGKRO^T5u| z;K|}TKJ<^Z*&cGD%z1G(23skjby^1WfzzJn&b7Y@zdw7wM4O3m$gY+mTaMG&`{dy4 zw|^h0+ryr^uj7-jx39x_EatDwXC1e-HWmNU!>RX4!&=r~bBg1b=Dp>&Et3}dj}IvmWDHrPdt1W77qBa9o2`MX47_4Xfeg|PxVFeTz>V~ zl&7D2?iOnxJUGWHv1!8KJcYH$E#F4j3j0=o7aH;ewm--I4f`Fo-}KyPO(qrP6eA`8 z-hMBG4qV)(-|bYkpwO_jzR>Kw=F2|gFt#2}#utpttuR>Q3+JK^P-qX^Gy8IMk8a#1 zTlFP>@fY4=yKYo$H`qMer%fg1OnlV>(YJS>tn=7j2v3D$-@}P4KBM@;`KQrgNnFf+ zrr0^Po)h0OdWJaG@QsWw6WdN{Upe}i+hgk``ZI`+qmpbz@w>=D@6YB`<|#N?aH4Nr z9iiX3o#e>)y5_fTz3ajmY>!iXPi9q9Id)BN&%85gsQE2GD+;{F#hawK$LY`(ijK6f ze(_&^$r&!${te+2$@?m^rpT09IQT8n!om%w(5qj_&_81&3*D3>lK>GWa9>!-l#HykB_j= z8h>cnd1yL)YV8bt1K7aL2tMqc9d_$>SG1Gi%Uy67zcAiXbn9{iU*8B!jxUvEj8NY8 zdB@Hy?Xo(GPlg=MLM#xueOzUUv8`xqUZ+CLv(KaW#Cd?l5r~MvNvdY)!NPNhBN|P` z9wa_m=)7J!V~enOXz4dZlRPD2DF5QW^owWL(Mb5b^@3Kp%^XXLaZGLd$bWrm7x5g+ z2t8xJjz-Jy*>4T~=8cuenQ-zf9DfO4hQDY1sWbMLLs|{jJQVme#1?#UXH=fDgNrKJ5vEwvd+Jc&GXpn*S5VM>ChP5pZH<>{F2%Num_(YmYOu z+6^D}u79?4pIU#cZ{1tM9v-O~oDpsFZY@(qhAl4I!pQc9fhixSNBAy;w%9J>ZBF$jngK`X%v8eSb9j z91k5;tlB+Y{vR^kw$Y?V~xUQ^*?P?eF zYgVSLMs#51LSxTaN+cYm?GOL;RxW&f)K9#}H^thsxEMct@CqSkF^F-wZhLuT|Bju% z`mg`S!}>szIcGCj+$u{BFA-^N*e$0q{d6qt#i8F=D|1Q=#BLoMOaGi@gP5M{zqbVT z_0PR0GAAzQaAG_c=tBFj4m3P)9i!y}nBhmPPC0jL(_9QADHY*LdXB_Wm8K_Ma>isq zc8rSsCn4Gzz33>gi4rfmDcd@H84BCAO9Dm{I^+!;kionlWsY zkX06r-iabYhnsdRw)i=d=2#=_FK3w4O7%PBhSNvG7mhP$;>*~9#O2&?z4h%M{iC6A zjBw}^@v&;e&ezQszT%p~Ck$NqspBJ#)TE-UDMAtxZ$G}=nrTgjsw4x4pPKlTBazQ! zO`=NF%=#$?>Pz$~9}J#p;k9KUV0p=H6LSrEmevTYNLWN#netP_rNq~>zCAv_iHZyB zy%dWwH{8V6XQHY2#QcP~v>zA8t!)uVDt~$J#nbw zBU;7W+B4*YKU4@|SXlV<- zhZ7dZ#m=7aF!9;e@`UEAn>=6m#2T&c>7XT4^9wEfz8=*6+At4 z66|dCbU?c65asLu6^q2Pd4JbuF$UlIZNKkc@cF&T{4|oS=n~@=hevc0A9M+L$6$yv zV{M;G3Vp2YDbHEqsY!Y({3)EP8e6;V0do>i&fh+A(yv4)lNDzQp`3Z} zk4=1uGr@9tkSI|(c6?3Fax{UX!k)VB*+iOAn&-qhl?*idSI@L}Q}Fw;aM-F1+qDcz zS&yZrdwlB_i^Pc|+c<~z`n~^oWUt9`$vpOFB!iIupXYhQ+ z3d5N!@i?QO@~qQ4pVYc}&9NfuJ6}EZ#DONnzWS{0J;1_~>!sF7xN~Q}F;*RIjP0Y} z^F4=8y!BdP*%yIzLYTH#kowH^XoxZERkUquEwTaaJLerD(QXfS>fYKN(W?3_5Q!27 zQM1xQn6=&A^ZC@7SWoP(?dGlzV{7yPYbtcR#TY!mc1t{)_|Q|CV-%M@Pyr+i=YD*( zpDZNeT*NKeO}suEy`~m`2WxE)jgd@Wqp+vynPZkP$HRNDxJ>n7E88gWd{AH$C7ute zZ#bW43SgBbH>JIoN#@+{fI(Ei!b>XK!^=!Vl6unSpM#TDIoQGm3%t24l!QTh9(BY3 zocPBwkZ_jp8B;j%G@Kl3<`RM~b;F(uKKtnh?u;FrYZE){?tQt%AI#P3%k`DBOZsfr z@hv3;wq9!89`4jVKlWhJ8(pyHFKs5e!omep96TqAYA6YwH_ zXupR1HZmpz9GRW6o$O#0_S7{W`y5}MImOkwoXOBO=}r{DVq?pRjoNVzY*xbAb)4|@ zDe>gF$Cq_wLlSF3v_eCom21C=$dr^77tUxS#I(*i=c_h(=E$+1f~T;4CI#sjS(wPfEMbw|}VYhZ76Uz`-!M2Q#Y zyluTcivqN%GHk3()VPGy=NlU_l~qXCGVK`F_2GrIRhgv*+#L&k-Mhxdy%hW6**RiM zLgd86IR-q(l>a^-YR}Hyo{5&T*W^!a?G{`Aq1#YYq@Si!Xtw zXXoIPv!CM`I5hKEeA$&kYua#D*qQC9mW_@n>!oGDT1ZQo*a^PCSWm1K+OpITJ z{kE-d_(F6VZFpim)-ky53!gM5_leIt!gri{ic{~2WO}TcZKA|uHQn1_cNCDcGLmw3 zP9AqfAI?WKfBcWJHObbUgOB6NQqnp`_SoL4v5PC57P`AvcEdw=Z5X6Fw0 z5Ifu0Y@v4q-4m=U`W0j8Du?*fo*moDF!}?=U-^|s@kMgYR(EPo2-(c^MdqKS*ptk{ z)yFY)-H+SEM~v>D^9#5C1Dxj9Joh2iZ9bw;)>t3<-otaB_~x81%_S>Gtc8Xb3oaHV zk8nV0eUSle3VvUfnY8BcsZU%IO3q)Lh>^IRrA3$vd+%|%#|mn*grF>vpGI86VN_zQU2=d)cdBy^ zp1RfClG9vmo0+1f(j~x$og*ql($sxJ8cmK^o;lVe^Ud*v7*adZu4$Xo(kSKti%hf) z2gm&!NP{rSIzMb;cqU1m!Uw&5NQ}W8MVbm0v^}m}CbFCHHeHu*+@_eS{Tx0GXo97v zs0Wc{ZDz5sI5%A0hxv}W(V+|9^-Ie_EjN}OwSxfguafjK{P;Eped z=lwhEj!|yl&!E62N}NHwt?7%A0$9nv{gq$Y*t<_@n%m!T=da;6efgI^ka!k*H>JTn zm4>VZSpHs%PBg+M_$1SkoeDqOp;si+Y#LAgxLELfR?pbVxn!WV-qJOejA5DfUUnSX zYM-zvBeTy;h)YWVk317n3VwdF`2MmS5u1YbDeW(Tj|0`!)hFEu!NoGsY(M?nb2sRE zEl`gPRrL+t`^~>QoW@vm%&|3uEuM0O z>f6rUNAlW+FWB#{lHUZ299E0f%DNPeJQpLTuemlzk`6>5- zudSCUXQQdIQ*hKa@UceLvb(%~KJdxhcb8+Tu#Im?lOuxVvtHC-Djt6_~A?+1$U`^k%u8>w?tUmi# zKP+}WgFiLHG_hhuSVd?ZuSzDLtUq_g_Q|e7yNeZ-D?dVn+qLc0=GrNb(DL@lmaS*0 zUOpp!rKkc&YDb0RVms$T*t5O!VOzUeIB{)Uu?Iz?NAJha2}eB0CGiD|+i-Eb>F3*; z!%EX;PjvN0(dhURpAgy`0QNTthgD|kxiX>*AMem~&sEx&qhW}B|NNg%!Xqwtel@f9 zu&A75mg>|lq(!K=C}kbtu0#uBZ~t3VCt`;7zrm4skPIoG!w04iUi}vuU9%d|h@&Gb zB5|f#Nwk~#8^NbtS}ugnn^T8A7UV7UZ>r+3G(Q6xc+G$6}hdL0(e;`{FYb8CIHx z^!mx&Ts>?Sg4~|$KEfG|BDMLK)H2$_jgy0eHKbB2!^(Fd#P!K(*Vs6_Sj$jDJVC#T z6RV6(pzS^l!*wemge(!nuq?$A&M5(%TAb))Y_TUn+mdc=`?dAGH;4z;=6&oDfaMk=W9_f)>nZj5-cuj=ZJ*M9ZJT)a zw%=I)TUZNcw0=5=jl=g^%+eQE${)aH;Rdo>5hKcW~Z3R`(Lmy&t8a zxC@KbCHnV5gt3VdFT`2d`gDgB;NvV`|GBKpXcw`)wBIZf_ZcR)3za-Q+`;$rM# zel9*V%IsTHVhK$?*4Wr(t#Uqfd@M{XW!6Aw*!zL+@3G;WBRFpngVT~WQ+>8LoWB~a zM%WAE6eit%_a})Qku12;Mq<}XSTSb@)gG`enTI*+%2L}?o;~~5er>F?vjcLNe)zR*ZkKni29B5~q&Bu}Yx@PrP-U)qxQQ>Ec4|jf zUW;D=zd!oi^NDXX8_!fXh%6jodR*?SAAfv08~dYtDf`hBj#@?ptGGDB$;PED>1pj? z|A`UAwsZXAtqZVe?pbeGA6UhdnMb1LVN0}TnOW-~Q^)2S+ZN+{`0%h6v5ZmHaMrW% z=f3^g`%F^&iGmLdmO2(Y_^?^5TuWA&QM_~bb~s=E0BtYn;If2ow9jkmNkxHXm$cL_ z;m`GX9}Z96WBJaw4}20Vqb2pk+Hm+Kn$0~UmV0m1J!IzUUaH)FO7NC8JuTa1%Nq8D9Y$?98ORSP@x`YRdo47MX9bm~nIV$ki?;hRBEzKs8DUxzt0E5qy z&lTVHxlfhknddqy$Ver*C~dV*KchB>D!BV$V55j)<}e zv5NNBF`AL-c8}JTz%`YrY8{mE+A+R+dMApFlQJT-zN$u+HM;@&y)zXmxgSY$(6 z)98%4;tj3yGeE3v$}w_vqZ1o=FCWVucp~UWw%I01JhD@}4eT}rglHq`l& zofFklXKO^~FnHzJo-VBqSb40f1c-BOPAz9)iLmOXQO<;iK9ymBZd&ckf^;hU+D@o$ z>@?iQVm#mRk&hhMMUF?xdQE<9zvkMwA|!P0aKjm_J|wAC}m=m zvJa=6Ie>u^NoV%}^dT)1Vj4VmVC&9}Q%ZbwEka8?qUlarvs_)m=`QgRf2Qce5`ksC zR2)U(BUVr0yL4INx{={Az8Y#N#*TkMW4H0g=JS?zk;c=F)a+=aPvN39WMaq0{2X zt}~VG6ad$l(f@xXG^%+eC?GWtAyU>z@kYYxUse&!SLx&Df0o;mbmu5K+SI@o_e zU-_c*gRjr$u5H8T^TJtXu1;Sre4CXBi<$oVuElu}&HDQNJwyAhZU1EYKD1kng~3*c zN5INsX&i7aWjF%yu(!uF_*uf!P+2hkS;*m^7F*mJ_7!hoKc3a}#q?Y|+*{sq>$gI>Va8e_7^nJ*-YtQ0Yz|#L5 zpEFTPB464*>oi#HwZOajM2m{6;!FI_G9jJokdY2t;1OqHUz2uiEQt#Xs!x^8@47s6 z%kfPHDzI5N{El;5vWiav<{^MbDq-(G{Ez=&tOME1vN~(eyWacW2j=RMj+5c|wWr-N zaWngJ&N;*GImDg24o3PRq}-aQaXTaJPOXz7c_jHg706&aqGyVd*2ZA7ldDBcRih<( zva+5GXrQ5QKRG7xH8&_?; z!*_{sR+@n~1+(_&+%Z;IbNF^6y&mK3@mE^RU)Qni`h5+vZ&jb4I>rj4;x^8H3|IT# zM$(|~&$0UoKl@u@W?90i&vrQT$=JEJ-(Bz8)iHzXS*?4?@$E4BU+3}mbx-T-cE_%E zu&%m&GQjJ{nSD4F*AB-H-y95e;C0N^IX0qvp8emQv-B>{yRqFjzN>ET?qYPnc#kU)+@oj%=D6okVYpAu=eUVb2y>9P(TRNJk^qJ;}LBnm&f=iN&Mu_H9 zcqE4@+SkL;vedlp>XR;}Md2g*_ul^74)AHVY4#GIgq8i+SO_?lXeSJ{e?E0@ZfGbu z4$;atN0!*Zn{hj{{mf|+X{oY5yEu+13^&UTnRpuQXvpoVtp>kXu%?`65`yj65}$LH zFP&?aA2by1EdGKs1e~l_bL}ODO?#u^dY>>2Gn>)5fRk&@7I zEYN;LTO7Nfx4P=Hv}8XxQNfw_OXrL+cN?&pN)k$46$X;$R|dpJSNNZs&T)xsRUK@kl&C;*;REhNHLN{YzU$s(7d5 zdJwy#zQY!cS?+9IalGp3*WV|;xw+z~C6TIiTjB3^9^NJMt>r!Z{{9%gtS^Z!>uPR? z2M2Mf;*y;#>(8BYy^K1UB^>Wj@1c=3nQffS+pdqdch#=-E_LmFDNEU1S=Rf0dRNyt zvi|vY|M*g16D1zsxqpmCP8;R8x(6*0vQ|yeeQCmWW4mAN+iq+-oVzg_{oValx?(NP z!Fk>Hk^ZI@ZavPcSaa9y^;?-^uK#tM`Z-r`y5~1>rN+9Rr*L1>W1Z!$_O*_!;;Yv= zo{Dexe0ns~rFD$gHT1>U92MvE^>w@BSJU|Vc8R|q+pa%-?dkhn-5tI)?GoDBHs|6? zyrbvm6W?ebBF?$3cV*7m;fOJg3$UkKe=n{K4nAA8d~fmQ9p+bmnp! z$t?=dZoT{!uW0*7yrRVt+}rVkh`u#x_mev(a09 zy=t#IyBLGzn8PO?#B*6~M&Z|bGU9|~V2>y9(Z)`N?w*5BXk~7hmU!}<8KY|7Tu1hh z731RvE174`L&6uEkk?BIF>?W<~vrk;0}h@EzO6kpP#j*nGIIj!(gsAlykKL=l>Q@_tdj*nk=eJAWJ zdiiwv9G0a0Z@Z4~Gh4#kh%sr;SkHvB*WGD31 z4n$c^+FaYrU3*_jv^@9x(&gSKBk!S0_Nz81=sLiOXeA zYtwHnrVsMW_xbiu9$0G#UTTOcJ~*-;yE^A!*|n`Or^d(0Bl~hLHn?(Z&S+Q1m+*xr z!+8!(tqZh*!vQw9JJ^Y7cg``MgKsyEwmzcoJGjO;HBX1SZ++{@tio>HKXn5>T$A(+ zJWrkVTgg7}Xtf)kvO3aOfOZDZoA<8i%P|A}eekNtYNS-S2n`&{vX(Bn(G_vP!b zUH`^=eEN~^lw6m73sdW84bI(fV0sPg#*04--kWRtyrzDvz4l$W4_dD2 z*9*DsCQ7`JXJ_l%85FR`f_(B`cbC|GcBw031Z;ssyXByrx^$cb{NgeAMW^QOs#zyw zuYq&QulFZD@&(Q3_jjDkih6~|c$&XAs|&4J_~SQxk!-Oe$~(W|vv=8oy7H0R&;T~g zBj56xP*HX&ec!04x(({vMrI^A;^QtdX7N@>%dCz(i zNg`uhH*sYh+C>`WejG9qt;GkXPa!R5!tc-GZNPoiYkv8GrZlPcTsZ|F%ZKE%O+!Dq z)@XmhHO8x#y)5cT^r=itQMEAieVw|`7?RrmT2CU;T)XBGe%Y7cJH;ZHdvDatO%GU^y}d^&UZX8zC7Oj&Ua4Izi-#yKijt7)93qQof2PI-TKbl z!To*XV>v?4OYAQ)LXKRYe!nC2&#m*Qsx# z=J%#Izxkw3>_pLVyz=Hn8S34>s}w_^_5jyg+x>8{^! zVs$vG5e?xy7hkaG1&{gv+UGv&0mm4k6nyZ)`SFi_^uZhc%C9`oI%zOEf7V*$xG(tX zuGcSr)vJP)OSsNLV(Y!@QGCxl^QqB)V{q8Np?9*;Cu=kl`Kreu$2y`hUh_T(5vh!g z6HKm){>(ARx^b9u;ol?9cNNYWe6`)F*FC;d#Je@Pdwf+@u?EgaT@FlA0Mxa1m-u3P z(y~^B=dW37BV(#{dVKHw=HETLHfy_6ucaFK?(h5q2YQa|L31juY$w5UzMrO4bXjjZ zUsGTcC9Y|)T|F`s7-bPIF<=@RY>n-@Qsa6FxKl9rdh=8EnZlxA+2hqdRrvyCvD0(I zllb6V+BGG0Vxc@88NkN-cP^tGZA&31n7`Clh%%0e(lQ|l@|WsUwz z^``n8*`M0j&`?|mIIbN_76e-w%2uP^sP8t7e`MR*!`B}oZ{j=V*UnEq>rRcOFRNoo zw$L6Rk!7s)K@82*+8|b86Nszui9zXMPu+8>$o^#pF7=4oaCwG}w>GR1JEPkgkXV)N z;F4i!3VvTk^vSi-ixB1#>>6*Ab>TW?eR`K|=6i*i-QC8y_pU`6td!_2=Sc`)|RciZ3j-ZN1#r_`Li5=I;7; zaPf8UJOkWBiRXC&HuQHv0k4Cvd;O<9$m1?>fGOnijc(e5XIopl; z!B>1g{DWVJtqy$H{oKYX#2TIG{(knGKlk8t1aJ3v^0wlu*0^B58w_wbjZHRgg%b1T z80YHLayu_){hGeu6F>KJhk32_>oxJQ{&woKscY~vAu$nz#xm>W8k|d-`!Ak*?sGCZ zsQzN{v+dkjN`@TU+rRcdxKW?Q%rhFy+Mc-Itbk#u%(kpDBU|OY_DjCxfKSrm3ODfy zEx&4xeJmXg;nj&J&!cSmo1XiuLt6=M>sS(B!kjxx#5c%Uz6Fu5;*7DECb3I%nd?)=7-J|f;G*q*&FeqyNg1=McAe8x z_Ql!SF5MZrjLbD1z6#TR9tkLRi{95I0*pz~WxrlHMsiGz9fmucVr=|= zIFP;O%RXcE*bFlE*(h+Ebt&M8X|q;pHZr^0+aH8==s=*A7vcC*@V)#Mzvu=m-=nv_ z?|pIfYNwT`qWa5T{puU@*W)_{=2Dw}Z7f-IpJJ2?Q539O)GhB zGQzKT=9vcxlSa2%Gia_OMx$-Xm2=wI?z7eq2d%`X|JDbG8RztA4`2BG8mgMwT%cH; z;qcu(oW$wekEP;ciRr^%{dPF*unc7%=FSl_({R?cf$i->(`!BKKxOAMMrA26ZsM== zmt*X4soU2hb|wtt%eGlIcSd(x?q&VC#Kc%bEC;qe>v|VCh~8N?ho_bcyUzN-BG!l9 zHJn*D=Q;2H#@GDT!L$-}E_raE3FDzS+xS#b1WT8z1=SmHjtPKKFCr zQu7DKT%B5OY#D~knWR3G2@6f@uEzDbYqSij%-}u`u?8DV^ES29bk1XCt9zW*tM=xg zf7ai_m+Nr~U(RD_g!{((`Z zF#hbbbB-_KgzAHtYtE8>BP_1k3Q5iP($eoH!LbCO`zXFT&Ro~d$LN#zSj6;q6koRU zdqhX=meN^d{XE0BW_;Q1)N_t+jYH|FxU|df-O_j99Qb`(??-cO*Y*C{UEEvKr)ELB zi~hWrU~Zzsi)psDzD`hpzqBot?juY}Mjt-K{Kp`0? z!vwa-_ES&uj3} zM#~Om96de3Z5r!%CQf87W+0ubM(7DCvZf$^-Do zU*a>yDa%5Rk+{g&$=az$WO8e5gRVz#c`TGX*Twh!z#qFcPu^ue^rK%CW5{|_=lCrS z;PXDo_EY$Ppc+cLX5D%g${%nB@c9kKyQR@&4j(Y$J~-Ohr=uQX{d_+MP#vnk$=S3Q zn-3MXi^q-vn<(+vP5GlTV9raIj14$O7qJKol;Pw8MEJKZC@;z}K{$OBuMZ zj#zN98>|~gP#qp;Sb6Rqo%m?-50z#vI7jJQwwg+8R(-H=9jsb&Ae4Cu`}M_(Fu)11BDUBTU-(=Wt%x*QM37 zug+YZzT9}IL9np-Od16u0$9x=a465`&MXDqLFydaZ=<%GyKdtOS_v<2b9w`3?%x9! zN5o4w9Epd>uygB|H7SpM!9mnq&5zAewn+8B0C9XOFF?gtPV>`xzAx@V@pd zS=-O8{^cmJi4vEik4%wTN5_AES9V6g3<<7H5tV!A|YVm+rY$jG+)?J{ z_JOpKEACJd^Pxp&~h zmProH!ZO%Y9M%S{`WjqtR(!;jHL8y6^O|$){=8ogqsfie!0+pt8{!a}-UmMTWzU0e zX}^=cpXc2B*0Hi>Y}q-mwzhXeflZXS8@g^#Poltk-~D9MZao#r4o?EGZ$!&<)jsPE zFng_&YMw<7@P==PSLyn{s{$wC?aFS#x?S()^QAlePue7o+vl4QPQf>ZXPRC+8aFnm z(Q|j)>`#ApyMDI2ecp}bCWKvCt$Xu%e>X?JT|d;mhIJKl+C#WmXX@$AvTJx?U)_Rv z{rvW`o9lCgp2zz?j&G_zH^=5~8D4#UyKQLqn<%=2G4}hr#p8uyb5(w) zyX)&`(^t#*j>EfdKZ39Je`zmdtLMTYOKxm81zq@+wIgD{kw)Y)#5e{ zYW>~y)ve#r6r?P>Za=jq)LO#1dzP?9&d2eAay3CmWuC41yNWi`@8dZ*=rBi%xi(w* zMuB4#*hGnA6xyEO69qT|(XdF|$d~l4(aNDtOG`V+aK5C~zhqzHHQqEj@BYT$x%$&_ zNfE8f{W5o0`O_SdWJ5r8Mj;14|Z!gx#NM$L-Mphh&MaK&C?#e! zrgYfIM67(|E%$L;(hLjxBXtt)Ye|RFYLC_h`Td z^gGy|0CKU3(7#MBh|M_Xv!953w&vI{pOs_{FM*GLI<^+OIG5P4h7q4LKF(C)Qj#ro zJ>_^t^vpV^;zOs(0)vO;?6)IB^NKI_MaYR5IKzWJ>DY#?;e_IV4@=dLvErKwPhCQ5 zJ=+d1#?x4C4uknwhEI8f%N8Iha(Z?OKJ6Uu`j-FX278#!YQnr@e}_HJ8s;v{v~=a@ zrSTbuqj%=9R@0~6b942!cmBsw7fbASxYki3-VginP3_ukZc~IvIyiHv=|h>14z5)Q zv44**#g@LE7#K8l5G9E2S0`w;{ob&M+3z`g@QWF65G77cbCF^Md}5)_6-#=$)cX`8 z;NtKWaYVJk1+T?8h^2yyYQi1}7X*I`RAyR(FH{D%YFxIUrhs_eD_?o1W7BiL8wsh2 z|4YuommTE>?NkbEqQt3`+uB`_0>W?K{AHhUlBV~RZqQW9!X%t=Neg`n?722RH9lt} zNjB$TDBt-?e^ppimg;$ib@h3xS%HqV@thW1wj)O*m2ID^{2bG02b((qNVuYUkq3U_ zd0)h@7>??PO*1M>lq0p;LLd9K?uBr|Hxk19u5bC4gZL;6krj$anJhMV#51A`i-`OF zFYyVR?4RAfZ_W;Ty4io^_VZnd>{oM%OA}wfnFFvdm(|D4)98nN$0ANOS;80NPwT=| zThgo5(tmrV>t@Zw(D<%uQz*wtW4chCdLELLm$ z^Wn3e=hOsZ1#-)_3YQk&xV>YDQ+;3N?%`WnpV~E1D$*+Woyrj9AcA9=*J}Dl@WE+| zI9rRRWz9O7664gJ`+L|#9^#y&wR3g)a&azV%WO@#R?{!GL)={t^RDi_7w;3FOj`Ge zPi$X^=yyfH4erG#u!#~EBjpC@bP8NGZ5kH)lwEdD!RqrwV<}uw_K;IxtZ8H0HTdFl z_^MakIX=!7!Jf9y@93Tj$Hqg$SC77ah7hy%*mkaeK6?MUtE&gJmFSfRt#Z#AW3zV& zR-ZaKqsyDW^hVb;tk z_%bOd*jT|z|7STI?O8L9P@Y9NYb!eiACVx#@~_%&`C;b(6QXPfJMl>(x|U_5sm!%8 z_ufcaODM{bO_nfi>Q3gBCQH{TE}#tcEKw!rHG*b%0jFk>PT1&U*PP zUU6govacKg6Q_Qi;+$qXw#4NeyIv=Pi8JxYxLDS(+Y?n5i&!-x24O$GWBB~eB`xj~ z7cU~|9ui5?A+B1+x@s4_&5-+x0$l)>_(D8isCTJHiSO9_T?!xGz`NqU@Lj6kj|-el zlz3cb`UN{qEE6p%e`LPEcimAM^<+ahmh^1TDfnptGqGPE=YdViZMxuH>l~B)As*1K z(R}g4%`r?RL-o4N=QN3@`1N~rCyaWgRjk@j{iBhlNzHm|EIs_mvCKq}Y~P=!SQfDn zS+C(6)&z2<$#%{x5VqW^Wqp<&W5j}F9Tv3D9LUAaFweI<+c~p>uq@RwNg&~`k=)bxa-5vDhyjTw&!e#}6`$CaJ_#u? zLzc0GJAMASVQo55Bw<9Bu?ae0O=Fym_0PJ*Mlk`j)FZ!x1Z#aH4D5ZDPn~7iR+Yi%S|}Ty^3)*dZX{!IZMcE@MW8G&UG!E z1821DVXZ7UG>*a=tG8b(_l)n9chcB+Zq_=`v%Sx5KH2VGXcX8)i5J?8Ja+w|?fr^p zp1IRk1&uz($xU}-+SoMujO962^Yv~(Gw#+Nt`&I5n<>wf!?Hv1Tk`aHInYIwkh zi#F3a?_c)nS6`I26<@oKP{!Pk0p2zEcFqg`Mr`*7uA?{5s{2n}viXcRhlB&W zaaWz^C=$NT?Gs}wF8=a&g)8B5;>pi3XRZ-vj%<7EJom}UkWZh|k)*1}@c7KW=JlWU z3T#9-{#g0St0Du*JDn%*f`dB2TQviOKs(Ou)9%n4C;UacVSxtJlbN6B8+mAskL|>01l{G4sM(BJ^MbOtBxv%(Q zACQqoXVm$Qk9_2?F76tiSfPx~@m*uVJ)`SKn_APk?uhh98=P6e_u02oQrI;j@|kG2 z<|j~K6D3X{+ScS!6!6)I{R&z}8(D2?80DpK)XYD!@^JDdK0E*<3dt6D| z@_6_X>lK>S#u_CYpROJsb~l|`vfeO~QnAg`jqEi!zMt&2NVHB<<9I5*H~!7P`5-M* z2N;cJG`&3w(9?6RkH2+jZB#sYJ(UQx$M^mZeQ0=nVwaM=m^#~21^wO>0MtW+L)m_9TF0Xfc>0^+#tDarM|&@suri|h zR6R*cjk}6aivw#9<2PJ>Lmb;)^JSm$!0#dPTPuCNXP)`gsCNv0|J)oTT)4uqu2rh| zJg@cOTAOVv@un5f``5UX+r#2VzVDA5`jPm|D_Up0gt6vK1i|{Y`8llRew`=2{{6BB zPp$i~%UnivMJ!A@JF}$qzxN8`+-nY^)INEf3vX+`QDB4un7egd#9g1Y6ji%^oEiEMl4kx31-4T3XX8#hO z?GlMuS?=0zp{I>|@0qM8PHL)7Et91y>nFa*#@77c!^*gyo4Mb0Y>T(AGh&Slk!)W- z@nwG`_RYSa1DySi<#ql>tOk{KJwD@%OlPe>Np}uZ#5fsstS##~`?NIAIR~hK$oKmrxBJAY$InI zyIl8q=V`bjj$tB`)F_^;hdn#6M|(DjrJ^RhOuzdz{`?8QKhsdsBx9F|vDQDa#x`bH zQCkUS+|VUNdKzHtrUb0DFSN^cpf+~a9W2_pKG6KA)AG_=k>I9*M#EDb;A;O}({dA) zvGE+L`sbIvPkc1%4?Au5rE!6m_*et(0)7uO#Fw@nMz^!<+poP(e7zIbS~#QipD%p< z9x|i-U0aWJlM0E&c5T~J>Pr|Ns;*NSZ*A_J0-Gpt=Oo?`Oj7_GhfQ}t{AvWH0pIk@2Tvoqv@U-52fy&ix%~SD!1wSO3yQSVWw#q9ve(GiM>y+@%geNl- z7BS)3VY&~y_9<)Z*P+g9kFbylyFWAhV_V?Lkl{J!5Mi#_r)R$OR}HaH6(3gSO<(@& z0v8d%rysn#z$aAq>SaIwKp&O?+SpNitXj42OV>iG8#{K`Dh(^hK`ZfLsrvN-UZv+6 z>sfQGI?RLdWGn(0IX@YyT>I^}pU>CetFam8Dn-}24d2e%S+TW!LEidXg8oEG@}ZsIbze2dPPE``na}-hwDF#q%A%F^`m=2!dENFR8jO_b3cW*cKaDHoW zj3HuYCMti*zxZ>9&}Dek<-GOAx4mt@H_Jq$i6T3kXFrdHBnxNv<>~b4oU>q$RP= zZ*ZbZ7cB~h-9{gx>_q|30P`C*|L8MQWh;IGQD74#UONcf6Vb)K2Qu=|L{G6};h1v*JvnQ! zE{KoFbyuCLl<7yW{nk#swc`s%!PZXO#0M6qHzCzoCT>T%0GjMKz4^@##Ko+|hn1~i z+mYp`!OTe%x5EMxIQQABTxUikEaK7}C!9RT+S-q8iq%$+HXBPG)|`$G;;ItX$hqP> z7C!>7_Y13?aS>VcV+2puAyQc<=HVE=Y~#F!hP9n*8Ozq<>tU?9x7Muz;`^F*r`8P- zZ@&&sg|Vicb#MsnUeoZ5y0KHaG2YW3r@J-j8SbfD8=69aO_V6a*e>sr0>aLO@cHyd ze)Bsg$cd znHy_O%#r%NNC4l6hMeNl1SlYLxB1t+Pg{*Jz9tiL&}FDnQY zfyBZUM_#kCaVldesmXrx`P3QRgAJW}*vhOuF+?={wF)a!SPz<&H52dF=Tq0jWtpP9 zgQuP=WAX5tWI5wB$DzzNj>H`zr1#5Q+rB)L4$oO5vOe)vx_}f>S#~%EzV7O37#CPn z1J?Y*K?f;gC6XCq_GQhPL+Qx2duw2BbvFu}N`Xz3IF)i+yE7=@)9N$kQ+o!ORZThJ zU;(cN*@#uZXDaij7F*Ce1kF@mU5r)bp$(JPyI3Ue8P!nSS9PjHiEI8 z+p&fxJSSpcL*Z*{<_24M0=T11#0OX>tS(0@U!X#>Ft#~*>dDw9@dEIUfsicM_+In+ z*Wbz7=myvr}kwtPOPuku5T; z9jknS=PZTRCa2m9fIA9;Q?hsSF5B19>RW|-qQE9f+!LWUG)VzkEW6Mi(-vj@Q_t8Z zhEd_qlKeFm>Y|^0nsn@-g9a^}mzvQR+uK4hFN%u>)aQ?5K!5ap+C9Cnr+k{9edA{i zeLBT_ux4O8Y491d(a*T-7<^1DBF69)2IJk|_>D*DAeO|KYNj0`LyWf{%))*6=%de( zzr+_i@9yi~CBEF@H}Oe?iH+f$bEK;`9v}7zxlwxiMz}9}vV`SR<6GkC=GSUUXn(Y9tx!$PteN(&&~ zLae}b%cZNQX7OnyDXNfeYmfDqLm&$QF+bL0D?er(t=O@-bK}c;5$n~gaquM$XTKY< z6r)xi@ZcMzg)|IOV>pb{y2?I{p3SwjF@8p+xmt+AGG=B`28kK>`VAkH(WkJ!d~-Po;e(OCPu^dL0Nvv`L$2~ zvKybge*Xub{M_pSPFZsJ44;aNC;^T`IB2gu=l{+zA?rBHg}(+L78|=;ae*Ua&J-N; zTX8NqPW0_r#a{Kl{p&aSn|+_d2SzwO?CB8od%zR-vH``e&b3{+Ucwh)rlsx#_*+F{ zFLEr-xodEiEs4~TEozT%bWf6sk2uIu79q8JICB}3go@TRI+E=M`+B2IW4{of;+vyq z!~*R5WZjHd?jl3cz76{2c9!9=v@f?4Zy#3f_ehjZnjZ_p%}LB8&e<;Ud2b2_&T^tt zw$aCAy*(!)jXKTPN-tqLCqA$vb;5p(GGqC@N%l;v$Tr5f#_6cXSNF<^bQODi3BNzD zeTUhCw8Dt(_x6OdP9w#_1Wx1fE?S?k_^AUv&V;@(IZw{Md-eCCZ!8vTXjtKSXXa7G zyjlm&tm8cDY*%4y*Bb?nQD74#j!|fPejy5Yb@L_Vh;WyV0bZ3tWI5g>TMF+8>#%mH z_NbsKgeMQ52iHA&u@<)ajJRS3@~GQFMM&$O#lHkTpEGPojw8>)#<4=Q&e*0)jn5n< z4x*0u2&{UIHHlb)j-J>_BH*ZxcG5lZVMXmgp5^t7?eE7Yjz>RP64r!xC%#?s&iCD5 zDJsq$AC^E?GdPKDqfQtSYhv@XTihjhPtVG3qG!&9_<$SzTe!*J*y7#*dwhKO6V$2r zavzDg5a=EJ9XrPgm-w7>?~{*y&-bhzA2!}tryPtv@pC^Hw5#KTPan~A4LzwFq@ch8 zgBR-&u=3~KNqb)LVR?xY@J*>PO7mNneox{BZapXk#$NHAI)4S^;^p&=PrMLXQv0;; z)*@{d2Qu@}_^c1#L|L*}e&AyvnrU)YF_8&;zX4VZ*Q{2(O6V*kr7;Uj#Jx2%Vmozs*@7Pdovib+Aw+9(k8o^Gp4oTc;KF z==l6r(S51^u@+mOmgZ-4eEJBNdF=PasQu4%ED7w?_djF1RftZQ+f9Uh#IL3;M^y%>F>~L26d95Gk;#`9d zUYScq#xYkX+Ly7>;ztNyDa-L)0$;?I6xh4H3I6y?g;%Ffd&M`GF=uW(eR)sMaH`l> z;wv3&O6gB#lm@Uf<>SwlTOq1M%baS0#owp0jw?cuEP7IWi`R1Ky7 z=Bl<^!j}nebAIQxKdiVph4@`%n{)65j0$|_n)rGOzSyp@qs}b!pUym&wkGvxFc&;KaTF+ z7J}6QY#P(9^lRSpxetWqJEpARzKwN$>oxZN*=A7#V+qHaXRK*qttw-Jhu9VlS8IR` z)@AA9MCv!kv29JV;XxEzwHFWtHc{dQG#QUhSA^yYG4z>epW?Gv%@L}^>w5p1gtKaP zN7}Eg^ROlSV@%CYKuwm&LFN4){`}d`05*$XFgA)Ne+0MPgDcCeE&9@s(-`i;6Q@J( zB&gL-b)wG$2*Y8ee0Fl2^*onJi8e9o>E{5tC&D6e1m9eI$tn0ENBJHfv4xMi>{pd# zoZF?ivTK}Mj*Tz0zk8YpEzYE=oC{)XVWLw#a)?H?yCu$DBL zvGeerHZ^p+!l~Ea`(r-oamGdCOnp77(+`0RV zQ<|DTZHN-)CdX3G;vI-gbFin%jSGx9eA=+KBu-)@QHBWLhuTlU@5_=dOZc=A&nL!a zjRm5|wY%Rr5RRl9m~OwWlo&!3lFw`$#{(K9N3w30z}Y@KpA)+x7koY&iQ8FRhA>4wsaU@?pS`}1M3jUr zWKdGb%&3&*&b1G3dhWAsu{=H8x%-MQV-c_jl0rJLmdUOp{&VmJ>(c0_|NV}~Xnn}q z(v1Gtd`Vgv%y(<*sSu;qXzzsI>o==WAATtACCB8`UgIe;T-0+c?2^MorETJiaOV1; zU0WOAiM8ojTAxl%4sfY6O5-_vEG}3xu$B^hW?aa*`26nRI)+a?5p6S8(Y$sfx^0FC7~0~thKOdz;P`D?ZzEy)w8KIFmeje&r(9;BWAB&rO&`JMy=k8M^)*UQeO!Z& z+GA>`Wa^HL*I8OGbzPkrI-GRPe%Ie!KFjkOI%J!hC3sc`H@4d9vjI66UJXwy` z#2V{{=!JIq%(=dMM}L2{{nRzJLT3K6uFyD-*ng(=g;P}x@T(l;tnp{xNo zP}f{pf8HhN>)`C@dv%ICCt=f;O_V8U5LK9$o78K-d@b2#qI z0U9iit>qfLfIHD9918aMGLB-R+Hlbu{>rb6zVCd5Z#KJXG|Y8WbC%OwM;L6|vTBMA zx(VgxU070D|EIQU<3y2bsE?UoWuir-4PL`CMcts+P3uK$pVih2d>?q<@4dmMW{kDPP+|uX)Q2HfIiP_;gQ;!a_T0s#ImD zzfkq)qpk3PbR_>}by8b=#;6=a9;nK;H$3-6_+84XeH(6n7||KtNsPuLXw9>K>4el+_K zeAr#udF*Lu(A%fS=Vv4Y(fH7z@AusORr{o7T#e5i=P^FncY*I=bB6P&wx+=+^6fu; zL-;r(U}Zja^A%qGzYhpe~HgNJbhE|s49XP zul;7sbMFwm3dJ|ck(Ghv`_{pW{W@#J-CjYnfZvYj&) z%whNE#Di8Go2FPCZGu&6b=GRsXb2hj9kiUmhGF{}tk3`h-u->skNk;X8)+xkY*|G! zhQ$NhfsG;pnmhN{-wsvW3P1Jy@4fxEO+KG$qbx**)(#E%&u3EN9c$M5(5z`wZIeY^ z>jNEofi!$IXFvF~@G<5Yd@1TYY@*gwf2PW?>MT^?Vrgm}Y^ib%kG0%2rwc4F_I}M; zNm_Dn>HYdyB;cFUYUl3MYb;p$iS}TIg-k*DOM3SBuyjI#|DR^1TkB6wtc5T3vnaS+ z3e%k$A5m2Kl+#`Z;?^ygezU!Jq+(hqb9_X&w#8n=6TxR+i(3E}3d}PxF02?6^^%^x zZ@xX=Y2Gz(*%ODjKhbQqh=>$z|ME*Ff3L*b`W{f%!8&lvNp1=$kzaq868XO8Yri(K z3>uI3m$kCy-Bs~Y^Ncv@)i&Gp<3oW>lz4pRdgVwt>Uh;Ly81LueWF)_oUJZNTpC-e zQFOM!nGc9Y$-2Jlz3+X%i6p}1&V;+B`5eo-2F8*fa}VbneD*BB2GB-1k7%UU;FD-+Loy{fTA9}n$$T?c*Em40l(7-RkP4`%M8Z%r#2P78Yjd}>PJjZpQOwu)`l75!VN zlaDxpVPn+~%SXHBD;(i5??qwbnWuiZpkbc+V`SC)~V;j$Ek)-eN}NB!-o}3 z$0R=0BNoAM@M(REcC{nznuX82U=RC!JUDP1oBxS=b)t_!cz3%r{}<&YU@*kg-ie5CFhH&_G`+-RoK#vIL42d z_8A*D!iKNGqb{~e0^G*xKgeC2BN1+U?e~83YhfR&@6&6HG`K(XqhIu7B+uzezVx?rwQg^1> zp*UHJsH=X=!!92_al$%KXYP|s!;VD_fBpe{*#Drz9+wPNtOfAu^SkUA-mYbfNfz|b z*>+*wphv8g0esrVF3XKB=H3xoq`7WZh}gEVvCvo#VmfO=#YbeM>5tI!)}}sNx+Y4b zScN{ucj9lB>4)|89Uu9~ zjUAymAtkJlv+!GEOnfX@IcKc38NZQuvtHhxqVkc2E8);&*M9mPN|MZi3rE&-?RT-p zi80|eYeN$Wo(fnp7iIa>c|^pVYS)*g+F;7!)7QPkeI}NEb}!0b0_GA7^CS%br`@G` z+p~=VcR_(ol(-A3zUW~RBWUgKYaq0Xry5@GtB>d4;jALYK-ey4onu*M&cUTEb_A|u zFR-n)TRR6XhXJ1}$JE;3kiZG#qaXQ#z{_Fb7`_ZM#rD>NAw7}^H+IDsuupToz?qD6 zM5Kri<~>Ys`PFM@PQZy!%zZf)Fr9wS4NjvR%Rr`)pI0beEN)&tR4C_@(Qs zZLG3v{=Cx~s*k1Z>m|O(-QPdl-&c`jrplqXj&CXi**aw1hy=QaRyQvaUx+(5A5Gj{ zkB?(WUm?d=_wMNHB)*aLS;uF;jA{h|H`wS#Yq2(d7ZsPW6Y+?S=Hnl<;FJ1%>KYDp z9js(2LS$VcS!`?Ao)c}+fO8Vsr-qtS56eUqrLuj0POUwBM7tUXl65)Iu&@x791(NS z>WiClzs89x!+>*)BbK42E_gs)vDPA?|9-#JIcxCcxX(FT@0`^hu5jKv2jA9qqrme` zflZWnzGvVCho5$KDIwpbJ4|EflXPjgbMW}_``q$XpR04}a;ze&!KLBO!NXFW;>$b- zuP^88%&{RqmDWpWi~EWfxaYvZ$<7*)h&Ms<4bSUKoUtlGNv#c*3+zYT6XjA?Gptc# zJ0E=DeeVlxw!NFkk!JXN5&o79!4h93?OH zTL%ZeN=tLdLSVfRzux`McSgAPi9@>4Eb;XYg?{gxxK8ouzk#5;2}?(_PSoslU6qU8 zTH~Cj{01)ycm`PDYbu8&2gaGEtBaf18P>4N{f?KrVEq#@C7@oexjlUxD6okVkHakA z7~@sTUK2i#{2tqZt+`R*f;YliE3WJ73nJNr+Kpp3rfWQTe|daYg^st!dAIm>&~t3g z(|;LSJpFXO6fLuDu8*rS-no`pXLp{fb+dfF&2D@<+TLuFcQ@xeOOKs*aIa!KweE3x zuEn>*F^#W2tG&2=o&CD~q%XNDbF{m@uCes&x|XZw2j8xLH+$0G?dnX^Q{9`t!@l({ zOIVI?2kUm7TC%G%kFP$@-^rTxwXW-{h55U#oAcv*^Y5y(zxibPuGXLGzgD#l+NtNL zv|jp+f{}k$gZ--i2mgR+pJ{)@X=c8eMS$_$5 z_YAJoF!yKUoLhg{Zq58@BW@tarERRY>~}j)t-soC>UlS$!)+Z;o%fi#hHGwqtV?~A ztaIcyuYYyE;Hq?U9r*X-tn(Px`|^Ns=>7afgcI-kAN`yNZv{8UpY!N_;y0e?<)hdvo^-);%V&1a2GE91-l-mLS)6IniWuJhfu>+h|@IzO(b)>(uKkzeE0DC`+olV=k=Zjx*uPCK5}in=lmVhW?S$1yUgAcW>x}{% z1;#0`i4x%}!>7CPtLLk5f< zc7IPifxF#*;iSediec4`oL$+rD)gJsrRf^3BNy&T6ikF zW7kqH$R^ewXnd^g_I#tjMuA740-GrD=+E&s!i@qO1r{k_YcCdu$m-MBP^{?T5IfNB z?fy*6SOm22P&OH?mpuWn*}2z*b3Ne7dzq`&CJ%;3a^>&)fj@SD10ELJJ^Ww-#87f>@ZSAt(@BD_(eo~(Px)mIDTAgZ{5PGdF+Ah#7AK7M(WhL8O zHLDTPq~Gq0C4p7pV?X}mH^Px)OR|P&mxXa@pK3fc6@@B!OpYl|7Th)Ls*;7mg7J8GVpV z1pjclz1TGOS1)^6I6furJY$_@B04rMVV78Q;y`nJS!5@%;d1)J zVcZVa`oKbg4#)6~vcBrs>wzeNEho-MmI|M}C*bK5x*o&5^jy+au+bswMW(98{<4x} z*`>~KaimIk5$Q%&ssc(eGt@1-(fNvtZ9nygjXhfALAiDQg;uyAo2yCjYZ zOLE;0{-fXXAde~z)_U9ZMuCk2%M{o|iDmk1Pd5r|6nNe!z%N@|0al!rSNJR+f7_8c z%fo-JX7um8@|7V`@e?=)|;ODtYDqMr|IRO z5o7N4ZD#*!=<9K;B49{%J{HrDK^$u0TpLL6gVf$#(Ge}CZ8f7XYpg{Lt$ zXSS2;MKk@$JxE5;85ePh$RnX`jw#y{lk@}on`O>TWc(arqGzL4m)E`Wm4S<8Xih9h ze6;35gW=Nd9e?xR3tS)mKmO~X#G@<>=8HIajGaYq^Otk$Y@LYlu$J}V*q^QCB*=Z^ z+unAAPIK*ScPW`J%befhD4v984g7;|`kdA4INNXMjRG45Uf2}aM2Q#n{A|73DDW6l z!0yJhq1azx$_~5#o|E66ruy4I`bQtw-YXBW1=v9>Dt5CTST)vzwVK$6EsPfX^{wCb z`vP9D=moa81beDraqTjlM~Dusp1lOX7@2;){MbYN#*)fd^V+9>SscQE?abU9L?&Ue z4CJ^uD@ADY5u`a5Y(2}zk?Qdn{a9VHt-dxE|ABp8jtPc^`m%b@iX`5JCB&TO_=KC+dY8JcJ8Z{%w^$Xp&iPF&_3ej$@CzfJ$5w;Z z)Z7m3OEA_v3kSVsZJArIYhZ14HwtVNxE~5^qQw2sdBd_%V57il3P|Qi!|TWp-nUIw(^Oi ztWQrSi9z}7vv9osLmz5;Nc^VV4(m-bo?7I&v+O!+vf{|lB3 zJ`AgiemPh%KK$K(DEefl?LMTqAD8>^lW*g2Mkg@>G`mxEr^?jTRzF!Vrs|wphVI4@ zn)Jr{2aU`5lDpiS-;6!sSj2+C0WO{6a(+)8OW%&emj2J})SUC*w>$OznJ@iSJ8Rhl z0GYmwZzp0*R5+o?xa^#4J+lrb2;(+3)-!ms{1WH4ebYC^IM6kJBe6!>`o1)-?63EU z-yMf9G!)A-H>cU=5@)}==AidrDWv}rcs3Xt1@40an<#M~RNioG6nNw*z;B$NckU-} zsR0Wa_PNgkYsyh3opK`E&Kbyj(qHnk|0*&o$&M2~-HmmrPENP3SK=cQG>4@J@Pu}^ zgO7dTAae^(tH-fq4LU5Bj7$|4__8;NJ8=-(u-ocz1R^fEj`)+-&ry8*_S4~pV-=?? znU-W$sxX9*izVjY(@Pio@nd%fNyl(vLDv-+IS8zSE;Oz;5Ic};4DsWKe)NlW zBGVecU0uHZlHAt)D`d*)sztakrIKxP;FlA)66RtX+j=4C2t0&ldsCJfvC2LMO|au%7*EzjjiiETmSQ zksYBcN@657Hl2&WNKw!ATv#U>m{$A)g3rgaFlO?RiBeXq2Zftv}E~|;1#93h6XI7^k{_Xpn^B1Qq z^qSMC>pMp{YbE=ZXUTa-ErNcpH2n_;!}?ivzT+bwdGPjkyd!9h-kV5pdlt8njRKD= z1vXLQah>^X+#3bXra*I&c!2%EzU5&*gR=n_09bqu1be5JgVRGgTg184Kjl(VQQOB! znu-ksGq&Mkk!UQcd+dHbI|YX}EDrpziAC9e&PQSu-u=#Zwq%?CAS@#8)#DI#>iFPl zmX8A-Rtd>o?L~kMlnu)mZO^4|_J<$>jO@;=e*NR^rkGY)Kh;&SH8FGj*O@#Cj{;(v_3(dOqK8=fuwiH{+wH|emdqIE_9TUiRd&{R~hH75cyjD_kY)a6KyReVM8I3{PYHbwQC~%Adn<#ON zLfi9=0viPi1+b|0J;2`Qfju47h!#+bmz*gc{?t!B2w(NaG9^ow=c||f{EKW~MU1cy zfv`)kg-Bz&U2`&d`73_WlXesKoHlsWzkF(NG|n{KIR-2!mM69yYxq3RcAQo0_Pf?f zknNk#{Q(!hcw2p?15Ai}4PVbz+p8h^mOuv2!g7tv-XD_7_MxO$aBzEVyR87%;zJ{` z7Q%ideVyu}{+P#_MTzyyT8cQ0sYVlJoTfOQ>0{7lqD~fZgyYzhVh!t$cB6j&ulcgi z2wc`4k?EKil5=I8tSxY{46%A;pXSc25}dM#lD+fUTpQ&fmhIzCjMwjeseV2q!s*{` zPm?SiJHPJgliEHjpVHz^+MXeUe=Ic;A|IpSAh>~QPsN-KRJ=$PyT;#@GwDS-xB+?W zD6okVkKL4S1Kucb4h4L|UjB+##BR(o^kBu$f!DVMAFD${PgIG#_Kh`2M-EziY=Wel zr;0CF>epReJs=_o8wMMT$%%y}fY#Ucafvg}SfQRx?)zWwu`(QR5(hZ%eAi1uJ73v& zaTDnjBz{8pH`Yu2uuN2-GuPO3KIz)n-#`W&jy-*U;o^j&9x)}ij(F<8my?;l{vPC2AkCiT!(d&F!V3Wvuy6X+k4}Gd6K4ou8#8W|E}-aWdLeck&pfO zk00VrJR^3%htBSo{y4d@qV%+@ZCF-VAvmguJ*dxIXI89{cnJmLj-QmM??ekB=!(xU;FA`I_Zd2w>c!z6UA61I5Embmg{qLM|OL9XzQ>Bv)}bh z>`KQ!4rUp1;y1x!$XaviZ@%KG*F-OLw|4U2P+9A2*Bb>k3M^A#6D5}EvpwA?uujr=oHxX0N9FD|8VDsPojlVOl-8(o6{pO$f#L1!zwuC=Djj+V5>EnK2ak2AF zOU!vC%fe!@VhmZ8=s*j9fYaE%dQC)1XP;zyJrAdu1Tt`@z3s&p&MY;?09;lb;8>Zi z+1|$PHx2LrnkROl&tLz5;}>{BsGWrXb9}>Mx_7{z>xT15mb3JvD6@o<*a&X61COH< zb~oGA*pd*%+5@6MhJkCF&Rp4M?rd!(u6KUJXGa#OzOLAWtYf}pPhyG4G9uY%MkTb8 ziOkVkZW1rcRl>&le#u|_g@=3AGBGw;_Y&t!%nE02z2{om0T!-H;HWT&qr@PhDKW)w zQF7Z+QKiCKdu@H?`dr&~tG`iTqrekSY@)=r3N{LC6j(t4pIl*=j!W)H3$E}^Nh663 zd8C60_JRY4a>+AC*|a(|ZTN=sgO|_}%+W)eQHKfjILC;;7==GokCWGk7FHa|RQpjE)wTTU26LY@xTmQ4ep&ydLI?U&vfB7#s+OOziour9taELog2}Oxv8504f zb)(JS{>ra>Ak$Z+#RQ7gMX~q5o{nNOD$5&j8vAD>yK|2qulR`A+B8c+h^HCTL4+Dz zqsMSod>p9IFcbeKEsr^U9m7$dqhI~4CyqR`N?dAeiI+H-`&J|#XAX}nu37JrXU=|a ze#56fnFmYEDetmN_GyE)QQ%w(Y@)=u6x`Zx6nLyCAn*P6fAE73Wc-j3=Ktk(` zWy2O`9UA|tc^|>ek6P$-!f@s^f_90I16jh7<%th`){lDm@W?qFd-g0d*pkj>6EP-4 z4vtV+mSuaC&#eYmW6v1l@S#RoWY$LxB_=o|lV@p;V>s1|lpHzIDwq#eDvuGrZ^89SeR3;-+ z-XA-gFAj&c$s^tplaIC8o^KS`DDe1DU=t-CpSj*fx>4W)6p+`R-*z5Yj0=E0fDn6e z)e?jTTUyvuY-ei2v3Hz7PSxgdYQi3hC6M?NTb68gWzqVwQ?#%wEu4lb+);dD8V-ax zazQ-c@Fg6%@3$;JwDuAI(Z-s6Awt+kBk!YMNAUdJulwy0R}y~gdsvB2GFvplKC?_D z>klqbiNlc{wb$SS1N=P11i;g;yJ`s_I?*b}9td6ABg1>*Q~t%Ddyr$}=oBI26OC@n zX>*Q?ox&TgI<_zV%P$$4KdiY_21_jZ4SXHl;pBvLNgscGsbg`S^9+})T@z$ExyWys z)v4lyk7M2?$JKYx?=#7aPCTXbhtg{Y}22!{s=9yE_|*l{@t@4`tAFqq3!y5eZPEF z$G8s9GTf!7*R))!wLRM?uu))+0-Go?N13gBqrgUijsi4?_IY^Imw)*KjxhYEX~Hw? zw5xv7tt>I%arl_piYr4;J+r&=Nl02F+DNYDw%h{;M4*a_RJmxlF+{SOTRQM z2Q==c2LCu#Dlvr&cw+c~@UZ)Z_TpZf^0J%*5`zx9iw+tg0W z)+FoA9NYQDf9V$=Xe{347`msQdoE&izV^A#nyiav`ZJg0Rr~^nBi59p{V|+N&xMM^ z$9mM)lL-m^;k$&xANUd5(P%E45zCHwm5`N#o;As0vy^G7@8ih5-vRN66^I4RcpZLs zcM;;PPpq1e-5EiQW*V4bzE<4QP*uO?^}s;|B* zPg!r|jAafdt=9HUZ~nKMSacnmk?oBOY$El;mi#v2PTD=TTUXrzAvd!X z9Pu#MOT=n4T=0*IxkN}}fpIxF75QB+^-n_L6j9-_hDNu5!>VgzeT>AZWIya5A_aim zy_b^mX8k;SFWD|ryDZzz8wEBB+${w*QQ~gtyFuS5@CZ>L{J9(ZD{GIPlk=d}!xrZj zXqOnWEKSMYV@0NptLCtReLEE&cIf4={N!M%Qp+BAnqPRbq&?nDqPi9*%{{tRctV6F zxW)F=eHLmRTJ<%!#3`^GVQmu*I!bDaH4bs6WgLoUEzvFOMT!B+u-&8h%(sM{iN~3U zQbT%c0$d`1z7gGOn-P4p{nlG;$Ldi-$i-=3;j<2wm*zfMbmfxn9>XU-LWZw|<7}K2 zm-GWWSUd6d1MmC)*}MN(>$3AM>;vQvl8R7TXelKm+ZeJ+WVY4erMi&rG584Yp?Y@Ydvf2y`SIf_tY);Y|+=YvsK_Z zd~Kh#6^7l%0qIi<7hQ?cEQ2XOh&xX$iUs;2p>b_f);U~LS;BgJGPeE5-~7ENTxCA) z^l@&SH2y@TV&#~n1^6|^LK%O?Lv=uG63=wW){t}kq81_A+X^xVF>9{=UcOUcr@%A? zc2Q!QE_>-tft>=kD1f!GHv)D5oA2Nxplpg2xz%{Z4^{$fwFH}CDf+Wym_j92fSs&L zjF1fod~6Hn4H``>O^(7+9r{t&t`P^B9uZA8I#p+;0)xmR!xASaJ0oL}+cw|$<~Nr; zE)J~TbHypCy?5>!t7}`o(2Ik}5}n@pu6I3XHQRGeLOotiFz{siIicP}taA<`*zfcW zYfiN4VfVkWcA-agfGOfcYxU^sx4*N81!p)}h}wH|*F<7;$6~KxUpW}b@@0H)f8!f3 zUYm3MQ(iM*$!u2ENi0iC^)I={LCy1KTwAP2YjMXyM#Q~j&zKRs#BL5^#>S~|q|GID z%oSWjqlXu$Kt}t(fsiORmw{{)&S*VW>Zy!vqwUw$lQyGwpj-=MuYVm1?4rbV=(t1K zDexMmfKM(>GxpKJho6ShwWOSqR9<8F{QWt1*dM?rckaO6V{b;;!@h4~188ttZ9H69 z;)oVgxM{_ESWCZUyOI#JW~Dif$JjWFV7))~p%0yQp2im1cCGIHP{f)zb1lJd?{PxW zw(Ym9N!a)A{+XX40=|yLzu=ecY8I;kmqm;76p;?@ zh}NUKu#fgZoKI-lo~<((yRP+)Nq@xMcn5eM9h~KeN}k!&+0Q!#b_$G9U>7CE=(N|` zDeweSfX4Nk|I{yg`d#1lYfJd&Z!Q|!KU$B)85bw`@gwjrG`y zQ^+jV0X*zG4Kz^+JBQ_NkB_|kbq``8!7{JG$*H4?CWWo!IE0Q<{`NggT10&tb*7@7 z;#sCm2h*5SjPQAG^(ecB7<98j|zF%0}B^WMZ z1Tl#f0-gI^#pm#(T)&7uH^rIloPRzsf)iCAqRfhtwG+5=XJQv?(cJx%6Jv$!v&^I{ zXY_>cJsg}644s1(M?Ez5OpNYhF@jUP z6@1oL>cd5J84&@~X7o%0O@$#oWRUHhC>o5H8|9*K+pFKPOqj=hSZ+rIH`kWc3 zXVtvj7p~wHm&E$Ynw+xzyi?%X6xcJ+}G~R6+n%L|QfCUDdXcNu0wj)`w zuqQW#{_1lc;tSv6aV_hQ^`g#HxO&~LuiP7PnqmE_aNhidv!V9M)Wq^K6|KITLhVj2e=^|1JTd%uMYyHD(zy5E*r6EY^XW$m?ofF*uoN<_3hT`W#xyOp=l z?_RSmQ5UenSwrSKM`6LM!9e=moe;jcM3c?6xcXw9u3<6?Ds@B4py9WT0;d$aS9;bPHqy>w>Pv0mfTZ!ZMT3px@- ziTg|K*WM$Ra`qD5O}x(YU|evog?Fw#@vkOCKCVOKx+&u7eQWKr<>s0!QRl|8-`^?l zq)}iOC7!ey-^aUC;4B4bxH$k!9b4lQ=8#HK$`?AUWq+m}#k%$y_BnWS<+8PW?8CqN z5FgR7D?Prj^F$~0+iXO0u&2sv2Bm^WW*^oJY-Q^>)m%Stza;srG{CZPRvJmt+2W;D zH5PvEZFzr98*3-(?C9K%|GWRve=uwXdDEZ%9l60qT=|6Z=_*g^S8%;!e2lhlkL^RLA zO%WyX4q|0f2H!Q~8HmEh;fQDkhr?PqCoWG6VYrBDg9^a$TnY7`+Banw*j{m`z~_(x zyD0HFB>s+ir@-S!foH$-H$M2`4}bV{NH)poCh@gRDnj@SZawA>@( z;}nv%IScoe4T_W16bvGvFy_>E#!8LMX^W581JA>Sj5OG_%5F}~GYiR7|Jt&f!cxk5 z!>Z7;>uo*PwA*vIPGjvGUGN(VeuCSL&l=zabOHD~O|9cOd}5jIG`O%pg54>6EEa3_ z%^2+lk&)wMkMlmpmOgQKB)YTC$>_uqwq)5$yZyXVV5h)=0=p=&uY{cfI|Ysu5GLD) zQu?G~->#WZu=?7UEI2|pi7Nb)bI=6VZQKew{r%tnADznW`YR0coYsA6GFd%()@15; z%OdAK?LBsCjRQmvM^>u8^x{`NSURxdpZ{mX$z) z-oDP%Z_Z9Pg-z$QV!WJM=6wJA_BB&cX@kY@@Abd!WpQ)!dDTIVlN|9Z^tH!CUw`U1 zOBpM{IV%ydg*7SgC(ly|Hk5f@u)XHn@pu2P|N3ctBqB>iz?xCl!Uot=O=6WWCL(y7 zEEPNK*@1Ja?Uu4C1|7vC>oLy&aVnuteYCG>3J8G|!igpI*|_h))*Xe9tVVU_xDrARGuThrTx1){QA4r|KB25j zKKi5GfP)Wy4GTVwBWy)`{tMsz@6IL$9l?bS*5B6h06vEeN6T6VudwJZzIo;=M&P$~ zIC9m5sLgn=m^AhJp>FlhZ|(=@T1OmgeJlRB=JbP=Zu=B`vgHs}Vu^bG&-|((wwL1# zt#m9s*k#$E=CtC$Cx$>jY8*is?~!A7`XU{&_vSVw|wij-WpFkHrK(Y9d?*_ ztUb}9uY33R!9hxhGsm6G(cOBFN;qiQ)(#GFIO*$1EC}&Hy|v-eH9WpqGxkZD3U7bZ z%ge4LlHXA7aQIW(ZM8wEagiBm<6>`uJ}e%l5cSVLuxn&sd#zSJ{r+V@K| z%&a{UO=6>iBWG5RmbJq==U)0SdKL>oQ~bjp`szb}3*A^{Xm$fr97wALWifD1=h(>^0{k_9!9l|5B4A!)G4}q0x5X9u;+o0<1wqkR|6bVq4a{Y)!JA(Wnby z&say!y_mPurp+dXu0%yUvoE!|O?FX{8%cBjBjfv29@MTvbC z>=bxSPyh>wo$8OJt*^EYoPi#z8f|EM7SFI1Zr{$fXIT1HOWfi*UH1(){FaPz$~r*% zN(4CJqM6l(Vl@aXXZ>%*Bq7kzsB;c#MVgUus^3~&IG|Ohsj`bWe6-ksgWi(PDrcFIDJ=I! z&%JY;Invg6TYL|T7M(+$GV5q03sl<&TlYOzLT^JudwdY_z*SEeHrPZk@g=eB7(Y?C zg>~-Q^Q#?r)}iTQPnZJINjcsXnZZQfhCOOv9F!))6 zSgwBZV;_64#j+PTqi65ft$iKEvDe!v@HkUo7bPC&+1dNHQ(%|^GS&RtPyh4-Nh6(y z8?LAbNA*KL?#5iR*EHAs#8-fut$OHxVLuw!uCd(4m-55)ESSX``Z|RN4(0A=Jh$sD zf}Om1yZvEY=r{!*{zG}jKM8WO@WAG!-QjF&!M5^LUqa8rxvf2YjqdF{Xg4<(Sudxt zmhr8@cj*74G1WCN!_J0oZC?(1UOZ|)J)*X>ty{)5gG-B+-UmDfIKg=sdkcH&`hd^4 zrtY?7w`uX{PR8BO^`jcwuG%?n>5zV8PPak7+p5LaK8M!JwZ{f~=IQo&oL|d#d%V5` z&5XrZ9qqbV4{dL;Q{eHSz%EKW9`msGVyD0b6yRSizCd`i^L=XK-teY3JrE;7Ysy+6 z>yO;?|MY$DyNVcr3@j}61dxSDcxN1Q+B1K2buJ)VZ7LbPKoEN)5OImAZV=y@n8RI$l?S0N)tSz+n zwDwEuPD}wdk=CAuTD%2|Pz!JA+Vhm{151p@*>?6oVBx6=Y0J2b&01Q*k$arz#5rlB zziX^D7ICI&OE&aHUb9Vohne5uhq^J68y>mc&+qZnny^brm zy{B^hN}TPj^Ge{^aqJWrrNAyqj8bZ^w^QJ;r2tWagG7$|@G<6i^E03QV-M^-(E8nD zM`Qp}UiM13_!Ebv5(>?+2<#l3EwJ_uY@~7y7niaNGDihR&fou+f5&Mb0sim8y>oBt zR$%s5XmN2;5mru=XdQs+ehtmIi5+Vf9@b6lLi$V`o61>FBm|$xVYlVfUxQCvKv-wa zM8c}u%t9?J&Tnvtd$2~=;*$vY|M|@SP+^EknA``UL6{6mA9?xfs`J5G$?uk&Y4_2p1G^axA zF0@#a=$La_+Bp+_PCyqfIDDyZ9eSP?s@}Tg*&vqMR`MTz@@s1*xmy)xeh8HpCVn5z zoiiw(vQNy27&q5`FW)KfYNo(0O1zq9Z}0m~fl2{eS;@yvlkULEC2`|5NkpSqwDJ$I zj4eyy8k;z!`QCzG7NUcu@0p4RD^ioi9(SX*tr?uve|!sqrVXn@SWaiY{e41QEJ?=O zG+lKm+?6GT_=81#uK2|ZQC4QWEwJFLNif?R1J^_`ds6V_Cnlg{EDvG~_}KR$!K@0# z$GS9t;#QTKS;_nte|w^4dsB?blojBeFMLV$(O;Wj*tqo5TA0e-#BzYHZ6AmLQUVYM zM%O2C$PU@G=y0KF3)3}G$XxdLYP-@$F-v`$so!?5W*Hy~*d}f6xmC{T=d-`?3n#cN zK<3z%b`Os+6;n9cGS)EGoPN!n=X0WI-`_32y{q?ZR9h&ww|Sf?u!|Cp^X%+>+bJ+a0ZC0|?U5IBBT3~v&jUw$s!I~FMsM&4`@q;gkoRXuuq@;{N~88)!~RG z02^1gNNYk%!6s7BYW|ZKUwmD$^t_G(iLhPw-hOj)V&x_WFi~hsHQcnr!`Y|JHq%jyE#o5Wld$w#RGPG$K{%vY^bt?#pEhYr}JieMAx(T8UxX z`I8J*$S)XZ8hmxzwI1-qZ}UMU0Mk*E+!j`~t%_9>s$X~iE_FWovH!d}L2&@$?3A`` zo4M;B_}(wQMMt}?&()R9s#=iB9D6>9RvF*OHhe`~vCd9y?6Q20R{_fjd@Nwe^3}pUcg>nX)bf5iSJ(ZT{-NMw0UN1* zX&vH{_uqYF&bzd4d;6UN_f3Ial(=sS@9=jD+&cwm(P=0-8qqq_w%y0R5{&p*QS3n2 zkesnWSjTG&7R!j$k!UpVr_NZ@OYuo+%CC9~zOuR(K{=o9Uw}|-heVzySPfikF&3+4 z(m9rIjgz2#$BNRpPT^8kvkjH8)UeMZz5~0xZh&yns<(J(zAM{(gllZX6HY3!EvZZV zz&@|R#!Az&=-`NfAU@RHkdOG{gw~dgW)_o_)j^DCMI`HhLl|88oQz8PV6AertD)27 zH*3bIRicLc9{UZJXSiBz3TqnLU{i_B?mKGyMbl<O23$%U`$3Qoyl}80|3s8B>3jt*u|Lte>s8+qb#&w}i79n>pK(;cHDFDqV|{7)P=9 ze&6@a(qawFz3xtd$AJR7DDgPV!rq6S0vAz$W?l#_cE_ha2R7>>z+GdG4m93z2#SU` zXX3;fjdlnA)=w-oXNSP9=Mp3F{B!?SVbvZ!h;Mk^4(wLT^*>k|ruKWNNm7sC?6jM{ ztjF-E1qU?R>%hvmSPalNHQ)4>Uva9FvbAR{%N<-a^<`Jz9u`Na^r^ph?l<;2VjmcF zhwvZz$_Fe0ez$vc1Xl1>c0Yv!XO2GD`1BPnu?ZZ=l4O>DeG5t&g6x06deGimy2fT( zU%^GZh_62mJd9_}$;1`duBJj`Y&jbJjXL&EeA@`#oqg8e^eNXvm6W zXXl5VgH-U@!$Uu)CdA+n1?$+#dCKu?YHXn)(abYFRY&>*?0jIGOVlf<0A6)Z<_}b+0W$V@{$}J)U5=4ctg+B4o96IMW zb|IO7w%~P_&<6N!?EjMaJ8Ye*_-~IxM+4NPAX}J~=4OeC`uJZN+Ebhd9U7 zKAb^_rL|RB`pbWwlhl+W$#eE)IIfp==jU4vUuW>K41wXGwO(@GImCe+oTe&{BkZ?5 z`)q|#tzBpESz8B(s2{(T&14le4^6yI^?3V;=7xI#C{VCS40SX_)v2x!K*0!^(-^ug*$ zEGwhdoDaO6t!2UzUsJWP$2lFvCv2By6)SB1r&#+IR-Au0xg>7n<{N+Uu&7is6O}$n zI(tq7jQ!^ZQC|0*-S@J_NLm`_x!e@{#7ybh%amMrH_BjP+NTtzEQFN zZO{IzkJf&{D>iB>#6I-s&oaq<&pZE}68rJU!q?YnfA{!=e7E<{UqAD-IXGYMP=f`T zcqSRIb?V(P+EzW!8gXk)8_#N+C2mdIz3xtd5en?0#0Zu4Iy(g(TM96)GycEr&0qMm zaLXV1$VXOvW<~}kp5VEn@s)s+vq4x28cG^)?CS`eb9H>~u$PB0)kh(+93N^D(4zwc zyx7=F_RsJcm(eI=p((#9`%8T4)1N-YnHY2KI}2|=H-@SElG_p?0t@?I`hN=+md3bJ z$9dFYrNzl9CqC!qiip{>Rm$S{goVwVjwCK^adFm4Xgo_*Viejs##v)p%6PE6!o7(u z>SOQQx=Yu@JM?iBZTaN>1jd={KX85YM}PFRF0rJQzDr2TJ^t*(EDm&Nv$QWnwJeo! znJa94Kyzv${(4?gHg`_!2gjPF#KxWt31i_R<~%&IphOu~5alD`%pA)&H?{qp>YL10 zU^?pV>&HqgZ063KdiPAV-BLyb3qCPd#CWMB%%f+YbJZ5Ob8R@~#i1(eip9`-po;W! z4KFC?Bc#{o_Zhz zlaDklZ_SKjUx#w|>?#wZK?8mXZt)62G_i1DDSX=PS%9U$CJ7}@#v1jAIBRjD7hiUo zRSsH?%4U}FROgwpHCzjmh8nwPynN*ocH7ojx|YpI+mm3y)7wD)pd z6OEQSHGDaRGv^v>ab#<)zwdbKTW`^guYV0oo@uYRIwKVrEKv*9>MbIqalNl-?yZ-; zJ~}VW4SI6QQpa|4efvAVOTU)b*Qqts!pM5zC^eF90o%S5&TPv%KWs<2 z73SVG0ntI$+&sZ2)a{ z8dk|36H6?&Zu9vUZF_0lE3BrKN$7wdeHX z;Oib7?RgD8@e8sswT@w6VZ(&O*Gx^v&Q}d@)0f~J#RpbpkvkYzia#H0iVhcQ+K zzen(i0b;#~<~V12R+z)7`G|!_U6vgVFIdE6t(t0+vZ-|-1b)s@OuJYXjOm20T(`TI z2Y!DR4}f*mmgrKmTutm_(Y9;xu|y^VSC@gBJD1(Av*jKUEmnx^maOhM9J$XYn#Q=r zV#S8Q9g5RmKr~obb|)V_y0{cdq>_ zVen}~)SMGTB$!P-_&B9ud6ntoh!aq|8#E`SKAgPmXNynBJj+$ZU)Cu{wzlrvb&Icc zCYq}M@{j-ciFO&M^P0FJ78-5r(c#&i!jm#PTDN#y6N}fxF9?%27oMZQojO}P)dE;> z1gAB&2G7AGhgsPBuv1_Y z1+a%K2C^&hVYV-TPgP;F7C+f*k;%?LKLazAA8u>*&|;T>F$i zSDe&5-{+j7h?)^!JOi8$x3c)7)?wCEJJ4TZRn0i}^G<=sfdacI@i@%F-iMt6n<&6< zpJBc|Mtr45kztDt`Q(3nb!+p|@?j_PVEA|J(Rd|C|M?w_HdM(25f1 z)WhHMn}vy!Px{3PPbf0+P+9!nSrE3sZ`+82NXRL+de_k721sF_sa zC*7a1r+?(xXE%S6leDv12TJxE@Uay=p7!_m{Q2Lsi4}z_2|QbP=kTpbf{FH6A%6N; z%N9?r&*8(ujqDWw7ZIgeX*!>F8hQP8&<@wVDw;Yp)aj3NtMLYR>JOVd7vEu>v~4(T zweYe1HD;mdLtB69nkE~q+JhFJ1>#|sOtTjimG+TKe_Z6FAzUmi{A7ST#dikK_2eCN6G)Ph7JgNtBB=;&#qy zhI^B0y_l`6xcVV?6we<;7^fcR1uWBk+NAbi%V&p*HabY`mw$E{`VJWv;X-P{1!Ir3oSZJ1;?;z z!8p>ht*2ij4o)=V83QL78F1|0EmM{2RvVA4vcq)h_xM;f`tkI?CA8F@ShAD}NV~d= z_;GC7Y{A##?|*YF(WgFC7cL@1J(p{2>=K*a!=3y6uIGPErC&dmc6`>#H@^AJXK6Wy zBjvK1)mW1W+65n6Yj*Jt?5Xox|LngpyEfo6pVt^ya3|AM#^Ri#Iy=>jkDI;Zd7$bu>`UPI<`7bt;M_7-zo4)QeYP)UP)^2xONI$jRF!t%2b2Rpkb%Q zv%@d#9ZjrVV%516A<@L=x6cez*{IHHP!l?CL@4fq);#eG(IV3bOAr~2vU7WEL5KCk zGKIx-=13)}DONLD+g4jlia{)>a#Z;!-lKK$LS~iXz{HPlY)~)$2bB9>x}J- zCYM7PQ3F2K3ULhLCH&^g9)JJ9n>$M;$|4l)ID8-d(cg1t_wgEga%9^QOT$N;rWno< z2PgdnKZkWlMB?1n_m4=c9!ET(+PP1}Nc)t{ovlB8<2ZCv;)CqIK3A7WsBXX29?OFA zx|hgY<6}APId)d&rlqRFfz2(rZNlCn`h8_hJy6QUNiD%xbI-E~m&1F7CC?EHg?Ydl zuYtK#mqTH%N-$^X!+q?Rb93`~r*%(;dJ~x8h zo9z_%oKs*IB|hg>zQcIu0G#mcVtM!k7C(TbJ0p9vAUJSY_Hzow38l;-G!)>(HR~ zaOZy8H=%D&?32;SxMkdt*$FEM2JIz_K@z2EOFKJ~lu`z$8X9|~AvTx@k&x4B**MZu z*m}zXqG7}fI4l0SK^@E6v7u8Nn;~mh_6W%QmGIAu(jVykG*eTt?I z;fj|U(P^D&e&xKUeZ*2M9az~`3p_d~cn)zMf4IV@t+o{v+zy$jeC}OiS?X4Dr7u{< z%Ki=_l4-}bX4$Vos5O>3to9rj;jHbr=jyfP=7>m=cG%j&GMr^euoL|M)6ThtJ9Vx9 z#7K3C4}8g|>a}GxWVcv)T0FUK>rGwDz@x5AT2tAY`Zg?M$?(LvOvWdc3Jzf4^!5AS zLXJQ9fe)M=lXkI4fXhd|g+-e_g0F3}bY157#A+XW_-`jvO>D;!j5YUeigiT)SzG!n zLBy8qyRio5QeATeUloat75~`_UsAWigv(m1+XgNHS#hsA+!cHg0f{WOz|#kZXJZAp zvsH=lft|Bw@1;+C;;MMk5vm!EhMoueq{y`8-In|Nd8fduodUZk@oJyJz5h=T1z03# zkvdN z`LT~Z;6%~Kbky+JBR6Kf>f)l_v^elnMmxMHPJk5)PJP;uwR z1?N~@jG-6X(BW3t`vrJ3Wfi%l#qQuqKpR@xW1_pg`ezHT!tEf)MihKEMdaviul@;N z=FYhdS$3U!#0hC1RwL*dpl9m<$Z3p4l(?CUZpLTNj;`tEpAx>tidWZQfgc+BEmKvy zW;Y{h8YVoy$43=#ADOCxnnlaI+?cD=T5jy;odQ>+z%EK$k(fKEodWku0Y-44i1{2j zOT>!BZ4Z9^*`Hubk2tA_7pP7SO}MAeZ5tlh|JG(rVnr;cez)V{R7DG!!xm=5h8iv% zF(#q3<%}&pH^!g&>>qm+|4_zbT-srUXj}XC#vgzFe!L_2k{ch3r0+}HUU5px_*qM6 z(Dg?<*8Kk@n#uHXZQ<)NoX|9hlgXr6JI!@h4BpSkGa_rL%68-B@I zIwZcOw@3sQZLskT(KC%^&NU%wfn^N+xY9%>E92Mqlx;odIb@}of|2#A-lf`Z&z*iR z&5IZVv>e%9aH_v;S>}z!sx8lT&74(mS+8@nPTi3+d~@qp=)Q4#Zm)q$f9jl-{(G*h z7a|cFXWYhBcb^~W{9qk4J6UiG?#$=Zb2td{Y&G=qKK1^#<}Y1`XA$ld9xDp$qQql0 z9eZb9TNIGg5Ucv=c%k~m-^VD44)1;cZ$D|*eCmbNUh9*XKGRwXg{GB+qd#cbQg`dw zCr`+#BcZ+U(Ck0=?O#+gM%9qk;NOae5?|7xrQT$I1l?g2nkLIeVe?O7^5y$V8yVW;%wK(T72`L z`eii*06%k%GGNj!+pi!yZfhgV+dX@CjNtPOao#gW4(wi>pB|Qku(tGLu`&+3D+eZ= zw9MKFb3NzOmUcDF_eei%VC(Et6D^l}6(%|#|2VgC&@-;SAE`5S7RsFG35(gsgVb>9 zBfhZjERU6TrJRPn#xP6)pD5FBeRPyQpJZQOgO1ygKZtwYHPXVk277`0&$K(6T)ZtS>6Ge3JInHqn zw!SAa4Jcpv;PBZOcaUwY`ubac6T`4dadHSgT0^W2ymfDbtq7WScJZaH4vw5TvdA^> zP&TLWZa|&A6dytpiS5M#%EFbf;p(hXAY^MiyBnVcxKM%EC%c!v3y-G#%$8u{Ig$m5 z^M#OB*_9T+&E6uC)C8%=wI?%ELW3*YS=gs!;4!a6nfHF*mz`MfS-=xF%vIJO%_|LP z9Qr(8)=Qf}H?*u)i3xafMPSN`#(@m|B=2QuY3paZzE~bS8(FU-aVub<2P$fW) z!q#Hne(tCL;4H2a?hEnJGPe!^C%D}>vHY^GT!K#AMl|x+z&TvX$&EMNNPDdG^b&nOK zjXjBk#gQ%It-5O*oAiqqXM4f)H|ICK*`o_oI@75HFO-s(bn_6CN5b=_l>K?yI=1VxK9e~qQret zcZdCiQ^2QqtCnmn>1csD=5U~qpjFnRbK8fl#i728N?4Du`0(Wr+PnqkTpMCbyW=e< z1WqarEKOxO<{Gak$M)IB!Cng)i!-O3fT@|1N?CqyIosFaFTMCxr+pP#Ty<}NBT%8j z+2U>s9&N0k>XLZ53~PbmX8VhU#-&OhpM zE?bL^lNO7PxyK40ejlLNs!#dNG5nY_Q&`irj$03lrr!FCvkgl}FWM1dj00^Y->rM1 zl;12MH0+~egNp-I=7!T-oWWWc)k;v{Tl*2ugyxT=9|#<4kaaD%%vbL9XO%Wplqb9Jv#PSn}08P}lX4&&8K zfnAh%HP7DO_s5(9ECQSt9IfB|wY$ywzyYDv!n)q-L^9VuY>12?mGvx|URz4>-NvS2 z`LQ?G@&T4OQAl~T&$M{0pL@buFLeTN-`I@5`O|;! z*0x;9eLn)ftz%rbwstETb?bw3Q1C^&DJ(fMV7HSI1Y_QQCVU$#FKqQgMp2Zqtvo zXRJQo`--E%XP4^2G9AhbU4>l>XGRI~;k z3m{yMHq+J`7^8LTJ_==B!^P5g$=V*l$@&Ku$HM!zI&g?%-LnSQ#;%NvrL99X!BNoI zuKhh}@#NYXgo{;iowY3x)}519!~1Vt%Wbtem-?OwDUrS`cvKT|l-Ubhcj9kMhFMBlI>EoI+7K*krFe`go zoHt~w5gLu9q6wuLO`T*4T7w767wur+)N^66t7f|cmR+*d!G3qQ7k?TPmi88&0!P_l z{$Bq8POL#qGFqUO@>BgHCJy1F@x;ooOtrYon@^?B7H1^cwsJpWQH$?i{Or%3#m70S z&DaDUKmKHC3PFev!DTM37tS%Z>QgS}fip|$&z;*oYbQDD)I7!6i0A}2?KW#nwp!Gd z7(9n3GUrxAY}>Zi)`4tLEMQCGNHTqaYo5gw zuoMO#=b3A48<)O4S28IDc7OJ~*Znn8`3h$&WOX=d=xR=dDN`R@0mU5<*ktiLGE6zD z)$2~Jy_%;j`UBlXpz?`Z> zV<~n4TZ>-8Ygr+tw2APArcSLgG4AC=$I`#i^@7EYb;{UkwxqJ|rurr( zDG6}35qpxgLc>4RrY)0HnUjrvzVn4Iso~;r4r}Y>x;pHjbz!ZE1lik#_I<>15GNk< zc1}yl`bY#duW%KOZC#hfd9H2x%V~?1ZKQs-^-G9;9Q0TV1MiyW>g-l%S)6hn)G+?C zt~@K(es8AFocgjhoUN_<5?SD+-pSfBu0D)E{krB^x&tETebMlmecfTbdMU7r60hF5 z+k5`lQXsEwXP<0G%L-q!iff^NVnD z;`r<@{K6r;_NbjSKJ;T9>9v;C$A9A>KlAagc|gORF$k^3#vc;te&~S^b9*u5e$HFa z5hYt(#4__k9ON`WgyEl#%_HWt+U+&?suhXDO0lr$uTXdv4|uFY?7Kv*V8SU*!ZrG0 z<+Sm8kmh^}_#rrnX5NT_@&Y+3du_v&_eXOMAn;49zmn*wIP&qK|iRAAQKm^+y0 z3ZD?{?$({3v6`rlZD*thXCWd)3xt0 zV7p7>psE1Nvp{^LCh5}@T!KqmV~SH_o@l%)!Y+ z{KTa=w3q3HmNh>2!XvR){5`R~!Fi26CD1J_BYarE#0yk*_n7@BI4>C=haapgSlD3f zVGA$5_?)h26V+?)uervyOJWNKZ9c7j-@dZ)MFij!^#kAgr5E9g|2xaZ)Sdwnh@vmH z*?PtLZW*6(v2I|4kCv)C<1$Y9`|Erb4JlR|E^Jg!gLA)$tJ>O0dnyBw_8j4exixIM z{0rKs(PPF=A?t1HV5@01Jk;=d2kV3DteW zH+|CsPGzpO?_tjUu1;4)`>)-cb*yb0d~vWMvU&E~I&;??^xzTuVO@`~vaAxUTr7p= zaZUUNF^BkU&gc3xSI(-|io$dC+H&hda_qM3KCg*(MELu}vBTaeu$BV5D6y7ed;OgP zXDQ(Gc~is($EdISS)g5o7DvsqFV}uKYHt0m?j=7D{Xf8N@$}a>WvrLtIrRNJuHfD} zR%1Gc%Xp^7ekdCs=^UQBun#cd&G?+xjIqZBmg7;M541htc&h##t{>67y?3ZF{eKEq zTk`5B)@$OVvEu;L;^?nOLUuagrg;bN^wo%USm zcN`y_`qkb)cWrLc*R-F*1y)Ve+Tg2g&mlS9|GA(3>03K>>+2G(OYn)?X`f^3FxI%a z?y?N9KDG2{lef;blPKESN;}UQr@p!RDQ8*ziNF8#r*_(ptQXd}6Ru+28;wWcEa}~u zwxjr}_)+@CGE@^QX8;3dPKMr{aItRmNqMJ0a602qy4Io*SjF-2fqhpvsHW_3KOY>Y z9__tz-~ILOUt^uR@BZxYZ&P3wC2rGW|Fu)#LJD9(vGCXc2dA0V_xD2Zx0?Hmb6~+1 zU`zSp`vfxbuW{zM1+@ENJTkR#QUJR@=d+FVxrXS%StjEMyHkDb;XblIIoJICU4KV^ z3LO=r(2j>#fo|~=H(u9CH0t*#{I(vL_8>^V&5y0%tP8&M{Ov1k%lL%&Cfv2wZvmaT z2CK#iH1ot!qzxOcS$B^2jTguy61IlUpg7%z%NBF#xARm?L_6+e7t$y7N9NBnWK8Wm z>F3<~PQBW<(p&4-no=J8r$J0yfSq}AJ~)W|lS#3tHb=`+ug+O1@C^734l!#2#jT?B z*I5RxTQ?_uq<)+ig$hD~J&sF$SM>mz9^W?{ z@9O>!mjGR8>pjrEZ{GQ?Uw?07OE>cDWDZk)pXd9Uulwxt4*S(ZfnAh%^~}}Y(^mxr ziq`dY{LphmyM4{h3lFsOv#`_3{*_O!PrgHFA0P5QOmqu=cnZ7mAlr!r$JF_`qOl$E zgP!V#a9}YPb9f|U6`I=P6}meGZ>bC}$z@wu5_zHIR|pFvk{mrP#88pI!+*4I{GeVf+ufK66UJcfF8 zJGUb~YqW3M{}wy2w3{;@{LoOGO&|K5j84YAl-RY$JNG*~v4dBikEWtEN^|a>we`gp z>` zHjR^nSNdu|0}VJOw0k?y`>o*K@@>DihCgGY zWOh1jt5wE}CQGpUdn|5$@5la&+FuEYr*<9}Zf!gXO$|P|{Cjvyzp?n$!ckyx;+adN zsg5~~!_HQG$p%FOKgaSBwX3$ClTrDzeIvTkq>r-NLX17Ho+a@TOHNJLPqfU~6LJmT z67AZ18J7%BOQGc2eC$IXs`wHxdAj$Q5hx;m$A0konGhe>Np(s1gn6;1c5I|YU*u!|DIq}nU) z6gWo#Y`0IK-1anFANb+dJ_4%DUMz+XW!bPK2odAjh z6C6H;w#1_mm2Js^ExdI1Y0f@xf8!e;eEJ{$q0?As+UK^~!(PlM3;w2Givf{Dukd^3}>YP+#PmblR7bEyGS6K9yKlQ0aTv+5J(QSR*CHS;u z+z39bta|3c^R_fL8GO`5^Oo(^Pvs+7)TkLeTk4;S#Str{UbT81JBFr`3EvCf`5P;2?-CAw;+f!bP@znX z<&rYAuWFbhj^Og_&poS`;M10s4SbG*&Ysmd@%U0)d;2Go0=p>jWX}6O>U*I8tq<*w z!)J-sa7oL!wG+ia{(y*poyeysXRH_8ML+et-YmZw7x_)k?b}7GP1a98>L#|IP3Itq3MgYaRYM>O2k35KInX~Ti(HuT%S%$IsKX5@G)$zX~1 zw`))T3N46q=fZZ)4{KR#W#Y63#^`R)-V@RaOgV4$Jw&s*Rb+ER84=vPHpLfT63HYx z^;~8x5g|S6BjMI-JJ2SUjs{(#UC(>nCUPynGxhCZ)4ofDl@nK3qRuqkd|P<6wLxUB z`-v3QL3sxat;;P)StM+eYRx#xd+*+GZRy|M=CwhAU6goj%;P?a$ASVhPBew>F`9pU z?Eq-`5-!;K{Br2HCQj@L>qp&6tT=D9xN>c8g3+ujbK`ti*mhfy(b@;DGlwc#JmcuH1?Xc}cCc8uTT3n8Vaa2kDoQX1h-=>FH5`s4UC2CIBd+MnwnlyFWcIJ1kU0ZIw zi+8}XqsN@6%5uc=*W$=EmQ*Nj3wP<-drH{ zm`i1K?KN}9k%|@1duXZM)_W|T)&#N4^B&P`E4;nkPJzdX0=p>jIL*f1msdFjY^UZE zg|)`|I%qbRw9#vajD4y*As1g~pZI7{Z=DNEdZ}?98_XBJbN#p6(m6!Oa`T`1?U_Hk@YN~jro>9L^<8Uo2HjIw z(6n=H?Gbq-YwX+7PFAP5vOdx#TiN-I-R|4;ztM@4l4EI?Ub}nu>r=QXc${#8zlXo& zw-^G>NP(GioRfmDW;@z~plQR2tHz=n*94k!Vq*;yD6&6k(B`6ku}CDc-kti^eyW}Ov4n0wycU) zg&rUL!7r~OW_xeyx{6jse5GU6o>QndNqRKcAfrRGWX88 z-D>+UYg4)k^c!{V+m3B-`6N?d7bTwL+26;0MGDY%2@ieg;~%dFCnMp#j%#YUgl%J$ zXej5jc$_lgsB_5>IZ?4Tz=JiyX31D1X)0$Lzbji=?qe@xtJ;E(=1y3896eY*rm_Kn zjop|0lm@;!AZEhu4X4c|XUS~A7OUg^Rjr-H0>bjSW-X8QLH0KI9O4$d-@MmG z+pn#6Q{FQbUl|jtGKaObFxJ#(jkPAOy)M@DVXyn-P+%7&o}9VfM|y7*I5^P!$_H2i z2Zw-bBNIo!Ifb8l!fR<;u~^u7<&Jkf|NO;U;uY)418gPO!Z~R?9YSrLYr@iU&wU4+ zTgHn;;1ou55Q`!C=@eV#b0~B-`1D~ddx13)P6>vjuJ+zo>N62gn0as#-g^ul6+{1_?-nkeRV*E`N2}@`;I(*k}u3`JOAdE%g*1759+Dmg;vkVnH zo{h3zPc*MvIhJzHR-6FEy6DeZ9I4N0R2>Wp49`f)orzSj@(9^h<{9bhDIfjg5R>^4 zE7Z0lZss0KihJd`m5+SN>MrJOUFci$4M(R(%I`G`*_ zzMZYT)5My1R;hxng-f4&7r?B}cLlFCNKBFpnj_q3zt`6D{V_L=IK-`mvDe=zu!aJ= zD6xiGd)=J^lN6wR=9tnRwCsMjKKPT3R}^9&`0{cD%4a?2h!sEZy(tF4Dnxp_{h?b3%qujw4*ZG|Gc7!OjgtV<3W@z)K*vPQ0KKGI+j=-I& zvlNGsPm^=-xy~nf3f^29tzBjLj_h_|CySHcwENf=M|`!FuZ_(RhIsmvAH@lpV@@-3 z{li|gu^A)mQ|7$Y_`_@d3hRAVyK23J4p{fOzMaZD+MyTrgr@#fXXZkT1#3|7InPu* z6%!vgzR={-RIg!8>OK?&#~OSP3-O&x*n8?&b1}EBM#k5G8M?eJPTUq#>v|i4GG3`V z0v&1CK7W;_jfgQ0alU3NI#k*+eK#$ogo zWh{^GPQ_*zW3hx>m5#wUH9qg5k#|F2tU1>N(arNc`W{+?qpe%s$FGY@y)j@wXl&mYrv7ICqIvXg_=T>^fJ6i{g7tJ2VQn zt%r4>{p1J`t$N_MpY^_sCVkY2C-@lQqh(!!9bC?Bz0s|)d^-6+_aWV2IXH(AXJlS# zodN6(qAa2Wwthq;tp4Q9=d|tjNGNan`5zHoTAGAbKH;21dJ9Ns-=151?KWg>o%WiO z9kJ2DuYJuY+}f&+KMj3a*jK-9Sl`g*>NpC(xiIg##;%OeGs^PRuB(i#mR%Vi^+n)M zt+i{Sqe&;G&f(h{8#UL*SVHFT4LyHzz$eP*EE9PTjieNq06Sa63MjU1YJBZhk+T3c z+_0~UZ2@bje0l!FD6!C6w;RB%YT{jF%&Qvhm9Ij9U6i;A8FwI0E(Ngd5^p+w?1#Sc zs@f&5MSkM^?P+u!*u78u-T(5ijZ%44W9Odx_Ak09W0MeZdHSpKLJ32xEKq@6S%>ob zqaXQwPYYA5u>;5ZwC>OUum9%Rn1g<0+!UX78qI_}+FlFpIhItorv$KAdTlr;VaZ3u zr5aXQeAj7@`A7HtnfmQ>m_0ix&O!_E2qQS{9=s-eQ^HVAK%sf?`Mos;Wok?*BW?zl zb7E{-W-e zY0m;_4Wk(`x=*VReAl99O=nPW3AdiI&Ir5Kmz%*90_SLu`N@2>^*PU>v1268rENQR z9eQCo`#QA`Wo*oxcm< zPc{8)NfvG_o_A`ODpHi+`f<&Dhk{#}w{#fQl3M1=dW364{Sp{k&eRjO8g#hc0%LD; zIR$o6;&K}95T0lXU@vK~`NwCcYv*?N+e--~Q_Fik@H=bn^Sr8kCcLh5(1w5NAN`{t zCz`4LB=ZgSu7xec8oPuIcXou0Qcv!ApUNqiQ)R*$X|1%)0S5M4IPg>(Wi-FpjHWOhmG+`P{7|{l#XqaObXN zhBEJS_vgwv0loi)FDdKg-#gH!g!SG>x&ZiJ`ot$5B-2sWt267@T>rL|dlqYK#&K;C zJMB$^Rm)o20(Ytn=L+nZaflzWJG)R??5?as*B)Puk0`iUb%?rcpMvk22|wR{b5qx$ z6b}!ZZQF!Ln@b_{*L2LW({-g;NjTe%Eh3^gqPG0fYpkdD80(s`!Y`q+_k?GD^c}IL zAN{tD)};M;YJANaK!bIy#kYLhw>{t-#yM0Pf)PvQAo#u7PJt5zc2VL)iv41zz!nPl z4A7k0s{jkhO5mVrQ+_Rx!Y5oJO`q^7gLTSeJnNEUcZ)upKJ&WTNaVmXbrI7t;4N5TEanYK|8WlV}EL%OHd2q@A zHDZ|n2V36K39b>=>*rs7`GgN%WA6La|7IPkR-2>a5T~U%T*gGaB6>P{+QZqyzw^7W zwCYLT%TgwM`%axwP_>24oN2=kU7L1akEyn|%8vNX#*Q58!Q&7g6EU}SJi7&Z^2Z_O zVQ#Ms>roc5gz&ShQ6IFRIL={>g+;gJQ`amgfBX%b70>z%9^XP`jy2iu zM(CM3)0yO8Ndjni!kTl05VgYu-X9`=49i?Hj_X17S z9ILmse=GqUEI#;w4?GaA>(fC?ilvIyH|Mpmr|KtwocL>L)frrM8|pQ4`5Nw#`ZoO&KMT61Db@Z?N0 zI|b`p8S%i{0!Q-P-a(GYUeh0x`!(jIoYkKbO_%V=fR?!;N^=)SgdW2z!x!M8OW znT8xJqV*ImPFtR_;Byu~wN+H9F){^){V@7*`E6cXT&_##^X7GF{n_n&?pbxes$~kE z;QfMs_CHPbZK}Up%Dgu#yIt0;Oh>PTh2UCFXj{g+w|R0Xu!|B;&Rp*!y@&!>$>QHy z;}&S<9pC?#e@A7feIBBrbWNj`I%~Bh@L<=YS&Eit>MV;+Uyd#0#8bqINeyRD1it+( z4D13J4lE(o*THvKXtiruWZX}_bnV4&%zd_R;XG=tuofJMiq>=Z^Q~@WSqnZmNBo&3 z&!s^g*>gdzd-d(_;GmT@=d`czUCZ|!J+P_Ct50KIowo`Oe(JQN2|v9>yNy4D=~|cO zD>?1?Y}?74wFJA7JpLO|oSNxB5sB5IkAM02_kCFv131ZD3t!+o`@##8>xhHSweX>x zu}#GggaEePyB0oO#69yX$GU#~nPud_eM7AM6X zR#k@-IxShkSn7`N&dp;BUKzaV9xCNKl(P0L^P2C8G!RQ`s%^7$Ef$frPyaoOsmn?z z6Q90Z3s+#2p>kEtM_2*T8ZCwBOlV5wvACB#Sr#-HPk9b47n>Y!$ zIQ-DYx;`COK!|-0o1L+I^Pl|uO1nE*S1!OuY$Fz7F+w{IVm|K)rM1OW`ry139mgWh zDS_n^%X$>?3p?1am#z3%AF9^*xE46v#Q_IIjteaZJLSwNkY2CwAZ*U0FWzENRb5Q2qIvhWekeIQ#VHH<4n573ILj0Yv;m z+OnebEW=5w-8^re4Q*H^yaSr9&0e;BczrYfv1*xkd9?O$K4oX^~` zI&A{HH`*!i7*JprB_4xG*t_sbQUD7;j3_MJ(5HS&-#E-LX4ecM$4?eEgSB9Uomqpk zvfaf8H=o6@7x_Jv6=eC1M*A#W zF=O0-rZ#~qyLyAw;}d@Q(kEVD**a_H5`41$5K{u9#YgMKr?_S9GQOINe!~D;RuBxD z&x~&>5oFkOY~~tt*0an)5hwWTYfsE%spDid1+O}#m1kp(gAVOn4WT~H6>-9|Ch2n4 z%Tz*5+EFy7X&!m*v3Y#iTXBK~lCz<)u#TBOG=sk#pLWq=E?FWa)MdXJ2);yu2TXmN>O5ogn<-~$^@BCvD2X2lq#{<$;Jl0(l@y{s2F z+e7*HeDaf5e^-01TQQF1&b#G4aAn*any4upJoQw)c4>1sF($ z(N>?;jWEtNqt)ajP&Jf=y^bHQ-#_`Wk1c*q&%y5R)5>xz3Xc9vv){Mb@>?blaSL_Z zwWDuwP4p?@U^!~2;<0=S{I(6&tg_KZcpO;xj??O=UUmMW38&@0;1A14;~cv2dlxDk zkhJQw%Qd(D6~@P5D!4dYv5Ev2i%&~m^$zS+YOFikv9~${6gaFBan^wIxj*;ChkZki zV@jW#*I_x-=Ks#hqCKZOVv{>?wy_f8cUpRH!G{g2Z0!+O8yYPcC7472&H+TTB&1co z7N7RXHpU?@qJ**4-QBU|1+Kj|l(8)NO~g(gm5=@rC$+cc+jYo+gQJXlANVU@QLO^R zh8jv<$~d&3Yt7^|h|Kz^j%dYUtKU(ahys@|cOnbxZQTQ=l-Wj(qXkwxYw8?I)~m#J zc((ic)&+bHaY7lZwX-%8nt9=T36?kz>!rsh)kE;ra}nd7E6>DQT;L{F1{`8!kIQed zGu|QQ&LOMVRK1Zh-w`4pk$vr6J|pn=Iy(inP+%7&woq+v^D3nPCkYv3_}<=E0zRan z-J)%c=9R{t?>lFiYm$!U(iT9ppsgnF*7X$-vV7QELI#o+Hg-wuLR)qTIwA|rb3V_3 zi#4OEwSR_v7*bx(8xays_zedZR=_nUw8|zHxLBpIjH$N;ovb&+A=>-Efsd6U?7+if zA;I3|{^`6DA@L{{h`_*NvL2x?aGc_jKs0qu=N0&{nb^Hk+d@BZ+h$x7u zPL=NP75<*@_zfqP6aBEfEiTV@&16;XxsN43I295(pQ#%7z*{VshevnDt`2P;(diM4L)jGF(x&gr zIgIU??zRnG??MT_HP)9F*VGumI>5b)?@%(|pT5PpIXK{%$CvweY0_iL@AuU}4xw;3M&fUFs zdDhxZrR}w$Ps4j9aZUH&)mgfW60gpA+k3s50(^C8*HXHA$eM8yf^ z(XQD?UwBSSaR6%j$^s&er#g2W>#m7g5MNRgl9sk;NFyo36-SsBk83*?bNp%NG-K@R zPW`5RZ})}JPitkWZq-nBe)hrTJk_QxlVL1tf@Mlbd3(RTt`3A{?5WQwFY86w2-}r# ze-0^A&N0Eq5sw3qF`zZ)Dh_eg;jom6!CBgJF>v$?uGZP@bg#KcJcV~knY&7Zwv6+p zcTm%0^3wQZxtYA#6ef1FMj3>vr_>nER(kAgiaIuh$WUflvt!Hsc z<(Vm=<==3AzPBRT+dY;P*hPuQa$5FIU6%qr)BJtqW#`nwfGu3~i!a#5ZLWv>h~u-3 z9d3_@ozub`jL~wRyms#etauF-J>p%%ik`zMOpudK#>00STgTDj2fp`94-O?4?(dL3 z5E+OjVh6CSSU@o-e2zH;VbRl%w7KNW=_Bys%PqSS#~hhmI9=raC3rZ<2vIjr;BgK@ zSL}WqbBsIq#2;+M2Uq<3`Kfcf;b;Y(>^<=7M-g=&1Rsa1E%@N#{~a0;)mk`2Pxlg2 zP($6z?=ASS%NZA#)}G|K!DoG7?cIymaf;(8S&&k1(<{meXTfZ|R!T9ET-| zo6l`&7kv5%XZq=#wuGa-7pK9tZ0`Cee(I+l(Y=g|WbxI+A}Q8l!>eBNo*awxkgM&Z$&nWu32luNuy zW15i3z!$EWWTGi^-WtwsJXp3Cmd~HxwjARdj^*XAF4-tck9%u1ZP-9Q@WibamuprD zpYL>pG*OzeIAGHU` z&~qf?P;hhE2Ycv5Hni5j<8HMTg% zXb9h0B4KsxS%Gb0Sgxjbw`s|vY42}R;>KMyw(onDQ#(-Y?uyzm=^@%JQ3z%?K zXP+zMBOYOcE3U$teeegryoSj8Y|yIZbL{@)^zKm9AuO3!-OF`_yUA!upA1!PWM?t=Bp;g_hnmafq`QMq7XG+78me#Yr-3dtX2G zjKe+aie*b%^dA+I>T_G#u<*^{0>8({+G@_=8x`Tf;XugIj|JNAxxZ@ZIKD^n&LI+d z=U^G<+D(;P_q6d-_flp}Rcl!pPw3CdFmsuD*F1BClf;;l8Yi(Nw!S8g9o9|(6xc0nKm^Xa}88E7qJRl@exqiAS2Rb65snCqL*lg~tO)g7pdQ;^@6BpjT= zr~gSV3O+WK20e)!v1@Gt%PAP6Wmpc{@{EhWJ3sW+X~#WTu{aEk!kwzexg-uhU~-0` zp`=}I9cgM`%3NZ3rr?Z}aj5&*lF26H!ct(#uxXMzp2|xc)_AB53QpHWDMy z_k%xso1~XhOxQ-wG{Hp+jD4jxP-Y#i)s#xg8V5xbp| z{q}qjBZ$h{dT&f=ko$~T>awm8jn}Z-HHN3O)!uuf7dg{M{rZ3Z`9H6g2;=7nw}t(! z{Y1Z#F>k~HiAZ({OE2RVYeUAdpLYs8X%yH+i6?EwpUCn0MEjgMu!nXYty<1|paw0d zB(OebG`4k1t#Zv-pxt)r8mO^UVRLA_SX=m#N0h0C?Q0+^FgPz&ZCV)%oJp7pJUqth zxCS~&E(4DSlqlB1&At*_wDv6=*Zk4j{#ze?9lqE4Qdlw!Nuu~n43!Pdj9$9B!>k~9nNgIzxMpLH-F&?4qVtXto>WQ^;e(D zgC`C_Tk+9SXDy;(#?^~{VjIFP!(E*LR}75!%wiDnKrBO^;W+<*i=`)WNKDJQL)p~a zE}=%Jyf?JR#uJ~l5elrV!Y#hqFBENLd9`(gjkD@ zrkL0j9M0H3PEGn#=vD7wi{JMbe@oS(3fWKnQGD2TaU|`IqlT-}p;UbYnRP ztaf}O^lUZ#Ra_}C34zs~;R_pDceOv>q*afoG?KYZHX)8K#2g}Ii=(}6$H$2(_uBDA zq{_Xyv$(P92sc-ul(+b7r&h(Lg*K_?}S~ zjKnVx1KUvkHE@^e)>yLRvzBo=d-j*wZM|oYjTVn<)-wCAY=ybh&by|H>&+PNOEgPA zVxcs)cRl}WuDT|Jqd(i*h=@%M)!%zkfA6BilREQH%(#6ry>74w?Lkv^?IeR<(KTV> zQF|9wBWKP5TiNY?!fYGWY7bc=?BZ%iUfUtj=EGZkj{C+df9rEid}&;T?X@kP(7_f= z9IY-%3JadkZA~@W*N8nO~3yuFSKtvBNPj<>$`L_;v(YFTe=ev1oBl`KbV$9IYuIQCbD zo)(w!SrZu(_S*beFX*AKp~1PcPj^k^TX10=#JjXKO#AG<&iRg&4nAz+8Y@VBZxn~D z+V8UFr`92SR2A8E+8iq*LTV@e#_w}~oa|&A3XNJ(sBcTVuby!7(Am0GHp*^WYr%z$ z?Vsh~sb6q z#%Z0c!374^d5vWV9jtv8r)!Fv`m+B$o)p+ciN|wZ_Fi3$0$6Lie7^VnzrBV&bAp%( zquh$n=K=f0nF8C6wDcpHxUNeBuF2~jRs=yCE?O6$W$2yK8KKIEellrIg8GPpXufF(ICp_8_eWo%iiD#%W z3{AgIcuHBdpq#;%b&z&M=ec=kcX3Yr_S`$JO`*o(MYOCtNl#TOWwi6Hc%+}TuO%jf zm4F&yrs;0f=kWDbsJ{K0-*0>N*~-#~ zZd2#OUJz@bF4wJT2WFlFj7A`LzB4=rKbR`-9LHp#n-jRHM1xMpQMbmFf@GCDWYgGbEds3 z{<~NE%yZC@aRhvyBxj!qnpiAk%J^QVe_PK)7Y-(HaDI{LNB;XbQeg3u=|(;5_*Puv zS+L*1H8)S@G&p5e>SGN?@WX|@M62o`a;A~K2u)SrJ`-oE-@1*45i5wXt>%(L0CpN( zdkb03$6x3^iVqvf$u9FHWVOYmQ`V=r528-bc4~|EG6LNiz|0R?O7$~eeKi_u4o;qp z{+{2$REd!GMVPwZmYL%khJVC2C9)Ecvb|Z`HrLwJ*rfq{ed5}w%ujK`3H<&HpFJQ( z=8uKc8X-Gp{^e&ofpK|lh*~>9yy-GbSTH;->hMpg) z3?fqIV+3E|ojVh)Y0d*1Jy|AWbz>>m@_w1aiw&*xF7$*`MlRwgQL2Wgmkf1tZMT-U zd%(acES*Kdd+d_;^;~~A7-f%%&~?o_a}MrOx$mx6lpOalt|k2Yy;q(ByD0I>(|pJH zic!F4j`q|sy1mtw&hb6w5WzP)uWK48WyLRblEvDBr)pO1^ejH$`1$M4{AjOXET@@c zeDC2;{f6_V_~Wnj!EI?!Gtf+oN%F{QVVStMhOKXLabCibW-P2crSD}I?XRy}ezl>j zspE|9YsEsJ)2ewt_MgvwtUCH6HU!%be%;5S(6QY@uib;99mCvpD`py3i%Xj}tUDTI zu{9Z|>@st(`*N&i>##9p;R!xtLklokd=+mGc;>g~Dg7jhU~Su~wD;DcI^q{{uO@&# z(ti{m(URZ1`q=5Ios%m|c7$sVBl3k9yRp1MKE#Qvc5!%8fJcMkKLbW(iI&PKIc?8)8tmMci8TH zbc-bWj|(ZVixL+SaBu#&Qh=3!;rMOe@+}X1a&vI<@G52GzgFMj5(dYx>oeq>Bfsa^ zk+pqz%eQ{(A>8_iBk-{wG->$_fBn4{!LdFc{+q(qwm#D|t^B`R+SPp>j-PqB*PDqDgk2Pj)+B6@IgRs-w*rJlK&MPMkR?j$1?s>m*I%Y*TTS7y@ihGF7$ndsF5+ zZD!9+|72_7#6S#d>(?;g;~c@olBSM+wd3vEwJ>rW`iqsBa&ECsvv$yrvrO7nN1>*_ zBlu`k)td_!%wc+mY9)Ak#qQv7rM|%%fQve&#J2m<3GqoDnOatM- ztQ|Ln{mL%8EnDVu;Zw{&uU>M*&w!gOHV#I9nq4d#m^rY5v~E7dIl`K%Lu6{hh$j&o z3$Z-5Sz<{E{G4ggSp}xHVB(Au@kM>?)5Bwa`XziU4S`QQ;lO|u5`H;1M>N_p)bwG) ztrN=}PV-_Ma`nr6HCHd?96*H8r;m5esW@}ya3YRm=Ju^(bPrn(k8RtCzr@{GD{eKo z;0Nt%)z%k0L|b^`Lk%oPr8?FNaHJ{cxEdy1aGCEZvGAJsvhAMrH077=8FY{1T>7~6 zoGeP>ZRRx5bqDdX?(8G76_0kF2eB09WiC6KHDq2{vuwk;6`yxx5(Zn7o`W71u_Ei0 zZ+w4$&F`9}sf^3mdO8_*!iC*uIe9qnZdtWsBm7w*+hCUW*Sa&yiEc zvetTCv+W(Nbt5pB>QJ3oYxCCd-y_#C!h{Jo91D>(oS9rI% zVhx0Y=sEhHT{EV=?s^LBqQrWd?JXXA3iu?76~KCWy5xyM06eOhhFCa2iU)SWs< zi;4A~3k7f6nIHI;!dlz<{k6o4p^b3K{(k@4r+i)F%1019C$003we;xjb*{`eEh(#D zvQ%Y!)v4_|XiiXKb+R{rHPULmFR{{{5HuHw;h-{z<6G8?F{?RsD( zjI-V~oqxoCv{nf$wlx`czpkN62i3JFGO)N9W4r#BVvITvtAAqcJ#RUr;Fz=Qtp>g~ zc#ixQ9h6zv^cM}ZO(Q7r{G#aiI-dmV(a+K0T}g^iX8XyLdNzxJcJ(vPh;5ACMM z6M6 z--pBRbI{J;Z{Z$rKWfw7(}zcQF1c!H*26j6yVlrRS{~j%kMOnk`f)$1wdLyW_*z_j zKek|K+uYQ7c|^B%&hkvO^)J0PmLA7pOo!+1QuuT2Tb!=1J7;rP_wp$U?4raJRra#S zoC2*=#w*d)Yiawg(O0iajw$^i1a=F2S~Jcc?ZM*0nSz0w7Bm{P`2PFN`9?O9H3nvm zA+*KX)6~Tw%(X){6@J*94;(ajKlb8_5BQ+l^FV!8iBT;)XO^G%sh@hlV!}boIoDGT zTP^)qT++Ylh|}J@himiT^9!eA)XKA_D;&sV;4&Uz(CvJ#+X8GED{GnXSmUtQgLne- z$x@iL#Nlii%Gg~g254ZxwKiII;s#4q${n#9j6t2R#d$4rlrZbm=9}&w+Ft9OQInP) z#}GWuvV6g1Y&Axsy?Md)d%y3?PBarD&6z~<S~Irn8UbMU{lqv}??X~)s!&QanQD$D<#-&SKL(uZVZxliLV zpL=W}ziRm47N(s3-sibGjTC$wm+li!U_UJW*&HHXDSYr^aqa`3XTUDuIjm{;u#0m( z|L$-3vo_;XpXY=lkk}Uf;+*uhz@2KtAt%qDxC~+5cBfWGlri3G((@2@{7l_$tgzUp zaS*@ZbX*f3kr>Rkz4;4oiC>ldEzbh_UJD=Ao@fGJzn$WJ;Hz7Amj3tnib(i6B3af> zDdR-5L|HeuKce#h4kDO!Ad>iNFbt?7LFi(jlyM0>{0wQaUs!($v3cKPJBXPWFB zL?>GHOUB8^H+l2halqNEP>$`UZuMnomqe&@{ZCTeo+S+|@<6|qaMd=)`Kfc<0f7%K z%NB&2RTv$}r@TkdtjG76kAF?2WnbR6U;8`jvxn>Xv2Lf5|E|T+*3bGhPFvu4k5*h| z-lS&mE^6yvdTpNbo)Ow!MX2L@)N^_%{JwqO2U*X~2aD_F@97>_ep|y;#4Y1;o=aJed_9;3t@w{ZS%?R4`F>ik=G(BSVV~yk1)L zh!g2ArWsEhuOzBMhY+Y%5*c02rcn1zixt@|4Ke!N#R=M3Ozt7OZv+SdX{rl^P}DGNTq{vo^dh z0=Ijl)>op}o+dZ8LYxQL1&6lx?fmQup{WR%eb*Xg&wOFAuA3 z?X!LnN%}gzH!M~y9)0BeHj+Mnacc!XZrUcWxiKEahGy@&q3^BwPa@WOZg#+sZppALTFH#I!% zdIQ6_AWL;9@;#)zR`k@KxXTg2(3;n;6!q-I@7X1^zV!~!1ZI}2#GEu=nlK}zE~aGz_I3u1=JpWd+(JVm)~_4?7s5M z@9HE~+5}G3&QH|!ercci)OQ~B_3z&sGMiPa#st2;U%_WvPtKGQ_blK-5<8ro^N5`hq`sNtf3XkIrXP@9ZWUm{e<21$2 zmvU?}UsPOk^L!r4USp@gG6i-~Vwpbs-Fu?|7C^F8$677WnoiNUW>-#)Qs-C zFp7^Rl*S2;7|wI;+-_$z3UjWWy(Z+5Z{cTrDXYHfa{va*tCPVu{^H91fsX~Hp~RL= ze)oB2;P7AvTAJRt}6bzFG)VV@Vmm^tSHOwfBZC z0xPY`I+%+E3hdUpB6L~1HTc?X_Lh?J247&K3AG4(BU{ILZpD+ZNN7V8S`x8>FYl6@ z<1=Gj)!n|Ah*<8u^mqQiiFLLHI7Kbtt8_lD7Z$v_Oy zhCUM~fD0G2Jq+pjY_<6Ed8*sXEx;JPb;!A1lKhoV-qgMiK7nEDXlySLmC(?*Xyvge zb76YfjkwZxBVED9R)TSE*It>I^69VQ#VGNP^;jLyuHq`J;z)DrRsAjd1Z2Ms+T>U~ z&f$d5951nyQ!uj60kNoPN{64VE#rH`H+@sxdw>&~obprsnJVMU9JYCso;2CCx#mC! z^%UHxGW2M7GoBJX`gWYllK0=*?(a|irp3rUQ{kg`?c()bU z?p0^d<8Lab5qv$&x!-lqp90%_=_8c|H3;E~d-+a*ODM355|>bKZ@ZEL*oaHm+fj58SFkKTeb0YUG5OJoUEtj?7yZT=<{FF2L-F)LWo<6bGjkp$$E$U%kt8e@=_*-{r#M|~l z1%LRj|MkkA)Lp`F)js^dB1=$ftlAP|CaT>j>cy2L2DMi7e~BHg;>K}3)m?G#K)+jH zTQq${mv;U+R4C_Qbhp)`k~a2WM9ju&qem}7gx#B?XE0*h(7_?wQp#t~WegmQ`un5! zJckiKhs2kHuUb<&gw|nAT3W0WOLM4M#4~j2z^wY|-4*c!TR7MFxo`iXI@i^4ZLZQ_ zIT6U*pxNA-r)p4Vh@NNg62+eXGry|hMBgcvI`|SVbq3_w+i(&gjDZv7XB(EjeSSyM z0-Qy5tnlLd-~WI$C-cb(%d;DGL;w7a!d7l)?T7471| zNu_9=yt;ERDEn+&>vh9Xmh3m0vme$#jo(avdT`IDAxA$)}nm7T%TIEbkFJcH&`{LO#z^J_iq z*J#$DkA3K?9}xG!7rOgg5xqH?jf^k&YMj%l7xNdy3nBwt))o4^>-pzvXfvm*`Id(} z5go;lovZ72(!lvQz6!NQ;MS(9kAP_5n)j(u>ZD(Z`pOk1ItCF5iz zZB-0&u*|#uzV9#omZzokkWt9Q8 zaq7SIt^ZoZXE$%^0&N_>Eo75;|i|eqcIl;LX%uSLq~0s zj51rx~}K6egciTgeewy5L(oKIouY{get zg$EqL^x1)i#$~%Pu?i`7t~ihN>Q-TGrw6ZVBF!V6utV0~KmO#`J}^fuu1oQu6-_vH zPrtLyIqV3n%~-J0#*z^5^kL2T(2#~aZOxZ4Aqf$_9m_{v{<>OT9cqqraDNTH8dFi) zX6_Sn0UxdFVZA(a$AN>Wm^K_9)gSr8!e(s!)^_dqXqj7hv8ri%Gw9o|z4h-M!Pma` zW%te`Mj6K)QM94wloh4z+vqjFeXJjRSnIT_NuH1Rtf4!7NyQE4gl4xe?ct*jXj$V5 zHo%>0av0aaQPkR^R;n4=iuLs{?&~===>wX!F&tIBP{x(<$#6B-*RADZIO>*(1y9y8 zeB#^gYtMxr{_uw%v^ZHO`}MqaJag^9m!3k_#5JPW9K5}J8wGYzVjJD|R;ww%BH>eE z>zo|2j*Mi!Sc8mK9Gg==Kjqcw_1PXmXh*bTBRIXT<6P3>=&!LDoUSf$?(!+Gjw8)@ zz@^crsiUFd2i@YXi77`0($!%tVkHPHEU<*1tOadV+P>=7eC$(YdD;pC*EFN;uFKfM z0%!Kq`k9hL|JK|N;7Ap0XG=qC;iXT!eim2S5qU$4lr5djvo#i6w5c`zVX0AfFUe=S z;>n*_`&0kQT-*$@u|&`$s7*2PBE@)pO+ez zdsSpSt}pvFeJ8ePAc8`B94mEi@eFSey97kBpC&ILTe__T_F# z)zlYUECmmbTIK(ry}JRn?Ya-b{xT$!m`p9oT%nc)%LIKm1>AI3jdv~k;jDYgY)d7)E#MOu_F~pPh%Q$AF+|*~VR3)o)5yMF8iir&BDLbuEDV=y z^9XogrUAyN?a*D!tdm zY&cQQx$m4t#QYW5bz3)ry*4J08 zn>}Yu`szf!6Z7c$R3sKAYc6Q+#tbu}U%4Qm_TUI#pLkN0s1 z7F>r`avylUGZ|J|*D=X8_S)WIOtlZZNImC$99ma+FfQLyW2<>@19#mrzMoUyv}`$JJvp)t$cu1Y-!x0_L@1kVX~Kxva)Herac?esWyyCNKld3Btqpe)pqW4!8l3R zLyJ(t%J=4WI8r}YhfKDl&-mu2!+9Jd?dLoKi-w|{kD!azySz82&!M_GplKR^*)M|_ z>rG)_-XH3t_CA9xw%PC%{c`(t34Dw50nW8jTb*0 z#}u1_1eK~T&#zeLnD+pbliR6HZ8WTC1SSoI!x+I5tu1RUqS8R=n_wRdJf6Ir-ktOM zXp;L$RJ95-Fwj_u8OUl?`!K-ZASCp)Zfbq~)ZU}b2#JJp>ZhW%-=TUWjM#!&37YYd zW=DNLfv)v41fzRDvt#Ke%KS*}`=m(vBef0*sh`;rFIg(w?RexmBup{xQZir8C!TZh zC3T9Q_?6Um1VHBz2~)OT?~J=U57#7587HX^_$3jAmzv<79kAy}-nET+lqFn-M9S%z?&p2)V{Lz}-tHJ^tihej;`uJ%_-L@p4vA~vOUbT2qSAP81zIhi&xQAa_q=C= zyHof|67+~%Ch`?K>L4`&W-UAcfWsFg+6FvQVMaH6B=m=c7+lW@fcBmDXyXU8Ha0VE zfu2hneqHsgXRMu6fD_=(fmP1OUZ5fB^9jkBo6|VT`hicZVNC|=GVcX;Xh>pj+@&jW zzzlfuK1|Kcb0np<-c_#pLv5_DG6piCZiTctJg4aIWW*ziOYfYgTfT)hrz@_fh^GZOO7voW0IV_ z{LqS4l3%16%>vTL&j(lKAafR&$~kjD(!A;){g&+7Mk3Qa{Df!HV^4rctL!gnMc4mYGw@oY{BmJGr9 zXFZLb8@7OsWglF7kv}C~Gr3V#syt^*klUtt$@S0|?>z0BY?On1ivYX3z{7qqpfGMh zAcY_34wDfMEf2PZdm_8|UGSOllgLEllos*QfFT5OM*ooyB!ZaM2D)m%e(F=nFXZ2xCr;|C)9y74Gz?rl82Iwf`@)}l@t6JV zi+}UOFTQ#P-A39l&@k{q!@w%gY?9C>^6$R8@7~f#)XsH|HV!!pRt{`h3t(OE)1Ulg z8sBQHtFoqps9#g}+_^C9*@km5li^c6tniTVm{N32)dgo&FUEKA0_pG^Tt@pTc<`1b z&uiRSgE??wE*p3Q`^1`-aHhr;{SK8w@L~_e8tuAoyKn>E>YYfModR#3Ed?jwcIRO} zbN4{>Ra36sx%In#fceynq19;^Xc$-y1C5nf4wtQ`hJhC-2H3rymH$n4MI?n~s59e_ zVF?#ohHSOXvVqN7;!7&qb;0cRLt>u&86qN0BDrc)pXobXtHI(*-tE7ubh1Gp<#`>v!7FvR=S%Itd ze4+$r$sn6FfW{x|qO?C5L(wMH?19?3MLqAiy~j;<<$fGbTG=qrFmSbDps^BH+lsaE zHVm9O1_bkDP~w;Z&gERaxBb&^TpO88wb*{neUz~ba>l_IW4p%cip{U!m5F@Z)Xi=B zD)-#O7LbSsc?KpUm7c08pT!+7&psjrS*QMSWhywf(2xA{Kd~a!)@0#OTmCeWhRl3o zd_(p7#!Jeqor4X2pDi}S5)$sH$DoOgIvYNdBD5wRj!}>7ldj8U-9K08tSkIyTQwtA!62@NOS)ZJ#JHd$cG4C={b8X*! z^}IW=COJ!aVprpuKa{NOeeHI=msq$}px$CWvQPiW%{Lz{?OJV&VHC!|&HIKe+-EOL z&6~fNcVOS3z*BJA^M-+I83T=#xRzJH&Ank@5(8|SxsbBcpRByDz$RTAF1NL9)FV0m z^e6v%CY^Hhn2E+n5RsPDMQ$PKCYo#>BR@&SgG|fmqBc*rPq+%-`@jP^;!)JkB}s0@ zpKX|wf@x?EiOWz@L;mUEVCA^K3t!d^bdx)^_12U{UvYFV%L-1vg!;)8Li zaAgoc-fbfFp;QRj{~y(zOTNFHoR+#f5= z3oLCCnO{X>l1__?6y((nKlp_?BGDL}cotfMRR-L+d%7g?CY27-6)fygL|QuC;`ALF z0Hia6BW%U#8t2Zv`tFw=aj|CD6!YPtjg^}lDJL8CBs^jjy0G93X&q6Ouux*Zb5EWN zC;B#CQsz<%@P@a(^+=?w9B_91oHSa*sA#uqDNP!faRl4oIvY-axZ$& zoObH6lP6J)oC#CyUvvEVN2J?WiIvOgR10zMZ$1CcM;|>R@h4H&Hv1S%H$8_yCDeoUj4%HDs6HOaJ?ZoXspoR2(RC zPJ=YRu@r^KQNd}kVBs=dlCAP9Fto`&9s_VrZm-DSK6qSu1^>)$<^@Uq(?9vOIZp`< z1v_>|WGDTNpV0^yD|qnF2+5i}^2oPbc)q}u6nxmKGF{$y&qGgF^J4@TENF=YVv(0MtO%D< zb!B_*>2B~=ju7;|Kb4~)Y5(#6=f6u@kZ+spgZ;PqKlB4%P|dbrt-@zSWF4vPGP^Z6 zx|0W35b$VBYV7D`#}@cgo7+;}2bzSF-lorJFQjrqVh`biF%u04gB@h|`x72-yyK2p z+K6%_5uHj}vC=?|0T9hd7;c#pvV0dKy%HWg+RwBC)&typq||$0I9xO4!iJY^_c_O6 zaWRkmjWuE`M$aFqu`2+RHgY4O$Yh7 z3n|Q4%6PFq48clxv<9y^KF%5z?(zH!_LZtLB9aL?`mqlf>d2T*3lVFyRQ?72IHgNJ zHWUnph(xp%H7@EUUdGO`KW};S*JM_Z=7=#DSX{ho*u$~D`<{Wv&*Z5m|6+!NEu^u# z>&Yj#*0QhdPWO^VDex!9#GH427kF4ek;5K){P8o@x-ZhV3(c^bk=hTcprZMpGnb& z{`9H(edBd6X9zF)keRt;m5?~daO5Hwaj1v%W?VJYci;V+U-R#0ycj!8L)5|kFf8I* zl8tmybR#9rg!z|aiZ(SRV;p0q6^i;`A@yS}O0eL4UosWjhaKk1j%~#jVIN9Z&Ka?Z zR25^|r%b`Kw)&5K$4JcHFEugyz=^iBUgdni^DZSgi_9nBMCn_BrSwh0GA$joDd4oS zVW45)Y%tJRiL=3TgVHc?Kn!rBJyiDslQM!~pmNKrzvKdU*M`ylF!=E&&@E z8j?z787as090Q@6@no_$kxa;@OsXXq$VF^|c@9NCs_DC1hdXZ0Sy8}y@ZLX|N%wBu zp=);dTyAR&TWDL%i8PxgrPfCSY_99y_xC?wU$b%bqT(jh=?@iAYgH6)z>WQ-m(> zM(in?ArZCalH|{S?sGZULmz8weMC9KVg}y9!$Q{VjlS_SOv5fYe)^}rKD+mWjopk$ zXJGK%5unz7-bH&=!hY~4{?Gvvy)A0vC-D z&BxF|yMevy%Y)Ge&pj*%)+b0@Ft9389cHaaXYZE#nAD$vS53Z@fbSjC2R4Y&n>8)_ zlWD_ljs6u^Fh_q$s z;yKAJ9AItc1j`i2mCLcVBJnVm>2n9>e<6=RbF_m1=AxwcWk<|KSbk z$mayx7JbYqxd?|lG_tqbR-O^s4)bS|kA2%Ws3{j~{gwGd_-4>{&<=Pe5Uo^l-SO;l zjg>Cp+3TeDkFy^?z<<3#=}P}8*wGvam)8T95QSi>ZA5P|CXFKGqH~}T1jiX z$o?CGAO1n2vF)4PeF#Q*pGMEo@|=`dxwg+6D{*bFf1CYPivd9)85R18OGm}CrRPtIL*(~b|o#2-nu`ix+(nMkxDcu1-2Td|%?2?yI$ z!>1^NL&Jl`D`5z9 z%U1~7u@BjeMPb{oZ{&4oVM>p{*eIE?RKJzKZKr2?l;X`zs z_U%%^ZA)0rX_T;)4(qb3-%PjPcHJ=0FmQP=&{&DfgZBotVPH=TU?WK6NeuVwYO*=D zCDL*oSF$eH5b4JplK_eeTvtU^3ft@&DKApBv{5g*n>@{dZP1BPU5yhUMB z-!T*+soM{uBXyDXq9#54tG{|Q6!ixVNd4aR@GCM5k$aHrB>S~C>ast_7#YBH$5UQs zx7YQ2NI1tLOqK289O;``%+$y2|M<6Gpk0b*Wl4(p=tCD)Mn0*oJO(81Wu2OHb3^*x zdHa|5&I_F5v`<&YPZu}%PI#CN%Xl)InZ{dY^tN^k((IX7Cu?gT&hbs@pYO^=J>YSb zxBju7O6eSTF=Fb27yc@b{M?_+-xT{ZU}2wV3WwITufM5#+3zXMAz&!6XW;Cg;_ozc zT3h=XoMfnV;r4&>I~Q}2KMielC+&}%IHM5M5=ewtuL#+roC?I?_XvTa$`r6OE=N*Ml z_#y&$(Qo|v5ra3{Eca#IUecI~VW4lNzRQE33}eqjXhiI3zAGs|M?M%rHe;Z=UAwy^gA{7}@oiR@h zys+!neuz}(r;()e8(1Ix_{WnU%yyoJ8ZC|SFoZF;wBEfWAe(BEZFP{6Pkr>Gv&s6H zH|e~6qMtr3gLMsiZY<`M_9yxsDrrsfxD)$W*0ImCKgQZ81+HIn{Q14}FWXI>la_;b z`z^n11J9gkMcl_4iJG+pEMoyjzqCpGfxwRlRB?Z5%yXUx883QS7|4Nx+{Yd+sgW?Q z9P~K#4@MkVwp|U~rshhUheZ|~`1#~F^u{~yJeZ%v)L7c{hJhC%1{y2zLR{50rx!2= z%HXU>ZPAtvHC@3+W4gb+gurKSehvXH9TctXwTT()(;om2e$yu-e9fw2V0q zmlv~7hb+l`b+Ab6<4I`)M(isRZ|?FAT7<52ystlP?UC{sUj{Dg{9v*hcAA5F=0Ve? zz5vEUV=VjXp6kcBNz1{p?pW)@xAiobgZ9rB$`(%44ccd)_e6FmUB!ps^BH?%K4GH4JnxKq5^- zDObRuTp|M`k}C{EkQR&t;@Q$ASX^?!K{9cbMXJ}pLEioRlTRLr6HcIPM}7zXw4tq-8<`5+3^Pg0!MyDaFWF)#^7A0C6>i!>#bqnfEXazC6#v6 z1~Mc8eCWZK?7p>AGEx~Mlh`72 zmu66t$+)D+z+(0F-FHtau{_2^p4X5`U}ZRIeB49l*mCm^2ty)9M1A8a@8w{uYm$)* zbFD#B^g+vDtc*EG>EnK1X>VXX)aMVgi}ab+1luy@mjP$k6tGC{YuKW`F&i(|XUH|2 zo2vk~E5oykc^}GI(AQVZJyEp4>oeR+u(Hc_?2X$JTSH?})9+B7@4fCfWgH2Nntg}T z{&~@FeBTjQ+5prlQqF$72t!oWF;3DRQ6o${f$UE>C^`g{v zZe%KLVx0NBoc#J6EM#5OQz9vMAsNg0b@7NTl-exj6Qwr0z%u-DeqDGORxMb4pm$;Y-u0E2=sa$h z4FfMY3^Z2a1-GPaLN81V&_ei0XJ|B6Qau+VT#b=`vP=6pBnc*+WJol(ubTQfx=#AJ zXPb>2B$<~_!cTHd`?+=4hMUvU?xaKG_{9JHUteH=HS~P>`xmpj?DGBg_lP94H~XKi zQVB9-qA|%qj1ws*=LHv7WgIkNH3m;o{4o~eOXJ|GE-QLmcYP(0#-qp3z}NR_2Sof~ zz~!)>j<^<-nePH4sIh}lk~|s{;7&chl=7xHPCaNh8U`8$8V0Ta3^Z2a3Rr|Tgoc4} z3^1-hD(&meIS%FIDPIia;oE-Z8wSb4liK}fjFyJdu$3)HRs~pri|7Dnu^hp8rwThfoXd)#gsAh1Kdc3^WWJ76XlyI4pi!?+pVB zV1Qk*V}Y)!x?pF%O7B%weSu0B&lH}yktFXja8hS&SM4uW^`mZW&{$(a+94ss-kcii%uhk>(#Y^?)thlCUJwD$Y#f#|1| z4Fe4WmmdR-mAL$C&_>WOFoFR;@zP_-wJ;TnQXVc>1Fza zizdS)F5AZynX$RZ20SHc=roj7jhk-0^(g1PSdUlxskRKZN_Zr@j8n#9+P~v{?>jQC z$mm?QpMsZ3w8FqeTbe2od`6Fq#LDr` zJqM4`Sq`z_{Ymo({K1IW|nk~cPt=7s$EmaWw-8iUyS2qXto~mJGc(KDFvRbF}^D2(>=FmL})7e zjdOb(yTo@%RIIt#!n?5Iny3M5$*2k}YhxcieET8?_Z>a{z}FoV z&MsqXZ5jp|2I?@-Scy73wOS1W2fzR&e_xQv8c=?Ek>RzGgus$Pouk8KV zOlzCdpG~wEZ$BatafO|zzIjYSDK>wGm5e%+Q{;NiV4Lcxd+E97s>u=LYe{Z$Geg$}|WyTAr9(mewcy@##p?qF zd-qzWA^g%r^-R%$AOUfCB3*F~q*NkD(Pu0;gPd}a0a*fp`5zYoC zYqkTK;Cif0JOhUtb4X*2mElD^FH7tV_uH;f1~OIdjk0dvxpDRx-SojjnkFN!4epFf zmTX5F$0RK9yGB6pf?X)x(iu^4tuKwJXeQ>mC4*Yua-2M2=K}6SNoO;->eR4gN z^UG{d07@nNA;DwsDY@A1L;93PZ2=EkBtUDh%KyG}`ul()vf^;HngJg=bOPcNr`}q66|45#J=s!x4 zW8V4hcl(K)l(BqqEYACH;PZwLakkiJC zO~J;-L)ss%!^ZA8ss2N6|D!owVxM*^+D(-aHAov|iDt?+fi7 ztpsB#_hr4Fla_G`Chd(r^^MgF*fc!Yc~bVNKIT5BZP~y{aJ{$BKK0Zh`0E=B4WIpj zZP_=XS6~0_UcL zXyl^$*ebpZwQL*-HqylykuyVNCbH)>*I%E>0(RtJF>s>Uh&k7k*a)!L*AE#uF~)iN zSATW0PJAn5Io?HrAMmzLXi8^a5Y9`aE!{7#PjJJFT1pLgoLDn$f z4%uywl=V8@O2@cs<=`PnNw|mnR?Goe#l}CfY?tFa-T(HTO!BVDX;OI_pY0ka zSJLw_Hrncc-PXMzXHQz3KEr~Ap%m7oi$juU209^4@517v@%^uUHKU2D^)mvwNTMHF z4);jAX;x@&;#pa8`R2mB3tNFqi@vNqErj{Ry(k&X@!gS=WgksJ2`~DN(*Aws>A#W5 z{u&J$+)VvN{)m7Pr8zF`D}hzA<^e-#|1hrW?wzS|*F47_m9TEQ^~IAK!J2-D>I7>B zrs!8w+Q*@OYVTtyi3CIqt4X<7+i$DaFmR<}ps^BH>dLfnT}2oWl##^#-uwQu1KZ%H z^i@d${L?@6^&4gV8}InajbwHylp%Fxw%$ZXN|Jv+!wggIdB;0ON&C5Dc;lV_VNU;5 z8Xthg0OGn=zH$;4+%t;cB3hLeEcIQ&!ZP{UpRzkgQoZrcJC7cDETLaqo`Znd)LFS$gb=Y;PPXV`M`^ys3h7_wPn}tH13S2VlG%r#$MP(tc7EUZR~@O zB?`YBeP=268cvKOaEH{#iud(1=N>Y?+jm?m;rtEv+_Q63s_s~<6?RD9F_(bHFQcp7 z>bBqIns#dCOSSsGJoD_|T>Sge9am&7=T9{C&Uf3k>xO}bfqo1$R-zv-?QX-s3=ELI ziwrW?SV_d7xe-O)`1;ppH&#FQTz_kZiPYi?u#r60G)i?CykseOg2lL^#-;S02VZuS zWBsQ#$W#*H8gN{Yk!a=x)~oM+=_oly0|b8&U=*o`w5oxnPDF2FlruFiMhaYojm`0~ zU;ge3;Mc+mcx=z7_6XJ}uj?9?fShw-T}ql7tVq`Oj0_Tv8Rx{j7zFKNq?3(^(s;+< z{o;o{lyiW9MdHl|ri%6J89x@2!HP`Q-N)Eq^Y9$2B53Nq?x-=pl|kJKA`OQ*J^Ii7 z*am+B#4h1e(1k}6nPIQ@#J_&-f6jrHmy)grJfZ`vACW=q3HO#xS(nDW(I^ERMus=s z@WmGnVGtAnkFiF+6SBE`&$@eOscU=BxAu64JSMx0!CbuayR>a>8U~iYKw~AAz+&s; z8piqJ4H4`DMY_DzC5RUf38X>XroUjy$wX(smqS+V{lyqy0d`GVEAdnaqc^c)jp)Q!26r_;88YMs~Kpk z(=gC5Fo}W2N=%}tRcII(!vK;&nx$u+efDUs^mwe_zGsX_Qt+k?S@G|ld1ld{_Y$Ug zF&GJwg5k!TIP)aIbL_0M!O5DG0h7CNIi9|AQd!xf+5Km?-=xjxt5@D531{dMniH^) z>{Fy4ABK_z3A~hj_nl+=Hgyv$B=DGV4Ff3d=Y9SoHO6eB#pK9FYh2#(rZ)}l5iCG< zi(l7`al+}^-MdQjf~Ac#ibE`c)PUi?5pWoMVP)p}7tvrLr*q)o9;Acyfk(e}L-vbA zRk9jAO-66QsWH=Vlodr+y7TtiS5eN=c#zy8kx98$)|-1C@MOJ?wn*>DH)1TA1xs4P zc)m{=@MsJ81qtgw^AYQG-`y|W*(JX_F7H|R8wr$}bBwI_pE)sJ+b3Ii7hV>(boRG- z*%SO{y7l+Hrmec`$9~@icVPKG&G{D29fQ7U;vC0G6LLrw_PJwg^{)*KG*;r;Sm!p& zgJB@q7awIQ@D32zAOHBnuc-bxV~`?vsh{~;*Go3!jEv+)AO+5L?UiFSwy=XI$(w3M z(#Op=NDyVj{hw;%~mG{&rD7ywAy z#2_N(BuZ2whbD2+*PXl(`r&Ha_nbjlLtRi?XvLCwwmbJB?klMQ%H^n@#?)nQHVb74jjL%{Zaj~s^Pom$UlD~lY zG0Xz9#2pR64L|sWqwe%W{r252_rZqjPIF~!C`#3?`>zj{dpRR|np^Azr8yo-JAkw< zVKHnP8rw4WS5e;V3x=M{^f}y%eaA3NRJeUy{f8ke^JGbuZJ80ZUanRQG*;qjUAZ>y zD+B|g)lkfz)o_p&nIt^}fczdr-^syU&j8+A-uyM03}oX*UjAS?RG#<9`;btx00}X<@0ts(*IB@}&G1hL$wH+&JU1<&`x~h6iI?o3P+lq~) z4HV_daR}1XxHBtQHM&-*VW454VPG2rjg{EON4shmn1KO5?X{%ZjBW?3N~>M`JF9I23x zg)+~MA%zY5N=lrJdh+>L?IoV2AdLdSrUNOuM2|y#Fir_r@Htd(srxG!XEQfrX1i~G zt8%GX^!HQZa+iHFmpmV!T81lb0da!BQnWkhIrEd+M8F@>$9G0{Y;wk(Y%!uWy3Ig?ArV;0p=R zw|?$(rH}Qc@mj*jg^^Ns{md)&Etm|H`fxLLZ%AtaUO7L$88I)pD#rR8d)Bu;>UZZ? zt`~Rju|9WsCP=FJee~lWU+s9Mk5N+Km>5uVX>r>f?Ld|_;}OMqL(PYfcEb&jb)a6t!; z3+q(f^7*9{yPciA=tkN2^DOy<)Sb=EwazT)~lYqFV_(JlyZVlZp8u>G@xP6?5Wmz@TTK$MM^1 zcz3?`wVSbm!PrWGTdwlpnA2T9@>SX0w#H~^5a=47_W>&@$T35$m`jw;KK0b>=vRF( z%Y7DoM=A5VIWFbC4nBi6Y@P^S(g)J)IKWvVFGt@T-QyF|XFNGnzuj*bXc(x&Kw~B9 z@YHHG40JF+O3LPZ9?8||r1FwaMwYN962njLQug~p-~$%Y$RS$IKJLOpaN@pbJOL}p zl|(OUEL*boX4@XH*!A~CIa(5G32$ZCQ`fubU|2hD!~7c@H7x_4x%EZgVn8E?ziiRr znyf5I{lL!@JiY_`6x4t0@$b06{~?a2KKju-_A)Znrr-7cifim^jRd@JzeFE3r6^;J zhgRCAso9-7R|AhfhM=nE+dbxz#}7UDvd!M?&gYST{wK04_S`XH~p4%8`Hst1(&#(rFGn z`46~82N@J(S1yb3^q;Pcqblb2cW%7#yvRgd(wj@JwI9n95wIg5ul3`i8~4jn{($#A z^;eD_`T75RL$=Ba41b9=m?iCw{Q?(#hCw9SjI{!1iEYbKiFS426=^E^;=&#CqRHXn zEIL#?=f-ZXQD>LZ9upl(pU7;%Y{4x_e(X2@f~9iHlzqosltLlMd~KiXGMAw?U3l^? z+?HJE50ww$hJWy)92l6kg*M_W$b7+16P%4^X$TkXe#1b+z&s2zR$?B$TJ45`H3p7T zDxf$HkCa4;vejQ7NZLcfT=jnf4_tFSB+TeLU>|yp0dF<0m}}fUTzR?RP2m%}YHa1< z-ds$?{8r_5YyHLg#$GPVsWEQ%wz`V`0-iGNm3{Y}N1Ihowf|RpWeA4*`i)~39L7k7 zV3uKF(5?Bv-Y&SK4uGd>RiVY^TBJ1bb* zklR(mzzGH#D{+F0cGfUZiGi1-xc$6b5KjApr2#yc74Xs20S+VV|{R`&6#<} zDpfoqRR*uLeeY{V3&5Z#{ao)Q0X|M-1>5r}lIbmX{ehgtcwdP#%|cBoLfRnp=H^-G z;;I%N!!CH2wY}Si>%a9Ye>>++=zCV)tA$6)vf9UwAMyG5g@5^%whlpMN#D!9`_2W9 z?K3Rnqa%y9EZF=$x{h zoi1=eGs0kXS8dq>Lug$z3|y5MXspClxomCFmlp$K#@E<%FEvV;c9AKJc$VsOyq_e+ zOU4vu5p~2kB6BcuYkXY&PX+Ykk{m~Wt&_WX?HH@ffQMZ}u5nR~4B(8;YVs2>gp)6W zyY|T_hR>vnOC-(4NXJ_6eW=gxkx0?x3Oo#9^#mOALE?rH4LA&(9)IBLs`nal4_P~u z9U1Q4aL+wQkuf^vBO1~iI58i_Uf^+~jeAkb2!Gd){l3k7z+%KDm1NYJTaq+)@rkTv z)D%7`;Mh;t)O`)A>`(jUII*DET?TI=D#aSXBY!D5Lgvn;uYSg38h`X3rS}pXq<>$# zUGEW%SSL}bq%CxS(~OUH#b|3s%p~-N_pf2&>@m<-iL=LfgLqYAfGsqs{aH-Yd z*UoroO9GB;;j)vHbkkh!5_RpJ?|yg2>D}-Bw#|M-hU=F`$^7|+&>k$|)4(;^__O~^ zn)OdzfBkvyA~=kn?BhP|K_9+q?_tM<26*G^Uw;%@B;~{R-#^P=rxuxOfH?a|jI)?# z8E1bEJaN%Ua_9W?Mc#yL2Tk%=`Dcs*kdjb-?{&XvvK>Q(oYgngXUwDB9L-IZxNDop zmXdVp8?K+^h_|mkvI^`uc@dE2#!1>chR<&P3VAPf$#8Dn;3UYjzIlO_$+lFF)IH}B z>8n_A4?Ob|u(+=}79-YIy?ydmSo;`vj@@4XBzZRPcy9i4)~V=fbDCYdQ=i$Ddvo{P zIk3C=5cY*7>_Cw1+1+~$&%83MwEvXdc^AW)bsGB;&b$ZanprG3T<_PZ_hSyg96Wiw zJ~2yn0?t<1ZP2-v zzj|_h8`5HplOYn2Y^Sl26Ikuqiyxigo(%k2MA^w9?oE0f9MqF%rgwDg__Usk>L zW#CfAH{E;yq}M|@WH=-mR^L2^?v?PcCfB|4m7DegiVne&Ghr8!xP%8s@VU~a{F}@g z2Hwi@)1UfpcaB8Xg?H(>=W>?ovi~@jXN1Anob~GBl_oCw8!KrK=H^A|!iwwCNUIO# zCqMR=tG-py-&C1>EqeAk*uIPMWlZ%k=Q%ASjc0dmU>(Y+ia{T33z+tW?*Jp5IpdyN zSAb^`%U<*wzrHB`=8mJ)zxFWDScz+I&9C7Zvkl*O`V5>dRm56nz#;|HzWEp!71X3% z3le$%pT>(>f;`nv7{N@{o5D9ca6h+G{g@xu)_q-|=fJam`?|DCSynqgX}YHHHT4`k zQt~9-9^C8PSmBe6`c!{Y&+P-oJW|aux<%S>p!u%W?MFaT!=RTKkyEp4C+h8E5R{FZ*7NEbb(mARPH@Q}qgyXCdz*mM6yhwHL5 za06J>;CJ2%9rK3U>D-Oi*of|-MQTNb3J~0rJRpz@843cs;KI!{LQmGvZZ3juE zz>T$ScCE+llBCy9J*oMeez@b!aoyE7_eZdh6eP|y@MsQXX=dD#jVEdvr8+b|B-}`h zY!5%tj=__9433P^k&U`0+QnlTgS8=9k9&P3S5t;U$mD=yUNZI0afiR|`Y+1uhtkYh zA8F%Az&CKx+-MKRv3-aw0__4*0<)Y;bpK^bJ~Xu>%#$dE3+rbB{|&21pL-wp zx<%Z^OBm%j16HiNJu}x(wCtyyHw-ik^kbm068(5-cdvR3Fa+RdaOjC(Szh#O&mVpE zGyl_t`|kNqx5%iaVPhkLtiVPbdVUe@L?oZSs5MK@FB^X($)unBU0}WW>8Hxef7$F z#`n(meDj55Gd)>XJM1<%58nF+Gg-&yg%Q|~Km3ZS_H6s*RhS~9lL6cLd*1PmZ1iV7 z;M{rpm!HyLlym7jX9V`Pcb*@_J9}}#fw4v;-PTA1CoEcDf7ACaW_%bwT+v23_&v#! zp?^VBNxy3hw5H%$XZXRM1gnxs?sV^oU-)yGt=u>7NI8c0zxTa4NH4>pI7DrtNxD>F z$XWK)&Km|A2ChvEG*;r;T=#2W*4(I* z!`&NuB?m_hyd2DOJx1d+zpn90*$-GNt<1%oufe=(Vd*EkmnRq*kE!I6-e+9FSHhag z8GxjtS)k1*$5L}%u7UaW!O~AevdTC}^}rg#c^_0&u2I;FfaWC)rV(qf) zaHoy+%6U9eJz5t=AG>M|yYK|n?SMOX+dAN{`PaN7-S*`*gDrtxW3QC(;F`aLdqaDu z?5p;?d>`VS#bU?0x;M0kYGIW1N_Zg>VMK{-<31c?DBJ9F9`I;cxFf@{xq%~P;;wsR zXiUD*_Cm>q+9N~t_r2f!zI2WL_U*rH?;91lVSIbNUpL(F#Rr>P`TmvoEXTCudHLNS zCRnoXzP^0t{VvZDP4)G2E!}IZ#I>~A*VJ6aDR=Bb_RT>QmL&(?OrIa0pg93h=fF{y z)DvkRQc5Ma?796+)hChcX4ht*A|lUJ+ufe8T)p$W*3nMp8e%arlb#4mH#LTLyzhNS zmp=PF8IL#L{AH8%%l<@{kq<#spje0U`OtYb+)C?}*n&wMA`R>#nu1a??Yx#m2zXNl zOW5tue8haX8cX*_!ap>h?tNMw?fc--lzj1L{^k)wmVlvT0C&@^w^qX_VVa-Z8D(Z- zaIh%dd=;Yi94Ao$_PFe$_8c2zzgUlGW1h(1az1h-Shsz@IWj@(*ZgQx_-L3zz>0k^ z_Z^9`cS~Bw=({XwM!Zus?&94($F6Vt;oqI`c=FC;!+hJ8nok!tl8|fl`ycz$Nn7`p zga0OU;kWCCffqUk8Y}TaU*T(VewUJBp3w#gGDCzzB7xCSj{lz|UZh@ru4>@%J0SHR zvYYSg1FVRs64K@=)&+Tm-9Q#Zo4&rg_qwpKFe2WJ1vDf{6fAQCC*dOpMC^9nxn}#` zo!1nLM5@VXsDy=-t8sBm_WS3foq4fM&r;IPxLne7u>X&Y+|iElS8dQwY3QQe*#5u{ zb%Up_$n6(T8W5F3&l2Q>dvn7Ume1lGECxC;E+t9aE@W^>%I)XqzbqMB<$M|mM!>r* zNrYuQF6-5vle+4s@7+c=mh&oBcNQMB8eOx<7fvg|04`{Iv}p!KyV@;yw%xx9}KmxBu=nse9B_vf=O6>AOZsGO8-uCX#%z z?d|{gw{P0WeKAG8FowV;Wz9(%A9zSE@%XWLePb){Wi3N$gUp@sdARenug$6IVou2P z6`YHca5gX#8KXVT#O7Y-B)P_C$$r+dtmlN)--W#Tp&$5yOgi0u+wVFB&wIi!Zso_a zBC)s-f8agunOvvNkT;h(%t0FS&p-L(Rv#h&S(m^aat;a3&3w~%myL0L_=1*g?3y7^YYj_M->}TYB35S2y9AnX~zWXlL@Vtle*90u#6R>@Y%&COCh8J+6 z+*Ko1cHMWLQ~j;?I9OOQuy)mo^}#6b`K_ZxbTH(a`o5HH+j+yl5*TQ##1dF+eOy5p zASod^kK<57Bkd(=^|Rq8kVH~BBn94g9E_-tDtr;ZjpI=8vV=AL3?9)DPhj5i{-Mr;oT-U~wZs@=f7ZfF+L@He`Q*?(_mK==A_(f(xYR|a zjluJ72o4(>5%uk{Zn^S<7=w21+|c~YAqS>i@84KwrFRHytj*l_v^zi1w)7S6rFjUR zDu63ZoMS9&`}pk6oim5r=STkh3P)*Tn8BT^Q2mIrg0hWWY z(LSU@%`mfDb0S&%+-D!v;OfCZVTm(l_jGPBR1WFHu1TV#FA zi!sNc$%}xO?D%)WA^U%ZFSSNy!Iot^{&M0y=Zw%ID|aTdJLGmxlT){HaM$(vbsySG zNPUJ~@DP&jz(1IAuI#q&yVhT{iRU8mco#{}<5}*n+kd*o_Snxk(oS?RQkL*y9gw9% z{mVA(-HhYT+i#ymDub8TB8~M_xn3W>3`=R9M3XCVR-S{O@Vb~Nfy;AgOvf_~s0n)lK z1{lheycE31@Eg)r3iPG=?&~Lv9)i@{CF62mzkT%M% z7yuo5y<*Nn03Zdkdw$%kD3LvkOSr*CStC^qW1JxnjQ_A|kqNjo%E@GMur8Z&sHfHa zU`4vM;yv6vUz3orfBVXzjN@hBF<$rVP4mq#CX9^WFz~1>`y4wDV=?>qe9k3-?1KuJ4o7-$%n zfq}+K%s^JFdNp8xtD>K(GDzeh$DLhx6A2?%#G&8_4oaF0mPkxU41O8r%54^0QHh3J zfRQ&ObD2b`7$i1AjC^oT9Vu8Vvj4*${PrBJsvE^4XSt?5S+}hKm#btc?7EHYwCtv( zT9XYo`ns)%1OJ)7ck$&6RI;>IyF{vId+$nsA{Av9>eRN3%tPDIg(oFgtg~mNwh2eG z*E@yG&vO}qC%Z9nRMe&z!*#Fv&4c9D*i>cV=Dx0<5+1xpAWq}qo~(`%Yw_9p?%OG% zO?J~y{dD0;4;OQZk^#~jQoe7VG-T0VIY01XeoCy#zN9D3X1OLJWHC5n6c+EJu{`ti z(?{ObzVX)H6U_{6Vs5!C-hsGpt<l>RrEPj!lPimLw(G-?>p27hjy2{zh?}^ ze1qonJ5;v|j{%*4%6=cb*IdlGdJnaqd!JD)_R=9bU(-y?#bB#{HDaK#5?AA@wJ{H1 zK)N_&=vctgfu`q>3);?M2_o$nW1yja`|dL=L8_C=X8ZKMNw`&c?2vNzr$-l}c(T3KR0F3rlpWS$6hm5ZECGL*V5#=^2>Z z1vwpKfkUvcXg$?OZ$0zuvzxW+i(W*#5_`#*uoBwr%I=4;nD@@UD+kB_qYQkE<=sDM z>dLl12OnMo12^Y+NpmqL2h9lomU2A1oFk|87x1vX-5|jXw&rGP9~w)*qlJ9_(xq&W z7IRV(9qbY$cnV$#BhHcHL+{5>f7AC*d@}+LF~#>>|3ybjuwz~Lflsh}d+)pZr8~3e z-M;&-{VrgceM6UG`TF3rdkq6a7-+1-5N6u_s}KVz*k1k_7j%y_Xjf8Y&eW0X$TmM8 zp&^*leokFKv10dwg`5(Mc24`bb&=*^k;p&y_~Tn7+f-k#IsW{kcc#=+enzM2dQO^4 z8i|zl6JOJ&tdo5o(pYnjgYv<^IjOz5(O?Z!Idz)`Mj%qKkO|UW)f%*fmY_yzZJilK zA&c`EQ=nV9FW~glpayR63CR8y1JC2O$UiyAaKZ&`3(Zw>Yp#Y9{gD0+X~on5hsGs> z_KLN?Yw9z0=t#6ac;3hF`}?1_N$rPa&n$Qhbcgb~SH3c5aWB^ndxvEcXx197QVD); zk(mC*?_G$Y%D)jBEwFq8cGVt5pJhpt^ND};SJS(CHy@>qrUmX2o_A*XN*LuiHr_nq z@GVQC9D!(OePhZ#$In;k@C0S{jXv~uGY3;n!L*lWiNWujd~Bj-#AQA^oI|!HH)KYT~DN zZvG0AV)-4kFO5N>@4hmA?zxiAkx0^lAW6aE=2(-#7Fm{EW>cFS^`~z@J@-ES$xm)z z-FyEZzEF~SJ=Hh2u$qC(n(Wb1K7kcDyz}-ipB1~Eq%h!3xqZR~O-|ImEg5y$Yn#DW zoG^ydo^<}E*L}s{-oysck_0R%(Q4MKI|Rly`mmq7&s?vJNxt-a&LOZKmwmEpNCcWZ z@cQ;L{9zqpPFOx{Sl_s6??qo-jSJR}Axk`$QzEBYtnGN9UhXd$>_xV5q%RhUhQ%}N zSYl&(Gm8&&IdS{q1@F0UCs@ZRGq|~cMU3FPIMKnZ$NJ4iZt43;Tb1XV?qNseNSM`G zMP>O*%rD@4`s4r2h2u2GUG{nfxO3iDer!U2@T18oF>4yOIt>F?DFzxVag{Dy8+IQC zkWHj$htBq?f80Z+el=G{F2bKVj-L~ESPhBVSvfE~AesI{GSM+hJ z|L%#gUr=BZe#fW@F@XpG%{ zkn7P-*$408`|m%>cIzEj48ORQ&yy5y8?0a>%Q2Z3SZw@_tvrx-W!N z*Y)_FfX_v~_8l+B5t^I8sWtMw;G@APE$)LS3;VvTQs4U?Fv=3VT{KqZ+?;1oLuVh9 zaJ*;s5Cbl(VYmJ4>(2U>uxMj)cKF$y4mZG(moS_Q!px}e5Wh{{9qf{@4vtIDJ-3sg z;_!^xlZJtD3^Z0^96#;pRfPc(-?#nDHykwUbcv84EmzX&&sCAyW&643*-d!;6I_$V zYi|GD`X!kutFPzm&XI-OtGo60y(W2$pI`jg&P8|0<__E%2I%{Lr z_*{aOm>AOP@fzNg-&9p!6_se1f93!CUk{(-Vm%hdd9}uo6~k{;8U`*a1{y1IS<&Bs zFN*<^NyL>O#@u{AU^L#EJyj|9?w#};E$=p_J zcnSrlZfFNDE8MTz*Q`Sqmo@PE+r*ep4Q=a2x9#eBtIFo0oJ;KGvaG>rExc~MfaCt^ z*}1B}Ro%eF>37eVhvro?pVNalfmN>E5Day8gAo?7W`-2U6|k;0@i}VZe2ADTs0VIti)BbRBfQ+7e>%~3v+%##d?Ip5!7|6SuFiDd-APQAa@_T5X#_Lj{u`{E3D`|;ov1E6f^k;ZAP z`53Hy@*v^iyt)jcPVSDMk$_5bwSkk`*XB{6oSy@aK}W*l7A!_CwQi09kL~%`tQe>8 zQ6ZHs=fx=Jx#a6I*0y{W@MtB<``vRA<`NEL9Q!LrNa|J|xt@Z@CZ0<+`|*GWHtmAw zXLF2=7@%ZV_q5)9^8+iV>5H|BTUt6r52`C?ztfcQ45yTo748yug$Sao0UKHYWZUfjcGb>QZaf2R9~IvZQeh9 z6&Jea+h9K|VtBUDm|8cN9|Mh*xcqC-Mz9D5eA>iyr#N1u!--D7@>9Vu;1EAaC3qxK zChHSCB+D)lj>`U~&gFQBpF_DTrt0@S$HtHZPLb={2Iv} zgUy_{P-AO~;SQb#ysi(AK%af^;9Ok)c*cJG-uv#Wf@Kf^@7brG+G^7Wk6n7XR#;s@ za0e&T^B~}L@mazX1&M1izXS6Um{?S+xv*B?lhi)-Vcq4=h#`IKm%n?1v%p=H`(%c{ z`Q|U%^vkf%dy!qu*M?~UL-|C?;j(W~Q{=nvSlMs)9N8}i#Zpo6_q_*65j=qn=DTo3$WGuHYjMo0j#6V*u zuFQ35qneEYF4vhfo85AL1CdpHp4l?bZ#}<7CNtAd0=w<`t>?Fh91o<{OGj_%j)vr) z#wAFXxzVZgM@q(?yT-*evnmOf;7Rj0C%`H=S(;Bmt$iZ%y7TE?V*^8aUAum?Ycx9C zvjZ z85!S|5YsFOq|671t9iF=-*u4L{4yf4828-J_GKKW@sx9M{?LOj%Z5O-34Ls0A1wDG zABaAz2bL??`YsID)`(<(4vcNsE4HPtZ}*VfGCy;zebH}x-_fO+(Y>}|ojXTEN^))u zzn6Bks>RZvEOF6Zv;Y1vANf%q>d__Zt0lV*xbJ5?;2? zPGs4JT$>)^el^x4V9c?mGi0K`xTL}IJ}BduYL7(l{x7xpT#~ri34W;{az=;w(IUkZS%TDPy`=*Qq#qK%4F^{-b9`gIv zH=cH{VW45)>cK!`C9a;eY9k%R0GnU7+HsJmvH!(lj5gfip)9whoq0*N%*mEr3pz2YW^H&QC}@4F-D?KCC2hdRMnjrS z(#scNtlm=37y#Y%BVTo4Ut^TotnpHuF-oa1^uxkD_QZD{@vWiND-RKs1kIengY~=Z zhktkP{AisRSHzL~TfZ;mdm7|FgNzz@>eIT=(m5C~iE0)apl;vYYg(JEeVYQ7p2z;| z_EBEbiV8-}Z$j$7?m@Fv0V-F)l0@HFCiy|)5Ydd5@|GKKtnRM{Ev+()QqRq zX&7i2*c}6nmDnA3t>qPk0fq%whd4qac!)t_Ti9O5&#zJ@)kDD2XCG;i{TU7fknuq` zu;(Fdm$48ep@C0Q@+GtRcG1+u=ONN!l`waKhveq2`Nr43{)mM7ZSOo^9jQkOy}+e1 z=0`d%U0xi>Z2dZN2!IaBZB1&jfny~C9*qb%G-;*(MLe%6OXT&OKk+aA!Ui648c7;H z5mo)K=V_1E1vTzyIXZai!sOS&Sg1Vs4AuD6m~QE&-t9ZC8CeA@F_jag2ZdX=uo5Lp zZ*}L2v zky-n=R?Znwz+=<=nZI{&|CCJ=z{7a!!|(qy2hnuE zF4}ASPTuFz>GAG6xO=ko zM`njVL)mxVc}>c?ANqkW$neawH2$M$$RZ4xl+7cF7hF45Wd6I4xZ2WS(!|i()MycD z%ijOKKefRNACucs*7I8WXspEHaNBx5GYpVw-FU|xN8I?%0%#d=sK%&cpD|$_!Yx@!JnrS) z{!-lZ#pa!Z39hBCnH=i?OF#cv+Wea9ug_#$=ke^zDR?;|*LpmRtY!n~G}r|mcYbNn zWU=iYo$gOvmhc|=xj%VIb7jwl^d7kg4w5yb&!@^$5Vmi8;63lj#wy6q1pCy!V5pJ< ze@;l&8P4Ruo!i}?x(e6u+-LC=Ig8cKuIW9nWNr2yF>rCz8)L0B1m-96H+O9$aNnFn zJ^JaN{Mzg~d}(L*cr5t-K?;|2TI0Sj3Gu#TGqJThA#p~dfQMb2(QN0czUO?4B4y;x9kqrH z16L3R8Y^)HEk+y4t{BLH1yjnQvy!Gowq5q`yN-EIJ8}3)J&{Ip+U;5w`FED^?s(Jh z&!qG7pW82EILUVy^Nh|tkw)g%4VD^h7+-LuN7~1|vIK__7GoYHPx-9(d<{>#zUK^% z7~ANtG$bRnJx=>tP66XZPSOj^66jw)uRu-e%Wqyn5)nJUd{3ZMM%OS4`=gcIaYO6YkFOlb*@1 zS*_<+F5!uUGrxy^T;ut?mq8oLeFMVCY12?BS>Ht|%BO^jp%ukuq@pa(I2f!}9l%jX4 zRA1(-*onNzWMnG)C4(z+wUl--kU-u$YJ*j(M`B&huXmtpa)-;a{RO{B-)NUzje8qz z-!!&p6XkY|bLTP+3-Sv2%XJq%qK$hs(T1krF&YW7J^GHagy$YoxjWYG^>NagM0@4! zxBRx8vDltrc!cD4zuR};y*Iq|t&`+kq!*06lFM%J*Mzr{ZHc;A41smKa~Qj(nJVYk zH}38|-=UN%A*I8cA3aXy@Hz1KRP-4Njm@cihBfYkhc&Xt<*$g`8t!2W%l326`}QXd zSeX+@=6Y*3#G1~9$p^@O9CA;8-`)ftKlQs+H*Pkbqr>xT#s8j()hk=NT-0O9#SzPk_Egd z``T~cJ24^(cVBzf0X`!gu=+^*?!BBUZ@cr;SH=7osAae5;@)Q51NXcmIn7*(TGW?P z?-am+egaOEfls7;1h_@QWn27Hoetli*@!-)jJ~mx_C!RhqM_P{jWCMxeF%88TL12u zXA=3j^1=L(wO`=&QTLJ?!OKu@Os*xAoC5aHV0Ou@YC>inQ_U zfB}m6n#d?SbluxT-2P90@{_$}T5p3rZz9J?1o!Nwy15{s3{V6kt!}hOMeyMCRX6?C zGnu0Rl;E9(u^6=Qh1drk5}l^VQIi=M3k%*HsV=Fdane|5>p<#Di#N5jF~8%ap%}B< zNA~i+pt;EGc!EJwfR%C30A&gI1i*kNmqnM9#=7%~KnimSkGasI-S7`yv~ee%0I~I$ zQIZVPjI^Hm$VZO!Q;vV;XY$ZI*^)ne70ye?^ z%Is@`VJ)z=!Yd55S>nF$c3&uoCHiiCGz>HhoDBvVD{(e>Zcq+}0XDq*xH!%o7*c|? z{*bn|<(NclK|YAIQrG)gQDe&cwdcq@XfK zXpyzXWE`FDappWtk-u=Qmvhg=zY)2d)N~B143)ig|~8D zE?b;Fhfh-SB(?5^Czvd<)KRi2V2Q$X^UYtDN#fN$pLV+kkBfTOCrSh;@1`KESRnmJ zN#k-`GE@@rDP}?p&tpEjTnu z$zaFcui5isV~F=*tN%4T+ABD%&4)1tC!PhHTf*~>FbIn}OO|lty|s>xhu;47qf2L` zU0HG-t*?fGD;WcgmAH~ur;V)w1ER)ExtCUSQ(eVRAZciIgFP$qBOnbOvh&@u`%}#y zzW@Fsl4!#7K^S}>@p4p~ zp~~y~jG`h(KyBU=q}$D}{Teefs#&;@w_hFqJG zr5l^|M8e6vz<;H5rJMqbtjLjS)+8V@wohtoUI7aZkCRc&n7>LFt{etj<-^__$;rSf zNZM}wteshtjq1%cq{DUusZJ_CMADkqN)vQ(Es6fzPDXp|^J8xf!Q$_e*@QGNEYUvE zfUHrt&G*^2OQXilKE^cUztP3}*mc&pq_KB{2P5@!q^R|J?K7_uK6utJ4B}`)b7%s4 zHEBR-LWbC=vY*=XnBQGL@>SfN&);(6mu~RKk72I=#T4A^ik@(W4T~Hnf|U}}NS8O& ze$I2OQu+OZS4Lt2&d%wlwyv~=@%|})`lr4=vq-h==hkC9X|EF>r0?QE=2VW!`^P|R z?suu|qn$Sl?2CcMO6-fl*8V^k5HZK6_)tGF<*=W7{5x`#5^jf?#4H~!d@hE0#GwbT zCNBYFDx}|BqZ~tzpqJrN$-a7ON`7*K^qWs2t=w=KXMFTipPGbaELe{iAM$BxeDVA? zc^}fxwzf!yxIb0WUIe_FC`Ii5MY6$;frsQmj!(gf=l1d(c#+Ln8&c@GvGwP`L;hze zeu4!aUjtJ97<)}=kHvl&3Ra7)jQk<$?vv-C+|zygQwRdlqhfF5@Bv9PQZtvnIz#X- zB|{n6rD-GJF3&4zb@nlE0>6ZJoLFV|1kJU_Y42h$J@w>Y%xUX9-{-G{2cI;2*rtHv z8^yn6j%Z+PJmAHC*_7)jTu*)E&sV?4V=&5R<@{n!;ClB%vsKpL_ng~(z){+Z)-JJt zv)-%Pap>2&(Z2b1V5w`{wt5W%OJJa}5=&sQ^|2!cNcXsu3Ub*ef!?v(+>At)wA=+a zq@g*5R{9Acxg<5;r~k68vC9|>21AXv-1P@GKa~t=YJzx{{f(W2m0iBmJT^bW>+|=& z_uG;b@!>^(t|EJK2}_{T<3!@@8=b10CvqLzh&(D`F)Ue1x?IA`d!vIxxdWtED(fwE zZho=957zPMrEV7VqbybK5dxq zPwjYeJRBwB|Kz*jd*1UZ$tM39yT_MITCJZEDohTxarngkG}8k ze_js6>|5ja&o^LbeC>Y2z>*kfti+PoY<*2&z$b2^%3_b%;!+cen1vK z_MtzU>vD+>GO|1VvFo4v#Huj#QjbJ6(zuQF-F?O#mk&z7Qg(5PJjO1- z9X1MCzbpBnT?q@>UP^qE)bxCe2T8iG?~k;UCA?CbWnS27&#`L}q_&HrIoDW$fX65- z*d+c00x)~VO?i&CY79POLjEfH#kQy`P;=HwqB?z-RmV3habyTSm@ z_u|lL>LMJ3wW292$8v~s;S%H+Ih5$-5MyqA9u5PIl{g%3ThChj>xNEGKGb@lUqs34A5Ge&3Z!jYuebvEUn*mq9d_KD<7TOWgE;*t+%RAOBBseSR#FXJM;C&xLOT_QHGu zmiMJby91xmMljo7`?vpgP_rXYD?bf**9N@US2aO{4P8cNeqsSr>3#3M+wNgE%X--O zx%&!CYY{#ft2IB#O_Hzeg|)$x>s|I0IOU%;@jdp96kNNUhjV!J%rR$Ql`w|RfBfNB zX@aO4MSi)!PFDBwB}%YL!FcdP_;qB^UWg zu{>8wUG_J2PU@K@$7}m?_OO2P^3UT~!yTVxx6c9=c;?rYGVj>^sbcYN+0_{z1Ws20OQ&aq{5d}U{7k|{TN&(I(2FcHR3V%F(9J1=~IL=?t2pUI~xZXDsSe7hVns zi|kAXypS+6@Iwl&0_Sd%sP;9sl&_1i%-z$BdOq>5{%RgOGMa6A%tIM-Aztt;z}9ug zJooy;AN=+VBl<4k(Izp3s!Fw23AhiQZ~^j7BpmNOpK=)Xw>?+l&N?P`eo;69Ge=Nc z@0Z$lFxu`?GH#3Y>U$q{>2s`&JTl;!8{Cf7UE&!HGLgff)@z9|wm!}V1C5n98$4gb zP%b5jY^^kJUo;_?*uFmT;qO{J@<=&05}$~tpMT<$a-OYfS!c;}q7P1lip6*ZGqSZn2YCC0MhK+dSk#EU$OL%krd%{?S!PKb% zr5g<6VqZl`@?7Kg-q#;vAJSxV^o>ym-ngC8T$FQBhp`YJqkW8L7~trO|4^E5_Q7!N z0M8o9!ysj5t&wBj+~QuqOJM@Cwjp4d%PI}tvgBgGD{;4SxBk+DeA~h()IQ+rk^Qn_ z@&_nmel$e1F$Vy#)n&jtbeg{aYR@fYU>AM1vSFYG1C5oa!BVTU2L_OOq=zKi<*~$1 zZQro&dD|b!Kkf0e8|9K(5q8AvijfY{f0PV>Nbr`7)=~bn2b?I`_xIsppL;{_1R6br z9c~vMscw=OAF`$@jyJ)w$HxnK^|85hk7YThBtZn-4z{@G@x{W*6-T9U9y#L%y zOO~+@7QA9y6>sFt(}Vb zSy!p?u!TL1cD~h}AEP>cQJ2;%@6H_$G{n9}XaUiiLF0{+^_rk@7@a+W9_)E*2I@Luh5{xSQ%n<`|i6gv+m~TE`60c_sTW_ zuhwuOTxGSJVn1kr7+zo<0}fmwnQOH}flu-J$0Ubr+~pAnI7(U+E{>h)4_?YJjeOBI zOGXE3+JiJ5B`o8c>+c{EQ}i(e&%7A@M7>-lQaId~2Zq>tyL3zMg9lb4?|_lZ6h?s- zZ*HLAm{VDXi||(e32$-Dc#KPQ*=C>fU;MxaHv2PR2n#_wA*{jDuGcgk-XpMlGw%4& zude>??mEv)iA}*$)_#w6h11$K4D?~3u@Ze)Y4;Yw0K+CffoG8$0{N6PCMT(?pQ44P zFxX9Yhg?f=bMV)}n)^RMq9dC_n#NUiPOnHtWba%4&;Phevd9KDU@7Oo!{THqe<|Ib z7;N4j>-yT(Xmqe)Sb%I3ziw;&k^B?j&Sj&Yb(`!SJJldPFWrj+H;j-fnr^}40zK9S zsm8{-l;WEgSoXnK+jy2We(U+s?!^6Wi9N{?z}9umi}blhGf?)$KOso8A+`)Fy2TDB zHUi1K3(3s~#8{=iL$;M;aSn$eQcw@scswI&S+_rP(@*}s6c4klcrx4~5zcUs!+XNg zJq=q45ALjE)@Zkw7kfTWHL(Qq|H|Dpi;Rcq4_9~9D@4oY0;c<_C z|9jtigyh;HDf@sF_qdk|SX%l$TuM?|+6c!le&|C-NH`K_-<~*-`CD+(`z07-$%{axu_Yi7R()UH~JL=88KqW1~44d%5}v5Xp$B z)=-I-fKndWz%8+sYy&H@{c7i^K|SeNMCVYF5I*+Acb<}vp6V-+FJ-$dfjm{|^o=e& zaBu|2cZ6_zvR6=h$&xW^!gA<0|b{52Oy8~Mt_v#PSQ=3oxBlZl&DbDxa0 z_HfKgDPmC{JW_QLbaM2jq)qCpQ*)0$1vvJyGWnKl^<{5(ABh!WTw~gjDVX-BHL$-K z-*w?_QbZ=wUO-`U+}8-mn2cYb-I83lUcNcz1(y9TkH8#XjK2ADW5=%Wor&ip_1GBu zkgtmrgaT4Cpcyloi}qus@qW+sU$lWmTg11GNXGZWb72^6|0ln*cQ1|g-SssQvub-NGU|?CdSh9lc%X_4=XKVT$s>4@-@l@<}CD?H-WWW;h3iw$0 z=sU`Q_t@j#v9*Vy%|2!K`vRulC)%ExbdPgkd#CIL0p7ui-%NPkH&x=?v9$XA7-+0S zKVI71MKC~9faI$&SlI_4mcvhK{(L4$UBnFMR!7o(-K$=;`H9>IPPApaP9ivT_~~8p zry>WS9D(#K+KT{LQWFjolG{Sam2H!atcm7FX2t~IrcpI8!NGUIqFv;vAccwzF^tnFz;j- zv^jid8bMKIU$EVhB9yIt^ z^-S<>!oXpSmw(c#l8ubw@H!pA|l7=15<6$)OO{)ta^n@a{Q92#v2?=0M|*`b>sx7DFBEqR}D12XBJ2Vb@!4WoZ{ z&P&f3+u4oA6Yxa2VknoR+QiL0O1R`Q&wjqQo3#z~;Pc)_!#AJ$=tuL~b^Ql6E!ObL z=Scmu*4bc1UKS+3qb6k-k`N2h%{Sux4|nY2B$f4?(G?fu2k-rZ8{6)%>9*f$qHmEV zOA}ba<3oV_)mN4_E90P{jPv4otr!3-V|&rBJ%8jqv~L*v#QR^4-#yn@Mgff-i~$|B zxgTQvqW^BmPTgLyepn*W)oT0R?1AEh4~h3Y;9wyf*35dG#rMr{7)*g%4DFz&0>B=M zRXXUvTjvb}S0Dx&D{%!b$_ryiB%Mp9g`2_(7rwF7UsLVpJSWlQs$C=Xozw47U6NEJ z%2~n70X@$U9#>NH%aNmokg(wWh`w{E2w$g%dNE6wPv4(+r`Dx0B-(R&M^-)a^wUQo z0?`mSL_Dftalp&Lk$V6l*^r~s8*(M)QtRN>Z_OQCHe*d5`ME#Y&vF%ggLl)dFTTKS zT&M#Ph_FCNeBlqGU-jZ#&|j(FXcpMYizes1W)J9xYqkEPKPAl;O@z5Qf^ATK=I>ps z3fx;vYjcWy0ndBErTx`+zce3nG%1_TDo(gtx3p#zh-y_D1{wx>G0<3vUbM8E`(r@l zk*bJB`-3`%j9_rUmG<5TzV3o-xQBJE>qt5iS;4464vht-bE*q&jwJ~b=jJPI)Tsu0 z-e6D@FuLU-B!{HE3`7DJV;gR?jv&d9e98D_F-@0!0^Vmo{Xd>c$tJ=IvX?Q-o~Yb3 z4|vFP(XHZv^jX}7jYTA-xW7-C_Wo8T<}9&acZgL8JTEmrMp9*#`G9xb^7Vtz}#e`8}gYk0I=v~ePIEuH#rY@Y4e<;6f_B`z=iU%=2sZ#ood-9;N%6#ooK z7{qj+FMUmN${>LK^_{ohesFkxdcb?*9bb9ypSp~zH9S8DOXIGuzidbaCYEW&AAcO0 zno?T4a6M^{yj!~Ddk7NYKRP! zjZPBlK?0B6N=7YJlJ0=VXe|0w(l+Tcd^UdKM}PE41gi`u-8a9Fe(F<`^C|mu{@CM> z=YEIa(QMrO+TW2$#DK-Gp>{r9c<>hF7^A6p9@?~c#@GqFU88x3KKe@0sm#sT8RA48 zL0Drx%BMc^k(63(8Fn{ka!-Bo??4V(GvBv>hXsvsEK%}NDf^FaPQYVCx&)lEj~t;Y z%@GWmD(vVX;gztR$9}=$(j;}CFMa)UpZx17s@VUwdH|VG9 zJTULO`|gAO6qUvre&&;b$U)(CNeQ;uy?9vaW(S{#AYHbGTEJuLfAF890v>;iLx&|O z_Y1#(T6pSVMY_ip?)om)^CL;773r%|-n;2_ugh-IIe2J+n;<32x_#%_04u$Bq}aFo z&=2*(GZ#id$Qx|`rtbot93x!X`(Tv!_}6GJ>pAkJe8x4ND|Tqs0#20PCEp}kn(FV; zdW`bxFaV=XA;DK>m+v$d{deC}?2UJ)8qRtmL^1l>I=X%)>3R11Nn^lr?n8bv*5iwj zUHyx>V6EMcL!j>x*ZZF5Z1PF#%Al)}q5ER9wQm?$VW6=RZ6_QE0}OAtSe{i>r68Ty zVAB*NQfSL~0XwsA8mAIfSpwVn=_UO^QaF$=Pki{hCP|dC&$09D2E2x+U+JQ@e)iL| z>~k>I$&6sv^A0+)w0G<2_60u$^|_=+w4!Jm<#vnHSLOWd z&(k$lcce>{RxsAOuVm~r)IRPH!E@iY#zA{z{9PDW0q)uknu{94k1jlUYD8_~vcA>- zn%pPt<_|)nwM&?SGKHIdLIbx2E4`mI30%GRb;mBhXI&WZ<@-{m=ZkgNwd{juF2~8e zJZYM?Si@aml3Ojq_(d+wc*?ff{+=Tg#a*0yW)8U~iYKw~AAz+&s8 z2LmfZlU4li8l#h*E@s|Z!8(Z-KC}7E>c{DmeFE!Z{z;nI#jNU=8(s}LIcq|l%Dc|5 z+itZcoz8nM%l)}~o7i(b08ZVCx~sefrydtKjyrp++-EgoU9Hs#yrH$5f-}^v7-~K} z-OqLn{hY@4$@o_7x7&Ucfoll^jg`2TR{PB6DmWnx zz#*kDz4_O(agG==jNJas!w;V? zk0s&sPPW}_7-$$+90QG&SRAFT!*k~-*mD5L6a+H;%vJn!?uWG-@3><_s=e!7M;Def zWa=AP3C~aPLvR11ljKNWKehL|@Sga;{>w9k2dA}T-KAc4eu7iZ?MF0}9Id3HVfJe& zNHs3NQ_UCwkQ+Gc|izyU9eNXdhTK6#0RMvz{ zmWG0<3vYjfQXJZmm(-07BfnHw55a)JvYN#mi}2xFk=50}a@JME$I&b=S= z1M@84<&4T}(()|eAyHs%-$axu=M?B0_%VFKK_a=!?u3BG$u z5tM=~%?2~;ebsj_k#|+BSvkL8uvc`ZEd z=G=)LwR@uvzV9b-0gce+NbJ$a|Mh>rSuYWf`r0meFCu=yC9TSmefRYhEPm}i=<8$7 zy_jE(=50>DLv_y@9)CA;lU;mhOznQdz!Df}ti%#nY<+Yv;HO8#6p?C{kYaVZKXu7{ zGm{P6K2z{aWakYxd~ph*`1uF>go>$7dRQJ(Rvju$o-}m<3+c_p+ricD6haobQNlZm zV1gv^nJi2=JqvheJ--s(E&*5PF6r(61i!Z?v2Rj;KyvMv zkXXUN7f39}IF1XKV%bihwu>wq+|~-nypXHTZ}xundG?&O_L{Xn&b{w@=Sb_UwPwvc z&&)Hk)_%TcZTq$LYhu*x`*0WUe1tfACu)f6+77R*&*I^m0-r2HXxb}^tVFe|E{QGi zL_%12AAI4D*Kp`3@Bfmz@5(V2j~(y>7sg6!&)QnW;O)E$$NqsI_>V7&TP8IYMxn~A z7V*vdu@C*{bsrG>kX&lqtc9<5J`?fie=upW3u@igWA}< zcjtbu=lW7}*OxWZ8|xw?mHEaB>uVgnIq-c$j8FD7-^})?Ybt$r;(eVu&y6X{_0?Vn zwQaJl9DN^vXU@LQ72lS94eOEd8W+(i=;e$SaWBgky2kNy{@I`C5?hG`=FWP_hwtZI zt=SoJjN(CoL4lPN7^1{Vf;}45KmEREUM4^KCD`7ULP7#x@-MzvHR^W}QcFwkCy-;s zCAFi*C1D~>I{)cQ(^5i9>`fVH9JOh;r}iXFI-^I<|WXRm+G!{&@xwG z*ZmAid)p%RZ-@9qjXjv`%uJyjsP&U3m_~Sw=9d<=CMTYOor}TcaAAjU?W#pFSLnEY z%eViE>g?4~^`4XrF;_j#VN)*FOA&L*MlbKJ*jY zAWp-$tPi5&VTgyHqURL8AA0}$uVSrpI%0`AN9-GX?RJes5U~=*#F4L8Bsn~?%~SXy zp3S#HqH>m9G?cLX)GZ9#KEOvrwRI|7#>ApSL~QGx<38pH9?6xxzYi{$5Z3ta|D|94 z_;)=0^sTt4bBt%ec=#wVM2UxQ&d$(dSzxgB4xhU-fSd$@9bzcY8A5!*k7b1?k(5G4 zX$}@bH)-zL?;6MYxo;)we=M?G|1hB{v(-~`;c zGb{Lr-4?c4xJbS*^{Z9_zi@i9``# zkoG{{B|g>%juUq^KIQR(r(}Cw$_g-qg8@ zO-*0hTd^Q9ch0OT){otaIgl8aa`1D#scU0w;kWm%`?g=S2q*BN1(9Do4yR6gQn-&p zwxt&St^1CC2flNt6(mZ{v|(Y1JwK{ts)!hY?-k$jvlr=gYdlSX1iX>ii|F4vOcC<~ zU$u@j?T@+Ts6_-4;@!3vLjj-ojZ@SnE!??#eXyn^5}vCw$_E7o1?DL*M2UIYJc3VX zg6@LFMmj$Me!f5V$v<@Q&ojSd&NNt+-n|7*nO^LXpE?@DV>OR!@QXLtghj4tb$ne8 zCwGYt8-17fXg~+PKJjdSu*atpP@xm=!9PT(c zP$_SZ>mriJehv3aVqO-z9(-*b_*s?Xm}2=kIn_+|n$@adEI)t3!Y-}Vu6qD%e`28K?jq$C)heb9Q-H5gotHoivHf!8P z2LM(>A~J^{;<$Yau&dbF<>njTaN_k`?A71-=toP8!oT;pj6e39dFS{2)*VfPHoNm8 z!W&<7F*zUp;UB*Gi4T9c#^<A-d+^VSxCi)sU;>?2e)RVt-w|-d+s+hGfp7t>dl08jm=66T2EU{jh&)A+$ zyj70P@{Q*()WW;eJx8RnPVWrgrN%Mf4hlSK6d0n!qc)?n<0;NYkIDWcvyq>U*@|a+ zP>u9kZQPH)_hlFV{QHRs%^&vZl1^MSr; zISUa!C1f$9$Ko}(uXul}g>if{hrD`B7oetmC*@pELUjzZ;m9?R4<77rZn z`&?p8;+i^~oMg3u;~T%}OCA@Z9f?gnr}>}4k;^uk=|5F7$K>*S9r(bk?`L7PT|318 zyyts=^W)gr9yLeclXWWN$k#&o!8X3YV=UUPO^eWj58VO}Cn4Y1W5bgNkJjIM zvrfD#>TK5Va=cEaO?w^Vi!DXe-RJ#gXp=pKpiX1|loi4r|sP z3*IeGXv*IE!4J;PbA~&2&0z~1mTPkpxW4sI|IEF$Kpo|o7_T{4J>~g*4_wq2E!|jp z{2p67JtF2D?iL?hbMH~4K47VvgSS+!uYPCHi>JdJE^s&%avU2Lt0k&F+|P$7@o>*y zXYXl}Y@K%OPg^INUA4$oQ+L7Lm0s?`=h{!JpP|cPxq)0WrWAafWX_qi@CL*Fm7l!O zLuyeHd5z4QB$`FYb*$(1d`OgxZd6On%z6=;UbYy!`tbXIXB|(5&oRk2 zA`Z!id*-i-tPMVRh4ED4Cf%asg7E!(|rMEkX6Hj433#c+0`dg@UF z>_h6i!`h1<0{_8TF_BhCBymow8>Y7!ltTDJ^E}VP%8~E(pJ6E4&tFYx)5(~%rs_?h? z_}oAH4}Q;V`wTbmab(Knag9B$8G)w8v&C1p@{0xw=c)ON`N=GOou^`%i1}h+S}Pvg z`$9IQm;=dXf8$B0sAnIScBuLL5-{^QuLp5PoSaVizW3m}E-@J9 zHm3w=`@QCVZ~h9sjLH5TuYb*#t)8DAyu?qIa+VjW62~bt>%lqd3<|thC@@5c7sX7| zSkcJ&3A~E|W9$4h$U#2volb#4&(mp`A)G=_T1FX>-tz6=UU%zlvm<3ZaV^uA_n~T= zqp;)5N1Y>`wtL4yTiR;%d*hQhU15Fs-e)}6*Xx&5+Ra zix2+SE?n^T;4}7jyyaKz#zU*m$&(er7G9jW@)!Uvsi0X9~|K z8x(kFO->B)epvBKa=%yzpCZ>%GLRy{5<*`f9xOsKu`h?TgxHSo*G5Zhb_=8To^P)m%4y@L+R)aEi$FuiAW-o$i&NJF#bsuZj$XI2?V1(8DFRWG*IUEghqN;6u-rj^_{`0#BR+AIomUGF*a= z@R>gjXgQ7rF8gv^V&5Zv`c+&lIN)*kh8&ySDrk1-Z2bi;=YR3f|M}71t3!C=84my< z^BfY1$s;x)C^OD?L4hGk+y#}7!Xa$X_HIWtUfJkcPJf@+q8YqoGF9PxLc)Yu!sqjT z?6y%$L&PFw|6*CBE){Mr8ExKaTL`I8)}=lk^Bmk@1<{ZX^QC?g(GcC>sT$Y9diK)r zLD6tJ){HTHsNuuv_pS|l1K728&91-LTG@7t3%=_P?Ypu2nI@uzhO>8k#L4QwQ}A(0 zS@P3D0jx++@uE8ma0 zwc!L3)&OFZJ!aH1CLypy9oCvdVq^UiG|epMAUJ#)b8JD1xYb8Vd%n|GZ!W}OMYR))QoocF;OzHl<0=$pQSEiHuart3Oh zs)YBv=co*60ekAqngmB2)`KwmOQjwW)@)~b-v%)?EG?J%?U@?yC>s<&fgwt~VAcUY zUtw*VWPVT7%p8}{&=_zrz(3~m7N5H~pYwAku251;<+@;#+mV<<@bKqcb)C$J<~C+ zS)gKPXZSX(Z9KiqIfu-0wz&+J8>lry&X&!%3w%TYVheiS1-`o;-@}BD7{IY^O)82! zJ|XKjD{20UwT>07?q5;ny*iW4SKoV5-?`F;swZ#VCq!8`DccoF0g#XU&RS=^_}=!J z&+No64o5=Zg+o^nv3%daS~@sJ^;+iJ@7WUU{G13P%TiBDfFNkG@?5nJgBd0oqm>L? zelPM-(-;Jx6=(1JXFv0qnx-LevAil9PF5=jIjXLvzz`)?lkD6Ld^WJ0d}_}PdkP+W zKL6(mpPRzbmieT%8n(91Iq$Jg@A%GNeR0+>`fPmlXCn$Oe7;M35-HLa-qrZ96SRgx zBriUZqdaTo8OgpFuPj99~-%{Ix_E!CJ1k$iW8T8@1s^;_17Yh+ z+wpY{XID)oGB_Nc`}7|!=jSf^TNHF-oenF=t}_?7@SZvD>i6{mLl__j1zzwJ7^1}U zIz@ioWvn=-Hu=zi;1Wt1A%TbLp5vM9Mv_e4#ZN}f-oiloi%&fmZM>}PlQKL3%gz4-c+8$ZUk z=6%PK)4tN4f9#W=+#O5wt-1KO^LGdzM<#2T746M${k3(z&3UlQu4$>4@X@pyA1Bb3 zO(gzuo-q!Ymei{_%GtH}3gtqL-S@&4=D=M8#>^-ekE#E6#r`y29d5hdqIDJ3etH zwnjX6sCA1G-5s2_+Gpc_P+(BtGztt+;xzi5&>DMPHLvB9DirbrP+d)hX~$0y?O9i& z6DmEQ_{3nG(DkmSd3@;F)#!vu)mf^n6+iR!I005qQ(F(&A0*aHPn%QfU|0A_-xWUe z3|wpGPiQZ#@!GMj#m7;FBZ;}@f4wFSMBnUr5o@Jp$XV#HB+8q5H!S5E%L(x>=qYr3 z4jpR(jkNIIO7}88XIl6*Y+j~k(fUu~v+qOIv~OARzz5e`o_cE49Aipzi{KdO(3#(Fgu8MUOu{4~J zILS`I=`CaVit#zak?&p0hIK9x`*=N zGMvP!(^QF_fak%=ShYj&$Fo6!LlhXI#3AZj>{a7mc!lhSQvgfPDX@Cl`k4sFn7hEIPsQ({()6zI(M+1d zyTT_l{x0zupEYwA>(l(j`0ff{te0MVtcVhfUKf_fK@<_Y(z6Ah@$ohHPK3r^S^Qz4 zBxEH{aSFovM!ZzEewcq2g=lx^ya@;XUXmP@Oqr-T3t1E!^0ZgEr_h%iq^IeD(UESST<=iOHF=Wz$_~M`);MV`zcSk!WvnFsbl$ulog6bH+G+=rfv{ z9DOO-Wi|L0P3(uBefHv?$c^#;k5ujBCHJEUg2)_#Ejecvs(?Lx-e@IdQki~!-crs5 z#`mp%`e&~C={~j>So)2&wY(2^@K<9O%J{Ib^4wn05@@>a&9=-b>`-u!T+iqP#CpKM7)Cv0$x2|Ldheh%K6^6TP^ z1uiz8r6Bf`fR9LFYt1RAt2H=sUHx2HE?Pp6H{ynW(zXg-#MyPqdb>>ff$#;vREo- zG0o+3*F<(OglXG)&LJg1jsau2iS5tyhqZ!8SAOZ8=9SYbet< zu5JHz3GKdjCyC+ z-LYAYIEnm=FMdPKx+Bb&rq{u#Yr}HBedqi{{NT`H$7xAQYgVgL))RNcQAm4-K1=2q z_fu!$1Xj0*QaiEbx@g_itzo#+!`hX~8WeDgCFu&AzHwlB^UG>H(Y7yQKa{tQhjkmY z!t$d@k2X5vW6@yM!Ny91z>>A=OL35rMGC%fUVFtie{=PHH+SY_H;4ll#IAsep2RXo zl(pyT>tC!^LcNJOGN3K(YM#-f0r58AvKEEok#+dq4}S29LzCym507giL|I35_vj3- z=bA{wqW8Y{d}S4{h-$>^aPVc3@GKwavVwZY+BMPB=TT=+U{K(NMS&qoys+kD^kz`t zMM?qMcuq$`eI>%=LoaD!u!83zi&yfWy|NvG8N~d7T3B}|Kuxv#-di6&peSu8@u~v&3TMRj6xA(u7scl z?0mM508ysLiH&oSwpiXnd*iSKO^r$0$S&8jlW^T5qO>{W@W#R`ztBG8T zBOr}>y)U~92R#m*@0ZL);K-uH@hzOHv@}<9X@z6JH(A$9Yg-P~hRA zzz`)Kp1B%59Ta#0Q-H&T#F+fL#Ra(U?Jli2?d@yd_{OWHcG&REPS`j5AJEGF$dCSK zi>$W(5KlP4L`Ei>{Hc@&>gD|p_@_&Z$(o=ot#HryzzAG5V@>HR82tIen!@M3vdP;_ zHg-;Ck-*~B(A!cTR+K|HvGcKC196A;J#URiBTuUx`ybGpzv24&iuDBC%5rb}(37nC zngyfO*WZ9a%dD-k7`ORxop-!9&((@j%4-b8GhfrOUN&u!$p**wT_w6qTfYl5SNw|Kv$r?v6p zMVz~Y6ISknJvh4OIyW5Gg93vBt0*u;iB;qpl?Md|1@(3vyrp7eCDTqYSs~G*N+l<&QjAG*1Kz0#&tvC&DHhR zF-slJV6^*ov!-pgHi%-ZUTwQW_e56k%zw5xxR#|Va2{H8D2X_`ah?2P^Dp}pjw zG{-C^hY}%U$JL{8&UoUQMxQ9Mgfqf{wPUFf%G|O_wA{H~;1J`06{d|=R>P*35N<=R z1BNrLvt~52L_HdLq7jzVR)B?-D!yQrPAix}5<{Tx0#vNsLANn3zcn zWeK}3B24?7Bb0=(=mVl^@TErwJJY%Z4R-)SX1Jkox^L~P_bgfykAfB zr-kDlJ?gIH1t*88sXCmF>^nh};#7uybq-1!j$NGb9<10=naY7B%G#d77iH2TxE8*y zzJ7ksGhcIS9-{s@4+;zlJiineqQvt{|ABo_-~~>B>%y8uW*=uh^i$3a)zP9nkGdNt zq;-SiL{gns3ab*Vk+Yq4Bcl-3sW?PcO@FRauUDYM0ts!_j@W>;rV$R9eEaQeER;6N z^0{UST7j=RC~aA=5JiM_6B{zzxTf{Sy15rm&}Je!gc@6K9|CPzh;rRm{h6P23!m5m z;)G00+OWLjHcQt5^L}wA%Gyx$i+;qooQV$MWaeIM%o2|L+#1uKI2+|cfn!JOjH|tV z``dn92`T=SyLQq3(3#~ZLbb)8*qJ+=n85c=5Eod2GCnLiOWGmWQ_mzLroI3CN51xo z2#S>#=OO8_N+yw{pJD`JVU< z@lDAV&+|bakj_bgAxfN+ngiCLz#~NgECtry@v%>SvgCX(dEhVbM?=aF94pA#A`S^f z3%fY^(@%4YO^Zx3Yr4g$Cvdc9?0buomN}e+u%c^rz{M(I&*0(IuERY z34*jP zGYIXp%tzQTp{r9GSw8mI)MNU>A*F6lml|W4!ulfiqp-MFoM!5d&qdt8UU3p4dN^pe zWwz>Jr?6+X;Y#T0C2>f0r_j*Ssa%r9I~}iD8L*XlSgiSx3~3BlcscR;U(1 za}S?>$?PR*ZCF`YBC<1|eu&N3(YLg7t-UjGu!qHueg_<9*0VO2M>nlsW0UPH_Smqt z#jv#Ea5JpkEz5_wa!4hhZBIXU0FJy6|3S2a|40z4nn~U)L)xlFqw=7@pgt*)c~OtlQQfz2^FlK4R@^+B$cs#Xp`rSb6CLP$RvYP+&kO1KqA%(2hbJTUdzW&nMAAQT^)7MWtE|w)fV{sscWqjz(xuj)@`TY|Mu-hDhGMwq_ zCvL^JM0^2!H1ioRM>OMzBi2lHq?<;tSBjoh%qgYT2JwZ+Cpj+>m?&vJIiO_+tK0{c zyeV;b57G9P?KkHP zKlL0+<{WFV@xe-k=2h7m;$V-$mpND?Y?ACs0Y@F0(zc%Z$~=p@OXtvD>#uzpf{hjT z;8`9Iu?D+uD?G;r&xS6&>vB`am*I8dOJ8m3UAZ%}pQ~bly_~5Is z%lKFr)W^QZ{5aQyrZa$o9_F8K{!F!9F`C6VV&9AEsFcQMKF#5-QM#a2k64sL{fZ{& zZqD;~V!Zi$Xienx9dHkwBdj~d8D}g%R^0gXNye%vIJLacBSeI$I&E2=N8>`%h!sN1 z7{B=n%UZzLygN_SH5Fw`XX7v~YqL!apsl%$alh-ke?y&5bAgt|?pGtN&EhT4U$=@=s|*Pr;Gf+I^V+H%&HY2W%iGi+8-`SKZnv_(9R2((XItr#~iK zlXg05ZCOQ#X0+Ed~3N3$K5@n(Y=>gz*XqMPGBuS&I)pt#+=DHo}?V>)Fjaw{?y66I;T%gk>i-DYs9D zj6*CYSV9?(u!zJlF(Vv;Dm$)n=bwIJZ5>lPa~F2}Wo5iqLmU99 z{+F8A6009a8ykRS;*0|}_Qw%11+EWiqx0BokHu*uV8r=7-}{?ONc3weyNh1JUE`Uv zkWr$A@Yr2>20U>siDe{5)rQkc>kp6h#5x8{Y`@rrV7F?R_jarkDE5lPodg`(WFmot zy~2)bIBU~JHIZwn%kitVobcn1EHfD=F;3{Pu;z$8AaeAG zFI5C6_=pc#ybFh*z(a)CZtGUXH09_nforAF#8{?_HkK&(@^07FiLAf`7p@%3P{fKV zG9Dw6XL{Mo#JD)0p)XwG8I(=;b$#jHS}yR-VP1=^L)m%^`z<&Iasm=-!WvRDy%oHy z9@iyXT&(&1R;S+)A*GH8N5V~L{ONG#AVmC+Iitd$zL=|fj&fPW;Nr*?+eMxO?i>tO zvhv;7WfPmLH_Bfe6d0n!i({V0C||S`@X7b{eNOgcikN)UZKHFdPi`E+nz|RdYD+mRE!L12!1uh? z-8y|myPmxX)Xn{iIE`H9T0J5dkt3dA9m9c08MbR`Ou38|L|xg2Xy~y|&JLoH1hXN= z1fDq8y)+6=_cZC+WY|;JLh;oXF5cD`l1#J=7C7+4IcQ_dSmr7CZ5bA+ZrN9Ge)#>r zvxq>Oq^MXLjz}3cT0i;vFTZt!<5vrN?w+GptWBZ|vCKM&bs$|pbqt#WaAdi6Hj?wk zxT+XY!sm|!Ioa$%vA&hLt0EsfF)u6#ZAPw104oZu!Mv+Ge5^IZN>-f}pt`DJ=Yw|6 zbxX)^|Ms)eFL%T`G#=JCtbMG{T<6@^Qe(6t{JC)qc!L5<6d0n!5{1U|L4k*s0@z_$ zkS?|RbS4ZIUn0(6v4{lu`8lt@<1hc^!VYYbi*m}rLJJ>Vi-VZOA%yjT274+|B#S_G z$|!#1wsUjHa8!NhS0Lo56q%8NMl{e^-dDWxm4%i1`V!Lm8!Bs6{LX7~OyIAJrk0O5 zr=4DOLW(&J5$lA|)22I&QF#`hXoWdlNs!A&A1n0CyWic73k_wd;wU!d_*3_fC=OK{ zfBMaqdzD??mL1wwZfd;3gZ1CI@^k+4L>kXnQ-U>RWghwz1^Bda|3rDSK!_B<955YWaF>D4+Grn$G@T5 zO`QyT0Xmd9P?cPi+=>2KM|d}1vghvjgq7+c^!T>BhPpLo1~{!hcV z8Iv`1H-GTm9UbpxOasfqL4hGkJRI{hdO0X?1O;fP`6M6fJ4?gHu|`-jry1E@rnGLf zeN}r}w2m~7jy2u2l^OOa`)F4fMMEnbTX=GWk=hZP(~Re08#td-Eq@0>u@=|}{`mpl zeg|SN;yx?}H59lck*wB^(hu78&{Fr}t71ZF8(Mfi;$l6_2TO#*wsdf0{C3)|&OAhs zZTqF+NeW5h3l4Zeuc>y~s-r&burdxH?#=g9yIjU{qHnAbhcwtM9My5Fz!f7=bNiQe zwBMEX3+)4Eje{xe_`640dxk3#u-e|uS~cg|L<~ZWe`y;|+@phdnyK}!y19%P7}=-V zcg%d4-?{cJN|qfCP2Pp}S)MN;kVbzQG zVYjx+yqd3U`N?fgxt_w;lP&BNoZda}FbBV>`8);oXgetILZrYDC0>ZLGWs+qa54q_ zEE6R-*4!_7+?}!EJ%Aq+|7r0HwgL-Bkwfj|>n}-JcFs+~A`mYQwTovthJmxvTo$L# zeDNEu?*FAa60M1S3{Z$4DkoHXsgLO%e~NV@X$E2H}Qp+cOU&d^B2iP^`q_)u_>PY&rwR< zx&>aT#a~+}CiV~>?7Qdu`-MekKw=h%2A*@^>S667+%s5Q^oeAq;5cZw#h};AO-<|CyA(8vztZW$Ip5a zVb^ne8W4x7MPeHsid_~*0IzMRmiijcGy54VCsr5Bp$-kRkYcQ%*a7{L*~qi&61xy= zm;0qZu_{%|Tl!c-(G#6)K?Y|{)LNbgtD9jGErPDOeCpa<$6ge)qF82HWxHAvA;1wP z9COt=-#jQH48L;q&~*w&E)$-vF7~;dZ(9%U`n?^qm;>~1yyU0<)m09`6(XHsPhE?T z0F%hoKDX8bpNu(i9p@Ocd-Kq4U)REO&!@k(D|3*a%QhDKgM8yReMt?kl`4Sc#9AQ+ zm~+?4bK67L@aZ4g^ zq(g#W^fjLsam-2h^u36^SLUl-t0#|Z2cS4J6`bd`wRdep5s2| zZE6jk1O9;V@KRui5)bd(jh+t*9H4*%ux}~ZO5!+B_2fsa1>p$fr?eg?L!Ab|86(*L zz$M$vbnVkhSxYLe;^*(|z+#73(F>ge7p6 zHqK}*oS>CVP49WG!|<>}TtK2Aq3j!Jd)+bH^CF(e6qMT@a~(-|iOjjZHK`x8_`>w_vs12#o2)rmlT8g_ z%@ZklcDP@I|EV|qvMLru8`c-=fYtL7nU!Qxs-jUrNEHDW5uVr?eK_}-We!YEX+QM- z_n&R9&fU)e@6o5g5G5Y{IUXZ?QBr{BcayO6iN~GYUy2adH1$o0ZCA3?rbGG&)&^el zaC;kLYVmFEr`t@G&cWI5w~)!3Bl>q!HU;^1*&MDLIGZ`j_c34lZEwSFOE&Fp`mFtR zq3J%Ho9A!_uC{Fpd;0zc7CifPZl2|_MW1rn?Y8@n+->S?U~a~sdSb>p}st{bf5YEa<$r@#;;p8q)*Js1?Y00qADb+5a!FT$yy zU-MnRM!ZB#7TUsM@b-*0)_w(iuInzn#do}|yL8V_U)zXJ;PdX@!4e?>C(UMPw%c>v zYHJC}_PJ!2vEM-p$2|v+>_0wLr^beK;)^kX!>69(h{Ursyt$N3!Eej@GAiA>Cj^?~ zO0*9r6;3cTvQgfig(k7NypT(1K)*6x!dGPuUPtP zeN8g1>|8k2&tuD~<4??WIs1Navg-|nUyHZ3h+R5XBJgoWZ|XeNHp98OzQK2$*0a9$;)1_#Y_Q(O*V0Ly z=>MiqH*MD8+PCmz-1)k{@A|*|cZhE1#-%r4UYEIZ#JKd~61d9}i zW89B01%$)O=CO8X-zky^<(=Bvi9fsyKz!?KHlA(990?@(eV3o>)jq;BYeH9LRH|8> zQio`4Y2oIMUcYYr65`3+#Bzl#7Q*>AiYLx7eIzDudcksY%9%k15bO7dP`@SdxVs>?cHO4 zIZAO%$gD6Cz}^J8%@Uq?E}K=flR2o0UWJ}f)}A?*2n*$ek)H*;L+2`n6#8ptPXuD1 z`Wyv9G|DiENWr3Ff#cZ^{MFyGqx&Iz<~iDM=*e|CZ^g4XhZ7o>j_45OJ!_Ps9adKx zA;`q6z(@S!_!RKsYz~Df3w4h&>p3!YMf<7qYv1@4yEZxFkM|-(XE0+2^MJp1F8vjb zI|aWj(=Tnk2azGi+~mGXyvvd5Re$y+yT0x46RO7@KioPke&Xx)n#SF_E;z6t70Wn;RH|0au8D|*OM z22aIR)<#^%*?Ng*xv+9S(IM+=YRg=;z}ex4tzZcdJo~q&m^0*R|(kG7j;Ko zL)J&Eq1^7A*Tj0Fni!q6qR_GZut$zKrDNNVc7p;B0|kaC@i5HN=;NS37X?C_*;RR= zl0<_-x3ST4Rpr089|Uc`G z?o!VY`wUe46FByZir;*p4acc$K`QLo9xEDS!A=QV-iLAT-YyipebC(7rVlJ(k@=Cm zEU=}4#~JI0mCW^A+uCp!dfh&3?l~@j#c3!WaV{dd5ScPQ*R;^K%V zzdJL-+bd>qv!94<1|z}fUWIbQ$&uwLAIE-gjD<+n;u!aX0)qk%Ck2Km@o>)D==Gq$ z1O;SV!Omk(Y2cpvwr^X_R!o4MEV6alROTFxRp(?Az9q% zCw{V%6NPfV6BmB`y)P@8?B}j%slj9+;FFGS@DNe#SezXcXoO`k!R}9hzEy<%!UkeV zbLOGU0@6lv^uSi%0#FL7g#Zh@@W(-fBO2H=<-{)HR~#~Jc?KV+k124Rh>Tk-!PN6y z7UBZ7cFMP&bt>@9IZSZ`QAa#VhCO}#M5W=>)^ZU?6Wj0^BN||Pv8@i8;ZHpKb#?r8 zOT}pR-vwCZ;6o|{|qO4P{lFFORyh5n*N8}k$5GfPjrSF$*XX&|aP zh!w1Ky;--QCOX4c_p8|Ur8VgoN-uHt) zIJq9DaIYz|zBn@)r^9^82Dfgv^EDXn-Sb_NCCX9P_qyhz_MpH+Nr53sJe1QmIz1>b zMFFu3tO8;W;yCu_?7biSU=m-g`cQBNtUipDnyBnqY5&rONQKKRh_s z#lJj1&#7AfR%30p_-ZV}7G{e-?pc?z6Wts>DF>qe=H|MY4`tK^ zQ7_;}8~4^}4=#PMUbKlLzfZL+brf8&2T@lnfU;C&@e=)rRpvU+7g1=+vPk?;-{E=@ z!Au3I1p*&ym4vx7RZlMVU92e;%9?23uakjK0D#XojU(m6<54sya3>TPqQsq$c_0}S zxHJVgH5BbDEo@=GzkbJNT>`Ig-kgnvV+FRN6*qcuKlAQ)U(s^Q!%rj-J{p>Cu^#2; zvWgL$KkQ=r?BD-y>OIzf?JmZzd&3(xKlMc{!8*l3b587HnNY^!5kB?CD_m4LHbF+1 zp0RPhiO^E?;t*r6f*f)jFk5Z=ORt~!YK}3rf5n$QVki#by~cW$J{kk|HMG=KEGf=8 z6;^r9$}53H#=xr{YIT^fIL6Zq80Vcf)uIaZMP3r5XZpMg(YM_91TR7BW} zk7E{#N{BEVl2{{JFdX*ai}2s}dA<*PVmyrR$-nRmD@};EtN=wqAsX&d24EdC8;_p$ z-KaFOXUD=H3xvAZbo3yy5QD5u<+0WyenOdN#^il5ugV$pS{!i!Y5&BgkT4J=Zx?Ecf7PFamB`d=p$cS`+-HR?BfEyx((X4 zf0Z_+K1&R|bNFa9BVIuqi3FX!qG(uSuL*V2dsWxYApj>n zd%p4c)}1(ngU$2!YI=i%wNHPT@m0si(?K4DAhsvoy=;9w*yx0Z(h%wxs>d*bW$N4dbkG4;$&uM7du~6-S?U1#k7bk!A>Lit52?6D_lJRg1V$7Fy2}{ob zi%&oNSmhr7XLJG+D<&-ZLD7)1FvJ-2!=4z*X+_&Hz+sCk^?R}o(HPT|XDbE=wCa#j zV9Z$uGG1{Z>Juk&`P8+3wJ@=`%CXjYN9L*aQ5FeK<9`19#N!<4io?oozU^G>XBRDd zMFseJ;xmi`i*48G=o9TR@kDvA2m~K1QNWlxS4Xsh6Fc@3fA{ZRaf;#~qaThQHpaU^ zRIQ=>bBMO(#G%;NBXDfa&(*jf>wHODE)$nFbK%o%7p!yZ8g6q*i>|ETxngFWSSh85SBeghkO$A}#XvnFod9-)SnXHD4@0H%Zk*F8#;lsXWt-bjta)~40 z7<0Szri50}dg6FAS0~Erc8Mj%1-nU%4v|`%hILs*n37!7{6w4fEa|UsaAI)#?Dl<0 zT3gll_=j8bd4(}pFGg(>gHl3}&iL?vPU#3cRXcvnTWLc<6*Cw_`$ zs1~mTzNsc+Ws_?AKX=b5ro|PoB&n2nNm5QCz+3{( z9?drVSi-zd1h%DWskeO(y!nhh7ZVcBK%Ujau5$xQ6+&#Q z$g-_hShhua8XIRV_~61Ka`K^JjXuPgXu}5{9xP*?KVh7~I;WxBy3Av%8HLI^V~q(M z`HY3f=CYJv)8iRUGdB4a@QH#N1F)rs@Ll_91RQ7Uuv&1ob@H>g#-2}ro-8sK+MyK@ z;xk|Th8;f6YS>-0VvJ7nS zv6^6SSw&*J#FkyR&UUfE=iQMZE$CI>{j=e?0VI!Lcz zoDcLQq}Zp!m?tZosgO8Fg%$BUYu=}0xm@vJf;#`_wF7ux<{kU5G5Xs zc^sp7(NN&J7{VPi%u9n-C+aWNZJ*ON+S)50o3j0yn||!qyJ@puck1rCjOC{OZ8$Mb z=czvJx1GY!mfhfh`*8pF>)!5b3w!@Q*YE596LoF_O77bRRy>(IZ~BW)nO3f~-@w=( zQ-+k=Y;fnNQ`a}JVjea)a^0Ic8~B@Nd2G2nw`{L-|KMx;cs;R}w8^+O{cIsm-`|GU z*1H*7jK4j%eQY7zzRxgko?fT1f!Wdm-c6mGHhou5Y}ZO#d()5H|G;}g)3(j>{q=So zg5hun&~boUMWJLaZBOnmQlqZ?1o+ zUi&Q5F~eA*X|CJCd@2nA?K>=J8RFh=>~w2*zoplz*5C$~m_hJPWs zeXZBmqEQP6gWb6}tjzhpbMT<);}AkCwk8~o|MLz<8lN&5bE5aMN3s6ckAO2qw86Gj zZFN*Ua27AXk`Zk==45|(A*P(SXwq|R2prKSt`DHuJ=$e`Dr1p_$4}s&e8tb$!4i%g zS&qQx|?i+w1- zv)!G3u;6g^se4$I@{BWIql-O7Xg8z(OXtYc1Wv>!aIOlt_F>>u6!&pfF6-G6{PsD` za)nv&g%cVp7Pyh&idBWRN*T*s8;jGzT)LO(&Dsi!*Mnk7Vg1ZbPM#_A4O)UXo(~EP z3OxT57^1}UKL?`+g94otus=ZTHrtgKzZ33MH5naZ3n_+WY;H|rT3C0i+fa6riCzw^D&EQK_t%9X*886D0zcI0)~Gtp}sb z*mm3f6`XH*>fc!um53kOOC^w#hvxNjzlb;CWJvTRsszqBTie8|sWIg;?7BAIwIYJl z@zt1~B2p_Cm(q$RHCgg@J3w{d>h<%Nu?a5i#F@mkdt2rOyPx~$S~fH9p(Bd`ZM~^` z-z`a5pM3q7@95N%QeXVwCkx#*CikC&`=BS3@xcLx#qDgSf1ER zf`%VH^{~IaS&5=Ekua6kT%uHa6wq)Rt3yb6&68iQ#VG{*D!P?2TgXj?yaxPy*0v^N zEbWLZ@r*c;*kBU9c=|nGxi=SZhPVbFv9N_vv;G+4 z-}>!cW30(TOMMoa>KC2>S}!qnEIC%cT|-#W>eN#E`<~s{6PUfr*f*igz61_2&A6n08g|ce`P6mQz}NAW#J1@#=oD+ndU*fe{!c31S+O$CDiRbJ)@lMVc_}-T zxyy-V#gGIYBU!7))?>u&7)v-{)Fi6~MqSU7SZ)eypGsXJ|JJGuPH3mCtU~QNhtvKY z^&KrDwIt2PTXDg;@W;Xub@jc9?qzMTo*l}{v=KEnVqLVaV&VcmkK;R)tvNknx$l{D z3vsO7Ge@iwakmFo$1p~fL4g-E1%@c`f}Wnyxj})TfDARVP%zL0m}AW*zvc0&u`qTXA7!c2ZeB?g3GnTF2WV} zR#O`q+x8rqW9QzjvcLyBC-X!vEDc(H_XmF0Ro)?YYWzF8ED^IjkMV(Np3pbOk>|~I zj9Zv=(83u#(YXU%&EymP3(E=z9pk#yemlRtuy3{B7SNbSf9Ev*k?kq=yqJS^j&fn` z#%@jYSOiYN$x1ghpU1!i%Usv6{22hAo8R7T=w`swV()ETE9E^4C$6ZkFEumQ_Psny zveNB(^Dcxn2R^v0AARW&yP}Uv=XjrpNH2f&tLysDV9%9#r$QZKzIhA8o9&g&S}BS(SjZAhJS zXdo{&NI2?GQen^Id?}n8xHotLU!2d4H$ptYH_m(?I5*=z7j^U4HZYB;#k;vb2hQ90 z0_qa2WBkrN{fRML*Yy%u_Rmve4LsTfj7`~o+uL`Wv1xbv*-gpjd8WgDyBnN0ZKkho zU}ytwyB=>~-PDco#aW$Ay=g2_+5;Wg;zv$>waug3D<^{sihfu&FFm_ZD@`+Z)< zHEIkB3<^AY6d0n!qc^8xM2{Q=_$)Kj$8m|=>X~SgLeijxHvap6=&`Cz&vn{stc8P* zdi(6qJxw7W?TpVg3xTlEwk>~u4mhFp(^j}}PuMP4mGvo_Pwgv=t<@v$<+SI2t}cf% z;nq|2s;{?72mUz1@Xt3MIq^A32ovWd72|cbcLm3*x@W?c?)>qEz}CGpO4;YiHd+Tz z)khMDg2yQ>JDYKq3M-4}bv#=J@h<; z4}R;f#aA__+s=jMOr5-KVB7b&uUg%*PB`>D{oeQPt;M`HDtn*S6o(=4G&S72lyfYL zH5IU@&f+n|TIgTVoL77i=AGAuIH9^#-gXY#F}F!$r*K7C-PdG`k3&;+6x+g;ur=03 z%%{1Im`FeIEGGDCi`f=#*X1TO9em zIwNk^f^Ww&?|%2*y2*9Vag8RNuJ17!ZxrBr&F|J$ zQvCNBuj}?kWoPDdcS`JnFjH(B2PwYL4(u-$0t-Kbo zg^o-;r{cwa@jIVmN$jP7eaig^oP5^N#%|bAwsf|SLe0}ZQ!#ecv3}+?eC)Bh{LsyBh=JHPaVqMy?c00!=0pH-A9V}`j_ggu z)46%9jwc=X>eg}-+rhyV8Jw`%f9tb|IE6ZGm)NnsLVlx$Qi!^ zem)mA`!V6ZLfdos)V23KVh@Oy)`NMlUkW&IdSA-*6{nmsXOB5Ml$i^A7^u^3bvK1i z8MTBgS2A7^6F63dszPi--YWzB>$*U@a&R6KT5mhaIrxXlr`Cg10p7!W-2Y3zyzWsG z{W=75JR1}k6u5f|3{m3lX+AIx3aq36Rtd|}yN%aM@O#zas+!TlCK4Sk;oF-L%BX_n zeDL|t*Ce-C!Vf+Bbw{x#8SdP*-F#_Mu_X^mB1)P=;)R%#xi-g^%PwVXulh4TYsW_N zYo@ivmc8c9Z{EX6OK;n;IM(|3+ld>#7A`ha=zLfp?5=Aow+K(x#(`}lqV|UNV$ksE@+n}=XGH3 zwG-+*MIT|({M3z?NNDHiw|(X_RRrHSLS^WE*GpnpSxYaY(x7HQb8E^`;{p0m99@eg z!P?0dx1)iLhJylw0%xPZ5GBq=$^mFl;8GO8QgXcEBtx@a!=f*QjE0z#OK2alDOf#L z2`uS3(aL3C9e4_k`z5I+-1axXLSgr?`N9~9H`m3V8~YaQB39}3ulcgavERLYs#%MQ zHWdrqW~Z?;H`YjUSlN_}pHmT*lhcBXF&rd>X_mHoBETGeeFbN3kA~P`TRm2ZoG2E| zNVdvBrYEE^c9|wpO!dk7(I1(KXxQ6265L7xY@9ug8FiQ20&fmqBxs$|K4;u$yYact zrPJ5uBqs-sTC7^=SVNMdaONyrxz5eZ^IS|^VkydVZvTw1-09Dh6)VF)_nL(%7tUN0 z56n?z&wu21)b;4Pih0{z#M|)b49Xg^&gZN}xvpz-POM|qc-Pavsft3bWdKu7Jd)L{ zZRg$^VmU#B9tW-|7;PB~9IF_MOMb@riFF5U&bbSBp=%A#w;tMqnAW=oN$%^?HSv~e z$hRd9)=t^#mfDWzg90x^3Jg)=g*YptPlEz`6evzBbsILUTbodGui_bQ#3E#EH7A%4 z|L_k_vJz)NcaLVe{Rt<^(9Uv-nZm&*80$avY^h9=OZbQc*ey;YfuDx_4cA|~gKON_ zR@-$EGbH@92gVYPc#dVD#m4^B1ft~uk>I+BMb&q^^n=*KLhz~o>%ZUiBWu{Pg%V2! z%tPl_{n?jP82rQAdRQ^+dW_ZCJ->U~0NNrD+o+rcV@W#=e&BoaTi-g_zt4Q}8!CGr zI61^*jWR#{5GAwartN#&FX4Mjb$;47g%LI3B?gNf2#X$Pp%{C>ojXT@S+sfWo8ELK zZXxOtZ)xVmVsMyZ843D^@ z6u3(YFzO2d6{g71;*#NYwvXB+7A!0E@zlrm-VGu zManbJg=mI&V~>T%=TF4pY^F}%PTm_PV}#dk;`)$?;gi7bIeJqwG?5kt5{HAp%r{YP7X-k1y;47uQlD9 zhTxs=dRKJ_%N87>XuHiHeDJc+an^I~Fn;e-o`*x%=-BRF&p8U)3!eBEj(Eh%h_O-b zJ@}DgEur2x1i$y07$9*SdE6`kEJqF^TraFqZ%|-R;Q67z5G9@;x(}>_0xKxMzgvbM z+UYqh;RLw7A zm)<*KtvCdt319u_FVRy>3+sgE0Tb=KKl)^Cg{6Y_eiBJjNbCd4dq-S$%?ZZZjjHHo zP1rXh@aD5M&Iv2(A3EpvopX_$vSY2b?Tpv+&wcvSbtiMyI`{d#Y1>TQbB5v+){1FU zbw}Btz@Wejp8`Xac;U~{=;0$nfvM2RHG{+6X2*cJwnf}{OtM96D1ZDOXO$KfgS;#< ze9t*=g!4mN5A5s^g{8r6we=3&fBYwZ@`@u$W{0r2EsKJFAquFARg@u#7PvP&T-Y-< zGjP$q#x)Tp5@{M&O|Dw-aKhn?gZ_4P#vWD+$Du)OpM_%zwvTv3Ganl0inp-!_X|t@ z_P71IT|e5e>AJgc>BkZ}ebz5OVytl{c4YCCW<4CNa{H-k>}tTN)|RG{ zwVLM@4Pr}0nrDaPuwYb+NYlQ(EtY~=d{}*sR^d1n{hB-%n)f)!iC>BK?f7K8if6R- ztcGG$WXDoYoZv`vzs#*Q*N$cC{{H{=Z|~-Z1qPc=EMw`2buTUf9ilzal&J32c)^~B3+a?>f`O&MI_D-RY25xaHM!eT92%K%*MegZ(&9fBL!Q@3?5hk_^A zz?ySDRiqJ)F&pO;E-dm?>_Y-mFC*pz4jOIYn|bsd!O>!2sG z>~*jE1r--s9U318upW8_KIfQA;vi8Je#h&+?H5%Gm-(CfhJ@uN!&|z>_SS4wMGRsY zlsziGFT`H`kNqVwEMhfBtD4RrKj^#`d*U6=)mg2qCEp!c-C{2tVr$*azI-Rny(a|m zp7W@2=7R-qYF+349(xU^_!RLt-X$UqOX;!jM!P|Q=aT|Mlz2X=KXAXuDB$N=s3QOI z&G~W!2t07z9OuW!@q(r_aK!cId|rICw1Fq+N5iOWb6$r}-_1!Jxpk)B25%7|s*}la za|cJzMt@yn_rVTa%EFN)%GAmI>bZ`7f`dak5 zTQFU3?w8-GJTBKgW7CHX{F`;)K9_B-?@j)BcIf$JqnW}reVzd^iU$Q=@Dv!L#0!3k zMh71a3edvQ%F)POM!R@JC%A5DW-p}qqwx#PSZIiEaNqRl99NZoTP>V12Fzpd)p;m1 zp*eKjYLtV9G=3S^=K6`M2@L#8=MC%|wkBAxSO;LjV;mV5xS_cX+AiVpe5Zk}R-79c z@YeAbJg!@qF}4jodob-=19FSa*lxx=g)92m(mum!ucJ-SE}pme)C*cV@7k?+59a8x z8Oz49u_imSb51hwX5n^q{S_pXUnvb5tyP-*xtUJdX z#QLvmzR+(Sl>XW=o>=$&Esv|2@x@#n2dS@Btb_Jl=xaS{4GNr(0z;HIA1w!@L4gAl z5WZ;pFUP6j*9Xwt<6F#^Skg;=M=C}UZ8lD~*e)wfi#V?3ikrk+S{)Ao&ua?cp z_F<9iHTojV9BWk*-A3W|EW}rMs&>tz$y>he5$7V@TO3bYF~ZuX=vB9xE3|CG%v%_7 zFHUFY%PtF5Ng^80oXrhd2`3glo}UTv7HiRlPv<)Abxqb=`X_vs-@fP8p*3Osi3O3w zx5Z_rZ`qB+Q?xJ--HU;+{_L?J-pT#|;&;qTtUKS23`2jsch*&YH*=l2YgwjZ-CKY6 zONhAr2yFK!)L*!_B+nAW3Ym9^VL-zisy_$6Ew}r7;1gdVHb|(l7%OQaV$T-w8Dd0) zFZZsE7MJ@6C7$bppZNV(IT^CGA?|9;)|OMS50#07kct9LBaOqM`s3N4z{5^~Axb># zGdTMHkWc`7FN_k4Cn4pfy8V`?o~qkDMSgfc-G2VDHs{2NJ%+1hGAcN*Gm>IDglJcm z;Y&N*2MseTvm5J~@Ku($;DS%6YYTP_zU!C2{OSw;&mVn!?vOCnnBcOD zv9MsQqpUo_vOn>E{Ogf^VaeSz?sK#^Mpt#BbhL2sTO=YBn!T& zahWTbuUc{TGMEO`L4j!s3{hg5E~9i%;LH@Tt(C2v;yC7~lNOjpK911h<*wRh>TZ z;%D@jEGsgMM0@8a@Bfl!8@DHRTgK&jyX)Msh3lFrrm+ryBS~oSkFM5`*v1X(flZZl zCU8~OZUIL;W)%uHMmDCJktwkpEIL(tzby;xa^10dTlVB%_=StCcHpa;@X}8IWvG!P zRYn>akfQ%_R+m;&TggbHymzZJ?LyR{)pj3$RYRJ_33k;Ezvn$yG8M_jr3~we=F#rZ zxo$_dowuw(HpXE6V+>Pg;)IN3hj8R)HOX!1qfA^6iq<)vu`mUERyZtU&8)N?FIp?3 zagR1jXBM4!k2cOZn-;Mb1ANv|SObXHw#P&FV=Rv~Gtsuc5+uWB-ie_VSIP24%v|bs z?{mAX2hCnl_BJ85)NE+zTN1aH??^YTdT`eLP6}+}!Oq9_d$t2ScUGoNe~N=$&~v*5 zq)cgc74PZq+B&R(fiKQ&*0!}U57jq6GH3-Zmf1@?RT8O;v$@f^>n3C;m$ zz_=U*hA444+78Hr0$U2us{0($#yPvc6md90RJLdP6Y6K?Qbu+^|9+ZjH#a%zpSzOC zwI*#m^8h(VRA;9$9@@T0_F1eGn;m*jz(*^K4R%OyiH637cGo2YlcgYouRq_ppyS%l zzTEj7A83NX;M~O7D4cfcF3Jro8gPj~o8p7FSS5R8(B9i?fpd=r*tgJp8=EA-E$dOk zTFbM82+~82Kg0nEOo_Vb(LD|`#>(nocX48iadQlc-HkcI*~=ieo4e-V1eY!9h$^|B z_X;keO_V<j-NDkOqhJMB75C|6&ii8Vlkh{kAYOz0HvS3EoQY-~h^>;y-Ag)5hxiYMB` zXG_GmCnj-z>)q}#+V!4^&72XVzWX*lD4w0;Y%Nr$w*pLp@g6JUIq-sEyUtT$X4dqB zb@ozLq_!W{e6J&E+xD1y@7rC8W5@JwO`9P~tfAJZJ1B4z1vukyYAWX|TAixhJ_^!u zBaQ=DHEe>P)A|!$*akks%dOYd_{7IPRvnz+$oSk7Db_)oukCZonZ{Oh!2;V7tXd1U z6?kB84*zi-B?n3(X$TnHyW@i6db(!QMNwsyJw` z-}}K2mL0YK$m1*t?!&=}Nb~-``P=GvwX;5I;S)Tlhzqnc&VoYer z@__ZX{UFgSZ`;Q|e+Try1vVWlN7i8biNsD53Rg+_Wan=ftwnqGt!E z2SptE{vUd5yY=03XMKlbJp?}Mhw~sBT0e91aBR6X52rmUyi?XY2R@dNHS^MoUyO^f zfFF*6914xI7si@;L{6fLF%oO<7g5r8W(|zBb?djP#0>duS=)Bh9~3wj1%@bbE=mqK zg95V@@Y7{0F9v?whq(^+bhgov58&E4>=cc+pVFGagm`g*zbY~m99Tro9@t_y-S?uy zb_)0W{71g_D$caRZTlFPpK79nU4+ro_%`EgTTI`xnnXY2O#I3k%X+;XhiB2Bim#M& zqM3%iUmC1MrAg^`dj`9P1eZP;mo2MecVuUxhjQY;KIGeX=03vh(O>L^gH{h7l#%;h;T zkES-BIR}n7=VynmqyM>1Uav8({QRWrW_-lI>X=x{y-)49PQq0h8=twi-dM6?yw;m} zIu-t{_GKL2-@M*>#qwLQItq-(cfp)_@8i4C(e$_~Wqk1+TD#VcLFhfb$UGg>nbGDT z1%@bbkUZnzpujmOz-h$6mvwEXp|$t}Pi3#M*yXq`TH3X6dg^cBZg6CLo9iA3H+8@Y z`Z!ldo14dbdEz~f`Zst}<%LUaZ`$SlZyIiJ-;_sR_S-EW**@NHa|1K*$C%x_&QPQ6 z=A55QU*~=Y4Fg8t%jXPN+YSEy^%C{le(PO4%lMSVk;~Oz`nyhZwz=MKk;lFd`-bLI z?e?ME-0k=OraZ3Vx$||icB3rn_MJEV+`!#$u|Kx`XUE@d@LgBB?n8;OO+DMM*?%_& zGsbpKTza*E7jv&<^QRwEG>+%Tp0Bs-GsBLu%@~h`Jm0Q`3tz@R-*P+|6gV#hhA44f zdJbrV0uvPA&?NC8=K&5g?|S+-6$g#t9CGGAP68aL7}sfJ^Kner^`Y*S{y!#hXy6c^ z!S6bU3Lk2}_F3^&*=U)IlMD@b_{!I$jxp9wplHu0y*)U3@IU6`9NK%%ECH4#&iom3 zO)wb`4xG(4u_tdp)P9J*q*)Ckd{)ML-Ursv9$|O4!_?((J2yrao6ch(va`6Pc&P~S5vI1uI zLFc5!0Zh4b?#CfI^*t-BXn{+stv>63Ia1~j|Ir7huU2TyS-?1cg+rhBSxm}H&mLp7 z86PJw8HZwALfZR!ysp;!e(n1|MALx(-rbZQrP;-}99l=b!DI<@dP{%KBOn zhvL0^kWvGzhqT;+IANXmR$SVdN^e+>#T!Xxz1+L=)ZoYp?|GmND3_qX5G5`_)d6!* zV2=XWYd@n{?>K~0?o~R&4UvUExlettCBYt?Gijx+3LzH)8}M6vmGxgaCSn1n5U}D9 zVBk8f9zHi75%+?HEIx!Ha9>V6DX~1lwAlcOo<=5=3g#!fwlq2Yb!I zNcO6Vr?eBE+Jnxz5}Oh2u=HYDz>Tup2P~w#{<%!O-bA3Bjsh+*gczZpId-9it$r@^ zeuQXvXpH~wGtX3~B5j2|2Ma3Ok+L{A%?J%8uH-V=pL#;BqYrg2iPBg7jj&d+ERiHO z#$7X=l}55Qak6rV!3j9MXMGKrSYYj9?~N!w=DnK3ANcGIB5RmA4>nktKAbaSRN#}O z6)x*4Z+xB`|AZ5_}4AOBU&W)%0%bx($8zph$#u;o!7 z&NUfRtSu~|_Cl1ggb#{#lt=^*7E3mpjN3IjKKc4DU!5t4Sf_oCVbBK}XXVCD4DcSX zEZF+(sVASjI!2r(W>wl0xLMpWmvCAC=gf*y=dHlaq29*geRdp^^!3ncv963Ghp}V5 z%@=m@P}{Z7B${RM@NEcDu5JTZwA`Qg*nhXT_SeFms!tpeEP}qd)GcC`BCFG0 zy))g&(uRKd5D#@G@VCJDhRE*adn~cD?}zW$t%{@Id8fb-C7$;w7#(=DDIlwd&pVbR zvj;t+_H!ph(9XnumX_ESnQC&@9Be{w9(_M|ZC-TR?YZ`T`eXls(3m=N0O=VgR#JT$ zWX?i0F>1Mg_V4}fTjQ*`^-C5O^=UHYYUik6M{H$1dpyK`1Ql-~KFx7B7ZD+X{q*w{ zbl_MdJUHfxcE9$Sg<;{(&#k%=o7wf=PgnU*JqKHAY&)dFRli2UT6X&|O zO85Bg-IFPaIF$FV0IRUsd-?U}GNeUDpWbA}#d)MA8qRx4z=uwG&x-4p|7*AAj%GS? z{tNllZt0IR9J*PXxh`j?Oi$OsdBa(UV_sMjHtjbSjSOSznqNKfF~Xg(zu@lP#S$W4 zX|l-WAV@RqKt z^d+iXnLxump)tA^1Q_Prfk}kOe*Au}ZF8m_wx-u7pSCb)h5a-xi75lWvvFlyF%HgM!Mc@s>|&#g0ZbMg zNjJgbJd*KAWIM_Sa5P#;6zW3P*eMx1U_}=@I*Or8Z6E?ZJF}j zv0X#%^V+D+O?&kM%FMac)vv8HRn5Q#rZQFvYrR0PTIl~7Rt4q{lov?d-bxbRD)Q^6-w$8fNOTwp}!&;pZ`{2f2zvp{@a}_57--qA- zJ8K+IS1eD1PX@2e`yC^jKC5Fqj&+uqjd-oE#?^ysx(9uw)`E4;;cbqFeUL`2L4k*h z0z;H|xaMp0_MxCa`N^w4xeV#AfAzn~4|&H195VnG1EHo4>{x3sXSMKi83zX=nzglI znpk?y2cbO-cvvDy09i>m3D9u+X_l}zmF_$5mQ_7>~Q0j9c9%~- z)~4=BuyxuAi@`b25|2WAi)|x@1zh4q)v^{#OBwr)zG#kxT5|XbIC19u6IyWE)hLg1 z!TmB~8JwQ0Pjn;FD8oXU>t5#vJ1=X;3h_sqouC7!aqX-fB?D2^#|nO^#7($2=hYb~ z8qJ)m=DV;IoNH#P)MA=s^GnpZU*dMMrk4c+JK7anQ(zd|A$rgR$Na(xU)#5sleYZU zeQBR9676#qpuknw%&y1A1s5FXx7nxVxo)Sdxv?yjvueU;m-3~$ z`Q80>{EnIXMBOz8Qh3-$go%@qFyea zx(U!s43uP3ZH=;<8o<6Pi8Moi&oXKWFwkBui@#X;0SqbbWxz>0I=(>B(N zXV#rL1;@MnhU+gaR+cY&%r=eQ2jvcqRo-2ps;vbhaOATwL|Y;Ac6?_Yh!A#TX+>Cl zArgoy5C)s!EnPn-);#Z2;B3z#a`>;Cm+2ycG?5 zVT^i%0uKuXhA8o{%+%=T3y=bSF76kWjMn-mKK$XMGSc)6%g;MUsrV_3GnON>aXqkl z>eN_*!kQ2#gdvK#;P?P9cIBJ~=Wv7(!p(6a`bX@F>@L{sUeP4l&z*%|8%Jc3$@PdW z)d`|J#}emf$6o6@hnzWwpd07<_f|2#)Cut&4eV{;;A{of0=j!d5@I{mL8t`amSwEn z$3jRwm_)?d+=3U*sU@~hoqn3 zC?YEl_6WVfcF@53saH4H$_(?+bvwQo2bKeSgjG@2`tXNyOmzY~gdjbmLFRk}p1u;z zh=9TF#Tkpr2R${tSGCclUCox2A1qu~(X+I>F4h;?XVt}`L?4`GVQRqyzFMz+c;Mdp z&+M$K$|42@yfzb(IP)!!YCd@+Gfd`GNtxN6p`+-7@S@ax06{WG@sh()YLwD#r;9_w3t7bn9B zbVn5tInX5^QAVUW3h04GoK35!wOtp+<-Ir7EM^}7vHDTn3#7!VPz6{wJ&tCpk&Fg| z0t*xvqQnA$#^XVOX$o*`qSc+#;7zwWzO=Ho<)?8fR5qYu-L4DkxG~;epY&5nql|6G zRs{Tf#ztVf{WRC3u=GM-*U&@wv!B`628)9cQKV&sI6l}Gtg;<}Z+osgZs(58T*Kq1 zU0;a@L=;XM?rE)~8S7Hi^NbaS^VG{<{pzZ9=J($7b6r`#(%yJjLa^E5JHW(BVVSUm zQI7_is3u2z^d+2H`nFZuI1*h8{oVd?m||fvhFCuwa)>N{qsW|5FV1!B##~tFD4RPQ zhl2(^*U3(2m3{@*oVB6Gl62w|-vQocFNd$a;y8kxui2Bfeev8jVYxr~+L~6Uimmy< z#I^V!bsb@ai2HoDmgd;R*vF&CnTpDR^I(SU+B?RXkOp2@tuk>-R2glz!{S!M@7sqH z_YaDQLcEHU4lD)P3DY_cQBMp}b_n8_7|vU3#OjG|+$#FqKmH%Mp z!NxvFZ0N|DLV8%SUVPO*x@ZouyOJqJJ8TPUidiyH_nTE=IqH#ELm|x-T<{tjG-PC=} zW>v;kch;V&ur0GioPNc4oLRGC9rcMW_wH3Eunv6Q38GBKSJ>2B^gqz$d)6XHoPbd? zKg~4w@DKl|SFtyO`qL|I!g4fWk9^Tpy(Elj1IvWRye zGR@(Ma{HZdSd6Swd47};x3RqH)G-z7gK)ek`uW$d~;J7`$T%Om@q3cZ_Z{x@%8E({{8k@c!qH(m%u&1swKJfB= zj3v_vTy^r%i|bzfo1TlnwK;EUp5y-^8s%}F0~Z)mb7f4|9p8#$`;g~k-CP`takLv0 zxJ`i}O5CQ&C>a!3pukH@p7MMwR6qGKi$`_3DGoXkl3p^Ow-8(!)xhV>i6lZvPaaGI zz@8F;i-n;&O>smj2aRb{{p8!gR~(M2IEI$j<)jwZCaWuK26GFz=5 z1s67>IHT<}uwrUz=xS((%X6C8*kfQ3XK7~~trdeaj6TpX3ll~Ihx?$tbKDCbuFn?` z+FwLFjz-2Lv|RQsY+8mFI4b_)&=J<8Xuo^DfR80fU5;*fTew~OW<0W2Wrv;Um-;dP zhfvbihZRfT^8uD^`HF>NSFS#ItPmTkS7Dt?JDT=9|A%m@8{=V>W0kT-VjL_JVk6Yi zH!)Q4zQQ#Z+cAgJym8)h%++C$s4*yszVIk-oe}3?tSztfO8Cs>S{S4LpunKO-B4hN z5_d!Af#s2+0H1e`4sjH1(AinuOfU{7!CpAam=df-u6M3$?0jaMu)K5r?x*6kJzR|! zDC3g7CLB08vmA0_ITmmAfiJkwv9M8BRV>@GEbuSE#X7Bt2dP`9C4Au9fteGT5ZQ1X z3B0o6aG;5`g$|s1da~nG7~8S%!OmHuIFK3hr%JXZ^{~BSYO25cwtwb4)|s&VV2z`1 z)8`GltnK}h%?y3{-rr)u%NojSibIvKc`-0iUgAfV*~Qey=xgqGTPD7Mb4)lMdUtC! zvNGmN;1jl->BND}-Yohh6h1$jyxxwF1E2LdcMuK1v^HX`V7If{LF~jNj$F9K*Y-PC zaV-@=;B0B^o~WTt+pfL0Hq6&l+K9HU`&a#$pS4>b)oM}NiYw~f$H%sLS`oHHXrh~ z9nCBj-GLTDafPB{8QQw-y-;golN~BQlR1f|Tl`Cew2D2je?@5YWdjmos=Tt~=(O-B z&IG=x@O5mW&`#`GWL=B~QKyO(Er471Pt-kC+G2@UY1Y9Z23fb*YOG;f_swtp zwRQYGnr`D0|Ij**m@hQs-~X1Uo~j9T-Nzmmfs^Q|j`b4aOgx)9$4=V9n&Y|kW^KrF z6kE@UCm^ngV~E%S?^1Q#DgCgYLbN}0j`bP%>e?=$^48^b)HFPuQ<_!TTpJOB{>17KwVN+SRcBfHplj5%^^j+r-N3 z$7jX!rE|m(!8f(d-%|TE&%L{>SJu01TWi!DYw$!}^UUhmem~>+CC)^-BK8k)?h?d3 z6U-q>oQaSF%!`Erv{W+fT>6vE>OgxP+FdLX(Piz=Xz#q#p{z>*o0AV# z60M9Ce2!QVBJLh8nrorQ(Jr4Oe7M6tKTl=}LZTDE{z;2IChPivU#L?XT(6#TXwcFOg zl2ciDXIMj79lg6m;y{I2&JJw3rnH^BVvB25SI1PT(ceaNF?PsosE$GQ`BvVP%#1+`N zeZD5jz~g71g#!DTpT~97Q+MvKup+y*SRZ}F9tDgz)2oTJD|0V_O;#)@XW;)jJhOT za@0qiHTck~b;z)PM8FDsZ~69Lu}8C-h;%#mU99%D-z^;DT#K*WyCZ0|f8IDq-EVg3 zuKS6UzU)}Q(D|Ir~o)gZ6>nx{JLPh$~r)%yg>rvZaG-q6M})XdUO4b-jhG z6i`jnEg1_O@4k#v#O!#-^rO-`;S2crtlpB=J@8xq_TsQjVE%3xL+oUY#m?~C>t08t zz*)Cs+^#dQSZsd$nP+z2G2<3z(tZnA41FX1fA;P+=)Ug03;Nws#k8X$Gia#~ErK#A z>ga%EXmj1@;2^`OD4mLkj4v2{;|Sv`l)m_rT$Yevg)|`dEv89Y+XO=rI-$vgq^UJ2 zeYg{2dlO@FH!o@?)>0z&1rz?#j_b3|`JQW?b?tr5-sj=}aR1kt*|YXpd#&GZt>0dI z@89#XjbvagDZO{wF;@qxiH`8HQoZ|oetosL^v}>lm&qy_;DV3UW^LxMNuUQReUIEL z$EDRT2hfg7g93XL7^1`;ImVwsfnyY)vF0F=B(PXojuh80z+!>05<4~t8$t70N_a0F zzaZ%$_+Sga{?x}mUbh6R{{E#N8{a!{9HZBv>v53iSyw*#tOVwHu12@TPgDZ4$ES_Y zaGbgNZO~|(h0^9&n#eTw%KjtmIC35*qT&Qo9e;?Kw3zDC*LaDOak@(1sWN@0=BIs} zS+pZ^fSn_K^IxLQDI9H``=?)Zz-upT&(W>Yw6OQu<)FnT20ttbL2LMQxi+^IS__s2 z*Z8oX8z;KA?`dCI?1z5J~=#4@4?jeKh@)X&-=T4&9$HsxGw$RXILUSoB#UWlFE z?_z2=SS@wdJQeTgHz@G5Q(%Y^Px~Dl#r_3GSY<3t-5Rer&*WOmPNl9br{XtuNSc!tPVnV&oSeXI z+qHOdUfKS1JW#f9SX;7{m+cf=ku>dyij_?)j&#J8;NwW-+5uGc66de}iRl9Z>%b}805xl9Rio4s+ z*w3jpVS(#fKZIy|%{}8(#lbDKNf~D@O^H%9{-&D~)|{$i=`)cbiT%Jn2bZ!}8xWIT zC{`gfrB-m9&-iO)Mq7X5W~wZZ-V%5PC)xdMSwIdK*z&fnhmrqSmfJ2& zHrJHc8nEI6&$1$mQmegIM^r`Iw(rzA+H=I2x~5JDy!9v}RoW!MXX|8weeK1pv~NrI zqt{&LI)8rmj98hvlyH^#>@xn)@^diD^|^8`o0n&e)sFMgTwn1FH9n;@U_YKApGN(ju}{t3bD#Hukt`VYfcd!eG_0pct!!pi^UP z>*5!VJx@LMOn+=>>afw~t)8Ptt0~3Gs>j-;{d~FcwXfRnrjhVpY${O@F4|GzR4Ya_ z&7XV!U;5`Zn$)&(>YTV&#x~`1O`Gf48vlL%- zKAD*xD@gFo(J?re;v;f`IYm!9p$Ffwq`6D+MXZ_{HqI}y=| z@NWyHM_HO+|6nZ1ow778O!dJS_mB z>Ul=0DyqzFX+qr=&eC(??C9Vn}_U6~X3SVRs%y@On(a}jGf{b0-7 zE3BEdi5}KVn@#5cy60GGB3ellid8@}mIzy`h1OV@D()Fu@yTapa}UwgdO3uny#{u* z&J_`CN~93JTSdcyBkM}Z=bZ>4jX?|V*m(_OEo1)JhhBS-oiwIhTti{mu#m@KPhDSv zk4nOvY7E8{kQH^oXT9_(90G=jL6m4k_*`3Cw(Dg|^Um@}{jn@5@G|V40nU-usr925 zR@=KKYs33%8MI}sYO02^Zaixoo3eMx()Y$PEO)KEi&$H!fy~KsCmzZEG#maC8XWlh+@K5Wx#4bu@0uzMq5|dgQE^EEpGD?=8Xkx;W*EMURCp5SZ-{$!yK)bK1{iVJ#}8ifi3J0 z{gr>YI%8QA!KExB%E?ZJEl&5`eK3BaNSw85R-;mfoz!OltuiX&>0EoGgENuG=vj z0kQJ(CN!LwXFe5wakQFCPP`KMz2>aAR0ga(5!Jmz);?4B03WH~wCkQZ;^~70_M(1l$V2BdeqU?bC6i`4c0mCnr?%KnZ6TT(^EuT(Wz9b~Q23{IwT3??s zpHEsj?8!B7VGB4!z{M$nhMQ0R<}-SEj6Ty9SIHiPJ)l{o?K=mbb->R&M?Q~L{3@SI z+C^G7P8J--;943p5ejf(ki-@&A+=!SFn;ybnXT0OZ2HfErnV_s{kOdJt&7$uZGE($ zfxou2U0=q1tjzrg4wjd)25LCB1iEOLYbmu+Pm{_jvDES5%VnPjYdLU~tY7Jm!GB17CQJ zIs*&p-N0h-jkgcJnfOp+qDm|l#~<|z!5wW@9nq}sAN#SozQlNhw#4+K09H1LmY%pr zos;*J5cnldpDWQlXPdpuM@enOij?)a7T&S;z9H5LQAy^ha~!{p;mY-6R^zlA<+DYB zAxb=3_j}BCj{?{(?2fSLN0IQ4mb>~`7l(^A{>6KPUhv1Kg;R&n!>T>nl?ysIP58NI zpM$1)@3dNV3cFo>w98gfoOa;!f3M+zKR!=2Uw`@h(pWp6x*xLEn3vCV%^gRe7+CGLVVuHfwynKnJaT`~GV=ltuC?n^Rwnc7*K^&6V>|E6U&wdnAFB@=UF`VzCIUMo+tNl$<~tR;PX6KGrnW}`{f+C zbA5etSli*6`u@!I8}-jUH$;iC3Z7vK{OHG?udk_1d0j#TS7q@@XXgY!CBM-{WfMc?!NMT){b2w)i$z(_(_i2G->jYa>(jlaYBw-8@TTS#`kg|z zDSU_Eg^tSUu}njpt36K3L-U@4I$wWV@7MIb$=o*cpNFyLN{g?hS3B>vehuvTwww9o zdW-9FYb4N>`}nRw#}-@Q-)^7d;J-W_@301Ea9xLvcley$ZrAFgz_pM0>HIfDiKp|v zjZvRX0S*a8vw6c&MLCuA;%p>a01`;nklUgOriF9JByuVEEgb&mHAjEZHgaC54ij6P zZM#dI^RqS|+A7&dgm2P-S8d~^kZj@5Vpc6`nOj07V+Coot4q;x<`8ag4djq*MYa)s z+|{Y1%q?M{{M4(%&I)>d`UAh^#xA`=cR98hXVyTt{*+cK1AUP>N$Hn#Ae|#tFx9>yvz744l1|;M&Z)uuTP)xR@!2FZV}? zJC?&7Ja@(ESKc`7u7iDhN{-e<1TNmD! zHFn~hfsejVJq3m+@zmeKF@e)4faSKEG()s4olb+Ytd|g7c&dTDusQaGSk`lWtL&Bw z*Y3CyzL#Epx%#eu)x-a?IvfQb2cq>OKiq?pg3w(!*InfYpIDQ3 zy#1S=k54!DgNU&X+H`A<2tL6?FJaZ8oy3?}Q*jehEafz`=~}+ie6=OqC02r&c?F+R z*!+)w@eiLhFAhgP_|ezx*fov`vR1Wt>^;J1VQGe_=~+GZoG?>tC)n14dr}?A9zLv2 zEJE331v@#ef8-&~f5J?u}=yib!R?92nE?Ql;!hiB%AeEUv{pyEGlA z#w!yWT*Qe>WWbujkveLGJlofxWALD%_hF5PDH$MXBnO_INOc5DR)JJ=etjRD&zq}XT1H7x5XS~yze_!!Gf zlB^`cC0RU-k#8{gtrmEvd7PWXL_HCq=Cdz!lJx`|g>9=(P#M{^cDmI6WWmSSQ4G^YK5W`;oC7DwFZ&7+YamH?V1it)UJ0)_x7Z*<{Yd^L?vs&y7i15hqkPRlmpZo=Gh*ff8`H8FWmbY-x{=J zJ%Z1iyhE?;>u(<3*N&{iWfU5%UX2tOqQtB5u8uh!p#WzlnoL_2T{C-=PrT2uPj)^l zoPRih92w&3%RU`6e!!f3HhsauMxI)9bzny?W*@lzRN zu)eaP1U5|=_`)Ysw*(&zeV+lS-?i91i{=vS+rnDM`E3n8eUj)E+d|ZW8;#l|o2)IF zgZ)3k*7#J*g`aE3=12QV?9o1~^84QZ{wj9aJ3@AumZym3-ZTR{I<|Z*K znEjpyvd)EWAN%l!cWXCeIi-{!7_B(i_4m)aAlmgc`=}fn2die?8)Tv(^#qZk7uhD7 zUZjdhagFcVMFuy3T%7_#l(;&H2ZAS$0+)11n3|m}@34W4!2GtAj!WY8i4FXOqxIz_ zaANQ4He|c-z6a0O`4#-eusiGmUD{w#rf71k4xCNjgb{8^^>cmi#x>Vsy59JETIK)d z+H}9oSRO3Cj~hpxyZIPz<`}we?%_1P^xw>Lx=m_(`eh!$w^@4`OL|R}%kE*|qFXz@ zyL+<1wP`nnr|hg;c5?3fPq!?V2G3Agi~XA3pV$6adRV!C;Plt-|G!)3y+c+z$3yEg z>+DdQ;B8@|ng5Q>`pA8oUwfeqdiLWDj=nUm9)6yKzRY;q`+nD^hdlq^*x;?l_ML%F zbu9J1+|Bh4{$1UMPV?~BUa@}QZ|RV^s=wqrd+pdp`$2)Hg#tsAcv|k%80Up3Kzqwi zouS)yV2<0@zy4^_)3xY1SqMp^{i3a+9aHWdYk|(R|H;pOcDF?l_|ZJ?;ez`DKP(H4 zZ1A9CwD`eCBVEr=z{i1!Ly;qS`>_;~p$cP+|2-=O2P9f)X!y*pL0277O6cd*eK9{_ z&TyCqt#cnEk+AP2IPGshV+M{l~5|UPC3$G=x`!?pqF$XRo_w6`q z?8O!zC!^Ln(!*KupB9n^zRs&?$%|%kk3~06*UXQWdkM@#ZLA5f&@k4Uw5OqU&EdXf zUF7;HWlb7d#xG96JEKjsX)h9%0rRsCPl4a}F?R4AEK4juZ5y#duEp7}z1E*E@hgYV zIuMhQcmi-(VY2hIIcnqiV{z&&1jWjF;~GmvKZgE4e4FH+`&T$^d1w91r#@vz`MMX0 zXU<$%L}>KwSyT7aDED4{eaGyBx%ZC+m@^oQ4oj5X;5{3xR{i_pVX@5I7AqPYEk5_5 zb=JFwq^YQ}5#rLy_#7;d?wNTt%`SS%N!MN~-Y+!;#YLe{17qc3g_yoNYM(_43{m1) zyz9^Oob4hjI|(++7EsqD1{S1-TNkGaKH)wqYj-|1UcT(dGI9vFu3>juT<|%t0c-r6 zWmeG-yEf%#k5x6V>de&P3eL*PE+DL!fOgDqV!P9olNHTi;Lep*v`MD}C1;Z!Msmz! zrK;mpVFB7U=Go8n;g`R>8{+|VC1z!lai@i6j_`0mP>Zl*?YnL#G+yw(~pR;G#7qx-QV@oA6){cEji)H1XTCTDC~55O}tO&b*x0sw(s0I zx=mg4474#!$M9jVTiQ&)R5ynYEqn3$7|vuf>TwDE7V0X-U8$d3Nb|M!P|QM?rxhN<0cG4;)Vr z1!%o|E`$JHQ(I)pcB#e6)5iHsVo8^qOHAmrpalc%QVe%BR3Q)Q48KCQB{& zA|BPmswIBmP`^{JZM29Fp0|^CV8AD9Q2wteu@OR7B~Nv*fQT)ik)?UXnhDixVZw*L zZQYV{A@($fSbgj-%`7+$;h6AaRr{S}PsJxbhX|VtK>Vr2N=PtU#SOTN_LSzA=<;yu zE6>lnpYxLW!6&@6hC4SM#JE`hw7hTm^Z#g_FAEcv4IXnoWqk_w*1z9w&YXYo|M??3 zu|+#p$12X3Ib)s0%QQng>C_sjq4H%ew?+Kxfrwk;>%W%O?&0MBJ70XUt~*w&Yiomv z^62%-XTEKhWhvvhbjf1o9mR6n#R-CmP6VLRvKHz zT|=CJ&wGR=hq&ros)IHbsU6>?&b=qR_b&x+z)U1k!p@gWPMcVVqBVsZ z`?vz#bHqFS_+rx#hGpWYg5A!1u%@wG#6ie4n*25R;0R15=PCW;M5E4UI_$X2EO4h! z-Q!{fLYASbSuJqEV4e8nr~d6tmZ8Gts*ANlR;(JXyKReASn9B|xo;n@x!z^4sTP|p zoQ&Pt=afdoVVPiw$~ElPnjMvsyfk|@U=OPkOM??W{^vxax-Ob|t-)n25(kX+6My}8 z?e4c-hBJO8eC>bd*6UIl=C+z!-V>}2sekd${hD1p>n8eH8$$SHzEVc>ojyxS@1cy_ z^P~^)EY5NEMR0%l?|t)CSfSV8@9{}@ zaH(utQ{$*!16vw$R`dRtYImtRsS~JMJYV_#TnhH99nKIXUhQ{y%&(yUR)U7_+M#s( zazFQ_FWn>&D>mj>Mx;u$t_4%U)-e(j;7l7(!ppUgMs%#XtgHXa`0 znOKuP;}XAr77XGxRv%k3H=aZFSdBJ2(_B0P>k<`3O$u79KN}0iA;{C$3R}N>u4&Vc zWoII4VV_SG0c_Kl`)2&pqdKj&u$*N}&_8-3-hu_9s!|G@2;at{^&ii!ky9|nVw(9V zS;VXrYvvS~hx)K^x47zgz4w3N;<$`MmMG6EJj82z{3O%dp>dvbO>7hk^f?Y-=incG zpIizIQR2zHH)CWcQ$Sdl*WSqhR&?}P#HP`l%k93R>#C-lZp1ENm*Q(I!DY0;0hV^0 zRdN_(%c7?3REgb!r#j(ue(3Ph{__Xc*C%DJ9p|no_-&m}pD|`Wvt`+nHO!!)rM>D&8PcQliF^_sDCuR zLVp8K+2U*Kowxg~bl~s(MVGd}B&~LOXPHVp)}PpJ?pZ&sSvxGetR(jU&s2B^-4<&= z=z}&6@Vr;VV7XuEPLv{Q!gEfF0IHqLC%BfppE8cMC6SXOZH~Ewt!v?q_JaaXDg}lp z@uc3FF|HFRz=4TYoJGLFf#U=)i@VUKwwfs6frm7(ivdqGyMq<#Vs_~I6bxNHe*_nAqVQS}RiBst#LRt}U?? z7$@hOz%-{Hc>6c)M93B&%S#;7jMM%Lt#zw9xVdoPM^q0kr8tPTc@;~yw8U<^SHxA? z`gYG1mWd^zHIb;rXDp%mpFHt+sOFP!7^QtD;jz4&1%V&EbVrOPS{N%4rKTU)wl{ZT#u7Sau2uYM z=12XKu~QFJ2f#EgW#W50|GmnjX*PSxIgGPetX9FL?6YId0y*IVyvA$o@Ay>*e(>fg>#(wqlJYFEDAGTW1GsSvM=b!UogSRmY#A&Ewk{JW8K_EQz^s+VRQ9POfpV!rB{eE2an^#M;}M zDw@cusV`YkIO}8#VYRX6sXJEkpN9|2(6+5vj?@>Qb@hQAzwWkf4L&R^HYwJX;Is~m z!~MZ(VD*KtXZtbjXlIErOEt~bqSz508yz;I-6@sYqwfg3f$J=j-@A7nZ&8I{ie=|$U?eppTf>6Jnt=@`y3LSfvO4%8zW0r?EnbZ|C>O=>*z%0Epl`e6fAD#KO|{Ri z;SqhO+N`NNhR-{Is%6bRI)wiaKHmWE$<|7E2-cEohw%OJH~fm{*R1t5_@r!TRS^1m zUoL$oF2SXZcauct-os+Y&PkHJ6sL9(-D6!{)Aw0zJ4A_R_0GR4=Piuywy;o+Yfo^4 zqd0hQ-q;=2!D?Q2IPUN*-AQ-ts5WpP>~`wS|IHZILR->)GpEfw(|%KTJ8>^V57(xy z2f|c2KbH+Y*YDuo(RzdDuFd{fr!Z|2wC29`?OS+hlX5f0{r;(&#+2H5nr&L#&F3(@ zvid)I5p+0@)N{ovfdzN7c%{}Q-!Z31iV`lkN2&|#n6huhCzy-&;ew&!2D zz1X+xqF-TPAGVrk*xiR-ia^1y5UYZd(#g zi$0uH^euGRTOSXLA9^2-Dyt1`b9*F2yUmi6b6kVdd)pY9re~otT-HiXx1}dthRGwSsEHUg^%-=1jS(4 zW;0_`vUGB6G7e6FoZ?t7+BS#Idwia;IBs$zjK#Buq5feLN6}xO`_Ecz9^NDFN2OP> zq6yEB)8bs;Q|dFn+!LiZuTx+x>0?a=7bj8o=TcUdDf}!2La!ai(f~~LA9b%z3Jg)= z)p=i^!C7%+umkEb7EcJ?xnhQ_K|a^mC!fKe`^q1?_bKJ$Ok0PA@X3A6-~Gx>*w!_- z$Gn0IJ_pvTg&`Rx);rfzVrkki-nP%&`EIA`EiSOYCZ_4bk-{e{I5?E?-9{7mT4$#@ z9I3bO00)i0YRhED;Y{RH$%#YwX)88h6R~eK8EAkWD1|>Ju`HHaI44#QUhE*p6fBbe z9N&6S)q!n$ZHe!54bv^{!K+)k6@YC*%FK6etHkPjwC%q{R254Sm~kdT8xAq4SI!yk zTYO8B6<24h?fBYQ2Xt)l&9Sv;$f-r!C7NxlVAv+CwfiHPZ0b@*6zuPxwNpdq%X(U} z`!?}C&mLzfY%|faotyJY_a0gK-_o=8nP}5j5l^u3J&yK2XDaK#(Ppcv`z37ehipte z?79CO$FS;EY%l%2(?0pqOPgof^T6u5h09S)(ppE&^tG*DyI+TQMs=(z{W-B@Y+orU z7Q_n$UtfCOtCQCX2T`!a(SXHSkQf+z%7;ZnaesSU{;zqT1*TL6#Ce(f-v6HW?B+ibU)Zz$QmUI&6OeGEH5B{}p-hFl?GW^(wKU~EVtcOqP6f1!3#g6ke zo&kNY+9$8Z36xJ7HmUv8u}ZWoTYEj{O8(U5e-HRXMP&yHt`^U+bF3EjLc8GYV;&|@ z)hh7;$KdBWwnIp-Pp$h+ETCZ~Y7k-gJkQ~|tG{E3u-DcBpL2;E!3ZuTZRTCSyR#}b zMQppPo;+*C2AQJLNIXl=-VXWo3U{qzYbz(tt+C7@@ge0A@EEzD%X3Qp^R$e#I&-=ICu^_12OS}dVwD+=~; zHCS)Ye(+V=2T4;V-e(CuA=9+x+P2~Az}W^cxRg`tlawcTjkH&KLJO~flM@Incs?CUy9{3g7q*c$ET0Rwd=}S!3NtoV93mKpfYhaARp6Ke zM~lzrIBk}eoHjPw+HIUPz_YDc`Wt_{UqKvoI3)=`OuHo|O`(3oAPyWfde~X43Y-DvIpK{_kgp-&_?33lH?Tgh83!SxrHRWhzy_i37V2zfNSb#lZMWR*i#}x}e z?p3U5EN1HZP|Ci4|6h6o_r6-Gx{Kyu9ky%3j^fySlvTL?E)VXaP zeez#<$`)u9R$be7`*6UOt(`;c5GAfd z*8z1k1!7sC(RQ4=F9eNm4%(2pf%tsVTw-@<@_Ik^xqfYZVJkMD!vY5^&PeU`oM+c- zxcKK*R;skm^(CdxK5eS4qu_B&&5sYOb~Js-v?5!T@h!oljq!rP@Apt2tZNlp3T#%N zw5`7O3-J*jC7Q)@(x}lcJElItSaIxk?{B};4rc%d%DN3#85?>;Q|X*#g#(MVlkPrY zQ;q{m@1Hq<%YmuxH&84nU=m^B%weAc_3oK%`GQ|56B%vd62E>9SYUF1qitj*14Gsr zaE&87$ZAJ)>(@{j^9DxQ)=l3-KhG&REIU)}WcbS3*G_B-r?Rx=&{4N>EZ3&2BWag1 zW1qvP-NrHP!Sh5#Ys!JWv!){UF5Z5v^u<<(ju}VF#7c1Vu?-+r-x$pmF7Fj0_+rg( z!B?Gof}knAH)!y|ZY@B@h92k`t#{My@aX}3SqF8e?FlF+i7+$xh!WO@xh+`-6T_LQ zdQjCUEKzCT{kVD^r1O!o##NO$V6#0});C<{#ge^v9UQ@v%Qbsgfek)2i#1@q!*z`~ zkd2l6JNEXUxwd7)Tscy*7|F1=X#bfIXq1nP0z;H|WTYO5<|$yyCk`N38=uR|eopS= z^=twEU$SGOK81m z{3@GU@bPC3%guofE;#3*-oK(h3q`fGl+S(GT=U7f(tmidzcnUj>$I~LU-nV37O;b2 zTHtpigcmMabS2A3wT^5x`fDspoKfNsLoCEn5i@L!h<&PB_A)n#jM0ypK#4_g?&uvd z?o)76Euqj$;m);#&;2m3 zs=03(a#nP$f8LrVt4}jO@1*vgV}p$WzDtJJ1|Nqi&WRatjsGfiByulZ3+a2P6hD&p zVcvh7Li9N%5XAGAtYwSn4PkAO@ z`OLS~IIz4QFNq5*^b^15pw_wc{krscpH+B6lz3L}{1Y{APC1+et~n@LnitA$yLD@@ zqH(9S6lU1_TrV91=N^tfH1H0ZAfGMHHgMH#s>&h!M5!{1Q9Z_r6rvjgdOXT!^ zlvcs>)vp(#MYSZA_NRU)ymPHrbFx;IVj|pI*3gzd^?$i8zTqV6%NCfdaP?hLU@776 z*8+#RSY^?n9mm@9Z@hi*&C(xy=zafEweVu|S(m-5S*FFpB~+g3MEkWk4z;(Aykpxr zJNIt7mNmh9T3kwd)N}`j20E@?oB~6XxHw4%!1WZM(es*f%w=C$5A}Wz{=iu6Xi%|w z5^)~d&h9>xN7Ip=G7jaO^c+Hdi(e$e~{XRIR2M}N4bOj+^urvcZx)mR;K zjyeP#`&K?nyDTf%HsXo7w6wEcGQYL>z;u7GQncEfi)i?Wg0@c!+%|N##n*!EDz-aI|%;Cc4E}`yNBIywp2DZC5LV)xF%luQOU9{VC?mqaYLXBqjtKM(dh+C|g{{n)nUZtWS<%?IDKa*qm~m^G1uyc{@(` zHE|QJwXA<&tSXy#MDVs^x!4)?XejpP?DtlOkLn=d<^1lkoxvxbX4PCK8rPVJf-`G? z=-^o^?0mgfsdvBDs9Ppb)hh21bP6pqm-fGH1TTRrV<1|=nS({d;Xkc~XnCw)09z z>Sj)N_&04XgL8vp1G|OWj-#zVgtZ@M#=qZZdLC`xCHUI*?R|lBzyD?MwK%5EH{&tW zX?&Y8-Cfi2@b9i|dyiW9aJJWO8;jNEd;??pKJ?h{@C8l_qs5!^v;(Padn!I~=I$$; zcQoF|x52IVq2Jy7dRTh3bjemV^XKjq2Cm|+a0V;-S-IndV=WzC8On*4j zXLG(!hqOz%8B=;3E~jvAaBS{l=9lp+UC)vF!&vsO>YK4D;b`mDjDH{U^uHE%i}&t) zy5Fg_bNEi7)zUHKK82Ro#I*_0>A0&;{C4FRj z@>Z>UqY>i((mR~=ea`t$3q{K&*M&%}DAOv|JqQZ!>X(1cBuZaSBtrV+ z&d(8N1I|nw1UL$q<2m>^OVRKqF=S}gm$cFesXYZB%ak$4*#UgC^|x>%$*wB*QIp}j2D7!D^lw&`1gCvD-&_`#WarZ~)LQ}-cA#}mtHwJET$ z6m$O4Ra;-$Tp}5yEKtTCe75LnbN9QqHEyORo_(;hY@Im0=Q!4J=7N*9To`t1nW%dl z6iZCM*UPE+^kZ3(>{Z-Ab!=>aB{pNtws1}N*;8pR%&o=ee#R+~1#hqgNKtCf8Y2Z zt2@=mF4Im;KwCoLeRes(^mF51jm|zPLgwMCS)5cZ{1IKmn8Z)HrB}{5cnhsZMpd%EsHrA-@OA$}pEq4!F=1G&9Iu=$Rb_S3NM*M8zBe@~qQ z6$f#UlVF@cy~}Dgv-{mnuBqugw)AP&gnLz03MWIJ?r2R}C%bs6f}7J8x=r`EOs#wF z`EgJMT?Xoaa!CpdQR0%+9IzHrK*(PC{M^vwd98Z|zvFFhySc_INuteYv0EQmuU;C? zp1oN(g~?_hD$gkECwKeV zD%$RePsfJsItQOQ$-wi%|L{-N5L@GSwALdu&aS+!$w;(zT3DAi|H)rjcB1{w6+Sq* zKh9+`!ukrQ&;4{AJAhSRi_dP)xgX&7_@@42#m~w30;f<*2fEgbVcYvx=eiAN$4(_w zf9<>uwdWI_xzfmr0dbtdqV;&%3~9LrZrxL((060)%$3ti?yjtOM4|lyzKgsx61K{eP>s+tv^K>-3o`F|IzFqCzNv>ZSUlmF2 z(Cp4Gjj!&@vb{fj!fHq_egCX^Rsr;?7U^O+&im8cSYceN7Lnbo&pUyWm?T{S*g4ww zr!7Cv!*aj3wJEly$7k)X6EE+>9yRAvV2Bdu({MmoO#vah(dc+K}WH#=Z0MsRYx! z`fIENu2t(qfANI>jO8-OxL>xGs^Q(`9E~NK6|(7CJx+k%cERRGGina)k@d1y*LGfc zj?JJ+1~0IQwP;H7O55+=SufhuU8Psz(3~TcM6JOkGZlOs*2=oSr`@yn#BK2_KgDsN zCT%R4V~iJDLnB+00T+Ddh?i5-CZ@#{Jz3X1^Nd|if%kFKt*VYDW3cTi5?$vI7H5|F%GoGpEeK)2BClbwizQAI6 zsQ0YECi13xLgp_qzht6|6C~@R^};$GeCot6Tx5N2=@jv=T3$+B`>bVtm%_(V)7}>r z5cH)gkrJTpQBuaxTY)nFH6?L`rL7J1=dcSOyh$rTd}+tD21nbLYOGI9;+^Ha;{5_I zOEeW^+i#pd9tsRm;_=Y^8NtdJDIp|wMuI=Dw`)xuuA2O^ygDmuSxVaNhlLfmG)@Nn zWNrzrlr>rC1rfqkS&HqvXh>^5_U(V4i%VK_h&q{zlE_D^?UR^#M(zGJfBKZ~oJcsL zC5JstxkQ@5S;dCak?eX#G@(fju3UtZ*a>HUAFd#F7@+iZXGoG{{8>D=f(kiRpcz!!NwXyeDXgBk%Kc2mZXQ@|F0qmd=hL< z{ILD4a~Ewp3yCcaZL!74jK4hG<~;#ouZoicYmzm%ccJ{juGYk?&0PNUfBpB}YuB#j z9>&yv&LS+RL|f|~OX!)b)`$j|Ihfn~-v9nRr?|Z_O(AINzWnLmyIX^ti{>22u-g() z=bq8dvUoajPnS6T#mOmf`ck4^*_*sa`Kiyn_r#tpVe6*wdCqMaI<++=yy`iQvD6u# z!H3`m3`rk5M3!MvW&M&lw z5^kohe$H2nf37K(`Q`uZ->tZ4MLDHpj}PPEOav#5*C|d#Jzgvxc-WSyt@E%+0goU5RM%>KDA%>YURyir9H)fmXXoi&lg%-WCJ(C*7gkK!9_H-6aypzE z>s+1u?FrqsPB|0yaV@mIaM2PQ2hnL4`%%V1&9GF(z>$I0oK_o)ojGt|)NN3ymrQAxP1e!W=QwrkIeXmZ2P-F!Gk z`rlsvfwzCtO`cC7(pcU0IxRJPEDUY^p>x)x@=n>q)3&$mS!h6{AksTHe8tLeNrx8i z@w#6}nMd&U<>k~Eo0-;_tFnhgkH>!^&NUopSm3?8z1wpTBQIfXYVY4T9~5{L6d0n! zqoDFLi^FGuR*)8{?s9rXN-)OTo~atKiy<4k-5O&riQ2rXpMAjtXSS-kR2)P*R^nK8 zE|#^zXS6y|72H_2w(TX(Ik})A);gbTiT>1KN0({?myRX)T9F8A$)FDBAx)>P%7hMs zu@qmS=j()-qNV1oewiMn+9H3EEd1A zE@f-3I236o_M@_B1&^^AD~pA(Ep=4G;&V7KuXf*sDry%voP$_EvPB+eJXycOGP%cD z(;X=$|Jp6w_l!#gPf++Zm$8&?G?VyPm ztO~Bz%u?;eiSz8RUPEfT>r^A5DR>oU3Ax37!prIsM=bZkdh0;k`&<3NjjdQV`in1= zEjHn>CalLQ9&YglzmoX9bh|UzjPz+OX4Zjv>sc!g2BI39$PErq7S$>OX z+29lVBy?U~i>sgessBCCU`y2?n_ku<%T(rdD(tC##82u9I31oX_o5g3ruv>*=Q*JA zaa{WSeCn7+zh{dALzH;7?)M{{txx{n{NfjH+6)=jL|@sg^6B%*#v)x47k0$w+Nahb zUg4749Q9+bBFYw5a8^IqzTt)bXQx|Pu97$u+k1+({Me8<8PUYjwqn_dX_aj&^QB#F zwadp~ExBGb?*%sN4zWTX;mvVYnaZNH1nyj$B+ji)HBF+N~8bLAN|oAA`LMGi*H>S3)YuYV%zu7Id)0keur6^j%t>q zaxH82PziVIH$RW_K^5JJ*xFbhoSj;yzZMTsKWtV?qKGyu8S{8HSIiHq%hBbwu+8af z{XG2gmv4ws#NEW95EVJou-4d8&yg6PI<6xKLr<28>NHlyS$DxMR=l<=OG4e*`T%{G zh@N6{LND)5;kplJL<8cHc@gtt0jkMd3mu7Pz1Y@{H|KdCh~;Y%9usx6U+d^d+%NBx zo8Fl*;~C|kz@WhS6d0n!`7|65CMiH8hedN-BQafzlxB;A2A0TyE&Ts}{^zfP7cMMF zaJ40e9Zm@C*v>sS4>*V}KH)7s=he6WTqN86Soy-%RvZ#_ayTx@@mJsY#y#f{Vp8B@ ztvFb{@|ka|bk`nhm+St=l5k*pvzM^JfBX%brg{S2^cskaVh0d6nUu+E7_ z!n^}VxreWYo^Cps?_RqJSZ$T@(ImT%sps^=sCnVT8(`bLDp2*lS{Eo(*G0E6Js?sO>toRm!ad?-u@Q$79i}u+O{aE{P{n??w5G9_Sd;JJ!ioNt{xrPC` zp9V*b80l%#d+li7{@nkZwP+5j_OSSYClC4|xI%TYG1znCJ=C_n#wm<702_(b^~v@L z7s}eUbIw79^N53{Fk{4;7B*yJHP<->9f~1ZiciQodgk6`{+>mmHC)1QS$%SCs;v0Rnwz@T z*1^}a$LK|C50|5|`faD~oJjS>vXxxbRCm|b6(8aaNpds5B}BPdT`ud zF23l-wFDpQm9`wtIM8|S{HI2sF33GRBsz0I)Mp8so%N}G4$u30KlIxA?8ecGUwu!T z{6|ad=#mU^nGamfSy$rF2%hsQ5*BMwFLt}`edgU%x5HF%p5XiR&;$z4{kwnX@9cbI zX%nYDyQj>Vb~kgOWo=nNjw887+`!s!cxaC$TFG`?VK>1QzL~AKw82)?q_gE3jivh; z{3&U9v5`w(<+Sqo#7%u_h!}x=sAM5nntYI$1XkG5cK%^wf{RE-WJ7lcr=wicw(UE2 z9(+@wlr27NV&BGgP<;8}T$0(y7_3R{dsy|FaOZN(F&B1R!|S(g=H}Gavv$nkBZ|33 zteuNl!bW9GE!?^D;G1%c2|jBID>DT{crEKCoKx5Px;$s!{wKa_uRZn*3knVqrtg>k z)+N@SzkKQrl`<6>KXE0NT<IH9LlhN3VX@=l6g@T zU^(HPTQ4kUt}lHafaACG9D=pv z8tbz65U~#qhx?1hm&VnO=ks6qLXF|E_G=cq_To6dIt7L(adi?ui3n=emGVmS>A3dz zd~T|iylAkzUV0zyOXA0p)z@UfRYO`!**a8Q5*Jv)VuPnGFO3I08s5qVl{qmo*HG8( zJPWJ4>eG5`e8h;#1{e7J>#;;Wzg25r>ZTHG9)o%4y7`2DLRQmOXtVlDYL0uxI)oS-o)P_9kE*sI&Z@m|@hFMqH_+2Szx#7?x1dit>3ti=~67wZKZ?_MT5 zQ#(JRLpv_>s}|ZqgX-kfUYt0W>VUXFtcT10j8!Qfi(^&xuAxF{vzu{rOnY;mKyk0C zT1ktjRMj#ChyK()-T|D*bfAt%mO82e7Ae+X=h_puS}VdmBu#~^aKyQar#y$=XYCx) z)V6sw)x&m7nU`-@O<i$h6V@oANdJD>YjnXsL|Nj` z1wjr_PdNpKDDjlv!1cqY-MN&Oxyvo1xJ->O;r{)mMxodD$cDj7} ztAA@@@#orJqTam164ku#M}Tg$+MA@fQ%*BSx<6Nq(b$@zDdZ@?seple%29y!JlR)_ z(Wjf%|Bw?+i{JXF2}Mg^+EMH+A7Zd1^o;XFb#y9D5!Lcku7w^c)2@X*bza$rZGZT% zQ8oT!TW9Ui5OTZ;Y^63n{m%JZFBLc1y!P<<{8p{;79V&Vg@P*vdYbj>%u%jOEDPrn zvAC}LEo_b()&l1l4lS=_htVR3S?!xBv%f1jq1nVWRs1>X$jyRslY8h$hj18Z1 zg6whmFB@Fo$~+{+LygPX!jq+`e|K}KkiVMCfayDZP?vzv{+Fk1B+b|dYFZ&iJ zyG!8;95@qRo_Tp^%vqDt@0_we$6LC3Pc2z%=ioC(-v-ChZ_7FOM&Bo!0z;H|vhU9Y zN9T1<^Fc$%-~1GocFk~TeDfMr*4?ew09euXVbLImeXf4?#R-5&G37Y30_<$lh%M$t z+le)Djy1eSCa2)T`qIX+8YIh6%_y@SDfR}-!LN9B@Cz;kpKY+x9Bgys06}X=3mF!W z29E(dZP%8xbM9A*kGQK2PFiy34h|kPh1i~@;gD0~p)qXnX;U?V1s|M_WWXUd(VQOh z{hz{hc;wli$DFx;m3;!~Pr_o<)XA$*I0Pc(ezi{DV`MNDYlE7ifU zncpcI*9mmbRV%?3Zo*r=(;}K>JmPMMZc{MU)Op@&qT7NKY>q0{p*6x`1gjsGyiG|E z95mj&1tHhwN+Pwb{XAQF4$LpH6IgEbvOC35Ca}C~Iho}Dk`mmA=H^9h5!$9dXBpS} zM73-2aRT+81}le{l;BAz#*CQcJ<}!=hmVya?Ut0cvTto;$%;6$OMKk-B}Sn+x^ilD z{S?u_|c013Gi!mG4tVj*x?!cj77}fOhr>5$chxb*(AH$rXq@ZseATYS#J3S4W3rW`BG!72552XI0nIy%AvCk~~uH$Cu) z|MCB>;nu~fl=!3;-wOw_)7VnYxR%fp`h>ps?2q?F=4UNbQLP&uH(OSi!g5 z3xW2(1&8H?kM%`4m60ekSmNLWHkjUh;LYLld=O1(@mmKj@0{usRQfKpCKy+I^=m@p zW`dcYUCX`mJe!=E#0(MDSj&u)MK;fW(|z*PebL9klGuut?u&ZXorGsw6R`wxK9X{R zDq#}%OeGaSz7d=Fu?IKa_t|e^DrSIshK7Fz9aJKq|= zMoi=h!x~*4F$IPw@rVij_|Vaoc`YXA`85roKDk7es@>Z9p0>#hFHME`EYqy;Ev`;6 zh0VP*9`HE2U}+@@41Aw-pE)dmZMjl^iINkB3@g3ZQk`%LyWeg>b}l~m4ck5yUs8AT zEn38We}r>z5(#Jku?{_)80;^JuWrd!@Nt;x@%hivgN?xQnJ*2YGRIV(Lv^uM&EbnS zGMZVR{+uhZ|3orJn?-95XKh=YvwD`7V?`Bv3QY9o^Ny~xv+Y3({X${EE6Y~e&EZKs zcCWubLU4EexA>;kQ(!2Gfc1URR!*FAHfr%v17MlNeB2hTGnE489xNd7eFDU;YGa|@ z{hd6)##$4?xH+g`m55%gQ&7iSw-)1 z%{;qHyT7#C`tK@^oM^v!pWJVVj@W8L*ZvLJ>VkW02xLDadht_6$&-=dpf3b*DVqV*>T6VV2 z2Pd++Pf+=9-n~Ov+wYunmSoPH=;uJoYvIZ`J=^ZJ^BYJ3E+-)p{byqpmoOzAA7AH1vnb4q5zgK`#kuRuIjv^vHL3Y9Lp6J zxOKQ%0qST|pI+h1G@95s)(JSVcT21hM-em)WY4LN6$Q4q2A66HUwL+DW1HrSPJKr0 zwZd;3ThjVy`=px-HYUL0#Iqz8Cw)?KjuMKQdr~!(#bHJEEt-Gt04z(Z9c7Wweu>Bu zJW3WZ&OTsb?H!f%+`ennvv9$8TiAUTuCQ%HfM|nTn1{}*(^nZMaR^OmTxqDS`I<(e zT&p-snfVJzB@V9DE*nSifyLs|j;+SElzN$P#I-o68kTCzTUa@_w%jlHu-vS2uCdf& ziPf*&rY*4NO02Kv2yFKfEoHm%zs60?bv&b<6~jAO;`Fhq$*PxCV|a*Q|= z#NzHg`}}noN#jJYxbwn>jJnk?yf|Oo7FMG=0c<~`3qh=ER+E($_?$R|+;Xy@;i|0H zDoD#3OSZ_c-fCndZN(BVgRs06v4KUVPnPNPg_T3B;{DR50WGF8h>iDx(t<$ene z6xKivc(C4lx3M;0_q}||Dr?YRIPl*mo>ib~rwd%L9qtQfj><+A&2Q@v5pbRh+Wh=K zR`R*#ryU&T!l|ULfd(%}k||i&1I}?%*QV;~ehgp7oKHFApf3lhD#}a&@9Jpm zvAmfZRuFryS!IKpad47r=}C<6eqjk(`d+I}fm@i?Uo0@<7g+Byj+AjCd|2YCs+Pa> zt@P>UG4+?mJ?jpL_xgXH@$zo3+28Dc;oeBwur_(^ zQp|;ZuWLizSs1ACmi8a*1_drffgwsyI2YttBZ^D{Jy6NQ zfQ=H9LGzF0E+RtB#&w`uy+kYi<+7CC8s)x z6u4kJhzrCN7FZhne%{C6AG*$REG|i04N;S*x5nA#5bSYnP+(ACP~Z|2_=c}~&5wWm zul`@Y{%e2u>n{Orz!(%56c`kE925}OP?Pj7KwjGxb`#6#d_k98lMRldT`lbM=Ga}D zJlVic+q?4ESh2($6YuamSmb=v_i6is*$ zN+0*_4*%wCQ@6Rsp{bqsX?TL+LG{_szm)ga!G;dY=@v-)WlN*1%?)ku*3KLa_8~3% zXS`@v4@s`;?+zDQ<{mDCxW1J&;Aq#-dMKmEpunKO$2i`$}L4g?xaNeMe zeRzRw;$~3PEMvBhqqUEN4$bq@t*_?bPSu-F{P^GX{lD>f*D}V4aZE5iftJ3 z+?dzYo0GLkWDtY2)X{7Wj?s2dU{K)IO@Sdwyt?o2nBAbjQ$_*e0|%6)JGXLjk(~w0 zw|1&n?V5?Ju!O=QkL`_tJ+2vqN_%YNQha>SYog1-CSpBh=#hEI{unj9wF4whMEJVv z6;HvnCbov7jeZh!CQc$@>aEtCU$rd{`Sqh8)>UXW|L)I!;R}20dN;(~U+ERFzi^dI zO}jp~+gO6OPOL=cL)^ilyMHx)X$Nus#JVs}G)%Z|S`%jy@}H!)wD%mSrZR-3-=Q*V zV`_g7p{>LrVoP0nh2Gk8a5@C@(6xxP#rf@CJY3D#WX+WWZgD7y6FCI)(6vJU8;)81 z8uaYRx@8@pX;14zFppgm|IsIo<@Cm3ZE6?lWAMiHL4iSm$4`MFN<991Fh(#aFetF2 zz!V#`yLiD9e)W9bu}i|NTbB128myM>>uPxRvBu>(7C7uWw$mX2`>aSQfK8q9;b*OQ&ENgXjaY?tY-PNQG9D%Kj_g`V_)McNoW0$y z&N%t{*KOC(pokx|;n>(UnU!eih0O|U&D>d0KK7y4R(e`L;!uS1nzK2{D%G>g>3h6< zP^}GV#|mTYlEcE0cno-0ag1xtd;&KmI4nd$nuV~#mHVP3UaA)}(PrRE|FtDr5w((- zx#IuQe^TeCuD3R=g@33$hbHUn-2D^|;TYEj1qKDKPk|vyT%X1R!=S*Rz+wsrZN=uV z*`|;txH^Qa*d1CZw8W^{%ISk2{B1`=euWcbOT(gKyOvIjHA8#W37UWB!lR33_};cu zG`F07T0BIL?|$!lckJ^ayquJ1uWxU^rtXBC3|?@yTR37NjbRT@No*?d?oxIbJ1Dad zQKIc*8?oSXPwl|YDM(#SMqR8Dc7bmD9GX+E5hcLrw>OiJRlT`ywDude+qyW<2Y=%z z#d*mXh$_S<^M%iWy(e0phi5ax%!4BwYg^_55BhP`;_wuDoLdro?8A|{q~v5JdGWdM zr~0#|5Fv;$?$2v}>F=$2-lt%WxWS~hC|zC?G1G0AM85|h>+U%e&i#SJ+pK08{YU2927-pYl&1e)HMA1 zIzNU!PFSrY$xA==?-i@W@4d$wVvC6m{Qo)8)Ue;JBTbx=9kjo1nBh=@cABC3$-_Q)O$Ev){Wm}8gp zwqCX&u!#~Z1rpo7^zzI0HmM>-+ji|aM>`fW;oHO>&S!Agt3pV*V`*&3QrwBTsHp(T zyu^GE2bcCgwVgFd+`Xg-cIsG0zd?aPfzv54M2XXBH+l{V3<_L~0vui5{f=K(_ZQ%F zWXrFQ{^*Zhh0TBq>q(=G&1a#Y(Z(Jw-M(zi#A)Th{+_qIWsg1cpH+lrekzV(4UV>L zc150Y9*W};OGMiz=UCZfDI#Y5^#Ac6?6K`>*Ou7!inFaO=U7CnI5wDLkpG;-IHcL; z3{4!a!?P40r=O`^z!PpxRC)8C{FR$A73%>U?lqCo{Wg|vs0aThj)aI}TR2AzA>UeK zSY?a}PWN<8+|3kXj#84d8h5N9#5KW==L2N74YSaR?=JmN-j{(}O80%uS_81hS>{&dxlbBdwe&9TG*Wi4u~y(~hq z?+6!eSz8tY4l+w%(rhmav%QNCz%JX-nIp`nKK?szBonpg!NUXlXUP6UL!B&6$HtcV zeAl1(RnoQRfKX3#uvS#xe*|J5~Cfm0A%f3svC`dgp<{Wrpv?K6?ND2Zd7pw{ekZ9Z_i zS1qiZn@5X>h$AjVV$*X(3o#}{uoiZ3D$&5Yn6fw^bwR;4$*=}kA++NO?m zMO@U|zxOvxrXV@?K>=g<5{i|~mN zNZ2lxm-WRVKBI^6mw)(o+;GIfDrRi5ABnAqMty30sr%$hFIA^0tS=lbdyidH&jE*n z6DOUzU&U6NZ>u$|l;&X$fAfofxU%7ykG%@S8=S^!Kd7+@!J%XU zA$GQjQ;9+5vm`#Gh1H)EKg^LBWW3@?k|E1{ICrr&T`H>*D~r7_h<0_i>+QWE0?Xn> zwCu-t&i{}8$dA-LNXq5cksaHxtnW$iTe7EC{A@8>l{l& z)=S_WFa`yl6$%Vd;#s-VW1fQoXHfuKPt!_!>{#Lte(KJ-G8M_&AMGed7>-C&+It#) zF%=*Ek$=5rAX+m?S$Ap}Zqf3xNMN-&%lzp7{I6BquYBe+m2ES3qJ(RlW->12Qhc)S zy!gVet=p?{(!o-tFHs0AKKSt57Kb0&_29ws|M55ciW=IRK8H%0Y8q{GdiaaKxW{@5 zQlb;jc+XY?<_NL34taurOld> zMF9+3tM&ajg|+)UV7+B|S|XD6u<{?>S;_i6GK456WE|^n2lLZJqhQ*ULb=`h)Do+0 zX&4I{kwPeTuAf~NRvtZ*DJqsLEWUV_OIssYi-eIA&m^H0MlL&5n{DjuIW+*hgIL8* zeHS&L#_6EIpuh|ThA1&Zm{C0_FeuPc01N7KENN$}>2BLx;v5@HG{`Q(*c|@uNi;fz z1FJx!n7VeT&d$$RC^)df*@4=BW9a*`w9s;ETX#b48_fJCHrPpjZ;Vt_j~2pMrI5S^b)@wy?=doPf+L zxQb}`ys_^Mru;GQ%%LyMxzKY_%mTRY{PTalj-6u>KjLlBm=Cj61b!Z&oNowp+S$~-ekx=HgCF|`F?73@1K$ZwjZ-4uTfB1FR zvbN0OAN7L*&jJO8DDf=ZBdlAlRl){Po{eSsW~D+y0z$Y`hqduug3vQ&s_1j$Y&4;(PaJKl?hq>S*8A z&z(y+3l}R5XPmIK*eN1V4gD^ijf?oDy!FJh7C$LDzNar5fv<`XMf3j3%imhtXU zh(s&}HDgnIp`2IosKjSfxSdlK(MSlf*nl|c$uQM{xcArI7siK`&qYpX#3PPy89SVI zz_x~15>9n?0s~B$#)x^hoZZUUg9i;+KU`a5(aG5Rl7k=EM5EyLOrRC#E>2^8pL70; zIboF}>RK1}uV9rp2j=KIC@?7ScquSMiN{O-fqhWmDWCxDsC@a3YdZ*Vnz5TPKWeO{ zM5Wgvkq~Q6K(~cmqrJB;!OK7W`>ukIMq3E4c$3UWYy#)4{6AE309mp%*~20h@Y~0# zwU!K41I{IL>@)UQs4r0p?S*I8(C3SdP8&M7sdwZP_dlh_aV zs#bpmtYc02x0@HZ8!OMIhgbrv`z3eTwl>@^mV#;x*$xhEZ6n7LLlccf)%GV|`GYra zc+=OG1fySDM5qTlZM%CQ{QTJ5^mBh&M=|0j2Q~E)3uFEH{`=n^mE|vvO6|HjhEL4S zCZo~6S;s~csc}48*sNJ_fUV>(CleBvTo`~UL4uwl!JHW_=y=Ux_`vk^1^Z2N`f3k!%9<}~66OQ=oL<>XUNH}*fE z9p_ln1Hxw0Jl7<%{l)EnZAx6n%@Su5jyUE8*Hm1KdC}}o;d7s5molD8`{E!qGo)R0 zbtWqJX;;ymwDZ7b&8<-A%3@+On}dVU zC2?^Q;g|ylKk@2dQ}%!0TtzD^+Y;^erEtdQ-j2}X5?Z)4z{X+Qy0-q*^O{(*z_WkD zRJN`7_=^=e6-V2T@4NAH@M-JOm?r+=Fa8(xx-2@v-D%X_=cz+t*%rpsdD!=M{PtUj z2ESc5?Kz7K=OpuY?AlKuY3i!jRpv*``rMbkbmOR*uS%Wgg0)8s1E(|&mlpTJ zaV(%bOWCe3byKA@16i9HKkE*;EvnEW?XukEK}s0ZA&hpz+syLBk}j5F4$ocvof8(vFmWhzuvss}P_SBvd2o*XUm`vPpDhACXK)ZtS~`gx zdhyTwnz|>4eK|OcrQbQ_!xCS^f`y)*ZHaaWz#V-D1qKD44GIiV;@P;*W0r#g zCr|);kL{&x&T;9GPs3v2pkb~_NsHqMt#c0ASbp#)41cl<-*EosfzJ<~RREiajiyDH zT=Y^nU-<9;@rnzZ`N<#qv9edh-@eEG35B*T*}8Dx$>}|3KAg9TmiZo@^q_$@|Ce5V zdB^(a+WPWA6%*$Aam0D$bAO=5ObF}Nj*}IZoJL=)2*)ALZF4Z%dhB^0Q(;ZA6xgSN zNMUYbH-7M=uiGRQe)wPQ?sx7xOG&X99L0&X<^TsaQHfa8?~M?cO8AL}oT!9<=XjyS z8j0U%=Q4HfSzwuAHEH2G|LDhm=kvsB2WKbZ-&`yMv0#luB2hwI4e?eC29b2h?&?!> z7d%u1|$J;V?;m8i_BkeZpM(++>y7tdZO! zxE=1B>(-G{ov{#y*WhT|x;H(}tjR;AF@zo|H|x2DK7DSyn{gZa^ntGKwKW>}N_gRM zep}{0jbYRd3cPwLP@b9H`~1~A%O`7QLzH;3?#CF-puiJE0W6Ie1qn+zRtTYe-v>T$ z!{_^w!+cmAENJXGt+dV~Viu~SM@bfXsW<|f)`S9s4F~p%f4fAb+2L4%(!!>l?chsm;T~I(9qm%>DI@H2!$5DFG(#c)H`4NhI{uSu-a1C{3aRf zb9F7q^d(uVJx4-kRxHnlE%)5xn#{FWLUKJNOGyiR$+_gR>ED*^eQxV>&N5}LOK`Mp zJRcnLZeRb^&x`MI*!qvNT>dLrhlGFIx5ROc3~TMU#`&PYpum$$fgwsfx%Xy_Y*64a zQUD7j*Zj4e9I$*?>G<~xf#&?ex#}9z7b1(@;=lw?SVGQC*TDDU3%|BH@Pv-l=>-0g z=<^!*u$YpC26nZ0ToE75e2WhYTIsuBe#Ap`&oQ^>MdqRu_8)vQS`qo+6DCZ&5Rc)W zP99rPBkXp~kkkU7IJecDScf83zE12y`W`EVcfbF=|6<)YL@2-ZYs5Ft1*flfzvI_E zFLt3nh`siMAR5{&okLTe86ws@-uccdI!#PzR1FFY3JeNdlmbJPxF|UXut9-W3kB?Z z@b>TfzMD^c;@_xQX|QOLbx!&EpF3yTQ(>@j^wTI~y=XzP#eCZBEPO7G^rxMsy%rkV z#wM^puo%4Z+23EoUG0mIKBtxp?R6iH!cvCK>tSKN&GQo3qU?|?qP0Wf2R~uR-w` zEDo$%GArE{#~xd{iL(jpltdrrEGBb!QjZ??#*j^kqY=>si_VG_cqv&UB!m@KpBGj<$mWg95L93Jg)= z)qju23+3qv4J{-~4UUw= z)LuO3@#H@%kAB2KY;Wq75~Ye02;8mJ*3uEHw&JD6_{Voi&~Is9UA ztdGE0TU!76bNZ8I%3NDKL?&ysb-Y@O<52szzVVG!w3h^zHRPPR9KtiM4GIhjygDc_ zM2T0&eH^nG6qulZOgWNKa;$j6cYMbl+cnXAS4GQPokz-Ux4X)jld9=1T4q{VTc3sH zdHeVMvq#xpAd4UNjU=V|*^UnDgzcQlpO38*S}R)*kpSDzi6l5@hPYQvJZy+0QdP~%ye3&e@KYjPR$A%`V&#iknD(zU{J z)#I~H!KnK%G~;MbSwbXO)sD!sRFtT(AH^}rGn7~WPI2JKutYTBC`UYM58}5rM8q}L zE%6?#aQeRM`+sB2_$9u=|G5J$&w%Vpy(rhxVD8+y3~W{$_*qu6mrP2QB;sD^FbD5g zJ;yB~(rvLEu>yMjkM$YX2L%QNo@@#XQR2zIKVx)*0w+)adoNT{GE8g`hl?rJd=4}x zAZ)K=6OR1#&(qBMFNc5MuWFPF%hmdiOM=?B>;FqUX)TtXN@7J8iId={r|ues#}`iAH7nv0Vu7)NbC`^12sA!B=BSCSbHx+5ccJ zyqD#JxMBP~K7Rg01J{UJEEp~9sq>nkx3p*J5b`_KM%+!0Poh&IqbV%vv< z)l%yYn5-WX=@LKT6FcJE!Ff$6d3M~+e3a-<^gdM#@IK1^AT8YfoaJtbBU($Jx%0Bt zZal-DbL-8ygLOwr0LQ}%q&?{IvkI2?@#I?V+m8Q(0#7RihA8p0-nlXEL4kV|kY}Ft zL3!`uQ&2%n5mNhI-}w(cFSq@*hD@xYQ4Sl11;^H|+475}VoI!<8naMgY!ViYrrQ1j zSTxp^srGFhHk2dJ4}IiYHtcD8apIh&SN0zG>}`-HKlsW0GKI- zkE0*4f>k5U+(T?y{-?x#^Y!N(l(+@`g*0ma`86e~gC)T~g&@e9k2wnP=8 zyge?&Z@f^F3v;AmaXAG~=Ifq^rp8Y66Bm-}*C>fh94O~R&}-lvFb4$&1+GVdAxd12 zwgd8@z$2gl_N~|WUK%YNG7r7`dwzY@uybUx8}g;`fM+f>cA&5|Sigk0R<`ZJ2>;6W ze(#OBfQ?1N<|f0E|Cd6?0V8$YNe#OJr)k zM2$7t>to{z`^&o2#xEEL+7c^{!B};@z%(ywBXe^v+xytUb5C;^3;Lsb%C_y)xxU&H zecUHnbyg#RK2c=f=FMb6c6GdBr-=mn>+;#7YxdOwD0Qo#%X? zJ}=!6&6e~XZ3YDf1)g0B3{m3Qz4v40Pdx=VZd}^}!Poo8zxao5uxhe7VP)jyzg8$V zhpezpaD{d1osKSX^Ed*v&K)g2I}2X|M_?4zedjC_R?|G|TnkHjC^;02FxGKraz!%*5Q#k+`mNu6--wrh4) z&d$;8{XP`)5E!;!3v6W#?cS~xmKaVY9P`k?Q8PttuWP@<_shS$Fr;`R0=D-n=M`4D ze^dX7R^pUC@)JL?H~-9It`wHv&adKV#?R5t`pn_Eq~(`^X%6P0dT}wtp5W~9iETL4 z|CDRis5LlsoC0t3859^4czhHXqQv8){lGjZa3%%#@ZT1;4@+?@VJhbxZ5^keC9?1> zZds-|CnOFGamcC8HYXso1y+m8`Bc1z?XiSyr>*@3)}rI|LgUQIrfx4*SVN(#JsjKf z@w2Yl=&~)@u`uSozyEx;i7(heVaco?*j?eTY0s%CdtPAgZp%Hhzd^1qEjjVTm!I=c z&DvDxD2665#7ETq4odz1{pJ7s`PPvqZI+f*<6hvn=fnkDow+Y{o4x{s7?juvv~DdJ zOL3$fC#5*%eC6T8O1s2T2sP&b6)}A%Wxvb}iVZxqMrxd<_t`#D5hyxxm&e~(Ksv=#%V@~B6yB8;tzNeINqT|d+{IM27 z+n@QVpE~(I&5Z?YG_xLswpS;^@_hfp^6gN?ThLqMa{m|5Z8L;Ny5F;c5)vjth=NfkPiNn~$O>8cvZf<_+ zYurX*C()zr+kxNvYyX5~h_W|DF5a?A6*wG#QYV%}wj%6lozIqa6Qwu60s0 z%{z8o@>N=A77^kGHaqi?G}ag$?|;vGc4N%A=1OdB6{`w9Vp`gXVPL5+4`O!aVq8Rt zJ`-5_9V=@lqjIlX3){nz?MD`(&HdkKp@}V*5O=dZ9mgeO!+N(Fk1E@5E4lKl92@(R z>!Fh^4z+*jn10z;Iz7G($AL4m7M02|Cfq(5k$FO678r`-RoRup^Y|0TmR z?JoR6$xKu8@^@O;;KOG1@gWj$+P-Z~4u7mGHn*LVM3LCs$}(=}hYfz~?bp=Zg@uF* zajl`?Wqhrp&@r0R^7ptBYHe5O%#URNizptYZSQ;&`g;u1bI*wtXyXT1bs{Q-1rEL* zNB*04)leTt_Z)41`YCT4CpuzYWxri37LT+^SskF# zcty$Dk@*o9%=fm;v86E_+E_p0T8!ls%LENQYe4YLmGDV}uvFBWo36D!heN$*g1Bq1 zlGZXa2k&V8mQFoR&XeMy)~ti0xJm=oycQQHJ{i=6m@nzSwha{vN4^{!8rQ-Y?FR*( zHVO<;;%U2MW4wa`l>#({oOE(rL-WP?fseV6VF@m26Latrw@uuoZU;QkiU&tuVsD8X zm%y3KP;jucaJHb~wjV}|_q*Tw-m~Y(8LYB1Tl&e&lQ~G38hqHpbK(cUq@9da2VB~0 zaSJRD*xS~>ztys5JZno%a3mZ^RRD>T2tq z3(w}flijJO%44__8xzn70z`>q^}(_+Z1eNKG|Iv@lR6PO3c*N0#0EO5DP2u^wpSI7JgJ{sBDfH8Cu7 zCDO1e!CRf=w)GwKjxzvTH8Bf%=s!Z8N-wWW`c;W&AkEG={9+X3-EUc;oMXi_5k+OX92A_$|BjLUG{2R$|v^$>E!GT;jY!6p+oz@$k!EK1zq|jyp}0JySoof^YLl;uo_ zp2^x(6BBP~36JN{{tw2{(%@3(#58z_+O|<#vvuPsV|eQu-*_V?gJa$){YRfcfkA=C zOMxLuJYM<_?1KU)Qb2Yd;h+wh?Q5|2G>{TSCNvovS6RoRVaDDp(XuX`p3vYLBSC|` zWt&(b65gyXi7}VLoND*5gv(Y(kgXOOONhP4!i9B{VF*6;Q!tj+p$|44ELI6tfhk9{ zngFx3IhN4(7~Yx*slXu?9CDQ6D{f4QpEw40?z%8mZ0Df_yqs593$#1ek;U4x9;U`a zJhV4Q-E*R}v$q8?_#Ac{9mQU7?CEhz#)^iH`b=!ktC&+nLrROwuGuq#HNej0t(B`Ega7Ez!|(r#<+y$QSJ!pUA~Cf#(?`fU(N;J= zF_B{xktQ^#Wg1_x@*Jpum$u0U^NH6IoVhqWi?3 zml~-Zl35F)g{_+E!m3~+UAq(>a6b35KU*`$aE4+1pg|Va9Zmb3_WqPH)E&aHfkpEU zSL;|rgRj2bP5BfY>EqsP;u0S0WdX`{^Tvu@ik{*Xa-XK0n6TqZEC3nbu`;YQt$gc% z#)(V*dDq+ckYn(UUAMKJ`St6;I#CZ7XPjejmR{#bBQAl1ljqG55toyb^>U5puR0Nx zwP|hSex~&NnSXn1cB+2qhUntiNx5{MZ9C5g7&+`q0+;td?srP}-8JtC$!uFV=bl?n zo=u1Qs%+QTx$sB-L4iSm^C>VyiSubVAPfpT1`7D3VzYD57+zx-w10LyW+7-bkYWuk zjgKb%o~E;~QSes$GYDl^vQ1sxLT{!`TaC@+xRE*0u$qs#a(;oYO_rLzw4v}|<94*j z#hH?2qQ;bL=V5=2)-f&Hb;TkQFhBX(&+Z(Of{O(z*XK%M$Z5~NpB6c7C49w-CZE50 zn&_I;wDI-Ff)WF!wDEzJvSz_5q5_Lj70Z@&6_&oFdx?fYe^aQ z^>2Lg->Q2{(3YEnc%IB#nRc9XSVTB`ErH*+2^l4Xv=xa~w$eu~toJ*=r!a;H~dHmXVc! zBSzovl>gW#b08YnH)9G0(T2lHeB`IvEU6QZfb|qBz>#XMPaQ{LRS&h9>zjIG63 zPrr>@+N9KG&Jjk8#gcnVQ;gj{CdyW;M44Os;mbJ79uNczysnJR5NXYA-<&D5i0E;DZ87so;(7;Axdz#JW>V9(Y4?Em$jR`}>8 zjM{dIVp~q#hhq*;>g_K={FebLb)y^<7!-JVDKJEdr}y5Ckw3E(z%I!Ogsr-E7L$tW zm;PR57qK%~n{!#~LpScWXm#OXC3y3ndQCk)H-=yiA3t$yo864DWHh~Sv-r&QIkjGh zFIJFaQTD~K&w>9BOZ>*EaOe8bT=S<-9D__$98v5j-uJ=QiPsQo&>u@OSP7ZMo+tJ~ zNHm8Rj!3d55!>vVjP=H9w+UkHdM)n2-Wu>Ze#!ZYis}0prhHb z>_k|4_jN7&zCCswzMe&I+Yv2jzB$me^{1V0c9u@YA2co2m6B!XcT_fB%z|)dV-l)- z8jibOwDehnZRuXIY|!MVUeaQ94k7UG{m4h^K?FHwAqqyS0kJMQ-uHpuatFVNXGEId zJ5&-yrkwPGXRaht3y*iO8c-!TXN8gMH;0T_-UANSNn^Rh`kfnBThBq!nz}}+i5AZ| z9~2lASVn;%N-U$(Xf-JC1W`aPd%L(gIAGAKUt@TjQG`5teZCkaa>r%tP1v z@p9_nj}8~@Jxz6T*8A@$+RS@eUrspKFV0FY|L_0)Vmc&r7d|1;Mcce_=-A?)!pCR4 ziW%GSXw%PQ>c9KNQA&H_l4P)HM|-c{VXuJH%T{GezR<&UF%7vsSF(7t$xyL?ZQD2h zH~--7exfJQMzUSbYZX^v3DJ^8Xo;2`yo|qQt7?&z^LCeX^Sfnr+Rj%g)6vv@?ZE8* zw)CgH-(4st)u+F?f8e+DW3^cAw9~-r@g-coolo$gMe7vRj`PqtQ2~q`$yz2GRYdJv zQxfIg_x|_q-Qz>>`)lBO)-neoJL`}&(9>@dQ}gg-T*2)=an$PJ=fCHX#qg3D;)oyC z8RxgmRoOp}8RI2O_t08#^v~lZ@eEjl0*{LVLzH-2lplBp1x}`b&urc2!CnvBLxR)d z<9zmKz8w_nN7W=_rwWV7p$fZCyzt*A|Co<_3-8c5Jh6Vr#6!y+Uuqg%mIF9H`C~tJ z_8j@hgC!=BlfMtN?$vf!Wr z%hOz&CH2^AaE@uf#YGT}TI&V68;9f1{>6V{(KuoS;q)^%K0AKf*6nZnum4WXC?pmG zyZrJ`|GqlbU0g${n}d=1kA3*J*FM%tpQWj=rr#oNgBXSeInbZALd=T$DfYmB?0MkM zmF@>8La@x0r6UJRQgEwLlq|MO@<5Bj2S@d;?LS=S_W3XTp}lS6+BP-D zW{dCtZ|~k?ZcDPluwN4=VIsvMK{z;o0UJURArs4aTGO{J2L;=*1(F{^cK#sw!}%wQ zm4_2Wv67DG3n4>d^-K@W+y+c{^XzCe)5aJkXkH;O5a@Zx?isa)1|`mDLU?pA8YaUH zj8eZkXPvuFeYN-5`<(sVzPE3E(*E||wX4>uT2*_W^{$ec=m;h)>cO$C&?vX-UDv%F zO1-D={JzP$9LZ?W%`I)SxCM!7ttVNQ(1(cX=;6p^*j%t$D?!v@^}&V{8LTPv;PAsT zWOr`i@5MQ=j#)^2w3|Fn&`EucRaLY`r+qzR!Iq!xAUA=JNd3Y)|4^kB9Pn|b^6p%e zAa#$A^(<&WM6owVtXmF&VwQShceGi2-s2-`v5H(3OI27w;E_sV%XlU5V;uR+Vz%Uc zT0E|OMKscXu|W=F6dN)p+ONei>JJJ$NE8^N#Dg>&qc4L3Cn=B}SUATlaY8u>bfSVS zu{g`*pI@5ZtwX)R!%|}PD!W;nA7mH8R=B4Tp4)P5qWj%q^pZ3-#&lKK48Gth*AV4B zn*)Crz{oV`_F+S#+AGLwyUcu)@(JtV4tVp!N+=q-Oc?t`})NE zpeM&T&LolDjKw571jcoQq|fywk^>uOojQDcv%_)neh%BMTO4u3cv$nCxojsm z<#1#T2mNC`=lh<+*%Wq}qk8Z-Jek`#s4<8calGlxZx*XktXrRV&1y4+OSy4gm3NON zLSkmLb-&sH4Y2VyC@?7Sno?kh60a%w2l7FIwG@!arD#D*CYs&(CNrVbEz|a_Td*h+ zvTnT%l`-&RuQ>J^=g>UId8_T)(3>VV#^WplkTCMtF9J(NOj|oQz4~>RV>B$|;&hbv zUcl_{ zhEKB8*s?9QS=*iebZxVAJyq+)e*WgxXC7~^cMvs-h@tPNhK$9I#JMq^bI)rMC!+^i z8VAwO^{Mxq3Lx6C8uoVSJ~dw2`>mWLPvIKR2L%QNZiWIwl(-qX4yc0yH$(xMVQBF4 zv1NE-?zUJ7zT#M;e0n|F{46 zmC&Q;OI1{e-%G_}>Q zKI19Z!lR?!)H&Et_^^cGD3bBP5$!+wxz8Q8w2MAEb7m?1IEjyQQna&OV!#c2*!Rc^ z#OfqzY}9w=e55~1_eThpU1p=mUdG`~79sGkw5&1WQvwg?CTq)DG568OCFj5=p={Lu zl}~>106TEx^QpLzfYUn%KF)+3v1sXIY#E2^>U30+2``za9xi2p4?NC;`Yu+Z#f66A zaHilaFTZESiHu%V#47j$r~CJQ@PixRx;H-G%jaKyxz7ES_ic|)`h{LrwdLbEt_csv zupFmieT%(vM5eLhR^x6^;Nhmg5G5Y&`5V1|q$nUG48MIb3>-NeM2oGpdVGDsZ4+Te zSf=pkZP<$4^x0nlOXH|<7=^v%eDaa^yz#cU0orFV8*ACX-r}hJE^Gvr7Q39;>Zp%} z)0gsW726r@^O+M2SoSr@<#COT5i*aq+cKF?(w z3OI*z5uwn~?$qrOtZ~#8pJB_osqpBw4;b+L2y`{-jeyHP0%e8GCNAk-wBWf%1}xo)*Y zUBhs_z}^+1#m1(`b;Y&v{yOyQB}AR#^jF`D7*Ei6&AnKs>(Hl5WJj|uBK-lJflrzD z(epTVJ9>XhW$DrTT#+R+eWmW9`ATEC@?6n4F!fMu?-yultF&NUy92SW+Sz8$QcVVJ42+|`z<&tP(CE}1Vs^;!b^?H{w&D6{&+AjPfy1;|o z!>ZV_&3%srZ9x{R%z@QwpY`ssm#h-3D-ml_bMqH`buWuvqf$nbUbo}gvsg8@Vxn#o z)6#Pe!Is5t(4K*%9E%opYkA2oWk+h6dLj%R9c)?l>mT^QNh?W+nCfS^L8FY1$e10w zVxC!RmW!uPRO>)>tlA^w#KA!a+YJ_$uv#bzD;gSeXHH1^ko$S+I#~H^<*6)lp^^SS z{T*L)FX?N!x1Qee_Af4uJ8xdRO5$}K>x1pR6GELGv*0qOyoPeSbFYPS6T4zO&YaIUNZp-E%6u-e zeLG@j{q_}H-Z}5%g&Y0I60{1R=15dok|}@_ z<$j+09NLvwdzy0RXTI;Nx_vkPJ4HLIjs&L@V7tZG=DIs_X=_ZM{Q19Bw69-Z!lb!pPV8vg#r;)rIMPq7P2TT8rX%cP)N?LBOOGwt9xb=Y*ZkP^lY>>Bt76%x z7L|-oJPc8c_<_x1A8Wvpu5sR>L=pa7k94D1u`P5&o z?XAb$H=+p^4s6x}n&%i7&HFif6!4EXU+CDf(^zo09J`pC-Q3z|;@q76;uOGZSweJ* zaR*NC#-*YQhd6M(vzeCY7kHeBz&RUOuDj+MEVRsRi0SrQ34BD;cJGl~f6cY`IOAh2 z0zYuE4x;V3^VRl+RfUBNtNm)pddIQ~7wgi`zVdIMy)IfjanEW?gtcDY@H@X<*Q0MV z(LL&HedbKaazyle@|kCDpQ94kx=#b%qeX!sN<3QgJw|(@6!1C6;Pl!p;4olR4zIKZR!=`IP9224(KId4B4}b9AI`l(EsOsb-1bg3yIKQ`S zB^MTyYe-T-UF-X{l{V>F9$Uceomp#odtf9`hfCr}z{r-Un#{L2ti`kTjCPXa?%2mC zR%rBh8HdoaCiX2oAui{202gbOY(Ojq#NVYM)`x-ZFTz=iGh(4hU60_#csK;vLU1eA zG|~`&&tWg#J@@Dv+eEhgJonyvcSo$%e$5!C_%F20@7lTWd+odv*lb5xlQ+iF_i&H0 zL4iks0z;H|6lQsh@jg)iE9|Ft3vJb~_@o1q_$%(()5)XWuc{IgpV$&j- zDQ6?|stwkdXcG<^+VtSJWwBFtj04NoZrN7jDE91?HB`0R`}=4|ENEf3_t@N=JXDB& zkL}DXdBD%-s?A;CL{oQ2-uIL9O3xunSt}L=E^8NzULV`BxevCQqg1f=xgK$;#)&kv zti4KGc}V{TYt$tjXBzID9rU)>mm(7&`wq} ziEx+FF2jrKz~|e+LPfl8-$CnLmZ~jc83JF7p*_R>|NZhWFM5w-40GpJ1(NZxygc{x z)0^Sa4^{~xYaG7I#8@dE&J{1`#@9ZhTDc#1TD;@_(Wbx&oS$r=n8Du1&%B>> zKfhbx0E4(Rz zQNvwJyuqdTuxfm#jjK92bvkJLF^{!xojOa4hzBuWu{~XG3s)V}eqJwKs}ikR6FoY? z6&8Y+S9EO;>sH;|&#CK}7uGmy1l~CEZr<%WZSU1RwVSg(z*pPutpzbpOV5X7zzcV~ z_8!OltvOrA<~tv~b=}gnHxBF7drfr9^M?+XzVG@e{)Fg94AtMfFOEJd^RD!^ifsFT z>fXAF@5o$T^)7uE=HQQKg95K-3Jg)=^*p;{Z1$!Je!L|{wVZ^+Qfz^fRyWvgVWzF)2D<7;FX5)Ja&6Z1L(*CB zIB%4BI`W?v`poh|)7`?vf(FgoXH)kQU&@q3>)YbeCPFmp-WGcx#o?=lu+zlDw{(1W zOC}uJYZd|i_HCcS;fUi!>=?~j5!=lfHwUkl{w zR+74{VsUIzChJn0{VCv1osEr;eR0w{!beLU^CFgq!<;o6Wvn!jX3b#4tVVg0Az_sXq9Di{Z`HSp<6ul|;0 zLfF0YYqP#>*@xE}7Wz}MvCO%?>HfU01YgNiR~>#!;_nfLe$y1cb!&(cH&5e%;gO&K z7D)&zUw;0#GXC7uz_ERqMe;MuiDs*;C%NBz_^~%T7WiAL#U-*qt!2wH+}?HIvZq4U zsP00_pIFTX_HaZQjcbG@XI!59*)wJ#qESDeu{s>2B(TiobJy53mK9;uxt=-WG}hZc zgGGbW3zpFw_cZ?*R$Q}=>3?;0io*T*u7sOKyPTbytX&E+0#oI!=SPvZ9qTab@Ay-l*cnx7MfH2V7Z9$e16M!eb-@^=3L&pwl>6> z1kK+2!4DpAsV}C$4$ko1Njr(N7!=2gnw_dzCJN1KNNf*DDwAL{aP_t_E9_2vj*S%8 z1HU+x7<06hMJoLN6ZmVFMcY`%HkI5s@QFJhYR7KX@E9{|0a4}};E4h(ns&XAe;UYG zL$uH8#X&?pYww2wLthImj{0Z~S(~24Se#i+(0!>@=h`s$_^1m!i*=CKD?GyYIbf|# z7Tn^iiC+u6fAD|(do{~dUI!IVu?Eucw(ZWn7Xtw(ww;s; zu?{M(vW6sSy{T_Bap~Rf`nozMX%M7QxM_{N^+x>LI)<&K<+vXdXels6iIyVceo){n z3Sh_lq+>~6`NSt~vrP0HS+p`hHvSt@u_2cJWet8po_CTPy=g`q+Q$n>1Ubf1V6HY)xXzxJYF&uj%rh&uJ+pd21CbW#u z;4(+I`P}c;;;mcODvdn{BN=q^m@m2J;N$(m=EBAD;9W4E%2_S0${r>8Aa+E!weuQw zzlzHRF4%E|_!Rffb7jO1;+gN#sj(9$iS7ZLg_FgNDCzLda}FXNMOmeJrw>A>PvP^N z3W7Mkq;9W{Z=H9>y6Qc@i85lfQ23jGJzx$BJj@grqQt{Id!z3U5Cv$ru+@&6);`g4 z+wu%6;lSdA&-<1n!OCN~X?fuZpMBSyb(UzXm-dGP0~7YwOwF6+9Z>_L+MLgjpl7 z;l1Z;%K~4|*^5Yx-4?nltX{^UsY8fu&xxq*k0iFnmtzsE9gZ`3eX7$QXp-ybOAEif zue-dLxF=hV(WbMi#5RqU&c(kUO&v0EaX^ZL)dUOfo`s>!2G!znPZZ*mC06Iq|3bs( zzUxbC#c_r-s-Q0lPuFe$bE>&EX|Da@*LU zIkc5EU;ej!eswIV2|0^avTk{n8=ZO03c>#yF0ljrn6p)f7TB{p$pPpbRN^~09-yDO z5);rmIn-T~3;TxUzBF92hIW>I^PRSnZ`)^KwGpGx3-NC2&)o~hM?)IssnFHB&0Xma z*7g#=YL*l}=;j2g-1f_U2jUj2PvawUv~||pn|H9VhTA z-@YpOdEWoS*jY$~BVVem#$LC$;H_Dv3Ldx|`sGZ#w}qTIhn}NX?(>>!?+0svJvcZ# zwN7HS5Bqn+yuyFUI%@mo9m&VD&wNGw##GUGPlLQQVcUMId*2`5A>*ydcDDjN0f!2M z7ztmyr^rN;i;B(j-B(||eO0V!BP{tMQjW%j0z;G-%V1Do6AH*dUw?M_XP0AlzX|9g zAT;$YuE2X$ein`@UwpG0OTcf>+m7)VSB9O(Yy6iEE5qx})sBw4{_L)=V_e#gUA=o_ z>%lvPFZ$bS+x9H#8rO9Fd;Q7n+rIARJJ;x4--FeoNrt)W+YUx9KVDZo9J%ZszFZpD zSM?p()!AJaO@H%lZiz3$n!;nO`qHCM^!wiV-uQBT*HiHI!Lu0eJy@r&=3wTrgngJI?Pqa=p`FIa;Tgw#~Ws!f0!l?;g)h;-gs(_%h*e+KT(F$46sNd(D!=DMh$^ z*8a}nlQD;r32R2(k@^I2#R}A&k_!!UmZK&3Xna|mf)4e30$eIVeT@sq>w^wSaXh!{tj)gYFQE!ePxE3qQv)#xr5J5}FEOp2CsK zLn|*H!n@!Z_-)^#o^hZdysU<-KpEbtYnH1@_X0;Bt(9J!QsA@S%O&wc_BW6n3@+c3 zus%i~j8p81b;S`WTkcMcW$w8?=e5QW%vzeVZq313`Yg8J6C+f&{af1b%z7+G=5I^K z%rm<(nhgrPt|%}>iPzP1j={V(6d*>}VK)wW_HRm@fU9N(*|TZEl3i_}y=$o_!C+0M zS#Zg^!%tnJO6-h%E!z5*-ba$z7eD$XmRraz4(%g$Vfsa#A^XQgPa{E}c`#`A2ac!5Fb|=aZM>u)3Fxq>#9EAnF zR+pGa9174;uNTi+EAK!*H5patq?gL-D{y6r?awNV1Jn<6E#RFxe|bsPyCls16rAyVP+%fUl(J<~)N2Sz0ZrOl-v0KQ#~%B@claiE*2b!YCN!UgD;I}>Rjf*H zJh_bKv$DNK^9z^lzCx4lzL(H6w_kg0UxFCZPyO^a)Np5EyHN)JTA1_oi$lkLY$3K( zEqog9eEW4zC7*;3YZ+_|b}8abwjSRv|LU*SwZal1qmtfM&F_AElD)1Q!F+8jHP+b} zh5tYQ{a^ifSY4E17qO1Ydxu1w!rkYw96!YtdhcQ$*PdDTh~aD4)|y;*Ps6qNj_Mx| zW%5qEQnbVB6RDRTV#Bn}mbXbn`vEM;2Q4jUh+rF!8lG`OV=q%puocdn zDkR&CM4ZZM==B-!a}(AZ$tMFBi;EC;<5iEQE!Y&|)1})=+3E|ffB2h!^>JG2mbKHL z2q#vr=A+LYUU@BHB{)|TRwJ2uYUp~YqaN0GC5of!w7TF-W#Qo*C!E}|1wPJ7M3sIsd^JzjTXe5@e7R#ikQV-GQdb~`lCz52__o~Yr+clQ=CVfaF?iAVNz zK*KmLU7Pw7;-2@6btBeK;K*lvP$v#qtynFWjcZqHr=s}co#R;=i#hc!=EHX;<9AJz z)7NMJ*6*)-lq|*V{bt$lei7}e=-D)8#i7zr$H5VbxEeBinJ71v&Li#j37_3oqnG`3r}h=5{LVjDm2!vFby{5y3_#s#i(9CP>8iZX{RJtKzn z;IkH5E4_m?DwnT$N9|?dokv3!B!@Vb9N&|=H!cSS1_d5q3Jg)=0iK-EwTFQMv}v)Q zL_SYxg)bS?hkx?xZ?g^ro08Ai6hF_Gz>l_fiv|`u6QV@MldmOCSz_ZX0`v&=bpg$(|%{ed|A6>6!7NmxRcy0{r%KqF;M<9!nS;UC!4o*J-fA#qk!ty+J zcV7*=j6b(o8qZWj&YFfGw~cG;{+;jsvYIs6dpCFVEK^g%6Rs`&`)5RgMOyy1`Kc+lEEU zKf!S=_jzT1PXSu+ob$n*@BESj9{J8~vla)fKD>EX;{{;5RfrN{eUPW#F1D?xP#s4K zzUolYg|fU*T#LTbzzR?9wXEU1rH*aKde7IC)i{9yi?dI!oq4Smf>I~q40`pJmc8=w zH`mbRDQCBu!KmQE(yw9trtv@Vc#S=os1Qy^IixpNwJW@Jiw0G^$qzf%wElMS?#&%B z!@OFr;ULFJM%ErXadW~6=cjmYIw+c=ohaC@7h9}x?jfq!S)2I8c`o({IlwQpb}bXs z5;5wTANc1E@10mEonc*yOlt6X|E9$J7vAx;b*&k9Gm!3W-X{Y->X4dcYv1oY4zweF z3svWA={;TRKt?p9D&WY)5j1~ih{2p49Z!G97u7BKx}CnD&>kQ}L1N_A-{=dvHJT0z zJk%5zR*8pt`bOtpdkXl;_OmLa(jnwCYk#-wp=6sf-Q=J5%BB>}@|Ji4mgJc&uDI`Q zw-)Wzo?m#!J8st)fyBPlZJN^kR=eX0)oj^4KG(8U#n_yCtm@ivsoy%h*!GM#Q(65q zzNN`fPxT?=EN>;<<@EKONphpG~Og&Ij`yNOgukbxn&6m+c{Q%mA%Dr>UuSmz0ZtiaBe~=9V~-4&%XE-6skVCHNT&4d$3YoQTAbTXz8qy3=gl%gc{l zqpjcC$Cx<8@d$jOmo&a&K{x~{+sXnGH2Un_ySKsROc72mpE!f#RQ|q3k}-dO`_@vG9DiD>$e8uooMk8UyRQ| zeZk4{(D;H28QNN0Xj$j;0=QF6g|D-2a#$lO2{E6lyLCCWh!~Nr;SP9%0*@jEhA8nU z&iWYhh7@3dpy7;Tt1Xp!T`a6t&Ftdmo+i}6@ZPI?Ng2OyY+YoL$@px?wRNT{;>i&U z({rYn8IZ7B+s$Pbd8)eo0kCL6zk^13PtPcS!|(j|BweeurqqXX3Qv^N_y$bcY>peO z5!g1Fu;Q8Rz_9$$F4mP~m@Ft!FP=}HXVwPiJE8b(8ys zbq{>T&spUFJN5NYmK%4kJB~K{dezcUVAi$NT|?7tyQoSbF(vvIXLV(OBAy+}dShwI zYRLZ-qJ25*B8R)YHjIk}Byc(BxnhA3*CT6}v-lLE+Zs!yt?dGShMmj2r>r5DZl&nm zA|7Yham@j+tV!!~v*gxO-xu4*ZI!AbPb(`?jK^6xIqOn>zjOUdul?p^=derRjrM~A zYbh{9iM13P^#=uxDBx$lhG3R3OpX>u4>!Ewpi+I#3wwec#Cl>Ol-)f;^LeUkM3ivW z2u*r8O~~$as{NYh&p!9uj=e7q8LS~R^4jxD*N2+jXbr#SpBY2x|T*L^ygXP4a4l1n|%!U%j@)03rx z#=phKF^9FTW(R5?rhEIA2=^9(n~Endt%=@VF?QDG=*OxP+1tDu5}OhMomsD{I8tDs z(-J2j^?vo0S8Ct9_e4*7xDdm_AQX3zzIb@cD`7BTd7+70S>i(;nurxQ@L9 zhyYvQ!xmxzY4Bd{ButIH320s0>`v$47vc;TANXEqsvUG=jD@DxSk}Oa7E3HN`cKna zGc`rybdH^`IV82a_i}9WJTm(%*_%KnA>*6EXB~M?yopd!V<8rc#nBJX9d&ED)YtCi zUi;p|SGD&=>;=pG*|}Uuc35_lqYdX4+S^4=M|*sAm+jOcW2ly*{d2I1YN2H}7O^l` z=XLQSJ8Oo_MbL4=5}Q!13~-hA4-Qre;on89cszDBPu*L7x$nR90P)@%OB`?+7psyn zF9+POnJq=uSFC;bh3FGy;Oe(tU+SI8IAeaKIGq z1_h=lFhq$d%8c?sfs+)V(dLM9QyE;^4oHy58u8H|`Ik5QS#NQ~ea#m2-p8sI)b&k; zk_SGPh+lj8o2%b67KUb4T*&vl^PM*`)|?%O_Lf5lmWD6C@kYOTXItt?E?N|_i>wz! zJbN8r|2P1F@2GJD#Sy2c(GR@#)(CBIWm)6F%$aXH`nAbr!x|%0IycuBVamkE@Xtq+ zCDsINwGiQ&)H4Gu*MU#wBadj7Y4Xi0_?3k$u?-)5|KpF>o!l3Q!o&^jTE`S)AOe|3 zA}%_!zyzCbj|K0Ko#~ry&fZFl3dVo*ZJ&SCj`n;vAIB*WXzV?k6G7(?5=SIij%2<< zp{WeaXQfx^i*+PrfNk=Odk)^IXB?t9;lbx!i&z$-J<(92+pV&T>5H@t#v$It{1~fw z@s6D8>$%URj1arxy}d~X)N}hc+6@Xk>=YQH#KS&=qyMiV1!%}QQ`{+e(w!YQ8OkRX z0Q-^;)`_)F1xrZAX(22g8J}x@_1URm3B0*FcZ-9T#EvbVxUb=*g{7H;vGf^jG*&xc z*K=WK`Az3C?9Eb`bM>&GH04LIpZdbbiA^y={@8W7irIS%h{5B6Lc@!5Kdbx_~G@*cv&>MHEK z#=gt}IpZ1Y3ekx5W9Llu>$k^tb!QNAG#L~~6d0mJBF4BJ6j(uZUFCF*|V6Myk>@8h|1vvt1)yg`B20|kaC@p_oWF^1QO z0)Cz(g5~Vu?1&KMt-FL3>?Q1^DGVR(t#H!1R+hELR+ch2TR-(na1+OZm2t+}2qzC` z2xZvAwazJ1eX9Qd`?0x>%qldwvgbr!_`FZSSW_OXC^oR&KCRyG`?3|jNCry`Y5SS+ zZDn~eKF%sVEppMu9}LTjKZmF8g&1epy=&sZTr$|eA@MDX9TwRh8gUSbIa*aicQgC(+%}uAhxi3^0QNj~oSt zDDlWm`55qa6eya(;xGTcn!%}M5x2v#+op~4RCbiHEgk=RKJHs?r-sGjFO5CqFI|&T zmTUKMVEeXm3aANZ3v4VrO(EZJXNR$IhQZEYnYP%0+8Dw&d}@w3e5eO!8wNeqW~v;V ztFnF9)N|!TQt<=oUhd2N#7TT`3Ul59AF_Q@51dT{sTX`h9lk-bIhBlDb?7Y;tsgX_ z)kj^HoIR^3foH@REFDi<4=V&7);2WqQHQp;X30vdX)nQP)L(LzlYiI$&s|#+L^C2_ zjL*5nCrN4vwXNGNF41NOJ$r1lOia}wuY{Gu#ZrPzWu;(ok|jt6sp!wu331E#17FVy zCu}=l#hLgL*S+&8nxY4>y0s)2M~L;t1$LZ!c$8taPvP_2I+tZA){6{rtRYiyd*yJg z`4$laSwY%(p@1z3wD)xPu70jsKhVUzELP`at0JCxF9JuLy?1N&!&pMosPq0G|Dy+d z=G8lIzlV6f^;xoEaYVA1SxZ~PAMgeR9ux`;QQ|?FiP4XTi~@dw*rnK(wd1UJ>R6JVAL4KdYYb=mJXJlu?2IoeMDk+e zC&nn33Cw&24OxhUYA;RlYfZHD41D$6^e5(98#G+=-hgMH2yKaJk%_8@&-16A{DMlm zz_o0?j-cJD`$?2Bm;)A;SmQ;&anS(mULR;&`!J)}}42NfN zxVP??&R4Gkwr6DV3z+%b;xn#Ga0Pxzh%=tJ&iH!CjHCTIXWwmWA|KQ+?$@rvbMTKg zg97)90z;IzUz8qrUS|{_3ShtTAv|_bUi_7`VD)HynEeDggznBcm#jo}i&#W#3HZIU z>^^JZPSvkY1N*VaHD9#nH4Bz=jU9+_3Ju-hk6(F=sj}(`Ce3}sO7O*BDmo<)*DhKHFJM`6Y_V2@o;2rENo}jxQ2O*+pZQGPf!S{A zwC+SU2X=OMW_8KefJ@x*xZV~T=f7|A3BY~vK6c5!B4pKNgE1BpLroT}qS(&YKs zm*B9?O@AhuTecPBw_P0{3T&)HXxZYdOrp+naYVZsnw!`nP7N|3i4lo9H5|OKQai18 zl&(9QqsSy<423m$9BagxBtzmGE{4PSYFKuDxbT`o4*2kZ<9celJ)x}Xv79X39XWSn zmo#&S3mn=}Y;Ud;*Vr{IzIv=RQNHvnEH>J+@XUoUTSK-VgG)I0QXILS-JH=6ZE}B! z2I2Sa%xQ;pK;1aBWsPvyJ40H547+#D8mGUqFnKPq=d;iKfyZU0QXU5!aUJKjOeUrt zj$FoKqP_JdoSUOv3&-qAy7 z+jthIzS~QZ9IwZAuZ4Ig&ad}M#?_#}>ze{Ylz4s5?-*UB0G5Mu!m^*c^NB{&D&KtM zXlJE3AKG+FY@jSHSR6m)_LiWD4i;;=<@zVWStZc~9}80wwcci=zX^QAJ3skg6USsA zqV?neqzrA>kD%XzqfNwwaBvc088tt9_in{E$Btp6YLeGzbmGiZe86RSAyNTA%C&+ed7VSGE@_kllF zS$K(S(Z{&NgV@tyN7G%5hFFD5G7%YDo$r#RNhUhia1nW{b62^ivUtPq{I+fXF19>W zhqFD->f}_ety4*K-E#z#NH>w>qDy@y68dKO9SAi7>y1(yDFQ8*Ii4?|8>;8d!Twocfu<>R^4&?E?o5{{F!VV8N#> zEi}V4&+4wlhvlw*?zF6X79h@dik7&A$Fe{J8+F>}bN6BwUitW6sBN)G!piw;$Bx4- zz8WIFuMeL#bMW&sY-X^D+Hw%#+(k1QqEq0rhFG&|EJvwJ3oYz)O)h-vA!u?{LX0E6 z2CEz8#6xQ}L^b8ux)1%#&#Yp3hz}BYYR{^Y*MDTE!b-9D*iRzxV}tpzXF4spmTdcb0_z{RXruKl*Ijar<{=uiqBAF*9LgQIh#tiJbF|(e#_o12|Mi3T zMU-TzfWuhgj8$f3Su&Phod_*&<8Xwj$9VFYLst*Z(lYA{JQl2Qz_VVeb*q$DTt!{s z9mI+g*)r_4Y(Fajn8XL~gmG3&NZ=|CdRt~n61JY{m9J%o+cK#&x>%nwdSyJWIRf5P z=(Us_zD=?AiMluKY{ok5H;n5*@as6mfV2Bcr)_gyLqxHGlX-9W{+BZc48c3{B zW#P(CFLqOW!(0gOxqZowEJOeST>{4vLInQA+tKWSF+_|oqwMyS5X6-z_#X5n-X&s8-iTEEO%QWvb zPNfGVmE9@gt2NSNVL3^8*M(O{6YCVaUE7o$&26)24aqvx8-KJroXgmztYMW+E$^II zCiOWSvD|qNW$wxE%K_f0i-NDlpEUJi4T&Ej#z?@6ZY*jtp`FXl`EHo&>|kVW!s3Z; z)-HCRm?O@m>DubsSYx)qyeeX~bzuI*hd5Y3#EYy3Jk!A1s0n4u+^|fYlXYqa{Iy4% zoQO(Wejl$P83ls^4+sT@DDi+y#OTINP{7YCjq(=ZrE~qGi7IT{E=x~g1!#A%bABqf z)(*iHzSzW-_Gn`Z=K6i=84aY2MS+7A#BS7a?|hDCc?xg!x8IKm4GWu7*$J#@;M@w| zxBn0Sr&fFG(YnaZ1{P?AUb)PFmg=A6f<*&CdXO*h!E}c5qxmr5(jahfwCC2J;e!ux2iiGWXBGdBBPct=`m#njkxIfqjg zI2NCP+nzbv**U#fTo3fkFwVL5?h&OkE>;&J**TW0C4I0)teyfnXEKnn_K zcFh;oohUKIZgL3Q0v|0qIISa2+f_~$>Tx8CFxs}xqWk^YI2Csg;kKvxbsFbRggB*n zjqq$*;fq+c{rJQS%=L}s0zHLf&(%BitU4PNI1zTe3%&l#A%z1?cm2lf`-L5kv<`7z zYDv-;V$NI$d5t3|-=ja8w=?7!0S$Ssy@w`pH~Sjg=VsRIQSkb-%@)g`;g?KG^tsOu5Wekh%(MK ztkTq&l=H*Jve8iXEGx8lSgEZv_t+QlCB5+D=%>DHfiElp**Xw7@|hNUZCpfdAJ_lI zoBomN;526i@IDCFCpJa9bI%-rILgemFCs)`-R9~neJ1Sq)Vn3T96QfxjOM#Nv|sM5 z^Tfdt(e{xOnNW2adm?x1BY$*X-X%_TEsXY_C4lOH$dnH;1YtQK4vWbU`_xcrtg$UD5ZFMhzMo#Ks1&OC~cin_gFsKUCBA;>WWE`7!wVxBjM|F^{1W*Tc<6a zvNYT+u^8xQJVKK>%v@qAG-q|slTL@I3x4Pazow4Knl-+{!YxQp+5xsNf{VzW*O+;- zbzG`EOVEK!8%edyYz2iv78tMpE{TRdXow3La#u6azg7a43 zlJy81OdUfkA-1rxsTYS>nyAkjatytHCr-sT${{^Pw8!4Frev7;8L{6HVuxpa4AZ4j z9B@2s!=gDmaSD{NjYTwPTVe%O)>=xZv0Q&*9r`Ys6V7p$uFLBfM#-SSYfFJ4O1!r8 zAJ}h<0{e_Dx7Q?oJf}0&IiTdZ4+FiQR3Vpjr{8k`&a|OPqcsdifPBV=vslbDU-gJ) z8@mJt)?1bktTge=y>iY%tAI{amPC~H6HMd7LglKljoViEu$ge7nXEq<#opIbw|*z4I@grfF!aW}$4!5itL_ZE``kwKG z#Ujqe`q%#J-&M0P5f8LI06OD>bxfjG$J8D#XMi0|1_cHM?gIseC~+T%JW$+k3h-ej zLNK07^67d@$DbGvZ5_6gW;Xxqx~4&+W!?fWEf&A_(9&^$z$RdW_{Fmd#J&SGvssH8 zeO+^|TicZ}(9}&?3D7uVD)N2;Yw)PcG7w_eDY37L7^Pp;_kJy+eSP7T=7-<=sZdeR z(fX8@f37R$4Ewbtb_SbEOv9$l!JL2g#H`a+Zi}A!cS*J!?aTT+5+O2gB}}L4iSm?IfHNj=MZzsd1elN`wR=y!;{N|2g_d5wmt8Dmy)Aq86u!4vii1MwhJT+No zVSmAubt~9tn$_6T!m~p6lkDn}@$y#(51STjCKiYCtblEN9<}dXcG$RrIZY z^Y_l;jy7AJ>v}HV0jd~Oa^vI*!o<3P7h1{vFG6EGv|#{;hp@*v5z#) zbMQ_*k# zmRO3O{*Esy;k93T(fIVi{9)zo0G&1AEC|A`m2>v-e){$pAAfs?=W(5V@Oxv6GNMat zf5&3y{eJhmzOFjtz*cy`!Hz^&Hx}O5Xw*5*_?chh4}Gd%oKN^@V_~lT`sW>O35SiUKH~+a z9QUVSEqRV5q+xv53omrx1xq#cd*p+%KmIglSG2_VB!dcDXKk=LY z$z+{e|I{@HH!!E_VSBNg=Y){=V9L6KX1!-vYH1FHx%Qr)Q)cTzTNP3fwC&N*J2*%0 zw#|}zn)0%R9$RR;>2u zg+tfA6sv61y(%I|3v=$CgBWXIua0jXOHx>MoF#A-uHCCY$_51n1ztM}3{m2>qy4~q zBNV{mV>296nPs*@Op|6yteVZGI63(#7GJ2Hbl^HV`#CRLRYl!P&uEy( zb%bjlijC#S8rn;6nyVHEA9Oz4GQ02*_q=5h#i`Kx3uj&$_DR&VV{pbt6bnr`&2o!R zNGSFk3mffZFIqBY>`=6Ee$%)9V+UN=q6iiC%=oNXu`0PuTra^V>1ov0-z+lZo!He8 z1Ax9!zdfJACtfD|@6*gj8`^5@XMMg81J8BywTxs-Vj5%a$P z%(B2@NDPR6&YlyT7#oK*@g5m&51;<6*3u_@@BPWIuiuiYE+}&~h0pximt;xCrgp5c z9y09SweeY}!qj)7XbWlCea%qSjHQQ<1!jxw6NxFlA&#pe<`A>v@I8%Z%X&X*4GIhj zOi^Hn5>u2J>5ZP3d2w!l}NhzhJZ z$DA$k85j0-Zcl}+@P(%QloOD>L9Canm73jlG9?QJ=ae`b_4wqsFxK7gVqLIh^X+!^ zE{(CF2{*nuI7CeKhv>WNy5;e-2vZ(B}*y?%p_s%t* zqQJ}BP1;$TmBsfgM0r+G)==-LwQMWBd?-7$#^IL%&bOGD%X&*3WZ5;Qd;}eGS$mBJ zy)m_AELquElQ_iDj@n0}YvJqp5ayomr9mL_)bCUa{08o2Bdgzo8-P7P4hq}^1%@bb z6I2~AUqcFTBDgB-7%ivcedRaurn*Cm9LXLzRyc|_c9umwr!O>)&;G67UlU8h;U_il zIkO;~19z&8pR)Y3eW&am3_sB<%9TyEUsDbjaW80>HOBTJu$O>m@UMY8SvS$HxBVP# zCL?v|`gSemWJHy2h7*Jk+k7KbT(JB!XAKO}9PoR@_UXz9(*6h5q~cd}*| zTL^Qmns*#NY;rIDi8X{}Y!1enXSNlj@_-ZjZareV?Q7y70(++1!GcxGKl7P-ALY(- zqH^w&_Y6!i8TDI|-Vx=F_M6iyJNG*j-s0Uykk8 zEo@4Qr*|*A$egBjabF4>5%7r)SYU^pc4bb9XIKkc#4Uz9Q4!k|FxyaCtT(M;ur2)3 zC6B#G)11aPHAi(zwFUJihICYHkAWCp7155K?rt(ZG`=LmQFe}+Vtq2csl6^b-Ru@g zJrJ5}?0dVzHYby+b>52{GFNp0ttfo@sV_XxlvbS&{U_i5fj0`t4u>G?6WzDiKF*kG z2B+kl^U}xu{4QhFz6?HNBO0z4%4AhqaZXjNo3VcK89oVLISn~jzy3@e^O!gVHYlj8DypondRt#=bs(?=^b}*b5-G zzsvZhjQzoW7#9{YTQp(}bLSe$Ug!_!FMRmJ6*s&+eDC?$pRI8Xo?~CJ9&@nUXLaw5 z(ocQ0PZqO@_ufi^UQ1|^m{^P6hfK4O| z*yyOt+L5>uT;hcL-zu8(hz${rU1Kd&Uio*$>R8}P-W%*Q`YJDeH@+}JMH}?<2vAN@EK3w za<&emFNZdlTKDt{8_iF=PdAxvWJ!NUokx!XLzH;*=KLWVvA72NH{6dd`yBi9 z?|y35W7M98mS66v)+`$}31|6Pt1QW>2F-IxB_)T|SI!($ti5JpM`8;-8n#~E_y+vZ zt*x)hz7^H^p|ZdQ=TrtHp|)6h{gRY)ZB`;85-qF*p4hd!MZ*g>i^x%@$_=)cV-#Fi z#QWL_Hg0bUU%(IofOkU z6ryDn?uv!uL~}Z>A_3`KhDg%J4AIhK+l1{}7uFK-wgX~Sk@Z;DT59FnBy1TT;!zcW z)ALI%rB`V2z2Ub$dvH2z*XJd;w6*PD#^?Luz4LBZua{WKdVS;2=etEj@(!^0pyRpg zy@w;0QM-Uo+^v?>a_zoNSC1{Mgm!3JPgvJ^!Y?{n8;9@SIkTq));ro@JK|Y+H;%NHw2YpI2E5jbh7;#j zgWZaK{JxFN5%}N{Qxvf<@!WX=&7qix6!nQg?V2Z=#dG*LA#o6lGGmHs=L6u03*vCp zZ`VVJy73%)pX*on3EziaeDN?36X;6u6NkAAdZ}w16?S_6>R8gf=fCm~{@rR>;&?fY zBRxs|3w+i~(Ciw$_Vr)<=r>L7OrM}%zijXsSG?aXz1-W^CTqa21Dr3ZTSOK(V3{9j z8(ha>d@O0^c8l+sOmIXZaD|qOu_4~ZagN1reuTGfu?P~C+Vy#=-6hYdLn5QuC2&UD zL4k89Fhq%Ss5aW1Ndej|Y|A-X-L)wBH<#UhE1Sl*n)tG1_NM3$2Mb#_5m)?N<%70) ziIpN?ojRjmb%xpFVF94Qbk#CQSRz9-*(0XxRGFHC*OX1Dc zGq#v7aBYbd^|3Crq+8*meXnr@`>~0^$k}&fuaUut2E973L`Tj%7nbmid0ogbmO-(4 z(4f-{Pwi)cMMsAvmV<6^aV{^ggCMA2#b9X}~n(gR$g=l8z# zt+&NN1P*8K0c+~r4PrsWBn0fwmPFRA1Cls}E%3xB#a7w*{bL{b#-lOSaaGaiyj{=( z{?u6tkig0E=bIy@XR7`d<-Rq3`=;W+w!kr93<|Uq7@|Z=k#T>66cFl$RS_b+*4KP( zwOGhM2^-LYR{I>1a+#hzcF7UwQe3ZlHiGk)c`hpE!13%Xl|B>g5 z143^-)_F_)$uQPj*PXogaIk23Tp7+5_-I5qQlS%8c5dxmijSyi?bU;3xnxYs<7uxs zVMP6_VyW>ivTE=nr#)`#xA*qUi1}Z##)b4p{qDI%(A>v+#X;~?NHHgxaMbDH$__*g zn!|12GjGI?I5Jx(D-(;DWY;}B%81?C9so~V1iltdIG__w#Vs6oC7s6lvcRycD9iQi{vO{JA>niD`3PCLDKKwa103Qwzj-&{Qm=}<1y|r> zy=Ix@SlPna;y%w8^-n9pZ-FDjKr8PRD`kv1KOfhF0$WmGh!R^;bHF-50gfT^d)E-u z6Ll|njK%TuQMVK;tPX8q_J_X&cCW3UD^>}b*P0Qh{6uZJWn9KdE6PF;p>=*LXiaH( zbGntKh^@V#J6lK6$~+fI8CMGU7|9G`SJa{ zgm!scam^wH=T%uR-ZjosTjK~^&eSZ%M}5#!Pvp8=U8Chzum+q#fpaJ@M2T~#Hrkv? z0Y4qsoVBdOIq1X`gzr7=*-hfaF2Tj$w`#e{nbvfxpWi(GxmbyFSfvcRy>4SAux7%J z!>>8A^|bBIxyQ0$Y0!`37Dp}Yyz;=erSEZMZ-TFCyZ4=DZ3%pBSnoOG(xz^YRq((? zB*<{YE-0_h8rDD8TXP+;CC1URO%W@RvujzWPd)hsb;~Z6C)Z0^)SSJkXO)oANI&3{ zp^LZ|e*emi+u7V>jZcj;uZMQMx9c^-cirCmE}X(w9o$MEX|;1*n)|6(31TLvug{k3 zM0JJZF{Ho? z-fEenJ~nDgd<_59K`RiXvy4YH_gjjd@BZ6=xN5HxntEB*WzDoyv-7kMFSxI9GfjKA zh)gU2TgLA|)59kg!G0j#`=%=w-+HHuhQ7iTgYa`Nzg!)su+c;f$ChlO7$J@}L?!!4 zR0o`5t#gip6O?CY;ovZXO~+hspTWX4jlTx z+7|t)dwwiISn8DTb9F#nSp85<7A7$vEKWZd(1Xoh8$nyRy;Jn)&fsSs{(0^hq{c%e)0CAfBVb|J#p*SR)z zos>1ov23p1sb{fPPR$FAy*fD=F4$ro;ybiN%n5pe<@?h1j~{wi3i;pr{y4lJGFnBZ zuvp)1`}W?P5lhtPEQP&~I$|Wm^>~)G`2o9kj<$1Wn)7N!DR67XDZ6WzLr8SMZ1GVi z5RtvF@f~uW!;_z}>JZV0XMrQ1;RjnC#}W%mhCg-9N|*5zHO3)R+*F%O%89zxoFuf= zKI_8oFTp(84hq~q3Jg)={?Ymo#)J)(3{lwIO_9M*jZd~@h2d;OD~es@Sg|$Uyo+)E zX^QJD#U;B4M}kuDIXqxFuw{Xd=6;JY7CIiBwP=>HTao$7&mezoIO~qd8z79`nfZeE z*adraY>6*erm0Oi!2wpkLEyQn(_p8h}e zbNohIm56jC-tp>`4=4bLR|u z>RKY^*w17NR;=TA&IwVQe5{SvBF1tO>mAfVaS$a~KBoFV$_E7o1zuYU3{m2>rT-&> z-OuAL8DwE|RUZnAcv7d*C7=amF5>-H+Wi|0X`Z1J}>$?dqO~ zEsrVQ{iWyg=Jxs+;M0%j%e^u6>#o5Yu$Izjn)c_>cWJBLn0Dj6$+fk#Kcn8mNr53s zJe>3P5cOJ^H9uZjt@=|zt5|%D-AjtMWgG)s5-EaB8`wMNxQ0u}2v>~b)cF!T+D?tD zbA6EJ$?-e&%G;}f2oakcXp&n`ai$48qT zuxeQC#JHo*7Uy=p%co7Eq*U1znh)$`@Z^4%Y)ZpL-5S9Ge`nQo*_P&f(YzdDn%EM*oGEOGF7DKjSFh;=Bu?{VyM{*>&~+CASTF z`kCJXPr$Hk8Q7T}ksT`V)$I+V@+RjeDyzb0Kfq;tThE`L=$fQ7kd?FdJjN%bN)swnQO^$|KV@`)yH3W z$2*F%-&Z{Lq=eU9;kNGju)?cHPxif3`g%5cicbCSA$bjhb)h_?6)?InJeBspMl z&Z_LFc!u&kAKoD>_uH;snx(B7n*^@n(RxmnbFh{?6GC4dRQEWoU0dcYX?t!R>(F}uw>>7# zg)`a>3M`?(5G9sSY1FxZ0@Wfx%UupX^=m@&)}qb0Btr~?`Ay*5X$GIDNf)D^@PybD zWm}zNJgr7@H?}7h`F$_#OTgUG^Lnf3%k?Mcm7tlqnwlppT-9Wq2fowlc;4Cy@a@)4 z`(*unz?z~Zh*4)f^y@7G)*QZFt0y+pZqL&*@Lliixh1=A$yDp0!PN7$<+0v(aId#U z>s_7e&(1B0cY>G=Z#~L4E&AepXuk#L;)-^=HM9Fx$G5XR!>bQl;f`-+#$(Ju)1ZUr zTmF9Ne{VjcTp!?sf2*+uou|(5z_%OoRIP2xcQkmSW*FNR@-#H_=$Njt?GtpppZK=z z{lfumh!PLSJUyhnlr87!|M8!xT4mw7A9>FkOZJqT(Z<-bqJ#+7@Wg5yFPbwUf-<>m z9kOR!!j2;Z@vVRJ_tqq>w6n@C*@?NGY9XD~8c^WM29ra`X<4HWFTV8WR%;d^lhJi)R)|^HP?Yp4*Ooh z&}yA1w3g&|4cw`^!c&b;a^hYcq0mBmx55`|e+r*Iz_k^=ST92Kd*iEH8J6`T^HBR7 z{q5c?g@CQfghaP>+IxE_pr`k`6+OYTyS(SIK9xyN@cgr1S(CKJT61PSXyLc_z8!F> zZ_kCCbeVN9;ITNcs0fLUvY3mw@6n*e=RRkKW0|u{xPH9vJ#UVU5!%i|wIq{Q0wbz~%K=LG zN9)1RiXFxV=bvJIV<1ursjU?&oKlm$vG(M zMcb(}mNK$??X-^fYG!;bJ)@SSyMfC&%BRlNDW%kV<>PaI z&pY3Fz(;JUPKc#1F&8o%^{~VN`+483ja}SQy^iu?4aoQTJJh4wT-np#@kMpZ%iQm{j&qEe+Tw?P z@N15~LvygFp82*@Re&F`1D7+E6ERdG;VBqv%l(deCwj+PSfl=+z*-6nQDQB{9;x~g zO8SgDmiRDVN*s9j6vuHXyi@J5`hm;&RQ*ez6KU=@zWkGXY5z~PXMu|G$&G)ie*2u2 z<*I1yYOeXh67t1v>%@I!kBgWHA6AmZE1oSm)7WB<&?ov#M585d+yC%0Us3mbh?-IasoCzVkg_{lX`L&?WZG$m=}DC)TEj`_%%YK5g``z5LBbV?Tw@oCU5t=RJIIMP27} z_FJ)&bifd=5dF*e#5CAqkY!0-TJCdL>r87Ry@)|lDS4FsE*(D@4*1AR- zHa^PCm$|+q){Ph!^o(;Ji?!%IfvZ}xg5YPK^Lokn%#nB->Iv)OoVC&GlJ~)H3hPlE z#J}fXe)-xrrPqFx4GKK06d0n!!#Z=D_giKIVS}3kZUC|#{MKhaQ-9{$-0)4+i*IQO ztHyF4?p9gHi=p-bFeD2dK7=L~obUaKKfa1J>qEX?`X4^|V9;;cQnOs0uDxd+VQ5Mc~_t#efg& zJB6*Uu*=q8vG+m7wFL6Yymb(4e4zQYg;nge#EI@JCcCgp~Q)S)*4sUP~DKk!? z-$du~9&W}C%czraxHezJGGf!I{;qv){aaT;xT(Y(uYUBSn~i5}e@6X5f!7-ahA8oR zo6%L{;ZsR#$dDaB$EyG=tPBqehi%=emaAyrWOZ4qNj`-R9!7g?cHp526W4$Zra7eb zJq2gUb3aAl-~ks+GFC9OcHtu~E6oykb9HERIXS^2`^PIE|M=}z>{Exc4|eI&cogsh z*W0ds_awcdPu2NpZU8gSI4N;D;skbzjlg!Q%innh;Ol-M#aU737F>udIJ1vkQ%M|LL;Q7@w?Ir&`E(wv-N2usBLu zo4@}5{%1#VLv8)~KK%VZTUqV)*_!*B9j)Ml)7q=fTO9z#g&hw^Jnv^{qrH#50~}jA zO^=G|x6<0&<0OV~W>U#kazr-HVd&0@=IXD1UUhgRLixtAtP#bd{Sokqo8lBxRW8v+ zdCKCOOtWRxFNOEiCse2hJQ+IU^E79j<*_*TiMyVO#UBeBu)| zBbV=`?~lHSXK*ggZ#N=hfV;-JnEO1g2L&E<3Jg)=L7$-;?O$w*v)iWTMp500ZGdWs zUD15qG|m>^O>V!l8_RCHBiDiZCdPPmw1w1e&Z2C0-j!|lzc@i~dPsjU-mS-XwIAE= zd=FuiZ;dbJ=^m!tRgAAKkNc~AKi90OfVpcMkap*QcUxiLyMN-StvjI|#cr$1nr-t@^Dorl9$~zOBb~4C5Y- zxZ3rhKc0BDgApy}&(UU@9(`N^bqy|U0^jbeWcPPG?t64w3qRo9JMX@eaSvX*_O8l$ zxg1Ym*!3MuN4plb!?VM;gLi$7c6g&t+n)2d&~kUbEsho%9QOlHi+9{Vm=qYI#Dh66 z_o-JLIA~~kVVGNYfztyGl(4h^`_=#8aSj+;;{cC_jg=*EMd%yfeSWN4;{5WGUp|Ln zMra*vnrj-wE%DX;5DMJD6X(FU<^BYDz5>sW{hdEjeebu%SAKpU6S^CCoZ(`%s#$OL z^xSHE)_lh0x;Ts6E{+_vJjNHk|Ci2Tw#I=S76a?lePuIh`Sd%%dYJv^XZy6LSB&u& zKK$Xj?qmJHDGa~oyhIDn*@V`g6H)b{pG8(4HuzMpna2uSqYs| z)9!rdmmJoQ=jJou*e?Lht)=Ks;EMK9w|_3IVS(xRxtCwA&P}YHQ3k&GsbSIT5dru0 zonQRuH&y(L_kSsEqJQw!{Z-O?@B3+fq7EE!ExXd%eFTIiNBftGB`2=qoZ%~rdroO( ze~km3`?R}WYTny11+EM~$|?=ZxE8Pf>nO|b!asl8HwLV0$9}#zb8!+gpKEE@L0Loo zV+{E$TbWF9ms*cooZdNcQNEMr##KLDRTq zg4vG`8Pn!EWGe&*I$bmW}&4?BSMJSth2o?YZ|Hx2)Tr z`sr^t9L5m|WAl1G7sqJ#Xj5Q_5|7mU)6DScjpJ8ed8PQA?%1}i2|>e4E7E={u@i;e zIQ$fEftTN`9bn<2WfUffMGW?|vZPxe+QXkae`P%b~FTGUR`no?sr!mbrt$yUN?^Ib; zRQFpbw493Xs9A-|^R?N9raFP`#Txw1|MJIjS zc@+Y@h1PX9h9|0((GKaXOZ?d5s5?>@TFOGT6~6jz6_`Sph3~fCm$(-*Bb;VfUmvpb;J!+OV7Tp_y}zETDA~t zD={%khw(i7+;fxtW`!lX)o<(GI)N>|`|7KcZKlSatwvLIayhls$3FCD>O6=E5%S)% zs^vD9Ui;=qsvUEFRUM=*g*xBf_xGui=o`*zp_#9~$Dy#***@~s-ub<-1vzPXBgVDoKfT^ z62OM$uo~(2{by|V6Ik=l7q(`DVI4rQMxbGJO`ucarz@Ubp!>7*9;B%f?MA1wD&rk=L`viqIyd@n;4 zjlK7S=6M;c)3su2v-^e5`zH>e6$d!onY$&~olcMHUX6I)WrVs{T14&N_>KCF>sjw| zy|ve}&w1xLOcAdfzST9^b9vuyLbR5(DXgl#HSs-ljjk-aj&m~^^=L2|OHgpeZ-%v1 z@ZKhZ$FW4*?BRRY3oqOzCb5cywTY;l#~OI^nTRN+!|_X>{`Aor=;4VnR+UiouoB9a zg?=nZ#BUa+*tY{deQO=RSZ@N}7H8izTiQ8@xvY1ty))D)TYeX|=#(FD1^xPk~?40apF4*BX z+Yvg~-Qqfr3(G6teZ^fgu#DkbIj!hxjf2>atA@0evJbuZV$ItR=c(}tsb3TJ2Z#A` zj_t?bi}6k2^WN8ISdB8zOXzIgVxIDM!!FEGUY!~mU_wA+z7N)krcUoR&s%;T{Wk78 z*7Ojr{2U$L^&>xW@*9Wlm6kt`C(}YC&E0Rp2zl`R47qD&UG$M;5d5h9VqjY*Wb~6mgn$!&e12< zt20gi`gd|C=5n;twub2zSohZ3K7>3%$;;V%W{c2K|4-ZyL%cVK@$JW(#}V`7Y>wmo sJOO(0vGo?~&w4;li2wiq literal 0 HcmV?d00001 diff --git a/paddle/trainer/tests/sample_trainer_config_opt_a.conf b/paddle/trainer/tests/sample_trainer_config_opt_a.conf index 61d2c62d4296a..f5b1988ddaf5f 100644 --- a/paddle/trainer/tests/sample_trainer_config_opt_a.conf +++ b/paddle/trainer/tests/sample_trainer_config_opt_a.conf @@ -12,32 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -#Todo(luotao02) This config is only used for unitest. It is out of date now, and will be updated later. +from paddle.trainer_config_helpers import * ################################### Data Configuration ################################### -TrainData(ProtoData(files = "train.list")) +TrainData(ProtoData(files = "trainer/tests/mnist.list")) ################################### Algorithm Configuration ################################### -Settings( - learning_rate_decay_a = 0.0, - learning_rate_decay_b = 0.0, - learning_rate = 1e-03, - batch_size = 1000, - algorithm = 'sgd', - num_batches_per_send_parameter = 1, - num_batches_per_get_parameter = 1, - learning_method='sparse_momentum', -) -default_momentum(0.5) +settings(batch_size = 1000, + learning_method = MomentumOptimizer(momentum=0.5, sparse=False)) ################################### Network Configuration ################################### -Layer(type = "data", name = "input", size = 784) -Layer(inputs = [Input("input", parameter_name = "_layer1.w")], name = "layer1", bias = Bias(parameter_name = "_layer1.bias"), active_type = "sigmoid", type = "fc", size = 800) -Layer(inputs = [Input("layer1", parameter_name = "_layer2.w")], name = "layer2", bias = Bias(parameter_name = "_layer2.bias"), active_type = "sigmoid", type = "fc", size = 800) -#Layer(inputs = [Input("layer2", parameter_name = "_layer_output.w", decay_rate = 0.02)], name = "output", bias = Bias(parameter_name = "_layer_output.bias"), active_type = "margin", type = "fc", size = 10) -#Layer(inputs = [Input("layer2", parameter_name = "_layer_output.w", decay_rate = 0.02)], name = "output", bias = Bias(parameter_name = "_layer_output.bias"), type = "fc", size = 10) -Layer(inputs = [Input("layer2", parameter_name = "_layer_output.w")], name = "output", bias = Bias(parameter_name = "_layer_output.bias"), active_type = "softmax", type = "fc", size = 10) -Layer(type = "data", name = "label", size = 1) -Layer(inputs = [Input("output"), Input("label")], type = "multi-class-cross-entropy", name = "cost") -#Layer(inputs = [Input("output"), Input("label")], type = "huber", name = "cost") -Evaluator(inputs=["output", "label"], type = "classification_error", name = "classification_error") -Inputs("input", "label") -Outputs("cost") +data = data_layer(name ="input", size=784) + +fc1 = fc_layer(input=data, size=800, + bias_attr=True, + act=SigmoidActivation()) + +fc2 = fc_layer(input=fc1, size=800, + bias_attr=True, + act=SigmoidActivation()) + +output = fc_layer(input=[fc1, fc2], size=10, + bias_attr=True, + act=SoftmaxActivation()) + +lbl = data_layer(name ="label", size=1) + +cost = classification_cost(input=output, label=lbl) +outputs(cost) diff --git a/paddle/trainer/tests/sample_trainer_config_opt_b.conf b/paddle/trainer/tests/sample_trainer_config_opt_b.conf index 82d547dd8a0d0..f5b1988ddaf5f 100644 --- a/paddle/trainer/tests/sample_trainer_config_opt_b.conf +++ b/paddle/trainer/tests/sample_trainer_config_opt_b.conf @@ -12,32 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -#Todo(luotao02) This config is only used for unitest. It is out of date now, and will be updated later. +from paddle.trainer_config_helpers import * ################################### Data Configuration ################################### -TrainData(ProtoData(files = "train.list")) +TrainData(ProtoData(files = "trainer/tests/mnist.list")) ################################### Algorithm Configuration ################################### -Settings( - learning_rate_decay_a = 0.0, - learning_rate_decay_b = 0.0, - learning_rate = 1e-03, - batch_size = 1000, - algorithm = 'sgd', - num_batches_per_send_parameter = 1, - num_batches_per_get_parameter = 1, - learning_method='momentum', -) -default_momentum(0.5) +settings(batch_size = 1000, + learning_method = MomentumOptimizer(momentum=0.5, sparse=False)) ################################### Network Configuration ################################### -Layer(type = "data", name = "input", size = 784) -Layer(inputs = [Input("input", parameter_name = "_layer1.w")], name = "layer1", bias = Bias(parameter_name = "_layer1.bias"), active_type = "sigmoid", type = "fc", size = 800) -Layer(inputs = [Input("layer1", parameter_name = "_layer2.w")], name = "layer2", bias = Bias(parameter_name = "_layer2.bias"), active_type = "sigmoid", type = "fc", size = 800) -#Layer(inputs = [Input("layer2", parameter_name = "_layer_output.w", decay_rate = 0.02)], name = "output", bias = Bias(parameter_name = "_layer_output.bias"), active_type = "margin", type = "fc", size = 10) -#Layer(inputs = [Input("layer2", parameter_name = "_layer_output.w", decay_rate = 0.02)], name = "output", bias = Bias(parameter_name = "_layer_output.bias"), type = "fc", size = 10) -Layer(inputs = [Input("layer2", parameter_name = "_layer_output.w")], name = "output", bias = Bias(parameter_name = "_layer_output.bias"), active_type = "softmax", type = "fc", size = 10) -Layer(type = "data", name = "label", size = 1) -Layer(inputs = [Input("output"), Input("label")], type = "multi-class-cross-entropy", name = "cost") -#Layer(inputs = [Input("output"), Input("label")], type = "huber", name = "cost") -Evaluator(inputs=["output", "label"], type = "classification_error", name = "classification_error") -Inputs("input", "label") -Outputs("cost") +data = data_layer(name ="input", size=784) + +fc1 = fc_layer(input=data, size=800, + bias_attr=True, + act=SigmoidActivation()) + +fc2 = fc_layer(input=fc1, size=800, + bias_attr=True, + act=SigmoidActivation()) + +output = fc_layer(input=[fc1, fc2], size=10, + bias_attr=True, + act=SoftmaxActivation()) + +lbl = data_layer(name ="label", size=1) + +cost = classification_cost(input=output, label=lbl) +outputs(cost) diff --git a/python/paddle/trainer_config_helpers/optimizers.py b/python/paddle/trainer_config_helpers/optimizers.py index af85f745f63e5..4660a6b5003da 100644 --- a/python/paddle/trainer_config_helpers/optimizers.py +++ b/python/paddle/trainer_config_helpers/optimizers.py @@ -71,16 +71,41 @@ def to_setting_kwargs(self): class MomentumOptimizer(BaseSGDOptimizer): + """ + MomentumOptimizer. + + When sparse=True, the update scheme: + + .. math:: + + \\alpha_t &= \\alpha_{t-1} / k \\\\ + \\beta_t &= \\beta_{t-1} / (1 + \\lambda \\gamma_t) \\\\ + u_t &= u_{t-1} - \\alpha_t \\gamma_t g_t \\\\ + v_t &= v_{t-1} + \\tau_{t-1} \\alpha_t \\gamma_t g_t \\\\ + \\tau_t &= \\tau_{t-1} + \\beta_t / \\alpha_t + + where :math:`k` is momentum, :math:`\\lambda` is decay rate, + :math:`\\gamma_t` is learning rate at the t'th step. + + :param sparse: with sparse support or not. + :type sparse: bool + """ def extra_settings(self): default_momentum(self.momentum) def to_setting_kwargs(self): - return { - 'learning_method': 'momentum' - } + if self.sparse: + return { + 'learning_method': 'sparse_momentum' + } + else: + return { + 'learning_method': 'momentum' + } - def __init__(self, momentum=None): + def __init__(self, momentum=None, sparse=False): self.momentum = momentum + self.sparse = sparse class AdamOptimizer(BaseSGDOptimizer): From cbb904356cec0b90effe46892ae3af2b073af9e4 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 29 Sep 2016 15:30:21 +0800 Subject: [PATCH 134/324] Remove main function in some unittest. --- paddle/math/tests/test_CpuGpuVector.cpp | 6 ------ paddle/math/tests/test_matrixCompare.cpp | 5 ----- paddle/math/tests/test_perturbation.cpp | 11 ----------- paddle/math/tests/test_sparseMatrixCompare.cpp | 6 ------ paddle/utils/tests/test_StringUtils.cpp | 5 ----- 5 files changed, 33 deletions(-) diff --git a/paddle/math/tests/test_CpuGpuVector.cpp b/paddle/math/tests/test_CpuGpuVector.cpp index 61b424e3c6647..7b50b020cda93 100644 --- a/paddle/math/tests/test_CpuGpuVector.cpp +++ b/paddle/math/tests/test_CpuGpuVector.cpp @@ -84,10 +84,4 @@ int main(int argc, char** argv) { return ret; } -#else - -int main(int argc, char const* argv[]) { - return 0; -} - #endif diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index ac50e7b7499d8..fe8eacc2efbc5 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -1851,10 +1851,5 @@ int main(int argc, char** argv) { initMain(argc, argv); return RUN_ALL_TESTS(); } -#else - -int main(int argc, char const* argv[]) { - return 0; -} #endif diff --git a/paddle/math/tests/test_perturbation.cpp b/paddle/math/tests/test_perturbation.cpp index 050f2ca9ced80..4fa9bc72013da 100644 --- a/paddle/math/tests/test_perturbation.cpp +++ b/paddle/math/tests/test_perturbation.cpp @@ -249,15 +249,4 @@ TEST_F(PerturbationTest, scale_test) { } } -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} - -#else - -int main(int argc, char const* argv[]) { - return 0; -} - #endif diff --git a/paddle/math/tests/test_sparseMatrixCompare.cpp b/paddle/math/tests/test_sparseMatrixCompare.cpp index b3467e4982e24..6048dd8112229 100644 --- a/paddle/math/tests/test_sparseMatrixCompare.cpp +++ b/paddle/math/tests/test_sparseMatrixCompare.cpp @@ -178,10 +178,4 @@ int main(int argc, char** argv) { return ret; } -#else - -int main(int argc, char const* argv[]) { - return 0; -} - #endif diff --git a/paddle/utils/tests/test_StringUtils.cpp b/paddle/utils/tests/test_StringUtils.cpp index 95290005ae983..b8636709e9b42 100644 --- a/paddle/utils/tests/test_StringUtils.cpp +++ b/paddle/utils/tests/test_StringUtils.cpp @@ -22,8 +22,3 @@ TEST(StringUtil, to) { ASSERT_DEATH(paddle::str::to("12.45x23"), ".*"); ASSERT_DEATH(paddle::str::to(""), ".*"); } - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} From 8ddc5faac162137678761d5c38fd9e80b70e87c7 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 29 Sep 2016 19:38:07 +0800 Subject: [PATCH 135/324] Update Mac OS X port * follow comments to fix bugs --- cmake/cblas.cmake | 4 +- cmake/util.cmake | 4 +- doc/build/build_from_source.md | 385 ++++++++++-------- paddle/api/Matrix.cpp | 8 +- .../gradientmachines/NeuralNetwork.cpp | 9 +- paddle/trainer/tests/test_Trainer.cpp | 4 + paddle/utils/PythonUtil.cpp | 4 +- paddle/utils/PythonUtil.h | 7 - paddle/utils/Stat.cpp | 18 +- paddle/utils/Thread.h | 20 +- paddle/utils/ThreadLocal.cpp | 10 +- paddle/utils/Util.cpp | 13 + paddle/utils/Util.h | 5 + 13 files changed, 259 insertions(+), 232 deletions(-) diff --git a/cmake/cblas.cmake b/cmake/cblas.cmake index 5568f927572f5..529b4b9d15d09 100644 --- a/cmake/cblas.cmake +++ b/cmake/cblas.cmake @@ -44,8 +44,8 @@ set(ATLAS_LIB_SEARCH_PATHS /usr/lib /usr/lib/blas/atlas /usr/lib/atlas - /usr/lib/atlas-base) # special for ubuntu 14.04. - + /usr/lib/atlas-base # special for ubuntu 14.04. + ) find_path(ATLAS_INC_DIR NAMES cblas.h PATHS ${ATLAS_INCLUDE_SEARCH_PATHS}) find_library(ATLAS_CBLAS_LIB NAMES cblas libcblas.so.3 diff --git a/cmake/util.cmake b/cmake/util.cmake index 4e9efd3c187b0..d776c3ae49952 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -24,7 +24,9 @@ function(target_circle_link_libraries TARGET_NAME) list(APPEND libsInArgn ${arg}) endif() endforeach() - + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + list(APPEND LIBS "-undefined dynamic_lookup") + endif() list(REVERSE libsInArgn) target_link_libraries(${TARGET_NAME} ${LIBS} diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index 6f9e03f2c28f1..f9899086bf060 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -1,148 +1,189 @@ -Build and Install +Installing from Sources ================= -* [1. Requirement](#Requirement) -* [2. Build on Ubuntu](#ubuntu) -* [3. Build on Mac OS X](#mac) +* [1. Download and Setup](#download) +* [2. Requirements](#requirements) +* [3. Build on Ubuntu](#ubuntu) +* [4. Build on Mac OS X](#mac) -## Requirement +## Download and Setup +You can download PaddlePaddle from the [github source](https://github.com/gangliao/Paddle). -### Dependents +```bash +git clone https://github.com/baidu/Paddle paddle +``` -- **CMake**: required for 2.8+ version -- **g++**: a recent c++ compiler supporting c++11, >= 4.6, < 5 -- **BLAS library**: such as openBLAS, MKL, ATLAS -- **protobuf**: required for 2.4+ version, 3.x is not supported -- **python**: currently only 2.7 version is supported +## Requirements -### Optional +To compile the source code, your computer must be equipped with GCC >=4.6 or Clang Compiler. +### Dependencies -PaddlePaddle also support some build options, you have to install related libraries. +- **CMake**: version >= 2.8 +- **BLAS**: MKL, OpenBlas or ATLAS +- **protobuf**: version >= 2.4, **Note: 3.x is not supported** +- **python**: only python 2.7 is supported currently -- **WITH_GPU**: Compile with gpu mode - - The GPU version works best with Cuda Toolkit 7.5 and cuDNN v5 - - Other versions Cuda Toolkit 6.5, 7.0 and cuDNN v2, v3, v4 are also supported - - Note: to utilize cuDNN v5, Cuda Toolkit 7.5 is prerequisite and vice versa -- **WITH_DOUBLE**: Compile with double precision, otherwise use single precision -- **WITH_GLOG**: Compile with glog, otherwise use a log implement internally -- **WITH_GFLAGS**: Compile with gflags, otherwise use a flag implement internally -- **WITH_TESTING**: Compile with gtest and run unittest for PaddlePaddle -- **WITH_DOC**: Compile with documentation -- **WITH_SWIG_PY**: Compile with python predict api -- **WITH_STYLE_CHECK**: Style check for source code +### Options +PaddlePaddle supports some build options. To enable it, first you need to install the related libraries. -## Building on Ubuntu14.04 + Optional | Description + ------------ | :----------- + **WITH_GPU** | Compile with GPU mode. + **WITH_DOUBLE** | Compile with double precision floating-point, default: single precision. | + **WITH_GLOG** | Compile with glog. If not found, default: an internal log implementation. + **WITH_GFLAGS** | Compile with gflags. If not found, default: an internal flag implementation. + **WITH_TESTING** | Compile with gtest for PaddlePaddle's unit testing. + **WITH_DOC** | Compile to generate PaddlePaddle's docs, default: disabled (OFF). + **WITH_SWIG_PY** | Compile with python predict API, default: disabled (OFF). + **WITH_STYLE_CHECK**| Compile with code style check, default: enabled (ON). +| -### Install Dependencies +**Note:** + - The GPU version works best with Cuda Toolkit 7.5 and cuDNN v5. + - Other versions like Cuda Toolkit 6.5, 7.0, 8.0 and cuDNN v2, v3, v4 are also supported. + - **To utilize cuDNN v5, Cuda Toolkit 7.5 is prerequisite and vice versa.** -- **CPU Dependencies** +As a simple example, consider the following: -```bash -# necessary -sudo apt-get update -sudo apt-get install -y g++ make cmake build-essential libatlas-base-dev python python-pip libpython-dev m4 libprotobuf-dev protobuf-compiler python-protobuf python-numpy git -# optional -sudo apt-get install libgoogle-glog-dev -sudo apt-get install libgflags-dev -sudo apt-get install libgtest-dev -sudo pip install wheel -pushd /usr/src/gtest -cmake . -make -sudo cp *.a /usr/lib -popd -``` - +1. **Python Dependencies(optional)** -- **GPU Dependencies(optional)** + To compile PaddlePaddle with python predict API, make sure swig installed and set `-DWITH_SWIG_PY=ON` as follows: -If you need to build GPU version, the first thing you need is a machine that has GPU and CUDA installed. -And you also need to install cuDNN. + ```bash + # install swig on ubuntu + sudo apt-get install swig + # install swig on Mac OS X + brew install swig -You can download CUDA toolkit and cuDNN from nvidia website: - -```bash -https://developer.nvidia.com/cuda-downloads -https://developer.nvidia.com/cudnn -``` -You can copy cuDNN files into the CUDA toolkit directory, such as: + # active swig in cmake + cmake .. -DWITH_SWIG_PY=ON + ``` -```bash -sudo tar -xzf cudnn-7.5-linux-x64-v5.1.tgz -C /usr/local -sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* -``` -Then you need to set LD\_LIBRARY\_PATH, CUDA\_HOME and PATH environment variables in ~/.bashrc. +2. **Doc Dependencies(optional)** -```bash -export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH -export CUDA_HOME=/usr/local/cuda -export PATH=/usr/local/cuda/bin:$PATH -``` -- **Python Dependencies(optional)** + To generate PaddlePaddle's documentation, install dependencies and set `-DWITH_DOC=ON` as follows: -If you want to compile PaddlePaddle with python predict api, you need to add -DWITH_SWIG_PY=ON in cmake command and install these first: + ```bash + pip install 'sphinx>=1.4.0' + pip install sphinx_rtd_theme breathe recommonmark -```bash -sudo apt-get install swig -``` + # install doxygen on Ubuntu + sudo apt-get install doxygen + # install doxygen on Mac OS X + brew install doxygen -- **Doc Dependencies(optional)** + # active docs in cmake + cmake .. -DWITH_DOC=ON` + ``` -If you want to compile PaddlePaddle with doc, you need to add -DWITH_DOC=ON in cmake command and install these first: +## Build on Ubuntu 14.04 -```bash -pip install 'sphinx>=1.4.0' -pip install sphinx_rtd_theme breathe recommonmark -sudo apt-get install doxygen -``` +### Install Dependencies -### Build and Install +- **CPU Dependencies** -CMake will find dependent libraries in system default paths first. After installing some optional libraries, corresponding build option will automatically be on(such as glog, gtest and gflags). And if libraries are not found, you have to set following variables manually in cmake command(CUDNN_ROOT, ATLAS_ROOT, MKL_ROOT, OPENBLAS_ROOT). + ```bash + # necessary + sudo apt-get update + sudo apt-get install -y g++ make cmake build-essential libatlas-base-dev python python-pip libpython-dev m4 libprotobuf-dev protobuf-compiler python-protobuf python-numpy git + # optional + sudo apt-get install libgoogle-glog-dev + sudo apt-get install libgflags-dev + sudo apt-get install libgtest-dev + sudo pip install wheel + pushd /usr/src/gtest + cmake . + make + sudo cp *.a /usr/lib + popd + ``` + +- **GPU Dependencies (optional)** -Here are some examples of cmake command with different options: + To build GPU version, you will need the following installed: -**only cpu** + 1. a CUDA-capable GPU + 2. A supported version of Linux with a gcc compiler and toolchain + 3. NVIDIA CUDA Toolkit (available at http://developer.nvidia.com/cuda-downloads) + 4. NVIDIA cuDNN Library (availabel at https://developer.nvidia.com/cudnn) -```bash -cmake -DWITH_GPU=OFF -DWITH_DOC=OFF -``` + The CUDA development environment relies on tight integration with the host development environment, + including the host compiler and C runtime libraries, and is therefore only supported on + distribution versions that have been qualified for this CUDA Toolkit release. + + After downloading cuDNN library, issue the following commands: -**gpu** + ```bash + sudo tar -xzf cudnn-7.5-linux-x64-v5.1.tgz -C /usr/local + sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* + ``` + Then you need to set LD\_LIBRARY\_PATH, CUDA\_HOME and PATH environment variables in ~/.bashrc. + + ```bash + export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH + export CUDA_HOME=/usr/local/cuda + export PATH=/usr/local/cuda/bin:$PATH + ``` + +### Build and Install + +As usual, the best option is to create build folder under paddle project directory. ```bash -cmake -DWITH_GPU=ON -DWITH_DOC=OFF +mkdir build && cd build +cmake .. ``` -**gpu with doc and swig** +CMake first check PaddlePaddle's dependecies in system default path. After installing some optional +libraries, corresponding build option will be set automatically (for instance, glog, gtest and gflags). +If still not found, you can manually set it based on CMake error information from your screen. -```bash -cmake -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON -``` +As a simple example, consider the following: + +- **Only CPU** + + ```bash + cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF + ``` +- **GPU** + + ```bash + cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF + ``` + +- **GPU with doc and swig** + + ```bash + cmake .. -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON + ``` Finally, you can download source code and build: ```bash -git clone https://github.com/baidu/Paddle paddle -cd paddle -mkdir build -cd build # you can add build option here, such as: -cmake -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX= .. +cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX= # please use sudo make install, if you want # to install PaddlePaddle into the system make -j `nproc` && make install -# PaddlePaddle installation path +# set PaddlePaddle installation path in ~/.bashrc export PATH=/bin:$PATH ``` -**Note** -And if you set WITH_SWIG_PY=ON, you have to install related python predict api at the same time: +**Note:** + +If you set `WITH_SWIG_PY=ON`, related python dependencies also need to be installed. +Otherwise, PaddlePaddle will automatically install python dependencies +at first time when user run paddle commands, such as `paddle version`, `paddle train`. +It may require sudo privileges: ```bash -pip install /opt/paddle/share/wheels/*.whl +# you can run +sudo pip install /opt/paddle/share/wheels/*.whl +# or just run +sudo paddle version ``` + ## Building on Mac OS X ### Prerequisites @@ -150,7 +191,7 @@ This guide is based on Mac OS X 10.11 (El Capitan). Note that if you are running you will already have Python 2.7.10 and Numpy 1.8 installed. The best option is to use the package manager homebrew to handle installations and upgrades for you. -To install homebrew, first open a terminal window (you can find Terminal in the Utilities folder in Applications), and issue the command: +To install [homebrew](http://brew.sh/), first open a terminal window (you can find Terminal in the Utilities folder in Applications), and issue the command: ```bash # install brew @@ -163,109 +204,103 @@ easy_install pip - **CPU Dependencies** -```bash -# Install fundamental dependents -brew install glog gflags cmake protobuf openblas - -# Install google test on Mac OS X -# Download gtest 1.7.0 -wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz -tar -xvf googletest-release-1.7.0.tar.gz && cd googletest-release-1.7.0 -# Build gtest -mkdir build && cmake .. -make -# Install gtest library -sudo cp -r ../include/gtest /usr/local/include/ -sudo cp lib*.a /usr/local/lib -``` - - + ```bash + # Install fundamental dependents + brew install glog gflags cmake protobuf openblas + + # Install google test on Mac OS X + # Download gtest 1.7.0 + wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz + tar -xvf googletest-release-1.7.0.tar.gz && cd googletest-release-1.7.0 + # Build gtest + mkdir build && cmake .. + make + # Install gtest library + sudo cp -r ../include/gtest /usr/local/include/ + sudo cp lib*.a /usr/local/lib + ``` + - **GPU Dependencies(optional)** -If you need to build GPU version, the first thing you need is a machine that has NVIDIA GPU and CUDA installed. -And you also need to install cuDNN. + To build GPU version, you will need the following installed: -You can download CUDA toolkit and cuDNN from nvidia website: - -```bash -https://developer.nvidia.com/cuda-downloads -https://developer.nvidia.com/cudnn -``` -You can copy cuDNN files into the CUDA toolkit directory, for instance: - -```bash -sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C /usr/local -sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* -``` -Then you need to set DYLD\_LIBRARY\_PATH, CUDA\_HOME and PATH environment variables in ~/.bashrc. + 1. a CUDA-capable GPU + 2. Mac OS X 10.11 or later + 2. the Clang compiler and toolchain installed using Xcode + 3. NVIDIA CUDA Toolkit (available at http://developer.nvidia.com/cuda-downloads) + 4. NVIDIA cuDNN Library (availabel at https://developer.nvidia.com/cudnn) -```bash -export DYLD_LIBRARY_PATH=/usr/local/cuda/lib:$DYLD_LIBRARY_PATH -export PATH=/usr/local/cuda/bin:$PATH -``` -- **Python Dependencies(optional)** + The CUDA development environment relies on tight integration with the host development environment, + including the host compiler and C runtime libraries, and is therefore only supported on + distribution versions that have been qualified for this CUDA Toolkit release. + + 1. After downloading cuDNN library, issue the following commands: -If you want to compile PaddlePaddle with python predict API, you need to add -DWITH_SWIG_PY=ON in cmake command and install these first: + ```bash + sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C /usr/local + sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* + ``` + 2. Then you need to set DYLD\_LIBRARY\_PATH, CUDA\_HOME and PATH environment variables in ~/.bashrc. -```bash -brew install swig -``` + ```bash + export DYLD_LIBRARY_PATH=/usr/local/cuda/lib:$DYLD_LIBRARY_PATH + export PATH=/usr/local/cuda/bin:$PATH + ``` -- **Doc Dependencies(optional)** +### Build and Install -If you want to compile PaddlePaddle with doc, you need to add -DWITH_DOC=ON in cmake command and install these first: +As usual, the best option is to create build folder under paddle project directory. ```bash -pip install 'sphinx>=1.4.0' -pip install sphinx_rtd_theme breathe recommonmark -brew install doxygen +mkdir build && cd build +cmake .. ``` -### Build and Install - -CMake can find dependent libraries in system default paths firstly. -After installing some optional libraries, corresponding build option will be on automatically (for instance, glog, gtest and gflags). -If not found, you have to set following variables manually via CMake command (CUDNN_ROOT, ATLAS_ROOT, MKL_ROOT, OPENBLAS_ROOT). - -Here are some examples of CMake command with different options: +CMake first check PaddlePaddle's dependecies in system default path. After installing some optional +libraries, corresponding build option will be set automatically (for instance, glog, gtest and gflags). +If still not found, you can manually set it based on CMake error information from your screen. -**only cpu** +As a simple example, consider the following: -```bash -cmake -DWITH_GPU=OFF -DWITH_DOC=OFF -``` +- **Only CPU** -**gpu** + ```bash + cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF + ``` +- **GPU** -```bash -cmake -DWITH_GPU=ON -DWITH_DOC=OFF -``` + ```bash + cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF + ``` -**gpu with doc and swig** +- **GPU with doc and swig** -```bash -cmake -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON -``` + ```bash + cmake .. -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON + ``` -Finally, you can download source code and build: +Finally, you can build PaddlePaddle: ```bash -git clone https://github.com/baidu/Paddle paddle -cd paddle -mkdir build -cd build # you can add build option here, such as: -cmake -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX= .. -# please use sudo make install, if you want -# to install PaddlePaddle into the system +cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX= +# please use sudo make install, if you want to install PaddlePaddle into the system make -j `nproc` && make install -# PaddlePaddle installation path -export PATH=/bin:$PATH +# set PaddlePaddle installation path in ~/.bashrc +export PATH=/bin:$PATH ``` -**Note** -And if you set WITH_SWIG_PY=ON, you have to install related python predict api at the same time: + +**Note:** + +If you set `WITH_SWIG_PY=ON`, related python dependencies also need to be installed. +Otherwise, PaddlePaddle will automatically install python dependencies +at first time when user run paddle commands, such as `paddle version`, `paddle train`. +It may require sudo privileges: ```bash +# you can run sudo pip install /opt/paddle/share/wheels/*.whl +# or just run +sudo paddle version ``` \ No newline at end of file diff --git a/paddle/api/Matrix.cpp b/paddle/api/Matrix.cpp index 9ae3716fa862c..6a79f83495a56 100644 --- a/paddle/api/Matrix.cpp +++ b/paddle/api/Matrix.cpp @@ -95,7 +95,7 @@ float Matrix::get(size_t x, size_t y) const throw(RangeError) { } void Matrix::set(size_t x, size_t y, float val) throw(RangeError, - UnsupportError) { + UnsupportError) { if (x > this->getWidth() || y > this->getHeight()) { RangeError e; throw e; @@ -239,7 +239,7 @@ void Matrix::toNumpyMatInplace(float** view_data, int* dim1, } void Matrix::copyToNumpyMat(float** view_m_data, int* dim1, int* dim2) throw(UnsupportError) { - static_assert(sizeof(float) == sizeof(float), + static_assert(sizeof(paddle::real) == sizeof(float), "Currently PaddleAPI only support for single " "precision version of paddle."); if (this->isSparse()) { @@ -251,12 +251,12 @@ void Matrix::copyToNumpyMat(float** view_m_data, int* dim1, if (auto cpuMat = dynamic_cast(m->mat.get())) { auto src = cpuMat->getData(); auto dest = *view_m_data; - std::memcpy(dest, src, sizeof(float) * (*dim1) * (*dim2)); + std::memcpy(dest, src, sizeof(paddle::real) * (*dim1) * (*dim2)); } else if (auto gpuMat = dynamic_cast(m->mat.get())) { auto src = gpuMat->getData(); auto dest = *view_m_data; hl_memcpy_device2host(dest, src, - sizeof(float) * (*dim1) * (*dim2)); + sizeof(paddle::real) * (*dim1) * (*dim2)); } else { LOG(WARNING) << "Unexpected Situation"; throw UnsupportError(); diff --git a/paddle/gserver/gradientmachines/NeuralNetwork.cpp b/paddle/gserver/gradientmachines/NeuralNetwork.cpp index 0f497e44d4c25..3127b4dd9a2fd 100644 --- a/paddle/gserver/gradientmachines/NeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/NeuralNetwork.cpp @@ -385,10 +385,17 @@ void NeuralNetwork::setOutputGrad(const std::vector& args) { } } +extern NeuralNetwork* newCustomNerualNetwork( + const std::string& name, NeuralNetwork* network) __attribute__((weak)); + NeuralNetwork* NeuralNetwork::newNeuralNetwork( const std::string& name, NeuralNetwork* rootNetwork) { - return new NeuralNetwork(name, rootNetwork); + if (newCustomNerualNetwork) { + return newCustomNerualNetwork(name, rootNetwork); + } else { + return new NeuralNetwork(name, rootNetwork); + } } } // namespace paddle diff --git a/paddle/trainer/tests/test_Trainer.cpp b/paddle/trainer/tests/test_Trainer.cpp index 2044279c2151f..ad2a715ef89c6 100644 --- a/paddle/trainer/tests/test_Trainer.cpp +++ b/paddle/trainer/tests/test_Trainer.cpp @@ -94,7 +94,11 @@ TEST(checkGradient, multi) { TEST(checkGradient, hsigmoid) { checkGradientTest(configFile2, false, false); } TEST(checkGradient, chunk) { +#if defined(__APPLE__) || defined (__OSX__) EXPECT_EQ(0, system("python trainer/tests/gen_proto_data.py")); +#else + EXPECT_EQ(0, system("python2 trainer/tests/gen_proto_data.py")); +#endif checkGradientTest(configFile3, false, false); #ifndef PADDLE_ONLY_CPU checkGradientTest(configFile3, true, true); diff --git a/paddle/utils/PythonUtil.cpp b/paddle/utils/PythonUtil.cpp index 9ee7a29aad0b6..78c3a80674f9c 100644 --- a/paddle/utils/PythonUtil.cpp +++ b/paddle/utils/PythonUtil.cpp @@ -144,12 +144,12 @@ PyObjectPtr createPythonClass( const std::map& kwargs) { PyGuard guard; PyObjectPtr pyModule(PyImport_ImportModule(moduleName.c_str())); - // LOG(INFO) << "createPythonClass moduleName.c_str:" << moduleName.c_str(); + LOG(INFO) << "createPythonClass moduleName.c_str:" << moduleName.c_str(); CHECK_PY(pyModule) << "Import module " << moduleName << " failed."; PyObjectPtr pyDict(PyModule_GetDict(pyModule.get())); CHECK_PY(pyDict) << "Get Dict failed."; PyObjectPtr pyClass(PyDict_GetItemString(pyDict.get(), className.c_str())); - // LOG(INFO) << "createPythonClass className.c_str():" << className.c_str(); + LOG(INFO) << "createPythonClass className.c_str():" << className.c_str(); CHECK_PY(pyClass) << "Import class " << className << " failed."; PyObjectPtr argsObjectList(PyTuple_New(args.size())); for (size_t i = 0; i < args.size(); ++i) { diff --git a/paddle/utils/PythonUtil.h b/paddle/utils/PythonUtil.h index 2808338fbdf59..db02d1252b405 100644 --- a/paddle/utils/PythonUtil.h +++ b/paddle/utils/PythonUtil.h @@ -35,13 +35,6 @@ limitations under the License. */ #include #include -// #ifndef _POSIX_C_SOURCE -// #warning "no _POSIX_C_SOURCE defined in Python.h" -// #endif -// #ifndef _XOPEN_SOURCE -// #warning "no _XOPEN_SOURCE defined in Python.h" -// #endif - #endif #include "paddle/utils/Util.h" diff --git a/paddle/utils/Stat.cpp b/paddle/utils/Stat.cpp index ff6e8ade2cd48..d7b20ca5eb2f4 100644 --- a/paddle/utils/Stat.cpp +++ b/paddle/utils/Stat.cpp @@ -13,28 +13,12 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "Stat.h" - -#include // for syscall() -#include +#include "Util.h" #include #include namespace paddle { -// return the thread id used by glog -pid_t getTID() { - #if defined(__APPLE__) || defined(__OSX__) - pid_t tid = syscall(SYS_thread_selfid); - #else - #ifndef __NR_gettid - #define __NR_gettid 224 - #endif - pid_t tid = syscall(__NR_gettid); - #endif - CHECK_NE(tid, -1); - return tid; -} - StatSet globalStat("GlobalStatInfo"); void Stat::addSample(uint64_t value) { diff --git a/paddle/utils/Thread.h b/paddle/utils/Thread.h index f1352e75d73a0..f6c826a1eeb65 100644 --- a/paddle/utils/Thread.h +++ b/paddle/utils/Thread.h @@ -13,24 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +#include "Util.h" #include "Logging.h" #include -#include -#include -inline pid_t gettid() { -#if defined(__APPLE__) || defined(__OSX__) - pid_t tid = syscall(SYS_thread_selfid); -#else - #ifndef __NR_gettid - #define __NR_gettid 224 - #endif - pid_t tid = syscall(__NR_gettid); -#endif - CHECK_NE(tid, -1); - return tid; -} - #include "Queue.h" #include "ThreadLocal.h" @@ -186,7 +172,7 @@ class SyncThreadPool { jobFinishBarrier_(numWorkers + 1), jobFunc_(nullptr), checkOwner_(checkOwner) { - ownerThreadId_ = ::gettid(); + ownerThreadId_ = getTID(); workers_.resize(numWorkers); start(); } @@ -210,7 +196,7 @@ class SyncThreadPool { */ void exec(JobFunc jobFunc, JobFunc ownerFunc = nullptr) { if (checkOwner_) { - CHECK_EQ(ownerThreadId_, ::gettid()) + CHECK_EQ(ownerThreadId_, getTID()) << "this sync thread pool should be used in one thread"; } diff --git a/paddle/utils/ThreadLocal.cpp b/paddle/utils/ThreadLocal.cpp index a4b399d144ee3..0f948f1029af8 100644 --- a/paddle/utils/ThreadLocal.cpp +++ b/paddle/utils/ThreadLocal.cpp @@ -12,10 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include "Util.h" #include "ThreadLocal.h" - -#include "Thread.h" - #include "CommandLineParser.h" P_DEFINE_bool(thread_local_rand_use_global_seed, false, @@ -31,11 +29,11 @@ unsigned int* ThreadLocalRand::getSeed() { if (!p) { // init seed if (FLAGS_thread_local_rand_use_global_seed) { p = new unsigned int(defaultSeed_); - } else if (getpid() == gettid()) { // main thread + } else if (getpid() == getTID()) { // main thread // deterministic, but differs from global srand() p = new unsigned int(defaultSeed_ - 1); } else { - p = new unsigned int(defaultSeed_ + gettid()); + p = new unsigned int(defaultSeed_ + getTID()); LOG(INFO) << "thread use undeterministic rand seed:" << *p; } seed_.set(p); @@ -51,7 +49,7 @@ std::default_random_engine& ThreadLocalRandomEngine::get() { int defaultSeed = ThreadLocalRand::getDefaultSeed(); engine->seed(FLAGS_thread_local_rand_use_global_seed ? defaultSeed - : defaultSeed + gettid()); + : defaultSeed + getTID()); engine_.set(engine); } return *engine; diff --git a/paddle/utils/Util.cpp b/paddle/utils/Util.cpp index d8c3376fb18c4..c3c76f907d40e 100644 --- a/paddle/utils/Util.cpp +++ b/paddle/utils/Util.cpp @@ -93,6 +93,19 @@ static void installProfilerSwitch() {} namespace paddle { +pid_t getTID() { + #if defined(__APPLE__) || defined(__OSX__) + pid_t tid = syscall(SYS_thread_selfid); + #else + #ifndef __NR_gettid + #define __NR_gettid 224 + #endif + pid_t tid = syscall(__NR_gettid); + #endif + CHECK_NE(tid, -1); + return tid; +} + static bool g_initialized = false; typedef std::pair> PriorityFuncPair; typedef std::vector InitFuncList; diff --git a/paddle/utils/Util.h b/paddle/utils/Util.h index 11a03e141dec5..a059ca50a538d 100644 --- a/paddle/utils/Util.h +++ b/paddle/utils/Util.h @@ -24,6 +24,8 @@ limitations under the License. */ #include #include #include +#include // for syscall() +#include #include "CommandLineParser.h" #include "Logging.h" @@ -63,6 +65,9 @@ limitations under the License. */ namespace paddle { +// return the thread id used by glog +pid_t getTID(); + /** * return the 1-based index of the highest bit set * From efea5c84607cd73b9a318915624ba16b7448f8cf Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 29 Sep 2016 19:52:40 +0800 Subject: [PATCH 136/324] Revise some word in build doc --- doc/build/build_from_source.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index f9899086bf060..a6090d6819162 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -135,7 +135,7 @@ mkdir build && cd build cmake .. ``` -CMake first check PaddlePaddle's dependecies in system default path. After installing some optional +CMake first check PaddlePaddle's dependencies in system default path. After installing some optional libraries, corresponding build option will be set automatically (for instance, glog, gtest and gflags). If still not found, you can manually set it based on CMake error information from your screen. @@ -256,7 +256,7 @@ mkdir build && cd build cmake .. ``` -CMake first check PaddlePaddle's dependecies in system default path. After installing some optional +CMake first check PaddlePaddle's dependencies in system default path. After installing some optional libraries, corresponding build option will be set automatically (for instance, glog, gtest and gflags). If still not found, you can manually set it based on CMake error information from your screen. From 90234791f41dbda698f626145fffac785671c27b Mon Sep 17 00:00:00 2001 From: gangliao Date: Fri, 30 Sep 2016 01:48:36 +0800 Subject: [PATCH 137/324] Add automatic check AVX in CMake (#145) * Add automatic check AVX in CMake * Revise table format and some words in build docs --- CMakeLists.txt | 7 ++-- cmake/FindAVX.cmake | 65 ++++++++++++++++++++++++++++++++++ doc/build/build_from_source.md | 37 +++++++++---------- 3 files changed, 86 insertions(+), 23 deletions(-) create mode 100644 cmake/FindAVX.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 92c866da8fc7c..af6a13efbde9e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ find_package(PythonInterp 2.7 REQUIRED) find_package(ZLIB REQUIRED) find_package(NumPy REQUIRED) find_package(Threads REQUIRED) +find_package(AVX QUIET) find_package(Glog) find_package(Gflags QUIET) find_package(GTest) @@ -28,7 +29,7 @@ find_program(M4_EXECUTABLE m4) option(WITH_DSO "Compile PaddlePaddle with dynamic linked libraries" ON) option(WITH_GPU "Compile PaddlePaddle with gpu" ${CUDA_FOUND}) option(WITH_DOUBLE "Compile PaddlePaddle with double precision, otherwise use single precision" OFF) -option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ON) # TODO(yuyang18): Check AVX is supported or not as default value +option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND}) option(WITH_PYTHON "Compile PaddlePaddle with python interpreter" ON) option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ${PYTHONINTERP_FOUND}) option(WITH_RDMA "Compile PaddlePaddle with rdma support" OFF) @@ -101,8 +102,8 @@ if(NOT WITH_TIMER) endif(NOT WITH_TIMER) if(WITH_AVX) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${AVX_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${AVX_FLAGS}") else(WITH_AVX) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse3") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse3") diff --git a/cmake/FindAVX.cmake b/cmake/FindAVX.cmake new file mode 100644 index 0000000000000..e0f1b7bff5162 --- /dev/null +++ b/cmake/FindAVX.cmake @@ -0,0 +1,65 @@ +# This file is use to check all support level of AVX on your machine +# so that PaddlePaddle can unleash the vectorization power of muticore. + +INCLUDE(CheckCXXSourceRuns) + +SET(FIND_AVX_10) +SET(FIND_AVX_20) +SET(AVX_FLAGS) +SET(AVX_FOUND) + +# Check AVX 2 +SET(CMAKE_REQUIRED_FLAGS) +IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + SET(CMAKE_REQUIRED_FLAGS "-mavx2") +ELSEIF(MSVC AND NOT CMAKE_CL_64) # reserve for WINDOWS + SET(CMAKE_REQUIRED_FLAGS "/arch:AVX2") +ENDIF() + +CHECK_CXX_SOURCE_RUNS(" +#include +int main() +{ + __m256i a = _mm256_set_epi32 (-1, 2, -3, 4, -1, 2, -3, 4); + __m256i result = _mm256_abs_epi32 (a); + return 0; +}" FIND_AVX_20) + +# Check AVX +SET(CMAKE_REQUIRED_FLAGS) +IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + SET(CMAKE_REQUIRED_FLAGS "-mavx") +ELSEIF(MSVC AND NOT CMAKE_CL_64) + SET(CMAKE_REQUIRED_FLAGS "/arch:AVX") +endif() + +CHECK_CXX_SOURCE_RUNS(" +#include +int main() +{ + __m256 a = _mm256_set_ps (-1.0f, 2.0f, -3.0f, 4.0f, -1.0f, 2.0f, -3.0f, 4.0f); + __m256 b = _mm256_set_ps (1.0f, 2.0f, 3.0f, 4.0f, 1.0f, 2.0f, 3.0f, 4.0f); + __m256 result = _mm256_add_ps (a, b); + return 0; +}" FIND_AVX_10) + +IF(${FIND_AVX_20}) + IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + SET(AVX_FLAGS "${AVX_FLAGS} -mavx2") + ELSEIF(MSVC) + SET(AVX_FLAGS "${AVX_FLAGS} /arch:AVX2") + ENDIF() +ENDIF() + +IF(${FIND_AVX_10}) + IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") + SET(AVX_FLAGS "${AVX_FLAGS} -mavx") + ELSEIF(MSVC) + SET(AVX_FLAGS "${AVX_FLAGS} /arch:AVX") + ENDIF() +ENDIF() + +IF(${FIND_AVX_10} OR ${FIND_AVX_20}) + SET(AVX_FOUND TRUE) + MESSAGE(STATUS "Find CPU supports ${AVX_FLAGS}.") +ENDIF() \ No newline at end of file diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index a6090d6819162..948f84abae86f 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -11,11 +11,12 @@ You can download PaddlePaddle from the [github source](https://github.com/gangli ```bash git clone https://github.com/baidu/Paddle paddle +cd paddle ``` ## Requirements -To compile the source code, your computer must be equipped with GCC >=4.6 or Clang Compiler. +To compile the source code, your computer must be equipped with GCC >=4.6 or Clang compiler. ### Dependencies - **CMake**: version >= 2.8 @@ -27,17 +28,17 @@ To compile the source code, your computer must be equipped with GCC >=4.6 or Cla PaddlePaddle supports some build options. To enable it, first you need to install the related libraries. - Optional | Description - ------------ | :----------- - **WITH_GPU** | Compile with GPU mode. - **WITH_DOUBLE** | Compile with double precision floating-point, default: single precision. | - **WITH_GLOG** | Compile with glog. If not found, default: an internal log implementation. - **WITH_GFLAGS** | Compile with gflags. If not found, default: an internal flag implementation. - **WITH_TESTING** | Compile with gtest for PaddlePaddle's unit testing. - **WITH_DOC** | Compile to generate PaddlePaddle's docs, default: disabled (OFF). - **WITH_SWIG_PY** | Compile with python predict API, default: disabled (OFF). - **WITH_STYLE_CHECK**| Compile with code style check, default: enabled (ON). -| + +| Optional | Description | +| -------------------- | :--------------------------------------------------------------------------- | +| **WITH_GPU** | Compile with GPU mode. | +| **WITH_DOUBLE** | Compile with double precision floating-point, default: single precision. | +| **WITH_GLOG** | Compile with glog. If not found, default: an internal log implementation. | +| **WITH_GFLAGS** | Compile with gflags. If not found, default: an internal flag implementation. | +| **WITH_TESTING** | Compile with gtest for PaddlePaddle's unit testing. | +| **WITH_DOC** | Compile to generate PaddlePaddle's docs, default: disabled (OFF). | +| **WITH_SWIG_PY** | Compile with python predict API, default: disabled (OFF). | +| **WITH_STYLE_CHECK** | Compile with code style check, default: enabled (ON). | **Note:** - The GPU version works best with Cuda Toolkit 7.5 and cuDNN v5. @@ -118,11 +119,10 @@ As a simple example, consider the following: sudo tar -xzf cudnn-7.5-linux-x64-v5.1.tgz -C /usr/local sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* ``` - Then you need to set LD\_LIBRARY\_PATH, CUDA\_HOME and PATH environment variables in ~/.bashrc. + Then you need to set LD\_LIBRARY\_PATH, PATH environment variables in ~/.bashrc. ```bash export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH - export CUDA_HOME=/usr/local/cuda export PATH=/usr/local/cuda/bin:$PATH ``` @@ -158,13 +158,12 @@ As a simple example, consider the following: cmake .. -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON ``` -Finally, you can download source code and build: +Finally, you can build PaddlePaddle: ```bash # you can add build option here, such as: cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX= -# please use sudo make install, if you want -# to install PaddlePaddle into the system +# please use sudo make install, if you want to install PaddlePaddle into the system make -j `nproc` && make install # set PaddlePaddle installation path in ~/.bashrc export PATH=/bin:$PATH @@ -240,7 +239,7 @@ easy_install pip sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C /usr/local sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* ``` - 2. Then you need to set DYLD\_LIBRARY\_PATH, CUDA\_HOME and PATH environment variables in ~/.bashrc. + 2. Then you need to set DYLD\_LIBRARY\_PATH, PATH environment variables in ~/.bashrc. ```bash export DYLD_LIBRARY_PATH=/usr/local/cuda/lib:$DYLD_LIBRARY_PATH @@ -289,8 +288,6 @@ make -j `nproc` && make install # set PaddlePaddle installation path in ~/.bashrc export PATH=/bin:$PATH ``` - - **Note:** If you set `WITH_SWIG_PY=ON`, related python dependencies also need to be installed. From 6ea794872ffc29641c183f93fedd4ba536d98bae Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Thu, 29 Sep 2016 15:08:43 -0700 Subject: [PATCH 138/324] Fix cmake/FindAVX.cmake --- cmake/FindAVX.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/FindAVX.cmake b/cmake/FindAVX.cmake index e0f1b7bff5162..58b89918ec622 100644 --- a/cmake/FindAVX.cmake +++ b/cmake/FindAVX.cmake @@ -59,7 +59,7 @@ IF(${FIND_AVX_10}) ENDIF() ENDIF() -IF(${FIND_AVX_10} OR ${FIND_AVX_20}) +IF("${FIND_AVX_10}" OR "${FIND_AVX_20}") SET(AVX_FOUND TRUE) MESSAGE(STATUS "Find CPU supports ${AVX_FLAGS}.") -ENDIF() \ No newline at end of file +ENDIF() From 12b619343b0e8e35cda6335f8230c4168277b8dd Mon Sep 17 00:00:00 2001 From: gangliao Date: Fri, 30 Sep 2016 10:39:20 +0800 Subject: [PATCH 139/324] Update build docs (#148) * Add automatic check AVX in CMake * Add indent in FindAVX.cmake * Revise table format and some words in build docs * Update build docs --- doc/build/build_from_source.md | 56 +++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index 948f84abae86f..c671f483863c7 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -28,17 +28,51 @@ To compile the source code, your computer must be equipped with GCC >=4.6 or Cla PaddlePaddle supports some build options. To enable it, first you need to install the related libraries. - -| Optional | Description | -| -------------------- | :--------------------------------------------------------------------------- | -| **WITH_GPU** | Compile with GPU mode. | -| **WITH_DOUBLE** | Compile with double precision floating-point, default: single precision. | -| **WITH_GLOG** | Compile with glog. If not found, default: an internal log implementation. | -| **WITH_GFLAGS** | Compile with gflags. If not found, default: an internal flag implementation. | -| **WITH_TESTING** | Compile with gtest for PaddlePaddle's unit testing. | -| **WITH_DOC** | Compile to generate PaddlePaddle's docs, default: disabled (OFF). | -| **WITH_SWIG_PY** | Compile with python predict API, default: disabled (OFF). | -| **WITH_STYLE_CHECK** | Compile with code style check, default: enabled (ON). | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionalDescription
WITH_GPUCompile with GPU mode.
WITH_DOUBLECompile with double precision floating-point, default: single precision.
WITH_GLOGCompile with glog. If not found, default: an internal log implementation.
WITH_GFLAGSCompile with gflags. If not found, default: an internal flag implementation.
WITH_TESTINGCompile with gtest for PaddlePaddle's unit testing.
WITH_DOCCompile to generate PaddlePaddle's docs, default: disabled (OFF)
WITH_SWIG_PYCompile with python predict API, default: disabled (OFF).
WITH_STYLE_CHECKCompile with code style check, default: enabled (ON).
**Note:** - The GPU version works best with Cuda Toolkit 7.5 and cuDNN v5. From 3ca5df0f479db82a39ace4962ede193bc18d9152 Mon Sep 17 00:00:00 2001 From: gangliao Date: Fri, 30 Sep 2016 13:53:23 +0800 Subject: [PATCH 140/324] Fix bug when only support AVX 2 (#150) In some situation, for instance, in the virtual machine, it could happen. --- cmake/FindAVX.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/FindAVX.cmake b/cmake/FindAVX.cmake index 58b89918ec622..f6103c6e667e8 100644 --- a/cmake/FindAVX.cmake +++ b/cmake/FindAVX.cmake @@ -59,7 +59,7 @@ IF(${FIND_AVX_10}) ENDIF() ENDIF() -IF("${FIND_AVX_10}" OR "${FIND_AVX_20}") +IF(${FIND_AVX_10}) SET(AVX_FOUND TRUE) MESSAGE(STATUS "Find CPU supports ${AVX_FLAGS}.") ENDIF() From 0276f15a4537cfc0a58a41a3a9eb695aab5a0fea Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 30 Sep 2016 14:58:02 +0800 Subject: [PATCH 141/324] add scripts to build ubuntu install package. (#132) * also refine install docs, too --- .../install/ubuntu_install.rst | 29 ++++++++++++++- paddle/scripts/deb/build_scripts/.gitignore | 1 + paddle/scripts/deb/build_scripts/Dockerfile | 5 +++ paddle/scripts/deb/build_scripts/build.sh | 37 +++++++++++++++++++ paddle/scripts/deb/build_scripts/build_deb.sh | 8 ++++ 5 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 paddle/scripts/deb/build_scripts/.gitignore create mode 100644 paddle/scripts/deb/build_scripts/Dockerfile create mode 100755 paddle/scripts/deb/build_scripts/build.sh create mode 100755 paddle/scripts/deb/build_scripts/build_deb.sh diff --git a/doc_cn/build_and_install/install/ubuntu_install.rst b/doc_cn/build_and_install/install/ubuntu_install.rst index 7cdd470677eb8..a813d9da2e52d 100644 --- a/doc_cn/build_and_install/install/ubuntu_install.rst +++ b/doc_cn/build_and_install/install/ubuntu_install.rst @@ -16,14 +16,41 @@ https://github.com/baidu/Paddle/releases/tag/V0.8.0b0 .. code-block:: shell - dpkg -i paddle-0.8.0b-cpu.deb + dpkg -i paddle-*-cpu.deb apt-get install -f 在 :code:`dpkg -i` 的时候如果报一些依赖未找到的错误是正常的， 在 :code:`apt-get install -f` 里会继续安装 PaddlePaddle。 + +或者使用下面一条命令安装. + +.. code-block:: shell + + gdebi paddle-*-cpu.deb + +如果 :code:`gdebi` 没有安装,则需要使用 :code:`sudo apt-get install gdebi`, 来安装 :code:`gdebi` + + 需要注意的是，如果使用GPU版本的PaddlePaddle，请安装CUDA 7.5 和CUDNN 5到本地环境中， 并设置好对应的环境变量(LD_LIBRARY_PATH等等)。 +安装完成后,可以使用命令 :code:`paddle version` 查看安装后的paddle 版本。可能的输出为 + +.. code-block:: text + + PaddlePaddle 0.8.0b1, compiled with + with_avx: ON + with_gpu: OFF + with_double: OFF + with_python: ON + with_rdma: OFF + with_glog: ON + with_gflags: ON + with_metric_learning: + with_timer: OFF + with_predict_sdk: + + 可能遇到的问题 -------------- diff --git a/paddle/scripts/deb/build_scripts/.gitignore b/paddle/scripts/deb/build_scripts/.gitignore new file mode 100644 index 0000000000000..1521c8b7652b1 --- /dev/null +++ b/paddle/scripts/deb/build_scripts/.gitignore @@ -0,0 +1 @@ +dist diff --git a/paddle/scripts/deb/build_scripts/Dockerfile b/paddle/scripts/deb/build_scripts/Dockerfile new file mode 100644 index 0000000000000..db365a65b7d33 --- /dev/null +++ b/paddle/scripts/deb/build_scripts/Dockerfile @@ -0,0 +1,5 @@ +FROM paddledev/paddle:gpu-latest +MAINTAINER PaddlePaddle Dev Team +COPY build.sh /root/ +CMD cd /root/ && bash build.sh + diff --git a/paddle/scripts/deb/build_scripts/build.sh b/paddle/scripts/deb/build_scripts/build.sh new file mode 100755 index 0000000000000..662d2a9103f7d --- /dev/null +++ b/paddle/scripts/deb/build_scripts/build.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -e +apt-get install -y dh-make +cd ~ +mkdir -p ~/dist/gpu +mkdir -p ~/dist/cpu +mkdir -p ~/dist/cpu-noavx +mkdir -p ~/dist/gpu-noavx +git clone https://github.com/baidu/Paddle.git paddle +cd paddle +mkdir build +cd build +cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON +make -j `nproc` +cpack -D CPACK_GENERATOR='DEB' .. +mv *.deb ~/dist/cpu + +rm -rf * +cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ +make -j `nproc` +cpack -D CPACK_GENERATOR='DEB' .. +mv *.deb ~/dist/gpu + + +rm -rf * +cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF +make -j `nproc` +cpack -D CPACK_GENERATOR='DEB' .. +mv *.deb ~/dist/cpu-noavx + +rm -rf * +cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ +make -j `nproc` +cpack -D CPACK_GENERATOR='DEB' .. +mv *.deb ~/dist/gpu-noavx + + diff --git a/paddle/scripts/deb/build_scripts/build_deb.sh b/paddle/scripts/deb/build_scripts/build_deb.sh new file mode 100755 index 0000000000000..1331c1249d5a7 --- /dev/null +++ b/paddle/scripts/deb/build_scripts/build_deb.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e +docker build -t build_paddle_deb . +rm -rf dist +mkdir -p dist +docker run -v$PWD/dist:/root/dist --name tmp_build_deb_container build_paddle_deb +docker rm tmp_build_deb_container +docker rmi build_paddle_deb From b52039bd11f76142e2cf418e636f933fb91c91a7 Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Fri, 30 Sep 2016 15:00:32 +0800 Subject: [PATCH 142/324] some bug fix for sparse matrix (#133) * some bug fix for sparse matrix * a minor bug fix --- paddle/cuda/include/hl_sparse.h | 4 ++ paddle/cuda/src/hl_cuda_sparse.cu | 107 ++++++++++------------------- paddle/cuda/src/hl_cuda_sparse.cuh | 48 ++++++++++--- paddle/gserver/layers/Layer.cpp | 27 ++++++-- 4 files changed, 98 insertions(+), 88 deletions(-) diff --git a/paddle/cuda/include/hl_sparse.h b/paddle/cuda/include/hl_sparse.h index 22f7a228e0ad6..9acdebdebf377 100644 --- a/paddle/cuda/include/hl_sparse.h +++ b/paddle/cuda/include/hl_sparse.h @@ -223,6 +223,7 @@ extern void hl_matrix_csc2dense(hl_sparse_matrix_s A_d, * @param[in] dimK width of op(A) & height of op(B) * @param[in] alpha scalar used for multiplication. * @param[in] beta scalar used for multiplication. + * If beta is zero, C does not have to be a valid input. * * @note transb is not support HPPL_OP_T. * @@ -251,6 +252,7 @@ extern void hl_matrix_csr_mul_dense(hl_sparse_matrix_s A_d, * @param[in] dimK width of op(A) & height of op(B) * @param[in] alpha scalar used for multiplication. * @param[in] beta scalar used for multiplication. + * If beta is zero, C does not have to be a valid input. * * @note transb is not support HPPL_OP_T. * @@ -275,6 +277,7 @@ extern void hl_matrix_csc_mul_dense(hl_sparse_matrix_s A_d, * @param[in] dimK width of op(A) & height of op(B) * @param[in] alpha scalar used for multiplication. * @param[in] beta scalar used for multiplication. + * If beta is zero, C does not have to be a valid input. * * @note transa is not support HPPL_OP_T. * @@ -327,6 +330,7 @@ extern void hl_sparse_matrix_mul(real* A_d, hl_trans_op_t transa, * @param[in] dimK width of op(A) & height of op(B) * @param[in] alpha scalar used for multiplication. * @param[in] beta scalar used for multiplication. + * If beta is zero, C does not have to be a valid input. * * * @note transa is not support HPPL_OP_T. diff --git a/paddle/cuda/src/hl_cuda_sparse.cu b/paddle/cuda/src/hl_cuda_sparse.cu index b42568afdaaf5..1687fcc221ab8 100644 --- a/paddle/cuda/src/hl_cuda_sparse.cu +++ b/paddle/cuda/src/hl_cuda_sparse.cu @@ -562,6 +562,22 @@ void hl_memcpy_sparse_matrix(hl_sparse_matrix_s dst, } } +/** + * Calculate beta * C, if beta is zero, C does not have to be a valid input. + */ +static void _beta_mul_c(real *c, int dimM, int dimN, real beta) { + if (beta == 0.0) { + hl_gpu_apply_unary_op(unary::Zero(), c, dimM, dimN, dimN); + } else { + if (beta != 1.0){ + hl_gpu_apply_unary_op( + unary::mul_scalar(beta), c, dimM, dimN, dimN); + } + } + + return; +} + void hl_matrix_csr_mul_dense(hl_sparse_matrix_s A_d, hl_trans_op_t transa, real *B_d, hl_trans_op_t transb, real *C_d, @@ -580,15 +596,8 @@ void hl_matrix_csr_mul_dense(hl_sparse_matrix_s A_d, hl_trans_op_t transa, } if (A_d->nnz == 0) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } else { - return; - } + _beta_mul_c(C_d, dimM, dimN, beta); + return; } /* nnz != 0 */ @@ -633,13 +642,7 @@ void hl_matrix_csr_mul_dense(hl_sparse_matrix_s A_d, hl_trans_op_t transa, beta); } } else if (HPPL_OP_T == transa) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } + _beta_mul_c(C_d, dimM, dimN, beta); int blocksX = (dimN + CU_CSC_MUL_DENSE_BLOCK_N - 1) / CU_CSC_MUL_DENSE_BLOCK_N; @@ -699,15 +702,8 @@ void hl_matrix_dense_mul_csc(real *A_d, hl_trans_op_t transa, << "matrix format error!"; if (B_d->nnz == 0) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } else { - return; - } + _beta_mul_c(C_d, dimM, dimN, beta); + return; } /* nnz != 0 */ @@ -750,13 +746,7 @@ void hl_matrix_dense_mul_csc(real *A_d, hl_trans_op_t transa, beta); } } else if (transb == HPPL_OP_T) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } + _beta_mul_c(C_d, dimM, dimN, beta); int blocksX = 1 + (dimK-1)/CU_DM_CSR_THREAD_X; int blocksY = 1 + (dimM-1)/CU_DM_CSR_BLOCK_M; dim3 threads(CU_DM_CSR_THREAD_X, CU_DM_CSR_THREAD_Y); @@ -813,15 +803,8 @@ void hl_matrix_dense_mul_csr(real *A_d, hl_trans_op_t transa, << "matrix format error!"; if (B_d->nnz == 0) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } else { - return; - } + _beta_mul_c(C_d, dimM, dimN, beta); + return; } /* nnz != 0 */ @@ -833,14 +816,7 @@ void hl_matrix_dense_mul_csr(real *A_d, hl_trans_op_t transa, } if (transb == HPPL_OP_N) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } - + _beta_mul_c(C_d, dimM, dimN, beta); int blocksX = 1 + (dimK-1)/CU_DM_CSR_THREAD_X; int blocksY = 1 + (dimM-1)/CU_DM_CSR_BLOCK_M; dim3 threads(CU_DM_CSR_THREAD_X, CU_DM_CSR_THREAD_Y); @@ -925,15 +901,8 @@ void hl_matrix_csc_mul_dense(hl_sparse_matrix_s A_d, hl_trans_op_t transa, } if (A_d->nnz == 0) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } else { - return; - } + _beta_mul_c(C_d, dimM, dimN, beta); + return; } /* nnz != 0 */ @@ -945,13 +914,7 @@ void hl_matrix_csc_mul_dense(hl_sparse_matrix_s A_d, hl_trans_op_t transa, } if (HPPL_OP_N == transa) { - if (beta != 1.0) { - hl_gpu_apply_unary_op(unary::mul_scalar(beta), - C_d, - dimM, - dimN, - dimN); - } + _beta_mul_c(C_d, dimM, dimN, beta); int blocksX = (dimN + CU_CSC_MUL_DENSE_BLOCK_N -1)/CU_CSC_MUL_DENSE_BLOCK_N; int blocksY = (dimK + CU_CSC_MUL_DENSE_BLOCK_K -1)/CU_CSC_MUL_DENSE_BLOCK_K; @@ -1113,7 +1076,7 @@ void hl_sparse_matrix_mul(real *A_d, hl_trans_op_t transa, CHECK(!transA) << "Not supported A is trans and B is not trans!"; dim3 block(CU_BLOCK_SIZE, 1); - int avgNnzPerRow = C_d2->nnz_s / dimM; + int avgNnzPerRow = C_d->nnz / dimM; avgNnzPerRow = avgNnzPerRow > 0 ? avgNnzPerRow : 1; int gridx = DIVUP(avgNnzPerRow, CU_BLOCK_SIZE); dim3 grid(gridx, dimM); @@ -1242,9 +1205,9 @@ void hl_matrix_csr_column_sum(real* A_d, hl_sparse_matrix_s B_d, LOG(FATAL) << "parameter B is null!"; } - if (B_d2->nnz_s == 0) return; + if (B_d->nnz == 0) return; - int nnz = B_d2->nnz_s; + int nnz = B_d->nnz; int block = 512; int grid = DIVUP(nnz, 512); KeSMatrixCsrColumnSum<<>>( @@ -1273,9 +1236,9 @@ void hl_matrix_csr_add_bias(hl_sparse_matrix_s A_d, real* B_d, LOG(FATAL) << "parameter A_d is null!"; } - if (A_d2->nnz_s == 0) return; + if (A_d->nnz == 0) return; - int nnz = A_d2->nnz_s; + int nnz = A_d->nnz; int block = 512; int grid = DIVUP(nnz, 512); KeSMatrixCsrAddBias<<>>( @@ -1308,9 +1271,9 @@ void hl_matrix_csr_add_dense(hl_sparse_matrix_s A_d, real* B_d, int dimM, LOG(FATAL) << "parameter A_d is null!"; } - if (A_d2->nnz_s == 0) return; + if (A_d->nnz == 0) return; - int gridX = DIVUP((A_d2->nnz_s / dimM), 512); + int gridX = DIVUP((A_d->nnz / dimM), 512); gridX = gridX > 0 ? gridX : 1; dim3 block(512, 1); dim3 grid(gridX, dimM); diff --git a/paddle/cuda/src/hl_cuda_sparse.cuh b/paddle/cuda/src/hl_cuda_sparse.cuh index db5c9ce979885..13e89390d68c2 100644 --- a/paddle/cuda/src/hl_cuda_sparse.cuh +++ b/paddle/cuda/src/hl_cuda_sparse.cuh @@ -85,6 +85,15 @@ __global__ void KeSMatrixCsc2Dense(real * csc_val, C_d[row*dimN + col] = sum; } +__device__ __forceinline__ +void _calculate_c(real &c, real sum) { + c = sum; +} +__device__ __forceinline__ +void _calculate_c(real &c, real sum, real beta) { + c = sum + beta * c; +} + #define CU_CSRMM_N 4 #define CU_CSRMM_THREAD_X 32 #define CU_CSRMM_THREAD_Y 32 @@ -191,11 +200,19 @@ __global__ void KeSMatrixCsrMulDense(real *C_d, } C_d += __mul24(index_m, dimN); - #pragma unroll - for (int n = 0; n < CU_CSRMM_N; n++) { - if (index_n < dimN) { - C_d[index_n] = alpha*sum[n] + beta*C_d[index_n]; - index_n += CU_CSRMM_THREAD_X; + if (beta == 0.0) { + for (int n = 0; n < CU_CSRMM_N; n++) { + if (index_n < dimN) { + _calculate_c(C_d[index_n], alpha * sum[n]); + index_n += CU_CSRMM_THREAD_X; + } + } + } else { + for (int n = 0; n < CU_CSRMM_N; n++) { + if (index_n < dimN) { + _calculate_c(C_d[index_n], alpha * sum[n], beta); + index_n += CU_CSRMM_THREAD_X; + } } } } @@ -544,13 +561,22 @@ TEMP_TEST: int index_m_c = ibx + idy; int index_n_c = blockIdx.y*CU_CSCMM_BLOCK_N_BEST + idx; C_d += index_n_c + __mul24(index_m_c, dimN); - #pragma unroll - for (int m = 0; m < CU_CSCMM_THREAD_M_BEST; m++) { - if (index_m_c < dimM && index_n_c < dimN) { - C_d[0] = A_s[idy+m*32][idx] + beta*C_d[0]; + if (beta == 0.0) { + for (int m = 0; m < CU_CSCMM_THREAD_M_BEST; m++) { + if (index_m_c < dimM && index_n_c < dimN) { + _calculate_c(C_d[0], A_s[idy + m * 32][idx]); + } + index_m_c += 32; + C_d += dimN*32; + } + } else { + for (int m = 0; m < CU_CSCMM_THREAD_M_BEST; m++) { + if (index_m_c < dimM && index_n_c < dimN) { + _calculate_c(C_d[0], A_s[idy + m * 32][idx], beta); + } + index_m_c += 32; + C_d += dimN*32; } - index_m_c += 32; - C_d += dimN*32; } } diff --git a/paddle/gserver/layers/Layer.cpp b/paddle/gserver/layers/Layer.cpp index 44ea95c80ab08..78d15c553021d 100644 --- a/paddle/gserver/layers/Layer.cpp +++ b/paddle/gserver/layers/Layer.cpp @@ -16,6 +16,7 @@ limitations under the License. */ #include "paddle/utils/Util.h" #include "paddle/utils/Logging.h" +#include "paddle/math/SparseMatrix.h" #include "AddtoLayer.h" #include "CosSimLayer.h" @@ -290,14 +291,30 @@ void Layer::showOutputStats() { << " is 0, skip to show the statistics"; return; } - real mean = out->getSum() / out->getElementCnt(); - MatrixPtr outSquare = out->clone(); - outSquare->copyFrom(*out); + MatrixPtr outSquare; + if (dynamic_cast(out.get())) { + GpuSparseMatrix *tmp = dynamic_cast(out.get()); + outSquare = std::make_shared( + tmp->getHeight(), tmp->getWidth(), tmp->getElementCnt(), + tmp->getValueType(), tmp->getFormat()); + } else { + outSquare = out->clone(); + } + outSquare->copyFrom(*out, HPPL_STREAM_DEFAULT); + hl_stream_synchronize(HPPL_STREAM_DEFAULT); + + real mean = outSquare->getSum() / out->getElementCnt(); + real min; + real max; if (dynamic_cast(outSquare.get())) { auto tmpMat = dynamic_cast(outSquare.get()); + min = tmpMat->getMin(); + max = tmpMat->getMax(); tmpMat->square(); LOG(INFO) << "show statistics of [none zero values] in sparse matrix"; } else { + min = outSquare->getMin(); + max = outSquare->getMax(); outSquare->square(); } real std = (outSquare->getSum() / outSquare->getElementCnt()) - mean * mean; @@ -306,8 +323,8 @@ void Layer::showOutputStats() { << ", " << "std=" << std << ", " - << "min=" << out->getMin() << ", " - << "max=" << out->getMax(); + << "min=" << min << ", " + << "max=" << max; } void Layer::forwardActivation() { From 2a0a5391cae761107dd243cce801b3ad99723f70 Mon Sep 17 00:00:00 2001 From: gangliao Date: Fri, 30 Sep 2016 15:04:51 +0800 Subject: [PATCH 143/324] Update build docs (#149) * Add automatic check AVX in CMake * Add indent in FindAVX.cmake * Revise table format and some words in build docs * Update build docs * Update build docs --- doc/build/build_from_source.md | 67 +++++++++++----------------------- 1 file changed, 21 insertions(+), 46 deletions(-) diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index c671f483863c7..f7db0a9b92e67 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -1,5 +1,5 @@ Installing from Sources -================= +========================== * [1. Download and Setup](#download) * [2. Requirements](#requirements) @@ -28,51 +28,26 @@ To compile the source code, your computer must be equipped with GCC >=4.6 or Cla PaddlePaddle supports some build options. To enable it, first you need to install the related libraries. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +
OptionalDescription
WITH_GPUCompile with GPU mode.
WITH_DOUBLECompile with double precision floating-point, default: single precision.
WITH_GLOGCompile with glog. If not found, default: an internal log implementation.
WITH_GFLAGSCompile with gflags. If not found, default: an internal flag implementation.
WITH_TESTINGCompile with gtest for PaddlePaddle's unit testing.
WITH_DOCCompile to generate PaddlePaddle's docs, default: disabled (OFF)
WITH_SWIG_PYCompile with python predict API, default: disabled (OFF).
WITH_STYLE_CHECKCompile with code style check, default: enabled (ON).
+ + + + + + + + + + + + + + + +
OptionalDescription
WITH_GPUCompile with GPU mode.
WITH_DOUBLECompile with double precision floating-point, default: single precision.
WITH_GLOGCompile with glog. If not found, default: an internal log implementation.
WITH_GFLAGSCompile with gflags. If not found, default: an internal flag implementation.
WITH_TESTINGCompile with gtest for PaddlePaddle's unit testing.
WITH_DOC Compile to generate PaddlePaddle's docs, default: disabled (OFF).
WITH_SWIG_PYCompile with python predict API, default: disabled (OFF).
WITH_STYLE_CHECKCompile with code style check, default: enabled (ON).
+ **Note:** - The GPU version works best with Cuda Toolkit 7.5 and cuDNN v5. @@ -334,4 +309,4 @@ It may require sudo privileges: sudo pip install /opt/paddle/share/wheels/*.whl # or just run sudo paddle version -``` \ No newline at end of file +``` From b221509695f2181714b3b68fa35c1d200d795a06 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 30 Sep 2016 16:51:49 +0800 Subject: [PATCH 144/324] [DOC CHANGE] Rerange Build docs & emphasize them in README.md (#151) * Rerange Build docs & emphasize them in README.md * Rerange Build docs & emphasize them in README.md --- README.md | 4 +- doc/build/docker_install.md | 91 ------------- doc/build/docker_install.rst | 122 ++++++++++++++++++ doc/build/index.rst | 23 ++-- doc/build/ubuntu_install.md | 21 --- doc/build/ubuntu_install.rst | 25 ++++ doc_cn/build_and_install/index.rst | 25 +++- .../install/docker_install.rst | 47 +++++-- .../install/paddle_version.txt | 11 ++ .../install/ubuntu_install.rst | 42 ++---- 10 files changed, 234 insertions(+), 177 deletions(-) delete mode 100644 doc/build/docker_install.md create mode 100644 doc/build/docker_install.rst delete mode 100644 doc/build/ubuntu_install.md create mode 100644 doc/build/ubuntu_install.rst create mode 100644 doc_cn/build_and_install/install/paddle_version.txt diff --git a/README.md b/README.md index cc2fc68ac3143..05e544186a2d0 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,9 @@ Welcome to the PaddlePaddle GitHub. -The software will be released on Sept. 30 with full documentation and installation support. +Do you wanna try and play PaddlePaddle? Just following the [Install Guide](http://www.paddlepaddle.org/doc/build/index.html) and [Quick Start](http://www.paddlepaddle.org/doc/demo/quick_start/index_en.html). The chinese version is [Install Guide](http://www.paddlepaddle.org/doc_cn/build_and_install/index.html) and [Quick Start](http://www.paddlepaddle.org/doc_cn/demo/quick_start/index.html). -A pre-release version is available now for those who are eager to take a look. +Please refer to our [release log](https://github.com/baidu/Paddle/releases) to track the latest feature of PaddlePaddle. PaddlePaddle (PArallel Distributed Deep LEarning) is an easy-to-use, efficient, flexible and scalable deep learning platform, which is originally diff --git a/doc/build/docker_install.md b/doc/build/docker_install.md deleted file mode 100644 index 3cd9d1730a22b..0000000000000 --- a/doc/build/docker_install.md +++ /dev/null @@ -1,91 +0,0 @@ -Docker installation guide -==================== -PaddlePaddle provides some pre-compiled binary, including Docker images, ubuntu deb packages. It is welcomed to contributed more installation package of different linux distribution (such as ubuntu, centos, debian, gentoo and so on). We recommend to use Docker images to deploy PaddlePaddle. -## Docker installation - -Docker is a tool designed to make it easier to create, deploy, and run applications by using containers. - -### PaddlePaddle Docker images -There are six Docker images: - -- paddledev/paddle:cpu-latest: PaddlePaddle CPU binary image. -- paddledev/paddle:gpu-latest: PaddlePaddle GPU binary image. -- paddledev/paddle:cpu-devel-latest: PaddlePaddle CPU binary image plus source code. -- paddledev/paddle:gpu-devel-latest: PaddlePaddle GPU binary image plus source code. -- paddledev/paddle:cpu-demo-latest: PaddlePaddle CPU binary image plus source code and demo -- paddledev/paddle:gpu-demo-latest: PaddlePaddle GPU binary image plus source code and demo - -Tags with latest will be replaced by a released version. - -### Download and Run Docker images - -You have to install Docker in your machine which has linux kernel version 3.10+ first. You can refer to the official guide https://docs.docker.com/engine/installation/ for further information. - -You can use ```docker pull ```to download images first, or just launch a container with ```docker run```: -```bash -docker run -it paddledev/paddle:cpu-latest -``` - -If you want to launch container with GPU support, you need to set some environment variables at the same time: - -```bash -export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}" -export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') -docker run -it paddledev/paddle:gpu-latest -``` - -### Notice - -#### Performance - -Since Docker is based on the lightweight virtual containers, the CPU computing performance maintains well. And GPU driver and equipments are all mapped to the container, so the GPU computing performance would not be seriously affected. - -If you use high performance nic, such as RDMA(RoCE 40GbE or IB 56GbE), Ethernet(10GbE), it is recommended to use config "-net = host". - - - - -#### Remote access -If you want to enable ssh access background, you need to build an image by yourself. Please refer to official guide https://docs.docker.com/engine/reference/builder/ for further information. - -Following is a simple Dockerfile with ssh: -```bash -FROM paddledev/paddle - -MAINTAINER PaddlePaddle dev team - -RUN apt-get update -RUN apt-get install -y openssh-server -RUN mkdir /var/run/sshd -RUN echo 'root:root' | chpasswd - -RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config - -EXPOSE 22 - -CMD ["/usr/sbin/sshd", "-D"] -``` - -Then you can build an image with Dockerfile and launch a container: - -```bash -# cd into Dockerfile directory -docker build . -t paddle_ssh -# run container, and map host machine port 8022 to container port 22 -docker run -d -p 8022:22 --name paddle_ssh_machine paddle_ssh -``` -Now, you can ssh on port 8022 to access the container, username is root, password is also root: - -```bash -ssh -p 8022 root@YOUR_HOST_MACHINE -``` - - -You can stop and delete the container as following: -```bash -# stop -docker stop paddle_ssh_machine -# delete -docker rm paddle_ssh_machine -``` diff --git a/doc/build/docker_install.rst b/doc/build/docker_install.rst new file mode 100644 index 0000000000000..542b9bac27afb --- /dev/null +++ b/doc/build/docker_install.rst @@ -0,0 +1,122 @@ +Docker installation guide +========================== + +PaddlePaddle provide the `Docker `_ image. `Docker`_ is a lightweight container utilities. The performance of PaddlePaddle in `Docker`_ container is basically as same as run it in a normal linux. The `Docker`_ is a very convenient way to deliver the binary release for linux programs. + +.. note:: + + The `Docker`_ image is the recommended way to run PaddlePaddle + +PaddlePaddle Docker images +-------------------------- + +There are 12 `images `_ for PaddlePaddle, and the name is :code:`paddle-dev/paddle`, tags are\: + + ++-----------------+------------------+------------------------+-----------------------+ +| | normal | devel | demo | ++=================+==================+========================+=======================+ +| CPU | cpu-latest | cpu-devel-latest | cpu-demo-latest | ++-----------------+------------------+------------------------+-----------------------+ +| GPU | gpu-latest | gpu-devel-latest | gpu-demo-latest | ++-----------------+------------------+------------------------+-----------------------+ +| CPU WITHOUT AVX | cpu-noavx-latest | cpu-devel-noavx-latest | cpu-demo-noavx-latest | ++-----------------+------------------+------------------------+-----------------------+ +| GPU WITHOUT AVX | gpu-noavx-latest | gpu-devel-noavx-latest | gpu-demo-noavx-latest | ++-----------------+------------------+------------------------+-----------------------+ + +And the three columns are: + +* normal\: The docker image only contains binary of PaddlePaddle. +* devel\: The docker image contains PaddlePaddle binary, source code and essential build environment. +* demo\: The docker image contains the dependencies to run PaddlePaddle demo. + +And the four rows are: + +* CPU\: CPU Version. Support CPU which has :code:`AVX` instructions. +* GPU\: GPU Version. Support GPU, and cpu has :code:`AVX` instructions. +* CPU WITHOUT AVX\: CPU Version, which support most CPU even doesn't have :code:`AVX` instructions. +* GPU WITHOUT AVX\: GPU Version, which support most CPU even doesn't have :code:`AVX` instructions. + +User can choose any version depends on machine. The following script can help you to detect your CPU support :code:`AVX` or not. + +.. code-block:: bash + + if cat /proc/cpuinfo | grep -q avx ; then echo "Support AVX"; else echo "Not support AVX"; fi + +If the output is :code:`Support AVX`, then you can choose the AVX version of PaddlePaddle, otherwise, you need select :code:`noavx` version of PaddlePaddle. For example, the CPU develop version of PaddlePaddle is :code:`paddle-dev/paddle:cpu-devel-latest`. + +The PaddlePaddle images don't contain any entry command. You need to write your entry command to use this image. See :code:`Remote Access` part or just use following command to run a :code:`bash` + +.. code-block:: bash + + docker run -it paddledev/paddle:cpu-latest /bin/bash + + +Download and Run Docker images +------------------------------ + +You have to install Docker in your machine which has linux kernel version 3.10+ first. You can refer to the official guide https://docs.docker.com/engine/installation/ for further information. + +You can use :code:`docker pull ` to download images first, or just launch a container with :code:`docker run` \: + +.. code-block:: bash + + docker run -it paddledev/paddle:cpu-latest + + +If you want to launch container with GPU support, you need to set some environment variables at the same time: + +.. code-block:: bash + + export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}" + export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') + docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:gpu-latest + + +Some notes for docker +--------------------- + +Performance ++++++++++++ + +Since Docker is based on the lightweight virtual containers, the CPU computing performance maintains well. And GPU driver and equipments are all mapped to the container, so the GPU computing performance would not be seriously affected. + +If you use high performance nic, such as RDMA(RoCE 40GbE or IB 56GbE), Ethernet(10GbE), it is recommended to use config "-net = host". + + + + +Remote access ++++++++++++++ + + +If you want to enable ssh access background, you need to build an image by yourself. Please refer to official guide https://docs.docker.com/engine/reference/builder/ for further information. + +Following is a simple Dockerfile with ssh: + +.. literalinclude:: ../../doc_cn/build_and_install/install/paddle_ssh.Dockerfile + +Then you can build an image with Dockerfile and launch a container: + +.. code-block:: bash + + # cd into Dockerfile directory + docker build . -t paddle_ssh + # run container, and map host machine port 8022 to container port 22 + docker run -d -p 8022:22 --name paddle_ssh_machine paddle_ssh + +Now, you can ssh on port 8022 to access the container, username is root, password is also root: + +.. code-block:: bash + + ssh -p 8022 root@YOUR_HOST_MACHINE + +You can stop and delete the container as following: + +.. code-block:: bash + + # stop + docker stop paddle_ssh_machine + # delete + docker rm paddle_ssh_machine diff --git a/doc/build/index.rst b/doc/build/index.rst index d6d0d19e110fc..511cdea145c7f 100644 --- a/doc/build/index.rst +++ b/doc/build/index.rst @@ -10,31 +10,24 @@ Install PaddlePaddle install_* internal/install_from_jumbo.md + docker_install.rst + ubuntu_install.rst Build from Source ----------------- -If you want to hack and contribute PaddlePaddle source code, following guides can help you\: +.. warning:: -.. toctree:: - :maxdepth: 1 - :glob: + Please use :code:`deb` package or :code:`docker` image to install paddle. The building guide is used for hacking or contributing to PaddlePaddle. + - build_from_source.md - contribute_to_paddle.md - -Docker and Debian Package installation --------------------------------------- - -Note: The installation packages are still in pre-release -state and your experience of installation may not be smooth. +If you want to hack and contribute PaddlePaddle source code, following guides can help you\: -If you want to pack docker image, the following guide can help you\: .. toctree:: :maxdepth: 1 :glob: - docker_install.md - ubuntu_install.md + build_from_source.md + contribute_to_paddle.md diff --git a/doc/build/ubuntu_install.md b/doc/build/ubuntu_install.md deleted file mode 100644 index c30a8f6db5d9e..0000000000000 --- a/doc/build/ubuntu_install.md +++ /dev/null @@ -1,21 +0,0 @@ -Debian Package installation guide -================================= - -## Debian Package installation -Currently , PaddlePaddle only provides ubuntu14.04 debian packages. -There are two versions package, including CPU and GPU. The download address is: - -https://github.com/baidu/Paddle/releases/tag/V0.8.0b0 - - -After downloading PaddlePaddle deb packages, you can run: - -```bash -dpkg -i paddle-0.8.0b-cpu.deb -apt-get install -f -``` -And if you use GPU version deb package, you need to install CUDA toolkit and cuDNN, and set related environment variables(such as LD_LIBRARY_PATH) first. It is normal when `dpkg -i` get errors. `apt-get install -f` will continue install paddle, and install dependences. - -**Note** - -PaddlePaddle package only supports x86 CPU with AVX instructions. If not, you have to download and build from source code. diff --git a/doc/build/ubuntu_install.rst b/doc/build/ubuntu_install.rst new file mode 100644 index 0000000000000..ea8042085bf45 --- /dev/null +++ b/doc/build/ubuntu_install.rst @@ -0,0 +1,25 @@ +Debian Package installation guide +================================= + +PaddlePaddle supports :code:`deb` pacakge. The installation of this :code:`deb` package is tested in ubuntu 14.04, but it should be support other debian based linux, too. + +There are four versions of debian package, :code:`cpu`, :code:`gpu`, :code:`cpu-noavx`, :code:`gpu-noavx`. And :code:`noavx` version is used to support CPU which does not contain :code:`AVX` instructions. The download url of :code:`deb` package is \: https://github.com/baidu/Paddle/releases/ + + +After downloading PaddlePaddle deb packages, you can use :code:`gdebi` install. + +.. code-block:: bash + + gdebi paddle-*.deb + +If :code:`gdebi` is not installed, you can use :code:`sudo apt-get install gdebi` to install it. + +Or you can use following commands to install PaddlePaddle. + +.. code-block:: bash + + dpkg -i paddle-*.deb + apt-get install -f + +And if you use GPU version deb package, you need to install CUDA toolkit and cuDNN, and set related environment variables(such as LD_LIBRARY_PATH) first. It is normal when `dpkg -i` get errors. `apt-get install -f` will continue install paddle, and install dependences. + diff --git a/doc_cn/build_and_install/index.rst b/doc_cn/build_and_install/index.rst index e21fc98c63dcd..2205e282248c4 100644 --- a/doc_cn/build_and_install/index.rst +++ b/doc_cn/build_and_install/index.rst @@ -1,19 +1,32 @@ 编译与安装 ======================== -PaddlePaddle提供数个预编译的二进制来进行安装，包括Docker镜像，ubuntu的deb安装包等。我们推荐使用Docker镜像来部署环境，同时欢迎贡献更多的安装包。 - -Note: The intallation packages are still in pre-release state and your experience of installation may not be smooth. +安装 +++++ -注意：目前PaddlePaddle的安装包还处在pre-release的状态，使用起来或许会不是很顺畅。 +PaddlePaddle提供数个预编译的二进制来进行安装，包括Docker镜像，ubuntu的deb安装包等。我们推荐使用Docker镜像来部署环境，同时欢迎贡献更多的安装包。 .. toctree:: :maxdepth: 1 :glob: - 源码下载(对内) <../build/internal/download_paddle_source_zh_cn.rst> 使用Jumbo安装(对内) <../build/internal/install_from_jumbo.rst> - 从源码编译安装(对内) <../build/internal/build_from_source_zh_cn.rst> install/docker_install.rst install/ubuntu_install.rst + + + +编译 +++++ + +.. warning:: + + 编译选项主要推荐高级用户查看，普通用户请走安装流程。 + +.. toctree:: + :maxdepth: 1 + :glob: + + 源码下载(对内) <../build/internal/download_paddle_source_zh_cn.rst> + 从源码编译安装(对内) <../build/internal/build_from_source_zh_cn.rst> cmake/index.rst diff --git a/doc_cn/build_and_install/install/docker_install.rst b/doc_cn/build_and_install/install/docker_install.rst index 4154cb86d8d64..44aa2a0983f4f 100644 --- a/doc_cn/build_and_install/install/docker_install.rst +++ b/doc_cn/build_and_install/install/docker_install.rst @@ -14,20 +14,43 @@ PaddlePaddle提供了Docker的使用镜像。PaddlePaddle推荐使用Docker进 PaddlePaddle提供的Docker镜像版本 -------------------------------- -我们提供了6个Docker image\: +我们提供了12个 `Docker image `_ ，他们的image name都是 :code:`paddle-dev/paddle` ，tag分别为 -* paddledev/paddle\:cpu-latest\: PaddlePaddle的CPU二进制 -* paddledev/paddle\:gpu-latest\： PaddlePaddle的GPU二进制 -* paddledev/paddle\:cpu-devel-latest\: PaddlePaddle的CPU二进制,同时包含CPU开发环境和源码 -* paddledev/paddle\:gpu-devel-latest\: PaddlePaddle的GPU二进制,同时包含GPU开发环境和源码 -* paddledev/paddle\:cpu-demo-latest\: PaddlePaddle的CPU二进制,同时包含CPU开发环境、源码和运行demo的必要依赖 -* paddledev/paddle\:gpu-demo-latest\: PaddlePaddle的GPU二进制,同时包含GPU开发环境、源码和运行demo的必要依赖 ++-----------------+------------------+------------------------+-----------------------+ +| | normal | devel | demo | ++=================+==================+========================+=======================+ +| CPU | cpu-latest | cpu-devel-latest | cpu-demo-latest | ++-----------------+------------------+------------------------+-----------------------+ +| GPU | gpu-latest | gpu-devel-latest | gpu-demo-latest | ++-----------------+------------------+------------------------+-----------------------+ +| CPU WITHOUT AVX | cpu-noavx-latest | cpu-devel-noavx-latest | cpu-demo-noavx-latest | ++-----------------+------------------+------------------------+-----------------------+ +| GPU WITHOUT AVX | gpu-noavx-latest | gpu-devel-noavx-latest | gpu-demo-noavx-latest | ++-----------------+------------------+------------------------+-----------------------+ -同时，不同的稳定版本，会将latest替换成稳定版本的版本号。 +其中，横向包括三个版本，normal，devel和demo。 + +* Normal: 正常的Docker image，只包括paddle的二进制 +* Devel: 包括Paddle的二进制、编译环境和源代码 +* Demo: 包括Paddle运行demo所需要的依赖 + +纵向包括四个版本，他们是。 + +* CPU: CPU版本。需要支持AVX指令集的CPU +* GPU: GPU版本。需要支持AVX指令集的CPU +* CPU WITHOUT AVX: CPU版本，不支持AVX指令集的CPU也可以运行 +* GPU WITHOUT AVX: GPU版本，不需要AVX指令集的CPU也可以运行。 + +用户可以选择对应版本的docker image。使用如下脚本可以确定本机的CPU知否支持 :code:`AVX` 指令集\: + +.. code-block:: bash + + if cat /proc/cpuinfo | grep -q avx ; then echo "Support AVX"; else echo "Not support AVX"; fi + +如果输出 :code:`Support AVX`，则可以选择上表中的AVX版本PaddlePaddle。否则需要选择非AVX的PaddlePaddle。选择普通CPU版本的devel版本的image，则可以使用 :code:`paddle-dev/paddle:cpu-devel-latest` 来引用这个image。 PaddlePaddle提供的镜像并不包含任何命令运行，想要运行PaddlePaddle，您需要进入镜像运行PaddlePaddle -程序或者自定义一个含有启动脚本的image。具体请参考注意事项中的 -`使用ssh访问PaddlePaddle镜像` +程序或者自定义一个含有启动脚本的image。具体请参考注意事项中的 :code:`使用ssh访问PaddlePaddle镜像` 下载和运行Docker镜像 -------------------- @@ -44,7 +67,7 @@ mac osx或者是windows机器，请参考 .. code-block:: bash - $ docker run -it paddledev/paddlepaddle:latest-cpu + $ docker run -it paddledev/paddlepaddle:cpu-latest 即可启动和进入PaddlePaddle的container。如果运行GPU版本的PaddlePaddle，则需要先将 cuda相关的Driver和设备映射进container中，脚本类似于 @@ -53,7 +76,7 @@ cuda相关的Driver和设备映射进container中，脚本类似于 $ export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" $ export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') - $ docker run -it paddledev/paddlepaddle:latest-gpu + $ docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddlepaddle:latest-gpu 进入Docker container后，运行 :code:`paddle version` 即可打印出PaddlePaddle的版本和构建 信息。安装完成的PaddlePaddle主体包括三个部分， :code:`paddle` 脚本， python的 diff --git a/doc_cn/build_and_install/install/paddle_version.txt b/doc_cn/build_and_install/install/paddle_version.txt new file mode 100644 index 0000000000000..7b2bfd2b1b3a9 --- /dev/null +++ b/doc_cn/build_and_install/install/paddle_version.txt @@ -0,0 +1,11 @@ +PaddlePaddle 0.8.0b1, compiled with + with_avx: ON + with_gpu: OFF + with_double: OFF + with_python: ON + with_rdma: OFF + with_glog: ON + with_gflags: ON + with_metric_learning: + with_timer: OFF + with_predict_sdk: \ No newline at end of file diff --git a/doc_cn/build_and_install/install/ubuntu_install.rst b/doc_cn/build_and_install/install/ubuntu_install.rst index a813d9da2e52d..70ac5225bd82e 100644 --- a/doc_cn/build_and_install/install/ubuntu_install.rst +++ b/doc_cn/build_and_install/install/ubuntu_install.rst @@ -1,55 +1,37 @@ 使用deb包在Ubuntu上安装PaddlePaddle =================================== -PaddlePaddle目前支持ubuntu 14.04版本使用deb包安装。更多的安装包PaddlePaddle会在近期提供。 -欢迎大家贡献各个发行版的安装包(例如，ubuntu，centos，debian，gentoo)。 +PaddlePaddle目前支持使用deb包安装。Paddle的 :code:`deb` 安装包在ubuntu 14.04中正确，但理论上支持其他的 debian 发行版。 -PaddlePaddle的ubuntu安装包分为两个版本，即CPU版本，和GPU版本，他们的下载地址是\: -https://github.com/baidu/Paddle/releases/tag/V0.8.0b0 -需要注意的是，目前PaddlePaddle的安装包只支持 -`AVX `_ -指令集的X86 CPU。如果系统使用不支持 `AVX`_ 指令集的CPU运行PaddlePaddle，那么需要从源码 -编译PaddlePaddle，请参考 `编译文档 <../cmake/index.html>`_ 。 +PaddlePaddle的ubuntu安装包分为四个版本，他们是 cpu、gpu、cpu-noavx、gpu-noavx 四个版本。其中 noavx 用于不支持AVX指令集的cpu。安装包的下载地址是\: https://github.com/baidu/Paddle/releases/ -用户需要先将PaddlePaddle安装包下载到本地，然后执行如下命令即可完成安装。 + +用户需要先将PaddlePaddle安装包下载到本地，然后执行如下 :code:`gdebi` 命令即可完成安装。 .. code-block:: shell - dpkg -i paddle-*-cpu.deb - apt-get install -f + gdebi paddle-*-cpu.deb + +如果 :code:`gdebi` 没有安装,则需要使用 :code:`sudo apt-get install gdebi`, 来安装 :code:`gdebi` 。 -在 :code:`dpkg -i` 的时候如果报一些依赖未找到的错误是正常的， -在 :code:`apt-get install -f` 里会继续安装 PaddlePaddle。 或者使用下面一条命令安装. .. code-block:: shell - gdebi paddle-*-cpu.deb - -如果 :code:`gdebi` 没有安装,则需要使用 :code:`sudo apt-get install gdebi`, 来安装 :code:`gdebi` + dpkg -i paddle-*-cpu.deb + apt-get install -f +在 :code:`dpkg -i` 的时候如果报一些依赖未找到的错误是正常的， +在 :code:`apt-get install -f` 里会继续安装 PaddlePaddle。 需要注意的是，如果使用GPU版本的PaddlePaddle，请安装CUDA 7.5 和CUDNN 5到本地环境中， 并设置好对应的环境变量(LD_LIBRARY_PATH等等)。 安装完成后,可以使用命令 :code:`paddle version` 查看安装后的paddle 版本。可能的输出为 -.. code-block:: text - - PaddlePaddle 0.8.0b1, compiled with - with_avx: ON - with_gpu: OFF - with_double: OFF - with_python: ON - with_rdma: OFF - with_glog: ON - with_gflags: ON - with_metric_learning: - with_timer: OFF - with_predict_sdk: - +.. literalinclude:: paddle_version.txt 可能遇到的问题 -------------- From 29c16e22420b8b69d17572accd2f6fc0239c8c7e Mon Sep 17 00:00:00 2001 From: gangliao Date: Sat, 1 Oct 2016 10:55:13 +0800 Subject: [PATCH 145/324] Update Readme (#153) * Update Readme * Update readme * Update readme --- README.md | 69 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 05e544186a2d0..2162488f2f7de 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,65 @@ # PaddlePaddle -[![Build Status](https://travis-ci.org/baidu/Paddle.svg?branch=master)](https://travis-ci.org/baidu/Paddle) -Welcome to the PaddlePaddle GitHub. - -Do you wanna try and play PaddlePaddle? Just following the [Install Guide](http://www.paddlepaddle.org/doc/build/index.html) and [Quick Start](http://www.paddlepaddle.org/doc/demo/quick_start/index_en.html). The chinese version is [Install Guide](http://www.paddlepaddle.org/doc_cn/build_and_install/index.html) and [Quick Start](http://www.paddlepaddle.org/doc_cn/demo/quick_start/index.html). +| **`Linux`** | +|----------------| +|[![Build Status](https://travis-ci.org/baidu/Paddle.svg?branch=master)](https://travis-ci.org/baidu/Paddle)| -Please refer to our [release log](https://github.com/baidu/Paddle/releases) to track the latest feature of PaddlePaddle. +Welcome to the PaddlePaddle GitHub. PaddlePaddle (PArallel Distributed Deep LEarning) is an easy-to-use, efficient, flexible and scalable deep learning platform, which is originally developed by Baidu scientists and engineers for the purpose of applying deep learning to many products at Baidu. +Our vision is to enable deep learning for everyone via PaddlePaddle. +Please refer to our [release log](https://github.com/baidu/Paddle/releases) to track the latest feature of PaddlePaddle. + ## Features - **Flexibility** - PaddlePaddle supports a wide range of neural network architectures and - optimization algorithms. It is easy to configure complex models such as - neural machine translation model with attention mechanism or complex memory - connection. + PaddlePaddle supports a wide range of neural network architectures and + optimization algorithms. It is easy to configure complex models such as + neural machine translation model with attention mechanism or complex memory + connection. - **Efficiency** - In order to unleash the power of heterogeneous computing resource, - optimization occurs at different levels of PaddlePaddle, including - computing, memory, architecture and communication. The following are some - examples: - 1. Optimized math operations through SSE/AVX intrinsics, BLAS libraries - (e.g. MKL, ATLAS, cuBLAS) or customized CPU/GPU kernels. - 2. Highly optimized recurrent networks which can handle **variable-length** - sequence without padding. - 3. Optimized local and distributed training for models with high dimensional - sparse data. + In order to unleash the power of heterogeneous computing resource, + optimization occurs at different levels of PaddlePaddle, including + computing, memory, architecture and communication. The following are some + examples: + + - Optimized math operations through SSE/AVX intrinsics, BLAS libraries + (e.g. MKL, ATLAS, cuBLAS) or customized CPU/GPU kernels. + - Highly optimized recurrent networks which can handle **variable-length** + sequence without padding. + - Optimized local and distributed training for models with high dimensional + sparse data. - **Scalability** - With PaddlePaddle, it is easy to use many CPUs/GPUs and machines to speed - up your training. PaddlePaddle can achieve high throughput and performance - via optimized communication. + With PaddlePaddle, it is easy to use many CPUs/GPUs and machines to speed + up your training. PaddlePaddle can achieve high throughput and performance + via optimized communication. - **Connected to Products** - In addition, PaddlePaddle is also designed to be easily deployable. At Baidu, - PaddlePaddle has been deployed into products or service with a vast number - of users, including ad click-through rate (CTR) prediction, large-scale image - classification, optical character recognition(OCR), search ranking, computer - virus detection, recommendation, etc. It is widely utilized in products at - Baidu and it has achieved a significant impact. We hope you can also exploit - the capability of PaddlePaddle to make a huge impact for your product. + In addition, PaddlePaddle is also designed to be easily deployable. At Baidu, + PaddlePaddle has been deployed into products or service with a vast number + of users, including ad click-through rate (CTR) prediction, large-scale image + classification, optical character recognition(OCR), search ranking, computer + virus detection, recommendation, etc. It is widely utilized in products at + Baidu and it has achieved a significant impact. We hope you can also exploit + the capability of PaddlePaddle to make a huge impact for your product. ## Installation -See [Installation Guide](http://paddlepaddle.org/doc/build/) to install from pre-built package or build from the source code. (Note: The installation packages are still in pre-release state and your experience of installation may not be smooth.). - +Check out the [Install Guide](http://paddlepaddle.org/doc/build/) to install from +pre-built packages (**docker image**, **deb package**) or +directly build on **Linux** and **Mac OS X** from the source code. + ## Documentation -- [Chinese Documentation](http://paddlepaddle.org/doc_cn/)
+Both [English Docs](http://paddlepaddle.org/doc/) and [Chinese Docs](http://paddlepaddle.org/doc_cn/) are provided for our users and developers. - [Quick Start](http://paddlepaddle.org/doc/demo/quick_start/index_en)
You can follow the quick start tutorial to learn how use PaddlePaddle From d8e28741858be60718973c59e21211968f460cf9 Mon Sep 17 00:00:00 2001 From: gangliao Date: Sat, 8 Oct 2016 10:07:07 +0800 Subject: [PATCH 146/324] Fix CUDA_VERSION Comparsion (#165) --- cmake/flags.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 4b99e7f7fb6af..bcd734b7e4dbc 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -88,15 +88,15 @@ endfunction() # Common gpu architectures: Kepler, Maxwell foreach(capability 30 35 50) - list(APPEND __arch_flags " -gencode arch=compute_${capability},code=sm_${capability}") + list(APPEND __arch_flags " -gencode arch=compute_${capability},code=sm_${capability}") endforeach() -if (CUDA_VERSION VERSION_GREATER "7.0") +if (CUDA_VERSION VERSION_GREATER "7.0" OR CUDA_VERSION VERSION_EQUAL "7.0") list(APPEND __arch_flags " -gencode arch=compute_52,code=sm_52") endif() # Modern gpu architectures: Pascal -if (CUDA_VERSION VERSION_GREATER "8.0") +if (CUDA_VERSION VERSION_GREATER "8.0" OR CUDA_VERSION VERSION_EQUAL "8.0") list(APPEND __arch_flags " -gencode arch=compute_60,code=sm_60") endif() From 1c09e9d5d17bc40365d35b1a8bf7db1aad26e41b Mon Sep 17 00:00:00 2001 From: gangliao Date: Sat, 8 Oct 2016 10:37:50 +0800 Subject: [PATCH 147/324] Update readme (#155) * Update readme * Apache 2.0 --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2162488f2f7de..1cc0444c0617a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # PaddlePaddle -| **`Linux`** | -|----------------| -|[![Build Status](https://travis-ci.org/baidu/Paddle.svg?branch=master)](https://travis-ci.org/baidu/Paddle)| +| **`Linux`** | **`License`** | **`Chat Room`** | +|----------------|---------------|-----------------| +|[![Build Status](https://travis-ci.org/baidu/Paddle.svg?branch=master)](https://travis-ci.org/baidu/Paddle)|[![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](LICENSE)|[![Join the chat at https://gitter.im/PaddlePaddle/Deep_Learning](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/PaddlePaddle/Deep_Learning?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)| Welcome to the PaddlePaddle GitHub. @@ -86,9 +86,9 @@ Both [English Docs](http://paddlepaddle.org/doc/) and [Chinese Docs](http://padd - [Source Code Documents](http://paddlepaddle.org/doc/source/)
## Ask Questions - -If you want to ask questions and discuss about methods and models, welcome -to send email to paddle-dev@baidu.com. Framework development discussions and +Please join the [**gitter chat**](https://gitter.im/PaddlePaddle/Deep_Learning) or send email to +**paddle-dev@baidu.com** to ask questions and talk about methods and models. +Framework development discussions and bug reports are collected on [Issues](https://github.com/baidu/paddle/issues). ## Copyright and License From 1c2ebe467bb3c88b858f5b583ab48c8944c7f0f2 Mon Sep 17 00:00:00 2001 From: Zrachel Date: Sat, 8 Oct 2016 11:28:57 +0800 Subject: [PATCH 148/324] add interface and test of RecurrentGradientMachine (#156) * add interface and unittest of RecurrentGradientMachine for the function of multiple Subsequence inlinks with unequal token length --- paddle/gserver/tests/rnn_data_provider.py | 30 ++++- ...ce_nest_rnn_multi_unequalength_inputs.conf | 106 ++++++++++++++++++ ...equence_rnn_multi_unequalength_inputs.conf | 75 +++++++++++++ .../tests/test_RecurrentGradientMachine.cpp | 9 ++ .../paddle/trainer_config_helpers/layers.py | 30 ++++- 5 files changed, 246 insertions(+), 4 deletions(-) create mode 100644 paddle/gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.conf create mode 100644 paddle/gserver/tests/sequence_rnn_multi_unequalength_inputs.conf diff --git a/paddle/gserver/tests/rnn_data_provider.py b/paddle/gserver/tests/rnn_data_provider.py index 347d5891b906b..5c3b062309c51 100644 --- a/paddle/gserver/tests/rnn_data_provider.py +++ b/paddle/gserver/tests/rnn_data_provider.py @@ -21,7 +21,7 @@ @provider(input_types=[integer_value_sub_sequence(10), - integer_value(2)], + integer_value(3)], should_shuffle=False) def process_subseq(settings, file_name): for d in data: @@ -29,7 +29,7 @@ def process_subseq(settings, file_name): @provider(input_types=[integer_value_sequence(10), - integer_value(2)], + integer_value(3)], should_shuffle=False) def process_seq(settings, file_name): for d in data: @@ -37,3 +37,29 @@ def process_seq(settings, file_name): for subseq in d[0]: seq += subseq yield seq, d[1] + +data2 = [ + [[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]] ,0], + [[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]], 1], +] + +@provider(input_types=[integer_value_sub_sequence(10), + integer_value_sub_sequence(10), + integer_value(2)], + should_shuffle=False) +def process_unequalength_subseq(settings, file_name): + for d in data2: + yield d + + +@provider(input_types=[integer_value_sequence(10), + integer_value_sequence(10), + integer_value(2)], + should_shuffle=False) +def process_unequalength_seq(settings, file_name): + for d in data2: + words1=reduce(lambda x,y: x+y, d[0]) + words2=reduce(lambda x,y: x+y, d[1]) + yield words1, words2, d[2] + + diff --git a/paddle/gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.conf b/paddle/gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.conf new file mode 100644 index 0000000000000..d0b9450f4b9f9 --- /dev/null +++ b/paddle/gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.conf @@ -0,0 +1,106 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +######################## data source ################################ +define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', + test_list=None, + module='rnn_data_provider', + obj='process_unequalength_subseq') + + +settings(batch_size=2, learning_rate=0.01) +######################## network configure ################################ +dict_dim = 10 +word_dim = 8 +hidden_dim = 8 +label_dim = 2 + +speaker1 = data_layer(name="word1", size=dict_dim) +speaker2 = data_layer(name="word2", size=dict_dim) + +emb1 = embedding_layer(input=speaker1, size=word_dim) +emb2 = embedding_layer(input=speaker2, size=word_dim) + +# This hierachical RNN is designed to be equivalent to the simple RNN in +# sequence_rnn_multi_unequalength_inputs.conf + +def outer_step(x1, x2): + outer_mem1 = memory(name = "outer_rnn_state1", size = hidden_dim) + outer_mem2 = memory(name = "outer_rnn_state2", size = hidden_dim) + def inner_step1(y): + inner_mem = memory(name = 'inner_rnn_state_' + y.name, + size = hidden_dim, + boot_layer = outer_mem1) + out = fc_layer(input = [y, inner_mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'inner_rnn_state_' + y.name) + return out + + def inner_step2(y): + inner_mem = memory(name = 'inner_rnn_state_' + y.name, + size = hidden_dim, + boot_layer = outer_mem2) + out = fc_layer(input = [y, inner_mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'inner_rnn_state_' + y.name) + return out + + encoder1 = recurrent_group( + step = inner_step1, + name = 'inner1', + input = x1) + + encoder2 = recurrent_group( + step = inner_step2, + name = 'inner2', + input = x2) + + sentence_last_state1 = last_seq(input = encoder1, name = 'outer_rnn_state1') + sentence_last_state2_ = last_seq(input = encoder2, name = 'outer_rnn_state2') + + encoder1_expand = expand_layer(input = sentence_last_state1, + expand_as = encoder2) + + return [encoder1_expand, encoder2] + + +encoder1_rep, encoder2_rep = recurrent_group( + name="outer", + step=outer_step, + input=[SubsequenceInput(emb1), SubsequenceInput(emb2)], + targetInlink=emb2) + +encoder1_last = last_seq(input = encoder1_rep) +encoder1_expandlast = expand_layer(input = encoder1_last, + expand_as = encoder2_rep) +context = mixed_layer(input = [identity_projection(encoder1_expandlast), + identity_projection(encoder2_rep)], + size = hidden_dim) + +rep = last_seq(input=context) +prob = fc_layer(size=label_dim, + input=rep, + act=SoftmaxActivation(), + bias_attr=True) + +outputs(classification_cost(input=prob, + label=data_layer(name="label", size=label_dim))) + diff --git a/paddle/gserver/tests/sequence_rnn_multi_unequalength_inputs.conf b/paddle/gserver/tests/sequence_rnn_multi_unequalength_inputs.conf new file mode 100644 index 0000000000000..28b1cb98cf132 --- /dev/null +++ b/paddle/gserver/tests/sequence_rnn_multi_unequalength_inputs.conf @@ -0,0 +1,75 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +######################## data source ################################ +define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', + test_list=None, + module='rnn_data_provider', + obj='process_unequalength_seq') + + +settings(batch_size=2, learning_rate=0.01) +######################## network configure ################################ +dict_dim = 10 +word_dim = 8 +hidden_dim = 8 +label_dim = 2 + +speaker1 = data_layer(name="word1", size=dict_dim) +speaker2 = data_layer(name="word2", size=dict_dim) + +emb1 = embedding_layer(input=speaker1, size=word_dim) +emb2 = embedding_layer(input=speaker2, size=word_dim) + +# This hierachical RNN is designed to be equivalent to the RNN in +# sequence_nest_rnn_multi_unequalength_inputs.conf + +def step(x1, x2): + def calrnn(y): + mem = memory(name = 'rnn_state_' + y.name, size = hidden_dim) + out = fc_layer(input = [y, mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'rnn_state_' + y.name) + return out + + encoder1 = calrnn(x1) + encoder2 = calrnn(x2) + return [encoder1, encoder2] + +encoder1_rep, encoder2_rep = recurrent_group( + name="stepout", + step=step, + input=[emb1, emb2]) + +encoder1_last = last_seq(input = encoder1_rep) +encoder1_expandlast = expand_layer(input = encoder1_last, + expand_as = encoder2_rep) +context = mixed_layer(input = [identity_projection(encoder1_expandlast), + identity_projection(encoder2_rep)], + size = hidden_dim) + +rep = last_seq(input=context) +prob = fc_layer(size=label_dim, + input=rep, + act=SoftmaxActivation(), + bias_attr=True) + +outputs(classification_cost(input=prob, + label=data_layer(name="label", size=label_dim))) + diff --git a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp index 550df0a31844e..ae7f617371ca5 100644 --- a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp +++ b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp @@ -73,6 +73,7 @@ void CalCost(const string& conf, const string& dir, real* cost, *ThreadLocalRand::getSeed() = FLAGS_seed; vecW.randnorm(0, 0.1); + vecMomentum.randnorm(0, 0.1); trainer.startTrain(); for (int i = 0; i < num_passes; ++i) { @@ -140,6 +141,14 @@ TEST(RecurrentGradientMachine, rnn_multi_input) { } } +TEST(RecurrentGradientMachine, rnn_multi_unequalength_input) { + for (bool useGpu : {false, true}) { + test("gserver/tests/sequence_rnn_multi_unequalength_inputs.conf", + "gserver/tests/sequence_nest_rnn_multi_unequalength_inputs.conf", + 1e-6, useGpu); + } +} + int main(int argc, char** argv) { if (paddle::version::isWithPyDataProvider()) { if (!paddle::version::isWithGpu()) { diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index c355dc042ac18..47db197f422ea 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -2347,7 +2347,7 @@ def __init__(self, input): @wrap_name_default("recurrent_group") -def recurrent_group(step, input, reverse=False, name=None): +def recurrent_group(step, input, reverse=False, name=None, targetInlink=None): """ Recurrent layer group is an extremely flexible recurrent unit in PaddlePaddle. As long as the user defines the calculation done within a @@ -2401,6 +2401,17 @@ def step(input): :param reverse: If reverse is set true, the recurrent unit will process the input sequence in a reverse order. :type reverse: bool + + :param targetInlink: the input layer which share info with layer group's output + + Param input specifies multiple input layers. For + SubsequenceInput inputs, config should assign one input + layer that share info(the number of sentences and the number + of words in each sentence) with all layer group's outputs. + targetInlink should be one of the layer group's input. + + :type targetInlink: LayerOutput|SubsequenceInput + :return: LayerOutput object. :rtype: LayerOutput """ @@ -2419,6 +2430,20 @@ def is_in_links(x): in_links = filter(is_in_links, input) + def targetInlink_in_inlinks(): + for inlink in in_links: + if isinstance(inlink, SubsequenceInput): + if targetInlink == inlink.input: + return True + elif targetInlink == inlink: + return True + return False + + assert(targetInlink == None or targetInlink_in_inlinks()) + targetInlinkName = None if targetInlink == None \ + else targetInlink.name if isinstance(targetInlink, LayerOutput) \ + else targetInlink.input.name + contains_sub_seq = [False] def map_in_links(x): @@ -2430,7 +2455,8 @@ def map_in_links(x): RecurrentLayerGroupWithoutOutLinksBegin( name=name, in_links=map(map_in_links, in_links), - seq_reversed=reverse) + seq_reversed=reverse, + target_inlinkname=targetInlinkName) in_args = [] for each_input in input: assert is_single_input(each_input) From db38043439cc9beb1dec391302317510028e4ab7 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Sat, 8 Oct 2016 16:58:44 +0800 Subject: [PATCH 149/324] bug fix for dataprovider for quick start inference (#168) --- demo/quick_start/dataprovider_bow.py | 2 +- demo/quick_start/dataprovider_emb.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demo/quick_start/dataprovider_bow.py b/demo/quick_start/dataprovider_bow.py index 435e6d8175bd6..f8cde189cf87d 100644 --- a/demo/quick_start/dataprovider_bow.py +++ b/demo/quick_start/dataprovider_bow.py @@ -79,6 +79,6 @@ def predict_initializer(settings, dictionary, **kwargs): def process_predict(settings, file_name): with open(file_name, 'r') as f: for line in f: - comment = line.strip() + comment = line.strip().split() word_vector = [settings.word_dict.get(w, UNK_IDX) for w in comment] yield word_vector diff --git a/demo/quick_start/dataprovider_emb.py b/demo/quick_start/dataprovider_emb.py index e5030c5e71aa5..ca940a89e5477 100755 --- a/demo/quick_start/dataprovider_emb.py +++ b/demo/quick_start/dataprovider_emb.py @@ -47,6 +47,6 @@ def predict_initializer(settings, dictionary, **kwargs): def process_predict(settings, file_name): with open(file_name, 'r') as f: for line in f: - comment = line.strip() + comment = line.strip().split() word_slot = [settings.word_dict.get(w, UNK_IDX) for w in comment] yield word_slot From 0ab332242fbaad8f541566e5ecc602e2180c6591 Mon Sep 17 00:00:00 2001 From: gangliao Date: Sat, 8 Oct 2016 18:22:47 +0800 Subject: [PATCH 150/324] Support MAC OS Sierra (#169) --- cmake/flags.cmake | 5 +++++ paddle/cuda/src/hl_cuda_device.cc | 6 +++++- paddle/pserver/SocketChannel.cpp | 3 ++- paddle/setup.py.in | 2 ++ paddle/utils/ThreadLocal.h | 10 ++-------- paddle/utils/Util.cpp | 6 +++++- paddle/utils/arch/osx/Locks.cpp | 20 ++++++++++++++++++-- 7 files changed, 39 insertions(+), 13 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index bcd734b7e4dbc..cc59309ee7efa 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -71,6 +71,11 @@ foreach(flag ${COMMON_FLAGS}) safe_set_cxxflag(CMAKE_CXX_FLAGS ${flag}) endforeach() +# On Mac OS X build fat binaries with x86_64 architectures by default. +if (APPLE) + set (CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Build architectures for OSX" FORCE) +endif () + # Release/Debug flags set by cmake. Such as -O3 -g -DNDEBUG etc. # So, don't set these flags here. diff --git a/paddle/cuda/src/hl_cuda_device.cc b/paddle/cuda/src/hl_cuda_device.cc index acd8e2fe6afb4..f4c07367b485b 100644 --- a/paddle/cuda/src/hl_cuda_device.cc +++ b/paddle/cuda/src/hl_cuda_device.cc @@ -211,7 +211,11 @@ bool hl_start_flag = false; inline pid_t gettid() { #if defined(__APPLE__) || defined(__OSX__) - pid_t tid = syscall(SYS_thread_selfid); + // syscall is deprecated: first deprecated in macOS 10.12. + // syscall is unsupported; + // syscall pid_t tid = syscall(SYS_thread_selfid); + uint64_t tid; + pthread_threadid_np(NULL, &tid); #else #ifndef __NR_gettid #define __NR_gettid 224 diff --git a/paddle/pserver/SocketChannel.cpp b/paddle/pserver/SocketChannel.cpp index b9d542a296ddd..20295d7cdc22b 100644 --- a/paddle/pserver/SocketChannel.cpp +++ b/paddle/pserver/SocketChannel.cpp @@ -157,7 +157,8 @@ void SocketChannel::writeMessage(const std::vector& userIovs) { std::vector iovs; iovs.reserve(userIovs.size() + 2); iovs.push_back({&header, sizeof(header)}); - iovs.push_back({&iovLengths[0], sizeof(iovLengths[0]) * header.numIovs}); + iovs.push_back({&iovLengths[0], static_cast( + sizeof(iovLengths[0]) * header.numIovs)}); iovs.insert(iovs.end(), userIovs.begin(), userIovs.end()); header.totalLength = 0; diff --git a/paddle/setup.py.in b/paddle/setup.py.in index 02ea9067431c6..3341dd6f95969 100644 --- a/paddle/setup.py.in +++ b/paddle/setup.py.in @@ -18,6 +18,7 @@ from setuptools import setup, Extension import numpy as np import api.paddle_ld_flags import platform +import os system = platform.system().lower() @@ -45,6 +46,7 @@ except: if is_lin == True: extra_links = ["-Xlinker", '-start-group'] + extra_links + ["-Xlinker", "-end-group"] elif is_osx == True: + os.environ["ARCHFLAGS"] = "-arch x86_64" extra_links = ["-Wl,-all_load"] + extra_links include_dirs = [np.get_include(), "../"] # include numpy and paddle. diff --git a/paddle/utils/ThreadLocal.h b/paddle/utils/ThreadLocal.h index 686a1a99a4aa0..b91e4ad5472ca 100644 --- a/paddle/utils/ThreadLocal.h +++ b/paddle/utils/ThreadLocal.h @@ -22,6 +22,7 @@ limitations under the License. */ #include #include #include +#include "Util.h" #include "Logging.h" namespace paddle { @@ -156,14 +157,7 @@ class ThreadLocalD { static void dataDestructor(void* p) { delete (T*)p; } void updateMap(T* p) { -#if defined(__APPLE__) || defined(__OSX__) - pid_t tid = syscall(SYS_thread_selfid); -#else - #ifndef __NR_gettid - #define __NR_gettid 224 - #endif - pid_t tid = syscall(__NR_gettid); -#endif + pid_t tid = getTID(); CHECK_NE(tid, -1); std::lock_guard guard(mutex_); auto ret = threadMap_.insert(std::make_pair(tid, p)); diff --git a/paddle/utils/Util.cpp b/paddle/utils/Util.cpp index c3c76f907d40e..45251213d2d79 100644 --- a/paddle/utils/Util.cpp +++ b/paddle/utils/Util.cpp @@ -95,7 +95,11 @@ namespace paddle { pid_t getTID() { #if defined(__APPLE__) || defined(__OSX__) - pid_t tid = syscall(SYS_thread_selfid); + // syscall is deprecated: first deprecated in macOS 10.12. + // syscall is unsupported; + // syscall pid_t tid = syscall(SYS_thread_selfid); + uint64_t tid; + pthread_threadid_np(NULL, &tid); #else #ifndef __NR_gettid #define __NR_gettid 224 diff --git a/paddle/utils/arch/osx/Locks.cpp b/paddle/utils/arch/osx/Locks.cpp index 47e44e9d7c114..44bab7198d5f4 100644 --- a/paddle/utils/arch/osx/Locks.cpp +++ b/paddle/utils/arch/osx/Locks.cpp @@ -16,6 +16,11 @@ limitations under the License. */ #include "paddle/utils/Logging.h" #include #include + +#if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 +#include +#endif + namespace paddle { class SemaphorePrivate { @@ -50,21 +55,32 @@ void Semaphore::post() { class SpinLockPrivate { public: +#if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 + os_unfair_lock lock_; +#else SpinLockPrivate(): lock_(OS_SPINLOCK_INIT) {} - OSSpinLock lock_; - char padding_[64 - sizeof(OSSpinLock)]; // Padding to cache line size +#endif + char padding_[64 - sizeof(lock_)]; // Padding to cache line size }; SpinLock::SpinLock(): m(new SpinLockPrivate()) {} SpinLock::~SpinLock() { delete m; } void SpinLock::lock() { +#if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 + os_unfair_lock_lock(&m->lock_); +#else OSSpinLockLock(&m->lock_); +#endif } void SpinLock::unlock() { +#if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 + os_unfair_lock_unlock(&m->lock_); +#else OSSpinLockUnlock(&m->lock_); +#endif } From 8a044d2e2d607f183c173d8fb4d16c3db06b1418 Mon Sep 17 00:00:00 2001 From: stoneyang Date: Sun, 9 Oct 2016 10:27:12 +0800 Subject: [PATCH 151/324] typo in image classification demo (#167) --- demo/image_classification/vgg_16_cifar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 demo/image_classification/vgg_16_cifar.py diff --git a/demo/image_classification/vgg_16_cifar.py b/demo/image_classification/vgg_16_cifar.py old mode 100644 new mode 100755 index 238608c3cbede..e8b8af4bd313d --- a/demo/image_classification/vgg_16_cifar.py +++ b/demo/image_classification/vgg_16_cifar.py @@ -44,7 +44,7 @@ label_size=10 img = data_layer(name='image', size=data_size) -# small_vgg is predined in trainer_config_helpers.network +# small_vgg is predefined in trainer_config_helpers.networks predict = small_vgg(input_image=img, num_channels=3, num_classes=label_size) From 191fafe355717c506663b5ddd13d8a18ae944924 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Sun, 9 Oct 2016 13:53:30 +0800 Subject: [PATCH 152/324] support rectangle padding, stride, window and input for PoolProjection (#115) * support rectangle padding, stride, window and input for PoolProjection * Follow comments. 1. Remove start 2. refine img_pool_a/b.conf for test_NetworkCompare 3. Split unit test * Modify the test in img_layers.py --- paddle/cuda/include/hl_cnn.h | 88 ++++--- paddle/cuda/include/stub/hl_cnn_stub.h | 48 ++-- paddle/cuda/src/hl_cuda_cnn.cu | 237 +++++++++++------- paddle/gserver/layers/CudnnPoolLayer.cpp | 1 - paddle/gserver/layers/CudnnPoolLayer.h | 10 - paddle/gserver/layers/PoolLayer.cpp | 17 -- paddle/gserver/layers/PoolLayer.h | 12 +- paddle/gserver/layers/PoolProjectionLayer.cpp | 28 ++- paddle/gserver/tests/img_pool_a.conf | 46 ++++ paddle/gserver/tests/img_pool_b.conf | 44 ++++ paddle/gserver/tests/test_LayerGrad.cpp | 36 +-- paddle/gserver/tests/test_NetworkCompare.cpp | 14 ++ paddle/math/Matrix.cpp | 155 +++++++----- paddle/math/Matrix.h | 79 ++++-- paddle/math/tests/test_matrixCompare.cpp | 153 +++++++++++ proto/ModelConfig.proto.m4 | 3 +- python/paddle/trainer/config_parser.py | 25 +- .../paddle/trainer_config_helpers/layers.py | 43 +++- .../paddle/trainer_config_helpers/networks.py | 12 +- .../paddle/trainer_config_helpers/poolings.py | 23 +- .../tests/configs/check.md5 | 2 +- .../tests/configs/img_layers.py | 6 +- 22 files changed, 757 insertions(+), 325 deletions(-) create mode 100644 paddle/gserver/tests/img_pool_a.conf create mode 100644 paddle/gserver/tests/img_pool_b.conf diff --git a/paddle/cuda/include/hl_cnn.h b/paddle/cuda/include/hl_cnn.h index dcae62d06b26d..5d750333e1e35 100644 --- a/paddle/cuda/include/hl_cnn.h +++ b/paddle/cuda/include/hl_cnn.h @@ -84,16 +84,23 @@ extern void hl_expand_feature2col( * @param[in] width image width. * @param[in] pooledH output image height. * @param[in] pooledW output image width. - * @param[in] sizeX size of pooling window. - * @param[in] stride pooling stride. - * @param[in] start pooling start. + * @param[in] sizeX width of pooling window. + * @param[in] sizeY height of pooling window. + * @param[in] strideH pooling stride height. + * @param[in] strideW pooling stride width. + * @param[in] paddingH padding height. + * @param[in] paddingW padding width. * @param[out] tgtData output data. * */ extern void hl_maxpool_forward( - int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData); + const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, real* tgtData); /** * @brief Maximum pool backward. @@ -107,21 +114,28 @@ extern void hl_maxpool_forward( * @param[in] width image width. * @param[in] pooledH output image height. * @param[in] pooledW output image width. - * @param[in] sizeX size of pooling window. - * @param[in] stride pooling stride. - * @param[in] start pooling start. - * @param[out] targetGrad output grad. + * @param[in] sizeX width of pooling window. + * @param[in] sizeY height of pooling window. + * @param[in] strideH pooling stride height. + * @param[in] strideW pooling stride width. * @param[in] scaleA scale. * @param[in] scaleB scale. + * @param[in] paddingH padding height. + * @param[in] paddingW padding width. + * @param[out] targetGrad output grad. * */ extern void hl_maxpool_backward( - int frameCnt, const real* inputData, + const int frameCnt, const real* inputData, const real* outData, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* targetGrad, - real scaleA, real scaleB); + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real scaleA, real scaleB, + real* targetGrad); /** * @brief Averge pool forward. @@ -133,16 +147,23 @@ extern void hl_maxpool_backward( * @param[in] width image width. * @param[in] pooledH output image height. * @param[in] pooledW output image width. - * @param[in] sizeX size of pooling window. - * @param[in] stride pooling stride. - * @param[in] start pooling start. + * @param[in] sizeX width of pooling window. + * @param[in] sizeY height of pooling window. + * @param[in] strideH pooling stride height. + * @param[in] strideW pooling stride width. + * @param[in] paddingH padding height. + * @param[in] paddingW padding width. * @param[out] tgtData output data. * */ extern void hl_avgpool_forward( - int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData); + const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, real* tgtData); /** * @brief Maximum pool backward. @@ -154,20 +175,27 @@ extern void hl_avgpool_forward( * @param[in] width image width. * @param[in] pooledH output image height. * @param[in] pooledW output image width. - * @param[in] sizeX size of pooling window. - * @param[in] stride pooling stride. - * @param[in] start pooling start. - * @param[out] backGrad output grad. + * @param[in] sizeX width of pooling window. + * @param[in] sizeY height of pooling window. + * @param[in] strideH pooling stride height. + * @param[in] strideW pooling stride width. + * @param[in] paddingH padding height. + * @param[in] paddingW padding width. * @param[in] scaleA scale. * @param[in] scaleB scale. + * @param[out] backGrad output grad. * */ extern void hl_avgpool_backward( - int frameCnt, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* backGrad, - real scaleA, real scaleB); + const int frameCnt, const real* outGrad, + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + int paddingH, int paddingW, + real scaleA, real scaleB, + real* backGrad); /** * @brief Cross-map-respose normalize forward. diff --git a/paddle/cuda/include/stub/hl_cnn_stub.h b/paddle/cuda/include/stub/hl_cnn_stub.h index e4d46e4fb186e..38e359c3eb2f3 100644 --- a/paddle/cuda/include/stub/hl_cnn_stub.h +++ b/paddle/cuda/include/stub/hl_cnn_stub.h @@ -38,29 +38,45 @@ inline void hl_expand_feature2col( real* dataCol) {} inline void hl_maxpool_forward( - int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData) {} + const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, real* tgtData) {} inline void hl_maxpool_backward( - int frameCnt, const real* inputData, + const int frameCnt, const real* inputData, const real* outData, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* targetGrad, - real scaleA, real scaleB) {} + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real scaleA, real scaleB, + real* targetGrad) {} inline void hl_avgpool_forward( - int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData) {} + const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, real* tgtData) {} inline void hl_avgpool_backward( - int frameCnt, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* backGrad, - real scaleA, real scaleB) {} + const int frameCnt, const real* outGrad, + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + int paddingH, int paddingW, + real scaleA, real scaleB, + real* backGrad) {} inline void hl_CMRNorm_forward( size_t frameCnt, const real* in, real* scale, real* out, diff --git a/paddle/cuda/src/hl_cuda_cnn.cu b/paddle/cuda/src/hl_cuda_cnn.cu index b3695a2c7f88e..abac83a3e0447 100644 --- a/paddle/cuda/src/hl_cuda_cnn.cu +++ b/paddle/cuda/src/hl_cuda_cnn.cu @@ -145,24 +145,28 @@ void hl_shrink_col2feature(const real * dataCol, size_t channels, CHECK_SYNC("hl_shrink_col2feature failed"); } -__global__ void KeMaxPoolForward(int nthreads, const real* inputData, - int channels, int height, int width, - int pooledH, int pooledW, - int ksize, int stride, int start, +__global__ void KeMaxPoolForward(const int nthreads, const real* inputData, + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int ksizeW, const int ksizeH, + const int strideH, const int strideW, + const int offsetH, const int offsetW, real* tgtData) { - int index = blockIdx.y * blockDim.x + threadIdx.x; + int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int pw = index % pooledW; int ph = (index / pooledW) % pooledH; int c = (index / pooledW / pooledH) % channels; - int frameNum = blockIdx.x; - int hstart = ph * stride + start; - int hend = min(hstart + ksize, height); - int wstart = pw * stride + start; - int wend = min(wstart + ksize, width); + int frameNum = index / pooledW / pooledH / channels; + int hstart = ph * strideH - offsetH; + int wstart = pw * strideW - offsetW; + int hend = min(hstart + ksizeH, height); + int wend = min(wstart + ksizeW, width); + hstart = max(hstart, 0); + wstart = max(wstart, 0); real maxval = -FLT_MAX; inputData += (frameNum * channels + c) * height * width; - tgtData += (frameNum * channels) * pooledW * pooledH; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { if (maxval < inputData[h * width + w]) @@ -173,44 +177,54 @@ __global__ void KeMaxPoolForward(int nthreads, const real* inputData, } } -void hl_maxpool_forward(int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData) { - int num_kernels = pooledH * pooledW * channels; - int blocksX = frameCnt; - int blocksY = (num_kernels + 1024 -1) / 1024; +void hl_maxpool_forward(const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real* tgtData) { + + int num_kernels = pooledH * pooledW * channels * frameCnt; + int blocks = (num_kernels + 1024 - 1) / 1024; dim3 threads(1024, 1); - dim3 grid(blocksX, blocksY); + dim3 grid(blocks, 1); + KeMaxPoolForward<<< grid, threads, 0, STREAM_DEFAULT >>> (num_kernels, inputData, channels, height, width, - pooledH, pooledW, sizeX, stride, start, tgtData); + pooledH, pooledW, sizeX, sizeY, strideH, strideW, + paddingH, paddingW, tgtData); CHECK_SYNC("hl_maxpool_forward failed"); } -__global__ void KeMaxPoolBackward(int nthreads, const real* inputData, +__global__ void KeMaxPoolBackward(const int nthreads, const real* inputData, const real* outData, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* targetGrad, - real scaleA, real scaleB) { - int index = blockIdx.y * blockDim.x + threadIdx.x; + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int padH, const int padW, + real scaleA, real scaleB, + real* targetGrad) { + int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { // find out the local index // find out the local offset - int offsetW = index % width + start; - int offsetH = (index / width) % height + start; + int offsetW = index % width + padW; + int offsetH = (index / width) % height + padH; int offsetC = (index / width / height) % channels; - int frameNum = blockIdx.x; - int phstart = (offsetH < sizeX) ? 0 : (offsetH - sizeX) / stride + 1; - int phend = min(offsetH / stride + 1, pooledH); - int pwstart = (offsetW < sizeX) ? 0 : (offsetW - sizeX) / stride + 1; - int pwend = min(offsetW / stride + 1, pooledW); + + int frameNum = index / width / height / channels; + int phstart = (offsetH < sizeY) ? 0 : (offsetH - sizeY) / strideH + 1; + int pwstart = (offsetW < sizeX) ? 0 : (offsetW - sizeX) / strideW + 1; + int phend = offsetH >= 0 ? min(offsetH / strideH + 1, pooledH) : 0; + int pwend = offsetW >= 0 ? min(offsetW / strideW + 1, pooledW) : 0; real gradient = 0; - inputData += (frameNum * channels) * height * width; real input = inputData[index]; outData += (frameNum * channels + offsetC) * pooledH * pooledW; outGrad += (frameNum * channels + offsetC) * pooledH * pooledW; - targetGrad += (frameNum * channels) * height * width; for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { if (input == outData[ph * pooledW + pw]) { @@ -223,90 +237,114 @@ __global__ void KeMaxPoolBackward(int nthreads, const real* inputData, } } -void hl_maxpool_backward(int frameCnt, const real* inputData, +void hl_maxpool_backward(const int frameCnt, const real* inputData, const real* outData, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* targetGrad, - real scaleA, real scaleB) { - int num_kernels = (height - start) * (width - start) * channels; - int blocksX = frameCnt; - int blocksY = (num_kernels + 1024 -1) / 1024; - dim3 threads(1024, 1); - dim3 grid(blocksX, blocksY); + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real scaleA, real scaleB, + real* targetGrad) { - KeMaxPoolBackward<<< grid, threads, 0, STREAM_DEFAULT >>> + int num_kernels = height * width * channels * frameCnt; + int blocks = (num_kernels + 1024 - 1) / 1024; + + KeMaxPoolBackward<<< blocks, 1024, 0, STREAM_DEFAULT >>> (num_kernels, inputData, outData, outGrad, channels, - height, width, pooledH, pooledW, sizeX, stride, start, - targetGrad, scaleA, scaleB); + height, width, pooledH, pooledW, sizeX, sizeY, + strideH, strideW, + paddingH, paddingW, + scaleA, scaleB, + targetGrad); CHECK_SYNC("hl_maxpool_backward"); } -__global__ void KeAvePoolForward(int nthreads, const real* inputData, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* tgtData) { - int index = blockIdx.y * blockDim.x + threadIdx.x; +__global__ void KeAvgPoolForward(const int nthreads, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int padH, const int padW, + real* tgtData) { + int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int pw = index % pooledW; int ph = (index / pooledW) % pooledH; int c = (index / pooledW / pooledH) % channels; - int frameNum = blockIdx.x; - int hstart = ph * stride + start; - int hend = min(hstart + sizeX, height); - int wstart = pw * stride + start; - int wend = min(wstart + sizeX, width); + int frameNum = index / pooledW / pooledH / channels; + + int hstart = ph * strideH - padH; + int wstart = pw * strideW - padW; + int hend = min(hstart + sizeY, height + padH); + int wend = min(wstart + sizeX, width + padW); + int pool_size = (hend - hstart) * (wend - wstart); + hstart = max(hstart, 0); + wstart = max(wstart, 0); + hend = min(hend, height); + wend = min(wend, width); + real aveval = 0; inputData += (frameNum * channels + c) * height * width; - tgtData += (frameNum * channels) * pooledH * pooledW; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { aveval += inputData[h * width + w]; } } - tgtData[index] = aveval / ((hend - hstart) * (wend - wstart)); + tgtData[index] = aveval / pool_size; } } -void hl_avgpool_forward(int frameCnt, const real* inputData, int channels, - int height, int width, int pooledH, int pooledW, - int sizeX, int stride, int start, real* tgtData) { - int num_kernels = pooledH * pooledW * channels; - int blocksX = frameCnt; - int blocksY = (num_kernels + 1024 -1) / 1024; - dim3 threads(1024, 1); - dim3 grid(blocksX, blocksY); - KeAvePoolForward<<< grid, threads, 0, STREAM_DEFAULT >>> +void hl_avgpool_forward(const int frameCnt, const real* inputData, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, real* tgtData) { + int num_kernels = pooledH * pooledW * channels * frameCnt; + int blocks = (num_kernels + 1024 - 1) / 1024; + KeAvgPoolForward<<< blocks, 1024, 0, STREAM_DEFAULT >>> (num_kernels, inputData, channels, height, width, pooledH, pooledW, - sizeX, stride, start, tgtData); + sizeX, sizeY, strideH, strideW, + paddingH, paddingW, tgtData); CHECK_SYNC("hl_avgpool_forward failed"); } -__global__ void KeAvgPoolBackward(int nthreads, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* tgtGrad, - real scaleA, real scaleB) { - int index = blockIdx.y * blockDim.x + threadIdx.x; +__global__ void KeAvgPoolBackward(const int nthreads, const real* outGrad, + const int channels, const int height, + const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int padH, const int padW, + real scaleA, real scaleB, + real* tgtGrad) { + int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { - int offsetW = index % width + start; - int offsetH = (index / width) % height + start; + int offsetW = index % width + padW; + int offsetH = (index / width) % height + padH; int offsetC = (index / width / height) % channels; - int frameNum = blockIdx.x; - int phstart = (offsetH < sizeX) ? 0 : (offsetH - sizeX) / stride + 1; - int phend = min(offsetH / stride + 1, pooledH); - int pwstart = (offsetW < sizeX) ? 0 : (offsetW - sizeX) / stride + 1; - int pwend = min(offsetW / stride + 1, pooledW); + int frameNum = index / width / height / channels; + + int phstart = (offsetH < sizeY) ? 0 : (offsetH - sizeY) / strideH + 1; + int pwstart = (offsetW < sizeX) ? 0 : (offsetW - sizeX) / strideW + 1; + int phend = offsetH >= 0 ? min(offsetH / strideH + 1, pooledH) : 0; + int pwend = offsetW >= 0 ? min(offsetW / strideW + 1, pooledW) : 0; real gradient = 0; outGrad += (frameNum * channels + offsetC) * pooledH * pooledW; - tgtGrad += (frameNum * channels) * height * width; for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { // figure out the pooling size - int poolsize = (min(ph * stride + sizeX, height) - ph * stride) * - (min(pw * stride + sizeX, width) - pw * stride); + int hstart = ph * strideH - padH; + int wstart = pw * strideW - padW; + int hend = min(hstart + sizeY, height + padH); + int wend = min(wstart + sizeX, width + padW); + int poolsize = (hend - hstart) * (wend - wstart); gradient += outGrad[ph * pooledW + pw]/poolsize; } } @@ -314,20 +352,25 @@ __global__ void KeAvgPoolBackward(int nthreads, const real* outGrad, } } -void hl_avgpool_backward(int frameCnt, const real* outGrad, - int channels, int height, int width, - int pooledH, int pooledW, int sizeX, - int stride, int start, real* backGrad, - real scaleA, real scaleB) { - int num_kernels = (height - start) * (width - start) * channels; - int blocksX = frameCnt; - int blocksY = (num_kernels + 1024 -1) / 1024; - dim3 threads(1024, 1); - dim3 grid(blocksX, blocksY); +void hl_avgpool_backward(const int frameCnt, const real* outGrad, + const int channels, + const int height, const int width, + const int pooledH, const int pooledW, + const int sizeX, const int sizeY, + const int strideH, const int strideW, + const int paddingH, const int paddingW, + real scaleA, real scaleB, + real* backGrad) { + int num_kernels = height * width * channels * frameCnt; + int blocks = (num_kernels + 1024 - 1) / 1024; - KeAvgPoolBackward <<< grid, threads, 0, STREAM_DEFAULT >>> + KeAvgPoolBackward <<< blocks, 1024, 0, STREAM_DEFAULT >>> (num_kernels, outGrad, channels, height, width, - pooledH, pooledW, sizeX, stride, start, backGrad, scaleA, scaleB); + pooledH, pooledW, sizeX, sizeY, + strideH, strideW, + paddingH, paddingW, + scaleA, scaleB, + backGrad); CHECK_SYNC("hl_avgpool_backward failed"); } diff --git a/paddle/gserver/layers/CudnnPoolLayer.cpp b/paddle/gserver/layers/CudnnPoolLayer.cpp index 86c056ef5692a..4c733591b3779 100644 --- a/paddle/gserver/layers/CudnnPoolLayer.cpp +++ b/paddle/gserver/layers/CudnnPoolLayer.cpp @@ -51,7 +51,6 @@ bool CudnnPoolLayer::init(const LayerMap &layerMap, PoolLayer::init(layerMap, parameterMap); CHECK(useGpu_) << "CudnnPoolLayer only support gpu"; - CHECK_EQ(start_, 0) << poolType_ << " dose not support 'start'"; hl_create_tensor_descriptor(&inputDesc_); hl_create_tensor_descriptor(&outputDesc_); diff --git a/paddle/gserver/layers/CudnnPoolLayer.h b/paddle/gserver/layers/CudnnPoolLayer.h index df97ef2edfd01..2ef94720d2b9f 100644 --- a/paddle/gserver/layers/CudnnPoolLayer.h +++ b/paddle/gserver/layers/CudnnPoolLayer.h @@ -56,16 +56,6 @@ class CudnnPoolLayer : public PoolLayer { void reshape(int batchSize); virtual void forward(PassType passType); virtual void backward(const UpdateCallback& callback = nullptr); - - /** - * Calculate output size according window size of pooling. - */ - int outputSize(int imageSize, int windowSize, int padding, int stride) { - int outputSize; - outputSize = - (imageSize - windowSize + 2 * padding + stride - 1) / stride + 1; - return outputSize; - } }; } // namespace paddle diff --git a/paddle/gserver/layers/PoolLayer.cpp b/paddle/gserver/layers/PoolLayer.cpp index 0ff7f374abb4b..7fc27ac0bd8e0 100644 --- a/paddle/gserver/layers/PoolLayer.cpp +++ b/paddle/gserver/layers/PoolLayer.cpp @@ -35,7 +35,6 @@ bool PoolLayer::init(const LayerMap& layerMap, poolType_ = conf.pool_type(); channels_ = conf.channels(); sizeX_ = conf.size_x(); - start_ = conf.start(); stride_ = conf.stride(); outputX_ = conf.output_x(); imgSize_ = conf.img_size(); @@ -47,22 +46,6 @@ bool PoolLayer::init(const LayerMap& layerMap, confPaddingY_ = conf.has_padding_y() ? conf.padding_y() : conf.padding(); outputY_ = conf.has_output_y() ? conf.output_y() : conf.output_x(); - bool cudnnTypeCheck = true; -#ifndef PADDLE_ONLY_CPU - cudnnTypeCheck = !CudnnPoolLayer::typeCheck(poolType_); -#endif - - if ((sizeY_ != sizeX_ || imgSizeY_ != imgSize_ || strideY_ != stride_ || - confPaddingY_ != confPadding_ || outputY_ != outputX_) && - cudnnTypeCheck) { - LOG(FATAL) << poolType_ << " does not supported non-square " - "filter, image, stride or padding"; - } - - if (confPadding_ != 0 && cudnnTypeCheck) { - LOG(FATAL) << poolType_ << " does not supported 'padding'"; - } - return true; } diff --git a/paddle/gserver/layers/PoolLayer.h b/paddle/gserver/layers/PoolLayer.h index b7a1dfd7632f9..bde1f5b8dcbfd 100644 --- a/paddle/gserver/layers/PoolLayer.h +++ b/paddle/gserver/layers/PoolLayer.h @@ -28,7 +28,7 @@ namespace paddle { class PoolLayer : public Layer { protected: size_t channels_, sizeX_, stride_, outputX_, imgSize_; - int start_, confPadding_; + int confPadding_; size_t sizeY_; size_t imgSizeY_; @@ -47,6 +47,16 @@ class PoolLayer : public Layer { static Layer* create(const LayerConfig& config); virtual bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + /** + * Calculate output size according window size and padding size. + */ + int outputSize(int imageSize, int windowSize, int padding, int stride) { + int outputSize; + outputSize = + (imageSize - windowSize + 2 * padding + stride - 1) / stride + 1; + return outputSize; + } }; } // namespace paddle diff --git a/paddle/gserver/layers/PoolProjectionLayer.cpp b/paddle/gserver/layers/PoolProjectionLayer.cpp index 9c2d6d2164a3f..5a2e9afb6e164 100644 --- a/paddle/gserver/layers/PoolProjectionLayer.cpp +++ b/paddle/gserver/layers/PoolProjectionLayer.cpp @@ -25,13 +25,15 @@ size_t PoolProjectionLayer::getSize() { imgSizeH_ = inputLayers_[0]->getOutput().getFrameHeight(); imgSizeW_ = inputLayers_[0]->getOutput().getFrameWidth(); if (imgSizeH_ == 0) { - imgSizeH_ = imgSize_; + imgSizeH_ = imgSizeY_; } if (imgSizeW_ == 0) { imgSizeW_ = imgSize_; } - outputH_ = 1 + (imgSizeH_ - start_ - sizeX_ + stride_ - 1) / stride_; - outputW_ = 1 + (imgSizeW_ - start_ - sizeX_ + stride_ - 1) / stride_; + + outputH_ = outputSize(imgSizeH_, sizeY_, confPaddingY_, strideY_); + outputW_ = outputSize(imgSizeW_, sizeX_, confPadding_, stride_); + layerSize = outputH_ * outputW_ * channels_; getOutput().setFrameHeight(outputH_); @@ -51,8 +53,9 @@ void MaxPoolProjectionLayer::forward(PassType passType) { MatrixPtr outV = getOutputValue(); - outV->maxPoolForward(*input, imgSizeH_, imgSizeW_, channels_, sizeX_, start_, - stride_, outputH_, outputW_); + outV->maxPoolForward(*input, imgSizeH_, imgSizeW_, channels_, + sizeX_, sizeY_, strideY_, stride_, + outputH_, outputW_, confPaddingY_, confPadding_); } void MaxPoolProjectionLayer::backward(const UpdateCallback& callback) { @@ -69,7 +72,9 @@ void MaxPoolProjectionLayer::backward(const UpdateCallback& callback) { MatrixPtr inputGrad = getInputGrad(0); inputGrad->maxPoolBackward(*inputV, imgSizeH_, imgSizeW_, *outGrad, *outV, - sizeX_, start_, stride_, outputH_, outputW_, 1, 1); + sizeX_, sizeY_, + strideY_, stride_, outputH_, outputW_, 1, 1, + confPaddingY_, confPadding_); } void AvgPoolProjectionLayer::forward(PassType passType) { @@ -84,8 +89,9 @@ void AvgPoolProjectionLayer::forward(PassType passType) { MatrixPtr outV = getOutputValue(); - outV->avgPoolForward(*input, imgSizeH_, imgSizeW_, channels_, sizeX_, start_, - stride_, outputH_, outputW_); + outV->avgPoolForward(*input, imgSizeH_, imgSizeW_, channels_, + sizeX_, sizeY_, strideY_, stride_, + outputH_, outputW_, confPaddingY_, confPadding_); } void AvgPoolProjectionLayer::backward(const UpdateCallback& callback) { @@ -97,7 +103,9 @@ void AvgPoolProjectionLayer::backward(const UpdateCallback& callback) { /* Do derivation */ MatrixPtr outputGrad = getOutputGrad(); MatrixPtr inputGrad = getInputGrad(0); - inputGrad->avgPoolBackward(*outputGrad, imgSizeH_, imgSizeW_, sizeX_, start_, - stride_, outputH_, outputW_, 1, 1); + inputGrad->avgPoolBackward(*outputGrad, imgSizeH_, imgSizeW_, + sizeX_, sizeY_, strideY_, stride_, + outputH_, outputW_, 1, 1, + confPaddingY_, confPadding_); } } // namespace paddle diff --git a/paddle/gserver/tests/img_pool_a.conf b/paddle/gserver/tests/img_pool_a.conf new file mode 100644 index 0000000000000..5938e7611201c --- /dev/null +++ b/paddle/gserver/tests/img_pool_a.conf @@ -0,0 +1,46 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +settings(batch_size=10) +data = data_layer(name ="input", size=8*16*16) +conv = img_conv_layer(input=data, filter_size=1, filter_size_y=1, + num_channels=8, + num_filters=8,stride=1) +maxpool = img_pool_layer(input=conv, + pool_size=3, + pool_size_y=5, + num_channels=8, + stride=1, + stride_y=2, + padding=1, + padding_y=2, + img_width=16, + pool_type=MaxPooling(), +) +avgpool = img_pool_layer(input=conv, + pool_size=3, + pool_size_y=5, + num_channels=8, + stride=1, + stride_y=2, + padding=1, + padding_y=2, + img_width=16, + pool_type=AvgPooling(), +) + +outputs([maxpool, avgpool]) diff --git a/paddle/gserver/tests/img_pool_b.conf b/paddle/gserver/tests/img_pool_b.conf new file mode 100644 index 0000000000000..6ea9649b3f1ea --- /dev/null +++ b/paddle/gserver/tests/img_pool_b.conf @@ -0,0 +1,44 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +settings(batch_size=10) +data = data_layer(name ="input", size=8*16*16) +conv = img_conv_layer(input=data, filter_size=1, filter_size_y=1, + num_channels=8, num_filters=8, stride=1) +maxpool = img_pool_layer(input=conv, + pool_size=3, + pool_size_y=5, + num_channels=8, + stride=1, + stride_y=2, + padding=1, + padding_y=2, + pool_type=CudnnMaxPooling(), +) + +avgpool = img_pool_layer(input=conv, + pool_size=3, + pool_size_y=5, + num_channels=8, + stride=1, + stride_y=2, + padding=1, + padding_y=2, + pool_type=CudnnAvgPooling(), +) + +outputs([maxpool, avgpool]) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 3150c31e4900c..c5723f8574ab3 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -791,21 +791,24 @@ void setPoolConfig(TestConfig* config, PoolConfig* pool, (*config).biasSize = 0; (*config).layerConfig.set_type("pool"); (*config).layerConfig.set_num_filters(16); - (*config).layerConfig.set_partial_sum(1); - (*config).layerConfig.set_shared_biases(true); + int kw = 3, kh = 3; + int pw = 0, ph = 0; + int sw = 2, sh = 2; pool->set_pool_type(poolType); pool->set_channels(16); - pool->set_size_x(3); - if (poolType == "cudnn-max-pool" || poolType == "cudnn-avg-pool") { - pool->set_padding(0); - } else { - pool->set_start(0); - } - pool->set_stride(2); - pool->set_output_x((pool->img_size() - pool->start() - pool->size_x()) / - ((float)pool->stride()) + - 1.5); + pool->set_size_x(kw); + pool->set_size_y(kh); + pool->set_start(0); + pool->set_padding(pw); + pool->set_padding_y(ph); + pool->set_stride(sw); + pool->set_stride_y(sh); + + int ow = (pool->img_size() - kw + 2 * pw + sw - 1) / sw + 1; + int oh = (pool->img_size_y() - kh + 2 * ph + sh - 1) / sh + 1; + pool->set_output_x(ow); + pool->set_output_y(oh); } void testPoolLayer(const string& poolType, bool trans, bool useGpu) { @@ -814,9 +817,10 @@ void testPoolLayer(const string& poolType, bool trans, bool useGpu) { LayerInputConfig* input = config.layerConfig.add_inputs(); PoolConfig* pool = input->mutable_pool_conf(); - setPoolConfig(&config, pool, poolType); pool->set_img_size(14); - config.layerConfig.set_size(pool->output_x() * pool->output_x() * + pool->set_img_size_y(14); + setPoolConfig(&config, pool, poolType); + config.layerConfig.set_size(pool->output_x() * pool->output_y() * pool->channels()); testLayerGrad(config, "pool", 100, trans, useGpu); @@ -829,11 +833,11 @@ void testPoolLayer2(const string& poolType, bool trans, bool useGpu) { LayerInputConfig* input = config.layerConfig.add_inputs(); PoolConfig* pool = input->mutable_pool_conf(); - setPoolConfig(&config, pool, poolType); pool->set_size_y(4); pool->set_stride_y(3); pool->set_img_size(10); pool->set_img_size_y(20); + setPoolConfig(&config, pool, poolType); pool->set_output_y((pool->img_size_y() - pool->start() - pool->size_y()) / ((float)pool->stride_y()) + 1.5); @@ -1252,8 +1256,6 @@ TEST(Layer, MultiplexLayer) { } } - - int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/paddle/gserver/tests/test_NetworkCompare.cpp b/paddle/gserver/tests/test_NetworkCompare.cpp index 1c6a8b0017fc9..b3ef53067301b 100644 --- a/paddle/gserver/tests/test_NetworkCompare.cpp +++ b/paddle/gserver/tests/test_NetworkCompare.cpp @@ -116,6 +116,8 @@ void calcGradient(DataIn& in, DataOut& out, const std::string& configPath) { gradientMachine->start(trainer.getConfig(), nullptr); gradientMachine->forward(in.inArgs, &outArgs, PASS_TRAIN); for (size_t i = 0; i < in.outGrads.size(); i++) { + // If the all the layers in the config have no parameters, also + // not set NeedGradient(), the outArgs[i] will be nullptr. outArgs[i].grad->copyFrom(*in.outGrads[i]); } gradientMachine->backward(); @@ -225,6 +227,18 @@ TEST(Compare, concat_table) { compareNetwork(config_file_a, config_file_b); } +#ifndef PADDLE_ONLY_CPU +TEST(Compare, img_pool) { + std::string config_file_a = "./gserver/tests/img_pool_a.conf"; + std::string config_file_b = "./gserver/tests/img_pool_b.conf"; + bool useGpu = FLAGS_use_gpu; + FLAGS_use_gpu = true; + compareNetwork(config_file_a, config_file_b); + FLAGS_use_gpu = useGpu; +} +#endif + + P_DEFINE_string(config_file_a, "", "config of one network to compare"); P_DEFINE_string(config_file_b, "", "config of another network to compare"); TEST(Compare, network) { diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index e351bede724ac..a6ff2f3b35d04 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -860,9 +860,11 @@ void GpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, } void GpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, - size_t imgSizeW, size_t channels, size_t sizeX, - int start, size_t stride, size_t outputH, - size_t outputW) { + size_t imgSizeW, size_t channels, + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW) { CHECK(inputMat.useGpu_ == true) << "Matrix type are not equal"; real* inputData = inputMat.getData(); @@ -874,14 +876,17 @@ void GpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, CHECK(width_ == outputH * outputW * channels); hl_maxpool_forward(frameNum, inputData, channels, height, width, - outputH, outputW, sizeX, stride, start, data_); + outputH, outputW, sizeX, sizeY, strideH, strideW, + paddingH, paddingW, data_); } void GpuMatrix::maxPoolBackward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, Matrix& outGrad, Matrix& outV, - size_t sizeX, int start, size_t stride, + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput) { + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW) { CHECK(inputMat.useGpu_ == true && outGrad.useGpu_ == true && outV.useGpu_ == true) << "Matrix type are not equal"; @@ -899,15 +904,19 @@ void GpuMatrix::maxPoolBackward(Matrix& inputMat, size_t imgSizeH, CHECK(outGrad.getHeight() == outV.getHeight() && outGrad.getWidth() == outV.getWidth()); + hl_maxpool_backward(frameNum, inputData, outData, outDiff, channels, - height, width, outputH, outputW, sizeX, stride, - start, data_, scaleTargets, scaleOutput); + height, width, outputH, outputW, sizeX, sizeY, + strideH, strideW, paddingH, paddingW, + scaleTargets, scaleOutput, data_); } void GpuMatrix::avgPoolForward(Matrix& inputMat, size_t imgSizeH, - size_t imgSizeW, size_t channels, size_t sizeX, - int start, size_t stride, size_t outputH, - size_t outputW) { + size_t imgSizeW, size_t channels, + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW) { CHECK(inputMat.useGpu_ == true) << "Matrix type are not equal"; real* inputData = inputMat.getData(); @@ -919,13 +928,17 @@ void GpuMatrix::avgPoolForward(Matrix& inputMat, size_t imgSizeH, CHECK(width_ == outputH * outputW * channels); hl_avgpool_forward(frameNum, inputData, channels, height, width, - outputH, outputW, sizeX, stride, start, data_); + outputH, outputW, sizeX, sizeY, + strideH, strideW, + paddingH, paddingW, data_); } void GpuMatrix::avgPoolBackward(Matrix& outGrad, size_t imgSizeH, - size_t imgSizeW, size_t sizeX, int start, - size_t stride, size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput) { + size_t imgSizeW, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW) { CHECK(outGrad.useGpu_ == true) << "Matrix type are not equal"; real* outDiff = outGrad.getData(); @@ -938,8 +951,10 @@ void GpuMatrix::avgPoolBackward(Matrix& outGrad, size_t imgSizeH, CHECK(outGrad.getWidth() == outputH * outputW * channels); hl_avgpool_backward(frameNum, outDiff, channels, height, width, - outputH, outputW, sizeX, stride, start, data_, - scaleTargets, scaleOutput); + outputH, outputW, sizeX, sizeY, + strideH, strideW, paddingH, paddingW, + scaleTargets, scaleOutput, + data_); } void GpuMatrix::crossMapNormalFwd(Matrix& input, size_t imgSizeH, @@ -1450,19 +1465,23 @@ void CpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, } void CpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, - size_t imgSizeW, size_t channels, size_t sizeX, - int start, size_t stride, size_t outputH, - size_t outputW) { + size_t imgSizeW, size_t channels, + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW) { real* inputData = inputMat.getData(); real* outData = data_; size_t num = inputMat.getHeight(); size_t inWidth = imgSizeW; size_t inHeight = imgSizeH; CHECK(inHeight * inWidth == inputMat.getWidth() / channels); + CHECK_EQ(num, this->getHeight()); + CHECK_EQ(channels*outputH*outputW, this->getWidth()); /* initialize the data_ */ for (size_t i = 0; i < height_ * width_; i++) { - data_[i] = -FLT_MAX; + outData[i] = -(real)FLT_MAX; } /* pool max one by one */ @@ -1470,12 +1489,14 @@ void CpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, for (size_t c = 0; c < channels; ++c) { // channel by channel for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { - size_t hstart = ph * stride + start; - size_t wstart = pw * stride + start; - size_t hend = std::min(hstart + sizeX, inHeight); - size_t wend = std::min(wstart + sizeX, inWidth); - for (size_t h = hstart; h < hend; ++h) { - for (size_t w = wstart; w < wend; ++w) { + int hstart = ph * strideH - paddingH; + int wstart = pw * strideW - paddingW; + int hend = std::min(hstart + sizeY, inHeight); + int wend = std::min(wstart + sizeX, inWidth); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { outData[ph * outputW + pw] = std::max(outData[ph * outputW + pw], inputData[h * inWidth + w]); } @@ -1491,9 +1512,10 @@ void CpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, void CpuMatrix::maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, Matrix& outGrad, Matrix& outV, size_t sizeX, - int start, size_t stride, size_t outputH, - size_t outputW, real scaleTargets, - real scaleOutput) { + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW) { size_t num = image.getHeight(); size_t channels = size_t(width_ / imgSizeH / imgSizeW); CHECK(image.getWidth() == imgSizeH * imgSizeW * channels); @@ -1509,32 +1531,36 @@ void CpuMatrix::maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { - size_t hstart = ph * stride + start; - size_t wstart = pw * stride + start; - size_t hend = std::min(hstart + sizeX, imgSizeH); - size_t wend = std::min(wstart + sizeX, imgSizeW); - for (size_t h = hstart; h < hend; ++h) { - for (size_t w = wstart; w < wend; ++w) { + int hstart = ph * strideH - paddingH; + int wstart = pw * strideW - paddingW; + int hend = std::min(hstart + sizeY, imgSizeH); + int wend = std::min(wstart + sizeX, imgSizeW); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { tgtGrad[h * imgSizeW + w] = scaleTargets * tgtGrad[h * imgSizeW + w] + scaleOutput * otGrad[ph * outputW + pw] * - (inData[h * imgSizeW + w] == otData[ph * outputH + pw]); + (inData[h * imgSizeW + w] == otData[ph * outputW + pw]); } } } } // offset inData += imgSizeH * imgSizeW; - otData += outputH * outputW; tgtGrad += imgSizeH * imgSizeW; + otData += outputH * outputW; otGrad += outputH * outputW; } } } void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start, - size_t stride, size_t outputH, size_t outputW) { + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW) { // The main loop size_t num = input.getHeight(); size_t inHeight = imgSizeH; @@ -1548,17 +1574,24 @@ void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { - size_t hstart = ph * stride + start; - size_t wstart = pw * stride + start; - size_t hend = std::min(hstart + sizeX, inHeight); - size_t wend = std::min(wstart + sizeX, inWidth); + int hstart = ph * strideH - paddingH; + int wstart = pw * strideW - paddingW; + int hend = std::min(hstart + sizeY, inHeight + paddingH); + int wend = std::min(wstart + sizeX, inWidth + paddingW); + int poolSize = (hend - hstart) * (wend - wstart); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + hend = std::min(hend, static_cast(inHeight)); + wend = std::min(wend, static_cast(inWidth)); + + CHECK(poolSize); tgtData[ph * outputW + pw] = 0; // clear - for (size_t h = hstart; h < hend; ++h) { - for (size_t w = wstart; w < wend; ++w) { + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { tgtData[ph * outputW + pw] += inData[h * inWidth + w]; } } - tgtData[ph * outputW + pw] /= (hend - hstart) * (wend - wstart); + tgtData[ph * outputW + pw] /= poolSize; } } // compute offset @@ -1569,9 +1602,11 @@ void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, } void CpuMatrix::avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t sizeX, int start, size_t stride, + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput) { + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW) { size_t num = input.getHeight(); size_t channels = input.getWidth() / outputH / outputW; CHECK(imgSizeH * imgSizeW * channels == getWidth()); @@ -1582,14 +1617,20 @@ void CpuMatrix::avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { - size_t hstart = ph * stride + start; - size_t wstart = pw * stride + start; - size_t hend = std::min(hstart + sizeX, imgSizeH); - size_t wend = std::min(wstart + sizeX, imgSizeW); - size_t poolsize = (hend - hstart) * (wend - wstart); - for (size_t h = hstart; h < hend; ++h) { - for (size_t w = wstart; w < wend; ++w) { - outData[h * imgSizeW + w] += inData[ph * outputW + pw] / poolsize; + int hstart = ph * strideH - paddingH; + int wstart = pw * strideW - paddingW; + int hend = std::min(hstart + sizeY, imgSizeH + paddingH); + int wend = std::min(wstart + sizeX, imgSizeW + paddingW); + int poolSize = (hend - hstart) * (wend - wstart); + hstart = std::max(hstart, 0); + wstart = std::max(wstart, 0); + hend = std::min(hend, static_cast(imgSizeH)); + wend = std::min(wend, static_cast(imgSizeW)); + CHECK(poolSize); + + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + outData[h * imgSizeW + w] += inData[ph * outputW + pw] / poolSize; } } } diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index cfb30797fcf1b..5c15c94012816 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -742,31 +742,37 @@ class Matrix : public BaseMatrix { */ virtual void maxPoolForward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, size_t channels, size_t sizeX, - int start_, size_t stride, size_t outputH, - size_t outputW) { + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW) { LOG(FATAL) << "Not implemeted"; } /// Pooling backward operation. virtual void maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, Matrix& outGrad, Matrix& outV, size_t sizeX, - int start, size_t stride, size_t outputH, - size_t outputW, real scaleTargets, - real scaleOutput) { + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW) { LOG(FATAL) << "Not implemeted"; } /// Pooling forward operation, caculate the average of sizeX elements. virtual void avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start, - size_t stride, size_t outputH, size_t outputW) { + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW) { LOG(FATAL) << "Not implemeted"; } virtual void avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t sizeX, int start, size_t stride, + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput) { + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW) { LOG(FATAL) << "Not implemeted"; } @@ -1131,21 +1137,30 @@ class GpuMatrix : public Matrix { real alpha = 1.0f, real beta = 0.0f); void maxPoolForward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start_, size_t stride, - size_t outputH, size_t outputW); + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW); void maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, - Matrix& outGrad, Matrix& outV, size_t sizeX, int start, - size_t stride, size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput); + Matrix& outGrad, Matrix& outV, size_t sizeX, + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW); void avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start, size_t stride, - size_t outputH, size_t outputW); + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW); void avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t sizeX, int start, size_t stride, size_t outputH, - size_t outputW, real scaleTargets, real scaleOutput); + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW); void crossMapNormalFwd(Matrix& input, size_t imgSizeH, size_t imgSizeW, Matrix& denoms, size_t channels, size_t sizeX, @@ -1242,21 +1257,31 @@ class CpuMatrix : public Matrix { real alpha = 1.0f, real beta = 0.0f); void maxPoolForward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start_, size_t stride, - size_t outputH, size_t outputW); + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW); void maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, - Matrix& outGrad, Matrix& outV, size_t sizeX, int start, - size_t stride, size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput); + Matrix& outGrad, Matrix& outV, + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW); void avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t channels, size_t sizeX, int start, size_t stride, - size_t outputH, size_t outputW); + size_t channels, size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + size_t paddingH, size_t paddingW); void avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t sizeX, int start, size_t stride, size_t outputH, - size_t outputW, real scaleTargets, real scaleOutput); + size_t sizeX, size_t sizeY, + size_t strideH, size_t strideW, + size_t outputH, size_t outputW, + real scaleTargets, real scaleOutput, + size_t paddingH, size_t paddingW); void crossMapNormalFwd(Matrix& input, size_t imgSizeH, size_t imgSizeW, Matrix& denoms, size_t channels, size_t sizeX, diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index fe8eacc2efbc5..e1bda79a8acb1 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -1846,6 +1846,159 @@ TEST(Matrix, classificationError) { } } +void testMaxPoolFwdBwd(int numSamples, int channels, + int imgSizeH, int imgSizeW, + int ksizeH, int ksizeW, + int strideH, int strideW, + int padH, int padW) { + int outH = 0, outW = 0; + outH = (imgSizeH - ksizeH + 2 * padH + strideH - 1) / strideH + 1; + outW = (imgSizeW - ksizeW + 2 * padW + strideW - 1) / strideW + 1; + + int inWidth = imgSizeH * imgSizeW * channels; + MatrixPtr input = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpu = GpuMatrix::create(numSamples, inWidth, false, true); + + int outWidth = channels * outH * outW; + MatrixPtr target = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpu = GpuMatrix::create(numSamples, outWidth, false, true); + + input->randomizeUniform(); + target->randomizeUniform(); + inputGpu->copyFrom(*input); + targetGpu->copyFrom(*target); + + target->maxPoolForward(*input, imgSizeH, imgSizeW, + channels, ksizeW, ksizeH, + strideH, strideW, outH, outW, padH, padW); + targetGpu->maxPoolForward(*inputGpu, imgSizeH, imgSizeW, + channels, ksizeW, ksizeH, + strideH, strideW, outH, outW, padH, padW); + MatrixPtr targetCheck = CpuMatrix::create(numSamples, outWidth, false, false); + targetCheck->copyFrom(*targetGpu); + checkMatrixEqual(target, targetCheck); + + MatrixPtr inputGrad = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpuGrad = GpuMatrix::create(numSamples, inWidth, false, true); + MatrixPtr targetGrad = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpuGrad = GpuMatrix::create(numSamples, outWidth, + false, true); + + inputGrad->randomizeUniform(); + targetGrad->randomizeUniform(); + inputGpuGrad->copyFrom(*inputGrad); + targetGpuGrad->copyFrom(*targetGrad); + + inputGrad->maxPoolBackward(*input, imgSizeH, imgSizeW, + *targetGrad, *target, + ksizeW, ksizeH, + strideH, strideW, + outH, outW, 1.0, 1.0, padH, padW); + inputGpuGrad->maxPoolBackward(*inputGpu, imgSizeH, imgSizeW, + *targetGpuGrad, *targetGpu, + ksizeW, ksizeH, + strideH, strideW, + outH, outW, 1.0, 1.0, padH, padW); + MatrixPtr targetBwdCheck = CpuMatrix::create(numSamples, inWidth, + false, false); + targetBwdCheck->copyFrom(*inputGpuGrad); + checkMatrixEqual(inputGrad, targetBwdCheck); +} + +void testAvgPoolFwdBwd(int numSamples, int channels, + int imgSizeH, int imgSizeW, + int ksizeH, int ksizeW, + int strideH, int strideW, + int padH, int padW) { + int outH = 0, outW = 0; + outH = (imgSizeH - ksizeH + 2 * padH + strideH - 1) / strideH + 1; + outW = (imgSizeW - ksizeW + 2 * padW + strideW - 1) / strideW + 1; + + int inWidth = imgSizeH * imgSizeW * channels; + MatrixPtr input = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpu = GpuMatrix::create(numSamples, inWidth, false, true); + + int outWidth = channels * outH * outW; + MatrixPtr target = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpu = GpuMatrix::create(numSamples, outWidth, false, true); + + input->randomizeUniform(); + target->randomizeUniform(); + inputGpu->copyFrom(*input); + targetGpu->copyFrom(*target); + + target->avgPoolForward(*input, imgSizeH, imgSizeW, + channels, ksizeW, ksizeH, + strideH, strideW, outH, outW, padH, padW); + targetGpu->avgPoolForward(*inputGpu, imgSizeH, imgSizeW, + channels, ksizeW, ksizeH, + strideH, strideW, outH, outW, padH, padW); + MatrixPtr targetCheck = CpuMatrix::create(numSamples, outWidth, false, false); + targetCheck->copyFrom(*targetGpu); + MatrixCheckErr(*target, *targetCheck); + + MatrixPtr inputGrad = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpuGrad = GpuMatrix::create(numSamples, inWidth, false, true); + MatrixPtr targetGrad = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpuGrad = GpuMatrix::create(numSamples, outWidth, + false, true); + + inputGrad->randomizeUniform(); + targetGrad->randomizeUniform(); + inputGpuGrad->copyFrom(*inputGrad); + targetGpuGrad->copyFrom(*targetGrad); + + inputGrad->avgPoolBackward(*targetGrad, imgSizeH, imgSizeW, + ksizeW, ksizeH, + strideH, strideW, + outH, outW, 1.0, 1.0, padH, padW); + inputGpuGrad->avgPoolBackward(*targetGpuGrad, imgSizeH, imgSizeW, + ksizeW, ksizeH, + strideH, strideW, + outH, outW, 1.0, 1.0, padH, padW); + MatrixPtr targetBwdCheck = CpuMatrix::create(numSamples, inWidth, + false, false); + targetBwdCheck->copyFrom(*inputGpuGrad); + MatrixCheckErr(*inputGrad, *targetBwdCheck); +} + +TEST(Matrix, PoolFwdBwd) { + for (auto numSamples : {5, 32}) { + for (auto channels : {1, 9, 32}) { + for (auto imgSizeH : {14, 28}) { + for (auto imgSizeW : {16, 30}) { + for (auto sizeX : {2, 5}) { + for (auto sizeY : {2, 5}) { + for (auto sH : {1, 2}) { + for (auto sW : {1, 2}) { + for (auto pH : {0, (sizeY - 1)/2}) { + for (auto pW : {0, (sizeX - 1)/2}) { + VLOG(3) << " numSamples=" << numSamples + << " channels=" << channels + << " imgSizeH=" << imgSizeH + << " imgSizeW=" << imgSizeW + << " sizeX=" << sizeX + << " sizeY=" << sizeY + << " strideH=" << sH + << " strideW=" << sW + << " padingH=" << pH + << " padingW=" << pW; + testMaxPoolFwdBwd(numSamples, channels, imgSizeH, + imgSizeW, sizeX, sizeY, sH, sW, pH, pW); + testAvgPoolFwdBwd(numSamples, channels, imgSizeH, + imgSizeW, sizeX, sizeY, sH, sW, pH, pW); + } + } + } + } + } + } + } + } + } + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index b32f8b1ee9072..25e36f9c4c168 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -88,7 +88,8 @@ message PoolConfig { required uint32 size_x = 3; // Tell the net where in the input image to start the pooling. - required uint32 start = 4; + // start is deprecated now. + optional uint32 start = 4; // Defines the stride size between successive pooling squares. required uint32 stride = 5; diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 1f55298f24f07..fb47fd0c6f0c3 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -961,10 +961,6 @@ def parse_pool(pool, input_layer_name, pool_conf): "['max-projection', 'avg-projection', " "'cudnn-max-pool', 'cudnn-avg-pool']" % pool.pool_type) - if pool.size_y or pool.stride_y or pool.img_width or pool.padding_y: - config_assert(pool.pool_type.startswith('cudnn'), - "'size_y', 'stride_y' and 'img_width' and 'padding_y'" - "can only be used for cudnn") pool_conf.channels = pool.channels pool_conf.size_x = pool.size_x @@ -974,36 +970,25 @@ def parse_pool(pool, input_layer_name, pool_conf): pool_conf.stride_y = default(pool.stride_y, pool_conf.stride); img_pixels = g_layer_map[input_layer_name].size / pool.channels + # the img_width may be removed, + # and it can be calculated automatically later. pool_conf.img_size = default(pool.img_width, int(img_pixels ** 0.5)) pool_conf.img_size_y = img_pixels / pool_conf.img_size config_assert(pool_conf.img_size * pool_conf.img_size_y == img_pixels, "Incorrect input image size %d for input image pixels %d" % (pool_conf.img_size, img_pixels)) - if pool.start is not None: - config_assert(pool.padding is None, - 'At most one of start and padding can be set.') - pool_conf.start = pool.start - pool_conf.padding = 0 - pool_conf.output_x = int(math.ceil((pool_conf.img_size - \ - pool_conf.start - pool_conf.size_x) / \ - float(pool_conf.stride))) + 1 + config_assert(not pool.start, "start is deprecated in pooling.") - pool_conf.output_y = int(math.ceil((pool_conf.img_size_y - \ - pool_conf.start - pool_conf.size_y) / \ - float(pool_conf.stride_y))) + 1 - elif pool.padding is not None: + if pool.padding is not None: pool_conf.padding = pool.padding pool_conf.padding_y = default(pool.padding_y, pool_conf.padding) - pool_conf.start = 0 pool_conf.output_x = int(math.ceil((pool_conf.img_size + \ 2*pool_conf.padding - pool_conf.size_x) / \ float(pool_conf.stride))) + 1 pool_conf.output_y = int(math.ceil((pool_conf.img_size_y + \ 2*pool_conf.padding_y - pool_conf.size_y) / \ float(pool_conf.stride_y))) + 1 - else: - raise ValueError('At least one of start and padding should be set.') def parse_image(image, input_layer_name, image_conf): image_conf.channels = image.channels @@ -1603,7 +1588,7 @@ def __init__( pool_conf = self.config.inputs[input_index].pool_conf print("output size for %s is %d*%d " % ( name, pool_conf.output_y, pool_conf.output_x)) - self.set_layer_size((pool_conf.output_x ** 2) * pool_conf.channels) + self.set_layer_size((pool_conf.output_x * pool_conf.output_y) * pool_conf.channels) @config_layer('batch_norm') class BatchNormLayer(LayerBase): diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 47db197f422ea..5e7e66a908ee0 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -210,7 +210,7 @@ def __str__(self): def layer_support(*attrs): - attrs_list = list(attrs) + attrs_list = list(attrs) attrs_list.append(DEVICE) def decorator(method): @functools.wraps(method) @@ -1627,7 +1627,9 @@ def img_conv_layer(input, filter_size, num_filters, @layer_support() def img_pool_layer(input, pool_size, name=None, num_channels=None, pool_type=None, - stride=1, start=None, padding=0, layer_attr=None): + stride=1, start=None, padding=0, layer_attr=None, + pool_size_y=None, stride_y=None, padding_y=None, + img_width=None): """ Image pooling Layer. @@ -1635,25 +1637,34 @@ def img_pool_layer(input, pool_size, name=None, .. _pooling: http://ufldl.stanford.edu/tutorial/supervised/Pooling/ - :param padding: pooling padding + :param padding: pooling padding width. :type padding: int + :param padding_y: pooling padding height. It's equal to padding by default. + :type padding_y: int|None :param name: name of pooling layer :type name: basestring. :param input: layer's input :type input: LayerOutput - :param pool_size: pooling size + :param pool_size: pooling window width :type pool_size: int + :param pool_size_y: pooling window height. It's eaqual to pool_size by default. + :type pool_size_y: int|None :param num_channels: number of input channel. :type num_channels: int :param pool_type: pooling type. MaxPooling or AveragePooling. Default is MaxPooling. :type pool_type: BasePoolingType - :param stride: stride of pooling. + :param stride: stride width of pooling. :type stride: int - :param start: start position of pooling operation. - :type start: int + :param stride_y: stride height of pooling. It is equal to stride by default. + :type stride_y: int|None + :param start: start position of pooling operation. Note it is deprecated now. + :type start: int|None :param layer_attr: Extra Layer attribute. :type layer_attr: ExtraLayerAttribute + :param img_width: the width of input feature map. If it is None, the input feature + map should be square. + :type img_width: int|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -1666,17 +1677,29 @@ def img_pool_layer(input, pool_size, name=None, elif isinstance(pool_type, AvgPooling): pool_type.name = 'avg' + type_name = pool_type.name + '-projection' \ + if (isinstance(pool_type, AvgPooling) or isinstance(pool_type, MaxPooling)) \ + else pool_type.name + + pool_size_y = pool_size if pool_size_y is None else pool_size_y + stride_y = stride if stride_y is None else stride_y + padding_y = padding if padding_y is None else padding_y + Layer( name=name, type=LayerType.POOL_LAYER, inputs=[Input(input.name, pool=Pool( - pool_type=''.join([pool_type.name, '-projection']), + pool_type=type_name, channels=num_channels, size_x=pool_size, start=start, stride=stride, - padding=padding + padding=padding, + size_y=pool_size_y, + stride_y=stride_y, + padding_y=padding_y, + img_width=img_width ))], **ExtraLayerAttribute.to_kwargs(layer_attr) ) @@ -2751,7 +2774,7 @@ def __real_step__(*args): tmp = recurrent_group(step=__real_step__, input=real_input, reverse=False, name=name) - + return tmp diff --git a/python/paddle/trainer_config_helpers/networks.py b/python/paddle/trainer_config_helpers/networks.py index e59e93acbe33a..ab4057d9d6c6b 100644 --- a/python/paddle/trainer_config_helpers/networks.py +++ b/python/paddle/trainer_config_helpers/networks.py @@ -170,13 +170,13 @@ def simple_img_conv_pool(input, filter_size, num_filters, pool_size, name=None, :type shared_bias: bool :param conv_layer_attr: see img_conv_layer for details :type conv_layer_attr: ExtraLayerAttribute - :param pool_stride: see img_conv_layer for details + :param pool_stride: see img_pool_layer for details :type pool_stride: int - :param pool_start: see img_conv_layer for details + :param pool_start: see img_pool_layer for details. It is deprecated now. :type pool_start: int - :param pool_padding: see img_conv_layer for details + :param pool_padding: see img_pool_layer for details :type pool_padding: int - :param pool_layer_attr: see img_conv_layer for details + :param pool_layer_attr: see img_pool_layer for details :type pool_layer_attr: ExtraLayerAttribute :return: Layer's output :rtype: LayerOutput @@ -243,7 +243,7 @@ def img_conv_bn_pool(input, filter_size, num_filters, pool_size, name=None, :param bn_layer_attr: ParameterAttribute. :param pool_stride: see img_pool_layer's document. :type pool_stride: int - :param pool_start: see img_pool_layer's document. + :param pool_start: see img_pool_layer's document. It is deprecated now. :type pool_start: int :param pool_padding: see img_pool_layer's document. :type pool_padding: int @@ -555,7 +555,7 @@ def lstmemory_unit(input, name=None, size=None, param_attr=None, :type gate_act: BaseActivation :param state_act: lstm state activiation type. :type state_act: BaseActivation - :param mixed_bias_attr: bias parameter attribute of mixed layer. + :param mixed_bias_attr: bias parameter attribute of mixed layer. False means no bias, None means default bias. :type mixed_bias_attr: ParameterAttribute|False :param lstm_bias_attr: bias parameter attribute of lstm layer. diff --git a/python/paddle/trainer_config_helpers/poolings.py b/python/paddle/trainer_config_helpers/poolings.py index d627daab0c496..3d2320f3ffc42 100644 --- a/python/paddle/trainer_config_helpers/poolings.py +++ b/python/paddle/trainer_config_helpers/poolings.py @@ -19,6 +19,8 @@ "BasePoolingType", "MaxPooling", "AvgPooling", + "CudnnMaxPooling", + "CudnnAvgPooling", "SumPooling", "SquareRootNPooling" ] @@ -26,7 +28,7 @@ class BasePoolingType(object): """ - Base Pooling Type. + Base Pooling Type. Note these pooling types are used for sequence input, not for images. Each PoolingType contains one parameter: @@ -55,7 +57,24 @@ class MaxPooling(BasePoolingType): def __init__(self, output_max_index=None): BasePoolingType.__init__(self, "max") self.output_max_index = output_max_index - + + +class CudnnMaxPooling(BasePoolingType): + """ + Cudnn max pooling only support GPU. Return the maxinum value in the + pooling window. + """ + def __init__(self): + BasePoolingType.__init__(self, "cudnn-max-pool") + + +class CudnnAvgPooling(BasePoolingType): + """ + Cudnn average pooling only support GPU. Return the average value in the + pooling window. + """ + def __init__(self): + BasePoolingType.__init__(self, "cudnn-avg-pool") class AvgPooling(BasePoolingType): """ diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 index 29928b6f7b423..359652f3d09c7 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/check.md5 +++ b/python/paddle/trainer_config_helpers/tests/configs/check.md5 @@ -1,4 +1,4 @@ -7e6919d17562516e9a1d9a88de1fb3b9 img_layers.protostr +86c0815275a9d5eb902e23c6a592f58a img_layers.protostr a5d9259ff1fd7ca23d0ef090052cb1f2 last_first_seq.protostr 9c038249ec8ff719753a746cdb04c026 layer_activations.protostr 5913f87b39cee3b2701fa158270aca26 projections.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/img_layers.py b/python/paddle/trainer_config_helpers/tests/configs/img_layers.py index 6c8ba8be846e5..f33357c3906fd 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/img_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/img_layers.py @@ -7,8 +7,10 @@ img = data_layer(name='image', size=256*256) +# the parse_conv in config_parse.py is not strictly accurate when filter_size +# is not square. So here set square filter_size. img_conv = img_conv_layer(input=img, num_channels=1, num_filters=64, - filter_size=(32, 64), padding=(1, 0), stride=(1, 1), + filter_size=(32, 32), padding=(1, 1), stride=(1, 1), act=LinearActivation()) img_bn = batch_norm_layer(input=img_conv, act=ReluActivation()) @@ -17,4 +19,4 @@ img_pool = img_pool_layer(input=img_conv, pool_size=32, pool_type=MaxPooling()) -outputs(img_pool, img_norm) \ No newline at end of file +outputs(img_pool, img_norm) From 9e11ca8096a57cda6d91741c064b362180ff2a50 Mon Sep 17 00:00:00 2001 From: gangliao Date: Mon, 10 Oct 2016 10:15:07 +0800 Subject: [PATCH 153/324] Use C++ 11 atomic_flag in MacOS as spin lock (#175) * Use C++ 11 atomic_flag in MacOS as spin lock * Add unittest for it. --- paddle/trainer/tests/test_CompareSparse.cpp | 2 +- paddle/utils/arch/osx/Locks.cpp | 24 ++------- paddle/utils/tests/CMakeLists.txt | 1 + paddle/utils/tests/test_SpinLock.cpp | 57 +++++++++++++++++++++ 4 files changed, 63 insertions(+), 21 deletions(-) create mode 100644 paddle/utils/tests/test_SpinLock.cpp diff --git a/paddle/trainer/tests/test_CompareSparse.cpp b/paddle/trainer/tests/test_CompareSparse.cpp index ff37d7b364840..311dd333a1b16 100644 --- a/paddle/trainer/tests/test_CompareSparse.cpp +++ b/paddle/trainer/tests/test_CompareSparse.cpp @@ -57,7 +57,7 @@ std::vector trainerOnePassTest(const string& configFile, << " sparseUpdate=" << sparseUpdate; srand(FLAGS_seed); *ThreadLocalRand::getSeed() = FLAGS_seed; - + ThreadLocalRandomEngine::get().seed(FLAGS_seed); if (useGpu) { CHECK_LE(trainerCount, gNumDevices); } diff --git a/paddle/utils/arch/osx/Locks.cpp b/paddle/utils/arch/osx/Locks.cpp index 44bab7198d5f4..b3ec454976520 100644 --- a/paddle/utils/arch/osx/Locks.cpp +++ b/paddle/utils/arch/osx/Locks.cpp @@ -15,12 +15,9 @@ limitations under the License. */ #include "paddle/utils/Locks.h" #include "paddle/utils/Logging.h" #include +#include #include -#if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 -#include -#endif - namespace paddle { class SemaphorePrivate { @@ -55,12 +52,7 @@ void Semaphore::post() { class SpinLockPrivate { public: -#if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 - os_unfair_lock lock_; -#else - SpinLockPrivate(): lock_(OS_SPINLOCK_INIT) {} - OSSpinLock lock_; -#endif + std::atomic_flag lock_ = ATOMIC_FLAG_INIT; char padding_[64 - sizeof(lock_)]; // Padding to cache line size }; @@ -68,19 +60,11 @@ SpinLock::SpinLock(): m(new SpinLockPrivate()) {} SpinLock::~SpinLock() { delete m; } void SpinLock::lock() { -#if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 - os_unfair_lock_lock(&m->lock_); -#else - OSSpinLockLock(&m->lock_); -#endif + while (m->lock_.test_and_set(std::memory_order_acquire)) {} } void SpinLock::unlock() { -#if MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12 - os_unfair_lock_unlock(&m->lock_); -#else - OSSpinLockUnlock(&m->lock_); -#endif + m->lock_.clear(std::memory_order_release); } diff --git a/paddle/utils/tests/CMakeLists.txt b/paddle/utils/tests/CMakeLists.txt index 51f1889392845..adf489fafe722 100644 --- a/paddle/utils/tests/CMakeLists.txt +++ b/paddle/utils/tests/CMakeLists.txt @@ -4,6 +4,7 @@ add_simple_unittest(test_Thread) add_simple_unittest(test_StringUtils) add_simple_unittest(test_CustomStackTrace) add_simple_unittest(test_ThreadBarrier) +add_simple_unittest(test_SpinLock) add_executable( test_CustomStackTracePrint diff --git a/paddle/utils/tests/test_SpinLock.cpp b/paddle/utils/tests/test_SpinLock.cpp new file mode 100644 index 0000000000000..ebc84e0f52d82 --- /dev/null +++ b/paddle/utils/tests/test_SpinLock.cpp @@ -0,0 +1,57 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include "paddle/utils/Logging.h" +#include "paddle/utils/CommandLineParser.h" +#include "paddle/utils/Util.h" +#include "paddle/utils/Locks.h" + +P_DEFINE_int32(test_thread_num, 100, "testing thread number"); + +void testNormalImpl(size_t thread_num, const std::function + & callback) { + paddle::SpinLock mutex; + std::vector threads; + threads.reserve(thread_num); + + size_t count = 0; + for (size_t i = 0; i < thread_num; ++i) { + threads.emplace_back([&thread_num, &count, &mutex, &callback]{ + callback(thread_num, count, mutex); + }); + } + for (auto& thread : threads) { + thread.join(); + } + // Check whether all threads reach this point or not + CHECK_EQ(count, thread_num); +} + +TEST(ThreadSpinLock, normalTest) { + for (auto &thread_num : {10, 30, 50 , 100 , 300, 1000}) { + testNormalImpl(thread_num, [](size_t thread_num, + size_t& count, paddle::SpinLock& mutex){ + std::lock_guard lock(mutex); + ++count; + }); + } +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + paddle::initMain(argc, argv); + return RUN_ALL_TESTS(); +} From 86bb5ef1693c601fd72fc0d5f16a239cb9f8f68a Mon Sep 17 00:00:00 2001 From: QiJune Date: Mon, 10 Oct 2016 10:18:07 +0800 Subject: [PATCH 154/324] Read git sha1 when building Paddle, and add it to PADDLE_VERSION macro * save the model file including git sha1 --- CMakeLists.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index af6a13efbde9e..44e93f22c0eaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,6 +65,19 @@ else() Subversion_WC_INFO(${PROJ_ROOT} Project) add_definitions(-DPADDLE_VERSION=${Project_WC_REVISION}) endif() + elseif(EXISTS ${PROJ_ROOT}/.git/) + find_package(Git REQUIRED) + execute_process( + COMMAND ${GIT_EXECUTABLE} log -1 --format=%H + WORKING_DIRECTORY ${PROJ_ROOT} + OUTPUT_VARIABLE GIT_SHA1 + RESULT_VARIABLE GIT_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NOT ${GIT_RESULT}) + add_definitions(-DPADDLE_VERSION=\"${GIT_SHA1}\") + else() + message(WARNING "Cannot add paddle version from git tag") + endif() endif() endif() From 199a6a4b5c62583d05d4c3199a13891dcba576c5 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Mon, 10 Oct 2016 11:21:44 +0800 Subject: [PATCH 155/324] add weight for cost layer interface (#177) --- .../paddle/trainer_config_helpers/layers.py | 44 +++++++++++++++---- .../tests/configs/check.md5 | 1 + .../tests/configs/generate_protostr.sh | 2 +- .../configs/test_cost_layers_with_weight.py | 14 ++++++ 4 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 5e7e66a908ee0..7699c90db716c 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -2777,29 +2777,49 @@ def __real_step__(*args): return tmp +def __cost_input__(input, label, weight=None): + """ + inputs and parents for cost layers. + """ + ipts = [Input(input.name), Input(label.name)] + parents = [input, label] + if weight is not None: + assert weight.layer_type == LayerType.DATA + ipts.append(Input(weight.name)) + parents.append(weight) + return ipts, parents + @wrap_name_default() -def regression_cost(input, label, cost='square_error', name=None): +def regression_cost(input, label, weight=None, cost='square_error', name=None): """ Regression Layer. TODO(yuyang18): Complete this method. :param name: layer name. + :type name: basestring :param input: Network prediction. + :type input: LayerOutput :param label: Data label. + :type label: LayerOutput + :param weight: The weight affects the cost, namely the scale of cost. + It is an optional argument. + :type weight: LayerOutput :param cost: Cost method. + :type cost: basestring :return: LayerOutput object. + :rtype: LayerOutput """ - Layer(inputs=[Input(input.name), Input(label.name)], type=cost, name=name) - return LayerOutput( - name, LayerType.COST, parents=[input, label] - ) + ipts, parents = __cost_input__(input, label, weight) + + Layer(inputs=ipts, type=cost, name=name) + return LayerOutput(name, LayerType.COST, parents=parents) @wrap_name_default("cost") @layer_support() -def classification_cost(input, label, name=None, +def classification_cost(input, label, weight=None, name=None, cost="multi-class-cross-entropy", evaluator=classification_error_evaluator, layer_attr=None): @@ -2812,6 +2832,9 @@ def classification_cost(input, label, name=None, :type input: LayerOutput :param label: label layer name. data_layer often. :type label: LayerOutput + :param weight: The weight affects the cost, namely the scale of cost. + It is an optional argument. + :type weight: LayerOutput :param cost: cost method. :type cost: basestring :param evaluator: Evaluator method. @@ -2823,7 +2846,10 @@ def classification_cost(input, label, name=None, assert input.layer_type != LayerType.DATA assert isinstance(input.activation, SoftmaxActivation) assert label.layer_type == LayerType.DATA - Layer(name=name, type=cost, inputs=[Input(input.name), Input(label.name)], + + ipts, parents = __cost_input__(input, label, weight) + + Layer(name=name, type=cost, inputs=ipts, **ExtraLayerAttribute.to_kwargs(layer_attr)) def __add_evaluator__(e): @@ -2835,7 +2861,7 @@ def __add_evaluator__(e): assert isinstance(e.for_classification, bool) assert e.for_classification - e(name=e.__name__, input=input, label=label) + e(name=e.__name__, input=input, label=label, weight=weight) if not isinstance(evaluator, collections.Sequence): evaluator = [evaluator] @@ -2843,7 +2869,7 @@ def __add_evaluator__(e): for each_evaluator in evaluator: __add_evaluator__(each_evaluator) - return LayerOutput(name, LayerType.COST, parents=[input, label]) + return LayerOutput(name, LayerType.COST, parents=parents) def conv_operator(img, filter, filter_size, num_filters, diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 index 359652f3d09c7..3ecfff2071630 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/check.md5 +++ b/python/paddle/trainer_config_helpers/tests/configs/check.md5 @@ -4,6 +4,7 @@ a5d9259ff1fd7ca23d0ef090052cb1f2 last_first_seq.protostr 5913f87b39cee3b2701fa158270aca26 projections.protostr 6b39e34beea8dfb782bee9bd3dea9eb5 simple_rnn_layers.protostr 0fc1409600f1a3301da994ab9d28b0bf test_cost_layers.protostr +6cd5f28a3416344f20120698470e0a4c test_cost_layers_with_weight.protostr 144bc6d3a509de74115fa623741797ed test_expand_layer.protostr 2378518bdb71e8c6e888b1842923df58 test_fc.protostr 8bb44e1e5072d0c261572307e7672bda test_grumemory_layer.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index fc2acbd41ed90..5514ee65e5a62 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -8,7 +8,7 @@ configs=(test_fc layer_activations projections test_print_layer test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers util_layers simple_rnn_layers unused_layers test_cost_layers -test_rnn_group) +test_cost_layers_with_weight test_rnn_group) for conf in ${configs[*]} diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py new file mode 100644 index 0000000000000..29749cbb66637 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.py @@ -0,0 +1,14 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +data = data_layer(name='input', size=300) +lbl = data_layer(name='label', size=1) +wt = data_layer(name='weight', size=1) +fc = fc_layer(input=data, size=10, act=SoftmaxActivation()) + +outputs(classification_cost(input=fc, label=lbl, weight=wt), + regression_cost(input=fc, label=lbl, weight=wt)) From 9f244e4a392c3d528572413d4969c7265150dc50 Mon Sep 17 00:00:00 2001 From: Mark Date: Mon, 10 Oct 2016 13:14:26 +0800 Subject: [PATCH 156/324] Should not compile the two files if -DWITH_AVX=OFF. (#163) * If cmake -DWITH_AVX=OFF during configuration, should not compile the file src/hl_math.cc and src/hl_avx_functions.cc. --- paddle/cuda/CMakeLists.txt | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) mode change 100644 => 100755 paddle/cuda/CMakeLists.txt diff --git a/paddle/cuda/CMakeLists.txt b/paddle/cuda/CMakeLists.txt old mode 100644 new mode 100755 index e03a9a1baa004..cdb730bb3cec7 --- a/paddle/cuda/CMakeLists.txt +++ b/paddle/cuda/CMakeLists.txt @@ -2,10 +2,17 @@ set(AVX_SOURCES src/hl_math.cc src/hl_avx_functions.cc ) -set(CUDA_SOURCES - src/hl_time.cc - src/hl_cpu_functions.cc - ${AVX_SOURCES}) + +if(WITH_AVX) + set(CUDA_SOURCES + src/hl_time.cc + src/hl_cpu_functions.cc + ${AVX_SOURCES}) +else() + set(CUDA_SOURCES + src/hl_time.cc + src/hl_cpu_functions.cc) +endif() set(CUDA_CXX_WITH_GPU_SOURCES src/hl_cuda_cublas.cc From 3edc4202b0005ea9cbdf6fac03347bd5d560887c Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 11 Oct 2016 15:20:40 +0800 Subject: [PATCH 157/324] Add travis for osx (#189) --- .travis.yml | 11 +++++++++- paddle/.set_python_path.sh | 2 +- ...ore_install.sh => before_install.linux.sh} | 0 paddle/scripts/travis/before_install.osx.sh | 13 ++++++++++++ paddle/scripts/travis/build_and_test.sh | 21 ++++++++++++++++--- 5 files changed, 42 insertions(+), 5 deletions(-) rename paddle/scripts/travis/{before_install.sh => before_install.linux.sh} (100%) create mode 100755 paddle/scripts/travis/before_install.osx.sh diff --git a/.travis.yml b/.travis.yml index d3dae9efd416b..119d01a4fa8fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,9 +2,17 @@ language: cpp cache: ccache sudo: required dist: trusty +os: + - linux + - osx env: - JOB=DOCS - JOB=BUILD_AND_TEST +matrix: + exclude: + - os: osx + env: JOB=DOCS # Only generate documentation in linux + addons: apt: packages: @@ -28,8 +36,9 @@ addons: - libgflags-dev - libgtest-dev before_install: + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi - pip install wheel protobuf sphinx breathe recommonmark - - sudo paddle/scripts/travis/before_install.sh script: - paddle/scripts/travis/main.sh notifications: diff --git a/paddle/.set_python_path.sh b/paddle/.set_python_path.sh index f7019b27f8f02..657fdf65e92c9 100755 --- a/paddle/.set_python_path.sh +++ b/paddle/.set_python_path.sh @@ -33,7 +33,7 @@ if ! python -c "import paddle" >/dev/null 2>/dev/null; then esac done shift $(($OPTIND - 1)) - export PYTHONPATH=$PYPATH + export PYTHONPATH=$PYPATH:$PYTHONPATH $@ else echo "paddle package is already in your PYTHONPATH. But unittest need a clean environment." diff --git a/paddle/scripts/travis/before_install.sh b/paddle/scripts/travis/before_install.linux.sh similarity index 100% rename from paddle/scripts/travis/before_install.sh rename to paddle/scripts/travis/before_install.linux.sh diff --git a/paddle/scripts/travis/before_install.osx.sh b/paddle/scripts/travis/before_install.osx.sh new file mode 100755 index 0000000000000..f438e69b822aa --- /dev/null +++ b/paddle/scripts/travis/before_install.osx.sh @@ -0,0 +1,13 @@ +#!/bin/bash +brew update +brew tap homebrew/science +brew install python +sudo pip install --upgrade protobuf==2.6.0 +brew install homebrew/versions/protobuf260 --without-python +brew install cmake python glog gflags openblas wget md5sha1sum + +wget https://github.com/google/googletest/archive/release-1.8.0.tar.gz -O gtest.tar.gz +tar xf gtest.tar.gz +cd googletest-release-1.8.0/ +cmake . +make install diff --git a/paddle/scripts/travis/build_and_test.sh b/paddle/scripts/travis/build_and_test.sh index 3ea633be32702..a73c32344c8ab 100755 --- a/paddle/scripts/travis/build_and_test.sh +++ b/paddle/scripts/travis/build_and_test.sh @@ -1,7 +1,22 @@ #!/bin/bash source ./common.sh -cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -make -j `nproc` -env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j `nproc`" +CMAKE_EXTRA="" +if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + CMAKE_EXTRA="-DPYTHON_LIBRARY=/usr/local/Cellar/python/2.7.12_1/Frameworks/Python.framework/Versions/2.7/lib/python2.7/config/libpython2.7.dylib" +fi + + +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON ${CMAKE_EXTRA} + +NPROC=1 +if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then + NRPOC=`nproc` +elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + NPROC=`sysctl -n hw.ncpu` +fi + + +make -j $NPROC +env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j $NPROC" sudo make install sudo paddle version From 6f0d634e02646ec05eef5de4f8643d97da0e2845 Mon Sep 17 00:00:00 2001 From: backyes Date: Tue, 11 Oct 2016 16:49:29 +0800 Subject: [PATCH 158/324] set MKL search path with intel64 (#188) --- cmake/cblas.cmake | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cmake/cblas.cmake b/cmake/cblas.cmake index 529b4b9d15d09..57c32a54cd727 100644 --- a/cmake/cblas.cmake +++ b/cmake/cblas.cmake @@ -17,10 +17,17 @@ ## Find MKL First. set(MKL_ROOT $ENV{MKL_ROOT} CACHE PATH "Folder contains MKL") -find_path(MKL_INCLUDE_DIR mkl.h PATHS ${MKL_ROOT}/include) -find_library(MKL_CORE_LIB NAMES mkl_core PATHS ${MKL_ROOT}/lib) -find_library(MKL_SEQUENTIAL_LIB NAMES mkl_sequential PATHS ${MKL_ROOT}/lib) -find_library(MKL_INTEL_LP64 NAMES mkl_intel_lp64 PATHS ${MKL_ROOT}/lib) +find_path(MKL_INCLUDE_DIR mkl.h PATHS + ${MKL_ROOT}/include) +find_library(MKL_CORE_LIB NAMES mkl_core PATHS + ${MKL_ROOT}/lib + ${MKL_ROOT}/lib/intel64) +find_library(MKL_SEQUENTIAL_LIB NAMES mkl_sequential PATHS + ${MKL_ROOT}/lib + ${MKL_ROOT}/lib/intel64) +find_library(MKL_INTEL_LP64 NAMES mkl_intel_lp64 PATHS + ${MKL_ROOT}/lib + ${MKL_ROOT}/lib/intel64) if(MKL_INCLUDE_DIR AND MKL_CORE_LIB AND MKL_SEQUENTIAL_LIB AND MKL_INTEL_LP64) From e26f220df81546d360e7759b3d96a2aa27d06ffc Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Tue, 11 Oct 2016 21:29:52 -0700 Subject: [PATCH 159/324] Mnist demo (#162) * added mnist demo * modified .gitignore for .project files * normalize pixel in mnist_provider.py and set use_gpu=0 --- .gitignore | 4 ++- demo/mnist/.gitignore | 6 ++++ demo/mnist/data/generate_list.py | 21 +++++++++++++ demo/mnist/data/get_mnist_data.sh | 22 +++++++++++++ demo/mnist/mnist_provider.py | 33 ++++++++++++++++++++ demo/mnist/train.sh | 31 ++++++++++++++++++ demo/mnist/vgg_16_mnist.py | 52 +++++++++++++++++++++++++++++++ 7 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 demo/mnist/.gitignore create mode 100644 demo/mnist/data/generate_list.py create mode 100644 demo/mnist/data/get_mnist_data.sh create mode 100644 demo/mnist/mnist_provider.py create mode 100755 demo/mnist/train.sh create mode 100644 demo/mnist/vgg_16_mnist.py diff --git a/.gitignore b/.gitignore index 7e21ba0b750df..65ba217de37c8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ build/ *.user .vscode -.idea \ No newline at end of file +.idea +.project +.pydevproject diff --git a/demo/mnist/.gitignore b/demo/mnist/.gitignore new file mode 100644 index 0000000000000..810910fd5ca56 --- /dev/null +++ b/demo/mnist/.gitignore @@ -0,0 +1,6 @@ +data/raw_data +data/*.list +mnist_vgg_model +plot.png +train.log +*pyc diff --git a/demo/mnist/data/generate_list.py b/demo/mnist/data/generate_list.py new file mode 100644 index 0000000000000..1b929048b4d82 --- /dev/null +++ b/demo/mnist/data/generate_list.py @@ -0,0 +1,21 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +o = open("./" + "train.list", "w") +o.write("./data/raw_data/train" +"\n") +o.close() + +o = open("./" + "test.list", "w") +o.write("./data/raw_data/t10k" +"\n") +o.close() \ No newline at end of file diff --git a/demo/mnist/data/get_mnist_data.sh b/demo/mnist/data/get_mnist_data.sh new file mode 100644 index 0000000000000..c3ef99445049d --- /dev/null +++ b/demo/mnist/data/get_mnist_data.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh +# This scripts downloads the mnist data and unzips it. + +DIR="$( cd "$(dirname "$0")" ; pwd -P )" +rm -rf "$DIR/raw_data" +mkdir "$DIR/raw_data" +cd "$DIR/raw_data" + +echo "Downloading..." + +for fname in train-images-idx3-ubyte train-labels-idx1-ubyte t10k-images-idx3-ubyte t10k-labels-idx1-ubyte +do + if [ ! -e $fname ]; then + wget --no-check-certificate http://yann.lecun.com/exdb/mnist/${fname}.gz + gunzip ${fname}.gz + fi +done + +cd $DIR +rm -f *.list +python generate_list.py + diff --git a/demo/mnist/mnist_provider.py b/demo/mnist/mnist_provider.py new file mode 100644 index 0000000000000..0f14ded2dce93 --- /dev/null +++ b/demo/mnist/mnist_provider.py @@ -0,0 +1,33 @@ +from paddle.trainer.PyDataProvider2 import * + + +# Define a py data provider +@provider(input_types=[ + dense_vector(28 * 28), + integer_value(10) +]) +def process(settings, filename): # settings is not used currently. + imgf = filename + "-images-idx3-ubyte" + labelf = filename + "-labels-idx1-ubyte" + f = open(imgf, "rb") + l = open(labelf, "rb") + + f.read(16) + l.read(8) + + # Define number of samples for train/test + if "train" in filename: + n = 60000 + else: + n = 10000 + + for i in range(n): + label = ord(l.read(1)) + pixels = [] + for j in range(28*28): + pixels.append(float(ord(f.read(1))) / 255.0) + yield { "pixel": pixels, 'label': label } + + f.close() + l.close() + \ No newline at end of file diff --git a/demo/mnist/train.sh b/demo/mnist/train.sh new file mode 100755 index 0000000000000..084b32ac390b8 --- /dev/null +++ b/demo/mnist/train.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e +config=vgg_16_mnist.py +output=./mnist_vgg_model +log=train.log + +paddle train \ +--config=$config \ +--dot_period=10 \ +--log_period=100 \ +--test_all_data_in_one_period=1 \ +--use_gpu=0 \ +--trainer_count=1 \ +--num_passes=100 \ +--save_dir=$output \ +2>&1 | tee $log + +python -m paddle.utils.plotcurve -i $log > plot.png diff --git a/demo/mnist/vgg_16_mnist.py b/demo/mnist/vgg_16_mnist.py new file mode 100644 index 0000000000000..ad0a4de3215ca --- /dev/null +++ b/demo/mnist/vgg_16_mnist.py @@ -0,0 +1,52 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +is_predict = get_config_arg("is_predict", bool, False) + +####################Data Configuration ################## + + +if not is_predict: + data_dir='./data/' + define_py_data_sources2(train_list= data_dir + 'train.list', + test_list= data_dir + 'test.list', + module='mnist_provider', + obj='process') + +######################Algorithm Configuration ############# +settings( + batch_size = 128, + learning_rate = 0.1 / 128.0, + learning_method = MomentumOptimizer(0.9), + regularization = L2Regularization(0.0005 * 128) +) + +#######################Network Configuration ############# + +data_size=1*28*28 +label_size=10 +img = data_layer(name='pixel', size=data_size) + +# small_vgg is predined in trainer_config_helpers.network +predict = small_vgg(input_image=img, + num_channels=1, + num_classes=label_size) + +if not is_predict: + lbl = data_layer(name="label", size=label_size) + outputs(classification_cost(input=predict, label=lbl)) +else: + outputs(predict) From 43f7d7b7684b8c4cee4e396f57c4c841f41b2dbe Mon Sep 17 00:00:00 2001 From: luotao1 Date: Thu, 13 Oct 2016 15:11:52 +0800 Subject: [PATCH 160/324] add interface and unittest for nce layer (#180) * add interface and unittest for nce layer * follow comments --- doc/ui/api/trainer_config_helpers/layers.rst | 6 + paddle/gserver/layers/NCELayer.cpp | 13 +- paddle/trainer/tests/test_config.conf | 222 ++++++------------ .../paddle/trainer_config_helpers/layers.py | 89 ++++++- 4 files changed, 170 insertions(+), 160 deletions(-) diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index c1d7a7ce81530..5271262d20d55 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -371,6 +371,12 @@ ctc_layer :members: ctc_layer :noindex: +nce_layer +----------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: nce_layer + :noindex: + hsigmoid --------- .. automodule:: paddle.trainer_config_helpers.layers diff --git a/paddle/gserver/layers/NCELayer.cpp b/paddle/gserver/layers/NCELayer.cpp index a896e16a6027b..4faebe5d2ad6f 100644 --- a/paddle/gserver/layers/NCELayer.cpp +++ b/paddle/gserver/layers/NCELayer.cpp @@ -21,14 +21,18 @@ limitations under the License. */ namespace paddle { /** - * Noise-contrastive estimation + * Noise-contrastive estimation. * Implements the method in the following paper: - * A fast and simple algorithm for training neural probabilistic language models + * A fast and simple algorithm for training neural probabilistic language models. + * + * The config file api is nce_layer. */ class NCELayer : public Layer { int numClasses_; - int numInputs_; // number of input layer besides labelLayer and weightLayer + /// number of input layer besides labelLayer and weightLayer + int numInputs_; LayerPtr labelLayer_; + /// weight layer, can be None LayerPtr weightLayer_; WeightList weights_; std::unique_ptr biases_; @@ -43,7 +47,8 @@ class NCELayer : public Layer { real weight; }; std::vector samples_; - bool prepared_; // whether samples_ is prepared + /// whether samples_ is prepared + bool prepared_; Argument sampleOut_; IVectorPtr labelIds_; diff --git a/paddle/trainer/tests/test_config.conf b/paddle/trainer/tests/test_config.conf index 5d2e2ba9df5c7..664e18cb98681 100644 --- a/paddle/trainer/tests/test_config.conf +++ b/paddle/trainer/tests/test_config.conf @@ -13,157 +13,71 @@ # See the License for the specific language governing permissions and # limitations under the License. -#Todo(luotao02) This config is only used for unitest. It is out of date now, and will be updated later. - -default_initial_std(0.5) - -model_type("nn") - -DataLayer( - name = "input", - size = 3, -) - -DataLayer( - name = "weight", - size = 1, -) - -Layer( - name = "layer1_1", - type = "fc", - size = 5, - active_type = "sigmoid", - inputs = "input", -) - -Layer( - name = "layer1_2", - type = "fc", - size = 12, - active_type = "linear", - inputs = Input("input", parameter_name='sharew'), -) - -Layer( - name = "layer1_3", - type = "fc", - size = 3, - active_type = "tanh", - inputs = "input", -) - -Layer( - name = "layer1_5", - type = "fc", - size = 3, - active_type = "tanh", - inputs = Input("input", - learning_rate=0.01, - momentum=0.9, - decay_rate=0.05, - initial_mean=0.0, - initial_std=0.01, - format = "csc", - nnz = 4) -) - -FCLayer( - name = "layer1_4", - size = 5, - active_type = "square", - inputs = "input", - drop_rate = 0.5, -) - -Layer( - name = "pool", - type = "pool", - inputs = Input("layer1_2", - pool = Pool(pool_type="cudnn-avg-pool", - channels = 1, - size_x = 2, - size_y = 3, - img_width = 3, - padding = 1, - padding_y = 2, - stride = 2, - stride_y = 3)) -) - -Layer( - name = "concat", - type = "concat", - inputs = ["layer1_3", "layer1_4"], -) - -MixedLayer( - name = "output", - size = 3, - active_type = "softmax", - inputs = [ - FullMatrixProjection("layer1_1", - learning_rate=0.1), - TransposedFullMatrixProjection("layer1_2", parameter_name='sharew'), - FullMatrixProjection("concat"), - IdentityProjection("layer1_3"), - ], -) - -Layer( - name = "label", - type = "data", - size = 1, -) - -Layer( - name = "cost", - type = "multi-class-cross-entropy", - inputs = ["output", "label", "weight"], -) - -Layer( - name = "cost2", - type = "nce", - num_classes = 3, - active_type = "sigmoid", - neg_sampling_dist = [0.1, 0.3, 0.6], - inputs = ["layer1_2", "label", "weight"], -) - -Evaluator( - name = "error", - type = "classification_error", - inputs = ["output", "label", "weight"] -) - -Inputs("input", "label", "weight") -Outputs("cost", "cost2") - -TrainData( - ProtoData( - files = "dummy_list", - constant_slots = [1.0], - async_load_data = True, - ) -) - -TestData( - SimpleData( - files = "trainer/tests/sample_filelist.txt", - feat_dim = 3, - context_len = 0, - buffer_capacity = 1000000, - async_load_data = False, - ), -) - -Settings( - algorithm = "sgd", - num_batches_per_send_parameter = 1, - num_batches_per_get_parameter = 1, - batch_size = 100, - learning_rate = 0.001, - learning_rate_decay_a = 1e-5, - learning_rate_decay_b = 0.5, -) +from paddle.trainer_config_helpers import * + +TrainData(ProtoData( + files = "dummy_list", + constant_slots = [1.0], + async_load_data = True)) + +TestData(SimpleData( + files = "trainer/tests/sample_filelist.txt", + feat_dim = 3, + context_len = 0, + buffer_capacity = 1000000, + async_load_data = False)) + +settings(batch_size = 100) + +data = data_layer(name='input', size=3) + +wt = data_layer(name='weight', size=1) + +fc1 = fc_layer(input=data, size=5, + bias_attr=True, + act=SigmoidActivation()) + +fc2 = fc_layer(input=data, size=12, + bias_attr=True, + param_attr=ParamAttr(name='sharew'), + act=LinearActivation()) + +fc3 = fc_layer(input=data, size=3, + bias_attr=True, + act=TanhActivation()) + +fc4 = fc_layer(input=data, size=5, + bias_attr=True, + layer_attr=ExtraAttr(drop_rate=0.5), + act=SquareActivation()) + +pool = img_pool_layer(input=fc2, + pool_size=2, + pool_size_y=3, + num_channels=1, + padding=1, + padding_y=2, + stride=2, + stride_y=3, + img_width=3, + pool_type=CudnnAvgPooling()) + +concat = concat_layer(input=[fc3, fc4]) + +with mixed_layer(size=3, act=SoftmaxActivation()) as output: + output += full_matrix_projection(input=fc1) + output += trans_full_matrix_projection(input=fc2, + param_attr=ParamAttr(name='sharew')) + output += full_matrix_projection(input=concat) + output += identity_projection(input=fc3) + +lbl = data_layer(name='label', size=1) + +cost = classification_cost(input=output, label=lbl, weight=wt, + layer_attr=ExtraAttr(device=-1)) + +nce = nce_layer(input=fc2, label=lbl, weight=wt, + num_classes=3, + neg_distribution=[0.1, 0.3, 0.6]) + +outputs(cost, nce) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 7699c90db716c..745e61b2eb0bc 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -50,6 +50,7 @@ 'slope_intercept_layer', 'trans_full_matrix_projection', 'linear_comb_layer', 'convex_comb_layer', 'ctc_layer', 'crf_layer', 'crf_decoding_layer', + 'nce_layer', 'cross_entropy_with_selfnorm', 'cross_entropy', 'multi_binary_label_cross_entropy', 'rank_cost', 'lambda_cost', 'huber_cost', @@ -115,6 +116,7 @@ class LayerType(object): CTC_LAYER = "ctc" CRF_LAYER = "crf" CRF_DECODING_LAYER = "crf_decoding" + NCE_LAYER = 'nce' RANK_COST = "rank-cost" LAMBDA_COST = "lambda_cost" @@ -168,7 +170,7 @@ class LayerOutput(object): :param activation: Layer Activation. :type activation: BaseActivation. :param parents: Layer's parents. - :type parents: list|tuple|collection.Sequence + :type parents: list|tuple|collections.Sequence """ def __init__(self, name, layer_type, parents=None, activation=None, @@ -1988,10 +1990,16 @@ def concat_layer(input, act=None, name=None, layer_attr=None): Concat all input vector into one huge vector. Inputs can be list of LayerOutput or list of projection. + The example usage is: + + .. code-block:: python + + concat = concat_layer(input=[layer1, layer2]) + :param name: Layer name. :type name: basestring :param input: input layers or projections - :type input: list|tuple|collection.Sequence + :type input: list|tuple|collections.Sequence :param act: Activation type. :type act: BaseActivation :param layer_attr: Extra Layer Attribute. @@ -3488,6 +3496,83 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): parents.append(label) return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=size) +@wrap_bias_attr_default(has_bias=True) +@wrap_name_default() +@layer_support() +def nce_layer(input, label, num_classes, weight=None, + num_neg_samples=10, neg_distribution=None, + name=None, bias_attr=None, layer_attr=None): + """ + Noise-contrastive estimation. + Implements the method in the following paper: + A fast and simple algorithm for training neural probabilistic language models. + + The example usage is: + + .. code-block:: python + + cost = nce_layer(input=layer1, label=layer2, weight=layer3, + num_classes=3, neg_distribution=[0.1,0.3,0.6]) + + :param name: layer name + :type name: basestring + :param input: input layers. It could be a LayerOutput of list/tuple of LayerOutput. + :type input: LayerOutput|list|tuple|collections.Sequence + :param label: label layer + :type label: LayerOutput + :param weight: weight layer, can be None(default) + :type weight: LayerOutput + :param num_classes: number of classes. + :type num_classes: int + :param num_neg_samples: number of negative samples. Default is 10. + :type num_neg_samples: int + :param neg_distribution: The distribution for generating the random negative labels. + A uniform distribution will be used if not provided. + If not None, its length must be equal to num_classes. + :type neg_distribution: list|tuple|collections.Sequence|None + :param bias_attr: Bias parameter attribute. True if no bias. + :type bias_attr: ParameterAttribute|None|False + :param layer_attr: Extra Layer Attribute. + :type layer_attr: ExtraLayerAttribute + :return: layer name. + :rtype: LayerOutput + """ + if isinstance(input, LayerOutput): + input = [input] + assert isinstance(input, collections.Sequence) + assert isinstance(label, LayerOutput) + assert label.layer_type == LayerType.DATA + if neg_distribution is not None: + assert isinstance(neg_distribution, collections.Sequence) + assert len(neg_distribution) == num_classes + assert sum(neg_distribution) == 1 + + ipts_for_layer = [] + parents = [] + for each_input in input: + assert isinstance(each_input, LayerOutput) + ipts_for_layer.append(each_input.name) + parents.append(each_input) + ipts_for_layer.append(label.name) + parents.append(label) + + if weight is not None: + assert isinstance(weight, LayerOutput) + assert weight.layer_type == LayerType.DATA + ipts_for_layer.append(weight.name) + parents.append(weight) + + Layer( + name=name, + type=LayerType.NCE_LAYER, + num_classes=num_classes, + neg_sampling_dist=neg_distribution, + num_neg_samples=num_neg_samples, + inputs=ipts_for_layer, + bias=ParamAttr.to_bias(bias_attr), + **ExtraLayerAttribute.to_kwargs(layer_attr) + ) + return LayerOutput(name, LayerType.NCE_LAYER, parents=parents) """ following are cost Layers. From c2d418dbfde2db19ec575bfb3aeb091b37fbc8ef Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 14 Oct 2016 09:35:51 +0800 Subject: [PATCH 161/324] Merge internal changes (#198) * fix DataProvider create function args bug Change-Id: I9e3a1c535c805bf30204a14aea8d5143ff534784 * remove PserverForPython.h which is not used Change-Id: I2b27f1f3c11a42766a92fc689f0f5f1f73ee1d70 * add internal document script Change-Id: Ia0fec79456caea0b271f9903cc13e8a3d32e0774 --- paddle/gserver/dataproviders/DataProvider.h | 3 +- paddle/internals/scripts/docs.sh | 20 ++++ paddle/pserver/PserverForPython.h | 116 -------------------- 3 files changed, 22 insertions(+), 117 deletions(-) create mode 100755 paddle/internals/scripts/docs.sh delete mode 100644 paddle/pserver/PserverForPython.h diff --git a/paddle/gserver/dataproviders/DataProvider.h b/paddle/gserver/dataproviders/DataProvider.h index 534491d70d546..c24546374abf3 100644 --- a/paddle/gserver/dataproviders/DataProvider.h +++ b/paddle/gserver/dataproviders/DataProvider.h @@ -308,7 +308,8 @@ class DataProvider { /** * @brief create only used for unittest. */ - inline static DataProvider* create(const DataConfig &config, bool useGpu) { + inline static DataProvider* create(const DataConfig &config, + bool useGpu = FLAGS_use_gpu) { return create(config, ModelConfig(), useGpu); } diff --git a/paddle/internals/scripts/docs.sh b/paddle/internals/scripts/docs.sh new file mode 100755 index 0000000000000..517405c120a47 --- /dev/null +++ b/paddle/internals/scripts/docs.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved + +cd `dirname $0` + +# Add set -e, cd to directory. +set -e +mkdir -p $PWD/../../../build +cd $PWD/../../../build + +# Compile Documentation only. +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=ON +make paddle_docs paddle_docs_cn + +# remove old docs. mv new docs in deeplearning.baidu.com +scp -r doc/html paddle_doc@yq01-idl-gpu-offline42.yq01.baidu.com:/home/paddle_doc/www/doc_new +ssh paddle_doc@yq01-idl-gpu-offline42.yq01.baidu.com "cd ~/www/ && rm -r doc && mv doc_new doc" + +scp -r doc_cn/html paddle_doc@yq01-idl-gpu-offline42.yq01.baidu.com:/home/paddle_doc/www/doc_cn_new +ssh paddle_doc@yq01-idl-gpu-offline42.yq01.baidu.com "cd ~/www/ && rm -r doc_cn && mv doc_cn_new doc_cn" diff --git a/paddle/pserver/PserverForPython.h b/paddle/pserver/PserverForPython.h deleted file mode 100644 index 5bbeae8bd8b97..0000000000000 --- a/paddle/pserver/PserverForPython.h +++ /dev/null @@ -1,116 +0,0 @@ -/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once -#include "paddle/pserver/ParameterClient.h" -#include "paddle/pserver/ParameterServer.h" -#include "paddle/parameter/Parameter.h" -#include - -namespace paddle { - -struct PyObjectDeleter { - void operator()(PyObject* obj) { - if (obj) { - Py_DECREF(obj); - } - } -}; - -class ParameterClientPy : public ParameterClient { -protected: - typedef std::unique_ptr PyObjectPtr; - - std::vector parameter_; - int initArgc_; - char** initArgv_; - -public: - ParameterClientPy(std::vector configs, int argc, - std::vector argv, bool useGpu) { - initArgc_ = argc; - initArgv_ = new char* [argc]; - for (int i = 0; i < argc; i++) { - initArgv_[i] = new char[argv[i].size()]; - strcpy(initArgv_[i], // NOLINT - argv[i].c_str()); // NOLINT TODO(yuyang18): use snprintf instead. - } - ParameterConfig pyConfig; - ParameterPtr param; - for (auto& config : configs) { - pyConfig.ParseFromString(config); - param.reset(new Parameter(pyConfig, useGpu)); - parameter_.push_back(param); - } - Py_Initialize(); - CHECK(Py_IsInitialized()); - } - - ~ParameterClientPy() { - delete initArgv_; - Py_Finalize(); - } - - Parameter getParameter(int idx) { return *(parameter_[idx].get()); } - - void initClientPy() { - initMain(initArgc_, initArgv_); - CHECK(init(parameter_)) << "Init Client Failed."; - } - - void setConfigPy(std::string config) { - OptimizationConfig optConfig; - optConfig.ParseFromString(config); - setConfig(optConfig); - } - - bool inStatusPy(int status) { return inStatus(PServerStatus(status)); } - - void setStatusPy(int status) { setStatus(PServerStatus(status)); } - - void waitForStatusPy(int status) { waitForStatus(PServerStatus(status)); } - - void sendParameterPy(int updateMode, int parameterType, int numSamples, - real cost, bool sendBackParameter) { - sendParameter(ParameterUpdateMode(updateMode), ParameterType(parameterType), - int64_t(numSamples), real(cost), sendBackParameter); - } - - template - std::string asyncCallPy(const char* serviceName, const char* funcName, - const std::string in) { - ProtoIn protoIn; - ProtoOut protoOut; - std::mutex waitLock; - std::string data; - protoIn.ParseFromString(in); - waitLock.lock(); - auto callback = [&](ProtoOut* pOut, bool isSuccessful) { - if (isSuccessful) { - pOut->SerializeToString(&data); - } else { - LOG(INFO) << "Async Talk Failed."; - } - waitLock.unlock(); - }; - - ubClient_.asyncCall(serviceName, funcName, protoIn, - &protoOut, callback); - waitLock.lock(); - protoOut.SerializeToString(&data); - return data; - } -}; - -} // namespace paddle From cebdb66768832bf05d6b5f35d7752375361e5f0a Mon Sep 17 00:00:00 2001 From: luotao1 Date: Fri, 14 Oct 2016 09:51:30 +0800 Subject: [PATCH 162/324] hierarchical rnn document, add new config example (#106) * hierarchical rnn document, add new config example * update inputs_type of label * add check for unsupported config * refine hierarchical document * refine doc title * update docs, fix paddle to PaddlePaddle * follow comments --- doc/ui/api/trainer_config_helpers/layers.rst | 6 + doc_cn/algorithm/rnn/hierarchical-layer.md | 66 +++++ doc_cn/algorithm/rnn/hierarchical-rnn.md | 267 ++++++++++++++++++ doc_cn/algorithm/rnn/rnn-tutorial.md | 96 +++++++ doc_cn/index.rst | 5 +- .../RecurrentGradientMachine.cpp | 29 +- paddle/gserver/layers/AverageLayer.cpp | 10 +- .../layers/SequenceLastInstanceLayer.cpp | 9 +- paddle/gserver/tests/sequenceGen.py | 8 +- paddle/gserver/tests/sequence_nest_rnn.conf | 5 +- .../tests/sequence_nest_rnn_multi_input.conf | 5 +- .../tests/test_RecurrentGradientMachine.cpp | 12 +- 12 files changed, 480 insertions(+), 38 deletions(-) create mode 100644 doc_cn/algorithm/rnn/hierarchical-layer.md create mode 100644 doc_cn/algorithm/rnn/hierarchical-rnn.md create mode 100644 doc_cn/algorithm/rnn/rnn-tutorial.md diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index 5271262d20d55..55f5623b0faef 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -130,6 +130,12 @@ gru_step_layer Recurrent Layer Group ===================== +memory +------ +.. automodule:: paddle.trainer_config_helpers.layers + :members: memory + :noindex: + recurrent_group --------------- .. automodule:: paddle.trainer_config_helpers.layers diff --git a/doc_cn/algorithm/rnn/hierarchical-layer.md b/doc_cn/algorithm/rnn/hierarchical-layer.md new file mode 100644 index 0000000000000..5282bbbcb82d0 --- /dev/null +++ b/doc_cn/algorithm/rnn/hierarchical-layer.md @@ -0,0 +1,66 @@ +# 支持双层序列作为输入的Layer + +## 概述 + +在自然语言处理任务中，序列是一种常见的数据类型。一个独立的词语，可以看作是一个非序列输入，或者，我们称之为一个0层的序列；由词语构成的句子，是一个单层序列；若干个句子构成一个段落，是一个双层的序列。 + +双层序列是一个嵌套的序列，它的每一个元素，又是一个单层的序列。这是一种非常灵活的数据组织方式，帮助我们构造一些复杂的输入信息。 + +我们可以按照如下层次定义非序列，单层序列，以及双层序列。 + ++ 0层序列：一个独立的元素，类型可以是PaddlePaddle支持的任意输入数据类型 ++ 单层序列：排成一列的多个元素，每个元素是一个0层序列，元素之间的顺序是重要的输入信息 ++ 双层序列：排成一列的多个元素，每个元素是一个单层序列，称之为双层序列的一个子序列（subseq），subseq的每个元素是一个0层序列 + + +在 PaddlePaddle中，下面这些Layer能够接受双层序列作为输入，完成相应的计算。 +## pooling_layer + +pooling_layer的使用示例如下，详细见配置API。 +```python +seq_pool = pooling_layer(input=layer, + pooling_type=AvgPooling(), + agg_level=AggregateLevel.EACH_SEQUENCE) +``` +- `pooling_type` 目前支持两种，分别是：MaxPooling()和AvgPooling()。 +- `agg_level=AggregateLevel.TIMESTEP`时（默认值）： + - 作用：双层序列经过运算变成一个0层序列，或单层序列经过运算变成一个0层序列 + - 输入：一个双层序列，或一个单层序列 + - 输出：一个0层序列，即整个输入序列（单层或双层）的平均值（或最大值） +- `agg_level=AggregateLevel.EACH_SEQUENCE`时： + - 作用：一个双层序列经过运算变成一个单层序列 + - 输入：必须是一个双层序列 + - 输出：一个单层序列，序列的每个元素是原来双层序列每个subseq元素的平均值（或最大值） + +## last_seq 和 first_seq + +last_seq的使用示例如下（first_seq类似），详细见配置API。 +```python +last = last_seq(input=layer, + agg_level=AggregateLevel.EACH_SEQUENCE) +``` +- `agg_level=AggregateLevel.TIMESTEP`时（默认值）： + - 作用：一个双层序列经过运算变成一个0层序列，或一个单层序列经过运算变成一个0层序列 + - 输入：一个双层序列或一个单层序列 + - 输出：一个0层序列，即整个输入序列（双层或者单层）最后一个，或第一个元素。 +- `agg_level=AggregateLevel.EACH_SEQUENCE`时： + - 作用：一个双层序列经过运算变成一个单层序列 + - 输入：必须是一个双层序列 + - 输出：一个单层序列，其中每个元素是双层序列中每个subseq最后一个（或第一个）元素。 + +## expand_layer + +expand_layer的使用示例如下，详细见配置API。 +```python +expand = expand_layer(input=layer1, + expand_as=layer2, + expand_level=ExpandLevel.FROM_TIMESTEP) +``` +- `expand_level=ExpandLevel.FROM_TIMESTEP`时（默认值）： + - 作用：一个0层序列经过运算扩展成一个单层序列，或者一个双层序列 + - 输入：layer1必须是一个0层序列，是待扩展的数据；layer2可以是一个单层序列，或者是一个双层序列，提供扩展的长度信息 + - 输出：一个单层序列，或一个双层序列，输出序列的类型（双层序列，或单层序列）和序列中含有元素的数目同 layer2一致。若输出是单层序列，单层序列的每个元素（0层序列），都是对layer1元素的拷贝；若输出是双层序列，双层序列每个subseq中每个元素（0层序列），都是对layer1元素的拷贝 +- `expand_level=ExpandLevel.FROM_SEQUENCE`时： + - 作用：一个单层序列经过运算扩展成一个双层序列 + - 输入：layer1必须是一个单层序列，是待扩展的数据；layer2必须是一个双层序列，提供扩展的长度信息 + - 输出：一个双层序列，序列中含有元素的数目同layer2一致。要求单层序列含有元素的数目（0层序列），和双层序列含有subseq 的数目一致。单层序列第i个元素（0层序列），被扩展为一个单层序列，构成了输出双层序列的第i个subseq。 \ No newline at end of file diff --git a/doc_cn/algorithm/rnn/hierarchical-rnn.md b/doc_cn/algorithm/rnn/hierarchical-rnn.md new file mode 100644 index 0000000000000..979fe13e2ecbd --- /dev/null +++ b/doc_cn/algorithm/rnn/hierarchical-rnn.md @@ -0,0 +1,267 @@ +# 双层RNN配置与示例 + +我们在`paddle/gserver/tests/test_RecurrentGradientMachine`单测中，通过多组语义相同的单双层RNN配置，讲解如何使用双层RNN。 + +## 示例1：双进双出，subseq间无memory + +配置：单层RNN（`sequence_layer_group`）和双层RNN（`sequence_nest_layer_group`），语义完全相同。 + +### 读取双层序列的方法 + +首先，我们看一下单双层序列的不同数据组织形式（您也可以采用别的组织形式）： + +- 单层序列的数据（`Sequence/tour_train_wdseg`）如下，一共有10个样本。每个样本由两部分组成，一个label（此处都为2）和一个已经分词后的句子。 + +```text +2 酒店 有 很 舒适 的 床垫 子 ， 床上用品 也 应该 是 一人 一 换 ， 感觉 很 利落 对 卫生 很 放心 呀 。 +2 很 温馨 ， 也 挺 干净 的 * 地段 不错 ， 出来 就 有 全家 ， 离 地铁站 也 近 ， 交通 很方便 * 就是 都 不 给 刷牙 的 杯子 啊 ， 就 第一天 给 了 一次性杯子 * +2 位置 方便 ， 强烈推荐 ， 十一 出去玩 的 时候 选 的 ， 对面 就是 华润万家 ， 周围 吃饭 的 也 不少 。 +2 交通便利 ， 吃 很 便利 ， 乾 浄 、 安静 ， 商务 房 有 电脑 、 上网 快 ， 价格 可以 ， 就 早餐 不 好吃 。 整体 是 不错 的 。 適 合 出差 來 住 。 +2 本来 准备 住 两 晚 ， 第 2 天 一早 居然 停电 ， 且 无 通知 ， 只有 口头 道歉 。 总体来说 性价比 尚可 ， 房间 较 新 ， 还是 推荐 . +2 这个 酒店 去过 很多 次 了 ， 选择 的 主要原因 是 离 客户 最 便宜 相对 又 近 的 酒店 +2 挺好 的 汉庭 ， 前台 服务 很 热情 ， 卫生 很 整洁 ， 房间 安静 ， 水温 适中 ， 挺好 ！ +2 HowardJohnson 的 品质 ， 服务 相当 好 的 一 家 五星级 。 房间 不错 、 泳池 不错 、 楼层 安排 很 合理 。 还有 就是 地理位置 ， 简直 一 流 。 就 在 天一阁 、 月湖 旁边 ， 离 天一广场 也 不远 。 下次 来 宁波 还会 住 。 +2 酒店 很干净 ， 很安静 ， 很 温馨 ， 服务员 服务 好 ， 各方面 都 不错 * +2 挺好 的 ， 就是 没 窗户 ， 不过 对 得 起 这 价格 +``` + +- 双层序列的数据（`Sequence/tour_train_wdseg.nest`）如下，一共有4个样本。样本间用空行分开，代表不同的双层序列，序列数据和上面的完全一样。每个样本的子句数分别为2,3,2,3。 + +```text +2 酒店 有 很 舒适 的 床垫 子 ， 床上用品 也 应该 是 一人 一 换 ， 感觉 很 利落 对 卫生 很 放心 呀 。 +2 很 温馨 ， 也 挺 干净 的 * 地段 不错 ， 出来 就 有 全家 ， 离 地铁站 也 近 ， 交通 很方便 * 就是 都 不 给 刷牙 的 杯子 啊 ， 就 第一天 给 了 一次性杯子 * + +2 位置 方便 ， 强烈推荐 ， 十一 出去玩 的 时候 选 的 ， 对面 就是 华润万家 ， 周围 吃饭 的 也 不少 。 +2 交通便利 ， 吃 很 便利 ， 乾 浄 、 安静 ， 商务 房 有 电脑 、 上网 快 ， 价格 可以 ， 就 早餐 不 好吃 。 整体 是 不错 的 。 適 合 出差 來 住 。 +2 本来 准备 住 两 晚 ， 第 2 天 一早 居然 停电 ， 且 无 通知 ， 只有 口头 道歉 。 总体来说 性价比 尚可 ， 房间 较 新 ， 还是 推荐 . + +2 这个 酒店 去过 很多 次 了 ， 选择 的 主要原因 是 离 客户 最 便宜 相对 又 近 的 酒店 +2 挺好 的 汉庭 ， 前台 服务 很 热情 ， 卫生 很 整洁 ， 房间 安静 ， 水温 适中 ， 挺好 ！ + +2 HowardJohnson 的 品质 ， 服务 相当 好 的 一 家 五星级 。 房间 不错 、 泳池 不错 、 楼层 安排 很 合理 。 还有 就是 地理位置 ， 简直 一 流 。 就 在 天一阁 、 月湖 旁边 ， 离 天一广场 也 不远 。 下次 来 宁波 还会 住 。 +2 酒店 很干净 ， 很安静 ， 很 温馨 ， 服务员 服务 好 ， 各方面 都 不错 * +2 挺好 的 ， 就是 没 窗户 ， 不过 对 得 起 这 价格 +``` + +其次，我们看一下单双层序列的不同dataprovider（见`sequenceGen.py`）： + +- 单层序列的dataprovider如下： + - word_slot是integer_value_sequence类型，代表单层序列。 + - label是integer_value类型，代表一个向量。 + +```python +def hook(settings, dict_file, **kwargs): + settings.word_dict = dict_file + settings.input_types = [integer_value_sequence(len(settings.word_dict)), + integer_value(3)] + +@provider(init_hook=hook) +def process(settings, file_name): + with open(file_name, 'r') as fdata: + for line in fdata: + label, comment = line.strip().split('\t') + label = int(''.join(label.split())) + words = comment.split() + word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] + yield word_slot, label +``` + +- 双层序列的dataprovider如下： + - word_slot是integer_value_sub_sequence类型，代表双层序列。 + - label是integer_value_sequence类型，代表单层序列，即一个子句一个label。注意：也可以为integer_value类型，代表一个向量，即一个句子一个label。通常根据任务需求进行不同设置。 + - 关于dataprovider中input_types的详细用法，参见PyDataProvider2。 + +```python +def hook2(settings, dict_file, **kwargs): + settings.word_dict = dict_file + settings.input_types = [integer_value_sub_sequence(len(settings.word_dict)), + integer_value_sequence(3)] + +@provider(init_hook=hook2) +def process2(settings, file_name): + with open(file_name) as fdata: + label_list = [] + word_slot_list = [] + for line in fdata: + if (len(line)) > 1: + label,comment = line.strip().split('\t') + label = int(''.join(label.split())) + words = comment.split() + word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] + label_list.append(label) + word_slot_list.append(word_slot) + else: + yield word_slot_list, label_list + label_list = [] + word_slot_list = [] +``` + +### 模型中的配置 + +首先，我们看一下单层序列的配置（见`sequence_layer_group.conf`）。注意：batchsize=5表示一次过5句单层序列，因此2个batch就可以完成1个pass。 + +```python +settings(batch_size=5) + +data = data_layer(name="word", size=dict_dim) + +emb = embedding_layer(input=data, size=word_dim) + +# (lstm_input + lstm) is equal to lstmemory +with mixed_layer(size=hidden_dim*4) as lstm_input: + lstm_input += full_matrix_projection(input=emb) + +lstm = lstmemory_group(input=lstm_input, + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) + +lstm_last = last_seq(input=lstm) + +with mixed_layer(size=label_dim, + act=SoftmaxActivation(), + bias_attr=True) as output: + output += full_matrix_projection(input=lstm_last) + +outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) + +``` +其次，我们看一下语义相同的双层序列配置（见`sequence_nest_layer_group.conf`），并对其详细分析： + +- batchsize=2表示一次过2句双层序列。但从上面的数据格式可知，2句双层序列和5句单层序列的数据完全一样。 +- data_layer和embedding_layer不关心数据是否是序列格式，因此两个配置在这两层上的输出是一样的。 +- lstmemory: + - 单层序列过了一个mixed_layer和lstmemory_group。 + - 双层序列在同样的mixed_layer和lstmemory_group外，直接加了一层group。由于这个外层group里面没有memory，表示subseq间不存在联系，即起到的作用仅仅是把双层seq拆成单层，因此双层序列过完lstmemory的输出和单层的一样。 +- last_seq： + - 单层序列直接取了最后一个元素 + - 双层序列首先（last_seq层）取了每个subseq的最后一个元素，将其拼接成一个新的单层序列；接着（expand_layer层）将其扩展成一个新的双层序列，其中第i个subseq中的所有向量均为输入的单层序列中的第i个向量；最后（average_layer层）取了每个subseq的平均值。 + - 分析得出：第一个last_seq后，每个subseq的最后一个元素就等于单层序列的最后一个元素，而expand_layer和average_layer后，依然保持每个subseq最后一个元素的值不变（这两层仅是为了展示它们的用法，实际中并不需要）。因此单双层序列的输出是一样旳。 + +```python +settings(batch_size=2) + +data = data_layer(name="word", size=dict_dim) + +emb_group = embedding_layer(input=data, size=word_dim) + +# (lstm_input + lstm) is equal to lstmemory +def lstm_group(lstm_group_input): + with mixed_layer(size=hidden_dim*4) as group_input: + group_input += full_matrix_projection(input=lstm_group_input) + + lstm_output = lstmemory_group(input=group_input, + name="lstm_group", + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) + return lstm_output + +lstm_nest_group = recurrent_group(input=SubsequenceInput(emb_group), + step=lstm_group, + name="lstm_nest_group") +# hasSubseq ->(seqlastins) seq +lstm_last = last_seq(input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE) + +# seq ->(expand) hasSubseq +lstm_expand = expand_layer(input=lstm_last, expand_as=emb_group, expand_level=ExpandLevel.FROM_SEQUENCE) + +# hasSubseq ->(average) seq +lstm_average = pooling_layer(input=lstm_expand, + pooling_type=AvgPooling(), + agg_level=AggregateLevel.EACH_SEQUENCE) + +with mixed_layer(size=label_dim, + act=SoftmaxActivation(), + bias_attr=True) as output: + output += full_matrix_projection(input=lstm_average) + +outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) +``` +## 示例2：双进双出，subseq间有memory + +配置：单层RNN（`sequence_rnn.conf`），双层RNN（`sequence_nest_rnn.conf`和`sequence_nest_rnn_readonly_memory.conf`），语义完全相同。 + +### 读取双层序列的方法 + +我们看一下单双层序列的不同数据组织形式和dataprovider（见`rnn_data_provider.py`） +```python +data = [ + [[[1, 3, 2], [4, 5, 2]], 0], + [[[0, 2], [2, 5], [0, 1, 2]], 1], +] + +@provider(input_types=[integer_value_sub_sequence(10), + integer_value(3)]) +def process_subseq(settings, file_name): + for d in data: + yield d + +@provider(input_types=[integer_value_sequence(10), + integer_value(3)]) +def process_seq(settings, file_name): + for d in data: + seq = [] +``` +- 单层序列：有两句，分别为[1,3,2,4,5,2]和[0,2,2,5,0,1,2]。 +- 双层序列：有两句，分别为[[1,3,2],[4,5,2]]（2个子句）和[[0,2],[2,5],[0,1,2]]（3个子句）。 +- 单双层序列的label都分别是0和1 + +### 模型中的配置 + +我们选取单双层序列配置中的不同部分，来对比分析两者语义相同的原因。 + +- 单层序列：过了一个很简单的recurrent_group。每一个时间步，当前的输入y和上一个时间步的输出rnn_state做了一个全链接。 + +```python +def step(y): + mem = memory(name="rnn_state", size=hidden_dim) + return fc_layer(input=[y, mem], + size=hidden_dim, + act=TanhActivation(), + bias_attr=True, + name="rnn_state") + +out = recurrent_group(step=step, input=emb) +``` +- 双层序列，外层memory是一个元素： + - 内层inner_step的recurrent_group和单层序列的几乎一样。除了boot_layer=outer_mem，表示将外层的outer_mem作为内层memory的初始状态。外层outer_step中，outer_mem是一个子句的最后一个向量，即整个双层group是将前一个子句的最后一个向量，作为下一个子句memory的初始状态。 + - 从输入数据上看，单双层序列的句子是一样的，只是双层序列将其又做了子序列划分。因此双层序列的配置中，必须将前一个子句的最后一个元素，作为boot_layer传给下一个子句的memory，才能保证和单层序列的配置中“每一个时间步都用了上一个时间步的输出结果”一致。 + +```python +def outer_step(x): + outer_mem = memory(name="outer_rnn_state", size=hidden_dim) + def inner_step(y): + inner_mem = memory(name="inner_rnn_state", + size=hidden_dim, + boot_layer=outer_mem) + return fc_layer(input=[y, inner_mem], + size=hidden_dim, + act=TanhActivation(), + bias_attr=True, + name="inner_rnn_state") + + inner_rnn_output = recurrent_group( + step=inner_step, + input=x) + last = last_seq(input=inner_rnn_output, name="outer_rnn_state") + + return inner_rnn_output + +out = recurrent_group(step=outer_step, input=SubsequenceInput(emb)) +``` +- 双层序列，外层memory是单层序列： + - 由于外层每个时间步返回的是一个子句，这些子句的长度往往不等长。因此当外层有is_seq=True的memory时，内层是**无法直接使用**它的，即内层memory的boot_layer不能链接外层的这个memory。 + - 如果内层memory想**间接使用**这个外层memory，只能通过`pooling_layer`、`last_seq`或`first_seq`这三个layer将它先变成一个元素。但这种情况下，外层memory必须有boot_layer，否则在第0个时间步时，由于外层memory没有任何seq信息，因此上述三个layer的前向会报出“**Check failed: input.sequenceStartPositions**”的错误。 + +## 示例3：双进双出，输入不等长 + +TBD + +## 示例4：beam_search的生成 + +TBD \ No newline at end of file diff --git a/doc_cn/algorithm/rnn/rnn-tutorial.md b/doc_cn/algorithm/rnn/rnn-tutorial.md new file mode 100644 index 0000000000000..7a553054c8039 --- /dev/null +++ b/doc_cn/algorithm/rnn/rnn-tutorial.md @@ -0,0 +1,96 @@ +# Recurrent Group教程 + +## 概述 + +序列数据是自然语言处理任务面对的一种主要输入数据类型。 + +一句话是由词语构成的序列，多句话进一步构成了段落。因此，段落可以看作是一个嵌套的双层的序列，这个序列的每个元素又是一个序列。 + +双层序列是PaddlePaddle支持的一种非常灵活的数据组织方式，帮助我们更好地描述段落、多轮对话等更为复杂的语言数据。基于双层序列输入，我们可以设计搭建一个灵活的、层次化的RNN，分别从词语和句子级别编码输入数据，同时也能够引入更加复杂的记忆机制，更好地完成一些复杂的语言理解任务。 + +在PaddlePaddle中，`recurrent_group`是一种任意复杂的RNN单元，用户只需定义RNN在一个时间步内完成的计算，PaddlePaddle负责完成信息和误差在时间序列上的传播。 + +更进一步，`recurrent_group`同样可以扩展到双层序列的处理上。通过两个嵌套的`recurrent_group`分别定义子句级别和词语级别上需要完成的运算，最终实现一个层次化的复杂RNN。 + +目前，在PaddlePaddle中，能够对双向序列进行处理的有`recurrent_group`和部分Layer，具体可参考文档：支持双层序列作为输入的Layer。 + +## 相关概念 + +### 基本原理 +`recurrent_group` 是PaddlePaddle支持的一种任意复杂的RNN单元。使用者只需要关注于设计RNN在一个时间步之内完成的计算，PaddlePaddle负责完成信息和梯度在时间序列上的传播。 + +PaddlePaddle中，`recurrent_group`的一个简单调用如下： + +``` python +recurrent_group(step, input, reverse) +``` +- step：一个可调用的函数，定义一个时间步之内RNN单元完成的计算 +- input：输入，必须是一个单层序列，或者一个双层序列 +- reverse：是否以逆序处理输入序列 + +使用`recurrent_group`的核心是设计step函数的计算逻辑。step函数内部可以自由组合PaddlePaddle支持的各种layer，完成任意的运算逻辑。`recurrent_group` 的输入（即input）会成为step函数的输入，由于step 函数只关注于RNN一个时间步之内的计算，在这里`recurrent_group`替我们完成了原始输入数据的拆分。 + +### 输入 +`recurrent_group`处理的输入序列主要分为以下三种类型： + +- **数据输入**：一个双层序列进入`recurrent_group`会被拆解为一个单层序列，一个单层序列进入`recurrent_group`会被拆解为非序列，然后交给step函数，这一过程对用户是完全透明的。可以有以下两种：1）通过data_layer拿到的用户输入；2）其它layer的输出。 + +- **只读Memory输入**：`StaticInput` 定义了一个只读的Memory，由`StaticInput`指定的输入不会被`recurrent_group`拆解，`recurrent_group` 循环展开的每个时间步总是能够引用所有输入，可以是一个非序列，或者一个单层序列。 + +- **序列生成任务的输入**：`GeneratedInput`只用于在序列生成任务中指定输入数据。 + +### 输入示例 + +序列生成任务大多遵循encoder-decoer架构，encoder和decoder可以是能够处理序列的任意神经网络单元，而RNN是最流行的选择。 + +给定encoder输出和当前词，decoder每次预测产生下一个最可能的词语。在这种结构中，decoder接受两个输入： + +- 要生成的目标序列：是decoder的数据输入，也是decoder循环展开的依据，`recurrent_group`会对这类输入进行拆解。 + +- encoder输出，可以是一个非序列，或者一个单层序列：是一个unbounded memory，decoder循环展开的每一个时间步会引用全部结果，不应该被拆解，这种类型的输入必须通过`StaticInput`指定。关于Unbounded Memory的更多讨论请参考论文 [Neural Turning Machine](https://arxiv.org/abs/1410.5401)。 + +在序列生成任务中，decoder RNN总是引用上一时刻预测出的词的词向量，作为当前时刻输入。`GeneratedInput`自动完成这一过程。 + +### 输出 +`step`函数必须返回一个或多个Layer的输出，这个Layer的输出会作为整个`recurrent_group` 最终的输出结果。在输出的过程中，`recurrent_group` 会将每个时间步的输出拼接，这个过程对用户也是透明的。 + +### memory +memory只能在`recurrent_group`中定义和使用。memory不能独立存在，必须指向一个PaddlePaddle定义的Layer。引用memory得到这layer上一时刻输出，因此，可以将memory理解为一个时延操作。 + +可以显示地指定一个layer的输出用于初始化memory。不指定时，memory默认初始化为0。 + +## 双层RNN介绍 +`recurrent_group`帮助我们完成对输入序列的拆分，对输出的合并，以及计算逻辑在序列上的循环展开。 + +利用这种特性，两个嵌套的`recurrent_group`能够处理双层序列，实现词语和句子两个级别的双层RNN结构。 + +- 单层（word-level）RNN：每个状态（state）对应一个词（word）。 +- 双层（sequence-level）RNN：一个双层RNN由多个单层RNN组成，每个单层RNN（即双层RNN的每个状态）对应一个子句（subseq）。 + +为了描述方便，下文以NLP任务为例，将含有子句（subseq）的段落定义为一个双层序列，将含有词语的句子定义为一个单层序列，那么0层序列即为一个词语。 + +## 双层RNN的使用 + +### 训练流程的使用方法 +使用 `recurrent_group`需要遵循以下约定： + +- **单进单出**：输入和输出都是单层序列。 + - 如果有多个输入，不同输入序列含有的词语数必须严格相等。 + - 输出一个单层序列，输出序列的词语数和输入序列一致。 + - memory：在step函数中定义 memory指向一个layer，通过引用memory得到这个layer上一个时刻输出，形成recurrent 连接。memory的is_seq参数必须为false。如果没有定义memory，每个时间步之内的运算是独立的。 + - boot_layer：memory的初始状态，默认初始状为0，memory的is_seq参数必须为false。 + +- **双进双出**：输入和输出都是双层序列。 + - 如果有多个输入序列，不同输入含有的子句（subseq）数必须严格相等，但子句含有的词语数可以不相等。 + - 输出一个双层序列，子句（subseq）数、子句的单词数和指定的一个输入序列一致，默认为第一个输入。 + - memory：在step函数中定义memory，指向一个layer，通过引用memory得到这个layer上一个时刻的输出，形成recurrent连接。定义在外层`recurrent_group` step函数中的memory，能够记录上一个subseq 的状态，可以是一个单层序列（只作为read-only memory），也可以是一个词语。如果没有定义memory，那么 subseq 之间的运算是独立的。 + - boot_layer：memory 初始状态，可以是一个单层序列（只作为read-only memory）或一个向量。默认不设置，即初始状态为0。 + +- **双进单出**：目前还未支持，会报错"In hierachical RNN, all out links should be from sequences now"。 + + +### 生成流程的使用方法 +使用`beam_search`需要遵循以下约定： + +- 单层RNN：从一个word生成下一个word。 +- 双层RNN：即把单层RNN生成后的subseq给拼接成一个新的双层seq。从语义上看，也不存在一个subseq直接生成下一个subseq的情况。 \ No newline at end of file diff --git a/doc_cn/index.rst b/doc_cn/index.rst index 6cf5588b5b34f..1a4908be14684 100644 --- a/doc_cn/index.rst +++ b/doc_cn/index.rst @@ -16,4 +16,7 @@ PaddlePaddle文档 算法教程 -------- -* `RNN配置 <../doc/algorithm/rnn/rnn.html>`_ +* `Recurrent Group教程 `_ +* `单层RNN示例 <../doc/algorithm/rnn/rnn.html>`_ +* `双层RNN示例 `_ +* `支持双层序列作为输入的Layer `_ diff --git a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp index fc38bca3c403b..340cd1b9f8e92 100644 --- a/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/RecurrentGradientMachine.cpp @@ -544,6 +544,12 @@ void RecurrentGradientMachine::forward(const std::vector& inArgs, const std::vector inArgs; std::vector outArgs; frames_[i]->forward(inArgs, &outArgs, passType); + if (hasSubseq) { + for (auto& outFrameLine : outFrameLines_) { + CHECK(outFrameLine.frames[i]->getOutput().sequenceStartPositions) + << "In hierachical RNN, all out links should be from sequences."; + } + } } if (evaluator_ && passType == PASS_TEST) { this->eval(evaluator_.get()); @@ -635,16 +641,15 @@ void RecurrentGradientMachine::createInFrameInfo(int inlinkId, std::vector sequenceStartPositions; const int* subSequenceStartPositions = nullptr; - if (hasSubseq) { // for sequenceScatterAgentLayer - subSequenceStartPositions = - input.subSequenceStartPositions->getData(false); + if (hasSubseq) { // for sequenceScatterAgentLayer + subSequenceStartPositions = input.subSequenceStartPositions->getData(false); inlinkInfo->seqStartPosIndex.clear(); inlinkInfo->seqStartPosIndex.push_back(0); // first seqStartPosIndex = 0 } // maxSequenceLength_: max topLevelLength in allsamples for (int i = 0; i < maxSequenceLength_; ++i) { if (hasSubseq) { - sequenceStartPositions.push_back(0); // first element = 0 + sequenceStartPositions.push_back(0); // first element = 0 } int numSeqs = 0; for (size_t j = 0; j < numSequences; ++j) { @@ -676,9 +681,9 @@ void RecurrentGradientMachine::createInFrameInfo(int inlinkId, } if (hasSubseq) { // inFrameLine create sequenceStartPositions one time - CHECK_EQ(sequenceStartPositions.size(), - static_cast(maxSequenceLength_ + - input.getNumSubSequences())); + CHECK_EQ( + sequenceStartPositions.size(), + static_cast(maxSequenceLength_ + input.getNumSubSequences())); CHECK_EQ(inlinkInfo->seqStartPosIndex.size(), static_cast(maxSequenceLength_ + 1)); createSeqPos(sequenceStartPositions, &inlinkInfo->sequenceStartPositions); @@ -1102,10 +1107,12 @@ size_t RecurrentGradientMachine::beamShrink(std::vector& newPaths, newPaths.end(), Path::greaterPath); newPaths.resize(totalExpandCount + minNewPathSize); - real minPathLogProb = std::min_element(newPaths.end() - minNewPathSize, - newPaths.end())->logProb; - real maxPathLogProb = std::max_element(newPaths.end() - minNewPathSize, - newPaths.end())->logProb; + real minPathLogProb = + std::min_element(newPaths.end() - minNewPathSize, newPaths.end()) + ->logProb; + real maxPathLogProb = + std::max_element(newPaths.end() - minNewPathSize, newPaths.end()) + ->logProb; // Remove the already formed paths that are relatively short finalPaths_[seqId].erase( diff --git a/paddle/gserver/layers/AverageLayer.cpp b/paddle/gserver/layers/AverageLayer.cpp index 374117b7659bb..6e52217de4e63 100644 --- a/paddle/gserver/layers/AverageLayer.cpp +++ b/paddle/gserver/layers/AverageLayer.cpp @@ -64,6 +64,11 @@ void AverageLayer::forward(PassType passType) { size_t dim = getSize(); const Argument& input = getInput(0); + CHECK(input.sequenceStartPositions); + if (type_) { + CHECK(input.subSequenceStartPositions) + << "when trans_type = seq, input must hasSubseq"; + } int64_t newBatchSize = type_ ? input.getNumSubSequences() : input.getNumSequences(); ICpuGpuVectorPtr startPositions = @@ -75,11 +80,6 @@ void AverageLayer::forward(PassType passType) { // check CHECK_EQ(numSequences, (size_t)newBatchSize); CHECK_EQ(starts[numSequences], input.getBatchSize()); - if (type_) { - // when trans_type = seq, input must hasSubseq - CHECK_EQ(input.hasSubseq(), 1UL); - } - CHECK_EQ(dim, input.value->getWidth()); resetOutput(newBatchSize, dim); diff --git a/paddle/gserver/layers/SequenceLastInstanceLayer.cpp b/paddle/gserver/layers/SequenceLastInstanceLayer.cpp index 12831e3668802..f4d26ba21bed6 100644 --- a/paddle/gserver/layers/SequenceLastInstanceLayer.cpp +++ b/paddle/gserver/layers/SequenceLastInstanceLayer.cpp @@ -91,6 +91,11 @@ void SequenceLastInstanceLayer::forward(PassType passType) { const Argument& input = getInput(0); // check + CHECK(input.sequenceStartPositions); + if (type_) { + CHECK(input.subSequenceStartPositions) + << "when trans_type = seq, input must hasSubseq"; + } auto startPositions = type_ ? input.subSequenceStartPositions->getVector(false) : input.sequenceStartPositions->getVector(false); @@ -98,10 +103,6 @@ void SequenceLastInstanceLayer::forward(PassType passType) { CHECK_EQ(dim, input.value->getWidth()); CHECK_EQ(startPositions->getData()[height], input.getBatchSize()); CHECK_EQ(height, startPositions->getSize() - 1); - if (type_) { - // when trans_type = seq, input must hasSubseq - CHECK_EQ(input.hasSubseq(), 1UL); - } reserveOutput(height, dim); const int* starts = startPositions->getData(); diff --git a/paddle/gserver/tests/sequenceGen.py b/paddle/gserver/tests/sequenceGen.py index cbed1f15fc415..b166e778d7a33 100644 --- a/paddle/gserver/tests/sequenceGen.py +++ b/paddle/gserver/tests/sequenceGen.py @@ -21,7 +21,7 @@ def hook(settings, dict_file, **kwargs): settings.word_dict = dict_file settings.input_types = [integer_value_sequence(len(settings.word_dict)), - integer_value_sequence(3)] + integer_value(3)] settings.logger.info('dict len : %d' % (len(settings.word_dict))) @@ -34,14 +34,14 @@ def process(settings, file_name): words = comment.split() word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] - yield word_slot, [label] + yield word_slot, label ## for hierarchical sequence network def hook2(settings, dict_file, **kwargs): settings.word_dict = dict_file settings.input_types = [integer_value_sub_sequence(len(settings.word_dict)), - integer_value_sub_sequence(3)] + integer_value_sequence(3)] settings.logger.info('dict len : %d' % (len(settings.word_dict))) @@ -57,7 +57,7 @@ def process2(settings, file_name): words = comment.split() word_slot = [settings.word_dict[w] for w in words if w in settings.word_dict] - label_list.append([label]) + label_list.append(label) word_slot_list.append(word_slot) else: yield word_slot_list, label_list diff --git a/paddle/gserver/tests/sequence_nest_rnn.conf b/paddle/gserver/tests/sequence_nest_rnn.conf index 62b8c5d072d7b..93b08eb2f8746 100644 --- a/paddle/gserver/tests/sequence_nest_rnn.conf +++ b/paddle/gserver/tests/sequence_nest_rnn.conf @@ -56,9 +56,8 @@ def outer_step(x): last = last_seq(input=inner_rnn_output, name="outer_rnn_state") # "return last" should also work. But currently RecurrentGradientMachine - # does not handle it correctly. Current implementation requires that - # all the out links are from sequences. However, it does not report error - # when the out links are not sequences. + # does not handle it, and will report error: In hierachical RNN, all out + # links should be from sequences now. return inner_rnn_output out = recurrent_group( diff --git a/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf b/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf index e01b3f8e7aa5c..e8222cef525a8 100644 --- a/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf +++ b/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf @@ -57,9 +57,8 @@ def outer_step(wid, x): last = last_seq(input=inner_rnn_output, name="outer_rnn_state") # "return last" should also work. But currently RecurrentGradientMachine - # does not handle it correctly. Current implementation requires that - # all the out links are from sequences. However, it does not report error - # when the out links are not sequences. + # does not handle it, and will report error: In hierachical RNN, all out + # links should be from sequences now. return inner_rnn_output out = recurrent_group( diff --git a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp index ae7f617371ca5..d104db3e5b32d 100644 --- a/paddle/gserver/tests/test_RecurrentGradientMachine.cpp +++ b/paddle/gserver/tests/test_RecurrentGradientMachine.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include #include #include @@ -24,7 +23,7 @@ limitations under the License. */ P_DECLARE_int32(seed); using namespace paddle; // NOLINT -using namespace std; // NOLINT +using namespace std; // NOLINT class TrainerForTest : public paddle::Trainer { public: void startTrain() { @@ -44,11 +43,10 @@ class TrainerForTest : public paddle::Trainer { */ size_t getTotalParameterSize() const { auto p = const_cast(this); - auto & params = p->getGradientMachine()->getParameters(); - return std::accumulate(params.begin(), params.end(), 0UL, - [](size_t a, const ParameterPtr& p){ - return a+p->getSize(); - }); + auto& params = p->getGradientMachine()->getParameters(); + return std::accumulate( + params.begin(), params.end(), 0UL, + [](size_t a, const ParameterPtr& p) { return a + p->getSize(); }); } }; From 91df6062804093743e52e356824e503339c7f43e Mon Sep 17 00:00:00 2001 From: luotao1 Date: Fri, 14 Oct 2016 16:02:24 +0800 Subject: [PATCH 163/324] remove some copyfrom in AgentLayer and ExpandLayer, fix warning in seq2seq config (#183) --- demo/seqToseq/seqToseq_net.py | 8 +++--- paddle/cuda/include/hl_sequence.h | 2 +- paddle/cuda/include/stub/hl_sequence_stub.h | 2 +- paddle/cuda/src/hl_cuda_sequence.cu | 4 +-- paddle/gserver/layers/AgentLayer.cpp | 29 ++++++++----------- paddle/gserver/layers/AgentLayer.h | 6 +--- paddle/gserver/layers/ExpandLayer.cpp | 31 +++++++-------------- paddle/gserver/layers/ExpandLayer.h | 7 +---- paddle/math/Matrix.cpp | 8 +++--- paddle/math/Matrix.h | 6 ++-- 10 files changed, 39 insertions(+), 64 deletions(-) diff --git a/demo/seqToseq/seqToseq_net.py b/demo/seqToseq/seqToseq_net.py index 2b0c3f34648b0..edd6ad3f739b6 100644 --- a/demo/seqToseq/seqToseq_net.py +++ b/demo/seqToseq/seqToseq_net.py @@ -96,12 +96,12 @@ def gru_encoder_decoder(data_conf, encoded_vector = concat_layer(input=[src_forward, src_backward]) with mixed_layer(size=decoder_size) as encoded_proj: - encoded_proj += full_matrix_projection(encoded_vector) + encoded_proj += full_matrix_projection(input=encoded_vector) backward_first = first_seq(input=src_backward) with mixed_layer(size=decoder_size, act=TanhActivation(), ) as decoder_boot: - decoder_boot += full_matrix_projection(backward_first) + decoder_boot += full_matrix_projection(input=backward_first) def gru_decoder_with_attention(enc_vec, enc_proj, current_word): decoder_mem = memory(name='gru_decoder', @@ -113,8 +113,8 @@ def gru_decoder_with_attention(enc_vec, enc_proj, current_word): decoder_state=decoder_mem, ) with mixed_layer(size=decoder_size * 3) as decoder_inputs: - decoder_inputs += full_matrix_projection(context) - decoder_inputs += full_matrix_projection(current_word) + decoder_inputs += full_matrix_projection(input=context) + decoder_inputs += full_matrix_projection(input=current_word) gru_step = gru_step_layer(name='gru_decoder', input=decoder_inputs, diff --git a/paddle/cuda/include/hl_sequence.h b/paddle/cuda/include/hl_sequence.h index 828c21beb2fbd..46d86b2982f06 100644 --- a/paddle/cuda/include/hl_sequence.h +++ b/paddle/cuda/include/hl_sequence.h @@ -143,7 +143,7 @@ extern void hl_context_projection_backward_weight(real* outputGrad, */ extern void hl_sequence2batch_copy(real *batch, real *sequence, - int *batchIndex, + const int *batchIndex, int seqWidth, int batchCount, bool seq2batch); diff --git a/paddle/cuda/include/stub/hl_sequence_stub.h b/paddle/cuda/include/stub/hl_sequence_stub.h index 417f40e0a69f6..aabd956c37f7d 100644 --- a/paddle/cuda/include/stub/hl_sequence_stub.h +++ b/paddle/cuda/include/stub/hl_sequence_stub.h @@ -62,7 +62,7 @@ inline void hl_context_projection_backward_weight(real* outputGrad, inline void hl_sequence2batch_copy(real *batch, real *sequence, - int *batchIndex, + const int *batchIndex, int seqWidth, int batchCount, bool seq2batch) {} diff --git a/paddle/cuda/src/hl_cuda_sequence.cu b/paddle/cuda/src/hl_cuda_sequence.cu index e028880156e5b..63824eaa4c201 100644 --- a/paddle/cuda/src/hl_cuda_sequence.cu +++ b/paddle/cuda/src/hl_cuda_sequence.cu @@ -374,7 +374,7 @@ template __global__ void KeSequence2Batch(real *batch, real *sequence, - int *batchIndex, + const int *batchIndex, int seqWidth, int batchCount) { int idx = threadIdx.x; @@ -405,7 +405,7 @@ void KeSequence2Batch(real *batch, void hl_sequence2batch_copy(real *batch, real *sequence, - int *batchIndex, + const int *batchIndex, int seqWidth, int batchCount, bool seq2batch) { diff --git a/paddle/gserver/layers/AgentLayer.cpp b/paddle/gserver/layers/AgentLayer.cpp index 056e9568852ac..5e07446c71ff6 100644 --- a/paddle/gserver/layers/AgentLayer.cpp +++ b/paddle/gserver/layers/AgentLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "AgentLayer.h" #include "paddle/utils/Logging.h" @@ -62,8 +61,8 @@ void SequenceAgentLayer::forward(PassType passType) { // get Arguments from real layers if (numSamples_ > 0 && numSamples_ < realNumSequences) { - int numRows = realOutput.sequenceStartPositions-> - getData(false)[numSamples_]; + int numRows = + realOutput.sequenceStartPositions->getData(false)[numSamples_]; CHECK(!realOutput.ids) << "Not supported"; output_.subArgFrom(realOutput, /* offset */ 0, numRows, getSize(), useGpu_, /* trans */ false, /* seqFlag */ true, @@ -141,8 +140,8 @@ void ScatterAgentLayer::forward(PassType passType) { int width = this->getSize(); if (realOutArg_.value || realOutArg_.ids) { - output_.subArgFrom(realOutArg_, /* offset */ idIndex_, idSize_, - width, useGpu_); + output_.subArgFrom(realOutArg_, /* offset */ idIndex_, idSize_, width, + useGpu_); } else { // used in generation if (realLayer_->getOutput().ids) { IVector::resizeOrCreate(output_.ids, ids_->getSize(), useGpu_); @@ -224,8 +223,8 @@ void SequenceScatterAgentLayer::forward(PassType passType) { if (realOutArg_.value || realOutArg_.ids) { CHECK(realOutArg_.sequenceStartPositions); - output_.subArgFrom(realOutArg_, /* offset */ idIndex_, idSize_, - width, useGpu_, /* trans */ false, /* seqFlag */ true, + output_.subArgFrom(realOutArg_, /* offset */ idIndex_, idSize_, width, + useGpu_, /* trans */ false, /* seqFlag */ true, /* seqStart */ seqStartPosIndex_, /* seqSize */ numSequences_); } else { @@ -249,11 +248,12 @@ void SequenceScatterAgentLayer::forward(PassType passType) { CHECK_NE(input.sequenceStartPositions.get(), output_.sequenceStartPositions.get()); ICpuGpuVector::resizeOrCreate(output_.sequenceStartPositions, - numSequences + 1, false); + numSequences + 1, false); int* outStarts = output_.sequenceStartPositions->getMutableData(false); - IVector::resizeOrCreate(cpuInputStartPos_, height, false); - int* inStarts = cpuInputStartPos_->getData(); + ICpuGpuVector::resizeOrCreate(inputStartPos_, height, false); + int* inStarts = inputStartPos_->getMutableData(false); + size_t offsetOut = 0; for (size_t i = 0; i < numSequences; ++i) { outStarts[i] = offsetOut; @@ -266,13 +266,8 @@ void SequenceScatterAgentLayer::forward(PassType passType) { } outStarts[numSequences] = offsetOut; - if (useGpu_) { - IVector::resizeOrCreate(inputStartPos_, height, true); - inputStartPos_->copyFrom(*cpuInputStartPos_, HPPL_STREAM_DEFAULT); - } else { - inputStartPos_ = cpuInputStartPos_; - } - outputValue->copyByRowIndex(*input.value, *inputStartPos_); + outputValue->copyByRowIndex(*input.value, + *inputStartPos_->getVector(useGpu_)); } } diff --git a/paddle/gserver/layers/AgentLayer.h b/paddle/gserver/layers/AgentLayer.h index d82078dd93329..3d7bf55834070 100644 --- a/paddle/gserver/layers/AgentLayer.h +++ b/paddle/gserver/layers/AgentLayer.h @@ -191,11 +191,7 @@ class SequenceScatterAgentLayer : public ScatterAgentLayer { protected: // use to store expanded cpuStartPositions or subSequenceStartPositions // of real layer. - IVectorPtr cpuInputStartPos_; - - // point to cpuInputStartPos_ when useGpu_ is false - // copy from cpuInputStartPos_ when useGpu_ is true - IVectorPtr inputStartPos_; + ICpuGpuVectorPtr inputStartPos_; public: explicit SequenceScatterAgentLayer(const LayerConfig& config) diff --git a/paddle/gserver/layers/ExpandLayer.cpp b/paddle/gserver/layers/ExpandLayer.cpp index bbd0b53273b43..9290ce4f6d46c 100644 --- a/paddle/gserver/layers/ExpandLayer.cpp +++ b/paddle/gserver/layers/ExpandLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "ExpandLayer.h" #include "paddle/utils/Logging.h" #include "paddle/utils/Stat.h" @@ -53,9 +52,8 @@ void ExpandLayer::forward(PassType passType) { const Argument& shapeInput = getInput(1); const Argument& dataInput = getInput(0); size_t outputBatchSize = shapeInput.getBatchSize(); - auto startPositions = - type_ ? shapeInput.subSequenceStartPositions - : shapeInput.sequenceStartPositions; + auto startPositions = type_ ? shapeInput.subSequenceStartPositions + : shapeInput.sequenceStartPositions; size_t numSequences = startPositions->getSize() - 1; const int* starts = startPositions->getData(false); @@ -71,8 +69,7 @@ void ExpandLayer::forward(PassType passType) { // set output sequence info as shape sequence output_.sequenceStartPositions = shapeInput.sequenceStartPositions; if (shapeInput.hasSubseq()) { - output_.subSequenceStartPositions = - shapeInput.subSequenceStartPositions; + output_.subSequenceStartPositions = shapeInput.subSequenceStartPositions; } // reserve output: Expand output to batchsize of sequence data. @@ -81,8 +78,8 @@ void ExpandLayer::forward(PassType passType) { MatrixPtr inputValue = getInputValue(0); MatrixPtr outputValue = getOutputValue(); - IVector::resizeOrCreate(cpuExpandStartsPos_, outputBatchSize, false); - int* expandStarts = cpuExpandStartsPos_->getData(); + ICpuGpuVector::resizeOrCreate(expandStartsPos_, outputBatchSize, false); + int* expandStarts = expandStartsPos_->getMutableData(false); for (size_t sequenceId = 0; sequenceId < numSequences; ++sequenceId) { int sequenceLength = starts[sequenceId + 1] - starts[sequenceId]; for (int j = 0; j < sequenceLength; j++) { @@ -90,15 +87,8 @@ void ExpandLayer::forward(PassType passType) { } } - if (useGpu_) { - // TODO(Dangqingqing) move copyFrom - IVector::resizeOrCreate(expandStartsPos_, outputBatchSize, true); - expandStartsPos_->copyFrom(*cpuExpandStartsPos_, HPPL_STREAM_DEFAULT); - } else { - expandStartsPos_ = cpuExpandStartsPos_; - } - - outputValue->copyByRowIndex(*inputValue, *expandStartsPos_); + outputValue->copyByRowIndex(*inputValue, + *expandStartsPos_->getVector(useGpu_)); if (biases_.get() != NULL) { outputValue->addBias(*(biases_->getW()), 1); @@ -108,16 +98,15 @@ void ExpandLayer::forward(PassType passType) { void ExpandLayer::backward(const UpdateCallback& callback) { if (biases_ && biases_->getWGrad()) { biases_->getWGrad()->collectBias(*getOutputGrad(), 1); - /* Increasing the number of gradient */ + /* Increasing the number of gradient */ biases_->getParameterPtr()->incUpdate(callback); } if (!getInputGrad(0)) return; MatrixPtr inputGrad = getInputGrad(0); MatrixPtr outputGrad = getOutputGrad(); - auto cpuSeqStartPos = - type_ ? getInput(1).subSequenceStartPositions - : getInput(1).sequenceStartPositions; + auto cpuSeqStartPos = type_ ? getInput(1).subSequenceStartPositions + : getInput(1).sequenceStartPositions; size_t numSequences = cpuSeqStartPos->getSize() - 1; const int* starts = cpuSeqStartPos->getData(false); diff --git a/paddle/gserver/layers/ExpandLayer.h b/paddle/gserver/layers/ExpandLayer.h index 8a3eb1c973a47..fbe0ced9b1754 100644 --- a/paddle/gserver/layers/ExpandLayer.h +++ b/paddle/gserver/layers/ExpandLayer.h @@ -44,14 +44,9 @@ class ExpandLayer : public Layer { enum ExpandLevel { kNonSeq = 0, kSeq = 1 }; /// store the ExpandLevel int type_; - // TODO(luotao) use ICpuGpuVectorPtr to merge cpuExpandStartsPos_ - // and expandStartsPos_ /// expanded sequenceStartPositions or subSequenceStartPositions /// of input[1] - IVectorPtr cpuExpandStartsPos_; - /// point to cpuExpandStartsPos_ when useGpu_ is false, - /// copy from cpuExpandStartsPos_ when useGpu_ is true - IVectorPtr expandStartsPos_; + ICpuGpuVectorPtr expandStartsPos_; public: explicit ExpandLayer(const LayerConfig& config) : Layer(config) {} diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index a6ff2f3b35d04..78519ce7aa874 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -282,13 +282,13 @@ void GpuMatrix::copyFrom(const IVector& src) { copyFrom(matrix); } -void GpuMatrix::copyByRowIndex(Matrix& b, IVector& rowIndex) { +void GpuMatrix::copyByRowIndex(Matrix& b, const IVector& rowIndex) { size_t height = getHeight(); size_t width = getWidth(); CHECK_EQ(b.getWidth(), width); real* dst = getData(); real* src = b.getData(); - int* index = rowIndex.getData(); + const int* index = rowIndex.getData(); hl_sequence2batch_copy(dst, src, index, width, height, true); } @@ -1278,11 +1278,11 @@ void CpuMatrix::copyFrom(const IVector& src) { } } -void CpuMatrix::copyByRowIndex(Matrix& b, IVector& rowIndex) { +void CpuMatrix::copyByRowIndex(Matrix& b, const IVector& rowIndex) { size_t height = getHeight(); size_t width = getWidth(); CHECK_EQ(b.getWidth(), width); - int* index = rowIndex.getData(); + const int* index = rowIndex.getData(); for (size_t i = 0; i < height; i++) { CHECK_LT(static_cast(index[i]), b.getHeight()); real* src = b.getData() + index[i] * width; diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index 5c15c94012816..25104fe1c6d70 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -253,7 +253,7 @@ class Matrix : public BaseMatrix { LOG(FATAL) << "copy data from int vector only available on CpuMatrix."; } - virtual void copyByRowIndex(Matrix& b, IVector& rowIndex) { + virtual void copyByRowIndex(Matrix& b, const IVector& rowIndex) { LOG(FATAL) << "Not implemented"; } @@ -979,7 +979,7 @@ class GpuMatrix : public Matrix { void copyFrom(const IVector& src); - void copyByRowIndex(Matrix& b, IVector& rowIndex); + void copyByRowIndex(Matrix& b, const IVector& rowIndex); MatrixPtr clone(size_t height, size_t width, bool useGpu = false); @@ -1241,7 +1241,7 @@ class CpuMatrix : public Matrix { void copyFrom(CpuSparseMatrix& src); - void copyByRowIndex(Matrix& b, IVector& rowIndex); + void copyByRowIndex(Matrix& b, const IVector& rowIndex); MatrixPtr clone(size_t height, size_t width, bool useGpu = false); From c13bdb15cdabda2a68c1eca470d612f079538d27 Mon Sep 17 00:00:00 2001 From: gangliao Date: Sat, 15 Oct 2016 06:54:52 -0700 Subject: [PATCH 164/324] remove redundant HPPL_TYPE_DOUBLE (#200) --- CMakeLists.txt | 2 +- paddle/cuda/include/hl_base.h | 2 +- paddle/cuda/include/hl_cpu_gru.cuh | 2 +- paddle/cuda/include/hl_gpu_functions.cuh | 4 ++-- paddle/cuda/include/hl_matrix_base.cuh | 2 +- paddle/cuda/include/hl_matrix_type.cuh | 4 ++-- paddle/cuda/include/hl_sse_matrix_kernel.cuh | 4 ++-- paddle/cuda/src/hl_cuda_cublas.cc | 2 +- paddle/cuda/src/hl_cuda_cudnn.cc | 10 +++++----- paddle/cuda/src/hl_cuda_device.cc | 2 +- paddle/cuda/src/hl_cuda_matrix.cu | 4 ++-- paddle/cuda/src/hl_cuda_sparse.cuh | 2 +- 12 files changed, 20 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 44e93f22c0eaf..b85709f807bc3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,7 +104,7 @@ else() endif(NOT WITH_GPU) if(WITH_DOUBLE) - add_definitions(-DPADDLE_TYPE_DOUBLE -DHPPL_TYPE_DOUBLE) + add_definitions(-DPADDLE_TYPE_DOUBLE) set(ACCURACY double) else(WITH_DOUBLE) set(ACCURACY float) diff --git a/paddle/cuda/include/hl_base.h b/paddle/cuda/include/hl_base.h index 77e2649b17214..1fe2774cc5a29 100644 --- a/paddle/cuda/include/hl_base.h +++ b/paddle/cuda/include/hl_base.h @@ -185,7 +185,7 @@ typedef struct { size_t nnz; } _hl_sparse_matrix_s, *hl_sparse_matrix_s; -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE /** * HPPL data type: real (float or double) * diff --git a/paddle/cuda/include/hl_cpu_gru.cuh b/paddle/cuda/include/hl_cpu_gru.cuh index cba1c9f30da8d..d39cf67448b4f 100644 --- a/paddle/cuda/include/hl_cpu_gru.cuh +++ b/paddle/cuda/include/hl_cpu_gru.cuh @@ -20,7 +20,7 @@ limitations under the License. */ #include "paddle/math/MathFunctions.h" -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE #define CBLAS_GEMM paddle::gemm #else #define CBLAS_GEMM paddle::gemm diff --git a/paddle/cuda/include/hl_gpu_functions.cuh b/paddle/cuda/include/hl_gpu_functions.cuh index 38df4eb8958f2..a2c5ebd18a440 100644 --- a/paddle/cuda/include/hl_gpu_functions.cuh +++ b/paddle/cuda/include/hl_gpu_functions.cuh @@ -28,7 +28,7 @@ namespace hppl { const real min = SIGMOID_THRESHOLD_MIN; const real max = SIGMOID_THRESHOLD_MAX; real tmp = (a < min) ? min : ((a > max) ? max : a); -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE return __fdividef(1.0f, 1.0f + __expf(-tmp)); #else return 1.0 / (1.0 + exp(-tmp)); @@ -36,7 +36,7 @@ namespace hppl { } __device__ static real tanh(const real a) { -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE return __fdividef(2.0f, (1.0f + __expf(-2.0f*a))) - 1.0f; #else return (2.0 / (1.0 + exp(-2.0*a))) - 1.0; diff --git a/paddle/cuda/include/hl_matrix_base.cuh b/paddle/cuda/include/hl_matrix_base.cuh index 473d394c0c688..a3645ef51e6ef 100644 --- a/paddle/cuda/include/hl_matrix_base.cuh +++ b/paddle/cuda/include/hl_matrix_base.cuh @@ -30,7 +30,7 @@ limitations under the License. */ #define INLINE inline #endif -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE #define DEVICE_FMAX fmaxf #define DEVICE_FMIN fminf #else diff --git a/paddle/cuda/include/hl_matrix_type.cuh b/paddle/cuda/include/hl_matrix_type.cuh index 6917f36290141..51e483d1fb2ff 100644 --- a/paddle/cuda/include/hl_matrix_type.cuh +++ b/paddle/cuda/include/hl_matrix_type.cuh @@ -21,7 +21,7 @@ limitations under the License. */ #ifdef __CUDA_ARCH__ // typedef void* vecType; #include -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE typedef float4 vecType; #else typedef double2 vecType; @@ -30,7 +30,7 @@ typedef double2 vecType; #include #include #include -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE typedef __m128 vecType; #else typedef __m128d vecType; diff --git a/paddle/cuda/include/hl_sse_matrix_kernel.cuh b/paddle/cuda/include/hl_sse_matrix_kernel.cuh index c90d49e4adeb5..45db2f313e0d6 100644 --- a/paddle/cuda/include/hl_sse_matrix_kernel.cuh +++ b/paddle/cuda/include/hl_sse_matrix_kernel.cuh @@ -20,7 +20,7 @@ limitations under the License. */ #define VECTOR_SIZE 16 -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE /* number of float in vector */ #define VECTOR_LEN 4 #define VECTOR_SET _mm_set_ps1 @@ -41,7 +41,7 @@ inline bool hl_check_align(void *ptr) { return hl_check_align(reinterpret_cast(ptr)); } -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE template inline real hl_agg_op(Agg agg, vecType mm) { __m128 lo = _mm_unpacklo_ps(mm, mm); diff --git a/paddle/cuda/src/hl_cuda_cublas.cc b/paddle/cuda/src/hl_cuda_cublas.cc index dc109487ded20..b3c9001ba3973 100644 --- a/paddle/cuda/src/hl_cuda_cublas.cc +++ b/paddle/cuda/src/hl_cuda_cublas.cc @@ -84,7 +84,7 @@ CUBLAS_BLAS_ROUTINE_EACH(DYNAMIC_LOAD_CUBLAS_V2_WRAP) } /* namespace dynload */ -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE #define CUBLAS_GEAM dynload::cublasSgeam #define CUBLAS_GEMV dynload::cublasSgemv #define CUBLAS_GEMM dynload::cublasSgemm diff --git a/paddle/cuda/src/hl_cuda_cudnn.cc b/paddle/cuda/src/hl_cuda_cudnn.cc index c2dce1977bdf5..b215c0f6e33a1 100644 --- a/paddle/cuda/src/hl_cuda_cudnn.cc +++ b/paddle/cuda/src/hl_cuda_cudnn.cc @@ -340,7 +340,7 @@ void hl_create_tensor_descriptor(hl_tensor_descriptor* image_desc, (cudnn_tensor_descriptor)malloc(sizeof(_cudnn_tensor_descriptor)); CHECK_NOTNULL(hl_desc); -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else cudnnDataType_t data_type = CUDNN_DATA_DOUBLE; @@ -373,7 +373,7 @@ void hl_create_tensor_descriptor(hl_tensor_descriptor* image_desc) { (cudnn_tensor_descriptor)malloc(sizeof(_cudnn_tensor_descriptor)); CHECK_NOTNULL(hl_desc); -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else cudnnDataType_t data_type = CUDNN_DATA_DOUBLE; @@ -611,7 +611,7 @@ void hl_create_filter_descriptor(hl_filter_descriptor* filter, CHECK_CUDNN(dynload::cudnnCreateFilterDescriptor(&hl_filter->desc)); -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else cudnnDataType_t data_type = CUDNN_DATA_DOUBLE; @@ -921,7 +921,7 @@ void hl_softmax_forward(real *input, int height, int width) { -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else cudnnDataType_t data_type = CUDNN_DATA_DOUBLE; @@ -955,7 +955,7 @@ void hl_softmax_backward(real *output_value, int height, int width) { -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE cudnnDataType_t data_type = CUDNN_DATA_FLOAT; #else cudnnDataType_t data_type = CUDNN_DATA_DOUBLE; diff --git a/paddle/cuda/src/hl_cuda_device.cc b/paddle/cuda/src/hl_cuda_device.cc index f4c07367b485b..e9fe9f1c117a0 100644 --- a/paddle/cuda/src/hl_cuda_device.cc +++ b/paddle/cuda/src/hl_cuda_device.cc @@ -626,7 +626,7 @@ void hl_specify_devices_start(int* device, int number) { void hl_rand(real *dest_d, size_t num) { pthread_mutex_lock(t_resource.gen_mutex); CHECK_EQ( -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE dynload::curandGenerateUniform(t_resource.gen, dest_d, num), #else dynload::curandGenerateUniformDouble(t_resource.gen, dest_d, num), diff --git a/paddle/cuda/src/hl_cuda_matrix.cu b/paddle/cuda/src/hl_cuda_matrix.cu index 38e4f16217c2a..067e68c41e119 100644 --- a/paddle/cuda/src/hl_cuda_matrix.cu +++ b/paddle/cuda/src/hl_cuda_matrix.cu @@ -47,7 +47,7 @@ void hl_matrix_add(real *A_d, CHECK_SYNC("hl_matrix_add failed"); } -#ifdef HPPL_TYPE_DOUBLE +#ifdef PADDLE_TYPE_DOUBLE #define THRESHOLD 128 #else #define THRESHOLD 64 @@ -102,7 +102,7 @@ void subMaxAndExp(real* I, val = -THRESHOLD; } I[nextIdx] = val; -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE O[nextIdx] = __expf(val); #else O[nextIdx] = exp(val); diff --git a/paddle/cuda/src/hl_cuda_sparse.cuh b/paddle/cuda/src/hl_cuda_sparse.cuh index 13e89390d68c2..c3b98f4ebc38d 100644 --- a/paddle/cuda/src/hl_cuda_sparse.cuh +++ b/paddle/cuda/src/hl_cuda_sparse.cuh @@ -355,7 +355,7 @@ __global__ void KeSMatrixCscMulDense(real *C_d, } /* best perf */ -#ifndef HPPL_TYPE_DOUBLE +#ifndef PADDLE_TYPE_DOUBLE #define CU_CSCMM_THREAD_M_BEST 9 #else #define CU_CSCMM_THREAD_M_BEST 4 From 6d21ecef545a5c400e2e9a3ef812404132fb88ee Mon Sep 17 00:00:00 2001 From: Zrachel Date: Mon, 17 Oct 2016 10:18:30 +0800 Subject: [PATCH 165/324] add cost_type constraint to weighted_cost interface (#206) --- python/paddle/trainer/config_parser.py | 1 - python/paddle/trainer_config_helpers/layers.py | 15 +++++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index fb47fd0c6f0c3..18f0b1b4e497e 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1715,7 +1715,6 @@ def init(cls, name, inputs, device=None, coeff=1.): g_cost_map[cost_type] = cls define_cost('MultiClassCrossEntropy', 'multi-class-cross-entropy') -define_cost('ClassificationErrorLayer', 'classification_error') define_cost('RankingCost', 'rank-cost') define_cost('AucValidation', 'auc-validation') define_cost('PnpairValidation', 'pnpair-validation') diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 745e61b2eb0bc..686704cb7c9b0 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -2799,7 +2799,7 @@ def __cost_input__(input, label, weight=None): @wrap_name_default() -def regression_cost(input, label, weight=None, cost='square_error', name=None): +def regression_cost(input, label, weight=None, name=None): """ Regression Layer. @@ -2814,21 +2814,18 @@ def regression_cost(input, label, weight=None, cost='square_error', name=None): :param weight: The weight affects the cost, namely the scale of cost. It is an optional argument. :type weight: LayerOutput - :param cost: Cost method. - :type cost: basestring :return: LayerOutput object. :rtype: LayerOutput """ ipts, parents = __cost_input__(input, label, weight) - Layer(inputs=ipts, type=cost, name=name) + Layer(inputs=ipts, type="square_error", name=name) return LayerOutput(name, LayerType.COST, parents=parents) @wrap_name_default("cost") @layer_support() def classification_cost(input, label, weight=None, name=None, - cost="multi-class-cross-entropy", evaluator=classification_error_evaluator, layer_attr=None): """ @@ -2843,8 +2840,6 @@ def classification_cost(input, label, weight=None, name=None, :param weight: The weight affects the cost, namely the scale of cost. It is an optional argument. :type weight: LayerOutput - :param cost: cost method. - :type cost: basestring :param evaluator: Evaluator method. :param layer_attr: layer's extra attribute. :type layer_attr: ExtraLayerAttribute @@ -2857,7 +2852,7 @@ def classification_cost(input, label, weight=None, name=None, ipts, parents = __cost_input__(input, label, weight) - Layer(name=name, type=cost, inputs=ipts, + Layer(name=name, type="multi-class-cross-entropy", inputs=ipts, **ExtraLayerAttribute.to_kwargs(layer_attr)) def __add_evaluator__(e): @@ -3819,8 +3814,8 @@ def multi_binary_label_cross_entropy(input, label, name=None, coeff=1.0): if input.activation is None or \ not isinstance(input.activation, SigmoidActivation): logger.log(logging.WARN, - "%s is not recommend for batch normalization's activation, " - "maybe the relu is better" % repr(input.activation)) + "%s is not recommend for multi_binary_label_cross_entropy's activation, " + "maybe the sigmoid is better" % repr(input.activation)) Layer(name=name, type=LayerType.MULTI_BIN_LABEL_CROSS_ENTROPY, From 4e43a59a183cb09682dcf6ed1506d6210c06a7b7 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Mon, 17 Oct 2016 11:03:58 +0800 Subject: [PATCH 166/324] remove unmerged internal documents (#205) --- paddle/internals/scripts/docs.sh | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100755 paddle/internals/scripts/docs.sh diff --git a/paddle/internals/scripts/docs.sh b/paddle/internals/scripts/docs.sh deleted file mode 100755 index 517405c120a47..0000000000000 --- a/paddle/internals/scripts/docs.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Copyright (c) 2016 Baidu, Inc. All Rights Reserved - -cd `dirname $0` - -# Add set -e, cd to directory. -set -e -mkdir -p $PWD/../../../build -cd $PWD/../../../build - -# Compile Documentation only. -cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=ON -make paddle_docs paddle_docs_cn - -# remove old docs. mv new docs in deeplearning.baidu.com -scp -r doc/html paddle_doc@yq01-idl-gpu-offline42.yq01.baidu.com:/home/paddle_doc/www/doc_new -ssh paddle_doc@yq01-idl-gpu-offline42.yq01.baidu.com "cd ~/www/ && rm -r doc && mv doc_new doc" - -scp -r doc_cn/html paddle_doc@yq01-idl-gpu-offline42.yq01.baidu.com:/home/paddle_doc/www/doc_cn_new -ssh paddle_doc@yq01-idl-gpu-offline42.yq01.baidu.com "cd ~/www/ && rm -r doc_cn && mv doc_cn_new doc_cn" From e4952ca6cea9130258086b4b7f2dff7a8bf5f566 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 17 Oct 2016 03:30:10 +0000 Subject: [PATCH 167/324] Add FAQ (#128) * Init commit for doing FAQ * Add speed up training * Add graphviz to ci * Add shared paramter * Tiny refine --- .travis.yml | 1 + doc_cn/conf.py.in | 1 + doc_cn/faq/index.rst | 169 ++++++++++++++++++ doc_cn/faq/reduce_min_pool_size.py | 6 + doc_cn/faq/word2vec_config.py | 8 + doc_cn/faq/word2vec_dataprovider.py | 8 + doc_cn/index.rst | 7 + .../paddle/trainer_config_helpers/networks.py | 4 +- .../tests/configs/check.md5 | 2 + .../tests/configs/generate_protostr.sh | 2 +- .../tests/configs/shared_fc.py | 22 +++ .../tests/configs/shared_lstm.py | 29 +++ 12 files changed, 256 insertions(+), 3 deletions(-) create mode 100644 doc_cn/faq/index.rst create mode 100644 doc_cn/faq/reduce_min_pool_size.py create mode 100644 doc_cn/faq/word2vec_config.py create mode 100644 doc_cn/faq/word2vec_dataprovider.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/shared_fc.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py diff --git a/.travis.yml b/.travis.yml index 119d01a4fa8fd..bf0e0b7bbddd4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,7 @@ addons: - libgoogle-glog-dev - libgflags-dev - libgtest-dev + - graphviz before_install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi diff --git a/doc_cn/conf.py.in b/doc_cn/conf.py.in index 391f7981eab80..93242ace40600 100644 --- a/doc_cn/conf.py.in +++ b/doc_cn/conf.py.in @@ -47,6 +47,7 @@ extensions = [ 'sphinx.ext.autosummary', 'sphinx.ext.mathjax', 'sphinx.ext.napoleon', + 'sphinx.ext.graphviz' ] table_styling_embed_css = True diff --git a/doc_cn/faq/index.rst b/doc_cn/faq/index.rst new file mode 100644 index 0000000000000..283607957ce63 --- /dev/null +++ b/doc_cn/faq/index.rst @@ -0,0 +1,169 @@ +#################### +PaddlePaddle常见问题 +#################### + +.. contents:: + +1. 如何减少PaddlePaddle的内存占用 +--------------------------------- + +神经网络的训练本身是一个非常消耗内存和显存的工作。经常会消耗数十G的内存和数G的显存。 +PaddlePaddle的内存占用主要分为如下几个方面\: + +* DataProvider缓冲池内存 (只针对内存) +* 神经元激活内存 （针对内存和显存） +* 参数内存 (针对内存和显存) +* 其他内存杂项 + +这其中，其他内存杂项是指PaddlePaddle本身所用的一些内存，包括字符串分配，临时变量等等， +这些内存就不考虑如何缩减了。 + +其他的内存的减少方法依次为 + + +减少DataProvider缓冲池内存 +++++++++++++++++++++++++++ + +PyDataProvider使用的是异步加载，同时在内存里直接随即选取数据来做Shuffle。即 + +.. graphviz:: + + digraph { + rankdir=LR; + 数据文件 -> 内存池 -> PaddlePaddle训练 + } + +所以，减小这个内存池即可减小内存占用，同时也可以加速开始训练前数据载入的过程。但是，这 +个内存池实际上决定了shuffle的粒度。所以，如果将这个内存池减小，又要保证数据是随机的， +那么最好将数据文件在每次读取之前做一次shuffle。可能的代码为 + +.. literalinclude:: reduce_min_pool_size.py + +这样做可以极大的减少内存占用，并且可能会加速训练过程。 详细文档参考 `这里 +<../ui/data_provider/pydataprovider2.html#provider>`_ 。 + +神经元激活内存 +++++++++++++++ + +神经网络在训练的时候，会对每一个激活暂存一些数据，包括激活，參差等等。 +在反向传递的时候，这些数据会被用来更新参数。这些数据使用的内存主要和两个参数有关系， +一是batch size，另一个是每条序列(Sequence)长度。所以，其实也是和每个mini-batch中包含 +的时间步信息成正比。 + +所以，做法可以有两种。他们是 + +* 减小batch size。 即在网络配置中 :code:`settings(batch_size=1000)` 设置成一个小一些的值。但是batch size本身是神经网络的超参数，减小batch size可能会对训练结果产生影响。 +* 减小序列的长度，或者直接扔掉非常长的序列。比如，一个数据集大部分序列长度是100-200, + 但是突然有一个10000长的序列，就很容易导致内存超限。特别是在LSTM等RNN中。 + +参数内存 +++++++++ + +PaddlePaddle支持非常多的优化算法(Optimizer)，不同的优化算法需要使用不同大小的内存。 +例如如果使用 :code:`adadelta` 算法，则需要使用参数规模大约5倍的内存。 如果参数保存下来的 +文件为 :code:`100M`， 那么该优化算法至少需要 :code:`500M` 的内存。 + +可以考虑使用一些优化算法，例如 :code:`momentum`。 + +2. 如何加速PaddlePaddle的训练速度 +--------------------------------- + +PaddlePaddle是神经网络训练平台，加速PaddlePaddle训练有如下几个方面\： + +* 减少数据载入的耗时 +* 加速训练速度 +* 利用更多的计算资源 + +减少数据载入的耗时 +++++++++++++++++++ + +使用 :code:`pydataprovider`时，可以减少缓存池的大小，同时设置内存缓存功能，即可以极大的加速数据载入流程。 +:code:`DataProvider` 缓存池的减小，和之前减小通过减小缓存池来减小内存占用的原理一致。 + +.. literalinclude:: reduce_min_pool_size.py + +同时 :code:`@provider` 接口有一个 :code:`cache` 参数来控制缓存方法，将其设置成 :code:`CacheType.CACHE_PASS_IN_MEM` 的话，会将第一个 :code:`pass` (过完所有训练数据即为一个pass)生成的数据缓存在内存里，在之后的 :code:`pass` 中，不会再从 :code:`python` 端读取数据，而是直接从内存的缓存里读取数据。这也会极大减少数据读入的耗时。 + + +加速训练速度 +++++++++++++ + +PaddlePaddle支持Sparse的训练，sparse训练需要训练特征是 :code:`sparse_binary_vector` 、 :code:`sparse_vector` 、或者 :code:`integer_value` 的任一一种。同时，与这个训练数据交互的Layer，需要将其Parameter设置成 sparse 更新模式，即设置 :code:`sparse_update=True` + +这里使用简单的 :code:`word2vec` 训练语言模型距离，具体使用方法为\: + +使用一个词前两个词和后两个词，来预测这个中间的词。这个任务的DataProvider为\: + +.. literalinclude:: word2vec_dataprovider.py + +这个任务的配置为\: + +.. literalinclude:: word2vec_config.py + +更多关于sparse训练的内容请参考 `sparse训练的文档 `_ + +利用更多的计算资源 +++++++++++++++++++ + +利用更多的计算资源可以分为一下几个方式来进行\: + +* 单机CPU训练 + * 使用多线程训练。设置命令行参数 :code:`trainer_count`，即可以设置参与训练的线程数量。使用方法为 :code:`paddle train --trainer_count=4` +* 单机GPU训练 + * 使用显卡训练。设置命令行参数 :code:`use_gpu`。 使用方法为 :code:`paddle train --use_gpu=true` + * 使用多块显卡训练。设置命令行参数 :code:`use_gpu` 和 :code:`trainer_count`。使用 :code:`--use_gpu=True` 开启GPU训练，使用 :code:`trainer_count` 指定显卡数量。使用方法为 :code:`paddle train --use_gpu=true --trainer_count=4` +* 多机训练 + * 使用多机训练的方法也比较简单，需要先在每个节点启动 :code:`paddle pserver`，在使用 :code:`paddle train --pservers=192.168.100.1,192.168.100.2` 来指定每个pserver的ip地址 + * 具体的多机训练方法参考 `多机训练 `_ 文档。 + + +3. 遇到“非法指令”或者是“illegal instruction” +-------------------------------------------- + +paddle在进行计算的时候为了提升计算性能，使用了avx指令。部分老的cpu型号无法支持这样的指令。通常来说执行下grep avx /proc/cpuinfo看看是否有输出即可知道是否支持。（另：用此方法部分虚拟机可能检测到支持avx指令但是实际运行会挂掉，请当成是不支持，看下面的解决方案） + +解决办法是\: + +* 使用 NO_AVX的 `安装包 <../build_and_install/index.html>`_ 或者 `Docker image <../build_and_install/install/docker_install.html>`_ +* 或者，使用 :code:`-DWITH_AVX=OFF` 重新编译PaddlePaddle。 + + +4. 如何选择SGD算法的学习率 +-------------------------- + +在采用sgd/async_sgd进行训练时，一个重要的问题是选择正确的learning_rate。如果learning_rate太大，那么训练有可能不收敛，如果learning_rate太小，那么收敛可能很慢，导致训练时间过长。 + +通常做法是从一个比较大的learning_rate开始试，如果不收敛，那减少学习率10倍继续试验，直到训练收敛为止。那么如何判断训练不收敛呢？可以估计出如果模型采用不变的输出最小的cost0是多少。 + +如果训练过程的的cost明显高于这个常数输出的cost，那么我们可以判断为训练不收敛。举一个例子，假如我们是三分类问题，采用multi-class-cross-entropy作为cost，数据中0,1,2三类的比例为 :code:`0.2, 0.5, 0.3` , 那么常数输出所能达到的最小cost是 :code:`-(0.2*log(0.2)+0.5*log(0.5)+0.3*log(0.3))=1.03` 。如果训练一个pass（或者更早）后，cost还大于这个数，那么可以认为训练不收敛，应该降低学习率。 + + +5. 如何初始化参数 +----------------- + +默认情况下，PaddlePaddle使用均值0，标准差为 :math:`\frac{1}{\sqrt{d}}` 来初始化参数。其中 :math:`d` 为参数矩阵的宽度。这种初始化方式在一般情况下不会产生很差的结果。如果用户想要自定义初始化方式，PaddlePaddle目前提供两种参数初始化的方式\: + +* 高斯分布。将 :code:`param_attr` 设置成 :code:`param_attr=ParamAttr(initial_mean=0.0, initial_std=1.0)` +* 均匀分布。将 :code:`param_attr` 设置成 :code:`param_attr=ParamAttr(initial_max=1.0, initial_min=-1.0)` + +比如设置一个全连接层的参数初始化方式和bias初始化方式，可以使用如下代码。 + +.. code-block:: python + + hidden = fc_layer(input=ipt, param_attr=ParamAttr(initial_max=1.0, initial_min=-1.0), + bias_attr=ParamAttr(initial_mean=1.0, initial_std=0.0)) + +上述代码将bias全部初始化为1.0, 同时将参数初始化为 :code:`[1.0, -1.0]` 的均匀分布。 + +6. 如何共享参数 +--------------- + +PaddlePaddle的参数使用名字 :code:`name` 作为参数的ID，相同名字的参数，会共享参数。设置参数的名字，可以使用 :code:`ParamAttr(name="YOUR_PARAM_NAME")` 来设置。更方便的设置方式，是想要共享的参数使用同样的 :code:`ParamAttr` 对象。 + +简单的全连接网络，参数共享的配置示例为\: + +.. literalinclude:: ../../python/paddle/trainer_config_helpers/tests/configs/shared_fc.py + +这里 :code:`hidden_a` 和 :code:`hidden_b` 使用了同样的parameter和bias。并且softmax层的两个输入也使用了同样的参数 :code:`softmax_param`。 + + diff --git a/doc_cn/faq/reduce_min_pool_size.py b/doc_cn/faq/reduce_min_pool_size.py new file mode 100644 index 0000000000000..2811b134b66b1 --- /dev/null +++ b/doc_cn/faq/reduce_min_pool_size.py @@ -0,0 +1,6 @@ +@provider(min_pool_size=0, ...) +def process(settings, filename): + os.system('shuf %s > %s.shuf' % (filename, filename)) # shuffle before. + with open('%s.shuf' % filename, 'r') as f: + for line in f: + yield get_sample_from_line(line) \ No newline at end of file diff --git a/doc_cn/faq/word2vec_config.py b/doc_cn/faq/word2vec_config.py new file mode 100644 index 0000000000000..e347252476eab --- /dev/null +++ b/doc_cn/faq/word2vec_config.py @@ -0,0 +1,8 @@ +... # the settings and define data provider is omitted. +DICT_DIM=3000 # dictionary dimension. +word_ids=data_layer('word_ids', size=DICT_DIM) + +emb = embedding_layer(input=word_ids, size=256, param_attr=ParamAttr(sparse_update=True)) +emb_sum = pooling_layer(input=emb, pooling_type=SumPooling()) +predict = fc_layer(input=emb_sum, size=DICT_DIM, act=Softmax()) +outputs(classification_cost(input=predict, label=data_layer('label', size=DICT_DIM))) \ No newline at end of file diff --git a/doc_cn/faq/word2vec_dataprovider.py b/doc_cn/faq/word2vec_dataprovider.py new file mode 100644 index 0000000000000..a0a39080cece9 --- /dev/null +++ b/doc_cn/faq/word2vec_dataprovider.py @@ -0,0 +1,8 @@ +DICT_DIM=3000 +@provider(input_types=[integer_sequence(DICT_DIM), integer_value(DICT_DIM)]) +def process(settings, filename): + with open(filename) as f: + # yield word ids to predict inner word id + # such as [28, 29, 10, 4], 4 + # It means the sentance is 28, 29, 4, 10, 4. + yield read_next_from_file(f) \ No newline at end of file diff --git a/doc_cn/index.rst b/doc_cn/index.rst index 1a4908be14684..d2d50fbdb47f2 100644 --- a/doc_cn/index.rst +++ b/doc_cn/index.rst @@ -3,6 +3,7 @@ PaddlePaddle文档 使用指南 -------- + * `快速入门 `_ * `编译与安装 `_ * `用户接口 `_ @@ -16,7 +17,13 @@ PaddlePaddle文档 算法教程 -------- + * `Recurrent Group教程 `_ * `单层RNN示例 <../doc/algorithm/rnn/rnn.html>`_ * `双层RNN示例 `_ * `支持双层序列作为输入的Layer `_ + +常见问题 +-------- + +* `常见问题 `_ diff --git a/python/paddle/trainer_config_helpers/networks.py b/python/paddle/trainer_config_helpers/networks.py index ab4057d9d6c6b..c54ec3096989c 100644 --- a/python/paddle/trainer_config_helpers/networks.py +++ b/python/paddle/trainer_config_helpers/networks.py @@ -20,7 +20,7 @@ IdentityActivation, TanhActivation, SequenceSoftmaxActivation from attrs import ExtraAttr from default_decorators import wrap_name_default, wrap_act_default, \ - wrap_param_default + wrap_param_default, wrap_bias_attr_default, wrap_param_attr_default from layers import * # There are too many layers used in network, so import * from poolings import MaxPooling, SumPooling from paddle.trainer.config_parser import * @@ -505,7 +505,7 @@ def simple_lstm(input, size, name=None, reverse=False, mat_param_attr=None, def lstmemory_unit(input, name=None, size=None, param_attr=None, act=None, gate_act=None, state_act=None, mixed_bias_attr=None, lstm_bias_attr=None, - mixed_layer_attr=None,lstm_layer_attr=None, + mixed_layer_attr=None, lstm_layer_attr=None, get_output_layer_attr=None): """ Define calculations that a LSTM unit performs in a single time step. diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 index 3ecfff2071630..96bf3fb2e19d6 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/check.md5 +++ b/python/paddle/trainer_config_helpers/tests/configs/check.md5 @@ -2,6 +2,8 @@ a5d9259ff1fd7ca23d0ef090052cb1f2 last_first_seq.protostr 9c038249ec8ff719753a746cdb04c026 layer_activations.protostr 5913f87b39cee3b2701fa158270aca26 projections.protostr +7334ba0a4544f0623231330fc51d390d shared_fc.protostr +8b8b6bb128a7dfcc937be86145f53e2f shared_lstm.protostr 6b39e34beea8dfb782bee9bd3dea9eb5 simple_rnn_layers.protostr 0fc1409600f1a3301da994ab9d28b0bf test_cost_layers.protostr 6cd5f28a3416344f20120698470e0a4c test_cost_layers_with_weight.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index 5514ee65e5a62..7cdd682056fd4 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -8,7 +8,7 @@ configs=(test_fc layer_activations projections test_print_layer test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers util_layers simple_rnn_layers unused_layers test_cost_layers -test_cost_layers_with_weight test_rnn_group) +test_rnn_group shared_fc shared_lstm test_cost_layers_with_weight) for conf in ${configs[*]} diff --git a/python/paddle/trainer_config_helpers/tests/configs/shared_fc.py b/python/paddle/trainer_config_helpers/tests/configs/shared_fc.py new file mode 100644 index 0000000000000..202cf367fc7f2 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/shared_fc.py @@ -0,0 +1,22 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-4, + batch_size=1000 +) + +a = data_layer(name='feature_a', size=200) +b = data_layer(name='feature_b', size=200) + +fc_param = ParamAttr(name='fc_param', initial_max=1.0, initial_min=-1.0) +bias_param = ParamAttr(name='bias_param', initial_mean=0.0, initial_std=0.0) + +softmax_param = ParamAttr(name='softmax_param', initial_max=1.0, initial_min=-1.0) + +hidden_a = fc_layer(input=a, size=200, param_attr=fc_param, bias_attr=bias_param) +hidden_b = fc_layer(input=b, size=200, param_attr=fc_param, bias_attr=bias_param) + +predict = fc_layer(input=[hidden_a, hidden_b], param_attr=[softmax_param, softmax_param], + bias_attr=False, size=10, act=SoftmaxActivation()) + +outputs(classification_cost(input=predict, label=data_layer(name='label', size=10))) diff --git a/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py b/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py new file mode 100644 index 0000000000000..8557e9daaf66a --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.py @@ -0,0 +1,29 @@ +from paddle.trainer_config_helpers import * + +settings(learning_rate=1e-4, batch_size=1000) + +data_1 = data_layer(name='data_a', size=100) +data_2 = data_layer(name='data_b', size=100) + +mixed_param = ParamAttr(name='mixed_param') + +with mixed_layer(size=400, bias_attr=False) as m1: + m1 += full_matrix_projection(input=data_1, param_attr=mixed_param) + +with mixed_layer(size=400, bias_attr=False) as m2: + m2 += full_matrix_projection(input=data_2, param_attr=mixed_param) + +lstm_param = ParamAttr(name='lstm_param') +lstm_bias = ParamAttr(name='lstm_bias', initial_mean=0., initial_std=0.) + +lstm1 = lstmemory_group(input=m1, param_attr=lstm_param, lstm_bias_attr=lstm_bias, mixed_bias_attr=False) +lstm2 = lstmemory_group(input=m2, param_attr=lstm_param, lstm_bias_attr=lstm_bias, mixed_bias_attr=False) + +softmax_param = ParamAttr(name='softmax_param') + +predict = fc_layer(input=[last_seq(input=lstm1), last_seq(input=lstm2)], + size=10, + param_attr=[softmax_param, softmax_param], + bias_attr=False, + act=SoftmaxActivation()) +outputs(classification_cost(input=predict, label=data_layer(name='label', size=10))) From 2f82d72ede17822f52a789e92afca6f8112bc44e Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 17 Oct 2016 07:17:08 +0000 Subject: [PATCH 168/324] Fix bug in yield dictionary in DataProvider. (#197) * Fix bug in yield dictionary in DataProvider. * Also make virtualenv work in Paddle. --- CMakeLists.txt | 2 +- cmake/util.cmake | 17 ++++++++ demo/mnist/data/get_mnist_data.sh | 0 demo/mnist/mnist_provider.py | 19 ++++----- demo/mnist/vgg_16_mnist.py | 1 + .../ui/data_provider/mnist_provider.dict.py | 10 ++--- doc_cn/ui/data_provider/pydataprovider2.rst | 2 - .../gserver/dataproviders/PyDataProvider2.cpp | 3 +- paddle/gserver/tests/test_PyDataProvider2.cpp | 2 +- paddle/gserver/tests/test_PyDataProvider2.py | 2 +- paddle/utils/.gitignore | 1 + paddle/utils/CMakeLists.txt | 6 ++- paddle/utils/PythonUtil.cpp | 31 ++++++++++---- paddle/utils/PythonUtil.h | 2 + paddle/utils/enable_virtualenv.py | 10 +++++ python/paddle/trainer/PyDataProvider2.py | 24 ++++++----- python/paddle/trainer/config_parser.py | 4 ++ .../paddle/trainer_config_helpers/networks.py | 42 ++++++++++++++----- 18 files changed, 126 insertions(+), 52 deletions(-) mode change 100644 => 100755 demo/mnist/data/get_mnist_data.sh create mode 100644 paddle/utils/.gitignore create mode 100644 paddle/utils/enable_virtualenv.py diff --git a/CMakeLists.txt b/CMakeLists.txt index b85709f807bc3..4613155f7700b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 2.8) project(paddle CXX C) set(PADDLE_MAJOR_VERSION 0) set(PADDLE_MINOR_VERSION 8) -set(PADDLE_PATCH_VERSION 0b1) +set(PADDLE_PATCH_VERSION 0b2) set(PADDLE_VERSION ${PADDLE_MAJOR_VERSION}.${PADDLE_MINOR_VERSION}.${PADDLE_PATCH_VERSION}) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") diff --git a/cmake/util.cmake b/cmake/util.cmake index d776c3ae49952..0fa36f070cc11 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -184,3 +184,20 @@ macro(add_paddle_culib TARGET_NAME) cuda_add_library(${TARGET_NAME} STATIC ${ARGN}) set(CUDA_NVCC_FLAGS ${NVCC_FLAG}) endmacro() + + +# Creates C resources file from files in given resource file +function(create_resources res_file output) + # Create empty output file + file(WRITE ${output} "") + # Get short filename + string(REGEX MATCH "([^/]+)$" filename ${res_file}) + # Replace filename spaces & extension separator for C compatibility + string(REGEX REPLACE "\\.| |-" "_" filename ${filename}) + # Read hex data from file + file(READ ${res_file} filedata HEX) + # Convert hex data for C compatibility + string(REGEX REPLACE "([0-9a-f][0-9a-f])" "0x\\1," filedata ${filedata}) + # Append data to output file + file(APPEND ${output} "const unsigned char ${filename}[] = {${filedata}};\nconst unsigned ${filename}_size = sizeof(${filename});\n") +endfunction() diff --git a/demo/mnist/data/get_mnist_data.sh b/demo/mnist/data/get_mnist_data.sh old mode 100644 new mode 100755 diff --git a/demo/mnist/mnist_provider.py b/demo/mnist/mnist_provider.py index 0f14ded2dce93..32af29730a736 100644 --- a/demo/mnist/mnist_provider.py +++ b/demo/mnist/mnist_provider.py @@ -2,10 +2,10 @@ # Define a py data provider -@provider(input_types=[ - dense_vector(28 * 28), - integer_value(10) -]) +@provider(input_types={ + 'pixel': dense_vector(28 * 28), + 'label': integer_value(10) +}) def process(settings, filename): # settings is not used currently. imgf = filename + "-images-idx3-ubyte" labelf = filename + "-labels-idx1-ubyte" @@ -14,20 +14,19 @@ def process(settings, filename): # settings is not used currently. f.read(16) l.read(8) - + # Define number of samples for train/test if "train" in filename: n = 60000 else: n = 10000 - + for i in range(n): label = ord(l.read(1)) pixels = [] - for j in range(28*28): + for j in range(28 * 28): pixels.append(float(ord(f.read(1))) / 255.0) - yield { "pixel": pixels, 'label': label } - + yield {"pixel": pixels, 'label': label} + f.close() l.close() - \ No newline at end of file diff --git a/demo/mnist/vgg_16_mnist.py b/demo/mnist/vgg_16_mnist.py index ad0a4de3215ca..45a45bb061aa7 100644 --- a/demo/mnist/vgg_16_mnist.py +++ b/demo/mnist/vgg_16_mnist.py @@ -47,6 +47,7 @@ if not is_predict: lbl = data_layer(name="label", size=label_size) + inputs(img, lbl) outputs(classification_cost(input=predict, label=lbl)) else: outputs(predict) diff --git a/doc_cn/ui/data_provider/mnist_provider.dict.py b/doc_cn/ui/data_provider/mnist_provider.dict.py index 4eab5b1fd3b50..bf13b56372b56 100644 --- a/doc_cn/ui/data_provider/mnist_provider.dict.py +++ b/doc_cn/ui/data_provider/mnist_provider.dict.py @@ -2,10 +2,10 @@ # Define a py data provider -@provider(input_types=[ - dense_vector(28 * 28), - integer_value(10) -]) +@provider(input_types={ + 'pixel': dense_vector(28 * 28), + 'label': integer_value(10) +}) def process(settings, filename): # settings is not used currently. f = open(filename, 'r') # open one of training file @@ -20,6 +20,6 @@ def process(settings, filename): # settings is not used currently. pixels_float.append(float(each_pixel_str)) # give data to paddle. - yield { "pixel": pixels_float, 'label': int(label) } + yield {"pixel": pixels_float, 'label': int(label)} f.close() # close file diff --git a/doc_cn/ui/data_provider/pydataprovider2.rst b/doc_cn/ui/data_provider/pydataprovider2.rst index 9e1d8c531f5ba..80b40084d8f50 100644 --- a/doc_cn/ui/data_provider/pydataprovider2.rst +++ b/doc_cn/ui/data_provider/pydataprovider2.rst @@ -141,8 +141,6 @@ DataProvider创建的时候执行。这个初始化函数具有如下参数: 是一个batch size，但是有时为了计算均衡性，可以将一条数据设置成多个batch size * cache 是数据缓存的策略，参考 `cache`_ * init_hook 是初始化时调用的函数，参考 `init_hook`_ -* use_dynamic_order 如果是true的话，可以返回一个dict，key是data_layer的名字，value是特征值。同时，也可以 - 返回一个list或者tuple。如果是false的话，只能够返回list或者tuple * check 设置成true的话，会根据input_types检查数据的合法性。 * check_fail_continue 如果设置成true的话，即使在check中数据不合法，也会扔到这条数据，继续训练。 如果 check是false的话，没有作用。 diff --git a/paddle/gserver/dataproviders/PyDataProvider2.cpp b/paddle/gserver/dataproviders/PyDataProvider2.cpp index 2f9a1223c6e45..e3e472ac166c2 100644 --- a/paddle/gserver/dataproviders/PyDataProvider2.cpp +++ b/paddle/gserver/dataproviders/PyDataProvider2.cpp @@ -246,8 +246,7 @@ class PyDataProvider2 : public DataProvider { PyObjectPtr && kwargs) { LOG(INFO) << "loading dataprovider " << model <<"::" << className; - PyObjectPtr module(PyImport_ImportModule(model.c_str())); - CHECK_PY(module) << "Cannot imort module " << model.c_str(); + PyObjectPtr module = py::import(model); PyObjectPtr moduleDict(PyModule_GetDict(module.get())); CHECK_PY(moduleDict) << "Invoke module.__dict__ error"; PyObjectPtr cls(PyDict_GetItemString(moduleDict.get(), diff --git a/paddle/gserver/tests/test_PyDataProvider2.cpp b/paddle/gserver/tests/test_PyDataProvider2.cpp index e75e53ab7f431..6bf1e32925121 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.cpp +++ b/paddle/gserver/tests/test_PyDataProvider2.cpp @@ -117,7 +117,7 @@ TEST(PyDataProvider2, index_no_seq) { } TEST(PyDataProvider2, init_hook) { - paddle::PyObjectPtr pickle(PyImport_ImportModule("pickle")); + paddle::PyObjectPtr pickle = paddle::py::import("pickle"); paddle::PyObjectPtr globals( PyModule_GetDict(PyImport_AddModule("__main__"))); PyDict_SetItemString(globals.get(), "pickle", pickle.get()); diff --git a/paddle/gserver/tests/test_PyDataProvider2.py b/paddle/gserver/tests/test_PyDataProvider2.py index 145fe85cff7d8..71c3335231e52 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.py +++ b/paddle/gserver/tests/test_PyDataProvider2.py @@ -86,7 +86,7 @@ def test_can_over_batch_size(setting, filename): yield [random.randint(0, 100 - 1) for _ in xrange(seq_len)] -@provider(input_types=[index_slot(10), index_slot(10)]) +@provider(input_types={'input1':index_slot(10), 'input2': index_slot(10)}) def test_input_order(setting, filename): for _ in xrange(1000): yield { diff --git a/paddle/utils/.gitignore b/paddle/utils/.gitignore new file mode 100644 index 0000000000000..f2cfd7409412d --- /dev/null +++ b/paddle/utils/.gitignore @@ -0,0 +1 @@ +enable_virtualenv.c diff --git a/paddle/utils/CMakeLists.txt b/paddle/utils/CMakeLists.txt index 0557b01e36f07..45240b5002aa1 100644 --- a/paddle/utils/CMakeLists.txt +++ b/paddle/utils/CMakeLists.txt @@ -2,6 +2,9 @@ file(GLOB UTIL_HEADERS . *.h) file(GLOB UTIL_SOURCES . *.cpp) +create_resources(enable_virtualenv.py enable_virtualenv.c) +set(UTIL_RES enable_virtualenv.c) + if(APPLE) file(GLOB UTIL_ARCH_SOURCES . arch/osx/*.cpp) else() @@ -9,7 +12,8 @@ else() endif() add_library(paddle_utils STATIC ${UTIL_SOURCES} - ${UTIL_ARCH_SOURCES}) + ${UTIL_ARCH_SOURCES} + ${UTIL_RES}) add_style_check_target(paddle_utils ${UTIL_HEADERS}) add_style_check_target(paddle_utils ${UTIL_SOURCES} ${UTIL_ARCH_SOURCES}) diff --git a/paddle/utils/PythonUtil.cpp b/paddle/utils/PythonUtil.cpp index 78c3a80674f9c..90e5093f96ea4 100644 --- a/paddle/utils/PythonUtil.cpp +++ b/paddle/utils/PythonUtil.cpp @@ -77,11 +77,18 @@ static std::recursive_mutex g_pyMutex; PyGuard::PyGuard() : guard_(g_pyMutex) {} -static void printPyErrorStack(std::ostream& os, bool withEndl = false) { +static void printPyErrorStack(std::ostream& os, bool withEndl = false, + bool withPyPath = true) { PyObject * ptype, *pvalue, *ptraceback; PyErr_Fetch(&ptype, &pvalue, &ptraceback); PyErr_NormalizeException(&ptype, &pvalue, &ptraceback); PyErr_Clear(); + if (withPyPath) { + os << "Current PYTHONPATH: " << py::repr(PySys_GetObject(strdup("path"))); + if (withEndl) { + os << std::endl; + } + } PyTracebackObject* obj = (PyTracebackObject*)ptraceback; os << "Python Error: " << PyString_AsString(PyObject_Str(ptype)) @@ -114,10 +121,7 @@ PyObjectPtr callPythonFuncRetPyObj(const std::string& moduleName, const std::string& funcName, const std::vector& args) { PyGuard guard; - PyObjectPtr pyModuleName(PyString_FromString(moduleName.c_str())); - CHECK_PY(pyModuleName) << "Import PyModule failed" << moduleName; - PyObjectPtr pyModule(PyImport_Import(pyModuleName.get())); - CHECK_PY(pyModule) << "Import Python Module"<< moduleName << " failed."; + PyObjectPtr pyModule = py::import(moduleName); PyObjectPtr pyFunc(PyObject_GetAttrString(pyModule.get(), funcName.c_str())); CHECK_PY(pyFunc) << "GetAttrString failed."; PyObjectPtr pyArgs(PyTuple_New(args.size())); @@ -143,7 +147,7 @@ PyObjectPtr createPythonClass( const std::vector& args, const std::map& kwargs) { PyGuard guard; - PyObjectPtr pyModule(PyImport_ImportModule(moduleName.c_str())); + PyObjectPtr pyModule = py::import(moduleName); LOG(INFO) << "createPythonClass moduleName.c_str:" << moduleName.c_str(); CHECK_PY(pyModule) << "Import module " << moduleName << " failed."; PyObjectPtr pyDict(PyModule_GetDict(pyModule.get())); @@ -181,18 +185,29 @@ std::string getPyCallStack() { printPyErrorStack(os, true); return os.str(); } + +PyObjectPtr import(const std::string &moduleName) { + auto module = PyImport_ImportModule(moduleName.c_str()); + CHECK_PY(module) << "Import " << moduleName << "Error"; + return PyObjectPtr(module); +} + } // namespace py #endif - +extern "C" { +extern const char enable_virtualenv_py[]; +} void initPython(int argc, char** argv) { #ifndef PADDLE_NO_PYTHON Py_SetProgramName(argv[0]); Py_Initialize(); PySys_SetArgv(argc, argv); - // python blocks SIGINT. Need to enable it. signal(SIGINT, SIG_DFL); + + // Manually activate virtualenv when user is using virtualenv + PyRun_SimpleString(enable_virtualenv_py); #endif } diff --git a/paddle/utils/PythonUtil.h b/paddle/utils/PythonUtil.h index db02d1252b405..00fc177022ac3 100644 --- a/paddle/utils/PythonUtil.h +++ b/paddle/utils/PythonUtil.h @@ -87,6 +87,8 @@ PyObjectPtr createPythonClass(const std::string& moduleName, CHECK((x) != nullptr) << ::paddle::py::getPyCallStack() namespace py { +PyObjectPtr import(const std::string& moduleName); + /** * Cast a PyLong or PyInt to int type T. * @tparam T return type. diff --git a/paddle/utils/enable_virtualenv.py b/paddle/utils/enable_virtualenv.py new file mode 100644 index 0000000000000..99d822a4145cc --- /dev/null +++ b/paddle/utils/enable_virtualenv.py @@ -0,0 +1,10 @@ +import os + +def __activate_virtual_env__(): + __path__ = os.getenv('VIRTUAL_ENV') + if __path__ is None: + return + __script__ = os.path.join(__path__, 'bin', 'activate_this.py') + execfile(__script__, {'__file__': __script__}) + +__activate_virtual_env__() diff --git a/python/paddle/trainer/PyDataProvider2.py b/python/paddle/trainer/PyDataProvider2.py index 34f5dd41b7e68..53409b746d811 100644 --- a/python/paddle/trainer/PyDataProvider2.py +++ b/python/paddle/trainer/PyDataProvider2.py @@ -208,7 +208,6 @@ def provider(input_types=None, should_shuffle=None, pool_size=-1, calc_batch_size=None, cache=CacheType.NO_CACHE, check=False, check_fail_continue=False, - use_dynamic_order=True, init_hook=None, **kwargs): """ Provider decorator. Use it to make a function into PyDataProvider2 object. @@ -228,9 +227,15 @@ def process(settings, file_name): The configuration of data provider should be setup by\: :param input_types: Specify the input types, can also be set in init_hook. - It is a list of InputType object. For example, input_types= \ - [dense_vector(9), integer_value(2)]. - :type input_types: list|tuple + It could be a list of InputType object. For example, + input_types=[dense_vector(9), integer_value(2)]. Or user + can set a dict of InputType object, which key is + data_layer's name. For example, input_types=\ + {'img': img_features, 'label': label}. when using dict of + InputType, user could yield a dict of feature values, which + key is also data_layer's name. + + :type input_types: list|tuple|dict :param should_shuffle: True if data should shuffle. Pass None means shuffle when is training and not to shuffle when is testing. @@ -281,12 +286,6 @@ def process(settings, file_name): drop the wrong format data when it is True. Has no effect when check set to False. :type check_fail_continue: bool - - :param use_dynamic_order: Allow provider to yield a dictionary object, whose - key is a input data layer name, and value is the - feature value. The tuples are still allowed when - use_dynmaic_order is True. - :type use_dynamic_order: bool """ def __wrapper__(generator): @@ -340,6 +339,11 @@ def __init__(self, file_list, **kwargs): assert self.slots is not None assert self.generator is not None + use_dynamic_order = False + if isinstance(self.slots, dict): # reorder input_types + self.slots = [self.slots[ipt] for ipt in self.input_order] + use_dynamic_order = True + if len(self.slots) == 1: self.generator = SingleSlotWrapper(self.generator) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 18f0b1b4e497e..c1e74c7a2d8f7 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -216,6 +216,10 @@ def Inputs(*args): if g_current_submodel is g_root_submodel: g_config.model_config.input_layer_names.append(name) +@config_func +def HasInputsSet(): + return len(g_config.model_config.input_layer_names) != 0 + # Define the name of the output layers of the NeuralNetwork. # Usually the output is simply the cost layer. diff --git a/python/paddle/trainer_config_helpers/networks.py b/python/paddle/trainer_config_helpers/networks.py index c54ec3096989c..d8f96195020b4 100644 --- a/python/paddle/trainer_config_helpers/networks.py +++ b/python/paddle/trainer_config_helpers/networks.py @@ -30,7 +30,7 @@ 'lstmemory_unit', 'small_vgg', 'img_conv_group', 'vgg_16_network', 'gru_unit', 'gru_group', 'simple_gru', 'simple_attention', 'text_conv_pool', - 'bidirectional_lstm', 'outputs'] + 'bidirectional_lstm', 'inputs', 'outputs'] ###################################################### @@ -372,8 +372,8 @@ def __vgg__(ipt, num_filter, times, dropouts, num_channels_=None): tmp = __vgg__(tmp, 128, 2, [0.4, 0]) tmp = __vgg__(tmp, 256, 3, [0.4, 0.4, 0]) tmp = __vgg__(tmp, 512, 3, [0.4, 0.4, 0]) - tmp = img_pool_layer(input = tmp, stride = 2, - pool_size = 2, pool_type = MaxPooling()) + tmp = img_pool_layer(input=tmp, stride=2, + pool_size=2, pool_type=MaxPooling()) tmp = dropout_layer(input=tmp, dropout_rate=0.5) tmp = fc_layer(input=tmp, size=512, layer_attr=ExtraAttr(drop_rate=0.5), act=LinearActivation()) @@ -745,7 +745,6 @@ def gru_group(input, gru_bias_attr=None, act=None, gate_act=None, gru_layer_attr=None): - """ gru_group is a recurrent layer group version Gated Recurrent Unit. It does exactly the same calculation as the grumemory layer does. A promising @@ -919,12 +918,12 @@ def bidirectional_lstm(input, size, name=None, return_seq=False, fw = simple_lstm(name='%s_fw' % name, input=input, size=size, **dict((k[len('fwd_'):], v) for k, v in args.iteritems() - if k.startswith('fwd_'))) + if k.startswith('fwd_'))) bw = simple_lstm(name="%s_bw" % name, input=input, size=size, reverse=True, **dict((k[len('bwd_'):], v) for k, v in args.iteritems() - if k.startswith('bwd_'))) + if k.startswith('bwd_'))) if return_seq: return concat_layer(name=name, input=[fw, bw], layer_attr=concat_attr, @@ -1052,14 +1051,30 @@ def dropout_layer(input, dropout_rate, name=None): layer_attr=ExtraAttr(drop_rate=dropout_rate)) -def outputs(layers, *args): +def inputs(layers, *args): + """ + Declare the inputs of network. The order of input should be as same as + the data provider's return order. + + :param layers: Input Layers. + :type layers: list|tuple|LayerOutput. + :return: """ - Declare the end of network. Currently it will only calculate the - input/output order of network. It will calculate the predict network or - train network's output automatically. + if isinstance(layers, LayerOutput) or isinstance(layers, basestring): + layers = [layers] + if len(args) != 0: + layers.extend(args) - :param layers: + Inputs(*[l.name for l in layers]) + + +def outputs(layers, *args): + """ + Declare the outputs of network. If user have not defined the inputs of + network, this method will calculate the input order by dfs travel. + + :param layers: Output layers. :type layers: list|tuple|LayerOutput :return: """ @@ -1093,6 +1108,11 @@ def __dfs_travel__(layer, layers.extend(args) assert len(layers) > 0 + + if HasInputsSet(): # input already set + Outputs(*[l.name for l in layers]) + return # just return outputs. + if len(layers) != 1: logger.warning("`outputs` routine try to calculate network's" " inputs and outputs order. It might not work well." From b22e50ede3e3803e34ccea29ab096a8ad664c129 Mon Sep 17 00:00:00 2001 From: emailweixu Date: Mon, 17 Oct 2016 15:17:35 +0800 Subject: [PATCH 169/324] Update docker_instll.rst docker image name (#210) --- doc_cn/build_and_install/install/docker_install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc_cn/build_and_install/install/docker_install.rst b/doc_cn/build_and_install/install/docker_install.rst index 44aa2a0983f4f..a5f5fb117e11e 100644 --- a/doc_cn/build_and_install/install/docker_install.rst +++ b/doc_cn/build_and_install/install/docker_install.rst @@ -23,9 +23,9 @@ PaddlePaddle提供的Docker镜像版本 +-----------------+------------------+------------------------+-----------------------+ | GPU | gpu-latest | gpu-devel-latest | gpu-demo-latest | +-----------------+------------------+------------------------+-----------------------+ -| CPU WITHOUT AVX | cpu-noavx-latest | cpu-devel-noavx-latest | cpu-demo-noavx-latest | +| CPU WITHOUT AVX | cpu-noavx-latest | cpu-noavx-devel-latest | cpu-noavx-demo-latest | +-----------------+------------------+------------------------+-----------------------+ -| GPU WITHOUT AVX | gpu-noavx-latest | gpu-devel-noavx-latest | gpu-demo-noavx-latest | +| GPU WITHOUT AVX | gpu-noavx-latest | gpu-noavx-devel-latest | gpu-noavx-demo-latest | +-----------------+------------------+------------------------+-----------------------+ 其中，横向包括三个版本，normal，devel和demo。 From 28bc05b126b0c1fb5da4825e10c67e13609e1fde Mon Sep 17 00:00:00 2001 From: emailweixu Date: Mon, 17 Oct 2016 17:56:21 +0800 Subject: [PATCH 170/324] Fix sparse training for trainer_count=1 (#204) * Fix sparse training for trainer_count=1 For trainer_count=1, the gradient machine is NeuralNetwork, which does not create parameter buf for PARAMETER_GRADIENT for sparse update in Parameter::enableType. But gradient parameter buf is still used in SgdThreadUpdater. * Minor update to comment --- paddle/gserver/evaluators/ChunkEvaluator.cpp | 2 +- .../gradientmachines/MultiGradientMachine.cpp | 1 - paddle/parameter/Parameter.h | 6 ++++++ paddle/trainer/ThreadParameterUpdater.cpp | 16 +++++++++++++++- paddle/utils/Logging.h | 2 +- 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/paddle/gserver/evaluators/ChunkEvaluator.cpp b/paddle/gserver/evaluators/ChunkEvaluator.cpp index 273925ba55ee4..22579891f397a 100644 --- a/paddle/gserver/evaluators/ChunkEvaluator.cpp +++ b/paddle/gserver/evaluators/ChunkEvaluator.cpp @@ -75,7 +75,6 @@ class ChunkEvaluator : public Evaluator { public: virtual void init(const EvaluatorConfig& config) { - CHECK(!FLAGS_use_gpu) << "Not supported"; Evaluator::init(config); if (config.chunk_scheme() == "IOB") { numTagTypes_ = 2; @@ -137,6 +136,7 @@ class ChunkEvaluator : public Evaluator { CHECK_EQ(arguments.size(), (size_t)2); IVectorPtr& output = arguments[0].ids; IVectorPtr& label = arguments[1].ids; + CHECK(!output->useGpu() && !label->useGpu()) << "Not supported"; auto sequenceStartPositions = arguments[1].sequenceStartPositions->getVector(false); CHECK_EQ(output->getSize(), label->getSize()); diff --git a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp index 787ce703a08ae..0ded30eeb44e9 100644 --- a/paddle/gserver/gradientmachines/MultiGradientMachine.cpp +++ b/paddle/gserver/gradientmachines/MultiGradientMachine.cpp @@ -813,7 +813,6 @@ void TrainerThread::mergeGradSparse( para->getMat(PARAMETER_GRADIENT).get()); std::vector& ids = mainMat->getIds(threadId_); - ids.clear(); for (auto slaveParams : slaveParameters) { SparseRowCpuMatrix* mat = dynamic_cast((*slaveParams)[pid] diff --git a/paddle/parameter/Parameter.h b/paddle/parameter/Parameter.h index 2f9606dc68026..ff251fe89f9f8 100644 --- a/paddle/parameter/Parameter.h +++ b/paddle/parameter/Parameter.h @@ -146,6 +146,12 @@ class Parameter { } } + void enableBufType(ParameterType type) { + if (bufs_[type]) return; + bufs_[type] = Vector::createParallelVector(config_.size(), useGpu_); + bufs_[type]->zeroMem(); + } + void enableIntType(ParameterType type, size_t intStoreSize = 0) { if (!intBufs_[type]) { SetDevice device(deviceId_); diff --git a/paddle/trainer/ThreadParameterUpdater.cpp b/paddle/trainer/ThreadParameterUpdater.cpp index 91f7f4d29df93..a26e9239f987f 100644 --- a/paddle/trainer/ThreadParameterUpdater.cpp +++ b/paddle/trainer/ThreadParameterUpdater.cpp @@ -20,6 +20,8 @@ limitations under the License. */ #include "paddle/math/SparseRowMatrix.h" #include "paddle/utils/Thread.h" +P_DECLARE_int32(trainer_count); + namespace paddle { SgdThreadUpdater::SgdThreadUpdater(const OptimizationConfig& optConfig) @@ -48,6 +50,13 @@ void SgdThreadUpdater::init(std::vector& parameters) { false /*inPserver*/)); size_t numRows = para->isGradSparseUpdate() ? para->getConfig().dims(0) : 0; optimizers_[pid]->init(numRows, ¶->getConfig()); + if (para->isGradSparseUpdate() && FLAGS_trainer_count == 1) { + // For trainer_count=1, the gradient machine is NeuralNetwork, which does + // not create parameter buf for PARAMETER_GRADIENT for sparse update in + // Parameter::enableType(). But gradient parameter buf is still used + // in SgdThreadUpdater. We need to explicitly create it. + para->enableBufType(PARAMETER_GRADIENT); + } } } @@ -211,7 +220,7 @@ void SgdThreadUpdater::threadUpdateSparse( // From MultiGradientMachine SparseRowIdsCpuMatrix* mainMat = dynamic_cast( para->getMat(PARAMETER_GRADIENT).get()); - const std::vector& sparseIds = mainMat->getIds(tid); + std::vector& sparseIds = mainMat->getIds(tid); for (auto id : sparseIds) { // setup sub bufs @@ -221,6 +230,7 @@ void SgdThreadUpdater::threadUpdateSparse( optimizer->update(vecs, para->getConfig(), id); vecs[PARAMETER_GRADIENT]->zeroMem(); } + sparseIds.clear(); } else if (dynamic_cast( para->getMat(PARAMETER_GRADIENT).get())) { // From NeuralNetwork @@ -246,6 +256,10 @@ void SgdThreadUpdater::threadUpdateSparse( optimizer->update(vecs, para->getConfig(), id); vecs[PARAMETER_GRADIENT]->zeroMem(); } + // For numThreads > 1, MultiGradientMachine is used, which goes + // to the above branch. + CHECK_EQ(numThreads, 1); + mainMat->clearIndices(); } else { auto & m = *para->getMat(PARAMETER_GRADIENT).get(); LOG(FATAL) << "Internal error: " << para->getName() << " " diff --git a/paddle/utils/Logging.h b/paddle/utils/Logging.h index b3f439804686f..7fdfa3240c1de 100644 --- a/paddle/utils/Logging.h +++ b/paddle/utils/Logging.h @@ -191,7 +191,7 @@ void installFailureWriter(void(*callback)(const char*, int)); } #endif // PADDLE_USE_GLOG -#ifdef NDEBUG +#ifndef NDEBUG #define DEBUG_LEVEL 5 #define DBG VLOG(DEBUG_LEVEL) #else From 45280a07dab539ca62a2f392786b95c797c0aa88 Mon Sep 17 00:00:00 2001 From: Zrachel Date: Tue, 18 Oct 2016 10:35:24 +0800 Subject: [PATCH 171/324] Supplement doc for RNN (#214) --- doc_cn/algorithm/rnn/hierarchical-rnn.md | 138 ++++++++++++++++++++++- 1 file changed, 137 insertions(+), 1 deletion(-) diff --git a/doc_cn/algorithm/rnn/hierarchical-rnn.md b/doc_cn/algorithm/rnn/hierarchical-rnn.md index 979fe13e2ecbd..4a85cf336146e 100644 --- a/doc_cn/algorithm/rnn/hierarchical-rnn.md +++ b/doc_cn/algorithm/rnn/hierarchical-rnn.md @@ -260,7 +260,143 @@ out = recurrent_group(step=outer_step, input=SubsequenceInput(emb)) ## 示例3：双进双出，输入不等长 -TBD +**输入不等长**是指recurrent_group的多个输入在各时刻的长度可以不相等, 但需要指定一个和输出长度一致的input，用targetInlink表示。参考配置：单层RNN（`sequence_rnn_multi_unequalength_inputs.conf`），双层RNN（`sequence_nest_rnn_multi_unequalength_inputs.conf`） + +### 读取双层序列的方法 + +我们看一下单双层序列的数据组织形式和dataprovider（见`rnn_data_provider.py`） +```python +data2 = [ + [[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]] ,0], + [[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]], 1], +] + +@provider(input_types=[integer_value_sub_sequence(10), + integer_value_sub_sequence(10), + integer_value(2)], + should_shuffle=False) +def process_unequalength_subseq(settings, file_name): #双层RNN的dataprovider + for d in data2: + yield d + + +@provider(input_types=[integer_value_sequence(10), + integer_value_sequence(10), + integer_value(2)], + should_shuffle=False) +def process_unequalength_seq(settings, file_name): #单层RNN的dataprovider + for d in data2: + words1=reduce(lambda x,y: x+y, d[0]) + words2=reduce(lambda x,y: x+y, d[1]) + yield words1, words2, d[2] +``` + +data2 中有两个样本，每个样本有两个特征, 记fea1, fea2。 + +- 单层序列：两个样本分别为[[1, 2, 4, 5, 2], [5, 4, 1, 3, 1]] 和 [[0, 2, 2, 5, 0, 1, 2], [1, 5, 4, 2, 3, 6, 1]] +- 双层序列：两个样本分别为 + - **样本1**：[[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]]]。fea1和fea2都分别有2个子句，fea1=[[1, 2], [4, 5, 2]], fea2=[[5, 4, 1], [3, 1]] + - **样本2**：[[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]]]。fea1和fea2都分别有3个子句， fea1=[[0, 2], [2, 5], [0, 1, 2]], fea2=[[1, 5], [4], [2, 3, 6, 1]]。
+ - **注意**：每个样本中，各特征的子句数目需要相等。这里说的“双进双出，输入不等长”是指fea1在i时刻的输入的长度可以不等于fea2在i时刻的输入的长度。如对于第1个样本，时刻i=2, fea1[2]=[4, 5, 2]，fea2[2]=[3, 1]，3≠2。 +- 单双层序列中，两个样本的label都分别是0和1 + +### 模型中的配置 + +单层RNN（`sequence_rnn_multi_unequalength_inputs.conf`）和双层RNN（`sequence_nest_rnn_multi_unequalength_inputs.conf`）两个模型配置达到的效果完全一样，区别只在于输入为单层还是双层序列，现在我们来看它们内部分别是如何实现的。 + +- 单层序列： + - 过了一个简单的recurrent_group。每一个时间步，当前的输入y和上一个时间步的输出rnn_state做了一个全连接，功能与示例2中`sequence_rnn.conf`的`step`函数完全相同。这里，两个输入x1,x2分别通过calrnn返回最后时刻的状态。结果得到的encoder1_rep和encoder2_rep分别是单层序列，最后取encoder1_rep的最后一个时刻和encoder2_rep的所有时刻分别相加得到context。 + - 注意到这里recurrent_group输入的每个样本中，fea1和fea2的长度都分别相等，这并非偶然，而是因为recurrent_group要求输入为单层序列时，所有输入的长度都必须相等。 + +```python +def step(x1, x2): + def calrnn(y): + mem = memory(name = 'rnn_state_' + y.name, size = hidden_dim) + out = fc_layer(input = [y, mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'rnn_state_' + y.name) + return out + + encoder1 = calrnn(x1) + encoder2 = calrnn(x2) + return [encoder1, encoder2] + +encoder1_rep, encoder2_rep = recurrent_group( + name="stepout", + step=step, + input=[emb1, emb2]) + +encoder1_last = last_seq(input = encoder1_rep) +encoder1_expandlast = expand_layer(input = encoder1_last, + expand_as = encoder2_rep) +context = mixed_layer(input = [identity_projection(encoder1_expandlast), + identity_projection(encoder2_rep)], + size = hidden_dim) +``` +- 双层序列： + - 双层RNN中，对输入的两个特征分别求时序上的连续全连接(`inner_step1`和`inner_step2`分别处理fea1和fea2)，其功能与示例2中`sequence_nest_rnn.conf`的`outer_step`函数完全相同。不同之处是，此时输入`[SubsequenceInput(emb1), SubsequenceInput(emb2)]`在各时刻并不等长。 + - 函数`outer_step`中可以分别处理这两个特征，但我们需要用targetInlink指定recurrent_group的输出的格式（各子句长度）只能和其中一个保持一致，如这里选择了和emb2的长度一致。 + - 最后，依然是取encoder1_rep的最后一个时刻和encoder2_rep的所有时刻分别相加得到context。 + +```python +def outer_step(x1, x2): + outer_mem1 = memory(name = "outer_rnn_state1", size = hidden_dim) + outer_mem2 = memory(name = "outer_rnn_state2", size = hidden_dim) + def inner_step1(y): + inner_mem = memory(name = 'inner_rnn_state_' + y.name, + size = hidden_dim, + boot_layer = outer_mem1) + out = fc_layer(input = [y, inner_mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'inner_rnn_state_' + y.name) + return out + + def inner_step2(y): + inner_mem = memory(name = 'inner_rnn_state_' + y.name, + size = hidden_dim, + boot_layer = outer_mem2) + out = fc_layer(input = [y, inner_mem], + size = hidden_dim, + act = TanhActivation(), + bias_attr = True, + name = 'inner_rnn_state_' + y.name) + return out + + encoder1 = recurrent_group( + step = inner_step1, + name = 'inner1', + input = x1) + + encoder2 = recurrent_group( + step = inner_step2, + name = 'inner2', + input = x2) + + sentence_last_state1 = last_seq(input = encoder1, name = 'outer_rnn_state1') + sentence_last_state2_ = last_seq(input = encoder2, name = 'outer_rnn_state2') + + encoder1_expand = expand_layer(input = sentence_last_state1, + expand_as = encoder2) + + return [encoder1_expand, encoder2] + +encoder1_rep, encoder2_rep = recurrent_group( + name="outer", + step=outer_step, + input=[SubsequenceInput(emb1), SubsequenceInput(emb2)], + targetInlink=emb2) + +encoder1_last = last_seq(input = encoder1_rep) +encoder1_expandlast = expand_layer(input = encoder1_last, + expand_as = encoder2_rep) +context = mixed_layer(input = [identity_projection(encoder1_expandlast), + identity_projection(encoder2_rep)], + size = hidden_dim) +``` ## 示例4：beam_search的生成 From 58f896c3f40602116f68f6bfc58c96228d0f48bd Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 18 Oct 2016 03:12:03 +0000 Subject: [PATCH 172/324] Speed up PyDP2, support numpy.float array (#207) --- cmake/flags.cmake | 4 ++- demo/mnist/data/get_mnist_data.sh | 2 +- paddle/gserver/dataproviders/DataProvider.cpp | 8 +++-- paddle/gserver/dataproviders/DataProvider.h | 5 +-- .../gserver/dataproviders/PyDataProvider2.cpp | 36 +++++++++++++++---- paddle/utils/Queue.h | 15 ++++++++ .../trainer_config_helpers/data_sources.py | 1 + 7 files changed, 59 insertions(+), 12 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index cc59309ee7efa..dbad6be3f41b3 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -64,7 +64,9 @@ set(COMMON_FLAGS -Wdelete-non-virtual-dtor -Wno-unused-parameter -Wno-error=literal-suffix - -Wno-error=unused-local-typedefs) + -Wno-error=unused-local-typedefs + -Wno-error=unused-function # Warnings in Numpy Header. +) foreach(flag ${COMMON_FLAGS}) safe_set_cflag(CMAKE_C_FLAGS ${flag}) diff --git a/demo/mnist/data/get_mnist_data.sh b/demo/mnist/data/get_mnist_data.sh index c3ef99445049d..9099b5ab6fb85 100755 --- a/demo/mnist/data/get_mnist_data.sh +++ b/demo/mnist/data/get_mnist_data.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh # This scripts downloads the mnist data and unzips it. - +set -e DIR="$( cd "$(dirname "$0")" ; pwd -P )" rm -rf "$DIR/raw_data" mkdir "$DIR/raw_data" diff --git a/paddle/gserver/dataproviders/DataProvider.cpp b/paddle/gserver/dataproviders/DataProvider.cpp index c3b4769f7612b..8cefbb30ada46 100644 --- a/paddle/gserver/dataproviders/DataProvider.cpp +++ b/paddle/gserver/dataproviders/DataProvider.cpp @@ -57,7 +57,8 @@ void BufferBatch::clone(DataBatch* srcBatch, bool useGpu) { } } -DoubleBuffer::DoubleBuffer(DataProvider* dataPool, bool useGpu, +DoubleBuffer::DoubleBuffer(DataProvider *dataPool, + bool useGpu, int64_t batchSize) { batchSize_ = batchSize; dataPool_ = dataPool; @@ -110,6 +111,9 @@ void DoubleBuffer::removeOneBatch(DataBatch* dataBatch) { } void DoubleBuffer::insertOneBatch(DataBatch* batch) { + while (!bufferQueue_->waitNotEmptyFor(2 /* seconds */)) { // time out + if (stopping_) return; + } BufferBatch* bufBatch = bufferQueue_->dequeue(); // clone and copy the data from an Threadlocal Variable bufBatch->clone(batch, useGpu_); @@ -138,7 +142,7 @@ void DoubleBuffer::asyncLoadBatch() { actualSize = dataPool_->getNextBatchInternal(batchSize_, &newBatch); } insertOneBatch(&newBatch); - } while (actualSize > 0); + } while (actualSize > 0 && !stopping_); } } diff --git a/paddle/gserver/dataproviders/DataProvider.h b/paddle/gserver/dataproviders/DataProvider.h index c24546374abf3..112e45de1cb23 100644 --- a/paddle/gserver/dataproviders/DataProvider.h +++ b/paddle/gserver/dataproviders/DataProvider.h @@ -259,7 +259,9 @@ typedef Queue BufferBatchQueue; class DoubleBuffer { public: - DoubleBuffer(DataProvider* dataPool, bool useGpu, int64_t batchSize = 0); + DoubleBuffer(DataProvider* dataPool, + bool useGpu, + int64_t batchSize = 0); virtual ~DoubleBuffer(); void removeOneBatch(DataBatch* dataBatch); @@ -349,7 +351,6 @@ class DataProvider { */ virtual void reset() { if (doubleBuffer_ != nullptr) { - LOG(INFO) << "the double-buffer is starting ..."; doubleBuffer_->startAsyncLoad(); } } diff --git a/paddle/gserver/dataproviders/PyDataProvider2.cpp b/paddle/gserver/dataproviders/PyDataProvider2.cpp index e3e472ac166c2..c464d01fdefd1 100644 --- a/paddle/gserver/dataproviders/PyDataProvider2.cpp +++ b/paddle/gserver/dataproviders/PyDataProvider2.cpp @@ -18,9 +18,16 @@ limitations under the License. */ #include #include #include +#include +#include +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#include #include "DataProvider.h" + #include "paddle/utils/PythonUtil.h" +#include "paddle/utils/Locks.h" +#include "paddle/utils/Stat.h" namespace paddle { @@ -202,7 +209,10 @@ class PyDataProvider2 : public DataProvider { PyDataProvider2(const DataConfig& config, const ModelConfig& modelConfig, bool useGpu) - :DataProvider(config, useGpu), callingContextCreated_(2) { + :DataProvider(config, useGpu), + callingContextCreated_(2) { + if (PyArray_API == NULL) + import_array(); auto& args = config.load_data_args(); PyObjectPtr kwargs = PyObjectPtr(PyDict_New()); if (!args.empty()) { @@ -454,6 +464,7 @@ class PyDataProvider2 : public DataProvider { std::condition_variable pushCV_; std::condition_variable pullCV_; std::mutex mtx_; + ThreadBarrier callingContextCreated_; std::unique_ptr cache_; @@ -496,8 +507,8 @@ class PyDataProvider2 : public DataProvider { * Resetting the PyDataProvider. May start reading thread here. */ virtual void reset() { - DataProvider::reset(); resetImpl(true); + DataProvider::reset(); } /** @@ -518,6 +529,7 @@ class PyDataProvider2 : public DataProvider { * Loading a batch of data. */ int64_t getNextBatchInternal(int64_t size_, DataBatch *batch) { + REGISTER_TIMER("PyDP2.getNextBatchInternal") CHECK_GE(size_, 0); size_t size = (size_t) size_; if (loadThread_) { // loading from thread should wait for data pool ready. @@ -698,10 +710,22 @@ class DenseScanner: public IFieldScanner { */ virtual void fill(Argument &argument, PyObject *obj) { real* dat = argument.value->getData() + height_ * headerPtr_->dim; - py::SequenceHelper s(obj); - // TODO(yuyang18): Here we can use AVX or SSE to accelerate memory copy. - for (size_t i=0; i < headerPtr_->dim; ++i) { - dat[i] = (real) s.getDouble(i); + if (PyArray_Check(obj)) { + auto dtype = PyArray_DTYPE((PyArrayObject*)obj); + if (dtype->type == 'f' && dtype->elsize == sizeof(real)) { + real * data = (real*)PyArray_DATA((PyArrayObject*)obj); + auto sz = PyArray_SIZE((PyArrayObject*)obj); + std::copy(data, data + sz, dat); + } else { + LOG(FATAL) << "You should yield float" << sizeof(real) * 8 + << " array"; + } + } else { + py::SequenceHelper s(obj); + // TODO(yuyang18): Here we can use AVX or SSE to accelerate memory copy. + for (size_t i=0; i < headerPtr_->dim; ++i) { + dat[i] = (real) s.getDouble(i); + } } ++height_; } diff --git a/paddle/utils/Queue.h b/paddle/utils/Queue.h index d73f27d7fafd6..f952cf58778de 100644 --- a/paddle/utils/Queue.h +++ b/paddle/utils/Queue.h @@ -135,6 +135,21 @@ class Queue { queueCV_.wait(lock, [this]() { return numElements_ == 0; }); } + /** + * @brief wait queue is not empty at most for some seconds. + * @param seconds wait time limit. + * @return true if queue is not empty. false if timeout. + */ + bool waitNotEmptyFor(int seconds) { + std::unique_lock lock(queueLock_); + return queueCV_.wait_for( + lock, + std::chrono::seconds(seconds), + [this] { + return numElements_ != 0; + }); + } + private: std::deque elements_; int numElements_; diff --git a/python/paddle/trainer_config_helpers/data_sources.py b/python/paddle/trainer_config_helpers/data_sources.py index 8ada3903dc06b..3b5c17a271f02 100644 --- a/python/paddle/trainer_config_helpers/data_sources.py +++ b/python/paddle/trainer_config_helpers/data_sources.py @@ -84,6 +84,7 @@ def py_data2(files, load_data_module, load_data_object, load_data_args, data.load_data_module = load_data_module data.load_data_object = load_data_object data.load_data_args = load_data_args + data.async_load_data = True return data data_cls = py_data2 From 6bef8390b92b4931b6ef0ded73f73a389ff1bef6 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Tue, 18 Oct 2016 14:53:57 +0800 Subject: [PATCH 173/324] fix bug in some different python environment (#220) --- paddle/gserver/dataproviders/PyDataProvider2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/gserver/dataproviders/PyDataProvider2.cpp b/paddle/gserver/dataproviders/PyDataProvider2.cpp index c464d01fdefd1..ca8b07af49ca0 100644 --- a/paddle/gserver/dataproviders/PyDataProvider2.cpp +++ b/paddle/gserver/dataproviders/PyDataProvider2.cpp @@ -14,11 +14,11 @@ limitations under the License. */ #ifndef PADDLE_NO_PYTHON +#include #include #include #include #include -#include #include #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include From 76fb74dc2e542bcaffb726bb137ff112b213f82c Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 19 Oct 2016 15:13:31 +0800 Subject: [PATCH 174/324] Fix install_docker.rst and data_sources file open mode * Follow #223 * Fix #222 --- doc/build/docker_install.rst | 2 +- python/paddle/trainer_config_helpers/data_sources.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/build/docker_install.rst b/doc/build/docker_install.rst index 542b9bac27afb..e95de35f4da35 100644 --- a/doc/build/docker_install.rst +++ b/doc/build/docker_install.rst @@ -69,7 +69,7 @@ If you want to launch container with GPU support, you need to set some environme .. code-block:: bash - export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}" + export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:gpu-latest diff --git a/python/paddle/trainer_config_helpers/data_sources.py b/python/paddle/trainer_config_helpers/data_sources.py index 3b5c17a271f02..f51140656d0dc 100644 --- a/python/paddle/trainer_config_helpers/data_sources.py +++ b/python/paddle/trainer_config_helpers/data_sources.py @@ -68,7 +68,7 @@ def define_py_data_source(file_list, cls, module, file_list_name = 'train.list' if isinstance(cls, TestData): file_list_name = 'test.list' - with open(file_list_name, 'r') as f: + with open(file_list_name, 'w') as f: f.writelines(file_list) file_list = file_list_name From e1f57bfd66245f78d04f47d670cba5592d5734b2 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Wed, 19 Oct 2016 15:19:32 +0800 Subject: [PATCH 175/324] add base class for seqlastin/max/average layer (#187) --- paddle/gserver/layers/AverageLayer.cpp | 74 ++-------------- paddle/gserver/layers/AverageLayer.h | 19 ++--- paddle/gserver/layers/MaxLayer.cpp | 79 ++--------------- paddle/gserver/layers/MaxLayer.h | 19 +++-- .../layers/SequenceLastInstanceLayer.cpp | 82 +++--------------- paddle/gserver/layers/SequencePoolLayer.cpp | 84 +++++++++++++++++++ paddle/gserver/layers/SequencePoolLayer.h | 57 +++++++++++++ 7 files changed, 188 insertions(+), 226 deletions(-) create mode 100644 paddle/gserver/layers/SequencePoolLayer.cpp create mode 100644 paddle/gserver/layers/SequencePoolLayer.h diff --git a/paddle/gserver/layers/AverageLayer.cpp b/paddle/gserver/layers/AverageLayer.cpp index 6e52217de4e63..7401cdc9a516b 100644 --- a/paddle/gserver/layers/AverageLayer.cpp +++ b/paddle/gserver/layers/AverageLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "AverageLayer.h" #include "paddle/utils/Logging.h" @@ -25,13 +24,8 @@ REGISTER_LAYER(average, AverageLayer); bool AverageLayer::init(const LayerMap& layerMap, const ParameterMap& parameterMap) { - /* Initialize the basic parent class */ - Layer::init(layerMap, parameterMap); + SequencePoolLayer::init(layerMap, parameterMap); - /* initialize biases_ */ - if (biasParameter_.get() != NULL) { - biases_ = std::unique_ptr(new Weight(1, getSize(), biasParameter_)); - } dataMtx_ = Matrix::create(nullptr, 1, 1, false, useGpu_); outMtx_ = Matrix::create(nullptr, 1, getSize(), false, useGpu_); // average strategy @@ -44,57 +38,15 @@ bool AverageLayer::init(const LayerMap& layerMap, } else { LOG(FATAL) << "Unknown average strategy: " << config_.average_strategy(); } - // transform to which sequence type - if (config_.trans_type() == "non-seq") { - type_ = kNonSeq; - } else if (config_.trans_type() == "seq") { - type_ = kSeq; - } else { - LOG(FATAL) << "Unknown trans_type: " << config_.trans_type(); - } - setNeedSequenceInfo(false); return true; } void AverageLayer::forward(PassType passType) { - Layer::forward(passType); - - // average layer should have exactly 1 input - CHECK_EQ(1U, inputLayers_.size()); + SequencePoolLayer::forward(passType); - size_t dim = getSize(); - const Argument& input = getInput(0); - CHECK(input.sequenceStartPositions); - if (type_) { - CHECK(input.subSequenceStartPositions) - << "when trans_type = seq, input must hasSubseq"; - } - int64_t newBatchSize = - type_ ? input.getNumSubSequences() : input.getNumSequences(); - ICpuGpuVectorPtr startPositions = - type_ ? input.subSequenceStartPositions - : input.sequenceStartPositions; - const int* starts = startPositions->getData(false); - size_t numSequences = startPositions->getSize() - 1; - - // check - CHECK_EQ(numSequences, (size_t)newBatchSize); - CHECK_EQ(starts[numSequences], input.getBatchSize()); - CHECK_EQ(dim, input.value->getWidth()); - - resetOutput(newBatchSize, dim); - auto startsPos = startPositions->getVector(useGpu_); MatrixPtr inputValue = getInputValue(0); - getOutputValue()->sequenceAvgForward(*inputValue, *startsPos, mode_); - - /* If type_ = kNonSeq, both seq has or not has sub-seq degrade to a non-seq, - * thus, in this case, output_ has no sequenceStartPositions. - * If type_ = kSeq, seq has sub-seq degrades to a seq, thus, only in this - * case, we should compute the new sequenceStartPositions. - */ - if (type_) { - output_.degradeSequence(input, useGpu_); - } + getOutputValue()->sequenceAvgForward( + *inputValue, *startPositions_->getVector(useGpu_), mode_); /* add the bias-vector AFTER average operation */ if (biases_.get() != NULL) { @@ -106,26 +58,16 @@ void AverageLayer::forward(PassType passType) { } void AverageLayer::backward(const UpdateCallback& callback) { - const Argument& input = getInput(0); - ICpuGpuVectorPtr startPositions = - type_ ? input.subSequenceStartPositions - : input.sequenceStartPositions; - const int* starts = startPositions->getData(false); - /* Do derivation */ { backwardActivation(); } - - if (biases_ && biases_->getWGrad()) { - biases_->getWGrad()->collectBias(*getOutputGrad(), 1); - - // Increasing the number of gradient - biases_->getParameterPtr()->incUpdate(callback); - } + SequencePoolLayer::backward(callback); + const int* starts = startPositions_->getData(false); MatrixPtr grad = getInputGrad(0); + if (grad) { size_t dim = getSize(); real* gradientData = getInputGrad(0)->getData(); real* gradient = getOutputGrad()->getData(); - size_t numSequences = startPositions->getSize() - 1; + size_t numSequences = startPositions_->getSize() - 1; for (size_t sequenceId = 0; sequenceId < numSequences; ++sequenceId) { // TODO(Dangqingqing) optimization for GPU int sequenceLength = starts[sequenceId + 1] - starts[sequenceId]; diff --git a/paddle/gserver/layers/AverageLayer.h b/paddle/gserver/layers/AverageLayer.h index ae910ddefad13..1edc2ace492c5 100644 --- a/paddle/gserver/layers/AverageLayer.h +++ b/paddle/gserver/layers/AverageLayer.h @@ -12,10 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once -#include "Layer.h" +#include "SequencePoolLayer.h" #include "paddle/math/Matrix.h" namespace paddle { @@ -23,20 +22,21 @@ namespace paddle { /** * A layer for "internal average" for sequence input. * Input: one or more sequences. Each sequence contains some instances. - * If AverageLevel = kNonSeq: + * If SequenceLevel = kNonSeq: * Output: output size is the number of input sequences (NOT input instances) * output[i] = average_{for each instance in this sequence}{input[i]} - * If AverageLevel = kSeq: + * If SequenceLevel = kSeq: * Check input sequence must has sub-sequence * Output: output size is the number of input sub-sequences * output[i] = average_{for each instance in this sub-sequence}{input[i]} + * + * The config file api is pooling_layer. */ - -class AverageLayer : public Layer { +class AverageLayer : public SequencePoolLayer { public: enum AverageStrategy { kAverage = 0, kSum = 1, kAverageSquareRootN = 2 }; - enum AverageLevel { kNonSeq = 0, kSeq = 1 }; - explicit AverageLayer(const LayerConfig& config) : Layer(config) {} + explicit AverageLayer(const LayerConfig& config) + : SequencePoolLayer(config) {} ~AverageLayer() {} @@ -46,11 +46,8 @@ class AverageLayer : public Layer { void backward(const UpdateCallback& callback = nullptr); protected: - std::unique_ptr biases_; MatrixPtr outMtx_; MatrixPtr dataMtx_; int mode_; - int type_; }; - } // namespace paddle diff --git a/paddle/gserver/layers/MaxLayer.cpp b/paddle/gserver/layers/MaxLayer.cpp index 226e0ea87dbd4..c4ffe894eccd6 100644 --- a/paddle/gserver/layers/MaxLayer.cpp +++ b/paddle/gserver/layers/MaxLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "MaxLayer.h" #include "paddle/utils/Logging.h" #include "paddle/utils/Stat.h" @@ -21,55 +20,11 @@ namespace paddle { REGISTER_LAYER(max, MaxLayer); -bool MaxLayer::init(const LayerMap& layerMap, - const ParameterMap& parameterMap) { - /* Initialize the basic parent class */ - Layer::init(layerMap, parameterMap); - - /* initialize biases_ */ - if (biasParameter_.get() != NULL) { - biases_ = std::unique_ptr(new Weight(1, getSize(), biasParameter_)); - } - - // transform to which sequence type - if (config_.trans_type() == "non-seq") { - type_ = kNonSeq; - } else if (config_.trans_type() == "seq") { - type_ = kSeq; - } else { - LOG(FATAL) << "Unknown trans_type: " << config_.trans_type(); - } - setNeedSequenceInfo(false); - return true; -} - void MaxLayer::forward(PassType passType) { - Layer::forward(passType); - // max layer should have exactly 1 input - CHECK_EQ(1U, inputLayers_.size()); - - size_t dim = getSize(); - const Argument& input = getInput(0); - int64_t newBatchSize = - type_ ? input.getNumSubSequences() : input.getNumSequences(); - ICpuGpuVectorPtr startPositions = - type_ ? input.subSequenceStartPositions - : input.sequenceStartPositions; - auto starts = startPositions->getVector(useGpu_); - size_t numSequences = startPositions->getSize() - 1; + SequencePoolLayer::forward(passType); - CHECK_EQ(dim, input.value->getWidth()); - CHECK_EQ(numSequences, (size_t)newBatchSize); - CHECK_EQ(startPositions->getData(false)[numSequences], input.getBatchSize()); - if (type_) { - // when trans_type = seq, input must hasSubseq - CHECK_EQ(input.hasSubseq(), 1UL); - } - - // reset output: resize to "num of sequences", not "batch size". - resetOutput(newBatchSize, dim); - - IVector::resizeOrCreate(maxIndex_, newBatchSize * dim, useGpu(deviceId_)); + IVector::resizeOrCreate(maxIndex_, newBatchSize_ * getSize(), + useGpu(deviceId_)); maxIndex_->zeroMem(); MatrixPtr inputValue = getInputValue(0); @@ -77,16 +32,8 @@ void MaxLayer::forward(PassType passType) { { REGISTER_TIMER_INFO("MaxLayerForward", getName().c_str()); - outputValue->maxSequenceForward(*inputValue, *starts, *maxIndex_); - } - - /* If type_ = kNonSeq, both seq has or not has sub-seq degrade to a non-seq, - * thus, in this case, output_ has no cpuSequenceStartPositions. - * If type_ = kSeq, seq has sub-seq degrades to a seq, thus, only in this - * case, we should compute the new cpuSequenceStartPositions. - */ - if (type_) { - output_.degradeSequence(input, useGpu_); + outputValue->maxSequenceForward( + *inputValue, *startPositions_->getVector(useGpu_), *maxIndex_); } if (config_.output_max_index()) { @@ -104,24 +51,14 @@ void MaxLayer::forward(PassType passType) { void MaxLayer::backward(const UpdateCallback& callback) { CHECK(!config_.output_max_index()) << "backward is not available when output_max_index is set"; - /* Do derivation */ { backwardActivation(); } - - if (biases_ && biases_->getWGrad()) { - biases_->getWGrad()->collectBias(*getOutputGrad(), 1); - - // Increasing the number of gradient - biases_->getParameterPtr()->incUpdate(callback); - } + SequencePoolLayer::backward(callback); MatrixPtr inputGrad = getInputGrad(0); MatrixPtr outputGrad = getOutputGrad(); if (inputGrad) { - ICpuGpuVectorPtr starts = - type_ ? getInput(0).subSequenceStartPositions - : getInput(0).sequenceStartPositions; REGISTER_TIMER_INFO("MaxLayerBackward", getName().c_str()); - inputGrad->maxSequenceBackward(*outputGrad, - *(starts->getVector(useGpu_)), *maxIndex_); + inputGrad->maxSequenceBackward( + *outputGrad, *(startPositions_->getVector(useGpu_)), *maxIndex_); } } diff --git a/paddle/gserver/layers/MaxLayer.h b/paddle/gserver/layers/MaxLayer.h index b4c34e665d926..e6dcfe9c6759d 100644 --- a/paddle/gserver/layers/MaxLayer.h +++ b/paddle/gserver/layers/MaxLayer.h @@ -15,7 +15,7 @@ limitations under the License. */ #pragma once -#include "Layer.h" +#include "SequencePoolLayer.h" #include "paddle/math/Matrix.h" #include "paddle/utils/ThreadLocal.h" @@ -24,29 +24,30 @@ namespace paddle { /** * A layer for "internal max" for sequence input. * Input: one or more sequences. Each sequence contains some instances. - * If MaxLevel = kNonSeq: + * If SequenceLevel = kNonSeq: * Output: output size is the number of input sequences (NOT input instances) * output[i] = max_{for each instance in this sequence}{input[i]} - * If MaxLevel = kSeq: + * If SequenceLevel = kSeq: * Check input sequence must has sub-sequence * Output: output size is the number of input sub-sequences * output[i] = max_{for each instance in this sub-sequence}{input[i]} + * + * The config file api is pooling_layer. */ -class MaxLayer : public Layer { +class MaxLayer : public SequencePoolLayer { protected: - std::unique_ptr biases_; // maxIndex_[i][j] = k : the value at (i, j) is from input[k]. IVectorPtr maxIndex_; - int type_; public: - explicit MaxLayer(const LayerConfig& config) : Layer(config) {} - enum MaxLevel {kNonSeq = 0, kSeq = 1 }; + explicit MaxLayer(const LayerConfig& config) : SequencePoolLayer(config) {} ~MaxLayer() {} - bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap) { + return SequencePoolLayer::init(layerMap, parameterMap); + } void forward(PassType passType); void backward(const UpdateCallback& callback = nullptr); diff --git a/paddle/gserver/layers/SequenceLastInstanceLayer.cpp b/paddle/gserver/layers/SequenceLastInstanceLayer.cpp index f4d26ba21bed6..26d9536dd57aa 100644 --- a/paddle/gserver/layers/SequenceLastInstanceLayer.cpp +++ b/paddle/gserver/layers/SequenceLastInstanceLayer.cpp @@ -15,7 +15,7 @@ limitations under the License. */ #include "paddle/utils/Logging.h" -#include "Layer.h" +#include "SequencePoolLayer.h" #include "paddle/math/Matrix.h" #include "paddle/utils/Stat.h" @@ -29,20 +29,19 @@ namespace paddle { * If SequenceLevel = kSeq: * Check input sequence must has sub-sequence * Output: a sequence containing only the last instance of each sub-sequence - * of the input sequence + * of the input sequence + * + * The config file api is last_seq and first_seq. */ -class SequenceLastInstanceLayer : public Layer { +class SequenceLastInstanceLayer : public SequencePoolLayer { protected: - std::unique_ptr biases_; MatrixPtr tmpSrc_; MatrixPtr tmpDest_; - enum SequenceLevel { kNonSeq = 0, kSeq = 1 }; - int type_; public: explicit SequenceLastInstanceLayer(const LayerConfig& config) - : Layer(config) {} + : SequencePoolLayer(config) {} ~SequenceLastInstanceLayer() {} @@ -56,56 +55,20 @@ REGISTER_LAYER(seqlastins, SequenceLastInstanceLayer); bool SequenceLastInstanceLayer::init(const LayerMap& layerMap, const ParameterMap& parameterMap) { - /* Initialize the basic parent class */ - Layer::init(layerMap, parameterMap); - - // seqlastins layer should have exactly 1 input - CHECK_EQ(1U, inputLayers_.size()); - - /* initialize biases_ */ - if (biasParameter_.get() != NULL) { - biases_ = std::unique_ptr(new Weight(1, getSize(), biasParameter_)); - } + SequencePoolLayer::init(layerMap, parameterMap); tmpSrc_ = Matrix::create(nullptr, /* height= */ 1, 1, /* trans= */ false, useGpu_); tmpDest_ = Matrix::create(nullptr, /* height= */ 1, 1, /* trans= */ false, useGpu_); - // transform to which sequence type - if (config_.trans_type() == "non-seq") { - type_ = kNonSeq; - } else if (config_.trans_type() == "seq") { - type_ = kSeq; - } else { - LOG(FATAL) << "Unknown trans_type: " << config_.trans_type(); - } - setNeedSequenceInfo(false); return true; } void SequenceLastInstanceLayer::forward(PassType passType) { - Layer::forward(passType); - - size_t dim = getSize(); - const Argument& input = getInput(0); + SequencePoolLayer::forward(passType); - // check - CHECK(input.sequenceStartPositions); - if (type_) { - CHECK(input.subSequenceStartPositions) - << "when trans_type = seq, input must hasSubseq"; - } - auto startPositions = - type_ ? input.subSequenceStartPositions->getVector(false) - : input.sequenceStartPositions->getVector(false); - size_t height = type_ ? input.getNumSubSequences() : input.getNumSequences(); - CHECK_EQ(dim, input.value->getWidth()); - CHECK_EQ(startPositions->getData()[height], input.getBatchSize()); - CHECK_EQ(height, startPositions->getSize() - 1); - - reserveOutput(height, dim); - const int* starts = startPositions->getData(); + const int* starts = startPositions_->getData(false); MatrixPtr inputValue = getInputValue(0); MatrixPtr outputValue = getOutputValue(); @@ -113,21 +76,13 @@ void SequenceLastInstanceLayer::forward(PassType passType) { AsyncGpuBlock asyncGpuBlock; REGISTER_TIMER_INFO("SequenceLastInstanceLayerForward", getName().c_str()); - for (size_t seqId = 0; seqId < height; ++seqId) { + for (size_t seqId = 0; seqId < newBatchSize_; ++seqId) { int insId = config_.select_first() ? starts[seqId] : starts[seqId + 1] - 1; outputValue->subMatrix(seqId, 1, tmpDest_) ->assign(*(inputValue->subMatrix(insId, 1, tmpSrc_))); } - /* If type_ = kNonSeq, both seq has or not has sub-seq degrade to a non-seq, - * thus, in this case, output_ has no sequenceStartPositions. - * If type_ = kSeq, seq has sub-seq degrades to a seq, thus, only in this - * case, we should compute the new sequenceStartPositions. - */ - if (type_) { - output_.degradeSequence(input, useGpu_); - } } if (biases_.get() != NULL) { @@ -139,23 +94,12 @@ void SequenceLastInstanceLayer::forward(PassType passType) { } void SequenceLastInstanceLayer::backward(const UpdateCallback& callback) { - /* activation, should set to 'linear' in most cases */ - backwardActivation(); - - if (biases_ && biases_->getWGrad()) { - biases_->getWGrad()->collectBias(*getOutputGrad(), 1); - - // Increasing the number of gradient - biases_->getParameterPtr()->incUpdate(callback); - } + SequencePoolLayer::backward(callback); MatrixPtr inputGrad = getInputGrad(0); MatrixPtr outputGrad = getOutputGrad(); - auto startPositions = - type_ ? getInput(0).subSequenceStartPositions->getVector(false) - : getInput(0).sequenceStartPositions->getVector(false); - const int* starts = startPositions->getData(); - size_t numSequences = startPositions->getSize() - 1; + const int* starts = startPositions_->getData(false); + size_t numSequences = startPositions_->getSize() - 1; if (inputGrad) { AsyncGpuBlock asyncGpuBlock; diff --git a/paddle/gserver/layers/SequencePoolLayer.cpp b/paddle/gserver/layers/SequencePoolLayer.cpp new file mode 100644 index 0000000000000..55be73d363df1 --- /dev/null +++ b/paddle/gserver/layers/SequencePoolLayer.cpp @@ -0,0 +1,84 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/utils/Logging.h" +#include "SequencePoolLayer.h" + +namespace paddle { + +bool SequencePoolLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + /* Initialize the basic parent class */ + Layer::init(layerMap, parameterMap); + + // seqlastins/max/average layer should have exactly 1 input + CHECK_EQ(1U, inputLayers_.size()); + + /* initialize biases_ */ + if (biasParameter_.get() != NULL) { + biases_ = std::unique_ptr(new Weight(1, getSize(), biasParameter_)); + } + // transform to which sequence type + if (config_.trans_type() == "non-seq") { + type_ = kNonSeq; + } else if (config_.trans_type() == "seq") { + type_ = kSeq; + } else { + LOG(FATAL) << "Unknown trans_type: " << config_.trans_type(); + } + setNeedSequenceInfo(false); + return true; +} + +void SequencePoolLayer::forward(PassType passType) { + Layer::forward(passType); + + const Argument& input = getInput(0); + newBatchSize_ = type_ ? input.getNumSubSequences() : input.getNumSequences(); + size_t dim = getSize(); + // check + CHECK_EQ(dim, input.value->getWidth()); + startPositions_ = + type_ ? input.subSequenceStartPositions : input.sequenceStartPositions; + auto starts = startPositions_->getVector(false); + CHECK_EQ(starts->getData()[newBatchSize_], input.getBatchSize()); + CHECK_EQ(newBatchSize_, starts->getSize() - 1); + + resetOutput(newBatchSize_, dim); + if (type_) { + CHECK(input.subSequenceStartPositions) + << "when trans_type = seq, input must hasSubseq"; + } + /* If type_ = kNonSeq, both seq has or not has sub-seq degrade to a non-seq, + * thus, in this case, output_ has no sequenceStartPositions. + * If type_ = kSeq, seq has sub-seq degrades to a seq, thus, only in this + * case, we should compute the new sequenceStartPositions. + */ + if (type_) { + output_.degradeSequence(input, useGpu_); + } +} + +void SequencePoolLayer::backward(const UpdateCallback& callback) { + /* Do derivation */ { backwardActivation(); } + + if (biases_ && biases_->getWGrad()) { + biases_->getWGrad()->collectBias(*getOutputGrad(), 1); + + // Increasing the number of gradient + biases_->getParameterPtr()->incUpdate(callback); + } +} + +} // namespace paddle diff --git a/paddle/gserver/layers/SequencePoolLayer.h b/paddle/gserver/layers/SequencePoolLayer.h new file mode 100644 index 0000000000000..669af80e1d447 --- /dev/null +++ b/paddle/gserver/layers/SequencePoolLayer.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" +#include "paddle/math/Matrix.h" + +namespace paddle { +/** + * A base layer for SequenceLastInstanceLayer/AverageLayer/MaxLayer. + * + * Input: one or more sequences. Each sequence contains some instances. + * If SequenceLevel = kNonSeq: + * Output: output size is the number of input sequences (NOT input instances) + * output[i] = seqlastin/average/max_{for each instance in this + * sequence}{input[i]} + * If SequenceLevel = kSeq: + * Check input sequence must has sub-sequence + * Output: output size is the number of input sub-sequences + * output[i] = seqlastin/average/max_{for each instance in this + * sub-sequence}{input[i]} + * + * The config file api is pooling_layer. + */ + +class SequencePoolLayer : public Layer { +protected: + int type_; + std::unique_ptr biases_; + enum SequenceLevel { kNonSeq = 0, kSeq = 1 }; + size_t newBatchSize_; + ICpuGpuVectorPtr startPositions_; + +public: + explicit SequencePoolLayer(const LayerConfig& config) : Layer(config) {} + + virtual ~SequencePoolLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + void forward(PassType passType); + void backward(const UpdateCallback& callback = nullptr); +}; + +} // namespace paddle From e387cdba77018805fcc3a0f5897e9747253798d7 Mon Sep 17 00:00:00 2001 From: alvations Date: Fri, 21 Oct 2016 11:28:46 +0800 Subject: [PATCH 176/324] Added Bidi-LSTM and DB-LSTM to quick_start demo (#226) --- demo/quick_start/train.sh | 2 + demo/quick_start/trainer_config.bidi-lstm.py | 62 +++++++++++++++++ demo/quick_start/trainer_config.db-lstm.py | 73 ++++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 demo/quick_start/trainer_config.bidi-lstm.py create mode 100644 demo/quick_start/trainer_config.db-lstm.py diff --git a/demo/quick_start/train.sh b/demo/quick_start/train.sh index 1f0a137c8bd59..ea4e32249a3d0 100755 --- a/demo/quick_start/train.sh +++ b/demo/quick_start/train.sh @@ -18,6 +18,8 @@ cfg=trainer_config.lr.py #cfg=trainer_config.emb.py #cfg=trainer_config.cnn.py #cfg=trainer_config.lstm.py +#cfg=trainer_config.bidi-lstm.py +#cfg=trainer_config.db-lstm.py paddle train \ --config=$cfg \ --save_dir=./output \ diff --git a/demo/quick_start/trainer_config.bidi-lstm.py b/demo/quick_start/trainer_config.bidi-lstm.py new file mode 100644 index 0000000000000..3be3d37342271 --- /dev/null +++ b/demo/quick_start/trainer_config.bidi-lstm.py @@ -0,0 +1,62 @@ +# edit-mode: -*- python -*- + +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +dict_file = "./data/dict.txt" +word_dict = dict() +with open(dict_file, 'r') as f: + for i, line in enumerate(f): + w = line.strip().split()[0] + word_dict[w] = i + +is_predict = get_config_arg('is_predict', bool, False) +trn = 'data/train.list' if not is_predict else None +tst = 'data/test.list' if not is_predict else 'data/pred.list' +process = 'process' if not is_predict else 'process_predict' +define_py_data_sources2(train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) + +batch_size = 128 if not is_predict else 1 +settings( + batch_size=batch_size, + learning_rate=2e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(8e-4), + gradient_clipping_threshold=25 +) + +bias_attr = ParamAttr(initial_std=0.,l2_rate=0.) +data = data_layer(name="word", size=len(word_dict)) +emb = embedding_layer(input=data, size=128) + +bi_lstm = bidirectional_lstm(input=emb, size=128) +dropout = dropout_layer(input=bi_lstm, dropout_rate=0.5) + +output = fc_layer(input=dropout, size=2, + bias_attr=bias_attr, + act=SoftmaxActivation()) + +if is_predict: + maxid = maxid_layer(output) + outputs([maxid, output]) +else: + label = data_layer(name="label", size=2) + cls = classification_cost(input=output, label=label) + outputs(cls) diff --git a/demo/quick_start/trainer_config.db-lstm.py b/demo/quick_start/trainer_config.db-lstm.py new file mode 100644 index 0000000000000..b35bdf5a61b47 --- /dev/null +++ b/demo/quick_start/trainer_config.db-lstm.py @@ -0,0 +1,73 @@ +# edit-mode: -*- python -*- + +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +dict_file = "./data/dict.txt" +word_dict = dict() +with open(dict_file, 'r') as f: + for i, line in enumerate(f): + w = line.strip().split()[0] + word_dict[w] = i + +is_predict = get_config_arg('is_predict', bool, False) +trn = 'data/train.list' if not is_predict else None +tst = 'data/test.list' if not is_predict else 'data/pred.list' +process = 'process' if not is_predict else 'process_predict' +define_py_data_sources2(train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) + +batch_size = 128 if not is_predict else 1 +settings( + batch_size=batch_size, + learning_rate=2e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(8e-4), + gradient_clipping_threshold=25 +) + +bias_attr = ParamAttr(initial_std=0.,l2_rate=0.) + +data = data_layer(name="word", size=len(word_dict)) +emb = embedding_layer(input=data, size=128) + +hidden_0 = mixed_layer(size=128, input=[full_matrix_projection(input=emb)]) +lstm_0 = lstmemory(input=hidden_0, layer_attr=ExtraAttr(drop_rate=0.1)) + +input_layers = [hidden_0, lstm_0] + +for i in range(1,8): + fc = fc_layer(input=input_layers, size=128) + lstm = lstmemory(input=fc, layer_attr=ExtraAttr(drop_rate=0.1), + reverse=(i % 2) == 1,) + input_layers = [fc, lstm] + +lstm_last = pooling_layer(input=lstm, pooling_type=MaxPooling()) + +output = fc_layer(input=lstm_last, size=2, + bias_attr=bias_attr, + act=SoftmaxActivation()) + +if is_predict: + maxid = maxid_layer(output) + outputs([maxid, output]) +else: + label = data_layer(name="label", size=2) + cls = classification_cost(input=output, label=label) + outputs(cls) From ac383dd021b2c2479dd9e810892f47b12136b8b6 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Mon, 24 Oct 2016 10:00:18 +0800 Subject: [PATCH 177/324] add missing layer_attr (#234) --- .../paddle/trainer_config_helpers/layers.py | 106 +++++++++++++++--- 1 file changed, 88 insertions(+), 18 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 686704cb7c9b0..d45a9b53dcc94 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -2799,7 +2799,9 @@ def __cost_input__(input, label, weight=None): @wrap_name_default() -def regression_cost(input, label, weight=None, name=None): +@layer_support() +def regression_cost(input, label, weight=None, name=None, + layer_attr=None): """ Regression Layer. @@ -2814,12 +2816,15 @@ def regression_cost(input, label, weight=None, name=None): :param weight: The weight affects the cost, namely the scale of cost. It is an optional argument. :type weight: LayerOutput + :param layer_attr: layer's extra attribute. + :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput """ ipts, parents = __cost_input__(input, label, weight) - Layer(inputs=ipts, type="square_error", name=name) + Layer(inputs=ipts, type="square_error", name=name, + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.COST, parents=parents) @@ -2948,7 +2953,8 @@ def conv_operator(img, filter, filter_size, num_filters, @wrap_name_default() -def conv_shift_layer(a, b, name=None): +@layer_support() +def conv_shift_layer(a, b, name=None, layer_attr=None): """ This layer performs cyclic convolution for two input. For example: - a[in]: contains M elements. @@ -2977,6 +2983,8 @@ def conv_shift_layer(a, b, name=None): :type a: LayerOutput :param b: input layer b :type b: LayerOutput + :param layer_attr: layer's extra attribute. + :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput """ @@ -2986,6 +2994,7 @@ def conv_shift_layer(a, b, name=None): name=name, type=LayerType.CONV_SHIFT_LAYER, inputs=[a.name, b.name], + **ExtraLayerAttribute.to_kwargs(layer_attr) ) return LayerOutput(name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], @@ -3059,6 +3068,7 @@ def tensor_layer(a, b, size, act=None, name=None, @wrap_param_attr_default() @wrap_bias_attr_default() @wrap_act_default() +@layer_support() def selective_fc_layer(input, select, size, act=None, name=None, pass_generation=False, has_selected_colums=True, @@ -3131,7 +3141,8 @@ def selective_fc_layer(input, select, size, act=None, name=None, @wrap_name_default() -def sampling_id_layer(input, name=None): +@layer_support() +def sampling_id_layer(input, name=None, layer_attr=None): """ A layer for sampling id from multinomial distribution from the input layer. Sampling one id for one sample. @@ -3146,6 +3157,8 @@ def sampling_id_layer(input, name=None): :type input: LayerOutput :param name: The Layer Name. :type name: basestring + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3153,12 +3166,15 @@ def sampling_id_layer(input, name=None): name=name, type=LayerType.SAMPLING_ID_LAYER, inputs=[Input(input.name)], + **ExtraLayerAttribute.to_kwargs(layer_attr) ) return LayerOutput(name, LayerType.SAMPLING_ID_LAYER, input) @wrap_name_default() -def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0): +@layer_support() +def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0, + layer_attr=None): """ This layer for applying a slope and an intercept to the input element-wise. There is no activation and weight. @@ -3180,6 +3196,8 @@ def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0): :type slope: float. :param intercept: the offset. :type intercept: float. + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3189,12 +3207,15 @@ def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0): slope=slope, intercept=intercept, inputs=[Input(input.name)], + **ExtraLayerAttribute.to_kwargs(layer_attr) ) return LayerOutput(name, LayerType.SLOPE_INTERCEPT_LAYER, input) @wrap_name_default() -def linear_comb_layer(weights, vectors, size=None, name=None): +@layer_support() +def linear_comb_layer(weights, vectors, size=None, name=None, + layer_attr=None): """ A layer for weighted sum of vectors takes two inputs. - Input: size of weights is M @@ -3235,6 +3256,8 @@ def linear_comb_layer(weights, vectors, size=None, name=None): :type size: int :param name: The Layer Name. :type name: basestring + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3250,6 +3273,7 @@ def linear_comb_layer(weights, vectors, size=None, name=None): type=LayerType.LINEAR_COMBINATION_LAYER, size=size, inputs=[Input(weights.name), Input(vectors.name)], + **ExtraLayerAttribute.to_kwargs(layer_attr) ) return LayerOutput(name, LayerType.LINEAR_COMBINATION_LAYER, [weights, vectors], size=size) @@ -3259,6 +3283,7 @@ def linear_comb_layer(weights, vectors, size=None, name=None): @wrap_name_default() +@layer_support() def block_expand_layer(input, channel=0, block_x=0, @@ -3267,7 +3292,8 @@ def block_expand_layer(input, stride_y=0, padding_x=0, padding_y=0, - name=None): + name=None, + layer_attr=None): """ Expand feature map to minibatch matrix. - matrix width is: block_y * block_x * channel @@ -3314,6 +3340,8 @@ def block_expand_layer(input, :type padding_y: int :param name: The name of this layer, which can not specify. :type name: None|basestring. + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3328,13 +3356,16 @@ def block_expand_layer(input, padding_y=padding_y) ), type=LayerType.BLOCK_EXPAND, + **ExtraLayerAttribute.to_kwargs(layer_attr) ) return LayerOutput(name, LayerType.BLOCK_EXPAND, parents=[input]) @wrap_name_default() -def ctc_layer(input, label, size=None, name=None, norm_by_times=False): +@layer_support() +def ctc_layer(input, label, size=None, name=None, norm_by_times=False, + layer_attr=None): """ Connectionist Temporal Classification (CTC) is designed for temporal classication task. That is, for sequence labeling problems where the @@ -3371,6 +3402,8 @@ def ctc_layer(input, label, size=None, name=None, norm_by_times=False): :type name: basestring|None :param norm_by_times: Whether to normalization by times. False by default. :type norm_by_times: bool + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3386,14 +3419,17 @@ def ctc_layer(input, label, size=None, name=None, norm_by_times=False): type=LayerType.CTC_LAYER, size=size, norm_by_times=norm_by_times, - inputs=[input.name, label.name] + inputs=[input.name, label.name], + **ExtraLayerAttribute.to_kwargs(layer_attr) ) return LayerOutput(name, LayerType.CTC_LAYER, [input, label], size=size) @wrap_name_default() @wrap_param_attr_default() -def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None): +@layer_support() +def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, + layer_attr=None): """ A layer for calculating the cost of sequential conditional random field model. @@ -3419,6 +3455,8 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None): :type param_attr: ParameterAttribute :param name: The name of this layers. It is not necessary. :type name: None|basestring + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3442,6 +3480,7 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None): type=LayerType.CRF_LAYER, size=size, inputs=ipts, + **ExtraLayerAttribute.to_kwargs(layer_attr) ) parents = [input, label] if weight is not None: @@ -3451,7 +3490,9 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None): @wrap_name_default() @wrap_param_attr_default() -def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): +@layer_support() +def crf_decoding_layer(input, size, label=None, param_attr=None, name=None, + layer_attr=None): """ A layer for calculating the decoding sequence of sequential conditional random field model. The decoding sequence is stored in output.ids. @@ -3469,6 +3510,8 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): :type param_attr: ParameterAttribute :param name: The name of this layers. It is not necessary. :type name: None|basestring + :param layer_attr: Extra Layer config. + :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. :rtype: LayerOutput """ @@ -3485,6 +3528,7 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None): type=LayerType.CRF_DECODING_LAYER, size=size, inputs=ipts, + **ExtraLayerAttribute.to_kwargs(layer_attr) ) parents = [input] if label is not None: @@ -3575,7 +3619,8 @@ def nce_layer(input, label, num_classes, weight=None, @wrap_name_default() -def rank_cost(left, right, label, weight=None, name=None, coeff=1.0): +@layer_support() +def rank_cost(left, right, label, weight=None, name=None, coeff=1.0, layer_attr=None): """ A cost Layer for learning to rank using gradient descent. Details can refer to `papers Date: Mon, 24 Oct 2016 14:10:29 +0800 Subject: [PATCH 178/324] fix build bug in gcc46 (#236) --- paddle/trainer/ThreadParameterUpdater.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/trainer/ThreadParameterUpdater.cpp b/paddle/trainer/ThreadParameterUpdater.cpp index a26e9239f987f..d0fda1b6253e3 100644 --- a/paddle/trainer/ThreadParameterUpdater.cpp +++ b/paddle/trainer/ThreadParameterUpdater.cpp @@ -258,7 +258,7 @@ void SgdThreadUpdater::threadUpdateSparse( } // For numThreads > 1, MultiGradientMachine is used, which goes // to the above branch. - CHECK_EQ(numThreads, 1); + CHECK_EQ(numThreads, 1UL); mainMat->clearIndices(); } else { auto & m = *para->getMat(PARAMETER_GRADIENT).get(); From e83950b0d2809af70806e3397964343ade296242 Mon Sep 17 00:00:00 2001 From: wenboyang Date: Mon, 24 Oct 2016 14:27:32 +0800 Subject: [PATCH 179/324] error in doc of quick_start (#228) * fix error in doc of quick_start * There are some warning when execute preprocess.sh --- demo/quick_start/preprocess.sh | 2 ++ doc/demo/quick_start/index_en.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/demo/quick_start/preprocess.sh b/demo/quick_start/preprocess.sh index fb2bee98beb26..fe2acbbd74898 100755 --- a/demo/quick_start/preprocess.sh +++ b/demo/quick_start/preprocess.sh @@ -20,6 +20,8 @@ set -e +export LC_ALL=C + mkdir -p data/tmp python preprocess.py -i data/reviews_Electronics_5.json.gz # uniq and shuffle diff --git a/doc/demo/quick_start/index_en.md b/doc/demo/quick_start/index_en.md index ee3fa2a2166f4..e7d74512292c8 100644 --- a/doc/demo/quick_start/index_en.md +++ b/doc/demo/quick_start/index_en.md @@ -134,7 +134,7 @@ def process(settings, file_name): You need to add a data provider definition `define_py_data_sources2` in our network configuration. This definition specifies: - The path of the training and testing data (`data/train.list`, `data/test.list`). -- The location of the data provider file (`dataprovider_pow`). +- The location of the data provider file (`dataprovider_bow`). - The function to call to get data. (`process`). - Additional arguments or data. Here it passes the path of word dictionary. From 3dd8c9bea4761652a8e4a4730b52563954ebc42b Mon Sep 17 00:00:00 2001 From: luotao1 Date: Mon, 24 Oct 2016 21:00:12 +0800 Subject: [PATCH 180/324] add maxout layer, including interface and unittest (#229) * add maxout layer, including interface and unittest * follow maxout comments * auto setting channels * fix unittest bug in test_RecurrentGradientMachine --- doc/ui/api/trainer_config_helpers/layers.rst | 6 + paddle/cuda/include/hl_cnn.h | 32 ++++- paddle/cuda/include/stub/hl_cnn_stub.h | 8 ++ paddle/cuda/src/hl_cuda_cnn.cu | 59 +++++++++ paddle/gserver/layers/MaxOutLayer.cpp | 87 ++++++++++++ paddle/gserver/layers/MaxOutLayer.h | 54 ++++++++ paddle/gserver/tests/rnn_data_provider.py | 29 +++- .../tests/sequence_nest_rnn_multi_input.conf | 2 +- .../tests/sequence_rnn_multi_input.conf | 2 +- paddle/gserver/tests/test_LayerGrad.cpp | 18 +++ paddle/math/Matrix.cpp | 125 ++++++++++++++++++ paddle/math/Matrix.h | 34 ++++- paddle/math/tests/test_matrixCompare.cpp | 72 ++++++++++ proto/ModelConfig.proto.m4 | 10 ++ python/paddle/trainer/config_parser.py | 32 +++++ .../paddle/trainer_config_helpers/layers.py | 70 +++++++++- .../tests/configs/check.md5 | 1 + .../tests/configs/generate_protostr.sh | 3 +- .../tests/configs/test_maxout.py | 30 +++++ 19 files changed, 665 insertions(+), 9 deletions(-) create mode 100644 paddle/gserver/layers/MaxOutLayer.cpp create mode 100644 paddle/gserver/layers/MaxOutLayer.h create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_maxout.py diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index 55f5623b0faef..5bb88b0615c12 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -73,6 +73,12 @@ img_pool_layer :members: img_pool_layer :noindex: +maxout_layer +------------ +.. automodule:: paddle.trainer_config_helpers.layers + :members: maxout_layer + :noindex: + Norm Layer ========== diff --git a/paddle/cuda/include/hl_cnn.h b/paddle/cuda/include/hl_cnn.h index 5d750333e1e35..d19f4a4bb310a 100644 --- a/paddle/cuda/include/hl_cnn.h +++ b/paddle/cuda/include/hl_cnn.h @@ -169,7 +169,7 @@ extern void hl_avgpool_forward( * @brief Maximum pool backward. * * @param[in] frameCnt batch size of input image. - * @param[in] outGrad input data. + * @param[in] outGrad output grad data. * @param[in] channels number of channel. * @param[in] height image height. * @param[in] width image width. @@ -240,4 +240,34 @@ extern void hl_CMRNorm_backward( size_t channels, size_t height, size_t width, size_t sizeX, real alpha, real beta); +/** + * @brief MaxOut forward. + * + * @param[in] inData input data. + * @param[out] outData output data. + * @param[out] idData output maxId. + * @param[in] batchSize batchSize. + * @param[in] size number of channels * image height * image width. + * @param[in] featLen feature length = image height * image width. + * @param[in] groups number of groups. + */ +extern void hl_maxout_forward( + const real* inData, real* outData, int* idData, + size_t batchSize, size_t size, size_t featLen, size_t groups); + +/** + * @brief MaxOut backward. + * + * @param[out] inGrad input grad data. + * @param[in] outGrad output grad data. + * @param[in] idData output maxId. + * @param[in] batchSize batchSize. + * @param[in] size number of channels * image height * image width. + * @param[in] featLen feature length = image height * image width. + * @param[in] groups number of groups. + */ +extern void hl_maxout_backward( + real* inGrad, const real* outGrad, const int* idData, + size_t batchSize, size_t size, size_t featLen, size_t groups); + #endif /* HL_CNN_H_ */ diff --git a/paddle/cuda/include/stub/hl_cnn_stub.h b/paddle/cuda/include/stub/hl_cnn_stub.h index 38e359c3eb2f3..5f696986e3c8f 100644 --- a/paddle/cuda/include/stub/hl_cnn_stub.h +++ b/paddle/cuda/include/stub/hl_cnn_stub.h @@ -89,4 +89,12 @@ inline void hl_CMRNorm_backward( size_t channels, size_t height, size_t width, size_t sizeX, real alpha, real beta) {} +inline void hl_maxout_forward( + const real* inData, real* outData, int* idData, + size_t batchSize, size_t size, size_t featLen, size_t group) {} + +inline void hl_maxout_backward( + real* inGrad, const real* outGrad, const int* idData, + size_t batchSize, size_t size, size_t featLen, size_t group) {} + #endif // HL_CNN_STUB_H_ diff --git a/paddle/cuda/src/hl_cuda_cnn.cu b/paddle/cuda/src/hl_cuda_cnn.cu index abac83a3e0447..baa2fb0d27d74 100644 --- a/paddle/cuda/src/hl_cuda_cnn.cu +++ b/paddle/cuda/src/hl_cuda_cnn.cu @@ -531,3 +531,62 @@ void hl_CMRNorm_backward(size_t frameCnt, const real* inV, height, width, sizeX, alpha, beta, inDiff); CHECK_SYNC("hl_CMRNorm_backward"); } + +__global__ void maxoutFpCompute(size_t nthreads, const real * inData, + real * outData, int* idData, + size_t size, size_t featLen, size_t groups) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if(index < nthreads) { + size_t batch_idx = index / size; + size_t i = index % size; + size_t channel_idx = i / featLen; + size_t feat_idx = i % featLen; + size_t data_idx = (batch_idx * size + channel_idx * featLen) * groups + feat_idx; + real max = inData[data_idx]; + int maxId = 0; + for (size_t g = 1; g < groups; ++g) { + real tmp = inData[data_idx + g * featLen]; + if (tmp > max) { + max = tmp; + maxId = g; + } + } + outData[index] = max; + idData[index] = maxId; + } +} + +void hl_maxout_forward(const real* inData, real* outData, + int* idData, size_t batchSize, size_t size, + size_t featLen, size_t groups) { + int num_kernels = size * batchSize; + int blocks = (num_kernels + 1024 - 1) / 1024; + maxoutFpCompute<<< blocks, 1024, 0, STREAM_DEFAULT>>>( + num_kernels, inData, outData, idData, size, featLen, groups); + CHECK_SYNC("hl_maxout_forward failed"); +} + +__global__ void maxoutBpCompute(size_t nthreads, real* inGrad, + const real* outGrad, const int* idData, + size_t size, size_t featLen, size_t groups) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if(index < nthreads) { + size_t batch_idx = index / size; + size_t i = index % size; + size_t channel_idx = i / featLen; + size_t feat_idx = i % featLen; + size_t newIndex = batch_idx * size; + size_t gradIdx = (channel_idx * groups + (idData + newIndex)[i]) * featLen + feat_idx; + (inGrad + newIndex * groups)[gradIdx] += (outGrad + newIndex)[i]; + } +} + +void hl_maxout_backward(real* inGrad, const real* outGrad, + const int* idData, size_t batchSize, size_t size, + size_t featLen, size_t groups) { + int num_kernels = size * batchSize; + int blocks = (num_kernels + 1024 - 1) / 1024; + maxoutBpCompute<<< blocks, 1024, 0, STREAM_DEFAULT >>>( + num_kernels, inGrad, outGrad, idData, size, featLen, groups); + CHECK_SYNC("hl_maxout_backward failed"); +} diff --git a/paddle/gserver/layers/MaxOutLayer.cpp b/paddle/gserver/layers/MaxOutLayer.cpp new file mode 100644 index 0000000000000..106ab26ba1aae --- /dev/null +++ b/paddle/gserver/layers/MaxOutLayer.cpp @@ -0,0 +1,87 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "MaxOutLayer.h" +#include "hl_gpu.h" +#include "hl_cnn.h" + +namespace paddle { + +REGISTER_LAYER(maxout, MaxOutLayer); + +size_t MaxOutLayer::getSize() { + const MaxOutConfig& maxoutConf = config_.inputs(0).maxout_conf(); + imgSizeH_ = inputLayers_[0]->getOutput().getFrameHeight(); + imgSizeW_ = inputLayers_[0]->getOutput().getFrameWidth(); + if (imgSizeH_ == 0) { + imgSizeH_ = maxoutConf.img_size_y(); + } + if (imgSizeW_ == 0) { + imgSizeW_ = maxoutConf.img_size_x(); + } + + featLen_ = imgSizeH_ * imgSizeW_; + size_t layerSize = featLen_ * outputChannels_; + + getOutput().setFrameHeight(imgSizeH_); + getOutput().setFrameWidth(imgSizeW_); + + return layerSize; +} + +bool MaxOutLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + /* Initialize the basic parent class */ + Layer::init(layerMap, parameterMap); + + /* the size of inputs for maxout-layer is 1 */ + CHECK_EQ(config_.inputs_size(), 1UL); + + const MaxOutConfig& conf = config_.inputs(0).maxout_conf(); + groups_ = conf.groups(); + channels_ = conf.channels(); + CHECK_EQ(channels_ % groups_, 0UL); + outputChannels_ = channels_ / groups_; + + return true; +} + +void MaxOutLayer::forward(PassType passType) { + Layer::forward(passType); + + /* malloc memory for the output_ if necessary */ + /* note: one sample correspond to one column */ + size_t batchSize = getInput(0).getBatchSize(); + size_t size = getSize(); + resetOutput(batchSize, size); + MatrixPtr inputV = getInputValue(0); + MatrixPtr outV = getOutputValue(); + + IVector::resizeOrCreate(maxoutId_, size * batchSize, useGpu_); + outV->maxoutForward(*inputV, *maxoutId_, outputChannels_, groups_); +} + +void MaxOutLayer::backward(const UpdateCallback& callback) { + (void)callback; + + /* Do derivation */ + MatrixPtr inputG = getInputGrad(0); + MatrixPtr outG = getOutputGrad(); + + if (inputG) { + inputG->maxoutBackward(*outG, *maxoutId_, outputChannels_, groups_); + } +} + +} // namespace paddle diff --git a/paddle/gserver/layers/MaxOutLayer.h b/paddle/gserver/layers/MaxOutLayer.h new file mode 100644 index 0000000000000..9011a5c332b17 --- /dev/null +++ b/paddle/gserver/layers/MaxOutLayer.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" +#include "paddle/math/Matrix.h" + +namespace paddle { + +/** + * A layer to do max out on conv layer output. + * Input: output of a conv layer. + * Output: feature map size same as input. Channel is (input channel) / groups. + * So the num of channels should be able to devided by groups. + * + * The config file api is maxout_layer. + */ + +class MaxOutLayer : public Layer { +protected: + size_t groups_; + size_t imgSizeH_, imgSizeW_; + /// outputChannels_ = channels_ / groups_ + size_t channels_, outputChannels_; + /// feature length = imgSizeH_ * imgSizeW_ + size_t featLen_; + IVectorPtr maxoutId_; + +public: + /// return imgSizeH_ * imgSizeW_ * outputChannels_; + size_t getSize(); + + explicit MaxOutLayer(const LayerConfig& config) : Layer(config) {} + virtual ~MaxOutLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + void forward(PassType passType); + void backward(const UpdateCallback& callback = nullptr); +}; + +} // namespace paddle diff --git a/paddle/gserver/tests/rnn_data_provider.py b/paddle/gserver/tests/rnn_data_provider.py index 5c3b062309c51..321c78cb1741b 100644 --- a/paddle/gserver/tests/rnn_data_provider.py +++ b/paddle/gserver/tests/rnn_data_provider.py @@ -14,12 +14,15 @@ from paddle.trainer.PyDataProvider2 import * +# Note that each config should has an independent provider +# in current design of PyDataProvider2. +####################################################### data = [ [[[1, 3, 2], [4, 5, 2]], 0], [[[0, 2], [2, 5], [0, 1, 2]], 1], ] - +# Used for sequence_nest_rnn.conf @provider(input_types=[integer_value_sub_sequence(10), integer_value(3)], should_shuffle=False) @@ -27,7 +30,7 @@ def process_subseq(settings, file_name): for d in data: yield d - +# Used for sequence_rnn.conf @provider(input_types=[integer_value_sequence(10), integer_value(3)], should_shuffle=False) @@ -38,11 +41,32 @@ def process_seq(settings, file_name): seq += subseq yield seq, d[1] +# Used for sequence_nest_rnn_multi_input.conf +@provider(input_types=[integer_value_sub_sequence(10), + integer_value(3)], + should_shuffle=False) +def process_subseq2(settings, file_name): + for d in data: + yield d + +# Used for sequence_rnn_multi_input.conf +@provider(input_types=[integer_value_sequence(10), + integer_value(3)], + should_shuffle=False) +def process_seq2(settings, file_name): + for d in data: + seq = [] + for subseq in d[0]: + seq += subseq + yield seq, d[1] + +########################################################### data2 = [ [[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]] ,0], [[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]], 1], ] +# Used for sequence_nest_rnn_multi_unequalength_inputs.conf @provider(input_types=[integer_value_sub_sequence(10), integer_value_sub_sequence(10), integer_value(2)], @@ -52,6 +76,7 @@ def process_unequalength_subseq(settings, file_name): yield d +# Used for sequence_rnn_multi_unequalength_inputs.conf @provider(input_types=[integer_value_sequence(10), integer_value_sequence(10), integer_value(2)], diff --git a/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf b/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf index e8222cef525a8..0614958b4719d 100644 --- a/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf +++ b/paddle/gserver/tests/sequence_nest_rnn_multi_input.conf @@ -19,7 +19,7 @@ from paddle.trainer_config_helpers import * define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', test_list=None, module='rnn_data_provider', - obj='process_subseq') + obj='process_subseq2') settings(batch_size=2, learning_rate=0.01) diff --git a/paddle/gserver/tests/sequence_rnn_multi_input.conf b/paddle/gserver/tests/sequence_rnn_multi_input.conf index 968621cab59be..51881e21d971b 100644 --- a/paddle/gserver/tests/sequence_rnn_multi_input.conf +++ b/paddle/gserver/tests/sequence_rnn_multi_input.conf @@ -19,7 +19,7 @@ from paddle.trainer_config_helpers import * define_py_data_sources2(train_list='gserver/tests/Sequence/dummy.list', test_list=None, module='rnn_data_provider', - obj='process_seq') + obj='process_seq2') settings(batch_size=2, learning_rate=0.01) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index c5723f8574ab3..eab9bf84141a2 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -307,6 +307,24 @@ TEST(Layer, blockExpandLayer) { } } +TEST(Layer, maxoutLayer) { + TestConfig config; + config.biasSize = 0; + config.layerConfig.set_type("maxout"); + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 4096, 0}); + LayerInputConfig* input = config.layerConfig.add_inputs(); + MaxOutConfig* maxout = input->mutable_maxout_conf(); + + maxout->set_img_size_x(32); + maxout->set_img_size_y(32); + maxout->set_channels(4); + maxout->set_groups(2); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "maxout", 10, false, useGpu); + } +} void testFcLayer(string format, size_t nnz) { TestConfig config; config.biasSize = 4096; diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 78519ce7aa874..843eabc97d642 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -583,6 +583,42 @@ void GpuMatrix::colMax(Matrix& max) { max.maxCols(*this); } +void GpuMatrix::colMax(IVector& maxIds, Matrix& maxVal) { + LOG(FATAL) << "Is not supported"; +} + +void GpuMatrix::maxoutForward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + CHECK(dynamic_cast(&a)); + CHECK(dynamic_cast(&id)); + CHECK_EQ(a.getHeight(), getHeight()); + + size_t size = getWidth(); + size_t batchSize = getHeight(); + const real* input = a.getData(); + real* output = getData(); + int* idForGpu = id.getData(); + + hl_maxout_forward(input, output, idForGpu, batchSize, size, + size / channels, groups); +} + +void GpuMatrix::maxoutBackward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + CHECK(dynamic_cast(&a)); + CHECK(dynamic_cast(&id)); + CHECK_EQ(a.getHeight(), getHeight()); + + size_t size = a.getWidth(); + size_t batchSize = getHeight(); + real* input = getData(); + const real* output = a.getData(); + const int* idForGpu = id.getData(); + + hl_maxout_backward(input, output, idForGpu, batchSize, size, + size / channels, groups); +} + /*calulate the error of classification */ void GpuMatrix::classificationError(MatrixPtr output, IVectorPtr label) { GpuMatrixPtr output_ptr = std::dynamic_pointer_cast(output); @@ -2748,6 +2784,95 @@ void CpuMatrix::colMax(Matrix& max) { max.maxCols(*this); } +void CpuMatrix::colMax(IVector& maxIds, Matrix& maxVal) { + CHECK(isContiguous()); + CHECK(!maxIds.useGpu() && !maxVal.useGpu()) << "Matrix type are not equal"; + size_t numSamples = getWidth(); + size_t beam = maxVal.getHeight(); + CHECK_EQ(maxIds.getSize(), numSamples * beam); + CHECK_EQ(maxVal.getWidth(), numSamples); + + real* a = getData(); + int* s = maxIds.getData(); + real* t = maxVal.getData(); + size_t dim = getHeight(); + for (size_t i = 0; i < numSamples; i++) { + std::vector> vec; + for (size_t j = 0; j < dim; j++) { + vec.push_back(std::pair(a[i + j * numSamples], j)); + } + + std::partial_sort( + vec.begin(), vec.begin() + beam, vec.end(), + [](const std::pair& l, const std::pair& r) { + return l.first > r.first; + }); + for (size_t j = 0; j < beam; j++) { + t[i + j * numSamples] = vec[j].first; + s[i + j * numSamples] = vec[j].second; + } + } +} + +void CpuMatrix::maxoutForward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + CHECK(dynamic_cast(&a)); + CHECK(dynamic_cast(&id)); + CHECK_EQ(a.getHeight(), getHeight()); + + size_t size = getWidth(); + size_t batchSize = getHeight(); + size_t featLen = size / channels; + const real* input = a.getData(); + int* idForCpu = id.getData(); + + MatrixPtr maxInMat, maxOutMat; + Matrix::resizeOrCreate(maxInMat, groups, size, false, false); + Matrix::resizeOrCreate(maxOutMat, 1, size, false, false); + + for (size_t batch_idx = 0; batch_idx < batchSize; ++batch_idx) { + size_t newIndex = batch_idx * size; + IVectorPtr tmpId = IVector::create(idForCpu + newIndex, size, false); + + for (size_t i = 0; i < channels; ++i) { + size_t newFeatLen = i * featLen; + for (size_t j = 0; j < groups; ++j) { + maxInMat->subMatrix(j, j + 1, newFeatLen, newFeatLen + featLen) + ->copyFrom(input + (newIndex + newFeatLen) * groups + j * featLen, + featLen); + } + } + maxInMat->colMax(*tmpId, *maxOutMat); + this->subRowMatrix(batch_idx, batch_idx + 1)->copyFrom(*maxOutMat); + } +} + +void CpuMatrix::maxoutBackward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + CHECK(dynamic_cast(&a)); + CHECK(dynamic_cast(&id)); + CHECK_EQ(a.getHeight(), getHeight()); + + size_t size = a.getWidth(); + size_t batchSize = getHeight(); + size_t featLen = size / channels; + size_t newFeatLen = groups * featLen; + real* inputG = getData(); + const real* outG = a.getData(); + int* idForCpu = id.getData(); + + for (size_t batch_idx = 0; batch_idx < batchSize; ++batch_idx) { + size_t newIndex = batch_idx * size; + int* idData = idForCpu + newIndex; + + for (size_t i = 0; i < size; ++i) { + int gradIdx = + idData[i] * featLen + (i / featLen) * newFeatLen + i % featLen; + (inputG + newIndex * groups)[gradIdx] += (outG + newIndex)[i]; + } + } +} + void CpuMatrix::rowNormalizeL1(Matrix& out) { CHECK(!out.useGpu()); diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index 25104fe1c6d70..047c76a8604cc 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -493,16 +493,40 @@ class Matrix : public BaseMatrix { LOG(FATAL) << "Not implemeted"; } + /** + * set the max of each column of this to mat + */ virtual void colMax(Matrix& max) { LOG(FATAL) << "not implemented"; } + /** + * @brief Get the top k elements of each column of this matrix. + * + * The row ids and values of these elements are stored in + * maxIds and max respectively. where k is the size of maxIds. + * And note that the top k elements are not sorted. + */ + virtual void colMax(IVector& maxIds, Matrix& maxVal) { + LOG(FATAL) << "not implemented"; + } + + virtual void maxoutForward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + LOG(FATAL) << "not implemented"; + } + + virtual void maxoutBackward(Matrix& a, IVector& id, size_t channels, + size_t groups) { + LOG(FATAL) << "not implemented"; + } + virtual void rowMaxId(IVector& maxIds) { LOG(FATAL) << "Not implemented"; } /** * @brief Get the top k elements of each row of this matrix. * * The column ids and values of these elements are stored in - * maxIds and max respectively. Note that the top k - * elements are not sorted. + * maxIds and max respectively. where k is the size of maxIds. + * And note that the top k elements are not sorted. */ virtual void rowMax(IVector& maxIds, Matrix& max) { LOG(FATAL) << "Not implemented"; @@ -1085,6 +1109,9 @@ class GpuMatrix : public Matrix { void rowMax(Matrix& max); void rowMax(IVector& maxIds, Matrix& max); void colMax(Matrix& max); + void colMax(IVector& maxIds, Matrix& max); + void maxoutForward(Matrix& a, IVector& id, size_t channels, size_t groups); + void maxoutBackward(Matrix& a, IVector& id, size_t channels, size_t groups); void oneHotCrossEntropy(Matrix& output, IVector& label); void oneHotCrossEntropyBp(Matrix& outputV, IVector& label); @@ -1395,6 +1422,9 @@ class CpuMatrix : public Matrix { void rowMax(Matrix& max); void rowMax(IVector& maxIds, Matrix& maxVal); void colMax(Matrix& max); + void colMax(IVector& maxIds, Matrix& maxVal); + void maxoutForward(Matrix& a, IVector& id, size_t channels, size_t groups); + void maxoutBackward(Matrix& a, IVector& id, size_t channels, size_t groups); void rowNormalizeL1(Matrix& out); void oneHotCrossEntropy(Matrix& output, IVector& label); diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index e1bda79a8acb1..ac160479a9dfc 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -1999,6 +1999,78 @@ TEST(Matrix, PoolFwdBwd) { } } +void testMaxOutFwdBwd(int numSamples, int imgSizeH, int imgSizeW, + int channels, int groups) { + int inWidth = imgSizeH * imgSizeW * channels; + int outChannels = channels / groups; + int outWidth = imgSizeH * imgSizeW * outChannels; + + // forward + MatrixPtr input = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpu = GpuMatrix::create(numSamples, inWidth, false, true); + + MatrixPtr target = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpu = GpuMatrix::create(numSamples, outWidth, false, true); + MatrixPtr targetCheck = CpuMatrix::create(numSamples, outWidth, false, false); + + IVectorPtr id = CpuIVector::create(numSamples * outWidth, false); + IVectorPtr idGpu = GpuIVector::create(numSamples * outWidth, true); + IVectorPtr idCheck = CpuIVector::create(numSamples * outWidth, false); + + input->randomizeUniform(); + inputGpu->copyFrom(*input); + + target->maxoutForward(*input, *id, outChannels, groups); + targetGpu->maxoutForward(*inputGpu, *idGpu, outChannels, groups); + + // check + targetCheck->copyFrom(*targetGpu); + MatrixCheckErr(*target, *targetCheck); + idCheck->copyFrom(*idGpu); + VectorCheckEqual(*id, *idCheck); + + // backward + MatrixPtr inputGrad = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpuGrad = GpuMatrix::create(numSamples, inWidth, false, true); + + MatrixPtr targetGrad = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpuGrad = GpuMatrix::create(numSamples, outWidth, false, + true); + MatrixPtr targetCheckGrad = CpuMatrix::create(numSamples, inWidth, false, + false); + + inputGrad->randomizeUniform(); + targetGrad->randomizeUniform(); + inputGpuGrad->copyFrom(*inputGrad); + targetGpuGrad->copyFrom(*targetGrad); + + inputGrad->maxoutBackward(*targetGrad, *id, outChannels, groups); + inputGpuGrad->maxoutBackward(*targetGpuGrad, *idGpu, outChannels, groups); + + // check + targetCheckGrad->copyFrom(*inputGpuGrad); + MatrixCheckErr(*inputGrad, *targetCheckGrad); +} + +TEST(Matrix, MaxOutFwdBwd) { + for (auto numSamples : {5, 10}) { + for (auto channels : {8, 16}) { + for (auto imgSizeH : {14, 28}) { + for (auto imgSizeW : {16, 30}) { + for (auto groups : {2, 4}) { + VLOG(3) << " numSamples=" << numSamples + << " channels=" << channels + << " imgSizeH=" << imgSizeH + << " imgSizeW=" << imgSizeW + << " groups=" << groups; + testMaxOutFwdBwd(numSamples, imgSizeH, imgSizeW, channels, groups); + } + } + } + } + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index 25e36f9c4c168..70c1f8d563238 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -170,6 +170,15 @@ message BlockExpandConfig { required uint32 img_size_y = 11; } +message MaxOutConfig { + required uint32 channels = 1; + required uint32 groups = 2; + + // The size of input feature map. + required uint32 img_size_x = 3; + required uint32 img_size_y = 4; +} + message ProjectionConfig { required string type = 1; required string name = 2; @@ -225,6 +234,7 @@ message LayerInputConfig { // If the input layer has multi-output. // Set the argument name. optional string input_layer_argument = 9; + optional MaxOutConfig maxout_conf = 10; } message LayerConfig { diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index c1e74c7a2d8f7..fe8a5e5d48767 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -469,6 +469,7 @@ def __init__( pool=None, image=None, block_expand=None, + maxout=None, format=None, nnz=None, is_static=None, @@ -785,6 +786,16 @@ def __init__( output_y = 0): self.add_keys(locals()) +@config_class +class MaxOut(Cfg): + def __init__( + self, + channels, + groups, + img_size_x = 0, + img_size_y = 0): + self.add_keys(locals()) + def DataBase(async_load_data=False, constant_slots=None, data_ratio=1, @@ -1082,6 +1093,12 @@ def parse_block_expand(block_expand, input_layer_name, block_expand_conf): int(math.ceil((2 * block_expand.padding_y + block_expand.img_size_y \ - block_expand.block_y) / float(block_expand.stride_y))) +def parse_maxout(maxout, input_layer_name, maxout_conf): + maxout_conf.channels = maxout.channels + maxout_conf.groups = maxout.groups + maxout_conf.img_size_x = maxout.img_size_x + maxout_conf.img_size_y = maxout.img_size_y + # Define an evaluator @config_func def Evaluator( @@ -1705,6 +1722,21 @@ def __init__( self.set_layer_size(block_expand_conf.block_x * block_expand_conf.block_y * block_expand_conf.channels) +@config_layer('maxout') +class MaxOutLayer(LayerBase): + def __init__( + self, + name, + inputs, + **xargs): + super(MaxOutLayer, self).__init__(name, 'maxout', 0, inputs=inputs, **xargs) + input_layer = self.get_input_layer(0) + parse_maxout(self.inputs[0].maxout, + input_layer.name, + self.config.inputs[0].maxout_conf) + maxout_conf = self.config.inputs[0].maxout_conf + self.set_layer_size(g_layer_map[input_layer.name].size / maxout_conf.groups) + # key: cost type # value: cost class g_cost_map = {} diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index d45a9b53dcc94..c4e8fe4abc026 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -55,7 +55,7 @@ 'multi_binary_label_cross_entropy', 'rank_cost', 'lambda_cost', 'huber_cost', # 'block_expand_layer', # TODO(yuyang18): this layer is not correct - 'out_prod_layer', 'print_layer' + 'maxout_layer', 'out_prod_layer', 'print_layer' ] @@ -110,6 +110,7 @@ class LayerType(object): SLOPE_INTERCEPT_LAYER = "slope_intercept" LINEAR_COMBINATION_LAYER = "convex_comb" BLOCK_EXPAND = "blockexpand" + MAXOUT = "maxout" PRINT_LAYER = "print" @@ -3362,6 +3363,73 @@ def block_expand_layer(input, return LayerOutput(name, LayerType.BLOCK_EXPAND, parents=[input]) +@wrap_name_default() +@layer_support() +def maxout_layer(input, + groups, + num_channels=None, + size_x=None, + size_y=None, + name=None, + layer_attr=None): + """ + A layer to do max out on conv layer output. + - Input: output of a conv layer. + - Output: feature map size same as input. Channel is (input channel) / groups. + + So groups should be larger than 1, and the num of channels should be able + to devided by groups. + + Please refer to Paper: + - Maxout Networks: http://www.jmlr.org/proceedings/papers/v28/goodfellow13.pdf + - Multi-digit Number Recognition from Street View \ + Imagery using Deep Convolutional Neural Networks: \ + https://arxiv.org/pdf/1312.6082v4.pdf + + The simple usage is: + + .. code-block:: python + + maxout = maxout_layer(input, + num_channels=128, + groups=4) + + :param input: The input layer. + :type input: LayerOutput + :param num_channels: The channel number of input layer. If None will be set + automatically from previous output. + :type num_channels: int|None + :param groups: The group number of input layer. + :type groups: int + :param size_x: conv output width. If None will be set + automatically from previous output. + :type size_x: int|None + :param size_y: conv output height. If None will be set + automatically from previous output. + :type size_y: int|None + :param name: The name of this layer, which can not specify. + :type name: None|basestring. + :param layer_attr: Extra Layer attribute. + :type layer_attr: ExtraLayerAttribute + :return: LayerOutput object. + :rtype: LayerOutput + """ + assert input.layer_type == LayerType.CONV_LAYER + assert isinstance(input.activation, LinearActivation) + assert groups > 1 + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + assert num_channels % groups == 0 + Layer(name=name, + inputs=Input(input.name, + maxout=MaxOut(channels=num_channels, + groups=groups)), + type=LayerType.MAXOUT, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput(name, LayerType.MAXOUT, parents=[input]) + + @wrap_name_default() @layer_support() def ctc_layer(input, label, size=None, name=None, norm_by_times=False, diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 index 96bf3fb2e19d6..88ce5c129e552 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/check.md5 +++ b/python/paddle/trainer_config_helpers/tests/configs/check.md5 @@ -12,6 +12,7 @@ a5d9259ff1fd7ca23d0ef090052cb1f2 last_first_seq.protostr 8bb44e1e5072d0c261572307e7672bda test_grumemory_layer.protostr 1f3510672dce7a9ed25317fc58579ac7 test_hsigmoid.protostr d350bd91a0dc13e854b1364c3d9339c6 test_lstmemory_layer.protostr +6fa59551808ee7012bbd24f757e782d2 test_maxout.protostr 251a948ba41c1071afcd3d9cf9c233f7 test_ntm_layers.protostr e6ff04e70aea27c7b06d808cc49c9497 test_print_layer.protostr 2a75dd33b640c49a8821c2da6e574577 test_rnn_group.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index 7cdd682056fd4..4b1d2d3d41d52 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -8,7 +8,8 @@ configs=(test_fc layer_activations projections test_print_layer test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers util_layers simple_rnn_layers unused_layers test_cost_layers -test_rnn_group shared_fc shared_lstm test_cost_layers_with_weight) +test_rnn_group shared_fc shared_lstm test_cost_layers_with_weight +test_maxout) for conf in ${configs[*]} diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py b/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py new file mode 100644 index 0000000000000..079e2cf4c4320 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py @@ -0,0 +1,30 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +data = data_layer(name='data', size=2304) + +conv = img_conv_layer(input=data, + filter_size = 3, + num_channels=1, + num_filters=16, + padding=1, + act=LinearActivation(), + bias_attr=True) + +maxout = maxout_layer(input=conv, + num_channels=16, + groups=2) + +pool = img_pool_layer(input=maxout, + num_channels=8, + pool_size=2, + stride=2, + pool_type=MaxPooling()) + +fc = fc_layer(input=pool, size=384, bias_attr=False) + +outputs(fc) From 652b83478fd36ed17d0c911c3564c795a537f440 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Mon, 24 Oct 2016 21:06:42 +0800 Subject: [PATCH 181/324] remove deprecated start input in img_pool_layer (#237) --- python/paddle/trainer_config_helpers/layers.py | 6 ++---- python/paddle/trainer_config_helpers/networks.py | 12 ++++-------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index c4e8fe4abc026..f8c32dc91f10b 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1630,7 +1630,7 @@ def img_conv_layer(input, filter_size, num_filters, @layer_support() def img_pool_layer(input, pool_size, name=None, num_channels=None, pool_type=None, - stride=1, start=None, padding=0, layer_attr=None, + stride=1, padding=0, layer_attr=None, pool_size_y=None, stride_y=None, padding_y=None, img_width=None): """ @@ -1661,8 +1661,6 @@ def img_pool_layer(input, pool_size, name=None, :type stride: int :param stride_y: stride height of pooling. It is equal to stride by default. :type stride_y: int|None - :param start: start position of pooling operation. Note it is deprecated now. - :type start: int|None :param layer_attr: Extra Layer attribute. :type layer_attr: ExtraLayerAttribute :param img_width: the width of input feature map. If it is None, the input feature @@ -1696,7 +1694,7 @@ def img_pool_layer(input, pool_size, name=None, pool_type=type_name, channels=num_channels, size_x=pool_size, - start=start, + start=None, stride=stride, padding=padding, size_y=pool_size_y, diff --git a/python/paddle/trainer_config_helpers/networks.py b/python/paddle/trainer_config_helpers/networks.py index d8f96195020b4..65512b327cdc6 100644 --- a/python/paddle/trainer_config_helpers/networks.py +++ b/python/paddle/trainer_config_helpers/networks.py @@ -133,7 +133,7 @@ def simple_img_conv_pool(input, filter_size, num_filters, pool_size, name=None, pool_type=None, act=None, groups=1, conv_stride=1, conv_padding=0, bias_attr=None, num_channel=None, param_attr=None, shared_bias=True, - conv_layer_attr=None, pool_stride=1, pool_start=None, + conv_layer_attr=None, pool_stride=1, pool_padding=0, pool_layer_attr=None): """ Simple image convolution and pooling group. @@ -172,8 +172,6 @@ def simple_img_conv_pool(input, filter_size, num_filters, pool_size, name=None, :type conv_layer_attr: ExtraLayerAttribute :param pool_stride: see img_pool_layer for details :type pool_stride: int - :param pool_start: see img_pool_layer for details. It is deprecated now. - :type pool_start: int :param pool_padding: see img_pool_layer for details :type pool_padding: int :param pool_layer_attr: see img_pool_layer for details @@ -192,7 +190,7 @@ def simple_img_conv_pool(input, filter_size, num_filters, pool_size, name=None, return img_pool_layer(name="%s_pool" % name, input=_conv_, pool_size=pool_size, pool_type=pool_type, stride=pool_stride, - start=pool_start, padding=pool_padding, + padding=pool_padding, layer_attr=pool_layer_attr) @@ -203,7 +201,7 @@ def img_conv_bn_pool(input, filter_size, num_filters, pool_size, name=None, conv_param_attr=None, shared_bias=True, conv_layer_attr=None, bn_param_attr=None, bn_bias_attr=None, bn_layer_attr=None, pool_stride=1, - pool_start=None, pool_padding=0, pool_layer_attr=None): + pool_padding=0, pool_layer_attr=None): """ Convolution, batch normalization, pooling group. @@ -243,8 +241,6 @@ def img_conv_bn_pool(input, filter_size, num_filters, pool_size, name=None, :param bn_layer_attr: ParameterAttribute. :param pool_stride: see img_pool_layer's document. :type pool_stride: int - :param pool_start: see img_pool_layer's document. It is deprecated now. - :type pool_start: int :param pool_padding: see img_pool_layer's document. :type pool_padding: int :param pool_layer_attr: see img_pool_layer's document. @@ -268,7 +264,7 @@ def img_conv_bn_pool(input, filter_size, num_filters, pool_size, name=None, return img_pool_layer(name="%s_pool" % name, input=__bn__, pool_type=pool_type, pool_size=pool_size, stride=pool_stride, - start=pool_start, padding=pool_padding, + padding=pool_padding, layer_attr=pool_layer_attr) From 07b2e5d54c61956eee65ddda7c0f704cb89f7b2a Mon Sep 17 00:00:00 2001 From: Z-TAO Date: Tue, 25 Oct 2016 05:04:36 +0800 Subject: [PATCH 182/324] Fix dataprovider converter for sparse data --- paddle/py_paddle/dataprovider_converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/py_paddle/dataprovider_converter.py b/paddle/py_paddle/dataprovider_converter.py index 0366bb636c704..6d8f5da3e298f 100644 --- a/paddle/py_paddle/dataprovider_converter.py +++ b/paddle/py_paddle/dataprovider_converter.py @@ -63,7 +63,8 @@ def __init__(self, input_type, pos): def scan(self, dat): self.extend_cols(dat) - self.__rows__.append(len(dat)) + self.__rows__.append(len(dat) + self.__rows__[-1]) + self.__height__ += 1 def extend_cols(self, dat): self.__cols__.extend(dat) From 9c5c38fa2ab67fb09c0100637da5f5efbe5e71d5 Mon Sep 17 00:00:00 2001 From: gangliao Date: Mon, 24 Oct 2016 22:14:34 -0700 Subject: [PATCH 183/324] FIx check type unmatch in MaxOutLayer (#242) Compiled failed on gcc 4.6 --- paddle/gserver/layers/MaxOutLayer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/gserver/layers/MaxOutLayer.cpp b/paddle/gserver/layers/MaxOutLayer.cpp index 106ab26ba1aae..a3de069bf7a6c 100644 --- a/paddle/gserver/layers/MaxOutLayer.cpp +++ b/paddle/gserver/layers/MaxOutLayer.cpp @@ -46,7 +46,7 @@ bool MaxOutLayer::init(const LayerMap& layerMap, Layer::init(layerMap, parameterMap); /* the size of inputs for maxout-layer is 1 */ - CHECK_EQ(config_.inputs_size(), 1UL); + CHECK_EQ(config_.inputs_size(), 1); const MaxOutConfig& conf = config_.inputs(0).maxout_conf(); groups_ = conf.groups(); From d6944dec16933a48396e4dc03e13a365cba60fb8 Mon Sep 17 00:00:00 2001 From: emailweixu Date: Tue, 25 Oct 2016 18:40:05 -0700 Subject: [PATCH 184/324] Sequence tagging demo (#225) --- demo/sequence_tagging/data/get_data.sh | 21 ++ demo/sequence_tagging/data/test.list | 1 + demo/sequence_tagging/data/train.list | 1 + demo/sequence_tagging/dataprovider.py | 258 ++++++++++++++++++ demo/sequence_tagging/linear_crf.py | 84 ++++++ demo/sequence_tagging/readme.md | 45 +++ demo/sequence_tagging/rnn_crf.py | 130 +++++++++ demo/sequence_tagging/train.sh | 10 + demo/sequence_tagging/train_linear.sh | 9 + .../trainer_config_helpers/optimizers.py | 15 +- 10 files changed, 572 insertions(+), 2 deletions(-) create mode 100755 demo/sequence_tagging/data/get_data.sh create mode 100644 demo/sequence_tagging/data/test.list create mode 100644 demo/sequence_tagging/data/train.list create mode 100644 demo/sequence_tagging/dataprovider.py create mode 100644 demo/sequence_tagging/linear_crf.py create mode 100644 demo/sequence_tagging/readme.md create mode 100644 demo/sequence_tagging/rnn_crf.py create mode 100755 demo/sequence_tagging/train.sh create mode 100755 demo/sequence_tagging/train_linear.sh diff --git a/demo/sequence_tagging/data/get_data.sh b/demo/sequence_tagging/data/get_data.sh new file mode 100755 index 0000000000000..e579d6c46ce5e --- /dev/null +++ b/demo/sequence_tagging/data/get_data.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +DIR="$( cd "$(dirname "$0")" ; pwd -P )" +cd $DIR + +wget http://www.cnts.ua.ac.be/conll2000/chunking/train.txt.gz +wget http://www.cnts.ua.ac.be/conll2000/chunking/test.txt.gz diff --git a/demo/sequence_tagging/data/test.list b/demo/sequence_tagging/data/test.list new file mode 100644 index 0000000000000..073c0a0c9063a --- /dev/null +++ b/demo/sequence_tagging/data/test.list @@ -0,0 +1 @@ +data/test.txt.gz diff --git a/demo/sequence_tagging/data/train.list b/demo/sequence_tagging/data/train.list new file mode 100644 index 0000000000000..43c24d5f6484a --- /dev/null +++ b/demo/sequence_tagging/data/train.list @@ -0,0 +1 @@ +data/train.txt.gz diff --git a/demo/sequence_tagging/dataprovider.py b/demo/sequence_tagging/dataprovider.py new file mode 100644 index 0000000000000..6f412d6834be6 --- /dev/null +++ b/demo/sequence_tagging/dataprovider.py @@ -0,0 +1,258 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer.PyDataProvider2 import * +import gzip +import logging + +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', +) +logger = logging.getLogger('paddle') +logger.setLevel(logging.INFO) + +OOV_POLICY_IGNORE = 0 +OOV_POLICY_USE = 1 +OOV_POLICY_ERROR = 2 + +num_original_columns = 3 + +# Feature combination patterns. +# [[-1,0], [0,0]] means previous token at column 0 and current token at +# column 0 are combined as one feature. +patterns = [ + [[-2,0]], + [[-1,0]], + [[0,0]], + [[1,0]], + [[2,0]], + + [[-1,0], [0,0]], + [[0,0], [1,0]], + + [[-2,1]], + [[-1,1]], + [[0,1]], + [[1,1]], + [[2,1]], + [[-2,1], [-1,1]], + [[-1,1], [0,1]], + [[0,1], [1,1]], + [[1,1], [2,1]], + + [[-2,1], [-1,1], [0,1]], + [[-1,1], [0,1], [1,1]], + [[0,1], [1,1], [2,1]], +] + +dict_label = { + 'B-ADJP': 0, + 'I-ADJP': 1, + 'B-ADVP': 2, + 'I-ADVP': 3, + 'B-CONJP': 4, + 'I-CONJP': 5, + 'B-INTJ': 6, + 'I-INTJ': 7, + 'B-LST': 8, + 'I-LST': 9, + 'B-NP': 10, + 'I-NP': 11, + 'B-PP': 12, + 'I-PP': 13, + 'B-PRT': 14, + 'I-PRT': 15, + 'B-SBAR': 16, + 'I-SBAR': 17, + 'B-UCP': 18, + 'I-UCP': 19, + 'B-VP': 20, + 'I-VP': 21, + 'O': 22 +} + +def make_features(sequence): + length = len(sequence) + num_features = len(sequence[0]) + def get_features(pos): + if pos < 0: + return ['#B%s' % -pos] * num_features + if pos >= length: + return ['#E%s' % (pos - length + 1)] * num_features + return sequence[pos] + + for i in xrange(length): + for pattern in patterns: + fname = '/'.join([get_features(i+pos)[f] for pos, f in pattern]) + sequence[i].append(fname) + +''' +Source file format: +Each line is for one timestep. The features are separated by space. +An empty line indicates end of a sequence. + +cutoff: a list of numbers. If count of a feature is smaller than this, + it will be ignored. +if oov_policy[i] is OOV_POLICY_USE, id 0 is reserved for OOV features of +i-th column. + +return a list of dict for each column +''' +def create_dictionaries(filename, cutoff, oov_policy): + def add_to_dict(sequence, dicts): + num_features = len(dicts) + for features in sequence: + l = len(features) + assert l == num_features, "Wrong number of features " + line + for i in xrange(l): + if features[i] in dicts[i]: + dicts[i][features[i]] += 1 + else: + dicts[i][features[i]] = 1 + + num_features = len(cutoff) + dicts = [] + for i in xrange(num_features): + dicts.append(dict()) + + f = gzip.open(filename, 'rb') + + sequence = [] + + for line in f: + line = line.strip() + if not line: + make_features(sequence) + add_to_dict(sequence, dicts) + sequence = [] + continue + features = line.split(' ') + sequence.append(features) + + + for i in xrange(num_features): + dct = dicts[i] + n = 1 if oov_policy[i] == OOV_POLICY_USE else 0 + todo = [] + for k, v in dct.iteritems(): + if v < cutoff[i]: + todo.append(k) + else: + dct[k] = n + n += 1 + + if oov_policy[i] == OOV_POLICY_USE: + # placeholder so that len(dct) will be the number of features + # including OOV + dct['#OOV#'] = 0 + + logger.info('column %d dict size=%d, ignored %d' % (i, n, len(todo))) + for k in todo: + del dct[k] + + f.close() + return dicts + + +def initializer(settings, **xargs): + cutoff = [3, 1, 0] + cutoff += [3] * len(patterns) + oov_policy = [OOV_POLICY_IGNORE, OOV_POLICY_ERROR, OOV_POLICY_ERROR] + oov_policy += [OOV_POLICY_IGNORE] * len(patterns) + dicts = create_dictionaries('data/train.txt.gz', cutoff, oov_policy) + dicts[2] = dict_label + settings.dicts = dicts + settings.oov_policy = oov_policy + input_types = [] + num_features = len(dicts) + for i in xrange(num_original_columns): + input_types.append(integer_sequence(len(dicts[i]))) + logger.info("slot %s size=%s" % (i, len(dicts[i]))) + if patterns: + dim = 0 + for i in xrange(num_original_columns, num_features): + dim += len(dicts[i]) + input_types.append(sparse_binary_vector_sequence(dim)) + logger.info("feature size=%s" % dim) + settings.input_types = input_types + +''' +if oov_policy[i] == OOV_POLICY_USE, features in i-th column which are not +existed in dicts[i] will be assigned to id 0. +if oov_policy[i] == OOV_POLICY_ERROR, all features in i-th column MUST exist +in dicts[i]. +''' +@provider(init_hook=initializer, cache=CacheType.CACHE_PASS_IN_MEM) +def process(settings, filename): + input_file = filename + dicts = settings.dicts + oov_policy = settings.oov_policy + + def gen_sample(sequence): + num_features = len(dicts) + sample = [list() for i in xrange(num_original_columns)] + if patterns: + sample.append([]) + for features in sequence: + assert len(features) == num_features, \ + "Wrong number of features: " + line + for i in xrange(num_original_columns): + id = dicts[i].get(features[i], -1) + if id != -1: + sample[i].append(id) + elif oov_policy[i] == OOV_POLICY_IGNORE: + sample[i].append(0xffffffff) + elif oov_policy[i] == OOV_POLICY_ERROR: + logger.fatal("Unknown token: %s" % features[i]) + else: + sample[i].append(0) + + if patterns: + dim = 0 + vec = [] + for i in xrange(num_original_columns, num_features): + id = dicts[i].get(features[i], -1) + if id != -1: + vec.append(dim + id) + elif oov_policy[i] == OOV_POLICY_IGNORE: + pass + elif oov_policy[i] == OOV_POLICY_ERROR: + logger.fatal("Unknown token: %s" % features[i]) + else: + vec.ids.append(dim + 0) + + dim += len(dicts[i]) + sample[-1].append(vec) + return sample + + num_features = len(dicts) + f = gzip.open(input_file, 'rb') + + num_sequences = 0 + sequence = [] + for line in f: + line = line.strip() + if not line: + make_features(sequence) + yield gen_sample(sequence) + sequence = [] + num_sequences += 1 + continue + features = line.split(' ') + sequence.append(features) + + f.close() + + logger.info("num_sequences=%s" % num_sequences) + diff --git a/demo/sequence_tagging/linear_crf.py b/demo/sequence_tagging/linear_crf.py new file mode 100644 index 0000000000000..2bd1a20bc52fc --- /dev/null +++ b/demo/sequence_tagging/linear_crf.py @@ -0,0 +1,84 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +import math + +define_py_data_sources2(train_list="data/train.list", + test_list="data/test.list", + module="dataprovider", + obj="process") + + +batch_size = 1 +settings( + learning_method=MomentumOptimizer(), + batch_size=batch_size, + regularization=L2Regularization(batch_size * 1e-4), + average_window=0.5, + learning_rate=1e-1, + learning_rate_decay_a=1e-5, + learning_rate_decay_b=0.25, +) + +num_label_types=23 + +def get_simd_size(size): + return int(math.ceil(float(size) / 8)) * 8 + +# Currently, in order to use sparse_update=True, +# the size has to be aligned. +num_label_types = get_simd_size(num_label_types) + +features = data_layer(name="features", size=76328) +word = data_layer(name="word", size=6778) +pos = data_layer(name="pos", size=44) +chunk = data_layer(name="chunk", + size=num_label_types) + +crf_input = fc_layer( + input=features, + size=num_label_types, + act=LinearActivation(), + bias_attr=False, + param_attr=ParamAttr(initial_std=0, sparse_update=True)) + +crf=crf_layer( + input=crf_input, + label=chunk, + param_attr=ParamAttr(name="crfw", initial_std=0), +) + +crf_decoding=crf_decoding_layer( + size=num_label_types, + input=crf_input, + label=chunk, + param_attr=ParamAttr(name="crfw"), +) + +sum_evaluator( + name="error", + input=crf_decoding, +) + +chunk_evaluator( + name="chunk_f1", + input =[crf_decoding, chunk], + chunk_scheme="IOB", + num_chunk_types=11, +) + +inputs(word, pos, chunk, features) +outputs(crf) diff --git a/demo/sequence_tagging/readme.md b/demo/sequence_tagging/readme.md new file mode 100644 index 0000000000000..2e17fffb83c53 --- /dev/null +++ b/demo/sequence_tagging/readme.md @@ -0,0 +1,45 @@ +# Sequence Tagging + +This demo is a sequence model for assigning tags to each token in a sentence. The task is described at CONLL2000 Text Chunking task. + +## Download data +```bash +cd demo/sequence_tagging +./data/get_data.sh +``` + +## Train model +```bash +cd demo/sequence_tagging +./train.sh +``` + +## Model description + +We provide two models. One is a linear CRF model (linear_crf.py) with is equivalent to the one at leon.bottou.org/projects/sgd. The second one is a stacked bidirectional RNN and CRF model (rnn_crf.py). +
+ + + + + + + + + + + + + + + + + + + + + + +
Model nameNumber of parametersF1 score
linear_crf 1.8M 0.937
rnn_crf 960K 0.941
+
+
diff --git a/demo/sequence_tagging/rnn_crf.py b/demo/sequence_tagging/rnn_crf.py new file mode 100644 index 0000000000000..fb157bf3ea719 --- /dev/null +++ b/demo/sequence_tagging/rnn_crf.py @@ -0,0 +1,130 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +import math + +define_py_data_sources2(train_list="data/train.list", + test_list="data/test.list", + module="dataprovider", + obj="process") + +batch_size = 16 +settings( + learning_method=MomentumOptimizer(), + batch_size=batch_size, + regularization=L2Regularization(batch_size * 1e-5), + average_window=0.5, + learning_rate = 2e-3, + learning_rate_decay_a = 5e-7, + learning_rate_decay_b = 0.5, +) + +word_dim=128 +hidden_dim = 128 +with_rnn = True + +initial_std=1/math.sqrt(hidden_dim) +param_attr=ParamAttr(initial_std=initial_std) +cpu_layer_attr=ExtraLayerAttribute(device=-1) + +default_device(0) + +num_label_types=23 + +features = data_layer(name="features", size=76328) +word = data_layer(name="word", size=6778) +pos = data_layer(name="pos", size=44) +chunk = data_layer(name="chunk", + size=num_label_types, + layer_attr=cpu_layer_attr) + +emb = embedding_layer( + input=word, size=word_dim, param_attr=ParamAttr(initial_std=0)) + +hidden1 = mixed_layer( + size=hidden_dim, + act=STanhActivation(), + bias_attr=True, + input=[full_matrix_projection(emb), + table_projection(pos, param_attr=param_attr)] +) + +if with_rnn: + rnn1 = recurrent_layer( + act=ReluActivation(), + bias_attr=True, + input=hidden1, + param_attr=ParamAttr(initial_std=0), + ) + +hidden2 = mixed_layer( + size=hidden_dim, + act=STanhActivation(), + bias_attr=True, + input=[full_matrix_projection(hidden1) + ] + ([ + full_matrix_projection(rnn1, param_attr=ParamAttr(initial_std=0)) + ] if with_rnn else []), +) + +if with_rnn: + rnn2=recurrent_layer( + reverse=True, + act=ReluActivation(), + bias_attr=True, + input=hidden2, + param_attr=ParamAttr(initial_std=0), + ) + +crf_input = mixed_layer( + size=num_label_types, + bias_attr=False, + input=[ + full_matrix_projection(hidden2), + ] + ([ + full_matrix_projection(rnn2, param_attr=ParamAttr(initial_std=0)) + ] if with_rnn else []), +) + +crf = crf_layer( + input=crf_input, + label=chunk, + param_attr=ParamAttr(name="crfw", initial_std=0), + layer_attr=cpu_layer_attr, +) + +crf_decoding = crf_decoding_layer( + size=num_label_types, + input=crf_input, + label=chunk, + param_attr=ParamAttr(name="crfw"), + layer_attr=cpu_layer_attr, +) + +sum_evaluator( + name="error", + input=crf_decoding, +) + +chunk_evaluator( + name="chunk_f1", + input =[crf_decoding, chunk], + chunk_scheme="IOB", + num_chunk_types=11, +) + +inputs(word, pos, chunk, features) +outputs(crf) diff --git a/demo/sequence_tagging/train.sh b/demo/sequence_tagging/train.sh new file mode 100755 index 0000000000000..9a706b98d8686 --- /dev/null +++ b/demo/sequence_tagging/train.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +paddle train \ + --config rnn_crf.py \ + --parallel_nn=1 \ + --use_gpu=1 \ + --dot_period=10 \ + --log_period=1000 \ + --test_period=0 \ + --num_passes=10 diff --git a/demo/sequence_tagging/train_linear.sh b/demo/sequence_tagging/train_linear.sh new file mode 100755 index 0000000000000..597b5afea9c63 --- /dev/null +++ b/demo/sequence_tagging/train_linear.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +paddle train \ + --config linear_crf.py \ + --use_gpu=0 \ + --dot_period=100 \ + --log_period=10000 \ + --test_period=0 \ + --num_passes=10 diff --git a/python/paddle/trainer_config_helpers/optimizers.py b/python/paddle/trainer_config_helpers/optimizers.py index 4660a6b5003da..d4b947517b7d0 100644 --- a/python/paddle/trainer_config_helpers/optimizers.py +++ b/python/paddle/trainer_config_helpers/optimizers.py @@ -362,6 +362,13 @@ def __extends__(dict1, dict2): default_factory=lambda _: BaseRegularization()) def settings(batch_size, learning_rate=1e-3, + learning_rate_decay_a=0., + learning_rate_decay_b=0., + learning_rate_schedule='poly', + learning_rate_args='', + average_window=0, + do_average_in_cpu=False, + max_average_window=None, learning_method=None, regularization=None, is_async=False, @@ -408,10 +415,14 @@ def settings(batch_size, else: algorithm = 'owlqn' + args=['batch_size', 'learning_rate', 'learning_rate_decay_a', + 'learning_rate_decay_b', 'learning_rate_schedule', + 'learning_rate_args', 'average_window', 'do_average_in_cpu', + 'max_average_window'] kwargs = dict() - kwargs['batch_size'] = batch_size - kwargs['learning_rate'] = learning_rate kwargs['algorithm'] = algorithm + for arg in args: + kwargs[arg] = locals()[arg] kwargs = __extends__(kwargs, learning_method.to_setting_kwargs()) learning_method.extra_settings() From d1d52bb7d4c1f9ad4703aeb17027135bcfc41a7d Mon Sep 17 00:00:00 2001 From: emailweixu Date: Tue, 25 Oct 2016 18:49:40 -0700 Subject: [PATCH 185/324] Update contribute_to_paddle.md (#248) --- doc/build/contribute_to_paddle.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/build/contribute_to_paddle.md b/doc/build/contribute_to_paddle.md index 06fcff6172075..bbdbb4d4227d0 100644 --- a/doc/build/contribute_to_paddle.md +++ b/doc/build/contribute_to_paddle.md @@ -99,3 +99,7 @@ git pull --rebase upstream HEAD git push -f origin HEAD ``` Now your Pull Request is updated with the latest version. + +## Revise your pull request + +When you revise your pull request according to reviewer's comments, please use 'git commit' instead of 'git commit --amend' to commit your changes so that the reviewers can see the difference between the new pull requrest and the old pull request. From 46bd5f53e3e536e712f3ee36c700834206db7e7a Mon Sep 17 00:00:00 2001 From: backyes Date: Wed, 26 Oct 2016 15:43:15 +0800 Subject: [PATCH 186/324] add input sparse data check for sparse layer at runtime (#247) * add input sparse data check for sparse layer at runtime, to avoid invalid data access at pserver end while doing prefetch * remote sparse design support binary sparse and float saprse both --- paddle/math/SparseRowMatrix.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/paddle/math/SparseRowMatrix.cpp b/paddle/math/SparseRowMatrix.cpp index 0b5de252258a9..6986624d25c7a 100644 --- a/paddle/math/SparseRowMatrix.cpp +++ b/paddle/math/SparseRowMatrix.cpp @@ -227,12 +227,18 @@ void CacheRowCpuMatrix::mul(CpuSparseMatrix* a, CpuMatrix* b, real scaleAB, void SparsePrefetchRowCpuMatrix::addRows(const unsigned int* ids, size_t len) { std::vector& localIndices = indexDictHandle_->localIndices; + for (size_t i = 0; i < len; i ++) { + CHECK_LT(*(ids + i), this->getHeight()) + << "id:" << *(ids + i) << "Height:" << this->getHeight() + << "sparse id value exceeds the max input dimension, " + << "it could be caused invalid input data samples"; + } localIndices.insert(localIndices.end(), ids, ids + len); } void SparsePrefetchRowCpuMatrix::addRows(MatrixPtr input) { CpuSparseMatrix* mat = dynamic_cast(input.get()); - CHECK(mat) << "only support non value sparse matrix"; + CHECK(mat) << "only support sparse matrix"; addRows(reinterpret_cast(mat->getCols()), mat->getElementCnt()); } @@ -243,7 +249,13 @@ void SparsePrefetchRowCpuMatrix::addRows(IVectorPtr ids) { int* index = ids->getData(); for (size_t i = 0; i < numSamples; ++i) { if (index[i] == -1) continue; - localIndices.push_back((unsigned int)index[i]); + + unsigned int id = (unsigned int)index[i]; + CHECK_LT(id, this->getHeight()) + << "id:" << id << "Height:" << this->getHeight() + << "sparse id value exceeds the max input dimension, " + << "it could be caused invalid input data samples"; + localIndices.push_back(id); } } From cbe734b396a6f6844cafb94b0de8bd736360ef90 Mon Sep 17 00:00:00 2001 From: emailweixu Date: Thu, 27 Oct 2016 00:23:46 -0700 Subject: [PATCH 187/324] Python trainer api (#193) * Python trainer API and demo * Adding missing PaddleAPIPrivate.h * Adding api_train.sh * More comments * Bump up patch version to 0b3 --- CMakeLists.txt | 2 +- cmake/swig.cmake | 1 + demo/quick_start/api_train.py | 114 +++++++++++ demo/quick_start/api_train.sh | 29 +++ demo/quick_start/train.sh | 2 +- demo/quick_start/trainer_config.lr.py | 3 +- demo/sentiment/predict.py | 4 +- paddle/api/Arguments.cpp | 19 +- paddle/api/CMakeLists.txt | 3 + paddle/api/ConfigParser.cpp | 44 ++--- paddle/api/GradientMachine.cpp | 18 +- paddle/api/PaddleAPI.h | 40 ++-- paddle/api/PaddleAPIPrivate.h | 68 +++++++ paddle/api/ParameterOptimizer.cpp | 4 +- paddle/api/Trainer.cpp | 175 +++++++++-------- paddle/api/test/run_tests.sh | 2 +- paddle/api/test/testTrain.py | 3 +- paddle/api/test/testTrainer.py | 63 +++++++ paddle/py_paddle/dataprovider_converter.py | 2 +- paddle/py_paddle/util.py | 107 +++++++++-- paddle/trainer/Tester.cpp | 56 ++++-- paddle/trainer/Tester.h | 12 +- paddle/trainer/Trainer.cpp | 208 ++++++++++++--------- paddle/trainer/Trainer.h | 22 ++- paddle/trainer/TrainerInternal.cpp | 10 +- paddle/trainer/TrainerInternal.h | 4 +- proto/TrainerConfig.proto.m4 | 2 +- python/paddle/proto/__init__.py | 2 + 28 files changed, 707 insertions(+), 312 deletions(-) create mode 100644 demo/quick_start/api_train.py create mode 100755 demo/quick_start/api_train.sh create mode 100644 paddle/api/PaddleAPIPrivate.h create mode 100644 paddle/api/test/testTrainer.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 4613155f7700b..e96ce28248ee5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 2.8) project(paddle CXX C) set(PADDLE_MAJOR_VERSION 0) set(PADDLE_MINOR_VERSION 8) -set(PADDLE_PATCH_VERSION 0b2) +set(PADDLE_PATCH_VERSION 0b3) set(PADDLE_VERSION ${PADDLE_MAJOR_VERSION}.${PADDLE_MINOR_VERSION}.${PADDLE_PATCH_VERSION}) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") diff --git a/cmake/swig.cmake b/cmake/swig.cmake index f5c1bcc79b3dc..160d7ee56a9c6 100644 --- a/cmake/swig.cmake +++ b/cmake/swig.cmake @@ -27,6 +27,7 @@ function(generate_python_api target_name) COMMAND swig -python -c++ -outcurrentdir -I../ api/Paddle.swig && mv ${PROJ_ROOT}/paddle/swig_paddle.py ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py DEPENDS ${PROJ_ROOT}/paddle/api/Paddle.swig + ${PROJ_ROOT}/paddle/api/PaddleAPI.h WORKING_DIRECTORY ${PROJ_ROOT}/paddle COMMENT "Generate Python API from swig") add_custom_target(${target_name} ALL DEPENDS diff --git a/demo/quick_start/api_train.py b/demo/quick_start/api_train.py new file mode 100644 index 0000000000000..5ae19b8d26534 --- /dev/null +++ b/demo/quick_start/api_train.py @@ -0,0 +1,114 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import itertools +import random + +from paddle.trainer.config_parser import parse_config +from py_paddle import swig_paddle as api +from py_paddle import DataProviderConverter +from paddle.trainer.PyDataProvider2 \ + import integer_value, integer_value_sequence, sparse_binary_vector + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("--train_data", + type=str, required=False, help="train data file") + parser.add_argument("--test_data", type=str, help="test data file") + parser.add_argument("--config", + type=str, required=True, help="config file name") + parser.add_argument("--dict_file", required=True, help="dictionary file") + parser.add_argument("--seq", + default=1, type=int, + help="whether use sequence training") + parser.add_argument("--use_gpu", default=0, type=int, + help="whether use GPU for training") + parser.add_argument("--trainer_count", default=1, type=int, + help="Number of threads for training") + parser.add_argument("--num_passes", default=5, type=int, + help="Number of training passes") + return parser.parse_args() + +UNK_IDX = 0 + +def load_data(file_name, word_dict): + with open(file_name, 'r') as f: + for line in f: + label, comment = line.strip().split('\t') + words = comment.split() + word_slot = [word_dict.get(w, UNK_IDX) for w in words] + yield word_slot, int(label) + +def load_dict(dict_file): + word_dict = dict() + with open(dict_file, 'r') as f: + for i, line in enumerate(f): + w = line.strip().split()[0] + word_dict[w] = i + return word_dict + +def main(): + options = parse_arguments() + api.initPaddle("--use_gpu=%s" % options.use_gpu, + "--trainer_count=%s" % options.trainer_count) + + word_dict = load_dict(options.dict_file) + train_dataset = list(load_data(options.train_data, word_dict)) + if options.test_data: + test_dataset = list(load_data(options.test_data, word_dict)) + else: + test_dataset = None + + trainer_config = parse_config(options.config, + "dict_file=%s" % options.dict_file) + # No need to have data provider for trainer + trainer_config.ClearField('data_config') + trainer_config.ClearField('test_data_config') + + # create a GradientMachine from the model configuratin + model = api.GradientMachine.createFromConfigProto( + trainer_config.model_config) + # create a trainer for the gradient machine + trainer = api.Trainer.create(trainer_config, model) + + # create a data converter which converts data to PaddlePaddle + # internal format + input_types = [ + integer_value_sequence(len(word_dict)) if options.seq + else sparse_binary_vector(len(word_dict)), + integer_value(2)] + converter = DataProviderConverter(input_types) + + batch_size = trainer_config.opt_config.batch_size + trainer.startTrain() + for train_pass in xrange(options.num_passes): + trainer.startTrainPass() + random.shuffle(train_dataset) + for pos in xrange(0, len(train_dataset), batch_size): + batch = itertools.islice(train_dataset, pos, pos + batch_size) + size = min(batch_size, len(train_dataset) - pos) + trainer.trainOneDataBatch(size, converter(batch)) + trainer.finishTrainPass() + if test_dataset: + trainer.startTestPeriod(); + for pos in xrange(0, len(test_dataset), batch_size): + batch = itertools.islice(test_dataset, pos, pos + batch_size) + size = min(batch_size, len(test_dataset) - pos) + trainer.testOneDataBatch(size, converter(batch)) + trainer.finishTestPeriod() + trainer.finishTrain() + +if __name__ == '__main__': + main() diff --git a/demo/quick_start/api_train.sh b/demo/quick_start/api_train.sh new file mode 100755 index 0000000000000..40e9d0a09aaa6 --- /dev/null +++ b/demo/quick_start/api_train.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +# Note: if using trainer_config.emb.py, trainer_config.cnn.py +# or trainer_config.lstm.py, you need to change --seq to --seq=1 +# because they are sequence models. +python api_train.py \ + --config=trainer_config.lr.py \ + --trainer_count=2 \ + --num_passes=15 \ + --use_gpu=0 \ + --seq=0 \ + --train_data=data/train.txt \ + --test_data=data/test.txt \ + --dict_file=data/dict.txt \ + 2>&1 | tee 'train.log' diff --git a/demo/quick_start/train.sh b/demo/quick_start/train.sh index ea4e32249a3d0..49806292a4ec5 100755 --- a/demo/quick_start/train.sh +++ b/demo/quick_start/train.sh @@ -24,7 +24,7 @@ paddle train \ --config=$cfg \ --save_dir=./output \ --trainer_count=4 \ - --log_period=20 \ + --log_period=100 \ --num_passes=15 \ --use_gpu=false \ --show_parameter_stats_period=100 \ diff --git a/demo/quick_start/trainer_config.lr.py b/demo/quick_start/trainer_config.lr.py index 119e3849a4b7e..c6059947f30b3 100644 --- a/demo/quick_start/trainer_config.lr.py +++ b/demo/quick_start/trainer_config.lr.py @@ -16,7 +16,7 @@ from paddle.trainer_config_helpers import * -dict_file = "./data/dict.txt" +dict_file = get_config_arg('dict_file', str, "./data/dict.txt") word_dict = dict() with open(dict_file, 'r') as f: for i, line in enumerate(f): @@ -63,7 +63,6 @@ label = data_layer(name="label", size=2) # Define cross-entropy classification loss and error. - classification_cost(input=output, label=label) cls = classification_cost(input=output, label=label) outputs(cls) else: diff --git a/demo/sentiment/predict.py b/demo/sentiment/predict.py index c61628d34db4a..7d0baeabbba68 100755 --- a/demo/sentiment/predict.py +++ b/demo/sentiment/predict.py @@ -46,8 +46,8 @@ def __init__(self, train_conf, dict_file, model_dir=None, label_file = None): conf = parse_config(train_conf, "is_predict=1") self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) self.network.loadParameters(self.model_dir) - slots = [integer_value_sequence(self.dict_dim)] - self.converter = DataProviderConverter(slots) + input_types = [integer_value_sequence(self.dict_dim)] + self.converter = DataProviderConverter(input_types) def load_dict(self): """ diff --git a/paddle/api/Arguments.cpp b/paddle/api/Arguments.cpp index 8f73e7626042c..6f51d55120069 100644 --- a/paddle/api/Arguments.cpp +++ b/paddle/api/Arguments.cpp @@ -14,27 +14,10 @@ limitations under the License. */ #include "PaddleAPI.h" +#include "PaddleAPIPrivate.h" #include "paddle/parameter/Argument.h" -struct ArgumentsPrivate { - std::vector outputs; - - inline paddle::Argument& getArg(size_t idx) throw(RangeError) { - if (idx < outputs.size()) { - return outputs[idx]; - } else { - RangeError e; - throw e; - } - } - - template - std::shared_ptr& cast(void* rawPtr) const { - return *(std::shared_ptr*)(rawPtr); - } -}; - size_t Arguments::getSlotNum() const { return m->outputs.size(); } Arguments* Arguments::createArguments(size_t slotNum) { diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index fe0da763514a6..9b2d122a09ada 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -40,6 +40,8 @@ configure_file( generate_python_api(python_swig_sources) +file(GLOB PY_PADDLE_PYTHON_FILES ${PROJ_ROOT}/paddle/py_paddle/*.py) + # TODO(yuyang18) : make wheel name calculated by cmake add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/dist/.timestamp COMMAND ${PYTHON_EXECUTABLE} setup.py bdist_wheel @@ -55,6 +57,7 @@ add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/dist/.timestamp paddle_trainer paddle_api paddle_cuda + ${PY_PADDLE_PYTHON_FILES} ) install(DIRECTORY ${PROJ_ROOT}/paddle/dist/ diff --git a/paddle/api/ConfigParser.cpp b/paddle/api/ConfigParser.cpp index c5ee784a0bda0..25d94f5a6a125 100644 --- a/paddle/api/ConfigParser.cpp +++ b/paddle/api/ConfigParser.cpp @@ -14,17 +14,9 @@ limitations under the License. */ #include "PaddleAPI.h" +#include "PaddleAPIPrivate.h" #include "paddle/trainer/Trainer.h" -struct TrainerConfigPrivate { - std::shared_ptr conf; - TrainerConfigPrivate() : conf(std::make_shared()) {} -}; - -struct ModelConfigPrivate { - std::shared_ptr conf; -}; - struct ParameterConfigPrivate { paddle::ParameterPtr parameter; paddle::ParameterConfig config; @@ -39,19 +31,6 @@ struct ParameterConfigPrivate { } }; -struct OptimizationConfigPrivate { - std::shared_ptr trainer_config; - paddle::OptimizationConfig config; - - paddle::OptimizationConfig& getConfig() { - if (trainer_config != nullptr) { - return *trainer_config->mutable_opt_config(); - } else { - return config; - } - } -}; - TrainerConfig::TrainerConfig() : m(new TrainerConfigPrivate()) {} TrainerConfig::~TrainerConfig() { delete m; } @@ -59,10 +38,19 @@ TrainerConfig::~TrainerConfig() { delete m; } TrainerConfig* TrainerConfig::createFromTrainerConfigFile( const std::string& confPath) { LOG(INFO) << "load trainer config from " << confPath; - paddle::TrainerConfigHelper helper(confPath); - //! TODO(yuyang18): Make TrainerConfigPrivate to TrainerConfigHelper + auto conf = std::make_shared(confPath); auto retv = new TrainerConfig(); - *retv->m->conf = helper.getConfig(); + retv->m->conf = conf; + return retv; +} + +TrainerConfig* TrainerConfig::createFromProtoString( + const std::string& str) { + auto retv = new TrainerConfig(); + paddle::TrainerConfig trainerConfigProto; + auto conf = std::make_shared(trainerConfigProto); + CHECK(conf->getMutableConfig().ParseFromString(str)); + retv->m->conf = conf; return retv; } @@ -76,10 +64,6 @@ ModelConfig* TrainerConfig::getModelConfig() const { return retv; } -void* ModelConfig::getPaddleModelConfig() const { - return m->conf->mutable_model_config(); -} - ParameterConfig::ParameterConfig() : m(new ParameterConfigPrivate()) {} ParameterConfig::~ParameterConfig() { @@ -132,8 +116,6 @@ OptimizationConfig* TrainerConfig::getOptimizationConfig() const { return opt_config; } -void* OptimizationConfig::getRawPtr() { return &m->getConfig(); } - OptimizationConfig* OptimizationConfig::createFromProtoString( const std::string& str) { auto conf = new OptimizationConfig(); diff --git a/paddle/api/GradientMachine.cpp b/paddle/api/GradientMachine.cpp index 6f1d63575a80f..bef499c67858b 100644 --- a/paddle/api/GradientMachine.cpp +++ b/paddle/api/GradientMachine.cpp @@ -14,30 +14,22 @@ limitations under the License. */ #include "PaddleAPI.h" -#include "paddle/gserver/gradientmachines/GradientMachine.h" +#include "PaddleAPIPrivate.h" + #include "paddle/gserver/gradientmachines/NeuralNetwork.h" #include "Internal.h" std::vector GradientMachine::defaultParamTypes = { PARAMETER_VALUE, PARAMETER_GRADIENT, PARAMETER_MOMENTUM}; -struct GradientMachinePrivate { - std::shared_ptr machine; - - template - inline T& cast(void* ptr) { - return *(T*)(ptr); - } -}; - GradientMachine::GradientMachine() : m(new GradientMachinePrivate()) {} GradientMachine::~GradientMachine() { delete m; } GradientMachine* GradientMachine::createFromPaddleModelPtr( - void* confPtr, GradientMatchineCreateMode mode, + const void* confPtr, GradientMatchineCreateMode mode, const std::vector& types) { - auto& conf = *(paddle::ModelConfig*)(confPtr); + auto& conf = *(const paddle::ModelConfig*)(confPtr); std::vector realTypes; staticCastVector(&realTypes, types); auto machineRawPtr = paddle::GradientMachine::create(conf, mode, realTypes); @@ -66,7 +58,7 @@ GradientMachine* GradientMachine::createByConfigProtoStr( GradientMachine* GradientMachine::createByModelConfig( ModelConfig* conf, GradientMatchineCreateMode mode, const std::vector& types) { - auto confPtr = (paddle::ModelConfig*)conf->getPaddleModelConfig(); + auto confPtr = &conf->m->conf->getModelConfig(); return GradientMachine::createFromPaddleModelPtr(confPtr, mode, types); } diff --git a/paddle/api/PaddleAPI.h b/paddle/api/PaddleAPI.h index b3140617af188..cf790f2f8ef1d 100644 --- a/paddle/api/PaddleAPI.h +++ b/paddle/api/PaddleAPI.h @@ -446,7 +446,6 @@ struct OptimizationConfigPrivate; class OptimizationConfig { DISABLE_COPY_AND_ASSIGN(OptimizationConfig); OptimizationConfig(); - void* getRawPtr(); public: static OptimizationConfig* createFromProtoString(const std::string& str); @@ -462,6 +461,7 @@ class OptimizationConfig { friend class TrainerConfig; friend class ParameterOptimizer; + friend class Trainer; }; struct ParameterPrivate; @@ -515,8 +515,6 @@ class ModelConfig { virtual ~ModelConfig(); private: - void* getPaddleModelConfig() const; - ModelConfigPrivate* m; friend class TrainerConfig; friend struct TrainerConfigPrivate; @@ -539,6 +537,7 @@ class TrainerConfig { static TrainerConfig* createFromTrainerConfigFile( const std::string& configPath); + static TrainerConfig* createFromProtoString(const std::string& str); ModelConfig* getModelConfig() const; @@ -546,6 +545,7 @@ class TrainerConfig { private: TrainerConfigPrivate* m; + friend class Trainer; }; /** @@ -700,11 +700,12 @@ class GradientMachine { GradientMachinePrivate* m; static GradientMachine* createFromPaddleModelPtr( - void* confPtr, GradientMatchineCreateMode mode, + const void* confPtr, GradientMatchineCreateMode mode, const std::vector& types); // Not to use c++ 11 init-list, so we use static var as function default arg. static std::vector defaultParamTypes; + friend class Trainer; }; struct TrainerPrivate; @@ -712,6 +713,7 @@ class Trainer { private: TrainerPrivate* m; Trainer(); + Trainer(TrainerConfig* optConfig, GradientMachine* gm); DISABLE_COPY_AND_ASSIGN(Trainer); public: @@ -720,38 +722,42 @@ class Trainer { /// Create A Trainer By TrainerConfig. using paddle command line. static Trainer* createByCommandLine() throw(IOError); - /// Start Train. + static Trainer* create(TrainerConfig* optConfig, GradientMachine* gm) + throw(IOError); + + /// Start training void startTrain(); + + /// Finish training void finishTrain(); - /// Start Pass. + /// Start a pass. void startTrainPass(); - void finishTrainPass(); - void setBatchSize(size_t batchSize); + /// Finish a pass + void finishTrainPass(); /** * Train one batch, * - * @param batchSize -1 wiil use command line or batch size set before, - * otherwise use this batchSize for train. - * * @return true if all batch finished. */ - bool trainOneBatch(size_t batchSize = -1UL); + bool trainOneBatch(size_t batchSize); - bool prepareBatchData(size_t batchSize = -1UL); + void trainOneDataBatch(size_t batchSize, const Arguments& args); - void finishTrainOneBatch(); + void startTestPeriod(); + void testOneDataBatch(size_t batchSize, const Arguments& args); + void finishTestPeriod(); - void forwardOneBatch() throw(UnsupportError); + void forwardOneBatch(size_t batchSize); - Arguments* getNetworkOutput(); + Arguments* getForwardOutput(); Matrix* getLayerOutput(const std::string& layerName); }; -/// The N-Best results generated from one input sequence. +/// the N-Best results generated from one input sequence. class ISequenceResults { public: virtual ~ISequenceResults(); diff --git a/paddle/api/PaddleAPIPrivate.h b/paddle/api/PaddleAPIPrivate.h new file mode 100644 index 0000000000000..93cdca8c4beaa --- /dev/null +++ b/paddle/api/PaddleAPIPrivate.h @@ -0,0 +1,68 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/gserver/gradientmachines/GradientMachine.h" +#include "paddle/trainer/TrainerConfigHelper.h" + +#pragma once + +struct GradientMachinePrivate { + std::shared_ptr machine; + + template + inline T& cast(void* ptr) { + return *(T*)(ptr); + } +}; + +struct OptimizationConfigPrivate { + std::shared_ptr trainer_config; + paddle::OptimizationConfig config; + + const paddle::OptimizationConfig& getConfig() { + if (trainer_config != nullptr) { + return trainer_config->getOptConfig(); + } else { + return config; + } + } +}; + +struct TrainerConfigPrivate { + std::shared_ptr conf; + TrainerConfigPrivate() {} +}; + +struct ModelConfigPrivate { + std::shared_ptr conf; +}; + +struct ArgumentsPrivate { + std::vector outputs; + + inline paddle::Argument& getArg(size_t idx) throw(RangeError) { + if (idx < outputs.size()) { + return outputs[idx]; + } else { + RangeError e; + throw e; + } + } + + template + std::shared_ptr& cast(void* rawPtr) const { + return *(std::shared_ptr*)(rawPtr); + } +}; + diff --git a/paddle/api/ParameterOptimizer.cpp b/paddle/api/ParameterOptimizer.cpp index e087defc6043c..b13761ab0900d 100644 --- a/paddle/api/ParameterOptimizer.cpp +++ b/paddle/api/ParameterOptimizer.cpp @@ -14,6 +14,7 @@ limitations under the License. */ #include "PaddleAPI.h" +#include "PaddleAPIPrivate.h" #include "paddle/parameter/ParameterOptimizer.h" #include "Internal.h" #include @@ -60,10 +61,9 @@ ParameterOptimizer::~ParameterOptimizer() { ParameterOptimizer* ParameterOptimizer::create(OptimizationConfig* config) { CHECK(config != nullptr); - auto opt_config_ptr = (paddle::OptimizationConfig*)config->getRawPtr(); auto retOptimizer = new ParameterOptimizer(); retOptimizer->m->optimizer.reset( - paddle::ParameterOptimizer::create(*opt_config_ptr, false)); + paddle::ParameterOptimizer::create(config->m->getConfig(), false)); return retOptimizer; } diff --git a/paddle/api/Trainer.cpp b/paddle/api/Trainer.cpp index 95b578c8db9fd..b61f36f740d47 100644 --- a/paddle/api/Trainer.cpp +++ b/paddle/api/Trainer.cpp @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "PaddleAPI.h" +#include "PaddleAPIPrivate.h" #include #include @@ -30,31 +31,17 @@ P_DECLARE_string(config); P_DECLARE_string(init_model_path); P_DECLARE_int32(start_pass); -struct TrainPassContext { - int64_t batchId; - int32_t batchSize; - real avgTestCost; - int64_t numAvgTests; - int passInnerId; - paddle::DataBatch data; - std::vector forwardOutput; -}; - struct TrainerPrivate : public paddle::Trainer { - void startTrain(); - void finishTrain(); - - void startTrainPass(); - void finishTrainPass(); - - bool _trainOneBatch(); - - bool _prepareBatchData(); - void _forwardOneBatch() throw(UnsupportError); - + bool _trainOneBatch(size_t batchSize); + bool forwardOneBatch(size_t batchSize); + void forwardOneDataBatch(const std::vector& inArgs); + void setBatchSize(size_t batchSize); + std::vector& getForwardOutput(); + + void startTestPeriod(); + void finishTestPeriod(); + void testOneDataBatch(const paddle::DataBatch& dataBatch); TrainerPrivate() : paddle::Trainer() {} - - TrainPassContext trainPassContext; }; Trainer::Trainer() : m(new TrainerPrivate()) { @@ -75,61 +62,76 @@ Trainer* Trainer::createByCommandLine() throw(IOError) { } } -void Trainer::startTrain() { m->startTrain(); } +Trainer::Trainer(TrainerConfig* config, GradientMachine* gm) + : m(new TrainerPrivate()) { + m->init(config->m->conf, /* testing= */false, gm ? gm->m->machine : nullptr); +} -void TrainerPrivate::startTrain() { - srand(this->config_->getConfig().start_pass() + 1); - this->dataProvider_->reset(); - this->trainerInternal_.getGradientMachine()->start(*config_, dataProvider_); +Trainer* Trainer::create(TrainerConfig* config, GradientMachine* gm) + throw(IOError) +{ + auto retv = new Trainer(config, gm); + if (retv->m->getConfig().IsInitialized()) { + return retv; + } else { + retv->m->getConfig().CheckInitialized(); + throw IOError(); + } } -void Trainer::finishTrain() { m->finishTrain(); } +void Trainer::startTrain() { m->startTrain(); } -void TrainerPrivate::finishTrain() { - this->trainerInternal_.getGradientMachine()->finish(); -} +void Trainer::finishTrain() { m->finishTrain(); } void Trainer::startTrainPass() { m->startTrainPass(); } -void TrainerPrivate::startTrainPass() { - this->stats_.reset(); - this->trainPassContext.batchId = 0; - this->trainPassContext.batchSize = this->config_->getOptConfig().batch_size(); - this->trainPassContext.avgTestCost = 0; - this->trainPassContext.numAvgTests = 0; - this->trainPassContext.passInnerId = 0; - this->trainerInternal_.getParameterUpdater()->startPass(); - this->evaluator_->start(); -} - void Trainer::finishTrainPass() { m->finishTrainPass(); } -void TrainerPrivate::finishTrainPass() { - this->trainerInternal_.getGradientMachine()->onPassEnd(); - this->trainerInternal_.getParameterUpdater()->finishPass(); - evaluator_->finish(); +void Trainer::trainOneDataBatch(size_t batchSize, const Arguments& inArgs) { + paddle::DataBatch dataBatch; + dataBatch.getStreams() = inArgs.m->outputs; + dataBatch.setSize(batchSize); + m->trainOneDataBatch(dataBatch); } -void Trainer::setBatchSize(size_t batchSize) { - this->m->trainPassContext.batchSize = batchSize; +bool Trainer::trainOneBatch(size_t batchSize) { + return m->_trainOneBatch(batchSize); } -bool Trainer::trainOneBatch(size_t batchSize) { - if (batchSize == -1UL) { - this->setBatchSize(batchSize); +bool TrainerPrivate::_trainOneBatch(size_t batchSize) { + paddle::DataBatch dataBatch; + CHECK(dataProvider_) << "data_provider is not specified"; + int num = dataProvider_->getNextBatch(batchSize, &dataBatch); + if (num == 0) { + return false; } - return m->_trainOneBatch(); + trainOneDataBatch(dataBatch); + return false; } -bool TrainerPrivate::_trainOneBatch() { - if (this->_prepareBatchData()) { - return true; +void TrainerPrivate::startTestPeriod() { + if (!tester_) { + createTester(); } - this->trainerInternal_.trainOneBatch(this->trainPassContext.batchId, - this->trainPassContext.data); - return false; + tester_->startTestPeriod(); +} + +void Trainer::startTestPeriod() { m->startTestPeriod(); } + +void TrainerPrivate::testOneDataBatch(const paddle::DataBatch& dataBatch) { + tester_->testOneDataBatch(dataBatch, &forwardOutput_); +} + +void Trainer::testOneDataBatch(size_t batchSize, const Arguments& args) { + paddle::DataBatch dataBatch; + dataBatch.getStreams() = args.m->outputs; + dataBatch.setSize(batchSize); + m->testOneDataBatch(dataBatch); } +void TrainerPrivate::finishTestPeriod() { tester_->finishTestPeriod(); } +void Trainer::finishTestPeriod() { m->finishTestPeriod(); } + Matrix* Trainer::getLayerOutput(const std::string& layerName) { auto nn = std::dynamic_pointer_cast( this->m->getGradientMachine()); @@ -138,46 +140,37 @@ Matrix* Trainer::getLayerOutput(const std::string& layerName) { return Matrix::createByPaddleMatrixPtr(&m); } -bool Trainer::prepareBatchData(size_t batchSize) { - if (batchSize != -1UL) { - this->setBatchSize(batchSize); +void Trainer::forwardOneBatch(size_t batchSize) { m->forwardOneBatch(batchSize); } + +bool TrainerPrivate::forwardOneBatch(size_t batchSize) { + CHECK(dataProvider_) << "data_provider is not specified"; + paddle::DataBatch dataBatch; + int num = dataProvider_->getNextBatch(batchSize, &dataBatch); + if (num == 0) { + return false; } - return this->m->_prepareBatchData(); -} -bool TrainerPrivate::_prepareBatchData() { - int num = dataProvider_->getNextBatch(this->trainPassContext.batchSize, - &this->trainPassContext.data); - return num == 0; + forwardOneDataBatch(dataBatch.getStreams()); + return true; } -void Trainer::finishTrainOneBatch() { ++m->trainPassContext.batchId; } +void TrainerPrivate::forwardOneDataBatch( + const std::vector& inArgs) { -void Trainer::forwardOneBatch() throw(UnsupportError) { m->_forwardOneBatch(); } - -void TrainerPrivate::_forwardOneBatch() throw(UnsupportError) { - auto& dataBatch = this->trainPassContext.data; - - int64_t actualBatchSize = dataBatch.getSize(); - if (actualBatchSize == 0) { - return; - } - - const std::vector& inArgs = dataBatch.getStreams(); - std::vector& outArgs = this->trainPassContext.forwardOutput; - outArgs.clear(); - paddle::PassType passType = - this->trainerInternal_.getParameterUpdater()->startBatch(actualBatchSize); + std::vector& outArgs = forwardOutput_; if (config_->getOptConfig().use_sparse_remote_updater()) { - this->trainerInternal_.getGradientMachine()->prefetch(inArgs); - this->trainerInternal_.getParameterUpdater()->getParametersRemote(); + trainerInternal_.getGradientMachine()->prefetch(inArgs); + trainerInternal_.getParameterUpdater()->getParametersRemote(); } - this->trainerInternal_.getGradientMachine()->forward( - inArgs, &outArgs, passType); + trainerInternal_.getGradientMachine()->forward( + inArgs, &outArgs, paddle::PASS_TEST); +} + +Arguments* Trainer::getForwardOutput() { + return Arguments::createByPaddleArgumentVector(&m->getForwardOutput()); } -Arguments* Trainer::getNetworkOutput() { - return Arguments::createByPaddleArgumentVector( - &m->trainPassContext.forwardOutput); +std::vector& TrainerPrivate::getForwardOutput() { + return forwardOutput_; } diff --git a/paddle/api/test/run_tests.sh b/paddle/api/test/run_tests.sh index 1fc6fd5a8c185..a4814f98f89c2 100755 --- a/paddle/api/test/run_tests.sh +++ b/paddle/api/test/run_tests.sh @@ -30,7 +30,7 @@ source .test_env/bin/activate pip --timeout 600 install ../../dist/*.whl -test_list="testArguments.py testGradientMachine.py testMatrix.py testVector.py testTrain.py" +test_list="testArguments.py testGradientMachine.py testMatrix.py testVector.py testTrain.py testTrainer.py" export PYTHONPATH=$PWD/../../../python/ diff --git a/paddle/api/test/testTrain.py b/paddle/api/test/testTrain.py index 7f79c2701e9ed..7759118a3d9d1 100644 --- a/paddle/api/test/testTrain.py +++ b/paddle/api/test/testTrain.py @@ -12,9 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from py_paddle import swig_paddle, DataProviderWrapperConverter +from py_paddle import swig_paddle import paddle.trainer.config_parser -from paddle.trainer.PyDataProviderWrapper import DenseSlot, IndexSlot import numpy import util diff --git a/paddle/api/test/testTrainer.py b/paddle/api/test/testTrainer.py new file mode 100644 index 0000000000000..da69a60f84f4d --- /dev/null +++ b/paddle/api/test/testTrainer.py @@ -0,0 +1,63 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer.config_parser import parse_config +from paddle.trainer.config_parser import logger +from py_paddle import swig_paddle +import util + +def main(): + trainer_config = parse_config( + "./testTrainConfig.py", "") + model = swig_paddle.GradientMachine.createFromConfigProto( + trainer_config.model_config) + trainer = swig_paddle.Trainer.create(trainer_config, model) + trainer.startTrain() + for train_pass in xrange(2): + trainer.startTrainPass() + num = 0 + cost = 0 + while True: # Train one batch + batch_size = 1000 + data, atEnd = util.loadMNISTTrainData(batch_size) + if atEnd: + break + trainer.trainOneDataBatch(batch_size, data) + outs = trainer.getForwardOutput() + cost += sum(outs[0]['value']) + num += batch_size + trainer.finishTrainPass() + logger.info('train cost=%f' % (cost / num)) + + trainer.startTestPeriod() + num = 0 + cost = 0 + while True: # Test one batch + batch_size = 1000 + data, atEnd = util.loadMNISTTrainData(batch_size) + if atEnd: + break + trainer.testOneDataBatch(batch_size, data) + outs = trainer.getForwardOutput() + cost += sum(outs[0]['value']) + num += batch_size + trainer.finishTestPeriod() + logger.info('test cost=%f' % (cost / num)) + + trainer.finishTrain() + + +if __name__ == '__main__': + swig_paddle.initPaddle("--use_gpu=0", "--trainer_count=1") + main() diff --git a/paddle/py_paddle/dataprovider_converter.py b/paddle/py_paddle/dataprovider_converter.py index 6d8f5da3e298f..dd2e146d112c0 100644 --- a/paddle/py_paddle/dataprovider_converter.py +++ b/paddle/py_paddle/dataprovider_converter.py @@ -63,7 +63,7 @@ def __init__(self, input_type, pos): def scan(self, dat): self.extend_cols(dat) - self.__rows__.append(len(dat) + self.__rows__[-1]) + self.__rows__.append(len(self.__cols__)) self.__height__ += 1 def extend_cols(self, dat): diff --git a/paddle/py_paddle/util.py b/paddle/py_paddle/util.py index e6cf2710ef523..53f67a861e7d9 100644 --- a/paddle/py_paddle/util.py +++ b/paddle/py_paddle/util.py @@ -79,6 +79,20 @@ def wrap(callback): else: return __ParameterCallbackWrapper__(callback).__disown__() +def __arguments_to_numpy__(i, arg): + assert isinstance(arg, swig_paddle.Arguments) + value = arg.getSlotValue(i) + if value is not None: + assert isinstance(value, swig_paddle.Matrix) + value = value.copyToNumpyMat() + ids = arg.getSlotIds(i) + if ids is not None: + assert isinstance(ids, swig_paddle.IVector) + ids = ids.copyToNumpyArray() + return { + "value": value, + "id": ids + } def __monkeypatch_gradient_machine__(): """ @@ -88,20 +102,6 @@ def __monkeypatch_gradient_machine__(): swig_paddle.GradientMachine.loadFromConfigFile = \ staticmethod(loadGradientMachine) - def __arguments_to_numpy__(i, arg): - assert isinstance(arg, swig_paddle.Arguments) - value = arg.getSlotValue(i) - if value is not None: - assert isinstance(value, swig_paddle.Matrix) - value = value.copyToNumpyMat() - ids = arg.getSlotIds(i) - if ids is not None: - assert isinstance(ids, swig_paddle.IVector) - ids = ids.copyToNumpyArray() - return { - "value": value, - "id": ids - } def __matrix_to_numpy__(m): if isinstance(m, swig_paddle.Matrix): @@ -126,7 +126,7 @@ def createFromConfigProto(protoObj, :type paramTypes: list of int :return: paddle.GradientMachine """ - assert isinstance(protoObj, paddle.proto.ModelConfig_pb2.ModelConfig) + assert isinstance(protoObj, paddle.proto.ModelConfig) return swig_paddle.GradientMachine.createByConfigProtoStr( protoObj.SerializeToString(), createMode, paramTypes) @@ -460,13 +460,29 @@ def OptimizationConfig_createFromProto(protoObj): """ assert isinstance(protoObj, - paddle.proto.TrainerConfig_pb2.OptimizationConfig) + paddle.proto.OptimizationConfig) return swig_paddle.OptimizationConfig.createFromProtoString( protoObj.SerializeToString()) swig_paddle.OptimizationConfig.createFromProto = staticmethod( OptimizationConfig_createFromProto) + def TrainerConfig_createFromProto(protoObj): + """ + Create a new paddle.TrainerConfig from + proto.OptimizationConfig + + :param protoObj: proto.TrainerConfig + :return: paddle.TrainerConfig + """ + assert isinstance(protoObj, + paddle.proto.TrainerConfig) + return swig_paddle.TrainerConfig.createFromProtoString( + protoObj.SerializeToString()) + + swig_paddle.TrainerConfig.createFromProto = staticmethod( + TrainerConfig_createFromProto) + def __monkey_patch_parameter__(): def getBufs(self): @@ -483,9 +499,66 @@ def getBufs(self): swig_paddle.Parameter.getBufs = getBufs +def __monkey_patch_trainer__(): + swig_paddle.Trainer.__create__ = staticmethod(swig_paddle.Trainer.create) + + def Trainer_create(config, model=None): + """ + Create a trainer for model with TrainerCOnfig trainer_config + trainer_config.model_config will be ignored when model is supplied. + Trainer.trainOneBatch() and Trainer.forwardOneBatch() can be used only + when trainer_config.data_config is set. + + A typical usage for Trainer is: + .. code-block:: python + trainer = Trainer.create(trainer_config, model) + for p in xrange(num_passes) + while True: + data = get_next_batch(batch_size) + if not data: + break + trainer.trainOneDataBatch(batch_size, data) + trainer.finishTrainPass() + trainer.finishTrain() + + The trainer will take care of logging, model saving, distributed + training, etc. + + :param config: trainer configuration + :type config: paddle.proto.TrainerConfig + :param model: the model to be trained + :type model: swig_paddle.GradientMachine + :return: a trainer + :rtype swig_paddle.Trainer + + """ + assert isinstance(config, paddle.proto.TrainerConfig) + if model is not None: + assert isinstance(model, swig_paddle.GradientMachine) + return swig_paddle.Trainer.__create__( + swig_paddle.TrainerConfig.createFromProto(config), model) + swig_paddle.Trainer.create = staticmethod(Trainer_create) + + swig_paddle.Trainer.__getForwardOutput__ = \ + swig_paddle.Trainer.getForwardOutput + + def getForwardOutput(self): + """ + Get the netword outputs from the previous trainOneBatch(), + trainOneDataBatch(), testOneDataPatch(), or forwardOneBatch() call. + + :return: list of dictionary with keys ['id', 'value'], each value is a + numpy.ndarray. + """ + outArgs = self.__getForwardOutput__() + return [__arguments_to_numpy__(i, outArgs) for i in xrange( + outArgs.getSlotNum())] + + swig_paddle.Trainer.getForwardOutput = getForwardOutput + def monkeypatches(): patches = [__monkeypatch_init_paddle__, __monkeypatch_gradient_machine__, __monkey_patch_protobuf_objects__, - __monkey_patch_parameter__] + __monkey_patch_parameter__, __monkey_patch_trainer__] for patch in patches: patch() diff --git a/paddle/trainer/Tester.cpp b/paddle/trainer/Tester.cpp index ccf06e1d84edc..b1bb75654a9d5 100644 --- a/paddle/trainer/Tester.cpp +++ b/paddle/trainer/Tester.cpp @@ -71,24 +71,36 @@ Tester::Tester(const std::shared_ptr &config, parameterUpdater_)); } +void Tester::startTestPeriod() { + testEvaluator_->start(); + testContext_.cost = 0; + testContext_.numSamples = 0; + + parameterUpdater_->apply(); + if (intconfig_->prevBatchState) { + gradientMachine_->getState(*intconfig_->trainState); + gradientMachine_->setState(*intconfig_->testState); + } +} + +void Tester::testOneDataBatch( + const DataBatch& dataBatch, std::vector* outArgs) { + testContext_.cost += forwardOneBatch( + dataBatch, testEvaluator_.get(), outArgs); + testContext_.numSamples += dataBatch.getSize(); +} + void Tester::testOnePeriod() { DataBatch dataBatch; int64_t batchSize = config_->getOptConfig().batch_size(); - testEvaluator_->start(); - real cost = 0; - int64_t numSamples = 0; bool testAllData = intconfig_->testPeriod == 0 || intconfig_->testAllDataInOnePeriod; - int batches = testAllData ? std::numeric_limits::max() : intconfig_->testPeriod; - parameterUpdater_->apply(); - if (intconfig_->prevBatchState) { - gradientMachine_->getState(*intconfig_->trainState); - gradientMachine_->setState(*intconfig_->testState); - } + std::vector outArgs; + startTestPeriod(); for (int i = 0; i < batches; ++i) { int num = testDataProvider_->getNextBatch(batchSize, &dataBatch); if (num == 0) { @@ -102,13 +114,17 @@ void Tester::testOnePeriod() { num = testDataProvider_->getNextBatch(batchSize, &dataBatch); } } - cost += testOneBatch(dataBatch, testEvaluator_.get()); - numSamples += num; + testOneDataBatch(dataBatch, &outArgs); } +} + +void Tester::finishTestPeriod() { testEvaluator_->finish(); - CHECK_GT(numSamples, 0) << "There is no samples in your test batch. Possibly " - "wrong implementation of DataProvidor.reset()"; - LOG(INFO) << " Test samples=" << numSamples << " cost=" << cost / numSamples + CHECK_GT(testContext_.numSamples, 0) + << "There is no samples in your test batch. Possibly " + "wrong implementation of DataProvidor.reset()"; + LOG(INFO) << " Test samples=" << testContext_.numSamples + << " cost=" << testContext_.cost / testContext_.numSamples << " Eval: " << *testEvaluator_; parameterUpdater_->restore(); if (intconfig_->prevBatchState) { @@ -128,9 +144,11 @@ int64_t Tester::testOneBatchById(int64_t batchId) { return 0; } + std::vector outArgs; + stats_ += std::pair{ actualBatchSize, - testOneBatch(dataBatch, testEvaluator_.get())}; + forwardOneBatch(dataBatch, testEvaluator_.get(), &outArgs)}; if (((batchId + 1) % intconfig_->logPeriod) == 0) { LOG(INFO) << " Batch=" << batchId + 1 << " " << stats_.getStats(false); @@ -139,7 +157,10 @@ int64_t Tester::testOneBatchById(int64_t batchId) { return actualBatchSize; } -real Tester::testOneBatch(const DataBatch &dataBatch, Evaluator *evaluator) { +real Tester::forwardOneBatch(const DataBatch &dataBatch, + Evaluator *evaluator, + std::vector* pOutArgs) { + auto& outArgs = *pOutArgs; const std::vector& inArgs = dataBatch.getStreams(); if (intconfig_->loadsaveParametersInPserver) { REGISTER_TIMER("prefetch"); @@ -148,12 +169,11 @@ real Tester::testOneBatch(const DataBatch &dataBatch, Evaluator *evaluator) { true /*after apply*/); } - std::vector outArgs; gradientMachine_->forward(inArgs, &outArgs, PASS_TEST); // write features if set this flag and outArgs is not empty std::string featFile = intconfig_->featFile; - if (!featFile.empty() && !outArgs.empty()) { + if (!featFile.empty() && outArgs.empty()) { size_t numOutputs = outArgs.size(); std::vector featMatrices; featMatrices.resize(numOutputs); diff --git a/paddle/trainer/Tester.h b/paddle/trainer/Tester.h index 9663b8def9145..671ffc5220eba 100644 --- a/paddle/trainer/Tester.h +++ b/paddle/trainer/Tester.h @@ -68,6 +68,10 @@ class Tester { * is training at same time. */ void testOnePeriod(); + void startTestPeriod(); + void finishTestPeriod(); + void testOneDataBatch(const DataBatch& dataBatch, + std::vector* outArgs); /** * Test for given data batch. @@ -75,7 +79,9 @@ class Tester { * @param evaluator Evaluator * @return cost */ - real testOneBatch(const DataBatch &dataBatch, Evaluator *evaluator); + real forwardOneBatch(const DataBatch& dataBatch, + Evaluator* evaluator, + std::vector* outArgs); /** @@ -99,6 +105,10 @@ class Tester { std::ofstream os_; std::vector cpuMat_; std::vector cpuVec_; + struct { + int64_t numSamples; + real cost; + } testContext_; private: /** diff --git a/paddle/trainer/Trainer.cpp b/paddle/trainer/Trainer.cpp index 275150e12d12b..04535849eb4aa 100644 --- a/paddle/trainer/Trainer.cpp +++ b/paddle/trainer/Trainer.cpp @@ -196,7 +196,8 @@ void Trainer::init(const std::shared_ptr &config, if (!dataProvider_ && config_->hasDataConfig()) { dataProvider_.reset(DataProvider::create(*config_, *config_, gpuData)); } - if (dataProvider_) { + if (!testDataProvider_) { + // No evaluator_ if there is testDataProvider but no dataProvider. evaluator_.reset(trainerInternal_.getGradientMachine()->makeEvaluator()); currentEvaluator_.reset( trainerInternal_.getGradientMachine()->makeEvaluator()); @@ -215,10 +216,7 @@ void Trainer::init(const std::shared_ptr &config, DataProvider::create(config_->getTestDataConfig(), *config_, gpuData)); } if (testDataProvider_) { - tester_.reset(new Tester(config_, createTesterConfig(), - trainerInternal_.getGradientMachine(), - trainerInternal_.getParameterUpdater(), - testDataProvider_)); + createTester(); } if (!testing && @@ -258,34 +256,25 @@ void Trainer::init(const std::shared_ptr &config, } } - // set current evaluator and evalutor trainerInternal_.setCurrentEvaluator(currentEvaluator_.get()); trainerInternal_.setEvaluator(evaluator_.get()); } void Trainer::train(size_t numPasses) { - srand(config_->getConfig().start_pass() + 1); - dataProvider_->reset(); - - if (this->testDataProvider_) { - this->testDataProvider_->reset(); - } - - trainerInternal_.getGradientMachine()->start(*config_, dataProvider_); - + startTrain(); for (size_t i = 0; i < numPasses; ++i) { if (IGradientMachineMode::trainWholeDataInOneBatch(mode_)) { trainOnePassBatch(config_->getConfig().start_pass() + i); } else { - trainOnePass(config_->getConfig().start_pass() + i); + trainOnePass(); } if (i < numPasses - 1) { dataProvider_->reset(); } } - trainerInternal_.getGradientMachine()->finish(); + finishTrain(); } @@ -387,13 +376,30 @@ real Trainer::checkGradient() { return maxDiff; } -void Trainer::trainOnePass(int passId) { - this->stats_->reset(); - int64_t batchId = 0; - int32_t batchSize = config_->getOptConfig().batch_size(); - real avgTestCost = 0; - int64_t numAvgTests = 0; - int passInnerId = 1; +void Trainer::startTrain() { + trainPassContext_.passId = config_->getConfig().start_pass(); + srand(config_->getConfig().start_pass() + 1); + if (dataProvider_) { + dataProvider_->reset(); + } + + if (this->testDataProvider_) { + this->testDataProvider_->reset(); + } + + trainerInternal_.getGradientMachine()->start(*config_, dataProvider_); +} + +void Trainer::finishTrain() { + trainerInternal_.getGradientMachine()->finish(); +} + +void Trainer::startTrainPass() { + stats_->reset(); + trainPassContext_.batchId = 0; + trainPassContext_.avgTestCost = 0; + trainPassContext_.numAvgTests = 0; + trainPassContext_.passInnerId = 1; trainerInternal_.getParameterUpdater()->startPass(); evaluator_->start(); @@ -401,81 +407,83 @@ void Trainer::trainOnePass(int passId) { trainerInternal_.getGradientMachine()->resetState(); trainerInternal_.getGradientMachine()->getState(testState_); } - while (true) { - DataBatch dataBatch; - - int num = 0; - { - REGISTER_TIMER("getTrainBatch"); - num = dataProvider_->getNextBatch(batchSize, &dataBatch); - } - if (num == 0) break; +} - if (averageEvaluator_) { - int64_t mod = batchId % FLAGS_average_test_period; - if (mod >= FLAGS_average_test_period - FLAGS_log_period) { - if (mod == FLAGS_average_test_period - FLAGS_log_period) { - averageEvaluator_->start(); - } - trainerInternal_.getParameterUpdater()->apply(); - if (FLAGS_prev_batch_state) { - trainerInternal_.getGradientMachine()->getState(trainState_); - } - avgTestCost += - tester_->testOneBatch(dataBatch, averageEvaluator_.get()); - if (FLAGS_prev_batch_state) { - trainerInternal_.getGradientMachine()->setState(trainState_); - } - numAvgTests += num; - trainerInternal_.getParameterUpdater()->restore(); +void Trainer::trainOneDataBatch(DataBatch& dataBatch) { + int num = dataBatch.getSize(); + if (averageEvaluator_) { + int64_t mod = trainPassContext_.batchId % FLAGS_average_test_period; + if (mod >= FLAGS_average_test_period - FLAGS_log_period) { + if (mod == FLAGS_average_test_period - FLAGS_log_period) { + averageEvaluator_->start(); } + trainerInternal_.getParameterUpdater()->apply(); + if (FLAGS_prev_batch_state) { + trainerInternal_.getGradientMachine()->getState(trainState_); + } + trainPassContext_.avgTestCost += + tester_->forwardOneBatch( + dataBatch, averageEvaluator_.get(), &forwardOutput_); + if (FLAGS_prev_batch_state) { + trainerInternal_.getGradientMachine()->setState(trainState_); + } + trainPassContext_.numAvgTests += num; + trainerInternal_.getParameterUpdater()->restore(); } - { - REGISTER_TIMER("TrainBatch"); - trainerInternal_.trainOneBatch(batchId, dataBatch); - } + } + { + REGISTER_TIMER("TrainBatch"); + trainerInternal_.trainOneBatch( + trainPassContext_.batchId, dataBatch, &forwardOutput_); + } - if (averageEvaluator_ && - batchId % FLAGS_average_test_period == FLAGS_average_test_period - 1) { - averageEvaluator_->finish(); - LOG(INFO) << " Averaged parameter:" - << " cost=" << avgTestCost / numAvgTests - << " Eval: " << *averageEvaluator_; - numAvgTests = 0; - avgTestCost = 0; - } + if (averageEvaluator_ && + trainPassContext_.batchId % FLAGS_average_test_period + == FLAGS_average_test_period - 1) { + averageEvaluator_->finish(); + LOG(INFO) << " Averaged parameter:" + << " cost=" << trainPassContext_.avgTestCost + / trainPassContext_.numAvgTests + << " Eval: " << *averageEvaluator_; + trainPassContext_.numAvgTests = 0; + trainPassContext_.avgTestCost = 0; + } - ++batchId; + ++trainPassContext_.batchId; - if (batchId % FLAGS_log_period == 0) { - FOR_TIMING(globalStat.setThreadInfo(true)); - FOR_TIMING(globalStat.printAllStatus()); - FOR_TIMING(globalStat.reset()); - } + if (trainPassContext_.batchId % FLAGS_log_period == 0) { + FOR_TIMING(globalStat.setThreadInfo(true)); + FOR_TIMING(globalStat.printAllStatus()); + FOR_TIMING(globalStat.reset()); + } - if (testDataProvider_ && FLAGS_test_period > 0 && - batchId % FLAGS_test_period == 0) { - tester_->testOnePeriod(); - } + if (testDataProvider_ && FLAGS_test_period > 0 && + trainPassContext_.batchId % FLAGS_test_period == 0) { + tester_->testOnePeriod(); + } - if (FLAGS_saving_period_by_batches > 0 && - batchId > FLAGS_saving_period_by_batches * passInnerId && - 0 == FLAGS_trainer_id) { - trainerInternal_.getParameterUpdater()->catchUpWith(); - if (testDataProvider_) { - tester_->testOnePeriod(); - } - paramUtil_->saveParametersOnePass(passId, passInnerId); - ++passInnerId; + if (FLAGS_saving_period_by_batches > 0 && + trainPassContext_.batchId + > FLAGS_saving_period_by_batches * trainPassContext_.passInnerId && + 0 == FLAGS_trainer_id) { + trainerInternal_.getParameterUpdater()->catchUpWith(); + if (testDataProvider_) { + tester_->testOnePeriod(); } + paramUtil_->saveParametersOnePass( + trainPassContext_.passId, trainPassContext_.passInnerId); + ++trainPassContext_.passInnerId; } +} - if (batchId == 0) { +void Trainer::finishTrainPass() { + if (trainPassContext_.batchId == 0) { // This means no more data from DataProvider return; } - trainerInternal_.finishTrainPass(passId, batchId); + trainerInternal_.finishTrainPass( + trainPassContext_.passId, trainPassContext_.batchId); FOR_TIMING(globalStat.setThreadInfo(true)); FOR_TIMING(globalStat.printAllStatus()); @@ -485,9 +493,30 @@ void Trainer::trainOnePass(int passId) { tester_->testOnePeriod(); } - if (passId % FLAGS_saving_period == 0 && FLAGS_trainer_id == 0) { - paramUtil_->saveParametersOnePass(passId); + if (trainPassContext_.passId % FLAGS_saving_period == 0 + && FLAGS_trainer_id == 0) { + paramUtil_->saveParametersOnePass(trainPassContext_.passId); } + ++trainPassContext_.passId; +} + +void Trainer::trainOnePass() { + startTrainPass(); + size_t batchSize = config_->getOptConfig().batch_size(); + while (true) { + DataBatch dataBatch; + + int num = 0; + { + REGISTER_TIMER("getTrainBatch"); + num = dataProvider_->getNextBatch(batchSize, &dataBatch); + } + if (num == 0) break; + CHECK_EQ(num, dataBatch.getSize()); + trainOneDataBatch(dataBatch); + } + + finishTrainPass(); } void Trainer::trainOnePassBatch(int passId) { @@ -582,6 +611,13 @@ void Trainer::clearGradient() { int Trainer::getBatchSize() { return config_->getOptConfig().batch_size(); } +void Trainer::createTester() { + tester_.reset(new paddle::Tester(config_, createTesterConfig(), + trainerInternal_.getGradientMachine(), + trainerInternal_.getParameterUpdater(), + testDataProvider_)); +} + void Trainer::test() { tester_->test(); } diff --git a/paddle/trainer/Trainer.h b/paddle/trainer/Trainer.h index 9bfd6d107a204..4f4811a139e74 100644 --- a/paddle/trainer/Trainer.h +++ b/paddle/trainer/Trainer.h @@ -94,6 +94,11 @@ class Trainer { */ real checkGradient(); + void startTrain(); + void finishTrain(); + void startTrainPass(); + void finishTrainPass(); + void trainOneDataBatch(DataBatch& dataBatch); /** * given a dataBatch and the current parameter value @@ -144,11 +149,11 @@ class Trainer { protected: /** - * Train one pass of data. passId starts from 0 + * Train one pass of data. * * SGD Method. */ - void trainOnePass(int passId); + void trainOnePass(); /** * Train one pass in one batch. @@ -161,6 +166,8 @@ class Trainer { */ void clearGradient(); + void createTester(); + private: std::unique_ptr createTesterConfig(); @@ -173,6 +180,17 @@ class Trainer { MachineState trainState_; MachineState testState_; + struct TrainPassContext { + int64_t batchId; + real avgTestCost; + int64_t numAvgTests; + int passId; + int passInnerId; + }; + std::vector forwardOutput_; + + TrainPassContext trainPassContext_; + std::unique_ptr evaluator_; std::unique_ptr currentEvaluator_; std::unique_ptr averageEvaluator_; diff --git a/paddle/trainer/TrainerInternal.cpp b/paddle/trainer/TrainerInternal.cpp index 6029a4b2c1d0a..e23e42927c381 100644 --- a/paddle/trainer/TrainerInternal.cpp +++ b/paddle/trainer/TrainerInternal.cpp @@ -55,6 +55,8 @@ void TrainerInternal::init(const std::shared_ptr &config, gradientMachine_ = gradientMachine; if (!gradientMachine) { + CHECK(config_->getConfig().has_model_config()) + << "Missing model_config in trainer_config"; gradientMachine_.reset(GradientMachine::create( config_->getConfig().model_config(), intconfig_->mode, parameterUpdater_->getParameterTypes())); @@ -62,7 +64,8 @@ void TrainerInternal::init(const std::shared_ptr &config, } void TrainerInternal::trainOneBatch(int64_t batchId, - const DataBatch& dataBatch) { + const DataBatch& dataBatch, + std::vector* outArgs) { // true means updating parameter whenever gradient is ready during backward() bool doPipelineUpdate = (intconfig_->mode != GradientMachine::kSgdSparseCpuTraining) && @@ -84,7 +87,6 @@ void TrainerInternal::trainOneBatch(int64_t batchId, } const std::vector& inArgs = dataBatch.getStreams(); - std::vector outArgs; PassType passType = parameterUpdater_->startBatch(actualBatchSize); @@ -114,7 +116,7 @@ void TrainerInternal::trainOneBatch(int64_t batchId, timer.start(); #endif REGISTER_TIMER("forwardBackward"); - forwardBackwardBatch(inArgs, outArgs, passType, updateCallback, + forwardBackwardBatch(inArgs, *outArgs, passType, updateCallback, doPipelineUpdate); #ifndef PADDLE_DISABLE_TIMER timer.stop(); @@ -132,7 +134,7 @@ void TrainerInternal::trainOneBatch(int64_t batchId, real cost = 0; { REGISTER_TIMER("sumCost"); - cost = Argument::sumCosts(outArgs); + cost = Argument::sumCosts(*outArgs); } if (batchId % intconfig_->log_period == 0) { diff --git a/paddle/trainer/TrainerInternal.h b/paddle/trainer/TrainerInternal.h index 17011c4d2e46f..3a53aa1d17b31 100644 --- a/paddle/trainer/TrainerInternal.h +++ b/paddle/trainer/TrainerInternal.h @@ -81,7 +81,9 @@ class TrainerInternal { * @param batchId current batch id * @param dataBatch data for the batch */ - void trainOneBatch(int64_t batchId, const DataBatch& dataBatch); + void trainOneBatch(int64_t batchId, + const DataBatch& dataBatch, + std::vector* outArgs); /** * showParameterStats diff --git a/proto/TrainerConfig.proto.m4 b/proto/TrainerConfig.proto.m4 index a42ff88d54b5e..3b0e24f90bed8 100644 --- a/proto/TrainerConfig.proto.m4 +++ b/proto/TrainerConfig.proto.m4 @@ -130,7 +130,7 @@ message OptimizationConfig { }; message TrainerConfig { - required ModelConfig model_config = 1; + optional ModelConfig model_config = 1; optional DataConfig data_config = 2; required OptimizationConfig opt_config = 3; optional DataConfig test_data_config = 4; diff --git a/python/paddle/proto/__init__.py b/python/paddle/proto/__init__.py index 7f9e87eee6037..cd6a59ecbb095 100644 --- a/python/paddle/proto/__init__.py +++ b/python/paddle/proto/__init__.py @@ -12,3 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. +from paddle.proto.TrainerConfig_pb2 import OptimizationConfig, TrainerConfig +from paddle.proto.ModelConfig_pb2 import ModelConfig From 45dca19a8d8744127331af0e0fd326e4afc740a2 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 28 Oct 2016 11:07:31 +0800 Subject: [PATCH 188/324] Change contribute to paddle to fit new branching model (#275) * Change contribute to paddle to fit new branching model --- doc/build/contribute_to_paddle.md | 41 +++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/doc/build/contribute_to_paddle.md b/doc/build/contribute_to_paddle.md index bbdbb4d4227d0..a9ab69c5f42b8 100644 --- a/doc/build/contribute_to_paddle.md +++ b/doc/build/contribute_to_paddle.md @@ -4,7 +4,7 @@ We sincerely appreciate your contributions. You can use fork and pull request workflow to merge your code. ## Code Requirements -- Your code mush be fully documented by +- Your code must be fully documented by [doxygen](http://www.stack.nl/~dimitri/doxygen/) style. - Make sure the compiler option WITH\_STYLE\_CHECK is on and the compiler passes the code style check. @@ -20,16 +20,30 @@ It's just that simple. ## Clone +Paddle is currently using [git-flow branching model](http://nvie.com/posts/a-successful-git-branching-model/). +The **develop** is the main branch, and other user's branches are feature branches. + Once you've created a fork, you can use your favorite git client to clone your repo or just head straight to the command line: ```shell # Clone your fork to your local machine -git clone https://github.com/USERNAME/Paddle.git +git clone --branch develop https://github.com/USERNAME/Paddle.git +``` +If your repository doesn't contain **develop** branch, just create it by your own. + +```shell +git clone https://github.com/USERNAME/Paddle.git Paddle +cd Paddle +git checkout -b develop # create develop branch. +git remote add upstream https://github.com/baidu/Paddle.git # add upstream to baidu/Paddle +git pull upstream develop # update to upstream ``` + Then you can start to develop by making a local developement branch + ```shell -git checkout -b MY_COOL_STUFF_BRANCH origin/master +git checkout -b MY_COOL_STUFF_BRANCH ``` ## Commit @@ -41,7 +55,7 @@ Commit your changes by following command lines: git status # add modified files git add xx -git commit -m "commit info" +env EDITOR=vim git commit # You can write your comments by vim/nano/emacs. ``` The first line of commit infomation is the title. The second and later lines are the details if any. @@ -63,7 +77,7 @@ git remote -v Update your fork with the latest upstream changes: ```shell -git pull --rebase upstream HEAD +git pull --rebase upstream develop ``` If there are no unique commits locally, git will simply perform a fast-forward. @@ -76,7 +90,7 @@ Now, your local master branch is up-to-date with everything modified upstream. ```shell # push to your repository in Github -git push origin HEAD +git push -u origin MY_COOL_STUFF_BRANCH # create remote branch MY_COOL_STUFF_BRANCH to origin. ``` ## Pull Request @@ -93,13 +107,24 @@ of conflict, you need to do the update manually. You need to do the following on your local repository: ```shell git checkout MY_COOL_STUFF_BRANCH -git pull --rebase upstream HEAD +git pull upstream develop # You may need to resolve the conflict according to the git prompt. # Make and test your code. -git push -f origin HEAD +git push origin MY_COOL_STUFF_BRANCH ``` Now your Pull Request is updated with the latest version. ## Revise your pull request When you revise your pull request according to reviewer's comments, please use 'git commit' instead of 'git commit --amend' to commit your changes so that the reviewers can see the difference between the new pull requrest and the old pull request. + +The possible commands are + +```shell +git checkout MY_COOL_STUFF_BRANCH +git pull upstream develop # update local to newest code base. +# May be some conflicts will occured. +# And develop your cool stuff +env EDITOR=vim git commit # add your revise log +git push origin MY_COOL_STUFF_BRANCH +``` From 0e1a22d0fcc80ea3931b7006c891b844b07f29a2 Mon Sep 17 00:00:00 2001 From: backyes Date: Fri, 28 Oct 2016 12:27:40 +0800 Subject: [PATCH 189/324] set test_period default value to 0 (#279) --- paddle/trainer/Trainer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/trainer/Trainer.cpp b/paddle/trainer/Trainer.cpp index 04535849eb4aa..7fc48dd1fbec6 100644 --- a/paddle/trainer/Trainer.cpp +++ b/paddle/trainer/Trainer.cpp @@ -40,7 +40,7 @@ limitations under the License. */ #include "TrainerConfigHelper.h" P_DEFINE_string(config, "", "Trainer config file"); -P_DEFINE_int32(test_period, 1000, +P_DEFINE_int32(test_period, 0, "Run test every so many train batches." " 0 for testing after each pass." " If not 0, test log_period batches." From ca5a5ec480db806674043311b2c59fd318ad4f41 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Fri, 28 Oct 2016 12:51:35 +0800 Subject: [PATCH 190/324] Make Paddle --save_dir support a directory name (#277) * Also fix #243 --- paddle/trainer/ParamUtil.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paddle/trainer/ParamUtil.cpp b/paddle/trainer/ParamUtil.cpp index dae8b44b6db8e..bb309a54975a1 100644 --- a/paddle/trainer/ParamUtil.cpp +++ b/paddle/trainer/ParamUtil.cpp @@ -89,6 +89,9 @@ void ParameterUtil::saveParameters(int passId, int passInnerId) { } std::string basePath = config_->getSaveDir(); + if (basePath.find('/') == std::string::npos) { + basePath = "./" + basePath; + } mkDirRecursively(basePath.c_str()); std::string saveDir = path::join(basePath, buf); From fc9ca53ae421ec2eac5648ec8de6a1e7a993bfd0 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Fri, 28 Oct 2016 12:54:13 +0800 Subject: [PATCH 191/324] fix interface bug of block_expand_layer and add unittest (#265) * fix interface bug of block_expand_layer and add unittest * auto compute num_channels * default value of num_channels is None * adjust input order of block_expand --- .../paddle/trainer_config_helpers/layers.py | 34 ++++++++++--------- .../tests/configs/check.md5 | 2 +- .../tests/configs/test_maxout.py | 21 +++++++++++- 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index f8c32dc91f10b..7df9108ae82a4 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -54,7 +54,7 @@ 'cross_entropy_with_selfnorm', 'cross_entropy', 'multi_binary_label_cross_entropy', 'rank_cost', 'lambda_cost', 'huber_cost', - # 'block_expand_layer', # TODO(yuyang18): this layer is not correct + 'block_expand_layer', 'maxout_layer', 'out_prod_layer', 'print_layer' ] @@ -3284,18 +3284,18 @@ def linear_comb_layer(weights, vectors, size=None, name=None, @wrap_name_default() @layer_support() def block_expand_layer(input, - channel=0, block_x=0, block_y=0, stride_x=0, stride_y=0, padding_x=0, padding_y=0, + num_channels=None, name=None, layer_attr=None): """ Expand feature map to minibatch matrix. - - matrix width is: block_y * block_x * channel + - matrix width is: block_y * block_x * num_channels - matirx height is: outputH * outputW .. math:: @@ -3307,7 +3307,7 @@ def block_expand_layer(input, The expand method is the same with ExpandConvLayer, but saved the transposed value. After expanding, output.sequenceStartPositions will store timeline. The number of time steps are outputH * outputW and the dimension of each - time step is block_y * block_x * channel. This layer can be used after + time step is block_y * block_x * num_channels. This layer can be used after convolution neural network, and before recurrent neural network. The simple usage is: @@ -3315,7 +3315,7 @@ def block_expand_layer(input, .. code-block:: python block_expand = block_expand_layer(input, - channel=128, + num_channels=128, stride_x=1, stride_y=1, block_x=1, @@ -3323,8 +3323,8 @@ def block_expand_layer(input, :param input: The input layer. :type input: LayerOutput - :param channel: The channel number of input layer. - :type channel: int + :param num_channels: The channel number of input layer. + :type num_channels: int|None :param block_x: The width of sub block. :type block_x: int :param block_y: The width of sub block. @@ -3344,16 +3344,18 @@ def block_expand_layer(input, :return: LayerOutput object. :rtype: LayerOutput """ + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters Layer(name=name, - input=Input(input.name, - block_expand=BlockExpand(channels=channel, - block_x=block_x, - block_y=block_y, - stride_x=stride_x, - stride_y=stride_y, - padding_x=padding_x, - padding_y=padding_y) - ), + inputs=Input(input.name, + block_expand=BlockExpand(channels=num_channels, + block_x=block_x, + block_y=block_y, + stride_x=stride_x, + stride_y=stride_y, + padding_x=padding_x, + padding_y=padding_y)), type=LayerType.BLOCK_EXPAND, **ExtraLayerAttribute.to_kwargs(layer_attr) ) diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 index 88ce5c129e552..d1b22b34903df 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/check.md5 +++ b/python/paddle/trainer_config_helpers/tests/configs/check.md5 @@ -12,7 +12,7 @@ a5d9259ff1fd7ca23d0ef090052cb1f2 last_first_seq.protostr 8bb44e1e5072d0c261572307e7672bda test_grumemory_layer.protostr 1f3510672dce7a9ed25317fc58579ac7 test_hsigmoid.protostr d350bd91a0dc13e854b1364c3d9339c6 test_lstmemory_layer.protostr -6fa59551808ee7012bbd24f757e782d2 test_maxout.protostr +5433ed33d4e7414eaf658f2a55946186 test_maxout.protostr 251a948ba41c1071afcd3d9cf9c233f7 test_ntm_layers.protostr e6ff04e70aea27c7b06d808cc49c9497 test_print_layer.protostr 2a75dd33b640c49a8821c2da6e574577 test_rnn_group.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py b/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py index 079e2cf4c4320..7c1fb04766ae5 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_maxout.py @@ -25,6 +25,25 @@ stride=2, pool_type=MaxPooling()) -fc = fc_layer(input=pool, size=384, bias_attr=False) +conv2 = img_conv_layer(input=pool, + filter_size = 3, + num_channels=32, + num_filters=128, + padding=1, + act=LinearActivation(), + bias_attr=True) + +maxout2 = maxout_layer(input=conv, + num_channels=128, + groups=4) + +block = block_expand_layer(input=maxout, + num_channels=32, + stride_x=1, + stride_y=1, + block_x=1, + block_y=6) + +fc = fc_layer(input=block, size=384, bias_attr=False) outputs(fc) From fa24cbdbe1b18649a9140152b971473ba99079f7 Mon Sep 17 00:00:00 2001 From: backyes Date: Fri, 28 Oct 2016 13:10:04 +0800 Subject: [PATCH 192/324] Support empty Param Block in ParameterSever (#244) * Because in cluster maybe use a lot machine to train a model, and some parameter size could be too small for ParameterServer. Then some of pservers could not have any ParamBlock. * Also, because ports_num or ports_num_for_sparse is too large, then give a warning in runtime. --- paddle/pserver/ParameterServer2.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/paddle/pserver/ParameterServer2.cpp b/paddle/pserver/ParameterServer2.cpp index 8f72c1988d167..c8f37d0bf4f84 100644 --- a/paddle/pserver/ParameterServer2.cpp +++ b/paddle/pserver/ParameterServer2.cpp @@ -264,6 +264,15 @@ void ParameterServer2::setParameter(const SendParameterRequest& request, std::vector blockIds; blockIds.reserve(request.blocks_size()); int bufferIndex = 0; + + if (!request.blocks().size()) { + LOG(WARNING) + << "--ports_num or --ports_num_for_sparse might be too large, " + << "or total dense parameter size or sparse parameters size " + << "might be too small, this psever doesn't store any parameter."; + return; + } + for (const auto& block : request.blocks()) { /// block size for parameter(e.g. 128 for sparse row, 1K for dense) uint64_t blockSize = getParameterConfig(block).parameter_block_size(); From ddfff3a7fd0e746f8ff232945ed0a05a89b9da1c Mon Sep 17 00:00:00 2001 From: liaogang Date: Sun, 30 Oct 2016 01:04:06 +0800 Subject: [PATCH 193/324] Add bilinear interpolation layer --- doc/ui/api/trainer_config_helpers/layers.rst | 6 + paddle/cuda/include/hl_cnn.h | 56 +++++++ paddle/cuda/include/stub/hl_cnn_stub.h | 24 +++ paddle/cuda/src/hl_cuda_cnn.cu | 134 ++++++++++++++- paddle/gserver/layers/BilinearInterpLayer.cpp | 87 ++++++++++ paddle/gserver/layers/BilinearInterpLayer.h | 45 +++++ paddle/gserver/tests/test_LayerGrad.cpp | 20 +++ paddle/math/Matrix.cpp | 154 ++++++++++++++++++ paddle/math/Matrix.h | 44 +++++ paddle/math/tests/test_matrixCompare.cpp | 66 ++++++++ proto/ModelConfig.proto.m4 | 10 ++ python/paddle/trainer/config_parser.py | 35 ++++ .../paddle/trainer_config_helpers/layers.py | 69 +++++++- .../tests/configs/generate_protostr.sh | 2 +- .../tests/configs/test_bilinear_interp.py | 33 ++++ 15 files changed, 781 insertions(+), 4 deletions(-) create mode 100644 paddle/gserver/layers/BilinearInterpLayer.cpp create mode 100644 paddle/gserver/layers/BilinearInterpLayer.h create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index c1d7a7ce81530..01443466105b5 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -263,6 +263,12 @@ interpolation_layer :members: interpolation_layer :noindex: +bilinear_interp_layer +------------------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: bilinear_interp_layer + :noindex: + power_layer ----------- .. automodule:: paddle.trainer_config_helpers.layers diff --git a/paddle/cuda/include/hl_cnn.h b/paddle/cuda/include/hl_cnn.h index 5d750333e1e35..aa4720f6ca749 100644 --- a/paddle/cuda/include/hl_cnn.h +++ b/paddle/cuda/include/hl_cnn.h @@ -240,4 +240,60 @@ extern void hl_CMRNorm_backward( size_t channels, size_t height, size_t width, size_t sizeX, real alpha, real beta); +/** + * @brief Bilinear interpolation forward. + * + * @param[in] inData input value. + * @param[in] inImgH input image height. + * @param[in] inImgW input image width. + * @param[in] inputH input batchSize. + * @param[in] inputW input image data dim. + * @param[out] outData output value. + * @param[in] outImgH output image height. + * @param[in] outImgW output image width. + * @param[in] outputH output batchSize. + * @param[in] outputW output image data dim. + * @param[in] numChannels number of channels. + * + */ +extern void hl_bilinear_forward(const real* inData, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + real* outData, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels); + + /** + * @brief Bilinear interpolation backward. + * + * @param[out] inGrad input gradient. + * @param[in] inImgH input image height. + * @param[in] inImgW input image width. + * @param[in] inputH input batchSize. + * @param[in] inputW input image data dim. + * @param[in] outGrad output gradient. + * @param[in] outImgH output image height. + * @param[in] outImgW output image width. + * @param[in] outputH output batchSize. + * @param[in] outputW output image data dim. + * @param[in] numChannels number of channels. + * + */ +extern void hl_bilinear_backward(real* inGrad, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + const real* outGrad, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels); + #endif /* HL_CNN_H_ */ diff --git a/paddle/cuda/include/stub/hl_cnn_stub.h b/paddle/cuda/include/stub/hl_cnn_stub.h index 38e359c3eb2f3..aa9442fb80237 100644 --- a/paddle/cuda/include/stub/hl_cnn_stub.h +++ b/paddle/cuda/include/stub/hl_cnn_stub.h @@ -89,4 +89,28 @@ inline void hl_CMRNorm_backward( size_t channels, size_t height, size_t width, size_t sizeX, real alpha, real beta) {} +inline void hl_bilinear_forward(const real* inData, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + real* outData, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels) {} + +inline void hl_bilinear_backward(real* inGrad, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + const real* outGrad, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels) {} + #endif // HL_CNN_STUB_H_ diff --git a/paddle/cuda/src/hl_cuda_cnn.cu b/paddle/cuda/src/hl_cuda_cnn.cu index abac83a3e0447..f965adc13575c 100644 --- a/paddle/cuda/src/hl_cuda_cnn.cu +++ b/paddle/cuda/src/hl_cuda_cnn.cu @@ -522,7 +522,7 @@ void hl_CMRNorm_backward(size_t frameCnt, const real* inV, size_t height, size_t width, size_t sizeX, real alpha, real beta) { size_t threadsNum = frameCnt * height * width; - size_t blocksX = (threadsNum + 1024 -1) / 1024; + size_t blocksX = (threadsNum + 1024 - 1) / 1024; size_t blocksY = 1; dim3 threads(1024, 1); dim3 grid(blocksX, blocksY); @@ -531,3 +531,135 @@ void hl_CMRNorm_backward(size_t frameCnt, const real* inV, height, width, sizeX, alpha, beta, inDiff); CHECK_SYNC("hl_CMRNorm_backward"); } + +__global__ void KeBilinearInterpFw(const size_t nthreads, + const real* in, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + real* out, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + if(tid < nthreads) { + int outIdH = tid / (outputW / numChannels); + int outIdW = tid % (outputW / numChannels); + + int inIdH = ratioH * (outIdW / outImgW); + int hId = (inIdH < inImgH - 1) ? 1 : 0; + real hlambda = ratioH * (outIdW / outImgW) - inIdH; + + int inIdW = ratioW * (tid % outImgW); + int wId = (inIdW < inImgW - 1) ? 1 : 0; + real wlambda = ratioW * (tid % outImgW) - inIdW; + + const real* inPos = &in[outIdH * inputW + inIdH * inImgW + inIdW]; + real* outPos = &out[outIdH * outputW + outIdW]; + for (int c = 0; c < numChannels; ++c) { + // bilinear interpolation + outPos[0] = (1.f - hlambda) * + ((1.f - wlambda) * inPos[0] + wlambda * inPos[wId]) + + hlambda * ((1.f - wlambda) * inPos[hId * inImgW] + + wlambda * inPos[hId * inImgW + wId]); + inPos += inImgH * inImgW; + outPos += outImgH * outImgW; + } + } +} + +void hl_bilinear_forward(const real* inData, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + real* outData, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels) { + int threadNum = outputH * outImgH * outImgW; + int blocks = (threadNum + 1024 - 1) / 1024; + + real ratioH = (outImgH > 1) ? + static_cast(inImgH - 1) / (outImgH - 1) : 0.f; + real ratioW = (outImgW > 1) ? + static_cast(inImgW - 1) / (outImgW - 1) : 0.f; + + KeBilinearInterpFw<<< blocks, 1024, 0, STREAM_DEFAULT>>>( + threadNum, inData, inImgH, inImgW, inputH, inputW, outData, + outImgH, outImgW, outputH, outputW, numChannels, ratioH, ratioW); + CHECK_SYNC("hl_bilinear_forward failed"); +} + +__global__ void KeBilinearInterpBw(const size_t nthreads, + real* in, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + const real* out, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels, + const real ratioH, + const real ratioW) { + int tid = blockIdx.x * blockDim.x + threadIdx.x; + + if(tid < nthreads) { + int outIdH = tid / (outputW / numChannels); + int outIdW = tid % (outputW / numChannels); + + int inIdH = ratioH * (outIdW / outImgW); + int hId = (inIdH < inImgH - 1) ? 1 : 0; + real hlambda = ratioH * (outIdW / outImgW) - inIdH; + + int inIdW = ratioW * (tid % outImgW); + int wId = (inIdW < inImgW - 1) ? 1 : 0; + real wlambda = ratioW * (tid % outImgW) - inIdW; + + const real* outPos = &out[outIdH * outputW + outIdW]; + real* inPos = &in[outIdH * inputW + inIdH * inImgW + inIdW]; + for (int c = 0; c < numChannels; ++c) { + atomicAdd(&inPos[0], (1.f - hlambda) * (1.f - wlambda) * outPos[0]); + atomicAdd(&inPos[wId], (1.f - hlambda) * wlambda * outPos[0]); + atomicAdd(&inPos[hId * inImgW], hlambda * (1.f - wlambda) * outPos[0]); + atomicAdd(&inPos[hId * inImgW + wId], hlambda * wlambda * outPos[0]); + inPos += inImgH * inImgW; + outPos += outImgH * outImgW; + } + } +} + +void hl_bilinear_backward(real* inGrad, + const size_t inImgH, + const size_t inImgW, + const size_t inputH, + const size_t inputW, + const real* outGrad, + const size_t outImgH, + const size_t outImgW, + const size_t outputH, + const size_t outputW, + const size_t numChannels) { + int threadNum = outputH * outImgH * outImgW; + int blocks = (threadNum + 1024 - 1) / 1024; + + real ratioH = (outImgH > 1) ? + static_cast(inImgH - 1) / (outImgH - 1) : 0.f; + real ratioW = (outImgW > 1) ? + static_cast(inImgW - 1) / (outImgW - 1) : 0.f; + + KeBilinearInterpBw<<< blocks, 1024, 0, STREAM_DEFAULT>>>( + threadNum, inGrad, inImgH, inImgW, inputH, inputW, outGrad, + outImgH, outImgW, outputH, outputW, numChannels, ratioH, ratioW); + CHECK_SYNC("hl_bilinear_backward failed"); +} \ No newline at end of file diff --git a/paddle/gserver/layers/BilinearInterpLayer.cpp b/paddle/gserver/layers/BilinearInterpLayer.cpp new file mode 100644 index 0000000000000..f43086e585535 --- /dev/null +++ b/paddle/gserver/layers/BilinearInterpLayer.cpp @@ -0,0 +1,87 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "BilinearInterpLayer.h" +#include "paddle/utils/Logging.h" +#include "paddle/utils/Stat.h" + +namespace paddle { + +REGISTER_LAYER(bilinear_interp, BilinearInterpLayer); + +size_t BilinearInterpLayer::getDataDimSize() { + getOutput().setFrameHeight(outImgH_); + getOutput().setFrameWidth(outImgW_); + return outImgH_ * outImgW_ * numChannels_; +} + +bool BilinearInterpLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + /* Initialize the basic parent class */ + Layer::init(layerMap, parameterMap); + + CHECK_EQ(1, config_.inputs_size()); + + const BilinearInterpConfig& conf = config_.inputs(0).bilinear_interp_conf(); + inImgH_ = inputLayers_[0]->getOutput().getFrameHeight(); + inImgW_ = inputLayers_[0]->getOutput().getFrameWidth(); + if (inImgH_ == 0) { + inImgH_ = conf.img_size_y(); + } + if (inImgW_ == 0) { + inImgW_ = conf.img_size_x(); + } + outImgH_ = conf.out_size_y(); + outImgW_ = conf.out_size_x(); + numChannels_ = conf.num_channels(); + + CHECK(outImgH_ > 0 && outImgW_ > 0); + CHECK(inImgH_ > 0 && inImgW_ > 0); + CHECK(numChannels_); + + return true; +} + +void BilinearInterpLayer::forward(PassType passType) { + Layer::forward(passType); + size_t batchSize = getInput(0).getBatchSize(); + size_t size = getDataDimSize(); + { + REGISTER_TIMER_INFO("FwResetTimer", getName().c_str()); + resetOutput(batchSize, size); + } + + MatrixPtr inV = getInputValue(0); + MatrixPtr outV = getOutputValue(); + { + REGISTER_TIMER_INFO("FwBilinearInterpTimer", getName().c_str()); + outV->bilinearForward(*inV, inImgH_, inImgW_, outImgH_, outImgW_, + numChannels_); + } +} + +void BilinearInterpLayer::backward(const UpdateCallback& callback) { + (void) callback; + + MatrixPtr inputG = getInputGrad(0); + MatrixPtr outG = getOutputGrad(); + { + REGISTER_TIMER_INFO("BwBilinearInterpTimer", getName().c_str()); + if (inputG) { + inputG->bilinearBackward(*outG, outImgH_, outImgW_, inImgH_, inImgW_, + numChannels_); + } + } +} +} // namespace paddle diff --git a/paddle/gserver/layers/BilinearInterpLayer.h b/paddle/gserver/layers/BilinearInterpLayer.h new file mode 100644 index 0000000000000..24f5b99910405 --- /dev/null +++ b/paddle/gserver/layers/BilinearInterpLayer.h @@ -0,0 +1,45 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Layer.h" +#include "paddle/math/Matrix.h" + +namespace paddle { + +/** + * @brief A layer for bilinear interpolation which is + * used on conv layer output. + * + * @note The config file api is bilinear_interp_layer. + */ +class BilinearInterpLayer : public Layer { +protected: + size_t outImgH_, outImgW_; + size_t inImgH_, inImgW_; + size_t numChannels_; + +public: + explicit BilinearInterpLayer(const LayerConfig& config) : Layer(config) {} + + virtual ~BilinearInterpLayer() {} + + size_t getDataDimSize(); + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + void forward(PassType passType); + void backward(const UpdateCallback& callback = nullptr); +}; + +} // namespace paddle diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index c5723f8574ab3..425d669206cce 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -31,6 +31,26 @@ P_DECLARE_double(checkgrad_eps); P_DECLARE_bool(thread_local_rand_use_global_seed); P_DECLARE_bool(prev_batch_state); +TEST(Layer, BilinearInterpLayer) { + TestConfig config; + config.layerConfig.set_type("bilinear_interp"); + config.biasSize = 0; + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 4096, 0}); + LayerInputConfig* input = config.layerConfig.add_inputs(); + BilinearInterpConfig* bilinear = input->mutable_bilinear_interp_conf(); + + bilinear->set_img_size_x(32); + bilinear->set_img_size_y(32); + bilinear->set_out_size_x(64); + bilinear->set_out_size_y(64); + bilinear->set_num_channels(4); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "bilinear_interp", 10, false, useGpu); + } +} + TEST(Operator, dot_mul) { TestConfig config; config.layerConfig.set_size(10); diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index a6ff2f3b35d04..469255719701a 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -23,6 +23,7 @@ limitations under the License. */ #include "paddle/utils/Logging.h" #include +#include "hl_cnn.h" #include "hl_gpu.h" #include "hl_table_apply.h" #include "hl_top_k.h" @@ -1144,6 +1145,56 @@ void GpuMatrix::addColumnVector(const Matrix& b) { BaseMatrix::addColVector(const_cast(b)); } +void GpuMatrix::bilinearForward(const Matrix& in, + const size_t inImgH, + const size_t inImgW, + const size_t outImgH, + const size_t outImgW, + const size_t numChannels) { + CHECK(dynamic_cast(&in)); + + const size_t outputW = getWidth(); + const size_t outputH = getHeight(); + const size_t inputW = in.getWidth(); + const size_t inputH = in.getHeight(); + + real* outData = getData(); + const real* inData = in.getData(); + + if (inImgH == outImgW && inImgW == outImgW) { + this->copyFrom(in); + } else { + hl_bilinear_forward(inData, inImgH, inImgW, + inputH, inputW, outData, outImgH, outImgW, + outputH, outputW, numChannels); + } +} + +void GpuMatrix::bilinearBackward(const Matrix& out, + const size_t outImgH, + const size_t outImgW, + const size_t inImgH, + const size_t inImgW, + const size_t numChannels) { + CHECK(dynamic_cast(&out)); + + const size_t inputW = getWidth(); + const size_t inputH = getHeight(); + const size_t outputW = out.getWidth(); + const size_t outputH = out.getHeight(); + + real* inGrad = getData(); + const real* outGrad = out.getData(); + + if (outImgH == inImgH && outImgW == inImgW) { + this->copyFrom(out); + } else { + hl_bilinear_backward(inGrad, inImgH, inImgW, + inputH, inputW, outGrad, outImgH, outImgW, + outputH, outputW, numChannels); + } +} + /** * CpuMatrix */ @@ -3598,6 +3649,109 @@ void CpuMatrix::classificationErrorMulti(Matrix& output, Matrix& label, } } +void CpuMatrix::bilinearForward(const Matrix& in, + const size_t inImgH, + const size_t inImgW, + const size_t outImgH, + const size_t outImgW, + const size_t numChannels) { + CHECK(dynamic_cast(&in)); + + size_t outputW = getWidth(); + size_t outputH = getHeight(); + size_t inputW = in.getWidth(); + size_t inputH = in.getHeight(); + + real* outData = getData(); + const real* inData = in.getData(); + + const real ratioH = (outImgH > 1) ? + static_cast(inImgH - 1) / (outImgH - 1) : 0.f; + const real ratioW = (outImgW > 1) ? + static_cast(inImgW - 1) / (outImgW - 1) : 0.f; + + if (inImgH == outImgH && inImgW == outImgW) { + this->copyFrom(in); + } else { + for (int k = 0; k < outputH; ++k) { // loop for batches + for (int i = 0; i < outImgH; ++i) { // loop for images + int h = ratioH * i; + int hid = (h < inImgH - 1) ? 1 : 0; + real hlambda = ratioH * i - h; + + for (int j = 0; j < outImgW; ++j) { + int w = ratioW * j; + int wid = (w < inImgW - 1) ? 1 : 0; + real wlambda = ratioW * j - w; + // calculate four position for bilinear interpolation + const real* inPos = &inData[k * inputW + h * inImgW + w]; + real* outPos = &outData[k * outputW + i * outImgW + j]; + for (int c = 0; c < numChannels; ++c) { // loop for channels + // bilinear interpolation + outPos[0] = (1.f - hlambda) * + ((1.f - wlambda) * inPos[0] + wlambda * inPos[wid]) + + hlambda * ((1.f - wlambda) * inPos[hid * inImgW] + + wlambda * inPos[hid * inImgW + wid]); + inPos += inImgH * inImgW; + outPos += outImgH * outImgW; + } + } + } + } + } +} + +void CpuMatrix::bilinearBackward(const Matrix& out, + const size_t outImgH, + const size_t outImgW, + const size_t inImgH, + const size_t inImgW, + const size_t numChannels) { + CHECK(dynamic_cast(&out)); + + size_t inputW = getWidth(); + size_t inputH = getHeight(); + size_t outputW = out.getWidth(); + size_t outputH = out.getHeight(); + + real* inGrad = getData(); + const real* outGrad = out.getData(); + + const real ratioH = (outImgH > 1) ? + static_cast(inImgH - 1) / (outImgH - 1) : 0.f; + const real ratioW = (outImgW > 1) ? + static_cast(inImgW - 1) / (outImgW - 1) : 0.f; + + if (inImgH == outImgH && inImgW == outImgW) { + this->copyFrom(out); + } else { + for (int k = 0; k < outputH; ++k) { // loop for batches + for (int i = 0; i < outImgH; ++i) { // loop for images + int h = ratioH * i; + int hid = (h < inImgH - 1) ? 1 : 0; + real hlambda = ratioH * i - h; + + for (int j = 0; j < outImgW; ++j) { + int w = ratioW * j; + int wid = (w < inImgW - 1) ? 1 : 0; + real wlambda = ratioW * j - w; + + real* inPos = &inGrad[k * inputW + h * inImgW + w]; + const real* outPos = &outGrad[k * outputW + i * outImgW + j]; + for (int c = 0; c < numChannels; ++c) { // loop for channels + inPos[0] += (1.f - hlambda) * (1.f - wlambda) * outPos[0]; + inPos[wid] += (1.f - hlambda) * wlambda * outPos[0]; + inPos[hid * inImgW] += hlambda * (1.f - wlambda) * outPos[0]; + inPos[hid * inImgW + wid] += hlambda * wlambda * outPos[0]; + inPos += inImgH * inImgW; + outPos += outImgH * outImgW; + } + } + } + } + } +} + //////////////////////////////////////////////////////////////// // functions executed via cpu // //////////////////////////////////////////////////////////////// diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index 5c15c94012816..b4922d7e6f546 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -930,6 +930,22 @@ class Matrix : public BaseMatrix { virtual void paramReluBackwardDiff(Matrix& oGrad, Matrix& data, Matrix& W) { LOG(FATAL) << "Not implemented"; } + virtual void bilinearForward(const Matrix& in, + const size_t inImgH, + const size_t inImgW, + const size_t outImgH, + const size_t outImgW, + const size_t numChannels) { + LOG(FATAL) << "Not implemented"; + } + virtual void bilinearBackward(const Matrix& out, + const size_t outImgH, + const size_t outImgW, + const size_t inImgH, + const size_t inImgW, + const size_t numChannels) { + LOG(FATAL) << "Not implemented"; + } }; inline std::ostream& operator<<(std::ostream& os, const Matrix& mat) { @@ -1191,6 +1207,20 @@ class GpuMatrix : public Matrix { int contextLength, int contextStart, int totalPad, size_t beginPad); + + void bilinearForward(const Matrix& in, + const size_t inImgH, + const size_t inImgW, + const size_t outImgH, + const size_t outImgW, + const size_t numChannels); + + void bilinearBackward(const Matrix& out, + const size_t outImgH, + const size_t outImgW, + const size_t inImgH, + const size_t inImgW, + const size_t numChannels); }; class CpuMatrix : public Matrix { @@ -1469,6 +1499,20 @@ class CpuMatrix : public Matrix { void multiBinaryLabelCrossEntropy(Matrix& output, Matrix& label); void multiBinaryLabelCrossEntropyBp(Matrix& output, Matrix& label); void classificationErrorMulti(Matrix& output, Matrix& label, real threshold); + + void bilinearForward(const Matrix& in, + const size_t inImgH, + const size_t inImgW, + const size_t outImgH, + const size_t outImgW, + const size_t numChannels); + + void bilinearBackward(const Matrix& out, + const size_t outImgH, + const size_t outImgW, + const size_t inImgH, + const size_t inImgW, + const size_t numChannels); }; class SharedCpuMatrix : public CpuMatrix { diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index e1bda79a8acb1..2ff19e7b3f87c 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -88,6 +88,72 @@ void MatrixCheckErr(const Matrix& matrix1, const Matrix& matrix2) { EXPECT_EQ(count, 0) << "There are " << count << " different element."; } +void testBilinearFwdBwd(int numSamples, int imgSizeH, int imgSizeW, + int channels) { + int inWidth = imgSizeH * imgSizeW * channels; + int outWidth = 2 * imgSizeH * 2 * imgSizeW * channels; + + // forward + MatrixPtr input = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpu = GpuMatrix::create(numSamples, inWidth, false, true); + + MatrixPtr target = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpu = GpuMatrix::create(numSamples, outWidth, false, true); + MatrixPtr targetCheck = CpuMatrix::create(numSamples, outWidth, false, false); + + input->randomizeUniform(); + inputGpu->copyFrom(*input); + + target->bilinearForward(*input, imgSizeH, imgSizeW, + 2 * imgSizeH, 2 * imgSizeW, channels); + targetGpu->bilinearForward(*inputGpu, imgSizeH, imgSizeW, + 2 * imgSizeH, 2 * imgSizeW, channels); + + // check + targetCheck->copyFrom(*targetGpu); + MatrixCheckErr(*target, *targetCheck); + + // backward + MatrixPtr inputGrad = CpuMatrix::create(numSamples, inWidth, false, false); + MatrixPtr inputGpuGrad = GpuMatrix::create(numSamples, inWidth, false, true); + + MatrixPtr targetGrad = CpuMatrix::create(numSamples, outWidth, false, false); + MatrixPtr targetGpuGrad = GpuMatrix::create(numSamples, outWidth, false, + true); + MatrixPtr targetCheckGrad = + CpuMatrix::create(numSamples, inWidth, false, false); + + inputGrad->randomizeUniform(); + targetGrad->randomizeUniform(); + inputGpuGrad->copyFrom(*inputGrad); + targetGpuGrad->copyFrom(*targetGrad); + + inputGrad->bilinearBackward(*targetGrad, 2 * imgSizeH, 2 * imgSizeW, + imgSizeH, imgSizeW, channels); + inputGpuGrad->bilinearBackward(*targetGpuGrad, 2 * imgSizeH, 2 * imgSizeW, + imgSizeH, imgSizeW, channels); + + // check + targetCheckGrad->copyFrom(*inputGpuGrad); + MatrixCheckErr(*inputGrad, *targetCheckGrad); +} + +TEST(Matrix, BilinearFwdBwd) { + for (auto numSamples : {5, 10}) { + for (auto channels : {8, 16}) { + for (auto imgSizeH : {14, 28}) { + for (auto imgSizeW : {16, 30}) { + VLOG(3) << " numSamples=" << numSamples + << " channels=" << channels + << " imgSizeH=" << imgSizeH + << " imgSizeW=" << imgSizeW; + testBilinearFwdBwd(numSamples, imgSizeH, imgSizeW, channels); + } + } + } + } +} + void testMatrixProjectionForward(int contextStart, int contextLength, bool padding, int batchSize, int inputDim) { MatrixPtr cpuInput = std::make_shared(batchSize, inputDim); diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index 25e36f9c4c168..8bdcd70a417b8 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -203,6 +203,15 @@ message OperatorConfig { optional int32 num_filters = 7; } +message BilinearInterpConfig { + // The size if input feature map. + required uint32 img_size_x = 1; + required uint32 img_size_y = 2; + // The size if output feature map. + required uint32 out_size_x = 3; + required uint32 out_size_y = 4; + required uint32 num_channels = 5; +} message ImageConfig { // The image data dimensionality. @@ -225,6 +234,7 @@ message LayerInputConfig { // If the input layer has multi-output. // Set the argument name. optional string input_layer_argument = 9; + optional BilinearInterpConfig bilinear_interp_conf = 10; } message LayerConfig { diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index fb47fd0c6f0c3..82446e980d81c 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -461,6 +461,7 @@ def __init__( sparse_update=None, gradient_clipping_threshold=None, conv=None, + bilinear_interp=None, norm=None, pool=None, image=None, @@ -723,6 +724,18 @@ def __init__( if output_x is not None: config_assert(output_x <= 0) +# please refer to the comments in proto/ModelConfig.proto +@config_class +class BilinearInterp(Cfg): + def __init__( + self, + img_size_x = None, + img_size_y=None, + out_size_x = None, + out_size_y = None, + num_channels = None): + self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Pool(Cfg): @@ -953,6 +966,13 @@ def TestData(data_config, async_load_data=None): " Data definition") g_config.test_data_config.async_load_data = async_load_data +def parse_bilinear(bilinear, input_layer_name, bilinear_conf): + bilinear_conf.img_size_x = bilinear.img_size_x; + bilinear_conf.img_size_y = bilinear.img_size_y; + bilinear_conf.out_size_x = bilinear.out_size_x; + bilinear_conf.out_size_y = bilinear.out_size_y; + bilinear_conf.num_channels = bilinear.num_channels; + def parse_pool(pool, input_layer_name, pool_conf): pool_conf.pool_type = pool.pool_type config_assert(pool.pool_type in ['max-projection', 'avg-projection', @@ -2306,6 +2326,21 @@ def __init__( config_assert(input_layer1.size == input_layer2.size, 'the two vector inputs should be of the same size') +@config_layer('bilinear_interp') +class BilinearInterpLayer(LayerBase): + def __init__( + self, + name, + inputs, + device=None): + super(BilinearInterpLayer, self).__init__( + name, 'bilinear_interp', 0, inputs=inputs, device=device) + input_layer = self.get_input_layer(0) + self.set_layer_size(input_layer.size) + parse_bilinear(self.inputs[0].bilinear_interp, + input_layer.name, + self.config.inputs[0].bilinear_interp_conf); + @config_layer('sum_to_one_norm') class SumToOneNormLayer(LayerBase): def __init__( diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 5e7e66a908ee0..59df4646faae9 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -40,8 +40,8 @@ 'img_cmrnorm_layer', 'addto_layer', 'concat_layer', 'lstm_step_layer', 'recurrent_group', 'memory', 'StaticInput', 'expand_layer', 'scaling_layer', - 'power_layer', 'interpolation_layer', 'trans_layer', - 'sum_to_one_norm_layer', + 'power_layer', 'interpolation_layer', 'bilinear_interp_layer', + 'trans_layer', 'sum_to_one_norm_layer', 'get_output_layer', 'LayerType', 'context_projection', 'beam_search', 'maxid_layer', 'GeneratedInput', 'SubsequenceInput', 'gru_step_layer', 'recurrent_layer', @@ -92,6 +92,7 @@ class LayerType(object): EXPAND_LAYER = 'expand' INTERPOLATION_LAYER = 'interpolation' + BILINEAR_INTERP_LAYER = 'bilinear_interp' POWER_LAYER = 'power' SCALING_LAYER = 'scaling' TRANS_LAYER = 'trans' @@ -1252,6 +1253,70 @@ def interpolation_layer(input, weight, name=None, layer_attr=None): size=input[0].size) +@wrap_name_default() +@layer_support() +def bilinear_interp_layer(input, + img_size_x=None, + img_size_y=None, + out_size_x=None, + out_size_y=None, + num_channels=None, + name=None, + layer_attr=None): + """ + This layer is to implement bilinear interpolation on conv layer output. + + Please refer to Wikipedia: https://en.wikipedia.org/wiki/Bilinear_interpolation + + The simple usage is: + + .. code-block:: python + + bilinear = bilinear_interp_layer(input, + img_size_x, + img_size_y, + out_size_x, + out_size_y, + num_channels) + + :para input: A input layer. + :type input: LayerOutput. + :para img_size_x: previous layer output width. + :type img_size_x: int|None + :para img_size_y: previous layer output height. + :type img_size_y: int|None + :para out_size_x: bilinear interpolation output width. + :type out_size_x: int|None + :para out_size_y: bilinear interpolation output height. + :type out_size_y: int|None + :para num_channels: number of channels of input layer. If None, + it will be set automatically from previous output. + :type num_channels: int|None + :para name: The layer's name, which cna not be specified. + :type name: None|basestring + :para layer_attr: Extra Layer attribute. + :type layer_attr: ExtraLayerAttribute + :return: LayerOutput object. + :rtype: LayerOutput + """ + assert input.layer_type == LayerType.CONV_LAYER + assert isinstance(input.activation, LinearActivation) + assert img_size_x > 0 and img_size_y > 0 + assert out_size_x > 0 and out_size_y > 0 + if num_channels is None: + assert input.numfilters is not None + num_channels = input.num_filters + Layer(name=name, + inputs=Input(input.name, + bilinear_interp=BilinearInterp(img_size_x=img_size_x, + img_size_y=img_size_y, + out_size_x=out_size_x, + out_size_y=out_size_y, + num_channels=num_channels)), + type=LayerType.BILINEAR_INTERP_LAYER, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput(name, LayerType.BILINEAR_INTERP_LAYER, parents=[input]) + @wrap_name_default() @layer_support() def power_layer(input, weight, name=None, layer_attr=None): diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index fc2acbd41ed90..e8be0023e7013 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -8,7 +8,7 @@ configs=(test_fc layer_activations projections test_print_layer test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers util_layers simple_rnn_layers unused_layers test_cost_layers -test_rnn_group) +test_rnn_group test_bilinear_interp) for conf in ${configs[*]} diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py b/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py new file mode 100644 index 0000000000000..7815b34abcc25 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py @@ -0,0 +1,33 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +data = data_layer(name='data', size=2304) + +conv = img_conv_layer(input=data, + filter_size = 3, + num_channels=1, + num_filters=16, + padding=1, + act=LinearActivation(), + bias_attr=True) + +bilinear = bilinear_interp_layer(input=conv, + img_size_x=32, + img_size_y=32, + out_size_x=64, + out_size_y=64, + num_channels=16) + +pool = img_pool_layer(input=bilinear, + num_channels=4, + pool_size=2, + stride=2, + pool_type=MaxPooling()) + +fc = fc_layer(input=pool, size=384, bias_attr=False) + +outputs(fc) \ No newline at end of file From 460320a41c24c6f1150bd4d9a9e499474ca28935 Mon Sep 17 00:00:00 2001 From: liaogang Date: Sun, 30 Oct 2016 10:13:09 +0800 Subject: [PATCH 194/324] fix type unmatch on gcc --- paddle/math/Matrix.cpp | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 602d7db035deb..ce4d2ac39938f 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -3786,6 +3786,7 @@ void CpuMatrix::bilinearForward(const Matrix& in, size_t outputH = getHeight(); size_t inputW = in.getWidth(); size_t inputH = in.getHeight(); + (void)(inputH); real* outData = getData(); const real* inData = in.getData(); @@ -3798,20 +3799,20 @@ void CpuMatrix::bilinearForward(const Matrix& in, if (inImgH == outImgH && inImgW == outImgW) { this->copyFrom(in); } else { - for (int k = 0; k < outputH; ++k) { // loop for batches - for (int i = 0; i < outImgH; ++i) { // loop for images - int h = ratioH * i; - int hid = (h < inImgH - 1) ? 1 : 0; + for (size_t k = 0; k < outputH; ++k) { // loop for batches + for (size_t i = 0; i < outImgH; ++i) { // loop for images + size_t h = ratioH * i; + size_t hid = (h < inImgH - 1) ? 1 : 0; real hlambda = ratioH * i - h; - for (int j = 0; j < outImgW; ++j) { - int w = ratioW * j; - int wid = (w < inImgW - 1) ? 1 : 0; + for (size_t j = 0; j < outImgW; ++j) { + size_t w = ratioW * j; + size_t wid = (w < inImgW - 1) ? 1 : 0; real wlambda = ratioW * j - w; // calculate four position for bilinear interpolation const real* inPos = &inData[k * inputW + h * inImgW + w]; real* outPos = &outData[k * outputW + i * outImgW + j]; - for (int c = 0; c < numChannels; ++c) { // loop for channels + for (size_t c = 0; c < numChannels; ++c) { // loop for channels // bilinear interpolation outPos[0] = (1.f - hlambda) * ((1.f - wlambda) * inPos[0] + wlambda * inPos[wid]) + @@ -3838,6 +3839,7 @@ void CpuMatrix::bilinearBackward(const Matrix& out, size_t inputH = getHeight(); size_t outputW = out.getWidth(); size_t outputH = out.getHeight(); + (void)(inputH); real* inGrad = getData(); const real* outGrad = out.getData(); @@ -3850,20 +3852,20 @@ void CpuMatrix::bilinearBackward(const Matrix& out, if (inImgH == outImgH && inImgW == outImgW) { this->copyFrom(out); } else { - for (int k = 0; k < outputH; ++k) { // loop for batches - for (int i = 0; i < outImgH; ++i) { // loop for images - int h = ratioH * i; - int hid = (h < inImgH - 1) ? 1 : 0; + for (size_t k = 0; k < outputH; ++k) { // loop for batches + for (size_t i = 0; i < outImgH; ++i) { // loop for images + size_t h = ratioH * i; + size_t hid = (h < inImgH - 1) ? 1 : 0; real hlambda = ratioH * i - h; - for (int j = 0; j < outImgW; ++j) { - int w = ratioW * j; - int wid = (w < inImgW - 1) ? 1 : 0; + for (size_t j = 0; j < outImgW; ++j) { + size_t w = ratioW * j; + size_t wid = (w < inImgW - 1) ? 1 : 0; real wlambda = ratioW * j - w; real* inPos = &inGrad[k * inputW + h * inImgW + w]; const real* outPos = &outGrad[k * outputW + i * outImgW + j]; - for (int c = 0; c < numChannels; ++c) { // loop for channels + for (size_t c = 0; c < numChannels; ++c) { // loop for channels inPos[0] += (1.f - hlambda) * (1.f - wlambda) * outPos[0]; inPos[wid] += (1.f - hlambda) * wlambda * outPos[0]; inPos[hid * inImgW] += hlambda * (1.f - wlambda) * outPos[0]; From 212d3391029b49526eb617aa30c49119034c6724 Mon Sep 17 00:00:00 2001 From: zhouxiao-coder Date: Mon, 31 Oct 2016 16:36:17 +0800 Subject: [PATCH 195/324] Adding an introduction doc for Paddle to implement simplest linear regression. --- demo/introduction/README.md | 4 ++ demo/introduction/dataprovider.py | 24 +++++++ demo/introduction/evaluate_model.py | 36 ++++++++++ demo/introduction/train.sh | 21 ++++++ demo/introduction/trainer_config.py | 32 +++++++++ doc/index.md | 1 + doc/introduction/index.md | 101 ++++++++++++++++++++++++++ doc/introduction/parameters.png | 1 + doc_cn/index.rst | 2 +- doc_cn/introduction/index.md | 105 ++++++++++++++++++++++++++++ doc_cn/introduction/parameters.png | Bin 0 -> 44469 bytes 11 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 demo/introduction/README.md create mode 100644 demo/introduction/dataprovider.py create mode 100755 demo/introduction/evaluate_model.py create mode 100755 demo/introduction/train.sh create mode 100644 demo/introduction/trainer_config.py create mode 100644 doc/introduction/index.md create mode 120000 doc/introduction/parameters.png create mode 100644 doc_cn/introduction/index.md create mode 100644 doc_cn/introduction/parameters.png diff --git a/demo/introduction/README.md b/demo/introduction/README.md new file mode 100644 index 0000000000000..bebf1d090d986 --- /dev/null +++ b/demo/introduction/README.md @@ -0,0 +1,4 @@ +This folder contains scripts used in PaddlePaddle introduction. +- use `bash train.sh` to train a simple linear regression model +- use `python evaluate_model.py` to read model parameters. You can see that `w` and `b` are very close to [2, 0.3]. + diff --git a/demo/introduction/dataprovider.py b/demo/introduction/dataprovider.py new file mode 100644 index 0000000000000..be8c0bc89156c --- /dev/null +++ b/demo/introduction/dataprovider.py @@ -0,0 +1,24 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer.PyDataProvider2 import * +import random + +# define data types of input: 2 real numbers +@provider(input_types=[dense_vector(1), dense_vector(1)],use_seq=False) +def process(settings, input_file): + for i in xrange(2000): + x = random.random() + yield [x], [2*x+0.3] + diff --git a/demo/introduction/evaluate_model.py b/demo/introduction/evaluate_model.py new file mode 100755 index 0000000000000..8cfb843c42105 --- /dev/null +++ b/demo/introduction/evaluate_model.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Print model parameters in last model + +Usage: + python evaluate_model.py +""" +import numpy as np +import os + +def load(file_name): + with open(file_name, 'rb') as f: + f.read(16) # skip header for float type. + return np.fromfile(f, dtype=np.float32) + +def main(): + print 'w=%.6f, b=%.6f from pass 29' % (load('output/pass-00029/w'), + load('output/pass-00029/b')) + +if __name__ == '__main__': + main() diff --git a/demo/introduction/train.sh b/demo/introduction/train.sh new file mode 100755 index 0000000000000..06db8edd105ad --- /dev/null +++ b/demo/introduction/train.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +paddle train \ + --config=trainer_config.py \ + --save_dir=./output \ + --num_passes=30 \ + 2>&1 |tee 'train.log' diff --git a/demo/introduction/trainer_config.py b/demo/introduction/trainer_config.py new file mode 100644 index 0000000000000..3e3df5583282a --- /dev/null +++ b/demo/introduction/trainer_config.py @@ -0,0 +1,32 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +# 1. read data. Suppose you saved above python code as dataprovider.py +data_file = 'empty.list' +with open(data_file, 'w') as f: f.writelines(' ') +define_py_data_sources2(train_list=data_file, test_list=None, + module='dataprovider', obj='process',args={}) + +# 2. learning algorithm +settings(batch_size=12, learning_rate=1e-3, learning_method=MomentumOptimizer()) + +# 3. Network configuration +x = data_layer(name='x', size=1) +y = data_layer(name='y', size=1) +y_predict = fc_layer(input=x, param_attr=ParamAttr(name='w'), size=1, act=LinearActivation(), bias_attr=ParamAttr(name='b')) +cost = regression_cost(input=y_predict, label=y) +outputs(cost) + diff --git a/doc/index.md b/doc/index.md index df03a33fac98c..a4dffb0405a6b 100644 --- a/doc/index.md +++ b/doc/index.md @@ -3,6 +3,7 @@ PaddlePaddle Documentation User Guide ---------- +* [Introduction](introduction/index.md) * [Quick Start](demo/quick_start/index_en.md) * [Build and Installation](build/index.rst) * [Contribute Code](build/contribute_to_paddle.md) diff --git a/doc/introduction/index.md b/doc/introduction/index.md new file mode 100644 index 0000000000000..004ca07844da0 --- /dev/null +++ b/doc/introduction/index.md @@ -0,0 +1,101 @@ +# Introduction + +PaddlePaddle is a deep learning platform open-sourced by Baidu. With PaddlePaddle, you can easily train a classic neural network within a couple lines of configuration, or you can build sophisticated models that provide state-of-the-art performance on difficult learning tasks like sentiment analysis, machine translation, image caption and so on. + +## 1. A Classic Problem + +Now, to give you a hint of what using PaddlePaddle looks like, let's start with a fundamental learning problem - **simple linear regression** : you have observed a set of two-dimensional data points of `X` and `Y`, where `X` is an explanatory variable and `Y` is corresponding dependent variable, and you want to recover the underlying correlation between `X` and `Y`. Linear regression can be used in many practical scenarios. For example, `X` can be a variable about house size, and `Y` a variable about house price. You can build a model that captures relationship between them by observing real estate markets. + +## 2. Prepare the Data + +Suppose the true relationship can be characterized as `Y = 2X + 0.3`, let's see how to recover this pattern only from observed data. Here is a piece of python code that feeds synthetic data to PaddlePaddle. The code is pretty self-explanatory, the only extra thing you need to add for PaddlePaddle is a definition of input data types. + +```python +# dataprovider.py +from paddle.trainer.PyDataProvider2 import * +import random + +# define data types of input: 2 real numbers +@provider(input_types=[dense_vector(1), dense_vector(1)],use_seq=False) +def process(settings, input_file): + for i in xrange(2000): + x = random.random() + yield [x], [2*x+0.3] +``` + +## 3. Train a NeuralNetwork in PaddlePaddle + +To recover this relationship between `X` and `Y`, we use a neural network with one layer of linear activation units and a square error cost layer. Don't worry if you are not familiar with these terminologies, it's just saying that we are starting from a random line `Y' = wX + b` , then we gradually adapt `w` and `b` to minimize the difference between `Y'` and `Y`. Here is what it looks like in PaddlePaddle: + +```python +# trainer_config.py +from paddle.trainer_config_helpers import * + +# 1. read data. Suppose you saved above python code as dataprovider.py +data_file = 'empty.list' +with open(data_file, 'w') as f: f.writelines(' ') +define_py_data_sources2(train_list=data_file, test_list=None, + module='dataprovider', obj='process',args={}) + +# 2. learning algorithm +settings(batch_size=12, learning_rate=1e-3, learning_method=MomentumOptimizer()) + +# 3. Network configuration +x = data_layer(name='x', size=1) +y = data_layer(name='y', size=1) +y_predict = fc_layer(input=x, param_attr=ParamAttr(name='w'), size=1, act=LinearActivation(), bias_attr=ParamAttr(name='b')) +cost = regression_cost(input=y_predict, label=y) +outputs(cost) +``` + +Some of the most fundamental usages of PaddlePaddle are demonstrated: + +- The first part shows how to feed data into PaddlePaddle. In general cases, PaddlePaddle reads raw data from a list of files, and then do some user-defined process to get real input. In this case, we only need to create a placeholder file since we are generating synthetic data on the fly. + +- The second part describes learning algorithm. It defines in what ways adjustments are made to model parameters. PaddlePaddle provides a rich set of optimizers, but a simple momentum based optimizer will suffice here, and it processes 12 data points each time. + +- Finally, the network configuration. It usually is as simple as "stacking" layers. Three kinds of layers are used in this configuration: + - **Data Layer**: a network always starts with one or more data layers. They provide input data to the rest of the network. In this problem, two data layers are used respectively for `X` and `Y`. + - **FC Layer**: FC layer is short for Fully Connected Layer, which connects all the input units to current layer and does the actual computation specified as activation function. Computation layers like this are the fundamental building blocks of a deeper model. + - **Cost Layer**: in training phase, cost layers are usually the last layers of the network. They measure the performance of current model, and provide guidence to adjust parameters. + +Now that everything is ready, you can train the network with a simple command line call: + ``` + paddle train --config=trainer_config.py --save_dir=./output --num_passes=30 + ``` + +This means that PaddlePaddle will train this network on the synthectic dataset for 30 passes, and save all the models under path `./output`. You will see from the messages printed out during training phase that the model cost is decreasing as time goes by, which indicates we are getting a closer guess. + + +## 4. Evaluate the Model + +Usually, a different dataset that left out during training phase should be used to evalute the models. However, we are lucky enough to know the real answer: `w=2, b=0.3`, thus a better option is to check out model parameters directly. + +In PaddlePaddle, training is just to get a collection of model parameters, which are `w` and `b` in this case. Each parameter is saved in an individual file in the popular `numpy` array format. Here is the code that reads parameters from last pass. + +```python +import numpy as np +import os + +def load(file_name): + with open(file_name, 'rb') as f: + f.read(16) # skip header for float type. + return np.fromfile(f, dtype=np.float32) + +print 'w=%.6f, b=%.6f' % (load('output/pass-00029/w'), load('output/pass-00029/b')) +# w=1.999743, b=0.300137 +``` + +
![](./parameters.png)
+ +Although starts from a random guess, you can see that value of `w` changes quickly towards 2 and `b` changes quickly towards 0.3. In the end, the predicted line is almost identical with real answer. + +There, you have recovered the underlying pattern between `X` and `Y` only from observed data. + + +## 5. Where to Go from Here + +- Build and Installation +- Quick Start +- Example and Demo + diff --git a/doc/introduction/parameters.png b/doc/introduction/parameters.png new file mode 120000 index 0000000000000..f47e74c94fffa --- /dev/null +++ b/doc/introduction/parameters.png @@ -0,0 +1 @@ +../../doc_cn/introduction/parameters.png \ No newline at end of file diff --git a/doc_cn/index.rst b/doc_cn/index.rst index d2d50fbdb47f2..715da44fb41d4 100644 --- a/doc_cn/index.rst +++ b/doc_cn/index.rst @@ -3,7 +3,7 @@ PaddlePaddle文档 使用指南 -------- - +* `介绍 `_ * `快速入门 `_ * `编译与安装 `_ * `用户接口 `_ diff --git a/doc_cn/introduction/index.md b/doc_cn/introduction/index.md new file mode 100644 index 0000000000000..164cb7d4943df --- /dev/null +++ b/doc_cn/introduction/index.md @@ -0,0 +1,105 @@ +# 简介 + +PaddlePaddle 是起源于百度的开源深度学习平台。它是简单易用的：你可以通过简单的十数行配置搭建经典的神经网络模型；它也是高效强大的：PaddlePaddle可以支撑复杂集群环境下超大模型的训练，令你受益于深度学习的前沿成果。在百度内部，已经有大量产品线使用了基于PaddlePaddle的深度学习技术。 + +这份简短的介绍将像你展示如何利用PaddlePaddle解决一个经典的学习问题。 + +## 1. 一个经典的任务 + +让我们从一个基础问题开始：单变量的线性回归。问题假定观测到了一批二维空间上的点`(x, y) `，并且已知 `x` 和 `y` 之间存在着某种线性关系，我们的目标是通过观测数据还原这个线性关系。作为一个简单基础的模型，线性回归却有着广泛的应用场景。比如可以想象一个资产定价的简化场景，其中 `x` 对应于房屋的大小，`y` 对应于房屋价格。我们可以通过观察市场上房屋的情况获得二者之间的关系，从而为新房屋的定价提供参考。 + + +## 2. 准备数据 + +假设变量 `X` 和 `Y` 的真实关系为： `Y = 2X + 0.3`，这里展示如何使用观测数据还原这一线性关系。如下Python代码将随机产生2000个观测点，它们将被用作PaddlePaddle的输入。产生PaddlePaddle的输入数据和写一段普通的Python脚本几乎一样，你唯一需要增加的就是定义输入数据的类型。 + +```python +# -*- coding:utf-8 -*- +# dataprovider.py +from paddle.trainer.PyDataProvider2 import * +import random + +# 定义输入数据的类型: 2个浮点数 +@provider(input_types=[dense_vector(1), dense_vector(1)],use_seq=False) +def process(settings, input_file): + for i in xrange(2000): + x = random.random() + yield [x], [2*x+0.3] +``` + +## 3. 训练模型 + +为了还原 `Y = 2X + 0.3`，我们先从一条随机的直线 `Y' = wX + b` 开始，然后利用观测数据调整 `w` 和 `b` 使得 `Y'` 和 `Y` 的差距不断减小，最终趋于相同。这个过程就是模型的训练过程，而 `w` 和 `b` 就是模型的参数，即我们的训练目标。 + +在PaddlePaddle里，该模型的网络配置如下。 + +```python +# -*- coding:utf-8 -*- +# trainer_config.py +from paddle.trainer_config_helpers import * + +# 1. 定义数据来源，调用上面的process函数获得观测数据 +data_file = 'empty.list' +with open(data_file, 'w') as f: f.writelines(' ') +define_py_data_sources2(train_list=data_file, test_list=None, + module='dataprovider', obj='process',args={}) + +# 2. 学习算法。控制如何改变模型参数 w 和 b +settings(batch_size=12, learning_rate=1e-3, learning_method=MomentumOptimizer()) + +# 3. 神经网络配置 +x = data_layer(name='x', size=1) +y = data_layer(name='y', size=1) +# 线性计算单元: y_predict = wx + b +y_predict = fc_layer(input=x, param_attr=ParamAttr(name='w'), size=1, act=LinearActivation(), bias_attr=ParamAttr(name='b')) +# 损失计算，度量 y_predict 和真实 y 之间的差距 +cost = regression_cost(input=y_predict, label=y) +outputs(cost) +``` +这段简短的配置展示了PaddlePaddle的基本用法： + +- 首先，第一部分定义了数据输入。一般情况下，PaddlePaddle先从一个文件列表里获得数据文件地址，然后交给用户自定义的函数（例如上面的`process`函数）进行读入和预处理从而得到真实输入。本文中由于输入数据是随机生成的不需要读输入文件，所以放一个空列表（`empty.list`）即可。 + +- 第二部分主要是选择学习算法，它定义了模型参数如何改变。PaddlePaddle提供了很多优秀的学习算法，但这里使用一个简单的基于momentum的算法就足够了，它每次读取12个数据进行计算和模型更新。 + +- 最后一部分是神经网络的配置。由于PaddlePaddle已经实现了丰富的网络单元（Layer），所以很多时候你需要做的只是声明正确的网络单元并把它们拼接起来。这里使用了三种网络单元： + - **数据层**：数据层 `data_layer` 是神经网络的入口，它读入数据并将它们传输到下游的其它单元。这里数据层有两个，分别对应于变量 `X` 和 `Y`。 + - **全连接层**：全连接层 `fc_layer` 是基础的计算单元，这里利用它建模变量之间的线性关系。计算单元是神经网络的核心，PaddlePaddle支持大量的计算单元和任意深度的网络连接，从而可以挖掘复杂的数据关系。 + - **回归损失层**：回归损失层 `regression_cost`是众多损失函数层的一种，它们在训练过程作为网络的出口，用来计算模型的表现，并指导模型参数的改变。 + +这样定义了网络结构并保存为`trainer_config.py`之后，运行训练命令即可： + ``` + paddle train --config=trainer_config.py --save_dir=./output --num_passes=30 + ``` + +PaddlePaddle将在观测数据集上迭代训练30轮，并将每轮的模型结果存放在 `./output` 路径下。从输出日志可以看到，随着轮数增加损失函数的输出在不断的减小，这意味着模型在不断的改进，直到逼近真实解：` Y = 2X + 0.3 ` + +## 4. 模型检验 + +训练完成后，我们希望能够检验模型的好坏。一种常用的做法是用模型对另外一组数据进行预测，然后评价预测的效果。但在这个例子中，由于已经知道了真实答案，我们可以直接观察模型的参数是否符合预期来进行检验。 + +PaddlePaddle将每个模型参数作为一个numpy数组单独存为一个文件，所以可以利用如下方法读取模型的参数。 + +```python +import numpy as np +import os + +def load(file_name): + with open(file_name, 'rb') as f: + f.read(16) # skip header for float type. + return np.fromfile(f, dtype=np.float32) + +print 'w=%.6f, b=%.6f' % (load('output/pass-00029/w'), load('output/pass-00029/b')) +# w=1.999743, b=0.300137 +``` +
![](./parameters.png)
+ +从图中可以看到，虽然 `w` 和 `b` 都使用随机值初始化，但在起初的几轮训练中它们都在快速逼近真实值，并且后续仍在不断改进，使得最终得到的模型几乎与真实模型重合。 + +这样，我们就完成了对单变量线性回归问题的解决：将数据输入PaddlePaddle，训练模型，最后验证结果。 + +## 5. 推荐后续阅读 + +- 安装/编译：PaddlePaddle的安装与编译文档。 +- 快速入门 ：使用商品评论分类任务，系统性的介绍如何一步步改进，最终得到产品级的深度模型。 +- 示例：各种实用案例，涵盖图像、文本、推荐等多个领域。 diff --git a/doc_cn/introduction/parameters.png b/doc_cn/introduction/parameters.png new file mode 100644 index 0000000000000000000000000000000000000000..2ec67480951e21f0400bce1c34b3108dcd65c18c GIT binary patch literal 44469 zcmeGEWmr{P_dgE9ra`)-5u_XGMp9|%25CWR)1A^F2D#}}IwUvU-67rG-TW6v&;32W z_s`46>*A8V_KG>@nsba#j7gZXq6|7JF)9oU4Eh^c$#*a?AU_xw*b-z!;7H%1sXp)@ zEcl(wYnb9e(rw@aioL8h7zPH@_~{2WUo_7OI6>P=P0LA3L0-Vb?jx(Qsoi@sR=1D# zz}YY`LT&=UuOH2vj49nd+Sq~x+=QwAIYR*W{pm0p73DvtI9Ur*X(=dEO4vD?QF60# zv9eQ%pi)v&3OSmZ3%rw*{(Cv_Ntnvg$;n=Tjm_27mDTkXtDU0-8wWo>KN~wI8z(0V za0Uz5-PXz2jl~vB{pTkC-bd05Y~pBT?__0XOZjwPRgWXF(@={Zu=}dbg3wj~8)$ry}Aqnrt zJ4x*N%z|mNjsu;kasnq7>yr9u4N>j-_j$+x$N``L*~13J*9wV;-h*ojwx8Z0f8p$b z(lyntL%>Sci-NU7(7fxr6{A7q=djd9@R5Gl>>z2%_sEh`@(M+n(#`)l9PNi9=85&^ zjDL<}8NyyDVz;>%V1+hlc*!|yY^9Q}aDkErKTQe=xtHohpUv)2T`(Hq%8YeMN1=9w{p`nT@6dg;|i$1#-@^ZBlY@m}ua-7c}GdX`L_L64!! z2EOFhb#<5+m#Ij3McW>$`4pEsT>HwNog>y>AuRcz!~dV#+(3Td_(w;2v6P_~d!Hz_$P; z>e_`GpA9jff8r<>u!Ck@Tk9DVbj@vUa4B3MZN@y#_5LuHSBh!y9Oz2sZQF+W5VMVo z@HHr}4>3gFAmtT2%#B%TpCx$OL9*6=EH7Adv6L7p9zTy)?8Am)2AKyVrCHx8TfyIM z|5~UUvs-NR=DS||s{iQ2QqdU|8NRvP&Jjo;(*tM}98W@wW| zx*@Fkb$lZwLn>v3-|z)AL3p=F7^Ka>OpZ(y_B4OM>OZGKW1fV+@-~maYFfM#FL|G# zrP+&0@df6P8=dD6dWw<6^O3j=PKf#)%t=`K$)>2g&PCP7*-zQoFTXKq4(?m6zmb&{ z%aV?o6Z^a~QDE|Lcb%_WU5p##W$(r!C0?LW#O`%vuU4qVu@-2aN)rNe8|t{`mZzBV zRxziLmLF&75+aIo4L71^ie*%wlqTX$l~t}GS7^{6pxg8)D}mzxQXYW5vKVZy7!uhv z*Z^Su7u=s_190EsHk2pM}*Yi>g4Yp~j@^0+6YrW}$gsD!8P}A8R)5C8!dM_gjmY6P~Y2iQMSCa5-z`5wG3C974jh=oh6o zXUkrtHn5o$hps(!ZFn9Fu9fqKlq>M*BBMES@q?tx1wLM4eDsoJL)W95A&L}~=+)k) zSi`<}H_X#uETYR4&`}VsVrXQq4htEq%WY4@s|b4o`d}r}lB0^pMeR$P_Xw2ON8-wP zn4vcaX|GaIVH-LxV&62U26opTUX+JIJO`3CoSG)mio{pDVq6tED)Dx1smkf`n`X-U z25UD{cXiIg4Y$#X+p$?-BpSK{?*oR=4m7lSfh-$D0(<)qGE2rVeL}kLQ=(r_(DZP_ zH2pEuK>>zf@4h=`oImr{_whE3WVpvxpK;@qdVeKVS`w{!#pkr}?oXOlc7vxUpZR4S z!t-$k&mcE$%mH_6y3E9!SO)hx5u17m$m{TPrUjXQ^707WpnuZ=dmZ zw|oSyWs-5E%^jYdK95ez&Ag%qDn8l9Aa+rD9pXRFz`};74_A90jD!0OZ9PqsFUiGV zl~7WM48>dMoIcd?{x?^f|8^RSr?`OyZX*6meYx2`iQp}>zmc(c9}ywGGWL}PLJ($hN`QQmxOp(k1^jn{HJL^&au2dggS<`R1N7Ch8*8ES(@sI1gk0>MBxU0T zHzABhiV{(e zChuF9IiK78=~{4}T+BmK{AssYR;%=7WUsY*aoSSlxZ&!DlXycP{>c)?v_pBvZ7vfp>swnb)6(uaJwGwVN&q-=zsnLj)Fkr?V+2L&9e2Y8is%1A0I1-(T3iL{)YY?{aXW{YqR)+(;?9qAUc|b$3sn}#m@!>4{P!t zA1<3xLNoY&0!cSIEBY6y4bKK=F1lGx=Jr^gNqZ=E4Um7?&3f_XJa&t0^*vFuLMVKm z@6T4dX72XTrM1V1J|Gv{cfjIcid?1Fry`_xqtRq)b#{?^_Uq@Ny%aj>re^h?eC#B# zlLS(FOOD3oP+ANcYYmJ^XQab>$C*ki)<@vc%d0V>ol}OsQ5r=$%U|SJVf$LkMbF1o zwJI&K7ragt=zO$YCIijPwGW#feX6zEPS*R5`(F*E%fyfm&h*mgCKBup;x{@;v8}JJ zQ}hmzkgIfo+U7WLKC}7PzxL_5TK7M*o2l$cDR|_dwHWz>k0>KiwjM9sPdBd9d~UXF zXzrIoD5eRvMc$*FX#yemRPR2J7#07nr0Jo~cPLfZ zM|*XW?RD>BjvQf=7WF(08UgcBn)k(TRz3Iv-@Z5&AM6Cyy=1$_V?*Ej^TaoXP9ste zK(wj@W7j4=2rWR}!5hw0D|b`+t|@31sS3$F=MkUJBEAoH6zJDi`7XBx`;^Vq3iObM zv$FQpTTijN_V4z5j_xx3<#GN&(-zyu2fO+K2({OOPFqWaefbexM~&XMbwXP$UKHME zDw@2LhP7J{Sur9fzQ-Me2Ri4g20?j`08Z)KI8>xEH6ajlpXn$1%QW~jK$*C6bthEe z_J<9ZbsM@mot8vG_h(}--XDc-Oe_f{AjXs7#lm4T&>J>Fcj4#*5UD~T!O;%UlTJO( z1tAz@NwrQ4EVdAM_nq+{#m}gQk6Mr@mNt^@q%RNV^?j)B^oSC9Z|@vnB9Pxq36cDS z*o+b^hdQiw);ikGR-uD82Row41oZ^ef2#S1%DjB}vIg@Or`+&m5qOWJXID{hpUd8$ z5y)lr%zPzO$xv-O9at-LQ}^!Uc3dT=^+axKg2_T{xC5*8td1{}3X1*u4Ia+%yr=2~ zruM%ZHrqUZTeFc3rL!g#}52U7&=xq1o!Y;&NKD=t0{z~_FI$wD~*hNevC zE@kuO;Px48l$|2a1l3`Rt59u>!yC2FeJIHPn*!)y@ue#jqsRmaN}wmOlKEd;-#miS-qB$^n*C;Lsw6Gq4p^q|2O<1}0#YqGAVxl#^Sn^w zQ??R|Q(7czWyrdEKYI!aF0H9`U5;{AM|5Vyc6nl)RRu$pSznx@vcC8E#Gqr38#5gO z6&XvAM?aVa2}vLN_;9NT9ocyUE1NgA3|{6*E)vFazT1>gv204yxEk+FbP`=;uD=*x!fDycZBzIvSW6dxF?7ddG!aLn~1g{j&c=6zg9sE zt`YIG;mNI8Mc%qO-?cF5p|wzilMwIa(lQj-kSn0tU2)bij}|cJpO$%dU$gm5I1l0& zPQqXKu&^t4Q>G0gcSBZn^CyIe`Qs|XiXV5G2>gPq(|{W`hV+KaRW3qQyS#o0alY1Q zcy;+K>$w^l37qMfgZUK-9b_q*8$H!BDRMKa&!2S%YL=dc%0)g^pczw7O?sNH{+U7` z6>KC!Kc$Qq7UbA+{VCkUM+S!-*6k(k>A%#`ULGF$e#jFtB?+B!i#hf1E?rk1OK%dm z*$#OfK_PH+ti=)!m0krngj=nCo)x=~y^yd)LgyHi`*8`}sZRKl^%+`meP^}z0U*|F z40gT#mpcau!{V#8APn?~YAqe>=_^jL-)R+}<`Q|OY>^75KH8=vCgSQzgj0;3dHma3TnQt`q-Qk5BfM`+QBa^UtuxiB9}7n z&_!mezX+sB5+Da8>F(HcMiMPKTH^UMJwDXy-s}Nj23`}(eKm@=`InQR%N}yB_@jvT zP0cTl+5@(!ZT(8iSiHk`&q(>$b$_UNjow_|?3SzodC+AgHrBT3-X(tXDcZYp*;c{t zWy_)@z%akNivy;{Q%?H!9E9g)(rH?v=p|&o;0_C$|Dv3gwKIC6qS42pZJcYcMTmKIXx~9qFxONjj>F)bl_|9`A?Ob z_4Pw*%n#S&4Wnzpn*zq&FsL;UNz$qPd!TX00H!w0ZQ<;P@Q| z!B|S`$CjIqVjMYyeQ!Vf=N!JC-_y zQV8JpBeh@)~(jp(T(Wk9Qf*d%FtmoZrK+yYa2hW$^c8l`8lo!&~wf#1N+o6z)>G8OW z!WOF&0Q$r7%uxzSTn>RkmJ>|W_gD;Dmh)NMF;ouQqd5mPa`CJ+N3CcryH#=&of8F` zf{(@e^#YG!xOC(yufr9?ngJ zjzA0c??RBo?vB7`V{?dxMU|Qa*#3d^@?hetE|iL={KUQauuAc04~nApvO*AgygN;y z%yRSGNVJ%I1YzHD zyeXj6KXT$69OIEv1eRg!B8cf zmK@&{P;8>Dg#^S*N}`Hi5Z>J`K1Od0BwK7`Y;QTU8qWri0ZYZ$RjyF%&rAeDx}x8E z0PB57Bn-kaFld5Azry_tXGeN-3ZedVeJrq1G|<{I<+l3U;p!Wjb}^)|JvGNR907{p!bcFe2M^m zj;XNeAiCS6wnc5uRx}Nn!q7#q1n^dt?W^?u%w9?=JsWrDG@HqQ?a!egbsYzQ1zATVhyW;?G>F z1pUJFD@q!(M|q$eQ?p3Nlr8qdTk&SkbG+^#`rZ{&4yj73Kr2fO<2(HVDuj zXKT32?Xj+$mUF)X&5~#Ur}}r@-<2Jqz zL866eJBZWr|GV0?aMEGs#?fSBN5RqP{8F!(uJ4{>d6Ek6;WEwm8~Qu9!Rp6G9q)P* zV)+rvK%+ZG!Y^CV*o*)L(pF(1M&t$=NOyvO9H?LUahBjoEZ0qoXL5x&bP`#P<)T{_ z?M@bv_&i)&C|B-Jm%-_cxdM9ZRiEKQ0Kngtv%e}nv6WX5nVD>35*Xt#8nQgFPcj#v zMM@TEymM8m5iC=f23GKMp6XWA_kX8*vR`QZm7w3adS-b0NqnP>q`n;gpt8qI0I|L{ zvrn7kOO5!Q@OSdMRB^4SUFyy}Qwu<5R@~#rXbD6zS?i89dS%n#an9~@=T`b7&ibNQ zyQ0(k!+5?w09TLzB;BaH8vifICDE}w}vwmpS=@Wa9fSK0@B5kYXJdt1mGhtB!Ar1 z++5mGO!R^``-()wiw}S!fv1ClLsBT`w&w=cT~acXnFy2={d44sc5b%E;RhGmgMXGT zmIi!gxujO~dZyGcVVcQN8p;x7R7v9vYW2KJGH$@$ZZ&|!#tXG^0nvhpRuQswbFFg4 zqFw$HOo?1V!fxbu%Vjm%&P@VvP_p&Dc-MRQ-VR8;dRhS1^*`MaXo8Ma->pTPYA5Yag&%j`#i- z$p~O|9>#xsR~m9YjFM=AtXSR6T~;=aeCcW-M?pbx+y6DCm)2b8yj%C+6>s3dQ0KHA z0Ei{EA|5v7W=$=+wzVDFNKak>z%N_1(fFhKimDfObQ*qY6z$}?)!rV|>%*!Y9ZSYw zlJfbfWJxarYt-3Y=ytE7SaiA6xNR7KME!2HQ_Zi@lEvCl#Jr8HW_$p}WTebQ=F)$m??U!Gl?M(DJhd1n%_W_aP0{6r9Kl~@0%pV$B3vKW5?6B&0I~$Qn znKo$><(VDLlyL<*kO?>#gG=J#vLjt8t&o#og@($7y48^^8H*e6F!KF2 zlt4__9DLe_DY_>D3HTJf@?}y6ye#*R(p|c@cXKgoSC3J4SN=rT6NKE$UquwiuCe{> zV{0P4zr7rS20nXb@s~b`E#fbW1+-e9ZVV_oxd#*Wb^BB7T@v;>BqbPiU7FHAF1dLO zo;kOd+-!?Lo|nllYml$2;oSlvK5Ff|oVN$!OZ*frNww;vJE_!33m&^HQ@)wn73NQh zASXg*ZksR#{rv8)!8V>hlTGKjEdUSd59EW2z6y+Sc5$ANxj&^ekanL|($0~m3~*2TK#^u*UwbW`2Y$I9I2vEp%`lY_b>v7_YVKu6$?$ z7AkpTLUUms|Azc_tYM0)I?Rx8{7%7;^L4suj>Fb&QNBeJ{wc>@O2SkC(adt|QpgF= zFxUc<`s2#F{iStrEuxah^~)#0hwf~;>kxPuu37&4R0#u(T-am1)y8dzgalrvM1zTA z4Y>A*U1i`up^u&)UTtryac!LWG6@eUBi<9;R9z&IN=84sshV05W`{6N;g{Z0Qh+jx z_nlO8ZRB*2To-n;9&T(jnhp59eCG()))3gmYr@1X;k5EE^7=*(x*;F-4*RJmqW0-f zUBE8>rnpfpAB_*?oV*{_+)U%R_*U4OUBp)FE{?e1WuG` z%=_U>{l(a2`nv!*n@i=mG>lI;D4Wn@FXg~sQVN}OwCFalrxDficB#NWbr>hol;eWp z3(PWl%yu!4jk^RBQ1{bP zPrFZb7CH~IvyZjvd2%o`BJg-#3z!8`zTRR`hw&14wfUca%P6kJD6`#g$_VcW84IB8 zD9-xyqHz+7`YpGRR<4w#E1{w7B<1r7r%HZjAeP%2tu;N~AKJ`kaT%x_f(s;6;)_a8 zV`R~cj3uh>{=V!r>~yUY0iE-Ut4lGDcz*2e(|0HFOLF#`#FwRy>=!*0Qj}z?@!y;H z3?b4zrv@D}0SIWNUe}JiuQ~urUZu&$`-vu=>u-L^g>=OQ5-^O{$U}w=Oeg{a_V>r~h`hz;Og|W@`Zmctpx~ z$4N*OUc>0AHuH7dfDubj?tVQq&LUcZ(r9s==;9#P$L!a!AFb{F*AQSvhL3nbz$o>A z(L+(>8*&%&c}FgIF}}et{gc#nHp!PevTvgJJ|FP!Ug58en^Vg~JHuehMxjT`@#_v0MY?J*tK1DBggg}zES)h`?#u(o>!1d}w$DXe`X>~CR`X*`UY$-Q}41L-EFX|)JH8v+kt-z_pmAY z*wP+`dwkS}S$}Geqp!0;&4}wy4sfWWT;1<~X5P{jN)Qf@JHDfPRQ%LCasj6&I9`tn zt~@&5pPh=@%AvgN5`FLoYy~_>=K2!=8>rJm_E)ITTMoCLANyrBhG>!MPRoZC)R#Sy z7Vrb`CZ44H2nlP$QWMg9_Q-8zp`X*E-bc<-4&*zj_)>s?qV)d?D0|33UrU$OSO9y< zCCi;ZQ9A}kXG=p3{X^}r!xLlrmPLKZ=0on94RFD#3Fz(X`?@J^#;GZR(o~yq}$vg zT0leepkb=^EBkLtqPA>4DV=Ayep2C!oPQ_L%3HBu6_q%{zTrG23W(Q9PceOqhCt_t zTO^iO;bz?w-vStwZjeq^j(RMwdo9&R?_ppUI^}-fw#*0V)0p-G5E_p|-w;sM_Fkg7 z5TRp31OJV9+xGPm{ZfDvo}#eFnSGk_br5cZ^15o%$~#3CJMja+l>g&x`@dIsy5{Br z-))(C|2o}mg>E`l!ogTYua~I5hH_Bia)UV(!u*dUA;v+~oXawpWe$`I`oqA6)A8IQ z`pT=}57D*58y!RKG(G@{6d=+?Br|DNwah6U*kAOx{T1)gD0SsVpJcxeG|$W!m^r>P zzrg2!)p~84t=G}SpPB$nUIM(khl#b+1nYj@Wrn`SJ& z@QG=iV6?ns-c1E;;6Dn07(DJ0nl*rx6pv_Le(Q?=>g&Ws?sX*oMCz72@&XajUw1{5 zy)tydj_mu%3g7x>%?%+ToRnu`)G9$RR-sgE%*y0>v9V0O^w?-a`BTuP4!#a#c<}<3 zgM)*ZhWSEgu#{dNo3NGHPs$69*%Rnrc&c9`d+poS%dd@{U82mL>r;y%9IhgthD2_k z#p}BydWJc^VF62vp=^P(Hl==9@YCDSUpXD(`x=M>wMT;YXKghLX94%%f>pp}-qN60 z4u2Vlgo(I`o_)IUefB&Un?TWwA@XJ;4$!*5StiE7tdoR?^VWxUb{gv}PS zm93$irL7?@MZRTtwIZE~j$>OV5iwTLOCVgV0*D-*E@$KXgZ8tIo@8lb@(@*R)QW|S z6HeIY;o(o50gD5ke#>%wN$#X-*0v3ZHhS|i=!c(FLl7hv0g4Mr3u_8>XFgF=@g}EV zqDN4bG!nj^(42(9L(PtzWk%DTwG1hr|t4@3Y_6^GLvbM5&; zldqfsI;h)aof5k3WhyZOOzL85xUrhGFLDqVuk#7b`I#1gb+@i`L~y)sM+P)gwpXHY zN=OyLQ&F0~c)1qY0&i>Wt(YO=F0V~Hx2FCl zw#K{8j>F+HIC77p7iZt%PpnJsNuI2A&)Q&J-&{Q=3x5Pfv7$F0kk0ycp1VF=kd#{C zVo_kq`qt=>2`o8cs|<$0;DL#G4(21`^ArN*kwgI?woJ4^HbxkN20h&R-zKXo`94gXgLE(gw?^At=rE zO%fU+z%n>VfiaY={C#9rYJN|9$L9Kfez0fpfdgOY)r}R$>J={rnrR@xjSmx0yH0} zHPWj>dj0$pLNoVpPCFnrzgaxt0ZFG1qrgY9!zK@uf4Ana_|2;JQv)ebsR=nAGJ%`- zY%9{zkl1BUKk(%E9E+OmJPyzRJos7Kj_A?@6y=CnUT>+DU9GQi`K&UkH!IQSIlK;D z-{{)4oLB97@VU*lN~Agiib(06DnBai78lO9XUj}NNPhIym;n} zLqm@6mkI+zl`Jez+(t|b8~N@A6VK}SMbx+}^{!9q+(g+Z3nL4%-(19#jN4qHYQQY2*)XBW)W{9eD*$;mzF|?)# z27DeQ4ghui<1c?xc{!LSx`-lmF_ne?8DtL#dc}(r2&O;`3I_^Zr!yzVD2q37F9 z5a$Q_y@dv}{hfHJ0@`(lr6rduZ?mJTGj})R86l{PkBPmSMJ05rpU5AI)g3YHprk4y zstOp}#VLFaR0lv^joMGOp8el8x82k+OAA2#Qx>@1^)Q;mwx%Tt(cGKQ%>=M12E#RT z<%ujuofAYYCVIFBpW)%CEtjTWtU%UZ~su)yeN3I`3E*ZMBRV2n=QMqbBEZ|YRR6kRWVPV9%5_?%9H&-Bmp=6U zpe^GyX+xiguagezuR#YTQGs(X9TACL0`J`JP(DQNGetDcBuYBJ?*>ct+{z6xXY?hp zoz62fq^^@-liKirD3s>RbX#|T2rDnakB;%TR17(LrHQx_U4*9BA8uckI`>r996ltg zH0V&7Gv6Y_d-1^LtKgHc)jbOfYmR3ye9==)k|&eGN>y5C`5Mcm_?K;kfvQc|@B(+-;fHthwTO$gcHdXXjQ_LaGQMNtsmK z_S-g%M-8=jQLr>69cbB9;l1`1%0X7V>Tl`^QOJR|$|As<;reMa?{z-`S9!4x3K>R6Z&g+^DOdY8c8JTxbJu}@At74*sRiAT2G@v35-zmcNjyyz1vEkc>(R%rMF(k>%6RCu8Zy@)Po{~a@8Lt zytiLcZ00tFN|lQenPtie#%?0Icgr!w`8H}{zz<^i`e&K2M0BuS!&efiTrR&e!OvxE zhOVoM#!c)FokCh2Pd2#?y4)7H8oCzTmhDLg&6sRNfI^4lnsq>_6l=t`XUmUlRNCdS zY2mHVH1>RJwEY8cAm9Op+G>GDXma%^(mLDrX(e+|-^{*?!0Y=1chXH#rmk9bbA%kN z|HUncK-BPrG+hlL-3izIRc_zxD6{{yOWRx7*NkpXX$?BCa#}7;%1k}*U1<1po{Tbr zst#G6cGu%dW5ew81)`TbgkDj+(d5>8gP2!9+^bcW;^U>v%jO5)GwFDJ$dtw6~3ij(;y2p3Z1*YwGXhgTMLHloZbI7jP z8+98Ii)P}w?h+EG*Ogj@uC1wj)z@NHhUf+wr>{9muWhh>+x+StVc+v{RBD4B$I%&E z2-jI7PZ-xuh}rBYG06nhf(Tfa_1faUimQ5!lzGD6S0+o=I<9lFxD?7cI+p8_1$99` zF$D~}-}8@DD8AeW1@b%6`CVl#$bGGa9s3eaN6O@!fz9_0`Nr(Ja{lBu zLz?~xoGINqGgJ2^8A*+@(J{BBz<lN{j5KN=n7RzS|?6|H~9|D^&iNn1) zW=rYUEyh2iWFSeW1oWzWzvcmJNbcQ$rXBJMV`;)@^-#yXutJNd>CEqOkvAMEm%Isx zf;Sl$W;;gXs0ga)`IWw!jrI1@y=Yq@a7!H>rHA2-Ff@9@(bcO#CMfQ(X;tEYphs;a8pA>WrF7#qcT1r#rbKbZ!LzK_CRzI=h;Kj|B; zBdu?otlWRy#X@;upZB;f?%2YI6Tu#e9-`njHe2PrQ7z&tR`A&Gut@hw`EYU~99fwU zXQ^BZBG5k}(eG^@=#~v2 zFQ;M3QLYZ^4_yfO4BixwvG@b~C0>8j-Z%mZt~-ETAoEELzIVq+lg0WGfWyS!as?Qi zHD)-1&O2+9=j)e|R>G(KuWGh3BZf01kO0%`)f8xCrh7uRxMl@4(exhXNfT$*E^kK} z&|_m>YhLoy=>8p|?gIF%J1$zrK)&h@^V7V$KlFWUbKgwa%uF4>eyQgntu#(>EVKEQ z!QlPcS4LkJvsjP1=3inghDFV_HWjpP1Q~?-*!icBy}mc0T4pv!A*-gzb0}_UJ!0}{ zXANjVCbjxRWMpiUd=K5-qFv=~w_7S(l^-ThNg*xf%Yb`q1Z>+VHY6|duRIwk+ja#E z+jRWTeb9#ci&;LPKjIM!0At!|m5w2g0>E+awdZbOSLIb%?G?bF#Wn|1^y>f+*9n-@ zTZLctRP!D5jj(N7b4aB^DPLLd!X<>YVgNlu)&wO!R!S9k-(627t{g6h*tI>1xXZ;k zM6@WoJA90#WuDQ}$S!K``axl0aige7^XqIRcT*G!+c+kFObxuD7i-BU$#_J7|DY{a?yc=x21n|$>uV6?4L9)wl zQm+Yh&D!0=`a={T)Xp)66uE7?jIO2zMm))n1_mK)``8E z=O7$s4Sf!}k>3#^WmL@grLa-&wH7O*=dj?uNd`Ti)GbbopA^zLpC-N|izRK}3-!#Qb zpsF%Cj9;oBSad1w=y?vtW2^{c+*5jAx6@Qp_(5(!(Yoy#+o}-2AM^FJ5p}})wuuR~ zx|4wf`a<(QbWa|J+`nDuY0e>m1T3__aLFhnHQ=e36;QuTDp)baXmW{K!6Mo33WyRM zHomK_f!b2vokpsb)tr&a%V`HxHCAnr%o85lvWJXs91$4le!7aVewpR97(3Fa4=$!H zBAu~?fSJ)aK*kb&aBMubbD`n*Om#5?k)UXAI1*mlLWAEIq?9XwX+M(wA5GRF$|+uBVcsQ3k$wvcc6%v ziFYPmq1(pAyh&6CFPYmqjz7`KUPp+R&<7rHNIfWQ7YObVaJ?d+hQ#;p;s5w`Co#Zk z0M^*Doo_w*<9&(NZo9s{j$vZc9<%G-WtX6_W(khHSY`Fg^n~%b@%bA;qMz*(0@>=B%s<)H;!0Z78#h~v@nstU6kY{60MSs^ylt(A1T2zyTi;+XQ zECk2ZI9W1}2$u`q)T6!X(ua??I82i^otRmy(q60c32_f_@778nC8Wf+#1Y8fc2!NC zgIR`P>M0ORX8{-H#_@gAVWGBc{>`_ZL&;XIWdIXK?ptUa=n<3)+}(2BJoG`Wsz^=TG$RU(j# z4kbhmtUD-VQjqz61aIEm$>+&A8P*CRPI_zrI zw<>|49EHG#XeB1`MBaF0MNQB~oS+Urk>aW+>P^`cT@i@0@NN#Gc$n`Cy(c#CI+o)7$}gh? zltl%8wQ&4FszA+KN?;DaH*sm&g@6ARzzB+)Zxow$Eo?=+jZAR3daNToQG;S(*bZkz zd)-Gt`^$E1Sqc9B2RXw#6$gQ;+&!7Re8ny72{?}OAOf>gwry)l_d4cHS~~SLB>nHP zj?Hf@_4D_H66%NzXXRsQIDjBtl zj)^hG7hE>=$iGldAl#{%XKd8WzuCeYU*_ZSKttxv7 ztJh@ix=mR-Zs_gjk&qr0&k_Eg5&0s$g3)LAat7SmZM+|GmBkDF);-PeD1)v{LYZyZ znC2zo@Onb&>Q9G4`oAUXpZ!qo4>x;avO$zsZ^yS?^)44&vu}cBe~LAnc@H#@6Iy;U zhPbt(isAbLzF*fQWyCnsel zz?k@5raclG`uH2?VSq;rd~W5Q9gLxz@2W|8%or!O*W!JuEDVHYnR%h;Q+aI`wLmap zGlO2^2Q~35sd@LFweS`XbB-Y=Lp7Ah!ZwdUIB^if$zKe{bALbI6-I|R_)E!XslQ5k z2^n;Ax3fUvAEQ-@b}mTe30+a96DGg)VzQgroKBA<MdqbyL`dL#up0;UPAa&>ZA#X+?!8H< zwy>OJ+kW3QhLpTw_f7uLzLD$J{@^9PTVJ*k7LvF9y#(vWR}L$Z)ki`U&lM|sLS%hM z*jX;nNddVByKVK{7;TEgZk-b(B`((FYjv`Fdmkgxg|L|#@%1(}D1@R0AE|Gu)sLuY zX=A_MVe_+()^lE@ic#~duo9F{(wI_KihFya$cq=p<=L}2qjOB$p+D#b8b;&eU$d=J z2z6wED;Em5mD;#5H2FSbKfKTUJkmNkAQ7DT3F%#F5q9@?KanBgu+2BO7=o};q9PhO znOVxLB0Rgtc#jecR@1JOG-ENxD{}F#x2H{fX{?F(WMfx8$w3|V7j8HMyn)I?+vr9^ zpKFAtjle8wHb3gFRwCwaBh9kc5B94UU5X6#a1H?S;p9E9*@H*j!Z7LEGDK*;e(kPk(uVk_5JRx9)t4iDbVMtRV~#AcK?jM5A~aP zY$oz#@9h9|`a27yn(&pVCNOwJ)&L`u{QY)lpG>@7K)GAtBu*Azey`Al;JEh)8$m z3?)dHgrt&6w{(MaDIguv-2)86d(qGLx8Akp?>qNC=bR_^-ltF@^JvSdC$(nAe9Cht zb@#)%TC90Du`pNb`74%FYx8-4tB<%=e|;fGB};-zwQO0G6WFwOSqrcuuMXEl+8yR< zJDF0A*1Ckg@-N#ExdUmC$NR+9?(iW!**5mI;5`*dg9%o|Y>Tpd=XYQnq`kGm=0cS9 z%=V{q|3alrS_oTRF9&}vJ}#^E&i5@A?EtLLWabIIRrd=RQE%?ArLFHCLSA#jZJz ze0pX*UthMj+2cipN@LZXyMC2iX1Jg=>oScaf;F3olGJbym%$Sc1rWCV7s!7+GF?JA zZK+fggA|7tjMZ!!JHSgsmP@Ecbz&^EfEdBRF{yHKJ>=l5-}|Pr1UBWS+I4`wa&{Lx?yoed%?`sW@^u0Q-i#4lonDg_$xVZDz#@g=i z#bYwBM3J88t`iajXIS35(3cpIeu1 zO%w0*v_^@Qql0BhRn6pis9K#68gJ3`TWfl;fpy%qldRy$Y90$mM4R;QWs*N&x5Pdh zojAqCEkY_WsQY&Iu5p%QzC@3^a*W!rMFO^hJY%fL=`$yJ1oTS!n~&P|HoFT7Vn=F; z-p7fZM%YmlGR1>WOkR^e=pUpX$vPp1a{B~I-05G#@q&=OcJu7D|`Q|RsQcnmqPWn^FjlCA$|2&Q1J+F1scz%N~ z(oh@{%2%B$)^j>uD^_1CiuvpD;D8|^8417idpDF%wwW+3Y|{d$2V+NaiUe=g59~hS zirhR*x!#;}r>@^tRyph3iB6Hf#FsHp+6lc!5rOhsR${rA>)ayl%F}W%3XiH1?G#!HT<{pr1jWb5c4fswd-^woT6rP zOI|Qj1d=IEEs}k5p}tyOPb!A-Skm0uIX@aBa}&{J!BM}b{fqT)7shE=eXr1Gen`bx z^-Ow2k=^81WZPX8sLEzVNVI28ffcIc<>^_u8McISa$4SMRM5PInRftO@Z+j7f(JT$ zV9SSN-l5Gu3Fi`x6eLvYw8oDlr^RotClb9T6We-jU-4q%CRQ-->EGleD&ja7$?Wl%m-kMzG_hUlPG;gs z(^taiS@w|z!?(UE$Y+-MPSp`YJ6Bs#XUsVT=r*7Q4%IHB(K5xPeFhDYW-AdwNIL62 z-91KVHS>nfkBw(0Cdxy?2i8^VpDrHcD7Zd?=-KAB)h8Mp7e#=yj*6eoRVhd^Pr-JunHpIH>9(4_cFlqTwTwjO#oJB{GGUKeRiw z-%4gK)UdZ1HPJ9iqy_5?E zp#trDnDDZ=Ij^M{g59nMv`Ht2#F{aFe!sq}zewqijgcb5Uy(*7(ym21&5I4BiW&N9 zDlHt@rV%U6lRWhz5l4%+H!#eqCPNmZA5K2HW7$&9+LI#)5hXEq{3qvi{Q`PFz5{!u zf8MV&Y?u_?SJu1|_0?^n*-o}65;a%seqFm9`T`j~tLMwU1~1@!VvF?2%>%D}co1?d z2#q@ExgzGhD&05aS9DR+efL2${Zx9cna*J-$jtHg{Ri;)o(jyzX4XZ55QBxF`!zd- zfc1*)4}>?IW;cA}JW#T30jEfIZ14P&27>?Ha0dXqSVSUJ4Pgueu!FmscT@h)#8soV zmd;xc6b+S^28UgP+?i**AhJ=f{=s)Bk(faH_E(s)uEpV?INzcSaee`nR7oN6#zxZn zt4*%nIyXJ@iN>uw?U5`Py$3Uz+@qJR#Z9l23B2t$%UqgO4mko4Y*BS|@859IvFY4f zuEhtcq_0(&O8yA7ihpg3+oC^f9 zLIV4R8PT@REZoWiw`_cV!qZX&4;CiR-V<-SsBN$h^Mxv#0BzO{HGSb|DPE>rZZHW1 zK#0RU9Iy;(9it8=my0a&CX2P6QhY_6&Zg>uYUDFNF#jr{A2FqVmS^`lvm#ijMN0mf zEM2Jny}-_-3lu%zI&s7Ad3V3lYUHfwq}9T1W80@BvR6#dk7iBU5LsoR9_)qV z3vEhzrmqjbzWXi`Y;-k~Xqy*P%z}{}ryIy? zas491(I$?5j2cKNEx})q67g=IJpD2|ObN%-b1B$ZGr6VvW;)x=6$<`@T8U3*NFtY; zPp~pqXtYQttGn;Lb-&bItjv|t*xI-Dr;TN`a|iq6e7CGv%o0p-sno@lzT?yY5uLjc z`D}732NTT3FjdO6B>fOKAYE`#v;H|ZZ+tx$_w_5RqZUGpB2>*3XZP4uin#vGXeoO9 z6>Ht{cbyVWM32)Yga5q#>Bjj!doLT|I~?FRno({-N+nq&w40L(i{4W;t@st-Xk$$+ zP%ijaQA)spoAAh^bG38Jml{C&T$!{3ooPYL%Lk4~;<(s?VyF|K(+5K~`y~kOSAt?M z$tzCnOH_@Hq%~0Zz1dx_6GX(EjqWp*;3oGM*hsWoEj8$;d?(v@%a!WN8i&xl)4$9C%!WT4LhZbP;W4O;w;YsKz(#l<}YR{ z&1J*@i4(qqKPFbFS`%(mit`1iD$uK9yP0x6)#717Hnk&5kd+}vq9910!!e0?5+f8H z^gOTf+A6LEiDkq94PwNwD`*|@{ukvBgTk+A4YrT6vm3CIxP)m3Vp8dz9?-pqp?y)( zHj8Ep78hC`q-2SEdHl1AOCBvMi6DFwJX@rmo32+D$OXVGa-A`qr7g-! z&C6i`fj!ORNCoL#RV5>k*1FOI^D(>VKv6r?H$yI;QU$c57enNJtySLU;&BfzLm}s4 z9ex5D(P^$z&CCcMeNlE7eY;5ZRd}YJyd%4AG3b^&~9`X zi%Luas<|O1*@2Jrhb||GsUpz~^gN&gFc@;zoB+)j*xq)@L~gf+``!d;!JVJWtEfJO zxWy=};$W-y(=S@i8_h!;dl&hG@~>7a#7WxHY}b+|hKE`O7^9wQ!t-q$1|FMDF++7F zJc#_NV~gF?SO*g#WnwIVO^$-1qCf<4AF7tZ@DB7PRk%rckk07MMB=2ezg&F6aFFT7 z-E-f1CDFIUi$z(_(e<@;?v0iIV~JDs&cf|Spp(=#yta|C&UPORxLjD1|{%X3r;m`o)WrUZbGP1?)m4I^| zALmB%qQQmHc`K`9XOe1I7_t4mC1XbOJpK6WG0=kn#v@74XRB7ATIQAiprL3WMAFZ~ za$-Bd)iAHN&X^g~P|?E{;twC$87v2l7S1m0b1y{Pzfy#Y>|DIfcXH~vep~(S^_vA2 z1H6xKed;NymE@Dp*QY*w5?%@U0^|!TC;iBcF*J{yBhfA|w4zMqgzf)hA)!7HJ3F20 zv}2^Im{y1(*!TJ$!uGE6tnR1A^ZW3%OAIKQFAw^ckZrurzm&7i2U6M=>B!%2OH`L; zs0?$=;Xb2HQT#NPtOk{l9k&$Vs~kvOi}e#pPXSf5vE>C>K(Q^E@OK2Q*SMj5^$%DK zD$t#|p|dbiPBMVw&GEBjdIm;S>tII&zWz=Hu@{R#NuM7(Qh)IgtDw^5ie`b|LxRiZ zH)_Bq!A{zzQ<~gO)N8$iTk=iYlA6#ezGc(>%IF}PS8rc)&?qWO{fho-_H$6sHkk~Y zAA$Nctg^$-njFA10o|RJ`*ntN$d-oyxET_VJ$HNfH#nlw_~Q%R?(YY{ z$>e);X85eV066di%?K+Dz-~x)+NYBR9Ww(4rS*r=zsDty{yo*!v2>8(;<`s6WaK4T zSit!rriabi#CY>!{^AjQ&!7vkqHH$*`k2~;usHYfL^-J~kpCT0cQ|^ATgLN8u$w&& zjHOQMrS(#>fBfMeUf^kz%&?!Ipf&ol2?P-eV3QL;#|6so@Ka6e<+CR(RUS-DsdFC| zFTU_{UZC%HIwq@DgsRPxMr=0=@c@&vfm0_TcYee{lX!V;CZ^2JMzH7FK~@_idEbaJHkRF4Td`0P$8?y*?T7 z#0pCePV`OL7Z))w?_(rGYGAa1A^KoM59R;dC;<7bPVtFA-M(8{oaOI)r6 zT|~O{%0fV<^M7egSXofSG3R-7H;)WJ8u8XDRglZ2!z_l)H?k}C@9t635nnVD?zRmdXVC=K&hz>)@LeLc*Nq>&!=}h%1TwDYM+~b#|6V zTm9bdB3GsljaERECy0U3jHH9({8Ve{VG5EUE!Df60J?$JCU#M@UzYfgY)&OblVIst00*Cdj4=Pnif3EsbU8BZDrk}QVSPTpH;Ht2~2MjAvc z!wsFM5vmGCevjccUzTBnRfADdYZ3dur@meNwgR=w)1sCqZ!Rl+tnRPk2aeD~bTI_9 zR=O{z^*f`b5CplaTuRLE5Eds#))UplOJr90~U6Jj)ikF2Fa1V>)|CD{|vJ+NXh zW)h!+WmTm_#k5k(jG8gk@3)Wv0;doTwFrjc6|)6gblKddQD!@e=sbEimaC1c8Y(XU ze?}Y3ec_pF&u#-&Lz`AM?cC%9|I$qGIvsfZYpQUSgZ4ije-IC|8JH*9DVBKTp$|k1 zQL$J})B*AVZnT*1WDXfG88^lQhhNta$GPtXv5k;?aZUHMW`wv0R-Iq3^gJL?W2#+J%8?jT@<6;1f z1qqv_kFf!5=R_*T&FBL*CqfMfEVISl$+08$o$eXAo*YvHmjVbt-l}%b;5hTFSGg>W zE%DPp^6kyO(qI3*^v|tzWj{dD+GTa67@h4=z#-y7j5XHZomedFY-P2}w2?(G9c9mS z(KSj8u%A3hmI-Tp^K0G}4<0T%b@d1d~(Z30Qr>@U6a18h5Am3sW5OpS10qH>iGi4!xbv+sz$EOnF%X z=-KvZu=nv#SZZ2fY(K;TFWQm`+KC{1@gd~_w;f6}q){I6LyeCpnM(qDP`d2@Q$g;{ zh60G9JI-3H2NKAgb5nVgv3fe+4LXEmA8h#Dnv4YGB8B;>eEqqqQa=j zgWf1h^_nUvVP*U8n$Ff)@B(UO$ndM`nzu@+u4#2wJh(J!y-z2KepoM$MtmEmMZ5#J z>ld$9hHN@K-k_=|4Sti9HEVPQo%9zhu)dDhzbExIx`X}zsEp1#sZ{a z_lUbR0Y7rKtZ@RXbL#i5mg^eqRcwccnk1?K{>W<|U*LRZpz>I@yP*c_crv8p?g|y7 zb_Xe%oKQ$tnQqMfX#*1BgPYRW$4o=z*(3S%4>_4TOQ3PmU|8y<1$iCTt%+G|l*i}E zRP@H-RE@(#kn-v$1Y5H?gZ{Z%X;3+3j{ceMkL~5Yn3`R`0VV(y^Lk5+FUr)D#uFI- z0WRYON-5@dltq;irMCoTrjm$Hi}e;Pg~P@nKetaKKCp$&eL&YNdL=OUM7>M+gWhNN z-MEOw3lV39PXO|SV2>8vbG-GsP#)1Z}gNYzU zGhW>{7vI+}MXM~gXrA;W_hs+0SGDfEqEbtb+-x(%XI#93TQ(^}+OMU8OYese+;Hu9 zf_8NY%;9uo_y#v1L4TX+Y@wSCXSQ80kJp6V7Q`PgRuVL61;4 zHuu`|B%5}W#LcP1!H$1VIOr~YJ$trVp>LYvo>aJe2_Ht+Hd_uOSL`!Qf31=Z}XO! zhWoS3q)Bv^9>x1Mw2{?Y7v|$y6&U=pe_P^5aWuREGL%H#MQl6!fV-}nHp5lG5 zfnn`i;y{yxP+W0&xJy`l^xsD zZ0s@2G7K-rRBcxf>3iVOLSs#L=<+{HOeCi4U@W?&Ck)Mmy_*?$#^aoz(T8CRBD`-QX_wM|vjR&^_+Xlw@kD>wwwmefi`!vjjs|~KU7g>*P#>NC)zPrV(9v~Q)iT!JVxx4+Kj!KT)E z$|{J5{>wXaq}uv5cb))(L1PwxgwywxVC=WVrJy&gm?~AD8&&~EYgg&(nzy41 zSPII@-GF|RW}~a+qxx0?Aer<>6YgII^p}q>AojO{I3HL8 zRtq4wOG;=}2CZ=aqJO=nQ3O}}`GF?fGmrqn zzH!ZVUmYP?1wZ+{QS_!43R^$xcO!)ObUL^^nB5EvhYaB@&n#U9JYo0%u+ooXa7i5f zSy|5kL5>viXUxoK%l>DN=IR|XcKU_vDYC_U_`d0-{vz!;`4k$8a%IN5kV)_(1}b4^ zr&rNu;nWh69U!0k%AIO@+VBJ0$LC+*(y!YbbRWT-W}}O@+R+fc9@DwFkrE<=fgs%a zzZVDmk%rzoL9RDQ0Uoyw!a;8JY8lcWPhTzeF0?6v7|J#ZGz*afw|{c~tih+G^a~A( z!z0WHzg4_10eI_8&Jx=4%Iw-Vo1{J{#mlhaD!g4CZn5yME)%Q`7MK)XvGQ-$lmvXt zah-3?(|pCcPZ(g@0ih;+UQ zGzX;S&$0{h^Bn8Gnp8U=L1}jKCS6x%LC-XRiQRg{eN|oT1O-rLJAj7> zvfa4TQaNtPbttdg&x_(Aef_o<1L3b}<;LdF@jYsdgho(vOi1>TH1`!(_0h!e+D7+% z(kB#+j0~Q|pu-6MqMfjG%f(@T#D6BaHEObi6a~iq&VOb+>vOfJx4F37c(!yk>Tt7` z`{H8W|7i$i13Ut< zU#(9OilP8T2~3B2oNrI|L9UCB7Z}&Xp!I{QzwQCk#{m)_d5THQ%uD6QqRFYLl=Q`$ zqeDadOXQ1J+>q-i8@%G3TM~*}sfMlT05XMK5pbvF+ttKhKL(fICxkRBe^%-`3(#ZW zV#H+Wi-U?TJ-W|;gxR&>T>2xfrHqVnvvRVNv)gya`C|UQMgp`IeecyhgD?Qks$Js* z#Q61$nX*3ybO~=oq!zFN*vUS*#wS4(gq4Cc;uFD4CCBTVe6{vJ$(yl)jC0BgWss%Js=#)LhxHhTyDRa0=DU80p{Vq7=Asle1qQIH#;W{@ zSMm2oNz8a8%=iv&d_YR5*a@i$e4k)FVTyjSriGZVl5qhDHV|)#CyFysEgOkAJNAEX2@Jppma`K8`Xq*X>Ne-Nu7K z=?jawcB&&8(G;ps6(^ihzT+b^zpu5KA&{H|CO%k4lfle%IBs%BX1A6IN^i+!V$X^{ zP8l894iENM$!VuK$oTcFrhA`jF8cPiDA(jpf|(K#s{j3ux*R@U^ijXylm-uXMFQ|f zp_t>y+hlRzaq~_x%%5q{3;4#LgT83gy@Go=Hr+qWbcJupcgK9G6iH)09^p^0Fib2& z{M@s$x-33vI74~Z01tb>8D$pjctyyC`!Vb_m@mKhYp2ebsahacVZ_$hfnvCzFpX-% zXt#^HL_~gyS4U1f+ZHCpRGdkB&cOww0`tR28tvO<-mNRAp;I<$C=wt^m=Mz zCUfkv>rV)z%Dexsq3$U;1GsR$oHFJej2fEmY!QgE00NYymM_ z+zs~p?_>{NYW6ioaDt{ChUB|z<*fB3wQgY|-7VIce~*Vkt}ebArBpx8Khv2(z`8IU zoz*6dAj?KdUUn;SK7KV>e?_K9WzY4{a@BF3nsNe2p>3K@+ilHTtiLB#s*GL=q;IFv ziT%|B<(u!pH~F!;5)Cwa1i`XXdUTuCq7Z*8W?!u}lHLp9*rRD4I>bwk>L8c08V2r& z2iqy+Qh@hSjUEv(CqU`x4z8kMx999I6LRRaGU7dz9>i!l_mRZ43+CqN9Zt93bJ--h zM{n`1BPCNP9*kCefC5WA)Q_N%dRRJqy4+uFufq1m=5S})O;9Hfdm{;~y0d(^)od2X zR1*am0{2MZJHb&Jdfx&9B+_qZ9M}RC)xD~MgTVIbg38CwE9$GKoogH`00;P&X@00KqB>f#!Lu*=RG24oQKl@{P5qP zup-F}o3$)3s^oran#ulkR3T8~S9)C5JuLB5!C(;^G5I!GhlGxQG>*pjmf`vHx!UVI zSk2GxPyW5YGvEb2zvDGIN}x9j%fX#VY@qbM3xKKbq3UIQO-owu?&#(97AQmhR5aax z`GrbA@nSV}i8|kNwi}mLz7H&h{jjD~Zs2{CHM$}scYPj`rzoYcDR`hHDWc>sWJr{} z+@#}+@eWzeg`U}Ka+(<9h&?S=7lVpJve^=gT!&n$AxedOBBD*;T+l6GR1!v5Xg)V{ z1)hTmJcm7DN7N($714ILLLorzO$mE$OE%pHfrhKm#CWOiLKjk(yE^&?$XK%_r+7ox zxF0KFoGAcS=|h2mkH}L0F7m2M2bN@pd>lH=B_VyftYvQ}LhW^U@wVh@{-y&y>4EIs zn!n23RSj9-nzI|qG6km)*FPuG6@-LpCZwO>Bg6wD4y3qL8DU<(-Qj`o@^*}Pnd`)1 zaZvmsr#|&HvvSlboi_-grre_E{VWV<8`$9%!1&2vFCLbNUcF;!%!7O&l>l3sPYqxUP2*j)3~4*Ea-#MrA-8)Ttln=U!`%9jU$^(`HM6WA90(3^L())bU%D@J2?M2Knur9WtjG1*8y1jXpTVm%GHI$B~sP8?TBGLHjD8o#> z({dXBdZPeL{jnV5!)Ecs=^aq?6FYfn6NhHpgAKo12RZolo?@S5LKWf$h;*t4ym@N@ z{(RO%qxK(Tw-CO9ftkVtMVC&hC#0AgAwIy?1mfp8TzK=}j{1gDx;?cwa~UX-<&Jqpr#hAK8Ms)H$sQ}&y&qQHY23%2O=vFMx;LgXe&n8pD%(A zXxYAxVFzJS-095?4Cq|y6mS~BwjrZ9rxq=Z{$;A>1}_=SHcGqted(npPBOREBUXg; zz-Ei?U;jk^5E-EV9vKEDhKLLgy-n|3Z(KbAe_m3fd+eIgOrACXq8-Z-}9^cEi z_#g{*P_wK#d8w=Dd!Zi2*~K|Dvz`2;JTkvr*s)F}4-XQb2Lf;lfqt6NL(&42>1Nz! z{iA=QV@V8(y+=jsbK{EL?avMf_#?hm5tsb@kr2CXeESQN7{OEdl}~NP8r+nwj#6+XPvbv77oiRKxdhv5jUNs5CpF@3R>MehwIGUTkMD|F11uAg zF=mQu=stGpYmi6j_Nz*<`L3P%GFgA3iK1?H)vS8Q^lmuq1HB9ydMp7*aV+OWC*H$a zMF@dg?nRlch${i3&dCA&c}Mb&YsH;aQTO8W^8j%1coawShG+WpOc?$cZytlo{6SKKX9b@R_>4k65jEP&% za=QtNoFMw3V^D<|#lqy2myM(7f`}kUreMAQ_E>QGq-N@a;L}Ho`w6Z9Bi{C z$XfF?)as}Si+xkQH=?cVDI&Ix`a0lW6s4C3vY|gJB+R5#`xm!6EGsQ%&@su1!Ay)U zHu~+j*n~#EptWN6Qb&JY=LK(~-i!WLasKflOU84XX^bKw`#$3K=Esnh8|i#zmdP_x zX4%l+4=V`diM~K|23khh*HRC`EO`lF?||G7as02DoG*X#RUA!AImN7PxYo|NFZ?*$ zS{;hSB%^I5U>8TVPW>YrT>VKC~A0MbRr2E`_RzNG0%E-%e6{I{CxZ`xFKLTYpFAtC)haC1Y&m~+`JLIn9 z@_iQc+uf+!Op?QYiLM=A=IgL>M5Io^$*Xv)GH0&zXXBRtg)bE-F1X(t=yQ$CIx{HX zp3C94+)3REv673@frtYx)k{2|m1l{mNs2rF5T6{d2-d#Gwc%bn0+H`-u%++mAxPW9 zD%kJC)i*7NyrDMMH6{F~n}Tu)k~9UfsN3UTFb`Yy)LzQ^up)3Vaws3tJ17bVNMeW+ z)5hTFB_sExn-kbTA<)t9&xFV@Rmp4(wdnrS{hO6ezOa7(oz;0A6muqci72SRPKCI# zq;tJkp^;3hXt?8hg>q$y2WH}A(EFG@h6*~on4~+t3MY92h^Aae^FcKIrRaYt5M59J z#&P2z(gz`sb(nF`xk$iPmRJ18k6{Fia%L+XL2?S#tChFfy1H-RuoL|@e_`Lt{SQ-W zqN?whqpXX*YPsSvzw6D>y$Xbx=a%KvUro7Zr7{{snqJXtssy%EmY=iqA`4Oc@W-3a zlUq~I8R;9p8LD(ypXYaw2+8yxE5DXd6fHwK( zVN!>Tx+BHk?S5A?xw^SknRN!+4grEwgv0f~T|=}NE3(I_f34x2my2FSnZD`BpGT39Zzo&rR#XyUei}ELNJM99o zm4#s2?efcWAuhDmrrIwsQ4IJ(*xzK=%n1y5V=;zNWb2rm?CjViHIV`D_~!tpBRnEN z^mtH(XqWzhF+K6JFRxh&Wth~|KPZ7wo~F`vmZo{@Z35NA*M(ZE3UAe`P>HpJlK8#= z3ME}oHW3~3)cN_y<-2w4`&5O9uTfL}sNak2ERyx66EI)2=-0UmOZeVvJzNyBJ>O6v zoLTw^E%2f02U1Qj=yreHGCv%UQJr3?$0r)pc=C%FMO9fzX-;b+BO$@OC9J8va6UJf z&p=c_A!2*i{MMrS;I{Ug|G^v*Gx-qSQ44D31*MG#n@M|1ddDTb)PkkddQs(QQq;~N zcLF=I2ewVDPcUqa@mx<;6N1QJsbKOH^Wr zE({gBGm7hRRVsR>AJ=$!7Tr@&^%9b&G4|Jsu#!LRv)X|a%)J{H^Yzm14&!9V5<7|UOIe|D$kRIuxv(lIi!M~QT75Z5}WYBJSF zE8Emj&xvdw$pEG2k-nyx6uNOduqJ8cFTu5{yHnVjG+(-m; zTx^x$5jb=8_Q2IUFvUW8!zTBqfV#2OT*e5Xv5Fx=>|&MWL1F^A^H^WK-Oh`9(5mYm z^aXjO$Hzl~YZ-D)+XKk}HRabcSzRV@8)k>N?etd#^P2CSeO6J^Te>OnL)pN;Mv6Y$ z?x;)ZY@Uh}BJwj#-1J3Y$2^O)2!0Q{weQW~r>`_AF2s41xs^Lw&d&nKHp<->&IkHg*~hvd z++>k>mc^w0$215;QAw4@1|Hc&12M(3VSy#x24RU;xqN^7a8h9TD5yk@4tg=6g57a& z&X)Y1sNJ3qV>#9=g<(mc6QIg1W)Q-O|d~izYiI+3N-A0x+ zs_J$Qxkq2{!y9kXp@V$nTTtG4Oy^Tv8by{|IoeF3CGJLtcvX8|-xJenkY_1}leo4% zUY4y*>a_eGkOwCv&GCDjsC8LxE6B2f{x@v^5=kgjqUa*7PU+PLJ>*z(x8e&dG%E${ zh{AaXPx2P^o}RF7lg{$J*+7bH2n}%M2>EtnRB6O4`#h3;Bf+cy&X3exb4=wH<+o|6 zV^#AFn9I7OFA78{%O(@60`zbIrQgWu1N8OJ;WT}~U5qSi+{6;su@Qtn-xEpxbG5M~ zAPGIHYJx40d-Y)tt~OopRLJ^h0s(#vwRmt1+uCes{VNCdN49{(Yp!^%dLN7(F6Vn= z!r5w}`)NQzqv^3*(idKIx7Aa*&#$$yqmKc&L#!)@zFwp&gs&~LmMBj0l+3a9z?1Rj zr|eimjoC_c(qBIo-;n@hOx|yO-Y@^I&697dJ(}7zu@SRF^VG%5B@TK-p^#l77D}8E zR3>97Mx~e-NZmHfy{bm}LA$hLfp##Vt~0DbJ}y*pD2y92`#$-+Y4Wfdsa^Q)CCc4UYaGJ6}7A#NTsGsSJfJ=!!j7#6ji~0ySm5ZBvdzl+4d4lmxi$`~I zFZN^AJpvQbeczEq7p&Q}@8r3j^)-lGDx2+wDvRIXvs!E6B!~qsCb3nhO&$IBPLTP13aOq6#Qq zfOLI8{G%Zazn_pF77$_$DZ9+~D0*A{k{)if7V7(Xu`;2|-4kwMDnEV6X`Qgj3uuxe zBcI9-B+CDFBU0f0SR~YyDhuQ0c1777;=oI#gJds0lgEYX4?(b4i>2}x)9qFuTCx-A z4omB|Xl#7_eZ$Csa@ph0;DQZ_pC7JPWBAeYAVcfU%BC0RAllYDr`n1wnf5~mS(#UG zROspdTzo7wNEbvaI$d{`QMz5@aI@cJaxlPxOvsGX>W3UdAv|-$o*YXv$;`9$9H!+6 zEg7&lZd^afIzPMv=1*7&aqf`EH~n6{2Q2O~1$y$P%dzUrf###UkWTRV_R?a{&b3?R zf4&j;sK|$pVlbv`XZap}6ep=f*`_b$cWFRSOf4lBgJslM_*H?!m}&h&9EGSAvwa%r z7F%;|q|g1Ht#u6`J_6ldt`Al4#mvG1=jf4echgBK`qR(R!2u-nFc&vYx$vVG9JFMoH)+F0RbEph6+ z+o;SEiB_1pLLms|bNt@Ebm9DQEH$7!sD1Qx;f868{nP6z-fYS%ao-u`&T*In>mXZK zeV27lzwBpLZpzgAcS5@}s)AD+E@`BozJL8P=V8ps-w7d^KxatSyFZ#mUDikgcB(87 z|9a)LE*BBSrGaWM7WfTOnxXdIX5x^Bcao$s-5aN*$JEABI6FreyZvrgbYY@FH1K%y z^A@dBN5fevBQGyO0jj9Ws~i&03CoPs329G`=D!-wLJ6wVf$IoV^dzg4*p7Qe(&O)= zxY67*+o#mg<>Ju9vcK&veY*A8~!w%|I^k0CnV+>Sn9wMgzzi ztK9u6M;si-WAedp%JoL1I|{*`%_#L3-N`84?5zqMoP1lpD?yU$f<@;*Q#;2OQOd## z=i!(CS4(|<0rVOA)qws*)N$17xcTX_j&3}3U)YknmQFbCKsnY+c7#B=bQK@^>`7Df ziC4++CH?J>WJfMN9M3$YDWFo|)Xc!&i;`7_*u5NDXyZMkdgA@2%F|A`?Pg8JC}3pK z8&J%429*Aliq`aP&g4%>15u-d|5dga9uM72$IhJQz>4ojg{sJ0?NqZZ!j(lpA-aC^ zbww-48c@;GYa*FtT2q%<+c*|%GM&47CR6Vv42-RYMo-ON+B)HUICAp*otYI}e!}7-B1fw>ULha$GllZ7g^F|DvP_@PT!sPr`j(OeYnE&wJ-c1SQ3g zBQJQ-j#pWjE!xi%d)%TXUQ8);Hl+?0N=-PY^CB(6-VCUJ-{mJqnac^@5-gWn{3`KM z=jRLCMYJCS120ON9C7vwv|3SnXy;!GsgN`NEnSPLQDFA2`x?I%rxeXNdN7L9rp)GV zQTs(b!0zwkNKG&zBo|vw^sbPQ*cTOFzR(ubhwJ=EgUlBYDr~5^VYLg~XesKmiA$3B zL0&eRio||j@HQ=U6!q9|P7|ue-ZT|>+&)$u5+xVip{FI;AG^=tZjH~mrryicn*$p7 z7fFH)6s1a_z$^~va5p}&t}teY*Oo00rw1*}g3E_SvyhQRm922{4u=s8XOR%_b7KZ- z2?Ub8pHZlJn7wCShbeSmiTn8L5uyLy-DqY-K&x+>=fr?effoC@^()$R*?Et5{zn+oHsrh|(a(7|Gfc`ufggz5Z% z%v8)z(p45|yhc7G_sjIP>m>K5AjIr*Ik@F@HSaG`RMExX5mzi7jK4aFm-yU=qu*Bq zT97Lif@`*hyPYbUUW~EIV2lFyl!!-bX^t`&Xy1kZudS~kim5M;Dju|XVk#QbQ*wg} zW~(RRLIEyZf$i6HjIw0nlO1ux#>Ky=pyXjPAf3tb3=(xj*x|ly!_^za|9XC55gUDy zdhk2;d!&0N{uCaykwMv>jE(HmX{!(QMyYW%@3TtK-|S3Y0~eym#LvX7sdw=FtE+y# z0Z(h1r`Ew<#oZKCVZn+SC8SMrY1&u58H*pJaK^%>gaafIWZRQWl{o}~y5~EiY~ybi zKOzZuKStD2t~iMhag7qn9cZ+3#e!ZeV9y>MU zen!94@ODaKZESz=f0;w^FO=Z&bUux7BS?{!mtUif?BJuH{8+c3U&E4mBo>A|F9LWJ0)PEwvW z0&gs(tdlp^PI-&y8O51?oLS*+{NJhaeP}i_e*wl9KikqSU^a&Bi+n6jVcN^nq+^-H zKA#ZiJ3XJ3A3bli-6r9Wo5v-;?~X7pPU($bVBDTNHcB4vpr+ zJeO9T!By+bPg|wDpo{aJW#IDAqd@e9Kv4}1d0jAnP1KPWD#^Jx0s9G)NX*kIRqt_-Y;uBWU9xJ<<#vYIrKH7vp9TVVTn ztUnAwrqDd_Pg38BL8w+5`%*SA|K-7fs1KF?_zvj77W$vH`_Rn$&&(1T8dngx`Y$7K*rW{HHJrAJt-#d}q*)e3j*T&g|dFGWD(Kw2>QHuSL zmsTOtEqfg|CYgHl(HZd+*?Bs`L_tMmt)=Zf#((Lz^hcnN9J|2^1j;&=cO|Zp*F6Lf zk?dtHbx=Po?RPc91G{N3akC$v2G{!eN0o3^5xRBa^tl6u1mCJ3mzJxHM<_ksVn5t} zbtBzN33xR~$y(B{Enc&-#r0(}?KyA7=$UOc*ya1IYJCTIwFK1Ey|&BZ&gB11;dCCp zIVBHRf4DC%JM0|G-lM$Fvi=bx%=ZFI`~JCY{?`{vz?B3PKS!+AAFGNg&SAHKAnwdY zHjE>>;!e~*dvmwjV61z7J&sP1n~Mh9oe{B0J(tE4Fq<0E46mP^yOSy2*9D@^zc=(c z7~do~AaI!f>tlopJ@osLUjczSFiYPVZ+YdrNz8CZ5S|25Vp0&yn> z>TD=Cu$(OB02AT6d!c1sJjuhO;FdhZ*j0f^bM5@R(wriZjqn)K>T5A_Q1i$`znlur`QXf5pmw%^&H|?$@g~#48&{m7tk!JKvn*44eBMXttMvs`YUxamu(b_ z<*k7_@ZSLr@^Ne~;c%z?n*JG^h z0YHI^y#?F0_OQbo%7O>!@%%Da&j>V3a+HbIY@GU>W|l5z_cx)_%4h?$EF{SIK6$B) zJH4z)_1@vqq3d?(ig!$KEPsoL*kUDAowrbtLPF@u+dtsMI-ivIjaj&-e*IgJt7Tex ztm@iCk)dIY_jr=7qE)@3IuWIczMV z<8Q<1|22f)nHt4{K+wI_ActSnFPnF+<&3v#*|~uu+u7oc{`uNa;)k_&dszEe$Kf^) z6zrz_CtIS+et1H+_B#pmT8{cb+xCkrx=+ul@60lx3azq*o=GR5Jj%2Hk{gmYkzQ zAM3f`@eDQNvw^BaHm#1%b`CKHg~wV6=QRw&{0DSDK_QK!Lc>LrdDzMpb+93f(7CjuPKkdXFl4>pRT z{4b>oVPWj8vnDIjC29fA_ZIhky<3?P(`EC{X{5;6wEz2TVF14I$NKkNq@?(L>FbVS z&3Ip3v-Q3XL4(dB<>n)kHFYmSAH6D_*lXl?QAX!Xf-s#ony&`$EKhfq);kSTK86|g zjdot`d9K!dlzbzrTkk?K3a}U{-N5~gY*@~B#1_#ot%)DzCf~+g|8~)VKPHd{uo~86 zb68FG1g3G~BxJ)r2zZBJnQ`-#;J590b(qewL;mvu=Hzh9qD}buN*mM>M-eub$AMR*=KQIkFjyj-eRPPT1HDe%9BUNCI!oc zi_l2ctRm40(}FY5!IqMPVr|vxt#QcwehS*{8-7PuX}cMbzYx#T_HIWr^BrHY9q#`o zUmvB4o)W3+f^OP)?}Tl+%ySB^9Ndd>XN5Pc=OY|6S+dr~kp4?yrCt)`PIdDN&I5ZM zm%DID^dYCqiP`(DMpTi7)p72=GG`Z*oZR8Hzjad6rP=C7p;wWcsoWPXX9U^VxW$1h z21FlKNWZ&F+sO!%ZR?G2@f%-SCJ9&!81dmJ%yS;AJWMnRI>qAdgTO(K?J|92$vpYD zED1=UIEN*lX|gzV9zAh&tvMNLA7z6%Gf`xb;&OmLC=hJ!Z((DeD%-bBWU*Qb&$R zs^4or^-T8Xt~EEFXnaB>)u`Ho{MqY)8wHNR@$bU9rxXZ80iiVL#W{Yh=3I-`mVDr zMo@IT2#SuH*NW20h-jU`DgS`;)O81>xk*`l`svBSX6;0r&*TLY-;HL$AYt`Z%j7a9 z$6s$F+8*83|MZbgC-yp5v1t0Q64@f+B5i~45hlUL6?y)S&V7VhiZ70BB3Y_|}5hMq{)ALBo~qQ6!RvH(l=X7|}x#%_Au8!-**UPV2F ze%F*f9H=J-^vLuS-w@Pqg+dx69+T}?Iz|k(W+(9yl|4ys+U>~W$hbRjZgEh8YEr5~ zl-D|V#6G4`<@b|7cA?Scouvl=6}4q{=q_HoL{w^XDpiUkRz71lkM1efNQRqrvgOAH zALrw@WjnZ-2!z51mj-zb=w3s!;j~m1rF4xHcT1{y?D=mE><>)kST*li(h8#X`3*(Y zTcKuRaUC4HJlR@!qSUcU5!~ktqtU-g?jQ*S-!;lDNQlx9VSwK#sl_f2ahZx-KV>6n zCJ3E$G~CpKr9?C>?kk{};GM>tSA?R^9*C?*p?~qe3UKgupk0Vn<uM(SF(9~XnkUXY4)YM%Z&xcsdprDsiL&($$q1!X-&)Yv z+Ro>z8Ec#k#ziuoQLgW+Xk=P z2cdFDZgW8*-8Eo%ce3z-5tkZmvMlkSZ1VtVXO5JwUC%hvx{6Kb7afY^=fnSy@qrRL zuyy`Q$sHiA*vo8b%sCIFff$44r5@e*P$evPXhHfE*DLMqxO}8@&53f-Ig#Qbb5ynb zIZK$9Ojw#Pm2(}6&|hd>whUBkD)ufPk=;3y57mZ-wDWmW%Awl<T9{DfaoA4TU$}y^3H^r?&gK_~GjbVBi2cO_Qb)(^#4FTm5SE}5-!9JT- zXdHqJifd@MNFH`74Y;ng^Y+qppVpa@B1HRp67|iA9F%aK$>TD+L6(Mo5t*YggF%N$ zX#4Bf&`Jr?7GE)C)l|_)#9>S$!{4v0bS@g8fi#I&wlXTSrIo|2UOzSuZU?6gSKL}9 z>TqXT?JYXL9QWf31;??grE2W?D?JBSTUX9CTGfZ+9_wxC|J3#Tigs~>5zPs%Jzcu| zOO5kt{hfpA7i(SZ9@8F<7#HPhb2=S*YO$1LeQcx@D$`-lBtG7Md?9MIt8Y3w{0Pg9 z$B$0CF2zvKO_Zvd+fF~Yl`w1L|B}NOkF6xW|7S*L6o?HNuf6=C;M#m+GO`w!VOjv| zhFhAN330}B-vGutAJ}{W3E%JWo|8Mr6pzjXzHdN7`7JOZ_p2aLJHRU#$)!(IJ_o8c zvGl@;TWqv}JytnMke8Q&bg++aZv-yq=YqFIs-Ga?Tj^o@|YP=Q?ZeO^S*U$hEQjwsqb5H$|-(y z8K-=1==_Qw^P2Se<7+d zVvU$iM!e8#T_=u-jn7o-%mZfoafFsg<742?=mumtEqmWe`X%sB7QN4>n}$i#M`cfh zki>7?yj`L}$-9(Gc!OBV@76uNp<*vIk|^9LiI<8C85yG(cn$y%Ge#6IRhQPQihbDTE zri7>%Xc>;`UaYxl6_Eg{Y{MQ{2_3~RJv|A+yYF_sU(;)tGRcX5EVb-ArNVyHwfRaQ zcT5ci3+#oE;ED2g3dEP%B`*`X+1wU}yhTeo@A*mc6d4r%dtIGON%)hodd~}yV z)OnchC`FEq$;7|pyH;O%Q{zc;MdA?=u)+n*UWkZ@h{kr&Ei8emF=H`iosvzAV#MSL zJCG-Tk(wFEB zaB5yG@71;25LdW2bcPu@EQvp_fzfIc+j~-TS;L@?!Lp{ji6m9p z!W)-1Unb8F4Gv|CeDUT1sn1@ut;tACgY4-fCx(F2r%nO{0L)(8hNf?e4HiMCqu-g2 z@HnvmUE9!CUU=_cUwOBH4^k#@-T_vG#U(HxVbL=lSZIsc5MYVdzWFXvk=@b9h4#`= zb%NDp&nJo50UOOsQ%InX1XANFkHcLD#bAXY8fU>1=(h z5BxcC74KoTAP9qkEVNn;ABpm03$pX#Ho9}zc8khXI0rCNuE(9@IE%;+zzi+Sg`T(4 z!ma0N&wdZUv$Ky^2(4iKddN?v4Uib3;A_MdiQ%3vg(NQn%->jVqypc3nUL)pm~^q5 zgN?}uqCSXRj%A$sE{on*YXNF{ZQ5@m9Yaif#`5OHM!AWHb6&4oFchZg6m%vf^j83d zrwDC6mRKrek0zO~nySamR0qgJP%#APY5a+$kL!N@#M~A15kb0&ADabtH5YNj@xl4OThNWt^a|QX? zjT}<7a!&Nc7Flq3j^xXcuaA#QV!wak!0kKVTrxscbQm{c366G{4;CQ~#1@a_rr*9( zk`eV00YWeYTb=`Men*6hlPb03r+$hQOuUlp`mlW~xBEPIOMMm08=Z-Zd;LaZOJCV< z)Ai41OM)iw@QpjWZ3o${j|BOFaUjr^_S3*>spcLGa6#SuYmvwSf@+b$t^vN0ncYj2 zo6qBM8TVUIFPIa#xA*5~d)(6BDURc<3GTRTX}z65l%wxt@8$(w@xJ`0?+4t3Vg>Fr zH$B)CNLyyxbP0TaLNj%bvcDioKh%T60>4?o@^H&sRBy*w8w!QKb`0&xmA2m06kz?m zGe-@qf~u!Al%tCXc&Tk2!rpJ~9r%^sl&RyJyZa}GRv;qf20@E*7^tMRnYVT7gi#e@ zC72dCr{c|8)6WSNRWh(I`{*ob4n2g&m6lTMOl))FanmE+JCEAuXImSby3vmbw<@)% zQ7tvB5S5eHx&jeG1H{RREbOk->=Tl0snN9Gcu}=_<8jF^;Z*(KTBzs^4DW_5UKX#g zN|-H_hBS{pnHE{t?+d^Os|!@O>Js7}*{m@%|IVu(?@W&~ph7d-1rqsfTphXu?bY;~ z!}%OSt1VB2%?`XByZRDmtvtw*B`#bPYpaR5xa(|n(GUYM@0B;jRs-wl=-q*!#o<(z zKi_ab-H-xRVCjhC1nGJAZXQcA>$R!S1#68{B5bH1KXg+|d)}&fPJY%KFE&f1wp;Q< zW?NGGV=#^svT57<5D{C*N!QelSu9k*3i){cJi6st+fgy}erCEO4Yy*LXfK;Dbz%(a zN|-;{*#QT8(i+hiwLSrp9RZ>>c<-E`|eY$@ZS;z7?-%56zh6q1D5DSrWA-NMNn{-|N z>;m+#Sh6D-Scj)RnNkXmWzTCTk@<0do8aWM2i`LF#0DhBr^*Bpjk3M?T0cUI)H>3F zw~7+eT5_w?XL`q7ln=YRWrLMQX88Wz4aXm5H*nKo;!o4z%uAS*dx%dUi7zWjRXOE; zwhgM8^vmGn7J*q>aH8VV5uUB`mE*xxNe0(|mWzNy0mr#+T;^$;ADMsefD5i@@NiYB zMju8*#R72A`P{KC5K+*B8j*d=n>6XsP#lSp>f!v)# zc?Q#=#%E0(SH%o5`PoV{=Am~b%7YnP=wI+>wg~z z8?Tyuq|o`FcgZL6`oCOIftfyo6e%ol438K7d(1+Yj?*&6zZpI!+tys;xWHZYoB#BC zh~O_LH<=EBYDjYO?UUAobgtlE{~k1FU<9WEAv3MD^`+k^vf~+Q1Z1fHG2UXP;{R~d zdN@1kzfaUup3tTkK9>;_zc z;i3hQ-Bo26TeIeU8X`;Run`nQ!My;zCv-=p_iG`Eqa~WWld$082}zdmf7969Y@a06LO_0njSnQ9-O*L zkSGX^1^$h!e%nnY9NEQpba2b+2z-Mqqz|H|U|{t!S=ejHPdt4@0pJU9LL=h1LX+Az z>B>q4YEjO4QQskYJA`prKC-wQ8@CqtkzKv8Sn8d4Bugrkm4zi*dD_6W@ zYJ8TLXx|SHw0paw5csJjE^^+L^;D+v^5HqaTV7b4MjTDC@!0ZPkeInLrjb`q);{5p zm4yioUMp`JWJ;)S$UuMpi?}#K z{EN-<>^`yyyGBCaBjj2MQEv85(qm*4i5v^zVTThBGTB zPi^5{ltA;Rk7@QQGZC~r83AK6N6!Necnm9@okFJ5@n~MuMG7orc@HgE6{8cvMOpjn zbCK+tW|MVUP0A@J=cSN&cWBDGLm#ZGwP9TO`beDL5aQ7Wtdj4_n2cywS3`xa^m4l9 zh}C9IdHE#dO|@KAUaSNmv!ijr@OX`99_^E{Fzprq&dpdmbVRwVapebuhK63Me|or| z%>(Bb^7^FLI3}l&7h2B5q70bM*oo@LyzcJq8Q9@$LNSxaABL32AUcWr+?iHwUgyYr zjJI>k&vA;2j{wQz0gj2U9b=~BN6Rk88`PR_M7hdnE@kxip~VX|{RgKiP~klvnIHHhvsm4FJ^`d`C%QeUwAbo?%~aBWQ~<#(YNO(bP%l9 zhq|>f?gGd1XVV}-G`~|YE}ByhG~aAh!vMd!EGvC@bMY1pMW%A&R8gXUNhAQxj?~T6 zgYL99uuFSQs1xU+7Jx~6y-(ACU!b!-t!d?%*f)B$>f!S;A47qJ5QH(7Z9^yY|)dL=HNAcK_a~T z!7R0P`in0Ng1_Lz=pDy^ynUIC{tr>|($eFrE*ne0m*hRv6stM*R`T(I_>?z_u#ZWL zjobOs3o}^0i!Zh>d)!m_^X-0nE(}qH?YHK9YIZ4xC?-KMw@LjLus+!6D>+jtYmzxG zFOFw(%qVZqL&yMjYA_=hzOqUnbs@rO;gdRsVM$?jEXr|xqS_kjM^)PV`~_DA!m@yT z@PiBuw#M1vekxsmEL(1U6L71J?nx2fC>PXM3Ws~m-%oK^)AUpfKrQe2%{}9F1-c-X zz?sQf+z65@9cEV^uW;v1q)*t&(5J(}QkxX-QJX5cB599BGm#9G%Cnn>tI%jhmRr22^rCIJvY`4(Ki)>rxcD& zdk5|tlQJ-A2PR*<#GOl_H1M9(a|kPLJlVU?R;h?G6eyeIR9>W`c|L8*RD`YEfjJFG z1J&$-Ze=K$OT(V^HrvS6*_lJE6rsNT#XvJZyZeO>h1KodyNm~ZBeb%PKE-ZMyt$gP z&5|l0{j)&9W-Q?$k897MyyLqZ!SLB*9ackNe1o^TqhmBg5;Rm~wwA zp~hT|2WmZ6PP5;;)e$#X>{2cK?11bx5$S%+%7%$_=$f>%?JaAOAVI@52afyiC<)Sy-Q}d?Mh;I`w~fbf_7h&jl*!SM z_RxC-5P~=SqRU$YQ)y#3?ciRnd}EhB?w;tS+TFyAvBW>F+4rrlTv9T)rjomFA__F`(n|34XaDR3ZZFX zrzu&Jt}4dl^_1UvAsd*Wx_H{b!NeXpT~=O+Z!b{0TjS21I%4yfIa zqfZapb9i#)2_ePIfEi;XtsgYt@P^zW`o2F%dsY4Frv1mgKF{lK-(cQIxzP7Tter*9I9dM^J|kXk9{hjf3@{Q#7` z-_od+;~sEYNy*GaU(_D=X#jSR*-wh8$d7lvIXK@~FRLvPKN_R>j^atR%=s}rroJ&9 zu-66M95!-EPLy>*6^x^w6hOGwCIdiyAlWM|05?AE+QnbFnk*8%gGLrtkbBps=Xob; z88OTGSJC?HvppzU2eS4VvLTqtLfJZuX`z>XNe=30Sv{R!Ff$INjOdIwPr8W|Zt_@0 zRw_Gj=IRBENvg;a))a~pdvX;)5mR*+bK%87LE=#@nTIRvmI0emr;}pov!^D(>FSO6 z&9&96igJiKhS^q+cI|NGdtUam<640-ZgJxwba#-?tiy9qRN6VI_e{PZ$ui}GlN3 zsZ2YY`em5YCp&v>&5`1b4WkjKH{ZFU*0A>&7%K=E+9T* zfSWrSTmJ^@{1xWp@_@Q1J7f00Vh9j55l~p7L>&GJiFMdPV78ni^sk1^NuZp|%$E4@ z@9>W^a2do#57a{K{%HuJzyH)TqGGA58vkktVlqPJ_I9h`zY{XY!Z!@?p}Xe)8WK3V zlQ#;936%aFTq+O^(Sl@NEcw7c4MA+FLl9Qtmi!wY@f-IF*aT*P3IDH#AmRh6k1<{O j{~Ea`= Date: Mon, 31 Oct 2016 01:40:02 -0700 Subject: [PATCH 196/324] Add default cuda system path (#192) * DYLD_LIBRARY_PATH is disable after Mac OS X 10.11 * fix clang + gpu compile error on Mac OS * fix some words and errors in build docs --- cmake/flags.cmake | 51 ++++++++++++++++---- doc/build/build_from_source.md | 16 +++---- paddle/cuda/src/hl_dso_loader.cc | 81 +++++++++++++++++++++++--------- 3 files changed, 108 insertions(+), 40 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index dbad6be3f41b3..e087770991aef 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -21,12 +21,6 @@ function(safe_set_flag is_c src_list flag_name) endif() if(${safe_name}) set(${src_list} "${${src_list}} ${flag_name}" PARENT_SCOPE) - if(is_c) - set(CUDA_NVCC_FLAGS - --compiler-options;${flag_name} - ${CUDA_NVCC_FLAGS} - PARENT_SCOPE) - endif() endif() endfunction() @@ -40,6 +34,20 @@ macro(safe_set_cxxflag src_list flag_name) safe_set_flag(OFF ${src_list} ${flag_name}) endmacro() +# helper macro to set nvcc flag +macro(safe_set_nvflag flag_name) + string(REPLACE "-" "_" safe_name ${flag_name}) + string(REPLACE "=" "_" safe_name ${safe_name}) + CHECK_C_COMPILER_FLAG(${flag_name} C_COMPILER_SUPPORT_FLAG_${safe_name}) + set(safe_name C_COMPILER_SUPPORT_FLAG_${safe_name}) + if(${safe_name}) + set(CUDA_NVCC_FLAGS + --compiler-options;${flag_name} + ${CUDA_NVCC_FLAGS}) + endif() +endmacro() + + CHECK_CXX_SYMBOL_EXISTS(UINT64_MAX "stdint.h" UINT64_MAX_EXISTS) if(NOT UINT64_MAX_EXISTS) set(CMAKE_REQUIRED_DEFINITIONS -D__STDC_LIMIT_MACROS) @@ -63,20 +71,43 @@ set(COMMON_FLAGS -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter + -Wno-unused-function + -Wno-error=literal-suffix + -Wno-error=unused-local-typedefs) + +set(GPU_COMMON_FLAGS + -fPIC + -fno-omit-frame-pointer + -Wnon-virtual-dtor + -Wdelete-non-virtual-dtor + -Wno-unused-parameter + -Wno-unused-function -Wno-error=literal-suffix -Wno-error=unused-local-typedefs -Wno-error=unused-function # Warnings in Numpy Header. ) +if (APPLE) + # On Mac OS X build fat binaries with x86_64 architectures by default. + set (CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Build architectures for OSX" FORCE) +else() + set(GPU_COMMON_FLAGS + -Wall + -Wextra + -Werror + ${GPU_COMMON_FLAGS}) +endif() + + foreach(flag ${COMMON_FLAGS}) safe_set_cflag(CMAKE_C_FLAGS ${flag}) safe_set_cxxflag(CMAKE_CXX_FLAGS ${flag}) endforeach() -# On Mac OS X build fat binaries with x86_64 architectures by default. -if (APPLE) - set (CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Build architectures for OSX" FORCE) -endif () +foreach(flag ${GPU_COMMON_FLAGS}) + safe_set_nvflag(${flag}) +endforeach() + # Release/Debug flags set by cmake. Such as -O3 -g -DNDEBUG etc. # So, don't set these flags here. diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index f7db0a9b92e67..7727c8c3788b9 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -153,12 +153,12 @@ As a simple example, consider the following: - **Only CPU** ```bash - cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF + cmake .. -DWITH_GPU=OFF ``` - **GPU** ```bash - cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF + cmake .. -DWITH_GPU=ON ``` - **GPU with doc and swig** @@ -171,7 +171,7 @@ Finally, you can build PaddlePaddle: ```bash # you can add build option here, such as: -cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX= +cmake .. -DWITH_GPU=ON -DCMAKE_INSTALL_PREFIX= # please use sudo make install, if you want to install PaddlePaddle into the system make -j `nproc` && make install # set PaddlePaddle installation path in ~/.bashrc @@ -246,7 +246,7 @@ easy_install pip ```bash sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C /usr/local - sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib64/libcudnn* + sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib/libcudnn* ``` 2. Then you need to set DYLD\_LIBRARY\_PATH, PATH environment variables in ~/.bashrc. @@ -273,12 +273,12 @@ As a simple example, consider the following: - **Only CPU** ```bash - cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF + cmake .. -DWITH_GPU=OFF ``` - **GPU** ```bash - cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF + cmake .. -DWITH_GPU=ON ``` - **GPU with doc and swig** @@ -291,9 +291,9 @@ Finally, you can build PaddlePaddle: ```bash # you can add build option here, such as: -cmake .. -DWITH_GPU=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX= +cmake .. -DWITH_GPU=ON -DCMAKE_INSTALL_PREFIX= # please use sudo make install, if you want to install PaddlePaddle into the system -make -j `nproc` && make install +make -j `sysctl -n hw.ncpu` && make install # set PaddlePaddle installation path in ~/.bashrc export PATH=/bin:$PATH ``` diff --git a/paddle/cuda/src/hl_dso_loader.cc b/paddle/cuda/src/hl_dso_loader.cc index eee9984e07326..91c60d85a1e41 100644 --- a/paddle/cuda/src/hl_dso_loader.cc +++ b/paddle/cuda/src/hl_dso_loader.cc @@ -46,63 +46,100 @@ static inline std::string join(const std::string& part1, const std::string& part return ret; } -static inline void GetDsoHandleWithSearchPath( +static inline void GetDsoHandleFromDefaultPath( + std::string& dso_path, void** dso_handle, int dynload_flags) { + LOG(INFO) << "Try to find cuda library: " << dso_path + << "from default system path."; + // default search from LD_LIBRARY_PATH/DYLD_LIBRARY_PATH + *dso_handle = dlopen(dso_path.c_str(), dynload_flags); + + // DYLD_LIBRARY_PATH is disabled after Mac OS 10.11 to + // bring System Integrity Projection (SIP), if dso_handle + // is null, search from default package path in Mac OS. + #if defined(__APPLE__) or defined(__OSX__) + if (nullptr == *dso_handle) { + dso_path = join("/usr/local/cuda/lib/", dso_path); + *dso_handle = dlopen(dso_path.c_str(), dynload_flags); + if (nullptr == *dso_handle) { + if (dso_path == "libcudnn.dylib") { + LOG(FATAL) << "Note: [Recommend] copy cudnn into /usr/local/cuda/ \n" + << "For instance, sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C " + << "/usr/local \n sudo chmod a+r /usr/local/cuda/include/cudnn.h " + << "/usr/local/cuda/lib/libcudnn*"; + } + } + } + #endif +} + +static inline void GetDsoHandleFromSearchPath( const std::string& search_root, - const std::string& dso_path, + const std::string& dso_name, void** dso_handle) { int dynload_flags = RTLD_LAZY | RTLD_LOCAL; *dso_handle = nullptr; - std::string dlPath = dso_path; + std::string dlPath = dso_name; if (search_root.empty()) { - // default search xxx.so from LD_LIBRARY_PATH - *dso_handle = dlopen(dlPath.c_str(), dynload_flags); + GetDsoHandleFromDefaultPath(dlPath, dso_handle, dynload_flags); } else { // search xxx.so from custom path - dlPath = join(search_root, dso_path); + dlPath = join(search_root, dso_name); *dso_handle = dlopen(dlPath.c_str(), dynload_flags); - // then, search xxx.so from LD_LIBRARY_PATH - if (nullptr == *dso_handle) { - *dso_handle = dlopen(dso_path.c_str(), dynload_flags); + // if not found, search from default path + if (nullptr == dso_handle) { + LOG(WARNING) << "Failed to find cuda library: " << dlPath; + dlPath = dso_name; + GetDsoHandleFromDefaultPath(dlPath, dso_handle, dynload_flags); } } CHECK(nullptr != *dso_handle) - << "For Gpu version of PaddlePaddle, it couldn't find CUDA library: " - << dlPath.c_str() << ". Please make sure you already specify its path. " - << "Note: for training data on Cpu using Gpu version of PaddlePaddle, " - << "you must specify libcudart via export LD_LIBRARY_PATH for Linux or " - << "export DYLD_LIBRARY_PATH for MAC OS."; + << "Failed to find cuda library: " << dlPath << std::endl + << "Please specify its path correctly using one of the following ideas: \n" + + << "Idea 1. set cuda and cudnn lib path at runtime. " + << "http://www.paddlepaddle.org/doc/ui/cmd_argument/argument_outline.html \n" + << "For instance, issue command: paddle train --use_gpu=1 " + << "--cuda_dir=/usr/local/cudnn/lib --cudnn_dir=/usr/local/cudnn/lib ...\n" + + << "Idea 2. set environment variable LD_LIBRARY_PATH on Linux or " + << "DYLD_LIBRARY_PATH on Mac OS. \n" + << "For instance, issue command: export LD_LIBRARY_PATH=... \n" + + << "Note: After Mac OS 10.11, using the DYLD_LIBRARY_PATH is impossible " + << "unless System Integrity Protection (SIP) is disabled. However, @Idea 1" + << "always work well."; } void GetCublasDsoHandle(void** dso_handle) { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcublas.dylib", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.dylib", dso_handle); #else - GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcublas.so", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcublas.so", dso_handle); #endif } void GetCudnnDsoHandle(void** dso_handle) { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleWithSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.dylib", dso_handle); #else - GetDsoHandleWithSearchPath(FLAGS_cudnn_dir, "libcudnn.so", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cudnn_dir, "libcudnn.so", dso_handle); #endif } void GetCudartDsoHandle(void** dso_handle) { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleWithSearchPath("", "libcudart.dylib", dso_handle); + GetDsoHandleFromSearchPath("", "libcudart.dylib", dso_handle); #else - GetDsoHandleWithSearchPath("", "libcudart.so", dso_handle); + GetDsoHandleFromSearchPath("", "libcudart.so", dso_handle); #endif } void GetCurandDsoHandle(void** dso_handle) { #if defined(__APPLE__) || defined(__OSX__) - GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcurand.dylib", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.dylib", dso_handle); #else - GetDsoHandleWithSearchPath(FLAGS_cuda_dir, "libcurand.so", dso_handle); + GetDsoHandleFromSearchPath(FLAGS_cuda_dir, "libcurand.so", dso_handle); #endif } From 12945b2c9017c88d2581253a6df83d9e35e2b804 Mon Sep 17 00:00:00 2001 From: backyes Date: Mon, 31 Oct 2016 23:42:59 +0800 Subject: [PATCH 197/324] Add glog header path to include (#295) --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index e96ce28248ee5..527064e31000a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,6 +135,7 @@ endif() if(WITH_GLOG) add_definitions(-DPADDLE_USE_GLOG) + include_directories(${LIBGLOG_INCLUDE_DIR}) endif() if(WITH_GFLAGS) From cdac60f616f4950aecd69a5aa326b196aa05e135 Mon Sep 17 00:00:00 2001 From: qijun Date: Tue, 1 Nov 2016 06:40:26 +0000 Subject: [PATCH 198/324] add SpatialPyramidPoolLayer c++ support --- paddle/cuda/include/hl_cnn.h | 14 +- paddle/cuda/include/stub/hl_cnn_stub.h | 10 +- paddle/cuda/src/hl_cuda_cnn.cu | 40 +-- paddle/gserver/layers/PoolProjection.cpp | 81 +++++ paddle/gserver/layers/PoolProjection.h | 72 +++++ paddle/gserver/layers/Projection.h | 13 +- .../layers/SpatialPyramidPoolLayer.cpp | 128 ++++++++ .../gserver/layers/SpatialPyramidPoolLayer.h | 54 ++++ paddle/gserver/tests/test_LayerGrad.cpp | 32 +- paddle/math/Matrix.cpp | 288 +++++++++--------- proto/ModelConfig.proto.m4 | 12 + 11 files changed, 562 insertions(+), 182 deletions(-) create mode 100644 paddle/gserver/layers/PoolProjection.cpp create mode 100644 paddle/gserver/layers/PoolProjection.h create mode 100644 paddle/gserver/layers/SpatialPyramidPoolLayer.cpp create mode 100644 paddle/gserver/layers/SpatialPyramidPoolLayer.h diff --git a/paddle/cuda/include/hl_cnn.h b/paddle/cuda/include/hl_cnn.h index d19f4a4bb310a..4bd9d5e7c9e90 100644 --- a/paddle/cuda/include/hl_cnn.h +++ b/paddle/cuda/include/hl_cnn.h @@ -91,6 +91,7 @@ extern void hl_expand_feature2col( * @param[in] paddingH padding height. * @param[in] paddingW padding width. * @param[out] tgtData output data. + * @param[in] tgtStride output data stride. * */ extern void hl_maxpool_forward( @@ -100,7 +101,8 @@ extern void hl_maxpool_forward( const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, - const int paddingH, const int paddingW, real* tgtData); + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride); /** * @brief Maximum pool backward. @@ -123,6 +125,7 @@ extern void hl_maxpool_forward( * @param[in] paddingH padding height. * @param[in] paddingW padding width. * @param[out] targetGrad output grad. + * @param[in] outStride output grad data stride. * */ extern void hl_maxpool_backward( @@ -135,7 +138,7 @@ extern void hl_maxpool_backward( const int strideH, const int strideW, const int paddingH, const int paddingW, real scaleA, real scaleB, - real* targetGrad); + real* targetGrad, const int outStride); /** * @brief Averge pool forward. @@ -154,6 +157,7 @@ extern void hl_maxpool_backward( * @param[in] paddingH padding height. * @param[in] paddingW padding width. * @param[out] tgtData output data. + * @param[in] tgtStride output data stride. * */ extern void hl_avgpool_forward( @@ -163,7 +167,8 @@ extern void hl_avgpool_forward( const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, - const int paddingH, const int paddingW, real* tgtData); + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride); /** * @brief Maximum pool backward. @@ -184,6 +189,7 @@ extern void hl_avgpool_forward( * @param[in] scaleA scale. * @param[in] scaleB scale. * @param[out] backGrad output grad. + * @param[in] outStride output grad data stride. * */ extern void hl_avgpool_backward( @@ -195,7 +201,7 @@ extern void hl_avgpool_backward( const int strideH, const int strideW, int paddingH, int paddingW, real scaleA, real scaleB, - real* backGrad); + real* backGrad, const int outStride); /** * @brief Cross-map-respose normalize forward. diff --git a/paddle/cuda/include/stub/hl_cnn_stub.h b/paddle/cuda/include/stub/hl_cnn_stub.h index 5f696986e3c8f..4342c30376eeb 100644 --- a/paddle/cuda/include/stub/hl_cnn_stub.h +++ b/paddle/cuda/include/stub/hl_cnn_stub.h @@ -44,7 +44,8 @@ inline void hl_maxpool_forward( const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, - const int paddingH, const int paddingW, real* tgtData) {} + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride) {} inline void hl_maxpool_backward( const int frameCnt, const real* inputData, @@ -56,7 +57,7 @@ inline void hl_maxpool_backward( const int strideH, const int strideW, const int paddingH, const int paddingW, real scaleA, real scaleB, - real* targetGrad) {} + real* targetGrad, const int outStride) {} inline void hl_avgpool_forward( const int frameCnt, const real* inputData, @@ -65,7 +66,8 @@ inline void hl_avgpool_forward( const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, - const int paddingH, const int paddingW, real* tgtData) {} + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride) {} inline void hl_avgpool_backward( const int frameCnt, const real* outGrad, @@ -76,7 +78,7 @@ inline void hl_avgpool_backward( const int strideH, const int strideW, int paddingH, int paddingW, real scaleA, real scaleB, - real* backGrad) {} + real* backGrad, const int outStride) {} inline void hl_CMRNorm_forward( size_t frameCnt, const real* in, real* scale, real* out, diff --git a/paddle/cuda/src/hl_cuda_cnn.cu b/paddle/cuda/src/hl_cuda_cnn.cu index baa2fb0d27d74..fcef6a4436b5c 100644 --- a/paddle/cuda/src/hl_cuda_cnn.cu +++ b/paddle/cuda/src/hl_cuda_cnn.cu @@ -152,7 +152,7 @@ __global__ void KeMaxPoolForward(const int nthreads, const real* inputData, const int ksizeW, const int ksizeH, const int strideH, const int strideW, const int offsetH, const int offsetW, - real* tgtData) { + real* tgtData, const int tgtStride) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int pw = index % pooledW; @@ -173,7 +173,9 @@ __global__ void KeMaxPoolForward(const int nthreads, const real* inputData, maxval = inputData[h * width + w]; } } - tgtData[index] = maxval; + int tgtIndex = index % (pooledW * pooledH * channels) + + frameNum * tgtStride; + tgtData[tgtIndex] = maxval; } } @@ -184,7 +186,7 @@ void hl_maxpool_forward(const int frameCnt, const real* inputData, const int sizeX, const int sizeY, const int strideH, const int strideW, const int paddingH, const int paddingW, - real* tgtData) { + real* tgtData, const int tgtStride) { int num_kernels = pooledH * pooledW * channels * frameCnt; int blocks = (num_kernels + 1024 - 1) / 1024; @@ -194,7 +196,7 @@ void hl_maxpool_forward(const int frameCnt, const real* inputData, KeMaxPoolForward<<< grid, threads, 0, STREAM_DEFAULT >>> (num_kernels, inputData, channels, height, width, pooledH, pooledW, sizeX, sizeY, strideH, strideW, - paddingH, paddingW, tgtData); + paddingH, paddingW, tgtData, tgtStride); CHECK_SYNC("hl_maxpool_forward failed"); } @@ -207,7 +209,7 @@ __global__ void KeMaxPoolBackward(const int nthreads, const real* inputData, const int strideH, const int strideW, const int padH, const int padW, real scaleA, real scaleB, - real* targetGrad) { + real* targetGrad, const int outStride) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { // find out the local index @@ -223,8 +225,8 @@ __global__ void KeMaxPoolBackward(const int nthreads, const real* inputData, int pwend = offsetW >= 0 ? min(offsetW / strideW + 1, pooledW) : 0; real gradient = 0; real input = inputData[index]; - outData += (frameNum * channels + offsetC) * pooledH * pooledW; - outGrad += (frameNum * channels + offsetC) * pooledH * pooledW; + outData += (frameNum * outStride + offsetC * pooledH * pooledW); + outGrad += (frameNum * outStride + offsetC * pooledH * pooledW); for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { if (input == outData[ph * pooledW + pw]) { @@ -246,7 +248,7 @@ void hl_maxpool_backward(const int frameCnt, const real* inputData, const int strideH, const int strideW, const int paddingH, const int paddingW, real scaleA, real scaleB, - real* targetGrad) { + real* targetGrad, const int outStride) { int num_kernels = height * width * channels * frameCnt; int blocks = (num_kernels + 1024 - 1) / 1024; @@ -257,7 +259,7 @@ void hl_maxpool_backward(const int frameCnt, const real* inputData, strideH, strideW, paddingH, paddingW, scaleA, scaleB, - targetGrad); + targetGrad, outStride); CHECK_SYNC("hl_maxpool_backward"); } @@ -268,7 +270,7 @@ __global__ void KeAvgPoolForward(const int nthreads, const real* inputData, const int sizeX, const int sizeY, const int strideH, const int strideW, const int padH, const int padW, - real* tgtData) { + real* tgtData, const int tgtStride) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int pw = index % pooledW; @@ -293,7 +295,9 @@ __global__ void KeAvgPoolForward(const int nthreads, const real* inputData, aveval += inputData[h * width + w]; } } - tgtData[index] = aveval / pool_size; + int tgtIndex = index % (pooledW * pooledH * channels) + + frameNum * tgtStride; + tgtData[tgtIndex] = aveval / pool_size; } } @@ -303,14 +307,15 @@ void hl_avgpool_forward(const int frameCnt, const real* inputData, const int pooledH, const int pooledW, const int sizeX, const int sizeY, const int strideH, const int strideW, - const int paddingH, const int paddingW, real* tgtData) { + const int paddingH, const int paddingW, + real* tgtData, const int tgtStride) { int num_kernels = pooledH * pooledW * channels * frameCnt; int blocks = (num_kernels + 1024 - 1) / 1024; KeAvgPoolForward<<< blocks, 1024, 0, STREAM_DEFAULT >>> (num_kernels, inputData, channels, height, width, pooledH, pooledW, sizeX, sizeY, strideH, strideW, - paddingH, paddingW, tgtData); + paddingH, paddingW, tgtData, tgtStride); CHECK_SYNC("hl_avgpool_forward failed"); } @@ -322,7 +327,7 @@ __global__ void KeAvgPoolBackward(const int nthreads, const real* outGrad, const int strideH, const int strideW, const int padH, const int padW, real scaleA, real scaleB, - real* tgtGrad) { + real* tgtGrad, const int outStride) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int offsetW = index % width + padW; @@ -335,7 +340,8 @@ __global__ void KeAvgPoolBackward(const int nthreads, const real* outGrad, int phend = offsetH >= 0 ? min(offsetH / strideH + 1, pooledH) : 0; int pwend = offsetW >= 0 ? min(offsetW / strideW + 1, pooledW) : 0; real gradient = 0; - outGrad += (frameNum * channels + offsetC) * pooledH * pooledW; + outGrad += (frameNum * outStride + offsetC * pooledH * pooledW); + for (int ph = phstart; ph < phend; ++ph) { for (int pw = pwstart; pw < pwend; ++pw) { @@ -360,7 +366,7 @@ void hl_avgpool_backward(const int frameCnt, const real* outGrad, const int strideH, const int strideW, const int paddingH, const int paddingW, real scaleA, real scaleB, - real* backGrad) { + real* backGrad, const int outStride) { int num_kernels = height * width * channels * frameCnt; int blocks = (num_kernels + 1024 - 1) / 1024; @@ -370,7 +376,7 @@ void hl_avgpool_backward(const int frameCnt, const real* outGrad, strideH, strideW, paddingH, paddingW, scaleA, scaleB, - backGrad); + backGrad, outStride); CHECK_SYNC("hl_avgpool_backward failed"); } diff --git a/paddle/gserver/layers/PoolProjection.cpp b/paddle/gserver/layers/PoolProjection.cpp new file mode 100644 index 0000000000000..50059ee04d39b --- /dev/null +++ b/paddle/gserver/layers/PoolProjection.cpp @@ -0,0 +1,81 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "PoolProjection.h" + +namespace paddle { + +REGISTER_PROJECTION_CREATE_FUNC(pool2, &PoolProjection::create); + +PoolProjection* PoolProjection::create(const ProjectionConfig& config, + ParameterPtr parameter, bool useGpu) { + const std::string& pool = config.pool_conf().pool_type(); + if (pool == "max") { + return new MaxPoolProjection(config, parameter, useGpu); + } else if (pool == "avg") { + return new AvgPoolProjection(config, parameter, useGpu); + } else { + LOG(FATAL) << "Unknown pool type: " << pool; + return nullptr; + } +} + +void MaxPoolProjection::forward() { + MatrixPtr inputV = in_->value; + MatrixPtr outV = out_->value; + outV->maxPoolForward(*inputV, imgSizeY_, imgSize_, channels_, + sizeX_, sizeY_, strideY_, stride_, + outputY_, outputX_, confPaddingY_, confPadding_); +} + +void MaxPoolProjection::backward(const UpdateCallback& callback) { + (void)callback; + MatrixPtr outGrad = out_->grad; + MatrixPtr inputV = in_->value; + MatrixPtr outV = out_->value; + MatrixPtr inputGrad = in_->grad; + + if (NULL == inputGrad) { + return; + } + inputGrad->maxPoolBackward(*inputV, imgSizeY_, imgSize_, *outGrad, *outV, + sizeX_, sizeY_, + strideY_, stride_, outputY_, outputX_, 1, 1, + confPaddingY_, confPadding_); +} + +void AvgPoolProjection::forward() { + MatrixPtr inputV = in_->value; + MatrixPtr outV = out_->value; + outV->avgPoolForward(*inputV, imgSizeY_, imgSize_, channels_, + sizeX_, sizeY_, strideY_, stride_, + outputY_, outputX_, confPaddingY_, confPadding_); +} + +void AvgPoolProjection::backward(const UpdateCallback& callback) { + (void)callback; + + MatrixPtr outputGrad = out_->grad; + MatrixPtr inputGrad = in_->grad; + + if (NULL == inputGrad) { + return; + } + + inputGrad->avgPoolBackward(*outputGrad, imgSizeY_, imgSize_, + sizeX_, sizeY_, strideY_, stride_, + outputY_, outputX_, 1, 1, + confPaddingY_, confPadding_); +} +} // namespace paddle diff --git a/paddle/gserver/layers/PoolProjection.h b/paddle/gserver/layers/PoolProjection.h new file mode 100644 index 0000000000000..73d8a41aefabe --- /dev/null +++ b/paddle/gserver/layers/PoolProjection.h @@ -0,0 +1,72 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "Projection.h" + +namespace paddle { + +class PoolProjection : public Projection { +protected: + size_t imgSizeY_, imgSize_; + size_t outputY_, outputX_; + size_t strideY_, stride_; + size_t sizeY_, sizeX_; + int confPaddingY_, confPadding_; + size_t channels_; + std::string poolType_; + +public: + PoolProjection(const ProjectionConfig& config, ParameterPtr parameter, + bool useGpu) + : Projection(config, parameter, useGpu) { + const PoolConfig& conf = config_.pool_conf(); + poolType_ = conf.pool_type(); + channels_ = conf.channels(); + sizeX_ = conf.size_x(); + stride_ = conf.stride(); + outputX_ = conf.output_x(); + imgSize_ = conf.img_size(); + confPadding_ = conf.padding(); + + sizeY_ = conf.has_size_y() ? conf.size_y() : conf.size_x(); + imgSizeY_ = conf.has_img_size_y() ? conf.img_size_y() : conf.img_size(); + strideY_ = conf.has_stride_y() ? conf.stride_y() : conf.stride(); + confPaddingY_ = conf.has_padding_y() ? conf.padding_y() : conf.padding(); + outputY_ = conf.has_output_y() ? conf.output_y() : conf.output_x(); + } + static PoolProjection* create(const ProjectionConfig& config, + ParameterPtr parameter, bool useGpu); + const std::string& getPoolType() const { return poolType_; } +}; + +class MaxPoolProjection : public PoolProjection { +public: + MaxPoolProjection(const ProjectionConfig& config, ParameterPtr parameter, + bool useGpu) + : PoolProjection(config, parameter, useGpu) {} + virtual void forward(); + virtual void backward(const UpdateCallback& callback = nullptr); +}; + +class AvgPoolProjection : public PoolProjection { +public: + AvgPoolProjection(const ProjectionConfig& config, ParameterPtr parameter, + bool useGpu) + : PoolProjection(config, parameter, useGpu) {} + virtual void forward(); + virtual void backward(const UpdateCallback& callback = nullptr); +}; +} // namespace paddle diff --git a/paddle/gserver/layers/Projection.h b/paddle/gserver/layers/Projection.h index 3fa3a0cc230ac..203edc5396a53 100644 --- a/paddle/gserver/layers/Projection.h +++ b/paddle/gserver/layers/Projection.h @@ -12,12 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once -#include "paddle/parameter/Parameter.h" -#include "ModelConfig.pb.h" #include "Layer.h" +#include "ModelConfig.pb.h" +#include "paddle/parameter/Parameter.h" namespace paddle { @@ -28,6 +27,11 @@ namespace paddle { Projection::registrar_.registerClass<__class_name>(#__type_name); \ }) +#define REGISTER_PROJECTION_CREATE_FUNC(__type_name, createFunction) \ + static InitFunction __reg_type_##__type_name([]() { \ + Projection::registrar_.registerClass(#__type_name, createFunction); \ + }) + /** * A projection takes one Argument as input, calculate the result and add it * to output Argument. @@ -50,7 +54,8 @@ class Projection { registrar_; /** - * Forward propagation. If backward() will be called, in and out must be kept valid until then. + * Forward propagation. If backward() will be called, in and out must be kept + * valid until then. * @param in input of projection * @param out output of projection * @param passType PASS_TRAIN of PASS_TEST diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp new file mode 100644 index 0000000000000..bcdba5c151175 --- /dev/null +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp @@ -0,0 +1,128 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "SpatialPyramidPoolLayer.h" + +namespace paddle { + +REGISTER_LAYER(spp, SpatialPyramidPoolLayer); + +ProjectionConfig SpatialPyramidPoolLayer::getConfig(size_t imgSizeW, + size_t imgSizeH, + size_t channels, + size_t pyramidLevel, + std::string& poolType) { + ProjectionConfig config; + config.set_type("pool2"); + PoolConfig* conf = config.mutable_pool_conf(); + conf->set_channels(channels); + conf->set_img_size(imgSizeW); + conf->set_img_size_y(imgSizeH); + conf->set_pool_type(poolType); + + int numBins = std::pow(2, pyramidLevel); + + int sizeH = std::ceil(imgSizeH / static_cast(numBins)); + int remainderH = sizeH * numBins - imgSizeH; + int paddingH = (remainderH + 1) / 2; + int outSizeH = outputSize(imgSizeH, sizeH, paddingH, sizeH); + + int sizeW = std::ceil(imgSizeW / static_cast(numBins)); + int remainderW = sizeW * numBins - imgSizeW; + int paddingW = (remainderW + 1) / 2; + int outSizeW = outputSize(imgSizeW, sizeW, paddingW, sizeW); + + conf->set_stride(sizeW); + conf->set_stride_y(sizeH); + conf->set_size_x(sizeW); + conf->set_size_y(sizeH); + conf->set_padding(paddingW); + conf->set_padding_y(paddingH); + conf->set_output_x(outSizeW); + conf->set_output_y(outSizeH); + config.set_output_size(outSizeH * outSizeW * channels); + return config; +} + +void SpatialPyramidPoolLayer::splitInput(Argument& input, size_t height, + size_t width, bool useGpu) { + input.value = getInput(0).value; + if (passType_ != PASS_TEST && needGradient()) { + Matrix::resizeOrCreate(input.grad, height, width, /* trans */ false, + useGpu); + input.grad->zeroMem(); + } +} + +bool SpatialPyramidPoolLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + Layer::init(layerMap, parameterMap); + CHECK_EQ(config_.inputs_size(), 1); + + const SppConfig& sppConf = config_.inputs(0).spp_conf(); + pyramidHeight_ = sppConf.pyramid_height(); + poolType_ = sppConf.pool_type(); + + channels_ = sppConf.channels(); + imgSizeW_ = sppConf.img_size(); + imgSizeH_ = sppConf.has_img_size_y() ? sppConf.img_size_y() : imgSizeW_; + poolProjections_.reserve(pyramidHeight_); + projCol_.reserve(pyramidHeight_); + projInput_.reserve(pyramidHeight_); + projOutput_.resize(pyramidHeight_); + + size_t startCol = 0; + size_t endCol = 0; + for (size_t i = 0; i < pyramidHeight_; i++) { + poolProjections_.emplace_back(PoolProjection::create( + getConfig(imgSizeW_, imgSizeH_, channels_, i, poolType_), + nullptr, useGpu_)); + endCol += poolProjections_[i]->getOutputSize(); + projCol_.push_back(std::make_pair(startCol, endCol)); + startCol = endCol; + projInput_.emplace_back(Argument()); + } + outputSize_ = endCol; + return true; +} + +void SpatialPyramidPoolLayer::forward(PassType passType) { + Layer::forward(passType); + + int batchSize = getInput(0).getBatchSize(); + resetOutput(batchSize, outputSize_); + for (size_t i = 0; i < pyramidHeight_; i++) { + size_t startCol = projCol_[i].first; + size_t endCol = projCol_[i].second; + projOutput_[i].value = output_.value->subColMatrix(startCol, endCol); + projOutput_[i].grad = output_.grad->subColMatrix(startCol, endCol); + splitInput(projInput_[i], getInput(0).value->getHeight(), + getInput(0).value->getWidth(), useGpu_); + } + for (size_t i = 0; i < pyramidHeight_; i++) { + poolProjections_[i]->forward(&projInput_[i], &projOutput_[i], passType); + } +} + +void SpatialPyramidPoolLayer::backward(const UpdateCallback& callback) { + for (size_t i = 0; i < pyramidHeight_; i++) { + if (poolProjections_[i]) { + poolProjections_[i]->backward(callback); + getInput(0).grad->add(*projInput_[i].grad); + } + } +} + +} // namespace paddle + diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.h b/paddle/gserver/layers/SpatialPyramidPoolLayer.h new file mode 100644 index 0000000000000..de1fd4da07dd8 --- /dev/null +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#pragma once + +#include "Layer.h" +#include "PoolProjection.h" +#include "paddle/utils/Logging.h" + +namespace paddle { + +class SpatialPyramidPoolLayer : public Layer { +protected: + size_t channels_; + size_t imgSizeW_; + size_t imgSizeH_; + size_t pyramidHeight_; + size_t outputSize_; + std::string poolType_; + + std::vector> poolProjections_; + std::vector projInput_; + std::vector projOutput_; + std::vector> projCol_; + +public: + explicit SpatialPyramidPoolLayer(const LayerConfig& config) : Layer(config) {} + ~SpatialPyramidPoolLayer() {} + + virtual bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + ProjectionConfig getConfig(size_t sizeX_, size_t sizeY_, size_t channels, + size_t pyamidLevel_, std::string& poolType_); + + int outputSize(int imageSize, int windowSize, int padding, int stride) { + return (imageSize - windowSize + 2 * padding) / stride + 1; + } + + virtual void forward(PassType passType); + virtual void backward(const UpdateCallback& callback = nullptr); + void splitInput(Argument& input, size_t height, size_t width, bool useGpu); +}; +} // namespace paddle diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index eab9bf84141a2..3d633f4b72797 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -13,14 +13,14 @@ See the License for the specific language governing permissions and limitations under the License. */ #include -#include #include -#include "paddle/gserver/layers/DataLayer.h" +#include #include "ModelConfig.pb.h" +#include "paddle/gserver/layers/DataLayer.h" #include "paddle/trainer/Trainer.h" -#include "TestUtil.h" #include "LayerGradUtil.h" +#include "TestUtil.h" using namespace paddle; // NOLINT using namespace std; // NOLINT @@ -880,6 +880,32 @@ TEST(Layer, PoolLayer) { #endif } +void testSppLayer(const string& poolType, const int pyramidHeight, bool trans, + bool useGpu) { + TestConfig config; + config.layerConfig.set_type("spp"); + config.inputDefs.push_back({INPUT_DATA, "layer_0", 3200, 0}); + LayerInputConfig* input = config.layerConfig.add_inputs(); + SppConfig* sppConfig = input->mutable_spp_conf(); + sppConfig->set_pool_type(poolType); + sppConfig->set_pyramid_height(pyramidHeight); + sppConfig->set_channels(16); + sppConfig->set_img_size(10); + sppConfig->set_img_size_y(20); + testLayerGrad(config, "spp", 100, trans, useGpu); +} + +TEST(Layer, SpatialPyramidPoolLayer) { + for (auto useGpu : {false, true}) { + testSppLayer("avg", 1, false, useGpu); + testSppLayer("avg", 3, false, useGpu); + testSppLayer("avg", 5, false, useGpu); + testSppLayer("max", 1, false, useGpu); + testSppLayer("max", 3, false, useGpu); + testSppLayer("avg", 5, false, useGpu); + } +} + TEST(Layer, rankCostLayer) { TestConfig config; config.layerConfig.set_type("rank-cost"); diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 843eabc97d642..ddf99f6f2974c 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -13,19 +13,19 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "Matrix.h" +#include "MathFunctions.h" #include "SparseMatrix.h" #include "SparseRowMatrix.h" -#include "MathFunctions.h" -#include #include #include +#include -#include "paddle/utils/Logging.h" #include #include "hl_gpu.h" #include "hl_table_apply.h" #include "hl_top_k.h" +#include "paddle/utils/Logging.h" #include "paddle/utils/ThreadLocal.h" @@ -42,9 +42,9 @@ inline real _safelog(real a) { return a > 0.0f ? std::log(a) : -40.0f; } Matrix::Matrix(MemoryHandlePtr memHandle, size_t height, size_t width, bool trans, bool use_gpu) : BaseMatrix( - height, width, - memHandle ? (reinterpret_cast(memHandle->getBuf())) : nullptr, - trans, use_gpu) { + height, width, + memHandle ? (reinterpret_cast(memHandle->getBuf())) : nullptr, + trans, use_gpu) { elementCnt_ = width * height; memoryHandle_ = memHandle; } @@ -95,7 +95,7 @@ MatrixPtr Matrix::create(MemoryHandlePtr memHandle, size_t height, size_t width, if (auto gpuHandle = std::dynamic_pointer_cast(memHandle)) { return std::make_shared(gpuHandle, height, width, trans); } else if (auto cpuHandle = - std::dynamic_pointer_cast(memHandle)) { + std::dynamic_pointer_cast(memHandle)) { return std::make_shared(cpuHandle, height, width, trans); } else { LOG(FATAL) << "Wrong"; @@ -343,19 +343,17 @@ void GpuMatrix::addBias(Matrix& b, real scale) { void GpuMatrix::collectBias(Matrix& a, real scale) { CHECK_EQ(getHeight(), (size_t)1); CHECK_EQ(width_, a.getWidth()); - GpuSparseMatrix* sMatPtr = dynamic_cast(&a); + GpuSparseMatrix* sMatPtr = dynamic_cast(&a); if (!sMatPtr) { sumCols(a, scale); } else { real* data = getData(); hl_sparse_matrix_s A_d = sMatPtr->sMatrix_.get(); - hl_sparse_matrix_column_sum(data, A_d, sMatPtr->getHeight(), - width_, scale); + hl_sparse_matrix_column_sum(data, A_d, sMatPtr->getHeight(), width_, scale); } } -void GpuMatrix::sequenceAvgForward(Matrix& a, - const IVector& startsPos, +void GpuMatrix::sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode) { size_t height = getHeight(); size_t width = getWidth(); @@ -401,8 +399,8 @@ void GpuMatrix::mul(const GpuMatrix& a, const GpuMatrix& b, real scaleAB, hl_trans_op_t transa = !a.isTransposed() ? HPPL_OP_N : HPPL_OP_T; hl_trans_op_t transb = !b.isTransposed() ? HPPL_OP_N : HPPL_OP_T; - hl_matrix_mul(A_d, transa, B_d, transb, C_d, dimM, dimN, dimK, - scaleAB, scaleT, lda, ldb, ldc); + hl_matrix_mul(A_d, transa, B_d, transb, C_d, dimM, dimN, dimK, scaleAB, + scaleT, lda, ldb, ldc); } void GpuMatrix::mul(const GpuSparseMatrix& a, const GpuMatrix& b, real scaleAB, @@ -423,8 +421,8 @@ void GpuMatrix::mul(const GpuSparseMatrix& a, const GpuMatrix& b, real scaleAB, hl_sparse_matrix_s A_d = a.sMatrix_.get(); real* B_d = b.data_; real* C_d = data_; - hl_matrix_csr_mul_dense(A_d, transA, B_d, HPPL_OP_N, C_d, height_, - width_, b.height_, scaleAB, scaleT); + hl_matrix_csr_mul_dense(A_d, transA, B_d, HPPL_OP_N, C_d, height_, width_, + b.height_, scaleAB, scaleT); } void GpuMatrix::mul(const GpuMatrix& a, const GpuSparseMatrix& b, real scaleAB, @@ -445,11 +443,11 @@ void GpuMatrix::mul(const GpuMatrix& a, const GpuSparseMatrix& b, real scaleAB, << "Matrix dimensions are not equal"; } if (b.format_ == SPARSE_CSC) { - hl_matrix_dense_mul_csc(A_d, HPPL_OP_N, B_d, transB, C_d, height_, - width_, a.width_, scaleAB, scaleT); + hl_matrix_dense_mul_csc(A_d, HPPL_OP_N, B_d, transB, C_d, height_, width_, + a.width_, scaleAB, scaleT); } else { - hl_matrix_dense_mul_csr(A_d, HPPL_OP_N, B_d, transB, C_d, height_, - width_, a.width_, scaleAB, scaleT); + hl_matrix_dense_mul_csr(A_d, HPPL_OP_N, B_d, transB, C_d, height_, width_, + a.width_, scaleAB, scaleT); } } @@ -511,8 +509,8 @@ void GpuMatrix::selectRows(Matrix& table, IVector& ids) { size_t tableSize = table.getHeight(); int* index = ids.getData(); - hl_matrix_select_rows(a, stride_, table.getData(), table.stride_, - index, numSamples, tableSize, dim); + hl_matrix_select_rows(a, stride_, table.getData(), table.stride_, index, + numSamples, tableSize, dim); #endif } @@ -529,8 +527,8 @@ void GpuMatrix::addToRows(Matrix& table, IVector& ids) { size_t tableSize = table.getHeight(); int* index = ids.getData(); - hl_matrix_add_to_rows(table.getData(), table.stride_, a, stride_, - index, numSamples, tableSize, dim); + hl_matrix_add_to_rows(table.getData(), table.stride_, a, stride_, index, + numSamples, tableSize, dim); #endif } @@ -565,13 +563,8 @@ void GpuMatrix::rowMax(IVector& maxIds, Matrix& maxVal) { CHECK_EQ(maxIds.getSize(), numSamples * beam); CHECK_EQ(maxVal.getHeight(), numSamples); - hl_matrix_top_k(maxVal.getData(), - maxVal.getStride(), - maxIds.getData(), - this->getData(), - this->getStride(), - this->getWidth(), - beam, + hl_matrix_top_k(maxVal.getData(), maxVal.getStride(), maxIds.getData(), + this->getData(), this->getStride(), this->getWidth(), beam, numSamples); #endif } @@ -595,12 +588,12 @@ void GpuMatrix::maxoutForward(Matrix& a, IVector& id, size_t channels, size_t size = getWidth(); size_t batchSize = getHeight(); - const real* input = a.getData(); + const real* input = a.getData(); real* output = getData(); int* idForGpu = id.getData(); - hl_maxout_forward(input, output, idForGpu, batchSize, size, - size / channels, groups); + hl_maxout_forward(input, output, idForGpu, batchSize, size, size / channels, + groups); } void GpuMatrix::maxoutBackward(Matrix& a, IVector& id, size_t channels, @@ -611,12 +604,12 @@ void GpuMatrix::maxoutBackward(Matrix& a, IVector& id, size_t channels, size_t size = a.getWidth(); size_t batchSize = getHeight(); - real* input = getData(); + real* input = getData(); const real* output = a.getData(); const int* idForGpu = id.getData(); - hl_maxout_backward(input, output, idForGpu, batchSize, size, - size / channels, groups); + hl_maxout_backward(input, output, idForGpu, batchSize, size, size / channels, + groups); } /*calulate the error of classification */ @@ -632,8 +625,8 @@ void GpuMatrix::classificationError(MatrixPtr output, IVectorPtr label) { real* recResult_d = data_; int* label_d = label_ptr->getData(); - hl_matrix_classification_error(output_d, label_d, recResult_d, - height_, output_ptr->width_); + hl_matrix_classification_error(output_d, label_d, recResult_d, height_, + output_ptr->width_); } /* copy -log(output[i * width + label]) to this->data[i] */ @@ -702,8 +695,7 @@ void GpuMatrix::sequenceSoftmax(Matrix& output, const IVector& index) { real* outputData = output.getData(); auto starts = index.getData(); int numSequences = index.getSize() - 1; - hl_sequence_softmax_forward(inputData, outputData, - starts, numSequences); + hl_sequence_softmax_forward(inputData, outputData, starts, numSequences); } void GpuMatrix::softmaxDerivative(Matrix& output, Matrix& sftmaxSum) { @@ -717,8 +709,7 @@ void GpuMatrix::softmaxDerivative(Matrix& output, Matrix& sftmaxSum) { real* output_d = output.data_; real* sftmaxSum_d = sftmaxSum.data_; real* grad_d = data_; - hl_matrix_softmax_derivative(grad_d, output_d, sftmaxSum_d, height_, - width_); + hl_matrix_softmax_derivative(grad_d, output_d, sftmaxSum_d, height_, width_); } void GpuMatrix::softmaxBackward(Matrix& outputV) { @@ -769,7 +760,7 @@ void GpuMatrix::scaledTanh(Matrix& output, real p1, real p2) { } void GpuMatrix::cosSim(Matrix& output1, Matrix& output2, real scale) { CHECK(output1.useGpu_ == true && output2.useGpu_ == true) - << "Matrix type are not equal"; + << "Matrix type are not equal"; size_t numSamples = getHeight(); size_t dim = output1.getWidth(); CHECK_EQ(getWidth(), 1UL); @@ -778,15 +769,15 @@ void GpuMatrix::cosSim(Matrix& output1, Matrix& output2, real scale) { real* out = getData(); real* x = output1.getData(); real* y = output2.getData(); - hl_cossim(out, x, y, - dim, output1.getHeight(), output2.getHeight(), scale); + hl_cossim(out, x, y, dim, output1.getHeight(), output2.getHeight(), scale); } void GpuMatrix::cosSimDerivative(Matrix& output, Matrix& prevOut1, Matrix& prevOut2, Matrix& prevGrad1, Matrix& prevGrad2, real scale) { CHECK(output.useGpu_ == true && prevOut1.useGpu_ == true && prevOut2.useGpu_ == true && prevGrad1.useGpu_ == true && - prevGrad2.useGpu_ == true) << "Matrix type are not equal"; + prevGrad2.useGpu_ == true) + << "Matrix type are not equal"; CHECK_EQ(getWidth(), 1UL); CHECK_EQ(output.getWidth(), 1UL); @@ -806,9 +797,8 @@ void GpuMatrix::cosSimDerivative(Matrix& output, Matrix& prevOut1, real* prevOutY = prevOut2.getData(); real* prevGradX = prevGrad1.getData(); real* prevGradY = prevGrad2.getData(); - hl_cossim_derivative(grad, out, prevOutX, prevOutY, - prevGradX, prevGradY, dim, - prevOut1.getHeight(), prevOut2.getHeight(), scale); + hl_cossim_derivative(grad, out, prevOutX, prevOutY, prevGradX, prevGradY, dim, + prevOut1.getHeight(), prevOut2.getHeight(), scale); } void GpuMatrix::randomizeUniform() { @@ -859,8 +849,8 @@ void GpuMatrix::check(std::ostream& os, Matrix& refMat, bool printDiff) { void GpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, int channels, int blockH, int blockW, int strideH, - int strideW, int paddingH, int paddingW, - int outputH, int outputW) { + int strideW, int paddingH, int paddingW, int outputH, + int outputW) { CHECK(feature.useGpu_ == true) << "Matrix type are not equal"; CHECK_EQ(size_t(feaImgHeight * feaImgWidth * channels), @@ -870,17 +860,16 @@ void GpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, size_t elemCnt = outputH * outputW * blockH * blockW * channels; CHECK_EQ(elemCnt, height_ * width_) << "Matrix dimensions are not equal"; - hl_expand_feature2col(feature.getData(), channels, feaImgHeight, - feaImgWidth, blockH, blockW, strideH, strideW, - paddingH, paddingW, outputH, outputW, - getData()); + hl_expand_feature2col(feature.getData(), channels, feaImgHeight, feaImgWidth, + blockH, blockW, strideH, strideW, paddingH, paddingW, + outputH, outputW, getData()); } void GpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, int thisImgWidth, int channels, int blockH, int blockW, int strideH, int strideW, int paddingH, - int paddingW, int outputH, int outputW, - real alpha, real beta) { + int paddingW, int outputH, int outputW, real alpha, + real beta) { CHECK(expandFeat.useGpu_ == true) << "Matrix type are not equal"; CHECK_EQ(size_t(thisImgHeight * thisImgWidth * channels), getHeight() * getWidth()) @@ -889,18 +878,17 @@ void GpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, size_t elemCnt = outputH * outputW * blockW * blockH * channels; CHECK(elemCnt == expandFeat.getHeight() * expandFeat.getWidth()) << "Matrix dimensions are not equal"; - hl_shrink_col2feature( - expandFeat.getData(), channels, thisImgHeight, thisImgWidth, blockH, - blockW, strideH, strideW, paddingH, paddingW, outputH, outputW, - getData(), alpha, beta); + hl_shrink_col2feature(expandFeat.getData(), channels, thisImgHeight, + thisImgWidth, blockH, blockW, strideH, strideW, + paddingH, paddingW, outputH, outputW, getData(), alpha, + beta); } void GpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, - size_t imgSizeW, size_t channels, - size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, - size_t paddingH, size_t paddingW) { + size_t imgSizeW, size_t channels, size_t sizeX, + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, size_t paddingH, + size_t paddingW) { CHECK(inputMat.useGpu_ == true) << "Matrix type are not equal"; real* inputData = inputMat.getData(); @@ -911,16 +899,15 @@ void GpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, CHECK(height_ == inputMat.getHeight()); CHECK(width_ == outputH * outputW * channels); - hl_maxpool_forward(frameNum, inputData, channels, height, width, - outputH, outputW, sizeX, sizeY, strideH, strideW, - paddingH, paddingW, data_); + hl_maxpool_forward(frameNum, inputData, channels, height, width, outputH, + outputW, sizeX, sizeY, strideH, strideW, paddingH, + paddingW, data_, getStride()); } void GpuMatrix::maxPoolBackward(Matrix& inputMat, size_t imgSizeH, size_t imgSizeW, Matrix& outGrad, Matrix& outV, - size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, + size_t sizeX, size_t sizeY, size_t strideH, + size_t strideW, size_t outputH, size_t outputW, real scaleTargets, real scaleOutput, size_t paddingH, size_t paddingW) { CHECK(inputMat.useGpu_ == true && outGrad.useGpu_ == true && @@ -940,19 +927,17 @@ void GpuMatrix::maxPoolBackward(Matrix& inputMat, size_t imgSizeH, CHECK(outGrad.getHeight() == outV.getHeight() && outGrad.getWidth() == outV.getWidth()); - - hl_maxpool_backward(frameNum, inputData, outData, outDiff, channels, - height, width, outputH, outputW, sizeX, sizeY, - strideH, strideW, paddingH, paddingW, - scaleTargets, scaleOutput, data_); + hl_maxpool_backward(frameNum, inputData, outData, outDiff, channels, height, + width, outputH, outputW, sizeX, sizeY, strideH, strideW, + paddingH, paddingW, scaleTargets, scaleOutput, data_, + outGrad.getStride()); } void GpuMatrix::avgPoolForward(Matrix& inputMat, size_t imgSizeH, - size_t imgSizeW, size_t channels, - size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, - size_t paddingH, size_t paddingW) { + size_t imgSizeW, size_t channels, size_t sizeX, + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, size_t paddingH, + size_t paddingW) { CHECK(inputMat.useGpu_ == true) << "Matrix type are not equal"; real* inputData = inputMat.getData(); @@ -963,18 +948,17 @@ void GpuMatrix::avgPoolForward(Matrix& inputMat, size_t imgSizeH, CHECK(height_ == inputMat.getHeight()); CHECK(width_ == outputH * outputW * channels); - hl_avgpool_forward(frameNum, inputData, channels, height, width, - outputH, outputW, sizeX, sizeY, - strideH, strideW, - paddingH, paddingW, data_); + hl_avgpool_forward(frameNum, inputData, channels, height, width, outputH, + outputW, sizeX, sizeY, strideH, strideW, paddingH, + paddingW, data_, getStride()); } void GpuMatrix::avgPoolBackward(Matrix& outGrad, size_t imgSizeH, size_t imgSizeW, size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, - real scaleTargets, real scaleOutput, - size_t paddingH, size_t paddingW) { + size_t strideH, size_t strideW, size_t outputH, + size_t outputW, real scaleTargets, + real scaleOutput, size_t paddingH, + size_t paddingW) { CHECK(outGrad.useGpu_ == true) << "Matrix type are not equal"; real* outDiff = outGrad.getData(); @@ -986,11 +970,10 @@ void GpuMatrix::avgPoolBackward(Matrix& outGrad, size_t imgSizeH, CHECK(height_ == outGrad.getHeight()); CHECK(outGrad.getWidth() == outputH * outputW * channels); - hl_avgpool_backward(frameNum, outDiff, channels, height, width, - outputH, outputW, sizeX, sizeY, - strideH, strideW, paddingH, paddingW, - scaleTargets, scaleOutput, - data_); + hl_avgpool_backward(frameNum, outDiff, channels, height, width, outputH, + outputW, sizeX, sizeY, strideH, strideW, paddingH, + paddingW, scaleTargets, scaleOutput, data_, + outGrad.getStride()); } void GpuMatrix::crossMapNormalFwd(Matrix& input, size_t imgSizeH, @@ -1005,8 +988,8 @@ void GpuMatrix::crossMapNormalFwd(Matrix& input, size_t imgSizeH, CHECK(denoms.getHeight() == input.getHeight() && denoms.getWidth() == input.getWidth() && input.getHeight() == height_ && input.getWidth() == width_); - hl_CMRNorm_forward(num, input.getData(), denoms.getData(), data_, - channels, height, width, sizeX, scale, -pow); + hl_CMRNorm_forward(num, input.getData(), denoms.getData(), data_, channels, + height, width, sizeX, scale, -pow); } void GpuMatrix::crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, @@ -1026,13 +1009,11 @@ void GpuMatrix::crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, denoms.getWidth() == localGrad.getWidth()); hl_CMRNorm_backward(num, preOutV.getData(), denoms.getData(), - localOutV.getData(), localGrad.getData(), data_, - channels, height, width, sizeX, -pow, - 2.0f * pow * scale); + localOutV.getData(), localGrad.getData(), data_, channels, + height, width, sizeX, -pow, 2.0f * pow * scale); } -void GpuMatrix::maxSequenceForward(Matrix& input, - const IVector& sequence, +void GpuMatrix::maxSequenceForward(Matrix& input, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&input)); CHECK(dynamic_cast(&sequence)); @@ -1049,12 +1030,11 @@ void GpuMatrix::maxSequenceForward(Matrix& input, CHECK_EQ(numSequences, sequence.getSize() - 1); CHECK_EQ(numSequences * dim, index.getSize()); - hl_max_sequence_forward(inputData, starts, outData, maxIndex, - numSequences, dim); + hl_max_sequence_forward(inputData, starts, outData, maxIndex, numSequences, + dim); } -void GpuMatrix::maxSequenceBackward(Matrix& outputGrad, - const IVector& sequence, +void GpuMatrix::maxSequenceBackward(Matrix& outputGrad, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&outputGrad)); CHECK(dynamic_cast(&sequence)); @@ -1111,9 +1091,8 @@ void GpuMatrix::contextProjectionBackwardData(MatrixPtr inputGrad, real* inGrad = inputGrad->getData(); const int* starts = sequence.getData(); - hl_context_projection_backward_data(outGrad, starts, inGrad, - numSequences, inputDim, - contextLength, contextStart); + hl_context_projection_backward_data(outGrad, starts, inGrad, numSequences, + inputDim, contextLength, contextStart); } void GpuMatrix::contextProjectionBackwardWeight(MatrixPtr weightGrad, @@ -1133,9 +1112,9 @@ void GpuMatrix::contextProjectionBackwardWeight(MatrixPtr weightGrad, real* wtGrad = weightGrad->getData(); const int* starts = sequence.getData(); - hl_context_projection_backward_weight( - outGrad, starts, wtGrad, numSequences, weightDim, totalPad, contextLength, - contextStart, beginPad); + hl_context_projection_backward_weight(outGrad, starts, wtGrad, numSequences, + weightDim, totalPad, contextLength, + contextStart, beginPad); } void GpuMatrix::paramReluForward(Matrix& data, Matrix& W) { @@ -1147,8 +1126,7 @@ void GpuMatrix::paramReluForward(Matrix& data, Matrix& W) { size_t numSamples = data.getHeight(); size_t partial_sum = numElements / (W.getHeight() * W.getWidth()); real* output = getData(); - hl_param_relu_forward(output, input, w, numElements, numSamples, - partial_sum); + hl_param_relu_forward(output, input, w, numElements, numSamples, partial_sum); } void GpuMatrix::paramReluBackwardW(Matrix& oGrad, Matrix& data) { @@ -1160,8 +1138,8 @@ void GpuMatrix::paramReluBackwardW(Matrix& oGrad, Matrix& data) { size_t numElements = data.getWidth(); size_t numSamples = data.getHeight(); size_t partial_sum = numElements / (this->getHeight() * this->getWidth()); - hl_param_relu_backward_w(wgrad, ograd, input, - numElements, numSamples, partial_sum); + hl_param_relu_backward_w(wgrad, ograd, input, numElements, numSamples, + partial_sum); } void GpuMatrix::paramReluBackwardDiff(Matrix& oGrad, Matrix& data, Matrix& W) { @@ -1172,8 +1150,8 @@ void GpuMatrix::paramReluBackwardDiff(Matrix& oGrad, Matrix& data, Matrix& W) { size_t numElements = data.getWidth(); size_t numSamples = data.getHeight(); size_t partial_sum = numElements / (W.getHeight() * W.getWidth()); - hl_param_relu_backward_diff(ograd, input, w, diff, - numElements, numSamples, partial_sum); + hl_param_relu_backward_diff(ograd, input, w, diff, numElements, numSamples, + partial_sum); } void GpuMatrix::addColumnVector(const Matrix& b) { @@ -1422,8 +1400,8 @@ void CpuMatrix::transpose(MatrixPtr matTrans, bool memAlloc) { void CpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, int channels, int blockH, int blockW, int strideH, - int strideW, int paddingH, int paddingW, - int outputH, int outputW) { + int strideW, int paddingH, int paddingW, int outputH, + int outputW) { CHECK(feature.useGpu_ == false) << "Matrix type are not equal"; CHECK_EQ(size_t(feaImgHeight * feaImgWidth * channels), @@ -1463,8 +1441,8 @@ void CpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, void CpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, int thisImgWidth, int channels, int blockH, int blockW, int strideH, int strideW, int paddingH, - int paddingW, int outputH, int outputW, - real alpha, real beta) { + int paddingW, int outputH, int outputW, real alpha, + real beta) { CHECK(expandFeat.useGpu_ == false) << "Matrix type are not equal"; CHECK_EQ(size_t(thisImgHeight * thisImgWidth * channels), getHeight() * getWidth()) @@ -1501,11 +1479,10 @@ void CpuMatrix::convShrink(Matrix& expandFeat, int thisImgHeight, } void CpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, - size_t imgSizeW, size_t channels, - size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, - size_t paddingH, size_t paddingW) { + size_t imgSizeW, size_t channels, size_t sizeX, + size_t sizeY, size_t strideH, size_t strideW, + size_t outputH, size_t outputW, size_t paddingH, + size_t paddingW) { real* inputData = inputMat.getData(); real* outData = data_; size_t num = inputMat.getHeight(); @@ -1513,15 +1490,20 @@ void CpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, size_t inHeight = imgSizeH; CHECK(inHeight * inWidth == inputMat.getWidth() / channels); CHECK_EQ(num, this->getHeight()); - CHECK_EQ(channels*outputH*outputW, this->getWidth()); + CHECK_EQ(channels * outputH * outputW, this->getWidth()); /* initialize the data_ */ - for (size_t i = 0; i < height_ * width_; i++) { - outData[i] = -(real)FLT_MAX; + for (size_t i = 0; i < height_; i++) { + for (size_t j = 0; j < width_; j++) { + outData[i * getStride() + j] = -(real)FLT_MAX; + } } /* pool max one by one */ - for (size_t n = 0; n < num; ++n) { // frame by frame + for (size_t n = 0; n < num; ++n) { // frame by frame + if (!isContiguous()) { + outData = data_ + n * getStride(); + } for (size_t c = 0; c < channels; ++c) { // channel by channel for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { @@ -1564,6 +1546,10 @@ void CpuMatrix::maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, real* otData = outV.getData(); real* otGrad = outGrad.getData(); for (size_t n = 0; n < num; ++n) { + if (!outV.isContiguous()) { + otData = outV.getData() + n * outV.getStride(); + otGrad = outGrad.getData() + n * outGrad.getStride(); + } for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { @@ -1594,9 +1580,9 @@ void CpuMatrix::maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, size_t channels, size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, - size_t paddingH, size_t paddingW) { + size_t strideH, size_t strideW, size_t outputH, + size_t outputW, size_t paddingH, + size_t paddingW) { // The main loop size_t num = input.getHeight(); size_t inHeight = imgSizeH; @@ -1607,6 +1593,9 @@ void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, real* inData = input.getData(); for (size_t n = 0; n < num; ++n) { + if (!isContiguous()) { + tgtData = data_ + n * getStride(); + } for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { @@ -1638,9 +1627,8 @@ void CpuMatrix::avgPoolForward(Matrix& input, size_t imgSizeH, size_t imgSizeW, } void CpuMatrix::avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, - size_t sizeX, size_t sizeY, - size_t strideH, size_t strideW, - size_t outputH, size_t outputW, + size_t sizeX, size_t sizeY, size_t strideH, + size_t strideW, size_t outputH, size_t outputW, real scaleTargets, real scaleOutput, size_t paddingH, size_t paddingW) { size_t num = input.getHeight(); @@ -1650,6 +1638,9 @@ void CpuMatrix::avgPoolBackward(Matrix& input, size_t imgSizeH, size_t imgSizeW, real* outData = getData(); for (size_t n = 0; n < num; ++n) { + if (!input.isContiguous()) { + inData = input.getData() + n * input.getStride(); + } for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { for (size_t pw = 0; pw < outputW; ++pw) { @@ -1752,8 +1743,7 @@ void CpuMatrix::crossMapNormalBwd(Matrix& localGrad, Matrix& denoms, * Output: output size is the number of input sequences (NOT input instances). * output[i] is set to max_{for each instance in this sequence}{input[i]} */ -void CpuMatrix::maxSequenceForward(Matrix& input, - const IVector& sequence, +void CpuMatrix::maxSequenceForward(Matrix& input, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&input)); CHECK(dynamic_cast(&sequence)); @@ -1794,8 +1784,7 @@ void CpuMatrix::maxSequenceForward(Matrix& input, } } -void CpuMatrix::maxSequenceBackward(Matrix& outputGrad, - const IVector& sequence, +void CpuMatrix::maxSequenceBackward(Matrix& outputGrad, const IVector& sequence, IVector& index) { CHECK(dynamic_cast(&outputGrad)); CHECK(dynamic_cast(&sequence)); @@ -2000,8 +1989,7 @@ void CpuMatrix::collectBias(Matrix& a, real scale) { } } -void CpuMatrix::sequenceAvgForward(Matrix& a, - const IVector& startsPos, +void CpuMatrix::sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode) { size_t height = getHeight(); size_t width = getWidth(); @@ -2592,7 +2580,7 @@ void SharedCpuMatrix::mul(CpuSparseMatrix* a, CpuMatrix* b, real scaleAB, blockSeq.push_back(k); } std::shuffle(blockSeq.begin(), blockSeq.end(), - ThreadLocalRandomEngine::get()); + ThreadLocalRandomEngine::get()); } std::vector& localBufRows = *localBufRows_; int* cols = a->getCols(); @@ -2823,7 +2811,7 @@ void CpuMatrix::maxoutForward(Matrix& a, IVector& id, size_t channels, size_t size = getWidth(); size_t batchSize = getHeight(); size_t featLen = size / channels; - const real* input = a.getData(); + const real* input = a.getData(); int* idForCpu = id.getData(); MatrixPtr maxInMat, maxOutMat; @@ -2857,8 +2845,8 @@ void CpuMatrix::maxoutBackward(Matrix& a, IVector& id, size_t channels, size_t batchSize = getHeight(); size_t featLen = size / channels; size_t newFeatLen = groups * featLen; - real* inputG = getData(); - const real* outG = a.getData(); + real* inputG = getData(); + const real* outG = a.getData(); int* idForCpu = id.getData(); for (size_t batch_idx = 0; batch_idx < batchSize; ++batch_idx) { @@ -3082,9 +3070,9 @@ void CpuMatrix::sequenceSoftmax(Matrix& output, const IVector& index) { CHECK(isContiguous()); MatrixPtr inTmp = Matrix::create(nullptr, /* height= */ 1, 1, - /* trans= */ false, false); + /* trans= */ false, false); MatrixPtr outTmp = Matrix::create(nullptr, /* height= */ 1, 1, - /* trans= */ false, false); + /* trans= */ false, false); size_t numSequences = index.getSize() - 1; auto starts = index.getData(); for (size_t i = 0; i < numSequences; ++i) { diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index 70c1f8d563238..5dac2f8204190 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -120,6 +120,14 @@ message PoolConfig { optional uint32 padding_y = 13 [default = 0]; } +message SppConfig { + required string pool_type = 1; + required uint32 pyramid_height = 2; + required uint32 channels = 3; + required uint32 img_size = 4; + optional uint32 img_size_y = 5; +} + message NormConfig { // rnorm or cmrnorm required string norm_type = 1; @@ -194,6 +202,9 @@ message ProjectionConfig { optional ConvConfig conv_conf = 8; optional int32 num_filters = 9; + // For pool + optional PoolConfig pool_conf = 10; + // For IdentityOffsetProjection optional uint64 offset = 11 [default = 0]; } @@ -235,6 +246,7 @@ message LayerInputConfig { // Set the argument name. optional string input_layer_argument = 9; optional MaxOutConfig maxout_conf = 10; + optional SppConfig spp_conf = 11; } message LayerConfig { From 45c81a414f912b93c6ad34fa4045e8e80fd396f6 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Wed, 2 Nov 2016 09:56:20 +0800 Subject: [PATCH 199/324] Add job=time in trainer, refine cudnn_conv to reduce gpu memory and speed up training. (#218) * Add benchmark for PaddlePaddle, tensorflow and caffe * ConvProjection to reduce memory for goolenet * Add unit test for ConvProjection. 1. unit test in test_LayerGrad. 2. compare the ConvPorjection and CudnnConvLayer, also compare the concat_layer+img_conv_layer and concat_layer_conv_projection. * Reduce cudnn_conv memory and add benchmark document. 1. Use TmpMatrix as the workspace in cudnn_conv to reduce gpu memory. It reduce lots of memory. 2. Add benchmark document. 3. fix smallnet_mnist_cifar.py in paddle. * Add job=time and refine cudnn_conv to reduce gpu memroy and speed up * Refine cudnn_conv and shared biases operation in concat_layer and mixed_layer. * follow comments * follow comments * Use unique_ptr to prevent memory leaks in CudnnConvLayer. --- doc/ui/cmd_argument/argument_outline.md | 7 +- doc/ui/cmd_argument/detail_introduction.md | 4 + paddle/cuda/include/hl_device_functions.cuh | 19 ++ paddle/cuda/include/hl_matrix.h | 36 +++ paddle/cuda/include/stub/hl_matrix_stub.h | 13 + paddle/cuda/src/hl_cuda_cudnn.cc | 7 +- paddle/cuda/src/hl_cuda_matrix.cu | 87 ++++++ paddle/cuda/src/hl_cuda_sparse.cuh | 18 -- paddle/gserver/layers/ConcatenateLayer.cpp | 33 ++- paddle/gserver/layers/ConvBaseLayer.cpp | 51 ++-- paddle/gserver/layers/ConvBaseLayer.h | 24 +- paddle/gserver/layers/ConvProjection.cpp | 210 ++++++++++++++ paddle/gserver/layers/ConvProjection.h | 125 +++++++++ paddle/gserver/layers/CudnnConvLayer.cpp | 256 +++--------------- paddle/gserver/layers/CudnnConvLayer.h | 73 +---- paddle/gserver/layers/ExpandConvLayer.cpp | 39 ++- paddle/gserver/layers/ExpandConvLayer.h | 10 +- paddle/gserver/layers/MixedLayer.cpp | 20 +- paddle/gserver/layers/MixedLayer.h | 1 + paddle/gserver/tests/LayerGradUtil.cpp | 6 +- paddle/gserver/tests/LayerGradUtil.h | 3 +- paddle/gserver/tests/img_conv_a.conf | 39 +++ paddle/gserver/tests/img_conv_b.conf | 32 +++ paddle/gserver/tests/test_LayerGrad.cpp | 39 +++ paddle/gserver/tests/test_NetworkCompare.cpp | 9 + paddle/math/Matrix.cpp | 52 ++++ paddle/math/Matrix.h | 28 ++ paddle/math/tests/test_matrixCompare.cpp | 56 ++++ paddle/trainer/CMakeLists.txt | 1 + paddle/trainer/Trainer.h | 1 + paddle/trainer/TrainerBenchmark.cpp | 71 +++++ paddle/trainer/TrainerMain.cpp | 2 + proto/ModelConfig.proto.m4 | 5 +- python/paddle/trainer/config_parser.py | 68 ++++- .../paddle/trainer_config_helpers/layers.py | 105 ++++++- .../paddle/trainer_config_helpers/networks.py | 165 ++++++++++- .../tests/configs/check.md5 | 9 +- .../tests/configs/generate_protostr.sh | 2 +- .../tests/configs/test_bi_grumemory.py | 10 + 39 files changed, 1340 insertions(+), 396 deletions(-) create mode 100644 paddle/gserver/layers/ConvProjection.cpp create mode 100644 paddle/gserver/layers/ConvProjection.h create mode 100644 paddle/gserver/tests/img_conv_a.conf create mode 100644 paddle/gserver/tests/img_conv_b.conf create mode 100644 paddle/trainer/TrainerBenchmark.cpp create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py diff --git a/doc/ui/cmd_argument/argument_outline.md b/doc/ui/cmd_argument/argument_outline.md index 98dadc270dcac..d6cc2c6ed7cc1 100644 --- a/doc/ui/cmd_argument/argument_outline.md +++ b/doc/ui/cmd_argument/argument_outline.md @@ -183,7 +183,7 @@ It looks like there are a lot of arguments. However, most of them are for develo -GPUgpu_id +GPUgpu_id √√√√ @@ -207,6 +207,11 @@ It looks like there are a lot of arguments. However, most of them are for develo √√√√ + +cudnn_conv_workspace_limit_in_mb +√√√√ + + RNN beam_size diff --git a/doc/ui/cmd_argument/detail_introduction.md b/doc/ui/cmd_argument/detail_introduction.md index 0d0362d022a72..07608e5edf740 100644 --- a/doc/ui/cmd_argument/detail_introduction.md +++ b/doc/ui/cmd_argument/detail_introduction.md @@ -163,6 +163,10 @@ - Choose path to dynamic load NVIDIA CUDA library, for instance, /usr/local/cuda/lib64. [Default]: LD_LIBRARY_PATH - type: string (default: "", null) +* `--cudnn_conv_workspace_limit_in_mb` + - Specify cuDNN max workspace limit, in units MB, 4096MB=4GB by default. + - type: int32 (default: 4096MB=4GB) + ## NLP: RNN/LSTM/GRU * `--rnn_use_batch` - Whether to use batch method for calculation in simple RecurrentLayer. diff --git a/paddle/cuda/include/hl_device_functions.cuh b/paddle/cuda/include/hl_device_functions.cuh index 88d950d6c1713..159c26f443cb1 100755 --- a/paddle/cuda/include/hl_device_functions.cuh +++ b/paddle/cuda/include/hl_device_functions.cuh @@ -48,5 +48,24 @@ inline __device__ double paddleAtomicAdd(double* address, double val) { } } // namespace paddle +/** + * @brief sum reduction + * + * @param[in,out] smem input data, better to use __shared__ memory. + * @param[in] tid thread index. + * @param[in] threads the total thread number used to reduce, + * such as, blockDim.x. + * + * @return smem[0]: the sum of each elements in smem. + */ +__device__ __forceinline__ +void simpleReduce(real* smem, int tid, int threads) { + for (unsigned int s = threads / 2; s > 0; s >>= 1) { + if (tid < s) { + smem[tid] += smem[tid + s]; + } + __syncthreads(); + } +} #endif /* HL_DEVICE_FUNCTIONS_CUH_ */ diff --git a/paddle/cuda/include/hl_matrix.h b/paddle/cuda/include/hl_matrix.h index 17419790471a7..71e8f8e3a60c9 100644 --- a/paddle/cuda/include/hl_matrix.h +++ b/paddle/cuda/include/hl_matrix.h @@ -229,4 +229,40 @@ extern void hl_cossim_derivative(real* grad, int input2_height, real scale); +/** + * @brief Matrix addition: A_d[i][j] += scale * B_d[j/channel]. + * + * @param[in] A_d input matrix (M x N). + * @param[in] B_d input matrix (1 x channel). + * @param[in] channel width of B. + * @param[in] dimM height of A. + * @param[in] dimN width of A. + * @param[in] scale scalar used for addition. + * + */ +extern void hl_matrix_add_shared_bias(real* A_d, + real* B_d, + const int channel, + const int dimM, + const int dimN, + real scale); + +/** + * @brief Matrix addition: A_d[i][j] += scale * B_d[j/channel]. + * + * @param[in] B_d input matrix (1 x channel). + * @param[in] A_d input matrix (M x N). + * @param[in] channel width of B. + * @param[in] dimM height of A. + * @param[in] dimN width of A. + * @param[in] scale scalar used for addition. + * + */ +extern void hl_matrix_collect_shared_bias(real* B_d, + real* A_d, + const int channel, + const int dimM, + const int dimN, + real scale); + #endif /* HL_MATRIX_H_ */ diff --git a/paddle/cuda/include/stub/hl_matrix_stub.h b/paddle/cuda/include/stub/hl_matrix_stub.h index f1f1020c84d46..e37b1275432ca 100644 --- a/paddle/cuda/include/stub/hl_matrix_stub.h +++ b/paddle/cuda/include/stub/hl_matrix_stub.h @@ -101,4 +101,17 @@ inline void hl_cossim_derivative(real* grad, int input2_height, real scale) {} +inline void hl_matrix_add_shared_bias(real* A_d, + real* B_d, + const int channel, + const int dimM, + const int dimN, + real scale) {} + +inline void hl_matrix_collect_shared_bias(real* B_d, + real* A_d, + const int channel, + const int dimM, + const int dimN, + real scale) {} #endif // HL_MATRIX_STUB_H_ diff --git a/paddle/cuda/src/hl_cuda_cudnn.cc b/paddle/cuda/src/hl_cuda_cudnn.cc index b215c0f6e33a1..7810d0d10053d 100644 --- a/paddle/cuda/src/hl_cuda_cudnn.cc +++ b/paddle/cuda/src/hl_cuda_cudnn.cc @@ -20,6 +20,11 @@ limitations under the License. */ #include "hl_thread.ph" #include "hl_dso_loader.h" #include "paddle/utils/Logging.h" +#include "paddle/utils/CommandLineParser.h" + +P_DEFINE_int32(cudnn_conv_workspace_limit_in_mb, 4096, + "Specify cuDNN max workspace limit, in units MB, " + "4096MB=4GB by default."); namespace dynload { @@ -242,7 +247,7 @@ void hl_conv_workspace(hl_tensor_descriptor input, CHECK_NOTNULL(conv); // Specify workspace limit directly - size_t memoryLimitBytes = 8 * 1024 * 1024; + size_t memoryLimitBytes = (1LL << 20) * FLAGS_cudnn_conv_workspace_limit_in_mb; // cudnn convolution forward configuration cudnnTensorDescriptor_t fwd_src_desc = GET_TENSOR_DESCRIPTOR(input); diff --git a/paddle/cuda/src/hl_cuda_matrix.cu b/paddle/cuda/src/hl_cuda_matrix.cu index 067e68c41e119..3df9f63f9e4b7 100644 --- a/paddle/cuda/src/hl_cuda_matrix.cu +++ b/paddle/cuda/src/hl_cuda_matrix.cu @@ -20,6 +20,7 @@ limitations under the License. */ #include "hl_sequence.h" #include "paddle/utils/Logging.h" #include "hl_device_functions.cuh" +#include "hl_gpu_matrix_kernel.cuh" DEFINE_MATRIX_UNARY_OP(Zero, a = 0); DEFINE_MATRIX_TERNARY_PARAMETER_OP(_add, TWO_PARAMETER, c = p1*a + p2*b); @@ -673,3 +674,89 @@ void hl_cossim_derivative(real* grad, input1_height, input2_height, scale); CHECK_SYNC("hl_cossim_derivate failed"); } + +__global__ void KeMatrixAddSharedBias(real* A, + real* B, + const int channel, + const int M, + const int N, + real scale) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + int dim = N / channel; + if (index < M * N) { + int i = index % N; + i = i / dim; + A[index] += scale * B[i]; + } +} + +void hl_matrix_add_shared_bias(real* A_d, + real* B_d, + const int channel, + const int dimM, + const int dimN, + real scale) { + const int blocks = 512; + const int grids = DIVUP(dimM * dimN, blocks); + KeMatrixAddSharedBias<<>> + (A_d, B_d, channel, dimM, dimN, scale); + CHECK_SYNC("hl_matrix_add_shared_bias failed"); +} + + +template +__global__ void KeMatrixCollectSharedBias(real *B, + real *A, + const int channel, + const int M, + const int N, + const int dim, + const int limit, + real scale) { + if (dim < limit) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < channel) { + real sum = 0.0; + for (int i = 0; i < M; ++i) { + for (int j = 0; j < dim; ++j) { + sum += A[i * N + index * dim + j]; + } + } + B[index] += scale * sum; + } + } else { + const int tid = threadIdx.x; + const int bid = blockIdx.x; + __shared__ real smem[blockSize]; + real sum = 0.0; + for (int j = 0; j < ((dim * M + blockSize - 1) / blockSize); ++j) { + int n = j * blockSize + tid; + int m = n / dim; + int w = n % dim; + smem[tid] = (m < M && w < dim) ? A[m * N + bid * dim + w] : 0.0; + __syncthreads(); + simpleReduce(smem, tid, blockSize); + sum += smem[0]; + } + if (tid == 0) { + B[bid] += scale * sum; + } + } +} + +void hl_matrix_collect_shared_bias(real* B_d, + real* A_d, + const int channel, + const int dimM, + const int dimN, + real scale) { + const int dim = dimN / channel; + const int blocks = 256; + const int limit = 64; + int grids = (dimM * dim) < limit ? DIVUP(channel, blocks) : channel; + + KeMatrixCollectSharedBias + <<< grids, blocks, 0, STREAM_DEFAULT>>> + (B_d, A_d, channel, dimM, dimN, dim, limit, scale); + CHECK_SYNC("hl_matrix_collect_shared_bias failed"); +} diff --git a/paddle/cuda/src/hl_cuda_sparse.cuh b/paddle/cuda/src/hl_cuda_sparse.cuh index c3b98f4ebc38d..9cf2d5a843343 100644 --- a/paddle/cuda/src/hl_cuda_sparse.cuh +++ b/paddle/cuda/src/hl_cuda_sparse.cuh @@ -908,24 +908,6 @@ int findIndex(int* indice, int num, int index) { return (end - 1); } -/** - * @brief sum reduction - * - * @param[in,out] smem input data, better to use __shared__ memory. - * @param[in] tid local thread index. - * @param[in] blockDimX the size of blockDim.x. - * - * note: return smem[0]: the sum of each elements of smem. - */ -__device__ __forceinline__ -void reduce(real* smem, int tid, int blockDimX) { - for (unsigned int s = blockDimX / 2; s > 0; s >>= 1) { - if (tid < s) { - smem[tid] += smem[tid + s]; - } - __syncthreads(); - } -} /** * @brief sum columns of csr sparse matrix (csr_val), then add to a_val. diff --git a/paddle/gserver/layers/ConcatenateLayer.cpp b/paddle/gserver/layers/ConcatenateLayer.cpp index 52a7cb6f777c3..bb6709b8df330 100644 --- a/paddle/gserver/layers/ConcatenateLayer.cpp +++ b/paddle/gserver/layers/ConcatenateLayer.cpp @@ -97,7 +97,8 @@ void ConcatenateLayer::backward(const UpdateCallback& callback) { */ class ConcatenateLayer2 : public Layer { public: - explicit ConcatenateLayer2(const LayerConfig& config) : Layer(config) {} + explicit ConcatenateLayer2(const LayerConfig& config) : + Layer(config) {} ~ConcatenateLayer2() {} @@ -110,6 +111,8 @@ class ConcatenateLayer2 : public Layer { std::vector> projections_; std::vector projOutput_; std::vector> projCol_; + bool sharedBias_; + std::unique_ptr biases_; }; REGISTER_LAYER(concat2, ConcatenateLayer2); @@ -119,7 +122,6 @@ bool ConcatenateLayer2::init(const LayerMap& layerMap, /* Initialize the basic parent class */ if (!Layer::init(layerMap, parameterMap)) return false; - CHECK(!biasParameter_); CHECK_EQ(inputLayers_.size(), parameters_.size()); projections_.reserve(inputLayers_.size()); projCol_.reserve(inputLayers_.size()); @@ -137,6 +139,13 @@ bool ConcatenateLayer2::init(const LayerMap& layerMap, } CHECK_EQ(getSize(), endCol); + /* initialize biases_ */ + if (biasParameter_.get() != NULL) { + sharedBias_ = config_.shared_biases(); + size_t psize = config_.bias_size(); + biases_ = std::unique_ptr(new Weight(1, psize, biasParameter_)); + } + return true; } @@ -154,8 +163,17 @@ void ConcatenateLayer2::forward(PassType passType) { projOutput_[i].grad = output_.grad->subColMatrix(startCol, endCol); } - for (size_t i = 0; i != inputLayers_.size(); ++i) { - projections_[i]->forward(&getInput(i), &projOutput_[i], passType); + { + AsyncGpuBlock block; + for (size_t i = 0; i != inputLayers_.size(); ++i) { + projections_[i]->forward(&getInput(i), &projOutput_[i], passType); + } + } + + /* add the bias-vector */ + if (biases_) { + REGISTER_TIMER_INFO("FwBiasTimer", getName().c_str()); + output_.value->addBias(*(biases_->getW()), 1, sharedBias_); } /* activation */ { @@ -170,6 +188,13 @@ void ConcatenateLayer2::backward(const UpdateCallback& callback) { backwardActivation(); } + AsyncGpuBlock block; + if (biases_ && biases_->getWGrad()) { + REGISTER_TIMER_INFO("Concat2BpBiasTimer", getName().c_str()); + biases_->getWGrad()->collectBias(*getOutputGrad(), 1, sharedBias_); + biases_->getParameterPtr()->incUpdate(callback); + } + for (size_t i = 0; i != inputLayers_.size(); ++i) { if (projections_[i]) { projections_[i]->backward(callback); diff --git a/paddle/gserver/layers/ConvBaseLayer.cpp b/paddle/gserver/layers/ConvBaseLayer.cpp index 9ed9572139dc8..040510b7ad211 100644 --- a/paddle/gserver/layers/ConvBaseLayer.cpp +++ b/paddle/gserver/layers/ConvBaseLayer.cpp @@ -35,25 +35,12 @@ bool ConvBaseLayer::init(const LayerMap& layerMap, filterSizeY_.push_back(conf.filter_size_y()); filterPixels_.push_back(filterSize_.back() * filterSizeY_.back()); channels_.push_back(conf.channels()); - imgSize_.push_back(conf.img_size()); - imgPixels_.push_back(imgSize_.back() * imgSize_.back()); + imgSizeH_.push_back(conf.img_size()); + imgSizeW_.push_back(conf.img_size()); groups_.push_back(conf.groups()); filterChannels_.push_back(conf.filter_channels()); - outputX_.push_back(conf.output_x()); - outputs_.push_back(outputX_.back() * outputX_.back()); - } - - /* initialize the weightList */ - CHECK(inputLayers_.size() == parameters_.size()); - for (size_t i = 0; i < inputLayers_.size(); i++) { - size_t height, width; - height = filterPixels_[i] * filterChannels_[i]; - width = numFilters_; - - // create a new weight - CHECK_EQ(parameters_[i]->getSize(), width * height); - Weight* w = new Weight(height, width, parameters_[i]); - weights_.emplace_back(w); + outputH_.push_back(conf.output_x()); + outputW_.push_back(conf.output_x()); } /* initialize the biases_ */ @@ -74,4 +61,34 @@ bool ConvBaseLayer::init(const LayerMap& layerMap, return true; } +size_t ConvBaseLayer::calOutputSize() { + auto clearAndReserve = [this](IntV* vec) { + vec->clear(); + vec->reserve(this->inputLayers_.size()); + }; + clearAndReserve(&imgSizeH_); + clearAndReserve(&imgSizeW_); + clearAndReserve(&outputH_); + clearAndReserve(&outputW_); + size_t layerSize = 0; + for (size_t i = 0; i < inputLayers_.size(); i++) { + imgSizeH_.push_back(inputLayers_[i]->getOutput().getFrameHeight()); + imgSizeW_.push_back(inputLayers_[i]->getOutput().getFrameWidth()); + if (imgSizeH_[i] == 0) + imgSizeH_[i] = config_.inputs(i).conv_conf().img_size(); + if (imgSizeW_[i] == 0) + imgSizeW_[i] = config_.inputs(i).conv_conf().img_size(); + outputH_.push_back( + outputSize(imgSizeH_[i], filterSizeY_[i], paddingY_[i], strideY_[i])); + outputW_.push_back( + outputSize(imgSizeW_[i], filterSize_[i], padding_[i], stride_[i])); + CHECK_EQ(outputH_[i], outputH_[0]); + CHECK_EQ(outputW_[i], outputW_[0]); + } + getOutput().setFrameHeight(outputH_[0]); + getOutput().setFrameWidth(outputW_[0]); + layerSize = outputH_[0] * outputW_[0] * size_t(numFilters_); + return layerSize; +} + } // namespace paddle diff --git a/paddle/gserver/layers/ConvBaseLayer.h b/paddle/gserver/layers/ConvBaseLayer.h index eaeaebf43be25..316514acf1a0d 100644 --- a/paddle/gserver/layers/ConvBaseLayer.h +++ b/paddle/gserver/layers/ConvBaseLayer.h @@ -43,19 +43,18 @@ class ConvBaseLayer : public Layer { IntV filterSizeY_; /// The spatial dimensions of the convolution input. IntV channels_; - /// The spatial dimensions of input feature map. - IntV imgSize_; - /// The total pixel size of input feature map. - /// imgPixels_ = imgSizeX_ * imgSizeY_. - IntV imgPixels_; + /// The spatial dimensions of input feature map height. + IntV imgSizeH_; + /// The spatial dimensions of input feature map width. + IntV imgSizeW_; /// filterPixels_ = filterSizeX_ * filterSizeY_. IntV filterPixels_; /// filterChannels_ = channels_/groups_. IntV filterChannels_; - /// The spatial dimensions of output feature map. - IntV outputX_; - /// The spatial dimensions of output feature map. - IntV outputs_; + /// The spatial dimensions of output feature map height. + IntV outputH_; + /// The spatial dimensions of output feature map width. + IntV outputW_; /// Group size, refer to grouped convolution in /// Alex Krizhevsky's paper: when group=2, the first half of the /// filters are only connected to the first half of the input channels, @@ -80,6 +79,13 @@ class ConvBaseLayer : public Layer { virtual bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + /** + * imgSizeH_ and imgSizeW_ will be set according to the previous input layers + * in this function. Then it will calculate outputH_ and outputW_ and set them + * into output argument. + */ + virtual size_t calOutputSize(); + Weight& getWeight(int idx) { return *weights_[idx]; } /** diff --git a/paddle/gserver/layers/ConvProjection.cpp b/paddle/gserver/layers/ConvProjection.cpp new file mode 100644 index 0000000000000..d1ce53fe26351 --- /dev/null +++ b/paddle/gserver/layers/ConvProjection.cpp @@ -0,0 +1,210 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#include "paddle/utils/Stat.h" +#include "ConvProjection.h" + +namespace paddle { + +REGISTER_PROJECTION(conv, ConvProjection); + +ThreadLocalD> ConvProjection::convMem_; + +ConvProjection::ConvProjection(const ProjectionConfig& config, + ParameterPtr parameter, bool useGpu) + : Projection(config, parameter, useGpu) { + + CHECK(useGpu); // only support GPU + getConvParams(); + initCudnn(); + + size_t height = filterH_ * filterW_ * channels_ / groups_; + size_t width = numFilters_; + weight_.reset(new Weight(height, width, parameter)); + weightOffset_ = height * width / groups_; +} + +void ConvProjection::getConvParams() { + const ConvConfig &conf = config_.conv_conf(); + paddingH_ = conf.padding_y(); + paddingW_ = conf.padding(); + + strideH_ = conf.stride_y(); + strideW_ = conf.stride(); + + filterH_ = conf.filter_size_y(); + filterW_ = conf.filter_size(); + + configImgH_ = conf.img_size(); + configImgW_ = conf.img_size(); + + channels_ = conf.channels(); + numFilters_ = config_.num_filters(); + + groups_ = conf.groups(); + CHECK_EQ(channels_ % groups_, 0); + CHECK_EQ(numFilters_ % groups_, 0); +} + +void ConvProjection::initCudnn() { + hl_create_filter_descriptor(&filterDesc_, channels_, numFilters_, + filterH_, filterW_); + hl_create_tensor_descriptor(&inputDesc_); + hl_create_tensor_descriptor(&outputDesc_); + hl_create_convolution_descriptor(&convDesc_, inputDesc_, filterDesc_, + paddingH_, paddingW_, strideH_, strideW_); + + // initialize all to default algorithms + fwdAlgo_ = 0; + bwdFilterAlgo_ = 0; + bwdDataAlgo_ = 0; + fwdLimitBytes_ = 0; + bwdDataLimitBytes_ = 0; + bwdFilterLimitBytes_ = 0; + workSpaceInBytes_ = 0; + + batchNum_ = 0; + isSelectAlgo_ = false; +} + +void ConvProjection::reshapeTensorDesc(int batchSize) { + hl_tensor_reshape(inputDesc_, batchSize, channels_, imageH_, imageW_, + channels_ * imageH_ * imageW_, imageH_ * imageW_, + imageW_, 1); + hl_reset_convolution_descriptor(convDesc_, inputDesc_, filterDesc_, + paddingH_, paddingW_, strideH_, strideW_); + + // The stride between two consecutive images in ConvProjection may not be 1, + // for example, in the case of layer ConcatenateLayer2 with two + // ConvProjection, the stride is the output_size of layer ConcatenateLayer2. + // So the calculation of nStride is different from CudnnConvLayer. + // In fact, only "nStride = out_->value->getStride()" is ok. + size_t nStride = numFilters_ * outputH_ * outputW_; + if (out_->value->isContiguous()) { + CHECK_EQ(nStride, out_->value->getWidth()); + } else { + nStride = out_->value->getStride(); + } + + hl_tensor_reshape(outputDesc_, batchSize, numFilters_, outputH_, outputW_, + nStride, outputH_ * outputW_, outputW_, 1); +} + +void ConvProjection::reshape(int batchSize) { + size_t width = calOutputSize(); + CHECK_EQ(width, out_->value->getWidth()); + + isSelectAlgo_ = (batchSize == batchNum_); + batchNum_ = batchSize; + + if (!isSelectAlgo_) { + reshapeTensorDesc(batchSize); + hl_conv_workspace(inputDesc_, outputDesc_, filterDesc_, + convDesc_, &fwdAlgo_, &fwdLimitBytes_, + &bwdDataAlgo_, &bwdDataLimitBytes_, + &bwdFilterAlgo_, &bwdFilterLimitBytes_); + + size_t maxWorkSpace = 0; + maxWorkSpace = std::max(fwdLimitBytes_, bwdDataLimitBytes_); + maxWorkSpace = std::max(maxWorkSpace, bwdFilterLimitBytes_); + workSpaceInBytes_ = maxWorkSpace; + + + VLOG(3) << getName() << " Fwd / BwdData / BwdFilter algo: " << fwdAlgo_ + << " / " << bwdDataAlgo_ + << " / " << bwdFilterAlgo_; + } + + isSelectAlgo_ = true; +} + +void ConvProjection::forward() { + int batchSize = in_->value->getHeight(); + reshape(batchSize); + + void* workSpace = NULL; + if (workSpaceInBytes_ > 0) { + workSpace = getSpaceBytes(workSpaceInBytes_); + } + + for (int g = 0; g < groups_; ++g) { + REGISTER_TIMER_INFO("CudnnConvFwTimer", getName().c_str()); + + real *inputData = in_->value->getData() + g * inputOffset_; + real *wgtData = weight_->getW()->getData() + g * weightOffset_; + real *outData = out_->value->getData() + g * outputOffset_; + hl_convolution_forward(inputDesc_, inputData, outputDesc_, + outData, filterDesc_, wgtData, + convDesc_, workSpace, + fwdLimitBytes_, fwdAlgo_); + } +} + +void ConvProjection::backward(const UpdateCallback& callback) { + REGISTER_TIMER_INFO("CudnnConvBpTimer", getName().c_str()); + + void* workSpace = NULL; + if (workSpaceInBytes_ > 0) { + workSpace = getSpaceBytes(workSpaceInBytes_); + } + + for (int g = 0; g < groups_; ++g) { + real *outGrad = out_->grad->getData() + g * outputOffset_; + if (weight_->getWGrad()) { + real *inputData = in_->value->getData() + g * inputOffset_; + real *weightGrad = weight_->getWGrad()->getData() + g * weightOffset_; + hl_convolution_backward_filter( + inputDesc_, inputData, outputDesc_, outGrad, filterDesc_, + weightGrad, convDesc_, workSpace, bwdFilterLimitBytes_, + bwdFilterAlgo_); + } + + MatrixPtr preGrad = in_->grad; + if (NULL != preGrad) { + real *inputGrad = preGrad->getData() + g * inputOffset_; + real *wgtData = weight_->getW()->getData() + g* weightOffset_; + hl_convolution_backward_data( + inputDesc_, inputGrad, outputDesc_, outGrad, filterDesc_, + wgtData, convDesc_, workSpace, bwdDataLimitBytes_, + bwdDataAlgo_); + } + } + + weight_->getParameterPtr()->incUpdate(callback); +} + +void* ConvProjection::getSpaceBytes(size_t size) { + std::vector& convMem = *convMem_; + if (convMem.empty()) { + int numDevices = hl_get_device_count(); + convMem.resize(numDevices); + } + + int devId = hl_get_device(); + MemoryHandle** localMem = &(convMem[devId]); + if (NULL == *localMem || size > (*localMem)->getAllocSize()) { + *localMem = new GpuMemoryHandle(size); + } + return (*localMem)->getBuf(); +} + +ConvProjection::~ConvProjection() { + hl_destroy_tensor_descriptor(inputDesc_); + hl_destroy_tensor_descriptor(outputDesc_); + hl_destroy_filter_descriptor(filterDesc_); + hl_destroy_convolution_descriptor(convDesc_); +} + +} // namespace paddle diff --git a/paddle/gserver/layers/ConvProjection.h b/paddle/gserver/layers/ConvProjection.h new file mode 100644 index 0000000000000..41a100ac3c50f --- /dev/null +++ b/paddle/gserver/layers/ConvProjection.h @@ -0,0 +1,125 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#pragma once + +#include "Projection.h" + +namespace paddle { + +/** + * @brief Convolution projection do the same calculation with CudnnConvLayer. + */ +class ConvProjection : public Projection { +public: + /** + * Constructor. + */ + ConvProjection(const ProjectionConfig& config, ParameterPtr parameter, + bool useGpu); + + ~ConvProjection(); + + virtual void forward(); + virtual void backward(const UpdateCallback& callback); + +protected: + void getConvParams(); + void initCudnn(); + + void reshapeTensorDesc(int batchSize); + void reshape(int batchSize); + + int outputSize(int imageSize, int filterSize, int padding, int stride) { + return (imageSize - filterSize + 2 * padding) / stride + 1; + } + + size_t calOutputSize() { + imageH_ = in_->getFrameHeight(); + imageW_ = in_->getFrameWidth(); + if (imageH_ == 0) imageH_ = configImgH_; + if (imageW_ == 0) imageW_ = configImgW_; + outputH_ = outputSize(imageH_, filterH_, paddingH_, strideH_); + outputW_ = outputSize(imageW_, filterW_, paddingW_, strideW_); + + const_cast(out_)->setFrameHeight(outputH_); + const_cast(out_)->setFrameWidth(outputW_); + + inputOffset_ = (channels_ / groups_) * imageH_ * imageW_; + outputOffset_ = (numFilters_ / groups_) * outputH_ * outputW_; + return outputH_ * outputW_ * numFilters_; + } + + static void* getSpaceBytes(size_t size); + + /// imageH_ and imageW_ is calculated from the input layer. + int imageH_, imageW_; + /// configImgH_ and configImgW_ is obtained from config. + int configImgH_, configImgW_; + int outputH_, outputW_; + int channels_, numFilters_; + int paddingH_, paddingW_; + int strideH_, strideW_; + int filterH_, filterW_; + /// One group offset of input data. + int inputOffset_; + /// One group offset of output data. + int outputOffset_; + /// One group offset of weight. + int weightOffset_; + int groups_; + + /// Cudnn tensor descriptor for input. + hl_tensor_descriptor inputDesc_; + /// Cudnn tensor descriptor for output. + hl_tensor_descriptor outputDesc_; + /// Cudnn tensor descriptor for filter. + hl_filter_descriptor filterDesc_; + /// Cudnn tensor descriptor for a convolution operation. + hl_convolution_descriptor convDesc_; + + /// Record the algorithm for forward convolution, which is obtained by cudnn + /// api to search the best suited algorithm. + int fwdAlgo_; + /// Record the algorithm for computing convolution gradient with respect to + /// filter coefficients. + int bwdFilterAlgo_; + /// Record the algorithm for computing convolution gradient with respect to + /// the output. + int bwdDataAlgo_; + /// Amount of GPU memory needed as workspace to be able to execute a + /// forward convolution with the specified algo. + size_t fwdLimitBytes_; + /// Amount of GPU memory needed as workspace to be able to execute a + /// backwardFilter with the specified algo. + size_t bwdDataLimitBytes_; + /// Amount of GPU memory needed as workspace to be able to execute a + /// backwardData with the specified algo. + size_t bwdFilterLimitBytes_; + /// Size of total work space. + size_t workSpaceInBytes_; + + /// Whether to call cuDNN api to choose conv algorithm. + bool isSelectAlgo_; + /// batchNum is used to record batch size. If the batch size is changed, + /// the selection algorithm will be called. + int batchNum_; + bool bias_; + + std::unique_ptr weight_; + static ThreadLocalD> convMem_; +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/CudnnConvLayer.cpp b/paddle/gserver/layers/CudnnConvLayer.cpp index 0f932f960f6ba..e77216f17c3be 100644 --- a/paddle/gserver/layers/CudnnConvLayer.cpp +++ b/paddle/gserver/layers/CudnnConvLayer.cpp @@ -22,215 +22,64 @@ REGISTER_LAYER(cudnn_conv, CudnnConvLayer); bool CudnnConvLayer::init(const LayerMap &layerMap, const ParameterMap ¶meterMap) { - ConvBaseLayer::init(layerMap, parameterMap); + if (!ConvBaseLayer::init(layerMap, parameterMap)) return false; CHECK(useGpu_) << "CudnnConvLayer only support gpu"; - maxGroups_ = 0; - for (size_t i = 0; i < inputLayers_.size(); i++) { - CHECK_EQ(channels_[i] % groups_[i], 0); - CHECK_EQ(numFilters_ % groups_[i], 0); - - hl_filter_descriptor filter; - hl_create_filter_descriptor(&filter, channels_[i] / groups_[i], - numFilters_ / groups_[i], filterSizeY_[i], - filterSize_[i]); - filterDesc_.push_back(filter); - - hl_tensor_descriptor input; - hl_create_tensor_descriptor(&input); - inputDesc_.push_back(input); - - hl_tensor_descriptor output; - int outputX = - outputSize(imgSize_[i], filterSize_[i], padding_[i], stride_[i]); - CHECK_EQ(outputX, outputX_[i]); - hl_create_tensor_descriptor(&output); - outputDesc_.push_back(output); + CHECK_EQ(inputLayers_.size(), parameters_.size()); + projections_.reserve(inputLayers_.size()); + projConf_.reserve(inputLayers_.size()); - hl_convolution_descriptor conv; - hl_create_convolution_descriptor(&conv, input, filter, paddingY_[i], - padding_[i], strideY_[i], stride_[i]); - convDesc_.push_back(conv); - - weightOffset_.push_back((numFilters_ / groups_[i]) * - (channels_[i] / groups_[i]) * filterPixels_[i]); - inputOffset_.push_back((channels_[i] / groups_[i]) * imgSize_[i] * - imgSize_[i]); - outputOffset_.push_back((numFilters_ / groups_[i]) * outputX_[i] * - outputX_[i]); - - // initialize all to default algorithms - fwdAlgo_.push_back(0); - bwdFilterAlgo_.push_back(0); - bwdDataAlgo_.push_back(0); - fwdLimitBytes_.push_back(0); - bwdFilterLimitBytes_.push_back(0); - bwdDataLimitBytes_.push_back(0); - - // cudnn streams per group equal to 1 - if (groups_[i] > maxGroups_) { - maxGroups_ = groups_[i]; - } - } - - workSpaceInBytes_ = 0; - workSpaceData_ = NULL; - for (int i = 0; i < maxGroups_; ++i) { - workSpace_.push_back(NULL); + numFilters_ = config_.num_filters(); + CHECK(config_.shared_biases()); + for (size_t i = 0; i < inputLayers_.size(); i++) { + ProjectionConfig* conf = new ProjectionConfig(); + conf->set_type("conv"); + conf->set_num_filters(numFilters_); + conf->set_allocated_conv_conf( + config_.mutable_inputs(i)->mutable_conv_conf()); + conf->set_input_size(getPrev(i)->getSize()); + conf->set_output_size(getSize()); + projConf_.emplace_back(conf); + projections_.emplace_back(Projection::create(*projConf_[i], + parameters_[i], useGpu_)); } if (biases_.get() && sharedBiases_) { hl_create_tensor_descriptor(&biasDesc_); + hl_create_tensor_descriptor(&outputDesc_); hl_tensor_reshape(biasDesc_, 1, numFilters_ / groups_[0], 1, 1); biasOffset_ = numFilters_ / groups_[0]; } - batchNum_ = 0; - isSelectAlgo_ = false; return true; } -void CudnnConvLayer::allocConvWorkSpace(size_t maxWorkSpace) { - size_t totalWorkSpace = maxWorkSpace * maxGroups_; - - if (totalWorkSpace > workSpaceInBytes_) { - if (workSpaceInBytes_ != 0) { - hl_free_mem_device(workSpaceData_); - } - // total amount of storage needed over all groups - workSpaceData_ = hl_malloc_device(totalWorkSpace); - - // update work space address for each group - for (int i = 0; i < maxGroups_; ++i) { - workSpace_[i] = reinterpret_cast(workSpaceData_) - + i * maxWorkSpace; - } - workSpaceInBytes_ = totalWorkSpace; - } -} - -void CudnnConvLayer::reshape(int batchSize) { - CHECK_NE(inputLayers_.size(), 0UL); - imageH_ = inputLayers_[0]->getOutput().getFrameHeight(); - imageW_ = inputLayers_[0]->getOutput().getFrameWidth(); - if (imageH_ == 0) imageH_ = imgSize_[0]; - if (imageW_ == 0) imageW_ = imgSize_[0]; - - for (size_t i = 1; i < inputLayers_.size(); i++) { - int imageH = inputLayers_[i]->getOutput().getFrameHeight(); - int imageW = inputLayers_[i]->getOutput().getFrameWidth(); - if (imageH) { - CHECK_EQ(imageH_, imageH) << "Inputs must have same height."; - } - if (imageW) { - CHECK_EQ(imageW_, imageW) << "Inputs must have same width."; - } - } - - outputH_ = outputSize(imageH_, filterSizeY_[0], paddingY_[0], strideY_[0]); - outputW_ = outputSize(imageW_, filterSize_[0], padding_[0], stride_[0]); - // check outputH & outputW - getOutput().setFrameHeight(outputH_); - getOutput().setFrameWidth(outputW_); - - // if the batchSize remains the same, set isSelectAlgo_ true. - // Otherwise, set isSelectAlgo_ false and select algo again. - isSelectAlgo_ = (batchSize == batchNum_); - batchNum_ = batchSize; - - size_t maxWorkSpace = 0; - for (size_t i = 0; i < inputLayers_.size(); i++) { - CHECK_EQ(inputLayers_[i]->getOutput().value->getWidth(), - (size_t)(channels_[i] * imageH_ * imageW_)); - - hl_tensor_reshape(inputDesc_[i], batchSize, channels_[i] / groups_[i], - imageH_, imageW_, channels_[i] * imageH_ * imageW_, - imageH_ * imageW_, imageW_, 1); - - hl_tensor_reshape(outputDesc_[i], batchSize, numFilters_ / groups_[i], - outputH_, outputW_, numFilters_ * outputH_ * outputW_, - outputH_ * outputW_, outputW_, 1); - - hl_reset_convolution_descriptor(convDesc_[i], inputDesc_[i], - filterDesc_[i], paddingY_[i], - padding_[i], strideY_[i], stride_[i]); - - inputOffset_[i] = (channels_[i] / groups_[i]) * imageH_ * imageW_; - outputOffset_[i] = (numFilters_ / groups_[i]) * outputH_ * outputW_; - - if (!isSelectAlgo_) { - hl_conv_workspace(inputDesc_[i], outputDesc_[i], filterDesc_[i], - convDesc_[i], &fwdAlgo_[i], &fwdLimitBytes_[i], - &bwdDataAlgo_[i], &bwdDataLimitBytes_[i], - &bwdFilterAlgo_[i], &bwdFilterLimitBytes_[i]); - - maxWorkSpace = std::max(fwdLimitBytes_[i], bwdDataLimitBytes_[i]); - maxWorkSpace = std::max(maxWorkSpace, bwdFilterLimitBytes_[i]); - - VLOG(3) << getName() << " Fwd / BwdData / BwdFilter algo: " << fwdAlgo_[i] - << " / " << bwdDataAlgo_[i] - << " / " << bwdFilterAlgo_[i]; - } - } - - if (!isSelectAlgo_) { - allocConvWorkSpace(maxWorkSpace); - } - - isSelectAlgo_ = true; -} - void CudnnConvLayer::forward(PassType passType) { Layer::forward(passType); - int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); - reshape(batchSize); - resetOutput(batchSize, outputH_ * outputW_ * numFilters_); + + int batchSize = getInput(0).getBatchSize(); + resetOutput(batchSize, calOutputSize()); for (size_t i = 0; i != inputLayers_.size(); ++i) { - REGISTER_TIMER_INFO("CudnnConvFwTimer", getName().c_str()); - for (int g = 0; g < groups_[i]; ++g) { - real *inputData = getInputValue(i)->getData() + inputOffset_[i] * g; - real *wgtData = weights_[i]->getW()->getData() + weightOffset_[i] * g; - real *outData = getOutputValue()->getData() + outputOffset_[i] * g; - hl_convolution_forward(inputDesc_[i], inputData, outputDesc_[i], - outData, filterDesc_[i], wgtData, - convDesc_[i], workSpace_[g], - fwdLimitBytes_[i], fwdAlgo_[i]); - } + projections_[i]->forward(&getInput(i), &getOutput(), passType); } if (biases_) { REGISTER_TIMER_INFO("CudnnConvBiasTimer", getName().c_str()); - addBiases(); - } - - forwardActivation(); -} - -void CudnnConvLayer::addBiases() { - if (sharedBiases_) { + int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); + hl_tensor_reshape(outputDesc_, batchSize, numFilters_ / groups_[0], + outputH_[0], outputW_[0], numFilters_ * outputH_[0] * outputW_[0], + outputH_[0] * outputW_[0], outputW_[0], 1); + outputOffset_ = getOutputValue()->getWidth() / groups_[0]; for (int g = 0; g < groups_[0]; ++g) { real *biasData = biases_->getW()->getData() + biasOffset_ * g; - real *outData = getOutputValue()->getData() + outputOffset_[0] * g; + real *outData = getOutputValue()->getData() + outputOffset_ * g; hl_convolution_forward_add_bias(biasDesc_, biasData, - outputDesc_[0], outData); + outputDesc_, outData); } - } else { - LOG(FATAL) << "Not supported"; } -} -void CudnnConvLayer::bpropBiases() { - if (sharedBiases_) { - for (int g = 0; g < groups_[0]; ++g) { - real *biasGrad = biases_->getWGrad()->getData() + biasOffset_ * g; - real *outGrad = getOutputGrad()->getData() + outputOffset_[0] * g; - hl_convolution_backward_bias(biasDesc_, biasGrad, - outputDesc_[0], outGrad); - } - } else { - LOG(FATAL) << "Not supported"; - } + forwardActivation(); } void CudnnConvLayer::backward(const UpdateCallback &callback) { @@ -238,52 +87,23 @@ void CudnnConvLayer::backward(const UpdateCallback &callback) { if (biases_ && biases_->getWGrad()) { REGISTER_TIMER_INFO("CudnnConvBpBiasTimer", getName().c_str()); - bpropBiases(); + for (int g = 0; g < groups_[0]; ++g) { + real *biasGrad = biases_->getWGrad()->getData() + biasOffset_ * g; + real *outGrad = getOutputGrad()->getData() + outputOffset_ * g; + hl_convolution_backward_bias(biasDesc_, biasGrad, outputDesc_, outGrad); + } biases_->getParameterPtr()->incUpdate(callback); } for (size_t i = 0; i != inputLayers_.size(); ++i) { - REGISTER_TIMER_INFO("CudnnConvBpTimer", getName().c_str()); - for (int g = 0; g < groups_[i]; ++g) { - real *outGrad = getOutputGrad()->getData() + outputOffset_[i] * g; - if (weights_[i]->getWGrad()) { - real *inputData = getInputValue(i)->getData() + inputOffset_[i] * g; - real *weightGrad = - weights_[i]->getWGrad()->getData() + weightOffset_[i] * g; - hl_convolution_backward_filter( - inputDesc_[i], inputData, outputDesc_[i], outGrad, filterDesc_[i], - weightGrad, convDesc_[i], workSpace_[g], bwdFilterLimitBytes_[i], - bwdFilterAlgo_[i]); - } - - MatrixPtr preGrad = getInputGrad(i); - if (NULL != preGrad) { - real *inputGrad = preGrad->getData() + inputOffset_[i] * g; - real *wgtData = weights_[i]->getW()->getData() + weightOffset_[i] * g; - hl_convolution_backward_data( - inputDesc_[i], inputGrad, outputDesc_[i], outGrad, filterDesc_[i], - wgtData, convDesc_[i], workSpace_[g], bwdDataLimitBytes_[i], - bwdDataAlgo_[i]); - } - } - weights_[i]->getParameterPtr()->incUpdate(callback); + projections_[i]->backward(callback); } } CudnnConvLayer::~CudnnConvLayer() { - if (biasDesc_) { + if (biases_) { hl_destroy_tensor_descriptor(biasDesc_); - } - - for (size_t i = 0; i < inputDesc_.size(); i++) { - hl_destroy_tensor_descriptor(inputDesc_[i]); - hl_destroy_tensor_descriptor(outputDesc_[i]); - hl_destroy_filter_descriptor(filterDesc_[i]); - hl_destroy_convolution_descriptor(convDesc_[i]); - } - if (workSpaceInBytes_ != 0) { - hl_free_mem_device(workSpaceData_); - workSpaceInBytes_ = 0; + hl_destroy_tensor_descriptor(outputDesc_); } } diff --git a/paddle/gserver/layers/CudnnConvLayer.h b/paddle/gserver/layers/CudnnConvLayer.h index a6dadba10daa4..6390d96315cc4 100644 --- a/paddle/gserver/layers/CudnnConvLayer.h +++ b/paddle/gserver/layers/CudnnConvLayer.h @@ -17,12 +17,13 @@ limitations under the License. */ #include "ConvBaseLayer.h" #include "paddle/math/Matrix.h" +#include "Projection.h" #include namespace paddle { /** - * @brief A subclass of ConvBaseLayer by cuDNN implementation. It only + * @brief A 2-dimension conv layer implemented by cuDNN. It only * supports GPU mode. We automatic select CudnnConvLayer for GPU * mode and ExpandConvLayer for CPU mode if you set type of "conv". * User also can specfiy type of "exconv" or "cudnn_conv" for @@ -31,81 +32,21 @@ namespace paddle { * The config file api is img_conv_layer. */ class CudnnConvLayer : public ConvBaseLayer { -private: - /// resize Cudnn workspace size - void allocConvWorkSpace(size_t maxWorkSpace); - protected: - int imageH_, imageW_, outputH_, outputW_; - /// Cudnn tensor descriptor for bias. + std::vector> projConf_; + std::vector> projections_; + hl_tensor_descriptor biasDesc_; - /// Cudnn tensor descriptor for input. - std::vector inputDesc_; - /// Cudnn tensor descriptor for output. - std::vector outputDesc_; - /// Cudnn tensor descriptor for filter. - std::vector filterDesc_; - /// Cudnn tensor descriptor for a convolution operation. - std::vector convDesc_; - /// One sample offset of input data. - IntV inputOffset_; - /// One sample offset of output data. - IntV outputOffset_; - /// One group offset of weight. - IntV weightOffset_; - /// One group offset of bias. + hl_tensor_descriptor outputDesc_; int biasOffset_; - - /// Save the algorithm for forward convolution, which is obtained by cudnn - /// api to search the best suited algorithm. - std::vector fwdAlgo_; - /// Save the algorithm for computing convolution gradient with respect to - /// filter coefficients. - std::vector bwdFilterAlgo_; - /// Save the algorithm for computing convolution gradient with respect to - /// the output. - std::vector bwdDataAlgo_; - /// Amount of GPU memory needed as workspace to be able to execute a - /// forward convolution with the specified algo. - std::vector fwdLimitBytes_; - /// Amount of GPU memory needed as workspace to be able to execute a - /// backwardFilter with the specified algo. - std::vector bwdFilterLimitBytes_; - /// Amount of GPU memory needed as workspace to be able to execute a - /// backwardData with the specified algo. - std::vector bwdDataLimitBytes_; - - /// Device work space address for each group. - std::vector workSpace_; - /// Max number of groups. - int maxGroups_; - /// Total work space address in device for all groups. - void* workSpaceData_; - /// Size of total work space. - size_t workSpaceInBytes_; - - /// Is or not select conv algorihtm. - bool isSelectAlgo_; - - /// batchNum is used to record batch size. If the batch size is changed, - /// the selection algorithm will be called. - int batchNum_; + int outputOffset_; public: explicit CudnnConvLayer(const LayerConfig& config) : ConvBaseLayer(config) {} ~CudnnConvLayer(); - /** - * Intialization. Initialize member variables and create tenor descriptor. - */ bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); - /** - * Reshape is done each forward. Reshape tensor decriptor - * inputDesc_, outputDesc_, convDesc_. And search the faster algo - * or the fastest algo within a given memeory limit. - */ - void reshape(int batchSize); void forward(PassType passType); void backward(const UpdateCallback& callback); void addBiases(); diff --git a/paddle/gserver/layers/ExpandConvLayer.cpp b/paddle/gserver/layers/ExpandConvLayer.cpp index df79c3e3037cf..80a6a62b5c0de 100644 --- a/paddle/gserver/layers/ExpandConvLayer.cpp +++ b/paddle/gserver/layers/ExpandConvLayer.cpp @@ -37,32 +37,29 @@ bool ExpandConvLayer::init(const LayerMap &layerMap, caffeMode_ = conf.caffe_mode(); } + /* initialize the weightList */ + CHECK(inputLayers_.size() == parameters_.size()); + for (size_t i = 0; i < inputLayers_.size(); i++) { + size_t height, width; + height = filterPixels_[i] * filterChannels_[i]; + width = numFilters_; + + // create a new weight + CHECK_EQ(parameters_[i]->getSize(), width * height); + Weight* w = new Weight(height, width, parameters_[i]); + weights_.emplace_back(w); + } + return true; } -size_t ExpandConvLayer::getSize() { +size_t ExpandConvLayer::getOutputSize() { CHECK_NE(inputLayers_.size(), 0UL); - imgSizeH_.clear(); - imgSizeW_.clear(); - outputH_.clear(); - outputW_.clear(); + size_t layerSize = ConvBaseLayer::calOutputSize(); subN_.clear(); - size_t layerSize = 0; for (size_t i = 0; i < inputLayers_.size(); i++) { - imgSizeH_.push_back(inputLayers_[i]->getOutput().getFrameHeight()); - imgSizeW_.push_back(inputLayers_[i]->getOutput().getFrameWidth()); - if (imgSizeH_[i] == 0) imgSizeH_[i] = imgSize_[i]; - if (imgSizeW_[i] == 0) imgSizeW_[i] = imgSize_[i]; - outputH_.push_back( - outputSize(imgSizeH_[i], filterSize_[i], padding_[i], stride_[i])); - outputW_.push_back( - outputSize(imgSizeW_[i], filterSize_[i], padding_[i], stride_[i])); subN_.push_back(outputH_[i] * outputW_[i]); - CHECK(layerSize == 0 || subN_[i] * size_t(numFilters_) == layerSize); - layerSize = subN_[i] * numFilters_; } - getOutput().setFrameHeight(outputH_[0]); - getOutput().setFrameWidth(outputW_[0]); return layerSize; } @@ -119,7 +116,7 @@ void ExpandConvLayer::expandFwdOnce(MatrixPtr image, int inIdx, int startIdx) { } void ExpandConvLayer::addSharedBias() { - size_t mapW = getSize() / numFilters_; + size_t mapW = getOutputValue()->getWidth() / numFilters_; size_t mapH = getOutputValue()->getElementCnt() / mapW; MatrixPtr out = Matrix::create(getOutputValue()->getData(), mapH, mapW, false, useGpu_); @@ -158,7 +155,7 @@ void ExpandConvLayer::forward(PassType passType) { * transOutValue correspond sample to one row */ int batchSize = inputLayers_[0]->getOutputValue()->getWidth(); batchSize = inputLayers_[0]->getOutputValue()->getHeight(); - resetOutput(batchSize, getSize()); + resetOutput(batchSize, getOutputSize()); MatrixPtr image = nullptr; for (size_t i = 0; i != inputLayers_.size(); ++i) { @@ -183,7 +180,7 @@ void ExpandConvLayer::forward(PassType passType) { } void ExpandConvLayer::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { - size_t mapW = getSize() / numFilters_; + size_t mapW = v->getWidth() / numFilters_; size_t mapH = v->getElementCnt() / mapW; MatrixPtr vTmp = Matrix::create(v->getData(), mapH, mapW, false, useGpu_); diff --git a/paddle/gserver/layers/ExpandConvLayer.h b/paddle/gserver/layers/ExpandConvLayer.h index fc3d69b1b7d14..030a3ba397ff4 100644 --- a/paddle/gserver/layers/ExpandConvLayer.h +++ b/paddle/gserver/layers/ExpandConvLayer.h @@ -37,14 +37,6 @@ class ExpandConvLayer : public ConvBaseLayer { IntV subN_; /// subK_ = channels_ * filterPixels_ * groups_. IntV subK_; - /// The spatial dimensions of height of input feature map. - IntV imgSizeH_; - /// The spatial dimensions of width of input feature map. - IntV imgSizeW_; - /// The spatial dimensions of height of output feature map. - IntV outputH_; - /// The spatial dimensions of width of output feature map. - IntV outputW_; /// Expand one sample at a time. shape: /// (numChannels * filterPixels_, outputSizeH * outputSizeW) MatrixPtr expandInput_; @@ -58,7 +50,7 @@ class ExpandConvLayer : public ConvBaseLayer { bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); - size_t getSize(); + size_t getOutputSize(); /** * Create or resize expandInput_. diff --git a/paddle/gserver/layers/MixedLayer.cpp b/paddle/gserver/layers/MixedLayer.cpp index 054ddd3a228ed..26b1360290ffb 100644 --- a/paddle/gserver/layers/MixedLayer.cpp +++ b/paddle/gserver/layers/MixedLayer.cpp @@ -41,9 +41,13 @@ bool MixedLayer::init(const LayerMap& layerMap, } operators_.emplace_back(Operator::create(operator_conf, useGpu_)); } + /* initialize biases_ */ if (biasParameter_.get() != NULL) { - biases_ = std::unique_ptr(new Weight(1, getSize(), biasParameter_)); + sharedBias_ = config_.shared_biases(); + size_t psize = config_.bias_size(); + biases_ = std::unique_ptr( + new Weight(1, psize, biasParameter_)); } return true; @@ -119,12 +123,6 @@ void MixedLayer::forward(PassType passType) { MatrixPtr outV = getOutputValue(); - /* add the bias-vector */ - if (biases_.get() != NULL) { - REGISTER_TIMER_INFO("FwBiasTimer", getName().c_str()); - outV->addBias(*(biases_->getW()), 1); - } - for (size_t i = 0; i != inputLayers_.size(); ++i) { if (projections_[i]) { projections_[i]->forward(&getInput(i), &output_, passType); @@ -140,6 +138,12 @@ void MixedLayer::forward(PassType passType) { op->forward(ins, &output_, passType); } + /* add the bias-vector */ + if (biases_.get() != NULL) { + REGISTER_TIMER_INFO("FwBiasTimer", getName().c_str()); + outV->addBias(*(biases_->getW()), 1, sharedBias_); + } + /* activation */ { REGISTER_TIMER_INFO("FwAtvTimer", getName().c_str()); forwardActivation(); @@ -154,7 +158,7 @@ void MixedLayer::backward(const UpdateCallback& callback) { if (biases_ && biases_->getWGrad()) { REGISTER_TIMER_INFO("BpBiasTimer", getName().c_str()); - biases_->getWGrad()->collectBias(*getOutputGrad(), 1); + biases_->getWGrad()->collectBias(*getOutputGrad(), 1, sharedBias_); /* Increasing the number of gradient */ biases_->getParameterPtr()->incUpdate(callback); diff --git a/paddle/gserver/layers/MixedLayer.h b/paddle/gserver/layers/MixedLayer.h index 9bac1355bd21f..5842e51e1d79d 100644 --- a/paddle/gserver/layers/MixedLayer.h +++ b/paddle/gserver/layers/MixedLayer.h @@ -58,5 +58,6 @@ class MixedLayer : public Layer { /// the matrix size of projection state std::vector projectionStateMatrixSize_; std::unique_ptr biases_; + bool sharedBias_; }; } // namespace paddle diff --git a/paddle/gserver/tests/LayerGradUtil.cpp b/paddle/gserver/tests/LayerGradUtil.cpp index 552a6c5b41c7f..bc7bee0e4bbc8 100644 --- a/paddle/gserver/tests/LayerGradUtil.cpp +++ b/paddle/gserver/tests/LayerGradUtil.cpp @@ -669,12 +669,14 @@ void testLayerGrad(TestConfig testConf, string testLayerName, size_t batchSize, void testProjectionGrad(ProjectionConfig conf, InputType inputType, size_t parameterSize, size_t batchSize, bool useGpu, - bool testState) { + bool testState, int biasSize, bool sharedBias) { TestConfig config; conf.set_name(conf.type()); config.layerConfig.set_type("mixed"); config.layerConfig.set_size(conf.output_size()); - config.biasSize = config.layerConfig.size(); + config.biasSize = biasSize == 0 ? config.layerConfig.size() : biasSize; + config.layerConfig.set_bias_size(config.biasSize); + config.layerConfig.set_shared_biases(sharedBias); config.inputDefs.push_back( {inputType, "layer_0", conf.input_size(), parameterSize}); *config.layerConfig.add_inputs()->mutable_proj_conf() = conf; diff --git a/paddle/gserver/tests/LayerGradUtil.h b/paddle/gserver/tests/LayerGradUtil.h index 1e608dc0620ab..3b9ec803959b3 100644 --- a/paddle/gserver/tests/LayerGradUtil.h +++ b/paddle/gserver/tests/LayerGradUtil.h @@ -217,7 +217,8 @@ void testLayerGrad(TestConfig testConf, string testLayerName, size_t batchSize, void testProjectionGrad(ProjectionConfig conf, InputType inputType, size_t parameterSize, size_t batchSize, bool useGpu, - bool testState = false); + bool testState = false, int biasSize = 0, + bool sharedBias = false); void testOperatorGrad(TestConfig& config, OperatorConfig& operatorConf, size_t batchSize, bool useGpu, bool testState = false); diff --git a/paddle/gserver/tests/img_conv_a.conf b/paddle/gserver/tests/img_conv_a.conf new file mode 100644 index 0000000000000..940589ed9ac24 --- /dev/null +++ b/paddle/gserver/tests/img_conv_a.conf @@ -0,0 +1,39 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +settings(batch_size=10) +data = data_layer(name ="input", size=8*16*16) +conv1 = img_conv_layer(input=data, filter_size=1, filter_size_y=1, + num_channels=8, + num_filters=16, stride=1, + bias_attr=False, + act=ReluActivation()) +conv2 = img_conv_layer(input=data, filter_size=1, filter_size_y=1, + num_channels=8, + num_filters=16, stride=1, + bias_attr=False, + act=ReluActivation()) + +concat = concat_layer(input=[conv1, conv2]) + +conv = img_conv_layer(input=data, filter_size=1, filter_size_y=1, + num_channels=8, + num_filters=16, stride=1, + bias_attr=True, + act=LinearActivation()) + +outputs(concat, conv) diff --git a/paddle/gserver/tests/img_conv_b.conf b/paddle/gserver/tests/img_conv_b.conf new file mode 100644 index 0000000000000..8ca9c94541504 --- /dev/null +++ b/paddle/gserver/tests/img_conv_b.conf @@ -0,0 +1,32 @@ +#edit-mode: -*- python -*- +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle.trainer_config_helpers import * + +settings(batch_size=10) +data = data_layer(name ="input", size=8*16*16) +proj1 = conv_projection(input=data, filter_size=1, filter_size_y=1, + num_channels=8, num_filters=16, stride=1) +proj2 = conv_projection(input=data, filter_size=1, filter_size_y=1, + num_channels=8, num_filters=16, stride=1) +concat = concat_layer(input=[proj1, proj2], bias_attr=False, act=ReluActivation()) + +proj = conv_projection(input=data, filter_size=1, filter_size_y=1, + num_channels=8, num_filters=16, stride=1) + +with mixed_layer(bias_attr=True, act=LinearActivation()) as conv: + conv += proj + +outputs(concat, conv) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index eab9bf84141a2..bf2c2e0499941 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -134,6 +134,45 @@ TEST(Projection, identity) { } } + +#ifndef PADDLE_ONLY_CPU +TEST(Projection, conv) { + const int NUM_FILTERS = 16; + const int FILTER_SIZE = 2; + const int FILTER_SIZE_Y = 3; + const int CHANNELS = 3; + const int IMAGE_SIZE = 16; + + ProjectionConfig conf; + conf.set_type("conv"); + conf.set_num_filters(NUM_FILTERS); + + ConvConfig* conv = conf.mutable_conv_conf(); + conv->set_filter_size(FILTER_SIZE); + conv->set_filter_size_y(FILTER_SIZE_Y); + conv->set_channels(CHANNELS); + conv->set_padding(0); + conv->set_padding_y(1); + conv->set_stride(2); + conv->set_stride_y(2); + conv->set_groups(1); + conv->set_filter_channels(conv->channels() / conv->groups()); + conv->set_img_size(IMAGE_SIZE); + int outputSize = (2 * conv->padding() + conv->img_size() - + conv->filter_size()) / conv->stride() + 1; + int outputSizeY = (2 * conv->padding_y() + conv->img_size() - + conv->filter_size_y()) / conv->stride_y() + 1; + conv->set_output_x(outputSize); + conf.set_input_size(IMAGE_SIZE * IMAGE_SIZE * CHANNELS); + conf.set_output_size(outputSize * outputSizeY * NUM_FILTERS); + + testProjectionGrad(conf, INPUT_DATA, + /* parameterSize */ NUM_FILTERS * CHANNELS * FILTER_SIZE * FILTER_SIZE_Y, + /* batchSize */ 100, true, false, NUM_FILTERS, true); +} +#endif + + TEST(Layer, concat) { TestConfig config; config.biasSize = 0; diff --git a/paddle/gserver/tests/test_NetworkCompare.cpp b/paddle/gserver/tests/test_NetworkCompare.cpp index b3ef53067301b..8d3eac5aca8d1 100644 --- a/paddle/gserver/tests/test_NetworkCompare.cpp +++ b/paddle/gserver/tests/test_NetworkCompare.cpp @@ -236,6 +236,15 @@ TEST(Compare, img_pool) { compareNetwork(config_file_a, config_file_b); FLAGS_use_gpu = useGpu; } + +TEST(Compare, img_conv) { + std::string config_file_a = "./gserver/tests/img_conv_a.conf"; + std::string config_file_b = "./gserver/tests/img_conv_b.conf"; + bool useGpu = FLAGS_use_gpu; + FLAGS_use_gpu = true; + compareNetwork(config_file_a, config_file_b); + FLAGS_use_gpu = useGpu; +} #endif diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 843eabc97d642..aaeae98f0d28b 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -340,6 +340,15 @@ void GpuMatrix::addBias(Matrix& b, real scale) { BaseMatrix::addBias(b, scale); } +void GpuMatrix::addSharedBias(Matrix& b, real scale) { + CHECK(b.getHeight() == 1) << "the Bias should be a vector"; + CHECK_LE(b.getWidth(), getWidth()); + CHECK_EQ(getWidth() % b.getWidth(), 0UL); + hl_matrix_add_shared_bias(getData(), b.getData(), b.getWidth(), + getHeight(), getWidth(), scale); +} + + void GpuMatrix::collectBias(Matrix& a, real scale) { CHECK_EQ(getHeight(), (size_t)1); CHECK_EQ(width_, a.getWidth()); @@ -354,6 +363,14 @@ void GpuMatrix::collectBias(Matrix& a, real scale) { } } +void GpuMatrix::collectSharedBias(Matrix& a, real scale) { + CHECK_EQ(getHeight(), (size_t)1); + CHECK_EQ(a.getWidth() % getWidth(), 0UL); + hl_matrix_collect_shared_bias(getData(), a.getData(), getWidth(), + a.getHeight(), a.getWidth(), scale); +} + + void GpuMatrix::sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode) { @@ -1983,6 +2000,24 @@ void CpuMatrix::addBias(Matrix& b, real scale) { } } +void CpuMatrix::addSharedBias(Matrix& b, real scale) { + CHECK_EQ(b.getHeight(), (size_t)1); + real* aData = getData(); + real* bData = b.getData(); + size_t numSamples = getHeight(); + size_t channel = b.getWidth(); + CHECK_EQ(getWidth() % channel, 0UL); + size_t dim = getWidth() / channel; + + for (size_t i = 0; i < numSamples; i++) { + for (size_t c = 0; c < channel; c++) { + for (size_t j = 0; j < dim; j++) { + aData[i * getStride() + c * dim + j] += scale * bData[c]; + } + } + } +} + void CpuMatrix::collectBias(Matrix& a, real scale) { CHECK_EQ(getHeight(), (size_t)1); CHECK_EQ(width_, a.getWidth()); @@ -2000,6 +2035,23 @@ void CpuMatrix::collectBias(Matrix& a, real scale) { } } +void CpuMatrix::collectSharedBias(Matrix& a, real scale) { + CHECK_EQ(getHeight(), (size_t)1); + real* B = getData(); + real* A = a.getData(); + size_t numSamples = a.getHeight(); + size_t channel = getWidth(); + CHECK_EQ(a.getWidth() % channel, 0UL); + size_t dim = a.getWidth() / channel; + for (size_t i = 0; i < numSamples; i++) { + for (size_t c = 0; c < channel; c++) { + for (size_t j = 0; j < dim; j++) { + B[c] += scale * A[i * channel * dim + c * dim + j]; + } + } + } +} + void CpuMatrix::sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode) { diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index 047c76a8604cc..52cbed528ca8b 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -343,11 +343,35 @@ class Matrix : public BaseMatrix { LOG(FATAL) << "Not implemented"; } + virtual void addSharedBias(Matrix& b, real scale) { + LOG(FATAL) << "Not implemented"; + } + + virtual void addBias(Matrix& b, real scale, bool sharedBias) { + if (!sharedBias) { + addBias(b, scale); + } else { + addSharedBias(b, scale); + } + } + /// add each sample from a to this. virtual void collectBias(Matrix& a, real scale) { LOG(FATAL) << "Not implemented"; } + virtual void collectSharedBias(Matrix& a, real scale) { + LOG(FATAL) << "Not implemented"; + } + + virtual void collectBias(Matrix& a, real scale, bool sharedBias) { + if (!sharedBias) { + collectBias(a, scale); + } else { + collectSharedBias(a, scale); + } + } + virtual void sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode) { LOG(FATAL) << "Not implemented"; @@ -1021,6 +1045,7 @@ class GpuMatrix : public Matrix { /// add b to each sample of this. void addBias(Matrix& b, real scale); + void addSharedBias(Matrix& b, real scale); /** * @code @@ -1028,6 +1053,7 @@ class GpuMatrix : public Matrix { * @endcode */ void collectBias(Matrix& a, real scale); + void collectSharedBias(Matrix& a, real scale); void sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode); @@ -1341,9 +1367,11 @@ class CpuMatrix : public Matrix { public: /// add b to each sample of this. void addBias(Matrix& b, real scale); + void addSharedBias(Matrix& b, real scale); /// add each sample of a to this. void collectBias(Matrix& a, real scale); + void collectSharedBias(Matrix& a, real scale); void sequenceAvgForward(Matrix& a, const IVector& startsPos, int mode); diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index ac160479a9dfc..0ddf7e0dfc386 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -21,6 +21,8 @@ limitations under the License. */ #include "paddle/math/SparseMatrix.h" #include #include "paddle/gserver/tests/TestUtil.h" +#include "paddle/utils/Stat.h" + using namespace paddle; // NOLINT using namespace std; // NOLINT @@ -2071,6 +2073,60 @@ TEST(Matrix, MaxOutFwdBwd) { } } +void testAddSharedBias(int numSamples, int dim, int channel) { + MatrixPtr cpuData = std::make_shared(numSamples, dim); + MatrixPtr gpuData = std::make_shared(numSamples, dim); + + MatrixPtr cpuBias = std::make_shared(1, channel); + MatrixPtr gpuBias = std::make_shared(1, channel); + + cpuData->randomizeUniform(); + gpuData->copyFrom(*cpuData); + cpuBias->randomizeUniform(); + gpuBias->copyFrom(*cpuBias); + + cpuData->addSharedBias(*cpuBias, 1.0); + gpuData->addSharedBias(*gpuBias, 1.0); + + MatrixPtr check = std::make_shared(numSamples, dim); + check->copyFrom(*gpuData); + MatrixCheckErr(*cpuData, *check); +} + +void testCollectSharedBias(int numSamples, int dim, int channel) { + MatrixPtr cpuData = std::make_shared(numSamples, dim); + MatrixPtr gpuData = std::make_shared(numSamples, dim); + + MatrixPtr cpuBias = std::make_shared(1, channel); + MatrixPtr gpuBias = std::make_shared(1, channel); + + cpuData->randomizeUniform(); + gpuData->copyFrom(*cpuData); + cpuBias->randomizeUniform(); + gpuBias->copyFrom(*cpuBias); + + cpuBias->collectSharedBias(*cpuData, 1.0); + gpuBias->collectSharedBias(*gpuData, 1.0); + + MatrixPtr check = std::make_shared(1, channel); + check->copyFrom(*gpuBias); + MatrixCheckErr(*cpuBias, *check); +} + + +TEST(Matrix, sharedBias) { + for (auto numSamples : {1, 100, 520}) { + for (auto dim : {100 * 16, 100 * 32}) { + for (auto channel : {8, 16}) { + VLOG(3) << " numSamples=" << numSamples << " dim=" << dim + << " channel=" << channel; + testAddSharedBias(numSamples, dim, channel); + testCollectSharedBias(numSamples, dim, channel); + } + } + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/paddle/trainer/CMakeLists.txt b/paddle/trainer/CMakeLists.txt index 08b411d2ccbae..06c019f0a9775 100644 --- a/paddle/trainer/CMakeLists.txt +++ b/paddle/trainer/CMakeLists.txt @@ -7,6 +7,7 @@ set(TRAINER_SOURCES Tester.cpp Trainer.cpp TrainerInternal.cpp + TrainerBenchmark.cpp ThreadParameterUpdater.cpp TrainerInternalConfig.cpp TrainerConfigHelper.cpp) diff --git a/paddle/trainer/Trainer.h b/paddle/trainer/Trainer.h index 4f4811a139e74..7762722456c44 100644 --- a/paddle/trainer/Trainer.h +++ b/paddle/trainer/Trainer.h @@ -99,6 +99,7 @@ class Trainer { void startTrainPass(); void finishTrainPass(); void trainOneDataBatch(DataBatch& dataBatch); + void time(); /** * given a dataBatch and the current parameter value diff --git a/paddle/trainer/TrainerBenchmark.cpp b/paddle/trainer/TrainerBenchmark.cpp new file mode 100644 index 0000000000000..54862e95b4a73 --- /dev/null +++ b/paddle/trainer/TrainerBenchmark.cpp @@ -0,0 +1,71 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#undef PADDLE_DISABLE_TIMER + +#include "Trainer.h" +#include "paddle/utils/Stat.h" +#include "paddle/utils/Util.h" + +P_DECLARE_int32(test_period); + +P_DEFINE_bool(feed_data, false, "Wether to read data from DataProvider."); + +namespace paddle { + +void Trainer::time() { + startTrain(); + + trainerInternal_.getParameterUpdater()->startPass(); + evaluator_->start(); + + DataBatch dataBatch; + int32_t batchSize = config_->getOptConfig().batch_size(); + int32_t num = dataProvider_->getNextBatch(batchSize, &dataBatch); + CHECK_EQ(num, batchSize) << "The sample number is less than batch size " + << num << " != " << batchSize; + + CHECK(dataBatch.getSize()) << "No data from data provider"; + + std::vector outputs; + // burning time + LOG(INFO) << "Burning time..."; + for (int n = 0; n < 10; ++n) { + trainerInternal_.trainOneBatch(n, dataBatch, &outputs); + } + LOG(INFO) << "Burning time end."; + + for (int n = 0; n < FLAGS_test_period; n++) { + if (FLAGS_feed_data) { + REGISTER_TIMER("GetData"); + num = dataProvider_->getNextBatch(batchSize, &dataBatch); + } + + if (num != batchSize) { + break; + } + + { + REGISTER_TIMER("FwdBwd"); + trainerInternal_.trainOneBatch(n, dataBatch, &outputs); + } + } + globalStat.setThreadInfo(true); + globalStat.printSegTimerStatus(); + globalStat.reset(); + + finishTrain(); +} + +} // namespace paddle diff --git a/paddle/trainer/TrainerMain.cpp b/paddle/trainer/TrainerMain.cpp index 94266639f94ad..a486cc383ace6 100644 --- a/paddle/trainer/TrainerMain.cpp +++ b/paddle/trainer/TrainerMain.cpp @@ -103,6 +103,8 @@ int main(int argc, char** argv) { trainer.checkGradient(); } else if (FLAGS_job == "test") { trainer.test(); + } else if (FLAGS_job == "time") { + trainer.time(); } else { LOG(FATAL) << "Unknown job type: " << FLAGS_job; } diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index 70c1f8d563238..79e76b6bf1bdd 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -255,7 +255,7 @@ sinclude(`ModelConfigLayer.proto.m4') // (which is how convnets are usually trained). Setting this to // false will untie the biases, yielding a separate bias for // every location at which the filter is applied. - optional bool shared_biases = 8; + optional bool shared_biases = 8 [default = false]; // Valid values are ones that divide the area of the output // grid in this convolutional layer. For example if this layer @@ -379,6 +379,9 @@ sinclude(`ModelConfigLayer.proto.m4') // use to compute moving mean and variance. optional real moving_average_fraction = 47 [default = 0.9]; + + // bias size + optional uint32 bias_size = 48 [default = 0]; } message EvaluatorConfig { diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index fe8a5e5d48767..e9098943165fd 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -632,6 +632,44 @@ def calc_parameter_dims(self, input_size, output_size): _total_pad = 0 +@config_class +class ConvProjection(Projection): + type = 'conv' + + def __init__( + self, + input_layer_name, + num_filters=None, + conv_conf=None, + **xargs): + super(ConvProjection, self).__init__(input_layer_name, **xargs) + + if num_filters is not None: + self.proj_conf.num_filters = num_filters + + parse_conv(conv_conf, + input_layer_name, + self.proj_conf.conv_conf) + # TODO: support rectangle input + self.proj_conf.output_size = (self.proj_conf.conv_conf.output_x ** 2) * num_filters + + def calc_output_size(self, input_layer_config): + return self.proj_conf.output_size + + def calc_parameter_size(self, input_size, output_size): + co = self.proj_conf.num_filters + ci = self.proj_conf.conv_conf.channels + fh = self.proj_conf.conv_conf.filter_size + fw = self.proj_conf.conv_conf.filter_size_y + return co * ci * fh * fw + + def calc_bias_size(self): + return self.proj_conf.num_filters + + def calc_parameter_dims(self, input_size, output_size): + return None + + # Define a operator for mixed layer @config_class class Operator(Cfg): @@ -2528,8 +2566,15 @@ def __init__( record_operator_conf = self.config.operator_confs.add() record_operator_conf.CopyFrom(operator_conf) + psize = self.config.size + if isinstance(self.inputs[0], ConvProjection): + self.config.shared_biases = True + psize = 0 + for input in self.inputs: + psize += input.calc_bias_size() - self.create_bias_parameter(bias, self.config.size) + self.config.bias_size = psize + self.create_bias_parameter(bias, psize) if error_clipping_threshold is not None: self.config.error_clipping_threshold = error_clipping_threshold @@ -2547,8 +2592,10 @@ def __init__( self, name, inputs, + bias=False, **xargs): config_assert(inputs, 'inputs cannot be empty') + config_assert(not bias, 'ConcatenateLayer cannot support bias.') super(ConcatenateLayer, self).__init__( name, 'concat', 0, inputs=inputs, **xargs) size = 0 @@ -2567,10 +2614,19 @@ def __init__( self, name, inputs, + bias=False, **xargs): config_assert(inputs, 'inputs cannot be empty') super(ConcatenateLayer2, self).__init__( name, 'concat2', 0, inputs=inputs, **xargs) + + if isinstance(self.inputs[0], ConvProjection): + for input_index in xrange(len(self.inputs) - 1): + input = self.inputs[input_index + 1] + config_assert(isinstance(input, ConvProjection), + "The first input of ConcatenateLayer2 is ConvProjection, " + "the other inputs should also be ConvProjection.") + size = 0 for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) @@ -2596,6 +2652,16 @@ def __init__( input.proj_conf.output_size) self.create_input_parameter(input_index, psize, dims) + psize = self.config.size + if isinstance(self.inputs[0], ConvProjection): + self.config.shared_biases = True + psize = 0 + for input in self.inputs: + psize += input.calc_bias_size() + + self.config.bias_size = psize + self.create_bias_parameter(bias, psize) + @config_layer('recurrent') class RecurrentLayer(LayerBase): def __init__( diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 7df9108ae82a4..9a23c02431d18 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -34,7 +34,7 @@ "table_projection", "mixed_layer", "data_layer", "embedding_layer", "fc_layer", "grumemory", "pooling_layer", "lstmemory", "last_seq", "first_seq", - "cos_sim", "hsigmoid", + "cos_sim", "hsigmoid", "conv_projection", "regression_cost", 'classification_cost', "LayerOutput", 'img_conv_layer', 'img_pool_layer', 'batch_norm_layer', 'img_cmrnorm_layer', 'addto_layer', @@ -1984,7 +1984,7 @@ def addto_layer(input, act=None, name=None, bias_attr=None, @wrap_act_default(act=IdentityActivation()) @wrap_name_default("concat") @layer_support() -def concat_layer(input, act=None, name=None, layer_attr=None): +def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None): """ Concat all input vector into one huge vector. Inputs can be list of LayerOutput or list of projection. @@ -2043,10 +2043,14 @@ def __reduce_concat_type__(a, b): layer_type = (LayerType.CONCAT_LAYER if is_concat_layer else LayerType.CONCAT_PROJ_LAYER) + if layer_type == LayerType.CONCAT_LAYER: + assert not bias_attr + Layer( name=name, type=layer_type, inputs=[x.name for x in input] if is_concat_layer else input, active_type=act.name, + bias=ParamAttr.to_bias(bias_attr), **ExtraLayerAttribute.to_kwargs(layer_attr) ) @@ -2950,6 +2954,103 @@ def conv_operator(img, filter, filter_size, num_filters, op.origin = [img, filter] return op +@wrap_param_attr_default() +def conv_projection(input, filter_size, num_filters, + num_channels=None, stride=1, padding=0, + filter_size_y=None, stride_y=None, padding_y=None, + groups=1, param_attr=None): + """ + ConvProjection with a layer as input. + It performs element-wise multiplication with weight. + + Different from img_conv_layer and conv_op, conv_projection is an Projection, + which can be used in mixed_layer and conat_layer. It use cudnn to implement + conv and only support GPU mode. + + The example usage is: + + .. code-block:: python + + proj = conv_projection(img=input1, + filter_size=3, + num_filters=64, + num_channels=64) + + :param input: input layer + :type input: LayerOutput + :param filter_size: The x dimension of a filter kernel. + :type filter_size: int + :param filter_size_y: The y dimension of a filter kernel. Since + PaddlePaddle now supports rectangular filters, + the filter's shape can be (filter_size, filter_size_y). + :type filter_size_y: int + :param num_filters: channel of output data. + :type num_filters: int + :param num_channel: channel of input data. + :type num_channel: int + :param stride: The x dimension of the stride. + :type stride: int + :param stride_y: The y dimension of the stride. + :type stride_y: int + :param padding: The x dimension of padding. + :type padding: int + :param padding_y: The y dimension of padding. + :type padding_y: int + :param groups: The group number. + :type groups: int + :param param_attr: Convolution param attribute. None means default attribute + :type param_attr: ParameterAttribute + :return: A DotMulProjection Object. + :rtype: DotMulProjection + """ + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + + if filter_size_y is None: + if isinstance(filter_size, collections.Sequence): + assert len(filter_size) == 2 + filter_size, filter_size_y = filter_size + else: + filter_size_y = filter_size + + if stride_y is None: + if isinstance(stride, collections.Sequence): + assert len(stride) == 2 + stride, stride_y = stride + else: + stride_y = stride + + if padding_y is None: + if isinstance(padding, collections.Sequence): + assert len(padding) == 2 + padding, padding_y = padding + else: + padding_y = padding + + if param_attr.attr.get('initial_smart'): + # special initial for conv layers. + init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 + param_attr.attr["initial_mean"] = 0.0 + param_attr.attr["initial_std"] = init_w + param_attr.attr["initial_strategy"] = 0 + param_attr.attr["initial_smart"] = False + + proj = ConvProjection(input_layer_name=input.name, + num_filters=num_filters, + conv_conf=Conv(filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y, + groups=groups), + **param_attr.attr) + + proj.origin = input + return proj + @wrap_name_default() @layer_support() diff --git a/python/paddle/trainer_config_helpers/networks.py b/python/paddle/trainer_config_helpers/networks.py index 65512b327cdc6..bce88f93626ec 100644 --- a/python/paddle/trainer_config_helpers/networks.py +++ b/python/paddle/trainer_config_helpers/networks.py @@ -29,7 +29,7 @@ "img_conv_bn_pool", 'dropout_layer', 'lstmemory_group', 'lstmemory_unit', 'small_vgg', 'img_conv_group', 'vgg_16_network', 'gru_unit', 'gru_group', 'simple_gru', 'simple_attention', - 'text_conv_pool', + 'simple_gru2', 'bidirectional_gru', 'text_conv_pool', 'bidirectional_lstm', 'inputs', 'outputs'] @@ -811,22 +811,37 @@ def simple_gru(input, gru_layer_attr=None ): """ - simple_gru is also a recurrent layer group version Gated Recurrent Unit as - gru_group. The difference only lies in implemention details. + You maybe see gru_step_layer, grumemory in layers.py, gru_unit, gru_group, + simple_gru in network.py. The reason why there are so many interfaces is + that we have two ways to implement recurrent neural network. One way is to + use one complete layer to implement rnn (including simple rnn, gru and lstm) + with multiple time steps, such as recurrent_layer, lstmemory, grumemory. But, + the multiplication operation :math:`W x_t` is not computed in these layers. + See details in their interfaces in layers.py. + The other implementation is to use an recurrent group which can ensemble a + series of layers to compute rnn step by step. This way is flexible for + attenion mechanism or other complex connections. + + - gru_step_layer: only compute rnn by one step. It needs an memory as input + and can be used in recurrent group. + - gru_unit: a wrapper of gru_step_layer with memory. + - gru_group: a GRU cell implemented by a combination of multiple layers in + recurrent group. + But :math:`W x_t` is not done in group. + - gru_memory: a GRU cell implemented by one layer, which does same calculation + with gru_group and is faster than gru_group. + - simple_gru: a complete GRU implementation inlcuding :math:`W x_t` and + gru_group. :math:`W` contains :math:`W_r`, :math:`W_z` and :math:`W`, see + formula in grumemory. + The computational speed is that, grumemory is relatively better than gru_group, and gru_group is relatively better than simple_gru. - simple_gru does exactly the same calculation as the grumemory layer does. - Please see grumemory in layers.py for more detail about the maths. - The example usage is: .. code-block:: python - gru = gur_group(input=[layer1], - size=256, - act=TanhActivation(), - gate_act=SigmoidActivation()) + gru = simple_gru(input=[layer1], size=256) :param input: input layer name. :type input: LayerOutput @@ -863,6 +878,132 @@ def simple_gru(input, gru_layer_attr=gru_layer_attr) +@wrap_name_default('simple_gru2') +def simple_gru2(input, + size, + name=None, + reverse=False, + mixed_param_attr=None, + mixed_bias_attr=None, + gru_param_attr=None, + gru_bias_attr=None, + act=None, + gate_act=None, + mixed_layer_attr=None, + gru_cell_attr=None + ): + """ + simple_gru2 is the same with simple_gru, but using grumemory instead + Please see grumemory in layers.py for more detail about the maths. + simple_gru2 is faster than simple_gru. + + The example usage is: + + .. code-block:: python + + gru = simple_gru2(input=[layer1], size=256) + + :param input: input layer name. + :type input: LayerOutput + :param name: name of the gru group. + :type name: basestring + :param size: hidden size of the gru. + :type size: int + :param reverse: whether to process the input data in a reverse order + :type reverse: bool + :param act: type of the activiation + :type act: BaseActivation + :param gate_act: type of the gate activiation + :type gate_act: BaseActivation + :param gru_bias_attr: bias. False means no bias, None means default bias. + :type gru_bias_attr: ParameterAttribute|False + :param gru_layer_attr: Extra parameter attribute of the gru layer. + :type gru_layer_attr: ParameterAttribute|False + :return: the gru group. + :rtype: LayerOutput + """ + with mixed_layer(name='%s_transform' % name, + size=size * 3, + bias_attr=mixed_bias_attr, + layer_attr=mixed_layer_attr) as m: + m += full_matrix_projection(input=input, param_attr=mixed_param_attr) + + return grumemory(name=name, + size=size, + input=m, + reverse=reverse, + bias_attr=gru_bias_attr, + param_attr=gru_param_attr, + act=act, + gate_act=gate_act, + layer_attr=gru_cell_attr) + + +@wrap_name_default("bidirectional_gru") +def bidirectional_gru(input, size, name=None, return_seq=False, + fwd_mixed_param_attr=None, fwd_mixed_bias_attr=None, + fwd_gru_param_attr=None, fwd_gru_bias_attr=None, + fwd_act=None, fwd_gate_act=None, + fwd_mixed_layer_attr=None, fwd_gru_cell_attr=None, + + bwd_mixed_param_attr=None, bwd_mixed_bias_attr=None, + bwd_gru_param_attr=None, bwd_gru_bias_attr=None, + bwd_act=None, bwd_gate_act=None, + bwd_mixed_layer_attr=None, bwd_gru_cell_attr=None, + + last_seq_attr=None, first_seq_attr=None, + concat_attr=None, concat_act=None): + """ + A bidirectional_gru is a recurrent unit that iterates over the input + sequence both in forward and bardward orders, and then concatenate two + outputs to form a final output. However, concatenation of two outputs + is not the only way to form the final output, you can also, for example, + just add them together. + + The example usage is: + + .. code-block:: python + + bi_gru = bidirectional_gru(input=[input1], size=512) + + :param name: bidirectional gru layer name. + :type name: basestring + :param input: input layer. + :type input: LayerOutput + :param size: gru layer size. + :type size: int + :param return_seq: If set False, outputs of the last time step are + concatenated and returned. + If set True, the entire output sequences that are + processed in forward and backward directions are + concatenated and returned. + :type return_seq: bool + :return: LayerOutput object. + :rtype: LayerOutput + """ + args = locals() + + fw = simple_gru2(name='%s_fw' % name, input=input, size=size, + **dict((k[len('fwd_'):], v) for k, v in args.iteritems() + if k.startswith('fwd_'))) + + bw = simple_gru2(name="%s_bw" % name, input=input, size=size, + reverse=True, + **dict((k[len('bwd_'):], v) for k, v in args.iteritems() + if k.startswith('bwd_'))) + + if return_seq: + return concat_layer(name=name, input=[fw, bw], layer_attr=concat_attr, + act=concat_act) + else: + fw_seq = last_seq(name="%s_fw_last" % name, input=fw, + layer_attr=last_seq_attr) + bw_seq = first_seq(name="%s_bw_last" % name, input=bw, + layer_attr=first_seq_attr) + return concat_layer(name=name, input=[fw_seq, bw_seq], + layer_attr=concat_attr, act=concat_act) + + @wrap_name_default("bidirectional_lstm") def bidirectional_lstm(input, size, name=None, return_seq=False, fwd_mat_param_attr=None, fwd_bias_param_attr=None, @@ -893,7 +1034,7 @@ def bidirectional_lstm(input, size, name=None, return_seq=False, .. code-block:: python - lstm_step = bidirectional_lstm(input=[input1], size=512) + bi_lstm = bidirectional_lstm(input=[input1], size=512) :param name: bidirectional lstm layer name. :type name: basestring @@ -907,7 +1048,7 @@ def bidirectional_lstm(input, size, name=None, return_seq=False, processed in forward and backward directions are concatenated and returned. :type return_seq: bool - :return: lstm layer name. + :return: LayerOutput object accroding to the return_seq. :rtype: LayerOutput """ args = locals() diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 index d1b22b34903df..72dfdad7bdd40 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/check.md5 +++ b/python/paddle/trainer_config_helpers/tests/configs/check.md5 @@ -1,10 +1,11 @@ 86c0815275a9d5eb902e23c6a592f58a img_layers.protostr a5d9259ff1fd7ca23d0ef090052cb1f2 last_first_seq.protostr 9c038249ec8ff719753a746cdb04c026 layer_activations.protostr -5913f87b39cee3b2701fa158270aca26 projections.protostr +34e04043cbb12931c47fa44ec50eeffc projections.protostr 7334ba0a4544f0623231330fc51d390d shared_fc.protostr -8b8b6bb128a7dfcc937be86145f53e2f shared_lstm.protostr +bb8e233b05b8e07f9ed386b7aee4f2c6 shared_lstm.protostr 6b39e34beea8dfb782bee9bd3dea9eb5 simple_rnn_layers.protostr +f98e79e1630d5eb827c300e64836d269 test_bi_grumemory.protostr 0fc1409600f1a3301da994ab9d28b0bf test_cost_layers.protostr 6cd5f28a3416344f20120698470e0a4c test_cost_layers_with_weight.protostr 144bc6d3a509de74115fa623741797ed test_expand_layer.protostr @@ -15,7 +16,7 @@ d350bd91a0dc13e854b1364c3d9339c6 test_lstmemory_layer.protostr 5433ed33d4e7414eaf658f2a55946186 test_maxout.protostr 251a948ba41c1071afcd3d9cf9c233f7 test_ntm_layers.protostr e6ff04e70aea27c7b06d808cc49c9497 test_print_layer.protostr -2a75dd33b640c49a8821c2da6e574577 test_rnn_group.protostr +fded24727338fb8ce44d9951ed8aea08 test_rnn_group.protostr 67d6fde3afb54f389d0ce4ff14726fe1 test_sequence_pooling.protostr f586a548ef4350ba1ed47a81859a64cb unused_layers.protostr -8122477f4f65244580cec09edc590041 util_layers.protostr +f937a5a6e7e8864b4d8cf56b0f7c7f44 util_layers.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index 4b1d2d3d41d52..6a31ceabdf36d 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -9,7 +9,7 @@ test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers util_layers simple_rnn_layers unused_layers test_cost_layers test_rnn_group shared_fc shared_lstm test_cost_layers_with_weight -test_maxout) +test_maxout test_bi_grumemory) for conf in ${configs[*]} diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py b/python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py new file mode 100644 index 0000000000000..ab9f7c4948b85 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_bi_grumemory.py @@ -0,0 +1,10 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=1000, + learning_rate=1e-4 +) + +din = data_layer(name='data', size=120) + +outputs(bidirectional_gru(input=din, size=40, return_seq=True)) From 7180b4246d085913390e4a338bc3f1decf77c44d Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 1 Nov 2016 23:55:46 -0500 Subject: [PATCH 200/324] Add some concepts documents to guide user for using paddle (#249) --- demo/quick_start/dataprovider_emb.py | 1 + demo/quick_start/trainer_config.lstm.py | 15 +- doc_cn/concepts/nn.rst | 3 + doc_cn/concepts/program_concepts.rst | 4 + doc_cn/concepts/pserver_topology.dot | 68 +++++++++ doc_cn/concepts/trainer_config.py | 23 +++ doc_cn/concepts/use_concepts.rst | 191 ++++++++++++++++++++++++ doc_cn/faq/index.rst | 10 ++ doc_cn/index.rst | 1 + 9 files changed, 305 insertions(+), 11 deletions(-) create mode 100644 doc_cn/concepts/nn.rst create mode 100644 doc_cn/concepts/program_concepts.rst create mode 100644 doc_cn/concepts/pserver_topology.dot create mode 100644 doc_cn/concepts/trainer_config.py create mode 100644 doc_cn/concepts/use_concepts.rst diff --git a/demo/quick_start/dataprovider_emb.py b/demo/quick_start/dataprovider_emb.py index ca940a89e5477..f5632d5f3f8bd 100755 --- a/demo/quick_start/dataprovider_emb.py +++ b/demo/quick_start/dataprovider_emb.py @@ -16,6 +16,7 @@ UNK_IDX = 0 + def initializer(settings, dictionary, **kwargs): settings.word_dict = dictionary settings.input_types = [ diff --git a/demo/quick_start/trainer_config.lstm.py b/demo/quick_start/trainer_config.lstm.py index ec8a2cb00abd1..b412a9cbd914d 100644 --- a/demo/quick_start/trainer_config.lstm.py +++ b/demo/quick_start/trainer_config.lstm.py @@ -42,20 +42,13 @@ gradient_clipping_threshold=25 ) -bias_attr = ParamAttr(initial_std=0.,l2_rate=0.) data = data_layer(name="word", size=len(word_dict)) emb = embedding_layer(input=data, size=128) -fc = fc_layer(input=emb, size=512, - act=LinearActivation(), - bias_attr=bias_attr, - layer_attr=ExtraAttr(drop_rate=0.1)) -lstm = lstmemory(input=fc, act=TanhActivation(), - bias_attr=bias_attr, - layer_attr=ExtraAttr(drop_rate=0.25)) -lstm_last = pooling_layer(input=lstm, pooling_type=MaxPooling()) -output = fc_layer(input=lstm_last, size=2, - bias_attr=bias_attr, +lstm = simple_lstm(input=emb, size=128, + lstm_cell_attr=ExtraAttr(drop_rate=0.25)) +lstm_max = pooling_layer(input=lstm, pooling_type=MaxPooling()) +output = fc_layer(input=lstm_max, size=2, act=SoftmaxActivation()) if is_predict: maxid = maxid_layer(output) diff --git a/doc_cn/concepts/nn.rst b/doc_cn/concepts/nn.rst new file mode 100644 index 0000000000000..f4d2cf490d147 --- /dev/null +++ b/doc_cn/concepts/nn.rst @@ -0,0 +1,3 @@ +TBD + +目前正在书写中。敬请期待。 \ No newline at end of file diff --git a/doc_cn/concepts/program_concepts.rst b/doc_cn/concepts/program_concepts.rst new file mode 100644 index 0000000000000..af5bbdac260af --- /dev/null +++ b/doc_cn/concepts/program_concepts.rst @@ -0,0 +1,4 @@ +TBD +### + +目前正在书写中。敬请期待。 \ No newline at end of file diff --git a/doc_cn/concepts/pserver_topology.dot b/doc_cn/concepts/pserver_topology.dot new file mode 100644 index 0000000000000..9ff658b849503 --- /dev/null +++ b/doc_cn/concepts/pserver_topology.dot @@ -0,0 +1,68 @@ +graph pp_topology { + rankdir=BT; + subgraph cluster_node0 { + style=filled; + color=lightgrey; + node [style=filled, color=white, shape=box]; + label = "机器0" + + pserver0 [label="Parameter \n Server 0"] + trainer0 [label="Trainer 0"] + } + subgraph cluster_node1 { + style=filled; + color=lightgrey; + node [style=filled, color=white, shape=box]; + label = "机器1" + + pserver1 [label="Parameter \n Server 1"] + trainer1 [label="Trainer 1"] + } + + subgraph cluster_node2 { + style=filled; + color=lightgrey; + node [style=filled, color=white, shape=box]; + label = "机器2" + + pserver2 [label="Parameter \n Server 2"] + trainer2 [label="Trainer 2"] + } + + subgraph cluster_node3 { + style=filled; + color=lightgrey; + node [style=filled, color=white, shape=box]; + label = "机器3" + + pserver3 [label="Parameter \n Server 3"] + trainer3 [label="Trainer 3"] + } + + data [label="数据", shape=hexagon] + + trainer0 -- pserver0 + trainer0 -- pserver1 + trainer0 -- pserver2 + trainer0 -- pserver3 + + trainer1 -- pserver0 + trainer1 -- pserver1 + trainer1 -- pserver2 + trainer1 -- pserver3 + + trainer2 -- pserver0 + trainer2 -- pserver1 + trainer2 -- pserver2 + trainer2 -- pserver3 + + trainer3 -- pserver0 + trainer3 -- pserver1 + trainer3 -- pserver2 + trainer3 -- pserver3 + + data -- trainer0 + data -- trainer1 + data -- trainer2 + data -- trainer3 +} diff --git a/doc_cn/concepts/trainer_config.py b/doc_cn/concepts/trainer_config.py new file mode 100644 index 0000000000000..8d8c79fb39e0c --- /dev/null +++ b/doc_cn/concepts/trainer_config.py @@ -0,0 +1,23 @@ +from paddle.trainer_config_helpers import * + +define_py_data_sources2(train_list='train.list', + test_list='test.list', + module='provider', + obj='process') +settings( + batch_size=128, + learning_rate=1e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(0.5) +) + +img = data_layer(name='pixel', size=28 * 28) + +hidden1 = simple_img_conv_pool(input=img, filter_size=3, num_filters=32, pool_size=3, + num_channel=1) + +hidden2 = fc_layer(input=hidden1, size=200, act=TanhActivation(), + layer_attr=ExtraAttr(drop_rate=0.5)) +predict = fc_layer(input=hidden2, size=10, act=SoftmaxActivation()) + +outputs(classification_cost(input=predict, label=data_layer(name='label', size=10))) diff --git a/doc_cn/concepts/use_concepts.rst b/doc_cn/concepts/use_concepts.rst new file mode 100644 index 0000000000000..67e98edabc0c2 --- /dev/null +++ b/doc_cn/concepts/use_concepts.rst @@ -0,0 +1,191 @@ +######################### +PaddlePaddle 基本使用概念 +######################### + +PaddlePaddle是一个神经网络学习框架。其单机进程为 :code:`paddle train`。 单机的所有设备使用，均在单机进程内调度完成。 而多机辅助进程 :code:`paddle pserver` 负责联合多个单机进程进行通信，进而充分利用集群的计算资源。 PaddlePaddle同时以 :code:`swig api` 的形式，提供训练结果模型预测的方法和自定义训练流程。 + +下面我们会分别介绍主要进程 :code:`paddle train` 中的一些概念。这些概念会对如何使用PaddlePaddle有一定的帮助。 了解这些概念的前提是，读者已经了解 `基本的神经网络/机器学习原理和概念 `_ 。同时，如果想要了解PaddlePaddle实现中的一些概念，请参考 `PaddlePaddle 编程中的基本概念 `_ 。 + +.. contents:: + +PaddlePaddle 的进程模型 +======================= + +PaddlePaddle进程内嵌了一个 :code:`python` 解释器。 这个 :code:`python` 解释器负责解析用户定义的神经网络配置，和解析用户数据，并将用户数据传入给 PaddlePaddle。 + +.. graphviz:: + + digraph pp_process { + rankdir=LR; + config_file [label="用户神经网络配置"]; + subgraph cluster_pp { + style=filled; + color=lightgrey; + node [style=filled, color=white, shape=box]; + label = "PaddlePaddle C++"; + py [label="Python解释器"]; + } + data_provider [label="用户数据解析"]; + config_file -> py; + py -> data_provider [dir="back"]; + } + +所以，PaddlePaddle单机训练进程，:code:`paddle train` , 对于用户的主要接口语言为 python。 主要需要用户配置的两个文件为 :code:`DataProvider` 和训练文件 :code:`TrainerConfig` 。 + + +DataProvider +============ + +DataProvider是 :code:`paddle train` 的数据提供器。 它负责将用户的原始数据转换成 PaddlePaddle 可以识别的数据类型。每当 PaddlePaddle 需要新的数据训练时，都会调用 DataProvider 返回数据。 当所有数据读取完一轮后，DataProvider 便返回空数据通知 PaddlePaddle。PaddlePaddle负责在下一轮训练开始前，将DataProvider重置。 + +需要注意的是，DataProvider在PaddlePaddle中是被训练逻辑调用的关系， 而不是新的数据驱动训练。并且所有的 :code:`shuffle` , 和一些随机化的噪声添加，都应该在 DataProvider 阶段完成。 + +为了方便用户使用自己的数据格式， PaddlePaddle 提供了 `PyDataProvider`_ 来处理数据。 并且在这个Provider中，PaddlePaddle的 C++ 部分接管了如何shuffle，处理 batch，GPU/CPU通信，双缓冲，异步读取等问题。 用户可以参考 `PyDataProvider`_ 的相关文档，继续深入了解 DataProvider 的使用。 + + +训练文件 +======== + +训练文件是PaddlePaddle中配置神经网络结构、学习优化算法、数据传入方式的地方。 训练文件是一个python文件，使用命令行参数 :code:`--config` 传给 paddle 的主程序。 例如\: + +.. code-block:: bash + + paddle train --config=trainer_config.py + +一个典型简单的训练文件可能为 + +.. literalinclude:: trainer_config.py + :linenos: + +下面我们详细的介绍一下训练文件中各个模块的概念。 + + +trainer_config_helpers +---------------------- + +PaddlePaddle的配置文件与PaddlePaddle C++端通信的最基础协议是 :code:`protobuf` 。而为了避免用户直接写比较难写的 protobuf string，我们书写了一个helpers来生成这个protobuf包。所以在文件的开始，import这些helpers函数。 + +需要注意的是，这个 :code:`paddle.trainer_config_helpers` 包是标准的python包，这意味着用户可以选择自己喜欢的 :code:`ide` 或者编辑器来编写Paddle的配置文件，这个python包注释文档比较完善，并且考虑了IDE的代码提示与类型注释。 + +data_sources +------------ + +data_sources是配置神经网络的数据源。这里使用的函数是 :code:`define_py_data_sources2` ，这个函数是定义了使用 `PyDataProvider`_ 作为数据源。 而后缀 :code:`2` 是Paddle历史遗留问题，因为Paddle之前使用的 PyDataProvider 性能较差，所以完全重构了一个新的 `PyDataProvider`_ 。 + +data_sources里面的 train_list 和 test_list 指定的是训练文件列表和测试文件列表。 如果传入一个字符串的话，是指一个训练列表文件。这个训练列表文件中包含的是每一个训练或者测试文件的路径。如果传入一个list的话，则会默认生成一个 list 文件，再传入给 train.list 或者 test.list 。 + +而 :code:`module` 和 :code:`obj` 指定了 DataProvider 的模块名和函数名。 + +更具体的使用，请参考 `PyDataProvider`_ 。 + +settings +-------- + +`settings`_ 是神经网络训练算法相关的设置项。包括学习率，batch_size，优化算法，正则方法等等。具体的使用方法请参考 `settings`_ 文档。 + +网络配置 +-------- + +上述网络配置中余下的部分均是神经网络配置。第一行是定义一个名字叫 "pixel" 的 :code:`data_layer` 。每一个layer返回的都是一个 :code:`LayerOutput` 对象。 这里第一层的输出对象是 :code:`img` 。然后这个对象传输给了另一个 layer 函数， +:code:`simple_img_conv_pool` 。:code:`simple_img_conv_pool` 是一个组合层， +包括了图像的卷积 (convolution) 和池化(pooling)， +并继续接了一个全连接层( :code:`fc_layer` )，然后再接了一个Softmax的全连接层。 + +最终，网络配置输出了 :code:`classification_cost` 。标记网络输出的函数为 +:code:`outputs` 。网络的输出是神经网络的优化目标，神经网络训练的时候，实际上就是 +要最小化这个输出。 + +在神经网络进行预测的时候，实际上网络的输出也是通过 :code:`outputs` 标记。 + + +Layer、Projection、Operator +=========================== + +PaddlePaddle的网络基本上是基于Layer来配置的。所谓的Layer即是神经网络的某一层， +而神经网络的某一层，一般是封装了许多复杂操作的操作集合。比如最简单的 +:code:`fc_layer` ，也包括矩阵乘法，多输入的求和，和activation。 + +.. code-block:: python + + data = data_layer(name='data', size=200) + out = fc_layer(input=data, size=200, act=TanhActivation()) + +而对于更灵活配置需求，可能这样基于Layer的配置是不灵活的。于是 PaddlePaddle 提供 +了基于 Projection 或者 Operator 的配置。使用Projection和Operator需要与 +:code:`mixed_layer` 配合使用。 :code:`mixed_layer` 是将layer中的元素累加求和， +并且做一个 :code:`activation` ， 而这个layer具体如何计算，是交由内部的Projection +和 Operator 定义。Projection是指含有可学习参数的操作，而Operator不含有可学习的 +参数，输入全是其他Layer的输出。 + + +例如，和 :code:`fc_layer` 同样功能的 :code:`mixed_layer` 。 + +.. code-block:: python + + data = data_layer(name='data', size=200) + with mixed_layer(size=200) as out: + out += full_matrix_projection(input=data) + +PaddlePaddle可以使用的mixed layer 配置出非常复杂的网络，甚至可以直接配置一个完整的LSTM。 +用户可以参考 `mixed_layer`_ 的相关文档进行配置。 + +如何利用单机的所有GPU或所有CPU核心 +================================== + +PaddlePaddle的单机进程 :code:`paddle train` 可以充分利用一台计算机上所有的GPU资 +源或者CPU。 + +如果要使用机器上多块GPU，使用如下命令即可\: + +.. code-block:: bash + + paddle train --use_gpu=true --trainer_count=4 # use 4 gpu card, 0, 1, 2, 3 + +如果要使用机器上多块CPU, 使用如下命令即可\: + +.. code-block:: bash + + paddle train --trainer_config=4 # use 4 cpu cores. + +对于其他设置GPU的选择情况，例如选择第0、2号GPU显卡，则可以使用 :code:`CUDA_VISIBLE_DEVICES` 环境变量来选择部分的显卡。 具体可以参考连接`masking-gpus`_ 。 可以使用的命令为 + +.. code-block:: bash + + env CUDA_VISIBLE_DEVICES=0,2 paddle train --use_gpu=true --trainer_config=2 + +如何利用多台机器的计算资源训练神经网络 +====================================== + +PaddlePaddle多机使用的经典方法是通过 :code:`Parameter Server` 来对多机的 :code:`paddle train` 进行同步。 而多机训练神经网络，首先要讲数据切分到不同的机器上。 切分数据文件的方式在PaddlePaddle的开源实现中并没有提供工具包。 但是切分数据并不是一件非常复杂的事情，也不是神经网络实现的重点。 + +多机训练过程中，经典的拓扑结构如下\: + +.. graphviz:: pserver_topology.dot + +图中每个灰色方块是一台机器，在每个机器中，先去启动一个 :code:`paddle pserver` 进程，并确定整体的端口号。可能的参数是\: + +.. code-block:: bash + + paddle pserver --port=5000 --num_gradient_servers=4 --nics='eth0' + +这里说明系统的 :code:`paddle pserver` 的起始端口是 :code:`5000` ，并且有四个训练进程(:code:`gradient_servers`，Paddle同时将 :code:`paddle train` 进程称作 :code:`GradientServer` 。因为其为负责提供Gradient的进程)。 而对于训练进程的话，则需要在 :code:`paddle pserver` 启动之后，再在各个节点上运行如下命令\: + +.. code-block:: bash + + paddle train --port=5000 --pservers=192.168.100.101,192.168.100.102,192.168.100.103,192.168.100.104 --config=... + +对于简单的多机协同使用上述方式即可。同时，pserver/train 通常在高级情况下，还有两个参数需要设置，他们是 + +* --ports_num\: 一个 pserver进程共绑定多少个端口用来做稠密更新。默认是1 +* --ports_num_for_sparse\: 一个pserver进程共绑定多少端口用来做稀疏更新，默认是0 + +使用手工指定端口数量，是因为Paddle的网络通信中，使用了 :code:`int32` 作为消息长度，比较容易在大模型下溢出。所以，在 :code:`paddle pserver` 进程中可以启动多个子线程去接受 trainer 的数据，这样单个子线程的长度就不会溢出了。但是这个值不可以调的过大，因为增加这个值，还是对性能，尤其是内存占用有一定的开销的，另外稀疏更新的端口如果太大的话，很容易某一个参数服务器没有分配到任何参数。 + +详细的说明可以参考，使用 `集群训练Paddle`_ 。 + + +.. _PyDataProvider: ../ui/data_provider/pydataprovider2.html +.. _settings: ../../doc/ui/api/trainer_config_helpers/optimizers.html#settings +.. _mixed_layer: ../../doc/ui/api/trainer_config_helpers/layers.html#mixed-layer +.. _masking-gpu: http://www.acceleware.com/blog/cudavisibledevices-masking-gpus +.. _集群训练Paddle: ../cluster/index.html diff --git a/doc_cn/faq/index.rst b/doc_cn/faq/index.rst index 283607957ce63..db28b4436fe5e 100644 --- a/doc_cn/faq/index.rst +++ b/doc_cn/faq/index.rst @@ -166,4 +166,14 @@ PaddlePaddle的参数使用名字 :code:`name` 作为参数的ID，相同名字 这里 :code:`hidden_a` 和 :code:`hidden_b` 使用了同样的parameter和bias。并且softmax层的两个输入也使用了同样的参数 :code:`softmax_param`。 +7. *-cp27mu-linux_x86_64.whl is not a supported wheel on this platform. +----------------------------------------------------------------------- + +出现这个问题的主要原因是，系统编译wheel包的时候，使用的 :code:`wheel` 包是最新的， +而系统中的 :code:`pip` 包比较老。具体的解决方法是，更新 :code:`pip` 包并重新编译PaddlePaddle。 +更新 :code:`pip` 包的方法是\: + +.. code-block:: bash + + pip install --upgrade pip diff --git a/doc_cn/index.rst b/doc_cn/index.rst index 715da44fb41d4..e74b0942a6cc0 100644 --- a/doc_cn/index.rst +++ b/doc_cn/index.rst @@ -5,6 +5,7 @@ PaddlePaddle文档 -------- * `介绍 `_ * `快速入门 `_ +* `基本使用概念 `_ * `编译与安装 `_ * `用户接口 `_ * `使用示例 `_ From fcf177fc4bcba5f83ed6c53ede9ae5caed5a0bc0 Mon Sep 17 00:00:00 2001 From: qijun Date: Wed, 2 Nov 2016 07:53:43 +0000 Subject: [PATCH 201/324] reuse code of PoolProjection in PoolProjectionLayer --- paddle/gserver/layers/PoolLayer.cpp | 6 +- paddle/gserver/layers/PoolProjection.cpp | 4 +- paddle/gserver/layers/PoolProjectionLayer.cpp | 64 ++----------------- paddle/gserver/layers/PoolProjectionLayer.h | 35 +++------- paddle/gserver/tests/test_LayerGrad.cpp | 10 ++- 5 files changed, 25 insertions(+), 94 deletions(-) diff --git a/paddle/gserver/layers/PoolLayer.cpp b/paddle/gserver/layers/PoolLayer.cpp index 7fc27ac0bd8e0..2fbc9001f1161 100644 --- a/paddle/gserver/layers/PoolLayer.cpp +++ b/paddle/gserver/layers/PoolLayer.cpp @@ -52,10 +52,8 @@ bool PoolLayer::init(const LayerMap& layerMap, Layer* PoolLayer::create(const LayerConfig& config) { CHECK_EQ(config.inputs_size(), 1); const std::string& pool = config.inputs(0).pool_conf().pool_type(); - if (pool == "max-projection") { - return new MaxPoolProjectionLayer(config); - } else if (pool == "avg-projection") { - return new AvgPoolProjectionLayer(config); + if (pool == "max-projection" || pool == "avg-projection") { + return new PoolProjectionLayer(config); #ifndef PADDLE_ONLY_CPU } else if (CudnnPoolLayer::typeCheck(pool)) { return new CudnnPoolLayer(config); diff --git a/paddle/gserver/layers/PoolProjection.cpp b/paddle/gserver/layers/PoolProjection.cpp index 50059ee04d39b..468ca6f1b7d2d 100644 --- a/paddle/gserver/layers/PoolProjection.cpp +++ b/paddle/gserver/layers/PoolProjection.cpp @@ -21,9 +21,9 @@ REGISTER_PROJECTION_CREATE_FUNC(pool2, &PoolProjection::create); PoolProjection* PoolProjection::create(const ProjectionConfig& config, ParameterPtr parameter, bool useGpu) { const std::string& pool = config.pool_conf().pool_type(); - if (pool == "max") { + if (pool == "max-projection") { return new MaxPoolProjection(config, parameter, useGpu); - } else if (pool == "avg") { + } else if (pool == "avg-projection") { return new AvgPoolProjection(config, parameter, useGpu); } else { LOG(FATAL) << "Unknown pool type: " << pool; diff --git a/paddle/gserver/layers/PoolProjectionLayer.cpp b/paddle/gserver/layers/PoolProjectionLayer.cpp index 5a2e9afb6e164..3a54c51cfc1ac 100644 --- a/paddle/gserver/layers/PoolProjectionLayer.cpp +++ b/paddle/gserver/layers/PoolProjectionLayer.cpp @@ -19,6 +19,7 @@ limitations under the License. */ namespace paddle { + size_t PoolProjectionLayer::getSize() { CHECK_EQ(inputLayers_.size(), 1UL); size_t layerSize = 0; @@ -41,71 +42,20 @@ size_t PoolProjectionLayer::getSize() { return layerSize; } -void MaxPoolProjectionLayer::forward(PassType passType) { - Layer::forward(passType); - - /* malloc memory for the output_ if necessary */ - /* note: one sample correspond to one ROW */ - MatrixPtr input = getInputValue(0); - int batchSize = input->getHeight(); - int size = getSize(); - resetOutput(batchSize, size); - - MatrixPtr outV = getOutputValue(); - - outV->maxPoolForward(*input, imgSizeH_, imgSizeW_, channels_, - sizeX_, sizeY_, strideY_, stride_, - outputH_, outputW_, confPaddingY_, confPadding_); -} - -void MaxPoolProjectionLayer::backward(const UpdateCallback& callback) { - (void)callback; - - if (NULL == getInputGrad(0)) { - return; - } - - /* Do derivation */ - MatrixPtr outGrad = getOutputGrad(); - MatrixPtr inputV = getInputValue(0); - MatrixPtr outV = getOutputValue(); - MatrixPtr inputGrad = getInputGrad(0); - - inputGrad->maxPoolBackward(*inputV, imgSizeH_, imgSizeW_, *outGrad, *outV, - sizeX_, sizeY_, - strideY_, stride_, outputH_, outputW_, 1, 1, - confPaddingY_, confPadding_); -} - -void AvgPoolProjectionLayer::forward(PassType passType) { +void PoolProjectionLayer::forward(PassType passType) { Layer::forward(passType); - - /* malloc memory for the output_ if necessary */ - /* note: one sample correspond to one ROW */ - MatrixPtr input = getInputValue(0); - int batchSize = input->getHeight(); + const Argument& in = getInput(0); + int batchSize = in.value->getHeight(); int size = getSize(); resetOutput(batchSize, size); - - MatrixPtr outV = getOutputValue(); - - outV->avgPoolForward(*input, imgSizeH_, imgSizeW_, channels_, - sizeX_, sizeY_, strideY_, stride_, - outputH_, outputW_, confPaddingY_, confPadding_); + poolProjection_->forward(&in, &output_, passType); } -void AvgPoolProjectionLayer::backward(const UpdateCallback& callback) { +void PoolProjectionLayer::backward(const UpdateCallback& callback) { (void)callback; - if (NULL == getInputGrad(0)) { return; } - /* Do derivation */ - MatrixPtr outputGrad = getOutputGrad(); - MatrixPtr inputGrad = getInputGrad(0); - inputGrad->avgPoolBackward(*outputGrad, imgSizeH_, imgSizeW_, - sizeX_, sizeY_, strideY_, stride_, - outputH_, outputW_, 1, 1, - confPaddingY_, confPadding_); + poolProjection_->backward(callback); } } // namespace paddle diff --git a/paddle/gserver/layers/PoolProjectionLayer.h b/paddle/gserver/layers/PoolProjectionLayer.h index 42bbc83c62246..6e336f79e9043 100644 --- a/paddle/gserver/layers/PoolProjectionLayer.h +++ b/paddle/gserver/layers/PoolProjectionLayer.h @@ -16,6 +16,7 @@ limitations under the License. */ #pragma once #include "PoolLayer.h" +#include "PoolProjection.h" #include "paddle/math/Matrix.h" #include @@ -27,35 +28,19 @@ class PoolProjectionLayer : public PoolLayer { protected: size_t imgSizeH_, imgSizeW_; size_t outputH_, outputW_; + std::unique_ptr poolProjection_; + ProjectionConfig projectionConfig_; public: size_t getSize(); - explicit PoolProjectionLayer(const LayerConfig& config) : PoolLayer(config) {} -}; -/** - * @brief A layer for max pooling - */ -class MaxPoolProjectionLayer : public PoolProjectionLayer { -public: - explicit MaxPoolProjectionLayer(const LayerConfig& config) - : PoolProjectionLayer(config) {} - - ~MaxPoolProjectionLayer() {} - - virtual void forward(PassType passType); - virtual void backward(const UpdateCallback& callback = nullptr); -}; -/** - * @brief A layer for average pooling - */ -class AvgPoolProjectionLayer : public PoolProjectionLayer { -public: - explicit AvgPoolProjectionLayer(const LayerConfig& config) - : PoolProjectionLayer(config) {} - - ~AvgPoolProjectionLayer() {} - virtual void forward(PassType passType); virtual void backward(const UpdateCallback& callback = nullptr); + explicit PoolProjectionLayer(const LayerConfig& config) + : PoolLayer(config) { + PoolConfig* conf = projectionConfig_.mutable_pool_conf(); + *conf = config_.inputs(0).pool_conf(); + poolProjection_.reset(PoolProjection::create(projectionConfig_, nullptr, + useGpu_)); + } }; } // namespace paddle diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 3d633f4b72797..fc11a9d39aa85 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -897,12 +897,10 @@ void testSppLayer(const string& poolType, const int pyramidHeight, bool trans, TEST(Layer, SpatialPyramidPoolLayer) { for (auto useGpu : {false, true}) { - testSppLayer("avg", 1, false, useGpu); - testSppLayer("avg", 3, false, useGpu); - testSppLayer("avg", 5, false, useGpu); - testSppLayer("max", 1, false, useGpu); - testSppLayer("max", 3, false, useGpu); - testSppLayer("avg", 5, false, useGpu); + for (auto pyramidHeight : {1, 2, 3}) { + testSppLayer("avg-projection", pyramidHeight, false, useGpu); + testSppLayer("max-projection", pyramidHeight, false, useGpu); + } } } From ee028bb5eaf9368bc411a0d9f6d40197866b7aec Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 2 Nov 2016 04:14:35 -0500 Subject: [PATCH 202/324] Add How to build docs (#312) --- doc_cn/howto/how_to_write_docs/index.rst | 63 +++++++++++++++++++ doc_cn/index.rst | 1 + paddle/scripts/tools/build_docs/.gitignore | 2 + paddle/scripts/tools/build_docs/Dockerfile | 6 ++ paddle/scripts/tools/build_docs/build.sh | 13 ++++ paddle/scripts/tools/build_docs/build_docs.sh | 4 ++ 6 files changed, 89 insertions(+) create mode 100644 doc_cn/howto/how_to_write_docs/index.rst create mode 100644 paddle/scripts/tools/build_docs/.gitignore create mode 100644 paddle/scripts/tools/build_docs/Dockerfile create mode 100755 paddle/scripts/tools/build_docs/build.sh create mode 100755 paddle/scripts/tools/build_docs/build_docs.sh diff --git a/doc_cn/howto/how_to_write_docs/index.rst b/doc_cn/howto/how_to_write_docs/index.rst new file mode 100644 index 0000000000000..869ef747f9f88 --- /dev/null +++ b/doc_cn/howto/how_to_write_docs/index.rst @@ -0,0 +1,63 @@ +############################### +如何贡献/修改PaddlePaddle的文档 +############################### + +PaddlePaddle的文档使用 `cmake`_ 驱动 `sphinx`_ 生成。公有两个文档，:code:`doc` 和 :code:`doc_cn` 。这两者会在 `cmake`_ 中进行编译，生成后的文档会存储在服务器的 :code:`doc` 和 :code:`doc_cn` 两个目录下。 + +下面分几个部分介绍一下PaddlePaddle文档的贡献方法。 + +如何书写PaddlePaddle的文档 +========================== + +TBD + +如何构建PaddlePaddle的文档 +========================== + +构建PaddlePaddle文档，需要使用构建Paddle的全部环境。准备这个环境相对来说比较复杂，所以本文档提供两种方式构建PaddlePaddle的文档，即 + +* 使用Docker构建PaddlePaddle的文档 +* 直接构建PaddlePaddle的文档。 + +并且，我们推荐使用Docker来构建PaddlePaddle的文档。 + + +使用Docker构建PaddlePaddle的文档 +-------------------------------- + +使用Docker构建PaddlePaddle的文档，首先要求在系统里安装好Docker工具包。安装Docker请参考 `Docker的官网 `_ 。 + +安装好Docker之后可以使用源码目录下的脚本构建文档，即 + +.. code-block:: bash + + cd TO_YOUR_PADDLE_CLONE_PATH + cd paddle/scripts/tools/build_docs + bash build_docs.sh + +执行完这个脚本后，该目录下会生成两个目录，分别是\: + +* doc 目录，英文文档地址 +* doc_cn 目录，中文文档地址 + +打开浏览器访问对应目录下的index.html即可访问本地文档。 + +.. code-block:: bash + + open doc_cn/index.html + + +直接构建PaddlePaddle的文档 +-------------------------- + +TBD + + +如何更新www.paddlepaddle.org文档 +================================ + +TBD + + +.. _cmake: https://cmake.org/ +.. _sphinx: http://www.sphinx-doc.org/en/1.4.8/ \ No newline at end of file diff --git a/doc_cn/index.rst b/doc_cn/index.rst index e74b0942a6cc0..f1398206fddff 100644 --- a/doc_cn/index.rst +++ b/doc_cn/index.rst @@ -15,6 +15,7 @@ PaddlePaddle文档 开发指南 -------- * `新写Layer <../doc/dev/new_layer/index.html>`_ +* `如何贡献文档 `_ 算法教程 -------- diff --git a/paddle/scripts/tools/build_docs/.gitignore b/paddle/scripts/tools/build_docs/.gitignore new file mode 100644 index 0000000000000..6ec14c8f5bc37 --- /dev/null +++ b/paddle/scripts/tools/build_docs/.gitignore @@ -0,0 +1,2 @@ +doc +doc_cn diff --git a/paddle/scripts/tools/build_docs/Dockerfile b/paddle/scripts/tools/build_docs/Dockerfile new file mode 100644 index 0000000000000..5db0b29c47399 --- /dev/null +++ b/paddle/scripts/tools/build_docs/Dockerfile @@ -0,0 +1,6 @@ +FROM paddledev/paddle:cpu-devel-latest +COPY build.sh / +RUN pip install sphinx &&\ + apt install -y doxygen graphviz &&\ + pip install breathe recommonmark numpy protobuf==2.6.1 +CMD /build.sh diff --git a/paddle/scripts/tools/build_docs/build.sh b/paddle/scripts/tools/build_docs/build.sh new file mode 100755 index 0000000000000..a23b6e61d4592 --- /dev/null +++ b/paddle/scripts/tools/build_docs/build.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -ex + +mkdir -p /build +cd /build +cmake /paddle -DWITH_DOC=ON +make paddle_docs paddle_docs_cn -j `nproc` +mkdir -p /output/doc +mkdir -p /output/doc_cn +cp -r doc/html/* /output/doc/ +cp -r doc_cn/html/* /output/doc_cn/ +cd / +rm -rf /paddle/build diff --git a/paddle/scripts/tools/build_docs/build_docs.sh b/paddle/scripts/tools/build_docs/build_docs.sh new file mode 100755 index 0000000000000..9f8b80435c8fb --- /dev/null +++ b/paddle/scripts/tools/build_docs/build_docs.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +docker build . -t paddle_build_doc +docker run --rm -v $PWD/../../../../:/paddle -v $PWD:/output paddle_build_doc From 5acf136615c9f1b6e73cf9a9bdbaac770e2e9669 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Wed, 2 Nov 2016 19:07:04 +0800 Subject: [PATCH 203/324] Bug fix in CudnnConvLayer, which will lead to destruction error. (#317) --- paddle/gserver/layers/CudnnConvLayer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/gserver/layers/CudnnConvLayer.cpp b/paddle/gserver/layers/CudnnConvLayer.cpp index e77216f17c3be..23ba2341185d1 100644 --- a/paddle/gserver/layers/CudnnConvLayer.cpp +++ b/paddle/gserver/layers/CudnnConvLayer.cpp @@ -35,8 +35,8 @@ bool CudnnConvLayer::init(const LayerMap &layerMap, ProjectionConfig* conf = new ProjectionConfig(); conf->set_type("conv"); conf->set_num_filters(numFilters_); - conf->set_allocated_conv_conf( - config_.mutable_inputs(i)->mutable_conv_conf()); + ConvConfig* convConf = conf->mutable_conv_conf(); + *convConf = *(config_.mutable_inputs(i)->mutable_conv_conf()); conf->set_input_size(getPrev(i)->getSize()); conf->set_output_size(getSize()); projConf_.emplace_back(conf); From 968464cc605ed67c658af6c65d617b7036bfecfb Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 2 Nov 2016 12:47:13 -0500 Subject: [PATCH 204/324] Fix a bug in testOnePeriod. (#322) * Forget to finishTestPeriod in testOnePeriod. * Fix #318 --- paddle/trainer/Tester.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/trainer/Tester.cpp b/paddle/trainer/Tester.cpp index b1bb75654a9d5..d3b88019faa04 100644 --- a/paddle/trainer/Tester.cpp +++ b/paddle/trainer/Tester.cpp @@ -116,6 +116,7 @@ void Tester::testOnePeriod() { } testOneDataBatch(dataBatch, &outArgs); } + finishTestPeriod(); } void Tester::finishTestPeriod() { From d412a5ea21bff386f47ef79ef182d3e86466c175 Mon Sep 17 00:00:00 2001 From: Haichao-Zhang Date: Wed, 2 Nov 2016 13:05:59 -0700 Subject: [PATCH 205/324] add user_arg to LayerConfig (#315) --- proto/ModelConfig.proto.m4 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index 79e76b6bf1bdd..3fb96dd5fe00c 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -382,6 +382,15 @@ sinclude(`ModelConfigLayer.proto.m4') // bias size optional uint32 bias_size = 48 [default = 0]; + + // this parameter can be used as a user-defined parameter when necessary, + // without changing the proto file. + // e.g., when a new layer with a user-defined parameter is implemented, + // it can be used to pass that parameter, without modifying the proto file. + // string type is used for flexibility: different types can be converted + // to string and reinterpreted in the user's own layer implementation. + optional string user_arg = 49; + } message EvaluatorConfig { From 9f9b4afcdb5f0dfb0dd203a0d02af04163a23ab7 Mon Sep 17 00:00:00 2001 From: emailweixu Date: Wed, 2 Nov 2016 20:20:40 -0700 Subject: [PATCH 206/324] install the right python package version (#326) For multiple installation of paddle, there might be multiple versions of python package at opt/paddle/share/wheels/. We should install the right version. Ideally, we should remove the wrong versions when install. But it's not easy to do this with cmake. Change-Id: Ida8a8d60643ad9e42cf1c85776de9122d5ba1392 --- paddle/scripts/submit_local.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/scripts/submit_local.sh.in b/paddle/scripts/submit_local.sh.in index 4cf5f41f195df..213cf2f1cc7e4 100644 --- a/paddle/scripts/submit_local.sh.in +++ b/paddle/scripts/submit_local.sh.in @@ -68,7 +68,7 @@ EOF if [ $? -eq 1 ]; then # Older version installed, or not installed at all echo "First time run paddle, need to install some python dependencies." BASEDIR=$(dirname "$0") - pip install ${BASEDIR}/../opt/paddle/share/wheels/*.whl + pip install ${BASEDIR}/../opt/paddle/share/wheels/*-@PADDLE_VERSION@-*.whl if [ $? -ne 0 ]; then echo "pip install wheels failed. " echo "Please use 'sudo paddle' at the first time you use PaddlePaddle" From 5f2059db05bbc0590f4971198e034a56d1aa5915 Mon Sep 17 00:00:00 2001 From: lzhao4ever Date: Thu, 3 Nov 2016 09:24:16 -0700 Subject: [PATCH 207/324] Add matrix inverse (#240) * Add matrix inverse --- cmake/cblas.cmake | 16 ++++- paddle/cuda/include/hl_cuda_cublas.h | 24 +++++-- .../cuda/include/stub/hl_cuda_cublas_stub.h | 6 ++ paddle/cuda/src/hl_cuda_cublas.cc | 55 ++++++++++++++++ paddle/math/MathFunctions.cpp | 40 ++++++++++++ paddle/math/MathFunctions.h | 15 +++++ paddle/math/Matrix.cpp | 65 +++++++++++++++++++ paddle/math/Matrix.h | 20 ++++++ paddle/math/tests/test_matrixCompare.cpp | 29 ++++++++- 9 files changed, 261 insertions(+), 9 deletions(-) diff --git a/cmake/cblas.cmake b/cmake/cblas.cmake index 57c32a54cd727..685334c658506 100644 --- a/cmake/cblas.cmake +++ b/cmake/cblas.cmake @@ -1,4 +1,4 @@ -# Find the CBlas libraries +# Find the CBlas and lapack libraries # # It will search MKL, atlas, OpenBlas, reference-cblas in order. # @@ -19,6 +19,8 @@ set(MKL_ROOT $ENV{MKL_ROOT} CACHE PATH "Folder contains MKL") find_path(MKL_INCLUDE_DIR mkl.h PATHS ${MKL_ROOT}/include) +find_path(MKL_INCLUDE_DIR mkl_lapacke.h PATHS + ${MKL_ROOT}/include) find_library(MKL_CORE_LIB NAMES mkl_core PATHS ${MKL_ROOT}/lib ${MKL_ROOT}/lib/intel64) @@ -37,6 +39,7 @@ if(MKL_INCLUDE_DIR AND MKL_CORE_LIB AND MKL_SEQUENTIAL_LIB AND MKL_INTEL_LP64) ${MKL_SEQUENTIAL_LIB} ${MKL_CORE_LIB}) add_definitions(-DPADDLE_USE_MKL) + message(STATUS "Found MKL (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})") return() # return file. endif() @@ -55,15 +58,19 @@ set(ATLAS_LIB_SEARCH_PATHS ) find_path(ATLAS_INC_DIR NAMES cblas.h PATHS ${ATLAS_INCLUDE_SEARCH_PATHS}) +find_path(ATLAS_CLAPACK_INC_DIR NAMES clapack.h + PATHS ${ATLAS_INCLUDE_SEARCH_PATHS}) find_library(ATLAS_CBLAS_LIB NAMES cblas libcblas.so.3 PATHS ${ATLAS_LIB_SEARCH_PATHS}) -find_library(ATLAS_LIB NAMES atlas libatlas.so.3 +find_library(ATLAS_LIB NAMES lapack_atlas liblapack_atlas.so.3 PATHS ${ATLAS_LIB_SEARCH_PATHS}) if(ATLAS_INC_DIR AND ATLAS_CBLAS_LIB AND ATLAS_LIB) set(CBLAS_PROVIDER ATLAS) - set(CBLAS_INC_DIR ${ATLAS_INC_DIR}) + set(CBLAS_INC_DIR ${ATLAS_INC_DIR} ${ATLAS_CLAPACK_INC_DIR}) set(CBLAS_LIBS ${ATLAS_LIB} ${ATLAS_CBLAS_LIB}) + add_definitions(-DPADDLE_USE_ATLAS) + message(STATUS "Found Atlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})") return() endif() @@ -83,6 +90,8 @@ set(OPENBLAS_LIB_SEARCH_PATHS find_path(OPENBLAS_INC_DIR NAMES cblas.h PATHS ${OPENBLAS_INCLUDE_SEARCH_PATHS}) +find_path(OPENBLAS_LAPACKE_INC_DIR NAMES lapacke.h + PATHS ${OPENBLAS_INCLUDE_SEARCH_PATHS}) find_library(OPENBLAS_LIB NAMES openblas PATHS ${OPENBLAS_LIB_SEARCH_PATHS}) @@ -90,6 +99,7 @@ if(OPENBLAS_INC_DIR AND OPENBLAS_LIB) set(CBLAS_PROVIDER OPENBLAS) set(CBLAS_INC_DIR ${OPENBLAS_INC_DIR}) set(CBLAS_LIBS ${OPENBLAS_LIB}) + message(STATUS "Found OpenBlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})") return() endif() diff --git a/paddle/cuda/include/hl_cuda_cublas.h b/paddle/cuda/include/hl_cuda_cublas.h index 0ffbed18b5f9e..d757317eb4a97 100644 --- a/paddle/cuda/include/hl_cuda_cublas.h +++ b/paddle/cuda/include/hl_cuda_cublas.h @@ -21,8 +21,8 @@ limitations under the License. */ /** * @brief Matrix transpose: C_d = T(A_d) * - * @param[in] A_d input matrix (M x N). - * @param[out] C_d output matrix (N x M). + * @param[in] A_d input matrix (dimM x dimN). + * @param[out] C_d output matrix (dimN x dimM). * @param[in] dimM matrix height. * @param[in] dimN matrix width. * @param[in] lda the first dimension of A_d. @@ -39,8 +39,8 @@ extern void hl_matrix_transpose(real *A_d, /* * @brief Matrix transpose, while lda = dimN, ldc = dimM. * - * @param[in] A_d input matrix (M x N). - * @param[out] C_d output matrix (N x M). + * @param[in] A_d input matrix (dimM x dimN). + * @param[out] C_d output matrix (dimN x dimM). * @param[in] dimM matrix height. * @param[in] dimN matrix width. * @@ -50,6 +50,22 @@ extern void hl_matrix_transpose(real *A_d, int dimM, int dimN); +/* + * @brief Matrix inverse + * + * @param[in] A_d input matrix (dimN x dimN). + * @param[out] C_d output matrix (dimN x dimN). + * @param[in] dimN matrix height = matrix width + * @param[in] lda the first dimension of A_d + * @param[in] ldc the first dimension of C_d + * + */ +extern void hl_matrix_inverse(real *A_d, + real *C_d, + int dimN, + int lda, + int ldc); + /** * @brief C_d = alpha*(op(A_d) * op(B_d)) + beta*C_d * diff --git a/paddle/cuda/include/stub/hl_cuda_cublas_stub.h b/paddle/cuda/include/stub/hl_cuda_cublas_stub.h index 4a5e2a25a71b3..903dcbe8355d6 100644 --- a/paddle/cuda/include/stub/hl_cuda_cublas_stub.h +++ b/paddle/cuda/include/stub/hl_cuda_cublas_stub.h @@ -30,6 +30,12 @@ inline void hl_matrix_transpose(real *A_d, int dimM, int dimN) {} +inline void hl_matrix_inverse(real *A_d, + real *C_d, + int dimN, + int lda, + int ldc) {} + inline void hl_matrix_mul(real *A_d, hl_trans_op_t transa, real *B_d, hl_trans_op_t transb, real *C_d, diff --git a/paddle/cuda/src/hl_cuda_cublas.cc b/paddle/cuda/src/hl_cuda_cublas.cc index b3c9001ba3973..724ea490e8ea9 100644 --- a/paddle/cuda/src/hl_cuda_cublas.cc +++ b/paddle/cuda/src/hl_cuda_cublas.cc @@ -15,6 +15,7 @@ limitations under the License. */ #include #include +#include "hl_cuda.h" #include "hl_cuda_cublas.h" #include "hl_thread.ph" #include "hl_dso_loader.h" @@ -75,6 +76,8 @@ DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgemmBatched) DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgemmBatched) DYNAMIC_LOAD_CUBLAS_WRAP(cublasCgemmBatched) DYNAMIC_LOAD_CUBLAS_WRAP(cublasZgemmBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgetrfBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgetriBatched) CUBLAS_BLAS_ROUTINE_EACH(DYNAMIC_LOAD_CUBLAS_V2_WRAP) #undef DYNAMIC_LOAD_CUBLAS_WRAP @@ -88,10 +91,14 @@ CUBLAS_BLAS_ROUTINE_EACH(DYNAMIC_LOAD_CUBLAS_V2_WRAP) #define CUBLAS_GEAM dynload::cublasSgeam #define CUBLAS_GEMV dynload::cublasSgemv #define CUBLAS_GEMM dynload::cublasSgemm +#define CUBLAS_GETRF dynload::cublasSgetrfBatched +#define CUBLAS_GETRI dynload::cublasSgetriBatched #else #define CUBLAS_GEAM dynload::cublasDgeam #define CUBLAS_GEMV dynload::cublasDgemv #define CUBLAS_GEMM dynload::cublasDgemm +#define CUBLAS_GETRF dynload::cublasDgetrfBatched +#define CUBLAS_GETRI dynload::cublasDgetriBatched #endif const char* hl_cublas_get_error_string(cublasStatus_t status) { @@ -162,6 +169,54 @@ void hl_matrix_transpose(real *A_d, real *C_d, int dimM, int dimN) { hl_matrix_transpose(A_d, C_d, dimM, dimN, dimN, dimM); } +void hl_matrix_inverse(real *A_d, real *C_d, int dimN, int lda, int ldc) { + /* Solve Ax = I */ + CHECK_NOTNULL(A_d); + CHECK_NOTNULL(C_d); + + /* Step 1: Compute the LU decomposition of matrix A */ + real **inout_h = &A_d; + real **inout_d = (real **)hl_malloc_device(sizeof(real *)); + hl_memcpy(inout_d, inout_h, sizeof(real *)); + + int *pivot_d = (int *)hl_malloc_device(dimN*sizeof(int)); + int *info_d = (int *)t_resource.gpu_mem; + + /* Note: cublasSgetrfBatched is used to calculate a number of + small-sized matrices. There may be a better way to reconstruct + the API for better performance. + */ + CHECK_CUBLAS(CUBLAS_GETRF(t_resource.handle, + dimN, inout_d, lda, pivot_d, + info_d, 1)); + + int info_h; + hl_memcpy(&info_h, info_d, sizeof(int)); + if (info_h != 0) { + LOG(FATAL) << "Factorization of matrix failed: matrix may be singular.\n"; + } + + /* Step 2: Compute the inverse of the matrix given its LU decomposition */ + real **out_h = &C_d; + real **out_d = (real **)hl_malloc_device(sizeof(real *)); + hl_memcpy(out_d, out_h, sizeof(real *)); + + CHECK_CUBLAS(CUBLAS_GETRI(t_resource.handle, + dimN, (const real **)inout_d, lda, pivot_d, + out_d, ldc, info_d, 1)); + + hl_memcpy(&info_h, info_d, sizeof(int)); + if (info_h != 0) { + LOG(FATAL) << "Inversion of matrix failed: matrix may be singular.\n"; + } + + hl_free_mem_device(inout_d); + hl_free_mem_device(pivot_d); + hl_free_mem_device(out_d); + + CHECK_SYNC("hl_matrix_inverse failed"); +} + void hl_matrix_mul(real *A_d, hl_trans_op_t transa, real *B_d, hl_trans_op_t transb, real *C_d, diff --git a/paddle/math/MathFunctions.cpp b/paddle/math/MathFunctions.cpp index da493379e3a37..f8132066477db 100644 --- a/paddle/math/MathFunctions.cpp +++ b/paddle/math/MathFunctions.cpp @@ -39,6 +39,46 @@ void gemm(const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, beta, C, ldc); } +template<> +int getrf(const CBLAS_ORDER order, const int M, const int N, + float *A, const int lda, int *ipiv) { +#ifdef PADDLE_USE_ATLAS + return clapack_sgetrf(order, M, N, A, lda, ipiv); +#else + return LAPACKE_sgetrf(order, M, N, A, lda, ipiv); +#endif +} + +template<> +int getrf(const CBLAS_ORDER order, const int M, const int N, + double *A, const int lda, int *ipiv) { +#ifdef PADDLE_USE_ATLAS + return clapack_dgetrf(order, M, N, A, lda, ipiv); +#else + return LAPACKE_dgetrf(order, M, N, A, lda, ipiv); +#endif +} + +template<> +int getri(const CBLAS_ORDER order, const int N, float *A, + const int lda, const int *ipiv) { +#ifdef PADDLE_USE_ATLAS + return clapack_sgetri(order, N, A, lda, ipiv); +#else + return LAPACKE_sgetri(order, N, A, lda, ipiv); +#endif +} + +template<> +int getri(const CBLAS_ORDER order, const int N, double *A, + const int lda, const int *ipiv) { +#ifdef PADDLE_USE_ATLAS + return clapack_dgetri(order, N, A, lda, ipiv); +#else + return LAPACKE_dgetri(order, N, A, lda, ipiv); +#endif +} + template<> void axpy(const int n, const float alpha, const float* x, float* y) { cblas_saxpy(n, alpha, x, 1, y, 1); diff --git a/paddle/math/MathFunctions.h b/paddle/math/MathFunctions.h index 43075977dc9ce..cad0e4740b8c1 100644 --- a/paddle/math/MathFunctions.h +++ b/paddle/math/MathFunctions.h @@ -21,6 +21,13 @@ limitations under the License. */ extern "C" { #include } +#ifdef PADDLE_USE_ATLAS +extern "C" { +#include +} +#else +#include +#endif #endif #include @@ -34,6 +41,14 @@ void gemm(const CBLAS_TRANSPOSE transA, const CBLAS_TRANSPOSE transB, const T* B, const int ldb, const T beta, T* C, const int ldc); +template +int getrf(const CBLAS_ORDER Order, const int M, const int N, + T *A, const int lda, int *ipiv); + +template +int getri(const CBLAS_ORDER Order, const int N, T *A, + const int lda, const int *ipiv); + template void axpy(const int n, const T alpha, const T* x, T* y); diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index aaeae98f0d28b..d901ba93492ac 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -335,6 +335,30 @@ void GpuMatrix::transpose(MatrixPtr matTrans, bool memAlloc) { hl_matrix_transpose(data, dataTrans, height_, width_, lda, ldc); } + +MatrixPtr GpuMatrix::getInverse() { + MatrixPtr matInv; + inverse(matInv, true); + return matInv; +} + +void GpuMatrix::inverse(MatrixPtr matInv, bool memAlloc) { + CHECK_EQ(height_, width_); + + if (memAlloc) { + matInv = std::make_shared(height_, width_); + } else { + CHECK(matInv != NULL); + } + + real* data = getData(); + real* dataInv = matInv->getData(); + int lda = getStride(); + int ldc = matInv->getStride(); + + hl_matrix_inverse(data, dataInv, height_, lda, ldc); +} + void GpuMatrix::addBias(Matrix& b, real scale) { CHECK(b.getHeight() == 1) << "the Bias should be a vector"; BaseMatrix::addBias(b, scale); @@ -1437,6 +1461,47 @@ void CpuMatrix::transpose(MatrixPtr matTrans, bool memAlloc) { } } + +MatrixPtr CpuMatrix::getInverse() { + MatrixPtr matInv; + inverse(matInv, true); + return matInv; +} + +void CpuMatrix::inverse(MatrixPtr matInv, bool memAlloc) { + CHECK_EQ(height_, width_); + + if (memAlloc) { + matInv = std::make_shared(height_, width_); + } else { + CHECK(matInv != NULL); + } + + CHECK_EQ(height_, matInv->getHeight()); + CHECK_EQ(width_, matInv->getWidth()); + matInv->copyFrom(*this); + + real* data = getData(); + real* dataInv = matInv->getData(); + int ldc = matInv->getStride(); + + if (height_ == 1) { + CHECK_NE(*data, 0); + *dataInv = 1.0 / (*data); + return; + } + + /* Compute the LU decomposition of the matrix */ + std::vector ipiv(height_); + CBLAS_ORDER order = (matInv->isTransposed() ? CblasColMajor : CblasRowMajor); + int info = getrf(order, height_, height_, dataInv, ldc, ipiv.data()); + CHECK_EQ(info, 0); + + /* Compute the inverse of the matrix given its LU decompsotion */ + info = getri(order, height_, dataInv, ldc, ipiv.data()); + CHECK_EQ(info, 0); +} + void CpuMatrix::convExpand(Matrix& feature, int feaImgHeight, int feaImgWidth, int channels, int blockH, int blockW, int strideH, int strideW, int paddingH, int paddingW, diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index 52cbed528ca8b..293d13f4d6d5a 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -328,6 +328,20 @@ class Matrix : public BaseMatrix { LOG(FATAL) << "Not implemented"; } + virtual MatrixPtr getInverse() { + LOG(FATAL) << "Not implemented"; + } + + /** + * @brief inverse. + * + * if allocate matInv's memory outside, then set memAlloc as false; + * else set as true. + */ + virtual void inverse(MatrixPtr matInv, bool memAlloc) { + LOG(FATAL) << "Not implemented"; + } + public: /// Only set all variables to 0 or NULL but not free them. virtual void clear() { @@ -1043,6 +1057,9 @@ class GpuMatrix : public Matrix { MatrixPtr getTranspose(); void transpose(MatrixPtr matTrans, bool memAlloc); + MatrixPtr getInverse(); + void inverse(MatrixPtr matInv, bool memAlloc); + /// add b to each sample of this. void addBias(Matrix& b, real scale); void addSharedBias(Matrix& b, real scale); @@ -1282,6 +1299,9 @@ class CpuMatrix : public Matrix { MatrixPtr getTranspose(); void transpose(MatrixPtr matTrans, bool memAlloc); + MatrixPtr getInverse(); + void inverse(MatrixPtr matInv, bool memAlloc); + void copyFrom(const Matrix& src); void copyFrom(const Matrix& src, hl_stream_t stream); diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index 0ddf7e0dfc386..b887cccaaa14e 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -641,9 +641,32 @@ void testMatrixTranspose(int height, int width) { MatrixCheckEqual(*cpuT, *outputCheck); } +void testMatrixInverse(int height) { + MatrixPtr cpu = std::make_shared(height, height); + MatrixPtr gpu = std::make_shared(height, height); + MatrixPtr cpuI = std::make_shared(height, height); + MatrixPtr gpuI = std::make_shared(height, height); + + cpu->randomizeUniform(); + gpu->copyFrom(*cpu); + cpu->inverse(cpuI, false); + gpu->inverse(gpuI, false); + + MatrixPtr outputCheck = std::make_shared(height, height); + outputCheck->copyFrom(*gpuI); + MatrixCheckErr(*cpuI, *outputCheck); + + outputCheck->mul(cpu, cpuI); + cpu->zeroMem(); + for (int i = 0; i < height; i++) { + cpu->getRowBuf(i)[i] = 1.0; + } + MatrixCheckErr(*cpu, *outputCheck); +} + TEST(Matrix, unary) { - for (auto height : {1, 11, 73, 128, 200, 330}) { - for (auto width : {1, 32, 100, 512, 1000, 3210}) { + for (auto height : {1, 3, 11, 73, 128, 200, 330}) { + for (auto width : {1, 3, 32, 100, 512, 1000, 3210}) { VLOG(3) << " height=" << height << " width=" << width; // applyUnary @@ -675,6 +698,8 @@ TEST(Matrix, unary) { // transpose testMatrixTranspose(height, width); } + // inverse + testMatrixInverse(height); } } From 1de75c039fb7e6c9bb433f91c574d264eb397f9c Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Fri, 4 Nov 2016 00:40:37 +0800 Subject: [PATCH 208/324] report error when use parallel_nn to train recurrent_nn model (#335) --- paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp b/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp index 952df60a7d786..22698f5867017 100644 --- a/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp +++ b/paddle/gserver/gradientmachines/ParallelNeuralNetwork.cpp @@ -28,6 +28,12 @@ void ParallelNeuralNetwork::init( const std::vector& parameterTypes, bool useGpu) { NeuralNetwork::init(config, callback, parameterTypes, useGpu); + if (config.type() == "recurrent_nn") { + LOG(FATAL) + << "You can not add `--parallel_nn=true` on the command line, " + << "parallel_nn training mode does not support the recurrent_nn model."; + } + useGpu_ = useGpu; numDevices_ = 0; if (useGpu_) { From f6f80b93788ad6ba56158df070f007c6ccef28aa Mon Sep 17 00:00:00 2001 From: emailweixu Date: Thu, 3 Nov 2016 19:35:18 -0700 Subject: [PATCH 209/324] install the right python package version (#340) For multiple installation of paddle, there might be multiple versions of python package at opt/paddle/share/wheels/. We should install the right version. Ideally, we should remove the wrong versions when install. But it's not easy to do this with cmake. Change-Id: Ida8a8d60643ad9e42cf1c85776de9122d5ba1392 From ed83a1d6b68d2bd1f573a0c4cabf811e9c32aead Mon Sep 17 00:00:00 2001 From: wangkuiyi Date: Thu, 3 Nov 2016 22:15:20 -0700 Subject: [PATCH 210/324] Fix minor errors in instructions of building Paddle on Mac OS X (#347) --- doc/build/build_from_source.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index 7727c8c3788b9..c37234d3ef14d 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -219,10 +219,9 @@ easy_install pip # Install google test on Mac OS X # Download gtest 1.7.0 wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz - tar -xvf googletest-release-1.7.0.tar.gz && cd googletest-release-1.7.0 + tar -xzf googletest-release-1.7.0.tar.gz && cd googletest-release-1.7.0 # Build gtest - mkdir build && cmake .. - make + mkdir build && cd build && cmake .. && make # Install gtest library sudo cp -r ../include/gtest /usr/local/include/ sudo cp lib*.a /usr/local/lib From 3424a4c0d898820442b648c31b510403fea7c994 Mon Sep 17 00:00:00 2001 From: gangliao Date: Thu, 3 Nov 2016 22:39:39 -0700 Subject: [PATCH 211/324] Fix bug and redundant code in hl_dso_loader.cc (#306) --- paddle/cuda/src/hl_cuda_cudnn.cc | 73 ++++++++----------------------- paddle/cuda/src/hl_cuda_device.cc | 31 +++---------- paddle/cuda/src/hl_dso_loader.cc | 4 +- 3 files changed, 27 insertions(+), 81 deletions(-) diff --git a/paddle/cuda/src/hl_cuda_cudnn.cc b/paddle/cuda/src/hl_cuda_cudnn.cc index 7810d0d10053d..92b28e4345c3d 100644 --- a/paddle/cuda/src/hl_cuda_cudnn.cc +++ b/paddle/cuda/src/hl_cuda_cudnn.cc @@ -41,65 +41,28 @@ void* cudnn_dso_handle = nullptr; #ifdef PADDLE_USE_DSO -#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - cudnnStatus_t operator()(Args... args) { \ - typedef cudnnStatus_t (*cudnnFunc)(Args...); \ - std::call_once(cudnn_dso_flag, GetCudnnDsoHandle, \ - &cudnn_dso_handle); \ - void* p_##__name = dlsym(cudnn_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ - } \ +#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + using cudnn_func = decltype(__name(args...))(*)(Args...); \ + std::call_once(cudnn_dso_flag, GetCudnnDsoHandle, \ + &cudnn_dso_handle); \ + void* p_##__name = dlsym(cudnn_dso_handle, #__name); \ + return reinterpret_cast(p_##__name)(args...); \ + } \ } __name; /* struct DynLoad__##__name */ -struct DynLoad__cudnnGetVersion { - template - size_t operator()(Args... args) { - typedef size_t (*cudnnFunc)(Args...); - std::call_once(cudnn_dso_flag, GetCudnnDsoHandle, - &cudnn_dso_handle); - void* p_name = dlsym(cudnn_dso_handle, "cudnnGetVersion"); - return reinterpret_cast(p_name)(args...); - } -} cudnnGetVersion; /* struct DynLoad__##__name */ - -struct DynLoad__cudnnGetErrorString { - template - const char* operator()(Args... args) { - typedef const char* (*cudnnFunc)(Args...); - std::call_once(cudnn_dso_flag, GetCudnnDsoHandle, - &cudnn_dso_handle); - void* p_name = dlsym(cudnn_dso_handle, "cudnnGetErrorString"); - return reinterpret_cast(p_name)(args...); - } -} cudnnGetErrorString; /* struct DynLoad__##__name */ - - #else -#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ - struct DynLoad__##__name { \ - template \ - cudnnStatus_t operator()(Args... args) { \ - return __name(args...); \ - } \ +#define DYNAMIC_LOAD_CUDNN_WRAP(__name) \ + struct DynLoad__##__name { \ + template \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + return __name(args...); \ + } \ } __name; /* struct DynLoad__##__name */ -struct DynLoad__cudnnGetVersion { - template - size_t operator()(Args... args) { - return cudnnGetVersion(args...); - } -} cudnnGetVersion; /* struct DynLoad__##__name */ - -struct DynLoad__cudnnGetErrorString { - template - const char* operator()(Args... args) { - return cudnnGetErrorString(args...); - } -} cudnnGetErrorString; /* struct DynLoad__##__name */ - #endif /** @@ -133,7 +96,9 @@ struct DynLoad__cudnnGetErrorString { __macro(cudnnPoolingForward) \ __macro(cudnnPoolingBackward) \ __macro(cudnnSoftmaxBackward) \ - __macro(cudnnSoftmaxForward) + __macro(cudnnSoftmaxForward) \ + __macro(cudnnGetVersion) \ + __macro(cudnnGetErrorString) CUDNN_DNN_ROUTINE_EACH(DYNAMIC_LOAD_CUDNN_WRAP) #define CUDNN_DNN_ROUTINE_EACH_R2(__macro) \ diff --git a/paddle/cuda/src/hl_cuda_device.cc b/paddle/cuda/src/hl_cuda_device.cc index e9fe9f1c117a0..3ea2c91bd5a41 100644 --- a/paddle/cuda/src/hl_cuda_device.cc +++ b/paddle/cuda/src/hl_cuda_device.cc @@ -85,44 +85,24 @@ void* cudart_dso_handle = nullptr; #define DYNAMIC_LOAD_CUDART_WRAP(__name) \ struct DynLoad__##__name { \ template \ - cudaError_t operator()(Args... args) { \ - typedef cudaError_t (*cudartFunc)(Args...); \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ + using cudart_func = decltype(__name(args...))(*)(Args...); \ std::call_once(cudart_dso_flag, GetCudartDsoHandle, \ &cudart_dso_handle); \ void* p_##__name = dlsym(cudart_dso_handle, #__name); \ - return reinterpret_cast(p_##__name)(args...); \ + return reinterpret_cast(p_##__name)(args...); \ } \ } __name; /* struct DynLoad__##__name */ #else #define DYNAMIC_LOAD_CUDART_WRAP(__name) \ struct DynLoad__##__name { \ template \ - cudaError_t operator()(Args... args) { \ + auto operator()(Args... args) -> decltype(__name(args...)) { \ return __name(args...); \ } \ } __name; /* struct DynLoad__##__name */ #endif -#ifdef PADDLE_USE_DSO - struct DynLoad__cudaGetErrorString { - template - const char* operator()(Args... args) { - typedef const char* (*cudaFunc)(Args...); - std::call_once(cudart_dso_flag, GetCudartDsoHandle, - &cudart_dso_handle); - void* p_func = dlsym(cudart_dso_handle, "cudaGetErrorString"); - return reinterpret_cast(p_func)(args...); - } - } cudaGetErrorString; /* struct DynLoad__cudaGetErrorString */ -#else -struct DynLoad__cudaGetErrorString { - template - const char* operator()(Args... args) { - return cudaGetErrorString(args...); - } -} cudaGetErrorString; /* struct DynLoad__cudaGetErrorString */ -#endif - /* include all needed cuda functions in HPPL */ #define CUDA_ROUTINE_EACH(__macro) \ __macro(cudaMalloc) \ @@ -152,7 +132,8 @@ struct DynLoad__cudaGetErrorString { __macro(cudaSetDeviceFlags) \ __macro(cudaGetLastError) \ __macro(cudaFuncSetCacheConfig) \ - __macro(cudaRuntimeGetVersion) + __macro(cudaRuntimeGetVersion) \ + __macro(cudaGetErrorString) CUDA_ROUTINE_EACH(DYNAMIC_LOAD_CUDART_WRAP) diff --git a/paddle/cuda/src/hl_dso_loader.cc b/paddle/cuda/src/hl_dso_loader.cc index 91c60d85a1e41..c0b5d6e357fc7 100644 --- a/paddle/cuda/src/hl_dso_loader.cc +++ b/paddle/cuda/src/hl_dso_loader.cc @@ -49,14 +49,14 @@ static inline std::string join(const std::string& part1, const std::string& part static inline void GetDsoHandleFromDefaultPath( std::string& dso_path, void** dso_handle, int dynload_flags) { LOG(INFO) << "Try to find cuda library: " << dso_path - << "from default system path."; + << " from default system path."; // default search from LD_LIBRARY_PATH/DYLD_LIBRARY_PATH *dso_handle = dlopen(dso_path.c_str(), dynload_flags); // DYLD_LIBRARY_PATH is disabled after Mac OS 10.11 to // bring System Integrity Projection (SIP), if dso_handle // is null, search from default package path in Mac OS. - #if defined(__APPLE__) or defined(__OSX__) + #if defined(__APPLE__) || defined(__OSX__) if (nullptr == *dso_handle) { dso_path = join("/usr/local/cuda/lib/", dso_path); *dso_handle = dlopen(dso_path.c_str(), dynload_flags); From 33004ecfb75d15a91eedb9ec7a5bc182ff96c6f8 Mon Sep 17 00:00:00 2001 From: gangliao Date: Fri, 4 Nov 2016 00:43:24 -0700 Subject: [PATCH 212/324] Fix glog check type unmatch in Util.cpp (#353) * Fix glog check type unmatch in Util.cpp #352 --- paddle/utils/Util.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/utils/Util.cpp b/paddle/utils/Util.cpp index 45251213d2d79..2cdff9d1aca92 100644 --- a/paddle/utils/Util.cpp +++ b/paddle/utils/Util.cpp @@ -106,7 +106,7 @@ pid_t getTID() { #endif pid_t tid = syscall(__NR_gettid); #endif - CHECK_NE(tid, -1); + CHECK_NE((int)tid, -1); return tid; } From 3e2dc77cc715c4b5f78a42f87e3800de0a0f3623 Mon Sep 17 00:00:00 2001 From: gangliao Date: Fri, 4 Nov 2016 02:52:53 -0700 Subject: [PATCH 213/324] Add code coverage and coveralls (#296) --- .travis.yml | 2 + CMakeLists.txt | 4 + README.md | 8 +- cmake/coveralls.cmake | 103 ++++++ cmake/coverallsGcovJsons.cmake | 403 ++++++++++++++++++++++++ paddle/scripts/travis/build_and_test.sh | 8 +- 6 files changed, 522 insertions(+), 6 deletions(-) create mode 100644 cmake/coveralls.cmake create mode 100644 cmake/coverallsGcovJsons.cmake diff --git a/.travis.yml b/.travis.yml index bf0e0b7bbddd4..7812ac0283789 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,6 +35,8 @@ addons: - libgoogle-glog-dev - libgflags-dev - libgtest-dev + - curl + - lcov - graphviz before_install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 527064e31000a..93cdcd4a75b70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,9 @@ option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ${GTEST_FOUND}) option(WITH_DOC "Compile PaddlePaddle with documentation" OFF) option(WITH_SWIG_PY "Compile PaddlePaddle with py PaddlePaddle prediction api" ${SWIG_FOUND}) option(ON_TRAVIS "Running test on travis-ci or not." OFF) +option(ON_COVERALLS "Generating code coverage data on coveralls or not." OFF) +option(COVERALLS_UPLOAD "Uploading the generated coveralls json." ON) + if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel" @@ -54,6 +57,7 @@ include(flags) include(cudnn) include(FindPythonModule) include(check_packages) +include(coveralls) # add PaddlePaddle version if(DEFINED ENV{PADDLE_VERSION}) diff --git a/README.md b/README.md index 1cc0444c0617a..66767d7ff8e4a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ # PaddlePaddle -| **`Linux`** | **`License`** | **`Chat Room`** | -|----------------|---------------|-----------------| -|[![Build Status](https://travis-ci.org/baidu/Paddle.svg?branch=master)](https://travis-ci.org/baidu/Paddle)|[![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](LICENSE)|[![Join the chat at https://gitter.im/PaddlePaddle/Deep_Learning](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/PaddlePaddle/Deep_Learning?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)| + +[![Build Status](https://travis-ci.org/baidu/Paddle.svg?branch=master)](https://travis-ci.org/baidu/Paddle) +[![Coverage Status](https://coveralls.io/repos/github/baidu/Paddle/badge.svg?branch=develop)](https://coveralls.io/github/baidu/Paddle?branch=develop) +[![Join the chat at https://gitter.im/PaddlePaddle/Deep_Learning](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/PaddlePaddle/Deep_Learning?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![License](https://img.shields.io/badge/license-Apache%202.0-green.svg)](LICENSE) Welcome to the PaddlePaddle GitHub. diff --git a/cmake/coveralls.cmake b/cmake/coveralls.cmake new file mode 100644 index 0000000000000..9be7643819efd --- /dev/null +++ b/cmake/coveralls.cmake @@ -0,0 +1,103 @@ +# CMake script for code coverage. +# If _COVERALLS_UPLOAD is ON, it will upload json files to overalls.io automatically. + +# Param _COVERAGE_SRCS A list of coverage source files. +# Param _COVERALLS_UPLOAD Upload the result to coveralls. +# Param _CMAKE_SCRIPT_PATH CMake script path. +function(code_coverage _COVERAGE_SRCS _COVERALLS_UPLOAD _CMAKE_SCRIPT_PATH) + # clean previous gcov data. + file(REMOVE_RECURSE ${PROJECT_BINARY_DIR}/*.gcda) + + # find curl for upload JSON soon. + if (_COVERALLS_UPLOAD) + find_program(CURL_EXECUTABLE curl) + if (NOT CURL_EXECUTABLE) + message(FATAL_ERROR "Coveralls: curl not found!") + endif() + endif() + + # When passing a CMake list to an external process, the list + # will be converted from the format "1;2;3" to "1 2 3". + set(COVERAGE_SRCS "") + foreach (SINGLE_SRC ${_COVERAGE_SRCS}) + set(COVERAGE_SRCS "${COVERAGE_SRCS}*${SINGLE_SRC}") + endforeach() + + # query number of logical cores + cmake_host_system_information(RESULT core_size QUERY NUMBER_OF_LOGICAL_CORES) + # coveralls json file. + set(COVERALLS_FILE ${PROJECT_BINARY_DIR}/coveralls.json) + add_custom_target(coveralls_generate + # Run regress tests. + COMMAND ${CMAKE_CTEST_COMMAND} + -j ${core_size} + --output-on-failure + # Generate Gcov and translate it into coveralls JSON. + COMMAND ${CMAKE_COMMAND} + -DCOVERAGE_SRCS="${COVERAGE_SRCS}" + -DCOVERALLS_OUTPUT_FILE="${COVERALLS_FILE}" + -DCOV_PATH="${PROJECT_BINARY_DIR}" + -DPROJECT_ROOT="${PROJECT_SOURCE_DIR}" + -P "${_CMAKE_SCRIPT_PATH}/coverallsGcovJsons.cmake" + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + COMMENT "Coveralls: generating coveralls output..." + ) + + if (_COVERALLS_UPLOAD) + message("COVERALLS UPLOAD: ON") + # Upload the JSON to coveralls. + add_custom_target(coveralls_upload + COMMAND ${CURL_EXECUTABLE} + -S -F json_file=@${COVERALLS_FILE} + https://coveralls.io/api/v1/jobs + DEPENDS coveralls_generate + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + COMMENT "Coveralls: uploading coveralls output...") + + add_custom_target(coveralls DEPENDS coveralls_upload) + else() + message("COVERALLS UPLOAD: OFF") + add_custom_target(coveralls DEPENDS coveralls_generate) + endif() +endfunction() + +if(ON_COVERALLS) + set(CMAKE_BUILD_TYPE "Debug") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O0 -fprofile-arcs -ftest-coverage") + + set(EXCLUDE_DIRS + "demo/" + "build/" + "tests/" + ".test_env/" + ) + + if(WITH_GPU) + file(GLOB_RECURSE PADDLE_SOURCES RELATIVE "${PROJECT_SOURCE_DIR}" "*.cpp" "*.cc" ".c" "*.cu") + else() + file(GLOB_RECURSE PADDLE_SOURCES RELATIVE "${PROJECT_SOURCE_DIR}" "*.cpp" "*.cc" "*.c") + endif() + + # exclude trivial files in PADDLE_SOURCES + foreach(EXCLUDE_DIR ${EXCLUDE_DIRS}) + foreach(TMP_PATH ${PADDLE_SOURCES}) + string(FIND ${TMP_PATH} ${EXCLUDE_DIR} EXCLUDE_DIR_FOUND) + if(NOT ${EXCLUDE_DIR_FOUND} EQUAL -1) + list(REMOVE_ITEM PADDLE_SOURCES ${TMP_PATH}) + endif() + endforeach(TMP_PATH) + endforeach() + + # convert to absolute path + set(PADDLE_SRCS "") + foreach(PADDLE_SRC ${PADDLE_SOURCES}) + set(PADDLE_SRCS "${PADDLE_SRCS};${PROJECT_SOURCE_DIR}/${PADDLE_SRC}") + endforeach() + + code_coverage( + "${PADDLE_SRCS}" + ${COVERALLS_UPLOAD} + "${PROJECT_SOURCE_DIR}/cmake" + ) +endif() diff --git a/cmake/coverallsGcovJsons.cmake b/cmake/coverallsGcovJsons.cmake new file mode 100644 index 0000000000000..ae3530c3a0eeb --- /dev/null +++ b/cmake/coverallsGcovJsons.cmake @@ -0,0 +1,403 @@ +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Copyright (C) 2014 Joakim Söderberg +# +# This is intended to be run by a custom target in a CMake project like this. +# 0. Compile program with coverage support. +# 1. Clear coverage data. (Recursively delete *.gcda in build dir) +# 2. Run the unit tests. +# 3. Run this script specifying which source files the coverage should be performed on. +# +# This script will then use gcov to generate .gcov files in the directory specified +# via the COV_PATH var. This should probably be the same as your cmake build dir. +# +# It then parses the .gcov files to convert them into the Coveralls JSON format: +# https://coveralls.io/docs/api +# + +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) + +# Since it's not possible to pass a CMake list properly in the +# "1;2;3" format to an external process, we have replaced the +# ";" with "*", so reverse that here so we get it back into the +# CMake list format. +string(REGEX REPLACE "\\*" ";" COVERAGE_SRCS ${COVERAGE_SRCS}) + +find_program(GCOV_EXECUTABLE gcov) +if (NOT GCOV_EXECUTABLE) + message(FATAL_ERROR "gcov not found! Aborting...") +endif() + +find_package(Git) + +# TODO: Add these git things to the coveralls json. +if (GIT_FOUND) + # Branch. + execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE GIT_BRANCH + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + macro (git_log_format FORMAT_CHARS VAR_NAME) + execute_process( + COMMAND ${GIT_EXECUTABLE} log -1 --pretty=format:%${FORMAT_CHARS} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + OUTPUT_VARIABLE ${VAR_NAME} + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + endmacro() + + git_log_format(an GIT_AUTHOR_EMAIL) + git_log_format(ae GIT_AUTHOR_EMAIL) + git_log_format(cn GIT_COMMITTER_NAME) + git_log_format(ce GIT_COMMITTER_EMAIL) + git_log_format(B GIT_COMMIT_MESSAGE) + + message("Git exe: ${GIT_EXECUTABLE}") + message("Git branch: ${GIT_BRANCH}") + message("Git author: ${GIT_AUTHOR_NAME}") + message("Git e-mail: ${GIT_AUTHOR_EMAIL}") + message("Git commiter name: ${GIT_COMMITTER_NAME}") + message("Git commiter e-mail: ${GIT_COMMITTER_EMAIL}") + message("Git commit message: ${GIT_COMMIT_MESSAGE}") + +endif() + +############################# Macros ######################################### + +# +# This macro converts from the full path format gcov outputs: +# +# /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov +# +# to the original source file path the .gcov is for: +# +# /path/to/project/root/subdir/the_file.c +# +macro(get_source_path_from_gcov_filename _SRC_FILENAME _GCOV_FILENAME) + + # /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov + # -> + # #path#to#project#root#subdir#the_file.c.gcov + get_filename_component(_GCOV_FILENAME_WEXT ${_GCOV_FILENAME} NAME) + + # #path#to#project#root#subdir#the_file.c.gcov -> /path/to/project/root/subdir/the_file.c + string(REGEX REPLACE "\\.gcov$" "" SRC_FILENAME_TMP ${_GCOV_FILENAME_WEXT}) + string(REGEX REPLACE "\#" "/" SRC_FILENAME_TMP ${SRC_FILENAME_TMP}) + set(${_SRC_FILENAME} "${SRC_FILENAME_TMP}") +endmacro() + +############################################################################## + +# Get the coverage data. +file(GLOB_RECURSE GCDA_FILES "${COV_PATH}" "*.gcda") +message("GCDA files:") + +# Get a list of all the object directories needed by gcov +# (The directories the .gcda files and .o files are found in) +# and run gcov on those. +foreach(GCDA ${GCDA_FILES}) + message("Process: ${GCDA}") + message("------------------------------------------------------------------------------") + get_filename_component(GCDA_DIR ${GCDA} PATH) + + # + # The -p below refers to "Preserve path components", + # This means that the generated gcov filename of a source file will + # keep the original files entire filepath, but / is replaced with #. + # Example: + # + # /path/to/project/root/build/CMakeFiles/the_file.dir/subdir/the_file.c.gcda + # ------------------------------------------------------------------------------ + # File '/path/to/project/root/subdir/the_file.c' + # Lines executed:68.34% of 199 + # /path/to/project/root/subdir/the_file.c:creating '#path#to#project#root#subdir#the_file.c.gcov' + # + # If -p is not specified then the file is named only "the_file.c.gcov" + # + execute_process( + COMMAND ${GCOV_EXECUTABLE} -p -o ${GCDA_DIR} ${GCDA} + WORKING_DIRECTORY ${GCDA_DIR} + ) +endforeach() + +# TODO: Make these be absolute path +file(GLOB_RECURSE ALL_GCOV_FILES "${COV_PATH}" "*.gcov") + +# Get only the filenames to use for filtering. +#set(COVERAGE_SRCS_NAMES "") +#foreach (COVSRC ${COVERAGE_SRCS}) +# get_filename_component(COVSRC_NAME ${COVSRC} NAME) +# message("${COVSRC} -> ${COVSRC_NAME}") +# list(APPEND COVERAGE_SRCS_NAMES "${COVSRC_NAME}") +#endforeach() + +# +# Filter out all but the gcov files we want. +# +# We do this by comparing the list of COVERAGE_SRCS filepaths that the +# user wants the coverage data for with the paths of the generated .gcov files, +# so that we only keep the relevant gcov files. +# +# Example: +# COVERAGE_SRCS = +# /path/to/project/root/subdir/the_file.c +# +# ALL_GCOV_FILES = +# /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov +# /path/to/project/root/build/#path#to#project#root#subdir#other_file.c.gcov +# +# Result should be: +# GCOV_FILES = +# /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov +# +set(GCOV_FILES "") +#message("Look in coverage sources: ${COVERAGE_SRCS}") +message("\nFilter out unwanted GCOV files:") +message("===============================") + +set(COVERAGE_SRCS_REMAINING ${COVERAGE_SRCS}) + +foreach (GCOV_FILE ${ALL_GCOV_FILES}) + + # + # /path/to/project/root/build/#path#to#project#root#subdir#the_file.c.gcov + # -> + # /path/to/project/root/subdir/the_file.c + get_source_path_from_gcov_filename(GCOV_SRC_PATH ${GCOV_FILE}) + + # Is this in the list of source files? + # TODO: We want to match against relative path filenames from the source file root... + list(FIND COVERAGE_SRCS ${GCOV_SRC_PATH} WAS_FOUND) + + if (NOT WAS_FOUND EQUAL -1) + message("YES: ${GCOV_FILE}") + list(APPEND GCOV_FILES ${GCOV_FILE}) + + # We remove it from the list, so we don't bother searching for it again. + # Also files left in COVERAGE_SRCS_REMAINING after this loop ends should + # have coverage data generated from them (no lines are covered). + list(REMOVE_ITEM COVERAGE_SRCS_REMAINING ${GCOV_SRC_PATH}) + else() + message("NO: ${GCOV_FILE}") + endif() +endforeach() + +# TODO: Enable setting these +set(JSON_SERVICE_NAME "travis-ci") +set(JSON_SERVICE_JOB_ID $ENV{TRAVIS_JOB_ID}) + +set(JSON_TEMPLATE +"{ + \"service_name\": \"\@JSON_SERVICE_NAME\@\", + \"service_job_id\": \"\@JSON_SERVICE_JOB_ID\@\", + \"source_files\": \@JSON_GCOV_FILES\@ +}" +) + +set(SRC_FILE_TEMPLATE +"{ + \"name\": \"\@GCOV_SRC_REL_PATH\@\", + \"source_digest\": \"\@GCOV_CONTENTS_MD5\@\", + \"coverage\": \@GCOV_FILE_COVERAGE\@ + }" +) + +message("\nGenerate JSON for files:") +message("=========================") + +set(JSON_GCOV_FILES "[") + +# Read the GCOV files line by line and get the coverage data. +foreach (GCOV_FILE ${GCOV_FILES}) + + get_source_path_from_gcov_filename(GCOV_SRC_PATH ${GCOV_FILE}) + file(RELATIVE_PATH GCOV_SRC_REL_PATH "${PROJECT_ROOT}" "${GCOV_SRC_PATH}") + + # The new coveralls API doesn't need the entire source (Yay!) + # However, still keeping that part for now. Will cleanup in the future. + file(MD5 "${GCOV_SRC_PATH}" GCOV_CONTENTS_MD5) + message("MD5: ${GCOV_SRC_PATH} = ${GCOV_CONTENTS_MD5}") + + # Loads the gcov file as a list of lines. + # (We first open the file and replace all occurences of [] with _ + # because CMake will fail to parse a line containing unmatched brackets... + # also the \ to escaped \n in macros screws up things.) + # https://public.kitware.com/Bug/view.php?id=15369 + file(READ ${GCOV_FILE} GCOV_CONTENTS) + string(REPLACE "[" "_" GCOV_CONTENTS "${GCOV_CONTENTS}") + string(REPLACE "]" "_" GCOV_CONTENTS "${GCOV_CONTENTS}") + string(REPLACE "\\" "_" GCOV_CONTENTS "${GCOV_CONTENTS}") + file(WRITE ${GCOV_FILE}_tmp "${GCOV_CONTENTS}") + + file(STRINGS ${GCOV_FILE}_tmp GCOV_LINES) + list(LENGTH GCOV_LINES LINE_COUNT) + + # Instead of trying to parse the source from the + # gcov file, simply read the file contents from the source file. + # (Parsing it from the gcov is hard because C-code uses ; in many places + # which also happens to be the same as the CMake list delimeter). + file(READ ${GCOV_SRC_PATH} GCOV_FILE_SOURCE) + + string(REPLACE "\\" "\\\\" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + string(REGEX REPLACE "\"" "\\\\\"" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + string(REPLACE "\t" "\\\\t" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + string(REPLACE "\r" "\\\\r" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + string(REPLACE "\n" "\\\\n" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + # According to http://json.org/ these should be escaped as well. + # Don't know how to do that in CMake however... + #string(REPLACE "\b" "\\\\b" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + #string(REPLACE "\f" "\\\\f" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + #string(REGEX REPLACE "\u([a-fA-F0-9]{4})" "\\\\u\\1" GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}") + + # We want a json array of coverage data as a single string + # start building them from the contents of the .gcov + set(GCOV_FILE_COVERAGE "[") + + set(GCOV_LINE_COUNT 1) # Line number for the .gcov. + set(DO_SKIP 0) + foreach (GCOV_LINE ${GCOV_LINES}) + #message("${GCOV_LINE}") + # Example of what we're parsing: + # Hitcount |Line | Source + # " 8: 26: if (!allowed || (strlen(allowed) == 0))" + string(REGEX REPLACE + "^([^:]*):([^:]*):(.*)$" + "\\1;\\2;\\3" + RES + "${GCOV_LINE}") + + # Check if we should exclude lines using the Lcov syntax. + string(REGEX MATCH "LCOV_EXCL_START" START_SKIP "${GCOV_LINE}") + string(REGEX MATCH "LCOV_EXCL_END" END_SKIP "${GCOV_LINE}") + string(REGEX MATCH "LCOV_EXCL_LINE" LINE_SKIP "${GCOV_LINE}") + + set(RESET_SKIP 0) + if (LINE_SKIP AND NOT DO_SKIP) + set(DO_SKIP 1) + set(RESET_SKIP 1) + endif() + + if (START_SKIP) + set(DO_SKIP 1) + message("${GCOV_LINE_COUNT}: Start skip") + endif() + + if (END_SKIP) + set(DO_SKIP 0) + endif() + + list(LENGTH RES RES_COUNT) + + if (RES_COUNT GREATER 2) + list(GET RES 0 HITCOUNT) + list(GET RES 1 LINE) + list(GET RES 2 SOURCE) + + string(STRIP ${HITCOUNT} HITCOUNT) + string(STRIP ${LINE} LINE) + + # Lines with 0 line numbers are metadata and can be ignored. + if (NOT ${LINE} EQUAL 0) + + if (DO_SKIP) + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}null, ") + else() + # Translate the hitcount into valid JSON values. + if (${HITCOUNT} STREQUAL "#####") + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}0, ") + elseif (${HITCOUNT} STREQUAL "-") + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}null, ") + else() + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}${HITCOUNT}, ") + endif() + endif() + endif() + else() + message(WARNING "Failed to properly parse line (RES_COUNT = ${RES_COUNT}) ${GCOV_FILE}:${GCOV_LINE_COUNT}\n-->${GCOV_LINE}") + endif() + + if (RESET_SKIP) + set(DO_SKIP 0) + endif() + math(EXPR GCOV_LINE_COUNT "${GCOV_LINE_COUNT}+1") + endforeach() + + message("${GCOV_LINE_COUNT} of ${LINE_COUNT} lines read!") + + # Advanced way of removing the trailing comma in the JSON array. + # "[1, 2, 3, " -> "[1, 2, 3" + string(REGEX REPLACE ",[ ]*$" "" GCOV_FILE_COVERAGE ${GCOV_FILE_COVERAGE}) + + # Append the trailing ] to complete the JSON array. + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}]") + + # Generate the final JSON for this file. + message("Generate JSON for file: ${GCOV_SRC_REL_PATH}...") + string(CONFIGURE ${SRC_FILE_TEMPLATE} FILE_JSON) + + set(JSON_GCOV_FILES "${JSON_GCOV_FILES}${FILE_JSON}, ") +endforeach() + +# Loop through all files we couldn't find any coverage for +# as well, and generate JSON for those as well with 0% coverage. +foreach(NOT_COVERED_SRC ${COVERAGE_SRCS_REMAINING}) + + # Loads the source file as a list of lines. + file(STRINGS ${NOT_COVERED_SRC} SRC_LINES) + + set(GCOV_FILE_COVERAGE "[") + set(GCOV_FILE_SOURCE "") + + foreach (SOURCE ${SRC_LINES}) + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}0, ") + + string(REPLACE "\\" "\\\\" SOURCE "${SOURCE}") + string(REGEX REPLACE "\"" "\\\\\"" SOURCE "${SOURCE}") + string(REPLACE "\t" "\\\\t" SOURCE "${SOURCE}") + string(REPLACE "\r" "\\\\r" SOURCE "${SOURCE}") + set(GCOV_FILE_SOURCE "${GCOV_FILE_SOURCE}${SOURCE}\\n") + endforeach() + + # Remove trailing comma, and complete JSON array with ] + string(REGEX REPLACE ",[ ]*$" "" GCOV_FILE_COVERAGE ${GCOV_FILE_COVERAGE}) + set(GCOV_FILE_COVERAGE "${GCOV_FILE_COVERAGE}]") + + # Generate the final JSON for this file. + message("Generate JSON for non-gcov file: ${NOT_COVERED_SRC}...") + string(CONFIGURE ${SRC_FILE_TEMPLATE} FILE_JSON) + set(JSON_GCOV_FILES "${JSON_GCOV_FILES}${FILE_JSON}, ") +endforeach() + +# Get rid of trailing comma. +string(REGEX REPLACE ",[ ]*$" "" JSON_GCOV_FILES ${JSON_GCOV_FILES}) +set(JSON_GCOV_FILES "${JSON_GCOV_FILES}]") + +# Generate the final complete JSON! +message("Generate final JSON...") +string(CONFIGURE ${JSON_TEMPLATE} JSON) + +file(WRITE "${COVERALLS_OUTPUT_FILE}" "${JSON}") +message("###########################################################################") +message("Generated coveralls JSON containing coverage data:") +message("${COVERALLS_OUTPUT_FILE}") +message("###########################################################################") diff --git a/paddle/scripts/travis/build_and_test.sh b/paddle/scripts/travis/build_and_test.sh index a73c32344c8ab..54e3320c8c158 100755 --- a/paddle/scripts/travis/build_and_test.sh +++ b/paddle/scripts/travis/build_and_test.sh @@ -6,17 +6,19 @@ if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then fi -cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON ${CMAKE_EXTRA} +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -DON_COVERALLS=ON ${CMAKE_EXTRA} NPROC=1 if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then NRPOC=`nproc` + make -j $NPROC + make coveralls elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then NPROC=`sysctl -n hw.ncpu` + make -j $NPROC + env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j $NPROC" fi -make -j $NPROC -env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j $NPROC" sudo make install sudo paddle version From 568d9cff1d9afaddffda79a3844475c365c22d06 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 4 Nov 2016 06:13:35 -0500 Subject: [PATCH 214/324] Add Issue template to guide user submit good issue (#354) * Add issue template * Update ISSUE_TEMPLATE.md * Update ISSUE_TEMPLATE.md * Rename * Rename * Typo * Typo * Typo * Typo * Follow comments * Follow comments --- ISSUE_TEMPLATE.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 ISSUE_TEMPLATE.md diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 0000000000000..66d6b8b4dd9a5 --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,13 @@ +Thank you for contributing to PaddlePaddle. Submitting an issue is a great help for us. +Both Chinese and English issues are welcome. + +Before submitting the issue, look over the following criteria before handing your request in. + +- [ ] Was there a similar issue submitted or resolved before? You could search issue in the github. +- [ ] Did you go to the search engine for your question? +- [ ] Is my description of issue clear enough to reproduce this problem? + * If there are some errors occured, we need details about `how do you run your code?`, `what system do you use?`, `Are you using GPU or not?`, etc. + * If you could provide an [asciinema](https://asciinema.org/) record, that's awesome! We could help you solve the problem more quickly. +- [ ] Is my description of issue use the github markdown correctly? + * Please use correct markdown syntax for code, header, etc. + * You can reference [this page](https://guides.github.com/features/mastering-markdown/) for markdown syntax. From 6c3a678c9a537a6b37c189490c646790860757c0 Mon Sep 17 00:00:00 2001 From: emailweixu Date: Fri, 4 Nov 2016 10:37:22 -0700 Subject: [PATCH 215/324] Add elementwise math operations (#343) * Add elementwise math operations This allows use to use expressions like: y=log(1+exp(x)) Also added unittests for ActivationFunction * Enforce keyword arguments for non-positional arguments * Add LogActivation to doc --- .../trainer_config_helpers/activations.rst | 7 ++ .../activations/ActivationFunction.cpp | 29 ++++++++ .../gserver/activations/ActivationFunction.h | 2 + paddle/gserver/tests/CMakeLists.txt | 7 ++ paddle/gserver/tests/test_ActivationGrad.cpp | 66 +++++++++++++++++++ python/paddle/trainer/config_parser.py | 10 +-- .../trainer_config_helpers/activations.py | 9 +++ .../default_decorators.py | 9 ++- .../paddle/trainer_config_helpers/layers.py | 2 +- python/paddle/trainer_config_helpers/math.py | 64 ++++++++++++++++++ .../tests/configs/check.md5 | 11 ++-- .../tests/configs/generate_protostr.sh | 2 +- .../tests/configs/math_ops.py | 24 +++++++ 13 files changed, 229 insertions(+), 13 deletions(-) create mode 100644 paddle/gserver/tests/test_ActivationGrad.cpp create mode 100644 python/paddle/trainer_config_helpers/math.py create mode 100644 python/paddle/trainer_config_helpers/tests/configs/math_ops.py diff --git a/doc/ui/api/trainer_config_helpers/activations.rst b/doc/ui/api/trainer_config_helpers/activations.rst index c4e14ed779efb..070ed03ab6cc9 100644 --- a/doc/ui/api/trainer_config_helpers/activations.rst +++ b/doc/ui/api/trainer_config_helpers/activations.rst @@ -32,6 +32,13 @@ LinearActivation .. automodule:: paddle.trainer_config_helpers.activations :members: LinearActivation :noindex: + +LogActivation +================== + +.. automodule:: paddle.trainer_config_helpers.activations + :members: LogActivation + :noindex: SquareActivation ================ diff --git a/paddle/gserver/activations/ActivationFunction.cpp b/paddle/gserver/activations/ActivationFunction.cpp index 9918d20d9082a..27eed75d4d76c 100644 --- a/paddle/gserver/activations/ActivationFunction.cpp +++ b/paddle/gserver/activations/ActivationFunction.cpp @@ -295,6 +295,7 @@ void forward(Argument& act) { void backward(Argument& act) { act.grad->squareDerivative(*act.in); } END_DEFINE_ACTIVATION(square) + /** * @brief Exponential Activation. * \f[ @@ -307,8 +308,36 @@ void forward(Argument& act) { act.value->exp(*act.value); } void backward(Argument& act) { act.grad->expDerivative(*act.value); } END_DEFINE_ACTIVATION(exponential) +/** + * @brief Logarithm Activation. + * \f[ + * f(z) = log(z) + * \f] + */ +BEGIN_DEFINE_ACTIVATION(log) +void forward(Argument& act) { + SetDevice device(act.deviceId); + Matrix::resizeOrCreate(act.in, act.value->getHeight(), act.value->getWidth(), + /* trans */ false, useGpu(act.deviceId)); + + act.in->copyFrom(*act.value); + act.value->log(*act.value); +} + +void backward(Argument& act) { act.grad->dotDiv(*act.grad, *act.in); } +END_DEFINE_ACTIVATION(log) + ActivationFunction* ActivationFunction::create(const std::string& type) { return gActivationRegistrar.createByType(type); } +std::vector ActivationFunction::getAllRegisteredTypes() { + std::vector types; + gActivationRegistrar.forEachType([&](const std::string& type) { + types.push_back(type); + }); + return types; +} + + } // namespace paddle diff --git a/paddle/gserver/activations/ActivationFunction.h b/paddle/gserver/activations/ActivationFunction.h index 29860b4a736c3..c483372256c03 100644 --- a/paddle/gserver/activations/ActivationFunction.h +++ b/paddle/gserver/activations/ActivationFunction.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include +#include namespace paddle { @@ -32,6 +33,7 @@ struct Argument; class ActivationFunction { public: static ActivationFunction* create(const std::string& type); + static std::vector getAllRegisteredTypes(); ActivationFunction() {} diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index ff2abf7697317..26ee2b3aae64a 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -20,6 +20,13 @@ add_unittest_without_exec(test_LayerGrad add_test(NAME test_LayerGrad COMMAND test_LayerGrad) +add_unittest_without_exec(test_ActivationGrad + test_ActivationGrad.cpp + LayerGradUtil.cpp + TestUtil.cpp) +add_test(NAME test_ActivationGrad + COMMAND test_ActivationGrad) + ################## test_Evaluator ####################### add_unittest(test_Evaluator test_Evaluator.cpp diff --git a/paddle/gserver/tests/test_ActivationGrad.cpp b/paddle/gserver/tests/test_ActivationGrad.cpp new file mode 100644 index 0000000000000..2c5d17090dfc7 --- /dev/null +++ b/paddle/gserver/tests/test_ActivationGrad.cpp @@ -0,0 +1,66 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include +#include "paddle/gserver/layers/DataLayer.h" +#include "ModelConfig.pb.h" +#include "paddle/trainer/Trainer.h" + +#include "TestUtil.h" +#include "LayerGradUtil.h" + +using namespace paddle; // NOLINT +using namespace std; // NOLINT + +P_DECLARE_bool(use_gpu); +P_DECLARE_bool(thread_local_rand_use_global_seed); + +void testActivation(const string& act) { + LOG(INFO) << "test activation: " << act; + size_t size = 10; + TestConfig config; + config.biasSize = 0; + config.layerConfig.set_type("addto"); + config.layerConfig.set_size(size); + config.layerConfig.set_active_type(act); + config.inputDefs.push_back({INPUT_DATA, "layer_0", size, 0}); + config.layerConfig.add_inputs(); + for (auto useGpu : {false, true}) { + testLayerGrad(config, + act + "_activation", + 100, + /* trans= */false, + useGpu, + /* useWeight */true); + } +} + +TEST(Activation, activation) { + auto types = ActivationFunction::getAllRegisteredTypes(); + std::set excluded{"sequence_softmax"}; + for (auto type : types) { + if (excluded.count(type)) continue; + testActivation(type); + } +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + initMain(argc, argv); + FLAGS_thread_local_rand_use_global_seed = true; + srand(1); + return RUN_ALL_TESTS(); +} diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index e9098943165fd..e9038fea8a208 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2573,8 +2573,9 @@ def __init__( for input in self.inputs: psize += input.calc_bias_size() - self.config.bias_size = psize - self.create_bias_parameter(bias, psize) + if bias: + self.config.bias_size = psize + self.create_bias_parameter(bias, psize) if error_clipping_threshold is not None: self.config.error_clipping_threshold = error_clipping_threshold @@ -2659,8 +2660,9 @@ def __init__( for input in self.inputs: psize += input.calc_bias_size() - self.config.bias_size = psize - self.create_bias_parameter(bias, psize) + if bias: + self.config.bias_size = psize + self.create_bias_parameter(bias, psize) @config_layer('recurrent') class RecurrentLayer(LayerBase): diff --git a/python/paddle/trainer_config_helpers/activations.py b/python/paddle/trainer_config_helpers/activations.py index 292014519374e..ad5cdc0a0eb13 100644 --- a/python/paddle/trainer_config_helpers/activations.py +++ b/python/paddle/trainer_config_helpers/activations.py @@ -199,3 +199,12 @@ class ExpActivation(BaseActivation): f(z) = e^z. """ def __init__(self): BaseActivation.__init__(self, 'exponential', False) + +class LogActivation(BaseActivation): + """ + Logarithm Activation. + + .. math:: + f(z) = log(z) + """ + def __init__(self): BaseActivation.__init__(self, 'log', False) diff --git a/python/paddle/trainer_config_helpers/default_decorators.py b/python/paddle/trainer_config_helpers/default_decorators.py index b20aebc685fe5..be00f48b457c1 100644 --- a/python/paddle/trainer_config_helpers/default_decorators.py +++ b/python/paddle/trainer_config_helpers/default_decorators.py @@ -13,6 +13,7 @@ # limitations under the License. import functools +import inspect from .attrs import ParamAttr from .activations import TanhActivation from paddle.trainer.config_parser import * @@ -37,8 +38,12 @@ def __impl__(func): @functools.wraps(func) def __wrapper__(*args, **kwargs): if len(args) != 0: - logger.warning("please use keyword arguments in paddle config.") - + argspec = inspect.getargspec(func) + num_positional = len(argspec.args) + if argspec.defaults: + num_positional -= len(argspec.defaults) + if not argspec.varargs and len(args) > num_positional: + logger.fatal("Must use keyword arguments for non-positional args") for name in param_names: if not_set_callback(kwargs, name): # Not set kwargs[name] = default_factory(func) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 9a23c02431d18..49f0ff3289db7 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -564,7 +564,7 @@ def __init__(self, name, size, act, bias_attr, layer_attr, self.inputs = [] self.finalized = False - def __add__(self, other): + def __iadd__(self, other): """ + += operator :param other: Other projection. diff --git a/python/paddle/trainer_config_helpers/math.py b/python/paddle/trainer_config_helpers/math.py new file mode 100644 index 0000000000000..e35849b77ac53 --- /dev/null +++ b/python/paddle/trainer_config_helpers/math.py @@ -0,0 +1,64 @@ +# Copyright (c) 2016 Baidu, Inc. All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .layers import LayerOutput, mixed_layer, identity_projection, \ + slope_intercept_layer +from .attrs import is_compatible_with +from .default_decorators import * +import activations as act + +__all__ = [] + +def register_unary_math_op(op_name, act): + def op(input, name=None): + return mixed_layer(input=[identity_projection(input=input)], + name=name, + act=act) + op = wrap_name_default(op_name)(op) + op.__doc__ = type(act).__doc__ + globals()[op_name] = op + __all__.append(op_name) + +register_unary_math_op('exp', act.ExpActivation()) +register_unary_math_op('log', act.LogActivation()) +register_unary_math_op('abs', act.AbsActivation()) +register_unary_math_op('sigmoid', act.SigmoidActivation()) +register_unary_math_op('tanh', act.TanhActivation()) +register_unary_math_op('square', act.SquareActivation()) + +def add(layeroutput, other): + if is_compatible_with(other, float): + return slope_intercept_layer(input=layeroutput, intercept=other) + assert isinstance(other, LayerOutput) + return mixed_layer(input=[identity_projection(input=layeroutput), + identity_projection(input=other)]) + +LayerOutput.__radd__ = add +LayerOutput.__add__ = add + +def sub(layeroutput, other): + if is_compatible_with(other, float): + return slope_intercept_layer(input=layeroutput, intercept=other) + assert isinstance(other, LayerOutput) + neg = slope_intercept_layer(input=other, slope=-1.0) + return mixed_layer(input=[identity_projection(input=layeroutput), + identity_projection(input=neg)]) + +LayerOutput.__sub__ = sub + +def rsub(layeroutput, other): + neg = slope_intercept_layer(input=layeroutput, slope=-1.0) + return add(neg, other) + +LayerOutput.__rsub__ = rsub diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 index 72dfdad7bdd40..93d129b765e19 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/check.md5 +++ b/python/paddle/trainer_config_helpers/tests/configs/check.md5 @@ -1,11 +1,11 @@ 86c0815275a9d5eb902e23c6a592f58a img_layers.protostr a5d9259ff1fd7ca23d0ef090052cb1f2 last_first_seq.protostr 9c038249ec8ff719753a746cdb04c026 layer_activations.protostr -34e04043cbb12931c47fa44ec50eeffc projections.protostr +5913f87b39cee3b2701fa158270aca26 projections.protostr 7334ba0a4544f0623231330fc51d390d shared_fc.protostr -bb8e233b05b8e07f9ed386b7aee4f2c6 shared_lstm.protostr +8b8b6bb128a7dfcc937be86145f53e2f shared_lstm.protostr 6b39e34beea8dfb782bee9bd3dea9eb5 simple_rnn_layers.protostr -f98e79e1630d5eb827c300e64836d269 test_bi_grumemory.protostr +4e78f0ded79f6fefb58ca0c104b57c79 test_bi_grumemory.protostr 0fc1409600f1a3301da994ab9d28b0bf test_cost_layers.protostr 6cd5f28a3416344f20120698470e0a4c test_cost_layers_with_weight.protostr 144bc6d3a509de74115fa623741797ed test_expand_layer.protostr @@ -16,7 +16,8 @@ d350bd91a0dc13e854b1364c3d9339c6 test_lstmemory_layer.protostr 5433ed33d4e7414eaf658f2a55946186 test_maxout.protostr 251a948ba41c1071afcd3d9cf9c233f7 test_ntm_layers.protostr e6ff04e70aea27c7b06d808cc49c9497 test_print_layer.protostr -fded24727338fb8ce44d9951ed8aea08 test_rnn_group.protostr +2a75dd33b640c49a8821c2da6e574577 test_rnn_group.protostr 67d6fde3afb54f389d0ce4ff14726fe1 test_sequence_pooling.protostr f586a548ef4350ba1ed47a81859a64cb unused_layers.protostr -f937a5a6e7e8864b4d8cf56b0f7c7f44 util_layers.protostr +8122477f4f65244580cec09edc590041 util_layers.protostr +dcd76bebb5f9c755f481c26192917818 math_ops.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index 6a31ceabdf36d..9e23bd1fe2bf5 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -9,7 +9,7 @@ test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers util_layers simple_rnn_layers unused_layers test_cost_layers test_rnn_group shared_fc shared_lstm test_cost_layers_with_weight -test_maxout test_bi_grumemory) +test_maxout test_bi_grumemory math_ops) for conf in ${configs[*]} diff --git a/python/paddle/trainer_config_helpers/tests/configs/math_ops.py b/python/paddle/trainer_config_helpers/tests/configs/math_ops.py new file mode 100644 index 0000000000000..fe515b7029336 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/math_ops.py @@ -0,0 +1,24 @@ +from paddle.trainer_config_helpers import * +from paddle.trainer_config_helpers import math + +settings( + batch_size=1000, + learning_rate=1e-5 +) + +x = data_layer(name='data', size=100) +x = math.exp(x) +x = math.log(x) +x = math.abs(x) +x = math.sigmoid(x) +x = math.square(x) +x = math.square(x) +y = 1 + x +y = y + 1 +y = x + y +y = y - x +y = y - 2 +y = 2 - y + +outputs(y) + From 36bda94eb4a686297f234bbb7d46423ab344a60e Mon Sep 17 00:00:00 2001 From: lzhao4ever Date: Fri, 4 Nov 2016 17:02:04 -0700 Subject: [PATCH 216/324] include mkl_lapacke.h (#359) --- paddle/math/MathFunctions.h | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/math/MathFunctions.h b/paddle/math/MathFunctions.h index cad0e4740b8c1..b322bd2bd7194 100644 --- a/paddle/math/MathFunctions.h +++ b/paddle/math/MathFunctions.h @@ -17,6 +17,7 @@ limitations under the License. */ #ifdef PADDLE_USE_MKL #include +#include #else extern "C" { #include From 8ff9aa3d2adb20ea4bafd8b0d9e0ee0411038955 Mon Sep 17 00:00:00 2001 From: gangliao Date: Sat, 5 Nov 2016 00:39:47 -0700 Subject: [PATCH 217/324] Update ISSUE_TEMPLATE.md (#357) --- ISSUE_TEMPLATE.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index 66d6b8b4dd9a5..b70d66dc259af 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -1,13 +1,14 @@ Thank you for contributing to PaddlePaddle. Submitting an issue is a great help for us. Both Chinese and English issues are welcome. +It's hard to solve a problem when important details are missing. Before submitting the issue, look over the following criteria before handing your request in. -- [ ] Was there a similar issue submitted or resolved before? You could search issue in the github. -- [ ] Did you go to the search engine for your question? -- [ ] Is my description of issue clear enough to reproduce this problem? - * If there are some errors occured, we need details about `how do you run your code?`, `what system do you use?`, `Are you using GPU or not?`, etc. - * If you could provide an [asciinema](https://asciinema.org/) record, that's awesome! We could help you solve the problem more quickly. -- [ ] Is my description of issue use the github markdown correctly? - * Please use correct markdown syntax for code, header, etc. - * You can reference [this page](https://guides.github.com/features/mastering-markdown/) for markdown syntax. +- [ ] Was there a similar issue submitted or resolved before ? You could search issue in the github. +- [ ] Did you retrieve your issue from widespread search engines ? +- [ ] Is my description of the issue clear enough to reproduce this problem? + * If some errors occured, we need details about `how do you run your code?`, `what system do you use?`, `Are you using GPU or not?`, etc. + * If you use an recording [asciinema](https://asciinema.org/) to show what you are doing to make it happen, that's awesome! We could help you solve the problem more quickly. +- [ ] Is my description of the issue use the github markdown correctly? + * Please use the proper markdown syntaxes for styling all forms of writing, e.g, source code, error information, etc. + * Check out [this page](https://guides.github.com/features/mastering-markdown/) to find out much more about markdown. From 744dba4abdedd74504c613d408221c075acc8cb9 Mon Sep 17 00:00:00 2001 From: backyes Date: Sat, 5 Nov 2016 16:01:41 +0800 Subject: [PATCH 218/324] add rdma cmake support (#284) * add rdma cmake support * move rdma related code to rdma.cmake --- CMakeLists.txt | 11 +++++-- cmake/rdma.cmake | 76 ++++++++++++++++++++++++++++++++++++++++++++++++ cmake/util.cmake | 10 +++++++ 3 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 cmake/rdma.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 93cdcd4a75b70..dd45f49b2f91a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,6 +52,9 @@ endif() include(enableCXX11) include(cpplint) include(ccache) +if(WITH_RDMA) + include(rdma) +endif() include(util) include(flags) include(cudnn) @@ -133,9 +136,11 @@ else(WITH_PYTHON) add_definitions(-DPADDLE_NO_PYTHON) endif(WITH_PYTHON) -if(NOT WITH_RDMA) - add_definitions(-DPADDLE_DISABLE_RDMA) -endif() +if(WITH_RDMA) + include_directories("${RDMA_INC_DIR}") +else(WITH_RDMA) + add_definitions(-DPADDLE_DISABLE_RDMA) +endif(WITH_RDMA) if(WITH_GLOG) add_definitions(-DPADDLE_USE_GLOG) diff --git a/cmake/rdma.cmake b/cmake/rdma.cmake new file mode 100644 index 0000000000000..e9a4da79aa92a --- /dev/null +++ b/cmake/rdma.cmake @@ -0,0 +1,76 @@ +# user should download rdma first from subversion repository + +# execute following instruction to download svn mannally +# svn co https://svn.baidu.com/sys/ip/trunk/rdma/sockrdmav1 rdma/ +# svn co https://svn.baidu.com/sys/ip/trunk/rdma/thirdparty rdma/ +# we use static output in svn repositories to avoid implict bugs from not standard runtime env. + +set(RDMA_ROOT $ENV{RDMA_ROOT} CACHE PATH "Folder contains RDMA sock library and thirdparty library") + +function(generate_rdma_links) + #redirect to current DIR to isolate the pollution from system runtime environment + #it can benifits unified control for different gcc environment. + #e.g, by default gcc48 did not refer /usr/lib64 which could contain low version + #runtime libraries that will crash process while loading it. That redirect trick + #can fix it. + execute_process( + COMMAND mkdir -p librdma + COMMAND ln -s -f /usr/lib64/libibverbs.so.1.0.0 librdma/libibverbs.so.1 + COMMAND ln -s -f /usr/lib64/libibverbs.so.1.0.0 librdma/libibverbs.so + COMMAND ln -s -f /usr/lib64/librdmacm.so.1.0.0 librdma/librdmacm.so.1 + COMMAND ln -s -f /usr/lib64/librdmacm.so.1.0.0 librdma/librdmacm.so + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) +endfunction(generate_rdma_links) + + +#check and set headers +find_path(RDMA_INC_SXISOCK sxi_sock.h PATHS ${RDMA_ROOT}/sockrdmav1/output/include) +find_path(RDMA_INC_XIO libxio.h PATHS ${RDMA_ROOT}/thirdparty/output/accelio) +find_path(RDMA_INC_EVENT event2 PATHS ${RDMA_ROOT}/thirdparty/output/libevent) +find_path(RDMA_INC_NUMA numa.h PATHS ${RDMA_ROOT}/thirdparty/output/libnuma) + +#check and set libs +find_library(RDMA_LIB_SXISOCK NAMES sxisock PATHS ${RDMA_ROOT}/sockrdmav1/output) +find_library(RDMA_LIB_XIO NAMES xio PATHS ${RDMA_ROOT}/thirdparty/output/accelio) +find_library(RDMA_LIB_EVENT NAMES event PATHS ${RDMA_ROOT}/thirdparty/output/libevent) +find_library(RDMA_LIB_EVENT_CORE NAMES event_core PATHS ${RDMA_ROOT}/thirdparty/output/libevent) +find_library(RDMA_LIB_EVENT_EXTRA NAMES event_extra PATHS ${RDMA_ROOT}/thirdparty/output/libevent) +find_library(RDMA_LIB_EVENT_PTHREADS NAMES event_pthreads PATHS ${RDMA_ROOT}/thirdparty/output/libevent) +find_library(RDMA_LIB_NUMA NAMES numa PATHS ${RDMA_ROOT}/thirdparty/output/libnuma) + +if( + RDMA_INC_SXISOCK AND + RDMA_INC_XIO AND + RDMA_INC_EVENT AND + RDMA_INC_NUMA AND + RDMA_LIB_SXISOCK AND + RDMA_LIB_XIO AND + RDMA_LIB_EVENT AND + RDMA_LIB_EVENT_CORE AND + RDMA_LIB_EVENT_EXTRA AND + RDMA_LIB_EVENT_PTHREADS AND + RDMA_LIB_NUMA + ) + + set(RDMA_INC_DIR + ${RDMA_INC_SXISOCK} + ${RDMA_INC_XIO} + ${RDMA_INC_EVENT} + ${RDMA_INC_NUMA}) + set(RDMA_LIBS + ${RDMA_LIB_SXISOCK} + ${RDMA_LIB_XIO} + ${RDMA_LIB_EVENT} + ${RDMA_LIB_EVENT_CORE} + ${RDMA_LIB_EVENT_EXTRA} + ${RDMA_LIB_EVENT_PTHREADS} + ${RDMA_LIB_NUMA} + ) + set(RDMA_LD_FLAGS "-L./librdma -libverbs -lrdmacm -Xlinker -rpath ./librdma") + return() +endif() + +#if this module is not called, RDMA_INC_DIR RDMA_LIBS will be null, so top module always refer this variable + +message(FATAL_ERROR, "RDMA libraries are not found, try to set RDMA_ROOT or check all related libraries.") diff --git a/cmake/util.cmake b/cmake/util.cmake index 0fa36f070cc11..3f78cd08c3905 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -67,6 +67,10 @@ endmacro() # # It will handle WITH_PYTHON/WITH_GLOG etc. function(link_paddle_exe TARGET_NAME) + if(WITH_RDMA) + generate_rdma_links() + endif() + if(WITH_METRIC) if(WITH_GPU) set(METRIC_LIBS paddle_metric_learning paddle_dserver_lib metric metric_cpu) @@ -109,6 +113,12 @@ function(link_paddle_exe TARGET_NAME) ${ZLIB_LIBRARIES} ${INTERAL_LIBS} ${CMAKE_DL_LIBS}) + + if(WITH_RDMA) + target_link_libraries(${TARGET_NAME} + ${RDMA_LD_FLAGS} + ${RDMA_LIBS}) + endif() if(WITH_PYTHON) target_link_libraries(${TARGET_NAME} From 93e4d0cce6d3048f77aa46dc9d4ee5c3fb777d0c Mon Sep 17 00:00:00 2001 From: backyes Date: Sat, 5 Nov 2016 16:38:11 +0800 Subject: [PATCH 219/324] using find_package for swig (#334) --- CMakeLists.txt | 3 ++- cmake/swig.cmake | 22 ---------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dd45f49b2f91a..282e3e199ef44 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ set(PADDLE_VERSION ${PADDLE_MAJOR_VERSION}.${PADDLE_MINOR_VERSION}.${PADDLE_PATC set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") set(PROJ_ROOT ${CMAKE_SOURCE_DIR}) include(package) -include(swig) +find_package(SWIG 2.0) find_package(CUDA QUIET) find_package(Protobuf REQUIRED) find_package(PythonLibs 2.7 REQUIRED) @@ -60,6 +60,7 @@ include(flags) include(cudnn) include(FindPythonModule) include(check_packages) +include(swig) include(coveralls) # add PaddlePaddle version diff --git a/cmake/swig.cmake b/cmake/swig.cmake index 160d7ee56a9c6..97e87aa947791 100644 --- a/cmake/swig.cmake +++ b/cmake/swig.cmake @@ -1,25 +1,3 @@ -find_program( - SWIG_BINARY_PATH - swig) - -if(${SWIG_BINARY_PATH} STREQUAL "SWIG_BINARY_PATH-NOTFOUND") - set(SWIG_FOUND OFF) -else() - set(SWIG_FOUND ON) -endif() - -set(MIN_SWIG_VERSION 2) -if(SWIG_FOUND) - execute_process(COMMAND sh -c "${SWIG_BINARY_PATH} -version | grep Version | cut -f3 -d' '" - OUTPUT_VARIABLE _SWIG_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(${_SWIG_VERSION} VERSION_LESS ${MIN_SWIG_VERSION}) - message("swig version ${MIN_SWIG_VERSION} or greater is needed for generating python api. " - "Only version ${_SWIG_VERSION} is found. Set SWIG_FOUND to FALSE") - set(SWIG_FOUND FALSE) - endif(${_SWIG_VERSION} VERSION_LESS ${MIN_SWIG_VERSION}) -endif(SWIG_FOUND) - function(generate_python_api target_name) add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py ${PROJ_ROOT}/paddle/Paddle_wrap.cxx From c64cd6feb4b8f7c052ed8f4934dc0b9059969d84 Mon Sep 17 00:00:00 2001 From: wenboyang Date: Mon, 7 Nov 2016 12:53:41 +0800 Subject: [PATCH 220/324] Use diff to compare config unittest (#363) Fix #342 --- .../tests/configs/check.md5 | 23 - .../tests/configs/generate_protostr.sh | 4 +- .../configs/protostr/img_layers.protostr | 176 +++++ .../configs/protostr/last_first_seq.protostr | 69 ++ .../protostr/layer_activations.protostr | 423 ++++++++++++ .../tests/configs/protostr/math_ops.protostr | 235 +++++++ .../configs/protostr/projections.protostr | 315 +++++++++ .../tests/configs/protostr/shared_fc.protostr | 125 ++++ .../configs/protostr/shared_lstm.protostr | 393 +++++++++++ .../protostr/simple_rnn_layers.protostr | 418 +++++++++++ .../protostr/test_bi_grumemory.protostr | 152 ++++ .../protostr/test_cost_layers.protostr | 289 ++++++++ .../test_cost_layers_with_weight.protostr | 111 +++ .../protostr/test_expand_layer.protostr | 56 ++ .../tests/configs/protostr/test_fc.protostr | 98 +++ .../protostr/test_grumemory_layer.protostr | 51 ++ .../configs/protostr/test_hsigmoid.protostr | 62 ++ .../protostr/test_lstmemory_layer.protostr | 53 ++ .../configs/protostr/test_maxout.protostr | 209 ++++++ .../configs/protostr/test_ntm_layers.protostr | 225 ++++++ .../protostr/test_print_layer.protostr | 26 + .../configs/protostr/test_rnn_group.protostr | 650 ++++++++++++++++++ .../protostr/test_sequence_pooling.protostr | 111 +++ .../configs/protostr/unused_layers.protostr | 27 + .../configs/protostr/util_layers.protostr | 81 +++ .../tests/configs/run_tests.sh | 14 +- 26 files changed, 4371 insertions(+), 25 deletions(-) delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/check.md5 create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/img_layers.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/last_first_seq.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/layer_activations.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/projections.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/shared_fc.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/shared_lstm.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/simple_rnn_layers.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_bi_grumemory.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_expand_layer.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_fc.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_grumemory_layer.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_hsigmoid.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_lstmemory_layer.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_ntm_layers.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_print_layer.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_rnn_group.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_sequence_pooling.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/unused_layers.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/util_layers.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 deleted file mode 100644 index 93d129b765e19..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/check.md5 +++ /dev/null @@ -1,23 +0,0 @@ -86c0815275a9d5eb902e23c6a592f58a img_layers.protostr -a5d9259ff1fd7ca23d0ef090052cb1f2 last_first_seq.protostr -9c038249ec8ff719753a746cdb04c026 layer_activations.protostr -5913f87b39cee3b2701fa158270aca26 projections.protostr -7334ba0a4544f0623231330fc51d390d shared_fc.protostr -8b8b6bb128a7dfcc937be86145f53e2f shared_lstm.protostr -6b39e34beea8dfb782bee9bd3dea9eb5 simple_rnn_layers.protostr -4e78f0ded79f6fefb58ca0c104b57c79 test_bi_grumemory.protostr -0fc1409600f1a3301da994ab9d28b0bf test_cost_layers.protostr -6cd5f28a3416344f20120698470e0a4c test_cost_layers_with_weight.protostr -144bc6d3a509de74115fa623741797ed test_expand_layer.protostr -2378518bdb71e8c6e888b1842923df58 test_fc.protostr -8bb44e1e5072d0c261572307e7672bda test_grumemory_layer.protostr -1f3510672dce7a9ed25317fc58579ac7 test_hsigmoid.protostr -d350bd91a0dc13e854b1364c3d9339c6 test_lstmemory_layer.protostr -5433ed33d4e7414eaf658f2a55946186 test_maxout.protostr -251a948ba41c1071afcd3d9cf9c233f7 test_ntm_layers.protostr -e6ff04e70aea27c7b06d808cc49c9497 test_print_layer.protostr -2a75dd33b640c49a8821c2da6e574577 test_rnn_group.protostr -67d6fde3afb54f389d0ce4ff14726fe1 test_sequence_pooling.protostr -f586a548ef4350ba1ed47a81859a64cb unused_layers.protostr -8122477f4f65244580cec09edc590041 util_layers.protostr -dcd76bebb5f9c755f481c26192917818 math_ops.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index 9e23bd1fe2bf5..77774f6fcfafd 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -4,6 +4,8 @@ set -e cd `dirname $0` export PYTHONPATH=$PWD/../../../../ +protostr=$PWD/protostr + configs=(test_fc layer_activations projections test_print_layer test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid @@ -15,5 +17,5 @@ test_maxout test_bi_grumemory math_ops) for conf in ${configs[*]} do echo "Generating " $conf - python -m paddle.utils.dump_config $conf.py > $conf.protostr + python -m paddle.utils.dump_config $conf.py > $protostr/$conf.protostr.unitest done diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/img_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/img_layers.protostr new file mode 100644 index 0000000000000..1f262af21126c --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/img_layers.protostr @@ -0,0 +1,176 @@ +type: "nn" +layers { + name: "image" + type: "data" + size: 65536 + active_type: "" +} +layers { + name: "__conv_0__" + type: "exconv" + size: 3297856 + active_type: "" + inputs { + input_layer_name: "image" + input_parameter_name: "___conv_0__.w0" + conv_conf { + filter_size: 32 + channels: 1 + stride: 1 + padding: 1 + groups: 1 + filter_channels: 1 + output_x: 227 + img_size: 256 + caffe_mode: true + filter_size_y: 32 + padding_y: 1 + stride_y: 1 + } + } + bias_parameter_name: "___conv_0__.wbias" + num_filters: 64 + shared_biases: true +} +layers { + name: "__batch_norm_0__" + type: "batch_norm" + size: 3297856 + active_type: "relu" + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w0" + image_conf { + channels: 64 + img_size: 227 + } + } + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w1" + } + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w2" + } + bias_parameter_name: "___batch_norm_0__.wbias" + moving_average_fraction: 0.9 +} +layers { + name: "__crmnorm_0__" + type: "norm" + size: 3297856 + active_type: "" + inputs { + input_layer_name: "__batch_norm_0__" + norm_conf { + norm_type: "cmrnorm-projection" + channels: 64 + size: 32 + scale: 0.0004 + pow: 0.75 + output_x: 227 + img_size: 227 + blocked: false + } + } +} +layers { + name: "__pool_0__" + type: "pool" + size: 2458624 + active_type: "" + inputs { + input_layer_name: "__conv_0__" + pool_conf { + pool_type: "max-projection" + channels: 64 + size_x: 32 + stride: 1 + output_x: 196 + img_size: 227 + padding: 0 + size_y: 32 + stride_y: 1 + output_y: 196 + img_size_y: 227 + padding_y: 0 + } + } +} +parameters { + name: "___conv_0__.w0" + size: 65536 + initial_mean: 0.0 + initial_std: 0.0441941738242 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___conv_0__.wbias" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 64 + dims: 1 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___batch_norm_0__.w0" + size: 64 + initial_mean: 1.0 + initial_std: 0.0 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___batch_norm_0__.w1" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false + is_static: true + is_shared: true +} +parameters { + name: "___batch_norm_0__.w2" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false + is_static: true + is_shared: true +} +parameters { + name: "___batch_norm_0__.wbias" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "image" +output_layer_names: "__pool_0__" +output_layer_names: "__crmnorm_0__" +sub_models { + name: "root" + layer_names: "image" + layer_names: "__conv_0__" + layer_names: "__batch_norm_0__" + layer_names: "__crmnorm_0__" + layer_names: "__pool_0__" + input_layer_names: "image" + output_layer_names: "__pool_0__" + output_layer_names: "__crmnorm_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/last_first_seq.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/last_first_seq.protostr new file mode 100644 index 0000000000000..7b2911f8e367e --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/last_first_seq.protostr @@ -0,0 +1,69 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 30 + active_type: "" +} +layers { + name: "__first_seq_0__" + type: "seqlastins" + size: 30 + active_type: "linear" + inputs { + input_layer_name: "data" + } + select_first: true + trans_type: "seq" +} +layers { + name: "__first_seq_1__" + type: "seqlastins" + size: 30 + active_type: "linear" + inputs { + input_layer_name: "data" + } + select_first: true + trans_type: "non-seq" +} +layers { + name: "__last_seq_0__" + type: "seqlastins" + size: 30 + active_type: "linear" + inputs { + input_layer_name: "data" + } + trans_type: "seq" +} +layers { + name: "__last_seq_1__" + type: "seqlastins" + size: 30 + active_type: "linear" + inputs { + input_layer_name: "data" + } + trans_type: "non-seq" +} +input_layer_names: "data" +output_layer_names: "__first_seq_0__" +output_layer_names: "__first_seq_1__" +output_layer_names: "__last_seq_0__" +output_layer_names: "__last_seq_1__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__first_seq_0__" + layer_names: "__first_seq_1__" + layer_names: "__last_seq_0__" + layer_names: "__last_seq_1__" + input_layer_names: "data" + output_layer_names: "__first_seq_0__" + output_layer_names: "__first_seq_1__" + output_layer_names: "__last_seq_0__" + output_layer_names: "__last_seq_1__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/layer_activations.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/layer_activations.protostr new file mode 100644 index 0000000000000..ecf39e4d32167 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/layer_activations.protostr @@ -0,0 +1,423 @@ +type: "nn" +layers { + name: "input" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "layer_0" + type: "fc" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_0.w0" + } + bias_parameter_name: "_layer_0.wbias" +} +layers { + name: "layer_1" + type: "fc" + size: 100 + active_type: "sigmoid" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_1.w0" + } + bias_parameter_name: "_layer_1.wbias" +} +layers { + name: "layer_2" + type: "fc" + size: 100 + active_type: "softmax" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_2.w0" + } + bias_parameter_name: "_layer_2.wbias" +} +layers { + name: "layer_3" + type: "fc" + size: 100 + active_type: "" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_3.w0" + } + bias_parameter_name: "_layer_3.wbias" +} +layers { + name: "layer_4" + type: "fc" + size: 100 + active_type: "" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_4.w0" + } + bias_parameter_name: "_layer_4.wbias" +} +layers { + name: "layer_5" + type: "fc" + size: 100 + active_type: "exponential" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_5.w0" + } + bias_parameter_name: "_layer_5.wbias" +} +layers { + name: "layer_6" + type: "fc" + size: 100 + active_type: "relu" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_6.w0" + } + bias_parameter_name: "_layer_6.wbias" +} +layers { + name: "layer_7" + type: "fc" + size: 100 + active_type: "brelu" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_7.w0" + } + bias_parameter_name: "_layer_7.wbias" +} +layers { + name: "layer_8" + type: "fc" + size: 100 + active_type: "softrelu" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_8.w0" + } + bias_parameter_name: "_layer_8.wbias" +} +layers { + name: "layer_9" + type: "fc" + size: 100 + active_type: "stanh" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_9.w0" + } + bias_parameter_name: "_layer_9.wbias" +} +layers { + name: "layer_10" + type: "fc" + size: 100 + active_type: "abs" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_10.w0" + } + bias_parameter_name: "_layer_10.wbias" +} +layers { + name: "layer_11" + type: "fc" + size: 100 + active_type: "square" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_11.w0" + } + bias_parameter_name: "_layer_11.wbias" +} +parameters { + name: "_layer_0.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_0.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_1.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_1.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_2.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_2.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_3.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_3.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_4.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_4.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_5.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_5.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_6.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_6.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_7.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_7.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_8.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_8.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_9.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_9.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_10.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_10.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_11.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_11.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "input" +output_layer_names: "layer_0" +output_layer_names: "layer_1" +output_layer_names: "layer_2" +output_layer_names: "layer_3" +output_layer_names: "layer_4" +output_layer_names: "layer_5" +output_layer_names: "layer_6" +output_layer_names: "layer_7" +output_layer_names: "layer_8" +output_layer_names: "layer_9" +output_layer_names: "layer_10" +output_layer_names: "layer_11" +sub_models { + name: "root" + layer_names: "input" + layer_names: "layer_0" + layer_names: "layer_1" + layer_names: "layer_2" + layer_names: "layer_3" + layer_names: "layer_4" + layer_names: "layer_5" + layer_names: "layer_6" + layer_names: "layer_7" + layer_names: "layer_8" + layer_names: "layer_9" + layer_names: "layer_10" + layer_names: "layer_11" + input_layer_names: "input" + output_layer_names: "layer_0" + output_layer_names: "layer_1" + output_layer_names: "layer_2" + output_layer_names: "layer_3" + output_layer_names: "layer_4" + output_layer_names: "layer_5" + output_layer_names: "layer_6" + output_layer_names: "layer_7" + output_layer_names: "layer_8" + output_layer_names: "layer_9" + output_layer_names: "layer_10" + output_layer_names: "layer_11" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr new file mode 100644 index 0000000000000..1767445c44bf5 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr @@ -0,0 +1,235 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__exp_0__" + type: "mixed" + size: 100 + active_type: "exponential" + inputs { + input_layer_name: "data" + proj_conf { + type: "identity" + name: "___exp_0__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__log_0__" + type: "mixed" + size: 100 + active_type: "log" + inputs { + input_layer_name: "__exp_0__" + proj_conf { + type: "identity" + name: "___log_0__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__abs_0__" + type: "mixed" + size: 100 + active_type: "abs" + inputs { + input_layer_name: "__log_0__" + proj_conf { + type: "identity" + name: "___abs_0__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__sigmoid_0__" + type: "mixed" + size: 100 + active_type: "sigmoid" + inputs { + input_layer_name: "__abs_0__" + proj_conf { + type: "identity" + name: "___sigmoid_0__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__square_0__" + type: "mixed" + size: 100 + active_type: "square" + inputs { + input_layer_name: "__sigmoid_0__" + proj_conf { + type: "identity" + name: "___square_0__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__square_1__" + type: "mixed" + size: 100 + active_type: "square" + inputs { + input_layer_name: "__square_0__" + proj_conf { + type: "identity" + name: "___square_1__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__slope_intercept_layer_0__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "__square_1__" + } + slope: 1.0 + intercept: 1 +} +layers { + name: "__slope_intercept_layer_1__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "__slope_intercept_layer_0__" + } + slope: 1.0 + intercept: 1 +} +layers { + name: "__mixed_0__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__square_1__" + proj_conf { + type: "identity" + name: "___mixed_0__.w0" + input_size: 100 + output_size: 100 + } + } + inputs { + input_layer_name: "__slope_intercept_layer_1__" + proj_conf { + type: "identity" + name: "___mixed_0__.w1" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__slope_intercept_layer_2__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "__square_1__" + } + slope: -1.0 + intercept: 0.0 +} +layers { + name: "__mixed_1__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_0__" + proj_conf { + type: "identity" + name: "___mixed_1__.w0" + input_size: 100 + output_size: 100 + } + } + inputs { + input_layer_name: "__slope_intercept_layer_2__" + proj_conf { + type: "identity" + name: "___mixed_1__.w1" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__slope_intercept_layer_3__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_1__" + } + slope: 1.0 + intercept: 2 +} +layers { + name: "__slope_intercept_layer_4__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "__slope_intercept_layer_3__" + } + slope: -1.0 + intercept: 0.0 +} +layers { + name: "__slope_intercept_layer_5__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "__slope_intercept_layer_4__" + } + slope: 1.0 + intercept: 2 +} +input_layer_names: "data" +output_layer_names: "__slope_intercept_layer_5__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__exp_0__" + layer_names: "__log_0__" + layer_names: "__abs_0__" + layer_names: "__sigmoid_0__" + layer_names: "__square_0__" + layer_names: "__square_1__" + layer_names: "__slope_intercept_layer_0__" + layer_names: "__slope_intercept_layer_1__" + layer_names: "__mixed_0__" + layer_names: "__slope_intercept_layer_2__" + layer_names: "__mixed_1__" + layer_names: "__slope_intercept_layer_3__" + layer_names: "__slope_intercept_layer_4__" + layer_names: "__slope_intercept_layer_5__" + input_layer_names: "data" + output_layer_names: "__slope_intercept_layer_5__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/projections.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/projections.protostr new file mode 100644 index 0000000000000..e47e531a2223d --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/projections.protostr @@ -0,0 +1,315 @@ +type: "nn" +layers { + name: "test" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__embedding_0__" + type: "mixed" + size: 256 + active_type: "" + inputs { + input_layer_name: "test" + input_parameter_name: "___embedding_0__.w0" + proj_conf { + type: "table" + name: "___embedding_0__.w0" + input_size: 100 + output_size: 256 + } + } +} +layers { + name: "__mixed_0__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__embedding_0__" + input_parameter_name: "___mixed_0__.w0" + proj_conf { + type: "fc" + name: "___mixed_0__.w0" + input_size: 256 + output_size: 100 + } + } +} +layers { + name: "__mixed_1__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_0__" + input_parameter_name: "___mixed_1__.w0" + proj_conf { + type: "table" + name: "___mixed_1__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__mixed_2__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_1__" + proj_conf { + type: "identity" + name: "___mixed_2__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__mixed_3__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_2__" + input_parameter_name: "___mixed_3__.w0" + proj_conf { + type: "dot_mul" + name: "___mixed_3__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__mixed_4__" + type: "mixed" + size: 300 + active_type: "" + inputs { + input_layer_name: "__mixed_3__" + input_parameter_name: "___mixed_4__.w0" + proj_conf { + type: "context" + name: "___mixed_4__.w0" + input_size: 100 + output_size: 300 + context_start: -1 + context_length: 3 + trainable_padding: true + } + } +} +layers { + name: "__mixed_5__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_2__" + } + inputs { + input_layer_name: "__mixed_3__" + } + operator_confs { + type: "dot_mul" + input_indices: 0 + input_indices: 1 + input_sizes: 100 + input_sizes: 100 + output_size: 100 + dotmul_scale: 1 + } +} +layers { + name: "img" + type: "data" + size: 1024 + active_type: "" +} +layers { + name: "filter" + type: "data" + size: 576 + active_type: "" +} +layers { + name: "__mixed_6__" + type: "mixed" + size: 57600 + active_type: "" + inputs { + input_layer_name: "img" + } + inputs { + input_layer_name: "filter" + } + operator_confs { + type: "conv" + input_indices: 0 + input_indices: 1 + input_sizes: 1024 + input_sizes: 576 + output_size: 57600 + conv_conf { + filter_size: 3 + channels: 1 + stride: 1 + padding: 0 + groups: 1 + filter_channels: 1 + output_x: 30 + img_size: 32 + caffe_mode: true + filter_size_y: 3 + padding_y: 0 + stride_y: 1 + } + num_filters: 64 + } +} +layers { + name: "__mixed_7__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_4__" + input_parameter_name: "___mixed_7__.w0" + proj_conf { + type: "fc" + name: "___mixed_7__.w0" + input_size: 300 + output_size: 100 + } + } + inputs { + input_layer_name: "__mixed_5__" + input_parameter_name: "___mixed_7__.w1" + proj_conf { + type: "trans_fc" + name: "___mixed_7__.w1" + input_size: 100 + output_size: 100 + } + } + inputs { + input_layer_name: "__mixed_6__" + input_parameter_name: "___mixed_7__.w2" + proj_conf { + type: "fc" + name: "___mixed_7__.w2" + input_size: 57600 + output_size: 100 + } + } + drop_rate: 0.5 +} +parameters { + name: "___embedding_0__.w0" + size: 25600 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 256 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_0__.w0" + size: 25600 + initial_mean: 0.0 + initial_std: 0.0625 + dims: 256 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_1__.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_3__.w0" + size: 100 + initial_mean: 0.0 + initial_std: 1.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_4__.w0" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 2 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___mixed_7__.w0" + size: 30000 + initial_mean: 0.0 + initial_std: 0.057735026919 + dims: 300 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_7__.w1" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_7__.w2" + size: 5760000 + initial_mean: 0.0 + initial_std: 0.00416666666667 + dims: 57600 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +input_layer_names: "test" +input_layer_names: "img" +input_layer_names: "filter" +output_layer_names: "__mixed_7__" +sub_models { + name: "root" + layer_names: "test" + layer_names: "__embedding_0__" + layer_names: "__mixed_0__" + layer_names: "__mixed_1__" + layer_names: "__mixed_2__" + layer_names: "__mixed_3__" + layer_names: "__mixed_4__" + layer_names: "__mixed_5__" + layer_names: "img" + layer_names: "filter" + layer_names: "__mixed_6__" + layer_names: "__mixed_7__" + input_layer_names: "test" + input_layer_names: "img" + input_layer_names: "filter" + output_layer_names: "__mixed_7__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/shared_fc.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/shared_fc.protostr new file mode 100644 index 0000000000000..3e8633b079831 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/shared_fc.protostr @@ -0,0 +1,125 @@ +type: "nn" +layers { + name: "feature_a" + type: "data" + size: 200 + active_type: "" +} +layers { + name: "feature_b" + type: "data" + size: 200 + active_type: "" +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 200 + active_type: "tanh" + inputs { + input_layer_name: "feature_a" + input_parameter_name: "fc_param" + } + bias_parameter_name: "bias_param" +} +layers { + name: "__fc_layer_1__" + type: "fc" + size: 200 + active_type: "tanh" + inputs { + input_layer_name: "feature_b" + input_parameter_name: "fc_param" + } + bias_parameter_name: "bias_param" +} +layers { + name: "__fc_layer_2__" + type: "fc" + size: 10 + active_type: "softmax" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "softmax_param" + } + inputs { + input_layer_name: "__fc_layer_1__" + input_parameter_name: "softmax_param" + } +} +layers { + name: "label" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "__cost_0__" + type: "multi-class-cross-entropy" + size: 1 + active_type: "" + inputs { + input_layer_name: "__fc_layer_2__" + } + inputs { + input_layer_name: "label" + } + coeff: 1.0 +} +parameters { + name: "fc_param" + size: 40000 + initial_mean: 0.0 + initial_std: 1.0 + dims: 200 + dims: 200 + initial_strategy: 1 + initial_smart: false +} +parameters { + name: "bias_param" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "softmax_param" + size: 2000 + initial_mean: 0.0 + initial_std: 1.0 + dims: 200 + dims: 10 + initial_strategy: 1 + initial_smart: false +} +input_layer_names: "feature_a" +input_layer_names: "feature_b" +input_layer_names: "label" +output_layer_names: "__cost_0__" +evaluators { + name: "classification_error_evaluator" + type: "classification_error" + input_layers: "__fc_layer_2__" + input_layers: "label" +} +sub_models { + name: "root" + layer_names: "feature_a" + layer_names: "feature_b" + layer_names: "__fc_layer_0__" + layer_names: "__fc_layer_1__" + layer_names: "__fc_layer_2__" + layer_names: "label" + layer_names: "__cost_0__" + input_layer_names: "feature_a" + input_layer_names: "feature_b" + input_layer_names: "label" + output_layer_names: "__cost_0__" + evaluator_names: "classification_error_evaluator" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/shared_lstm.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/shared_lstm.protostr new file mode 100644 index 0000000000000..0a83499b72480 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/shared_lstm.protostr @@ -0,0 +1,393 @@ +type: "recurrent_nn" +layers { + name: "data_a" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "data_b" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__mixed_0__" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "data_a" + input_parameter_name: "mixed_param" + proj_conf { + type: "fc" + name: "___mixed_0__.w0" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__mixed_1__" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "data_b" + input_parameter_name: "mixed_param" + proj_conf { + type: "fc" + name: "___mixed_1__.w0" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__lstm_group_0___recurrent_group" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "__mixed_0__@__lstm_group_0___recurrent_group" + type: "scatter_agent" + size: 400 + active_type: "" +} +layers { + name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "__mixed_0__@__lstm_group_0___recurrent_group" + proj_conf { + type: "identity" + name: "___lstm_group_0___input_recurrent.w0" + input_size: 400 + output_size: 400 + } + } + inputs { + input_layer_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + input_parameter_name: "lstm_param" + proj_conf { + type: "fc" + name: "___lstm_group_0___input_recurrent.w1" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + type: "lstm_step" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + } + inputs { + input_layer_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + } + bias_parameter_name: "lstm_bias" + active_gate_type: "sigmoid" + active_state_type: "sigmoid" +} +layers { + name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + type: "get_output" + size: 100 + active_type: "" + inputs { + input_layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + input_layer_argument: "state" + } +} +layers { + name: "__lstm_group_0__" + type: "gather_agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_1___recurrent_group" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "__mixed_1__@__lstm_group_1___recurrent_group" + type: "scatter_agent" + size: 400 + active_type: "" +} +layers { + name: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_1___input_recurrent@__lstm_group_1___recurrent_group" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "__mixed_1__@__lstm_group_1___recurrent_group" + proj_conf { + type: "identity" + name: "___lstm_group_1___input_recurrent.w0" + input_size: 400 + output_size: 400 + } + } + inputs { + input_layer_name: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" + input_parameter_name: "lstm_param" + proj_conf { + type: "fc" + name: "___lstm_group_1___input_recurrent.w1" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__lstm_group_1__@__lstm_group_1___recurrent_group" + type: "lstm_step" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "__lstm_group_1___input_recurrent@__lstm_group_1___recurrent_group" + } + inputs { + input_layer_name: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" + } + bias_parameter_name: "lstm_bias" + active_gate_type: "sigmoid" + active_state_type: "sigmoid" +} +layers { + name: "__lstm_group_1___state@__lstm_group_1___recurrent_group" + type: "get_output" + size: 100 + active_type: "" + inputs { + input_layer_name: "__lstm_group_1__@__lstm_group_1___recurrent_group" + input_layer_argument: "state" + } +} +layers { + name: "__lstm_group_1__" + type: "gather_agent" + size: 100 + active_type: "" +} +layers { + name: "__last_seq_0__" + type: "seqlastins" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "__lstm_group_0__" + } + trans_type: "non-seq" +} +layers { + name: "__last_seq_1__" + type: "seqlastins" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "__lstm_group_1__" + } + trans_type: "non-seq" +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 10 + active_type: "softmax" + inputs { + input_layer_name: "__last_seq_0__" + input_parameter_name: "softmax_param" + } + inputs { + input_layer_name: "__last_seq_1__" + input_parameter_name: "softmax_param" + } +} +layers { + name: "label" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "__cost_0__" + type: "multi-class-cross-entropy" + size: 1 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + } + inputs { + input_layer_name: "label" + } + coeff: 1.0 +} +parameters { + name: "mixed_param" + size: 40000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 400 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "lstm_param" + size: 40000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 400 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "lstm_bias" + size: 300 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 300 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "softmax_param" + size: 1000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 10 + initial_strategy: 0 + initial_smart: true +} +input_layer_names: "data_a" +input_layer_names: "data_b" +input_layer_names: "label" +output_layer_names: "__cost_0__" +evaluators { + name: "classification_error_evaluator" + type: "classification_error" + input_layers: "__fc_layer_0__" + input_layers: "label" +} +sub_models { + name: "root" + layer_names: "data_a" + layer_names: "data_b" + layer_names: "__mixed_0__" + layer_names: "__mixed_1__" + layer_names: "__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__" + layer_names: "__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1__" + layer_names: "__last_seq_0__" + layer_names: "__last_seq_1__" + layer_names: "__fc_layer_0__" + layer_names: "label" + layer_names: "__cost_0__" + input_layer_names: "data_a" + input_layer_names: "data_b" + input_layer_names: "label" + output_layer_names: "__cost_0__" + evaluator_names: "classification_error_evaluator" + is_recurrent_layer_group: false +} +sub_models { + name: "__lstm_group_0___recurrent_group" + layer_names: "__mixed_0__@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + is_sequence: false + } + memories { + layer_name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + is_sequence: false + } + in_links { + layer_name: "__mixed_0__" + link_name: "__mixed_0__@__lstm_group_0___recurrent_group" + has_subseq: false + } + out_links { + layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0__" + has_subseq: false + } + target_inlinkid: -1 +} +sub_models { + name: "__lstm_group_1___recurrent_group" + layer_names: "__mixed_1__@__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1___input_recurrent@__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1__@__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1___state@__lstm_group_1___recurrent_group" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "__lstm_group_1__@__lstm_group_1___recurrent_group" + link_name: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" + is_sequence: false + } + memories { + layer_name: "__lstm_group_1___state@__lstm_group_1___recurrent_group" + link_name: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" + is_sequence: false + } + in_links { + layer_name: "__mixed_1__" + link_name: "__mixed_1__@__lstm_group_1___recurrent_group" + has_subseq: false + } + out_links { + layer_name: "__lstm_group_1__@__lstm_group_1___recurrent_group" + link_name: "__lstm_group_1__" + has_subseq: false + } + target_inlinkid: -1 +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/simple_rnn_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/simple_rnn_layers.protostr new file mode 100644 index 0000000000000..dacb40185f863 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/simple_rnn_layers.protostr @@ -0,0 +1,418 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 200 + active_type: "" +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "data" + input_parameter_name: "___fc_layer_0__.w0" + } + bias_parameter_name: "___fc_layer_0__.wbias" +} +layers { + name: "__recurrent_layer_0__" + type: "recurrent" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___recurrent_layer_0__.w0" + } + bias_parameter_name: "___recurrent_layer_0__.wbias" + reversed: false +} +layers { + name: "__recurrent_layer_1__" + type: "recurrent" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___recurrent_layer_1__.w0" + } + bias_parameter_name: "___recurrent_layer_1__.wbias" + reversed: true +} +layers { + name: "__fc_layer_1__" + type: "fc" + size: 800 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___fc_layer_1__.w0" + } +} +layers { + name: "__lstmemory_0__" + type: "lstmemory" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_1__" + input_parameter_name: "___lstmemory_0__.w0" + } + bias_parameter_name: "___lstmemory_0__.wbias" + reversed: false + active_gate_type: "sigmoid" + active_state_type: "tanh" +} +layers { + name: "__fc_layer_2__" + type: "fc" + size: 800 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___fc_layer_2__.w0" + } +} +layers { + name: "__lstmemory_1__" + type: "lstmemory" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_2__" + input_parameter_name: "___lstmemory_1__.w0" + } + bias_parameter_name: "___lstmemory_1__.wbias" + reversed: true + active_gate_type: "sigmoid" + active_state_type: "tanh" +} +layers { + name: "__fc_layer_3__" + type: "fc" + size: 600 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___fc_layer_3__.w0" + } +} +layers { + name: "__gru_0__" + type: "gated_recurrent" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_3__" + input_parameter_name: "___gru_0__.w0" + } + bias_parameter_name: "___gru_0__.wbias" + reversed: false + active_gate_type: "sigmoid" +} +layers { + name: "__fc_layer_4__" + type: "fc" + size: 600 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___fc_layer_4__.w0" + } +} +layers { + name: "__gru_1__" + type: "gated_recurrent" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_4__" + input_parameter_name: "___gru_1__.w0" + } + bias_parameter_name: "___gru_1__.wbias" + reversed: true + active_gate_type: "sigmoid" +} +layers { + name: "__last_seq_0__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__recurrent_layer_0__" + } + trans_type: "non-seq" +} +layers { + name: "__first_seq_0__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__recurrent_layer_1__" + } + select_first: true + trans_type: "non-seq" +} +layers { + name: "__last_seq_1__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__lstmemory_0__" + } + trans_type: "non-seq" +} +layers { + name: "__first_seq_1__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__lstmemory_1__" + } + select_first: true + trans_type: "non-seq" +} +layers { + name: "__last_seq_2__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__gru_0__" + } + trans_type: "non-seq" +} +layers { + name: "__first_seq_2__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__gru_1__" + } + select_first: true + trans_type: "non-seq" +} +parameters { + name: "___fc_layer_0__.w0" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___fc_layer_0__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___recurrent_layer_0__.w0" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___recurrent_layer_0__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___recurrent_layer_1__.w0" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___recurrent_layer_1__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___fc_layer_1__.w0" + size: 160000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 800 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstmemory_0__.w0" + size: 160000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 200 + dims: 4 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstmemory_0__.wbias" + size: 1400 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 1400 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___fc_layer_2__.w0" + size: 160000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 800 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstmemory_1__.w0" + size: 160000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 200 + dims: 4 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstmemory_1__.wbias" + size: 1400 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 1400 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___fc_layer_3__.w0" + size: 120000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 600 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___gru_0__.w0" + size: 120000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 600 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___gru_0__.wbias" + size: 600 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 600 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___fc_layer_4__.w0" + size: 120000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 600 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___gru_1__.w0" + size: 120000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 600 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___gru_1__.wbias" + size: 600 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 600 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +output_layer_names: "__last_seq_0__" +output_layer_names: "__first_seq_0__" +output_layer_names: "__last_seq_1__" +output_layer_names: "__first_seq_1__" +output_layer_names: "__last_seq_2__" +output_layer_names: "__first_seq_2__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__fc_layer_0__" + layer_names: "__recurrent_layer_0__" + layer_names: "__recurrent_layer_1__" + layer_names: "__fc_layer_1__" + layer_names: "__lstmemory_0__" + layer_names: "__fc_layer_2__" + layer_names: "__lstmemory_1__" + layer_names: "__fc_layer_3__" + layer_names: "__gru_0__" + layer_names: "__fc_layer_4__" + layer_names: "__gru_1__" + layer_names: "__last_seq_0__" + layer_names: "__first_seq_0__" + layer_names: "__last_seq_1__" + layer_names: "__first_seq_1__" + layer_names: "__last_seq_2__" + layer_names: "__first_seq_2__" + input_layer_names: "data" + output_layer_names: "__last_seq_0__" + output_layer_names: "__first_seq_0__" + output_layer_names: "__last_seq_1__" + output_layer_names: "__first_seq_1__" + output_layer_names: "__last_seq_2__" + output_layer_names: "__first_seq_2__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bi_grumemory.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bi_grumemory.protostr new file mode 100644 index 0000000000000..b110e91498ce7 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bi_grumemory.protostr @@ -0,0 +1,152 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 120 + active_type: "" +} +layers { + name: "__bidirectional_gru_0___fw_transform" + type: "mixed" + size: 120 + active_type: "" + inputs { + input_layer_name: "data" + input_parameter_name: "___bidirectional_gru_0___fw_transform.w0" + proj_conf { + type: "fc" + name: "___bidirectional_gru_0___fw_transform.w0" + input_size: 120 + output_size: 120 + } + } +} +layers { + name: "__bidirectional_gru_0___fw" + type: "gated_recurrent" + size: 40 + active_type: "tanh" + inputs { + input_layer_name: "__bidirectional_gru_0___fw_transform" + input_parameter_name: "___bidirectional_gru_0___fw.w0" + } + bias_parameter_name: "___bidirectional_gru_0___fw.wbias" + reversed: false + active_gate_type: "sigmoid" +} +layers { + name: "__bidirectional_gru_0___bw_transform" + type: "mixed" + size: 120 + active_type: "" + inputs { + input_layer_name: "data" + input_parameter_name: "___bidirectional_gru_0___bw_transform.w0" + proj_conf { + type: "fc" + name: "___bidirectional_gru_0___bw_transform.w0" + input_size: 120 + output_size: 120 + } + } +} +layers { + name: "__bidirectional_gru_0___bw" + type: "gated_recurrent" + size: 40 + active_type: "tanh" + inputs { + input_layer_name: "__bidirectional_gru_0___bw_transform" + input_parameter_name: "___bidirectional_gru_0___bw.w0" + } + bias_parameter_name: "___bidirectional_gru_0___bw.wbias" + reversed: true + active_gate_type: "sigmoid" +} +layers { + name: "__bidirectional_gru_0__" + type: "concat" + size: 80 + active_type: "" + inputs { + input_layer_name: "__bidirectional_gru_0___fw" + } + inputs { + input_layer_name: "__bidirectional_gru_0___bw" + } +} +parameters { + name: "___bidirectional_gru_0___fw_transform.w0" + size: 14400 + initial_mean: 0.0 + initial_std: 0.0912870929175 + dims: 120 + dims: 120 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___bidirectional_gru_0___fw.w0" + size: 4800 + initial_mean: 0.0 + initial_std: 0.158113883008 + dims: 40 + dims: 120 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___bidirectional_gru_0___fw.wbias" + size: 120 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 120 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___bidirectional_gru_0___bw_transform.w0" + size: 14400 + initial_mean: 0.0 + initial_std: 0.0912870929175 + dims: 120 + dims: 120 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___bidirectional_gru_0___bw.w0" + size: 4800 + initial_mean: 0.0 + initial_std: 0.158113883008 + dims: 40 + dims: 120 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___bidirectional_gru_0___bw.wbias" + size: 120 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 120 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +output_layer_names: "__bidirectional_gru_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__bidirectional_gru_0___fw_transform" + layer_names: "__bidirectional_gru_0___fw" + layer_names: "__bidirectional_gru_0___bw_transform" + layer_names: "__bidirectional_gru_0___bw" + layer_names: "__bidirectional_gru_0__" + input_layer_names: "data" + output_layer_names: "__bidirectional_gru_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr new file mode 100644 index 0000000000000..5261cf0c44943 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr @@ -0,0 +1,289 @@ +type: "nn" +layers { + name: "input" + type: "data" + size: 200 + active_type: "" +} +layers { + name: "labels" + type: "data" + size: 5000 + active_type: "" +} +layers { + name: "probs" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "xe-label" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "__ctc_layer_0__" + type: "ctc" + size: 5001 + active_type: "" + inputs { + input_layer_name: "input" + } + inputs { + input_layer_name: "labels" + } + norm_by_times: false +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 4 + active_type: "tanh" + inputs { + input_layer_name: "input" + input_parameter_name: "___fc_layer_0__.w0" + } + bias_parameter_name: "___fc_layer_0__.wbias" +} +layers { + name: "crf_label" + type: "data" + size: 4 + active_type: "" +} +layers { + name: "__crf_layer_0__" + type: "crf" + size: 4 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___crf_layer_0__.w0" + } + inputs { + input_layer_name: "crf_label" + } + coeff: 1.0 +} +layers { + name: "left" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "right" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "label" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__rank_cost_0__" + type: "rank-cost" + size: 1 + active_type: "" + inputs { + input_layer_name: "left" + } + inputs { + input_layer_name: "right" + } + inputs { + input_layer_name: "label" + } + coeff: 1.0 +} +layers { + name: "list_feature" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "list_scores" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__lambda_cost_0__" + type: "lambda_cost" + size: 1 + active_type: "" + inputs { + input_layer_name: "list_feature" + } + inputs { + input_layer_name: "list_scores" + } + NDCG_num: 5 + max_sort_size: -1 +} +layers { + name: "__cross_entropy_0__" + type: "multi-class-cross-entropy" + size: 1 + active_type: "" + inputs { + input_layer_name: "probs" + } + inputs { + input_layer_name: "xe-label" + } + coeff: 1.0 +} +layers { + name: "__cross_entropy_with_selfnorm_0__" + type: "multi_class_cross_entropy_with_selfnorm" + active_type: "" + inputs { + input_layer_name: "probs" + } + inputs { + input_layer_name: "xe-label" + } + softmax_selfnorm_alpha: 0.1 + coeff: 1.0 +} +layers { + name: "huber_probs" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "huber_label" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__huber_cost_0__" + type: "huber" + size: 1 + active_type: "" + inputs { + input_layer_name: "huber_probs" + } + inputs { + input_layer_name: "huber_label" + } + coeff: 1.0 +} +layers { + name: "__multi_binary_label_cross_entropy_0__" + type: "multi_binary_label_cross_entropy" + size: 1 + active_type: "" + inputs { + input_layer_name: "probs" + } + inputs { + input_layer_name: "xe-label" + } + coeff: 1.0 +} +parameters { + name: "___fc_layer_0__.w0" + size: 800 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 4 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___fc_layer_0__.wbias" + size: 4 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 4 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___crf_layer_0__.w0" + size: 24 + initial_mean: 0.0 + initial_std: 0.5 + dims: 4 + dims: 6 + initial_strategy: 0 + initial_smart: true +} +input_layer_names: "input" +input_layer_names: "labels" +input_layer_names: "crf_label" +input_layer_names: "left" +input_layer_names: "right" +input_layer_names: "label" +input_layer_names: "list_feature" +input_layer_names: "list_scores" +input_layer_names: "probs" +input_layer_names: "xe-label" +input_layer_names: "huber_probs" +input_layer_names: "huber_label" +output_layer_names: "__ctc_layer_0__" +output_layer_names: "__crf_layer_0__" +output_layer_names: "__rank_cost_0__" +output_layer_names: "__lambda_cost_0__" +output_layer_names: "__cross_entropy_0__" +output_layer_names: "__cross_entropy_with_selfnorm_0__" +output_layer_names: "__huber_cost_0__" +output_layer_names: "__multi_binary_label_cross_entropy_0__" +sub_models { + name: "root" + layer_names: "input" + layer_names: "labels" + layer_names: "probs" + layer_names: "xe-label" + layer_names: "__ctc_layer_0__" + layer_names: "__fc_layer_0__" + layer_names: "crf_label" + layer_names: "__crf_layer_0__" + layer_names: "left" + layer_names: "right" + layer_names: "label" + layer_names: "__rank_cost_0__" + layer_names: "list_feature" + layer_names: "list_scores" + layer_names: "__lambda_cost_0__" + layer_names: "__cross_entropy_0__" + layer_names: "__cross_entropy_with_selfnorm_0__" + layer_names: "huber_probs" + layer_names: "huber_label" + layer_names: "__huber_cost_0__" + layer_names: "__multi_binary_label_cross_entropy_0__" + input_layer_names: "input" + input_layer_names: "labels" + input_layer_names: "crf_label" + input_layer_names: "left" + input_layer_names: "right" + input_layer_names: "label" + input_layer_names: "list_feature" + input_layer_names: "list_scores" + input_layer_names: "probs" + input_layer_names: "xe-label" + input_layer_names: "huber_probs" + input_layer_names: "huber_label" + output_layer_names: "__ctc_layer_0__" + output_layer_names: "__crf_layer_0__" + output_layer_names: "__rank_cost_0__" + output_layer_names: "__lambda_cost_0__" + output_layer_names: "__cross_entropy_0__" + output_layer_names: "__cross_entropy_with_selfnorm_0__" + output_layer_names: "__huber_cost_0__" + output_layer_names: "__multi_binary_label_cross_entropy_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr new file mode 100644 index 0000000000000..811b38ae4a51e --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers_with_weight.protostr @@ -0,0 +1,111 @@ +type: "nn" +layers { + name: "input" + type: "data" + size: 300 + active_type: "" +} +layers { + name: "label" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "weight" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 10 + active_type: "softmax" + inputs { + input_layer_name: "input" + input_parameter_name: "___fc_layer_0__.w0" + } + bias_parameter_name: "___fc_layer_0__.wbias" +} +layers { + name: "__cost_0__" + type: "multi-class-cross-entropy" + size: 1 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + } + inputs { + input_layer_name: "label" + } + inputs { + input_layer_name: "weight" + } + coeff: 1.0 +} +layers { + name: "__regression_cost_0__" + type: "square_error" + size: 1 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + } + inputs { + input_layer_name: "label" + } + inputs { + input_layer_name: "weight" + } + coeff: 1.0 +} +parameters { + name: "___fc_layer_0__.w0" + size: 3000 + initial_mean: 0.0 + initial_std: 0.057735026919 + dims: 300 + dims: 10 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___fc_layer_0__.wbias" + size: 10 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 10 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "input" +input_layer_names: "label" +input_layer_names: "weight" +output_layer_names: "__cost_0__" +output_layer_names: "__regression_cost_0__" +evaluators { + name: "classification_error_evaluator" + type: "classification_error" + input_layers: "__fc_layer_0__" + input_layers: "label" + input_layers: "weight" +} +sub_models { + name: "root" + layer_names: "input" + layer_names: "label" + layer_names: "weight" + layer_names: "__fc_layer_0__" + layer_names: "__cost_0__" + layer_names: "__regression_cost_0__" + input_layer_names: "input" + input_layer_names: "label" + input_layer_names: "weight" + output_layer_names: "__cost_0__" + output_layer_names: "__regression_cost_0__" + evaluator_names: "classification_error_evaluator" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_expand_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_expand_layer.protostr new file mode 100644 index 0000000000000..f4b36052264bc --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_expand_layer.protostr @@ -0,0 +1,56 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 30 + active_type: "" +} +layers { + name: "data_seq" + type: "data" + size: 30 + active_type: "" +} +layers { + name: "__expand_layer_0__" + type: "expand" + size: 30 + active_type: "" + inputs { + input_layer_name: "data" + } + inputs { + input_layer_name: "data_seq" + } + trans_type: "seq" +} +layers { + name: "__expand_layer_1__" + type: "expand" + size: 30 + active_type: "" + inputs { + input_layer_name: "data" + } + inputs { + input_layer_name: "data_seq" + } + trans_type: "non-seq" +} +input_layer_names: "data" +input_layer_names: "data_seq" +output_layer_names: "__expand_layer_0__" +output_layer_names: "__expand_layer_1__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "data_seq" + layer_names: "__expand_layer_0__" + layer_names: "__expand_layer_1__" + input_layer_names: "data" + input_layer_names: "data_seq" + output_layer_names: "__expand_layer_0__" + output_layer_names: "__expand_layer_1__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_fc.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_fc.protostr new file mode 100644 index 0000000000000..8151898832ded --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_fc.protostr @@ -0,0 +1,98 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__trans_layer_0__" + type: "trans" + size: 100 + active_type: "" + inputs { + input_layer_name: "data" + } +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "__trans_layer_0__" + input_parameter_name: "___fc_layer_0__.w0" + } +} +layers { + name: "mask" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__selective_fc_layer_0__" + type: "selective_fc" + size: 100 + active_type: "sigmoid" + inputs { + input_layer_name: "data" + input_parameter_name: "___selective_fc_layer_0__.w0" + } + inputs { + input_layer_name: "mask" + } + bias_parameter_name: "___selective_fc_layer_0__.wbias" + selective_fc_pass_generation: false + has_selected_colums: true + selective_fc_full_mul_ratio: 0.02 +} +parameters { + name: "___fc_layer_0__.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___selective_fc_layer_0__.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true + is_sparse: false +} +parameters { + name: "___selective_fc_layer_0__.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +input_layer_names: "mask" +output_layer_names: "__fc_layer_0__" +output_layer_names: "__selective_fc_layer_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__trans_layer_0__" + layer_names: "__fc_layer_0__" + layer_names: "mask" + layer_names: "__selective_fc_layer_0__" + input_layer_names: "data" + input_layer_names: "mask" + output_layer_names: "__fc_layer_0__" + output_layer_names: "__selective_fc_layer_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_grumemory_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_grumemory_layer.protostr new file mode 100644 index 0000000000000..2c19b2fd120e7 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_grumemory_layer.protostr @@ -0,0 +1,51 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 120 + active_type: "" +} +layers { + name: "__gru_0__" + type: "gated_recurrent" + size: 40 + active_type: "sigmoid" + inputs { + input_layer_name: "data" + input_parameter_name: "___gru_0__.w0" + } + bias_parameter_name: "___gru_0__.wbias" + reversed: true + active_gate_type: "tanh" +} +parameters { + name: "___gru_0__.w0" + size: 4800 + initial_mean: 0.0 + initial_std: 0.158113883008 + dims: 40 + dims: 120 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___gru_0__.wbias" + size: 120 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 120 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +output_layer_names: "__gru_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__gru_0__" + input_layer_names: "data" + output_layer_names: "__gru_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_hsigmoid.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_hsigmoid.protostr new file mode 100644 index 0000000000000..e81fcb13c4c6e --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_hsigmoid.protostr @@ -0,0 +1,62 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "label" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "__hsigmoid_0__" + type: "hsigmoid" + size: 1 + active_type: "" + inputs { + input_layer_name: "data" + input_parameter_name: "___hsigmoid_0__.w0" + } + inputs { + input_layer_name: "label" + } + bias_parameter_name: "___hsigmoid_0__.wbias" + num_classes: 10 +} +parameters { + name: "___hsigmoid_0__.w0" + size: 900 + initial_mean: 0.0 + initial_std: 0.333333333333 + dims: 9 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___hsigmoid_0__.wbias" + size: 9 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 9 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +input_layer_names: "label" +output_layer_names: "__hsigmoid_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "label" + layer_names: "__hsigmoid_0__" + input_layer_names: "data" + input_layer_names: "label" + output_layer_names: "__hsigmoid_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_lstmemory_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_lstmemory_layer.protostr new file mode 100644 index 0000000000000..76a4afab82c59 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_lstmemory_layer.protostr @@ -0,0 +1,53 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 128 + active_type: "" +} +layers { + name: "__lstmemory_0__" + type: "lstmemory" + size: 32 + active_type: "tanh" + inputs { + input_layer_name: "data" + input_parameter_name: "___lstmemory_0__.w0" + } + bias_parameter_name: "___lstmemory_0__.wbias" + reversed: true + active_gate_type: "tanh" + active_state_type: "tanh" +} +parameters { + name: "___lstmemory_0__.w0" + size: 4096 + initial_mean: 0.0 + initial_std: 0.176776695297 + dims: 32 + dims: 32 + dims: 4 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstmemory_0__.wbias" + size: 224 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 224 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +output_layer_names: "__lstmemory_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__lstmemory_0__" + input_layer_names: "data" + output_layer_names: "__lstmemory_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr new file mode 100644 index 0000000000000..1be2a7ceebfb7 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_maxout.protostr @@ -0,0 +1,209 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 2304 + active_type: "" +} +layers { + name: "__conv_0__" + type: "exconv" + size: 36864 + active_type: "" + inputs { + input_layer_name: "data" + input_parameter_name: "___conv_0__.w0" + conv_conf { + filter_size: 3 + channels: 1 + stride: 1 + padding: 1 + groups: 1 + filter_channels: 1 + output_x: 48 + img_size: 48 + caffe_mode: true + filter_size_y: 3 + padding_y: 1 + stride_y: 1 + } + } + bias_parameter_name: "___conv_0__.wbias" + num_filters: 16 + shared_biases: true +} +layers { + name: "__maxout_layer_0__" + type: "maxout" + size: 18432 + active_type: "" + inputs { + input_layer_name: "__conv_0__" + maxout_conf { + channels: 16 + groups: 2 + img_size_x: 0 + img_size_y: 0 + } + } +} +layers { + name: "__pool_0__" + type: "pool" + size: 4608 + active_type: "" + inputs { + input_layer_name: "__maxout_layer_0__" + pool_conf { + pool_type: "max-projection" + channels: 8 + size_x: 2 + stride: 2 + output_x: 24 + img_size: 48 + padding: 0 + size_y: 2 + stride_y: 2 + output_y: 24 + img_size_y: 48 + padding_y: 0 + } + } +} +layers { + name: "__conv_1__" + type: "exconv" + size: 18432 + active_type: "" + inputs { + input_layer_name: "__pool_0__" + input_parameter_name: "___conv_1__.w0" + conv_conf { + filter_size: 3 + channels: 32 + stride: 1 + padding: 1 + groups: 1 + filter_channels: 32 + output_x: 12 + img_size: 12 + caffe_mode: true + filter_size_y: 3 + padding_y: 1 + stride_y: 1 + } + } + bias_parameter_name: "___conv_1__.wbias" + num_filters: 128 + shared_biases: true +} +layers { + name: "__maxout_layer_1__" + type: "maxout" + size: 9216 + active_type: "" + inputs { + input_layer_name: "__conv_0__" + maxout_conf { + channels: 128 + groups: 4 + img_size_x: 0 + img_size_y: 0 + } + } +} +layers { + name: "__block_expand_layer_0__" + type: "blockexpand" + size: 192 + active_type: "" + inputs { + input_layer_name: "__maxout_layer_0__" + block_expand_conf { + channels: 32 + stride_x: 1 + stride_y: 1 + padding_x: 0 + padding_y: 0 + block_x: 1 + block_y: 6 + output_x: 0 + output_y: 0 + img_size_x: 0 + img_size_y: 0 + } + } +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 384 + active_type: "tanh" + inputs { + input_layer_name: "__block_expand_layer_0__" + input_parameter_name: "___fc_layer_0__.w0" + } +} +parameters { + name: "___conv_0__.w0" + size: 144 + initial_mean: 0.0 + initial_std: 0.471404520791 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___conv_0__.wbias" + size: 16 + initial_mean: 0.0 + initial_std: 0.0 + dims: 16 + dims: 1 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___conv_1__.w0" + size: 36864 + initial_mean: 0.0 + initial_std: 0.0833333333333 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___conv_1__.wbias" + size: 128 + initial_mean: 0.0 + initial_std: 0.0 + dims: 128 + dims: 1 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___fc_layer_0__.w0" + size: 73728 + initial_mean: 0.0 + initial_std: 0.0721687836487 + dims: 192 + dims: 384 + initial_strategy: 0 + initial_smart: true +} +input_layer_names: "data" +output_layer_names: "__fc_layer_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__conv_0__" + layer_names: "__maxout_layer_0__" + layer_names: "__pool_0__" + layer_names: "__conv_1__" + layer_names: "__maxout_layer_1__" + layer_names: "__block_expand_layer_0__" + layer_names: "__fc_layer_0__" + input_layer_names: "data" + output_layer_names: "__fc_layer_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_ntm_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_ntm_layers.protostr new file mode 100644 index 0000000000000..b30bbb2a4e24d --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_ntm_layers.protostr @@ -0,0 +1,225 @@ +type: "nn" +layers { + name: "w" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "a" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "b" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "c" + type: "data" + size: 200 + active_type: "" +} +layers { + name: "d" + type: "data" + size: 31 + active_type: "" +} +layers { + name: "__interpolation_layer_0__" + type: "interpolation" + size: 100 + active_type: "" + inputs { + input_layer_name: "w" + } + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "b" + } +} +layers { + name: "__power_layer_0__" + type: "power" + size: 100 + active_type: "" + inputs { + input_layer_name: "w" + } + inputs { + input_layer_name: "a" + } +} +layers { + name: "__scaling_layer_0__" + type: "scaling" + size: 100 + active_type: "" + inputs { + input_layer_name: "w" + } + inputs { + input_layer_name: "a" + } +} +layers { + name: "__cos_sim_0__" + type: "cos" + size: 1 + active_type: "" + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "b" + } + cos_scale: 5 +} +layers { + name: "__cos_sim_1__" + type: "cos_vm" + size: 2 + active_type: "" + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "c" + } + cos_scale: 5 +} +layers { + name: "__sum_to_one_norm_layer_0__" + type: "sum_to_one_norm" + size: 100 + active_type: "" + inputs { + input_layer_name: "a" + } +} +layers { + name: "__conv_shift_layer_0__" + type: "conv_shift" + size: 100 + active_type: "" + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "d" + } +} +layers { + name: "__tensor_layer_0__" + type: "tensor" + size: 1000 + active_type: "" + inputs { + input_layer_name: "a" + input_parameter_name: "___tensor_layer_0__.w0" + } + inputs { + input_layer_name: "b" + } + bias_parameter_name: "___tensor_layer_0__.wbias" +} +layers { + name: "__slope_intercept_layer_0__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "a" + } + slope: 0.7 + intercept: 0.9 +} +layers { + name: "__linear_comb_layer_0__" + type: "convex_comb" + size: 2 + active_type: "" + inputs { + input_layer_name: "b" + } + inputs { + input_layer_name: "c" + } +} +parameters { + name: "___tensor_layer_0__.w0" + size: 10000000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 100 + dims: 1000 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___tensor_layer_0__.wbias" + size: 1000 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 1000 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "w" +input_layer_names: "a" +input_layer_names: "b" +input_layer_names: "c" +input_layer_names: "d" +output_layer_names: "__interpolation_layer_0__" +output_layer_names: "__power_layer_0__" +output_layer_names: "__scaling_layer_0__" +output_layer_names: "__cos_sim_0__" +output_layer_names: "__cos_sim_1__" +output_layer_names: "__sum_to_one_norm_layer_0__" +output_layer_names: "__conv_shift_layer_0__" +output_layer_names: "__tensor_layer_0__" +output_layer_names: "__slope_intercept_layer_0__" +output_layer_names: "__linear_comb_layer_0__" +sub_models { + name: "root" + layer_names: "w" + layer_names: "a" + layer_names: "b" + layer_names: "c" + layer_names: "d" + layer_names: "__interpolation_layer_0__" + layer_names: "__power_layer_0__" + layer_names: "__scaling_layer_0__" + layer_names: "__cos_sim_0__" + layer_names: "__cos_sim_1__" + layer_names: "__sum_to_one_norm_layer_0__" + layer_names: "__conv_shift_layer_0__" + layer_names: "__tensor_layer_0__" + layer_names: "__slope_intercept_layer_0__" + layer_names: "__linear_comb_layer_0__" + input_layer_names: "w" + input_layer_names: "a" + input_layer_names: "b" + input_layer_names: "c" + input_layer_names: "d" + output_layer_names: "__interpolation_layer_0__" + output_layer_names: "__power_layer_0__" + output_layer_names: "__scaling_layer_0__" + output_layer_names: "__cos_sim_0__" + output_layer_names: "__cos_sim_1__" + output_layer_names: "__sum_to_one_norm_layer_0__" + output_layer_names: "__conv_shift_layer_0__" + output_layer_names: "__tensor_layer_0__" + output_layer_names: "__slope_intercept_layer_0__" + output_layer_names: "__linear_comb_layer_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_print_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_print_layer.protostr new file mode 100644 index 0000000000000..c402aff174ab7 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_print_layer.protostr @@ -0,0 +1,26 @@ +type: "nn" +layers { + name: "input" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__print_0__" + type: "print" + active_type: "" + inputs { + input_layer_name: "input" + } +} +input_layer_names: "input" +output_layer_names: "input" +sub_models { + name: "root" + layer_names: "input" + layer_names: "__print_0__" + input_layer_names: "input" + output_layer_names: "input" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_rnn_group.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_rnn_group.protostr new file mode 100644 index 0000000000000..41d2e2f2671f5 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_rnn_group.protostr @@ -0,0 +1,650 @@ +type: "recurrent_nn" +layers { + name: "seq_input" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "sub_seq_input" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "label" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__mixed_0__" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "seq_input" + input_parameter_name: "___mixed_0__.w0" + proj_conf { + type: "fc" + name: "___mixed_0__.w0" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__mixed_1__" + type: "mixed" + size: 300 + active_type: "" + inputs { + input_layer_name: "seq_input" + input_parameter_name: "___mixed_1__.w0" + proj_conf { + type: "fc" + name: "___mixed_1__.w0" + input_size: 100 + output_size: 300 + } + } +} +layers { + name: "__recurrent_group_0__" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "seq_input@__recurrent_group_0__" + type: "scatter_agent" + size: 100 + active_type: "" +} +layers { + name: "rnn_forward+delay1@__recurrent_group_0__" + type: "agent" + size: 200 + active_type: "" +} +layers { + name: "rnn_forward@__recurrent_group_0__" + type: "fc" + size: 200 + active_type: "tanh" + inputs { + input_layer_name: "seq_input@__recurrent_group_0__" + input_parameter_name: "_rnn_forward@__recurrent_group_0__.w0" + } + inputs { + input_layer_name: "rnn_forward+delay1@__recurrent_group_0__" + input_parameter_name: "_rnn_forward@__recurrent_group_0__.w1" + } + bias_parameter_name: "_rnn_forward@__recurrent_group_0__.wbias" +} +layers { + name: "rnn_forward" + type: "gather_agent" + size: 200 + active_type: "" +} +layers { + name: "__last_seq_0__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "rnn_forward" + } + trans_type: "non-seq" +} +layers { + name: "__recurrent_group_1__" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "seq_input@__recurrent_group_1__" + type: "scatter_agent" + size: 100 + active_type: "" +} +layers { + name: "rnn_back+delay1@__recurrent_group_1__" + type: "agent" + size: 200 + active_type: "" +} +layers { + name: "rnn_back@__recurrent_group_1__" + type: "fc" + size: 200 + active_type: "tanh" + inputs { + input_layer_name: "seq_input@__recurrent_group_1__" + input_parameter_name: "_rnn_back@__recurrent_group_1__.w0" + } + inputs { + input_layer_name: "rnn_back+delay1@__recurrent_group_1__" + input_parameter_name: "_rnn_back@__recurrent_group_1__.w1" + } + bias_parameter_name: "_rnn_back@__recurrent_group_1__.wbias" +} +layers { + name: "rnn_back" + type: "gather_agent" + size: 200 + active_type: "" +} +layers { + name: "__first_seq_0__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "rnn_back" + } + select_first: true + trans_type: "non-seq" +} +layers { + name: "__recurrent_group_2__" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "sub_seq_input@__recurrent_group_2__" + type: "sequence_scatter_agent" + size: 100 + active_type: "" +} +layers { + name: "rnn_subseq_forward+delay1@__recurrent_group_2__" + type: "agent" + size: 200 + active_type: "" +} +layers { + name: "rnn_subseq_forward@__recurrent_group_2__" + type: "fc" + size: 200 + active_type: "tanh" + inputs { + input_layer_name: "sub_seq_input@__recurrent_group_2__" + input_parameter_name: "_rnn_subseq_forward@__recurrent_group_2__.w0" + } + inputs { + input_layer_name: "rnn_subseq_forward+delay1@__recurrent_group_2__" + input_parameter_name: "_rnn_subseq_forward@__recurrent_group_2__.w1" + } + bias_parameter_name: "_rnn_subseq_forward@__recurrent_group_2__.wbias" +} +layers { + name: "rnn_subseq_forward" + type: "sequence_gather_agent" + size: 200 + active_type: "" +} +layers { + name: "__last_seq_1__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "rnn_subseq_forward" + } + trans_type: "non-seq" +} +layers { + name: "__lstm_group_0___recurrent_group" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "__mixed_0__@__lstm_group_0___recurrent_group" + type: "scatter_agent" + size: 400 + active_type: "" +} +layers { + name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "__mixed_0__@__lstm_group_0___recurrent_group" + proj_conf { + type: "identity" + name: "___lstm_group_0___input_recurrent.w0" + input_size: 400 + output_size: 400 + } + } + inputs { + input_layer_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + input_parameter_name: "___lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group.w1" + proj_conf { + type: "fc" + name: "___lstm_group_0___input_recurrent.w1" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + type: "lstm_step" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + } + inputs { + input_layer_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + } + bias_parameter_name: "___lstm_group_0__@__lstm_group_0___recurrent_group.wbias" + active_gate_type: "sigmoid" + active_state_type: "sigmoid" +} +layers { + name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + type: "get_output" + size: 100 + active_type: "" + inputs { + input_layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + input_layer_argument: "state" + } +} +layers { + name: "__lstm_group_0__" + type: "gather_agent" + size: 100 + active_type: "" +} +layers { + name: "__last_seq_2__" + type: "seqlastins" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "__lstm_group_0__" + } + trans_type: "non-seq" +} +layers { + name: "__gru_group_0___recurrent_group" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "__mixed_1__@__gru_group_0___recurrent_group" + type: "scatter_agent" + size: 300 + active_type: "" +} +layers { + name: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__gru_group_0__@__gru_group_0___recurrent_group" + type: "gru_step" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "__mixed_1__@__gru_group_0___recurrent_group" + input_parameter_name: "___gru_group_0__@__gru_group_0___recurrent_group.w0" + } + inputs { + input_layer_name: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" + } + bias_parameter_name: "___gru_group_0__@__gru_group_0___recurrent_group.wbias" + active_gate_type: "sigmoid" +} +layers { + name: "__gru_group_0__" + type: "gather_agent" + size: 100 + active_type: "" +} +layers { + name: "__last_seq_3__" + type: "seqlastins" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "__gru_group_0__" + } + trans_type: "non-seq" +} +parameters { + name: "___mixed_0__.w0" + size: 40000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 400 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_1__.w0" + size: 30000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 300 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_forward@__recurrent_group_0__.w0" + size: 20000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_forward@__recurrent_group_0__.w1" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_forward@__recurrent_group_0__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_rnn_back@__recurrent_group_1__.w0" + size: 20000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_back@__recurrent_group_1__.w1" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_back@__recurrent_group_1__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_rnn_subseq_forward@__recurrent_group_2__.w0" + size: 20000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_subseq_forward@__recurrent_group_2__.w1" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106781187 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_subseq_forward@__recurrent_group_2__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group.w1" + size: 40000 + initial_mean: 0.0 + initial_std: 0.1 + dims: 100 + dims: 400 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstm_group_0__@__lstm_group_0___recurrent_group.wbias" + size: 300 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 300 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___gru_group_0__@__gru_group_0___recurrent_group.w0" + size: 30000 + initial_mean: 0.0 + initial_std: 0.01 + dims: 100 + dims: 300 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___gru_group_0__@__gru_group_0___recurrent_group.wbias" + size: 300 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 300 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "seq_input" +input_layer_names: "sub_seq_input" +output_layer_names: "__last_seq_0__" +output_layer_names: "__first_seq_0__" +output_layer_names: "__last_seq_1__" +output_layer_names: "__last_seq_2__" +output_layer_names: "__last_seq_3__" +sub_models { + name: "root" + layer_names: "seq_input" + layer_names: "sub_seq_input" + layer_names: "label" + layer_names: "__mixed_0__" + layer_names: "__mixed_1__" + layer_names: "__recurrent_group_0__" + layer_names: "rnn_forward" + layer_names: "__last_seq_0__" + layer_names: "__recurrent_group_1__" + layer_names: "rnn_back" + layer_names: "__first_seq_0__" + layer_names: "__recurrent_group_2__" + layer_names: "rnn_subseq_forward" + layer_names: "__last_seq_1__" + layer_names: "__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__" + layer_names: "__last_seq_2__" + layer_names: "__gru_group_0___recurrent_group" + layer_names: "__gru_group_0__" + layer_names: "__last_seq_3__" + input_layer_names: "seq_input" + input_layer_names: "sub_seq_input" + output_layer_names: "__last_seq_0__" + output_layer_names: "__first_seq_0__" + output_layer_names: "__last_seq_1__" + output_layer_names: "__last_seq_2__" + output_layer_names: "__last_seq_3__" + is_recurrent_layer_group: false +} +sub_models { + name: "__recurrent_group_0__" + layer_names: "seq_input@__recurrent_group_0__" + layer_names: "rnn_forward+delay1@__recurrent_group_0__" + layer_names: "rnn_forward@__recurrent_group_0__" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "rnn_forward@__recurrent_group_0__" + link_name: "rnn_forward+delay1@__recurrent_group_0__" + is_sequence: false + } + in_links { + layer_name: "seq_input" + link_name: "seq_input@__recurrent_group_0__" + has_subseq: false + } + out_links { + layer_name: "rnn_forward@__recurrent_group_0__" + link_name: "rnn_forward" + has_subseq: false + } + target_inlinkid: -1 +} +sub_models { + name: "__recurrent_group_1__" + layer_names: "seq_input@__recurrent_group_1__" + layer_names: "rnn_back+delay1@__recurrent_group_1__" + layer_names: "rnn_back@__recurrent_group_1__" + is_recurrent_layer_group: true + reversed: true + memories { + layer_name: "rnn_back@__recurrent_group_1__" + link_name: "rnn_back+delay1@__recurrent_group_1__" + is_sequence: false + } + in_links { + layer_name: "seq_input" + link_name: "seq_input@__recurrent_group_1__" + has_subseq: false + } + out_links { + layer_name: "rnn_back@__recurrent_group_1__" + link_name: "rnn_back" + has_subseq: false + } + target_inlinkid: -1 +} +sub_models { + name: "__recurrent_group_2__" + layer_names: "sub_seq_input@__recurrent_group_2__" + layer_names: "rnn_subseq_forward+delay1@__recurrent_group_2__" + layer_names: "rnn_subseq_forward@__recurrent_group_2__" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "rnn_subseq_forward@__recurrent_group_2__" + link_name: "rnn_subseq_forward+delay1@__recurrent_group_2__" + is_sequence: false + } + in_links { + layer_name: "sub_seq_input" + link_name: "sub_seq_input@__recurrent_group_2__" + has_subseq: true + } + out_links { + layer_name: "rnn_subseq_forward@__recurrent_group_2__" + link_name: "rnn_subseq_forward" + has_subseq: true + } + target_inlinkid: -1 +} +sub_models { + name: "__lstm_group_0___recurrent_group" + layer_names: "__mixed_0__@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + is_sequence: false + } + memories { + layer_name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + is_sequence: false + } + in_links { + layer_name: "__mixed_0__" + link_name: "__mixed_0__@__lstm_group_0___recurrent_group" + has_subseq: false + } + out_links { + layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0__" + has_subseq: false + } + target_inlinkid: -1 +} +sub_models { + name: "__gru_group_0___recurrent_group" + layer_names: "__mixed_1__@__gru_group_0___recurrent_group" + layer_names: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" + layer_names: "__gru_group_0__@__gru_group_0___recurrent_group" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "__gru_group_0__@__gru_group_0___recurrent_group" + link_name: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" + is_sequence: false + } + in_links { + layer_name: "__mixed_1__" + link_name: "__mixed_1__@__gru_group_0___recurrent_group" + has_subseq: false + } + out_links { + layer_name: "__gru_group_0__@__gru_group_0___recurrent_group" + link_name: "__gru_group_0__" + has_subseq: false + } + target_inlinkid: -1 +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_sequence_pooling.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_sequence_pooling.protostr new file mode 100644 index 0000000000000..1999c006d237e --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_sequence_pooling.protostr @@ -0,0 +1,111 @@ +type: "nn" +layers { + name: "dat_in" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__seq_pooling_0__" + type: "max" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + trans_type: "seq" +} +layers { + name: "__seq_pooling_1__" + type: "max" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + trans_type: "non-seq" +} +layers { + name: "__seq_pooling_2__" + type: "average" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + average_strategy: "average" + trans_type: "seq" +} +layers { + name: "__seq_pooling_3__" + type: "average" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + average_strategy: "average" + trans_type: "non-seq" +} +layers { + name: "__seq_pooling_4__" + type: "average" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + average_strategy: "sum" + trans_type: "seq" +} +layers { + name: "__seq_pooling_5__" + type: "average" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + average_strategy: "sum" + trans_type: "non-seq" +} +layers { + name: "__seq_pooling_6__" + type: "max" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + output_max_index: true + trans_type: "non-seq" +} +input_layer_names: "dat_in" +output_layer_names: "__seq_pooling_0__" +output_layer_names: "__seq_pooling_1__" +output_layer_names: "__seq_pooling_2__" +output_layer_names: "__seq_pooling_3__" +output_layer_names: "__seq_pooling_4__" +output_layer_names: "__seq_pooling_5__" +output_layer_names: "__seq_pooling_6__" +sub_models { + name: "root" + layer_names: "dat_in" + layer_names: "__seq_pooling_0__" + layer_names: "__seq_pooling_1__" + layer_names: "__seq_pooling_2__" + layer_names: "__seq_pooling_3__" + layer_names: "__seq_pooling_4__" + layer_names: "__seq_pooling_5__" + layer_names: "__seq_pooling_6__" + input_layer_names: "dat_in" + output_layer_names: "__seq_pooling_0__" + output_layer_names: "__seq_pooling_1__" + output_layer_names: "__seq_pooling_2__" + output_layer_names: "__seq_pooling_3__" + output_layer_names: "__seq_pooling_4__" + output_layer_names: "__seq_pooling_5__" + output_layer_names: "__seq_pooling_6__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/unused_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/unused_layers.protostr new file mode 100644 index 0000000000000..89ed28406e553 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/unused_layers.protostr @@ -0,0 +1,27 @@ +type: "nn" +layers { + name: "probs" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__sampling_id_layer_0__" + type: "sampling_id" + size: 100 + active_type: "" + inputs { + input_layer_name: "probs" + } +} +input_layer_names: "probs" +output_layer_names: "__sampling_id_layer_0__" +sub_models { + name: "root" + layer_names: "probs" + layer_names: "__sampling_id_layer_0__" + input_layer_names: "probs" + output_layer_names: "__sampling_id_layer_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/util_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/util_layers.protostr new file mode 100644 index 0000000000000..d0ad388165007 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/util_layers.protostr @@ -0,0 +1,81 @@ +type: "nn" +layers { + name: "a" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "b" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "__addto_0__" + type: "addto" + size: 10 + active_type: "" + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "b" + } +} +layers { + name: "__concat_0__" + type: "concat" + size: 20 + active_type: "" + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "b" + } +} +layers { + name: "__concat_1__" + type: "concat2" + size: 20 + active_type: "" + inputs { + input_layer_name: "a" + proj_conf { + type: "identity" + name: "___concat_1__.w0" + input_size: 10 + output_size: 10 + } + } + inputs { + input_layer_name: "b" + proj_conf { + type: "identity" + name: "___concat_1__.w1" + input_size: 10 + output_size: 10 + } + } +} +input_layer_names: "a" +input_layer_names: "b" +output_layer_names: "__addto_0__" +output_layer_names: "__concat_0__" +output_layer_names: "__concat_1__" +sub_models { + name: "root" + layer_names: "a" + layer_names: "b" + layer_names: "__addto_0__" + layer_names: "__concat_0__" + layer_names: "__concat_1__" + input_layer_names: "a" + input_layer_names: "b" + output_layer_names: "__addto_0__" + output_layer_names: "__concat_0__" + output_layer_names: "__concat_1__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh b/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh index 78114ce32b019..f05fc46cd5520 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh @@ -1,5 +1,17 @@ #!/bin/bash cd `dirname $0` + set -e + +protostr=`dirname $0`/protostr + +files=`ls $protostr | grep -v "unitest"` + ./generate_protostr.sh -md5sum -c check.md5 + +for file in $files +do + base_protostr=$protostr/$file + new_protostr=$protostr/$file.unitest + diff $base_protostr $new_protostr +done From e05f4ff26700dd34aa6d3c6da7061c62c5fa39c9 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sun, 6 Nov 2016 23:04:02 -0600 Subject: [PATCH 221/324] Fix SRL hang when exit. (#291) * Fix SRL hang when exit. * Error occurred when enable Async Load in TestDataProvider. * It because DataProvider is calling getNextBatchInternal in one thread, and destructing DataProvider in other thread. * Add wait routine in DataProvider destructing. * Also fix another bug, when destructing TestDataProvider and do not read any test data. Fix #286 * Follow comments, Use mutex is cool! --- demo/semantic_role_labeling/.gitignore | 10 ++++++++++ paddle/gserver/dataproviders/DataProvider.cpp | 3 ++- .../gserver/dataproviders/PyDataProvider2.cpp | 19 +++++++++++++++++-- paddle/gserver/tests/test_PyDataProvider2.cpp | 17 +++++++++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 demo/semantic_role_labeling/.gitignore diff --git a/demo/semantic_role_labeling/.gitignore b/demo/semantic_role_labeling/.gitignore new file mode 100644 index 0000000000000..cd90ca7bbe9be --- /dev/null +++ b/demo/semantic_role_labeling/.gitignore @@ -0,0 +1,10 @@ +*.pyc +train.log +data/feature +data/conll05st-release/ +data/src.dict +data/test.wsj.props +data/test.wsj.seq_pair +data/test.wsj.words +data/tgt.dict +output diff --git a/paddle/gserver/dataproviders/DataProvider.cpp b/paddle/gserver/dataproviders/DataProvider.cpp index 8cefbb30ada46..2cfb5a3a18c8a 100644 --- a/paddle/gserver/dataproviders/DataProvider.cpp +++ b/paddle/gserver/dataproviders/DataProvider.cpp @@ -131,9 +131,10 @@ void DoubleBuffer::asyncLoadBatch() { taskReadySem_.wait(); if (stopping_) break; - while (batchSize_ == 0) { + while (batchSize_ == 0 && !stopping_) { usleep(5); } + if (stopping_) break; do { DataBatch newBatch; diff --git a/paddle/gserver/dataproviders/PyDataProvider2.cpp b/paddle/gserver/dataproviders/PyDataProvider2.cpp index ca8b07af49ca0..90391a7c307d8 100644 --- a/paddle/gserver/dataproviders/PyDataProvider2.cpp +++ b/paddle/gserver/dataproviders/PyDataProvider2.cpp @@ -433,26 +433,34 @@ class PyDataProvider2 : public DataProvider { inline void resetImpl(bool startNewThread) { DBG << "Reseting " << startNewThread; + exit_.store(true); if (loadThread_) { // is loading. - exit_.store(true); loadThread_->join(); loadThread_.reset(); } { PyGuard g; callingContexts_.clear(); + this->pullCV_.notify_one(); + } + + std::lock_guard guard(mutexForReset_); + { + PyGuard g; dataPool_.clear(); } poolActualSize_ = 0; - exit_ = false; + if (startNewThread && cache_->reset()) { DBG << "Start new thread."; loadThread_.reset(new std::thread([this] { + exit_ = false; loadThread(); })); callingContextCreated_.wait(); } DBG << "Reset done"; + exit_ = false; } private: @@ -465,6 +473,8 @@ class PyDataProvider2 : public DataProvider { std::condition_variable pullCV_; std::mutex mtx_; + std::mutex mutexForReset_; + ThreadBarrier callingContextCreated_; std::unique_ptr cache_; @@ -529,6 +539,7 @@ class PyDataProvider2 : public DataProvider { * Loading a batch of data. */ int64_t getNextBatchInternal(int64_t size_, DataBatch *batch) { + std::lock_guard guard(mutexForReset_); REGISTER_TIMER("PyDP2.getNextBatchInternal") CHECK_GE(size_, 0); size_t size = (size_t) size_; @@ -554,6 +565,10 @@ class PyDataProvider2 : public DataProvider { } else { // loading from cache. poolPtr = this->cache_->load(); } + if (exit_) { + // PyDataProvider is destructing. + return 0; + } CHECK(poolPtr != nullptr); std::deque& pool = *poolPtr; diff --git a/paddle/gserver/tests/test_PyDataProvider2.cpp b/paddle/gserver/tests/test_PyDataProvider2.cpp index 6bf1e32925121..b9867a728d9b4 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.cpp +++ b/paddle/gserver/tests/test_PyDataProvider2.cpp @@ -353,6 +353,23 @@ TEST(PyDataProvider2, test_check) { } } +TEST(PyDataProvider2, multiThread) { + paddle::DataConfig config; + config.set_type("py2"); + config.set_files(FLAGS_train_list.c_str()); + config.set_load_data_module("test_PyDataProvider2"); + config.set_load_data_object("test_dense_no_seq"); + config.set_async_load_data(true); + + std::unique_ptr provider( + paddle::DataProvider::create(config, false)); + provider->reset(); + paddle::DataBatch batch; + provider->getNextBatch(100, &batch); + provider->reset(); + provider.reset(); +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); paddle::initMain(argc, argv); From 57348806b5a9f83536113578adc6746fe6b19421 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 7 Nov 2016 14:27:06 +0800 Subject: [PATCH 222/324] Follow comments --- cmake/flags.cmake | 6 +- doc/ui/api/trainer_config_helpers/layers.rst | 2 +- paddle/cuda/include/hl_cnn.h | 12 +- paddle/cuda/include/stub/hl_cnn_stub.h | 8 +- paddle/cuda/src/hl_cuda_cnn.cu | 118 +++++++++--------- paddle/gserver/layers/BilinearInterpLayer.cpp | 19 ++- paddle/gserver/layers/BilinearInterpLayer.h | 2 +- paddle/gserver/tests/test_LayerGrad.cpp | 2 - paddle/math/Matrix.cpp | 24 +++- proto/ModelConfig.proto.m4 | 9 +- python/paddle/trainer/config_parser.py | 13 +- .../paddle/trainer_config_helpers/layers.py | 28 +---- .../tests/configs/test_bilinear_interp.py | 5 +- 13 files changed, 121 insertions(+), 127 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index dbad6be3f41b3..8c5cb4cc49d93 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -57,9 +57,9 @@ endif() set(COMMON_FLAGS -fPIC -fno-omit-frame-pointer - -Wall - -Wextra - -Werror +# -Wall +# -Wextra +# -Werror -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index ab27c3bd6e8ad..c78682423e448 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -276,7 +276,7 @@ interpolation_layer :noindex: bilinear_interp_layer -------------------- +---------------------- .. automodule:: paddle.trainer_config_helpers.layers :members: bilinear_interp_layer :noindex: diff --git a/paddle/cuda/include/hl_cnn.h b/paddle/cuda/include/hl_cnn.h index b5240da0f398c..ac35727ac28c7 100644 --- a/paddle/cuda/include/hl_cnn.h +++ b/paddle/cuda/include/hl_cnn.h @@ -254,6 +254,8 @@ extern void hl_CMRNorm_backward( * @param[in] outputH output batchSize. * @param[in] outputW output image data dim. * @param[in] numChannels number of channels. + * @param[in] ratioH inImgH / outImgH. + * @param[in] ratioW inImgW / outImgW. * */ extern void hl_bilinear_forward(const real* inData, @@ -266,7 +268,9 @@ extern void hl_bilinear_forward(const real* inData, const size_t outImgW, const size_t outputH, const size_t outputW, - const size_t numChannels); + const size_t numChannels, + const real ratioH, + const real ratioW); /** * @brief Bilinear interpolation backward. @@ -282,6 +286,8 @@ extern void hl_bilinear_forward(const real* inData, * @param[in] outputH output batchSize. * @param[in] outputW output image data dim. * @param[in] numChannels number of channels. + * @param[in] ratioH inImgH / outImgH. + * @param[in] ratioW inImgW / outImgW. * */ extern void hl_bilinear_backward(real* inGrad, @@ -294,7 +300,9 @@ extern void hl_bilinear_backward(real* inGrad, const size_t outImgW, const size_t outputH, const size_t outputW, - const size_t numChannels); + const size_t numChannels, + const real ratioH, + const real ratioW); /** * @brief MaxOut forward. diff --git a/paddle/cuda/include/stub/hl_cnn_stub.h b/paddle/cuda/include/stub/hl_cnn_stub.h index cf79fad9004cd..50fddce584252 100644 --- a/paddle/cuda/include/stub/hl_cnn_stub.h +++ b/paddle/cuda/include/stub/hl_cnn_stub.h @@ -99,7 +99,9 @@ inline void hl_bilinear_forward(const real* inData, const size_t outImgW, const size_t outputH, const size_t outputW, - const size_t numChannels) {} + const size_t numChannels, + const real ratioH, + const real ratioW) {} inline void hl_bilinear_backward(real* inGrad, const size_t inImgH, @@ -111,7 +113,9 @@ inline void hl_bilinear_backward(real* inGrad, const size_t outImgW, const size_t outputH, const size_t outputW, - const size_t numChannels) {} + const size_t numChannels, + const real ratioH, + const real ratioW) {} inline void hl_maxout_forward( const real* inData, real* outData, int* idData, diff --git a/paddle/cuda/src/hl_cuda_cnn.cu b/paddle/cuda/src/hl_cuda_cnn.cu index 499b61195af5e..49c09334e086d 100644 --- a/paddle/cuda/src/hl_cuda_cnn.cu +++ b/paddle/cuda/src/hl_cuda_cnn.cu @@ -547,29 +547,32 @@ __global__ void KeBilinearInterpFw(const size_t nthreads, const real ratioH, const real ratioW) { int tid = blockIdx.x * blockDim.x + threadIdx.x; - if(tid < nthreads) { - int outIdH = tid / (outputW / numChannels); - int outIdW = tid % (outputW / numChannels); - - int inIdH = ratioH * (outIdW / outImgW); - int hId = (inIdH < inImgH - 1) ? 1 : 0; - real hlambda = ratioH * (outIdW / outImgW) - inIdH; - - int inIdW = ratioW * (tid % outImgW); - int wId = (inIdW < inImgW - 1) ? 1 : 0; - real wlambda = ratioW * (tid % outImgW) - inIdW; - - const real* inPos = &in[outIdH * inputW + inIdH * inImgW + inIdW]; - real* outPos = &out[outIdH * outputW + outIdW]; - for (int c = 0; c < numChannels; ++c) { - // bilinear interpolation - outPos[0] = (1.f - hlambda) * - ((1.f - wlambda) * inPos[0] + wlambda * inPos[wId]) + - hlambda * ((1.f - wlambda) * inPos[hId * inImgW] + - wlambda * inPos[hId * inImgW + wId]); - inPos += inImgH * inImgW; - outPos += outImgH * outImgW; - } + if (tid < nthreads) { + int outIdH = tid / outputW; + int outIdW = tid % outputW; + int inImgSize = inputW / numChannels; + int outImgSize = outputW / numChannels; + int channelId = outIdW / outImgSize; + + int outImgIdy = (outIdW % outImgSize) / outImgW; + int inImgIdy = ratioH * outImgIdy; + int hId = (inImgIdy < inImgH - 1) ? 1 : 0; + real h1lambda = ratioH * outImgIdy - inImgIdy; + real h2lambda = 1.f - h1lambda; + + int outImgIdx = tid % outImgW; + int inImgIdx = ratioW * outImgIdx; + int wId = (inImgIdx < inImgW - 1) ? 1 : 0; + real w1lambda = ratioW * outImgIdx - inImgIdx; + real w2lambda = 1.f - w1lambda; + + const real* inPos = + &in[outIdH * inputW + channelId * inImgSize + inImgIdy * inImgW + inImgIdx]; + + // bilinear interpolation + out[outIdH * outputW + outIdW] = + h2lambda * (w2lambda * inPos[0] + w1lambda * inPos[wId]) + + h1lambda * (w2lambda * inPos[hId * inImgW] + w1lambda * inPos[hId * inImgW + wId]); } } @@ -583,15 +586,12 @@ void hl_bilinear_forward(const real* inData, const size_t outImgW, const size_t outputH, const size_t outputW, - const size_t numChannels) { - int threadNum = outputH * outImgH * outImgW; + const size_t numChannels, + const real ratioH, + const real ratioW) { + int threadNum = outputH * outputW; int blocks = (threadNum + 1024 - 1) / 1024; - real ratioH = (outImgH > 1) ? - static_cast(inImgH - 1) / (outImgH - 1) : 0.f; - real ratioW = (outImgW > 1) ? - static_cast(inImgW - 1) / (outImgW - 1) : 0.f; - KeBilinearInterpFw<<< blocks, 1024, 0, STREAM_DEFAULT>>>( threadNum, inData, inImgH, inImgW, inputH, inputW, outData, outImgH, outImgW, outputH, outputW, numChannels, ratioH, ratioW); @@ -613,29 +613,32 @@ __global__ void KeBilinearInterpBw(const size_t nthreads, const real ratioH, const real ratioW) { int tid = blockIdx.x * blockDim.x + threadIdx.x; - - if(tid < nthreads) { - int outIdH = tid / (outputW / numChannels); - int outIdW = tid % (outputW / numChannels); - - int inIdH = ratioH * (outIdW / outImgW); - int hId = (inIdH < inImgH - 1) ? 1 : 0; - real hlambda = ratioH * (outIdW / outImgW) - inIdH; - - int inIdW = ratioW * (tid % outImgW); - int wId = (inIdW < inImgW - 1) ? 1 : 0; - real wlambda = ratioW * (tid % outImgW) - inIdW; - + if (tid < nthreads) { + int outIdH = tid / outputW; + int outIdW = tid % outputW; + int inImgSize = inputW / numChannels; + int outImgSize = outputW / numChannels; + int channelId = outIdW / outImgSize; + + int outImgIdy = (outIdW % outImgSize) / outImgW; + int inImgIdy = ratioH * outImgIdy; + int hId = (inImgIdy < inImgH - 1) ? 1 : 0; + real h1lambda = ratioH * outImgIdy - inImgIdy; + real h2lambda = 1.f - h1lambda; + + int outImgIdx = tid % outImgW; + int inImgIdx = ratioW * outImgIdx; + int wId = (inImgIdx < inImgW - 1) ? 1 : 0; + real w1lambda = ratioW * outImgIdx - inImgIdx; + real w2lambda = 1.f - w1lambda; + + real* inPos = + &in[outIdH * inputW + channelId * inImgSize + inImgIdy * inImgW + inImgIdx]; const real* outPos = &out[outIdH * outputW + outIdW]; - real* inPos = &in[outIdH * inputW + inIdH * inImgW + inIdW]; - for (int c = 0; c < numChannels; ++c) { - atomicAdd(&inPos[0], (1.f - hlambda) * (1.f - wlambda) * outPos[0]); - atomicAdd(&inPos[wId], (1.f - hlambda) * wlambda * outPos[0]); - atomicAdd(&inPos[hId * inImgW], hlambda * (1.f - wlambda) * outPos[0]); - atomicAdd(&inPos[hId * inImgW + wId], hlambda * wlambda * outPos[0]); - inPos += inImgH * inImgW; - outPos += outImgH * outImgW; - } + atomicAdd(&inPos[0], h2lambda * w2lambda * outPos[0]); + atomicAdd(&inPos[wId], h2lambda * w1lambda * outPos[0]); + atomicAdd(&inPos[hId * inImgW], h1lambda * w2lambda * outPos[0]); + atomicAdd(&inPos[hId * inImgW + wId], h1lambda * w1lambda * outPos[0]); } } @@ -649,14 +652,11 @@ void hl_bilinear_backward(real* inGrad, const size_t outImgW, const size_t outputH, const size_t outputW, - const size_t numChannels) { - int threadNum = outputH * outImgH * outImgW; + const size_t numChannels, + const real ratioH, + const real ratioW) { + int threadNum = outputH * outputW; int blocks = (threadNum + 1024 - 1) / 1024; - - real ratioH = (outImgH > 1) ? - static_cast(inImgH - 1) / (outImgH - 1) : 0.f; - real ratioW = (outImgW > 1) ? - static_cast(inImgW - 1) / (outImgW - 1) : 0.f; KeBilinearInterpBw<<< blocks, 1024, 0, STREAM_DEFAULT>>>( threadNum, inGrad, inImgH, inImgW, inputH, inputW, outGrad, diff --git a/paddle/gserver/layers/BilinearInterpLayer.cpp b/paddle/gserver/layers/BilinearInterpLayer.cpp index f43086e585535..9f0c01a838562 100644 --- a/paddle/gserver/layers/BilinearInterpLayer.cpp +++ b/paddle/gserver/layers/BilinearInterpLayer.cpp @@ -20,7 +20,11 @@ namespace paddle { REGISTER_LAYER(bilinear_interp, BilinearInterpLayer); -size_t BilinearInterpLayer::getDataDimSize() { +size_t BilinearInterpLayer::getSize() { + inImgH_ = inputLayers_[0]->getOutput().getFrameHeight(); + inImgW_ = inputLayers_[0]->getOutput().getFrameWidth(); + CHECK(inImgH_ > 0 && inImgW_ > 0); + getOutput().setFrameHeight(outImgH_); getOutput().setFrameWidth(outImgW_); return outImgH_ * outImgW_ * numChannels_; @@ -34,20 +38,12 @@ bool BilinearInterpLayer::init(const LayerMap& layerMap, CHECK_EQ(1, config_.inputs_size()); const BilinearInterpConfig& conf = config_.inputs(0).bilinear_interp_conf(); - inImgH_ = inputLayers_[0]->getOutput().getFrameHeight(); - inImgW_ = inputLayers_[0]->getOutput().getFrameWidth(); - if (inImgH_ == 0) { - inImgH_ = conf.img_size_y(); - } - if (inImgW_ == 0) { - inImgW_ = conf.img_size_x(); - } + outImgH_ = conf.out_size_y(); outImgW_ = conf.out_size_x(); numChannels_ = conf.num_channels(); CHECK(outImgH_ > 0 && outImgW_ > 0); - CHECK(inImgH_ > 0 && inImgW_ > 0); CHECK(numChannels_); return true; @@ -55,8 +51,9 @@ bool BilinearInterpLayer::init(const LayerMap& layerMap, void BilinearInterpLayer::forward(PassType passType) { Layer::forward(passType); + size_t batchSize = getInput(0).getBatchSize(); - size_t size = getDataDimSize(); + size_t size = getSize(); { REGISTER_TIMER_INFO("FwResetTimer", getName().c_str()); resetOutput(batchSize, size); diff --git a/paddle/gserver/layers/BilinearInterpLayer.h b/paddle/gserver/layers/BilinearInterpLayer.h index 24f5b99910405..33e0cb1220511 100644 --- a/paddle/gserver/layers/BilinearInterpLayer.h +++ b/paddle/gserver/layers/BilinearInterpLayer.h @@ -36,7 +36,7 @@ class BilinearInterpLayer : public Layer { virtual ~BilinearInterpLayer() {} - size_t getDataDimSize(); + size_t getSize(); bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); void forward(PassType passType); void backward(const UpdateCallback& callback = nullptr); diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index db48cc47a4a63..54a9aea024423 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -40,8 +40,6 @@ TEST(Layer, BilinearInterpLayer) { LayerInputConfig* input = config.layerConfig.add_inputs(); BilinearInterpConfig* bilinear = input->mutable_bilinear_interp_conf(); - bilinear->set_img_size_x(32); - bilinear->set_img_size_y(32); bilinear->set_out_size_x(64); bilinear->set_out_size_y(64); bilinear->set_num_channels(4); diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index ce4d2ac39938f..33bc8d280fe8c 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -1197,12 +1197,18 @@ void GpuMatrix::bilinearForward(const Matrix& in, real* outData = getData(); const real* inData = in.getData(); + real ratioH = (outImgH > 1) ? + static_cast(inImgH - 1) / (outImgH - 1) : 0.f; + real ratioW = (outImgW > 1) ? + static_cast(inImgW - 1) / (outImgW - 1) : 0.f; + if (inImgH == outImgW && inImgW == outImgW) { this->copyFrom(in); } else { - hl_bilinear_forward(inData, inImgH, inImgW, - inputH, inputW, outData, outImgH, outImgW, - outputH, outputW, numChannels); + hl_bilinear_forward( + inData, inImgH, inImgW, inputH, inputW, outData, + outImgH, outImgW, outputH, outputW, numChannels, + ratioH, ratioW); } } @@ -1222,12 +1228,18 @@ void GpuMatrix::bilinearBackward(const Matrix& out, real* inGrad = getData(); const real* outGrad = out.getData(); + real ratioH = (outImgH > 1) ? + static_cast(inImgH - 1) / (outImgH - 1) : 0.f; + real ratioW = (outImgW > 1) ? + static_cast(inImgW - 1) / (outImgW - 1) : 0.f; + if (outImgH == inImgH && outImgW == inImgW) { this->copyFrom(out); } else { - hl_bilinear_backward(inGrad, inImgH, inImgW, - inputH, inputW, outGrad, outImgH, outImgW, - outputH, outputW, numChannels); + hl_bilinear_backward( + inGrad, inImgH, inImgW, inputH, inputW, outGrad, + outImgH, outImgW, outputH, outputW, numChannels, + ratioH, ratioW); } } diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index 753fd0cac4223..a1eb11eccaeda 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -213,13 +213,10 @@ message OperatorConfig { } message BilinearInterpConfig { - // The size if input feature map. - required uint32 img_size_x = 1; - required uint32 img_size_y = 2; // The size if output feature map. - required uint32 out_size_x = 3; - required uint32 out_size_y = 4; - required uint32 num_channels = 5; + required uint32 out_size_x = 1; + required uint32 out_size_y = 2; + required uint32 num_channels = 3; } message ImageConfig { diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index c6cd4f62b91c9..574c02eefc35f 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -734,8 +734,6 @@ def __init__( class BilinearInterp(Cfg): def __init__( self, - img_size_x = None, - img_size_y=None, out_size_x = None, out_size_y = None, num_channels = None): @@ -982,8 +980,6 @@ def TestData(data_config, async_load_data=None): g_config.test_data_config.async_load_data = async_load_data def parse_bilinear(bilinear, input_layer_name, bilinear_conf): - bilinear_conf.img_size_x = bilinear.img_size_x; - bilinear_conf.img_size_y = bilinear.img_size_y; bilinear_conf.out_size_x = bilinear.out_size_x; bilinear_conf.out_size_y = bilinear.out_size_y; bilinear_conf.num_channels = bilinear.num_channels; @@ -2367,15 +2363,16 @@ def __init__( self, name, inputs, - device=None): + **xargs): super(BilinearInterpLayer, self).__init__( - name, 'bilinear_interp', 0, inputs=inputs, device=device) + name, 'bilinear_interp', 0, inputs=inputs, **xargs) input_layer = self.get_input_layer(0) - self.set_layer_size(input_layer.size) parse_bilinear(self.inputs[0].bilinear_interp, input_layer.name, self.config.inputs[0].bilinear_interp_conf); - + conf = self.inputs[0].bilinear_interp + self.set_layer_size(conf.out_size_x * conf.out_size_y * conf.num_channels) + @config_layer('sum_to_one_norm') class SumToOneNormLayer(LayerBase): def __init__( diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 8d249b140e8cd..6457c60a35392 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1259,11 +1259,8 @@ def interpolation_layer(input, weight, name=None, layer_attr=None): @wrap_name_default() @layer_support() def bilinear_interp_layer(input, - img_size_x=None, - img_size_y=None, out_size_x=None, out_size_y=None, - num_channels=None, name=None, layer_attr=None): """ @@ -1276,25 +1273,15 @@ def bilinear_interp_layer(input, .. code-block:: python bilinear = bilinear_interp_layer(input, - img_size_x, - img_size_y, out_size_x, - out_size_y, - num_channels) + out_size_y) :para input: A input layer. :type input: LayerOutput. - :para img_size_x: previous layer output width. - :type img_size_x: int|None - :para img_size_y: previous layer output height. - :type img_size_y: int|None :para out_size_x: bilinear interpolation output width. :type out_size_x: int|None :para out_size_y: bilinear interpolation output height. :type out_size_y: int|None - :para num_channels: number of channels of input layer. If None, - it will be set automatically from previous output. - :type num_channels: int|None :para name: The layer's name, which cna not be specified. :type name: None|basestring :para layer_attr: Extra Layer attribute. @@ -1304,21 +1291,18 @@ def bilinear_interp_layer(input, """ assert input.layer_type == LayerType.CONV_LAYER assert isinstance(input.activation, LinearActivation) - assert img_size_x > 0 and img_size_y > 0 assert out_size_x > 0 and out_size_y > 0 - if num_channels is None: - assert input.numfilters is not None - num_channels = input.num_filters + assert input.numfilters is not None + num_channels = input.num_filters Layer(name=name, inputs=Input(input.name, - bilinear_interp=BilinearInterp(img_size_x=img_size_x, - img_size_y=img_size_y, - out_size_x=out_size_x, + bilinear_interp=BilinearInterp(out_size_x=out_size_x, out_size_y=out_size_y, num_channels=num_channels)), type=LayerType.BILINEAR_INTERP_LAYER, **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.BILINEAR_INTERP_LAYER, parents=[input]) + return LayerOutput(name, LayerType.BILINEAR_INTERP_LAYER, parents=[input], + num_filters=num_filters) @wrap_name_default() @layer_support() diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py b/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py index 7815b34abcc25..5a61c5a395af2 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.py @@ -16,11 +16,8 @@ bias_attr=True) bilinear = bilinear_interp_layer(input=conv, - img_size_x=32, - img_size_y=32, out_size_x=64, - out_size_y=64, - num_channels=16) + out_size_y=64) pool = img_pool_layer(input=bilinear, num_channels=4, From 45b8c47e043e6163dc3e228bf2fb7a6f6cab9b43 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 7 Nov 2016 15:10:15 +0800 Subject: [PATCH 223/324] Add img_size for unit test --- cmake/flags.cmake | 6 ++--- paddle/gserver/layers/BilinearInterpLayer.cpp | 24 ++++++++++++------- paddle/gserver/tests/test_LayerGrad.cpp | 2 ++ proto/ModelConfig.proto.m4 | 11 +++++---- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 337db53b37fa3..e087770991aef 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -65,9 +65,9 @@ endif() set(COMMON_FLAGS -fPIC -fno-omit-frame-pointer -# -Wall -# -Wextra -# -Werror + -Wall + -Wextra + -Werror -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter diff --git a/paddle/gserver/layers/BilinearInterpLayer.cpp b/paddle/gserver/layers/BilinearInterpLayer.cpp index 9f0c01a838562..f37efc824a2ec 100644 --- a/paddle/gserver/layers/BilinearInterpLayer.cpp +++ b/paddle/gserver/layers/BilinearInterpLayer.cpp @@ -23,7 +23,22 @@ REGISTER_LAYER(bilinear_interp, BilinearInterpLayer); size_t BilinearInterpLayer::getSize() { inImgH_ = inputLayers_[0]->getOutput().getFrameHeight(); inImgW_ = inputLayers_[0]->getOutput().getFrameWidth(); + + const BilinearInterpConfig& conf = config_.inputs(0).bilinear_interp_conf(); + if (inImgH_ == 0) { + inImgH_ = conf.img_size_y(); + } + if (inImgW_ == 0) { + inImgW_ = conf.img_size_x(); + } + + outImgH_ = conf.out_size_y(); + outImgW_ = conf.out_size_x(); + numChannels_ = conf.num_channels(); + + CHECK(outImgH_ > 0 && outImgW_ > 0); CHECK(inImgH_ > 0 && inImgW_ > 0); + CHECK(numChannels_); getOutput().setFrameHeight(outImgH_); getOutput().setFrameWidth(outImgW_); @@ -37,15 +52,6 @@ bool BilinearInterpLayer::init(const LayerMap& layerMap, CHECK_EQ(1, config_.inputs_size()); - const BilinearInterpConfig& conf = config_.inputs(0).bilinear_interp_conf(); - - outImgH_ = conf.out_size_y(); - outImgW_ = conf.out_size_x(); - numChannels_ = conf.num_channels(); - - CHECK(outImgH_ > 0 && outImgW_ > 0); - CHECK(numChannels_); - return true; } diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 8ee89992ddf1e..b5b944833e47f 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -40,6 +40,8 @@ TEST(Layer, BilinearInterpLayer) { LayerInputConfig* input = config.layerConfig.add_inputs(); BilinearInterpConfig* bilinear = input->mutable_bilinear_interp_conf(); + bilinear->set_img_size_x(32); + bilinear->set_img_size_y(32); bilinear->set_out_size_x(64); bilinear->set_out_size_y(64); bilinear->set_num_channels(4); diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index 41d081942778e..479b457e55a7d 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -213,10 +213,13 @@ message OperatorConfig { } message BilinearInterpConfig { - // The size if output feature map. - required uint32 out_size_x = 1; - required uint32 out_size_y = 2; - required uint32 num_channels = 3; + // The size of input feature map. + optional uint32 img_size_x = 1; + optional uint32 img_size_y = 2; + // The size of output feature map. + required uint32 out_size_x = 3; + required uint32 out_size_y = 4; + required uint32 num_channels = 5; } message ImageConfig { From d373c10091205e2ec44f1a1781a73a8869166272 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 7 Nov 2016 15:53:22 +0800 Subject: [PATCH 224/324] Fix bilinear interp bug --- cmake/flags.cmake | 6 +++--- paddle/math/Matrix.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index e087770991aef..03407e72d60b2 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -65,9 +65,9 @@ endif() set(COMMON_FLAGS -fPIC -fno-omit-frame-pointer - -Wall - -Wextra - -Werror +# -Wall +# -Wextra +# -Werror -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index f1af9536ba5d6..3b11e831fe429 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -1275,7 +1275,7 @@ void GpuMatrix::bilinearBackward(const Matrix& out, static_cast(inImgW - 1) / (outImgW - 1) : 0.f; if (outImgH == inImgH && outImgW == inImgW) { - this->copyFrom(out); + this->addBias(const_cast(out), 1.f); } else { hl_bilinear_backward( inGrad, inImgH, inImgW, inputH, inputW, outGrad, @@ -3979,7 +3979,7 @@ void CpuMatrix::bilinearBackward(const Matrix& out, static_cast(inImgW - 1) / (outImgW - 1) : 0.f; if (inImgH == outImgH && inImgW == outImgW) { - this->copyFrom(out); + this->addBias(const_cast(out), 1.f); } else { for (size_t k = 0; k < outputH; ++k) { // loop for batches for (size_t i = 0; i < outImgH; ++i) { // loop for images From 6e12482e4fd7c920349e832e799c14b779ec9e8c Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 7 Nov 2016 15:53:48 +0800 Subject: [PATCH 225/324] revert flags.cmake --- cmake/flags.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 03407e72d60b2..e087770991aef 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -65,9 +65,9 @@ endif() set(COMMON_FLAGS -fPIC -fno-omit-frame-pointer -# -Wall -# -Wextra -# -Werror + -Wall + -Wextra + -Werror -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wno-unused-parameter From 724d6dd40a6af8b0f60ea206e34ca2ba1515eff5 Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 7 Nov 2016 16:08:36 +0800 Subject: [PATCH 226/324] Replace outputH to batchSize --- paddle/math/Matrix.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 3b11e831fe429..283733fe845d4 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -3912,7 +3912,7 @@ void CpuMatrix::bilinearForward(const Matrix& in, CHECK(dynamic_cast(&in)); size_t outputW = getWidth(); - size_t outputH = getHeight(); + size_t batchSize = getHeight(); size_t inputW = in.getWidth(); size_t inputH = in.getHeight(); (void)(inputH); @@ -3928,7 +3928,7 @@ void CpuMatrix::bilinearForward(const Matrix& in, if (inImgH == outImgH && inImgW == outImgW) { this->copyFrom(in); } else { - for (size_t k = 0; k < outputH; ++k) { // loop for batches + for (size_t k = 0; k < batchSize; ++k) { // loop for batches for (size_t i = 0; i < outImgH; ++i) { // loop for images size_t h = ratioH * i; size_t hid = (h < inImgH - 1) ? 1 : 0; @@ -3967,7 +3967,7 @@ void CpuMatrix::bilinearBackward(const Matrix& out, size_t inputW = getWidth(); size_t inputH = getHeight(); size_t outputW = out.getWidth(); - size_t outputH = out.getHeight(); + size_t batchSize = out.getHeight(); (void)(inputH); real* inGrad = getData(); @@ -3981,7 +3981,7 @@ void CpuMatrix::bilinearBackward(const Matrix& out, if (inImgH == outImgH && inImgW == outImgW) { this->addBias(const_cast(out), 1.f); } else { - for (size_t k = 0; k < outputH; ++k) { // loop for batches + for (size_t k = 0; k < batchSize; ++k) { // loop for batches for (size_t i = 0; i < outImgH; ++i) { // loop for images size_t h = ratioH * i; size_t hid = (h < inImgH - 1) ? 1 : 0; From 91a4064e35bb30dc73420f8e5f0c4525ac2765ed Mon Sep 17 00:00:00 2001 From: liaogang Date: Mon, 7 Nov 2016 16:17:36 +0800 Subject: [PATCH 227/324] Follow comments --- python/paddle/trainer_config_helpers/layers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index bf722fb1ba453..038f4d32a588e 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1292,7 +1292,7 @@ def bilinear_interp_layer(input, assert input.layer_type == LayerType.CONV_LAYER assert isinstance(input.activation, LinearActivation) assert out_size_x > 0 and out_size_y > 0 - assert input.numfilters is not None + assert input.num_filters is not None num_channels = input.num_filters Layer(name=name, inputs=Input(input.name, @@ -1301,8 +1301,7 @@ def bilinear_interp_layer(input, num_channels=num_channels)), type=LayerType.BILINEAR_INTERP_LAYER, **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.BILINEAR_INTERP_LAYER, parents=[input], - num_filters=num_filters) + return LayerOutput(name, LayerType.BILINEAR_INTERP_LAYER, parents=[input]) @wrap_name_default() @layer_support() From f9849ac92d3aa89b8bddbe9f08dd8ae9768132f5 Mon Sep 17 00:00:00 2001 From: gangliao Date: Mon, 7 Nov 2016 00:54:28 -0800 Subject: [PATCH 228/324] Revise one word in ISSUE_TEMPLATE.md (#371) --- ISSUE_TEMPLATE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md index b70d66dc259af..6b2614b101108 100644 --- a/ISSUE_TEMPLATE.md +++ b/ISSUE_TEMPLATE.md @@ -7,7 +7,7 @@ Before submitting the issue, look over the following criteria before handing you - [ ] Was there a similar issue submitted or resolved before ? You could search issue in the github. - [ ] Did you retrieve your issue from widespread search engines ? - [ ] Is my description of the issue clear enough to reproduce this problem? - * If some errors occured, we need details about `how do you run your code?`, `what system do you use?`, `Are you using GPU or not?`, etc. + * If some errors occurred, we need details about `how do you run your code?`, `what system do you use?`, `Are you using GPU or not?`, etc. * If you use an recording [asciinema](https://asciinema.org/) to show what you are doing to make it happen, that's awesome! We could help you solve the problem more quickly. - [ ] Is my description of the issue use the github markdown correctly? * Please use the proper markdown syntaxes for styling all forms of writing, e.g, source code, error information, etc. From e802471c58068669de78a3eaec143b705cb654b8 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Mon, 7 Nov 2016 19:41:20 +0800 Subject: [PATCH 229/324] abstract outputSize function in CNN-related layers (#314) --- paddle/gserver/layers/ConvBaseLayer.cpp | 9 ++- paddle/gserver/layers/ConvBaseLayer.h | 26 +------ paddle/gserver/layers/ConvOperator.cpp | 72 ++++++------------- paddle/gserver/layers/ConvProjection.h | 12 ++-- paddle/gserver/layers/CudnnPoolLayer.cpp | 20 +++--- paddle/gserver/layers/PoolLayer.h | 11 +-- paddle/gserver/layers/PoolProjectionLayer.cpp | 29 ++++---- paddle/gserver/tests/test_LayerGrad.cpp | 56 +++++++-------- paddle/math/MathUtils.cpp | 19 +++-- paddle/math/MathUtils.h | 16 +++++ python/paddle/trainer/config_parser.py | 53 +++++++------- 11 files changed, 139 insertions(+), 184 deletions(-) diff --git a/paddle/gserver/layers/ConvBaseLayer.cpp b/paddle/gserver/layers/ConvBaseLayer.cpp index 040510b7ad211..42ff0b70d86f7 100644 --- a/paddle/gserver/layers/ConvBaseLayer.cpp +++ b/paddle/gserver/layers/ConvBaseLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/utils/Logging.h" #include "ConvBaseLayer.h" namespace paddle { @@ -78,10 +77,10 @@ size_t ConvBaseLayer::calOutputSize() { imgSizeH_[i] = config_.inputs(i).conv_conf().img_size(); if (imgSizeW_[i] == 0) imgSizeW_[i] = config_.inputs(i).conv_conf().img_size(); - outputH_.push_back( - outputSize(imgSizeH_[i], filterSizeY_[i], paddingY_[i], strideY_[i])); - outputW_.push_back( - outputSize(imgSizeW_[i], filterSize_[i], padding_[i], stride_[i])); + outputH_.push_back(outputSize(imgSizeH_[i], filterSizeY_[i], paddingY_[i], + strideY_[i], caffeMode_)); + outputW_.push_back(outputSize(imgSizeW_[i], filterSize_[i], padding_[i], + stride_[i], caffeMode_)); CHECK_EQ(outputH_[i], outputH_[0]); CHECK_EQ(outputW_[i], outputW_[0]); } diff --git a/paddle/gserver/layers/ConvBaseLayer.h b/paddle/gserver/layers/ConvBaseLayer.h index 316514acf1a0d..e660a6d6f50ac 100644 --- a/paddle/gserver/layers/ConvBaseLayer.h +++ b/paddle/gserver/layers/ConvBaseLayer.h @@ -16,6 +16,7 @@ limitations under the License. */ #pragma once #include "Layer.h" +#include "paddle/math/MathUtils.h" namespace paddle { /** @@ -87,31 +88,6 @@ class ConvBaseLayer : public Layer { virtual size_t calOutputSize(); Weight& getWeight(int idx) { return *weights_[idx]; } - - /** - * Calculate output size based on caffeMode_. - * - input(+padding): 0123456789 - * - imageSize(+padding) = 10; - * - filterSize = 3; - * - stride = 2; - * - caffeMode_ is true: - - output: (012), (234), (456), (678) - - outputSize = 4; - * - caffeMode_ is false: - * - output: (012), (234), (456), (678), (9) - * - outputSize = 5; - */ - int outputSize(int imageSize, int filterSize, int padding, int stride) { - int outputSize; - if (!caffeMode_) { - outputSize = - (imageSize - filterSize + 2 * padding + stride - 1) / stride + 1; - } else { - outputSize = (imageSize - filterSize + 2 * padding) / stride + 1; - } - CHECK_GE(outputSize, 1); - return outputSize; - } }; } // namespace paddle diff --git a/paddle/gserver/layers/ConvOperator.cpp b/paddle/gserver/layers/ConvOperator.cpp index 8c72c1778451d..2d9c892fe595f 100644 --- a/paddle/gserver/layers/ConvOperator.cpp +++ b/paddle/gserver/layers/ConvOperator.cpp @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/math/Matrix.h" +#include "paddle/math/MathUtils.h" #include "Operator.h" namespace paddle { @@ -35,8 +35,8 @@ class ConvOperator : public Operator { */ virtual ~ConvOperator() { if (workSpaceInBytes_ != 0) { - hl_free_mem_device(workSpace_); - workSpaceInBytes_ = 0; + hl_free_mem_device(workSpace_); + workSpaceInBytes_ = 0; } hl_destroy_tensor_descriptor(inputDesc_); @@ -83,33 +83,6 @@ class ConvOperator : public Operator { filterSize_ * filterSizeY_ * channels_ * numFilters_); } - /** - * Calculate output size. - */ - int outputSize(int imageSize, int filterSize, int padding, int stride) { - int outputSize; - if (!caffeMode_) { - /* input(+padding): 0123456789 - * imageSize(+padding) = 10; - * filterSize = 3; - * stride = 2; - * output: (012), (234), (456), (678), (9) - * outputSize = 5; - */ - outputSize = - (imageSize - filterSize + 2 * padding + stride - 1) / stride + 1; - } else { - /* input(+padding): 0123456789 - * imageSize(+padding) = 10; - * filterSize = 3; - * stride = 2; - * output: (012), (234), (456), (678) - * outputSize = 4; - */ - outputSize = (imageSize - filterSize + 2 * padding) / stride + 1; - } - return outputSize; - } /// Most of member variables are same with CudnnConvLayer. /// There is no explanation here. int imageH_, imageW_, outputH_, outputW_; @@ -129,7 +102,7 @@ class ConvOperator : public Operator { int fwdAlgo_, bwdFilterAlgo_, bwdDataAlgo_; size_t fwdLimitBytes_, bwdDataLimitBytes_, bwdFilterLimitBytes_; size_t workSpaceInBytes_; - void* workSpace_; + void *workSpace_; bool isSelectAlgo_; }; @@ -160,7 +133,7 @@ ConvOperator::ConvOperator(const OperatorConfig &config, bool useGpu) void ConvOperator::allocConvWorkSpace(size_t maxWorkSpace) { if (maxWorkSpace > workSpaceInBytes_) { if (workSpaceInBytes_ != 0) { - hl_free_mem_device(workSpace_); + hl_free_mem_device(workSpace_); } // total amount of storage needed workSpace_ = hl_malloc_device(maxWorkSpace); @@ -168,14 +141,13 @@ void ConvOperator::allocConvWorkSpace(size_t maxWorkSpace) { } } - void ConvOperator::reshape(int batchSize) { imageH_ = ins_[0]->getFrameHeight(); imageW_ = ins_[0]->getFrameWidth(); if (imageH_ == 0) imageH_ = imgSize_; if (imageW_ == 0) imageW_ = imgSize_; - outputH_ = outputSize(imageH_, filterSizeY_, paddingY_, strideY_); - outputW_ = outputSize(imageW_, filterSize_, padding_, stride_); + outputH_ = outputSize(imageH_, filterSizeY_, paddingY_, strideY_, caffeMode_); + outputW_ = outputSize(imageW_, filterSize_, padding_, stride_, caffeMode_); out_->setFrameHeight(outputH_); out_->setFrameWidth(outputW_); @@ -183,10 +155,10 @@ void ConvOperator::reshape(int batchSize) { reshapeImageDescriptors(); if (!isSelectAlgo_) { - hl_conv_workspace(inputDesc_, outputDesc_, filterDesc_, - convDesc_, &fwdAlgo_, &fwdLimitBytes_, - &bwdDataAlgo_, &bwdDataLimitBytes_, - &bwdFilterAlgo_, &bwdFilterLimitBytes_); + hl_conv_workspace(inputDesc_, outputDesc_, filterDesc_, convDesc_, + &fwdAlgo_, &fwdLimitBytes_, &bwdDataAlgo_, + &bwdDataLimitBytes_, &bwdFilterAlgo_, + &bwdFilterLimitBytes_); size_t maxWorkSpace = 0; maxWorkSpace = std::max(fwdLimitBytes_, bwdDataLimitBytes_); @@ -202,7 +174,8 @@ void ConvOperator::computeConvSizes() { hl_create_filter_descriptor(&filterDesc_, channels_, numFilters_, filterSizeY_, filterSize_); hl_create_tensor_descriptor(&inputDesc_); - int outputX = outputSize(imgSize_, filterSize_, padding_, stride_); + int outputX = + outputSize(imgSize_, filterSize_, padding_, stride_, caffeMode_); CHECK_EQ(outputX, outputX_); hl_create_tensor_descriptor(&outputDesc_); hl_create_convolution_descriptor(&convDesc_, inputDesc_, filterDesc_, @@ -211,13 +184,13 @@ void ConvOperator::computeConvSizes() { void ConvOperator::reshapeImageDescriptors() { hl_tensor_reshape(inputDesc_, 1, channels_, imageH_, imageW_, - channels_ * imageH_ * imageW_, imageH_ * imageW_, - imageW_, 1); + channels_ * imageH_ * imageW_, imageH_ * imageW_, imageW_, + 1); hl_tensor_reshape(outputDesc_, 1, numFilters_, outputH_, outputW_, numFilters_ * outputH_ * outputW_, outputH_ * outputW_, outputW_, 1); - hl_reset_convolution_descriptor(convDesc_, inputDesc_, filterDesc_, - paddingY_, padding_, strideY_, stride_); + hl_reset_convolution_descriptor(convDesc_, inputDesc_, filterDesc_, paddingY_, + padding_, strideY_, stride_); inputOffset_ = channels_ * imageH_ * imageW_; outputOffset_ = numFilters_ * outputH_ * outputW_; weightOffset_ = numFilters_ * channels_ * filterSize_ * filterSize_; @@ -273,18 +246,17 @@ void ConvOperator::backward() { real *weightGrad = ins_[1]->grad->getData() + weightOffset_ * batchId; hl_convolution_backward_filter(inputDesc_, inputData, outputDesc_, outGrad, filterDesc_, weightGrad, - convDesc_, workSpace_, - workSpaceInBytes_, bwdFilterAlgo_); + convDesc_, workSpace_, workSpaceInBytes_, + bwdFilterAlgo_); } MatrixPtr preGrad = ins_[0]->grad; if (NULL != preGrad) { real *inputGrad = preGrad->getData() + inputOffset_ * batchId; real *wgtData = ins_[1]->value->getData() + weightOffset_ * batchId; - hl_convolution_backward_data(inputDesc_, inputGrad, outputDesc_, - outGrad, filterDesc_, wgtData, - convDesc_, workSpace_, - workSpaceInBytes_, bwdDataAlgo_); + hl_convolution_backward_data( + inputDesc_, inputGrad, outputDesc_, outGrad, filterDesc_, wgtData, + convDesc_, workSpace_, workSpaceInBytes_, bwdDataAlgo_); } } } diff --git a/paddle/gserver/layers/ConvProjection.h b/paddle/gserver/layers/ConvProjection.h index 41a100ac3c50f..d0bfe9a6edba0 100644 --- a/paddle/gserver/layers/ConvProjection.h +++ b/paddle/gserver/layers/ConvProjection.h @@ -12,10 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once #include "Projection.h" +#include "paddle/math/MathUtils.h" namespace paddle { @@ -42,17 +42,15 @@ class ConvProjection : public Projection { void reshapeTensorDesc(int batchSize); void reshape(int batchSize); - int outputSize(int imageSize, int filterSize, int padding, int stride) { - return (imageSize - filterSize + 2 * padding) / stride + 1; - } - size_t calOutputSize() { imageH_ = in_->getFrameHeight(); imageW_ = in_->getFrameWidth(); if (imageH_ == 0) imageH_ = configImgH_; if (imageW_ == 0) imageW_ = configImgW_; - outputH_ = outputSize(imageH_, filterH_, paddingH_, strideH_); - outputW_ = outputSize(imageW_, filterW_, paddingW_, strideW_); + outputH_ = outputSize(imageH_, filterH_, paddingH_, strideH_, + /* caffeMode */ true); + outputW_ = outputSize(imageW_, filterW_, paddingW_, strideW_, + /* caffeMode */ true); const_cast(out_)->setFrameHeight(outputH_); const_cast(out_)->setFrameWidth(outputW_); diff --git a/paddle/gserver/layers/CudnnPoolLayer.cpp b/paddle/gserver/layers/CudnnPoolLayer.cpp index 4c733591b3779..24adb50a985ff 100644 --- a/paddle/gserver/layers/CudnnPoolLayer.cpp +++ b/paddle/gserver/layers/CudnnPoolLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/utils/Logging.h" #include "paddle/utils/Stat.h" #include "paddle/math/Matrix.h" @@ -62,9 +61,9 @@ bool CudnnPoolLayer::init(const LayerMap &layerMap, strideHeight = strideY_; strideWidth = stride_; - hl_create_pooling_descriptor(&poolingDesc_, mode_, windowHeight, - windowWidth, heightPadding, widthPadding, - strideHeight, strideWidth); + hl_create_pooling_descriptor(&poolingDesc_, mode_, windowHeight, windowWidth, + heightPadding, widthPadding, strideHeight, + strideWidth); return true; } @@ -80,8 +79,10 @@ void CudnnPoolLayer::reshape(int batchSize) { } CHECK_EQ(inputLayers_[0]->getOutput().value->getWidth(), channels_ * imageH_ * imageW_); - outputH_ = outputSize(imageH_, sizeY_, confPaddingY_, strideY_); - outputW_ = outputSize(imageW_, sizeX_, confPadding_, stride_); + outputH_ = outputSize(imageH_, sizeY_, confPaddingY_, strideY_, + /* caffeMode */ false); + outputW_ = + outputSize(imageW_, sizeX_, confPadding_, stride_, /* caffeMode */ false); getOutput().setFrameHeight(outputH_); getOutput().setFrameWidth(outputW_); @@ -99,8 +100,7 @@ void CudnnPoolLayer::forward(PassType passType) { real *inputData = getInputValue(0)->getData(); real *outData = getOutputValue()->getData(); - hl_pooling_forward(inputDesc_, inputData, outputDesc_, outData, - poolingDesc_); + hl_pooling_forward(inputDesc_, inputData, outputDesc_, outData, poolingDesc_); } void CudnnPoolLayer::backward(const UpdateCallback &callback) { @@ -113,8 +113,8 @@ void CudnnPoolLayer::backward(const UpdateCallback &callback) { real *inputGrad = getInputGrad(0)->getData(); real *outData = getOutputValue()->getData(); real *outGrad = getOutputGrad()->getData(); - hl_pooling_backward(inputDesc_, inputData, inputGrad, outputDesc_, - outData, outGrad, poolingDesc_); + hl_pooling_backward(inputDesc_, inputData, inputGrad, outputDesc_, outData, + outGrad, poolingDesc_); } CudnnPoolLayer::~CudnnPoolLayer() { diff --git a/paddle/gserver/layers/PoolLayer.h b/paddle/gserver/layers/PoolLayer.h index bde1f5b8dcbfd..e87ad08251dd4 100644 --- a/paddle/gserver/layers/PoolLayer.h +++ b/paddle/gserver/layers/PoolLayer.h @@ -17,6 +17,7 @@ limitations under the License. */ #include "Layer.h" #include "paddle/math/Matrix.h" +#include "paddle/math/MathUtils.h" #include namespace paddle { @@ -47,16 +48,6 @@ class PoolLayer : public Layer { static Layer* create(const LayerConfig& config); virtual bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); - - /** - * Calculate output size according window size and padding size. - */ - int outputSize(int imageSize, int windowSize, int padding, int stride) { - int outputSize; - outputSize = - (imageSize - windowSize + 2 * padding + stride - 1) / stride + 1; - return outputSize; - } }; } // namespace paddle diff --git a/paddle/gserver/layers/PoolProjectionLayer.cpp b/paddle/gserver/layers/PoolProjectionLayer.cpp index 5a2e9afb6e164..9e8ce778501bb 100644 --- a/paddle/gserver/layers/PoolProjectionLayer.cpp +++ b/paddle/gserver/layers/PoolProjectionLayer.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "paddle/utils/Logging.h" #include "paddle/utils/Stat.h" #include "PoolProjectionLayer.h" @@ -31,8 +30,10 @@ size_t PoolProjectionLayer::getSize() { imgSizeW_ = imgSize_; } - outputH_ = outputSize(imgSizeH_, sizeY_, confPaddingY_, strideY_); - outputW_ = outputSize(imgSizeW_, sizeX_, confPadding_, stride_); + outputH_ = outputSize(imgSizeH_, sizeY_, confPaddingY_, strideY_, + /* caffeMode */ false); + outputW_ = outputSize(imgSizeW_, sizeX_, confPadding_, stride_, + /* caffeMode */ false); layerSize = outputH_ * outputW_ * channels_; @@ -53,9 +54,9 @@ void MaxPoolProjectionLayer::forward(PassType passType) { MatrixPtr outV = getOutputValue(); - outV->maxPoolForward(*input, imgSizeH_, imgSizeW_, channels_, - sizeX_, sizeY_, strideY_, stride_, - outputH_, outputW_, confPaddingY_, confPadding_); + outV->maxPoolForward(*input, imgSizeH_, imgSizeW_, channels_, sizeX_, sizeY_, + strideY_, stride_, outputH_, outputW_, confPaddingY_, + confPadding_); } void MaxPoolProjectionLayer::backward(const UpdateCallback& callback) { @@ -72,9 +73,8 @@ void MaxPoolProjectionLayer::backward(const UpdateCallback& callback) { MatrixPtr inputGrad = getInputGrad(0); inputGrad->maxPoolBackward(*inputV, imgSizeH_, imgSizeW_, *outGrad, *outV, - sizeX_, sizeY_, - strideY_, stride_, outputH_, outputW_, 1, 1, - confPaddingY_, confPadding_); + sizeX_, sizeY_, strideY_, stride_, outputH_, + outputW_, 1, 1, confPaddingY_, confPadding_); } void AvgPoolProjectionLayer::forward(PassType passType) { @@ -89,9 +89,9 @@ void AvgPoolProjectionLayer::forward(PassType passType) { MatrixPtr outV = getOutputValue(); - outV->avgPoolForward(*input, imgSizeH_, imgSizeW_, channels_, - sizeX_, sizeY_, strideY_, stride_, - outputH_, outputW_, confPaddingY_, confPadding_); + outV->avgPoolForward(*input, imgSizeH_, imgSizeW_, channels_, sizeX_, sizeY_, + strideY_, stride_, outputH_, outputW_, confPaddingY_, + confPadding_); } void AvgPoolProjectionLayer::backward(const UpdateCallback& callback) { @@ -103,9 +103,8 @@ void AvgPoolProjectionLayer::backward(const UpdateCallback& callback) { /* Do derivation */ MatrixPtr outputGrad = getOutputGrad(); MatrixPtr inputGrad = getInputGrad(0); - inputGrad->avgPoolBackward(*outputGrad, imgSizeH_, imgSizeW_, - sizeX_, sizeY_, strideY_, stride_, - outputH_, outputW_, 1, 1, + inputGrad->avgPoolBackward(*outputGrad, imgSizeH_, imgSizeW_, sizeX_, sizeY_, + strideY_, stride_, outputH_, outputW_, 1, 1, confPaddingY_, confPadding_); } } // namespace paddle diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index bf2c2e0499941..5397b952bced8 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -18,6 +18,7 @@ limitations under the License. */ #include "paddle/gserver/layers/DataLayer.h" #include "ModelConfig.pb.h" #include "paddle/trainer/Trainer.h" +#include "paddle/math/MathUtils.h" #include "TestUtil.h" #include "LayerGradUtil.h" @@ -134,7 +135,6 @@ TEST(Projection, identity) { } } - #ifndef PADDLE_ONLY_CPU TEST(Projection, conv) { const int NUM_FILTERS = 16; @@ -158,21 +158,23 @@ TEST(Projection, conv) { conv->set_groups(1); conv->set_filter_channels(conv->channels() / conv->groups()); conv->set_img_size(IMAGE_SIZE); - int outputSize = (2 * conv->padding() + conv->img_size() - - conv->filter_size()) / conv->stride() + 1; - int outputSizeY = (2 * conv->padding_y() + conv->img_size() - - conv->filter_size_y()) / conv->stride_y() + 1; - conv->set_output_x(outputSize); + int output_x = + outputSize(conv->img_size(), conv->filter_size(), conv->padding(), + conv->stride(), /* caffeMode */ true); + int output_y = + outputSize(conv->img_size(), conv->filter_size_y(), conv->padding_y(), + conv->stride_y(), /* caffeMode */ true); + conv->set_output_x(output_x); conf.set_input_size(IMAGE_SIZE * IMAGE_SIZE * CHANNELS); - conf.set_output_size(outputSize * outputSizeY * NUM_FILTERS); + conf.set_output_size(output_x * output_y * NUM_FILTERS); - testProjectionGrad(conf, INPUT_DATA, + testProjectionGrad( + conf, INPUT_DATA, /* parameterSize */ NUM_FILTERS * CHANNELS * FILTER_SIZE * FILTER_SIZE_Y, /* batchSize */ 100, true, false, NUM_FILTERS, true); } #endif - TEST(Layer, concat) { TestConfig config; config.biasSize = 0; @@ -293,10 +295,9 @@ void testConvLayer(const string& type, bool trans, bool useGpu) { conv->set_groups(1); conv->set_filter_channels(conv->channels() / conv->groups()); conv->set_img_size(16); - conv->set_output_x( - (2 * conv->padding() + conv->img_size() - conv->filter_size()) / - ((float)conv->stride()) + - 1.5); + conv->set_output_x(outputSize(conv->img_size(), conv->filter_size(), + conv->padding(), conv->stride(), + /* caffeMode */ true)); config.layerConfig.set_size(conv->output_x() * conv->output_x() * config.layerConfig.num_filters()); @@ -329,15 +330,13 @@ TEST(Layer, blockExpandLayer) { blockExpand->set_stride_x(2); blockExpand->set_stride_y(2); blockExpand->set_output_x( - 1 + - (2 * blockExpand->padding_x() + blockExpand->img_size_x() - - blockExpand->block_x() + blockExpand->stride_x() - 1) / - blockExpand->stride_x()); + outputSize(blockExpand->img_size_x(), blockExpand->block_x(), + blockExpand->padding_x(), blockExpand->stride_x(), + /* caffeMode */ false)); blockExpand->set_output_y( - 1 + - (2 * blockExpand->padding_y() + blockExpand->img_size_y() - - blockExpand->block_y() + blockExpand->stride_y() - 1) / - blockExpand->stride_y()); + outputSize(blockExpand->img_size_y(), blockExpand->block_y(), + blockExpand->padding_y(), blockExpand->stride_y(), + /* caffeMode */ false)); config.layerConfig.set_size(blockExpand->block_x() * blockExpand->block_y() * blockExpand->channels()); @@ -862,8 +861,8 @@ void setPoolConfig(TestConfig* config, PoolConfig* pool, pool->set_stride(sw); pool->set_stride_y(sh); - int ow = (pool->img_size() - kw + 2 * pw + sw - 1) / sw + 1; - int oh = (pool->img_size_y() - kh + 2 * ph + sh - 1) / sh + 1; + int ow = outputSize(pool->img_size(), kw, pw, sw, /* caffeMode */ false); + int oh = outputSize(pool->img_size_y(), kh, ph, sh, /* caffeMode */ false); pool->set_output_x(ow); pool->set_output_y(oh); } @@ -1255,12 +1254,11 @@ TEST(Operator, conv) { conv->set_groups(1); conv->set_filter_channels(conv->channels() / conv->groups()); conv->set_img_size(IMAGE_SIZE); - int outputSize = - int(1.0 * (2 * conv->padding() + conv->img_size() - conv->filter_size()) / - conv->stride()) + - 1; - conv->set_output_x(outputSize); - config.layerConfig.set_size(outputSize * outputSize * + int output_x = + outputSize(conv->img_size(), conv->filter_size(), conv->padding(), + conv->stride(), /* caffeMode */ true); + conv->set_output_x(output_x); + config.layerConfig.set_size(output_x * output_x * config.layerConfig.num_filters()); config.layerConfig.set_size(conv->output_x() * conv->output_x() * NUM_FILTERS); diff --git a/paddle/math/MathUtils.cpp b/paddle/math/MathUtils.cpp index 5b78ab1b07bda..c1af8628d03c5 100644 --- a/paddle/math/MathUtils.cpp +++ b/paddle/math/MathUtils.cpp @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #include "MathUtils.h" #include #include "paddle/utils/Logging.h" @@ -24,11 +23,7 @@ namespace paddle { * major is rows and minor is cols, according to * major value to initialize minor value" */ -void sparseRand(int* major, - int* minor, - int nnz, - int majorLen, - int minorMax, +void sparseRand(int* major, int* minor, int nnz, int majorLen, int minorMax, bool useGpu) { CHECK(size_t(nnz) > size_t(1)); int* cpuMajor; @@ -72,5 +67,17 @@ void sparseRand(int* major, } } +int outputSize(int imageSize, int filterSize, int padding, int stride, + bool caffeMode) { + int outputSize; + if (!caffeMode) { + outputSize = + (imageSize - filterSize + 2 * padding + stride - 1) / stride + 1; + } else { + outputSize = (imageSize - filterSize + 2 * padding) / stride + 1; + } + CHECK_GE(outputSize, 1); + return outputSize; +} } // namespace paddle diff --git a/paddle/math/MathUtils.h b/paddle/math/MathUtils.h index 83375022abbe2..49d0c10a8f5e4 100644 --- a/paddle/math/MathUtils.h +++ b/paddle/math/MathUtils.h @@ -44,4 +44,20 @@ namespace paddle { void sparseRand(int* major, int* minor, int nnz, int majorLen, int minorMax, bool useGpu); +/** + * Calculate output size based on caffeMode_. + * - input(+padding): 0123456789 + * - imageSize(+padding) = 10; + * - filterSize = 3; + * - stride = 2; + * - caffeMode is true: + - output: (012), (234), (456), (678) + - outputSize = 4; + * - caffeMode is false: + * - output: (012), (234), (456), (678), (9) + * - outputSize = 5; + */ +int outputSize(int imageSize, int filterSize, int padding, int stride, + bool caffeMode); + } // namespace paddle diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index e9038fea8a208..73631602a92be 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1006,6 +1006,17 @@ def TestData(data_config, async_load_data=None): " Data definition") g_config.test_data_config.async_load_data = async_load_data +''' +caffe_mode: compute the output size using floor instead of ceil, + which is consistent of caffe and CuDNN's convention. +''' +def cnn_output_size(img_size, filter_size, padding, stride, caffe_mode): + output = (2 * padding + img_size - filter_size) / float(stride) + if caffe_mode: + return 1 + int(math.floor(output)) + else: + return 1 + int(math.ceil(output)) + def parse_pool(pool, input_layer_name, pool_conf): pool_conf.pool_type = pool.pool_type config_assert(pool.pool_type in ['max-projection', 'avg-projection', @@ -1036,12 +1047,10 @@ def parse_pool(pool, input_layer_name, pool_conf): if pool.padding is not None: pool_conf.padding = pool.padding pool_conf.padding_y = default(pool.padding_y, pool_conf.padding) - pool_conf.output_x = int(math.ceil((pool_conf.img_size + \ - 2*pool_conf.padding - pool_conf.size_x) / \ - float(pool_conf.stride))) + 1 - pool_conf.output_y = int(math.ceil((pool_conf.img_size_y + \ - 2*pool_conf.padding_y - pool_conf.size_y) / \ - float(pool_conf.stride_y))) + 1 + pool_conf.output_x = cnn_output_size(pool_conf.img_size, pool_conf.size_x, + pool_conf.padding, pool_conf.stride, False) + pool_conf.output_y = cnn_output_size(pool_conf.img_size_y, pool_conf.size_y, + pool_conf.padding_y, pool_conf.stride_y, False) def parse_image(image, input_layer_name, image_conf): image_conf.channels = image.channels @@ -1072,10 +1081,7 @@ def parse_norm(norm, input_layer_name, norm_conf): norm_conf.scale /= norm.size else: norm_conf.scale /= norm.size ** 2 -''' -caffe_mode: compute the output size using floor instead of ceil, - which is consistent of caffe and CuDNN's convention. -''' + def parse_conv(conv, input_layer_name, conv_conf): conv_conf.filter_size = conv.filter_size conv_conf.filter_size_y = conv.filter_size_y @@ -1096,14 +1102,9 @@ def parse_conv(conv, input_layer_name, conv_conf): ("Input layer %s: Incorrect input image size %d for input " + "image pixels %d") % (input_layer_name, conv_conf.img_size, img_pixels)) - if conv.caffe_mode: - conv_conf.output_x = \ - 1 + int(math.floor((2 * conv.padding + conv_conf.img_size \ - - conv.filter_size) / float(conv.stride))) - else: - conv_conf.output_x = \ - 1 + int(math.ceil((2 * conv.padding + conv_conf.img_size \ - - conv.filter_size) / float(conv.stride))) + conv_conf.output_x = cnn_output_size(conv_conf.img_size, conv_conf.filter_size, + conv_conf.padding, conv_conf.stride, + conv_conf.caffe_mode) def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand_conf.channels = block_expand.channels @@ -1118,18 +1119,16 @@ def parse_block_expand(block_expand, input_layer_name, block_expand_conf): if block_expand_conf.img_size_x == 0: block_expand_conf.output_x = 0 else: - block_expand_conf.output_x = \ - 1 + \ - int(math.ceil((2 * block_expand.padding_x + block_expand.img_size_x \ - - block_expand.block_x) / float(block_expand.stride_x))) + block_expand_conf.output_x = cnn_output_size( + block_expand.img_size_x, block_expand.block_x, + block_expand.padding_x, block_expand.stride_x, False) if block_expand_conf.img_size_y == 0: - block_expand_conf.output_y = 0 + block_expand_conf.output_y = 0 else: - block_expand_conf.output_y = \ - 1 + \ - int(math.ceil((2 * block_expand.padding_y + block_expand.img_size_y \ - - block_expand.block_y) / float(block_expand.stride_y))) + block_expand_conf.output_y = cnn_output_size( + block_expand.img_size_y, block_expand.block_y, + block_expand.padding_y, block_expand.stride_y, False) def parse_maxout(maxout, input_layer_name, maxout_conf): maxout_conf.channels = maxout.channels From 4905751a22e5211defafcc56d16a26114e61ca25 Mon Sep 17 00:00:00 2001 From: lzhao4ever Date: Mon, 7 Nov 2016 14:09:20 -0800 Subject: [PATCH 230/324] Add define for double getrf, getri (#381) --- paddle/cuda/src/hl_cuda_cublas.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paddle/cuda/src/hl_cuda_cublas.cc b/paddle/cuda/src/hl_cuda_cublas.cc index 724ea490e8ea9..f16376ec937d3 100644 --- a/paddle/cuda/src/hl_cuda_cublas.cc +++ b/paddle/cuda/src/hl_cuda_cublas.cc @@ -78,6 +78,8 @@ DYNAMIC_LOAD_CUBLAS_WRAP(cublasCgemmBatched) DYNAMIC_LOAD_CUBLAS_WRAP(cublasZgemmBatched) DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgetrfBatched) DYNAMIC_LOAD_CUBLAS_WRAP(cublasSgetriBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgetrfBatched) +DYNAMIC_LOAD_CUBLAS_WRAP(cublasDgetriBatched) CUBLAS_BLAS_ROUTINE_EACH(DYNAMIC_LOAD_CUBLAS_V2_WRAP) #undef DYNAMIC_LOAD_CUBLAS_WRAP From ebad8e525d711230b63cd0112a55cac3f6cc751a Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Fri, 4 Nov 2016 13:04:28 -0700 Subject: [PATCH 231/324] Add SumCost This allows user to implement any type of cost by summing over the output of non-cost layers. Change-Id: Ic55aaabbf0c1299e70b8e48a0effcc91f8f5bd29 --- paddle/gserver/layers/CostLayer.cpp | 29 +++++++++++++++ paddle/gserver/tests/test_LayerGrad.cpp | 13 +++++++ python/paddle/trainer/config_parser.py | 1 + .../paddle/trainer_config_helpers/layers.py | 37 ++++++++++++++++--- .../tests/configs/test_cost_layers.py | 6 ++- 5 files changed, 79 insertions(+), 7 deletions(-) diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index 14ff8510f7b19..0bb8359a904c8 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -562,4 +562,33 @@ void HuberTwoClass::backwardImpIn( } } +class SumCostLayer : public Layer { +public: + explicit SumCostLayer(const LayerConfig& config) : Layer(config) {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap) { + bool ret = Layer::init(layerMap, parameterMap); + if (!ret) return ret; + CHECK_EQ(inputLayers_.size(), 1UL); + return true; + } + + virtual void forward(PassType passType) { + Layer::forward(passType); + const MatrixPtr& input = getInputValue(0); + + /* malloc memory for the output_ if necessary */ + int batchSize = input->getHeight(); + int size = 1; + resizeOutput(batchSize, size); + output_.value->sumRows(*input); + } + + virtual void backward(const UpdateCallback& callback = nullptr) { + getInputGrad(0)->add((real)1); + } +}; + +REGISTER_LAYER(sum_cost, SumCostLayer); + } // namespace paddle diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 5397b952bced8..61b89f5ec3c5c 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -935,6 +935,19 @@ TEST(Layer, rankCostLayer) { } } +TEST(Layer, sumCostLayer) { + TestConfig config; + config.layerConfig.set_type("sum_cost"); + config.biasSize = 0; + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 1, 0}); + config.layerConfig.add_inputs(); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "sum_cost", 100, false, useGpu); + } +} + TEST(Layer, weightedRankCostLayer) { TestConfig config; config.layerConfig.set_type("rank-cost"); diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 73631602a92be..5a0d5018f0d11 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1795,6 +1795,7 @@ def init(cls, name, inputs, device=None, coeff=1.): define_cost('MultiBinaryLabelCrossEntropy', 'multi_binary_label_cross_entropy') define_cost('SoftBinaryClassCrossEntropy', 'soft_binary_class_cross_entropy') define_cost('HuberTwoClass', 'huber') +define_cost('SumCost', 'sum_cost') @config_layer('hsigmoid') class HierarchicalSigmoidLayer(LayerBase): diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 49f0ff3289db7..c768a419c0bff 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -52,7 +52,7 @@ 'convex_comb_layer', 'ctc_layer', 'crf_layer', 'crf_decoding_layer', 'nce_layer', 'cross_entropy_with_selfnorm', 'cross_entropy', - 'multi_binary_label_cross_entropy', + 'multi_binary_label_cross_entropy', 'sum_cost', 'rank_cost', 'lambda_cost', 'huber_cost', 'block_expand_layer', 'maxout_layer', 'out_prod_layer', 'print_layer' @@ -126,6 +126,7 @@ class LayerType(object): CROSS_ENTROPY_WITH_SELFNORM = "multi_class_cross_entropy_with_selfnorm" SOFT_BIN_CLASS_CROSS_ENTROPY = "soft_binary_class_cross_entropy" MULTI_BIN_LABEL_CROSS_ENTROPY = "multi_binary_label_cross_entropy" + SUM_COST = "sum_cost" @staticmethod def is_layer_type(type_name): @@ -3924,8 +3925,6 @@ def cross_entropy(input, label, name=None, coeff=1.0, layer_attr=None): :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param type: The type of cost. - :type type: basestring. :param name: The name of this layers. It is not necessary. :type name: None|basestring. :param coeff: The coefficient affects the gradient in the backward. @@ -3961,8 +3960,6 @@ def cross_entropy_with_selfnorm(input, label, name=None, coeff=1.0, :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param type: The type of cost. - :type type: basestring. :param name: The name of this layers. It is not necessary. :type name: None|basestring. :param coeff: The coefficient affects the gradient in the backward. @@ -3987,6 +3984,36 @@ def cross_entropy_with_selfnorm(input, label, name=None, coeff=1.0, parents=[input, label]) +@wrap_name_default() +@layer_support() +def sum_cost(input, name=None, layer_attr=None): + """ + A loss layer which calculate the sum of the input as loss + + .. code-block:: python + + cost = sum_cost(input) + + :param input: The first input layer. + :type input: LayerOutput. + :param name: The name of this layers. It is not necessary. + :type name: None|basestring. + :param layer_attr: Extra Layer Attribute. + :type layer_attr: ExtraLayerAttribute + :return: LayerOutput object. + :rtype: LayerOutput. + """ + Layer(name=name, + type=LayerType.SUM_COST, + inputs=[input.name], + **ExtraLayerAttribute.to_kwargs(layer_attr) + ) + + return LayerOutput(name, + LayerType.SUM_COST, + parents=[input]) + + @wrap_name_default() @layer_support() def huber_cost(input, label, name=None, coeff=1.0, layer_attr=None): diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py index 64b45f4ded10b..f1b3365f84e3e 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py @@ -11,8 +11,9 @@ probs = data_layer(name='probs', size=10) xe_label = data_layer(name='xe-label', size=10) +hidden = fc_layer(input=seq_in, size=4) outputs(ctc_layer(input=seq_in, label=labels), - crf_layer(input=fc_layer(input=seq_in, size=4), + crf_layer(input=hidden, label=data_layer(name='crf_label', size=4)), rank_cost(left=data_layer(name='left', size=1), right=data_layer(name='right', size=1), @@ -23,4 +24,5 @@ cross_entropy_with_selfnorm(input=probs, label=xe_label), huber_cost(input=data_layer(name='huber_probs', size=1), label=data_layer(name='huber_label', size=1)), - multi_binary_label_cross_entropy(input=probs, label=xe_label)) + multi_binary_label_cross_entropy(input=probs, label=xe_label), + sum_cost(hidden)) From 38764bf908578dae66d2d4ce6bb5a2380f3c8cb4 Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Mon, 7 Nov 2016 14:28:23 -0800 Subject: [PATCH 232/324] Add sum_cost to document And rebase Change-Id: I7ea234b3aa8fc70675af15d91db08242c43fb5ff --- doc/source/gserver/layers/layer.rst | 5 +++ doc/ui/api/trainer_config_helpers/layers.rst | 6 +++ paddle/gserver/layers/CostLayer.cpp | 6 +++ paddle/gserver/layers/CostLayer.h | 2 +- .../protostr/test_cost_layers.protostr | 37 +++++++++++++------ 5 files changed, 43 insertions(+), 13 deletions(-) diff --git a/doc/source/gserver/layers/layer.rst b/doc/source/gserver/layers/layer.rst index 807b22ca140ee..4b8e149505f06 100644 --- a/doc/source/gserver/layers/layer.rst +++ b/doc/source/gserver/layers/layer.rst @@ -465,6 +465,11 @@ SumOfSquaresCostLayer .. doxygenclass:: paddle::SumOfSquaresCostLayer :members: +SumCostLayer +````````````````````` +.. doxygenclass:: paddle::SumCostLayer + :members: + CosSimLayer ----------- .. doxygenclass:: paddle::CosSimLayer diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index 5bb88b0615c12..c2e347d12b3f8 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -395,6 +395,12 @@ hsigmoid :members: hsigmoid :noindex: +sum_cost +--------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: sum_cost + :noindex: + Check Layer ============ diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index 0bb8359a904c8..949788be49787 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -562,6 +562,12 @@ void HuberTwoClass::backwardImpIn( } } +/** + * This cost layer compute the sum of its input as loss. + * \f[ + * o(i) = \sum_{j=1}^D y_{ij} + * \f] + */ class SumCostLayer : public Layer { public: explicit SumCostLayer(const LayerConfig& config) : Layer(config) {} diff --git a/paddle/gserver/layers/CostLayer.h b/paddle/gserver/layers/CostLayer.h index b464e16737ae5..f263c688213ae 100644 --- a/paddle/gserver/layers/CostLayer.h +++ b/paddle/gserver/layers/CostLayer.h @@ -129,7 +129,7 @@ class SoftBinaryClassCrossEntropy : public CostLayer { * This cost layer compute Euclidean (L2) loss for real-valued regression * tasks. * \f[ - * L = \frac{1}{2N} \sum_{i=1}^N {|| \hat{y}_i - y_i||_2^2} + * L = \sum_{i=1}^N {|| \hat{y}_i - y_i||_2^2} * \f] */ class SumOfSquaresCostLayer : public CostLayer { diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr index 5261cf0c44943..f6045fe1f6825 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_cost_layers.protostr @@ -23,6 +23,17 @@ layers { size: 10 active_type: "" } +layers { + name: "__fc_layer_0__" + type: "fc" + size: 4 + active_type: "tanh" + inputs { + input_layer_name: "input" + input_parameter_name: "___fc_layer_0__.w0" + } + bias_parameter_name: "___fc_layer_0__.wbias" +} layers { name: "__ctc_layer_0__" type: "ctc" @@ -36,17 +47,6 @@ layers { } norm_by_times: false } -layers { - name: "__fc_layer_0__" - type: "fc" - size: 4 - active_type: "tanh" - inputs { - input_layer_name: "input" - input_parameter_name: "___fc_layer_0__.w0" - } - bias_parameter_name: "___fc_layer_0__.wbias" -} layers { name: "crf_label" type: "data" @@ -191,6 +191,16 @@ layers { } coeff: 1.0 } +layers { + name: "__sum_cost_0__" + type: "sum_cost" + size: 1 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + } + coeff: 1.0 +} parameters { name: "___fc_layer_0__.w0" size: 800 @@ -241,14 +251,15 @@ output_layer_names: "__cross_entropy_0__" output_layer_names: "__cross_entropy_with_selfnorm_0__" output_layer_names: "__huber_cost_0__" output_layer_names: "__multi_binary_label_cross_entropy_0__" +output_layer_names: "__sum_cost_0__" sub_models { name: "root" layer_names: "input" layer_names: "labels" layer_names: "probs" layer_names: "xe-label" - layer_names: "__ctc_layer_0__" layer_names: "__fc_layer_0__" + layer_names: "__ctc_layer_0__" layer_names: "crf_label" layer_names: "__crf_layer_0__" layer_names: "left" @@ -264,6 +275,7 @@ sub_models { layer_names: "huber_label" layer_names: "__huber_cost_0__" layer_names: "__multi_binary_label_cross_entropy_0__" + layer_names: "__sum_cost_0__" input_layer_names: "input" input_layer_names: "labels" input_layer_names: "crf_label" @@ -284,6 +296,7 @@ sub_models { output_layer_names: "__cross_entropy_with_selfnorm_0__" output_layer_names: "__huber_cost_0__" output_layer_names: "__multi_binary_label_cross_entropy_0__" + output_layer_names: "__sum_cost_0__" is_recurrent_layer_group: false } From 2e1d968fc284fa733e6ed20191875b003116c083 Mon Sep 17 00:00:00 2001 From: gangliao Date: Mon, 7 Nov 2016 19:05:37 -0800 Subject: [PATCH 233/324] Remove Mac OS X build docs (#386) Currently, Paddle on Mac OS X is not deliberate testing through the different versions of Mac OS X and Clang. When all these things that we've done, we will reopen Mac build docs. --- doc/build/build_from_source.md | 119 --------------------------------- 1 file changed, 119 deletions(-) diff --git a/doc/build/build_from_source.md b/doc/build/build_from_source.md index c37234d3ef14d..b8f26f431eb7a 100644 --- a/doc/build/build_from_source.md +++ b/doc/build/build_from_source.md @@ -4,7 +4,6 @@ Installing from Sources * [1. Download and Setup](#download) * [2. Requirements](#requirements) * [3. Build on Ubuntu](#ubuntu) -* [4. Build on Mac OS X](#mac) ## Download and Setup You can download PaddlePaddle from the [github source](https://github.com/gangliao/Paddle). @@ -191,121 +190,3 @@ sudo pip install /opt/paddle/share/wheels/*.whl # or just run sudo paddle version ``` - -## Building on Mac OS X - -### Prerequisites -This guide is based on Mac OS X 10.11 (El Capitan). Note that if you are running an up to date version of OS X, -you will already have Python 2.7.10 and Numpy 1.8 installed. - -The best option is to use the package manager homebrew to handle installations and upgrades for you. -To install [homebrew](http://brew.sh/), first open a terminal window (you can find Terminal in the Utilities folder in Applications), and issue the command: - -```bash -# install brew -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -# install pip -easy_install pip -``` - -### Install Dependencies - -- **CPU Dependencies** - - ```bash - # Install fundamental dependents - brew install glog gflags cmake protobuf openblas - - # Install google test on Mac OS X - # Download gtest 1.7.0 - wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz - tar -xzf googletest-release-1.7.0.tar.gz && cd googletest-release-1.7.0 - # Build gtest - mkdir build && cd build && cmake .. && make - # Install gtest library - sudo cp -r ../include/gtest /usr/local/include/ - sudo cp lib*.a /usr/local/lib - ``` - -- **GPU Dependencies(optional)** - - To build GPU version, you will need the following installed: - - 1. a CUDA-capable GPU - 2. Mac OS X 10.11 or later - 2. the Clang compiler and toolchain installed using Xcode - 3. NVIDIA CUDA Toolkit (available at http://developer.nvidia.com/cuda-downloads) - 4. NVIDIA cuDNN Library (availabel at https://developer.nvidia.com/cudnn) - - The CUDA development environment relies on tight integration with the host development environment, - including the host compiler and C runtime libraries, and is therefore only supported on - distribution versions that have been qualified for this CUDA Toolkit release. - - 1. After downloading cuDNN library, issue the following commands: - - ```bash - sudo tar -xzf cudnn-7.5-osx-x64-v5.0-ga.tgz -C /usr/local - sudo chmod a+r /usr/local/cuda/include/cudnn.h /usr/local/cuda/lib/libcudnn* - ``` - 2. Then you need to set DYLD\_LIBRARY\_PATH, PATH environment variables in ~/.bashrc. - - ```bash - export DYLD_LIBRARY_PATH=/usr/local/cuda/lib:$DYLD_LIBRARY_PATH - export PATH=/usr/local/cuda/bin:$PATH - ``` - -### Build and Install - -As usual, the best option is to create build folder under paddle project directory. - -```bash -mkdir build && cd build -cmake .. -``` - -CMake first check PaddlePaddle's dependencies in system default path. After installing some optional -libraries, corresponding build option will be set automatically (for instance, glog, gtest and gflags). -If still not found, you can manually set it based on CMake error information from your screen. - -As a simple example, consider the following: - -- **Only CPU** - - ```bash - cmake .. -DWITH_GPU=OFF - ``` -- **GPU** - - ```bash - cmake .. -DWITH_GPU=ON - ``` - -- **GPU with doc and swig** - - ```bash - cmake .. -DWITH_GPU=ON -DWITH_DOC=ON -DWITH_SWIG_PY=ON - ``` - -Finally, you can build PaddlePaddle: - -```bash -# you can add build option here, such as: -cmake .. -DWITH_GPU=ON -DCMAKE_INSTALL_PREFIX= -# please use sudo make install, if you want to install PaddlePaddle into the system -make -j `sysctl -n hw.ncpu` && make install -# set PaddlePaddle installation path in ~/.bashrc -export PATH=/bin:$PATH -``` -**Note:** - -If you set `WITH_SWIG_PY=ON`, related python dependencies also need to be installed. -Otherwise, PaddlePaddle will automatically install python dependencies -at first time when user run paddle commands, such as `paddle version`, `paddle train`. -It may require sudo privileges: - -```bash -# you can run -sudo pip install /opt/paddle/share/wheels/*.whl -# or just run -sudo paddle version -``` From 5ece5c96ada7a14099408f072abefd213b08bbce Mon Sep 17 00:00:00 2001 From: qijun Date: Tue, 8 Nov 2016 03:36:02 +0000 Subject: [PATCH 234/324] add python wrap for sppLayer --- doc/ui/api/trainer_config_helpers/layers.rst | 12 ++++ paddle/gserver/layers/PoolProjection.cpp | 2 +- .../layers/SpatialPyramidPoolLayer.cpp | 6 +- .../gserver/layers/SpatialPyramidPoolLayer.h | 1 - paddle/gserver/tests/test_LayerGrad.cpp | 2 + paddle/math/Matrix.cpp | 14 +++-- proto/ModelConfig.proto.m4 | 6 +- python/paddle/trainer/config_parser.py | 46 +++++++++++++- .../paddle/trainer_config_helpers/layers.py | 61 ++++++++++++++++++- .../tests/configs/check.md5 | 1 + .../tests/configs/generate_protostr.sh | 2 +- .../tests/configs/test_spp_layer.py | 17 ++++++ 12 files changed, 155 insertions(+), 15 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index 5bb88b0615c12..a7cf57d01799d 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -46,6 +46,12 @@ conv_operator :members: conv_operator :noindex: +conv_projection +------------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: conv_projection + :noindex: + conv_shift_layer ------------------ .. automodule:: paddle.trainer_config_helpers.layers @@ -71,6 +77,12 @@ img_pool_layer -------------- .. automodule:: paddle.trainer_config_helpers.layers :members: img_pool_layer + :noindex: + +spp_layer +-------------- +.. automodule:: paddle.trainer_config_helpers.layers + :members: spp_layer :noindex: maxout_layer diff --git a/paddle/gserver/layers/PoolProjection.cpp b/paddle/gserver/layers/PoolProjection.cpp index 468ca6f1b7d2d..e10788e926470 100644 --- a/paddle/gserver/layers/PoolProjection.cpp +++ b/paddle/gserver/layers/PoolProjection.cpp @@ -16,7 +16,7 @@ limitations under the License. */ namespace paddle { -REGISTER_PROJECTION_CREATE_FUNC(pool2, &PoolProjection::create); +REGISTER_PROJECTION_CREATE_FUNC(pool, &PoolProjection::create); PoolProjection* PoolProjection::create(const ProjectionConfig& config, ParameterPtr parameter, bool useGpu) { diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp index bcdba5c151175..83334a59882ac 100644 --- a/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp @@ -24,7 +24,7 @@ ProjectionConfig SpatialPyramidPoolLayer::getConfig(size_t imgSizeW, size_t pyramidLevel, std::string& poolType) { ProjectionConfig config; - config.set_type("pool2"); + config.set_type("pool"); PoolConfig* conf = config.mutable_pool_conf(); conf->set_channels(channels); conf->set_img_size(imgSizeW); @@ -93,7 +93,7 @@ bool SpatialPyramidPoolLayer::init(const LayerMap& layerMap, startCol = endCol; projInput_.emplace_back(Argument()); } - outputSize_ = endCol; + CHECK_EQ(endCol, getSize()); return true; } @@ -101,7 +101,7 @@ void SpatialPyramidPoolLayer::forward(PassType passType) { Layer::forward(passType); int batchSize = getInput(0).getBatchSize(); - resetOutput(batchSize, outputSize_); + resetOutput(batchSize, getSize()); for (size_t i = 0; i < pyramidHeight_; i++) { size_t startCol = projCol_[i].first; size_t endCol = projCol_[i].second; diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.h b/paddle/gserver/layers/SpatialPyramidPoolLayer.h index de1fd4da07dd8..156581530a1bc 100644 --- a/paddle/gserver/layers/SpatialPyramidPoolLayer.h +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.h @@ -27,7 +27,6 @@ class SpatialPyramidPoolLayer : public Layer { size_t imgSizeW_; size_t imgSizeH_; size_t pyramidHeight_; - size_t outputSize_; std::string poolType_; std::vector> poolProjections_; diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index c3597f56070ef..595e20354ad5b 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -931,6 +931,8 @@ void testSppLayer(const string& poolType, const int pyramidHeight, bool trans, sppConfig->set_channels(16); sppConfig->set_img_size(10); sppConfig->set_img_size_y(20); + int outputSize = (std::pow(4, sppConfig->pyramid_height()) - 1) / (4 - 1); + config.layerConfig.set_size(outputSize * sppConfig->channels()); testLayerGrad(config, "spp", 100, trans, useGpu); } diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 721c3de59fbcd..607334aaa934b 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -1510,18 +1510,19 @@ void CpuMatrix::maxPoolForward(Matrix& inputMat, size_t imgSizeH, CHECK(inHeight * inWidth == inputMat.getWidth() / channels); CHECK_EQ(num, this->getHeight()); CHECK_EQ(channels * outputH * outputW, this->getWidth()); + size_t outStride = getStride(); /* initialize the data_ */ for (size_t i = 0; i < height_; i++) { for (size_t j = 0; j < width_; j++) { - outData[i * getStride() + j] = -(real)FLT_MAX; + outData[i * outStride + j] = -(real)FLT_MAX; } } /* pool max one by one */ for (size_t n = 0; n < num; ++n) { // frame by frame if (!isContiguous()) { - outData = data_ + n * getStride(); + outData = data_ + n * outStride; } for (size_t c = 0; c < channels; ++c) { // channel by channel for (size_t ph = 0; ph < outputH; ++ph) { @@ -1564,10 +1565,15 @@ void CpuMatrix::maxPoolBackward(Matrix& image, size_t imgSizeH, size_t imgSizeW, real* inData = image.getData(); real* otData = outV.getData(); real* otGrad = outGrad.getData(); + + size_t outStride = outV.getStride(); + real* origOutData = otData; + real* origOutGrad = otGrad; + for (size_t n = 0; n < num; ++n) { if (!outV.isContiguous()) { - otData = outV.getData() + n * outV.getStride(); - otGrad = outGrad.getData() + n * outGrad.getStride(); + otData = origOutData + n * outStride; + otGrad = origOutGrad + n * outStride; } for (size_t c = 0; c < channels; ++c) { for (size_t ph = 0; ph < outputH; ++ph) { diff --git a/proto/ModelConfig.proto.m4 b/proto/ModelConfig.proto.m4 index 4ef1550105492..a247f6f3e7ed4 100644 --- a/proto/ModelConfig.proto.m4 +++ b/proto/ModelConfig.proto.m4 @@ -202,11 +202,11 @@ message ProjectionConfig { optional ConvConfig conv_conf = 8; optional int32 num_filters = 9; - // For pool - optional PoolConfig pool_conf = 10; - // For IdentityOffsetProjection optional uint64 offset = 11 [default = 0]; + + // For pool + optional PoolConfig pool_conf = 12; } message OperatorConfig { diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index e9098943165fd..7ad2b7fd5ce83 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -470,6 +470,7 @@ def __init__( image=None, block_expand=None, maxout=None, + spp=None, format=None, nnz=None, is_static=None, @@ -669,7 +670,6 @@ def calc_bias_size(self): def calc_parameter_dims(self, input_size, output_size): return None - # Define a operator for mixed layer @config_class class Operator(Cfg): @@ -783,6 +783,15 @@ def __init__( padding_y = None): self.add_keys(locals()) +class SpatialPyramidPool(Cfg): + def __init__( + self, + pool_type, + pyramid_height, + channels, + img_width = None): + self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Norm(Cfg): @@ -1043,6 +1052,22 @@ def parse_pool(pool, input_layer_name, pool_conf): 2*pool_conf.padding_y - pool_conf.size_y) / \ float(pool_conf.stride_y))) + 1 +def parse_spp(spp, input_layer_name, spp_conf): + spp_conf.pool_type = spp.pool_type + config_assert(spp.pool_type in ['max-projection', 'avg-projection'], + "pool-type %s is not in " "['max-projection', 'avg-projection']" + % spp.pool_type) + spp_conf.pyramid_height = spp.pyramid_height + spp_conf.channels = spp.channels + + img_pixels = g_layer_map[input_layer_name].size / spp_conf.channels + + spp_conf.img_size = default(spp.img_width, int(img_pixels ** 0.5)) + spp_conf.img_size_y = img_pixels / spp_conf.img_size + config_assert(spp_conf.img_size * spp_conf.img_size_y == img_pixels, + "Incorrect input image size %d for input image pixels %d" + % (spp_conf.img_size, img_pixels)) + def parse_image(image, input_layer_name, image_conf): image_conf.channels = image.channels image_pixels = g_layer_map[input_layer_name].size / image_conf.channels @@ -1649,6 +1674,25 @@ def __init__( name, pool_conf.output_y, pool_conf.output_x)) self.set_layer_size((pool_conf.output_x * pool_conf.output_y) * pool_conf.channels) +@config_layer('spp') +class SpatialPyramidPoolLayer(LayerBase): + def __init__( + self, + name, + inputs, + device=None): + super(SpatialPyramidPoolLayer, self).__init__(name, 'spp', 0, inputs=inputs, device=device) + for input_index in xrange(len(self.inputs)): + input_layer = self.get_input_layer(input_index) + parse_spp( + self.inputs[input_index].spp, + input_layer.name, + self.config.inputs[input_index].spp_conf) + spp_conf = self.config.inputs[input_index].spp_conf + output_size = (pow(4, spp_conf.pyramid_height) - 1) / (4 - 1) + print("output size for %s is %d " % (name, output_size)) + self.set_layer_size(output_size * spp_conf.channels) + @config_layer('batch_norm') class BatchNormLayer(LayerBase): layer_type = 'batch_norm' diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 9a23c02431d18..03243c03b021b 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -55,7 +55,8 @@ 'multi_binary_label_cross_entropy', 'rank_cost', 'lambda_cost', 'huber_cost', 'block_expand_layer', - 'maxout_layer', 'out_prod_layer', 'print_layer' + 'maxout_layer', 'out_prod_layer', 'print_layer', + 'spp_layer', ] @@ -111,6 +112,7 @@ class LayerType(object): LINEAR_COMBINATION_LAYER = "convex_comb" BLOCK_EXPAND = "blockexpand" MAXOUT = "maxout" + SPP_LAYER = "spp" PRINT_LAYER = "print" @@ -868,6 +870,7 @@ def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, size=input.size) + @wrap_bias_attr_default() @wrap_param_attr_default() @wrap_act_default(param_names=['gate_act'], @@ -1708,6 +1711,62 @@ def img_pool_layer(input, pool_size, name=None, num_filters=num_channels) +@wrap_name_default("spp") +@layer_support() +def spp_layer(input, name=None, num_channels=None, pool_type=None, + pyramid_height=None, img_width=None, layer_attr=None): + pass + """ + Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition. + The details please refer to + `Kaiming He's paper `_. + + :param name: layer name. + :type name: basestring + :param input: layer's input. + :type input: LayerOutput + :param num_channels: number of input channel. + :type num_channels: int + :param pool_type: Pooling type. MaxPooling or AveragePooling. Default is MaxPooling. + :type scale: BasePoolingType + :param pyramid_height: pyramid height. + :type pyramid_height: int + :param img_width: the width of input feature map. If it is None, the input feature + map should be square. + :type img_width: int|None + :param layer_attr: Extra Layer Attribute. + :type layer_attr: ExtraLayerAttribute + :return: LayerOutput object. + :rtype: LayerOutput + """ + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + + if pool_type is None: + pool_type = MaxPooling() + elif isinstance(pool_type, AvgPooling): + pool_type.name = 'avg' + + type_name = pool_type.name + if (isinstance(pool_type, AvgPooling) or isinstance(pool_type, MaxPooling)): + type_name += '-projection' + + Layer( + name=name, + type=LayerType.SPP_LAYER, + inputs=Input(input.name, + spp=SpatialPyramidPool(pool_type=type_name, + channels=num_channels, + pyramid_height=pyramid_height, + img_width=img_width) + ), + **ExtraLayerAttribute.to_kwargs(layer_attr) + ) + return LayerOutput(name, LayerType.SPP_LAYER, parents=[input], + num_filters=num_channels) + + def __img_norm_layer__(name, input, size, norm_type, scale, power, num_channels, blocked, layer_attr): if num_channels is None: diff --git a/python/paddle/trainer_config_helpers/tests/configs/check.md5 b/python/paddle/trainer_config_helpers/tests/configs/check.md5 index 72dfdad7bdd40..bf0512420ed3c 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/check.md5 +++ b/python/paddle/trainer_config_helpers/tests/configs/check.md5 @@ -20,3 +20,4 @@ fded24727338fb8ce44d9951ed8aea08 test_rnn_group.protostr 67d6fde3afb54f389d0ce4ff14726fe1 test_sequence_pooling.protostr f586a548ef4350ba1ed47a81859a64cb unused_layers.protostr f937a5a6e7e8864b4d8cf56b0f7c7f44 util_layers.protostr +60c9a71e19bd4b2a1253712799d0ae70 test_spp_layer.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index 6a31ceabdf36d..6102c614de3b3 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -9,7 +9,7 @@ test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers util_layers simple_rnn_layers unused_layers test_cost_layers test_rnn_group shared_fc shared_lstm test_cost_layers_with_weight -test_maxout test_bi_grumemory) +test_maxout test_bi_grumemory test_spp_layer) for conf in ${configs[*]} diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py new file mode 100644 index 0000000000000..6786c27639ea8 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py @@ -0,0 +1,17 @@ +from paddle.trainer_config_helpers import * + +settings( + batch_size=100, + learning_rate=1e-5 +) + +data = data_layer(name='data', size=3200) + + +spp = spp_layer(input=data, + pyramid_height=2, + num_channels=16, + pool_type=MaxPooling(), + img_width=10) + +outputs(spp) From 098e7caa23bb18a512b5809d1128225e4e12ede6 Mon Sep 17 00:00:00 2001 From: gangliao Date: Mon, 7 Nov 2016 19:37:48 -0800 Subject: [PATCH 235/324] Cancelling Travis build with docs updates only. (#372) --- .travis.yml | 10 ++++++++++ README.md | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7812ac0283789..b5a00bb2ca0ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,16 @@ addons: - lcov - graphviz before_install: + - | + if [ ${JOB} == "BUILD_AND_TEST" ]; then + if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then + TRAVIS_COMMIT_RANGE="FETCH_HEAD...$TRAVIS_BRANCH" + fi + git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.md$)' || { + echo "Only markdown docs were updated, stopping build process." + exit + } + fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi - pip install wheel protobuf sphinx breathe recommonmark diff --git a/README.md b/README.md index 66767d7ff8e4a..81ff8c7122ab8 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ developed by Baidu scientists and engineers for the purpose of applying deep learning to many products at Baidu. Our vision is to enable deep learning for everyone via PaddlePaddle. -Please refer to our [release log](https://github.com/baidu/Paddle/releases) to track the latest feature of PaddlePaddle. +Please refer to our [release announcement](https://github.com/baidu/Paddle/releases) to track the latest feature of PaddlePaddle. ## Features From 8b5cb2955c3b656077a1d2c1ed8f7ec309b50472 Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Tue, 8 Nov 2016 12:57:21 +0800 Subject: [PATCH 236/324] fix deadlink in Chinese quick start doc. (#389) --- doc_cn/demo/quick_start/index.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc_cn/demo/quick_start/index.md b/doc_cn/demo/quick_start/index.md index aa6b66ca8c024..4d9b24ba851a7 100644 --- a/doc_cn/demo/quick_start/index.md +++ b/doc_cn/demo/quick_start/index.md @@ -134,9 +134,8 @@ define_py_data_sources2(train_list='data/train.list', * obj="process": 指定生成数据的函数 * args={"dictionary": word_dict}: 额外的参数，这里指定词典 -更详细用例请参考文档Python Use Case， -数据格式和详细文档请参考 -PyDataProviderWrapper。 +更详细数据格式和用例请参考 +PyDataProvider2。 ## 网络结构(Network Architecture) 本节我们将专注于网络结构的介绍。 From a275fe93aae77553bd833d995855d0da5f6a508f Mon Sep 17 00:00:00 2001 From: luotao1 Date: Tue, 8 Nov 2016 13:26:22 +0800 Subject: [PATCH 237/324] add python-related unittest problem in faq document (#377) --- doc_cn/faq/index.rst | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/doc_cn/faq/index.rst b/doc_cn/faq/index.rst index db28b4436fe5e..3eb0e10ae2228 100644 --- a/doc_cn/faq/index.rst +++ b/doc_cn/faq/index.rst @@ -177,3 +177,40 @@ PaddlePaddle的参数使用名字 :code:`name` 作为参数的ID，相同名字 pip install --upgrade pip +8. python相关的单元测试都过不了 +-------------------------------- + +如果出现以下python相关的单元测试都过不了的情况： + +.. code-block:: bash + + 24 - test_PyDataProvider (Failed) + 26 - test_RecurrentGradientMachine (Failed) + 27 - test_NetworkCompare (Failed) + 28 - test_PyDataProvider2 (Failed) + 32 - test_Prediction (Failed) + 33 - test_Compare (Failed) + 34 - test_Trainer (Failed) + 35 - test_TrainerOnePass (Failed) + 36 - test_CompareTwoNets (Failed) + 37 - test_CompareTwoOpts (Failed) + 38 - test_CompareSparse (Failed) + 39 - test_recurrent_machine_generation (Failed) + 40 - test_PyDataProviderWrapper (Failed) + 41 - test_config_parser (Failed) + 42 - test_swig_api (Failed) + 43 - layers_test (Failed) + +并且查询PaddlePaddle单元测试的日志，提示： + +.. code-block:: bash + + paddle package is already in your PYTHONPATH. But unittest need a clean environment. + Please uninstall paddle package before start unittest. Try to 'pip uninstall paddle'. + +解决办法是：卸载paddle包 :code:`pip uninstall paddle`。 + +原因是：单元测试使用了一个旧版本的python包，而没有测试到代码中实际修改的python包。即单元测试需要一个干净的环境： + +* 如果paddle包已经在python的site-packages里面了，那么单元测试时使用的paddle包，就是site-packages里面的python包，而不是源码目录里 :code:`/python` 目录下的python包。 +* 即便设置了 :code:`PYTHONPATH` 到 :code:`/python` 也没用，因为python的搜索路径是优先已经安装的python包。 \ No newline at end of file From 56b23d1838daf3757e7f6f5cac8abf3f295a2c15 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 7 Nov 2016 23:44:33 -0600 Subject: [PATCH 238/324] Fix macOS quick start preprocess script. (#390) * Use `gshuf` instead of `shuf` in macOS * Fix #388 --- demo/quick_start/preprocess.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/demo/quick_start/preprocess.sh b/demo/quick_start/preprocess.sh index fe2acbbd74898..58a72147c5e41 100755 --- a/demo/quick_start/preprocess.sh +++ b/demo/quick_start/preprocess.sh @@ -21,14 +21,21 @@ set -e export LC_ALL=C +UNAME_STR=`uname` + +if [[ ${UNAME_STR} == 'Linux' ]]; then + SHUF_PROG='shuf' +else + SHUF_PROG='gshuf' +fi mkdir -p data/tmp python preprocess.py -i data/reviews_Electronics_5.json.gz # uniq and shuffle cd data/tmp echo 'uniq and shuffle...' -cat pos_*|sort|uniq|shuf> pos.shuffed -cat neg_*|sort|uniq|shuf> neg.shuffed +cat pos_*|sort|uniq|${SHUF_PROG}> pos.shuffed +cat neg_*|sort|uniq|${SHUF_PROG}> neg.shuffed min_len=`sed -n '$=' neg.shuffed` test_num=$((min_len/10)) @@ -42,8 +49,8 @@ head -n$train_num neg.shuffed >train.neg tail -n$test_num pos.shuffed >test.pos tail -n$test_num neg.shuffed >test.neg -cat train.pos train.neg|shuf>../train.txt -cat test.pos test.neg|shuf>../test.txt +cat train.pos train.neg | ${SHUF_PROG} >../train.txt +cat test.pos test.neg | ${SHUF_PROG} >../test.txt cd - echo 'data/train.txt' > data/train.list From a07da94939695986e2c15858f61199ca60c37fec Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Tue, 8 Nov 2016 14:19:56 +0800 Subject: [PATCH 239/324] fix floating-point overflow problem of tanh (#355) --- paddle/cuda/include/hl_base.h | 9 ++ paddle/cuda/src/hl_avx_functions.cc | 2 + paddle/cuda/src/hl_cpu_functions.cc | 4 +- paddle/gserver/tests/test_LayerGrad.cpp | 2 +- paddle/gserver/tests/test_RecurrentLayer.cpp | 2 +- paddle/math/BaseMatrix.cu | 5 +- paddle/math/MathFunctions.cpp | 5 +- paddle/math/Matrix.cpp | 9 -- paddle/math/tests/CMakeLists.txt | 1 + paddle/math/tests/test_FPException.cpp | 94 ++++++++++++++++++++ 10 files changed, 119 insertions(+), 14 deletions(-) create mode 100644 paddle/math/tests/test_FPException.cpp diff --git a/paddle/cuda/include/hl_base.h b/paddle/cuda/include/hl_base.h index 1fe2774cc5a29..02fa6bc3ace32 100644 --- a/paddle/cuda/include/hl_base.h +++ b/paddle/cuda/include/hl_base.h @@ -209,6 +209,15 @@ typedef struct { #define HL_FLOAT_MIN 2.2250738585072014e-308 #endif + +/** + * The maximum input value for exp, used to avoid overflow problem. + * + * Currently only used for tanh function. + */ +#define EXP_MAX_INPUT 40.0 + + /** * @brief DIVUP(x, y) is similar to ceil(x / y). * @note For CUDA, DIVUP will be used to specify diff --git a/paddle/cuda/src/hl_avx_functions.cc b/paddle/cuda/src/hl_avx_functions.cc index 2d471206f61f2..08976180fff5b 100644 --- a/paddle/cuda/src/hl_avx_functions.cc +++ b/paddle/cuda/src/hl_avx_functions.cc @@ -38,7 +38,9 @@ namespace hppl { } __m256 tanh(const __m256 a) { + __m256 max = _mm256_set1_ps(EXP_MAX_INPUT); __m256 tmp = _mm256_mul_ps(_mm256_set1_ps(-2.0f), a); + tmp = _mm256_min_ps(tmp, max); tmp = exp(tmp); return _mm256_sub_ps( _mm256_div_ps(_mm256_set1_ps(2.0f), diff --git a/paddle/cuda/src/hl_cpu_functions.cc b/paddle/cuda/src/hl_cpu_functions.cc index 3fd6b278d0537..b8352c2d537fb 100644 --- a/paddle/cuda/src/hl_cpu_functions.cc +++ b/paddle/cuda/src/hl_cpu_functions.cc @@ -30,7 +30,9 @@ namespace hppl { } real tanh(const real a) { - return (2.0 / (1.0 + exp(-2.0*a))) - 1.0; + real tmp = -2.0 * a; + tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; + return (2.0 / (1.0 + exp(tmp))) - 1.0; } real linear(const real a) { diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 5397b952bced8..4e01fa91ed2ba 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -995,7 +995,7 @@ TEST(Layer, LstmLayer) { TestConfig config; config.layerConfig.set_type("lstmemory"); config.layerConfig.set_size(4); - config.layerConfig.set_active_type("sigmoid"); + config.layerConfig.set_active_type("tanh"); config.layerConfig.set_active_state_type("sigmoid"); config.layerConfig.set_active_gate_type("sigmoid"); config.biasSize = 28; diff --git a/paddle/gserver/tests/test_RecurrentLayer.cpp b/paddle/gserver/tests/test_RecurrentLayer.cpp index 9b933b153d158..1c8497e8c526f 100644 --- a/paddle/gserver/tests/test_RecurrentLayer.cpp +++ b/paddle/gserver/tests/test_RecurrentLayer.cpp @@ -369,7 +369,7 @@ TEST(Layer, LstmLayer) { LayerConfig layerConfig; layerConfig.set_type("lstmemory"); layerConfig.set_active_type("relu"); - layerConfig.set_active_state_type("sigmoid"); + layerConfig.set_active_state_type("tanh"); layerConfig.set_active_gate_type("sigmoid"); layerConfig.add_inputs(); diff --git a/paddle/math/BaseMatrix.cu b/paddle/math/BaseMatrix.cu index 8b888b1ee5e46..d81b99e544158 100644 --- a/paddle/math/BaseMatrix.cu +++ b/paddle/math/BaseMatrix.cu @@ -625,7 +625,10 @@ void BaseMatrixT::squareDerivative(BaseMatrixT& b) { applyBinary(binary::SquareDerivative(), b); } -DEFINE_MATRIX_BINARY_OP(Tanh, b = 2.0 / (1.0 + exp(-2 * a)) - 1.0); +DEFINE_MATRIX_BINARY_OP(Tanh, + T tmp = -2.0 * a; + tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; + b = 2.0 / (1.0 + std::exp(tmp)) - 1.0); template<> void BaseMatrixT::tanh(BaseMatrixT& b) { applyBinary(binary::Tanh(), b); diff --git a/paddle/math/MathFunctions.cpp b/paddle/math/MathFunctions.cpp index f8132066477db..e0b2a2bb5b2cd 100644 --- a/paddle/math/MathFunctions.cpp +++ b/paddle/math/MathFunctions.cpp @@ -200,7 +200,10 @@ void vLog1p(const int n, const T* a, T* r) { binary::vLog1p(), const_cast(a), r, 1, n, n, n); } -DEFINE_MATRIX_BINARY_OP(vTanh, b = 2.0 / (1.0 + std::exp(-2 * a)) - 1.0); +DEFINE_MATRIX_BINARY_OP(vTanh, + T tmp = -2.0 * a; + tmp = (tmp > EXP_MAX_INPUT) ? EXP_MAX_INPUT : tmp; + b = 2.0 / (1.0 + std::exp(tmp)) - 1.0); template void vTanh(const int n, const T* a, T* r) { hl_cpu_apply_binary_op, 0, 0>( diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index d901ba93492ac..4fc9b2d089366 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -3471,9 +3471,7 @@ void CpuMatrix::tanh(Matrix& output) { size_t dim = getWidth(); CHECK_EQ(output.getHeight(), numSamples); CHECK_EQ(output.getWidth(), dim); - errno = 0; vTanh(numSamples * dim, getData(), output.getData()); - CHECK_EQ(errno, 0) << "vTanh error"; } void CpuMatrix::tanhDerivative(Matrix& output) { @@ -3495,10 +3493,8 @@ void CpuMatrix::softrelu(Matrix& output) { out[j] = x; } } - errno = 0; vExp(numSamples * dim, output.getData(), output.getData()); vLog1p(numSamples * dim, output.getData(), output.getData()); - CHECK_EQ(errno, 0) << "vExp+vLog1p error"; } void CpuMatrix::softreluDerivative(Matrix& output) { @@ -3513,9 +3509,7 @@ void CpuMatrix::softreluDerivative(Matrix& output) { MatrixPtr tmpMat = Matrix::create(numSamples, dim); real* tmp = tmpMat->getData(); - errno = 0; vExp(size, output.getData(), tmpMat->getData()); - CHECK_EQ(errno, 0) << "vExp error"; for (size_t i = 0; i < size; ++i) { grad[i] *= (1.0 - 1.0 / tmp[i]); @@ -3538,10 +3532,7 @@ void CpuMatrix::scaledTanh(Matrix& output, real p1, real p2) { out[i] = p2 * in[i]; } - // out = tanh(out) - errno = 0; vTanh(numSamples * dim, out, out); - CHECK_EQ(errno, 0) << "vTanh error"; // out = p1 * out for (size_t i = 0; i < numSamples * dim; ++i) { diff --git a/paddle/math/tests/CMakeLists.txt b/paddle/math/tests/CMakeLists.txt index eb72f11e1c653..247be983ba329 100644 --- a/paddle/math/tests/CMakeLists.txt +++ b/paddle/math/tests/CMakeLists.txt @@ -13,3 +13,4 @@ add_simple_unittest(test_sparseMatrixCompare) add_simple_unittest(test_perturbation) add_simple_unittest(test_CpuGpuVector) add_simple_unittest(test_Allocator) +add_simple_unittest(test_FPException) diff --git a/paddle/math/tests/test_FPException.cpp b/paddle/math/tests/test_FPException.cpp new file mode 100644 index 0000000000000..174278c2aaac4 --- /dev/null +++ b/paddle/math/tests/test_FPException.cpp @@ -0,0 +1,94 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +/** + * This test is about floating point calculation exception. + * Paddle catches FE_INVALID, FE DIVBYZERO and FE_OVERFLOW exceptions. + * + * Some exceptions occur in the middle of a set of formulas, + * that can be circumvented by some tricks. + * For example, + * calculate tanh + * b = 2.0 / (1.0 + exp(-2 * a)) - 1.0 + * + * If the result of (-2 * a) is too large, + * a FE_OVERFLOW exception occurs when calculating exp. + * But the result of tanh is no overflow problem, + * so we can add some tricks to prevent exp calculate an excessive value. + * + */ +#include +#include +#include "paddle/math/Matrix.h" +#include "paddle/utils/Excepts.h" + +using namespace paddle; // NOLINT + +void SetTensorValue(Matrix& matrix, real value) { + int height = matrix.getHeight(); + int width = matrix.getWidth(); + int stride = matrix.getStride(); + real* data = matrix.getData(); + for (int i = 0; i < height; i++) { + int j = rand() % width; // NOLINT + if (typeid(matrix) == typeid(CpuMatrix)) { + data[i * stride + j] = value; + } else if (typeid(matrix) == typeid(GpuMatrix)) { + hl_memcpy(&data[i * stride + j], &value, sizeof(real)); + } else { + LOG(FATAL) << "should not reach here"; + } + } +} + +template +void testTanh(real illegal) { + MatrixPtr A = std::make_shared(10, 10); + MatrixPtr B = std::make_shared(10, 10); + A->randomizeUniform(); + B->randomizeUniform(); + + SetTensorValue(*A, illegal); + + A->tanh(*B); +} + +template +void testSigmoid(real illegal) { + MatrixPtr A = std::make_shared(10, 10); + MatrixPtr B = std::make_shared(10, 10); + A->randomizeUniform(); + B->randomizeUniform(); + + SetTensorValue(*A, illegal); + + A->sigmoid(*B); +} + +TEST(fp, overflow) { + for (auto illegal : {-90.0, 90.0}) { + LOG(INFO) << " illegal=" << illegal; + testTanh(illegal); + testSigmoid(illegal); + } +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + initMain(argc, argv); + + feenableexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW); + return RUN_ALL_TESTS(); +} From f06f4dfbaee1696b0d789693095b09babf003d90 Mon Sep 17 00:00:00 2001 From: backyes Date: Tue, 8 Nov 2016 14:52:37 +0800 Subject: [PATCH 240/324] py_paddle link zlib(#393) --- paddle/api/paddle_api_config.py.in | 1 + paddle/api/paddle_ld_flags.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/paddle/api/paddle_api_config.py.in b/paddle/api/paddle_api_config.py.in index 6531e5ccb3dba..27e71c86605b6 100644 --- a/paddle/api/paddle_api_config.py.in +++ b/paddle/api/paddle_api_config.py.in @@ -1,6 +1,7 @@ PADDLE_BUILD_DIR="@CMAKE_CURRENT_BINARY_DIR@/../" WITH_GPU="@WITH_GPU@" PROTOBUF_LIB="@PROTOBUF_LIBRARY@" +ZLIB_LIB="@ZLIB_LIBRARIES@" CMAKE_THREAD_LIB="@CMAKE_THREAD_LIBS_INIT@" CMAKE_DL_LIBS="@CMAKE_DL_LIBS@" diff --git a/paddle/api/paddle_ld_flags.py b/paddle/api/paddle_ld_flags.py index bc1afc5898e82..e51f65bcf341b 100644 --- a/paddle/api/paddle_ld_flags.py +++ b/paddle/api/paddle_ld_flags.py @@ -38,6 +38,7 @@ def __init__(self): self.paddle_build_dir = os.path.abspath(self.paddle_build_dir) self.with_gpu = PaddleLDFlag.cmake_bool(WITH_GPU) self.protolib = PROTOBUF_LIB + self.zlib = ZLIB_LIB self.thread = CMAKE_THREAD_LIB self.dl_libs = CMAKE_DL_LIBS self.with_python = PaddleLDFlag.cmake_bool(WITH_PYTHON) @@ -64,7 +65,7 @@ def libs_dir_str(self): def parent_dir_str(self): libdirs = PARENT_LIB_DIRS - return " ".join(map(lambda x: "-L" + os.path.join(self.paddle_build_dir, '..', x), + return " ".join(map(lambda x: "-L" + os.path.join(self.paddle_build_dir, '..', x), libdirs)) def libs_str(self): @@ -82,6 +83,7 @@ def libs_str(self): "-lpaddle_cuda", "-lpaddle_api", self.normalize_flag(self.protolib), + self.normalize_flag(self.zlib), self.normalize_flag(self.thread), self.normalize_flag(self.dl_libs), self.normalize_flag(self.cblas_libs), From 57bc6238d93b50632a644620b62fdb81698b5eb7 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 8 Nov 2016 01:35:21 -0600 Subject: [PATCH 241/324] enable swig unittest in travis-ci (#394) * Init * Add numpy deps * Refine --- .travis.yml | 3 ++- paddle/api/paddle_api_config.py.in | 1 + paddle/api/paddle_ld_flags.py | 11 ++++++++++- paddle/scripts/travis/build_and_test.sh | 2 ++ paddle/setup.py.in | 13 ++++++++++--- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index b5a00bb2ca0ec..74aa767febeb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,6 +38,7 @@ addons: - curl - lcov - graphviz + - swig before_install: - | if [ ${JOB} == "BUILD_AND_TEST" ]; then @@ -51,7 +52,7 @@ before_install: fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi - - pip install wheel protobuf sphinx breathe recommonmark + - pip install wheel protobuf sphinx breathe recommonmark virtualenv numpy script: - paddle/scripts/travis/main.sh notifications: diff --git a/paddle/api/paddle_api_config.py.in b/paddle/api/paddle_api_config.py.in index 27e71c86605b6..a2352250c31ef 100644 --- a/paddle/api/paddle_api_config.py.in +++ b/paddle/api/paddle_api_config.py.in @@ -16,3 +16,4 @@ GFLAGS_LOCATION="@GFLAGS_LOCATION@" CBLAS_LIBRARIES="@CBLAS_LIBS@" CUDA_LIBRARIES="@CUDA_LIBRARIES@" +WITH_COVERALLS="@ON_COVERALLS@" diff --git a/paddle/api/paddle_ld_flags.py b/paddle/api/paddle_ld_flags.py index e51f65bcf341b..05d741f8859ba 100644 --- a/paddle/api/paddle_ld_flags.py +++ b/paddle/api/paddle_ld_flags.py @@ -48,6 +48,7 @@ def __init__(self): self.glog_libs = LIBGLOG_LIBRARY self.with_gflags = PaddleLDFlag.cmake_bool(WITH_GFLAGS) + self.with_coverage = PaddleLDFlag.cmake_bool(WITH_COVERALLS) self.gflags_libs = GFLAGS_LIBRARIES self.gflags_location = GFLAGS_LOCATION self.cblas_libs = CBLAS_LIBRARIES @@ -97,6 +98,8 @@ def libs_str(self): libs.append(self.normalize_flag(self.gflags_libs)) if self.with_gpu: libs.append(self.normalize_flag(self.curt)) + if self.with_coverage: + libs.append("-fprofile-arcs") return " ".join(filter(lambda l: len(l) != 0, libs)) def normalize_flag(self, cmake_flag): @@ -133,8 +136,14 @@ def cmake_bool(cmake_str): return False else: return True - + def c_flag(self): + if self.with_coverage: + return ["-fprofile-arcs", "-ftest-coverage", "-O0", "-g"] + else: + return None except ImportError: class PaddleLDFlag(object): def ldflag_str(self): pass + def c_flag(self): + pass diff --git a/paddle/scripts/travis/build_and_test.sh b/paddle/scripts/travis/build_and_test.sh index 54e3320c8c158..242fd982aa001 100755 --- a/paddle/scripts/travis/build_and_test.sh +++ b/paddle/scripts/travis/build_and_test.sh @@ -3,6 +3,8 @@ source ./common.sh CMAKE_EXTRA="" if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then CMAKE_EXTRA="-DPYTHON_LIBRARY=/usr/local/Cellar/python/2.7.12_1/Frameworks/Python.framework/Versions/2.7/lib/python2.7/config/libpython2.7.dylib" +else + CMAKE_EXTRA="-DWITH_SWIG_PY=ON" fi diff --git a/paddle/setup.py.in b/paddle/setup.py.in index 3341dd6f95969..1a15eafd5528a 100644 --- a/paddle/setup.py.in +++ b/paddle/setup.py.in @@ -31,8 +31,8 @@ is_lin = (system == 'linux') # because generate paddle LDFLAGS is too complicated to do in setup.py # it just read COMAKE generated LDFLAGS. extra_links = [] -ldflags = api.paddle_ld_flags.PaddleLDFlag() -ldflags = ldflags.ldflag_str() +obj = api.paddle_ld_flags.PaddleLDFlag() +ldflags = obj.ldflag_str() if ldflags is not None: extra_links.extend(ldflags.split(" ")) @@ -51,13 +51,20 @@ elif is_osx == True: include_dirs = [np.get_include(), "../"] # include numpy and paddle. +extra_c = obj.c_flag() + +attr=dict() +if extra_c is not None: + attr["extra_compile_args"] = extra_c + setup(name="py_paddle", version="@PADDLE_VERSION@", ext_modules=[ Extension('py_paddle._swig_paddle', # Build SWIG Extension. ['Paddle_wrap.cxx'], include_dirs = include_dirs, - extra_link_args = extra_links + extra_link_args = extra_links, + **attr ) ], packages=['py_paddle'], From 125c19a3c31a6290d7feea3faf41dd20fb97e2f4 Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Tue, 8 Nov 2016 15:35:57 +0800 Subject: [PATCH 242/324] fix some nvcc compile options (#392) --- CMakeLists.txt | 15 +++++++++++++++ cmake/util.cmake | 8 -------- paddle/gserver/CMakeLists.txt | 2 +- paddle/math/CMakeLists.txt | 2 +- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 282e3e199ef44..39f876bc9ee4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,11 +95,26 @@ if(NOT WITH_GPU) add_definitions(-DHPPL_STUB_FUNC) list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu) else() + if(${CUDA_VERSION_MAJOR} GREATER 6) + if(COMPILER_SUPPORT_CXX11) + LIST(APPEND CUDA_NVCC_FLAGS -std=c++11) + endif() + endif() + # TODO(yuyang18): Change it to remove std=c++11 in cuda compile. set(CUDA_PROPAGATE_HOST_FLAGS OFF) if(NOT CUDNN_FOUND) message(FATAL_ERROR "Paddle need cudnn to compile") endif() + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-g -O3 --use_fast_math") + + if(WITH_AVX) + if(AVX_FOUND) + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler -mavx") + endif(AVX_FOUND) + else(WITH_AVX) + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler -msse3") + endif(WITH_AVX) if(WITH_DSO) set(CUDA_LIBRARIES "") diff --git a/cmake/util.cmake b/cmake/util.cmake index 3f78cd08c3905..a8282f07184c3 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -188,14 +188,6 @@ macro(add_simple_unittest TARGET_NAME) add_unittest(${TARGET_NAME} ${TARGET_NAME}.cpp) endmacro() -macro(add_paddle_culib TARGET_NAME) - set(NVCC_FLAG ${CUDA_NVCC_FLAGS}) - set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS};--use_fast_math) - cuda_add_library(${TARGET_NAME} STATIC ${ARGN}) - set(CUDA_NVCC_FLAGS ${NVCC_FLAG}) -endmacro() - - # Creates C resources file from files in given resource file function(create_resources res_file output) # Create empty output file diff --git a/paddle/gserver/CMakeLists.txt b/paddle/gserver/CMakeLists.txt index 9ac4d210f6d37..a066f80c221ee 100644 --- a/paddle/gserver/CMakeLists.txt +++ b/paddle/gserver/CMakeLists.txt @@ -50,7 +50,7 @@ if(NOT WITH_PYTHON) endif() if(WITH_GPU) - add_paddle_culib(paddle_gserver ${GSERVER_SOURCES}) + cuda_add_library(paddle_gserver ${GSERVER_SOURCES}) else() add_library(paddle_gserver STATIC ${GSERVER_SOURCES}) diff --git a/paddle/math/CMakeLists.txt b/paddle/math/CMakeLists.txt index db305812a7c03..93b1bf46a1007 100644 --- a/paddle/math/CMakeLists.txt +++ b/paddle/math/CMakeLists.txt @@ -23,7 +23,7 @@ if(NOT WITH_GPU) add_library(paddle_math STATIC ${MATH_SOURCES}) else() - add_paddle_culib(paddle_math ${MATH_SOURCES}) + cuda_add_library(paddle_math ${MATH_SOURCES}) endif() From db1757556e0712ab74b23c5f048768c952bf59a9 Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 8 Nov 2016 17:20:02 +0800 Subject: [PATCH 243/324] Follow comments --- paddle/cuda/src/hl_cuda_cnn.cu | 16 ++--- paddle/gserver/layers/BilinearInterpLayer.cpp | 9 ++- paddle/gserver/layers/BilinearInterpLayer.h | 1 + paddle/gserver/tests/test_LayerGrad.cpp | 10 +++ paddle/math/Matrix.cpp | 69 ++++++++----------- paddle/math/Matrix.h | 24 +++++-- paddle/math/tests/test_matrixCompare.cpp | 11 +-- .../paddle/trainer_config_helpers/layers.py | 17 +++-- 8 files changed, 88 insertions(+), 69 deletions(-) diff --git a/paddle/cuda/src/hl_cuda_cnn.cu b/paddle/cuda/src/hl_cuda_cnn.cu index 49c09334e086d..9eec44f77f27a 100644 --- a/paddle/cuda/src/hl_cuda_cnn.cu +++ b/paddle/cuda/src/hl_cuda_cnn.cu @@ -532,8 +532,7 @@ void hl_CMRNorm_backward(size_t frameCnt, const real* inV, CHECK_SYNC("hl_CMRNorm_backward"); } -__global__ void KeBilinearInterpFw(const size_t nthreads, - const real* in, +__global__ void KeBilinearInterpFw(const real* in, const size_t inImgH, const size_t inImgW, const size_t inputH, @@ -546,6 +545,7 @@ __global__ void KeBilinearInterpFw(const size_t nthreads, const size_t numChannels, const real ratioH, const real ratioW) { + int nthreads = outputH * outputW; int tid = blockIdx.x * blockDim.x + threadIdx.x; if (tid < nthreads) { int outIdH = tid / outputW; @@ -593,13 +593,12 @@ void hl_bilinear_forward(const real* inData, int blocks = (threadNum + 1024 - 1) / 1024; KeBilinearInterpFw<<< blocks, 1024, 0, STREAM_DEFAULT>>>( - threadNum, inData, inImgH, inImgW, inputH, inputW, outData, - outImgH, outImgW, outputH, outputW, numChannels, ratioH, ratioW); + inData, inImgH, inImgW, inputH, inputW, outData, outImgH, + outImgW, outputH, outputW, numChannels, ratioH, ratioW); CHECK_SYNC("hl_bilinear_forward failed"); } -__global__ void KeBilinearInterpBw(const size_t nthreads, - real* in, +__global__ void KeBilinearInterpBw(real* in, const size_t inImgH, const size_t inImgW, const size_t inputH, @@ -612,6 +611,7 @@ __global__ void KeBilinearInterpBw(const size_t nthreads, const size_t numChannels, const real ratioH, const real ratioW) { + int nthreads = outputH * outputW; int tid = blockIdx.x * blockDim.x + threadIdx.x; if (tid < nthreads) { int outIdH = tid / outputW; @@ -659,8 +659,8 @@ void hl_bilinear_backward(real* inGrad, int blocks = (threadNum + 1024 - 1) / 1024; KeBilinearInterpBw<<< blocks, 1024, 0, STREAM_DEFAULT>>>( - threadNum, inGrad, inImgH, inImgW, inputH, inputW, outGrad, - outImgH, outImgW, outputH, outputW, numChannels, ratioH, ratioW); + inGrad, inImgH, inImgW, inputH, inputW, outGrad, outImgH, + outImgW, outputH, outputW, numChannels, ratioH, ratioW); CHECK_SYNC("hl_bilinear_backward failed"); } diff --git a/paddle/gserver/layers/BilinearInterpLayer.cpp b/paddle/gserver/layers/BilinearInterpLayer.cpp index f37efc824a2ec..ac5f87be7af07 100644 --- a/paddle/gserver/layers/BilinearInterpLayer.cpp +++ b/paddle/gserver/layers/BilinearInterpLayer.cpp @@ -40,6 +40,11 @@ size_t BilinearInterpLayer::getSize() { CHECK(inImgH_ > 0 && inImgW_ > 0); CHECK(numChannels_); + ratioH_ = (outImgH_ > 1) ? + static_cast(inImgH_ - 1) / (outImgH_ - 1) : 0.f; + ratioW_ = (outImgW_ > 1) ? + static_cast(inImgW_ - 1) / (outImgW_ - 1) : 0.f; + getOutput().setFrameHeight(outImgH_); getOutput().setFrameWidth(outImgW_); return outImgH_ * outImgW_ * numChannels_; @@ -70,7 +75,7 @@ void BilinearInterpLayer::forward(PassType passType) { { REGISTER_TIMER_INFO("FwBilinearInterpTimer", getName().c_str()); outV->bilinearForward(*inV, inImgH_, inImgW_, outImgH_, outImgW_, - numChannels_); + numChannels_, ratioH_, ratioW_); } } @@ -83,7 +88,7 @@ void BilinearInterpLayer::backward(const UpdateCallback& callback) { REGISTER_TIMER_INFO("BwBilinearInterpTimer", getName().c_str()); if (inputG) { inputG->bilinearBackward(*outG, outImgH_, outImgW_, inImgH_, inImgW_, - numChannels_); + numChannels_, ratioH_, ratioW_); } } } diff --git a/paddle/gserver/layers/BilinearInterpLayer.h b/paddle/gserver/layers/BilinearInterpLayer.h index 33e0cb1220511..eba3c054fa8e7 100644 --- a/paddle/gserver/layers/BilinearInterpLayer.h +++ b/paddle/gserver/layers/BilinearInterpLayer.h @@ -29,6 +29,7 @@ class BilinearInterpLayer : public Layer { protected: size_t outImgH_, outImgW_; size_t inImgH_, inImgW_; + real ratioH_, ratioW_; size_t numChannels_; public: diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index c00190449984c..4d4e439dc6268 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -50,6 +50,16 @@ TEST(Layer, BilinearInterpLayer) { for (auto useGpu : {false, true}) { testLayerGrad(config, "bilinear_interp", 10, false, useGpu); } + + bilinear->set_img_size_x(32); + bilinear->set_img_size_y(32); + bilinear->set_out_size_x(32); + bilinear->set_out_size_y(32); + bilinear->set_num_channels(4); + + for (auto useGpu : {false, true}) { + testLayerGrad(config, "bilinear_interp", 10, false, useGpu); + } } TEST(Operator, dot_mul) { diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 283733fe845d4..9abcbba67ab7e 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -1227,7 +1227,9 @@ void GpuMatrix::bilinearForward(const Matrix& in, const size_t inImgW, const size_t outImgH, const size_t outImgW, - const size_t numChannels) { + const size_t numChannels, + const real ratioH, + const real ratioW) { CHECK(dynamic_cast(&in)); const size_t outputW = getWidth(); @@ -1238,11 +1240,6 @@ void GpuMatrix::bilinearForward(const Matrix& in, real* outData = getData(); const real* inData = in.getData(); - real ratioH = (outImgH > 1) ? - static_cast(inImgH - 1) / (outImgH - 1) : 0.f; - real ratioW = (outImgW > 1) ? - static_cast(inImgW - 1) / (outImgW - 1) : 0.f; - if (inImgH == outImgW && inImgW == outImgW) { this->copyFrom(in); } else { @@ -1258,7 +1255,9 @@ void GpuMatrix::bilinearBackward(const Matrix& out, const size_t outImgW, const size_t inImgH, const size_t inImgW, - const size_t numChannels) { + const size_t numChannels, + const real ratioH, + const real ratioW) { CHECK(dynamic_cast(&out)); const size_t inputW = getWidth(); @@ -1269,13 +1268,8 @@ void GpuMatrix::bilinearBackward(const Matrix& out, real* inGrad = getData(); const real* outGrad = out.getData(); - real ratioH = (outImgH > 1) ? - static_cast(inImgH - 1) / (outImgH - 1) : 0.f; - real ratioW = (outImgW > 1) ? - static_cast(inImgW - 1) / (outImgW - 1) : 0.f; - if (outImgH == inImgH && outImgW == inImgW) { - this->addBias(const_cast(out), 1.f); + this->add(const_cast(out)); } else { hl_bilinear_backward( inGrad, inImgH, inImgW, inputH, inputW, outGrad, @@ -3908,7 +3902,9 @@ void CpuMatrix::bilinearForward(const Matrix& in, const size_t inImgW, const size_t outImgH, const size_t outImgW, - const size_t numChannels) { + const size_t numChannels, + const real ratioH, + const real ratioW) { CHECK(dynamic_cast(&in)); size_t outputW = getWidth(); @@ -3920,11 +3916,6 @@ void CpuMatrix::bilinearForward(const Matrix& in, real* outData = getData(); const real* inData = in.getData(); - const real ratioH = (outImgH > 1) ? - static_cast(inImgH - 1) / (outImgH - 1) : 0.f; - const real ratioW = (outImgW > 1) ? - static_cast(inImgW - 1) / (outImgW - 1) : 0.f; - if (inImgH == outImgH && inImgW == outImgW) { this->copyFrom(in); } else { @@ -3932,21 +3923,23 @@ void CpuMatrix::bilinearForward(const Matrix& in, for (size_t i = 0; i < outImgH; ++i) { // loop for images size_t h = ratioH * i; size_t hid = (h < inImgH - 1) ? 1 : 0; - real hlambda = ratioH * i - h; + real h1lambda = ratioH * i - h; + real h2lambda = 1 - h1lambda; for (size_t j = 0; j < outImgW; ++j) { size_t w = ratioW * j; size_t wid = (w < inImgW - 1) ? 1 : 0; - real wlambda = ratioW * j - w; + real w1lambda = ratioW * j - w; + real w2lambda = 1 - w1lambda; // calculate four position for bilinear interpolation const real* inPos = &inData[k * inputW + h * inImgW + w]; real* outPos = &outData[k * outputW + i * outImgW + j]; for (size_t c = 0; c < numChannels; ++c) { // loop for channels // bilinear interpolation - outPos[0] = (1.f - hlambda) * - ((1.f - wlambda) * inPos[0] + wlambda * inPos[wid]) + - hlambda * ((1.f - wlambda) * inPos[hid * inImgW] + - wlambda * inPos[hid * inImgW + wid]); + outPos[0] = + h2lambda * (w2lambda * inPos[0] + w1lambda * inPos[wid]) + + h1lambda * (w2lambda * inPos[hid * inImgW] + + w1lambda * inPos[hid * inImgW + wid]); inPos += inImgH * inImgW; outPos += outImgH * outImgW; } @@ -3961,7 +3954,9 @@ void CpuMatrix::bilinearBackward(const Matrix& out, const size_t outImgW, const size_t inImgH, const size_t inImgW, - const size_t numChannels) { + const size_t numChannels, + const real ratioH, + const real ratioW) { CHECK(dynamic_cast(&out)); size_t inputW = getWidth(); @@ -3973,32 +3968,28 @@ void CpuMatrix::bilinearBackward(const Matrix& out, real* inGrad = getData(); const real* outGrad = out.getData(); - const real ratioH = (outImgH > 1) ? - static_cast(inImgH - 1) / (outImgH - 1) : 0.f; - const real ratioW = (outImgW > 1) ? - static_cast(inImgW - 1) / (outImgW - 1) : 0.f; - if (inImgH == outImgH && inImgW == outImgW) { - this->addBias(const_cast(out), 1.f); + this->add(const_cast(out)); } else { for (size_t k = 0; k < batchSize; ++k) { // loop for batches for (size_t i = 0; i < outImgH; ++i) { // loop for images size_t h = ratioH * i; size_t hid = (h < inImgH - 1) ? 1 : 0; - real hlambda = ratioH * i - h; - + real h1lambda = ratioH * i - h; + real h2lambda = 1 - h1lambda; for (size_t j = 0; j < outImgW; ++j) { size_t w = ratioW * j; size_t wid = (w < inImgW - 1) ? 1 : 0; - real wlambda = ratioW * j - w; + real w1lambda = ratioW * j - w; + real w2lambda = 1 - w1lambda; real* inPos = &inGrad[k * inputW + h * inImgW + w]; const real* outPos = &outGrad[k * outputW + i * outImgW + j]; for (size_t c = 0; c < numChannels; ++c) { // loop for channels - inPos[0] += (1.f - hlambda) * (1.f - wlambda) * outPos[0]; - inPos[wid] += (1.f - hlambda) * wlambda * outPos[0]; - inPos[hid * inImgW] += hlambda * (1.f - wlambda) * outPos[0]; - inPos[hid * inImgW + wid] += hlambda * wlambda * outPos[0]; + inPos[0] += h2lambda * w2lambda * outPos[0]; + inPos[wid] += h2lambda * w1lambda * outPos[0]; + inPos[hid * inImgW] += h1lambda * w2lambda * outPos[0]; + inPos[hid * inImgW + wid] += h1lambda * w1lambda * outPos[0]; inPos += inImgH * inImgW; outPos += outImgH * outImgW; } diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index 25748a15696e1..07a2aebf556ef 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -997,7 +997,9 @@ class Matrix : public BaseMatrix { const size_t inImgW, const size_t outImgH, const size_t outImgW, - const size_t numChannels) { + const size_t numChannels, + const real ratioH, + const real ratioW) { LOG(FATAL) << "Not implemented"; } virtual void bilinearBackward(const Matrix& out, @@ -1005,7 +1007,9 @@ class Matrix : public BaseMatrix { const size_t outImgW, const size_t inImgH, const size_t inImgW, - const size_t numChannels) { + const size_t numChannels, + const real ratioH, + const real ratioW) { LOG(FATAL) << "Not implemented"; } }; @@ -1283,14 +1287,18 @@ class GpuMatrix : public Matrix { const size_t inImgW, const size_t outImgH, const size_t outImgW, - const size_t numChannels); + const size_t numChannels, + const real ratioH, + const real ratioW); void bilinearBackward(const Matrix& out, const size_t outImgH, const size_t outImgW, const size_t inImgH, const size_t inImgW, - const size_t numChannels); + const size_t numChannels, + const real ratioH, + const real ratioW); }; class CpuMatrix : public Matrix { @@ -1583,14 +1591,18 @@ class CpuMatrix : public Matrix { const size_t inImgW, const size_t outImgH, const size_t outImgW, - const size_t numChannels); + const size_t numChannels, + const real ratioH, + const real ratioW); void bilinearBackward(const Matrix& out, const size_t outImgH, const size_t outImgW, const size_t inImgH, const size_t inImgW, - const size_t numChannels); + const size_t numChannels, + const real ratioH, + const real ratioW); }; class SharedCpuMatrix : public CpuMatrix { diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index ef22e2aa8dd17..017fddc799591 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -94,7 +94,8 @@ void testBilinearFwdBwd(int numSamples, int imgSizeH, int imgSizeW, int channels) { int inWidth = imgSizeH * imgSizeW * channels; int outWidth = 2 * imgSizeH * 2 * imgSizeW * channels; - + real ratioH = 0.5; + real ratioW = 0.5; // forward MatrixPtr input = CpuMatrix::create(numSamples, inWidth, false, false); MatrixPtr inputGpu = GpuMatrix::create(numSamples, inWidth, false, true); @@ -107,9 +108,9 @@ void testBilinearFwdBwd(int numSamples, int imgSizeH, int imgSizeW, inputGpu->copyFrom(*input); target->bilinearForward(*input, imgSizeH, imgSizeW, - 2 * imgSizeH, 2 * imgSizeW, channels); + 2 * imgSizeH, 2 * imgSizeW, channels, ratioH, ratioW); targetGpu->bilinearForward(*inputGpu, imgSizeH, imgSizeW, - 2 * imgSizeH, 2 * imgSizeW, channels); + 2 * imgSizeH, 2 * imgSizeW, channels, ratioH, ratioW); // check targetCheck->copyFrom(*targetGpu); @@ -131,9 +132,9 @@ void testBilinearFwdBwd(int numSamples, int imgSizeH, int imgSizeW, targetGpuGrad->copyFrom(*targetGrad); inputGrad->bilinearBackward(*targetGrad, 2 * imgSizeH, 2 * imgSizeW, - imgSizeH, imgSizeW, channels); + imgSizeH, imgSizeW, channels, ratioH, ratioW); inputGpuGrad->bilinearBackward(*targetGpuGrad, 2 * imgSizeH, 2 * imgSizeW, - imgSizeH, imgSizeW, channels); + imgSizeH, imgSizeW, channels, ratioH, ratioW); // check targetCheckGrad->copyFrom(*inputGpuGrad); diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 038f4d32a588e..ccfdb3ded3ba1 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1272,19 +1272,17 @@ def bilinear_interp_layer(input, .. code-block:: python - bilinear = bilinear_interp_layer(input, - out_size_x, - out_size_y) + bilinear = bilinear_interp_layer(input=layer1, out_size_x=64, out_size_y=64) - :para input: A input layer. + :param input: A input layer. :type input: LayerOutput. - :para out_size_x: bilinear interpolation output width. + :param out_size_x: bilinear interpolation output width. :type out_size_x: int|None - :para out_size_y: bilinear interpolation output height. + :param out_size_y: bilinear interpolation output height. :type out_size_y: int|None - :para name: The layer's name, which cna not be specified. + :param name: The layer's name, which cna not be specified. :type name: None|basestring - :para layer_attr: Extra Layer attribute. + :param layer_attr: Extra Layer attribute. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. :rtype: LayerOutput @@ -1301,7 +1299,8 @@ def bilinear_interp_layer(input, num_channels=num_channels)), type=LayerType.BILINEAR_INTERP_LAYER, **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.BILINEAR_INTERP_LAYER, parents=[input]) + return LayerOutput(name, LayerType.BILINEAR_INTERP_LAYER, parents=[input], + num_filters=num_channels) @wrap_name_default() @layer_support() From 65af9f949532347c3ed47ba1fe47604d9cf3e7b7 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Tue, 8 Nov 2016 17:39:27 +0800 Subject: [PATCH 244/324] modify the format of diff information in protostr (#398) --- python/paddle/trainer_config_helpers/tests/configs/run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh b/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh index f05fc46cd5520..9683288358426 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh @@ -13,5 +13,5 @@ for file in $files do base_protostr=$protostr/$file new_protostr=$protostr/$file.unitest - diff $base_protostr $new_protostr + diff $base_protostr $new_protostr -u done From 7f6e9acab05ff407371cec41661de84afd8d0f1d Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 8 Nov 2016 17:41:58 +0800 Subject: [PATCH 245/324] Fix minior bug * add patch does not trigger travis ci --- .travis.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74aa767febeb1..ffe3bc193b49e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,13 +42,11 @@ addons: before_install: - | if [ ${JOB} == "BUILD_AND_TEST" ]; then - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - TRAVIS_COMMIT_RANGE="FETCH_HEAD...$TRAVIS_BRANCH" - fi - git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.md$)' || { + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.md$)' + then echo "Only markdown docs were updated, stopping build process." exit - } + fi fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi From e2c071358914fdead0a86348fd91c8227e78f280 Mon Sep 17 00:00:00 2001 From: qijun Date: Tue, 8 Nov 2016 10:12:04 +0000 Subject: [PATCH 246/324] follow comments --- paddle/cuda/include/hl_cnn.h | 8 +- paddle/gserver/layers/PoolProjection.cpp | 22 ++-- paddle/gserver/layers/PoolProjection.h | 1 + .../layers/SpatialPyramidPoolLayer.cpp | 14 ++- .../paddle/trainer_config_helpers/layers.py | 112 +++++++++--------- .../tests/configs/generate_protostr.sh | 2 - 6 files changed, 81 insertions(+), 78 deletions(-) diff --git a/paddle/cuda/include/hl_cnn.h b/paddle/cuda/include/hl_cnn.h index 4bd9d5e7c9e90..de6c2fb8f281a 100644 --- a/paddle/cuda/include/hl_cnn.h +++ b/paddle/cuda/include/hl_cnn.h @@ -91,7 +91,7 @@ extern void hl_expand_feature2col( * @param[in] paddingH padding height. * @param[in] paddingW padding width. * @param[out] tgtData output data. - * @param[in] tgtStride output data stride. + * @param[in] tgtStride stride between output data samples. * */ extern void hl_maxpool_forward( @@ -125,7 +125,7 @@ extern void hl_maxpool_forward( * @param[in] paddingH padding height. * @param[in] paddingW padding width. * @param[out] targetGrad output grad. - * @param[in] outStride output grad data stride. + * @param[in] outStride stride between output data samples. * */ extern void hl_maxpool_backward( @@ -157,7 +157,7 @@ extern void hl_maxpool_backward( * @param[in] paddingH padding height. * @param[in] paddingW padding width. * @param[out] tgtData output data. - * @param[in] tgtStride output data stride. + * @param[in] tgtStride stride between output data samples. * */ extern void hl_avgpool_forward( @@ -189,7 +189,7 @@ extern void hl_avgpool_forward( * @param[in] scaleA scale. * @param[in] scaleB scale. * @param[out] backGrad output grad. - * @param[in] outStride output grad data stride. + * @param[in] outStride stride between output data samples. * */ extern void hl_avgpool_backward( diff --git a/paddle/gserver/layers/PoolProjection.cpp b/paddle/gserver/layers/PoolProjection.cpp index e10788e926470..8c7d027c07250 100644 --- a/paddle/gserver/layers/PoolProjection.cpp +++ b/paddle/gserver/layers/PoolProjection.cpp @@ -34,9 +34,9 @@ PoolProjection* PoolProjection::create(const ProjectionConfig& config, void MaxPoolProjection::forward() { MatrixPtr inputV = in_->value; MatrixPtr outV = out_->value; - outV->maxPoolForward(*inputV, imgSizeY_, imgSize_, channels_, - sizeX_, sizeY_, strideY_, stride_, - outputY_, outputX_, confPaddingY_, confPadding_); + outV->maxPoolForward(*inputV, imgSizeY_, imgSize_, channels_, sizeX_, sizeY_, + strideY_, stride_, outputY_, outputX_, confPaddingY_, + confPadding_); } void MaxPoolProjection::backward(const UpdateCallback& callback) { @@ -50,17 +50,16 @@ void MaxPoolProjection::backward(const UpdateCallback& callback) { return; } inputGrad->maxPoolBackward(*inputV, imgSizeY_, imgSize_, *outGrad, *outV, - sizeX_, sizeY_, - strideY_, stride_, outputY_, outputX_, 1, 1, - confPaddingY_, confPadding_); + sizeX_, sizeY_, strideY_, stride_, outputY_, + outputX_, 1, 1, confPaddingY_, confPadding_); } void AvgPoolProjection::forward() { MatrixPtr inputV = in_->value; MatrixPtr outV = out_->value; - outV->avgPoolForward(*inputV, imgSizeY_, imgSize_, channels_, - sizeX_, sizeY_, strideY_, stride_, - outputY_, outputX_, confPaddingY_, confPadding_); + outV->avgPoolForward(*inputV, imgSizeY_, imgSize_, channels_, sizeX_, sizeY_, + strideY_, stride_, outputY_, outputX_, confPaddingY_, + confPadding_); } void AvgPoolProjection::backward(const UpdateCallback& callback) { @@ -73,9 +72,8 @@ void AvgPoolProjection::backward(const UpdateCallback& callback) { return; } - inputGrad->avgPoolBackward(*outputGrad, imgSizeY_, imgSize_, - sizeX_, sizeY_, strideY_, stride_, - outputY_, outputX_, 1, 1, + inputGrad->avgPoolBackward(*outputGrad, imgSizeY_, imgSize_, sizeX_, sizeY_, + strideY_, stride_, outputY_, outputX_, 1, 1, confPaddingY_, confPadding_); } } // namespace paddle diff --git a/paddle/gserver/layers/PoolProjection.h b/paddle/gserver/layers/PoolProjection.h index 73d8a41aefabe..9fa16c1ea64c5 100644 --- a/paddle/gserver/layers/PoolProjection.h +++ b/paddle/gserver/layers/PoolProjection.h @@ -15,6 +15,7 @@ limitations under the License. */ #pragma once #include "Projection.h" +#include "paddle/math/MathUtils.h" namespace paddle { diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp index 7ec761364043b..846e2e0666030 100644 --- a/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp @@ -56,8 +56,15 @@ ProjectionConfig SpatialPyramidPoolLayer::getConfig(size_t imgSizeW, size_t SpatialPyramidPoolLayer::getSize() { CHECK_EQ(inputLayers_.size(), 1UL); size_t layerSize = 0; + const SppConfig& sppConf = config_.inputs(0).spp_conf(); imgSizeH_ = inputLayers_[0]->getOutput().getFrameHeight(); imgSizeW_ = inputLayers_[0]->getOutput().getFrameWidth(); + if (imgSizeH_ == 0) { + imgSizeH_ = sppConf.has_img_size_y() ? sppConf.img_size_y() : imgSizeW_; + } + if (imgSizeW_ == 0) { + imgSizeW_ = sppConf.img_size(); + } size_t outputH = 1; size_t outputW = (std::pow(4, pyramidHeight_) - 1) / (4 - 1); @@ -66,10 +73,10 @@ size_t SpatialPyramidPoolLayer::getSize() { getOutput().setFrameHeight(outputH); getOutput().setFrameWidth(outputW); + return layerSize; } - bool SpatialPyramidPoolLayer::init(const LayerMap& layerMap, const ParameterMap& parameterMap) { Layer::init(layerMap, parameterMap); @@ -90,8 +97,8 @@ bool SpatialPyramidPoolLayer::init(const LayerMap& layerMap, size_t endCol = 0; for (size_t i = 0; i < pyramidHeight_; i++) { poolProjections_.emplace_back(PoolProjection::create( - getConfig(imgSizeW_, imgSizeH_, channels_, i, poolType_), - nullptr, useGpu_)); + getConfig(imgSizeW_, imgSizeH_, channels_, i, poolType_), nullptr, + useGpu_)); endCol += poolProjections_[i]->getOutputSize(); projCol_.push_back(std::make_pair(startCol, endCol)); startCol = endCol; @@ -125,4 +132,3 @@ void SpatialPyramidPoolLayer::backward(const UpdateCallback& callback) { } } // namespace paddle - diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index e6338e804536a..1459c9a84a56f 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -56,7 +56,7 @@ 'rank_cost', 'lambda_cost', 'huber_cost', 'block_expand_layer', 'maxout_layer', 'out_prod_layer', 'print_layer', - # 'spp_layer', + 'spp_layer', ] @@ -112,7 +112,7 @@ class LayerType(object): LINEAR_COMBINATION_LAYER = "convex_comb" BLOCK_EXPAND = "blockexpand" MAXOUT = "maxout" - # SPP_LAYER = "spp" + SPP_LAYER = "spp" PRINT_LAYER = "print" @@ -1711,60 +1711,60 @@ def img_pool_layer(input, pool_size, name=None, num_filters=num_channels) -# @wrap_name_default("spp") -# @layer_support() -# def spp_layer(input, name=None, num_channels=None, pool_type=None, -# pyramid_height=None, img_width=None, layer_attr=None): -# pass -# """ -# Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition. -# The details please refer to -# `Kaiming He's paper `_. - -# :param name: layer name. -# :type name: basestring -# :param input: layer's input. -# :type input: LayerOutput -# :param num_channels: number of input channel. -# :type num_channels: int -# :param pool_type: Pooling type. MaxPooling or AveragePooling. Default is MaxPooling. -# :type scale: BasePoolingType -# :param pyramid_height: pyramid height. -# :type pyramid_height: int -# :param img_width: the width of input feature map. If it is None, the input feature -# map should be square. -# :type img_width: int|None -# :param layer_attr: Extra Layer Attribute. -# :type layer_attr: ExtraLayerAttribute -# :return: LayerOutput object. -# :rtype: LayerOutput -# """ -# if num_channels is None: -# assert input.num_filters is not None -# num_channels = input.num_filters - -# if pool_type is None: -# pool_type = MaxPooling() -# elif isinstance(pool_type, AvgPooling): -# pool_type.name = 'avg' - -# type_name = pool_type.name -# if (isinstance(pool_type, AvgPooling) or isinstance(pool_type, MaxPooling)): -# type_name += '-projection' - -# Layer( -# name=name, -# type=LayerType.SPP_LAYER, -# inputs=Input(input.name, -# spp=SpatialPyramidPool(pool_type=type_name, -# channels=num_channels, -# pyramid_height=pyramid_height, -# img_width=img_width) -# ), -# **ExtraLayerAttribute.to_kwargs(layer_attr) -# ) -# return LayerOutput(name, LayerType.SPP_LAYER, parents=[input], -# num_filters=num_channels) +@wrap_name_default("spp") +@layer_support() +def spp_layer(input, name=None, num_channels=None, pool_type=None, + pyramid_height=None, img_width=None, layer_attr=None): + pass + """ + Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition. + The details please refer to + `Kaiming He's paper `_. + + :param name: layer name. + :type name: basestring + :param input: layer's input. + :type input: LayerOutput + :param num_channels: number of input channel. + :type num_channels: int + :param pool_type: Pooling type. MaxPooling or AveragePooling. Default is MaxPooling. + :type scale: BasePoolingType + :param pyramid_height: pyramid height. + :type pyramid_height: int + :param img_width: the width of input feature map. If it is None, the input feature + map should be square. + :type img_width: int|None + :param layer_attr: Extra Layer Attribute. + :type layer_attr: ExtraLayerAttribute + :return: LayerOutput object. + :rtype: LayerOutput + """ + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + + if pool_type is None: + pool_type = MaxPooling() + elif isinstance(pool_type, AvgPooling): + pool_type.name = 'avg' + + type_name = pool_type.name + if (isinstance(pool_type, AvgPooling) or isinstance(pool_type, MaxPooling)): + type_name += '-projection' + + Layer( + name=name, + type=LayerType.SPP_LAYER, + inputs=Input(input.name, + spp=SpatialPyramidPool(pool_type=type_name, + channels=num_channels, + pyramid_height=pyramid_height, + img_width=img_width) + ), + **ExtraLayerAttribute.to_kwargs(layer_attr) + ) + return LayerOutput(name, LayerType.SPP_LAYER, parents=[input], + num_filters=num_channels) def __img_norm_layer__(name, input, size, norm_type, scale, power, diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index 7440181970af1..e1686742b778f 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -11,11 +11,9 @@ test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers util_layers simple_rnn_layers unused_layers test_cost_layers test_rnn_group shared_fc shared_lstm test_cost_layers_with_weight -# test_maxout test_bi_grumemory math_ops test_spp_layer) test_maxout test_bi_grumemory math_ops test_spp_layer) - for conf in ${configs[*]} do echo "Generating " $conf From 0c7ac3d9c2062572967f3a34bee68aba9fc82409 Mon Sep 17 00:00:00 2001 From: gangliao Date: Tue, 8 Nov 2016 03:00:42 -0800 Subject: [PATCH 247/324] Fix Travis Ci does not build when push patches (#399) --- .travis.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74aa767febeb1..ffe3bc193b49e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,13 +42,11 @@ addons: before_install: - | if [ ${JOB} == "BUILD_AND_TEST" ]; then - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then - TRAVIS_COMMIT_RANGE="FETCH_HEAD...$TRAVIS_BRANCH" - fi - git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.md$)' || { + if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.md$)' + then echo "Only markdown docs were updated, stopping build process." exit - } + fi fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi From 70e04683dddd4ae47f1947f95008413ce1e69fce Mon Sep 17 00:00:00 2001 From: qijun Date: Tue, 8 Nov 2016 13:26:41 +0000 Subject: [PATCH 248/324] add getSize method for PoolProjection --- paddle/gserver/layers/PoolProjection.cpp | 4 ++++ paddle/gserver/layers/PoolProjection.h | 20 +++++++++++++++++++ paddle/gserver/layers/PoolProjectionLayer.cpp | 2 -- .../layers/SpatialPyramidPoolLayer.cpp | 4 ---- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/paddle/gserver/layers/PoolProjection.cpp b/paddle/gserver/layers/PoolProjection.cpp index 8c7d027c07250..a4fb001ffa0e8 100644 --- a/paddle/gserver/layers/PoolProjection.cpp +++ b/paddle/gserver/layers/PoolProjection.cpp @@ -32,6 +32,8 @@ PoolProjection* PoolProjection::create(const ProjectionConfig& config, } void MaxPoolProjection::forward() { + size_t width = getSize(); + CHECK_EQ(width, out_->value->getWidth()); MatrixPtr inputV = in_->value; MatrixPtr outV = out_->value; outV->maxPoolForward(*inputV, imgSizeY_, imgSize_, channels_, sizeX_, sizeY_, @@ -55,6 +57,8 @@ void MaxPoolProjection::backward(const UpdateCallback& callback) { } void AvgPoolProjection::forward() { + size_t width = getSize(); + CHECK_EQ(width, out_->value->getWidth()); MatrixPtr inputV = in_->value; MatrixPtr outV = out_->value; outV->avgPoolForward(*inputV, imgSizeY_, imgSize_, channels_, sizeX_, sizeY_, diff --git a/paddle/gserver/layers/PoolProjection.h b/paddle/gserver/layers/PoolProjection.h index 9fa16c1ea64c5..04d592f51fb59 100644 --- a/paddle/gserver/layers/PoolProjection.h +++ b/paddle/gserver/layers/PoolProjection.h @@ -51,6 +51,26 @@ class PoolProjection : public Projection { static PoolProjection* create(const ProjectionConfig& config, ParameterPtr parameter, bool useGpu); const std::string& getPoolType() const { return poolType_; } + size_t getSize() { + imgSizeY_ = in_->getFrameHeight(); + imgSize_ = in_->getFrameWidth(); + const PoolConfig& conf = config_.pool_conf(); + if (imgSizeY_ == 0) { + imgSizeY_ = conf.has_img_size_y() ? conf.img_size_y() : conf.img_size(); + } + if (imgSize_ == 0) { + imgSize_ = conf.img_size(); + } + outputY_ = outputSize(imgSizeY_, sizeY_, confPaddingY_, strideY_, + /* caffeMode */ false); + outputX_ = outputSize(imgSize_, sizeX_, confPadding_, stride_, + /* caffeMode */ false); + + const_cast(out_)->setFrameHeight(outputY_); + const_cast(out_)->setFrameWidth(outputX_); + + return outputY_ * outputX_ * channels_; + } }; class MaxPoolProjection : public PoolProjection { diff --git a/paddle/gserver/layers/PoolProjectionLayer.cpp b/paddle/gserver/layers/PoolProjectionLayer.cpp index fbef55e4d49a0..cabb346d6c991 100644 --- a/paddle/gserver/layers/PoolProjectionLayer.cpp +++ b/paddle/gserver/layers/PoolProjectionLayer.cpp @@ -38,8 +38,6 @@ size_t PoolProjectionLayer::getSize() { layerSize = outputH_ * outputW_ * channels_; - getOutput().setFrameHeight(outputH_); - getOutput().setFrameWidth(outputW_); return layerSize; } diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp index 846e2e0666030..582abf78c84a4 100644 --- a/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.cpp @@ -70,10 +70,6 @@ size_t SpatialPyramidPoolLayer::getSize() { size_t outputW = (std::pow(4, pyramidHeight_) - 1) / (4 - 1); layerSize = outputH * outputW * channels_; - - getOutput().setFrameHeight(outputH); - getOutput().setFrameWidth(outputW); - return layerSize; } From 8c40bfd0fc37fe193f5d1c57a55a389f3871cfa6 Mon Sep 17 00:00:00 2001 From: Liang Zhao Date: Mon, 7 Nov 2016 14:26:28 -0800 Subject: [PATCH 249/324] Make matrix well-conditioned when unittest inverse --- paddle/math/Matrix.cpp | 21 +++++++++++++++++++++ paddle/math/Matrix.h | 6 ++++++ paddle/math/tests/test_matrixCompare.cpp | 13 ++++++++----- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 4fc9b2d089366..f0f5ebe3bd05c 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -202,6 +202,17 @@ void GpuMatrix::resetOne() { CHECK(data_ != NULL); one(); } + +void GpuMatrix::setDiag(real value) { + CHECK(data_ != NULL); + CHECK_EQ(height_, width_); + + zeroMem(); + for (size_t i = 0; i < height_; i++) { + hl_memcpy_host2device(&data_[i * stride_ + i], &value, sizeof(real)); + } +} + void GpuMatrix::resize(size_t newHeight, size_t newWidth) { size_t newSize = newHeight * newWidth; if (NULL == memoryHandle_.get() || @@ -1244,6 +1255,16 @@ void CpuMatrix::resetOne() { BaseMatrix::one(); } +void CpuMatrix::setDiag(real value) { + CHECK(data_ != NULL); + CHECK_EQ(height_, width_); + + zeroMem(); + for (size_t i = 0; i < height_; i++) { + data_[i * stride_ + i] = value; + } +} + void CpuMatrix::copyFrom(const Matrix& src) { CHECK(isContiguous()); if (typeid(src) == typeid(GpuMatrix)) { diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index 293d13f4d6d5a..9e15055c056a1 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -195,6 +195,8 @@ class Matrix : public BaseMatrix { virtual void resetOne() { LOG(FATAL) << "Not implemented"; } + virtual void setDiag(real value) { LOG(FATAL) << "Not implemented"; } + virtual void copyFrom(const Matrix& src) { LOG(FATAL) << "Not implemented"; } virtual void trimFrom(const CpuSparseMatrix& src) { @@ -330,6 +332,7 @@ class Matrix : public BaseMatrix { virtual MatrixPtr getInverse() { LOG(FATAL) << "Not implemented"; + return nullptr; } /** @@ -1016,6 +1019,7 @@ class GpuMatrix : public Matrix { void zeroMem(); void resetOne(); + void setDiag(real value); void resize(size_t newHeight, size_t newWidth); void resize(size_t newHeight, size_t newWidth, @@ -1280,6 +1284,8 @@ class CpuMatrix : public Matrix { void zeroMem(); void resetOne(); + void setDiag(real value); + void resize(size_t newHeight, size_t newWidth); void resize(size_t newHeight, size_t newWidth, size_t newNnz, /* used to allocate space */ diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index b887cccaaa14e..91a68006325f6 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -647,20 +647,23 @@ void testMatrixInverse(int height) { MatrixPtr cpuI = std::make_shared(height, height); MatrixPtr gpuI = std::make_shared(height, height); + /* Make matrix well conditioned: cpu * cpuT + Identity */ cpu->randomizeUniform(); + MatrixPtr cpuT = cpu->getTranspose(); + MatrixPtr outputCheck = std::make_shared(height, height); + outputCheck->mul(cpu, cpuT); + cpu->setDiag(1.0); + cpu->add(*outputCheck); + gpu->copyFrom(*cpu); cpu->inverse(cpuI, false); gpu->inverse(gpuI, false); - MatrixPtr outputCheck = std::make_shared(height, height); outputCheck->copyFrom(*gpuI); MatrixCheckErr(*cpuI, *outputCheck); outputCheck->mul(cpu, cpuI); - cpu->zeroMem(); - for (int i = 0; i < height; i++) { - cpu->getRowBuf(i)[i] = 1.0; - } + cpu->setDiag(1.0); MatrixCheckErr(*cpu, *outputCheck); } From 992ac8f9a1c54080bc273f7748510e2b85c7f8cc Mon Sep 17 00:00:00 2001 From: Liang Zhao Date: Tue, 8 Nov 2016 10:36:22 -0800 Subject: [PATCH 250/324] Implement setDiag() with BaseMatrix::assign() --- paddle/math/Matrix.cpp | 29 +++++++++-------------------- paddle/math/Matrix.h | 2 +- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index f0f5ebe3bd05c..a5b0d959536fa 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -187,6 +187,15 @@ MatrixPtr Matrix::subMatrix(size_t startRow, size_t endRow, size_t startCol, trans_, useGpu_); } +void Matrix::setDiag(real value) { + CHECK(data_ != NULL); + CHECK_EQ(height_, width_); + + zeroMem(); + BaseMatrix diag(height_, 1, stride_ + 1, data_, false, useGpu_); + diag.assign(value); +} + GpuMatrix::GpuMatrix(size_t height, size_t width, bool trans) : Matrix(std::make_shared(height * width * sizeof(real)), height, width, trans, true) {} @@ -203,16 +212,6 @@ void GpuMatrix::resetOne() { one(); } -void GpuMatrix::setDiag(real value) { - CHECK(data_ != NULL); - CHECK_EQ(height_, width_); - - zeroMem(); - for (size_t i = 0; i < height_; i++) { - hl_memcpy_host2device(&data_[i * stride_ + i], &value, sizeof(real)); - } -} - void GpuMatrix::resize(size_t newHeight, size_t newWidth) { size_t newSize = newHeight * newWidth; if (NULL == memoryHandle_.get() || @@ -1255,16 +1254,6 @@ void CpuMatrix::resetOne() { BaseMatrix::one(); } -void CpuMatrix::setDiag(real value) { - CHECK(data_ != NULL); - CHECK_EQ(height_, width_); - - zeroMem(); - for (size_t i = 0; i < height_; i++) { - data_[i * stride_ + i] = value; - } -} - void CpuMatrix::copyFrom(const Matrix& src) { CHECK(isContiguous()); if (typeid(src) == typeid(GpuMatrix)) { diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index 9e15055c056a1..120957f45d0c9 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -195,7 +195,7 @@ class Matrix : public BaseMatrix { virtual void resetOne() { LOG(FATAL) << "Not implemented"; } - virtual void setDiag(real value) { LOG(FATAL) << "Not implemented"; } + void setDiag(real value); virtual void copyFrom(const Matrix& src) { LOG(FATAL) << "Not implemented"; } From bc2b521c241f903b56596b9a52d50c6f59bc9002 Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 9 Nov 2016 12:07:19 +0800 Subject: [PATCH 251/324] Follow comments --- paddle/gserver/tests/test_LayerGrad.cpp | 18 +++++------------- paddle/math/Matrix.cpp | 12 ++++++++---- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 0c18611f01090..d8c45040d86bc 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -43,22 +43,14 @@ TEST(Layer, BilinearInterpLayer) { bilinear->set_img_size_x(32); bilinear->set_img_size_y(32); - bilinear->set_out_size_x(64); - bilinear->set_out_size_y(64); bilinear->set_num_channels(4); for (auto useGpu : {false, true}) { - testLayerGrad(config, "bilinear_interp", 10, false, useGpu); - } - - bilinear->set_img_size_x(32); - bilinear->set_img_size_y(32); - bilinear->set_out_size_x(32); - bilinear->set_out_size_y(32); - bilinear->set_num_channels(4); - - for (auto useGpu : {false, true}) { - testLayerGrad(config, "bilinear_interp", 10, false, useGpu); + for (auto out_size : {32, 64, 128}) { + bilinear->set_out_size_x(out_size); + bilinear->set_out_size_y(out_size); + testLayerGrad(config, "bilinear_interp", 10, false, useGpu); + } } } diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 4770a7203498d..80bf74bd4c51a 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -3902,6 +3902,8 @@ void CpuMatrix::bilinearForward(const Matrix& in, size_t batchSize = getHeight(); size_t inputW = in.getWidth(); size_t inputH = in.getHeight(); + size_t inPosOffset = inImgH * inImgW; + size_t outPosOffset = outImgH * outImgW; (void)(inputH); real* outData = getData(); @@ -3931,8 +3933,8 @@ void CpuMatrix::bilinearForward(const Matrix& in, h2lambda * (w2lambda * inPos[0] + w1lambda * inPos[wid]) + h1lambda * (w2lambda * inPos[hid * inImgW] + w1lambda * inPos[hid * inImgW + wid]); - inPos += inImgH * inImgW; - outPos += outImgH * outImgW; + inPos += inPosOffset; + outPos += outPosOffset; } } } @@ -3954,6 +3956,8 @@ void CpuMatrix::bilinearBackward(const Matrix& out, size_t inputH = getHeight(); size_t outputW = out.getWidth(); size_t batchSize = out.getHeight(); + size_t inPosOffset = inImgH * inImgW; + size_t outPosOffset = outImgH * outImgW; (void)(inputH); real* inGrad = getData(); @@ -3981,8 +3985,8 @@ void CpuMatrix::bilinearBackward(const Matrix& out, inPos[wid] += h2lambda * w1lambda * outPos[0]; inPos[hid * inImgW] += h1lambda * w2lambda * outPos[0]; inPos[hid * inImgW + wid] += h1lambda * w1lambda * outPos[0]; - inPos += inImgH * inImgW; - outPos += outImgH * outImgW; + inPos += inPosOffset; + outPos += outPosOffset; } } } From eaf3dec9c57a9028843767c339009c0882ba38e8 Mon Sep 17 00:00:00 2001 From: qijun Date: Wed, 9 Nov 2016 05:48:35 +0000 Subject: [PATCH 252/324] follow comments --- paddle/gserver/layers/PoolProjection.cpp | 40 +++++++++++++++++++ paddle/gserver/layers/PoolProjection.h | 40 +++---------------- paddle/gserver/layers/PoolProjectionLayer.h | 18 ++++----- .../gserver/layers/SpatialPyramidPoolLayer.h | 11 ++++- .../configs/protostr/test_spp_layer.protostr | 34 ++++++++++++++++ .../tests/configs/test_spp_layer.py | 2 +- 6 files changed, 98 insertions(+), 47 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_spp_layer.protostr diff --git a/paddle/gserver/layers/PoolProjection.cpp b/paddle/gserver/layers/PoolProjection.cpp index a4fb001ffa0e8..9be5aba3d57d2 100644 --- a/paddle/gserver/layers/PoolProjection.cpp +++ b/paddle/gserver/layers/PoolProjection.cpp @@ -18,6 +18,46 @@ namespace paddle { REGISTER_PROJECTION_CREATE_FUNC(pool, &PoolProjection::create); +PoolProjection::PoolProjection(const ProjectionConfig& config, + ParameterPtr parameter, bool useGpu) + : Projection(config, parameter, useGpu) { + const PoolConfig& conf = config_.pool_conf(); + poolType_ = conf.pool_type(); + channels_ = conf.channels(); + sizeX_ = conf.size_x(); + stride_ = conf.stride(); + outputX_ = conf.output_x(); + imgSize_ = conf.img_size(); + confPadding_ = conf.padding(); + + sizeY_ = conf.has_size_y() ? conf.size_y() : conf.size_x(); + imgSizeY_ = conf.has_img_size_y() ? conf.img_size_y() : conf.img_size(); + strideY_ = conf.has_stride_y() ? conf.stride_y() : conf.stride(); + confPaddingY_ = conf.has_padding_y() ? conf.padding_y() : conf.padding(); + outputY_ = conf.has_output_y() ? conf.output_y() : conf.output_x(); +} + +size_t PoolProjection::getSize() { + imgSizeY_ = in_->getFrameHeight(); + imgSize_ = in_->getFrameWidth(); + const PoolConfig& conf = config_.pool_conf(); + if (imgSizeY_ == 0) { + imgSizeY_ = conf.has_img_size_y() ? conf.img_size_y() : conf.img_size(); + } + if (imgSize_ == 0) { + imgSize_ = conf.img_size(); + } + outputY_ = outputSize(imgSizeY_, sizeY_, confPaddingY_, strideY_, + /* caffeMode */ false); + outputX_ = outputSize(imgSize_, sizeX_, confPadding_, stride_, + /* caffeMode */ false); + + const_cast(out_)->setFrameHeight(outputY_); + const_cast(out_)->setFrameWidth(outputX_); + + return outputY_ * outputX_ * channels_; +} + PoolProjection* PoolProjection::create(const ProjectionConfig& config, ParameterPtr parameter, bool useGpu) { const std::string& pool = config.pool_conf().pool_type(); diff --git a/paddle/gserver/layers/PoolProjection.h b/paddle/gserver/layers/PoolProjection.h index 04d592f51fb59..a11e25b729cb7 100644 --- a/paddle/gserver/layers/PoolProjection.h +++ b/paddle/gserver/layers/PoolProjection.h @@ -31,46 +31,14 @@ class PoolProjection : public Projection { public: PoolProjection(const ProjectionConfig& config, ParameterPtr parameter, - bool useGpu) - : Projection(config, parameter, useGpu) { - const PoolConfig& conf = config_.pool_conf(); - poolType_ = conf.pool_type(); - channels_ = conf.channels(); - sizeX_ = conf.size_x(); - stride_ = conf.stride(); - outputX_ = conf.output_x(); - imgSize_ = conf.img_size(); - confPadding_ = conf.padding(); + bool useGpu); - sizeY_ = conf.has_size_y() ? conf.size_y() : conf.size_x(); - imgSizeY_ = conf.has_img_size_y() ? conf.img_size_y() : conf.img_size(); - strideY_ = conf.has_stride_y() ? conf.stride_y() : conf.stride(); - confPaddingY_ = conf.has_padding_y() ? conf.padding_y() : conf.padding(); - outputY_ = conf.has_output_y() ? conf.output_y() : conf.output_x(); - } static PoolProjection* create(const ProjectionConfig& config, ParameterPtr parameter, bool useGpu); - const std::string& getPoolType() const { return poolType_; } - size_t getSize() { - imgSizeY_ = in_->getFrameHeight(); - imgSize_ = in_->getFrameWidth(); - const PoolConfig& conf = config_.pool_conf(); - if (imgSizeY_ == 0) { - imgSizeY_ = conf.has_img_size_y() ? conf.img_size_y() : conf.img_size(); - } - if (imgSize_ == 0) { - imgSize_ = conf.img_size(); - } - outputY_ = outputSize(imgSizeY_, sizeY_, confPaddingY_, strideY_, - /* caffeMode */ false); - outputX_ = outputSize(imgSize_, sizeX_, confPadding_, stride_, - /* caffeMode */ false); - const_cast(out_)->setFrameHeight(outputY_); - const_cast(out_)->setFrameWidth(outputX_); + const std::string& getPoolType() const { return poolType_; } - return outputY_ * outputX_ * channels_; - } + size_t getSize(); }; class MaxPoolProjection : public PoolProjection { @@ -78,6 +46,7 @@ class MaxPoolProjection : public PoolProjection { MaxPoolProjection(const ProjectionConfig& config, ParameterPtr parameter, bool useGpu) : PoolProjection(config, parameter, useGpu) {} + virtual void forward(); virtual void backward(const UpdateCallback& callback = nullptr); }; @@ -87,6 +56,7 @@ class AvgPoolProjection : public PoolProjection { AvgPoolProjection(const ProjectionConfig& config, ParameterPtr parameter, bool useGpu) : PoolProjection(config, parameter, useGpu) {} + virtual void forward(); virtual void backward(const UpdateCallback& callback = nullptr); }; diff --git a/paddle/gserver/layers/PoolProjectionLayer.h b/paddle/gserver/layers/PoolProjectionLayer.h index 6e336f79e9043..777b6f39e7cc4 100644 --- a/paddle/gserver/layers/PoolProjectionLayer.h +++ b/paddle/gserver/layers/PoolProjectionLayer.h @@ -12,13 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once +#include #include "PoolLayer.h" #include "PoolProjection.h" #include "paddle/math/Matrix.h" -#include namespace paddle { /** @@ -32,15 +31,16 @@ class PoolProjectionLayer : public PoolLayer { ProjectionConfig projectionConfig_; public: - size_t getSize(); - virtual void forward(PassType passType); - virtual void backward(const UpdateCallback& callback = nullptr); - explicit PoolProjectionLayer(const LayerConfig& config) - : PoolLayer(config) { + explicit PoolProjectionLayer(const LayerConfig& config) : PoolLayer(config) { PoolConfig* conf = projectionConfig_.mutable_pool_conf(); *conf = config_.inputs(0).pool_conf(); - poolProjection_.reset(PoolProjection::create(projectionConfig_, nullptr, - useGpu_)); + poolProjection_.reset( + PoolProjection::create(projectionConfig_, nullptr, useGpu_)); } + + size_t getSize(); + + virtual void forward(PassType passType); + virtual void backward(const UpdateCallback& callback = nullptr); }; } // namespace paddle diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.h b/paddle/gserver/layers/SpatialPyramidPoolLayer.h index 64f3fda8a0adf..8416a717d654e 100644 --- a/paddle/gserver/layers/SpatialPyramidPoolLayer.h +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.h @@ -12,15 +12,19 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ - #pragma once #include "Layer.h" #include "PoolProjection.h" -#include "paddle/utils/Logging.h" #include "paddle/math/MathUtils.h" +#include "paddle/utils/Logging.h" namespace paddle { +/** + * @brief A layer for spatial pyramid pooling on the input image by taking + * the max, average, etc. within regions, so that the result vector of + * different sized images are of the same size. + */ class SpatialPyramidPoolLayer : public Layer { protected: @@ -36,12 +40,15 @@ class SpatialPyramidPoolLayer : public Layer { public: explicit SpatialPyramidPoolLayer(const LayerConfig& config) : Layer(config) {} + ~SpatialPyramidPoolLayer() {} virtual bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + ProjectionConfig getConfig(size_t sizeX_, size_t sizeY_, size_t channels, size_t pyamidLevel_, std::string& poolType_); size_t getSize(); + virtual void forward(PassType passType); virtual void backward(const UpdateCallback& callback = nullptr); }; diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_spp_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_spp_layer.protostr new file mode 100644 index 0000000000000..8b0a8f2146b70 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_spp_layer.protostr @@ -0,0 +1,34 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 3200 + active_type: "" +} +layers { + name: "__spp_0__" + type: "spp" + size: 80 + active_type: "" + inputs { + input_layer_name: "data" + spp_conf { + pool_type: "max-projection" + pyramid_height: 2 + channels: 16 + img_size: 10 + img_size_y: 20 + } + } +} +input_layer_names: "data" +output_layer_names: "__spp_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__spp_0__" + input_layer_names: "data" + output_layer_names: "__spp_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py index 2cbc76ce20b8a..178387d3cf1d1 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_spp_layer.py @@ -1,7 +1,7 @@ from paddle.trainer_config_helpers import * settings( - batch_size=100, + batch_size=100, learning_rate=1e-5 ) From 14ba68f963d2a87bcc78c06f1835b40ff7bf3135 Mon Sep 17 00:00:00 2001 From: gangliao Date: Wed, 9 Nov 2016 14:05:05 +0800 Subject: [PATCH 253/324] Update FindAVX.cmake (#404) * make AVX_FOUND is default value to WITH AVX * let AVX_FLAG always keep -mavx flag since compiler can build binary with -mavx even CPU does not support avx. --- CMakeLists.txt | 14 +++---- cmake/FindAVX.cmake | 93 +++++++++++++++++++++++++-------------------- 2 files changed, 58 insertions(+), 49 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 39f876bc9ee4b..d7e7e49e9a038 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,11 +109,9 @@ else() set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-g -O3 --use_fast_math") if(WITH_AVX) - if(AVX_FOUND) - set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler -mavx") - endif(AVX_FOUND) + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${AVX_FLAG}") else(WITH_AVX) - set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler -msse3") + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${SSE3_FLAG}") endif(WITH_AVX) if(WITH_DSO) @@ -138,11 +136,11 @@ if(NOT WITH_TIMER) endif(NOT WITH_TIMER) if(WITH_AVX) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${AVX_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${AVX_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${AVX_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${AVX_FLAG}") else(WITH_AVX) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -msse3") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse3") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SSE3_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SSE3_FLAG}") endif(WITH_AVX) if(WITH_PYTHON) diff --git a/cmake/FindAVX.cmake b/cmake/FindAVX.cmake index f6103c6e667e8..d380c996dfa95 100644 --- a/cmake/FindAVX.cmake +++ b/cmake/FindAVX.cmake @@ -3,36 +3,55 @@ INCLUDE(CheckCXXSourceRuns) -SET(FIND_AVX_10) -SET(FIND_AVX_20) -SET(AVX_FLAGS) -SET(AVX_FOUND) - -# Check AVX 2 -SET(CMAKE_REQUIRED_FLAGS) IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - SET(CMAKE_REQUIRED_FLAGS "-mavx2") -ELSEIF(MSVC AND NOT CMAKE_CL_64) # reserve for WINDOWS - SET(CMAKE_REQUIRED_FLAGS "/arch:AVX2") + set(MMX_FLAG "-mmmx") + set(SSE2_FLAG "-msse2") + set(SSE3_FLAG "-msse3") + SET(AVX_FLAG "-mavx") + SET(AVX2_FLAG "-mavx2") +ELSEIF(MSVC) + set(MMX_FLAG "/arch:MMX") + set(SSE2_FLAG "/arch:SSE2") + set(SSE3_FLAG "/arch:SSE3") + SET(AVX_FLAG "/arch:AVX") + SET(AVX2_FLAG "/arch:AVX2") ENDIF() +# Check MMX +set(CMAKE_REQUIRED_FLAGS ${MMX_FLAG}) CHECK_CXX_SOURCE_RUNS(" -#include +#include int main() { - __m256i a = _mm256_set_epi32 (-1, 2, -3, 4, -1, 2, -3, 4); - __m256i result = _mm256_abs_epi32 (a); + _mm_setzero_si64(); return 0; -}" FIND_AVX_20) +}" MMX_FOUND) -# Check AVX -SET(CMAKE_REQUIRED_FLAGS) -IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - SET(CMAKE_REQUIRED_FLAGS "-mavx") -ELSEIF(MSVC AND NOT CMAKE_CL_64) - SET(CMAKE_REQUIRED_FLAGS "/arch:AVX") -endif() +# Check SSE2 +set(CMAKE_REQUIRED_FLAGS ${SSE2_FLAG}) +CHECK_CXX_SOURCE_RUNS(" +#include +int main() +{ + _mm_setzero_si128(); + return 0; +}" SSE2_FOUND) +# Check SSE3 +set(CMAKE_REQUIRED_FLAGS ${SSE3_FLAG}) +CHECK_CXX_SOURCE_RUNS(" +#include +int main() +{ + __m128d a = _mm_set1_pd(6.28); + __m128d b = _mm_set1_pd(3.14); + __m128d result = _mm_addsub_pd(a, b); + result = _mm_movedup_pd(result); + return 0; +}" SSE3_FOUND) + +# Check AVX +set(CMAKE_REQUIRED_FLAGS ${AVX_FLAG}) CHECK_CXX_SOURCE_RUNS(" #include int main() @@ -41,25 +60,17 @@ int main() __m256 b = _mm256_set_ps (1.0f, 2.0f, 3.0f, 4.0f, 1.0f, 2.0f, 3.0f, 4.0f); __m256 result = _mm256_add_ps (a, b); return 0; -}" FIND_AVX_10) - -IF(${FIND_AVX_20}) - IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - SET(AVX_FLAGS "${AVX_FLAGS} -mavx2") - ELSEIF(MSVC) - SET(AVX_FLAGS "${AVX_FLAGS} /arch:AVX2") - ENDIF() -ENDIF() +}" AVX_FOUND) -IF(${FIND_AVX_10}) - IF(CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") - SET(AVX_FLAGS "${AVX_FLAGS} -mavx") - ELSEIF(MSVC) - SET(AVX_FLAGS "${AVX_FLAGS} /arch:AVX") - ENDIF() -ENDIF() +# Check AVX 2 +set(CMAKE_REQUIRED_FLAGS ${AVX2_FLAG}) +CHECK_CXX_SOURCE_RUNS(" +#include +int main() +{ + __m256i a = _mm256_set_epi32 (-1, 2, -3, 4, -1, 2, -3, 4); + __m256i result = _mm256_abs_epi32 (a); + return 0; +}" AVX2_FOUND) -IF(${FIND_AVX_10}) - SET(AVX_FOUND TRUE) - MESSAGE(STATUS "Find CPU supports ${AVX_FLAGS}.") -ENDIF() +mark_as_advanced(MMX_FOUND SSE2_FOUND SSE3_FOUND AVX_FOUND AVX2_FOUND) From e6c83f4ec058c65b3ded7605a1c85910caf7a0b0 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Wed, 9 Nov 2016 14:59:02 +0800 Subject: [PATCH 254/324] some tiny fixs (#406) * some tiny fixs * use VLOG(3) --- paddle/cuda/src/hl_dso_loader.cc | 2 +- python/paddle/trainer_config_helpers/tests/configs/.gitignore | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/cuda/src/hl_dso_loader.cc b/paddle/cuda/src/hl_dso_loader.cc index c0b5d6e357fc7..b564b96903368 100644 --- a/paddle/cuda/src/hl_dso_loader.cc +++ b/paddle/cuda/src/hl_dso_loader.cc @@ -48,7 +48,7 @@ static inline std::string join(const std::string& part1, const std::string& part static inline void GetDsoHandleFromDefaultPath( std::string& dso_path, void** dso_handle, int dynload_flags) { - LOG(INFO) << "Try to find cuda library: " << dso_path + VLOG(3) << "Try to find cuda library: " << dso_path << " from default system path."; // default search from LD_LIBRARY_PATH/DYLD_LIBRARY_PATH *dso_handle = dlopen(dso_path.c_str(), dynload_flags); diff --git a/python/paddle/trainer_config_helpers/tests/configs/.gitignore b/python/paddle/trainer_config_helpers/tests/configs/.gitignore index 52378fe7a4865..eb646b4a71ec1 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/.gitignore +++ b/python/paddle/trainer_config_helpers/tests/configs/.gitignore @@ -1 +1 @@ -*protostr +protostr/*.unitest From bd50f93e63df045130bc8d95017fc9e07601f415 Mon Sep 17 00:00:00 2001 From: wangkuiyi Date: Tue, 8 Nov 2016 23:55:30 -0800 Subject: [PATCH 255/324] [Work in Progress] Update cluster_train.md (#391) Update cluster_train.md for easier understanding --- doc/cluster/opensource/cluster_train.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/doc/cluster/opensource/cluster_train.md b/doc/cluster/opensource/cluster_train.md index 4763ede39b049..cb493a88f0318 100644 --- a/doc/cluster/opensource/cluster_train.md +++ b/doc/cluster/opensource/cluster_train.md @@ -1,26 +1,24 @@ -# Cluster Training +# Distributed Training -We provide some simple scripts ```paddle/scripts/cluster_train``` to help you to launch cluster training Job to harness PaddlePaddle's distributed trainning. For MPI and other cluster scheduler refer this naive script to implement more robust cluster training platform by yourself. +In this article, we explain how to run distributed Paddle training jobs on clusters. We will create the distributed version of the single-process training example, [recommendation](https://github.com/baidu/Paddle/tree/develop/demo/recommendation). -The following cluster demo is based on RECOMMENDATION local training demo in PaddlePaddle ```demo/recommendation``` directory. Assuming you enter the ```paddle/scripts/cluster_train/``` directory. +[Scripts](https://github.com/baidu/Paddle/tree/develop/paddle/scripts/cluster_train) used in this article launch distributed jobs via SSH. They also work as a reference for users running more sophisticated cluster management systems like MPI and Kubernetes. -## Pre-requirements +## Prerequisite -Firstly, +1. Aforementioned scripts use a Python library [fabric](http://www.fabfile.org/) to run SSH commands. We can use `pip` to install fabric: -```bash + ```bash pip install fabric -``` - -Secondly, go through installing scripts to install PaddlePaddle at all nodes to make sure demo can run as local mode. For CUDA enabled training, we assume that CUDA is installed in ```/usr/local/cuda```, otherwise missed cuda runtime libraries error could be reported at cluster runtime. In one word, the local training environment should be well prepared for the simple scripts. + ``` -Then you should prepare same ROOT_DIR directory in all nodes. ROOT_DIR is from in cluster_train/conf.py. Assuming that the ROOT_DIR = /home/paddle, you can create ```paddle``` user account as well, at last ```paddle.py``` can ssh connections to all nodes with ```paddle``` user automatically. +1. We need to install PaddlePaddle on all nodes in the cluster. To enable GPUs, we need to install CUDA in `/usr/local/cuda`; otherwise Paddle would report errors at runtime. -At last you can create ssh mutual trust relationship between all nodes for easy ssh login, otherwise ```password``` should be provided at runtime from ```paddle.py```. +1. Set the `ROOT_DIR` variable in [`cluster_train/conf.py`] on all nodes. For convenience, we often create a Unix user `paddle` on all nodes and set `ROOT_DIR=/home/paddle`. In this way, we can write public SSH keys into `/home/paddle/.ssh/authorized_keys` so that user `paddle` can SSH to all nodes without password. ## Prepare Job Workspace -```Job workspace``` is defined as one package directory which contains dependency libraries, train data, test data, model config file and all other related file dependencies. +We refer to the directory where we put dependent libraries, config files, etc., as *workspace*. These ```train/test``` data should be prepared before launching cluster job. To satisfy the requirement that train/test data are placed in different directory from workspace, PADDLE refers train/test data according to index file named as ```train.list/test.list``` which are used in model config file. So the train/test data also contains train.list/test.list two list file. All local training demo already provides scripts to help you create these two files, and all nodes in cluster job will handle files with same logical code in normal condition. From 05204af1f2f1cb549c40880893ca0e2eb47e588f Mon Sep 17 00:00:00 2001 From: qingqing01 Date: Wed, 9 Nov 2016 16:12:52 +0800 Subject: [PATCH 256/324] Fix memory leak in image classification demo, which is caused by dataprovider (#323) * the memory leak is inside one pass. --- demo/image_classification/.gitignore | 2 + .../data/download_cifar.sh | 0 demo/image_classification/image_provider.py | 39 +++++++++++-------- demo/image_classification/preprocess.py | 2 + demo/image_classification/preprocess.sh | 3 ++ demo/image_classification/vgg_16_cifar.py | 4 +- 6 files changed, 31 insertions(+), 19 deletions(-) mode change 100644 => 100755 demo/image_classification/data/download_cifar.sh diff --git a/demo/image_classification/.gitignore b/demo/image_classification/.gitignore index 76961dd1436f8..6a05b8f6632db 100644 --- a/demo/image_classification/.gitignore +++ b/demo/image_classification/.gitignore @@ -5,3 +5,5 @@ plot.png train.log image_provider_copy_1.py *pyc +train.list +test.list diff --git a/demo/image_classification/data/download_cifar.sh b/demo/image_classification/data/download_cifar.sh old mode 100644 new mode 100755 diff --git a/demo/image_classification/image_provider.py b/demo/image_classification/image_provider.py index 9e2f8b8949b39..305efbcdc6bb1 100644 --- a/demo/image_classification/image_provider.py +++ b/demo/image_classification/image_provider.py @@ -58,24 +58,29 @@ def hook(settings, img_size, mean_img_size, num_classes, color, meta, use_jpeg, settings.logger.info('DataProvider Initialization finished') -@provider(init_hook=hook) -def processData(settings, file_name): +@provider(init_hook=hook, min_pool_size=0) +def processData(settings, file_list): """ The main function for loading data. Load the batch, iterate all the images and labels in this batch. - file_name: the batch file name. + file_list: the batch file list. """ - data = cPickle.load(io.open(file_name, 'rb')) - indexes = list(range(len(data['images']))) - if settings.is_train: - random.shuffle(indexes) - for i in indexes: - if settings.use_jpeg == 1: - img = image_util.decode_jpeg(data['images'][i]) - else: - img = data['images'][i] - img_feat = image_util.preprocess_img(img, settings.img_mean, - settings.img_size, settings.is_train, - settings.color) - label = data['labels'][i] - yield img_feat.tolist(), int(label) + with open(file_list, 'r') as fdata: + lines = [line.strip() for line in fdata] + random.shuffle(lines) + for file_name in lines: + with io.open(file_name.strip(), 'rb') as file: + data = cPickle.load(file) + indexes = list(range(len(data['images']))) + if settings.is_train: + random.shuffle(indexes) + for i in indexes: + if settings.use_jpeg == 1: + img = image_util.decode_jpeg(data['images'][i]) + else: + img = data['images'][i] + img_feat = image_util.preprocess_img(img, settings.img_mean, + settings.img_size, settings.is_train, + settings.color) + label = data['labels'][i] + yield img_feat.astype('float32'), int(label) diff --git a/demo/image_classification/preprocess.py b/demo/image_classification/preprocess.py index 0286a5d7e9dc8..fe7ea19bf0277 100755 --- a/demo/image_classification/preprocess.py +++ b/demo/image_classification/preprocess.py @@ -35,6 +35,8 @@ def option_parser(): data_creator = ImageClassificationDatasetCreater(data_dir, processed_image_size, color) + data_creator.train_list_name = "train.txt" + data_creator.test_list_name = "test.txt" data_creator.num_per_batch = 1000 data_creator.overwrite = True data_creator.create_batches() diff --git a/demo/image_classification/preprocess.sh b/demo/image_classification/preprocess.sh index dfe3eb95d1ab8..e3e86ff10675c 100755 --- a/demo/image_classification/preprocess.sh +++ b/demo/image_classification/preprocess.sh @@ -17,3 +17,6 @@ set -e data_dir=./data/cifar-out python preprocess.py -i $data_dir -s 32 -c 1 + +echo "data/cifar-out/batches/train.txt" > train.list +echo "data/cifar-out/batches/test.txt" > test.list diff --git a/demo/image_classification/vgg_16_cifar.py b/demo/image_classification/vgg_16_cifar.py index e8b8af4bd313d..edd6988c48acd 100755 --- a/demo/image_classification/vgg_16_cifar.py +++ b/demo/image_classification/vgg_16_cifar.py @@ -25,8 +25,8 @@ 'img_size': 32,'num_classes': 10, 'use_jpeg': 1,'color': "color"} - define_py_data_sources2(train_list=data_dir+"train.list", - test_list=data_dir+'test.list', + define_py_data_sources2(train_list="train.list", + test_list="train.list", module='image_provider', obj='processData', args=args) From 93dc44c9df5b56154e4da8fbacfbffb174352ebf Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 9 Nov 2016 17:53:49 +0800 Subject: [PATCH 257/324] Update --- .../tests/configs/img_layers.protostr | 176 +++++ .../tests/configs/last_first_seq.protostr | 69 ++ .../tests/configs/layer_activations.protostr | 423 ++++++++++++ .../tests/configs/projections.protostr | 315 +++++++++ .../tests/configs/shared_fc.protostr | 0 .../tests/configs/shared_lstm.protostr | 393 +++++++++++ .../tests/configs/simple_rnn_layers.protostr | 418 +++++++++++ .../configs/test_bilinear_interp.protostr | 125 ++++ .../tests/configs/test_cost_layers.protostr | 289 ++++++++ .../test_cost_layers_with_weight.protostr | 111 +++ .../tests/configs/test_expand_layer.protostr | 56 ++ .../tests/configs/test_fc.protostr | 98 +++ .../configs/test_grumemory_layer.protostr | 51 ++ .../tests/configs/test_hsigmoid.protostr | 62 ++ .../configs/test_lstmemory_layer.protostr | 53 ++ .../tests/configs/test_maxout.protostr | 0 .../tests/configs/test_ntm_layers.protostr | 225 ++++++ .../tests/configs/test_print_layer.protostr | 26 + .../tests/configs/test_rnn_group.protostr | 650 ++++++++++++++++++ .../configs/test_sequence_pooling.protostr | 111 +++ .../tests/configs/unused_layers.protostr | 27 + .../tests/configs/util_layers.protostr | 81 +++ 22 files changed, 3759 insertions(+) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/img_layers.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/last_first_seq.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/layer_activations.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/projections.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/shared_fc.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/shared_lstm.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_fc.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_maxout.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_print_layer.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/unused_layers.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/util_layers.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/img_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/img_layers.protostr new file mode 100644 index 0000000000000..899171ff1d00b --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/img_layers.protostr @@ -0,0 +1,176 @@ +type: "nn" +layers { + name: "image" + type: "data" + size: 65536 + active_type: "" +} +layers { + name: "__conv_0__" + type: "exconv" + size: 3297856 + active_type: "" + inputs { + input_layer_name: "image" + input_parameter_name: "___conv_0__.w0" + conv_conf { + filter_size: 32 + channels: 1 + stride: 1 + padding: 1 + groups: 1 + filter_channels: 1 + output_x: 227 + img_size: 256 + caffe_mode: true + filter_size_y: 32 + padding_y: 1 + stride_y: 1 + } + } + bias_parameter_name: "___conv_0__.wbias" + num_filters: 64 + shared_biases: true +} +layers { + name: "__batch_norm_0__" + type: "batch_norm" + size: 3297856 + active_type: "relu" + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w0" + image_conf { + channels: 64 + img_size: 227 + } + } + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w1" + } + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w2" + } + bias_parameter_name: "___batch_norm_0__.wbias" + moving_average_fraction: 0.899999976158 +} +layers { + name: "__crmnorm_0__" + type: "norm" + size: 3297856 + active_type: "" + inputs { + input_layer_name: "__batch_norm_0__" + norm_conf { + norm_type: "cmrnorm-projection" + channels: 64 + size: 32 + scale: 0.000399999989895 + pow: 0.75 + output_x: 227 + img_size: 227 + blocked: false + } + } +} +layers { + name: "__pool_0__" + type: "pool" + size: 2458624 + active_type: "" + inputs { + input_layer_name: "__conv_0__" + pool_conf { + pool_type: "max-projection" + channels: 64 + size_x: 32 + stride: 1 + output_x: 196 + img_size: 227 + padding: 0 + size_y: 32 + stride_y: 1 + output_y: 196 + img_size_y: 227 + padding_y: 0 + } + } +} +parameters { + name: "___conv_0__.w0" + size: 65536 + initial_mean: 0.0 + initial_std: 0.0441941730678 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___conv_0__.wbias" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 64 + dims: 1 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___batch_norm_0__.w0" + size: 64 + initial_mean: 1.0 + initial_std: 0.0 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___batch_norm_0__.w1" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false + is_static: true + is_shared: true +} +parameters { + name: "___batch_norm_0__.w2" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false + is_static: true + is_shared: true +} +parameters { + name: "___batch_norm_0__.wbias" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "image" +output_layer_names: "__pool_0__" +output_layer_names: "__crmnorm_0__" +sub_models { + name: "root" + layer_names: "image" + layer_names: "__conv_0__" + layer_names: "__batch_norm_0__" + layer_names: "__crmnorm_0__" + layer_names: "__pool_0__" + input_layer_names: "image" + output_layer_names: "__pool_0__" + output_layer_names: "__crmnorm_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.protostr b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.protostr new file mode 100644 index 0000000000000..7b2911f8e367e --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.protostr @@ -0,0 +1,69 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 30 + active_type: "" +} +layers { + name: "__first_seq_0__" + type: "seqlastins" + size: 30 + active_type: "linear" + inputs { + input_layer_name: "data" + } + select_first: true + trans_type: "seq" +} +layers { + name: "__first_seq_1__" + type: "seqlastins" + size: 30 + active_type: "linear" + inputs { + input_layer_name: "data" + } + select_first: true + trans_type: "non-seq" +} +layers { + name: "__last_seq_0__" + type: "seqlastins" + size: 30 + active_type: "linear" + inputs { + input_layer_name: "data" + } + trans_type: "seq" +} +layers { + name: "__last_seq_1__" + type: "seqlastins" + size: 30 + active_type: "linear" + inputs { + input_layer_name: "data" + } + trans_type: "non-seq" +} +input_layer_names: "data" +output_layer_names: "__first_seq_0__" +output_layer_names: "__first_seq_1__" +output_layer_names: "__last_seq_0__" +output_layer_names: "__last_seq_1__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__first_seq_0__" + layer_names: "__first_seq_1__" + layer_names: "__last_seq_0__" + layer_names: "__last_seq_1__" + input_layer_names: "data" + output_layer_names: "__first_seq_0__" + output_layer_names: "__first_seq_1__" + output_layer_names: "__last_seq_0__" + output_layer_names: "__last_seq_1__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/layer_activations.protostr b/python/paddle/trainer_config_helpers/tests/configs/layer_activations.protostr new file mode 100644 index 0000000000000..8ae2421727efe --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/layer_activations.protostr @@ -0,0 +1,423 @@ +type: "nn" +layers { + name: "input" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "layer_0" + type: "fc" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_0.w0" + } + bias_parameter_name: "_layer_0.wbias" +} +layers { + name: "layer_1" + type: "fc" + size: 100 + active_type: "sigmoid" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_1.w0" + } + bias_parameter_name: "_layer_1.wbias" +} +layers { + name: "layer_2" + type: "fc" + size: 100 + active_type: "softmax" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_2.w0" + } + bias_parameter_name: "_layer_2.wbias" +} +layers { + name: "layer_3" + type: "fc" + size: 100 + active_type: "" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_3.w0" + } + bias_parameter_name: "_layer_3.wbias" +} +layers { + name: "layer_4" + type: "fc" + size: 100 + active_type: "" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_4.w0" + } + bias_parameter_name: "_layer_4.wbias" +} +layers { + name: "layer_5" + type: "fc" + size: 100 + active_type: "exponential" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_5.w0" + } + bias_parameter_name: "_layer_5.wbias" +} +layers { + name: "layer_6" + type: "fc" + size: 100 + active_type: "relu" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_6.w0" + } + bias_parameter_name: "_layer_6.wbias" +} +layers { + name: "layer_7" + type: "fc" + size: 100 + active_type: "brelu" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_7.w0" + } + bias_parameter_name: "_layer_7.wbias" +} +layers { + name: "layer_8" + type: "fc" + size: 100 + active_type: "softrelu" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_8.w0" + } + bias_parameter_name: "_layer_8.wbias" +} +layers { + name: "layer_9" + type: "fc" + size: 100 + active_type: "stanh" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_9.w0" + } + bias_parameter_name: "_layer_9.wbias" +} +layers { + name: "layer_10" + type: "fc" + size: 100 + active_type: "abs" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_10.w0" + } + bias_parameter_name: "_layer_10.wbias" +} +layers { + name: "layer_11" + type: "fc" + size: 100 + active_type: "square" + inputs { + input_layer_name: "input" + input_parameter_name: "_layer_11.w0" + } + bias_parameter_name: "_layer_11.wbias" +} +parameters { + name: "_layer_0.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_0.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_1.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_1.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_2.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_2.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_3.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_3.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_4.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_4.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_5.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_5.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_6.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_6.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_7.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_7.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_8.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_8.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_9.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_9.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_10.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_10.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_layer_11.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_layer_11.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "input" +output_layer_names: "layer_0" +output_layer_names: "layer_1" +output_layer_names: "layer_2" +output_layer_names: "layer_3" +output_layer_names: "layer_4" +output_layer_names: "layer_5" +output_layer_names: "layer_6" +output_layer_names: "layer_7" +output_layer_names: "layer_8" +output_layer_names: "layer_9" +output_layer_names: "layer_10" +output_layer_names: "layer_11" +sub_models { + name: "root" + layer_names: "input" + layer_names: "layer_0" + layer_names: "layer_1" + layer_names: "layer_2" + layer_names: "layer_3" + layer_names: "layer_4" + layer_names: "layer_5" + layer_names: "layer_6" + layer_names: "layer_7" + layer_names: "layer_8" + layer_names: "layer_9" + layer_names: "layer_10" + layer_names: "layer_11" + input_layer_names: "input" + output_layer_names: "layer_0" + output_layer_names: "layer_1" + output_layer_names: "layer_2" + output_layer_names: "layer_3" + output_layer_names: "layer_4" + output_layer_names: "layer_5" + output_layer_names: "layer_6" + output_layer_names: "layer_7" + output_layer_names: "layer_8" + output_layer_names: "layer_9" + output_layer_names: "layer_10" + output_layer_names: "layer_11" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/projections.protostr b/python/paddle/trainer_config_helpers/tests/configs/projections.protostr new file mode 100644 index 0000000000000..a901af6b42431 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/projections.protostr @@ -0,0 +1,315 @@ +type: "nn" +layers { + name: "test" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__embedding_0__" + type: "mixed" + size: 256 + active_type: "" + inputs { + input_layer_name: "test" + input_parameter_name: "___embedding_0__.w0" + proj_conf { + type: "table" + name: "___embedding_0__.w0" + input_size: 100 + output_size: 256 + } + } +} +layers { + name: "__mixed_0__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__embedding_0__" + input_parameter_name: "___mixed_0__.w0" + proj_conf { + type: "fc" + name: "___mixed_0__.w0" + input_size: 256 + output_size: 100 + } + } +} +layers { + name: "__mixed_1__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_0__" + input_parameter_name: "___mixed_1__.w0" + proj_conf { + type: "table" + name: "___mixed_1__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__mixed_2__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_1__" + proj_conf { + type: "identity" + name: "___mixed_2__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__mixed_3__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_2__" + input_parameter_name: "___mixed_3__.w0" + proj_conf { + type: "dot_mul" + name: "___mixed_3__.w0" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__mixed_4__" + type: "mixed" + size: 300 + active_type: "" + inputs { + input_layer_name: "__mixed_3__" + input_parameter_name: "___mixed_4__.w0" + proj_conf { + type: "context" + name: "___mixed_4__.w0" + input_size: 100 + output_size: 300 + context_start: -1 + context_length: 3 + trainable_padding: true + } + } +} +layers { + name: "__mixed_5__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_2__" + } + inputs { + input_layer_name: "__mixed_3__" + } + operator_confs { + type: "dot_mul" + input_indices: 0 + input_indices: 1 + input_sizes: 100 + input_sizes: 100 + output_size: 100 + dotmul_scale: 1.0 + } +} +layers { + name: "img" + type: "data" + size: 1024 + active_type: "" +} +layers { + name: "filter" + type: "data" + size: 576 + active_type: "" +} +layers { + name: "__mixed_6__" + type: "mixed" + size: 57600 + active_type: "" + inputs { + input_layer_name: "img" + } + inputs { + input_layer_name: "filter" + } + operator_confs { + type: "conv" + input_indices: 0 + input_indices: 1 + input_sizes: 1024 + input_sizes: 576 + output_size: 57600 + conv_conf { + filter_size: 3 + channels: 1 + stride: 1 + padding: 0 + groups: 1 + filter_channels: 1 + output_x: 30 + img_size: 32 + caffe_mode: true + filter_size_y: 3 + padding_y: 0 + stride_y: 1 + } + num_filters: 64 + } +} +layers { + name: "__mixed_7__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_4__" + input_parameter_name: "___mixed_7__.w0" + proj_conf { + type: "fc" + name: "___mixed_7__.w0" + input_size: 300 + output_size: 100 + } + } + inputs { + input_layer_name: "__mixed_5__" + input_parameter_name: "___mixed_7__.w1" + proj_conf { + type: "trans_fc" + name: "___mixed_7__.w1" + input_size: 100 + output_size: 100 + } + } + inputs { + input_layer_name: "__mixed_6__" + input_parameter_name: "___mixed_7__.w2" + proj_conf { + type: "fc" + name: "___mixed_7__.w2" + input_size: 57600 + output_size: 100 + } + } + drop_rate: 0.5 +} +parameters { + name: "___embedding_0__.w0" + size: 25600 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 256 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_0__.w0" + size: 25600 + initial_mean: 0.0 + initial_std: 0.0625 + dims: 256 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_1__.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_3__.w0" + size: 100 + initial_mean: 0.0 + initial_std: 1.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_4__.w0" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 2 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___mixed_7__.w0" + size: 30000 + initial_mean: 0.0 + initial_std: 0.0577350258827 + dims: 300 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_7__.w1" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_7__.w2" + size: 5760000 + initial_mean: 0.0 + initial_std: 0.00416666688398 + dims: 57600 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +input_layer_names: "test" +input_layer_names: "img" +input_layer_names: "filter" +output_layer_names: "__mixed_7__" +sub_models { + name: "root" + layer_names: "test" + layer_names: "__embedding_0__" + layer_names: "__mixed_0__" + layer_names: "__mixed_1__" + layer_names: "__mixed_2__" + layer_names: "__mixed_3__" + layer_names: "__mixed_4__" + layer_names: "__mixed_5__" + layer_names: "img" + layer_names: "filter" + layer_names: "__mixed_6__" + layer_names: "__mixed_7__" + input_layer_names: "test" + input_layer_names: "img" + input_layer_names: "filter" + output_layer_names: "__mixed_7__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/shared_fc.protostr b/python/paddle/trainer_config_helpers/tests/configs/shared_fc.protostr new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.protostr b/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.protostr new file mode 100644 index 0000000000000..26eed43a459f5 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.protostr @@ -0,0 +1,393 @@ +type: "recurrent_nn" +layers { + name: "data_a" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "data_b" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__mixed_0__" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "data_a" + input_parameter_name: "mixed_param" + proj_conf { + type: "fc" + name: "___mixed_0__.w0" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__mixed_1__" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "data_b" + input_parameter_name: "mixed_param" + proj_conf { + type: "fc" + name: "___mixed_1__.w0" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__lstm_group_0___recurrent_group" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "__mixed_0__@__lstm_group_0___recurrent_group" + type: "scatter_agent" + size: 400 + active_type: "" +} +layers { + name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "__mixed_0__@__lstm_group_0___recurrent_group" + proj_conf { + type: "identity" + name: "___lstm_group_0___input_recurrent.w0" + input_size: 400 + output_size: 400 + } + } + inputs { + input_layer_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + input_parameter_name: "lstm_param" + proj_conf { + type: "fc" + name: "___lstm_group_0___input_recurrent.w1" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + type: "lstm_step" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + } + inputs { + input_layer_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + } + bias_parameter_name: "lstm_bias" + active_gate_type: "sigmoid" + active_state_type: "sigmoid" +} +layers { + name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + type: "get_output" + size: 100 + active_type: "" + inputs { + input_layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + input_layer_argument: "state" + } +} +layers { + name: "__lstm_group_0__" + type: "gather_agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_1___recurrent_group" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "__mixed_1__@__lstm_group_1___recurrent_group" + type: "scatter_agent" + size: 400 + active_type: "" +} +layers { + name: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_1___input_recurrent@__lstm_group_1___recurrent_group" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "__mixed_1__@__lstm_group_1___recurrent_group" + proj_conf { + type: "identity" + name: "___lstm_group_1___input_recurrent.w0" + input_size: 400 + output_size: 400 + } + } + inputs { + input_layer_name: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" + input_parameter_name: "lstm_param" + proj_conf { + type: "fc" + name: "___lstm_group_1___input_recurrent.w1" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__lstm_group_1__@__lstm_group_1___recurrent_group" + type: "lstm_step" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "__lstm_group_1___input_recurrent@__lstm_group_1___recurrent_group" + } + inputs { + input_layer_name: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" + } + bias_parameter_name: "lstm_bias" + active_gate_type: "sigmoid" + active_state_type: "sigmoid" +} +layers { + name: "__lstm_group_1___state@__lstm_group_1___recurrent_group" + type: "get_output" + size: 100 + active_type: "" + inputs { + input_layer_name: "__lstm_group_1__@__lstm_group_1___recurrent_group" + input_layer_argument: "state" + } +} +layers { + name: "__lstm_group_1__" + type: "gather_agent" + size: 100 + active_type: "" +} +layers { + name: "__last_seq_0__" + type: "seqlastins" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "__lstm_group_0__" + } + trans_type: "non-seq" +} +layers { + name: "__last_seq_1__" + type: "seqlastins" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "__lstm_group_1__" + } + trans_type: "non-seq" +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 10 + active_type: "softmax" + inputs { + input_layer_name: "__last_seq_0__" + input_parameter_name: "softmax_param" + } + inputs { + input_layer_name: "__last_seq_1__" + input_parameter_name: "softmax_param" + } +} +layers { + name: "label" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "__cost_0__" + type: "multi-class-cross-entropy" + size: 1 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + } + inputs { + input_layer_name: "label" + } + coeff: 1.0 +} +parameters { + name: "mixed_param" + size: 40000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 400 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "lstm_param" + size: 40000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 400 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "lstm_bias" + size: 300 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 300 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "softmax_param" + size: 1000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 10 + initial_strategy: 0 + initial_smart: true +} +input_layer_names: "data_a" +input_layer_names: "data_b" +input_layer_names: "label" +output_layer_names: "__cost_0__" +evaluators { + name: "classification_error_evaluator" + type: "classification_error" + input_layers: "__fc_layer_0__" + input_layers: "label" +} +sub_models { + name: "root" + layer_names: "data_a" + layer_names: "data_b" + layer_names: "__mixed_0__" + layer_names: "__mixed_1__" + layer_names: "__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__" + layer_names: "__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1__" + layer_names: "__last_seq_0__" + layer_names: "__last_seq_1__" + layer_names: "__fc_layer_0__" + layer_names: "label" + layer_names: "__cost_0__" + input_layer_names: "data_a" + input_layer_names: "data_b" + input_layer_names: "label" + output_layer_names: "__cost_0__" + evaluator_names: "classification_error_evaluator" + is_recurrent_layer_group: false +} +sub_models { + name: "__lstm_group_0___recurrent_group" + layer_names: "__mixed_0__@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + is_sequence: false + } + memories { + layer_name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + is_sequence: false + } + in_links { + layer_name: "__mixed_0__" + link_name: "__mixed_0__@__lstm_group_0___recurrent_group" + has_subseq: false + } + out_links { + layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0__" + has_subseq: false + } + target_inlinkid: -1 +} +sub_models { + name: "__lstm_group_1___recurrent_group" + layer_names: "__mixed_1__@__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1___input_recurrent@__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1__@__lstm_group_1___recurrent_group" + layer_names: "__lstm_group_1___state@__lstm_group_1___recurrent_group" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "__lstm_group_1__@__lstm_group_1___recurrent_group" + link_name: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" + is_sequence: false + } + memories { + layer_name: "__lstm_group_1___state@__lstm_group_1___recurrent_group" + link_name: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" + is_sequence: false + } + in_links { + layer_name: "__mixed_1__" + link_name: "__mixed_1__@__lstm_group_1___recurrent_group" + has_subseq: false + } + out_links { + layer_name: "__lstm_group_1__@__lstm_group_1___recurrent_group" + link_name: "__lstm_group_1__" + has_subseq: false + } + target_inlinkid: -1 +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.protostr new file mode 100644 index 0000000000000..57445243bd06f --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.protostr @@ -0,0 +1,418 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 200 + active_type: "" +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "data" + input_parameter_name: "___fc_layer_0__.w0" + } + bias_parameter_name: "___fc_layer_0__.wbias" +} +layers { + name: "__recurrent_layer_0__" + type: "recurrent" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___recurrent_layer_0__.w0" + } + bias_parameter_name: "___recurrent_layer_0__.wbias" + reversed: false +} +layers { + name: "__recurrent_layer_1__" + type: "recurrent" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___recurrent_layer_1__.w0" + } + bias_parameter_name: "___recurrent_layer_1__.wbias" + reversed: true +} +layers { + name: "__fc_layer_1__" + type: "fc" + size: 800 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___fc_layer_1__.w0" + } +} +layers { + name: "__lstmemory_0__" + type: "lstmemory" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_1__" + input_parameter_name: "___lstmemory_0__.w0" + } + bias_parameter_name: "___lstmemory_0__.wbias" + reversed: false + active_gate_type: "sigmoid" + active_state_type: "tanh" +} +layers { + name: "__fc_layer_2__" + type: "fc" + size: 800 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___fc_layer_2__.w0" + } +} +layers { + name: "__lstmemory_1__" + type: "lstmemory" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_2__" + input_parameter_name: "___lstmemory_1__.w0" + } + bias_parameter_name: "___lstmemory_1__.wbias" + reversed: true + active_gate_type: "sigmoid" + active_state_type: "tanh" +} +layers { + name: "__fc_layer_3__" + type: "fc" + size: 600 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___fc_layer_3__.w0" + } +} +layers { + name: "__gru_0__" + type: "gated_recurrent" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_3__" + input_parameter_name: "___gru_0__.w0" + } + bias_parameter_name: "___gru_0__.wbias" + reversed: false + active_gate_type: "sigmoid" +} +layers { + name: "__fc_layer_4__" + type: "fc" + size: 600 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___fc_layer_4__.w0" + } +} +layers { + name: "__gru_1__" + type: "gated_recurrent" + size: 200 + active_type: "sigmoid" + inputs { + input_layer_name: "__fc_layer_4__" + input_parameter_name: "___gru_1__.w0" + } + bias_parameter_name: "___gru_1__.wbias" + reversed: true + active_gate_type: "sigmoid" +} +layers { + name: "__last_seq_0__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__recurrent_layer_0__" + } + trans_type: "non-seq" +} +layers { + name: "__first_seq_0__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__recurrent_layer_1__" + } + select_first: true + trans_type: "non-seq" +} +layers { + name: "__last_seq_1__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__lstmemory_0__" + } + trans_type: "non-seq" +} +layers { + name: "__first_seq_1__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__lstmemory_1__" + } + select_first: true + trans_type: "non-seq" +} +layers { + name: "__last_seq_2__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__gru_0__" + } + trans_type: "non-seq" +} +layers { + name: "__first_seq_2__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "__gru_1__" + } + select_first: true + trans_type: "non-seq" +} +parameters { + name: "___fc_layer_0__.w0" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___fc_layer_0__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___recurrent_layer_0__.w0" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___recurrent_layer_0__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___recurrent_layer_1__.w0" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___recurrent_layer_1__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___fc_layer_1__.w0" + size: 160000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 800 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstmemory_0__.w0" + size: 160000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 200 + dims: 4 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstmemory_0__.wbias" + size: 1400 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 1400 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___fc_layer_2__.w0" + size: 160000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 800 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstmemory_1__.w0" + size: 160000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 200 + dims: 4 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstmemory_1__.wbias" + size: 1400 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 1400 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___fc_layer_3__.w0" + size: 120000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 600 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___gru_0__.w0" + size: 120000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 600 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___gru_0__.wbias" + size: 600 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 600 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___fc_layer_4__.w0" + size: 120000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 600 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___gru_1__.w0" + size: 120000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 600 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___gru_1__.wbias" + size: 600 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 600 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +output_layer_names: "__last_seq_0__" +output_layer_names: "__first_seq_0__" +output_layer_names: "__last_seq_1__" +output_layer_names: "__first_seq_1__" +output_layer_names: "__last_seq_2__" +output_layer_names: "__first_seq_2__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__fc_layer_0__" + layer_names: "__recurrent_layer_0__" + layer_names: "__recurrent_layer_1__" + layer_names: "__fc_layer_1__" + layer_names: "__lstmemory_0__" + layer_names: "__fc_layer_2__" + layer_names: "__lstmemory_1__" + layer_names: "__fc_layer_3__" + layer_names: "__gru_0__" + layer_names: "__fc_layer_4__" + layer_names: "__gru_1__" + layer_names: "__last_seq_0__" + layer_names: "__first_seq_0__" + layer_names: "__last_seq_1__" + layer_names: "__first_seq_1__" + layer_names: "__last_seq_2__" + layer_names: "__first_seq_2__" + input_layer_names: "data" + output_layer_names: "__last_seq_0__" + output_layer_names: "__first_seq_0__" + output_layer_names: "__last_seq_1__" + output_layer_names: "__first_seq_1__" + output_layer_names: "__last_seq_2__" + output_layer_names: "__first_seq_2__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.protostr new file mode 100644 index 0000000000000..278088d4abd50 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.protostr @@ -0,0 +1,125 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 2304 + active_type: "" +} +layers { + name: "__conv_0__" + type: "exconv" + size: 36864 + active_type: "" + inputs { + input_layer_name: "data" + input_parameter_name: "___conv_0__.w0" + conv_conf { + filter_size: 3 + channels: 1 + stride: 1 + padding: 1 + groups: 1 + filter_channels: 1 + output_x: 48 + img_size: 48 + caffe_mode: true + filter_size_y: 3 + padding_y: 1 + stride_y: 1 + } + } + bias_parameter_name: "___conv_0__.wbias" + num_filters: 16 + shared_biases: true +} +layers { + name: "__bilinear_interp_layer_0__" + type: "bilinear_interp" + size: 36864 + active_type: "" + inputs { + input_layer_name: "__conv_0__" + bilinear_interp_conf { + img_size_x: 32 + img_size_y: 32 + out_size_x: 64 + out_size_y: 64 + num_channels: 16 + } + } +} +layers { + name: "__pool_0__" + type: "pool" + size: 9216 + active_type: "" + inputs { + input_layer_name: "__bilinear_interp_layer_0__" + pool_conf { + pool_type: "max-projection" + channels: 4 + size_x: 2 + stride: 2 + output_x: 48 + img_size: 96 + padding: 0 + size_y: 2 + stride_y: 2 + output_y: 48 + img_size_y: 96 + padding_y: 0 + } + } +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 384 + active_type: "tanh" + inputs { + input_layer_name: "__pool_0__" + input_parameter_name: "___fc_layer_0__.w0" + } +} +parameters { + name: "___conv_0__.w0" + size: 144 + initial_mean: 0.0 + initial_std: 0.471404522657 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___conv_0__.wbias" + size: 16 + initial_mean: 0.0 + initial_std: 0.0 + dims: 16 + dims: 1 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___fc_layer_0__.w0" + size: 3538944 + initial_mean: 0.0 + initial_std: 0.0104166669771 + dims: 9216 + dims: 384 + initial_strategy: 0 + initial_smart: true +} +input_layer_names: "data" +output_layer_names: "__fc_layer_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__conv_0__" + layer_names: "__bilinear_interp_layer_0__" + layer_names: "__pool_0__" + layer_names: "__fc_layer_0__" + input_layer_names: "data" + output_layer_names: "__fc_layer_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.protostr new file mode 100644 index 0000000000000..c37586f4068e4 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.protostr @@ -0,0 +1,289 @@ +type: "nn" +layers { + name: "input" + type: "data" + size: 200 + active_type: "" +} +layers { + name: "labels" + type: "data" + size: 5000 + active_type: "" +} +layers { + name: "probs" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "xe-label" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "__ctc_layer_0__" + type: "ctc" + size: 5001 + active_type: "" + inputs { + input_layer_name: "input" + } + inputs { + input_layer_name: "labels" + } + norm_by_times: false +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 4 + active_type: "tanh" + inputs { + input_layer_name: "input" + input_parameter_name: "___fc_layer_0__.w0" + } + bias_parameter_name: "___fc_layer_0__.wbias" +} +layers { + name: "crf_label" + type: "data" + size: 4 + active_type: "" +} +layers { + name: "__crf_layer_0__" + type: "crf" + size: 4 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + input_parameter_name: "___crf_layer_0__.w0" + } + inputs { + input_layer_name: "crf_label" + } + coeff: 1.0 +} +layers { + name: "left" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "right" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "label" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__rank_cost_0__" + type: "rank-cost" + size: 1 + active_type: "" + inputs { + input_layer_name: "left" + } + inputs { + input_layer_name: "right" + } + inputs { + input_layer_name: "label" + } + coeff: 1.0 +} +layers { + name: "list_feature" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "list_scores" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__lambda_cost_0__" + type: "lambda_cost" + size: 1 + active_type: "" + inputs { + input_layer_name: "list_feature" + } + inputs { + input_layer_name: "list_scores" + } + NDCG_num: 5 + max_sort_size: -1 +} +layers { + name: "__cross_entropy_0__" + type: "multi-class-cross-entropy" + size: 1 + active_type: "" + inputs { + input_layer_name: "probs" + } + inputs { + input_layer_name: "xe-label" + } + coeff: 1.0 +} +layers { + name: "__cross_entropy_with_selfnorm_0__" + type: "multi_class_cross_entropy_with_selfnorm" + active_type: "" + inputs { + input_layer_name: "probs" + } + inputs { + input_layer_name: "xe-label" + } + softmax_selfnorm_alpha: 0.10000000149 + coeff: 1.0 +} +layers { + name: "huber_probs" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "huber_label" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__huber_cost_0__" + type: "huber" + size: 1 + active_type: "" + inputs { + input_layer_name: "huber_probs" + } + inputs { + input_layer_name: "huber_label" + } + coeff: 1.0 +} +layers { + name: "__multi_binary_label_cross_entropy_0__" + type: "multi_binary_label_cross_entropy" + size: 1 + active_type: "" + inputs { + input_layer_name: "probs" + } + inputs { + input_layer_name: "xe-label" + } + coeff: 1.0 +} +parameters { + name: "___fc_layer_0__.w0" + size: 800 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 4 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___fc_layer_0__.wbias" + size: 4 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 4 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___crf_layer_0__.w0" + size: 24 + initial_mean: 0.0 + initial_std: 0.5 + dims: 4 + dims: 6 + initial_strategy: 0 + initial_smart: true +} +input_layer_names: "input" +input_layer_names: "labels" +input_layer_names: "crf_label" +input_layer_names: "left" +input_layer_names: "right" +input_layer_names: "label" +input_layer_names: "list_feature" +input_layer_names: "list_scores" +input_layer_names: "probs" +input_layer_names: "xe-label" +input_layer_names: "huber_probs" +input_layer_names: "huber_label" +output_layer_names: "__ctc_layer_0__" +output_layer_names: "__crf_layer_0__" +output_layer_names: "__rank_cost_0__" +output_layer_names: "__lambda_cost_0__" +output_layer_names: "__cross_entropy_0__" +output_layer_names: "__cross_entropy_with_selfnorm_0__" +output_layer_names: "__huber_cost_0__" +output_layer_names: "__multi_binary_label_cross_entropy_0__" +sub_models { + name: "root" + layer_names: "input" + layer_names: "labels" + layer_names: "probs" + layer_names: "xe-label" + layer_names: "__ctc_layer_0__" + layer_names: "__fc_layer_0__" + layer_names: "crf_label" + layer_names: "__crf_layer_0__" + layer_names: "left" + layer_names: "right" + layer_names: "label" + layer_names: "__rank_cost_0__" + layer_names: "list_feature" + layer_names: "list_scores" + layer_names: "__lambda_cost_0__" + layer_names: "__cross_entropy_0__" + layer_names: "__cross_entropy_with_selfnorm_0__" + layer_names: "huber_probs" + layer_names: "huber_label" + layer_names: "__huber_cost_0__" + layer_names: "__multi_binary_label_cross_entropy_0__" + input_layer_names: "input" + input_layer_names: "labels" + input_layer_names: "crf_label" + input_layer_names: "left" + input_layer_names: "right" + input_layer_names: "label" + input_layer_names: "list_feature" + input_layer_names: "list_scores" + input_layer_names: "probs" + input_layer_names: "xe-label" + input_layer_names: "huber_probs" + input_layer_names: "huber_label" + output_layer_names: "__ctc_layer_0__" + output_layer_names: "__crf_layer_0__" + output_layer_names: "__rank_cost_0__" + output_layer_names: "__lambda_cost_0__" + output_layer_names: "__cross_entropy_0__" + output_layer_names: "__cross_entropy_with_selfnorm_0__" + output_layer_names: "__huber_cost_0__" + output_layer_names: "__multi_binary_label_cross_entropy_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.protostr new file mode 100644 index 0000000000000..de58f5c64969b --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.protostr @@ -0,0 +1,111 @@ +type: "nn" +layers { + name: "input" + type: "data" + size: 300 + active_type: "" +} +layers { + name: "label" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "weight" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 10 + active_type: "softmax" + inputs { + input_layer_name: "input" + input_parameter_name: "___fc_layer_0__.w0" + } + bias_parameter_name: "___fc_layer_0__.wbias" +} +layers { + name: "__cost_0__" + type: "multi-class-cross-entropy" + size: 1 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + } + inputs { + input_layer_name: "label" + } + inputs { + input_layer_name: "weight" + } + coeff: 1.0 +} +layers { + name: "__regression_cost_0__" + type: "square_error" + size: 1 + active_type: "" + inputs { + input_layer_name: "__fc_layer_0__" + } + inputs { + input_layer_name: "label" + } + inputs { + input_layer_name: "weight" + } + coeff: 1.0 +} +parameters { + name: "___fc_layer_0__.w0" + size: 3000 + initial_mean: 0.0 + initial_std: 0.0577350258827 + dims: 300 + dims: 10 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___fc_layer_0__.wbias" + size: 10 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 10 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "input" +input_layer_names: "label" +input_layer_names: "weight" +output_layer_names: "__cost_0__" +output_layer_names: "__regression_cost_0__" +evaluators { + name: "classification_error_evaluator" + type: "classification_error" + input_layers: "__fc_layer_0__" + input_layers: "label" + input_layers: "weight" +} +sub_models { + name: "root" + layer_names: "input" + layer_names: "label" + layer_names: "weight" + layer_names: "__fc_layer_0__" + layer_names: "__cost_0__" + layer_names: "__regression_cost_0__" + input_layer_names: "input" + input_layer_names: "label" + input_layer_names: "weight" + output_layer_names: "__cost_0__" + output_layer_names: "__regression_cost_0__" + evaluator_names: "classification_error_evaluator" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.protostr new file mode 100644 index 0000000000000..f4b36052264bc --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.protostr @@ -0,0 +1,56 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 30 + active_type: "" +} +layers { + name: "data_seq" + type: "data" + size: 30 + active_type: "" +} +layers { + name: "__expand_layer_0__" + type: "expand" + size: 30 + active_type: "" + inputs { + input_layer_name: "data" + } + inputs { + input_layer_name: "data_seq" + } + trans_type: "seq" +} +layers { + name: "__expand_layer_1__" + type: "expand" + size: 30 + active_type: "" + inputs { + input_layer_name: "data" + } + inputs { + input_layer_name: "data_seq" + } + trans_type: "non-seq" +} +input_layer_names: "data" +input_layer_names: "data_seq" +output_layer_names: "__expand_layer_0__" +output_layer_names: "__expand_layer_1__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "data_seq" + layer_names: "__expand_layer_0__" + layer_names: "__expand_layer_1__" + input_layer_names: "data" + input_layer_names: "data_seq" + output_layer_names: "__expand_layer_0__" + output_layer_names: "__expand_layer_1__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_fc.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_fc.protostr new file mode 100644 index 0000000000000..80b01246ba96f --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_fc.protostr @@ -0,0 +1,98 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__trans_layer_0__" + type: "trans" + size: 100 + active_type: "" + inputs { + input_layer_name: "data" + } +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "__trans_layer_0__" + input_parameter_name: "___fc_layer_0__.w0" + } +} +layers { + name: "mask" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__selective_fc_layer_0__" + type: "selective_fc" + size: 100 + active_type: "sigmoid" + inputs { + input_layer_name: "data" + input_parameter_name: "___selective_fc_layer_0__.w0" + } + inputs { + input_layer_name: "mask" + } + bias_parameter_name: "___selective_fc_layer_0__.wbias" + selective_fc_pass_generation: false + has_selected_colums: true + selective_fc_full_mul_ratio: 0.019999999553 +} +parameters { + name: "___fc_layer_0__.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___selective_fc_layer_0__.w0" + size: 10000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + initial_strategy: 0 + initial_smart: true + is_sparse: false +} +parameters { + name: "___selective_fc_layer_0__.wbias" + size: 100 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 100 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +input_layer_names: "mask" +output_layer_names: "__fc_layer_0__" +output_layer_names: "__selective_fc_layer_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__trans_layer_0__" + layer_names: "__fc_layer_0__" + layer_names: "mask" + layer_names: "__selective_fc_layer_0__" + input_layer_names: "data" + input_layer_names: "mask" + output_layer_names: "__fc_layer_0__" + output_layer_names: "__selective_fc_layer_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.protostr new file mode 100644 index 0000000000000..81577910ccf34 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.protostr @@ -0,0 +1,51 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 120 + active_type: "" +} +layers { + name: "__gru_0__" + type: "gated_recurrent" + size: 40 + active_type: "sigmoid" + inputs { + input_layer_name: "data" + input_parameter_name: "___gru_0__.w0" + } + bias_parameter_name: "___gru_0__.wbias" + reversed: true + active_gate_type: "tanh" +} +parameters { + name: "___gru_0__.w0" + size: 4800 + initial_mean: 0.0 + initial_std: 0.158113881946 + dims: 40 + dims: 120 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___gru_0__.wbias" + size: 120 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 120 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +output_layer_names: "__gru_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__gru_0__" + input_layer_names: "data" + output_layer_names: "__gru_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.protostr new file mode 100644 index 0000000000000..e8cc61b8c5410 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.protostr @@ -0,0 +1,62 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "label" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "__hsigmoid_0__" + type: "hsigmoid" + size: 1 + active_type: "" + inputs { + input_layer_name: "data" + input_parameter_name: "___hsigmoid_0__.w0" + } + inputs { + input_layer_name: "label" + } + bias_parameter_name: "___hsigmoid_0__.wbias" + num_classes: 10 +} +parameters { + name: "___hsigmoid_0__.w0" + size: 900 + initial_mean: 0.0 + initial_std: 0.333333343267 + dims: 9 + dims: 100 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___hsigmoid_0__.wbias" + size: 9 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 9 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +input_layer_names: "label" +output_layer_names: "__hsigmoid_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "label" + layer_names: "__hsigmoid_0__" + input_layer_names: "data" + input_layer_names: "label" + output_layer_names: "__hsigmoid_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.protostr new file mode 100644 index 0000000000000..8341cd2684746 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.protostr @@ -0,0 +1,53 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 128 + active_type: "" +} +layers { + name: "__lstmemory_0__" + type: "lstmemory" + size: 32 + active_type: "tanh" + inputs { + input_layer_name: "data" + input_parameter_name: "___lstmemory_0__.w0" + } + bias_parameter_name: "___lstmemory_0__.wbias" + reversed: true + active_gate_type: "tanh" + active_state_type: "tanh" +} +parameters { + name: "___lstmemory_0__.w0" + size: 4096 + initial_mean: 0.0 + initial_std: 0.176776692271 + dims: 32 + dims: 32 + dims: 4 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstmemory_0__.wbias" + size: 224 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 224 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "data" +output_layer_names: "__lstmemory_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__lstmemory_0__" + input_layer_names: "data" + output_layer_names: "__lstmemory_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_maxout.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_maxout.protostr new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.protostr new file mode 100644 index 0000000000000..44400e2c3a23d --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.protostr @@ -0,0 +1,225 @@ +type: "nn" +layers { + name: "w" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "a" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "b" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "c" + type: "data" + size: 200 + active_type: "" +} +layers { + name: "d" + type: "data" + size: 31 + active_type: "" +} +layers { + name: "__interpolation_layer_0__" + type: "interpolation" + size: 100 + active_type: "" + inputs { + input_layer_name: "w" + } + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "b" + } +} +layers { + name: "__power_layer_0__" + type: "power" + size: 100 + active_type: "" + inputs { + input_layer_name: "w" + } + inputs { + input_layer_name: "a" + } +} +layers { + name: "__scaling_layer_0__" + type: "scaling" + size: 100 + active_type: "" + inputs { + input_layer_name: "w" + } + inputs { + input_layer_name: "a" + } +} +layers { + name: "__cos_sim_0__" + type: "cos" + size: 1 + active_type: "" + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "b" + } + cos_scale: 5.0 +} +layers { + name: "__cos_sim_1__" + type: "cos_vm" + size: 2 + active_type: "" + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "c" + } + cos_scale: 5.0 +} +layers { + name: "__sum_to_one_norm_layer_0__" + type: "sum_to_one_norm" + size: 100 + active_type: "" + inputs { + input_layer_name: "a" + } +} +layers { + name: "__conv_shift_layer_0__" + type: "conv_shift" + size: 100 + active_type: "" + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "d" + } +} +layers { + name: "__tensor_layer_0__" + type: "tensor" + size: 1000 + active_type: "" + inputs { + input_layer_name: "a" + input_parameter_name: "___tensor_layer_0__.w0" + } + inputs { + input_layer_name: "b" + } + bias_parameter_name: "___tensor_layer_0__.wbias" +} +layers { + name: "__slope_intercept_layer_0__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "a" + } + slope: 0.699999988079 + intercept: 0.899999976158 +} +layers { + name: "__linear_comb_layer_0__" + type: "convex_comb" + size: 2 + active_type: "" + inputs { + input_layer_name: "b" + } + inputs { + input_layer_name: "c" + } +} +parameters { + name: "___tensor_layer_0__.w0" + size: 10000000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 100 + dims: 1000 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___tensor_layer_0__.wbias" + size: 1000 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 1000 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "w" +input_layer_names: "a" +input_layer_names: "b" +input_layer_names: "c" +input_layer_names: "d" +output_layer_names: "__interpolation_layer_0__" +output_layer_names: "__power_layer_0__" +output_layer_names: "__scaling_layer_0__" +output_layer_names: "__cos_sim_0__" +output_layer_names: "__cos_sim_1__" +output_layer_names: "__sum_to_one_norm_layer_0__" +output_layer_names: "__conv_shift_layer_0__" +output_layer_names: "__tensor_layer_0__" +output_layer_names: "__slope_intercept_layer_0__" +output_layer_names: "__linear_comb_layer_0__" +sub_models { + name: "root" + layer_names: "w" + layer_names: "a" + layer_names: "b" + layer_names: "c" + layer_names: "d" + layer_names: "__interpolation_layer_0__" + layer_names: "__power_layer_0__" + layer_names: "__scaling_layer_0__" + layer_names: "__cos_sim_0__" + layer_names: "__cos_sim_1__" + layer_names: "__sum_to_one_norm_layer_0__" + layer_names: "__conv_shift_layer_0__" + layer_names: "__tensor_layer_0__" + layer_names: "__slope_intercept_layer_0__" + layer_names: "__linear_comb_layer_0__" + input_layer_names: "w" + input_layer_names: "a" + input_layer_names: "b" + input_layer_names: "c" + input_layer_names: "d" + output_layer_names: "__interpolation_layer_0__" + output_layer_names: "__power_layer_0__" + output_layer_names: "__scaling_layer_0__" + output_layer_names: "__cos_sim_0__" + output_layer_names: "__cos_sim_1__" + output_layer_names: "__sum_to_one_norm_layer_0__" + output_layer_names: "__conv_shift_layer_0__" + output_layer_names: "__tensor_layer_0__" + output_layer_names: "__slope_intercept_layer_0__" + output_layer_names: "__linear_comb_layer_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.protostr new file mode 100644 index 0000000000000..c402aff174ab7 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.protostr @@ -0,0 +1,26 @@ +type: "nn" +layers { + name: "input" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__print_0__" + type: "print" + active_type: "" + inputs { + input_layer_name: "input" + } +} +input_layer_names: "input" +output_layer_names: "input" +sub_models { + name: "root" + layer_names: "input" + layer_names: "__print_0__" + input_layer_names: "input" + output_layer_names: "input" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.protostr new file mode 100644 index 0000000000000..dfb5ce20a31a0 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.protostr @@ -0,0 +1,650 @@ +type: "recurrent_nn" +layers { + name: "seq_input" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "sub_seq_input" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "label" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__mixed_0__" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "seq_input" + input_parameter_name: "___mixed_0__.w0" + proj_conf { + type: "fc" + name: "___mixed_0__.w0" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__mixed_1__" + type: "mixed" + size: 300 + active_type: "" + inputs { + input_layer_name: "seq_input" + input_parameter_name: "___mixed_1__.w0" + proj_conf { + type: "fc" + name: "___mixed_1__.w0" + input_size: 100 + output_size: 300 + } + } +} +layers { + name: "__recurrent_group_0__" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "seq_input@__recurrent_group_0__" + type: "scatter_agent" + size: 100 + active_type: "" +} +layers { + name: "rnn_forward+delay1@__recurrent_group_0__" + type: "agent" + size: 200 + active_type: "" +} +layers { + name: "rnn_forward@__recurrent_group_0__" + type: "fc" + size: 200 + active_type: "tanh" + inputs { + input_layer_name: "seq_input@__recurrent_group_0__" + input_parameter_name: "_rnn_forward@__recurrent_group_0__.w0" + } + inputs { + input_layer_name: "rnn_forward+delay1@__recurrent_group_0__" + input_parameter_name: "_rnn_forward@__recurrent_group_0__.w1" + } + bias_parameter_name: "_rnn_forward@__recurrent_group_0__.wbias" +} +layers { + name: "rnn_forward" + type: "gather_agent" + size: 200 + active_type: "" +} +layers { + name: "__last_seq_0__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "rnn_forward" + } + trans_type: "non-seq" +} +layers { + name: "__recurrent_group_1__" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "seq_input@__recurrent_group_1__" + type: "scatter_agent" + size: 100 + active_type: "" +} +layers { + name: "rnn_back+delay1@__recurrent_group_1__" + type: "agent" + size: 200 + active_type: "" +} +layers { + name: "rnn_back@__recurrent_group_1__" + type: "fc" + size: 200 + active_type: "tanh" + inputs { + input_layer_name: "seq_input@__recurrent_group_1__" + input_parameter_name: "_rnn_back@__recurrent_group_1__.w0" + } + inputs { + input_layer_name: "rnn_back+delay1@__recurrent_group_1__" + input_parameter_name: "_rnn_back@__recurrent_group_1__.w1" + } + bias_parameter_name: "_rnn_back@__recurrent_group_1__.wbias" +} +layers { + name: "rnn_back" + type: "gather_agent" + size: 200 + active_type: "" +} +layers { + name: "__first_seq_0__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "rnn_back" + } + select_first: true + trans_type: "non-seq" +} +layers { + name: "__recurrent_group_2__" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "sub_seq_input@__recurrent_group_2__" + type: "sequence_scatter_agent" + size: 100 + active_type: "" +} +layers { + name: "rnn_subseq_forward+delay1@__recurrent_group_2__" + type: "agent" + size: 200 + active_type: "" +} +layers { + name: "rnn_subseq_forward@__recurrent_group_2__" + type: "fc" + size: 200 + active_type: "tanh" + inputs { + input_layer_name: "sub_seq_input@__recurrent_group_2__" + input_parameter_name: "_rnn_subseq_forward@__recurrent_group_2__.w0" + } + inputs { + input_layer_name: "rnn_subseq_forward+delay1@__recurrent_group_2__" + input_parameter_name: "_rnn_subseq_forward@__recurrent_group_2__.w1" + } + bias_parameter_name: "_rnn_subseq_forward@__recurrent_group_2__.wbias" +} +layers { + name: "rnn_subseq_forward" + type: "sequence_gather_agent" + size: 200 + active_type: "" +} +layers { + name: "__last_seq_1__" + type: "seqlastins" + size: 200 + active_type: "linear" + inputs { + input_layer_name: "rnn_subseq_forward" + } + trans_type: "non-seq" +} +layers { + name: "__lstm_group_0___recurrent_group" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "__mixed_0__@__lstm_group_0___recurrent_group" + type: "scatter_agent" + size: 400 + active_type: "" +} +layers { + name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + type: "mixed" + size: 400 + active_type: "" + inputs { + input_layer_name: "__mixed_0__@__lstm_group_0___recurrent_group" + proj_conf { + type: "identity" + name: "___lstm_group_0___input_recurrent.w0" + input_size: 400 + output_size: 400 + } + } + inputs { + input_layer_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + input_parameter_name: "___lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group.w1" + proj_conf { + type: "fc" + name: "___lstm_group_0___input_recurrent.w1" + input_size: 100 + output_size: 400 + } + } +} +layers { + name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + type: "lstm_step" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + } + inputs { + input_layer_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + } + bias_parameter_name: "___lstm_group_0__@__lstm_group_0___recurrent_group.wbias" + active_gate_type: "sigmoid" + active_state_type: "sigmoid" +} +layers { + name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + type: "get_output" + size: 100 + active_type: "" + inputs { + input_layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + input_layer_argument: "state" + } +} +layers { + name: "__lstm_group_0__" + type: "gather_agent" + size: 100 + active_type: "" +} +layers { + name: "__last_seq_2__" + type: "seqlastins" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "__lstm_group_0__" + } + trans_type: "non-seq" +} +layers { + name: "__gru_group_0___recurrent_group" + type: "recurrent_layer_group" + active_type: "" +} +layers { + name: "__mixed_1__@__gru_group_0___recurrent_group" + type: "scatter_agent" + size: 300 + active_type: "" +} +layers { + name: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" + type: "agent" + size: 100 + active_type: "" +} +layers { + name: "__gru_group_0__@__gru_group_0___recurrent_group" + type: "gru_step" + size: 100 + active_type: "tanh" + inputs { + input_layer_name: "__mixed_1__@__gru_group_0___recurrent_group" + input_parameter_name: "___gru_group_0__@__gru_group_0___recurrent_group.w0" + } + inputs { + input_layer_name: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" + } + bias_parameter_name: "___gru_group_0__@__gru_group_0___recurrent_group.wbias" + active_gate_type: "sigmoid" +} +layers { + name: "__gru_group_0__" + type: "gather_agent" + size: 100 + active_type: "" +} +layers { + name: "__last_seq_3__" + type: "seqlastins" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "__gru_group_0__" + } + trans_type: "non-seq" +} +parameters { + name: "___mixed_0__.w0" + size: 40000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 400 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___mixed_1__.w0" + size: 30000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 300 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_forward@__recurrent_group_0__.w0" + size: 20000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_forward@__recurrent_group_0__.w1" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_forward@__recurrent_group_0__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_rnn_back@__recurrent_group_1__.w0" + size: 20000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_back@__recurrent_group_1__.w1" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_back@__recurrent_group_1__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "_rnn_subseq_forward@__recurrent_group_2__.w0" + size: 20000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_subseq_forward@__recurrent_group_2__.w1" + size: 40000 + initial_mean: 0.0 + initial_std: 0.0707106813788 + dims: 200 + dims: 200 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "_rnn_subseq_forward@__recurrent_group_2__.wbias" + size: 200 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 200 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group.w1" + size: 40000 + initial_mean: 0.0 + initial_std: 0.10000000149 + dims: 100 + dims: 400 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___lstm_group_0__@__lstm_group_0___recurrent_group.wbias" + size: 300 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 300 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___gru_group_0__@__gru_group_0___recurrent_group.w0" + size: 30000 + initial_mean: 0.0 + initial_std: 0.00999999977648 + dims: 100 + dims: 300 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___gru_group_0__@__gru_group_0___recurrent_group.wbias" + size: 300 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 300 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "seq_input" +input_layer_names: "sub_seq_input" +output_layer_names: "__last_seq_0__" +output_layer_names: "__first_seq_0__" +output_layer_names: "__last_seq_1__" +output_layer_names: "__last_seq_2__" +output_layer_names: "__last_seq_3__" +sub_models { + name: "root" + layer_names: "seq_input" + layer_names: "sub_seq_input" + layer_names: "label" + layer_names: "__mixed_0__" + layer_names: "__mixed_1__" + layer_names: "__recurrent_group_0__" + layer_names: "rnn_forward" + layer_names: "__last_seq_0__" + layer_names: "__recurrent_group_1__" + layer_names: "rnn_back" + layer_names: "__first_seq_0__" + layer_names: "__recurrent_group_2__" + layer_names: "rnn_subseq_forward" + layer_names: "__last_seq_1__" + layer_names: "__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__" + layer_names: "__last_seq_2__" + layer_names: "__gru_group_0___recurrent_group" + layer_names: "__gru_group_0__" + layer_names: "__last_seq_3__" + input_layer_names: "seq_input" + input_layer_names: "sub_seq_input" + output_layer_names: "__last_seq_0__" + output_layer_names: "__first_seq_0__" + output_layer_names: "__last_seq_1__" + output_layer_names: "__last_seq_2__" + output_layer_names: "__last_seq_3__" + is_recurrent_layer_group: false +} +sub_models { + name: "__recurrent_group_0__" + layer_names: "seq_input@__recurrent_group_0__" + layer_names: "rnn_forward+delay1@__recurrent_group_0__" + layer_names: "rnn_forward@__recurrent_group_0__" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "rnn_forward@__recurrent_group_0__" + link_name: "rnn_forward+delay1@__recurrent_group_0__" + is_sequence: false + } + in_links { + layer_name: "seq_input" + link_name: "seq_input@__recurrent_group_0__" + has_subseq: false + } + out_links { + layer_name: "rnn_forward@__recurrent_group_0__" + link_name: "rnn_forward" + has_subseq: false + } + target_inlinkid: -1 +} +sub_models { + name: "__recurrent_group_1__" + layer_names: "seq_input@__recurrent_group_1__" + layer_names: "rnn_back+delay1@__recurrent_group_1__" + layer_names: "rnn_back@__recurrent_group_1__" + is_recurrent_layer_group: true + reversed: true + memories { + layer_name: "rnn_back@__recurrent_group_1__" + link_name: "rnn_back+delay1@__recurrent_group_1__" + is_sequence: false + } + in_links { + layer_name: "seq_input" + link_name: "seq_input@__recurrent_group_1__" + has_subseq: false + } + out_links { + layer_name: "rnn_back@__recurrent_group_1__" + link_name: "rnn_back" + has_subseq: false + } + target_inlinkid: -1 +} +sub_models { + name: "__recurrent_group_2__" + layer_names: "sub_seq_input@__recurrent_group_2__" + layer_names: "rnn_subseq_forward+delay1@__recurrent_group_2__" + layer_names: "rnn_subseq_forward@__recurrent_group_2__" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "rnn_subseq_forward@__recurrent_group_2__" + link_name: "rnn_subseq_forward+delay1@__recurrent_group_2__" + is_sequence: false + } + in_links { + layer_name: "sub_seq_input" + link_name: "sub_seq_input@__recurrent_group_2__" + has_subseq: true + } + out_links { + layer_name: "rnn_subseq_forward@__recurrent_group_2__" + link_name: "rnn_subseq_forward" + has_subseq: true + } + target_inlinkid: -1 +} +sub_models { + name: "__lstm_group_0___recurrent_group" + layer_names: "__mixed_0__@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0__@__lstm_group_0___recurrent_group" + layer_names: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" + is_sequence: false + } + memories { + layer_name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" + is_sequence: false + } + in_links { + layer_name: "__mixed_0__" + link_name: "__mixed_0__@__lstm_group_0___recurrent_group" + has_subseq: false + } + out_links { + layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" + link_name: "__lstm_group_0__" + has_subseq: false + } + target_inlinkid: -1 +} +sub_models { + name: "__gru_group_0___recurrent_group" + layer_names: "__mixed_1__@__gru_group_0___recurrent_group" + layer_names: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" + layer_names: "__gru_group_0__@__gru_group_0___recurrent_group" + is_recurrent_layer_group: true + reversed: false + memories { + layer_name: "__gru_group_0__@__gru_group_0___recurrent_group" + link_name: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" + is_sequence: false + } + in_links { + layer_name: "__mixed_1__" + link_name: "__mixed_1__@__gru_group_0___recurrent_group" + has_subseq: false + } + out_links { + layer_name: "__gru_group_0__@__gru_group_0___recurrent_group" + link_name: "__gru_group_0__" + has_subseq: false + } + target_inlinkid: -1 +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.protostr new file mode 100644 index 0000000000000..1999c006d237e --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.protostr @@ -0,0 +1,111 @@ +type: "nn" +layers { + name: "dat_in" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__seq_pooling_0__" + type: "max" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + trans_type: "seq" +} +layers { + name: "__seq_pooling_1__" + type: "max" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + trans_type: "non-seq" +} +layers { + name: "__seq_pooling_2__" + type: "average" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + average_strategy: "average" + trans_type: "seq" +} +layers { + name: "__seq_pooling_3__" + type: "average" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + average_strategy: "average" + trans_type: "non-seq" +} +layers { + name: "__seq_pooling_4__" + type: "average" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + average_strategy: "sum" + trans_type: "seq" +} +layers { + name: "__seq_pooling_5__" + type: "average" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + average_strategy: "sum" + trans_type: "non-seq" +} +layers { + name: "__seq_pooling_6__" + type: "max" + size: 100 + active_type: "linear" + inputs { + input_layer_name: "dat_in" + } + output_max_index: true + trans_type: "non-seq" +} +input_layer_names: "dat_in" +output_layer_names: "__seq_pooling_0__" +output_layer_names: "__seq_pooling_1__" +output_layer_names: "__seq_pooling_2__" +output_layer_names: "__seq_pooling_3__" +output_layer_names: "__seq_pooling_4__" +output_layer_names: "__seq_pooling_5__" +output_layer_names: "__seq_pooling_6__" +sub_models { + name: "root" + layer_names: "dat_in" + layer_names: "__seq_pooling_0__" + layer_names: "__seq_pooling_1__" + layer_names: "__seq_pooling_2__" + layer_names: "__seq_pooling_3__" + layer_names: "__seq_pooling_4__" + layer_names: "__seq_pooling_5__" + layer_names: "__seq_pooling_6__" + input_layer_names: "dat_in" + output_layer_names: "__seq_pooling_0__" + output_layer_names: "__seq_pooling_1__" + output_layer_names: "__seq_pooling_2__" + output_layer_names: "__seq_pooling_3__" + output_layer_names: "__seq_pooling_4__" + output_layer_names: "__seq_pooling_5__" + output_layer_names: "__seq_pooling_6__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/unused_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/unused_layers.protostr new file mode 100644 index 0000000000000..89ed28406e553 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/unused_layers.protostr @@ -0,0 +1,27 @@ +type: "nn" +layers { + name: "probs" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__sampling_id_layer_0__" + type: "sampling_id" + size: 100 + active_type: "" + inputs { + input_layer_name: "probs" + } +} +input_layer_names: "probs" +output_layer_names: "__sampling_id_layer_0__" +sub_models { + name: "root" + layer_names: "probs" + layer_names: "__sampling_id_layer_0__" + input_layer_names: "probs" + output_layer_names: "__sampling_id_layer_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/util_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/util_layers.protostr new file mode 100644 index 0000000000000..d0ad388165007 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/util_layers.protostr @@ -0,0 +1,81 @@ +type: "nn" +layers { + name: "a" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "b" + type: "data" + size: 10 + active_type: "" +} +layers { + name: "__addto_0__" + type: "addto" + size: 10 + active_type: "" + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "b" + } +} +layers { + name: "__concat_0__" + type: "concat" + size: 20 + active_type: "" + inputs { + input_layer_name: "a" + } + inputs { + input_layer_name: "b" + } +} +layers { + name: "__concat_1__" + type: "concat2" + size: 20 + active_type: "" + inputs { + input_layer_name: "a" + proj_conf { + type: "identity" + name: "___concat_1__.w0" + input_size: 10 + output_size: 10 + } + } + inputs { + input_layer_name: "b" + proj_conf { + type: "identity" + name: "___concat_1__.w1" + input_size: 10 + output_size: 10 + } + } +} +input_layer_names: "a" +input_layer_names: "b" +output_layer_names: "__addto_0__" +output_layer_names: "__concat_0__" +output_layer_names: "__concat_1__" +sub_models { + name: "root" + layer_names: "a" + layer_names: "b" + layer_names: "__addto_0__" + layer_names: "__concat_0__" + layer_names: "__concat_1__" + input_layer_names: "a" + input_layer_names: "b" + output_layer_names: "__addto_0__" + output_layer_names: "__concat_0__" + output_layer_names: "__concat_1__" + is_recurrent_layer_group: false +} + From 4dada9c7ce40bb424ebea43ac57396d65995644f Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 9 Nov 2016 19:30:08 +0800 Subject: [PATCH 258/324] Delelte old protostr --- .../tests/configs/img_layers.protostr | 176 ----- .../tests/configs/last_first_seq.protostr | 69 -- .../tests/configs/layer_activations.protostr | 423 ------------ .../tests/configs/projections.protostr | 315 --------- .../tests/configs/shared_fc.protostr | 0 .../tests/configs/shared_lstm.protostr | 393 ----------- .../tests/configs/simple_rnn_layers.protostr | 418 ----------- .../configs/test_bilinear_interp.protostr | 125 ---- .../tests/configs/test_cost_layers.protostr | 289 -------- .../test_cost_layers_with_weight.protostr | 111 --- .../tests/configs/test_expand_layer.protostr | 56 -- .../tests/configs/test_fc.protostr | 98 --- .../configs/test_grumemory_layer.protostr | 51 -- .../tests/configs/test_hsigmoid.protostr | 62 -- .../configs/test_lstmemory_layer.protostr | 53 -- .../tests/configs/test_maxout.protostr | 0 .../tests/configs/test_ntm_layers.protostr | 225 ------ .../tests/configs/test_print_layer.protostr | 26 - .../tests/configs/test_rnn_group.protostr | 650 ------------------ .../configs/test_sequence_pooling.protostr | 111 --- .../tests/configs/unused_layers.protostr | 27 - .../tests/configs/util_layers.protostr | 81 --- 22 files changed, 3759 deletions(-) delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/img_layers.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/last_first_seq.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/layer_activations.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/projections.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/shared_fc.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/shared_lstm.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_fc.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_maxout.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_print_layer.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/unused_layers.protostr delete mode 100644 python/paddle/trainer_config_helpers/tests/configs/util_layers.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/img_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/img_layers.protostr deleted file mode 100644 index 899171ff1d00b..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/img_layers.protostr +++ /dev/null @@ -1,176 +0,0 @@ -type: "nn" -layers { - name: "image" - type: "data" - size: 65536 - active_type: "" -} -layers { - name: "__conv_0__" - type: "exconv" - size: 3297856 - active_type: "" - inputs { - input_layer_name: "image" - input_parameter_name: "___conv_0__.w0" - conv_conf { - filter_size: 32 - channels: 1 - stride: 1 - padding: 1 - groups: 1 - filter_channels: 1 - output_x: 227 - img_size: 256 - caffe_mode: true - filter_size_y: 32 - padding_y: 1 - stride_y: 1 - } - } - bias_parameter_name: "___conv_0__.wbias" - num_filters: 64 - shared_biases: true -} -layers { - name: "__batch_norm_0__" - type: "batch_norm" - size: 3297856 - active_type: "relu" - inputs { - input_layer_name: "__conv_0__" - input_parameter_name: "___batch_norm_0__.w0" - image_conf { - channels: 64 - img_size: 227 - } - } - inputs { - input_layer_name: "__conv_0__" - input_parameter_name: "___batch_norm_0__.w1" - } - inputs { - input_layer_name: "__conv_0__" - input_parameter_name: "___batch_norm_0__.w2" - } - bias_parameter_name: "___batch_norm_0__.wbias" - moving_average_fraction: 0.899999976158 -} -layers { - name: "__crmnorm_0__" - type: "norm" - size: 3297856 - active_type: "" - inputs { - input_layer_name: "__batch_norm_0__" - norm_conf { - norm_type: "cmrnorm-projection" - channels: 64 - size: 32 - scale: 0.000399999989895 - pow: 0.75 - output_x: 227 - img_size: 227 - blocked: false - } - } -} -layers { - name: "__pool_0__" - type: "pool" - size: 2458624 - active_type: "" - inputs { - input_layer_name: "__conv_0__" - pool_conf { - pool_type: "max-projection" - channels: 64 - size_x: 32 - stride: 1 - output_x: 196 - img_size: 227 - padding: 0 - size_y: 32 - stride_y: 1 - output_y: 196 - img_size_y: 227 - padding_y: 0 - } - } -} -parameters { - name: "___conv_0__.w0" - size: 65536 - initial_mean: 0.0 - initial_std: 0.0441941730678 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___conv_0__.wbias" - size: 64 - initial_mean: 0.0 - initial_std: 0.0 - dims: 64 - dims: 1 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___batch_norm_0__.w0" - size: 64 - initial_mean: 1.0 - initial_std: 0.0 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___batch_norm_0__.w1" - size: 64 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 64 - initial_strategy: 0 - initial_smart: false - is_static: true - is_shared: true -} -parameters { - name: "___batch_norm_0__.w2" - size: 64 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 64 - initial_strategy: 0 - initial_smart: false - is_static: true - is_shared: true -} -parameters { - name: "___batch_norm_0__.wbias" - size: 64 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 64 - initial_strategy: 0 - initial_smart: false -} -input_layer_names: "image" -output_layer_names: "__pool_0__" -output_layer_names: "__crmnorm_0__" -sub_models { - name: "root" - layer_names: "image" - layer_names: "__conv_0__" - layer_names: "__batch_norm_0__" - layer_names: "__crmnorm_0__" - layer_names: "__pool_0__" - input_layer_names: "image" - output_layer_names: "__pool_0__" - output_layer_names: "__crmnorm_0__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.protostr b/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.protostr deleted file mode 100644 index 7b2911f8e367e..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/last_first_seq.protostr +++ /dev/null @@ -1,69 +0,0 @@ -type: "nn" -layers { - name: "data" - type: "data" - size: 30 - active_type: "" -} -layers { - name: "__first_seq_0__" - type: "seqlastins" - size: 30 - active_type: "linear" - inputs { - input_layer_name: "data" - } - select_first: true - trans_type: "seq" -} -layers { - name: "__first_seq_1__" - type: "seqlastins" - size: 30 - active_type: "linear" - inputs { - input_layer_name: "data" - } - select_first: true - trans_type: "non-seq" -} -layers { - name: "__last_seq_0__" - type: "seqlastins" - size: 30 - active_type: "linear" - inputs { - input_layer_name: "data" - } - trans_type: "seq" -} -layers { - name: "__last_seq_1__" - type: "seqlastins" - size: 30 - active_type: "linear" - inputs { - input_layer_name: "data" - } - trans_type: "non-seq" -} -input_layer_names: "data" -output_layer_names: "__first_seq_0__" -output_layer_names: "__first_seq_1__" -output_layer_names: "__last_seq_0__" -output_layer_names: "__last_seq_1__" -sub_models { - name: "root" - layer_names: "data" - layer_names: "__first_seq_0__" - layer_names: "__first_seq_1__" - layer_names: "__last_seq_0__" - layer_names: "__last_seq_1__" - input_layer_names: "data" - output_layer_names: "__first_seq_0__" - output_layer_names: "__first_seq_1__" - output_layer_names: "__last_seq_0__" - output_layer_names: "__last_seq_1__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/layer_activations.protostr b/python/paddle/trainer_config_helpers/tests/configs/layer_activations.protostr deleted file mode 100644 index 8ae2421727efe..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/layer_activations.protostr +++ /dev/null @@ -1,423 +0,0 @@ -type: "nn" -layers { - name: "input" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "layer_0" - type: "fc" - size: 100 - active_type: "tanh" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_0.w0" - } - bias_parameter_name: "_layer_0.wbias" -} -layers { - name: "layer_1" - type: "fc" - size: 100 - active_type: "sigmoid" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_1.w0" - } - bias_parameter_name: "_layer_1.wbias" -} -layers { - name: "layer_2" - type: "fc" - size: 100 - active_type: "softmax" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_2.w0" - } - bias_parameter_name: "_layer_2.wbias" -} -layers { - name: "layer_3" - type: "fc" - size: 100 - active_type: "" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_3.w0" - } - bias_parameter_name: "_layer_3.wbias" -} -layers { - name: "layer_4" - type: "fc" - size: 100 - active_type: "" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_4.w0" - } - bias_parameter_name: "_layer_4.wbias" -} -layers { - name: "layer_5" - type: "fc" - size: 100 - active_type: "exponential" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_5.w0" - } - bias_parameter_name: "_layer_5.wbias" -} -layers { - name: "layer_6" - type: "fc" - size: 100 - active_type: "relu" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_6.w0" - } - bias_parameter_name: "_layer_6.wbias" -} -layers { - name: "layer_7" - type: "fc" - size: 100 - active_type: "brelu" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_7.w0" - } - bias_parameter_name: "_layer_7.wbias" -} -layers { - name: "layer_8" - type: "fc" - size: 100 - active_type: "softrelu" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_8.w0" - } - bias_parameter_name: "_layer_8.wbias" -} -layers { - name: "layer_9" - type: "fc" - size: 100 - active_type: "stanh" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_9.w0" - } - bias_parameter_name: "_layer_9.wbias" -} -layers { - name: "layer_10" - type: "fc" - size: 100 - active_type: "abs" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_10.w0" - } - bias_parameter_name: "_layer_10.wbias" -} -layers { - name: "layer_11" - type: "fc" - size: 100 - active_type: "square" - inputs { - input_layer_name: "input" - input_parameter_name: "_layer_11.w0" - } - bias_parameter_name: "_layer_11.wbias" -} -parameters { - name: "_layer_0.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_0.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_layer_1.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_1.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_layer_2.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_2.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_layer_3.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_3.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_layer_4.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_4.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_layer_5.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_5.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_layer_6.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_6.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_layer_7.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_7.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_layer_8.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_8.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_layer_9.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_9.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_layer_10.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_10.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_layer_11.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_layer_11.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -input_layer_names: "input" -output_layer_names: "layer_0" -output_layer_names: "layer_1" -output_layer_names: "layer_2" -output_layer_names: "layer_3" -output_layer_names: "layer_4" -output_layer_names: "layer_5" -output_layer_names: "layer_6" -output_layer_names: "layer_7" -output_layer_names: "layer_8" -output_layer_names: "layer_9" -output_layer_names: "layer_10" -output_layer_names: "layer_11" -sub_models { - name: "root" - layer_names: "input" - layer_names: "layer_0" - layer_names: "layer_1" - layer_names: "layer_2" - layer_names: "layer_3" - layer_names: "layer_4" - layer_names: "layer_5" - layer_names: "layer_6" - layer_names: "layer_7" - layer_names: "layer_8" - layer_names: "layer_9" - layer_names: "layer_10" - layer_names: "layer_11" - input_layer_names: "input" - output_layer_names: "layer_0" - output_layer_names: "layer_1" - output_layer_names: "layer_2" - output_layer_names: "layer_3" - output_layer_names: "layer_4" - output_layer_names: "layer_5" - output_layer_names: "layer_6" - output_layer_names: "layer_7" - output_layer_names: "layer_8" - output_layer_names: "layer_9" - output_layer_names: "layer_10" - output_layer_names: "layer_11" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/projections.protostr b/python/paddle/trainer_config_helpers/tests/configs/projections.protostr deleted file mode 100644 index a901af6b42431..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/projections.protostr +++ /dev/null @@ -1,315 +0,0 @@ -type: "nn" -layers { - name: "test" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "__embedding_0__" - type: "mixed" - size: 256 - active_type: "" - inputs { - input_layer_name: "test" - input_parameter_name: "___embedding_0__.w0" - proj_conf { - type: "table" - name: "___embedding_0__.w0" - input_size: 100 - output_size: 256 - } - } -} -layers { - name: "__mixed_0__" - type: "mixed" - size: 100 - active_type: "" - inputs { - input_layer_name: "__embedding_0__" - input_parameter_name: "___mixed_0__.w0" - proj_conf { - type: "fc" - name: "___mixed_0__.w0" - input_size: 256 - output_size: 100 - } - } -} -layers { - name: "__mixed_1__" - type: "mixed" - size: 100 - active_type: "" - inputs { - input_layer_name: "__mixed_0__" - input_parameter_name: "___mixed_1__.w0" - proj_conf { - type: "table" - name: "___mixed_1__.w0" - input_size: 100 - output_size: 100 - } - } -} -layers { - name: "__mixed_2__" - type: "mixed" - size: 100 - active_type: "" - inputs { - input_layer_name: "__mixed_1__" - proj_conf { - type: "identity" - name: "___mixed_2__.w0" - input_size: 100 - output_size: 100 - } - } -} -layers { - name: "__mixed_3__" - type: "mixed" - size: 100 - active_type: "" - inputs { - input_layer_name: "__mixed_2__" - input_parameter_name: "___mixed_3__.w0" - proj_conf { - type: "dot_mul" - name: "___mixed_3__.w0" - input_size: 100 - output_size: 100 - } - } -} -layers { - name: "__mixed_4__" - type: "mixed" - size: 300 - active_type: "" - inputs { - input_layer_name: "__mixed_3__" - input_parameter_name: "___mixed_4__.w0" - proj_conf { - type: "context" - name: "___mixed_4__.w0" - input_size: 100 - output_size: 300 - context_start: -1 - context_length: 3 - trainable_padding: true - } - } -} -layers { - name: "__mixed_5__" - type: "mixed" - size: 100 - active_type: "" - inputs { - input_layer_name: "__mixed_2__" - } - inputs { - input_layer_name: "__mixed_3__" - } - operator_confs { - type: "dot_mul" - input_indices: 0 - input_indices: 1 - input_sizes: 100 - input_sizes: 100 - output_size: 100 - dotmul_scale: 1.0 - } -} -layers { - name: "img" - type: "data" - size: 1024 - active_type: "" -} -layers { - name: "filter" - type: "data" - size: 576 - active_type: "" -} -layers { - name: "__mixed_6__" - type: "mixed" - size: 57600 - active_type: "" - inputs { - input_layer_name: "img" - } - inputs { - input_layer_name: "filter" - } - operator_confs { - type: "conv" - input_indices: 0 - input_indices: 1 - input_sizes: 1024 - input_sizes: 576 - output_size: 57600 - conv_conf { - filter_size: 3 - channels: 1 - stride: 1 - padding: 0 - groups: 1 - filter_channels: 1 - output_x: 30 - img_size: 32 - caffe_mode: true - filter_size_y: 3 - padding_y: 0 - stride_y: 1 - } - num_filters: 64 - } -} -layers { - name: "__mixed_7__" - type: "mixed" - size: 100 - active_type: "" - inputs { - input_layer_name: "__mixed_4__" - input_parameter_name: "___mixed_7__.w0" - proj_conf { - type: "fc" - name: "___mixed_7__.w0" - input_size: 300 - output_size: 100 - } - } - inputs { - input_layer_name: "__mixed_5__" - input_parameter_name: "___mixed_7__.w1" - proj_conf { - type: "trans_fc" - name: "___mixed_7__.w1" - input_size: 100 - output_size: 100 - } - } - inputs { - input_layer_name: "__mixed_6__" - input_parameter_name: "___mixed_7__.w2" - proj_conf { - type: "fc" - name: "___mixed_7__.w2" - input_size: 57600 - output_size: 100 - } - } - drop_rate: 0.5 -} -parameters { - name: "___embedding_0__.w0" - size: 25600 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 256 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___mixed_0__.w0" - size: 25600 - initial_mean: 0.0 - initial_std: 0.0625 - dims: 256 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___mixed_1__.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___mixed_3__.w0" - size: 100 - initial_mean: 0.0 - initial_std: 1.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___mixed_4__.w0" - size: 200 - initial_mean: 0.0 - initial_std: 0.0 - dims: 2 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___mixed_7__.w0" - size: 30000 - initial_mean: 0.0 - initial_std: 0.0577350258827 - dims: 300 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___mixed_7__.w1" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___mixed_7__.w2" - size: 5760000 - initial_mean: 0.0 - initial_std: 0.00416666688398 - dims: 57600 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -input_layer_names: "test" -input_layer_names: "img" -input_layer_names: "filter" -output_layer_names: "__mixed_7__" -sub_models { - name: "root" - layer_names: "test" - layer_names: "__embedding_0__" - layer_names: "__mixed_0__" - layer_names: "__mixed_1__" - layer_names: "__mixed_2__" - layer_names: "__mixed_3__" - layer_names: "__mixed_4__" - layer_names: "__mixed_5__" - layer_names: "img" - layer_names: "filter" - layer_names: "__mixed_6__" - layer_names: "__mixed_7__" - input_layer_names: "test" - input_layer_names: "img" - input_layer_names: "filter" - output_layer_names: "__mixed_7__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/shared_fc.protostr b/python/paddle/trainer_config_helpers/tests/configs/shared_fc.protostr deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.protostr b/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.protostr deleted file mode 100644 index 26eed43a459f5..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/shared_lstm.protostr +++ /dev/null @@ -1,393 +0,0 @@ -type: "recurrent_nn" -layers { - name: "data_a" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "data_b" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "__mixed_0__" - type: "mixed" - size: 400 - active_type: "" - inputs { - input_layer_name: "data_a" - input_parameter_name: "mixed_param" - proj_conf { - type: "fc" - name: "___mixed_0__.w0" - input_size: 100 - output_size: 400 - } - } -} -layers { - name: "__mixed_1__" - type: "mixed" - size: 400 - active_type: "" - inputs { - input_layer_name: "data_b" - input_parameter_name: "mixed_param" - proj_conf { - type: "fc" - name: "___mixed_1__.w0" - input_size: 100 - output_size: 400 - } - } -} -layers { - name: "__lstm_group_0___recurrent_group" - type: "recurrent_layer_group" - active_type: "" -} -layers { - name: "__mixed_0__@__lstm_group_0___recurrent_group" - type: "scatter_agent" - size: 400 - active_type: "" -} -layers { - name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" - type: "agent" - size: 100 - active_type: "" -} -layers { - name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" - type: "agent" - size: 100 - active_type: "" -} -layers { - name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" - type: "mixed" - size: 400 - active_type: "" - inputs { - input_layer_name: "__mixed_0__@__lstm_group_0___recurrent_group" - proj_conf { - type: "identity" - name: "___lstm_group_0___input_recurrent.w0" - input_size: 400 - output_size: 400 - } - } - inputs { - input_layer_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" - input_parameter_name: "lstm_param" - proj_conf { - type: "fc" - name: "___lstm_group_0___input_recurrent.w1" - input_size: 100 - output_size: 400 - } - } -} -layers { - name: "__lstm_group_0__@__lstm_group_0___recurrent_group" - type: "lstm_step" - size: 100 - active_type: "tanh" - inputs { - input_layer_name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" - } - inputs { - input_layer_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" - } - bias_parameter_name: "lstm_bias" - active_gate_type: "sigmoid" - active_state_type: "sigmoid" -} -layers { - name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" - type: "get_output" - size: 100 - active_type: "" - inputs { - input_layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" - input_layer_argument: "state" - } -} -layers { - name: "__lstm_group_0__" - type: "gather_agent" - size: 100 - active_type: "" -} -layers { - name: "__lstm_group_1___recurrent_group" - type: "recurrent_layer_group" - active_type: "" -} -layers { - name: "__mixed_1__@__lstm_group_1___recurrent_group" - type: "scatter_agent" - size: 400 - active_type: "" -} -layers { - name: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" - type: "agent" - size: 100 - active_type: "" -} -layers { - name: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" - type: "agent" - size: 100 - active_type: "" -} -layers { - name: "__lstm_group_1___input_recurrent@__lstm_group_1___recurrent_group" - type: "mixed" - size: 400 - active_type: "" - inputs { - input_layer_name: "__mixed_1__@__lstm_group_1___recurrent_group" - proj_conf { - type: "identity" - name: "___lstm_group_1___input_recurrent.w0" - input_size: 400 - output_size: 400 - } - } - inputs { - input_layer_name: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" - input_parameter_name: "lstm_param" - proj_conf { - type: "fc" - name: "___lstm_group_1___input_recurrent.w1" - input_size: 100 - output_size: 400 - } - } -} -layers { - name: "__lstm_group_1__@__lstm_group_1___recurrent_group" - type: "lstm_step" - size: 100 - active_type: "tanh" - inputs { - input_layer_name: "__lstm_group_1___input_recurrent@__lstm_group_1___recurrent_group" - } - inputs { - input_layer_name: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" - } - bias_parameter_name: "lstm_bias" - active_gate_type: "sigmoid" - active_state_type: "sigmoid" -} -layers { - name: "__lstm_group_1___state@__lstm_group_1___recurrent_group" - type: "get_output" - size: 100 - active_type: "" - inputs { - input_layer_name: "__lstm_group_1__@__lstm_group_1___recurrent_group" - input_layer_argument: "state" - } -} -layers { - name: "__lstm_group_1__" - type: "gather_agent" - size: 100 - active_type: "" -} -layers { - name: "__last_seq_0__" - type: "seqlastins" - size: 100 - active_type: "linear" - inputs { - input_layer_name: "__lstm_group_0__" - } - trans_type: "non-seq" -} -layers { - name: "__last_seq_1__" - type: "seqlastins" - size: 100 - active_type: "linear" - inputs { - input_layer_name: "__lstm_group_1__" - } - trans_type: "non-seq" -} -layers { - name: "__fc_layer_0__" - type: "fc" - size: 10 - active_type: "softmax" - inputs { - input_layer_name: "__last_seq_0__" - input_parameter_name: "softmax_param" - } - inputs { - input_layer_name: "__last_seq_1__" - input_parameter_name: "softmax_param" - } -} -layers { - name: "label" - type: "data" - size: 10 - active_type: "" -} -layers { - name: "__cost_0__" - type: "multi-class-cross-entropy" - size: 1 - active_type: "" - inputs { - input_layer_name: "__fc_layer_0__" - } - inputs { - input_layer_name: "label" - } - coeff: 1.0 -} -parameters { - name: "mixed_param" - size: 40000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 400 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "lstm_param" - size: 40000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 400 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "lstm_bias" - size: 300 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 300 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "softmax_param" - size: 1000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 10 - initial_strategy: 0 - initial_smart: true -} -input_layer_names: "data_a" -input_layer_names: "data_b" -input_layer_names: "label" -output_layer_names: "__cost_0__" -evaluators { - name: "classification_error_evaluator" - type: "classification_error" - input_layers: "__fc_layer_0__" - input_layers: "label" -} -sub_models { - name: "root" - layer_names: "data_a" - layer_names: "data_b" - layer_names: "__mixed_0__" - layer_names: "__mixed_1__" - layer_names: "__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0__" - layer_names: "__lstm_group_1___recurrent_group" - layer_names: "__lstm_group_1__" - layer_names: "__last_seq_0__" - layer_names: "__last_seq_1__" - layer_names: "__fc_layer_0__" - layer_names: "label" - layer_names: "__cost_0__" - input_layer_names: "data_a" - input_layer_names: "data_b" - input_layer_names: "label" - output_layer_names: "__cost_0__" - evaluator_names: "classification_error_evaluator" - is_recurrent_layer_group: false -} -sub_models { - name: "__lstm_group_0___recurrent_group" - layer_names: "__mixed_0__@__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0__@__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0___state@__lstm_group_0___recurrent_group" - is_recurrent_layer_group: true - reversed: false - memories { - layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" - link_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" - is_sequence: false - } - memories { - layer_name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" - link_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" - is_sequence: false - } - in_links { - layer_name: "__mixed_0__" - link_name: "__mixed_0__@__lstm_group_0___recurrent_group" - has_subseq: false - } - out_links { - layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" - link_name: "__lstm_group_0__" - has_subseq: false - } - target_inlinkid: -1 -} -sub_models { - name: "__lstm_group_1___recurrent_group" - layer_names: "__mixed_1__@__lstm_group_1___recurrent_group" - layer_names: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" - layer_names: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" - layer_names: "__lstm_group_1___input_recurrent@__lstm_group_1___recurrent_group" - layer_names: "__lstm_group_1__@__lstm_group_1___recurrent_group" - layer_names: "__lstm_group_1___state@__lstm_group_1___recurrent_group" - is_recurrent_layer_group: true - reversed: false - memories { - layer_name: "__lstm_group_1__@__lstm_group_1___recurrent_group" - link_name: "__lstm_group_1__+delay1@__lstm_group_1___recurrent_group" - is_sequence: false - } - memories { - layer_name: "__lstm_group_1___state@__lstm_group_1___recurrent_group" - link_name: "__lstm_group_1___state+delay1@__lstm_group_1___recurrent_group" - is_sequence: false - } - in_links { - layer_name: "__mixed_1__" - link_name: "__mixed_1__@__lstm_group_1___recurrent_group" - has_subseq: false - } - out_links { - layer_name: "__lstm_group_1__@__lstm_group_1___recurrent_group" - link_name: "__lstm_group_1__" - has_subseq: false - } - target_inlinkid: -1 -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.protostr deleted file mode 100644 index 57445243bd06f..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/simple_rnn_layers.protostr +++ /dev/null @@ -1,418 +0,0 @@ -type: "nn" -layers { - name: "data" - type: "data" - size: 200 - active_type: "" -} -layers { - name: "__fc_layer_0__" - type: "fc" - size: 200 - active_type: "sigmoid" - inputs { - input_layer_name: "data" - input_parameter_name: "___fc_layer_0__.w0" - } - bias_parameter_name: "___fc_layer_0__.wbias" -} -layers { - name: "__recurrent_layer_0__" - type: "recurrent" - size: 200 - active_type: "sigmoid" - inputs { - input_layer_name: "__fc_layer_0__" - input_parameter_name: "___recurrent_layer_0__.w0" - } - bias_parameter_name: "___recurrent_layer_0__.wbias" - reversed: false -} -layers { - name: "__recurrent_layer_1__" - type: "recurrent" - size: 200 - active_type: "sigmoid" - inputs { - input_layer_name: "__fc_layer_0__" - input_parameter_name: "___recurrent_layer_1__.w0" - } - bias_parameter_name: "___recurrent_layer_1__.wbias" - reversed: true -} -layers { - name: "__fc_layer_1__" - type: "fc" - size: 800 - active_type: "" - inputs { - input_layer_name: "__fc_layer_0__" - input_parameter_name: "___fc_layer_1__.w0" - } -} -layers { - name: "__lstmemory_0__" - type: "lstmemory" - size: 200 - active_type: "sigmoid" - inputs { - input_layer_name: "__fc_layer_1__" - input_parameter_name: "___lstmemory_0__.w0" - } - bias_parameter_name: "___lstmemory_0__.wbias" - reversed: false - active_gate_type: "sigmoid" - active_state_type: "tanh" -} -layers { - name: "__fc_layer_2__" - type: "fc" - size: 800 - active_type: "" - inputs { - input_layer_name: "__fc_layer_0__" - input_parameter_name: "___fc_layer_2__.w0" - } -} -layers { - name: "__lstmemory_1__" - type: "lstmemory" - size: 200 - active_type: "sigmoid" - inputs { - input_layer_name: "__fc_layer_2__" - input_parameter_name: "___lstmemory_1__.w0" - } - bias_parameter_name: "___lstmemory_1__.wbias" - reversed: true - active_gate_type: "sigmoid" - active_state_type: "tanh" -} -layers { - name: "__fc_layer_3__" - type: "fc" - size: 600 - active_type: "" - inputs { - input_layer_name: "__fc_layer_0__" - input_parameter_name: "___fc_layer_3__.w0" - } -} -layers { - name: "__gru_0__" - type: "gated_recurrent" - size: 200 - active_type: "sigmoid" - inputs { - input_layer_name: "__fc_layer_3__" - input_parameter_name: "___gru_0__.w0" - } - bias_parameter_name: "___gru_0__.wbias" - reversed: false - active_gate_type: "sigmoid" -} -layers { - name: "__fc_layer_4__" - type: "fc" - size: 600 - active_type: "" - inputs { - input_layer_name: "__fc_layer_0__" - input_parameter_name: "___fc_layer_4__.w0" - } -} -layers { - name: "__gru_1__" - type: "gated_recurrent" - size: 200 - active_type: "sigmoid" - inputs { - input_layer_name: "__fc_layer_4__" - input_parameter_name: "___gru_1__.w0" - } - bias_parameter_name: "___gru_1__.wbias" - reversed: true - active_gate_type: "sigmoid" -} -layers { - name: "__last_seq_0__" - type: "seqlastins" - size: 200 - active_type: "linear" - inputs { - input_layer_name: "__recurrent_layer_0__" - } - trans_type: "non-seq" -} -layers { - name: "__first_seq_0__" - type: "seqlastins" - size: 200 - active_type: "linear" - inputs { - input_layer_name: "__recurrent_layer_1__" - } - select_first: true - trans_type: "non-seq" -} -layers { - name: "__last_seq_1__" - type: "seqlastins" - size: 200 - active_type: "linear" - inputs { - input_layer_name: "__lstmemory_0__" - } - trans_type: "non-seq" -} -layers { - name: "__first_seq_1__" - type: "seqlastins" - size: 200 - active_type: "linear" - inputs { - input_layer_name: "__lstmemory_1__" - } - select_first: true - trans_type: "non-seq" -} -layers { - name: "__last_seq_2__" - type: "seqlastins" - size: 200 - active_type: "linear" - inputs { - input_layer_name: "__gru_0__" - } - trans_type: "non-seq" -} -layers { - name: "__first_seq_2__" - type: "seqlastins" - size: 200 - active_type: "linear" - inputs { - input_layer_name: "__gru_1__" - } - select_first: true - trans_type: "non-seq" -} -parameters { - name: "___fc_layer_0__.w0" - size: 40000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 200 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___fc_layer_0__.wbias" - size: 200 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 200 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___recurrent_layer_0__.w0" - size: 40000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 200 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___recurrent_layer_0__.wbias" - size: 200 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 200 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___recurrent_layer_1__.w0" - size: 40000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 200 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___recurrent_layer_1__.wbias" - size: 200 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 200 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___fc_layer_1__.w0" - size: 160000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 800 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___lstmemory_0__.w0" - size: 160000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 200 - dims: 4 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___lstmemory_0__.wbias" - size: 1400 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 1400 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___fc_layer_2__.w0" - size: 160000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 800 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___lstmemory_1__.w0" - size: 160000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 200 - dims: 4 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___lstmemory_1__.wbias" - size: 1400 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 1400 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___fc_layer_3__.w0" - size: 120000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 600 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___gru_0__.w0" - size: 120000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 600 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___gru_0__.wbias" - size: 600 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 600 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___fc_layer_4__.w0" - size: 120000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 600 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___gru_1__.w0" - size: 120000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 600 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___gru_1__.wbias" - size: 600 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 600 - initial_strategy: 0 - initial_smart: false -} -input_layer_names: "data" -output_layer_names: "__last_seq_0__" -output_layer_names: "__first_seq_0__" -output_layer_names: "__last_seq_1__" -output_layer_names: "__first_seq_1__" -output_layer_names: "__last_seq_2__" -output_layer_names: "__first_seq_2__" -sub_models { - name: "root" - layer_names: "data" - layer_names: "__fc_layer_0__" - layer_names: "__recurrent_layer_0__" - layer_names: "__recurrent_layer_1__" - layer_names: "__fc_layer_1__" - layer_names: "__lstmemory_0__" - layer_names: "__fc_layer_2__" - layer_names: "__lstmemory_1__" - layer_names: "__fc_layer_3__" - layer_names: "__gru_0__" - layer_names: "__fc_layer_4__" - layer_names: "__gru_1__" - layer_names: "__last_seq_0__" - layer_names: "__first_seq_0__" - layer_names: "__last_seq_1__" - layer_names: "__first_seq_1__" - layer_names: "__last_seq_2__" - layer_names: "__first_seq_2__" - input_layer_names: "data" - output_layer_names: "__last_seq_0__" - output_layer_names: "__first_seq_0__" - output_layer_names: "__last_seq_1__" - output_layer_names: "__first_seq_1__" - output_layer_names: "__last_seq_2__" - output_layer_names: "__first_seq_2__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.protostr deleted file mode 100644 index 278088d4abd50..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_bilinear_interp.protostr +++ /dev/null @@ -1,125 +0,0 @@ -type: "nn" -layers { - name: "data" - type: "data" - size: 2304 - active_type: "" -} -layers { - name: "__conv_0__" - type: "exconv" - size: 36864 - active_type: "" - inputs { - input_layer_name: "data" - input_parameter_name: "___conv_0__.w0" - conv_conf { - filter_size: 3 - channels: 1 - stride: 1 - padding: 1 - groups: 1 - filter_channels: 1 - output_x: 48 - img_size: 48 - caffe_mode: true - filter_size_y: 3 - padding_y: 1 - stride_y: 1 - } - } - bias_parameter_name: "___conv_0__.wbias" - num_filters: 16 - shared_biases: true -} -layers { - name: "__bilinear_interp_layer_0__" - type: "bilinear_interp" - size: 36864 - active_type: "" - inputs { - input_layer_name: "__conv_0__" - bilinear_interp_conf { - img_size_x: 32 - img_size_y: 32 - out_size_x: 64 - out_size_y: 64 - num_channels: 16 - } - } -} -layers { - name: "__pool_0__" - type: "pool" - size: 9216 - active_type: "" - inputs { - input_layer_name: "__bilinear_interp_layer_0__" - pool_conf { - pool_type: "max-projection" - channels: 4 - size_x: 2 - stride: 2 - output_x: 48 - img_size: 96 - padding: 0 - size_y: 2 - stride_y: 2 - output_y: 48 - img_size_y: 96 - padding_y: 0 - } - } -} -layers { - name: "__fc_layer_0__" - type: "fc" - size: 384 - active_type: "tanh" - inputs { - input_layer_name: "__pool_0__" - input_parameter_name: "___fc_layer_0__.w0" - } -} -parameters { - name: "___conv_0__.w0" - size: 144 - initial_mean: 0.0 - initial_std: 0.471404522657 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___conv_0__.wbias" - size: 16 - initial_mean: 0.0 - initial_std: 0.0 - dims: 16 - dims: 1 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___fc_layer_0__.w0" - size: 3538944 - initial_mean: 0.0 - initial_std: 0.0104166669771 - dims: 9216 - dims: 384 - initial_strategy: 0 - initial_smart: true -} -input_layer_names: "data" -output_layer_names: "__fc_layer_0__" -sub_models { - name: "root" - layer_names: "data" - layer_names: "__conv_0__" - layer_names: "__bilinear_interp_layer_0__" - layer_names: "__pool_0__" - layer_names: "__fc_layer_0__" - input_layer_names: "data" - output_layer_names: "__fc_layer_0__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.protostr deleted file mode 100644 index c37586f4068e4..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.protostr +++ /dev/null @@ -1,289 +0,0 @@ -type: "nn" -layers { - name: "input" - type: "data" - size: 200 - active_type: "" -} -layers { - name: "labels" - type: "data" - size: 5000 - active_type: "" -} -layers { - name: "probs" - type: "data" - size: 10 - active_type: "" -} -layers { - name: "xe-label" - type: "data" - size: 10 - active_type: "" -} -layers { - name: "__ctc_layer_0__" - type: "ctc" - size: 5001 - active_type: "" - inputs { - input_layer_name: "input" - } - inputs { - input_layer_name: "labels" - } - norm_by_times: false -} -layers { - name: "__fc_layer_0__" - type: "fc" - size: 4 - active_type: "tanh" - inputs { - input_layer_name: "input" - input_parameter_name: "___fc_layer_0__.w0" - } - bias_parameter_name: "___fc_layer_0__.wbias" -} -layers { - name: "crf_label" - type: "data" - size: 4 - active_type: "" -} -layers { - name: "__crf_layer_0__" - type: "crf" - size: 4 - active_type: "" - inputs { - input_layer_name: "__fc_layer_0__" - input_parameter_name: "___crf_layer_0__.w0" - } - inputs { - input_layer_name: "crf_label" - } - coeff: 1.0 -} -layers { - name: "left" - type: "data" - size: 1 - active_type: "" -} -layers { - name: "right" - type: "data" - size: 1 - active_type: "" -} -layers { - name: "label" - type: "data" - size: 1 - active_type: "" -} -layers { - name: "__rank_cost_0__" - type: "rank-cost" - size: 1 - active_type: "" - inputs { - input_layer_name: "left" - } - inputs { - input_layer_name: "right" - } - inputs { - input_layer_name: "label" - } - coeff: 1.0 -} -layers { - name: "list_feature" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "list_scores" - type: "data" - size: 1 - active_type: "" -} -layers { - name: "__lambda_cost_0__" - type: "lambda_cost" - size: 1 - active_type: "" - inputs { - input_layer_name: "list_feature" - } - inputs { - input_layer_name: "list_scores" - } - NDCG_num: 5 - max_sort_size: -1 -} -layers { - name: "__cross_entropy_0__" - type: "multi-class-cross-entropy" - size: 1 - active_type: "" - inputs { - input_layer_name: "probs" - } - inputs { - input_layer_name: "xe-label" - } - coeff: 1.0 -} -layers { - name: "__cross_entropy_with_selfnorm_0__" - type: "multi_class_cross_entropy_with_selfnorm" - active_type: "" - inputs { - input_layer_name: "probs" - } - inputs { - input_layer_name: "xe-label" - } - softmax_selfnorm_alpha: 0.10000000149 - coeff: 1.0 -} -layers { - name: "huber_probs" - type: "data" - size: 1 - active_type: "" -} -layers { - name: "huber_label" - type: "data" - size: 1 - active_type: "" -} -layers { - name: "__huber_cost_0__" - type: "huber" - size: 1 - active_type: "" - inputs { - input_layer_name: "huber_probs" - } - inputs { - input_layer_name: "huber_label" - } - coeff: 1.0 -} -layers { - name: "__multi_binary_label_cross_entropy_0__" - type: "multi_binary_label_cross_entropy" - size: 1 - active_type: "" - inputs { - input_layer_name: "probs" - } - inputs { - input_layer_name: "xe-label" - } - coeff: 1.0 -} -parameters { - name: "___fc_layer_0__.w0" - size: 800 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 4 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___fc_layer_0__.wbias" - size: 4 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 4 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___crf_layer_0__.w0" - size: 24 - initial_mean: 0.0 - initial_std: 0.5 - dims: 4 - dims: 6 - initial_strategy: 0 - initial_smart: true -} -input_layer_names: "input" -input_layer_names: "labels" -input_layer_names: "crf_label" -input_layer_names: "left" -input_layer_names: "right" -input_layer_names: "label" -input_layer_names: "list_feature" -input_layer_names: "list_scores" -input_layer_names: "probs" -input_layer_names: "xe-label" -input_layer_names: "huber_probs" -input_layer_names: "huber_label" -output_layer_names: "__ctc_layer_0__" -output_layer_names: "__crf_layer_0__" -output_layer_names: "__rank_cost_0__" -output_layer_names: "__lambda_cost_0__" -output_layer_names: "__cross_entropy_0__" -output_layer_names: "__cross_entropy_with_selfnorm_0__" -output_layer_names: "__huber_cost_0__" -output_layer_names: "__multi_binary_label_cross_entropy_0__" -sub_models { - name: "root" - layer_names: "input" - layer_names: "labels" - layer_names: "probs" - layer_names: "xe-label" - layer_names: "__ctc_layer_0__" - layer_names: "__fc_layer_0__" - layer_names: "crf_label" - layer_names: "__crf_layer_0__" - layer_names: "left" - layer_names: "right" - layer_names: "label" - layer_names: "__rank_cost_0__" - layer_names: "list_feature" - layer_names: "list_scores" - layer_names: "__lambda_cost_0__" - layer_names: "__cross_entropy_0__" - layer_names: "__cross_entropy_with_selfnorm_0__" - layer_names: "huber_probs" - layer_names: "huber_label" - layer_names: "__huber_cost_0__" - layer_names: "__multi_binary_label_cross_entropy_0__" - input_layer_names: "input" - input_layer_names: "labels" - input_layer_names: "crf_label" - input_layer_names: "left" - input_layer_names: "right" - input_layer_names: "label" - input_layer_names: "list_feature" - input_layer_names: "list_scores" - input_layer_names: "probs" - input_layer_names: "xe-label" - input_layer_names: "huber_probs" - input_layer_names: "huber_label" - output_layer_names: "__ctc_layer_0__" - output_layer_names: "__crf_layer_0__" - output_layer_names: "__rank_cost_0__" - output_layer_names: "__lambda_cost_0__" - output_layer_names: "__cross_entropy_0__" - output_layer_names: "__cross_entropy_with_selfnorm_0__" - output_layer_names: "__huber_cost_0__" - output_layer_names: "__multi_binary_label_cross_entropy_0__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.protostr deleted file mode 100644 index de58f5c64969b..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers_with_weight.protostr +++ /dev/null @@ -1,111 +0,0 @@ -type: "nn" -layers { - name: "input" - type: "data" - size: 300 - active_type: "" -} -layers { - name: "label" - type: "data" - size: 1 - active_type: "" -} -layers { - name: "weight" - type: "data" - size: 1 - active_type: "" -} -layers { - name: "__fc_layer_0__" - type: "fc" - size: 10 - active_type: "softmax" - inputs { - input_layer_name: "input" - input_parameter_name: "___fc_layer_0__.w0" - } - bias_parameter_name: "___fc_layer_0__.wbias" -} -layers { - name: "__cost_0__" - type: "multi-class-cross-entropy" - size: 1 - active_type: "" - inputs { - input_layer_name: "__fc_layer_0__" - } - inputs { - input_layer_name: "label" - } - inputs { - input_layer_name: "weight" - } - coeff: 1.0 -} -layers { - name: "__regression_cost_0__" - type: "square_error" - size: 1 - active_type: "" - inputs { - input_layer_name: "__fc_layer_0__" - } - inputs { - input_layer_name: "label" - } - inputs { - input_layer_name: "weight" - } - coeff: 1.0 -} -parameters { - name: "___fc_layer_0__.w0" - size: 3000 - initial_mean: 0.0 - initial_std: 0.0577350258827 - dims: 300 - dims: 10 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___fc_layer_0__.wbias" - size: 10 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 10 - initial_strategy: 0 - initial_smart: false -} -input_layer_names: "input" -input_layer_names: "label" -input_layer_names: "weight" -output_layer_names: "__cost_0__" -output_layer_names: "__regression_cost_0__" -evaluators { - name: "classification_error_evaluator" - type: "classification_error" - input_layers: "__fc_layer_0__" - input_layers: "label" - input_layers: "weight" -} -sub_models { - name: "root" - layer_names: "input" - layer_names: "label" - layer_names: "weight" - layer_names: "__fc_layer_0__" - layer_names: "__cost_0__" - layer_names: "__regression_cost_0__" - input_layer_names: "input" - input_layer_names: "label" - input_layer_names: "weight" - output_layer_names: "__cost_0__" - output_layer_names: "__regression_cost_0__" - evaluator_names: "classification_error_evaluator" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.protostr deleted file mode 100644 index f4b36052264bc..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_expand_layer.protostr +++ /dev/null @@ -1,56 +0,0 @@ -type: "nn" -layers { - name: "data" - type: "data" - size: 30 - active_type: "" -} -layers { - name: "data_seq" - type: "data" - size: 30 - active_type: "" -} -layers { - name: "__expand_layer_0__" - type: "expand" - size: 30 - active_type: "" - inputs { - input_layer_name: "data" - } - inputs { - input_layer_name: "data_seq" - } - trans_type: "seq" -} -layers { - name: "__expand_layer_1__" - type: "expand" - size: 30 - active_type: "" - inputs { - input_layer_name: "data" - } - inputs { - input_layer_name: "data_seq" - } - trans_type: "non-seq" -} -input_layer_names: "data" -input_layer_names: "data_seq" -output_layer_names: "__expand_layer_0__" -output_layer_names: "__expand_layer_1__" -sub_models { - name: "root" - layer_names: "data" - layer_names: "data_seq" - layer_names: "__expand_layer_0__" - layer_names: "__expand_layer_1__" - input_layer_names: "data" - input_layer_names: "data_seq" - output_layer_names: "__expand_layer_0__" - output_layer_names: "__expand_layer_1__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_fc.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_fc.protostr deleted file mode 100644 index 80b01246ba96f..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_fc.protostr +++ /dev/null @@ -1,98 +0,0 @@ -type: "nn" -layers { - name: "data" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "__trans_layer_0__" - type: "trans" - size: 100 - active_type: "" - inputs { - input_layer_name: "data" - } -} -layers { - name: "__fc_layer_0__" - type: "fc" - size: 100 - active_type: "tanh" - inputs { - input_layer_name: "__trans_layer_0__" - input_parameter_name: "___fc_layer_0__.w0" - } -} -layers { - name: "mask" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "__selective_fc_layer_0__" - type: "selective_fc" - size: 100 - active_type: "sigmoid" - inputs { - input_layer_name: "data" - input_parameter_name: "___selective_fc_layer_0__.w0" - } - inputs { - input_layer_name: "mask" - } - bias_parameter_name: "___selective_fc_layer_0__.wbias" - selective_fc_pass_generation: false - has_selected_colums: true - selective_fc_full_mul_ratio: 0.019999999553 -} -parameters { - name: "___fc_layer_0__.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___selective_fc_layer_0__.w0" - size: 10000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - initial_strategy: 0 - initial_smart: true - is_sparse: false -} -parameters { - name: "___selective_fc_layer_0__.wbias" - size: 100 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 100 - initial_strategy: 0 - initial_smart: false -} -input_layer_names: "data" -input_layer_names: "mask" -output_layer_names: "__fc_layer_0__" -output_layer_names: "__selective_fc_layer_0__" -sub_models { - name: "root" - layer_names: "data" - layer_names: "__trans_layer_0__" - layer_names: "__fc_layer_0__" - layer_names: "mask" - layer_names: "__selective_fc_layer_0__" - input_layer_names: "data" - input_layer_names: "mask" - output_layer_names: "__fc_layer_0__" - output_layer_names: "__selective_fc_layer_0__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.protostr deleted file mode 100644 index 81577910ccf34..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_grumemory_layer.protostr +++ /dev/null @@ -1,51 +0,0 @@ -type: "nn" -layers { - name: "data" - type: "data" - size: 120 - active_type: "" -} -layers { - name: "__gru_0__" - type: "gated_recurrent" - size: 40 - active_type: "sigmoid" - inputs { - input_layer_name: "data" - input_parameter_name: "___gru_0__.w0" - } - bias_parameter_name: "___gru_0__.wbias" - reversed: true - active_gate_type: "tanh" -} -parameters { - name: "___gru_0__.w0" - size: 4800 - initial_mean: 0.0 - initial_std: 0.158113881946 - dims: 40 - dims: 120 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___gru_0__.wbias" - size: 120 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 120 - initial_strategy: 0 - initial_smart: false -} -input_layer_names: "data" -output_layer_names: "__gru_0__" -sub_models { - name: "root" - layer_names: "data" - layer_names: "__gru_0__" - input_layer_names: "data" - output_layer_names: "__gru_0__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.protostr deleted file mode 100644 index e8cc61b8c5410..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_hsigmoid.protostr +++ /dev/null @@ -1,62 +0,0 @@ -type: "nn" -layers { - name: "data" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "label" - type: "data" - size: 10 - active_type: "" -} -layers { - name: "__hsigmoid_0__" - type: "hsigmoid" - size: 1 - active_type: "" - inputs { - input_layer_name: "data" - input_parameter_name: "___hsigmoid_0__.w0" - } - inputs { - input_layer_name: "label" - } - bias_parameter_name: "___hsigmoid_0__.wbias" - num_classes: 10 -} -parameters { - name: "___hsigmoid_0__.w0" - size: 900 - initial_mean: 0.0 - initial_std: 0.333333343267 - dims: 9 - dims: 100 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___hsigmoid_0__.wbias" - size: 9 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 9 - initial_strategy: 0 - initial_smart: false -} -input_layer_names: "data" -input_layer_names: "label" -output_layer_names: "__hsigmoid_0__" -sub_models { - name: "root" - layer_names: "data" - layer_names: "label" - layer_names: "__hsigmoid_0__" - input_layer_names: "data" - input_layer_names: "label" - output_layer_names: "__hsigmoid_0__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.protostr deleted file mode 100644 index 8341cd2684746..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_lstmemory_layer.protostr +++ /dev/null @@ -1,53 +0,0 @@ -type: "nn" -layers { - name: "data" - type: "data" - size: 128 - active_type: "" -} -layers { - name: "__lstmemory_0__" - type: "lstmemory" - size: 32 - active_type: "tanh" - inputs { - input_layer_name: "data" - input_parameter_name: "___lstmemory_0__.w0" - } - bias_parameter_name: "___lstmemory_0__.wbias" - reversed: true - active_gate_type: "tanh" - active_state_type: "tanh" -} -parameters { - name: "___lstmemory_0__.w0" - size: 4096 - initial_mean: 0.0 - initial_std: 0.176776692271 - dims: 32 - dims: 32 - dims: 4 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___lstmemory_0__.wbias" - size: 224 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 224 - initial_strategy: 0 - initial_smart: false -} -input_layer_names: "data" -output_layer_names: "__lstmemory_0__" -sub_models { - name: "root" - layer_names: "data" - layer_names: "__lstmemory_0__" - input_layer_names: "data" - output_layer_names: "__lstmemory_0__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_maxout.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_maxout.protostr deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.protostr deleted file mode 100644 index 44400e2c3a23d..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_ntm_layers.protostr +++ /dev/null @@ -1,225 +0,0 @@ -type: "nn" -layers { - name: "w" - type: "data" - size: 1 - active_type: "" -} -layers { - name: "a" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "b" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "c" - type: "data" - size: 200 - active_type: "" -} -layers { - name: "d" - type: "data" - size: 31 - active_type: "" -} -layers { - name: "__interpolation_layer_0__" - type: "interpolation" - size: 100 - active_type: "" - inputs { - input_layer_name: "w" - } - inputs { - input_layer_name: "a" - } - inputs { - input_layer_name: "b" - } -} -layers { - name: "__power_layer_0__" - type: "power" - size: 100 - active_type: "" - inputs { - input_layer_name: "w" - } - inputs { - input_layer_name: "a" - } -} -layers { - name: "__scaling_layer_0__" - type: "scaling" - size: 100 - active_type: "" - inputs { - input_layer_name: "w" - } - inputs { - input_layer_name: "a" - } -} -layers { - name: "__cos_sim_0__" - type: "cos" - size: 1 - active_type: "" - inputs { - input_layer_name: "a" - } - inputs { - input_layer_name: "b" - } - cos_scale: 5.0 -} -layers { - name: "__cos_sim_1__" - type: "cos_vm" - size: 2 - active_type: "" - inputs { - input_layer_name: "a" - } - inputs { - input_layer_name: "c" - } - cos_scale: 5.0 -} -layers { - name: "__sum_to_one_norm_layer_0__" - type: "sum_to_one_norm" - size: 100 - active_type: "" - inputs { - input_layer_name: "a" - } -} -layers { - name: "__conv_shift_layer_0__" - type: "conv_shift" - size: 100 - active_type: "" - inputs { - input_layer_name: "a" - } - inputs { - input_layer_name: "d" - } -} -layers { - name: "__tensor_layer_0__" - type: "tensor" - size: 1000 - active_type: "" - inputs { - input_layer_name: "a" - input_parameter_name: "___tensor_layer_0__.w0" - } - inputs { - input_layer_name: "b" - } - bias_parameter_name: "___tensor_layer_0__.wbias" -} -layers { - name: "__slope_intercept_layer_0__" - type: "slope_intercept" - size: 100 - active_type: "" - inputs { - input_layer_name: "a" - } - slope: 0.699999988079 - intercept: 0.899999976158 -} -layers { - name: "__linear_comb_layer_0__" - type: "convex_comb" - size: 2 - active_type: "" - inputs { - input_layer_name: "b" - } - inputs { - input_layer_name: "c" - } -} -parameters { - name: "___tensor_layer_0__.w0" - size: 10000000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 100 - dims: 1000 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___tensor_layer_0__.wbias" - size: 1000 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 1000 - initial_strategy: 0 - initial_smart: false -} -input_layer_names: "w" -input_layer_names: "a" -input_layer_names: "b" -input_layer_names: "c" -input_layer_names: "d" -output_layer_names: "__interpolation_layer_0__" -output_layer_names: "__power_layer_0__" -output_layer_names: "__scaling_layer_0__" -output_layer_names: "__cos_sim_0__" -output_layer_names: "__cos_sim_1__" -output_layer_names: "__sum_to_one_norm_layer_0__" -output_layer_names: "__conv_shift_layer_0__" -output_layer_names: "__tensor_layer_0__" -output_layer_names: "__slope_intercept_layer_0__" -output_layer_names: "__linear_comb_layer_0__" -sub_models { - name: "root" - layer_names: "w" - layer_names: "a" - layer_names: "b" - layer_names: "c" - layer_names: "d" - layer_names: "__interpolation_layer_0__" - layer_names: "__power_layer_0__" - layer_names: "__scaling_layer_0__" - layer_names: "__cos_sim_0__" - layer_names: "__cos_sim_1__" - layer_names: "__sum_to_one_norm_layer_0__" - layer_names: "__conv_shift_layer_0__" - layer_names: "__tensor_layer_0__" - layer_names: "__slope_intercept_layer_0__" - layer_names: "__linear_comb_layer_0__" - input_layer_names: "w" - input_layer_names: "a" - input_layer_names: "b" - input_layer_names: "c" - input_layer_names: "d" - output_layer_names: "__interpolation_layer_0__" - output_layer_names: "__power_layer_0__" - output_layer_names: "__scaling_layer_0__" - output_layer_names: "__cos_sim_0__" - output_layer_names: "__cos_sim_1__" - output_layer_names: "__sum_to_one_norm_layer_0__" - output_layer_names: "__conv_shift_layer_0__" - output_layer_names: "__tensor_layer_0__" - output_layer_names: "__slope_intercept_layer_0__" - output_layer_names: "__linear_comb_layer_0__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.protostr deleted file mode 100644 index c402aff174ab7..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_print_layer.protostr +++ /dev/null @@ -1,26 +0,0 @@ -type: "nn" -layers { - name: "input" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "__print_0__" - type: "print" - active_type: "" - inputs { - input_layer_name: "input" - } -} -input_layer_names: "input" -output_layer_names: "input" -sub_models { - name: "root" - layer_names: "input" - layer_names: "__print_0__" - input_layer_names: "input" - output_layer_names: "input" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.protostr deleted file mode 100644 index dfb5ce20a31a0..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_rnn_group.protostr +++ /dev/null @@ -1,650 +0,0 @@ -type: "recurrent_nn" -layers { - name: "seq_input" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "sub_seq_input" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "label" - type: "data" - size: 1 - active_type: "" -} -layers { - name: "__mixed_0__" - type: "mixed" - size: 400 - active_type: "" - inputs { - input_layer_name: "seq_input" - input_parameter_name: "___mixed_0__.w0" - proj_conf { - type: "fc" - name: "___mixed_0__.w0" - input_size: 100 - output_size: 400 - } - } -} -layers { - name: "__mixed_1__" - type: "mixed" - size: 300 - active_type: "" - inputs { - input_layer_name: "seq_input" - input_parameter_name: "___mixed_1__.w0" - proj_conf { - type: "fc" - name: "___mixed_1__.w0" - input_size: 100 - output_size: 300 - } - } -} -layers { - name: "__recurrent_group_0__" - type: "recurrent_layer_group" - active_type: "" -} -layers { - name: "seq_input@__recurrent_group_0__" - type: "scatter_agent" - size: 100 - active_type: "" -} -layers { - name: "rnn_forward+delay1@__recurrent_group_0__" - type: "agent" - size: 200 - active_type: "" -} -layers { - name: "rnn_forward@__recurrent_group_0__" - type: "fc" - size: 200 - active_type: "tanh" - inputs { - input_layer_name: "seq_input@__recurrent_group_0__" - input_parameter_name: "_rnn_forward@__recurrent_group_0__.w0" - } - inputs { - input_layer_name: "rnn_forward+delay1@__recurrent_group_0__" - input_parameter_name: "_rnn_forward@__recurrent_group_0__.w1" - } - bias_parameter_name: "_rnn_forward@__recurrent_group_0__.wbias" -} -layers { - name: "rnn_forward" - type: "gather_agent" - size: 200 - active_type: "" -} -layers { - name: "__last_seq_0__" - type: "seqlastins" - size: 200 - active_type: "linear" - inputs { - input_layer_name: "rnn_forward" - } - trans_type: "non-seq" -} -layers { - name: "__recurrent_group_1__" - type: "recurrent_layer_group" - active_type: "" -} -layers { - name: "seq_input@__recurrent_group_1__" - type: "scatter_agent" - size: 100 - active_type: "" -} -layers { - name: "rnn_back+delay1@__recurrent_group_1__" - type: "agent" - size: 200 - active_type: "" -} -layers { - name: "rnn_back@__recurrent_group_1__" - type: "fc" - size: 200 - active_type: "tanh" - inputs { - input_layer_name: "seq_input@__recurrent_group_1__" - input_parameter_name: "_rnn_back@__recurrent_group_1__.w0" - } - inputs { - input_layer_name: "rnn_back+delay1@__recurrent_group_1__" - input_parameter_name: "_rnn_back@__recurrent_group_1__.w1" - } - bias_parameter_name: "_rnn_back@__recurrent_group_1__.wbias" -} -layers { - name: "rnn_back" - type: "gather_agent" - size: 200 - active_type: "" -} -layers { - name: "__first_seq_0__" - type: "seqlastins" - size: 200 - active_type: "linear" - inputs { - input_layer_name: "rnn_back" - } - select_first: true - trans_type: "non-seq" -} -layers { - name: "__recurrent_group_2__" - type: "recurrent_layer_group" - active_type: "" -} -layers { - name: "sub_seq_input@__recurrent_group_2__" - type: "sequence_scatter_agent" - size: 100 - active_type: "" -} -layers { - name: "rnn_subseq_forward+delay1@__recurrent_group_2__" - type: "agent" - size: 200 - active_type: "" -} -layers { - name: "rnn_subseq_forward@__recurrent_group_2__" - type: "fc" - size: 200 - active_type: "tanh" - inputs { - input_layer_name: "sub_seq_input@__recurrent_group_2__" - input_parameter_name: "_rnn_subseq_forward@__recurrent_group_2__.w0" - } - inputs { - input_layer_name: "rnn_subseq_forward+delay1@__recurrent_group_2__" - input_parameter_name: "_rnn_subseq_forward@__recurrent_group_2__.w1" - } - bias_parameter_name: "_rnn_subseq_forward@__recurrent_group_2__.wbias" -} -layers { - name: "rnn_subseq_forward" - type: "sequence_gather_agent" - size: 200 - active_type: "" -} -layers { - name: "__last_seq_1__" - type: "seqlastins" - size: 200 - active_type: "linear" - inputs { - input_layer_name: "rnn_subseq_forward" - } - trans_type: "non-seq" -} -layers { - name: "__lstm_group_0___recurrent_group" - type: "recurrent_layer_group" - active_type: "" -} -layers { - name: "__mixed_0__@__lstm_group_0___recurrent_group" - type: "scatter_agent" - size: 400 - active_type: "" -} -layers { - name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" - type: "agent" - size: 100 - active_type: "" -} -layers { - name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" - type: "agent" - size: 100 - active_type: "" -} -layers { - name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" - type: "mixed" - size: 400 - active_type: "" - inputs { - input_layer_name: "__mixed_0__@__lstm_group_0___recurrent_group" - proj_conf { - type: "identity" - name: "___lstm_group_0___input_recurrent.w0" - input_size: 400 - output_size: 400 - } - } - inputs { - input_layer_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" - input_parameter_name: "___lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group.w1" - proj_conf { - type: "fc" - name: "___lstm_group_0___input_recurrent.w1" - input_size: 100 - output_size: 400 - } - } -} -layers { - name: "__lstm_group_0__@__lstm_group_0___recurrent_group" - type: "lstm_step" - size: 100 - active_type: "tanh" - inputs { - input_layer_name: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" - } - inputs { - input_layer_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" - } - bias_parameter_name: "___lstm_group_0__@__lstm_group_0___recurrent_group.wbias" - active_gate_type: "sigmoid" - active_state_type: "sigmoid" -} -layers { - name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" - type: "get_output" - size: 100 - active_type: "" - inputs { - input_layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" - input_layer_argument: "state" - } -} -layers { - name: "__lstm_group_0__" - type: "gather_agent" - size: 100 - active_type: "" -} -layers { - name: "__last_seq_2__" - type: "seqlastins" - size: 100 - active_type: "linear" - inputs { - input_layer_name: "__lstm_group_0__" - } - trans_type: "non-seq" -} -layers { - name: "__gru_group_0___recurrent_group" - type: "recurrent_layer_group" - active_type: "" -} -layers { - name: "__mixed_1__@__gru_group_0___recurrent_group" - type: "scatter_agent" - size: 300 - active_type: "" -} -layers { - name: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" - type: "agent" - size: 100 - active_type: "" -} -layers { - name: "__gru_group_0__@__gru_group_0___recurrent_group" - type: "gru_step" - size: 100 - active_type: "tanh" - inputs { - input_layer_name: "__mixed_1__@__gru_group_0___recurrent_group" - input_parameter_name: "___gru_group_0__@__gru_group_0___recurrent_group.w0" - } - inputs { - input_layer_name: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" - } - bias_parameter_name: "___gru_group_0__@__gru_group_0___recurrent_group.wbias" - active_gate_type: "sigmoid" -} -layers { - name: "__gru_group_0__" - type: "gather_agent" - size: 100 - active_type: "" -} -layers { - name: "__last_seq_3__" - type: "seqlastins" - size: 100 - active_type: "linear" - inputs { - input_layer_name: "__gru_group_0__" - } - trans_type: "non-seq" -} -parameters { - name: "___mixed_0__.w0" - size: 40000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 400 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___mixed_1__.w0" - size: 30000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 300 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_rnn_forward@__recurrent_group_0__.w0" - size: 20000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 200 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_rnn_forward@__recurrent_group_0__.w1" - size: 40000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 200 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_rnn_forward@__recurrent_group_0__.wbias" - size: 200 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 200 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_rnn_back@__recurrent_group_1__.w0" - size: 20000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 200 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_rnn_back@__recurrent_group_1__.w1" - size: 40000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 200 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_rnn_back@__recurrent_group_1__.wbias" - size: 200 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 200 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "_rnn_subseq_forward@__recurrent_group_2__.w0" - size: 20000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 200 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_rnn_subseq_forward@__recurrent_group_2__.w1" - size: 40000 - initial_mean: 0.0 - initial_std: 0.0707106813788 - dims: 200 - dims: 200 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "_rnn_subseq_forward@__recurrent_group_2__.wbias" - size: 200 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 200 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group.w1" - size: 40000 - initial_mean: 0.0 - initial_std: 0.10000000149 - dims: 100 - dims: 400 - initial_strategy: 0 - initial_smart: true -} -parameters { - name: "___lstm_group_0__@__lstm_group_0___recurrent_group.wbias" - size: 300 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 300 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___gru_group_0__@__gru_group_0___recurrent_group.w0" - size: 30000 - initial_mean: 0.0 - initial_std: 0.00999999977648 - dims: 100 - dims: 300 - initial_strategy: 0 - initial_smart: false -} -parameters { - name: "___gru_group_0__@__gru_group_0___recurrent_group.wbias" - size: 300 - initial_mean: 0.0 - initial_std: 0.0 - dims: 1 - dims: 300 - initial_strategy: 0 - initial_smart: false -} -input_layer_names: "seq_input" -input_layer_names: "sub_seq_input" -output_layer_names: "__last_seq_0__" -output_layer_names: "__first_seq_0__" -output_layer_names: "__last_seq_1__" -output_layer_names: "__last_seq_2__" -output_layer_names: "__last_seq_3__" -sub_models { - name: "root" - layer_names: "seq_input" - layer_names: "sub_seq_input" - layer_names: "label" - layer_names: "__mixed_0__" - layer_names: "__mixed_1__" - layer_names: "__recurrent_group_0__" - layer_names: "rnn_forward" - layer_names: "__last_seq_0__" - layer_names: "__recurrent_group_1__" - layer_names: "rnn_back" - layer_names: "__first_seq_0__" - layer_names: "__recurrent_group_2__" - layer_names: "rnn_subseq_forward" - layer_names: "__last_seq_1__" - layer_names: "__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0__" - layer_names: "__last_seq_2__" - layer_names: "__gru_group_0___recurrent_group" - layer_names: "__gru_group_0__" - layer_names: "__last_seq_3__" - input_layer_names: "seq_input" - input_layer_names: "sub_seq_input" - output_layer_names: "__last_seq_0__" - output_layer_names: "__first_seq_0__" - output_layer_names: "__last_seq_1__" - output_layer_names: "__last_seq_2__" - output_layer_names: "__last_seq_3__" - is_recurrent_layer_group: false -} -sub_models { - name: "__recurrent_group_0__" - layer_names: "seq_input@__recurrent_group_0__" - layer_names: "rnn_forward+delay1@__recurrent_group_0__" - layer_names: "rnn_forward@__recurrent_group_0__" - is_recurrent_layer_group: true - reversed: false - memories { - layer_name: "rnn_forward@__recurrent_group_0__" - link_name: "rnn_forward+delay1@__recurrent_group_0__" - is_sequence: false - } - in_links { - layer_name: "seq_input" - link_name: "seq_input@__recurrent_group_0__" - has_subseq: false - } - out_links { - layer_name: "rnn_forward@__recurrent_group_0__" - link_name: "rnn_forward" - has_subseq: false - } - target_inlinkid: -1 -} -sub_models { - name: "__recurrent_group_1__" - layer_names: "seq_input@__recurrent_group_1__" - layer_names: "rnn_back+delay1@__recurrent_group_1__" - layer_names: "rnn_back@__recurrent_group_1__" - is_recurrent_layer_group: true - reversed: true - memories { - layer_name: "rnn_back@__recurrent_group_1__" - link_name: "rnn_back+delay1@__recurrent_group_1__" - is_sequence: false - } - in_links { - layer_name: "seq_input" - link_name: "seq_input@__recurrent_group_1__" - has_subseq: false - } - out_links { - layer_name: "rnn_back@__recurrent_group_1__" - link_name: "rnn_back" - has_subseq: false - } - target_inlinkid: -1 -} -sub_models { - name: "__recurrent_group_2__" - layer_names: "sub_seq_input@__recurrent_group_2__" - layer_names: "rnn_subseq_forward+delay1@__recurrent_group_2__" - layer_names: "rnn_subseq_forward@__recurrent_group_2__" - is_recurrent_layer_group: true - reversed: false - memories { - layer_name: "rnn_subseq_forward@__recurrent_group_2__" - link_name: "rnn_subseq_forward+delay1@__recurrent_group_2__" - is_sequence: false - } - in_links { - layer_name: "sub_seq_input" - link_name: "sub_seq_input@__recurrent_group_2__" - has_subseq: true - } - out_links { - layer_name: "rnn_subseq_forward@__recurrent_group_2__" - link_name: "rnn_subseq_forward" - has_subseq: true - } - target_inlinkid: -1 -} -sub_models { - name: "__lstm_group_0___recurrent_group" - layer_names: "__mixed_0__@__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0___input_recurrent@__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0__@__lstm_group_0___recurrent_group" - layer_names: "__lstm_group_0___state@__lstm_group_0___recurrent_group" - is_recurrent_layer_group: true - reversed: false - memories { - layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" - link_name: "__lstm_group_0__+delay1@__lstm_group_0___recurrent_group" - is_sequence: false - } - memories { - layer_name: "__lstm_group_0___state@__lstm_group_0___recurrent_group" - link_name: "__lstm_group_0___state+delay1@__lstm_group_0___recurrent_group" - is_sequence: false - } - in_links { - layer_name: "__mixed_0__" - link_name: "__mixed_0__@__lstm_group_0___recurrent_group" - has_subseq: false - } - out_links { - layer_name: "__lstm_group_0__@__lstm_group_0___recurrent_group" - link_name: "__lstm_group_0__" - has_subseq: false - } - target_inlinkid: -1 -} -sub_models { - name: "__gru_group_0___recurrent_group" - layer_names: "__mixed_1__@__gru_group_0___recurrent_group" - layer_names: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" - layer_names: "__gru_group_0__@__gru_group_0___recurrent_group" - is_recurrent_layer_group: true - reversed: false - memories { - layer_name: "__gru_group_0__@__gru_group_0___recurrent_group" - link_name: "__gru_group_0__+delay1@__gru_group_0___recurrent_group" - is_sequence: false - } - in_links { - layer_name: "__mixed_1__" - link_name: "__mixed_1__@__gru_group_0___recurrent_group" - has_subseq: false - } - out_links { - layer_name: "__gru_group_0__@__gru_group_0___recurrent_group" - link_name: "__gru_group_0__" - has_subseq: false - } - target_inlinkid: -1 -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.protostr b/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.protostr deleted file mode 100644 index 1999c006d237e..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/test_sequence_pooling.protostr +++ /dev/null @@ -1,111 +0,0 @@ -type: "nn" -layers { - name: "dat_in" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "__seq_pooling_0__" - type: "max" - size: 100 - active_type: "linear" - inputs { - input_layer_name: "dat_in" - } - trans_type: "seq" -} -layers { - name: "__seq_pooling_1__" - type: "max" - size: 100 - active_type: "linear" - inputs { - input_layer_name: "dat_in" - } - trans_type: "non-seq" -} -layers { - name: "__seq_pooling_2__" - type: "average" - size: 100 - active_type: "linear" - inputs { - input_layer_name: "dat_in" - } - average_strategy: "average" - trans_type: "seq" -} -layers { - name: "__seq_pooling_3__" - type: "average" - size: 100 - active_type: "linear" - inputs { - input_layer_name: "dat_in" - } - average_strategy: "average" - trans_type: "non-seq" -} -layers { - name: "__seq_pooling_4__" - type: "average" - size: 100 - active_type: "linear" - inputs { - input_layer_name: "dat_in" - } - average_strategy: "sum" - trans_type: "seq" -} -layers { - name: "__seq_pooling_5__" - type: "average" - size: 100 - active_type: "linear" - inputs { - input_layer_name: "dat_in" - } - average_strategy: "sum" - trans_type: "non-seq" -} -layers { - name: "__seq_pooling_6__" - type: "max" - size: 100 - active_type: "linear" - inputs { - input_layer_name: "dat_in" - } - output_max_index: true - trans_type: "non-seq" -} -input_layer_names: "dat_in" -output_layer_names: "__seq_pooling_0__" -output_layer_names: "__seq_pooling_1__" -output_layer_names: "__seq_pooling_2__" -output_layer_names: "__seq_pooling_3__" -output_layer_names: "__seq_pooling_4__" -output_layer_names: "__seq_pooling_5__" -output_layer_names: "__seq_pooling_6__" -sub_models { - name: "root" - layer_names: "dat_in" - layer_names: "__seq_pooling_0__" - layer_names: "__seq_pooling_1__" - layer_names: "__seq_pooling_2__" - layer_names: "__seq_pooling_3__" - layer_names: "__seq_pooling_4__" - layer_names: "__seq_pooling_5__" - layer_names: "__seq_pooling_6__" - input_layer_names: "dat_in" - output_layer_names: "__seq_pooling_0__" - output_layer_names: "__seq_pooling_1__" - output_layer_names: "__seq_pooling_2__" - output_layer_names: "__seq_pooling_3__" - output_layer_names: "__seq_pooling_4__" - output_layer_names: "__seq_pooling_5__" - output_layer_names: "__seq_pooling_6__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/unused_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/unused_layers.protostr deleted file mode 100644 index 89ed28406e553..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/unused_layers.protostr +++ /dev/null @@ -1,27 +0,0 @@ -type: "nn" -layers { - name: "probs" - type: "data" - size: 100 - active_type: "" -} -layers { - name: "__sampling_id_layer_0__" - type: "sampling_id" - size: 100 - active_type: "" - inputs { - input_layer_name: "probs" - } -} -input_layer_names: "probs" -output_layer_names: "__sampling_id_layer_0__" -sub_models { - name: "root" - layer_names: "probs" - layer_names: "__sampling_id_layer_0__" - input_layer_names: "probs" - output_layer_names: "__sampling_id_layer_0__" - is_recurrent_layer_group: false -} - diff --git a/python/paddle/trainer_config_helpers/tests/configs/util_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/util_layers.protostr deleted file mode 100644 index d0ad388165007..0000000000000 --- a/python/paddle/trainer_config_helpers/tests/configs/util_layers.protostr +++ /dev/null @@ -1,81 +0,0 @@ -type: "nn" -layers { - name: "a" - type: "data" - size: 10 - active_type: "" -} -layers { - name: "b" - type: "data" - size: 10 - active_type: "" -} -layers { - name: "__addto_0__" - type: "addto" - size: 10 - active_type: "" - inputs { - input_layer_name: "a" - } - inputs { - input_layer_name: "b" - } -} -layers { - name: "__concat_0__" - type: "concat" - size: 20 - active_type: "" - inputs { - input_layer_name: "a" - } - inputs { - input_layer_name: "b" - } -} -layers { - name: "__concat_1__" - type: "concat2" - size: 20 - active_type: "" - inputs { - input_layer_name: "a" - proj_conf { - type: "identity" - name: "___concat_1__.w0" - input_size: 10 - output_size: 10 - } - } - inputs { - input_layer_name: "b" - proj_conf { - type: "identity" - name: "___concat_1__.w1" - input_size: 10 - output_size: 10 - } - } -} -input_layer_names: "a" -input_layer_names: "b" -output_layer_names: "__addto_0__" -output_layer_names: "__concat_0__" -output_layer_names: "__concat_1__" -sub_models { - name: "root" - layer_names: "a" - layer_names: "b" - layer_names: "__addto_0__" - layer_names: "__concat_0__" - layer_names: "__concat_1__" - input_layer_names: "a" - input_layer_names: "b" - output_layer_names: "__addto_0__" - output_layer_names: "__concat_0__" - output_layer_names: "__concat_1__" - is_recurrent_layer_group: false -} - From c8091ad80f0676c75a3859cda197e1bd2a7e262e Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 9 Nov 2016 19:46:00 +0800 Subject: [PATCH 259/324] Follow comments --- .../protostr/test_bilinear_interp.protostr | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr new file mode 100644 index 0000000000000..d4cbfc2389ac5 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr @@ -0,0 +1,123 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 2304 + active_type: "" +} +layers { + name: "__conv_0__" + type: "exconv" + size: 36864 + active_type: "" + inputs { + input_layer_name: "data" + input_parameter_name: "___conv_0__.w0" + conv_conf { + filter_size: 3 + channels: 1 + stride: 1 + padding: 1 + groups: 1 + filter_channels: 1 + output_x: 48 + img_size: 48 + caffe_mode: true + filter_size_y: 3 + padding_y: 1 + stride_y: 1 + } + } + bias_parameter_name: "___conv_0__.wbias" + num_filters: 16 + shared_biases: true +} +layers { + name: "__bilinear_interp_layer_0__" + type: "bilinear_interp" + size: 65536 + active_type: "" + inputs { + input_layer_name: "__conv_0__" + bilinear_interp_conf { + out_size_x: 64 + out_size_y: 64 + num_channels: 16 + } + } +} +layers { + name: "__pool_0__" + type: "pool" + size: 16384 + active_type: "" + inputs { + input_layer_name: "__bilinear_interp_layer_0__" + pool_conf { + pool_type: "max-projection" + channels: 4 + size_x: 2 + stride: 2 + output_x: 64 + img_size: 128 + padding: 0 + size_y: 2 + stride_y: 2 + output_y: 64 + img_size_y: 128 + padding_y: 0 + } + } +} +layers { + name: "__fc_layer_0__" + type: "fc" + size: 384 + active_type: "tanh" + inputs { + input_layer_name: "__pool_0__" + input_parameter_name: "___fc_layer_0__.w0" + } +} +parameters { + name: "___conv_0__.w0" + size: 144 + initial_mean: 0.0 + initial_std: 0.471404522657 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___conv_0__.wbias" + size: 16 + initial_mean: 0.0 + initial_std: 0.0 + dims: 16 + dims: 1 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___fc_layer_0__.w0" + size: 6291456 + initial_mean: 0.0 + initial_std: 0.0078125 + dims: 16384 + dims: 384 + initial_strategy: 0 + initial_smart: true +} +input_layer_names: "data" +output_layer_names: "__fc_layer_0__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__conv_0__" + layer_names: "__bilinear_interp_layer_0__" + layer_names: "__pool_0__" + layer_names: "__fc_layer_0__" + input_layer_names: "data" + output_layer_names: "__fc_layer_0__" + is_recurrent_layer_group: false +} + From dfbde28ad30243a4d7d6152c1c3abb82bb298e2e Mon Sep 17 00:00:00 2001 From: qijun Date: Wed, 9 Nov 2016 11:48:09 +0000 Subject: [PATCH 260/324] add some code comments for SppLayer --- paddle/gserver/layers/SpatialPyramidPoolLayer.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paddle/gserver/layers/SpatialPyramidPoolLayer.h b/paddle/gserver/layers/SpatialPyramidPoolLayer.h index 8416a717d654e..e15b6d2f85c6f 100644 --- a/paddle/gserver/layers/SpatialPyramidPoolLayer.h +++ b/paddle/gserver/layers/SpatialPyramidPoolLayer.h @@ -24,6 +24,8 @@ namespace paddle { * @brief A layer for spatial pyramid pooling on the input image by taking * the max, average, etc. within regions, so that the result vector of * different sized images are of the same size. + * + * The config file api is spp_layer. */ class SpatialPyramidPoolLayer : public Layer { From 1c9f6f7f9902ce05e71fd62edea9b10f895009e5 Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 9 Nov 2016 20:36:45 +0800 Subject: [PATCH 261/324] Update --- paddle/gserver/tests/test_LayerGrad.cpp | 47 +++++++++++++------------ 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index d8c45040d86bc..ad09d7d00e4f5 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -32,28 +32,6 @@ P_DECLARE_double(checkgrad_eps); P_DECLARE_bool(thread_local_rand_use_global_seed); P_DECLARE_bool(prev_batch_state); -TEST(Layer, BilinearInterpLayer) { - TestConfig config; - config.layerConfig.set_type("bilinear_interp"); - config.biasSize = 0; - - config.inputDefs.push_back({INPUT_DATA, "layer_0", 4096, 0}); - LayerInputConfig* input = config.layerConfig.add_inputs(); - BilinearInterpConfig* bilinear = input->mutable_bilinear_interp_conf(); - - bilinear->set_img_size_x(32); - bilinear->set_img_size_y(32); - bilinear->set_num_channels(4); - - for (auto useGpu : {false, true}) { - for (auto out_size : {32, 64, 128}) { - bilinear->set_out_size_x(out_size); - bilinear->set_out_size_y(out_size); - testLayerGrad(config, "bilinear_interp", 10, false, useGpu); - } - } -} - TEST(Operator, dot_mul) { TestConfig config; config.layerConfig.set_size(10); @@ -197,6 +175,31 @@ TEST(Projection, conv) { } #endif +TEST(Layer, BilinearInterpLayer) { + TestConfig config; + config.layerConfig.set_type("bilinear_interp"); + config.biasSize = 0; + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 4096, 0}); + + for (auto useGpu : {false, true}) { + for (auto out_size : {32, 64}) { + LOG(INFO) << " out_size_x=" << out_size + << " out_size_y=" << out_size; + LayerInputConfig* input + = config.layerConfig.add_inputs(); + BilinearInterpConfig* bilinear + = input->mutable_bilinear_interp_conf(); + bilinear->set_img_size_x(32); + bilinear->set_img_size_y(32); + bilinear->set_num_channels(4); + bilinear->set_out_size_x(out_size); + bilinear->set_out_size_y(out_size); + testLayerGrad(config, "bilinear_interp", 10, false, useGpu); + } + } +} + TEST(Layer, concat) { TestConfig config; config.biasSize = 0; From 65b8bb25833f94453f17d8539fe77203bc88b9bb Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 9 Nov 2016 21:07:25 +0800 Subject: [PATCH 262/324] Fix a bug --- paddle/gserver/tests/test_LayerGrad.cpp | 32 ++++++++++++------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index ad09d7d00e4f5..c1f7876c1722f 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -179,25 +179,23 @@ TEST(Layer, BilinearInterpLayer) { TestConfig config; config.layerConfig.set_type("bilinear_interp"); config.biasSize = 0; - config.inputDefs.push_back({INPUT_DATA, "layer_0", 4096, 0}); - for (auto useGpu : {false, true}) { - for (auto out_size : {32, 64}) { - LOG(INFO) << " out_size_x=" << out_size - << " out_size_y=" << out_size; - LayerInputConfig* input - = config.layerConfig.add_inputs(); - BilinearInterpConfig* bilinear - = input->mutable_bilinear_interp_conf(); - bilinear->set_img_size_x(32); - bilinear->set_img_size_y(32); - bilinear->set_num_channels(4); - bilinear->set_out_size_x(out_size); - bilinear->set_out_size_y(out_size); - testLayerGrad(config, "bilinear_interp", 10, false, useGpu); - } - } + LayerInputConfig* input = config.layerConfig.add_inputs(); + BilinearInterpConfig* bilinear = input->mutable_bilinear_interp_conf(); + bilinear->set_img_size_x(32); + bilinear->set_img_size_y(32); + bilinear->set_num_channels(4); + + bilinear->set_out_size_x(32); + bilinear->set_out_size_y(32); + testLayerGrad(config, "bilinear_interp", 10, false, false); + testLayerGrad(config, "bilinear_interp", 10, false, true); + + bilinear->set_out_size_x(64); + bilinear->set_out_size_y(64); + testLayerGrad(config, "bilinear_interp", 10, false, false); + testLayerGrad(config, "bilinear_interp", 10, false, true); } TEST(Layer, concat) { From 5c88f072621b52cd6069b365e85a0ae892ed8375 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Thu, 20 Oct 2016 17:37:10 -0700 Subject: [PATCH 263/324] initial take on deconv layers --- paddle/gserver/layers/ConvTransBaseLayer.cpp | 77 ++++ paddle/gserver/layers/ConvTransBaseLayer.h | 112 ++++++ .../gserver/layers/ExpandConvTransLayer.cpp | 332 ++++++++++++++++++ paddle/gserver/layers/ExpandConvTransLayer.h | 106 ++++++ paddle/gserver/tests/test_LayerGrad.cpp | 43 +++ 5 files changed, 670 insertions(+) create mode 100644 paddle/gserver/layers/ConvTransBaseLayer.cpp create mode 100644 paddle/gserver/layers/ConvTransBaseLayer.h create mode 100644 paddle/gserver/layers/ExpandConvTransLayer.cpp create mode 100644 paddle/gserver/layers/ExpandConvTransLayer.h diff --git a/paddle/gserver/layers/ConvTransBaseLayer.cpp b/paddle/gserver/layers/ConvTransBaseLayer.cpp new file mode 100644 index 0000000000000..68fb48a38b518 --- /dev/null +++ b/paddle/gserver/layers/ConvTransBaseLayer.cpp @@ -0,0 +1,77 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#include "paddle/utils/Logging.h" +#include "ConvTransBaseLayer.h" +namespace paddle { + +bool ConvTransBaseLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + /* Initialize the basic parent class */ + Layer::init(layerMap, parameterMap); + + /* Initialize the convolutional layer parameter */ + channel_ = config_.num_filters(); + sharedBiases_ = config_.shared_biases(); + for (auto& inputConfig : config_.inputs()) { + const ConvConfig& conf = inputConfig.conv_conf(); + padding_.push_back(conf.padding()); + stride_.push_back(conf.stride()); + filterSize_.push_back(conf.filter_size()); + paddingY_.push_back(conf.padding_y()); + strideY_.push_back(conf.stride_y()); + filterSizeY_.push_back(conf.filter_size_y()); + filterPixels_.push_back(filterSize_.back() * filterSizeY_.back()); + numFilters_.push_back(conf.channels()); + imgSize_.push_back(conf.img_size()); + imgPixels_.push_back(imgSize_.back() * imgSize_.back()); + groups_.push_back(conf.groups()); + filterChannels_.push_back(conf.filter_channels()); + outputX_.push_back(conf.output_x()); + outputs_.push_back(outputX_.back() * outputX_.back()); + } + + /* initialize the weightList */ + CHECK(inputLayers_.size() == parameters_.size()); + for (size_t i = 0; i < inputLayers_.size(); i++) { + size_t height, width; + height = filterPixels_[i] * filterChannels_[i]; + width = numFilters_[i]; + + // create a new weight + CHECK_EQ(parameters_[i]->getSize(), width * height); + Weight* w = new Weight(height, width, parameters_[i]); + weights_.emplace_back(w); + } + + /* initialize the biases_ */ + if (biasParameter_.get() != NULL) { + if (sharedBiases_) { + CHECK_EQ((size_t)channel_, biasParameter_->getSize()); + biases_ = + std::unique_ptr(new Weight(channel_, 1, biasParameter_)); + } else { + biases_ = + std::unique_ptr(new Weight(getSize(), 1, biasParameter_)); + } + } + + // default caffe model + caffeMode_ = true; + + return true; +} + +} // namespace paddle diff --git a/paddle/gserver/layers/ConvTransBaseLayer.h b/paddle/gserver/layers/ConvTransBaseLayer.h new file mode 100644 index 0000000000000..467a260569759 --- /dev/null +++ b/paddle/gserver/layers/ConvTransBaseLayer.h @@ -0,0 +1,112 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#pragma once + +#include "Layer.h" +namespace paddle { + +/** + * @brief A Base Convolution Layer, which convolves the input image + * with learned filters and (optionally) adds biases. + */ + +class ConvTransBaseLayer : public Layer { +protected: + typedef std::vector IntV; + + /// The number of channel in image (the output of the deconv layer). + int channel_; + /// The x dimension of the padding. + IntV padding_; + /// The y dimension of the padding. + IntV paddingY_; + /// The x dimension of the stride. + IntV stride_; + /// The y dimension of the stride. + IntV strideY_; + /// The x dimension of a filter kernel. + IntV filterSize_; + /// The y dimension of a filter kernel. + IntV filterSizeY_; + /// The number of filters(i.e. the number channels of the deconv layer input) + IntV numFilters_; + /// The spatial dimensions of input feature map. + IntV imgSize_; + /// The total pixel size of input feature map. + /// imgPixels_ = imgSizeX_ * imgSizeY_. + IntV imgPixels_; + /// filterPixels_ = filterSizeX_ * filterSizeY_. + IntV filterPixels_; + /// filterChannels_ = channels_/groups_. + IntV filterChannels_; + /// The spatial dimensions of output feature map. + IntV outputX_; + /// The spatial dimensions of output feature map. + IntV outputs_; + /// Group size, refer to grouped convolution in + /// Alex Krizhevsky's paper: when group=2, the first half of the + /// filters are only connected to the first half of the input channels, + /// and the second half only connected to the second half. + IntV groups_; + /// Whether the bias is shared for feature in each channel. + bool sharedBiases_; + + /// shape of weight: (numChannels * filterPixels_, numFilters) + WeightList weights_; + /// If shared_biases is false shape of bias: (numFilters_, 1) + /// If shared_biases is ture shape of bias: + /// (numFilters_ * outputX * outputY, 1) + std::unique_ptr biases_; + + /// True by default. The only difference is the calculation + /// of output size. + bool caffeMode_; + +public: + explicit ConvTransBaseLayer(const LayerConfig& config) : Layer(config) {} + + virtual bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + Weight& getWeight(int idx) { return *weights_[idx]; } + + /** + * Calculate image size based on caffeMode_ from outputSize. + * - input(+padding): 0123456789 + * - imageSize(+padding) = 10; + * - filterSize = 3; + * - stride = 2; + * - caffeMode_ is true: + - output: (012), (234), (456), (678) + - outputSize = 4; + * - caffeMode_ is false: + * - output: (012), (234), (456), (678), (9) + * - outputSize = 5; + */ + + int imageSize(int outputSize, int filterSize, int padding, int stride) { + int imageSize; + if (!caffeMode_) { + imageSize = + (outputSize - 1) * stride + filterSize - 2 * padding - stride + 1; + } else { + imageSize = (outputSize - 1) * stride + filterSize - 2 * padding; + } + CHECK_GE(imageSize, 1); + return imageSize; + } +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvTransLayer.cpp b/paddle/gserver/layers/ExpandConvTransLayer.cpp new file mode 100644 index 0000000000000..56cc042653b60 --- /dev/null +++ b/paddle/gserver/layers/ExpandConvTransLayer.cpp @@ -0,0 +1,332 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#include "paddle/utils/Logging.h" +#include "paddle/utils/Stat.h" +#include "ExpandConvTransLayer.h" + +namespace paddle { + +REGISTER_LAYER(exconvt, ExpandConvTransLayer); + +bool ExpandConvTransLayer::init(const LayerMap &layerMap, + const ParameterMap ¶meterMap) { + /* Initialize the basic convolutional parent class */ + ConvTransBaseLayer::init(layerMap, parameterMap); + + /* Initialize the projection */ + for (auto &inputConfig : config_.inputs()) { + const ConvConfig &conf = inputConfig.conv_conf(); + subM_.push_back(conf.channels() / conf.groups()); + subN_.push_back(conf.output_x() * conf.output_x()); + subK_.push_back(channel_ * conf.filter_size() * conf.filter_size() / + conf.groups()); + /* Consistent caffe mode for multiple input */ + caffeMode_ = conf.caffe_mode(); + } + + return true; +} + +// Why this is necessary after calling init? +size_t ExpandConvTransLayer::getSize() { + CHECK_NE(inputLayers_.size(), 0UL); + imgSizeH_.clear(); + imgSizeW_.clear(); + outputH_.clear(); + outputW_.clear(); + subN_.clear(); + size_t layerSize = 0; + for (size_t i = 0; i < inputLayers_.size(); i++) { + outputH_.push_back(inputLayers_[i]->getOutput().getFrameHeight()); + outputW_.push_back(inputLayers_[i]->getOutput().getFrameWidth()); + if (outputH_[i] == 0) outputH_[i] = outputX_[i]; + if (outputW_[i] == 0) outputW_[i] = outputX_[i]; + imgSizeH_.push_back( + imageSize(outputH_[i], filterSize_[i], padding_[i], stride_[i])); + imgSizeW_.push_back( + imageSize(outputW_[i], filterSize_[i], padding_[i], stride_[i])); + subN_.push_back(outputH_[i] * outputW_[i]); + CHECK(layerSize == 0 || + imgSizeH_[i] * imgSizeW_[i] * (size_t)channel_ == layerSize); + layerSize = imgSizeH_[i] * imgSizeW_[i] * channel_; + } + getOutput().setFrameHeight(imgSizeH_[0]); + getOutput().setFrameWidth(imgSizeW_[0]); + return layerSize; +} + +void ExpandConvTransLayer::resetExpandInput(size_t height, size_t width) { + Matrix::resizeOrCreate(expandInput_, height, width, false, useGpu_); +} + +/*void ExpandConvTransLayer::resetConvOutput(size_t batchSize, int inIdx) { + Matrix::resizeOrCreate(transOutValue_, batchSize * numFilters_, subN_[inIdx], + false, useGpu_); +}*/ + + +void ExpandConvTransLayer::addSharedBias() { + size_t mapW = getSize() / channel_; + size_t mapH = getOutputValue()->getElementCnt() / mapW; + MatrixPtr out = + Matrix::create(getOutputValue()->getData(), mapH, mapW, false, useGpu_); + + Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); + + out->transpose(transOutValue_, false); // false means no memory allocation + transOutValue_->reshape(transOutValue_->getElementCnt() / channel_, + channel_); + + MatrixPtr bias = + Matrix::create(biases_->getW()->getData(), 1, + biases_->getW()->getElementCnt(), false, useGpu_); + transOutValue_->addBias(*bias, 1.0f); + + transOutValue_->reshape(mapW, mapH); + transOutValue_->transpose(out, false); // false means no memory allocation + + out->clear(); + bias->clear(); +} + +void ExpandConvTransLayer::addUnsharedBias() { + MatrixPtr outValue = getOutputValue(); + MatrixPtr bias = + Matrix::create(biases_->getW()->getData(), 1, + biases_->getW()->getElementCnt(), false, useGpu_); + outValue->addBias(*bias, 1.0f); +} + + +void ExpandConvTransLayer::expandOneFrame(MatrixPtr image, size_t startIdx, + int inIdx) { + resetExpandInput(subK_[inIdx] * groups_[inIdx], subN_[inIdx]); + real *imgData = image->getData() + startIdx * image->getWidth(); + MatrixPtr imageTmp = Matrix::create( + imgData, 1, imgSizeH_[inIdx] * imgSizeW_[inIdx] * channel_, false, + useGpu_); + expandInput_->convExpand(*imageTmp, imgSizeH_[inIdx], imgSizeW_[inIdx], + channel_, filterSize_[inIdx], + filterSize_[inIdx], stride_[inIdx], stride_[inIdx], + padding_[inIdx], padding_[inIdx], + outputH_[inIdx], outputW_[inIdx]); + imageTmp->clear(); +} + +void ExpandConvTransLayer::expandBackOnce(MatrixPtr imageGrad, int inIdx, + int startIdx) { + int subM = subM_[inIdx]; + int subN = subN_[inIdx]; + int subK = subK_[inIdx]; + + LayerPtr prevLayer = getPrev(inIdx); + if (NULL == prevLayer->getOutputGrad()) { + return; + } + + expandOneFrame(imageGrad, startIdx, inIdx); + + real *outGradData = + prevLayer -> getOutputGrad()->getData() + + startIdx * subN * numFilters_[inIdx]; + + real *wgtData = weights_[inIdx]->getW()->getData(); + real *expInData = expandInput_->getData(); + for (int g = 0; g < groups_[inIdx]; ++g) { + MatrixPtr A = + Matrix::create(wgtData, subK, subM, true, useGpu_); // mark transpose + MatrixPtr B = Matrix::create(expInData, subK, subN, false, useGpu_); + MatrixPtr C = Matrix::create(outGradData, subM, subN, false, useGpu_); + C->mul(A, B, 1, 1); + + A->clear(); + B->clear(); + C->clear(); + wgtData += subK * subM; + expInData += subK * subN; + outGradData += subM * subN; + } +} + +void ExpandConvTransLayer::forward(PassType passType) { + Layer::forward(passType); + + /* malloc memory for the output_ if necessary */ + /* note: one sample correspond to one colum, and the + * transOutValue correspond sample to one row */ + int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); + resetOutput(batchSize, getSize()); + + MatrixPtr output = nullptr; + for (size_t i = 0; i != inputLayers_.size(); ++i) { + LayerPtr prevLayer = getPrev(i); + output = prevLayer->getOutputValue(); + REGISTER_TIMER_INFO("shrinkFwd", getName().c_str()); + shrinkFwd(output, i); + } + + /* add the bias-vector */ + if (biases_.get() != NULL) { + if (sharedBiases_) { + addSharedBias(); + } else { + addUnsharedBias(); + } + } + + /* activation */ + forwardActivation(); +} + +void ExpandConvTransLayer::shrinkFwd(MatrixPtr output, int inpIdx) { + int subM = subM_[inpIdx]; + int subN = subN_[inpIdx]; + int subK = subK_[inpIdx]; + + size_t batchSize = output->getHeight(); + MatrixPtr image = getOutputValue(); + + /* reset the expand-grad memory */ + resetExpandInput(subK * groups_[inpIdx], subN); + + real *localData = output->getData(); + real *imageData = image->getData(); + for (size_t n = 0; n < batchSize; n++) { + real *wgtData = weights_[inpIdx]->getW()->getData(); + real *expandInData = expandInput_->getData(); + + for (int g = 0; g < groups_[inpIdx]; g++) { + // create temporary matrix + MatrixPtr C = Matrix::create(expandInData, subK, subN, false, useGpu_); + MatrixPtr B = Matrix::create(localData, subM, subN, false, useGpu_); + MatrixPtr A = Matrix::create(wgtData, subK, subM, false, useGpu_); + C->mul(A, B); // mul + + // clear the temporary matrix + A->clear(); + B->clear(); + C->clear(); + + expandInData += subK * subN; + localData += subM * subN; + wgtData += subK * subM; + } + + // shrink one frame outGrad + MatrixPtr oneTmp = Matrix::create( + expandInput_->getData(), subK * groups_[inpIdx], subN, false, useGpu_); + MatrixPtr vTmp = Matrix::create( + imageData, 1, + imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channel_, false, + useGpu_); + vTmp->convShrink(*oneTmp, imgSizeH_[inpIdx], imgSizeW_[inpIdx], + channel_, filterSize_[inpIdx], + filterSize_[inpIdx], stride_[inpIdx], stride_[inpIdx], + padding_[inpIdx], padding_[inpIdx], + outputH_[inpIdx], outputW_[inpIdx], 1.0f, 1.0f); + vTmp->clear(); + oneTmp->clear(); + + // move the data-pointer + imageData += imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channel_; + } +} + +void ExpandConvTransLayer::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { + size_t mapW = getSize() / channel_; + size_t mapH = v->getElementCnt() / mapW; + MatrixPtr vTmp = Matrix::create(v->getData(), mapH, mapW, false, useGpu_); + + Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); + + vTmp->transpose(transOutValue_, false); // false means no memory allocation + vTmp->reshape(transOutValue_->getElementCnt() / channel_, channel_); + biases->collectBias(*vTmp, 1.0f); +} + +void ExpandConvTransLayer::bpropBiases(MatrixPtr v) { + MatrixPtr biases = + Matrix::create(biases_->getWGrad()->getData(), 1, + biases_->getWGrad()->getElementCnt(), false, useGpu_); + if (sharedBiases_) { + bpropSharedBias(biases, v); + } else { + biases->collectBias(*v, 1.0f); + } + biases->clear(); +} + +void ExpandConvTransLayer::backward(const UpdateCallback &callback) { + backwardActivation(); + + MatrixPtr imageGrad = getOutputGrad(); + if (biases_ && biases_->getWGrad()) { + bpropBiases(imageGrad); + /* Increasing the number of gradient */ + biases_->getParameterPtr()->incUpdate(callback); + } + + for (size_t i = 0; i != inputLayers_.size(); ++i) { + /* First, calculate the input layers error */ + for (size_t off = 0; off < imageGrad->getHeight(); off++) { + expandBackOnce(imageGrad, i, off); + } + if (weights_[i]->getWGrad()) { + /* Then, calculate the W-gradient for the current layer */ + bpropWeights(imageGrad, i); + /* Increasing the number of gradient */ + weights_[i]->getParameterPtr()->incUpdate(callback); + } + } +} + +void ExpandConvTransLayer::bpropWeights(MatrixPtr v, int inpIdx) { + MatrixPtr weightGrad = weights_[inpIdx]->getWGrad(); + MatrixPtr outputV = getPrev(inpIdx)->getOutputValue(); + + int subM = subM_[inpIdx]; + int subN = subN_[inpIdx]; + int subK = subK_[inpIdx]; + size_t batchSize = outputV->getHeight(); + resetExpandInput(subK * groups_[inpIdx], subN); + + real *outputData = outputV -> getData(); + + for (size_t n = 0; n < batchSize; n++) { // frame by frame + // expand + expandOneFrame(v, n, inpIdx); + real *wGradData = weightGrad->getData(); + real *expandInData = expandInput_->getData(); + + // expand-mul one-group by one + for (int g = 0; g < groups_[inpIdx]; g++) { + MatrixPtr A = Matrix::create(expandInData, subK, subN, false, useGpu_); + MatrixPtr B = Matrix::create(outputData, subM, subN, true, useGpu_); + MatrixPtr C = Matrix::create(wGradData, subK, subM, false, useGpu_); + C->mul(A, B, 1, 1); + + A->clear(); + B->clear(); + C->clear(); + outputData += subM * subN; + wGradData += subK * subM; + expandInData += subK * subN; + } + } +} + + +} // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvTransLayer.h b/paddle/gserver/layers/ExpandConvTransLayer.h new file mode 100644 index 0000000000000..f19aa81f3d3c2 --- /dev/null +++ b/paddle/gserver/layers/ExpandConvTransLayer.h @@ -0,0 +1,106 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#pragma once + +#include "ConvTransBaseLayer.h" +#include "paddle/math/Matrix.h" +#include + +namespace paddle { + +/** + * @brief A subclass of convolution layer. + * This layer expands input and use matrix multiplication to + * calculate convolution operation. + * + * The config file api is img_conv_layer. + */ +class ExpandConvTransLayer : public ConvTransBaseLayer { +protected: + /// For expand convolution. + /// subM_ = numFilters_ / groups_. + IntV subM_; + /// subN_ = outputH_ * outputW_. + IntV subN_; + /// subK_ = channels_ * filterPixels_ * groups_. + IntV subK_; + /// The spatial dimensions of height of input feature map. + IntV imgSizeH_; + /// The spatial dimensions of width of input feature map. + IntV imgSizeW_; + /// The spatial dimensions of height of output feature map. + IntV outputH_; + /// The spatial dimensions of width of output feature map. + IntV outputW_; + /// Expand one sample at a time. shape: + /// (numChannels * filterPixels_, outputSizeH * outputSizeW) + MatrixPtr expandInput_; + /// The transpose of output, which is an auxiliary matrix. + MatrixPtr transOutValue_; + +public: + explicit ExpandConvTransLayer(const LayerConfig& config) : + ConvTransBaseLayer(config) {} + + ~ExpandConvTransLayer() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + size_t getSize(); + + /** + * Create or resize expandInput_. + */ + void resetExpandInput(size_t height, size_t width); + + /** + * Create or resize transOutValue_. + */ + void resetConvOutput(size_t batchSize, int inIdx); + + /** + * Expand one input sample. + */ + void expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx); + + /** + * Expand one output image and perform matrix multiplication. + */ + void expandBackOnce(MatrixPtr image, int inIdx, int startIdx); + + /** + * Perform matrix multiplication on one output and then shrink. + */ + void shrinkFwd(MatrixPtr output, int inpIdx); + + /** + * Add shared bias. + */ + void addSharedBias(); + + /** + * Add unshared bias. + */ + void addUnsharedBias(); + void forward(PassType passType); + void bpropSharedBias(MatrixPtr biases, MatrixPtr v); + void bpropBiases(MatrixPtr v); + void backward(const UpdateCallback& callback); + void bpropWeights(MatrixPtr v, int inpIdx); + void bpropActs(MatrixPtr v, int inpIdx); +}; + +} // namespace paddle diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 4e01fa91ed2ba..d634d198c3be7 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -312,6 +312,49 @@ TEST(Layer, convLayer) { #endif } + +void testConvTransLayer(const string& type, bool trans, bool useGpu) { + TestConfig config; + config.biasSize = 3; + config.layerConfig.set_type(type); + config.layerConfig.set_num_filters(3); + config.layerConfig.set_partial_sum(1); + config.layerConfig.set_shared_biases(true); + + config.inputDefs.push_back({INPUT_DATA, "layer_0", 1024, 288}); + LayerInputConfig* input = config.layerConfig.add_inputs(); + ConvConfig* conv = input->mutable_conv_conf(); + conv->set_filter_size(2); + conv->set_filter_size_y(3); + conv->set_channels(16); + conv->set_padding(0); + conv->set_padding_y(1); + conv->set_stride(2); + conv->set_stride_y(2); + conv->set_groups(1); + conv->set_filter_channels(3 / conv->groups()); + conv->set_img_size(16); + conv->set_output_x( + (2 * conv->padding() + conv->img_size() - conv->filter_size()) / + ((float)conv->stride()) + + 1.5); + + config.layerConfig.set_size(conv->img_size() * conv->img_size() * + config.layerConfig.num_filters()); + + testLayerGrad(config, "convTrans", 100, trans, useGpu); +} + +TEST(Layer, convTransLayer) { + testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ false); +/* +#ifndef PADDLE_ONLY_CPU + testConvLayer("exconv", trans= false, useGpu= true); + testConvLayer("cudnn_conv", trans= false, useGpu= true); +#endif +*/ +} + TEST(Layer, blockExpandLayer) { TestConfig config; config.biasSize = 0; From 70e44732c2c1a2186d26a076c3b3be69b6a91bc4 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Tue, 25 Oct 2016 13:55:40 -0700 Subject: [PATCH 264/324] added convTrans test and python components --- .gitignore | 2 + paddle/gserver/tests/CMakeLists.txt | 8 + paddle/gserver/tests/test_ConvTrans.cpp | 139 ++++++++++++++++++ python/paddle/trainer/config_parser.py | 95 ++++++++++++ .../paddle/trainer_config_helpers/layers.py | 123 ++++++++++++++++ 5 files changed, 367 insertions(+) create mode 100644 paddle/gserver/tests/test_ConvTrans.cpp diff --git a/.gitignore b/.gitignore index 65ba217de37c8..ee8489c1d71bd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ build/ .vscode .idea .project +.cproject .pydevproject +Makefile diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index 26ee2b3aae64a..0651d0b4733ea 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -26,6 +26,14 @@ add_unittest_without_exec(test_ActivationGrad TestUtil.cpp) add_test(NAME test_ActivationGrad COMMAND test_ActivationGrad) +################# test_ConvTrans ####################### +add_unittest_without_exec(test_ConvTrans + test_ConvTrans.cpp + LayerGradUtil.cpp + TestUtil.cpp) + +add_test(NAME test_ConvTrans + COMMAND test_ConvTrans) ################## test_Evaluator ####################### add_unittest(test_Evaluator diff --git a/paddle/gserver/tests/test_ConvTrans.cpp b/paddle/gserver/tests/test_ConvTrans.cpp new file mode 100644 index 0000000000000..e7cbe2614faca --- /dev/null +++ b/paddle/gserver/tests/test_ConvTrans.cpp @@ -0,0 +1,139 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include +#include +#include "paddle/gserver/layers/DataLayer.h" +#include "ModelConfig.pb.h" +#include "paddle/trainer/Trainer.h" +#include "paddle/utils/GlobalConstants.h" +#include "paddle/gserver/layers/ExpandConvTransLayer.h" + +#include "TestUtil.h" +#include "LayerGradUtil.h" + +using namespace paddle; // NOLINT +using namespace std; // NOLINT + +P_DECLARE_bool(use_gpu); +P_DECLARE_int32(gpu_id); +P_DECLARE_double(checkgrad_eps); +P_DECLARE_bool(thread_local_rand_use_global_seed); +P_DECLARE_bool(prev_batch_state); + +TEST(Layer, convTransLayerFwd) { + TestConfig configt; + configt.biasSize = 3; + configt.layerConfig.set_type("exconvt"); + configt.layerConfig.set_num_filters(3); + configt.layerConfig.set_partial_sum(1); + configt.layerConfig.set_shared_biases(true); + + configt.inputDefs.push_back({INPUT_DATA, "layer_0", 1024, 288}); + LayerInputConfig* input = configt.layerConfig.add_inputs(); + ConvConfig* conv = input->mutable_conv_conf(); + conv->set_filter_size(2); + conv->set_filter_size_y(3); + conv->set_channels(16); + conv->set_padding(0); + conv->set_padding_y(1); + conv->set_stride(2); + conv->set_stride_y(2); + conv->set_groups(1); + conv->set_filter_channels(3 / conv->groups()); + conv->set_img_size(16); + conv->set_output_x( + (2 * conv->padding() + conv->img_size() - conv->filter_size()) / + ((float)conv->stride()) + + 1.5); + + configt.layerConfig.set_size(conv->img_size() * conv->img_size() * + configt.layerConfig.num_filters()); + configt.layerConfig.set_name("convTrans"); + + // data layer initialize + std::vector dataLayers; + LayerMap layerMap; + vector datas; + initDataLayer(configt, &dataLayers, &datas, &layerMap, "convTrans", + 100, false, useGpu); + // test layer initialize + std::vector parameters; + LayerPtr convtLayer; + initTestLayer(configt, &layerMap, ¶meters, &convtLayer); + convtLayer->getBiasParameter()->zeroMem(); + convtLayer->forward(PASS_GC); + + TestConfig config; + config.biasSize = 16; + config.layerConfig.set_type("exconv"); + config.layerConfig.set_num_filters(16); + config.layerConfig.set_partial_sum(1); + config.layerConfig.set_shared_biases(true); + + config.inputDefs.push_back({INPUT_DATA, "layer_1", 768, 288}); + input = config.layerConfig.add_inputs(); + conv = input->mutable_conv_conf(); + conv->set_filter_size(2); + conv->set_filter_size_y(3); + conv->set_channels(3); + conv->set_padding(0); + conv->set_padding_y(1); + conv->set_stride(2); + conv->set_stride_y(2); + conv->set_groups(1); + conv->set_filter_channels(conv->channels() / conv->groups()); + conv->set_img_size(16); + conv->set_output_x( + (2 * conv->padding() + conv->img_size() - conv->filter_size()) / + ((float)conv->stride()) + + 1.5); + config.layerConfig.set_size(conv->output_x() * conv->output_x() * + config.layerConfig.num_filters()); + config.layerConfig.set_name("conv"); + + // data layer initialize + std::vector dataLayers2; + LayerMap layerMap2; + vector datas2; + initDataLayer(config, &dataLayers2, &datas2, &layerMap2, "conv", + 100, false, useGpu); + // test layer initialize + std::vector parameters2; + LayerPtr convLayer; + initTestLayer(config, &layerMap2, ¶meters2, &convLayer); + + convLayer->getBiasParameter()->zeroMem(); + convLayer->getParameters()[0]->getBuf(PARAMETER_VALUE)->copyFrom( + *(convtLayer->getParameters()[0]->getBuf(PARAMETER_VALUE))); + + convLayer->forward(PASS_GC); + convLayer->getOutput().grad->copyFrom(*(dataLayers[0]->getOutputValue())); + + vector callbackFlags(parameters2.size(), 0); + auto callback = [&](Parameter* para) { ++callbackFlags[para->getID()]; }; + convLayer->backward(callback); + + checkMatrixEqual(convtLayer->getOutputValue(), + dataLayers2[0]->getOutputGrad()); +} + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + initMain(argc, argv); + FLAGS_thread_local_rand_use_global_seed = true; + srand(1); + return RUN_ALL_TESTS(); +} diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 73631602a92be..2d28b34999cb0 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1106,6 +1106,37 @@ def parse_conv(conv, input_layer_name, conv_conf): conv_conf.padding, conv_conf.stride, conv_conf.caffe_mode) + +def parse_convt(conv, input_layer_name, conv_conf): + conv_conf.filter_size = conv.filter_size + conv_conf.filter_size_y = conv.filter_size_y + conv_conf.channels = conv.channels + conv_conf.padding = conv.padding + conv_conf.padding_y = conv.padding_y + conv_conf.stride = conv.stride + conv_conf.stride_y = conv.stride_y + conv_conf.groups = conv.groups + conv_conf.filter_channels = conv.channels / conv.groups + conv_conf.caffe_mode = conv.caffe_mode + + outputSize = g_layer_map[input_layer_name].size / conv.channels + print('channels=%d size=%d'%(conv.channels, + g_layer_map[input_layer_name].size)) + conv_conf.output_x = int(outputSize ** 0.5) + config_assert((conv_conf.output_x ** 2) == outputSize, + ("Input layer %s: Incorrect input image size %d for input " + + "image pixels %d") + % (input_layer_name, conv_conf.img_size, img_pixels)) + if conv.caffe_mode: + conv_conf.img_size = \ + (conv_conf.output_x - 1) * conv.stride \ + + conv.filter_size - 2 * conv.padding + else: + conv_conf.img_size = \ + (conv_conf.output_x - 1) * conv.stride \ + + conv.filter_size - 2 * conv.padding + 1 + + def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand_conf.channels = block_expand.channels block_expand_conf.stride_x = block_expand.stride_x @@ -1612,6 +1643,70 @@ class ConvLayer(ConvLayerBase): class ConvLayer(ConvLayerBase): layer_type = 'cudnn_conv' + +@config_layer('convt') +class ConvTransLayerBase(LayerBase): + layer_type = 'convt' + def __init__( + self, + name, + inputs=[], + bias=True, + num_filters=None, + shared_biases=False, + **xargs): + super(ConvLayerBase, self).__init__( + name, self.layer_type, 0, inputs=inputs, **xargs) + + if num_filters is not None: + self.config.num_filters = num_filters + + use_gpu = int(g_command_config_args.get("use_gpu", 0)) + parallel_nn = int(g_command_config_args.get("parallel_nn", 0)) + + # Automatically select cudnn_type for GPU and exconv for CPU + # if set type=conv, but still reserve the way user specify + # exconv or cudnn_conv manually. + if self.layer_type == "cudnn_convt": + config_assert(use_gpu, "cudnn_convt only support GPU") + + if (use_gpu == 1 and self.layer_type != "exconvt" and + (parallel_nn == 0 or self.config.device > -1)): + self.layer_type = "cudnn_convt" + else: + self.layer_type = "exconvt" + # need to specify layer in config + self.config.type = self.layer_type + + if shared_biases is not None: + self.config.shared_biases = shared_biases + + for input_index in xrange(len(self.inputs)): + input_layer = self.get_input_layer(input_index) + parse_convt( + self.inputs[input_index].conv, + input_layer.name, + self.config.inputs[input_index].conv_conf) + conv_conf = self.config.inputs[input_index].conv_conf + psize = self.calc_parameter_size(conv_conf) + print("output size for %s is %d " % (name, conv_conf.output_x)) + self.create_input_parameter(input_index, psize) + self.set_layer_size( + (conv_conf.img_size ** 2) * self.config.num_filters) + + psize = self.config.size + if shared_biases: + psize = self.config.num_filters + self.create_bias_parameter(bias, psize, [psize, 1]) + + def calc_parameter_size(self, conv_conf): + return conv_conf.channels() * conv_conf.filter_channels \ + * (conv_conf.filter_size * conv_conf.filter_size_y) + +@config_layer('exconvt') +class ConvTransLayer(ConvTransLayerBase): + layer_type = 'exconvt' + @config_layer('norm') class NormLayer(LayerBase): def __init__( diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 49f0ff3289db7..853df8b83709d 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -78,6 +78,7 @@ class LayerType(object): COSINE_SIM = 'cos' HSIGMOID = 'hsigmoid' CONV_LAYER = "conv" + CONVTRANS_LAYER = "convt" POOL_LAYER = "pool" BATCH_NORM_LAYER = 'batch_norm' NORM_LAYER = 'norm' @@ -1625,6 +1626,128 @@ def img_conv_layer(input, filter_size, num_filters, return LayerOutput(name, LayerType.CONV_LAYER, parents=[input], activation=act, num_filters=num_filters) +@wrap_name_default("convt") +@wrap_param_attr_default() +@wrap_bias_attr_default() +@wrap_act_default(act=ReluActivation()) +@layer_support(DROPOUT) +def img_convTrans_layer(input, filter_size, num_filters, + name=None, num_channels=None, + act=None, groups=1, stride=1, padding=0, bias_attr=None, + param_attr=None, shared_biases=True, layer_attr=None, + filter_size_y=None, stride_y=None, padding_y=None): + """ + Convolution Transpose (deconv) layer for image. Paddle only support square + input currently and thus input image's width equals height. + + The details of convolution transpose layer, + please refer to the following explanation and references therein + `_ . + + The num_channel means input image's channel number. It may be 1 or 3 when + input is raw pixels of image(mono or RGB), or it may be the previous layer's + num_filters * num_group. + + There are several group of filter in PaddlePaddle implementation. + Each group will process some channel of the inputs. For example, if an input + num_channel = 256, group = 4, num_filter=32, the PaddlePaddle will create + 32*4 = 128 filters to process inputs. The channels will be split into 4 + pieces. First 256/4 = 64 channels will process by first 32 filters. The + rest channels will be processed by rest group of filters. + + :param name: Layer name. + :type name: basestring + :param input: Layer Input. + :type input: LayerOutput + :param filter_size: The x dimension of a filter kernel. Or input a tuple for + two image dimension. + :type filter_size: int|tuple|list + :param filter_size_y: The y dimension of a filter kernel. Since PaddlePaddle + currently supports rectangular filters, the filter's + shape will be (filter_size, filter_size_y). + :type filter_size_y: int|None + :param num_filters: Each filter group's number of filter + :param act: Activation type. Default is tanh + :type act: BaseActivation + :param groups: Group size of filters. + :type groups: int + :param stride: The x dimension of the stride. Or input a tuple for two image + dimension. + :type stride: int|tuple|list + :param stride_y: The y dimension of the stride. + :type stride_y: int + :param padding: The x dimension of the padding. Or input a tuple for two + image dimension + :type padding: int|tuple|list + :param padding_y: The y dimension of the padding. + :type padding_y: int + :param bias_attr: Convolution bias attribute. None means default bias. + False means no bias. + :type bias_attr: ParameterAttribute|False + :param num_channels: number of input channels. If None will be set + automatically from previous output. + :type num_channels: int + :param param_attr: Convolution param attribute. None means default attribute + :type param_attr: ParameterAttribute + :param shared_biases: Is biases will be shared between filters or not. + :type shared_biases: bool + :param layer_attr: Layer Extra Attribute. + :type layer_attr: ExtraLayerAttribute + :return: LayerOutput object. + :rtype: LayerOutput + """ + if num_channels is None: + assert input.num_filters is not None + num_channels = input.num_filters + + if filter_size_y is None: + if isinstance(filter_size, collections.Sequence): + assert len(filter_size) == 2 + filter_size, filter_size_y = filter_size + else: + filter_size_y = filter_size + + if stride_y is None: + if isinstance(stride, collections.Sequence): + assert len(stride) == 2 + stride, stride_y = stride + else: + stride_y = stride + + if padding_y is None: + if isinstance(padding, collections.Sequence): + assert len(padding) == 2 + padding, padding_y = padding + else: + padding_y = padding + + if param_attr.attr.get('initial_smart'): + # special initial for conv layers. + init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 + param_attr.attr["initial_mean"] = 0.0 + param_attr.attr["initial_std"] = init_w + param_attr.attr["initial_strategy"] = 0 + param_attr.attr["initial_smart"] = False + Layer( + name=name, + inputs=Input(input.name, conv=Conv( + filter_size=filter_size, padding=padding, stride=stride, + channels=num_channels, groups=groups, + filter_size_y=filter_size_y, padding_y=padding_y, + stride_y=stride_y), + **param_attr.attr), + active_type=act.name, + num_filters=num_filters, + bias=ParamAttr.to_bias(bias_attr), + shared_biases=shared_biases, + type=LayerType.CONVTRANS_LAYER, + **ExtraLayerAttribute.to_kwargs(layer_attr) + ) + return LayerOutput(name, LayerType.CONVTRANS_LAYER, parents=[input], + activation=act, num_filters=num_filters) + + @wrap_name_default("pool") @layer_support() From bda259bb18b484b88d0a144392c2ccb1b1530769 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Wed, 26 Oct 2016 16:36:54 -0700 Subject: [PATCH 265/324] added more test on convTrans layer and comments --- paddle/gserver/layers/ConvTransBaseLayer.cpp | 11 ++ paddle/gserver/layers/ConvTransBaseLayer.h | 5 + .../gserver/layers/ExpandConvTransLayer.cpp | 5 + paddle/gserver/layers/ExpandConvTransLayer.h | 2 +- paddle/gserver/tests/test_ConvTrans.cpp | 116 +++++++++++++++++- python/paddle/trainer/config_parser.py | 14 +-- .../paddle/trainer_config_helpers/layers.py | 2 +- 7 files changed, 144 insertions(+), 11 deletions(-) diff --git a/paddle/gserver/layers/ConvTransBaseLayer.cpp b/paddle/gserver/layers/ConvTransBaseLayer.cpp index 68fb48a38b518..1b58b7fed43d4 100644 --- a/paddle/gserver/layers/ConvTransBaseLayer.cpp +++ b/paddle/gserver/layers/ConvTransBaseLayer.cpp @@ -23,6 +23,17 @@ bool ConvTransBaseLayer::init(const LayerMap& layerMap, Layer::init(layerMap, parameterMap); /* Initialize the convolutional layer parameter */ + /* Everything is the same as ConvBaseLayer.cpp except that the meaning of + * num_filters and channel is switched. + * + * In the config, num_filters refer to the number of feature maps in the + * output of convTransLayer, and channel refer to the number of feature maps + * in the input of convTransLayer. + * + * However, within the convTrans class, the channel is related to the output + * and num_filters is related to the input, so that it is consistent with the + * settings in convLayer. + * */ channel_ = config_.num_filters(); sharedBiases_ = config_.shared_biases(); for (auto& inputConfig : config_.inputs()) { diff --git a/paddle/gserver/layers/ConvTransBaseLayer.h b/paddle/gserver/layers/ConvTransBaseLayer.h index 467a260569759..d7acc184cc9ac 100644 --- a/paddle/gserver/layers/ConvTransBaseLayer.h +++ b/paddle/gserver/layers/ConvTransBaseLayer.h @@ -96,6 +96,11 @@ class ConvTransBaseLayer : public Layer { * - outputSize = 5; */ + /* + * In order to be consistent with the convLayer, here the outputSize is + * actually the size of the input image of convTransLayer, and the image size + * is actually the size of the output image of convTransLayer + */ int imageSize(int outputSize, int filterSize, int padding, int stride) { int imageSize; if (!caffeMode_) { diff --git a/paddle/gserver/layers/ExpandConvTransLayer.cpp b/paddle/gserver/layers/ExpandConvTransLayer.cpp index 56cc042653b60..67c045821d173 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.cpp +++ b/paddle/gserver/layers/ExpandConvTransLayer.cpp @@ -17,6 +17,11 @@ limitations under the License. */ #include "paddle/utils/Stat.h" #include "ExpandConvTransLayer.h" +/* The implementation of the convTransLayer is basically a swap of forward and + * backward of the original convLayer. + * The variable naming follows the convention of the convLayer. + * */ + namespace paddle { REGISTER_LAYER(exconvt, ExpandConvTransLayer); diff --git a/paddle/gserver/layers/ExpandConvTransLayer.h b/paddle/gserver/layers/ExpandConvTransLayer.h index f19aa81f3d3c2..a6591fe1aa386 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.h +++ b/paddle/gserver/layers/ExpandConvTransLayer.h @@ -26,7 +26,7 @@ namespace paddle { * This layer expands input and use matrix multiplication to * calculate convolution operation. * - * The config file api is img_conv_layer. + * The config file api is img_convTrans_layer. */ class ExpandConvTransLayer : public ConvTransBaseLayer { protected: diff --git a/paddle/gserver/tests/test_ConvTrans.cpp b/paddle/gserver/tests/test_ConvTrans.cpp index e7cbe2614faca..787113d242391 100644 --- a/paddle/gserver/tests/test_ConvTrans.cpp +++ b/paddle/gserver/tests/test_ConvTrans.cpp @@ -33,7 +33,9 @@ P_DECLARE_double(checkgrad_eps); P_DECLARE_bool(thread_local_rand_use_global_seed); P_DECLARE_bool(prev_batch_state); +// Test that the convTrans forward is the same as conv backward TEST(Layer, convTransLayerFwd) { + // Setting up conv-trans layer TestConfig configt; configt.biasSize = 3; configt.layerConfig.set_type("exconvt"); @@ -68,7 +70,7 @@ TEST(Layer, convTransLayerFwd) { LayerMap layerMap; vector datas; initDataLayer(configt, &dataLayers, &datas, &layerMap, "convTrans", - 100, false, useGpu); + 100, false, false); // test layer initialize std::vector parameters; LayerPtr convtLayer; @@ -76,6 +78,7 @@ TEST(Layer, convTransLayerFwd) { convtLayer->getBiasParameter()->zeroMem(); convtLayer->forward(PASS_GC); + // Setting up conv-layer config TestConfig config; config.biasSize = 16; config.layerConfig.set_type("exconv"); @@ -109,16 +112,18 @@ TEST(Layer, convTransLayerFwd) { LayerMap layerMap2; vector datas2; initDataLayer(config, &dataLayers2, &datas2, &layerMap2, "conv", - 100, false, useGpu); + 100, false, false); // test layer initialize std::vector parameters2; LayerPtr convLayer; initTestLayer(config, &layerMap2, ¶meters2, &convLayer); + // Sync convLayer and convtLayer parameter convLayer->getBiasParameter()->zeroMem(); convLayer->getParameters()[0]->getBuf(PARAMETER_VALUE)->copyFrom( *(convtLayer->getParameters()[0]->getBuf(PARAMETER_VALUE))); + // Set convLayer outputGrad as convTransLayer input value convLayer->forward(PASS_GC); convLayer->getOutput().grad->copyFrom(*(dataLayers[0]->getOutputValue())); @@ -126,10 +131,117 @@ TEST(Layer, convTransLayerFwd) { auto callback = [&](Parameter* para) { ++callbackFlags[para->getID()]; }; convLayer->backward(callback); + // Check that the convLayer backward is the same as convTransLayer forward checkMatrixEqual(convtLayer->getOutputValue(), dataLayers2[0]->getOutputGrad()); } + +// Do one forward pass of convTrans layer and check to see if its output +// matches the given result +void doOneConvtTest(size_t imgSize, size_t output_x, size_t stride, + size_t padding, size_t filter_size, MatrixPtr& result) { + TestConfig configt; + configt.biasSize = 1; + configt.layerConfig.set_type("exconvt"); + configt.layerConfig.set_num_filters(1); + configt.layerConfig.set_partial_sum(1); + configt.layerConfig.set_shared_biases(true); + + configt.inputDefs.push_back({INPUT_DATA, "layer_0", output_x * output_x, + filter_size * filter_size}); + LayerInputConfig* input = configt.layerConfig.add_inputs(); + ConvConfig* conv = input->mutable_conv_conf(); + conv->set_filter_size(filter_size); + conv->set_filter_size_y(filter_size); + conv->set_channels(1); + conv->set_padding(padding); + conv->set_padding_y(padding); + conv->set_stride(stride); + conv->set_stride_y(stride); + conv->set_groups(1); + conv->set_filter_channels(1); + conv->set_img_size(imgSize); + conv->set_output_x(output_x); + + configt.layerConfig.set_size(conv->img_size() * conv->img_size() * + configt.layerConfig.num_filters()); + configt.layerConfig.set_name("convTrans"); + + std::vector dataLayers; + LayerMap layerMap; + vector datas; + initDataLayer(configt, &dataLayers, &datas, &layerMap, "convTrans", + 1, false, false); + dataLayers[0]->getOutputValue()->zeroMem(); + dataLayers[0]->getOutputValue()->add(1.0); + + // test layer initialize + std::vector parameters; + LayerPtr convtLayer; + initTestLayer(configt, &layerMap, ¶meters, &convtLayer); + convtLayer->getBiasParameter()->zeroMem(); + convtLayer->getParameters()[0]->zeroMem(); + convtLayer->getParameters()[0]->getBuf(PARAMETER_VALUE)->add(1.0); + convtLayer->forward(PASS_GC); + + checkMatrixEqual(convtLayer->getOutputValue(), result); +} + +TEST(Layer, convTransLayerFwd2) { + size_t imgSize, output_x, stride, padding, filter_size; + MatrixPtr result; + + imgSize = 5; + output_x = 1; + stride = 1; + padding = 0; + filter_size = 5; + result = Matrix::create(1, imgSize * imgSize, false, false); + result->zeroMem(); + result->add(1.0); + doOneConvtTest(imgSize, output_x, stride, padding, filter_size, result); + + imgSize = 5; + output_x = 2; + stride = 1; + padding = 0; + filter_size = 4; + float resultData[] = {1, 2, 2, 2, 1, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 1, 2, 2, 2, 1}; + result = Matrix::create(resultData, 1, imgSize * imgSize, false, false); + doOneConvtTest(imgSize, output_x, stride, padding, filter_size, result); + + imgSize = 5; + output_x = 2; + stride = 2; + padding = 1; + filter_size = 5; + float resultData2[] = {1, 2, 2, 2, 1, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 2, 4, 4, 4, 2, + 1, 2, 2, 2, 1}; + result = Matrix::create(resultData2, 1, imgSize * imgSize, false, false); + doOneConvtTest(imgSize, output_x, stride, padding, filter_size, result); + + imgSize = 5; + output_x = 2; + stride = 2; + padding = 0; + filter_size = 3; + float resultData3[] = {1, 1, 2, 1, 1, + 1, 1, 2, 1, 1, + 2, 2, 4, 2, 2, + 1, 1, 2, 1, 1, + 1, 1, 2, 1, 1}; + result = Matrix::create(resultData3, 1, imgSize * imgSize, false, false); + doOneConvtTest(imgSize, output_x, stride, padding, filter_size, result); +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 2d28b34999cb0..95c5f774c6cea 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1107,7 +1107,7 @@ def parse_conv(conv, input_layer_name, conv_conf): conv_conf.caffe_mode) -def parse_convt(conv, input_layer_name, conv_conf): +def parse_convt(conv, input_layer_name, conv_conf, num_filters): conv_conf.filter_size = conv.filter_size conv_conf.filter_size_y = conv.filter_size_y conv_conf.channels = conv.channels @@ -1116,7 +1116,7 @@ def parse_convt(conv, input_layer_name, conv_conf): conv_conf.stride = conv.stride conv_conf.stride_y = conv.stride_y conv_conf.groups = conv.groups - conv_conf.filter_channels = conv.channels / conv.groups + conv_conf.filter_channels = num_filters / conv.groups conv_conf.caffe_mode = conv.caffe_mode outputSize = g_layer_map[input_layer_name].size / conv.channels @@ -1126,14 +1126,14 @@ def parse_convt(conv, input_layer_name, conv_conf): config_assert((conv_conf.output_x ** 2) == outputSize, ("Input layer %s: Incorrect input image size %d for input " + "image pixels %d") - % (input_layer_name, conv_conf.img_size, img_pixels)) + % (input_layer_name, conv_conf.output_x, outputSize)) if conv.caffe_mode: conv_conf.img_size = \ (conv_conf.output_x - 1) * conv.stride \ + conv.filter_size - 2 * conv.padding else: conv_conf.img_size = \ - (conv_conf.output_x - 1) * conv.stride \ + (conv_conf.output_x - 2) * conv.stride \ + conv.filter_size - 2 * conv.padding + 1 @@ -1655,7 +1655,7 @@ def __init__( num_filters=None, shared_biases=False, **xargs): - super(ConvLayerBase, self).__init__( + super(ConvTransLayerBase, self).__init__( name, self.layer_type, 0, inputs=inputs, **xargs) if num_filters is not None: @@ -1686,7 +1686,7 @@ def __init__( parse_convt( self.inputs[input_index].conv, input_layer.name, - self.config.inputs[input_index].conv_conf) + self.config.inputs[input_index].conv_conf, num_filters) conv_conf = self.config.inputs[input_index].conv_conf psize = self.calc_parameter_size(conv_conf) print("output size for %s is %d " % (name, conv_conf.output_x)) @@ -1700,7 +1700,7 @@ def __init__( self.create_bias_parameter(bias, psize, [psize, 1]) def calc_parameter_size(self, conv_conf): - return conv_conf.channels() * conv_conf.filter_channels \ + return conv_conf.channels * conv_conf.filter_channels \ * (conv_conf.filter_size * conv_conf.filter_size_y) @config_layer('exconvt') diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 853df8b83709d..172d45a761ecc 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -36,7 +36,7 @@ "pooling_layer", "lstmemory", "last_seq", "first_seq", "cos_sim", "hsigmoid", "conv_projection", "regression_cost", 'classification_cost', "LayerOutput", - 'img_conv_layer', 'img_pool_layer', 'batch_norm_layer', + 'img_conv_layer', 'img_convTrans_layer', 'img_pool_layer', 'batch_norm_layer', 'img_cmrnorm_layer', 'addto_layer', 'concat_layer', 'lstm_step_layer', 'recurrent_group', 'memory', 'StaticInput', 'expand_layer', 'scaling_layer', From aa2cd2ce8f01072a2e604740bd350417c23485c4 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Tue, 1 Nov 2016 10:22:00 -0700 Subject: [PATCH 266/324] Refactor ExpandConvTransLayer to share codes with ExpandConvLayer --- paddle/gserver/layers/ConvBaseLayer.cpp | 7 ++ paddle/gserver/layers/ConvBaseLayer.h | 67 +++++++++++++++++++ paddle/gserver/layers/ExpandConvLayer.cpp | 18 ----- paddle/gserver/layers/ExpandConvLayer.h | 19 +----- .../gserver/layers/ExpandConvTransLayer.cpp | 40 ++--------- paddle/gserver/layers/ExpandConvTransLayer.h | 20 ++---- 6 files changed, 83 insertions(+), 88 deletions(-) diff --git a/paddle/gserver/layers/ConvBaseLayer.cpp b/paddle/gserver/layers/ConvBaseLayer.cpp index 42ff0b70d86f7..4346cb520ea01 100644 --- a/paddle/gserver/layers/ConvBaseLayer.cpp +++ b/paddle/gserver/layers/ConvBaseLayer.cpp @@ -21,6 +21,12 @@ bool ConvBaseLayer::init(const LayerMap& layerMap, /* Initialize the basic parent class */ Layer::init(layerMap, parameterMap); + if (config_.type() == "exconv" || config_.type() == "cudnn_conv") { + isConv_ = true; + } else { + isConv_ = false; + } + /* Initialize the convolutional layer parameter */ numFilters_ = config_.num_filters(); sharedBiases_ = config_.shared_biases(); @@ -88,6 +94,7 @@ size_t ConvBaseLayer::calOutputSize() { getOutput().setFrameWidth(outputW_[0]); layerSize = outputH_[0] * outputW_[0] * size_t(numFilters_); return layerSize; + } } // namespace paddle diff --git a/paddle/gserver/layers/ConvBaseLayer.h b/paddle/gserver/layers/ConvBaseLayer.h index e660a6d6f50ac..24927dec24d0c 100644 --- a/paddle/gserver/layers/ConvBaseLayer.h +++ b/paddle/gserver/layers/ConvBaseLayer.h @@ -28,6 +28,9 @@ class ConvBaseLayer : public Layer { protected: typedef std::vector IntV; + /// True if it's convolution layer, false if it's deconv layer + bool isConv_; + /// The number of filters. int numFilters_; /// The x dimension of the padding. @@ -75,6 +78,13 @@ class ConvBaseLayer : public Layer { /// of output size. bool caffeMode_; + /*The expandInput_ and transOutValue_ are used for CPU expand conv calc*/ + /// Expand one sample at a time. shape: + /// (numChannels * filterPixels_, outputSizeH * outputSizeW) + MatrixPtr expandInput_; + /// The transpose of output, which is an auxiliary matrix. + MatrixPtr transOutValue_; + public: explicit ConvBaseLayer(const LayerConfig& config) : Layer(config) {} @@ -88,6 +98,63 @@ class ConvBaseLayer : public Layer { virtual size_t calOutputSize(); Weight& getWeight(int idx) { return *weights_[idx]; } + + /** + * Calculate output size based on caffeMode_. + * - input(+padding): 0123456789 + * - imageSize(+padding) = 10; + * - filterSize = 3; + * - stride = 2; + * - caffeMode_ is true: + - output: (012), (234), (456), (678) + - outputSize = 4; + * - caffeMode_ is false: + * - output: (012), (234), (456), (678), (9) + * - outputSize = 5; + */ + int outputSize(int imageSize, int filterSize, int padding, int stride) { + int outputSize; + if (!caffeMode_) { + outputSize = + (imageSize - filterSize + 2 * padding + stride - 1) / stride + 1; + } else { + outputSize = (imageSize - filterSize + 2 * padding) / stride + 1; + } + CHECK_GE(outputSize, 1); + return outputSize; + } + + int imageSize(int outputSize, int filterSize, int padding, int stride) { + int imageSize; + if (!caffeMode_) { + imageSize = + (outputSize - 1) * stride + filterSize - 2 * padding - stride + 1; + } else { + imageSize = (outputSize - 1) * stride + filterSize - 2 * padding; + } + CHECK_GE(imageSize, 1); + return imageSize; + } + + /** + * Create or resize expandInput_. + */ + void resetExpandInput(size_t height, size_t width); + + /** + * Create or resize transOutValue_. + */ + void resetConvOutput(size_t batchSize, int inIdx); + + /** + * Add shared bias. + */ + void addSharedBias(); + + /** + * Add unshared bias. + */ + void addUnsharedBias(); }; } // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvLayer.cpp b/paddle/gserver/layers/ExpandConvLayer.cpp index 80a6a62b5c0de..866cd33c118d0 100644 --- a/paddle/gserver/layers/ExpandConvLayer.cpp +++ b/paddle/gserver/layers/ExpandConvLayer.cpp @@ -63,14 +63,6 @@ size_t ExpandConvLayer::getOutputSize() { return layerSize; } -void ExpandConvLayer::resetExpandInput(size_t height, size_t width) { - Matrix::resizeOrCreate(expandInput_, height, width, false, useGpu_); -} - -void ExpandConvLayer::resetConvOutput(size_t batchSize, int inIdx) { - Matrix::resizeOrCreate(transOutValue_, batchSize * numFilters_, subN_[inIdx], - false, useGpu_); -} void ExpandConvLayer::expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx) { @@ -135,17 +127,7 @@ void ExpandConvLayer::addSharedBias() { transOutValue_->reshape(mapW, mapH); transOutValue_->transpose(out, false); // false means no memory allocation - out->clear(); - bias->clear(); -} -void ExpandConvLayer::addUnsharedBias() { - MatrixPtr outValue = getOutputValue(); - MatrixPtr bias = - Matrix::create(biases_->getW()->getData(), 1, - biases_->getW()->getElementCnt(), false, useGpu_); - outValue->addBias(*bias, 1.0f); -} void ExpandConvLayer::forward(PassType passType) { Layer::forward(passType); diff --git a/paddle/gserver/layers/ExpandConvLayer.h b/paddle/gserver/layers/ExpandConvLayer.h index 030a3ba397ff4..f43b199498ec3 100644 --- a/paddle/gserver/layers/ExpandConvLayer.h +++ b/paddle/gserver/layers/ExpandConvLayer.h @@ -43,6 +43,7 @@ class ExpandConvLayer : public ConvBaseLayer { /// The transpose of output, which is an auxiliary matrix. MatrixPtr transOutValue_; + public: explicit ExpandConvLayer(const LayerConfig& config) : ConvBaseLayer(config) {} @@ -52,16 +53,6 @@ class ExpandConvLayer : public ConvBaseLayer { size_t getOutputSize(); - /** - * Create or resize expandInput_. - */ - void resetExpandInput(size_t height, size_t width); - - /** - * Create or resize transOutValue_. - */ - void resetConvOutput(size_t batchSize, int inIdx); - /** * Expand one input sample. */ @@ -72,15 +63,7 @@ class ExpandConvLayer : public ConvBaseLayer { */ void expandFwdOnce(MatrixPtr image, int inIdx, int startIdx); - /** - * Add shared bias. - */ - void addSharedBias(); - /** - * Add unshared bias. - */ - void addUnsharedBias(); void forward(PassType passType); void bpropSharedBias(MatrixPtr biases, MatrixPtr v); void bpropBiases(MatrixPtr v); diff --git a/paddle/gserver/layers/ExpandConvTransLayer.cpp b/paddle/gserver/layers/ExpandConvTransLayer.cpp index 67c045821d173..fb2e7fc4bd6e2 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.cpp +++ b/paddle/gserver/layers/ExpandConvTransLayer.cpp @@ -29,14 +29,14 @@ REGISTER_LAYER(exconvt, ExpandConvTransLayer); bool ExpandConvTransLayer::init(const LayerMap &layerMap, const ParameterMap ¶meterMap) { /* Initialize the basic convolutional parent class */ - ConvTransBaseLayer::init(layerMap, parameterMap); + ConvBaseLayer::init(layerMap, parameterMap); /* Initialize the projection */ for (auto &inputConfig : config_.inputs()) { const ConvConfig &conf = inputConfig.conv_conf(); subM_.push_back(conf.channels() / conf.groups()); subN_.push_back(conf.output_x() * conf.output_x()); - subK_.push_back(channel_ * conf.filter_size() * conf.filter_size() / + subK_.push_back(numFilters_ * conf.filter_size() * conf.filter_size() / conf.groups()); /* Consistent caffe mode for multiple input */ caffeMode_ = conf.caffe_mode(); @@ -65,8 +65,8 @@ size_t ExpandConvTransLayer::getSize() { imageSize(outputW_[i], filterSize_[i], padding_[i], stride_[i])); subN_.push_back(outputH_[i] * outputW_[i]); CHECK(layerSize == 0 || - imgSizeH_[i] * imgSizeW_[i] * (size_t)channel_ == layerSize); - layerSize = imgSizeH_[i] * imgSizeW_[i] * channel_; + imgSizeH_[i] * imgSizeW_[i] * (size_t)numFilters_ == layerSize); + layerSize = imgSizeH_[i] * imgSizeW_[i] * numFilters_; } getOutput().setFrameHeight(imgSizeH_[0]); getOutput().setFrameWidth(imgSizeW_[0]); @@ -83,38 +83,6 @@ void ExpandConvTransLayer::resetExpandInput(size_t height, size_t width) { }*/ -void ExpandConvTransLayer::addSharedBias() { - size_t mapW = getSize() / channel_; - size_t mapH = getOutputValue()->getElementCnt() / mapW; - MatrixPtr out = - Matrix::create(getOutputValue()->getData(), mapH, mapW, false, useGpu_); - - Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); - - out->transpose(transOutValue_, false); // false means no memory allocation - transOutValue_->reshape(transOutValue_->getElementCnt() / channel_, - channel_); - - MatrixPtr bias = - Matrix::create(biases_->getW()->getData(), 1, - biases_->getW()->getElementCnt(), false, useGpu_); - transOutValue_->addBias(*bias, 1.0f); - - transOutValue_->reshape(mapW, mapH); - transOutValue_->transpose(out, false); // false means no memory allocation - - out->clear(); - bias->clear(); -} - -void ExpandConvTransLayer::addUnsharedBias() { - MatrixPtr outValue = getOutputValue(); - MatrixPtr bias = - Matrix::create(biases_->getW()->getData(), 1, - biases_->getW()->getElementCnt(), false, useGpu_); - outValue->addBias(*bias, 1.0f); -} - void ExpandConvTransLayer::expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx) { diff --git a/paddle/gserver/layers/ExpandConvTransLayer.h b/paddle/gserver/layers/ExpandConvTransLayer.h index a6591fe1aa386..cbe4da8143cef 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.h +++ b/paddle/gserver/layers/ExpandConvTransLayer.h @@ -15,7 +15,7 @@ limitations under the License. */ #pragma once -#include "ConvTransBaseLayer.h" +#include "ConvBaseLayer.h" #include "paddle/math/Matrix.h" #include @@ -28,7 +28,7 @@ namespace paddle { * * The config file api is img_convTrans_layer. */ -class ExpandConvTransLayer : public ConvTransBaseLayer { +class ExpandConvTransLayer : public ConvBaseLayer { protected: /// For expand convolution. /// subM_ = numFilters_ / groups_. @@ -45,15 +45,11 @@ class ExpandConvTransLayer : public ConvTransBaseLayer { IntV outputH_; /// The spatial dimensions of width of output feature map. IntV outputW_; - /// Expand one sample at a time. shape: - /// (numChannels * filterPixels_, outputSizeH * outputSizeW) - MatrixPtr expandInput_; - /// The transpose of output, which is an auxiliary matrix. - MatrixPtr transOutValue_; + public: explicit ExpandConvTransLayer(const LayerConfig& config) : - ConvTransBaseLayer(config) {} + ConvBaseLayer(config) {} ~ExpandConvTransLayer() {} @@ -86,15 +82,7 @@ class ExpandConvTransLayer : public ConvTransBaseLayer { */ void shrinkFwd(MatrixPtr output, int inpIdx); - /** - * Add shared bias. - */ - void addSharedBias(); - /** - * Add unshared bias. - */ - void addUnsharedBias(); void forward(PassType passType); void bpropSharedBias(MatrixPtr biases, MatrixPtr v); void bpropBiases(MatrixPtr v); From 2575b74feefd10dde3990452c53e595f83f08d52 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Tue, 1 Nov 2016 15:12:17 -0700 Subject: [PATCH 267/324] refactored ExpandConvLayer and ExpandConvTransLayer with ConvBaseLayerCpu --- paddle/gserver/layers/ConvBaseLayer.h | 27 +- paddle/gserver/layers/ConvBaseLayerCpu.cpp | 241 ++++++++++++++++++ paddle/gserver/layers/ConvBaseLayerCpu.h | 91 +++++++ paddle/gserver/layers/ConvTransBaseLayer.cpp | 88 ------- paddle/gserver/layers/ConvTransBaseLayer.h | 117 --------- paddle/gserver/layers/ExpandConvLayer.cpp | 219 +--------------- paddle/gserver/layers/ExpandConvLayer.h | 36 +-- .../gserver/layers/ExpandConvTransLayer.cpp | 195 +------------- paddle/gserver/layers/ExpandConvTransLayer.h | 56 +--- paddle/gserver/tests/test_LayerGrad.cpp | 2 + 10 files changed, 355 insertions(+), 717 deletions(-) create mode 100644 paddle/gserver/layers/ConvBaseLayerCpu.cpp create mode 100644 paddle/gserver/layers/ConvBaseLayerCpu.h delete mode 100644 paddle/gserver/layers/ConvTransBaseLayer.cpp delete mode 100644 paddle/gserver/layers/ConvTransBaseLayer.h diff --git a/paddle/gserver/layers/ConvBaseLayer.h b/paddle/gserver/layers/ConvBaseLayer.h index 24927dec24d0c..ecdc119a94941 100644 --- a/paddle/gserver/layers/ConvBaseLayer.h +++ b/paddle/gserver/layers/ConvBaseLayer.h @@ -78,12 +78,7 @@ class ConvBaseLayer : public Layer { /// of output size. bool caffeMode_; - /*The expandInput_ and transOutValue_ are used for CPU expand conv calc*/ - /// Expand one sample at a time. shape: - /// (numChannels * filterPixels_, outputSizeH * outputSizeW) - MatrixPtr expandInput_; - /// The transpose of output, which is an auxiliary matrix. - MatrixPtr transOutValue_; + public: explicit ConvBaseLayer(const LayerConfig& config) : Layer(config) {} @@ -135,26 +130,6 @@ class ConvBaseLayer : public Layer { CHECK_GE(imageSize, 1); return imageSize; } - - /** - * Create or resize expandInput_. - */ - void resetExpandInput(size_t height, size_t width); - - /** - * Create or resize transOutValue_. - */ - void resetConvOutput(size_t batchSize, int inIdx); - - /** - * Add shared bias. - */ - void addSharedBias(); - - /** - * Add unshared bias. - */ - void addUnsharedBias(); }; } // namespace paddle diff --git a/paddle/gserver/layers/ConvBaseLayerCpu.cpp b/paddle/gserver/layers/ConvBaseLayerCpu.cpp new file mode 100644 index 0000000000000..0da92bf0485b0 --- /dev/null +++ b/paddle/gserver/layers/ConvBaseLayerCpu.cpp @@ -0,0 +1,241 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#include "paddle/utils/Logging.h" +#include "ConvBaseLayerCpu.h" +namespace paddle { + +bool ConvBaseLayerCpu::init(const LayerMap &layerMap, + const ParameterMap ¶meterMap) { + /* Initialize the basic convolutional parent class */ + ConvBaseLayer::init(layerMap, parameterMap); + + int channel; + /* Initialize the projection */ + for (auto &inputConfig : config_.inputs()) { + const ConvConfig &conf = inputConfig.conv_conf(); + subM_.push_back(numFilters_ / conf.groups()); + subN_.push_back(conf.output_x() * conf.output_x()); + channel = isConv_ ? conf.channels() : numFilters_; + subK_.push_back(channel * conf.filter_size() * conf.filter_size() / + conf.groups()); + /* Consistent caffe mode for multiple input */ + caffeMode_ = conf.caffe_mode(); + } + + return true; +} + +void ConvBaseLayerCpu::resetExpandInput(size_t height, size_t width) { + Matrix::resizeOrCreate(expandInput_, height, width, false, useGpu_); +} + +void ConvBaseLayerCpu::addSharedBias() { + size_t mapW = getSize() / numFilters_; + size_t mapH = getOutputValue()->getElementCnt() / mapW; + MatrixPtr out = + Matrix::create(getOutputValue()->getData(), mapH, mapW, false, useGpu_); + + Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); + + out->transpose(transOutValue_, false); // false means no memory allocation + transOutValue_->reshape(transOutValue_->getElementCnt() / numFilters_, + numFilters_); + + MatrixPtr bias = + Matrix::create(biases_->getW()->getData(), 1, + biases_->getW()->getElementCnt(), false, useGpu_); + transOutValue_->addBias(*bias, 1.0f); + + transOutValue_->reshape(mapW, mapH); + transOutValue_->transpose(out, false); // false means no memory allocation + + out->clear(); + bias->clear(); +} + +void ConvBaseLayerCpu::addUnsharedBias() { + MatrixPtr outValue = getOutputValue(); + MatrixPtr bias = + Matrix::create(biases_->getW()->getData(), 1, + biases_->getW()->getElementCnt(), false, useGpu_); + outValue->addBias(*bias, 1.0f); +} + + +void ConvBaseLayerCpu::expandOneFrame(MatrixPtr image, size_t startIdx, + int inIdx) { + int channel = isConv_ ? channels_[inIdx] : numFilters_; + + resetExpandInput(subK_[inIdx] * groups_[inIdx], subN_[inIdx]); + real *imgData = image->getData() + startIdx * image->getWidth(); + MatrixPtr imageTmp = Matrix::create( + imgData, 1, imgSizeH_[inIdx] * imgSizeW_[inIdx] * channel, false, + useGpu_); + expandInput_->convExpand(*imageTmp, imgSizeH_[inIdx], imgSizeW_[inIdx], + channel, filterSize_[inIdx], + filterSize_[inIdx], stride_[inIdx], stride_[inIdx], + padding_[inIdx], padding_[inIdx], + outputH_[inIdx], outputW_[inIdx]); + imageTmp->clear(); +} + +void ConvBaseLayerCpu::expandFwdOnce(MatrixPtr image, MatrixPtr out, + int inIdx, int startIdx) { + int subM = subM_[inIdx]; + int subN = subN_[inIdx]; + int subK = subK_[inIdx]; + + expandOneFrame(image, startIdx, inIdx); + + int nf = isConv_ ? numFilters_ : channels_[inIdx]; + + real *outData = + out->getData() + startIdx * subN * nf; + + real *wgtData = weights_[inIdx]->getW()->getData(); + real *expInData = expandInput_->getData(); + for (int g = 0; g < groups_[inIdx]; ++g) { + MatrixPtr A = + Matrix::create(wgtData, subK, subM, true, useGpu_); // mark transpose + MatrixPtr B = Matrix::create(expInData, subK, subN, false, useGpu_); + MatrixPtr C = Matrix::create(outData, subM, subN, false, useGpu_); + C->mul(A, B, 1, 1); + + A->clear(); + B->clear(); + C->clear(); + wgtData += subK * subM; + expInData += subK * subN; + outData += subM * subN; + } +} + +void ConvBaseLayerCpu::bpropActs(MatrixPtr image, MatrixPtr out, int inpIdx) { + int channel = isConv_ ? channels_[inpIdx] : numFilters_; + + int subM = subM_[inpIdx]; + int subN = subN_[inpIdx]; + int subK = subK_[inpIdx]; + size_t batchSize = image->getHeight(); + MatrixPtr tgtGrad = out; + + /* reset the expand-grad memory */ + resetExpandInput(subK * groups_[inpIdx], subN); + + real *localGradData = image->getData(); + real *tgtGradData = tgtGrad->getData(); + for (size_t n = 0; n < batchSize; n++) { + real *wgtData = weights_[inpIdx]->getW()->getData(); + real *expandInData = expandInput_->getData(); + + for (int g = 0; g < groups_[inpIdx]; g++) { + // create temporary matrix + MatrixPtr C = Matrix::create(expandInData, subK, subN, false, useGpu_); + MatrixPtr B = Matrix::create(localGradData, subM, subN, false, useGpu_); + MatrixPtr A = Matrix::create(wgtData, subK, subM, false, useGpu_); + C->mul(A, B); // mul + + // clear the temporary matrix + A->clear(); + B->clear(); + C->clear(); + + expandInData += subK * subN; + localGradData += subM * subN; + wgtData += subK * subM; + } + + // shrink one frame outGrad + MatrixPtr oneGradTmp = Matrix::create( + expandInput_->getData(), subK * groups_[inpIdx], subN, false, useGpu_); + MatrixPtr vTmp = Matrix::create( + tgtGradData, 1, + imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channel, false, + useGpu_); + vTmp->convShrink(*oneGradTmp, imgSizeH_[inpIdx], imgSizeW_[inpIdx], + channel, filterSize_[inpIdx], + filterSize_[inpIdx], stride_[inpIdx], stride_[inpIdx], + padding_[inpIdx], padding_[inpIdx], + outputH_[inpIdx], outputW_[inpIdx], 1.0f, 1.0f); + vTmp->clear(); + oneGradTmp->clear(); + + // move the data-pointer + tgtGradData += imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channel; + } +} + +void ConvBaseLayerCpu::bpropWeights(MatrixPtr image, MatrixPtr out, + int inpIdx) { + MatrixPtr weightGrad = weights_[inpIdx]->getWGrad(); + + int subM = subM_[inpIdx]; + int subN = subN_[inpIdx]; + int subK = subK_[inpIdx]; + size_t batchSize = image->getHeight(); + resetExpandInput(subK * groups_[inpIdx], subN); + + real *gradData = out->getData(); + + for (size_t n = 0; n < batchSize; n++) { // frame by frame + // expand + expandOneFrame(image, n, inpIdx); + real *wGradData = weightGrad->getData(); + real *expandInData = expandInput_->getData(); + + // expand-mul one-group by one + for (int g = 0; g < groups_[inpIdx]; g++) { + MatrixPtr A = Matrix::create(expandInData, subK, subN, false, useGpu_); + MatrixPtr B = Matrix::create(gradData, subM, subN, true, useGpu_); + MatrixPtr C = Matrix::create(wGradData, subK, subM, false, useGpu_); + C->mul(A, B, 1, 1); + + A->clear(); + B->clear(); + C->clear(); + gradData += subM * subN; + wGradData += subK * subM; + expandInData += subK * subN; + } + } +} + +void ConvBaseLayerCpu::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { + size_t mapW = getSize() / numFilters_; + size_t mapH = v->getElementCnt() / mapW; + MatrixPtr vTmp = Matrix::create(v->getData(), mapH, mapW, false, useGpu_); + + Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); + + vTmp->transpose(transOutValue_, false); // false means no memory allocation + transOutValue_->reshape(transOutValue_->getElementCnt() / numFilters_, + numFilters_); + biases->collectBias(*transOutValue_, 1.0f); +} + +void ConvBaseLayerCpu::bpropBiases(MatrixPtr v) { + MatrixPtr biases = + Matrix::create(biases_->getWGrad()->getData(), 1, + biases_->getWGrad()->getElementCnt(), false, useGpu_); + if (sharedBiases_) { + bpropSharedBias(biases, v); + } else { + biases->collectBias(*v, 1.0f); + } + biases->clear(); +} + +} // namespace paddle diff --git a/paddle/gserver/layers/ConvBaseLayerCpu.h b/paddle/gserver/layers/ConvBaseLayerCpu.h new file mode 100644 index 0000000000000..08a1426b473a0 --- /dev/null +++ b/paddle/gserver/layers/ConvBaseLayerCpu.h @@ -0,0 +1,91 @@ +/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + + +#pragma once + +#include "ConvBaseLayer.h" +#include "paddle/math/Matrix.h" +#include + +namespace paddle { + +/** + * @brief A subclass of ConvBaseLayer that is a superclass of both + * ExpandConvLayer and ExpandConvTransLayer + */ +class ConvBaseLayerCpu : public ConvBaseLayer { +protected: + /// For expand convolution. + /// subM_ = numFilters_ / groups_. + IntV subM_; + /// subN_ = outputH_ * outputW_. + IntV subN_; + /// subK_ = channels_ * filterPixels_ * groups_. + IntV subK_; + /// The spatial dimensions of height of input feature map. + IntV imgSizeH_; + /// The spatial dimensions of width of input feature map. + IntV imgSizeW_; + /// The spatial dimensions of height of output feature map. + IntV outputH_; + /// The spatial dimensions of width of output feature map. + IntV outputW_; + + /*The expandInput_ and transOutValue_ are used for CPU expand conv calc*/ + /// Expand one sample at a time. shape: + /// (numChannels * filterPixels_, outputSizeH * outputSizeW) + MatrixPtr expandInput_; + /// The transpose of output, which is an auxiliary matrix. + MatrixPtr transOutValue_; + +public: + explicit ConvBaseLayerCpu(const LayerConfig& config) + : ConvBaseLayer(config) {} + + ~ConvBaseLayerCpu() {} + + bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + + /** + * Create or resize expandInput_. + */ + void resetExpandInput(size_t height, size_t width); + + /** + * Add shared bias. + */ + void addSharedBias(); + + /** + * Add unshared bias. + */ + void addUnsharedBias(); + /** + * Expand one input sample. + */ + void expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx); + + /** + * Expand one input sample and perform matrix multiplication. + */ + void expandFwdOnce(MatrixPtr image, MatrixPtr out, int inIdx, int startIdx); + + void bpropSharedBias(MatrixPtr biases, MatrixPtr v); + void bpropBiases(MatrixPtr v); + void bpropWeights(MatrixPtr image, MatrixPtr out, int inpIdx); + void bpropActs(MatrixPtr image, MatrixPtr out, int inpIdx); +}; + +} // namespace paddle diff --git a/paddle/gserver/layers/ConvTransBaseLayer.cpp b/paddle/gserver/layers/ConvTransBaseLayer.cpp deleted file mode 100644 index 1b58b7fed43d4..0000000000000 --- a/paddle/gserver/layers/ConvTransBaseLayer.cpp +++ /dev/null @@ -1,88 +0,0 @@ -/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - - -#include "paddle/utils/Logging.h" -#include "ConvTransBaseLayer.h" -namespace paddle { - -bool ConvTransBaseLayer::init(const LayerMap& layerMap, - const ParameterMap& parameterMap) { - /* Initialize the basic parent class */ - Layer::init(layerMap, parameterMap); - - /* Initialize the convolutional layer parameter */ - /* Everything is the same as ConvBaseLayer.cpp except that the meaning of - * num_filters and channel is switched. - * - * In the config, num_filters refer to the number of feature maps in the - * output of convTransLayer, and channel refer to the number of feature maps - * in the input of convTransLayer. - * - * However, within the convTrans class, the channel is related to the output - * and num_filters is related to the input, so that it is consistent with the - * settings in convLayer. - * */ - channel_ = config_.num_filters(); - sharedBiases_ = config_.shared_biases(); - for (auto& inputConfig : config_.inputs()) { - const ConvConfig& conf = inputConfig.conv_conf(); - padding_.push_back(conf.padding()); - stride_.push_back(conf.stride()); - filterSize_.push_back(conf.filter_size()); - paddingY_.push_back(conf.padding_y()); - strideY_.push_back(conf.stride_y()); - filterSizeY_.push_back(conf.filter_size_y()); - filterPixels_.push_back(filterSize_.back() * filterSizeY_.back()); - numFilters_.push_back(conf.channels()); - imgSize_.push_back(conf.img_size()); - imgPixels_.push_back(imgSize_.back() * imgSize_.back()); - groups_.push_back(conf.groups()); - filterChannels_.push_back(conf.filter_channels()); - outputX_.push_back(conf.output_x()); - outputs_.push_back(outputX_.back() * outputX_.back()); - } - - /* initialize the weightList */ - CHECK(inputLayers_.size() == parameters_.size()); - for (size_t i = 0; i < inputLayers_.size(); i++) { - size_t height, width; - height = filterPixels_[i] * filterChannels_[i]; - width = numFilters_[i]; - - // create a new weight - CHECK_EQ(parameters_[i]->getSize(), width * height); - Weight* w = new Weight(height, width, parameters_[i]); - weights_.emplace_back(w); - } - - /* initialize the biases_ */ - if (biasParameter_.get() != NULL) { - if (sharedBiases_) { - CHECK_EQ((size_t)channel_, biasParameter_->getSize()); - biases_ = - std::unique_ptr(new Weight(channel_, 1, biasParameter_)); - } else { - biases_ = - std::unique_ptr(new Weight(getSize(), 1, biasParameter_)); - } - } - - // default caffe model - caffeMode_ = true; - - return true; -} - -} // namespace paddle diff --git a/paddle/gserver/layers/ConvTransBaseLayer.h b/paddle/gserver/layers/ConvTransBaseLayer.h deleted file mode 100644 index d7acc184cc9ac..0000000000000 --- a/paddle/gserver/layers/ConvTransBaseLayer.h +++ /dev/null @@ -1,117 +0,0 @@ -/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - - -#pragma once - -#include "Layer.h" -namespace paddle { - -/** - * @brief A Base Convolution Layer, which convolves the input image - * with learned filters and (optionally) adds biases. - */ - -class ConvTransBaseLayer : public Layer { -protected: - typedef std::vector IntV; - - /// The number of channel in image (the output of the deconv layer). - int channel_; - /// The x dimension of the padding. - IntV padding_; - /// The y dimension of the padding. - IntV paddingY_; - /// The x dimension of the stride. - IntV stride_; - /// The y dimension of the stride. - IntV strideY_; - /// The x dimension of a filter kernel. - IntV filterSize_; - /// The y dimension of a filter kernel. - IntV filterSizeY_; - /// The number of filters(i.e. the number channels of the deconv layer input) - IntV numFilters_; - /// The spatial dimensions of input feature map. - IntV imgSize_; - /// The total pixel size of input feature map. - /// imgPixels_ = imgSizeX_ * imgSizeY_. - IntV imgPixels_; - /// filterPixels_ = filterSizeX_ * filterSizeY_. - IntV filterPixels_; - /// filterChannels_ = channels_/groups_. - IntV filterChannels_; - /// The spatial dimensions of output feature map. - IntV outputX_; - /// The spatial dimensions of output feature map. - IntV outputs_; - /// Group size, refer to grouped convolution in - /// Alex Krizhevsky's paper: when group=2, the first half of the - /// filters are only connected to the first half of the input channels, - /// and the second half only connected to the second half. - IntV groups_; - /// Whether the bias is shared for feature in each channel. - bool sharedBiases_; - - /// shape of weight: (numChannels * filterPixels_, numFilters) - WeightList weights_; - /// If shared_biases is false shape of bias: (numFilters_, 1) - /// If shared_biases is ture shape of bias: - /// (numFilters_ * outputX * outputY, 1) - std::unique_ptr biases_; - - /// True by default. The only difference is the calculation - /// of output size. - bool caffeMode_; - -public: - explicit ConvTransBaseLayer(const LayerConfig& config) : Layer(config) {} - - virtual bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); - - Weight& getWeight(int idx) { return *weights_[idx]; } - - /** - * Calculate image size based on caffeMode_ from outputSize. - * - input(+padding): 0123456789 - * - imageSize(+padding) = 10; - * - filterSize = 3; - * - stride = 2; - * - caffeMode_ is true: - - output: (012), (234), (456), (678) - - outputSize = 4; - * - caffeMode_ is false: - * - output: (012), (234), (456), (678), (9) - * - outputSize = 5; - */ - - /* - * In order to be consistent with the convLayer, here the outputSize is - * actually the size of the input image of convTransLayer, and the image size - * is actually the size of the output image of convTransLayer - */ - int imageSize(int outputSize, int filterSize, int padding, int stride) { - int imageSize; - if (!caffeMode_) { - imageSize = - (outputSize - 1) * stride + filterSize - 2 * padding - stride + 1; - } else { - imageSize = (outputSize - 1) * stride + filterSize - 2 * padding; - } - CHECK_GE(imageSize, 1); - return imageSize; - } -}; - -} // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvLayer.cpp b/paddle/gserver/layers/ExpandConvLayer.cpp index 866cd33c118d0..5c30c5a1fec7b 100644 --- a/paddle/gserver/layers/ExpandConvLayer.cpp +++ b/paddle/gserver/layers/ExpandConvLayer.cpp @@ -24,32 +24,7 @@ REGISTER_LAYER(exconv, ExpandConvLayer); bool ExpandConvLayer::init(const LayerMap &layerMap, const ParameterMap ¶meterMap) { /* Initialize the basic convolutional parent class */ - ConvBaseLayer::init(layerMap, parameterMap); - - /* Initialize the projection */ - for (auto &inputConfig : config_.inputs()) { - const ConvConfig &conf = inputConfig.conv_conf(); - subM_.push_back(numFilters_ / conf.groups()); - subN_.push_back(conf.output_x() * conf.output_x()); - subK_.push_back(conf.channels() * conf.filter_size() * conf.filter_size() / - conf.groups()); - /* Consistent caffe mode for multiple input */ - caffeMode_ = conf.caffe_mode(); - } - - /* initialize the weightList */ - CHECK(inputLayers_.size() == parameters_.size()); - for (size_t i = 0; i < inputLayers_.size(); i++) { - size_t height, width; - height = filterPixels_[i] * filterChannels_[i]; - width = numFilters_; - - // create a new weight - CHECK_EQ(parameters_[i]->getSize(), width * height); - Weight* w = new Weight(height, width, parameters_[i]); - weights_.emplace_back(w); - } - + ConvBaseLayerCpu::init(layerMap, parameterMap); return true; } @@ -63,72 +38,6 @@ size_t ExpandConvLayer::getOutputSize() { return layerSize; } - -void ExpandConvLayer::expandOneFrame(MatrixPtr image, size_t startIdx, - int inIdx) { - resetExpandInput(subK_[inIdx] * groups_[inIdx], subN_[inIdx]); - real *imgData = image->getData() + startIdx * image->getWidth(); - MatrixPtr imageTmp = Matrix::create( - imgData, 1, imgSizeH_[inIdx] * imgSizeW_[inIdx] * channels_[inIdx], false, - useGpu_); - expandInput_->convExpand(*imageTmp, imgSizeH_[inIdx], imgSizeW_[inIdx], - channels_[inIdx], filterSize_[inIdx], - filterSize_[inIdx], stride_[inIdx], stride_[inIdx], - padding_[inIdx], padding_[inIdx], - outputH_[inIdx], outputW_[inIdx]); - imageTmp->clear(); -} - -void ExpandConvLayer::expandFwdOnce(MatrixPtr image, int inIdx, int startIdx) { - int subM = subM_[inIdx]; - int subN = subN_[inIdx]; - int subK = subK_[inIdx]; - - expandOneFrame(image, startIdx, inIdx); - - real *outData = - getOutputValue()->getData() + startIdx * subN * numFilters_; - - real *wgtData = weights_[inIdx]->getW()->getData(); - real *expInData = expandInput_->getData(); - for (int g = 0; g < groups_[inIdx]; ++g) { - MatrixPtr A = - Matrix::create(wgtData, subK, subM, true, useGpu_); // mark transpose - MatrixPtr B = Matrix::create(expInData, subK, subN, false, useGpu_); - MatrixPtr C = Matrix::create(outData, subM, subN, false, useGpu_); - C->mul(A, B, 1, 1); - - A->clear(); - B->clear(); - C->clear(); - wgtData += subK * subM; - expInData += subK * subN; - outData += subM * subN; - } -} - -void ExpandConvLayer::addSharedBias() { - size_t mapW = getOutputValue()->getWidth() / numFilters_; - size_t mapH = getOutputValue()->getElementCnt() / mapW; - MatrixPtr out = - Matrix::create(getOutputValue()->getData(), mapH, mapW, false, useGpu_); - - Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); - - out->transpose(transOutValue_, false); // false means no memory allocation - transOutValue_->reshape(transOutValue_->getElementCnt() / numFilters_, - numFilters_); - - MatrixPtr bias = - Matrix::create(biases_->getW()->getData(), 1, - biases_->getW()->getElementCnt(), false, useGpu_); - transOutValue_->addBias(*bias, 1.0f); - - transOutValue_->reshape(mapW, mapH); - transOutValue_->transpose(out, false); // false means no memory allocation - - - void ExpandConvLayer::forward(PassType passType) { Layer::forward(passType); @@ -145,7 +54,7 @@ void ExpandConvLayer::forward(PassType passType) { image = prevLayer->getOutputValue(); for (size_t off = 0; off < image->getHeight(); off++) { REGISTER_TIMER_INFO("expandFwdOnce", getName().c_str()); - expandFwdOnce(image, i, off); + expandFwdOnce(image, getOutputValue(), i, off); } } /* add the bias-vector */ @@ -161,29 +70,6 @@ void ExpandConvLayer::forward(PassType passType) { forwardActivation(); } -void ExpandConvLayer::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { - size_t mapW = v->getWidth() / numFilters_; - size_t mapH = v->getElementCnt() / mapW; - MatrixPtr vTmp = Matrix::create(v->getData(), mapH, mapW, false, useGpu_); - - Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); - - vTmp->transpose(transOutValue_, false); // false means no memory allocation - vTmp->reshape(transOutValue_->getElementCnt() / numFilters_, numFilters_); - biases->collectBias(*vTmp, 1.0f); -} - -void ExpandConvLayer::bpropBiases(MatrixPtr v) { - MatrixPtr biases = - Matrix::create(biases_->getWGrad()->getData(), 1, - biases_->getWGrad()->getElementCnt(), false, useGpu_); - if (sharedBiases_) { - bpropSharedBias(biases, v); - } else { - biases->collectBias(*v, 1.0f); - } - biases->clear(); -} void ExpandConvLayer::backward(const UpdateCallback &callback) { backwardActivation(); @@ -197,109 +83,16 @@ void ExpandConvLayer::backward(const UpdateCallback &callback) { for (size_t i = 0; i != inputLayers_.size(); ++i) { /* First, calculate the input layers error */ - bpropActs(outGrad, i); + if (NULL != getPrev(i)->getOutputGrad()) { + bpropActs(outGrad, getPrev(i)->getOutputGrad(), i); + } if (weights_[i]->getWGrad()) { /* Then, calculate the W-gradient for the current layer */ - bpropWeights(outGrad, i); + bpropWeights(getPrev(i)->getOutputValue(), outGrad, i); /* Increasing the number of gradient */ weights_[i]->getParameterPtr()->incUpdate(callback); } } } -void ExpandConvLayer::bpropWeights(MatrixPtr v, int inpIdx) { - MatrixPtr weightGrad = weights_[inpIdx]->getWGrad(); - MatrixPtr inputV = getPrev(inpIdx)->getOutputValue(); - - int subM = subM_[inpIdx]; - int subN = subN_[inpIdx]; - int subK = subK_[inpIdx]; - size_t batchSize = inputV->getHeight(); - resetExpandInput(subK * groups_[inpIdx], subN); - resetConvOutput(batchSize, inpIdx); - - real *gradData = v->getData(); - - for (size_t n = 0; n < batchSize; n++) { // frame by frame - // expand - expandOneFrame(inputV, n, inpIdx); - real *wGradData = weightGrad->getData(); - real *expandInData = expandInput_->getData(); - - // expand-mul one-group by one - for (int g = 0; g < groups_[inpIdx]; g++) { - MatrixPtr A = Matrix::create(expandInData, subK, subN, false, useGpu_); - MatrixPtr B = Matrix::create(gradData, subM, subN, true, useGpu_); - MatrixPtr C = Matrix::create(wGradData, subK, subM, false, useGpu_); - C->mul(A, B, 1, 1); - - A->clear(); - B->clear(); - C->clear(); - gradData += subM * subN; - wGradData += subK * subM; - expandInData += subK * subN; - } - } -} - -void ExpandConvLayer::bpropActs(MatrixPtr v, int inpIdx) { - LayerPtr prevLayer = getPrev(inpIdx); - if (NULL == prevLayer->getOutputGrad()) { - return; - } - - int subM = subM_[inpIdx]; - int subN = subN_[inpIdx]; - int subK = subK_[inpIdx]; - size_t batchSize = v->getHeight(); - MatrixPtr tgtGrad = prevLayer->getOutputGrad(); - - /* reset the expand-grad memory */ - resetExpandInput(subK * groups_[inpIdx], subN); - resetConvOutput(batchSize, inpIdx); - - real *localGradData = v->getData(); - real *tgtGradData = tgtGrad->getData(); - for (size_t n = 0; n < batchSize; n++) { - real *wgtData = weights_[inpIdx]->getW()->getData(); - real *expandInData = expandInput_->getData(); - - for (int g = 0; g < groups_[inpIdx]; g++) { - // create temporary matrix - MatrixPtr C = Matrix::create(expandInData, subK, subN, false, useGpu_); - MatrixPtr B = Matrix::create(localGradData, subM, subN, false, useGpu_); - MatrixPtr A = Matrix::create(wgtData, subK, subM, false, useGpu_); - C->mul(A, B); // mul - - // clear the temporary matrix - A->clear(); - B->clear(); - C->clear(); - - expandInData += subK * subN; - localGradData += subM * subN; - wgtData += subK * subM; - } - - // shrink one frame outGrad - MatrixPtr oneGradTmp = Matrix::create( - expandInput_->getData(), subK * groups_[inpIdx], subN, false, useGpu_); - MatrixPtr vTmp = Matrix::create( - tgtGradData, 1, - imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channels_[inpIdx], false, - useGpu_); - vTmp->convShrink(*oneGradTmp, imgSizeH_[inpIdx], imgSizeW_[inpIdx], - channels_[inpIdx], filterSize_[inpIdx], - filterSize_[inpIdx], stride_[inpIdx], stride_[inpIdx], - padding_[inpIdx], padding_[inpIdx], - outputH_[inpIdx], outputW_[inpIdx], 1.0f, 1.0f); - vTmp->clear(); - oneGradTmp->clear(); - - // move the data-pointer - tgtGradData += imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channels_[inpIdx]; - } -} - } // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvLayer.h b/paddle/gserver/layers/ExpandConvLayer.h index f43b199498ec3..5a4abec14e7d5 100644 --- a/paddle/gserver/layers/ExpandConvLayer.h +++ b/paddle/gserver/layers/ExpandConvLayer.h @@ -15,7 +15,7 @@ limitations under the License. */ #pragma once -#include "ConvBaseLayer.h" +#include "ConvBaseLayerCpu.h" #include "paddle/math/Matrix.h" #include @@ -28,24 +28,11 @@ namespace paddle { * * The config file api is img_conv_layer. */ -class ExpandConvLayer : public ConvBaseLayer { -protected: - /// For expand convolution. - /// subM_ = numFilters_ / groups_. - IntV subM_; - /// subN_ = outputH_ * outputW_. - IntV subN_; - /// subK_ = channels_ * filterPixels_ * groups_. - IntV subK_; - /// Expand one sample at a time. shape: - /// (numChannels * filterPixels_, outputSizeH * outputSizeW) - MatrixPtr expandInput_; - /// The transpose of output, which is an auxiliary matrix. - MatrixPtr transOutValue_; - +class ExpandConvLayer : public ConvBaseLayerCpu { public: - explicit ExpandConvLayer(const LayerConfig& config) : ConvBaseLayer(config) {} + explicit ExpandConvLayer(const LayerConfig& config) : + ConvBaseLayerCpu(config) {} ~ExpandConvLayer() {} @@ -53,23 +40,8 @@ class ExpandConvLayer : public ConvBaseLayer { size_t getOutputSize(); - /** - * Expand one input sample. - */ - void expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx); - - /** - * Expand one input sample and perform matrix multiplication. - */ - void expandFwdOnce(MatrixPtr image, int inIdx, int startIdx); - - void forward(PassType passType); - void bpropSharedBias(MatrixPtr biases, MatrixPtr v); - void bpropBiases(MatrixPtr v); void backward(const UpdateCallback& callback); - void bpropWeights(MatrixPtr v, int inpIdx); - void bpropActs(MatrixPtr v, int inpIdx); }; } // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvTransLayer.cpp b/paddle/gserver/layers/ExpandConvTransLayer.cpp index fb2e7fc4bd6e2..99eb18053d32c 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.cpp +++ b/paddle/gserver/layers/ExpandConvTransLayer.cpp @@ -29,18 +29,7 @@ REGISTER_LAYER(exconvt, ExpandConvTransLayer); bool ExpandConvTransLayer::init(const LayerMap &layerMap, const ParameterMap ¶meterMap) { /* Initialize the basic convolutional parent class */ - ConvBaseLayer::init(layerMap, parameterMap); - - /* Initialize the projection */ - for (auto &inputConfig : config_.inputs()) { - const ConvConfig &conf = inputConfig.conv_conf(); - subM_.push_back(conf.channels() / conf.groups()); - subN_.push_back(conf.output_x() * conf.output_x()); - subK_.push_back(numFilters_ * conf.filter_size() * conf.filter_size() / - conf.groups()); - /* Consistent caffe mode for multiple input */ - caffeMode_ = conf.caffe_mode(); - } + ConvBaseLayerCpu::init(layerMap, parameterMap); return true; } @@ -73,67 +62,6 @@ size_t ExpandConvTransLayer::getSize() { return layerSize; } -void ExpandConvTransLayer::resetExpandInput(size_t height, size_t width) { - Matrix::resizeOrCreate(expandInput_, height, width, false, useGpu_); -} - -/*void ExpandConvTransLayer::resetConvOutput(size_t batchSize, int inIdx) { - Matrix::resizeOrCreate(transOutValue_, batchSize * numFilters_, subN_[inIdx], - false, useGpu_); -}*/ - - - -void ExpandConvTransLayer::expandOneFrame(MatrixPtr image, size_t startIdx, - int inIdx) { - resetExpandInput(subK_[inIdx] * groups_[inIdx], subN_[inIdx]); - real *imgData = image->getData() + startIdx * image->getWidth(); - MatrixPtr imageTmp = Matrix::create( - imgData, 1, imgSizeH_[inIdx] * imgSizeW_[inIdx] * channel_, false, - useGpu_); - expandInput_->convExpand(*imageTmp, imgSizeH_[inIdx], imgSizeW_[inIdx], - channel_, filterSize_[inIdx], - filterSize_[inIdx], stride_[inIdx], stride_[inIdx], - padding_[inIdx], padding_[inIdx], - outputH_[inIdx], outputW_[inIdx]); - imageTmp->clear(); -} - -void ExpandConvTransLayer::expandBackOnce(MatrixPtr imageGrad, int inIdx, - int startIdx) { - int subM = subM_[inIdx]; - int subN = subN_[inIdx]; - int subK = subK_[inIdx]; - - LayerPtr prevLayer = getPrev(inIdx); - if (NULL == prevLayer->getOutputGrad()) { - return; - } - - expandOneFrame(imageGrad, startIdx, inIdx); - - real *outGradData = - prevLayer -> getOutputGrad()->getData() - + startIdx * subN * numFilters_[inIdx]; - - real *wgtData = weights_[inIdx]->getW()->getData(); - real *expInData = expandInput_->getData(); - for (int g = 0; g < groups_[inIdx]; ++g) { - MatrixPtr A = - Matrix::create(wgtData, subK, subM, true, useGpu_); // mark transpose - MatrixPtr B = Matrix::create(expInData, subK, subN, false, useGpu_); - MatrixPtr C = Matrix::create(outGradData, subM, subN, false, useGpu_); - C->mul(A, B, 1, 1); - - A->clear(); - B->clear(); - C->clear(); - wgtData += subK * subM; - expInData += subK * subN; - outGradData += subM * subN; - } -} - void ExpandConvTransLayer::forward(PassType passType) { Layer::forward(passType); @@ -148,7 +76,7 @@ void ExpandConvTransLayer::forward(PassType passType) { LayerPtr prevLayer = getPrev(i); output = prevLayer->getOutputValue(); REGISTER_TIMER_INFO("shrinkFwd", getName().c_str()); - shrinkFwd(output, i); + bpropActs(output, getOutputValue(), i); } /* add the bias-vector */ @@ -164,84 +92,6 @@ void ExpandConvTransLayer::forward(PassType passType) { forwardActivation(); } -void ExpandConvTransLayer::shrinkFwd(MatrixPtr output, int inpIdx) { - int subM = subM_[inpIdx]; - int subN = subN_[inpIdx]; - int subK = subK_[inpIdx]; - - size_t batchSize = output->getHeight(); - MatrixPtr image = getOutputValue(); - - /* reset the expand-grad memory */ - resetExpandInput(subK * groups_[inpIdx], subN); - - real *localData = output->getData(); - real *imageData = image->getData(); - for (size_t n = 0; n < batchSize; n++) { - real *wgtData = weights_[inpIdx]->getW()->getData(); - real *expandInData = expandInput_->getData(); - - for (int g = 0; g < groups_[inpIdx]; g++) { - // create temporary matrix - MatrixPtr C = Matrix::create(expandInData, subK, subN, false, useGpu_); - MatrixPtr B = Matrix::create(localData, subM, subN, false, useGpu_); - MatrixPtr A = Matrix::create(wgtData, subK, subM, false, useGpu_); - C->mul(A, B); // mul - - // clear the temporary matrix - A->clear(); - B->clear(); - C->clear(); - - expandInData += subK * subN; - localData += subM * subN; - wgtData += subK * subM; - } - - // shrink one frame outGrad - MatrixPtr oneTmp = Matrix::create( - expandInput_->getData(), subK * groups_[inpIdx], subN, false, useGpu_); - MatrixPtr vTmp = Matrix::create( - imageData, 1, - imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channel_, false, - useGpu_); - vTmp->convShrink(*oneTmp, imgSizeH_[inpIdx], imgSizeW_[inpIdx], - channel_, filterSize_[inpIdx], - filterSize_[inpIdx], stride_[inpIdx], stride_[inpIdx], - padding_[inpIdx], padding_[inpIdx], - outputH_[inpIdx], outputW_[inpIdx], 1.0f, 1.0f); - vTmp->clear(); - oneTmp->clear(); - - // move the data-pointer - imageData += imgSizeH_[inpIdx] * imgSizeW_[inpIdx] * channel_; - } -} - -void ExpandConvTransLayer::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { - size_t mapW = getSize() / channel_; - size_t mapH = v->getElementCnt() / mapW; - MatrixPtr vTmp = Matrix::create(v->getData(), mapH, mapW, false, useGpu_); - - Matrix::resizeOrCreate(transOutValue_, mapW, mapH, false, useGpu_); - - vTmp->transpose(transOutValue_, false); // false means no memory allocation - vTmp->reshape(transOutValue_->getElementCnt() / channel_, channel_); - biases->collectBias(*vTmp, 1.0f); -} - -void ExpandConvTransLayer::bpropBiases(MatrixPtr v) { - MatrixPtr biases = - Matrix::create(biases_->getWGrad()->getData(), 1, - biases_->getWGrad()->getElementCnt(), false, useGpu_); - if (sharedBiases_) { - bpropSharedBias(biases, v); - } else { - biases->collectBias(*v, 1.0f); - } - biases->clear(); -} - void ExpandConvTransLayer::backward(const UpdateCallback &callback) { backwardActivation(); @@ -255,51 +105,18 @@ void ExpandConvTransLayer::backward(const UpdateCallback &callback) { for (size_t i = 0; i != inputLayers_.size(); ++i) { /* First, calculate the input layers error */ for (size_t off = 0; off < imageGrad->getHeight(); off++) { - expandBackOnce(imageGrad, i, off); + if (NULL != getPrev(i)->getOutputGrad()) { + expandFwdOnce(imageGrad, getPrev(i)->getOutputGrad(), i, off); + } } if (weights_[i]->getWGrad()) { /* Then, calculate the W-gradient for the current layer */ - bpropWeights(imageGrad, i); + bpropWeights(imageGrad, getPrev(i)->getOutputValue(), i); /* Increasing the number of gradient */ weights_[i]->getParameterPtr()->incUpdate(callback); } } } -void ExpandConvTransLayer::bpropWeights(MatrixPtr v, int inpIdx) { - MatrixPtr weightGrad = weights_[inpIdx]->getWGrad(); - MatrixPtr outputV = getPrev(inpIdx)->getOutputValue(); - - int subM = subM_[inpIdx]; - int subN = subN_[inpIdx]; - int subK = subK_[inpIdx]; - size_t batchSize = outputV->getHeight(); - resetExpandInput(subK * groups_[inpIdx], subN); - - real *outputData = outputV -> getData(); - - for (size_t n = 0; n < batchSize; n++) { // frame by frame - // expand - expandOneFrame(v, n, inpIdx); - real *wGradData = weightGrad->getData(); - real *expandInData = expandInput_->getData(); - - // expand-mul one-group by one - for (int g = 0; g < groups_[inpIdx]; g++) { - MatrixPtr A = Matrix::create(expandInData, subK, subN, false, useGpu_); - MatrixPtr B = Matrix::create(outputData, subM, subN, true, useGpu_); - MatrixPtr C = Matrix::create(wGradData, subK, subM, false, useGpu_); - C->mul(A, B, 1, 1); - - A->clear(); - B->clear(); - C->clear(); - outputData += subM * subN; - wGradData += subK * subM; - expandInData += subK * subN; - } - } -} - } // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvTransLayer.h b/paddle/gserver/layers/ExpandConvTransLayer.h index cbe4da8143cef..214f460d65877 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.h +++ b/paddle/gserver/layers/ExpandConvTransLayer.h @@ -15,7 +15,7 @@ limitations under the License. */ #pragma once -#include "ConvBaseLayer.h" +#include "ConvBaseLayerCpu.h" #include "paddle/math/Matrix.h" #include @@ -24,32 +24,14 @@ namespace paddle { /** * @brief A subclass of convolution layer. * This layer expands input and use matrix multiplication to - * calculate convolution operation. + * calculate convolution transpose (deconv) operation. * * The config file api is img_convTrans_layer. */ -class ExpandConvTransLayer : public ConvBaseLayer { -protected: - /// For expand convolution. - /// subM_ = numFilters_ / groups_. - IntV subM_; - /// subN_ = outputH_ * outputW_. - IntV subN_; - /// subK_ = channels_ * filterPixels_ * groups_. - IntV subK_; - /// The spatial dimensions of height of input feature map. - IntV imgSizeH_; - /// The spatial dimensions of width of input feature map. - IntV imgSizeW_; - /// The spatial dimensions of height of output feature map. - IntV outputH_; - /// The spatial dimensions of width of output feature map. - IntV outputW_; - - +class ExpandConvTransLayer : public ConvBaseLayerCpu { public: explicit ExpandConvTransLayer(const LayerConfig& config) : - ConvBaseLayer(config) {} + ConvBaseLayerCpu(config) {} ~ExpandConvTransLayer() {} @@ -57,38 +39,8 @@ class ExpandConvTransLayer : public ConvBaseLayer { size_t getSize(); - /** - * Create or resize expandInput_. - */ - void resetExpandInput(size_t height, size_t width); - - /** - * Create or resize transOutValue_. - */ - void resetConvOutput(size_t batchSize, int inIdx); - - /** - * Expand one input sample. - */ - void expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx); - - /** - * Expand one output image and perform matrix multiplication. - */ - void expandBackOnce(MatrixPtr image, int inIdx, int startIdx); - - /** - * Perform matrix multiplication on one output and then shrink. - */ - void shrinkFwd(MatrixPtr output, int inpIdx); - - void forward(PassType passType); - void bpropSharedBias(MatrixPtr biases, MatrixPtr v); - void bpropBiases(MatrixPtr v); void backward(const UpdateCallback& callback); - void bpropWeights(MatrixPtr v, int inpIdx); - void bpropActs(MatrixPtr v, int inpIdx); }; } // namespace paddle diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index d634d198c3be7..1c27ee2d5f7a3 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -302,6 +302,8 @@ void testConvLayer(const string& type, bool trans, bool useGpu) { config.layerConfig.num_filters()); testLayerGrad(config, "conv", 100, trans, useGpu); + // Use small batch_size and useWeight=true to test biasGrad + testLayerGrad(config, "conv", 2, trans, useGpu, true, 0.02); } TEST(Layer, convLayer) { From e68b50ad0902531d0fc3fa3e9dddb25ad95fd5c3 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Tue, 1 Nov 2016 16:36:57 -0700 Subject: [PATCH 268/324] fixed a bug in refactoring ExpandConv/TransLayer --- paddle/gserver/layers/ConvBaseLayerCpu.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/paddle/gserver/layers/ConvBaseLayerCpu.cpp b/paddle/gserver/layers/ConvBaseLayerCpu.cpp index 0da92bf0485b0..13521bcd2b209 100644 --- a/paddle/gserver/layers/ConvBaseLayerCpu.cpp +++ b/paddle/gserver/layers/ConvBaseLayerCpu.cpp @@ -22,11 +22,20 @@ bool ConvBaseLayerCpu::init(const LayerMap &layerMap, /* Initialize the basic convolutional parent class */ ConvBaseLayer::init(layerMap, parameterMap); + /* The class fields channels_ and numFilters_ are the same as in the config + * i.e., channels_ is the for the input and numFilters_ is for the output + * + * But in order for the variables in convTrans having the same semantic + * meaning as in conv, we need to swap channels_ and numFilters here for + * convTrans, and in other functions too. + * */ int channel; + int nf; /* Initialize the projection */ for (auto &inputConfig : config_.inputs()) { const ConvConfig &conf = inputConfig.conv_conf(); - subM_.push_back(numFilters_ / conf.groups()); + nf = isConv_ ? numFilters_ : conf.channels(); + subM_.push_back(nf / conf.groups()); subN_.push_back(conf.output_x() * conf.output_x()); channel = isConv_ ? conf.channels() : numFilters_; subK_.push_back(channel * conf.filter_size() * conf.filter_size() / @@ -123,20 +132,19 @@ void ConvBaseLayerCpu::expandFwdOnce(MatrixPtr image, MatrixPtr out, } } -void ConvBaseLayerCpu::bpropActs(MatrixPtr image, MatrixPtr out, int inpIdx) { +void ConvBaseLayerCpu::bpropActs(MatrixPtr out, MatrixPtr image, int inpIdx) { int channel = isConv_ ? channels_[inpIdx] : numFilters_; int subM = subM_[inpIdx]; int subN = subN_[inpIdx]; int subK = subK_[inpIdx]; size_t batchSize = image->getHeight(); - MatrixPtr tgtGrad = out; /* reset the expand-grad memory */ resetExpandInput(subK * groups_[inpIdx], subN); - real *localGradData = image->getData(); - real *tgtGradData = tgtGrad->getData(); + real *localGradData = out->getData(); + real *tgtGradData = image->getData(); for (size_t n = 0; n < batchSize; n++) { real *wgtData = weights_[inpIdx]->getW()->getData(); real *expandInData = expandInput_->getData(); From 5fff96f5322d8275a1683cdf1efdd29b77c7ee20 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Tue, 1 Nov 2016 17:01:58 -0700 Subject: [PATCH 269/324] add another small test in test_LayerGrad for convTransLayer --- paddle/gserver/tests/test_LayerGrad.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 1c27ee2d5f7a3..42c7b13906254 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -345,6 +345,8 @@ void testConvTransLayer(const string& type, bool trans, bool useGpu) { config.layerConfig.num_filters()); testLayerGrad(config, "convTrans", 100, trans, useGpu); + // Use small batch_size and useWeight=true to test biasGrad + testLayerGrad(config, "convTrans", 2, trans, useGpu, true, 0.02); } TEST(Layer, convTransLayer) { From 5e4cc241ac2cace3f6564070014e4ac59471ab65 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Wed, 2 Nov 2016 11:02:30 -0700 Subject: [PATCH 270/324] Revised deconv implementations according to luotao1 --- paddle/gserver/layers/ConvBaseLayer.cpp | 4 +- paddle/gserver/layers/ConvBaseLayer.h | 4 +- ...seLayerCpu.cpp => ExpandConvBaseLayer.cpp} | 34 ++--- ...nvBaseLayerCpu.h => ExpandConvBaseLayer.h} | 13 +- paddle/gserver/layers/ExpandConvLayer.cpp | 13 +- paddle/gserver/layers/ExpandConvLayer.h | 6 +- .../gserver/layers/ExpandConvTransLayer.cpp | 10 +- paddle/gserver/layers/ExpandConvTransLayer.h | 6 +- python/paddle/trainer/config_parser.py | 4 +- .../paddle/trainer_config_helpers/layers.py | 132 ++---------------- 10 files changed, 62 insertions(+), 164 deletions(-) rename paddle/gserver/layers/{ConvBaseLayerCpu.cpp => ExpandConvBaseLayer.cpp} (88%) rename paddle/gserver/layers/{ConvBaseLayerCpu.h => ExpandConvBaseLayer.h} (90%) diff --git a/paddle/gserver/layers/ConvBaseLayer.cpp b/paddle/gserver/layers/ConvBaseLayer.cpp index 4346cb520ea01..607fb99cf6288 100644 --- a/paddle/gserver/layers/ConvBaseLayer.cpp +++ b/paddle/gserver/layers/ConvBaseLayer.cpp @@ -22,9 +22,9 @@ bool ConvBaseLayer::init(const LayerMap& layerMap, Layer::init(layerMap, parameterMap); if (config_.type() == "exconv" || config_.type() == "cudnn_conv") { - isConv_ = true; + isDeconv_ = false; } else { - isConv_ = false; + isDeconv_ = true; } /* Initialize the convolutional layer parameter */ diff --git a/paddle/gserver/layers/ConvBaseLayer.h b/paddle/gserver/layers/ConvBaseLayer.h index ecdc119a94941..2f2ce59ad9e6c 100644 --- a/paddle/gserver/layers/ConvBaseLayer.h +++ b/paddle/gserver/layers/ConvBaseLayer.h @@ -28,8 +28,8 @@ class ConvBaseLayer : public Layer { protected: typedef std::vector IntV; - /// True if it's convolution layer, false if it's deconv layer - bool isConv_; + /// True if it's deconv layer, false if it's convolution layer + bool isDeconv_; /// The number of filters. int numFilters_; diff --git a/paddle/gserver/layers/ConvBaseLayerCpu.cpp b/paddle/gserver/layers/ExpandConvBaseLayer.cpp similarity index 88% rename from paddle/gserver/layers/ConvBaseLayerCpu.cpp rename to paddle/gserver/layers/ExpandConvBaseLayer.cpp index 13521bcd2b209..9693ad82450f4 100644 --- a/paddle/gserver/layers/ConvBaseLayerCpu.cpp +++ b/paddle/gserver/layers/ExpandConvBaseLayer.cpp @@ -13,11 +13,12 @@ See the License for the specific language governing permissions and limitations under the License. */ +#include "ExpandConvBaseLayer.h" + #include "paddle/utils/Logging.h" -#include "ConvBaseLayerCpu.h" namespace paddle { -bool ConvBaseLayerCpu::init(const LayerMap &layerMap, +bool ExpandConvBaseLayer::init(const LayerMap &layerMap, const ParameterMap ¶meterMap) { /* Initialize the basic convolutional parent class */ ConvBaseLayer::init(layerMap, parameterMap); @@ -34,10 +35,10 @@ bool ConvBaseLayerCpu::init(const LayerMap &layerMap, /* Initialize the projection */ for (auto &inputConfig : config_.inputs()) { const ConvConfig &conf = inputConfig.conv_conf(); - nf = isConv_ ? numFilters_ : conf.channels(); + nf = (!isDeconv_) ? numFilters_ : conf.channels(); subM_.push_back(nf / conf.groups()); subN_.push_back(conf.output_x() * conf.output_x()); - channel = isConv_ ? conf.channels() : numFilters_; + channel = (!isDeconv_) ? conf.channels() : numFilters_; subK_.push_back(channel * conf.filter_size() * conf.filter_size() / conf.groups()); /* Consistent caffe mode for multiple input */ @@ -47,11 +48,11 @@ bool ConvBaseLayerCpu::init(const LayerMap &layerMap, return true; } -void ConvBaseLayerCpu::resetExpandInput(size_t height, size_t width) { +void ExpandConvBaseLayer::resetExpandInput(size_t height, size_t width) { Matrix::resizeOrCreate(expandInput_, height, width, false, useGpu_); } -void ConvBaseLayerCpu::addSharedBias() { +void ExpandConvBaseLayer::addSharedBias() { size_t mapW = getSize() / numFilters_; size_t mapH = getOutputValue()->getElementCnt() / mapW; MatrixPtr out = @@ -75,7 +76,7 @@ void ConvBaseLayerCpu::addSharedBias() { bias->clear(); } -void ConvBaseLayerCpu::addUnsharedBias() { +void ExpandConvBaseLayer::addUnsharedBias() { MatrixPtr outValue = getOutputValue(); MatrixPtr bias = Matrix::create(biases_->getW()->getData(), 1, @@ -84,9 +85,9 @@ void ConvBaseLayerCpu::addUnsharedBias() { } -void ConvBaseLayerCpu::expandOneFrame(MatrixPtr image, size_t startIdx, +void ExpandConvBaseLayer::expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx) { - int channel = isConv_ ? channels_[inIdx] : numFilters_; + int channel = (!isDeconv_) ? channels_[inIdx] : numFilters_; resetExpandInput(subK_[inIdx] * groups_[inIdx], subN_[inIdx]); real *imgData = image->getData() + startIdx * image->getWidth(); @@ -101,7 +102,7 @@ void ConvBaseLayerCpu::expandOneFrame(MatrixPtr image, size_t startIdx, imageTmp->clear(); } -void ConvBaseLayerCpu::expandFwdOnce(MatrixPtr image, MatrixPtr out, +void ExpandConvBaseLayer::expandFwdOnce(MatrixPtr image, MatrixPtr out, int inIdx, int startIdx) { int subM = subM_[inIdx]; int subN = subN_[inIdx]; @@ -109,7 +110,7 @@ void ConvBaseLayerCpu::expandFwdOnce(MatrixPtr image, MatrixPtr out, expandOneFrame(image, startIdx, inIdx); - int nf = isConv_ ? numFilters_ : channels_[inIdx]; + int nf = (!isDeconv_) ? numFilters_ : channels_[inIdx]; real *outData = out->getData() + startIdx * subN * nf; @@ -132,8 +133,9 @@ void ConvBaseLayerCpu::expandFwdOnce(MatrixPtr image, MatrixPtr out, } } -void ConvBaseLayerCpu::bpropActs(MatrixPtr out, MatrixPtr image, int inpIdx) { - int channel = isConv_ ? channels_[inpIdx] : numFilters_; +void ExpandConvBaseLayer::bpropActs(MatrixPtr out, MatrixPtr image, + int inpIdx) { + int channel = (!isDeconv_) ? channels_[inpIdx] : numFilters_; int subM = subM_[inpIdx]; int subN = subN_[inpIdx]; @@ -186,7 +188,7 @@ void ConvBaseLayerCpu::bpropActs(MatrixPtr out, MatrixPtr image, int inpIdx) { } } -void ConvBaseLayerCpu::bpropWeights(MatrixPtr image, MatrixPtr out, +void ExpandConvBaseLayer::bpropWeights(MatrixPtr image, MatrixPtr out, int inpIdx) { MatrixPtr weightGrad = weights_[inpIdx]->getWGrad(); @@ -221,7 +223,7 @@ void ConvBaseLayerCpu::bpropWeights(MatrixPtr image, MatrixPtr out, } } -void ConvBaseLayerCpu::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { +void ExpandConvBaseLayer::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { size_t mapW = getSize() / numFilters_; size_t mapH = v->getElementCnt() / mapW; MatrixPtr vTmp = Matrix::create(v->getData(), mapH, mapW, false, useGpu_); @@ -234,7 +236,7 @@ void ConvBaseLayerCpu::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { biases->collectBias(*transOutValue_, 1.0f); } -void ConvBaseLayerCpu::bpropBiases(MatrixPtr v) { +void ExpandConvBaseLayer::bpropBiases(MatrixPtr v) { MatrixPtr biases = Matrix::create(biases_->getWGrad()->getData(), 1, biases_->getWGrad()->getElementCnt(), false, useGpu_); diff --git a/paddle/gserver/layers/ConvBaseLayerCpu.h b/paddle/gserver/layers/ExpandConvBaseLayer.h similarity index 90% rename from paddle/gserver/layers/ConvBaseLayerCpu.h rename to paddle/gserver/layers/ExpandConvBaseLayer.h index 08a1426b473a0..418c9dd6ce2ad 100644 --- a/paddle/gserver/layers/ConvBaseLayerCpu.h +++ b/paddle/gserver/layers/ExpandConvBaseLayer.h @@ -25,7 +25,7 @@ namespace paddle { * @brief A subclass of ConvBaseLayer that is a superclass of both * ExpandConvLayer and ExpandConvTransLayer */ -class ConvBaseLayerCpu : public ConvBaseLayer { +class ExpandConvBaseLayer : public ConvBaseLayer { protected: /// For expand convolution. /// subM_ = numFilters_ / groups_. @@ -43,18 +43,19 @@ class ConvBaseLayerCpu : public ConvBaseLayer { /// The spatial dimensions of width of output feature map. IntV outputW_; - /*The expandInput_ and transOutValue_ are used for CPU expand conv calc*/ - /// Expand one sample at a time. shape: - /// (numChannels * filterPixels_, outputSizeH * outputSizeW) + /*The expandInput_ and transOutValue_ are used for CPU expand conv calc + * Expand one sample at a time. shape: + * (numChannels * filterPixels_, outputSizeH * outputSizeW) + * */ MatrixPtr expandInput_; /// The transpose of output, which is an auxiliary matrix. MatrixPtr transOutValue_; public: - explicit ConvBaseLayerCpu(const LayerConfig& config) + explicit ExpandConvBaseLayer(const LayerConfig& config) : ConvBaseLayer(config) {} - ~ConvBaseLayerCpu() {} + ~ExpandConvBaseLayer() {} bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); diff --git a/paddle/gserver/layers/ExpandConvLayer.cpp b/paddle/gserver/layers/ExpandConvLayer.cpp index 5c30c5a1fec7b..379823a6feb45 100644 --- a/paddle/gserver/layers/ExpandConvLayer.cpp +++ b/paddle/gserver/layers/ExpandConvLayer.cpp @@ -24,7 +24,7 @@ REGISTER_LAYER(exconv, ExpandConvLayer); bool ExpandConvLayer::init(const LayerMap &layerMap, const ParameterMap ¶meterMap) { /* Initialize the basic convolutional parent class */ - ConvBaseLayerCpu::init(layerMap, parameterMap); + ExpandConvBaseLayer::init(layerMap, parameterMap); return true; } @@ -49,16 +49,17 @@ void ExpandConvLayer::forward(PassType passType) { resetOutput(batchSize, getOutputSize()); MatrixPtr image = nullptr; - for (size_t i = 0; i != inputLayers_.size(); ++i) { + MatrixPtr outV = getOutputValue(); + for (size_t i = 0; i < inputLayers_.size(); ++i) { LayerPtr prevLayer = getPrev(i); image = prevLayer->getOutputValue(); for (size_t off = 0; off < image->getHeight(); off++) { REGISTER_TIMER_INFO("expandFwdOnce", getName().c_str()); - expandFwdOnce(image, getOutputValue(), i, off); + expandFwdOnce(image, outV, i, off); } } /* add the bias-vector */ - if (biases_.get() != NULL) { + if (biases_.get()) { if (sharedBiases_) { addSharedBias(); } else { @@ -81,9 +82,9 @@ void ExpandConvLayer::backward(const UpdateCallback &callback) { biases_->getParameterPtr()->incUpdate(callback); } - for (size_t i = 0; i != inputLayers_.size(); ++i) { + for (size_t i = 0; i < inputLayers_.size(); ++i) { /* First, calculate the input layers error */ - if (NULL != getPrev(i)->getOutputGrad()) { + if (getPrev(i)->getOutputGrad()) { bpropActs(outGrad, getPrev(i)->getOutputGrad(), i); } if (weights_[i]->getWGrad()) { diff --git a/paddle/gserver/layers/ExpandConvLayer.h b/paddle/gserver/layers/ExpandConvLayer.h index 5a4abec14e7d5..b5cb448bdfcde 100644 --- a/paddle/gserver/layers/ExpandConvLayer.h +++ b/paddle/gserver/layers/ExpandConvLayer.h @@ -15,9 +15,9 @@ limitations under the License. */ #pragma once -#include "ConvBaseLayerCpu.h" #include "paddle/math/Matrix.h" #include +#include "ExpandConvBaseLayer.h" namespace paddle { @@ -29,10 +29,10 @@ namespace paddle { * The config file api is img_conv_layer. */ -class ExpandConvLayer : public ConvBaseLayerCpu { +class ExpandConvLayer : public ExpandConvBaseLayer { public: explicit ExpandConvLayer(const LayerConfig& config) : - ConvBaseLayerCpu(config) {} + ExpandConvBaseLayer(config) {} ~ExpandConvLayer() {} diff --git a/paddle/gserver/layers/ExpandConvTransLayer.cpp b/paddle/gserver/layers/ExpandConvTransLayer.cpp index 99eb18053d32c..1d630a4ecd031 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.cpp +++ b/paddle/gserver/layers/ExpandConvTransLayer.cpp @@ -29,7 +29,7 @@ REGISTER_LAYER(exconvt, ExpandConvTransLayer); bool ExpandConvTransLayer::init(const LayerMap &layerMap, const ParameterMap ¶meterMap) { /* Initialize the basic convolutional parent class */ - ConvBaseLayerCpu::init(layerMap, parameterMap); + ExpandConvBaseLayer::init(layerMap, parameterMap); return true; } @@ -72,7 +72,7 @@ void ExpandConvTransLayer::forward(PassType passType) { resetOutput(batchSize, getSize()); MatrixPtr output = nullptr; - for (size_t i = 0; i != inputLayers_.size(); ++i) { + for (size_t i = 0; i < inputLayers_.size(); ++i) { LayerPtr prevLayer = getPrev(i); output = prevLayer->getOutputValue(); REGISTER_TIMER_INFO("shrinkFwd", getName().c_str()); @@ -80,7 +80,7 @@ void ExpandConvTransLayer::forward(PassType passType) { } /* add the bias-vector */ - if (biases_.get() != NULL) { + if (biases_.get()) { if (sharedBiases_) { addSharedBias(); } else { @@ -102,10 +102,10 @@ void ExpandConvTransLayer::backward(const UpdateCallback &callback) { biases_->getParameterPtr()->incUpdate(callback); } - for (size_t i = 0; i != inputLayers_.size(); ++i) { + for (size_t i = 0; i < inputLayers_.size(); ++i) { /* First, calculate the input layers error */ for (size_t off = 0; off < imageGrad->getHeight(); off++) { - if (NULL != getPrev(i)->getOutputGrad()) { + if (getPrev(i)->getOutputGrad()) { expandFwdOnce(imageGrad, getPrev(i)->getOutputGrad(), i, off); } } diff --git a/paddle/gserver/layers/ExpandConvTransLayer.h b/paddle/gserver/layers/ExpandConvTransLayer.h index 214f460d65877..ebcb34f073494 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.h +++ b/paddle/gserver/layers/ExpandConvTransLayer.h @@ -15,9 +15,9 @@ limitations under the License. */ #pragma once -#include "ConvBaseLayerCpu.h" #include "paddle/math/Matrix.h" #include +#include "ExpandConvBaseLayer.h" namespace paddle { @@ -28,10 +28,10 @@ namespace paddle { * * The config file api is img_convTrans_layer. */ -class ExpandConvTransLayer : public ConvBaseLayerCpu { +class ExpandConvTransLayer : public ExpandConvBaseLayer { public: explicit ExpandConvTransLayer(const LayerConfig& config) : - ConvBaseLayerCpu(config) {} + ExpandConvBaseLayer(config) {} ~ExpandConvTransLayer() {} diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 95c5f774c6cea..b17ec6c6f6bf7 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1107,7 +1107,7 @@ def parse_conv(conv, input_layer_name, conv_conf): conv_conf.caffe_mode) -def parse_convt(conv, input_layer_name, conv_conf, num_filters): +def parse_conv_trans(conv, input_layer_name, conv_conf, num_filters): conv_conf.filter_size = conv.filter_size conv_conf.filter_size_y = conv.filter_size_y conv_conf.channels = conv.channels @@ -1683,7 +1683,7 @@ def __init__( for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_convt( + parse_conv_trans( self.inputs[input_index].conv, input_layer.name, self.config.inputs[input_index].conv_conf, num_filters) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 172d45a761ecc..711c9ca993a23 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1515,7 +1515,8 @@ def img_conv_layer(input, filter_size, num_filters, name=None, num_channels=None, act=None, groups=1, stride=1, padding=0, bias_attr=None, param_attr=None, shared_biases=True, layer_attr=None, - filter_size_y=None, stride_y=None, padding_y=None): + filter_size_y=None, stride_y=None, padding_y=None, + trans=False): """ Convolution layer for image. Paddle only support square input currently and thus input image's width equals height. @@ -1523,120 +1524,7 @@ def img_conv_layer(input, filter_size, num_filters, The details of convolution layer, please refer UFLDL's `convolution `_ . - - The num_channel means input image's channel number. It may be 1 or 3 when - input is raw pixels of image(mono or RGB), or it may be the previous layer's - num_filters * num_group. - - There are several group of filter in PaddlePaddle implementation. - Each group will process some channel of the inputs. For example, if an input - num_channel = 256, group = 4, num_filter=32, the PaddlePaddle will create - 32*4 = 128 filters to process inputs. The channels will be split into 4 - pieces. First 256/4 = 64 channels will process by first 32 filters. The - rest channels will be processed by rest group of filters. - - :param name: Layer name. - :type name: basestring - :param input: Layer Input. - :type input: LayerOutput - :param filter_size: The x dimension of a filter kernel. Or input a tuple for - two image dimension. - :type filter_size: int|tuple|list - :param filter_size_y: The y dimension of a filter kernel. Since PaddlePaddle - currently supports rectangular filters, the filter's - shape will be (filter_size, filter_size_y). - :type filter_size_y: int|None - :param num_filters: Each filter group's number of filter - :param act: Activation type. Default is tanh - :type act: BaseActivation - :param groups: Group size of filters. - :type groups: int - :param stride: The x dimension of the stride. Or input a tuple for two image - dimension. - :type stride: int|tuple|list - :param stride_y: The y dimension of the stride. - :type stride_y: int - :param padding: The x dimension of the padding. Or input a tuple for two - image dimension - :type padding: int|tuple|list - :param padding_y: The y dimension of the padding. - :type padding_y: int - :param bias_attr: Convolution bias attribute. None means default bias. - False means no bias. - :type bias_attr: ParameterAttribute|False - :param num_channels: number of input channels. If None will be set - automatically from previous output. - :type num_channels: int - :param param_attr: Convolution param attribute. None means default attribute - :type param_attr: ParameterAttribute - :param shared_biases: Is biases will be shared between filters or not. - :type shared_biases: bool - :param layer_attr: Layer Extra Attribute. - :type layer_attr: ExtraLayerAttribute - :return: LayerOutput object. - :rtype: LayerOutput - """ - if num_channels is None: - assert input.num_filters is not None - num_channels = input.num_filters - - if filter_size_y is None: - if isinstance(filter_size, collections.Sequence): - assert len(filter_size) == 2 - filter_size, filter_size_y = filter_size - else: - filter_size_y = filter_size - - if stride_y is None: - if isinstance(stride, collections.Sequence): - assert len(stride) == 2 - stride, stride_y = stride - else: - stride_y = stride - - if padding_y is None: - if isinstance(padding, collections.Sequence): - assert len(padding) == 2 - padding, padding_y = padding - else: - padding_y = padding - - if param_attr.attr.get('initial_smart'): - # special initial for conv layers. - init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 - param_attr.attr["initial_mean"] = 0.0 - param_attr.attr["initial_std"] = init_w - param_attr.attr["initial_strategy"] = 0 - param_attr.attr["initial_smart"] = False - Layer( - name=name, - inputs=Input(input.name, conv=Conv( - filter_size=filter_size, padding=padding, stride=stride, - channels=num_channels, groups=groups, - filter_size_y=filter_size_y, padding_y=padding_y, - stride_y=stride_y), - **param_attr.attr), - active_type=act.name, - num_filters=num_filters, - bias=ParamAttr.to_bias(bias_attr), - shared_biases=shared_biases, - type=LayerType.CONV_LAYER, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.CONV_LAYER, parents=[input], - activation=act, num_filters=num_filters) - -@wrap_name_default("convt") -@wrap_param_attr_default() -@wrap_bias_attr_default() -@wrap_act_default(act=ReluActivation()) -@layer_support(DROPOUT) -def img_convTrans_layer(input, filter_size, num_filters, - name=None, num_channels=None, - act=None, groups=1, stride=1, padding=0, bias_attr=None, - param_attr=None, shared_biases=True, layer_attr=None, - filter_size_y=None, stride_y=None, padding_y=None): - """ + Convolution Transpose (deconv) layer for image. Paddle only support square input currently and thus input image's width equals height. @@ -1644,7 +1532,6 @@ def img_convTrans_layer(input, filter_size, num_filters, please refer to the following explanation and references therein `_ . - The num_channel means input image's channel number. It may be 1 or 3 when input is raw pixels of image(mono or RGB), or it may be the previous layer's num_filters * num_group. @@ -1694,6 +1581,8 @@ def img_convTrans_layer(input, filter_size, num_filters, :type shared_biases: bool :param layer_attr: Layer Extra Attribute. :type layer_attr: ExtraLayerAttribute + :param trans: true if it is a convTransLayer, false if it is a convLayer + :type trans: bool :return: LayerOutput object. :rtype: LayerOutput """ @@ -1729,6 +1618,12 @@ def img_convTrans_layer(input, filter_size, num_filters, param_attr.attr["initial_std"] = init_w param_attr.attr["initial_strategy"] = 0 param_attr.attr["initial_smart"] = False + + if trans: + lt = LayerType.CONVTRANS_LAYER + else: + lt = LayerType.CONV_LAYER + Layer( name=name, inputs=Input(input.name, conv=Conv( @@ -1741,14 +1636,13 @@ def img_convTrans_layer(input, filter_size, num_filters, num_filters=num_filters, bias=ParamAttr.to_bias(bias_attr), shared_biases=shared_biases, - type=LayerType.CONVTRANS_LAYER, + type=lt, **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.CONVTRANS_LAYER, parents=[input], + return LayerOutput(name, lt, parents=[input], activation=act, num_filters=num_filters) - @wrap_name_default("pool") @layer_support() def img_pool_layer(input, pool_size, name=None, From 3d72e94939c2d5afbbca8fa89c310d239d6c0a6c Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Wed, 2 Nov 2016 16:10:54 -0700 Subject: [PATCH 271/324] rebase deconv implementation with develop branch and resolve conflicts with pull#218 commit 45c81a414f9 --- paddle/gserver/layers/ConvBaseLayer.cpp | 69 ++++++++++++++----- paddle/gserver/layers/ExpandConvBaseLayer.cpp | 16 ++++- paddle/gserver/layers/ExpandConvBaseLayer.h | 9 +-- paddle/gserver/layers/ExpandConvLayer.cpp | 10 --- paddle/gserver/layers/ExpandConvLayer.h | 2 - .../gserver/layers/ExpandConvTransLayer.cpp | 30 +------- paddle/gserver/layers/ExpandConvTransLayer.h | 2 - paddle/gserver/tests/test_ConvTrans.cpp | 8 +-- python/paddle/trainer/config_parser.py | 12 ++-- 9 files changed, 78 insertions(+), 80 deletions(-) diff --git a/paddle/gserver/layers/ConvBaseLayer.cpp b/paddle/gserver/layers/ConvBaseLayer.cpp index 607fb99cf6288..5bc22f477932c 100644 --- a/paddle/gserver/layers/ConvBaseLayer.cpp +++ b/paddle/gserver/layers/ConvBaseLayer.cpp @@ -48,8 +48,20 @@ bool ConvBaseLayer::init(const LayerMap& layerMap, outputW_.push_back(conf.output_x()); } + CHECK(inputLayers_.size() == parameters_.size()); + for (size_t i = 0; i < inputLayers_.size(); i++) { + size_t height, width; + height = filterPixels_[i] * filterChannels_[i]; + width = (!isDeconv_) ? numFilters_ : channels_[i]; + + // create a new weight + CHECK_EQ(parameters_[i]->getSize(), width * height); + Weight* w = new Weight(height, width, parameters_[i]); + weights_.emplace_back(w); + } + /* initialize the biases_ */ - if (biasParameter_.get() != NULL) { + if (biasParameter_.get()) { if (sharedBiases_) { CHECK_EQ((size_t)numFilters_, biasParameter_->getSize()); biases_ = @@ -76,25 +88,46 @@ size_t ConvBaseLayer::calOutputSize() { clearAndReserve(&outputH_); clearAndReserve(&outputW_); size_t layerSize = 0; - for (size_t i = 0; i < inputLayers_.size(); i++) { - imgSizeH_.push_back(inputLayers_[i]->getOutput().getFrameHeight()); - imgSizeW_.push_back(inputLayers_[i]->getOutput().getFrameWidth()); - if (imgSizeH_[i] == 0) - imgSizeH_[i] = config_.inputs(i).conv_conf().img_size(); - if (imgSizeW_[i] == 0) - imgSizeW_[i] = config_.inputs(i).conv_conf().img_size(); - outputH_.push_back(outputSize(imgSizeH_[i], filterSizeY_[i], paddingY_[i], - strideY_[i], caffeMode_)); - outputW_.push_back(outputSize(imgSizeW_[i], filterSize_[i], padding_[i], - stride_[i], caffeMode_)); - CHECK_EQ(outputH_[i], outputH_[0]); - CHECK_EQ(outputW_[i], outputW_[0]); + + if (!isDeconv_) { + for (size_t i = 0; i < inputLayers_.size(); i++) { + imgSizeH_.push_back(inputLayers_[i]->getOutput().getFrameHeight()); + imgSizeW_.push_back(inputLayers_[i]->getOutput().getFrameWidth()); + if (imgSizeH_[i] == 0) + imgSizeH_[i] = config_.inputs(i).conv_conf().img_size(); + if (imgSizeW_[i] == 0) + imgSizeW_[i] = config_.inputs(i).conv_conf().img_size(); + outputH_.push_back( + outputSize(imgSizeH_[i], filterSizeY_[i], paddingY_[i], strideY_[i])); + outputW_.push_back( + outputSize(imgSizeW_[i], filterSize_[i], padding_[i], stride_[i])); + CHECK_EQ(outputH_[i], outputH_[0]); + CHECK_EQ(outputW_[i], outputW_[0]); + } + getOutput().setFrameHeight(outputH_[0]); + getOutput().setFrameWidth(outputW_[0]); + layerSize = outputH_[0] * outputW_[0] * size_t(numFilters_); + } else { + for (size_t i = 0; i < inputLayers_.size(); i++) { + outputH_.push_back(inputLayers_[i]->getOutput().getFrameHeight()); + outputW_.push_back(inputLayers_[i]->getOutput().getFrameWidth()); + if (outputH_[i] == 0) + outputH_[i] = config_.inputs(i).conv_conf().output_x(); + if (outputW_[i] == 0) + outputW_[i] = config_.inputs(i).conv_conf().output_x(); + imgSizeH_.push_back( + imageSize(outputH_[i], filterSizeY_[i], paddingY_[i], strideY_[i])); + imgSizeW_.push_back( + imageSize(outputW_[i], filterSize_[i], padding_[i], stride_[i])); + CHECK_EQ(imgSizeH_[i], imgSizeH_[0]); + CHECK_EQ(imgSizeW_[i], imgSizeW_[0]); + } + getOutput().setFrameHeight(imgSizeH_[0]); + getOutput().setFrameWidth(imgSizeW_[0]); + layerSize = imgSizeH_[0] * imgSizeW_[0] * size_t(numFilters_); } - getOutput().setFrameHeight(outputH_[0]); - getOutput().setFrameWidth(outputW_[0]); - layerSize = outputH_[0] * outputW_[0] * size_t(numFilters_); - return layerSize; + return layerSize; } } // namespace paddle diff --git a/paddle/gserver/layers/ExpandConvBaseLayer.cpp b/paddle/gserver/layers/ExpandConvBaseLayer.cpp index 9693ad82450f4..75ac8245d8829 100644 --- a/paddle/gserver/layers/ExpandConvBaseLayer.cpp +++ b/paddle/gserver/layers/ExpandConvBaseLayer.cpp @@ -45,15 +45,27 @@ bool ExpandConvBaseLayer::init(const LayerMap &layerMap, caffeMode_ = conf.caffe_mode(); } + getOutputSize(); + return true; } +size_t ExpandConvBaseLayer::getOutputSize() { + CHECK_NE(inputLayers_.size(), 0UL); + size_t layerSize = ConvBaseLayer::calOutputSize(); + subN_.clear(); + for (size_t i = 0; i < inputLayers_.size(); i++) { + subN_.push_back(outputH_[i] * outputW_[i]); + } + return layerSize; +} + void ExpandConvBaseLayer::resetExpandInput(size_t height, size_t width) { Matrix::resizeOrCreate(expandInput_, height, width, false, useGpu_); } void ExpandConvBaseLayer::addSharedBias() { - size_t mapW = getSize() / numFilters_; + size_t mapW = getOutputSize() / numFilters_; size_t mapH = getOutputValue()->getElementCnt() / mapW; MatrixPtr out = Matrix::create(getOutputValue()->getData(), mapH, mapW, false, useGpu_); @@ -224,7 +236,7 @@ void ExpandConvBaseLayer::bpropWeights(MatrixPtr image, MatrixPtr out, } void ExpandConvBaseLayer::bpropSharedBias(MatrixPtr biases, MatrixPtr v) { - size_t mapW = getSize() / numFilters_; + size_t mapW = getOutputSize() / numFilters_; size_t mapH = v->getElementCnt() / mapW; MatrixPtr vTmp = Matrix::create(v->getData(), mapH, mapW, false, useGpu_); diff --git a/paddle/gserver/layers/ExpandConvBaseLayer.h b/paddle/gserver/layers/ExpandConvBaseLayer.h index 418c9dd6ce2ad..9858fa348c3fc 100644 --- a/paddle/gserver/layers/ExpandConvBaseLayer.h +++ b/paddle/gserver/layers/ExpandConvBaseLayer.h @@ -34,14 +34,6 @@ class ExpandConvBaseLayer : public ConvBaseLayer { IntV subN_; /// subK_ = channels_ * filterPixels_ * groups_. IntV subK_; - /// The spatial dimensions of height of input feature map. - IntV imgSizeH_; - /// The spatial dimensions of width of input feature map. - IntV imgSizeW_; - /// The spatial dimensions of height of output feature map. - IntV outputH_; - /// The spatial dimensions of width of output feature map. - IntV outputW_; /*The expandInput_ and transOutValue_ are used for CPU expand conv calc * Expand one sample at a time. shape: @@ -59,6 +51,7 @@ class ExpandConvBaseLayer : public ConvBaseLayer { bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); + size_t getOutputSize(); /** * Create or resize expandInput_. */ diff --git a/paddle/gserver/layers/ExpandConvLayer.cpp b/paddle/gserver/layers/ExpandConvLayer.cpp index 379823a6feb45..9f30fcf00a422 100644 --- a/paddle/gserver/layers/ExpandConvLayer.cpp +++ b/paddle/gserver/layers/ExpandConvLayer.cpp @@ -28,16 +28,6 @@ bool ExpandConvLayer::init(const LayerMap &layerMap, return true; } -size_t ExpandConvLayer::getOutputSize() { - CHECK_NE(inputLayers_.size(), 0UL); - size_t layerSize = ConvBaseLayer::calOutputSize(); - subN_.clear(); - for (size_t i = 0; i < inputLayers_.size(); i++) { - subN_.push_back(outputH_[i] * outputW_[i]); - } - return layerSize; -} - void ExpandConvLayer::forward(PassType passType) { Layer::forward(passType); diff --git a/paddle/gserver/layers/ExpandConvLayer.h b/paddle/gserver/layers/ExpandConvLayer.h index b5cb448bdfcde..c07188a406183 100644 --- a/paddle/gserver/layers/ExpandConvLayer.h +++ b/paddle/gserver/layers/ExpandConvLayer.h @@ -38,8 +38,6 @@ class ExpandConvLayer : public ExpandConvBaseLayer { bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); - size_t getOutputSize(); - void forward(PassType passType); void backward(const UpdateCallback& callback); }; diff --git a/paddle/gserver/layers/ExpandConvTransLayer.cpp b/paddle/gserver/layers/ExpandConvTransLayer.cpp index 1d630a4ecd031..4c4016c30168f 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.cpp +++ b/paddle/gserver/layers/ExpandConvTransLayer.cpp @@ -34,34 +34,6 @@ bool ExpandConvTransLayer::init(const LayerMap &layerMap, return true; } -// Why this is necessary after calling init? -size_t ExpandConvTransLayer::getSize() { - CHECK_NE(inputLayers_.size(), 0UL); - imgSizeH_.clear(); - imgSizeW_.clear(); - outputH_.clear(); - outputW_.clear(); - subN_.clear(); - size_t layerSize = 0; - for (size_t i = 0; i < inputLayers_.size(); i++) { - outputH_.push_back(inputLayers_[i]->getOutput().getFrameHeight()); - outputW_.push_back(inputLayers_[i]->getOutput().getFrameWidth()); - if (outputH_[i] == 0) outputH_[i] = outputX_[i]; - if (outputW_[i] == 0) outputW_[i] = outputX_[i]; - imgSizeH_.push_back( - imageSize(outputH_[i], filterSize_[i], padding_[i], stride_[i])); - imgSizeW_.push_back( - imageSize(outputW_[i], filterSize_[i], padding_[i], stride_[i])); - subN_.push_back(outputH_[i] * outputW_[i]); - CHECK(layerSize == 0 || - imgSizeH_[i] * imgSizeW_[i] * (size_t)numFilters_ == layerSize); - layerSize = imgSizeH_[i] * imgSizeW_[i] * numFilters_; - } - getOutput().setFrameHeight(imgSizeH_[0]); - getOutput().setFrameWidth(imgSizeW_[0]); - return layerSize; -} - void ExpandConvTransLayer::forward(PassType passType) { Layer::forward(passType); @@ -69,7 +41,7 @@ void ExpandConvTransLayer::forward(PassType passType) { /* note: one sample correspond to one colum, and the * transOutValue correspond sample to one row */ int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); - resetOutput(batchSize, getSize()); + resetOutput(batchSize, getOutputSize()); MatrixPtr output = nullptr; for (size_t i = 0; i < inputLayers_.size(); ++i) { diff --git a/paddle/gserver/layers/ExpandConvTransLayer.h b/paddle/gserver/layers/ExpandConvTransLayer.h index ebcb34f073494..d0c0469c351aa 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.h +++ b/paddle/gserver/layers/ExpandConvTransLayer.h @@ -37,8 +37,6 @@ class ExpandConvTransLayer : public ExpandConvBaseLayer { bool init(const LayerMap& layerMap, const ParameterMap& parameterMap); - size_t getSize(); - void forward(PassType passType); void backward(const UpdateCallback& callback); }; diff --git a/paddle/gserver/tests/test_ConvTrans.cpp b/paddle/gserver/tests/test_ConvTrans.cpp index 787113d242391..756faf26516fe 100644 --- a/paddle/gserver/tests/test_ConvTrans.cpp +++ b/paddle/gserver/tests/test_ConvTrans.cpp @@ -43,11 +43,11 @@ TEST(Layer, convTransLayerFwd) { configt.layerConfig.set_partial_sum(1); configt.layerConfig.set_shared_biases(true); - configt.inputDefs.push_back({INPUT_DATA, "layer_0", 1024, 288}); + configt.inputDefs.push_back({INPUT_DATA, "layer_0", 1024, 384}); LayerInputConfig* input = configt.layerConfig.add_inputs(); ConvConfig* conv = input->mutable_conv_conf(); conv->set_filter_size(2); - conv->set_filter_size_y(3); + conv->set_filter_size_y(4); conv->set_channels(16); conv->set_padding(0); conv->set_padding_y(1); @@ -86,11 +86,11 @@ TEST(Layer, convTransLayerFwd) { config.layerConfig.set_partial_sum(1); config.layerConfig.set_shared_biases(true); - config.inputDefs.push_back({INPUT_DATA, "layer_1", 768, 288}); + config.inputDefs.push_back({INPUT_DATA, "layer_1", 768, 384}); input = config.layerConfig.add_inputs(); conv = input->mutable_conv_conf(); conv->set_filter_size(2); - conv->set_filter_size_y(3); + conv->set_filter_size_y(4); conv->set_channels(3); conv->set_padding(0); conv->set_padding_y(1); diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index b17ec6c6f6bf7..b3d17a47a965f 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1670,11 +1670,13 @@ def __init__( if self.layer_type == "cudnn_convt": config_assert(use_gpu, "cudnn_convt only support GPU") - if (use_gpu == 1 and self.layer_type != "exconvt" and - (parallel_nn == 0 or self.config.device > -1)): - self.layer_type = "cudnn_convt" - else: - self.layer_type = "exconvt" +# if (use_gpu == 1 and self.layer_type != "exconvt" and +# (parallel_nn == 0 or self.config.device > -1)): +# self.layer_type = "cudnn_convt" +# else: +# self.layer_type = "exconvt" + # cudnn_convt has not been implemented so use exconvt only + self.layer_type = "exconvt" # need to specify layer in config self.config.type = self.layer_type From fb20187aaa30b7c049c1dee2ad62bde7005b5af8 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Thu, 3 Nov 2016 11:25:34 -0700 Subject: [PATCH 272/324] deconv layer implementation modification following luotao1 comments --- paddle/gserver/layers/ConvBaseLayer.cpp | 65 +++++++------- paddle/gserver/layers/ConvBaseLayer.h | 2 - paddle/gserver/layers/ExpandConvBaseLayer.cpp | 16 ++-- paddle/gserver/tests/test_ConvTrans.cpp | 63 +++++++------- paddle/gserver/tests/test_LayerGrad.cpp | 6 +- python/paddle/trainer/config_parser.py | 87 +++++++++---------- .../paddle/trainer_config_helpers/layers.py | 2 +- 7 files changed, 115 insertions(+), 126 deletions(-) diff --git a/paddle/gserver/layers/ConvBaseLayer.cpp b/paddle/gserver/layers/ConvBaseLayer.cpp index 5bc22f477932c..733065a753772 100644 --- a/paddle/gserver/layers/ConvBaseLayer.cpp +++ b/paddle/gserver/layers/ConvBaseLayer.cpp @@ -89,42 +89,41 @@ size_t ConvBaseLayer::calOutputSize() { clearAndReserve(&outputW_); size_t layerSize = 0; - if (!isDeconv_) { + auto setLayerSize = [&](IntV& inH, IntV& inW, IntV& outH, IntV& outW) { for (size_t i = 0; i < inputLayers_.size(); i++) { - imgSizeH_.push_back(inputLayers_[i]->getOutput().getFrameHeight()); - imgSizeW_.push_back(inputLayers_[i]->getOutput().getFrameWidth()); - if (imgSizeH_[i] == 0) - imgSizeH_[i] = config_.inputs(i).conv_conf().img_size(); - if (imgSizeW_[i] == 0) - imgSizeW_[i] = config_.inputs(i).conv_conf().img_size(); - outputH_.push_back( - outputSize(imgSizeH_[i], filterSizeY_[i], paddingY_[i], strideY_[i])); - outputW_.push_back( - outputSize(imgSizeW_[i], filterSize_[i], padding_[i], stride_[i])); - CHECK_EQ(outputH_[i], outputH_[0]); - CHECK_EQ(outputW_[i], outputW_[0]); + inH.push_back(inputLayers_[i]->getOutput().getFrameHeight()); + inW.push_back(inputLayers_[i]->getOutput().getFrameWidth()); + if (isDeconv_) { + if (inH[i] == 0) + inH[i] = config_.inputs(i).conv_conf().output_x(); + if (inW[i] == 0) + inW[i] = config_.inputs(i).conv_conf().output_x(); + outH.push_back( + imageSize(inH[i], filterSizeY_[i], paddingY_[i], strideY_[i])); + outW.push_back( + imageSize(inW[i], filterSize_[i], padding_[i], stride_[i])); + } else { + if (inH[i] == 0) + inH[i] = config_.inputs(i).conv_conf().img_size(); + if (inW[i] == 0) + inW[i] = config_.inputs(i).conv_conf().img_size(); + outH.push_back( + outputSize(inH[i], filterSizeY_[i], paddingY_[i], strideY_[i])); + outW.push_back( + outputSize(inW[i], filterSize_[i], padding_[i], stride_[i])); + CHECK_EQ(outH[i], outH[0]); + CHECK_EQ(outW[i], outW[0]); + } } - getOutput().setFrameHeight(outputH_[0]); - getOutput().setFrameWidth(outputW_[0]); - layerSize = outputH_[0] * outputW_[0] * size_t(numFilters_); + getOutput().setFrameHeight(outH[0]); + getOutput().setFrameWidth(outW[0]); + layerSize = outH[0] * outW[0] * size_t(numFilters_); + }; + + if (isDeconv_) { + setLayerSize(outputH_, outputW_, imgSizeH_, imgSizeW_); } else { - for (size_t i = 0; i < inputLayers_.size(); i++) { - outputH_.push_back(inputLayers_[i]->getOutput().getFrameHeight()); - outputW_.push_back(inputLayers_[i]->getOutput().getFrameWidth()); - if (outputH_[i] == 0) - outputH_[i] = config_.inputs(i).conv_conf().output_x(); - if (outputW_[i] == 0) - outputW_[i] = config_.inputs(i).conv_conf().output_x(); - imgSizeH_.push_back( - imageSize(outputH_[i], filterSizeY_[i], paddingY_[i], strideY_[i])); - imgSizeW_.push_back( - imageSize(outputW_[i], filterSize_[i], padding_[i], stride_[i])); - CHECK_EQ(imgSizeH_[i], imgSizeH_[0]); - CHECK_EQ(imgSizeW_[i], imgSizeW_[0]); - } - getOutput().setFrameHeight(imgSizeH_[0]); - getOutput().setFrameWidth(imgSizeW_[0]); - layerSize = imgSizeH_[0] * imgSizeW_[0] * size_t(numFilters_); + setLayerSize(imgSizeH_, imgSizeW_, outputH_, outputW_); } return layerSize; diff --git a/paddle/gserver/layers/ConvBaseLayer.h b/paddle/gserver/layers/ConvBaseLayer.h index 2f2ce59ad9e6c..4d5b2b8d05af7 100644 --- a/paddle/gserver/layers/ConvBaseLayer.h +++ b/paddle/gserver/layers/ConvBaseLayer.h @@ -78,8 +78,6 @@ class ConvBaseLayer : public Layer { /// of output size. bool caffeMode_; - - public: explicit ConvBaseLayer(const LayerConfig& config) : Layer(config) {} diff --git a/paddle/gserver/layers/ExpandConvBaseLayer.cpp b/paddle/gserver/layers/ExpandConvBaseLayer.cpp index 75ac8245d8829..0bab0ca764f4f 100644 --- a/paddle/gserver/layers/ExpandConvBaseLayer.cpp +++ b/paddle/gserver/layers/ExpandConvBaseLayer.cpp @@ -31,14 +31,14 @@ bool ExpandConvBaseLayer::init(const LayerMap &layerMap, * convTrans, and in other functions too. * */ int channel; - int nf; + int numFilters; /* Initialize the projection */ for (auto &inputConfig : config_.inputs()) { const ConvConfig &conf = inputConfig.conv_conf(); - nf = (!isDeconv_) ? numFilters_ : conf.channels(); - subM_.push_back(nf / conf.groups()); + numFilters = isDeconv_ ? conf.channels() : numFilters_; + subM_.push_back(numFilters / conf.groups()); subN_.push_back(conf.output_x() * conf.output_x()); - channel = (!isDeconv_) ? conf.channels() : numFilters_; + channel = isDeconv_ ? numFilters_ : conf.channels(); subK_.push_back(channel * conf.filter_size() * conf.filter_size() / conf.groups()); /* Consistent caffe mode for multiple input */ @@ -99,7 +99,7 @@ void ExpandConvBaseLayer::addUnsharedBias() { void ExpandConvBaseLayer::expandOneFrame(MatrixPtr image, size_t startIdx, int inIdx) { - int channel = (!isDeconv_) ? channels_[inIdx] : numFilters_; + int channel = isDeconv_ ? numFilters_ : channels_[inIdx]; resetExpandInput(subK_[inIdx] * groups_[inIdx], subN_[inIdx]); real *imgData = image->getData() + startIdx * image->getWidth(); @@ -122,10 +122,10 @@ void ExpandConvBaseLayer::expandFwdOnce(MatrixPtr image, MatrixPtr out, expandOneFrame(image, startIdx, inIdx); - int nf = (!isDeconv_) ? numFilters_ : channels_[inIdx]; + int numFilters = isDeconv_ ? channels_[inIdx] : numFilters_; real *outData = - out->getData() + startIdx * subN * nf; + out->getData() + startIdx * subN * numFilters; real *wgtData = weights_[inIdx]->getW()->getData(); real *expInData = expandInput_->getData(); @@ -147,7 +147,7 @@ void ExpandConvBaseLayer::expandFwdOnce(MatrixPtr image, MatrixPtr out, void ExpandConvBaseLayer::bpropActs(MatrixPtr out, MatrixPtr image, int inpIdx) { - int channel = (!isDeconv_) ? channels_[inpIdx] : numFilters_; + int channel = isDeconv_ ? numFilters_ : channels_[inpIdx]; int subM = subM_[inpIdx]; int subN = subN_[inpIdx]; diff --git a/paddle/gserver/tests/test_ConvTrans.cpp b/paddle/gserver/tests/test_ConvTrans.cpp index 756faf26516fe..9246484ba22c2 100644 --- a/paddle/gserver/tests/test_ConvTrans.cpp +++ b/paddle/gserver/tests/test_ConvTrans.cpp @@ -189,58 +189,55 @@ void doOneConvtTest(size_t imgSize, size_t output_x, size_t stride, } TEST(Layer, convTransLayerFwd2) { - size_t imgSize, output_x, stride, padding, filter_size; MatrixPtr result; - - imgSize = 5; - output_x = 1; - stride = 1; - padding = 0; - filter_size = 5; - result = Matrix::create(1, imgSize * imgSize, false, false); + result = Matrix::create(1, 5 * 5, false, false); result->zeroMem(); result->add(1.0); - doOneConvtTest(imgSize, output_x, stride, padding, filter_size, result); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 1, + /* stride */ 1, + /* padding */ 0, + /* filter_size */ 5, + result); - imgSize = 5; - output_x = 2; - stride = 1; - padding = 0; - filter_size = 4; float resultData[] = {1, 2, 2, 2, 1, 2, 4, 4, 4, 2, 2, 4, 4, 4, 2, 2, 4, 4, 4, 2, 1, 2, 2, 2, 1}; - result = Matrix::create(resultData, 1, imgSize * imgSize, false, false); - doOneConvtTest(imgSize, output_x, stride, padding, filter_size, result); - - imgSize = 5; - output_x = 2; - stride = 2; - padding = 1; - filter_size = 5; + result->setData(resultData); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 2, + /* stride */ 1, + /* padding */ 0, + /* filter_size */ 4, + result); + float resultData2[] = {1, 2, 2, 2, 1, 2, 4, 4, 4, 2, 2, 4, 4, 4, 2, 2, 4, 4, 4, 2, 1, 2, 2, 2, 1}; - result = Matrix::create(resultData2, 1, imgSize * imgSize, false, false); - doOneConvtTest(imgSize, output_x, stride, padding, filter_size, result); - - imgSize = 5; - output_x = 2; - stride = 2; - padding = 0; - filter_size = 3; + result->setData(resultData2); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 2, + /* stride */ 2, + /* padding */ 1, + /* filter_size */ 5, + result); + float resultData3[] = {1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 2, 2, 4, 2, 2, 1, 1, 2, 1, 1, 1, 1, 2, 1, 1}; - result = Matrix::create(resultData3, 1, imgSize * imgSize, false, false); - doOneConvtTest(imgSize, output_x, stride, padding, filter_size, result); -} + result->setData(resultData3); + doOneConvtTest(/* imgSize */ 5, + /* output_x */ 2, + /* stride */ 2, + /* padding */ 0, + /* filter_size */ 3, + result);} int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 42c7b13906254..9e2e5ebaac24e 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -351,12 +351,10 @@ void testConvTransLayer(const string& type, bool trans, bool useGpu) { TEST(Layer, convTransLayer) { testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ false); -/* #ifndef PADDLE_ONLY_CPU - testConvLayer("exconv", trans= false, useGpu= true); - testConvLayer("cudnn_conv", trans= false, useGpu= true); + testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ true); + // testConvLayer("cudnn_conv", /* trans= */ false, /* useGpu= */ true); #endif -*/ } TEST(Layer, blockExpandLayer) { diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index b3d17a47a965f..3aa5576c3cf06 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1082,7 +1082,11 @@ def parse_norm(norm, input_layer_name, norm_conf): else: norm_conf.scale /= norm.size ** 2 -def parse_conv(conv, input_layer_name, conv_conf): +''' +caffe_mode: compute the output size using floor instead of ceil, + which is consistent of caffe and CuDNN's convention. +''' +def parse_conv(conv, input_layer_name, conv_conf, trans=False): conv_conf.filter_size = conv.filter_size conv_conf.filter_size_y = conv.filter_size_y conv_conf.channels = conv.channels @@ -1093,49 +1097,41 @@ def parse_conv(conv, input_layer_name, conv_conf): conv_conf.groups = conv.groups conv_conf.filter_channels = conv.channels / conv.groups conv_conf.caffe_mode = conv.caffe_mode - - img_pixels = g_layer_map[input_layer_name].size / conv.channels - print('channels=%d size=%d'%(conv.channels, - g_layer_map[input_layer_name].size)) - conv_conf.img_size = int(img_pixels ** 0.5) - config_assert((conv_conf.img_size ** 2) == img_pixels, - ("Input layer %s: Incorrect input image size %d for input " - + "image pixels %d") - % (input_layer_name, conv_conf.img_size, img_pixels)) - conv_conf.output_x = cnn_output_size(conv_conf.img_size, conv_conf.filter_size, - conv_conf.padding, conv_conf.stride, - conv_conf.caffe_mode) - - -def parse_conv_trans(conv, input_layer_name, conv_conf, num_filters): - conv_conf.filter_size = conv.filter_size - conv_conf.filter_size_y = conv.filter_size_y - conv_conf.channels = conv.channels - conv_conf.padding = conv.padding - conv_conf.padding_y = conv.padding_y - conv_conf.stride = conv.stride - conv_conf.stride_y = conv.stride_y - conv_conf.groups = conv.groups - conv_conf.filter_channels = num_filters / conv.groups - conv_conf.caffe_mode = conv.caffe_mode - - outputSize = g_layer_map[input_layer_name].size / conv.channels - print('channels=%d size=%d'%(conv.channels, - g_layer_map[input_layer_name].size)) - conv_conf.output_x = int(outputSize ** 0.5) - config_assert((conv_conf.output_x ** 2) == outputSize, - ("Input layer %s: Incorrect input image size %d for input " - + "image pixels %d") - % (input_layer_name, conv_conf.output_x, outputSize)) - if conv.caffe_mode: - conv_conf.img_size = \ - (conv_conf.output_x - 1) * conv.stride \ - + conv.filter_size - 2 * conv.padding + + if not trans: + img_pixels = g_layer_map[input_layer_name].size / conv.channels + print('channels=%d size=%d'%(conv.channels, + g_layer_map[input_layer_name].size)) + conv_conf.img_size = int(img_pixels ** 0.5) + config_assert((conv_conf.img_size ** 2) == img_pixels, + ("Input layer %s: Incorrect input image size %d for input " + + "image pixels %d") + % (input_layer_name, conv_conf.img_size, img_pixels)) + if conv.caffe_mode: + conv_conf.output_x = \ + 1 + int(math.floor((2 * conv.padding + conv_conf.img_size \ + - conv.filter_size) / float(conv.stride))) + else: + conv_conf.output_x = \ + 1 + int(math.ceil((2 * conv.padding + conv_conf.img_size \ + - conv.filter_size) / float(conv.stride))) else: - conv_conf.img_size = \ - (conv_conf.output_x - 2) * conv.stride \ - + conv.filter_size - 2 * conv.padding + 1 - + outputSize = g_layer_map[input_layer_name].size / conv.channels + print('channels=%d size=%d'%(conv.channels, + g_layer_map[input_layer_name].size)) + conv_conf.output_x = int(outputSize ** 0.5) + config_assert((conv_conf.output_x ** 2) == outputSize, + ("Input layer %s: Incorrect input image size %d for input " + + "image pixels %d") + % (input_layer_name, conv_conf.output_x, outputSize)) + if conv.caffe_mode: + conv_conf.img_size = \ + (conv_conf.output_x - 1) * conv.stride \ + + conv.filter_size - 2 * conv.padding + else: + conv_conf.img_size = \ + (conv_conf.output_x - 2) * conv.stride \ + + conv.filter_size - 2 * conv.padding + 1 def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand_conf.channels = block_expand.channels @@ -1685,10 +1681,11 @@ def __init__( for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_conv_trans( + parse_conv( self.inputs[input_index].conv, input_layer.name, - self.config.inputs[input_index].conv_conf, num_filters) + self.config.inputs[input_index].conv_conf, num_filters, + trans=True) conv_conf = self.config.inputs[input_index].conv_conf psize = self.calc_parameter_size(conv_conf) print("output size for %s is %d " % (name, conv_conf.output_x)) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 711c9ca993a23..1fdb1a849e86b 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -36,7 +36,7 @@ "pooling_layer", "lstmemory", "last_seq", "first_seq", "cos_sim", "hsigmoid", "conv_projection", "regression_cost", 'classification_cost', "LayerOutput", - 'img_conv_layer', 'img_convTrans_layer', 'img_pool_layer', 'batch_norm_layer', + 'img_conv_layer', 'img_pool_layer', 'batch_norm_layer', 'img_cmrnorm_layer', 'addto_layer', 'concat_layer', 'lstm_step_layer', 'recurrent_group', 'memory', 'StaticInput', 'expand_layer', 'scaling_layer', From d116b17f060cfad17c381a9f978d81dadf9fa81e Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Thu, 3 Nov 2016 17:30:02 -0700 Subject: [PATCH 273/324] fix a small bug in ConvTransLayerBase in config_parser.py --- python/paddle/trainer/config_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 3aa5576c3cf06..5ee46cd5f788b 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1684,7 +1684,7 @@ def __init__( parse_conv( self.inputs[input_index].conv, input_layer.name, - self.config.inputs[input_index].conv_conf, num_filters, + self.config.inputs[input_index].conv_conf, trans=True) conv_conf = self.config.inputs[input_index].conv_conf psize = self.calc_parameter_size(conv_conf) From 7a322df0a8e9bf06be32eb2433445d2bc4f22997 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Fri, 4 Nov 2016 10:15:57 -0700 Subject: [PATCH 274/324] deconv implementation mionr changes in ConvBaseLayer.cpp and config_parser.py --- paddle/gserver/layers/ConvBaseLayer.cpp | 12 ++++-------- python/paddle/trainer/config_parser.py | 5 ----- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/paddle/gserver/layers/ConvBaseLayer.cpp b/paddle/gserver/layers/ConvBaseLayer.cpp index 733065a753772..b9359867b9cce 100644 --- a/paddle/gserver/layers/ConvBaseLayer.cpp +++ b/paddle/gserver/layers/ConvBaseLayer.cpp @@ -20,12 +20,8 @@ bool ConvBaseLayer::init(const LayerMap& layerMap, const ParameterMap& parameterMap) { /* Initialize the basic parent class */ Layer::init(layerMap, parameterMap); - - if (config_.type() == "exconv" || config_.type() == "cudnn_conv") { - isDeconv_ = false; - } else { - isDeconv_ = true; - } + isDeconv_ = (config_.type() == "exconv" || config_.type() == "cudnn_conv") + ? false : true; /* Initialize the convolutional layer parameter */ numFilters_ = config_.num_filters(); @@ -111,9 +107,9 @@ size_t ConvBaseLayer::calOutputSize() { outputSize(inH[i], filterSizeY_[i], paddingY_[i], strideY_[i])); outW.push_back( outputSize(inW[i], filterSize_[i], padding_[i], stride_[i])); - CHECK_EQ(outH[i], outH[0]); - CHECK_EQ(outW[i], outW[0]); } + CHECK_EQ(outH[i], outH[0]); + CHECK_EQ(outW[i], outW[0]); } getOutput().setFrameHeight(outH[0]); getOutput().setFrameWidth(outW[0]); diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 5ee46cd5f788b..b75c2618411d3 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1666,11 +1666,6 @@ def __init__( if self.layer_type == "cudnn_convt": config_assert(use_gpu, "cudnn_convt only support GPU") -# if (use_gpu == 1 and self.layer_type != "exconvt" and -# (parallel_nn == 0 or self.config.device > -1)): -# self.layer_type = "cudnn_convt" -# else: -# self.layer_type = "exconvt" # cudnn_convt has not been implemented so use exconvt only self.layer_type = "exconvt" # need to specify layer in config From 03f4b1d4d2073248acedef00eb0d313f03ebf789 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Mon, 7 Nov 2016 09:49:33 -0800 Subject: [PATCH 275/324] minor changes on deconv per luotao1 comments --- paddle/gserver/tests/test_LayerGrad.cpp | 1 - python/paddle/trainer_config_helpers/layers.py | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 9e2e5ebaac24e..0fed97a73b0a4 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -353,7 +353,6 @@ TEST(Layer, convTransLayer) { testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ false); #ifndef PADDLE_ONLY_CPU testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ true); - // testConvLayer("cudnn_conv", /* trans= */ false, /* useGpu= */ true); #endif } diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 1fdb1a849e86b..ecc73c34a833c 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1619,10 +1619,7 @@ def img_conv_layer(input, filter_size, num_filters, param_attr.attr["initial_strategy"] = 0 param_attr.attr["initial_smart"] = False - if trans: - lt = LayerType.CONVTRANS_LAYER - else: - lt = LayerType.CONV_LAYER + lt = LayerType.CONVTRANS_LAYER if trans else LayerType.CONV_LAYER Layer( name=name, From 53e1629a43af7dc0e3b7d5d02745632e399aa6a8 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Tue, 8 Nov 2016 14:59:12 -0800 Subject: [PATCH 276/324] Refactored imageSize in ConvBaseLayer to MathUtil --- paddle/gserver/layers/ConvBaseLayer.cpp | 13 ++++++--- paddle/gserver/layers/ConvBaseLayer.h | 37 ------------------------- paddle/gserver/tests/test_ConvTrans.cpp | 16 +++++------ paddle/gserver/tests/test_LayerGrad.cpp | 7 ++--- paddle/math/MathUtils.cpp | 13 +++++++++ paddle/math/MathUtils.h | 3 ++ python/paddle/trainer/config_parser.py | 12 +++----- 7 files changed, 39 insertions(+), 62 deletions(-) diff --git a/paddle/gserver/layers/ConvBaseLayer.cpp b/paddle/gserver/layers/ConvBaseLayer.cpp index b9359867b9cce..6bc3b3b801796 100644 --- a/paddle/gserver/layers/ConvBaseLayer.cpp +++ b/paddle/gserver/layers/ConvBaseLayer.cpp @@ -14,6 +14,7 @@ limitations under the License. */ #include "paddle/utils/Logging.h" #include "ConvBaseLayer.h" +#include "paddle/math/MathUtils.h" namespace paddle { bool ConvBaseLayer::init(const LayerMap& layerMap, @@ -95,18 +96,22 @@ size_t ConvBaseLayer::calOutputSize() { if (inW[i] == 0) inW[i] = config_.inputs(i).conv_conf().output_x(); outH.push_back( - imageSize(inH[i], filterSizeY_[i], paddingY_[i], strideY_[i])); + imageSize(inH[i], filterSizeY_[i], paddingY_[i], strideY_[i], + caffeMode_)); outW.push_back( - imageSize(inW[i], filterSize_[i], padding_[i], stride_[i])); + imageSize(inW[i], filterSize_[i], padding_[i], stride_[i], + caffeMode_)); } else { if (inH[i] == 0) inH[i] = config_.inputs(i).conv_conf().img_size(); if (inW[i] == 0) inW[i] = config_.inputs(i).conv_conf().img_size(); outH.push_back( - outputSize(inH[i], filterSizeY_[i], paddingY_[i], strideY_[i])); + outputSize(inH[i], filterSizeY_[i], paddingY_[i], strideY_[i], + caffeMode_)); outW.push_back( - outputSize(inW[i], filterSize_[i], padding_[i], stride_[i])); + outputSize(inW[i], filterSize_[i], padding_[i], stride_[i], + caffeMode_)); } CHECK_EQ(outH[i], outH[0]); CHECK_EQ(outW[i], outW[0]); diff --git a/paddle/gserver/layers/ConvBaseLayer.h b/paddle/gserver/layers/ConvBaseLayer.h index 4d5b2b8d05af7..b80cab899585e 100644 --- a/paddle/gserver/layers/ConvBaseLayer.h +++ b/paddle/gserver/layers/ConvBaseLayer.h @@ -91,43 +91,6 @@ class ConvBaseLayer : public Layer { virtual size_t calOutputSize(); Weight& getWeight(int idx) { return *weights_[idx]; } - - /** - * Calculate output size based on caffeMode_. - * - input(+padding): 0123456789 - * - imageSize(+padding) = 10; - * - filterSize = 3; - * - stride = 2; - * - caffeMode_ is true: - - output: (012), (234), (456), (678) - - outputSize = 4; - * - caffeMode_ is false: - * - output: (012), (234), (456), (678), (9) - * - outputSize = 5; - */ - int outputSize(int imageSize, int filterSize, int padding, int stride) { - int outputSize; - if (!caffeMode_) { - outputSize = - (imageSize - filterSize + 2 * padding + stride - 1) / stride + 1; - } else { - outputSize = (imageSize - filterSize + 2 * padding) / stride + 1; - } - CHECK_GE(outputSize, 1); - return outputSize; - } - - int imageSize(int outputSize, int filterSize, int padding, int stride) { - int imageSize; - if (!caffeMode_) { - imageSize = - (outputSize - 1) * stride + filterSize - 2 * padding - stride + 1; - } else { - imageSize = (outputSize - 1) * stride + filterSize - 2 * padding; - } - CHECK_GE(imageSize, 1); - return imageSize; - } }; } // namespace paddle diff --git a/paddle/gserver/tests/test_ConvTrans.cpp b/paddle/gserver/tests/test_ConvTrans.cpp index 9246484ba22c2..bff7222b29907 100644 --- a/paddle/gserver/tests/test_ConvTrans.cpp +++ b/paddle/gserver/tests/test_ConvTrans.cpp @@ -20,6 +20,7 @@ limitations under the License. */ #include "paddle/trainer/Trainer.h" #include "paddle/utils/GlobalConstants.h" #include "paddle/gserver/layers/ExpandConvTransLayer.h" +#include "paddle/math/MathUtils.h" #include "TestUtil.h" #include "LayerGradUtil.h" @@ -56,11 +57,9 @@ TEST(Layer, convTransLayerFwd) { conv->set_groups(1); conv->set_filter_channels(3 / conv->groups()); conv->set_img_size(16); - conv->set_output_x( - (2 * conv->padding() + conv->img_size() - conv->filter_size()) / - ((float)conv->stride()) + - 1.5); - + conv->set_output_x(outputSize(conv->img_size(), conv->filter_size(), + conv->padding(), conv->stride(), + /* caffeMode */ true)); configt.layerConfig.set_size(conv->img_size() * conv->img_size() * configt.layerConfig.num_filters()); configt.layerConfig.set_name("convTrans"); @@ -99,10 +98,9 @@ TEST(Layer, convTransLayerFwd) { conv->set_groups(1); conv->set_filter_channels(conv->channels() / conv->groups()); conv->set_img_size(16); - conv->set_output_x( - (2 * conv->padding() + conv->img_size() - conv->filter_size()) / - ((float)conv->stride()) + - 1.5); + conv->set_output_x(outputSize(conv->img_size(), conv->filter_size(), + conv->padding(), conv->stride(), + /* caffeMode */ true)); config.layerConfig.set_size(conv->output_x() * conv->output_x() * config.layerConfig.num_filters()); config.layerConfig.set_name("conv"); diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 0fed97a73b0a4..02a30719f98e0 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -336,10 +336,9 @@ void testConvTransLayer(const string& type, bool trans, bool useGpu) { conv->set_groups(1); conv->set_filter_channels(3 / conv->groups()); conv->set_img_size(16); - conv->set_output_x( - (2 * conv->padding() + conv->img_size() - conv->filter_size()) / - ((float)conv->stride()) + - 1.5); + conv->set_output_x(outputSize(conv->img_size(), conv->filter_size(), + conv->padding(), conv->stride(), + /* caffeMode */ true)); config.layerConfig.set_size(conv->img_size() * conv->img_size() * config.layerConfig.num_filters()); diff --git a/paddle/math/MathUtils.cpp b/paddle/math/MathUtils.cpp index c1af8628d03c5..548f17936381c 100644 --- a/paddle/math/MathUtils.cpp +++ b/paddle/math/MathUtils.cpp @@ -80,4 +80,17 @@ int outputSize(int imageSize, int filterSize, int padding, int stride, return outputSize; } +int imageSize(int outputSize, int filterSize, int padding, int stride, + bool caffeMode) { + int imageSize; + if (!caffeMode) { + imageSize = + (outputSize - 1) * stride + filterSize - 2 * padding - stride + 1; + } else { + imageSize = (outputSize - 1) * stride + filterSize - 2 * padding; + } + CHECK_GE(imageSize, 1); + return imageSize; +} + } // namespace paddle diff --git a/paddle/math/MathUtils.h b/paddle/math/MathUtils.h index 49d0c10a8f5e4..ae035e55bcceb 100644 --- a/paddle/math/MathUtils.h +++ b/paddle/math/MathUtils.h @@ -60,4 +60,7 @@ void sparseRand(int* major, int* minor, int nnz, int majorLen, int minorMax, int outputSize(int imageSize, int filterSize, int padding, int stride, bool caffeMode); +int imageSize(int outputSize, int filterSize, int padding, int stride, + bool caffeMode); + } // namespace paddle diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index b75c2618411d3..8b13e4a142544 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1107,14 +1107,10 @@ def parse_conv(conv, input_layer_name, conv_conf, trans=False): ("Input layer %s: Incorrect input image size %d for input " + "image pixels %d") % (input_layer_name, conv_conf.img_size, img_pixels)) - if conv.caffe_mode: - conv_conf.output_x = \ - 1 + int(math.floor((2 * conv.padding + conv_conf.img_size \ - - conv.filter_size) / float(conv.stride))) - else: - conv_conf.output_x = \ - 1 + int(math.ceil((2 * conv.padding + conv_conf.img_size \ - - conv.filter_size) / float(conv.stride))) + + conv_conf.output_x = cnn_output_size( + conv_conf.img_size, conv_conf.filter_size, + conv_conf.padding, conv_conf.stride, conv_conf.caffe_mode) else: outputSize = g_layer_map[input_layer_name].size / conv.channels print('channels=%d size=%d'%(conv.channels, From 449120991ef9d4deb5e9635be3017824a0370a29 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Tue, 8 Nov 2016 15:02:49 -0800 Subject: [PATCH 277/324] minor change to convTransLayer test in test_LayerGrad --- paddle/gserver/tests/test_LayerGrad.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 02a30719f98e0..7b6e6fd3999ff 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -349,10 +349,9 @@ void testConvTransLayer(const string& type, bool trans, bool useGpu) { } TEST(Layer, convTransLayer) { - testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ false); -#ifndef PADDLE_ONLY_CPU - testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ true); -#endif + for (auto useGpu : {false, true}) { + testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ useGpu); + } } TEST(Layer, blockExpandLayer) { From af7a50c0d4e42d53920681c661921d4e78ff8e36 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Wed, 9 Nov 2016 11:26:28 -0800 Subject: [PATCH 278/324] minor changes on deconv implementation and add protostr test for deconv layer --- paddle/gserver/layers/ExpandConvLayer.cpp | 5 +--- .../gserver/layers/ExpandConvTransLayer.cpp | 2 -- paddle/gserver/layers/ExpandConvTransLayer.h | 2 +- paddle/math/MathUtils.h | 4 +++ python/paddle/trainer/config_parser.py | 28 +++++++++---------- .../tests/configs/generate_protostr.sh | 2 +- .../tests/configs/img_trans_layers.py | 22 +++++++++++++++ 7 files changed, 43 insertions(+), 22 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/img_trans_layers.py diff --git a/paddle/gserver/layers/ExpandConvLayer.cpp b/paddle/gserver/layers/ExpandConvLayer.cpp index 9f30fcf00a422..5ea1fdece5f7b 100644 --- a/paddle/gserver/layers/ExpandConvLayer.cpp +++ b/paddle/gserver/layers/ExpandConvLayer.cpp @@ -32,10 +32,7 @@ void ExpandConvLayer::forward(PassType passType) { Layer::forward(passType); /* malloc memory for the output_ if necessary */ - /* note: one sample correspond to one colum, and the - * transOutValue correspond sample to one row */ - int batchSize = inputLayers_[0]->getOutputValue()->getWidth(); - batchSize = inputLayers_[0]->getOutputValue()->getHeight(); + int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); resetOutput(batchSize, getOutputSize()); MatrixPtr image = nullptr; diff --git a/paddle/gserver/layers/ExpandConvTransLayer.cpp b/paddle/gserver/layers/ExpandConvTransLayer.cpp index 4c4016c30168f..a3e160f1f4eb5 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.cpp +++ b/paddle/gserver/layers/ExpandConvTransLayer.cpp @@ -38,8 +38,6 @@ void ExpandConvTransLayer::forward(PassType passType) { Layer::forward(passType); /* malloc memory for the output_ if necessary */ - /* note: one sample correspond to one colum, and the - * transOutValue correspond sample to one row */ int batchSize = inputLayers_[0]->getOutputValue()->getHeight(); resetOutput(batchSize, getOutputSize()); diff --git a/paddle/gserver/layers/ExpandConvTransLayer.h b/paddle/gserver/layers/ExpandConvTransLayer.h index d0c0469c351aa..87c464a97f2ed 100644 --- a/paddle/gserver/layers/ExpandConvTransLayer.h +++ b/paddle/gserver/layers/ExpandConvTransLayer.h @@ -26,7 +26,7 @@ namespace paddle { * This layer expands input and use matrix multiplication to * calculate convolution transpose (deconv) operation. * - * The config file api is img_convTrans_layer. + * The config file api is img_conv_layer with flag trans=True. */ class ExpandConvTransLayer : public ExpandConvBaseLayer { public: diff --git a/paddle/math/MathUtils.h b/paddle/math/MathUtils.h index ae035e55bcceb..91683dc3e9144 100644 --- a/paddle/math/MathUtils.h +++ b/paddle/math/MathUtils.h @@ -60,6 +60,10 @@ void sparseRand(int* major, int* minor, int nnz, int majorLen, int minorMax, int outputSize(int imageSize, int filterSize, int padding, int stride, bool caffeMode); +/** + * Calculate image size based on output size and caffeMode_. + * It is the reverse function of outputSize() + */ int imageSize(int outputSize, int filterSize, int padding, int stride, bool caffeMode); diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 8b13e4a142544..4a701326e4f9c 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1017,6 +1017,17 @@ def cnn_output_size(img_size, filter_size, padding, stride, caffe_mode): else: return 1 + int(math.ceil(output)) +''' +calcualte image_size based on output_size for convolution. +It is the reverse function of cnn_output_size +''' +def cnn_image_size(output_size, filter_size, padding, stride, caffe_mode): + if caffe_mode: + img_size = (output_size - 1) * stride + filter_size - 2 * padding + else: + img_size = (output_size - 2) * stride + filter_size - 2 * padding + 1 + return img_size + def parse_pool(pool, input_layer_name, pool_conf): pool_conf.pool_type = pool.pool_type config_assert(pool.pool_type in ['max-projection', 'avg-projection', @@ -1120,14 +1131,9 @@ def parse_conv(conv, input_layer_name, conv_conf, trans=False): ("Input layer %s: Incorrect input image size %d for input " + "image pixels %d") % (input_layer_name, conv_conf.output_x, outputSize)) - if conv.caffe_mode: - conv_conf.img_size = \ - (conv_conf.output_x - 1) * conv.stride \ - + conv.filter_size - 2 * conv.padding - else: - conv_conf.img_size = \ - (conv_conf.output_x - 2) * conv.stride \ - + conv.filter_size - 2 * conv.padding + 1 + conv_conf.img_size = cnn_image_size( + conv_conf.output_x, conv_conf.filter_size, + conv_conf.padding, conv_conf.stride, conv_conf.caffe_mode) def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand_conf.channels = block_expand.channels @@ -1656,12 +1662,6 @@ def __init__( use_gpu = int(g_command_config_args.get("use_gpu", 0)) parallel_nn = int(g_command_config_args.get("parallel_nn", 0)) - # Automatically select cudnn_type for GPU and exconv for CPU - # if set type=conv, but still reserve the way user specify - # exconv or cudnn_conv manually. - if self.layer_type == "cudnn_convt": - config_assert(use_gpu, "cudnn_convt only support GPU") - # cudnn_convt has not been implemented so use exconvt only self.layer_type = "exconvt" # need to specify layer in config diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index 77774f6fcfafd..b8687e1d48371 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -9,7 +9,7 @@ protostr=$PWD/protostr configs=(test_fc layer_activations projections test_print_layer test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid -img_layers util_layers simple_rnn_layers unused_layers test_cost_layers +img_layers img_trans_layers util_layers simple_rnn_layers unused_layers test_cost_layers test_rnn_group shared_fc shared_lstm test_cost_layers_with_weight test_maxout test_bi_grumemory math_ops) diff --git a/python/paddle/trainer_config_helpers/tests/configs/img_trans_layers.py b/python/paddle/trainer_config_helpers/tests/configs/img_trans_layers.py new file mode 100644 index 0000000000000..077c78d201648 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/img_trans_layers.py @@ -0,0 +1,22 @@ +from paddle.trainer_config_helpers import * + +settings( + learning_rate=1e-3, + batch_size=1000 +) + +img = data_layer(name='image', size=227*227) + +# the parse_conv in config_parse.py is not strictly accurate when filter_size +# is not square. So here set square filter_size. +img_conv = img_conv_layer(input=img, num_channels=1, num_filters=64, + filter_size=(32, 32), padding=(1, 1), stride=(1, 1), + act=LinearActivation(), trans=True) +img_bn = batch_norm_layer(input=img_conv, act=ReluActivation()) + +img_norm = img_cmrnorm_layer(input=img_bn, size=32) + +img_pool = img_pool_layer(input=img_conv, pool_size=32, pool_type=MaxPooling()) + + +outputs(img_pool, img_norm) From 1c58e27fdc94ebdec8522e36c47b3d636a835896 Mon Sep 17 00:00:00 2001 From: wangyang59 Date: Wed, 9 Nov 2016 11:43:40 -0800 Subject: [PATCH 279/324] fixed a bug in parse_conv in config_parser.py --- python/paddle/trainer/config_parser.py | 17 +- .../protostr/img_trans_layers.protostr | 176 ++++++++++++++++++ 2 files changed, 188 insertions(+), 5 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/img_trans_layers.protostr diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 4a701326e4f9c..958bfdaf2e283 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -649,7 +649,8 @@ def __init__( parse_conv(conv_conf, input_layer_name, - self.proj_conf.conv_conf) + self.proj_conf.conv_conf, + num_filters) # TODO: support rectangle input self.proj_conf.output_size = (self.proj_conf.conv_conf.output_x ** 2) * num_filters @@ -730,7 +731,8 @@ def __init__( parse_conv(conv_conf, MakeLayerNameInSubmodel(input_layer_names[0]), - self.operator_conf.conv_conf) + self.operator_conf.conv_conf, + num_filters) self.operator_conf.output_size = (self.operator_conf.conv_conf.output_x ** 2) * num_filters config_assert(len(input_layer_names) == 2, "Conv is binary operator") @@ -1097,7 +1099,7 @@ def parse_norm(norm, input_layer_name, norm_conf): caffe_mode: compute the output size using floor instead of ceil, which is consistent of caffe and CuDNN's convention. ''' -def parse_conv(conv, input_layer_name, conv_conf, trans=False): +def parse_conv(conv, input_layer_name, conv_conf, num_filters, trans=False): conv_conf.filter_size = conv.filter_size conv_conf.filter_size_y = conv.filter_size_y conv_conf.channels = conv.channels @@ -1106,10 +1108,11 @@ def parse_conv(conv, input_layer_name, conv_conf, trans=False): conv_conf.stride = conv.stride conv_conf.stride_y = conv.stride_y conv_conf.groups = conv.groups - conv_conf.filter_channels = conv.channels / conv.groups conv_conf.caffe_mode = conv.caffe_mode if not trans: + conv_conf.filter_channels = conv.channels / conv.groups + img_pixels = g_layer_map[input_layer_name].size / conv.channels print('channels=%d size=%d'%(conv.channels, g_layer_map[input_layer_name].size)) @@ -1123,6 +1126,8 @@ def parse_conv(conv, input_layer_name, conv_conf, trans=False): conv_conf.img_size, conv_conf.filter_size, conv_conf.padding, conv_conf.stride, conv_conf.caffe_mode) else: + conv_conf.filter_channels = num_filters / conv.groups + outputSize = g_layer_map[input_layer_name].size / conv.channels print('channels=%d size=%d'%(conv.channels, g_layer_map[input_layer_name].size)) @@ -1616,7 +1621,8 @@ def __init__( parse_conv( self.inputs[input_index].conv, input_layer.name, - self.config.inputs[input_index].conv_conf) + self.config.inputs[input_index].conv_conf, + num_filters) conv_conf = self.config.inputs[input_index].conv_conf psize = self.calc_parameter_size(conv_conf) print("output size for %s is %d " % (name, conv_conf.output_x)) @@ -1676,6 +1682,7 @@ def __init__( self.inputs[input_index].conv, input_layer.name, self.config.inputs[input_index].conv_conf, + num_filters, trans=True) conv_conf = self.config.inputs[input_index].conv_conf psize = self.calc_parameter_size(conv_conf) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/img_trans_layers.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/img_trans_layers.protostr new file mode 100644 index 0000000000000..38346354080b0 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/img_trans_layers.protostr @@ -0,0 +1,176 @@ +type: "nn" +layers { + name: "image" + type: "data" + size: 51529 + active_type: "" +} +layers { + name: "__conv_0__" + type: "exconvt" + size: 4194304 + active_type: "" + inputs { + input_layer_name: "image" + input_parameter_name: "___conv_0__.w0" + conv_conf { + filter_size: 32 + channels: 1 + stride: 1 + padding: 1 + groups: 1 + filter_channels: 64 + output_x: 227 + img_size: 256 + caffe_mode: true + filter_size_y: 32 + padding_y: 1 + stride_y: 1 + } + } + bias_parameter_name: "___conv_0__.wbias" + num_filters: 64 + shared_biases: true +} +layers { + name: "__batch_norm_0__" + type: "batch_norm" + size: 4194304 + active_type: "relu" + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w0" + image_conf { + channels: 64 + img_size: 256 + } + } + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w1" + } + inputs { + input_layer_name: "__conv_0__" + input_parameter_name: "___batch_norm_0__.w2" + } + bias_parameter_name: "___batch_norm_0__.wbias" + moving_average_fraction: 0.9 +} +layers { + name: "__crmnorm_0__" + type: "norm" + size: 4194304 + active_type: "" + inputs { + input_layer_name: "__batch_norm_0__" + norm_conf { + norm_type: "cmrnorm-projection" + channels: 64 + size: 32 + scale: 0.0004 + pow: 0.75 + output_x: 256 + img_size: 256 + blocked: false + } + } +} +layers { + name: "__pool_0__" + type: "pool" + size: 3240000 + active_type: "" + inputs { + input_layer_name: "__conv_0__" + pool_conf { + pool_type: "max-projection" + channels: 64 + size_x: 32 + stride: 1 + output_x: 225 + img_size: 256 + padding: 0 + size_y: 32 + stride_y: 1 + output_y: 225 + img_size_y: 256 + padding_y: 0 + } + } +} +parameters { + name: "___conv_0__.w0" + size: 65536 + initial_mean: 0.0 + initial_std: 0.0441941738242 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___conv_0__.wbias" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 64 + dims: 1 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___batch_norm_0__.w0" + size: 64 + initial_mean: 1.0 + initial_std: 0.0 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___batch_norm_0__.w1" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false + is_static: true + is_shared: true +} +parameters { + name: "___batch_norm_0__.w2" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false + is_static: true + is_shared: true +} +parameters { + name: "___batch_norm_0__.wbias" + size: 64 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 64 + initial_strategy: 0 + initial_smart: false +} +input_layer_names: "image" +output_layer_names: "__pool_0__" +output_layer_names: "__crmnorm_0__" +sub_models { + name: "root" + layer_names: "image" + layer_names: "__conv_0__" + layer_names: "__batch_norm_0__" + layer_names: "__crmnorm_0__" + layer_names: "__pool_0__" + input_layer_names: "image" + output_layer_names: "__pool_0__" + output_layer_names: "__crmnorm_0__" + is_recurrent_layer_group: false +} + From 9ff0db37feb08b0afe9e78c79379b6418c904f64 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 10 Nov 2016 09:23:48 +0800 Subject: [PATCH 280/324] Generate bilinear protostr via Linux --- .../tests/configs/protostr/test_bilinear_interp.protostr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr index d4cbfc2389ac5..13d0d477eb58f 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_bilinear_interp.protostr @@ -83,7 +83,7 @@ parameters { name: "___conv_0__.w0" size: 144 initial_mean: 0.0 - initial_std: 0.471404522657 + initial_std: 0.471404520791 initial_strategy: 0 initial_smart: false } From 8d4c453bbb7980491d539bbe574d5e729c90a097 Mon Sep 17 00:00:00 2001 From: Haonan Date: Wed, 9 Nov 2016 17:24:37 -0800 Subject: [PATCH 281/324] set mixedlayer output size according to input operator (#414) * set mixedlayer output size according to input operator * change from num_channel to num_channels for conv_operator (the old one is really misleading because all the others are num_channels) * also changed the arg name in projections.py --- .../paddle/trainer_config_helpers/layers.py | 43 ++++++++++--------- .../tests/configs/projections.py | 2 +- .../tests/layers_test_config.py | 4 +- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 49f0ff3289db7..6b5d39a47158b 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -590,7 +590,7 @@ def __enter__(self): def __exit__(self, *args, **kwargs): del args, kwargs # unused parameter to suppress warning assert len(self.inputs) != 0 - MixedLayer( + ml = MixedLayer( name=self.name, size=self.size, active_type=self.activation.name, @@ -598,6 +598,9 @@ def __exit__(self, *args, **kwargs): inputs=self.inputs, **ExtraLayerAttribute.to_kwargs(self.layer_attr) ) + # update the size which might be computed inside MixedLayer + # according to the operator's output size + self.size = ml.config.size @wrap_name_default("mixed") @@ -2045,7 +2048,7 @@ def __reduce_concat_type__(a, b): if layer_type == LayerType.CONCAT_LAYER: assert not bias_attr - + Layer( name=name, type=layer_type, inputs=[x.name for x in input] if is_concat_layer else input, @@ -2623,7 +2626,7 @@ def out_prod_layer(input1, input2, name=None, layer_attr=None): assert isinstance(input1, LayerOutput) assert isinstance(input2, LayerOutput) Layer(name=name, - type="out_prod", + type=LayerType.OUT_PROD_LAYER, inputs=[input1.name, input2.name], **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name=name, @@ -2790,7 +2793,7 @@ def __real_step__(*args): def __cost_input__(input, label, weight=None): """ - inputs and parents for cost layers. + inputs and parents for cost layers. """ ipts = [Input(input.name), Input(label.name)] parents = [input, label] @@ -2799,7 +2802,7 @@ def __cost_input__(input, label, weight=None): ipts.append(Input(weight.name)) parents.append(weight) return ipts, parents - + @wrap_name_default() @layer_support() @@ -2884,7 +2887,7 @@ def __add_evaluator__(e): def conv_operator(img, filter, filter_size, num_filters, - num_channel=None, stride=1, padding=0, + num_channels=None, stride=1, padding=0, filter_size_y=None, stride_y=None, padding_y=None): """ Different from img_conv_layer, conv_op is an Operator, which can be used @@ -2914,8 +2917,8 @@ def conv_operator(img, filter, filter_size, num_filters, :type filter_size_y: int :param num_filters: channel of output data. :type num_filters: int - :param num_channel: channel of input data. - :type num_channel: int + :param num_channels: channel of input data. + :type num_channels: int :param stride: The x dimension of the stride. :type stride: int :param stride_y: The y dimension of the stride. @@ -2934,19 +2937,19 @@ def conv_operator(img, filter, filter_size, num_filters, if padding_y is None: padding_y = padding - if num_channel is None: - num_channel = img.num_filters + if num_channels is None: + num_channels = img.num_filters assert isinstance(filter, LayerOutput) if filter.size is not None: - filter.size = filter_size * filter_size_y * num_filters * num_channel + filter.size = filter_size * filter_size_y * num_filters * num_channels op = ConvOperator(input_layer_names=[img.name, filter.name], num_filters=num_filters, conv_conf=Conv(filter_size=filter_size, padding=padding, stride=stride, - channels=num_channel, + channels=num_channels, filter_size_y=filter_size_y, padding_y=padding_y, stride_y=stride_y, @@ -2986,8 +2989,8 @@ def conv_projection(input, filter_size, num_filters, :type filter_size_y: int :param num_filters: channel of output data. :type num_filters: int - :param num_channel: channel of input data. - :type num_channel: int + :param num_channels: channel of input data. + :type num_channels: int :param stride: The x dimension of the stride. :type stride: int :param stride_y: The y dimension of the stride. @@ -3478,15 +3481,15 @@ def maxout_layer(input, - Input: output of a conv layer. - Output: feature map size same as input. Channel is (input channel) / groups. - So groups should be larger than 1, and the num of channels should be able + So groups should be larger than 1, and the num of channels should be able to devided by groups. - Please refer to Paper: + Please refer to Paper: - Maxout Networks: http://www.jmlr.org/proceedings/papers/v28/goodfellow13.pdf - Multi-digit Number Recognition from Street View \ Imagery using Deep Convolutional Neural Networks: \ https://arxiv.org/pdf/1312.6082v4.pdf - + The simple usage is: .. code-block:: python @@ -3731,9 +3734,9 @@ def nce_layer(input, label, num_classes, weight=None, :param weight: weight layer, can be None(default) :type weight: LayerOutput :param num_classes: number of classes. - :type num_classes: int + :type num_classes: int :param num_neg_samples: number of negative samples. Default is 10. - :type num_neg_samples: int + :type num_neg_samples: int :param neg_distribution: The distribution for generating the random negative labels. A uniform distribution will be used if not provided. If not None, its length must be equal to num_classes. @@ -3754,7 +3757,7 @@ def nce_layer(input, label, num_classes, weight=None, assert isinstance(neg_distribution, collections.Sequence) assert len(neg_distribution) == num_classes assert sum(neg_distribution) == 1 - + ipts_for_layer = [] parents = [] for each_input in input: diff --git a/python/paddle/trainer_config_helpers/tests/configs/projections.py b/python/paddle/trainer_config_helpers/tests/configs/projections.py index 4066c5bc6e0f0..51194b5a2a8ae 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/projections.py +++ b/python/paddle/trainer_config_helpers/tests/configs/projections.py @@ -35,7 +35,7 @@ with mixed_layer() as m7: m7 += conv_operator(img=img, filter=flt, num_filters=64, - num_channel=1, filter_size=3) + num_channels=1, filter_size=3) end = mixed_layer(input=[full_matrix_projection(input=m5), trans_full_matrix_projection(input=m6), diff --git a/python/paddle/trainer_config_helpers/tests/layers_test_config.py b/python/paddle/trainer_config_helpers/tests/layers_test_config.py index faaab9107d8fb..26be84f122180 100644 --- a/python/paddle/trainer_config_helpers/tests/layers_test_config.py +++ b/python/paddle/trainer_config_helpers/tests/layers_test_config.py @@ -29,9 +29,11 @@ filter=y1, filter_size=1, num_filters=5, - num_channel=5, + num_channels=5, stride=1)]) +assert z1.size > 0 + y2 = fc_layer(input=y, size=15) cos1 = cos_sim(a=x1, b=y1) From 880774d1418d53a3f4cf103891bc77ebed05e685 Mon Sep 17 00:00:00 2001 From: Haonan Date: Wed, 9 Nov 2016 17:48:17 -0800 Subject: [PATCH 282/324] change the act.name for LinearActivation() to "linear" so that it won't fail in hl_activetype; also fix the hasinputsset in submodel --- python/paddle/trainer/config_parser.py | 10 +++++----- python/paddle/trainer_config_helpers/activations.py | 8 ++++---- python/paddle/trainer_config_helpers/layers.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 73631602a92be..2abbf565039e4 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -218,7 +218,7 @@ def Inputs(*args): @config_func def HasInputsSet(): - return len(g_config.model_config.input_layer_names) != 0 + return len(g_current_submodel.input_layer_names) != 0 # Define the name of the output layers of the NeuralNetwork. @@ -1120,14 +1120,14 @@ def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand_conf.output_x = 0 else: block_expand_conf.output_x = cnn_output_size( - block_expand.img_size_x, block_expand.block_x, + block_expand.img_size_x, block_expand.block_x, block_expand.padding_x, block_expand.stride_x, False) if block_expand_conf.img_size_y == 0: block_expand_conf.output_y = 0 else: block_expand_conf.output_y = cnn_output_size( - block_expand.img_size_y, block_expand.block_y, + block_expand.img_size_y, block_expand.block_y, block_expand.padding_y, block_expand.stride_y, False) def parse_maxout(maxout, input_layer_name, maxout_conf): @@ -1135,7 +1135,7 @@ def parse_maxout(maxout, input_layer_name, maxout_conf): maxout_conf.groups = maxout.groups maxout_conf.img_size_x = maxout.img_size_x maxout_conf.img_size_y = maxout.img_size_y - + # Define an evaluator @config_func def Evaluator( @@ -1773,7 +1773,7 @@ def __init__( self.config.inputs[0].maxout_conf) maxout_conf = self.config.inputs[0].maxout_conf self.set_layer_size(g_layer_map[input_layer.name].size / maxout_conf.groups) - + # key: cost type # value: cost class g_cost_map = {} diff --git a/python/paddle/trainer_config_helpers/activations.py b/python/paddle/trainer_config_helpers/activations.py index ad5cdc0a0eb13..29b5437446d78 100644 --- a/python/paddle/trainer_config_helpers/activations.py +++ b/python/paddle/trainer_config_helpers/activations.py @@ -23,9 +23,9 @@ class BaseActivation(object): """ - A mark for activation class. + A mark for activation class. Each activation inherit BaseActivation, which has two parameters. - + :param name: activation name in paddle config. :type name: basestring :param support_hppl: True if supported by hppl. HPPL is a library used by paddle @@ -104,7 +104,7 @@ class IdentityActivation(BaseActivation): Just do nothing for output both forward/backward. """ - def __init__(self): BaseActivation.__init__(self, '', False) + def __init__(self): BaseActivation.__init__(self, 'linear', False) LinearActivation = IdentityActivation @@ -194,7 +194,7 @@ def __init__(self): BaseActivation.__init__(self, 'square', False) class ExpActivation(BaseActivation): """ Exponential Activation. - + .. math:: f(z) = e^z. """ diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 6b5d39a47158b..bf1d0631aa46a 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1657,7 +1657,7 @@ def img_pool_layer(input, pool_size, name=None, :type pool_size_y: int|None :param num_channels: number of input channel. :type num_channels: int - :param pool_type: pooling type. MaxPooling or AveragePooling. Default is + :param pool_type: pooling type. MaxPooling or AvgPooling. Default is MaxPooling. :type pool_type: BasePoolingType :param stride: stride width of pooling. From f27ff4d8a4e1370ceff5d3082bef8b712b1b23cb Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 10 Nov 2016 11:06:02 +0800 Subject: [PATCH 283/324] Revise code --- paddle/gserver/tests/test_LayerGrad.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index c1f7876c1722f..55a6f66ac5171 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -187,15 +187,13 @@ TEST(Layer, BilinearInterpLayer) { bilinear->set_img_size_y(32); bilinear->set_num_channels(4); - bilinear->set_out_size_x(32); - bilinear->set_out_size_y(32); - testLayerGrad(config, "bilinear_interp", 10, false, false); - testLayerGrad(config, "bilinear_interp", 10, false, true); - - bilinear->set_out_size_x(64); - bilinear->set_out_size_y(64); - testLayerGrad(config, "bilinear_interp", 10, false, false); - testLayerGrad(config, "bilinear_interp", 10, false, true); + for (auto useGpu : {false, true}) { + for (auto outSize : {32, 64}) { + bilinear->set_out_size_x(outSize); + bilinear->set_out_size_y(outSize); + testLayerGrad(config, "bilinear_interp", 10, false, useGpu); + } + } } TEST(Layer, concat) { From a45e6c95bd66e32a2d3a6e89172f1d198f2a9a81 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 10 Nov 2016 07:20:54 +0000 Subject: [PATCH 284/324] use yapf to format python code, add style config file --- .style.yapf | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .style.yapf diff --git a/.style.yapf b/.style.yapf new file mode 100644 index 0000000000000..c9cdc2e790a40 --- /dev/null +++ b/.style.yapf @@ -0,0 +1,5 @@ +[style] +based_on_style = google +indent_width = 4 +spaces_before_comment = 4 +split_before_logical_operator = true From 64b7561ce68440b168ba7bc20471cd41a2c63f58 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 10 Nov 2016 22:37:47 +0800 Subject: [PATCH 285/324] Add checkout name for Dockerfile * Because in dockerhub, we cannot set the `docker build `running directory, we could only use `git clone` command to get the latest code if we put `Dockerfile` in subdirectory * But the `git clone` will checkout the default branch only, so here we add a `ENV` in Dockerfile to checkout special branch or tag in git repo. We could change it to `V0.9.0` tag when it release. --- paddle/scripts/docker/Dockerfile.cpu | 1 + paddle/scripts/docker/Dockerfile.cpu-demo | 1 + paddle/scripts/docker/Dockerfile.cpu-devel | 1 + paddle/scripts/docker/Dockerfile.cpu-noavx | 1 + paddle/scripts/docker/Dockerfile.cpu-noavx-demo | 1 + paddle/scripts/docker/Dockerfile.cpu-noavx-devel | 1 + paddle/scripts/docker/Dockerfile.gpu | 1 + paddle/scripts/docker/Dockerfile.gpu-demo | 1 + paddle/scripts/docker/Dockerfile.gpu-devel | 1 + paddle/scripts/docker/Dockerfile.gpu-noavx | 1 + paddle/scripts/docker/Dockerfile.gpu-noavx-demo | 1 + paddle/scripts/docker/Dockerfile.gpu-noavx-devel | 1 + paddle/scripts/docker/Dockerfile.m4 | 1 + paddle/scripts/docker/build.sh | 1 + 14 files changed, 14 insertions(+) diff --git a/paddle/scripts/docker/Dockerfile.cpu b/paddle/scripts/docker/Dockerfile.cpu index 3aa8cb1a3a869..a833c69c66900 100644 --- a/paddle/scripts/docker/Dockerfile.cpu +++ b/paddle/scripts/docker/Dockerfile.cpu @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.cpu-demo b/paddle/scripts/docker/Dockerfile.cpu-demo index 22c0b9e701bfc..1fda1e472b290 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-demo +++ b/paddle/scripts/docker/Dockerfile.cpu-demo @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.cpu-devel b/paddle/scripts/docker/Dockerfile.cpu-devel index b40f3c0a30ba3..66bdc978ddcb4 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-devel +++ b/paddle/scripts/docker/Dockerfile.cpu-devel @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx b/paddle/scripts/docker/Dockerfile.cpu-noavx index 5cb5ac7dc4e68..d0ba30e55afb2 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx-demo b/paddle/scripts/docker/Dockerfile.cpu-noavx-demo index bec401960efb2..28439b4bdfab4 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx-demo +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx-demo @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx-devel b/paddle/scripts/docker/Dockerfile.cpu-noavx-devel index b7c3eaed97aa5..eb4739d6dc742 100644 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx-devel +++ b/paddle/scripts/docker/Dockerfile.cpu-noavx-devel @@ -1,6 +1,7 @@ FROM ubuntu:14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=OFF ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu b/paddle/scripts/docker/Dockerfile.gpu index b7f5b6d93df50..fa61cfeec851f 100644 --- a/paddle/scripts/docker/Dockerfile.gpu +++ b/paddle/scripts/docker/Dockerfile.gpu @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu-demo b/paddle/scripts/docker/Dockerfile.gpu-demo index 2d1411de09f2a..4f5417c1af072 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-demo +++ b/paddle/scripts/docker/Dockerfile.gpu-demo @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.gpu-devel b/paddle/scripts/docker/Dockerfile.gpu-devel index eb13f4304fa06..37cfced190886 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-devel +++ b/paddle/scripts/docker/Dockerfile.gpu-devel @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx b/paddle/scripts/docker/Dockerfile.gpu-noavx index 0944b0e152af3..95fb125b799e8 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=OFF ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx-demo b/paddle/scripts/docker/Dockerfile.gpu-noavx-demo index 2da2a55d696a3..b5fbe4b941d68 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx-demo +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx-demo @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=ON diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx-devel b/paddle/scripts/docker/Dockerfile.gpu-noavx-devel index 9f551462f206a..531c8ec7ae30c 100644 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx-devel +++ b/paddle/scripts/docker/Dockerfile.gpu-noavx-devel @@ -1,6 +1,7 @@ FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=ON ENV IS_DEVEL=ON ENV WITH_DEMO=OFF diff --git a/paddle/scripts/docker/Dockerfile.m4 b/paddle/scripts/docker/Dockerfile.m4 index 129d21b36abd9..57c8655844133 100644 --- a/paddle/scripts/docker/Dockerfile.m4 +++ b/paddle/scripts/docker/Dockerfile.m4 @@ -1,6 +1,7 @@ FROM PADDLE_BASE_IMAGE MAINTAINER PaddlePaddle Dev Team COPY build.sh /root/ +ENV GIT_CHECKOUT=develop ENV WITH_GPU=PADDLE_WITH_GPU ENV IS_DEVEL=PADDLE_IS_DEVEL ENV WITH_DEMO=PADDLE_WITH_DEMO diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 33689e736cda7..ec5f3bd967d35 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -23,6 +23,7 @@ fi cd ~ git clone https://github.com/baidu/Paddle.git paddle cd paddle +git checkout ${GIT_CHECKOUT} mkdir build cd build cmake .. -DWITH_DOC=OFF -DWITH_GPU=${WITH_GPU} -DWITH_SWIG_PY=ON\ From 36fa251756f583b9c40f067717990fff062bf9ae Mon Sep 17 00:00:00 2001 From: xuwei06 Date: Thu, 10 Nov 2016 10:07:13 -0800 Subject: [PATCH 286/324] '*' operator overload for LayerOutput Making '*' support the multiplication between a scalar and LayerOutput Also changing '+' to support adding between a vector and a scalar. Change-Id: I7daf35590dc2b2f855a29d9ef43ac57979442e0f --- doc/ui/api/trainer_config_helpers/layers.rst | 6 + python/paddle/trainer/config_parser.py | 2 +- .../paddle/trainer_config_helpers/__init__.py | 3 + .../paddle/trainer_config_helpers/layers.py | 146 +++++++++++++----- python/paddle/trainer_config_helpers/math.py | 43 +++++- .../tests/configs/math_ops.py | 8 +- .../tests/configs/protostr/math_ops.protostr | 135 +++++++++++++++- 7 files changed, 294 insertions(+), 49 deletions(-) diff --git a/doc/ui/api/trainer_config_helpers/layers.rst b/doc/ui/api/trainer_config_helpers/layers.rst index c78682423e448..a98e8f2f55c58 100644 --- a/doc/ui/api/trainer_config_helpers/layers.rst +++ b/doc/ui/api/trainer_config_helpers/layers.rst @@ -254,6 +254,12 @@ expand_layer :members: expand_layer :noindex: +repeat_layer +------------ +.. automodule:: paddle.trainer_config_helpers.layers + :members: repeat_layer + :noindex: + Math Layers =========== diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index c55579c960ecc..06ef35544590a 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -3015,7 +3015,7 @@ def Layer( layer_func = layers.get(type) config_assert(layer_func, "layer type '%s' not supported." % type) - layer_func(name, **xargs) + return layer_func(name, **xargs) @config_func def ParameterHook( diff --git a/python/paddle/trainer_config_helpers/__init__.py b/python/paddle/trainer_config_helpers/__init__.py index 451b9ac3396ea..adebebba2523f 100644 --- a/python/paddle/trainer_config_helpers/__init__.py +++ b/python/paddle/trainer_config_helpers/__init__.py @@ -20,3 +20,6 @@ from networks import * from optimizers import * from attrs import * + +# This will enable operator overload for LayerOutput +import math diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 59822180883c9..bd8e9f07b6aeb 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -31,6 +31,7 @@ __all__ = ["full_matrix_projection", "AggregateLevel", "ExpandLevel", "identity_projection", "dotmul_projection", "dotmul_operator", + "repeat_layer", "table_projection", "mixed_layer", "data_layer", "embedding_layer", "fc_layer", "grumemory", "pooling_layer", "lstmemory", "last_seq", "first_seq", @@ -99,6 +100,7 @@ class LayerType(object): SCALING_LAYER = 'scaling' TRANS_LAYER = 'trans' OUT_PROD_LAYER = 'out_prod' + FEATURE_MAP_EXPAND_LAYER = 'featmap_expand' MEMORY = 'memory' MAXID_LAYER = 'maxid' @@ -181,6 +183,7 @@ def __init__(self, name, layer_type, parents=None, activation=None, reverse=None): assert isinstance(name, basestring) assert isinstance(layer_type, basestring) + assert size is not None assert LayerType.is_layer_type(layer_type) self.name = name self.layer_type = layer_type @@ -1209,6 +1212,48 @@ def expand_layer(input, expand_as, parents=[input, expand_as]) +@wrap_name_default() +@layer_support() +def repeat_layer(input, num_repeats, + name=None, + layer_attr=None): + """ + A layer for repeating the input for num_repeats times. This is equivalent + to apply concat_layer() with num_repeats same input. + + .. math:: + y = [x, x, \cdots, x] + + The example usage is: + + .. code-block:: python + + expand = repeat_layer(layer, 4) + + :param input: Input layer + :type input: LayerOutput + :param num_repeats: Repeat the input so many times + :type num_repeats: int + :param name: Layer name. + :type name: basestring + :param layer_attr: extra layer attributes. + :type layer_attr: ExtraLayerAttribute. + :return: LayerOutput object. + :rtype: LayerOutput + """ + + l = Layer( + inputs=[input.name], + name=name, + num_filters=num_repeats, + type=LayerType.FEATURE_MAP_EXPAND_LAYER, + **ExtraAttr.to_kwargs(layer_attr) + ) + return LayerOutput(name=name, + size=l.config.size, + layer_type=LayerType.FEATURE_MAP_EXPAND_LAYER, + parents=[input]) + @wrap_name_default() @layer_support() def interpolation_layer(input, weight, name=None, layer_attr=None): @@ -1296,7 +1341,7 @@ def bilinear_interp_layer(input, assert out_size_x > 0 and out_size_y > 0 assert input.num_filters is not None num_channels = input.num_filters - Layer(name=name, + l = Layer(name=name, inputs=Input(input.name, bilinear_interp=BilinearInterp(out_size_x=out_size_x, out_size_y=out_size_y, @@ -1304,7 +1349,7 @@ def bilinear_interp_layer(input, type=LayerType.BILINEAR_INTERP_LAYER, **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.BILINEAR_INTERP_LAYER, parents=[input], - num_filters=num_channels) + num_filters=num_channels, size=l.config.size) @wrap_name_default() @layer_support() @@ -1482,7 +1527,7 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): inputs=[a.name, b.name], **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b]) + return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b], size=size) @wrap_name_default() @@ -1545,7 +1590,7 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, ipts_for_layer.append(label.name) parents.append(label) - Layer( + l = Layer( name=name, type=LayerType.HSIGMOID, num_classes=num_classes, @@ -1553,7 +1598,8 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, inputs=ipts_for_layer, **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.HSIGMOID, parents=parents) + return LayerOutput(name, LayerType.HSIGMOID, parents=parents, + size=l.config.size) @wrap_name_default("conv") @@ -1671,7 +1717,7 @@ def img_conv_layer(input, filter_size, num_filters, lt = LayerType.CONVTRANS_LAYER if trans else LayerType.CONV_LAYER - Layer( + l = Layer( name=name, inputs=Input(input.name, conv=Conv( filter_size=filter_size, padding=padding, stride=stride, @@ -1687,7 +1733,8 @@ def img_conv_layer(input, filter_size, num_filters, **ExtraLayerAttribute.to_kwargs(layer_attr) ) return LayerOutput(name, lt, parents=[input], - activation=act, num_filters=num_filters) + activation=act, num_filters=num_filters, + size=l.config.size) @wrap_name_default("pool") @@ -1750,7 +1797,7 @@ def img_pool_layer(input, pool_size, name=None, stride_y = stride if stride_y is None else stride_y padding_y = padding if padding_y is None else padding_y - Layer( + l = Layer( name=name, type=LayerType.POOL_LAYER, inputs=[Input(input.name, @@ -1769,7 +1816,7 @@ def img_pool_layer(input, pool_size, name=None, **ExtraLayerAttribute.to_kwargs(layer_attr) ) return LayerOutput(name, LayerType.POOL_LAYER, parents=[input], - num_filters=num_channels) + num_filters=num_channels, size=l.config.size) def __img_norm_layer__(name, input, size, norm_type, scale, power, @@ -1778,7 +1825,7 @@ def __img_norm_layer__(name, input, size, norm_type, scale, power, assert input.num_filters is not None num_channels = input.num_filters - Layer( + l = Layer( name=name, type=LayerType.NORM_LAYER, inputs=Input( input.name, norm=Norm(norm_type=norm_type, channels=num_channels, size=size, @@ -1788,7 +1835,8 @@ def __img_norm_layer__(name, input, size, norm_type, scale, power, **ExtraLayerAttribute.to_kwargs(layer_attr) ) return LayerOutput(name, layer_type=LayerType.NORM_LAYER, parents=[input], - num_filters=num_channels, img_norm_type=norm_type) + num_filters=num_channels, img_norm_type=norm_type, + size=l.config.size) @wrap_name_default("crmnorm") @@ -1913,7 +1961,7 @@ def batch_norm_layer(input, act=None, name=None, num_channels=None, num_channels = input.size assert (batch_norm_type is None) or (batch_norm_type == "batch_norm") or \ (batch_norm_type == "cudnn_batch_norm") - Layer( + l = Layer( name=name, inputs=Input(input.name, image=Image(channels=num_channels), @@ -1929,7 +1977,8 @@ def batch_norm_layer(input, act=None, name=None, num_channels=None, return LayerOutput(name=name, layer_type=LayerType.BATCH_NORM_LAYER, parents=[input], activation=act, - num_filters=num_channels) + num_filters=num_channels, + size=l.config.size) @wrap_name_default() @@ -2034,7 +2083,7 @@ def addto_layer(input, act=None, name=None, bias_attr=None, if each_input.num_filters is not None: num_filters = each_input.num_filters - Layer( + l = Layer( name=name, type=LayerType.ADDTO_LAYER, inputs=ipts_for_layer, bias=ParamAttr.to_bias(bias_attr), active_type=act.name, @@ -2042,7 +2091,8 @@ def addto_layer(input, act=None, name=None, bias_attr=None, ) return LayerOutput(name, LayerType.ADDTO_LAYER, parents=input, - activation=act, num_filters=num_filters) + activation=act, num_filters=num_filters, + size=l.config.size) @wrap_act_default(act=IdentityActivation()) @@ -2651,13 +2701,14 @@ def maxid_layer(input, name=None, layer_attr=None): """ assert isinstance(input, LayerOutput) - Layer(name=name, + l = Layer(name=name, type='maxid', inputs=[input.name], **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name=name, layer_type=LayerType.MAXID_LAYER, - parents=[input]) + parents=[input], + size=l.config.size) @wrap_name_default() @@ -2686,13 +2737,14 @@ def out_prod_layer(input1, input2, name=None, layer_attr=None): assert isinstance(input1, LayerOutput) assert isinstance(input2, LayerOutput) - Layer(name=name, + l = Layer(name=name, type=LayerType.OUT_PROD_LAYER, inputs=[input1.name, input2.name], **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name=name, layer_type=LayerType.OUT_PROD_LAYER, - parents=[input1, input2]) + parents=[input1, input2], + size=l.config.size) @wrap_name_default() @@ -2721,13 +2773,14 @@ def eos_layer(input, eos_id, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - Layer(name=name, + l = Layer(name=name, type=LayerType.EOSID_LAYER, eos_id=eos_id, inputs=[input.name], **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name=name, layer_type=LayerType.EOSID_LAYER, - parents=[input]) + parents=[input], + size=l.config.size) @wrap_name_default() @@ -2892,7 +2945,7 @@ def regression_cost(input, label, weight=None, name=None, Layer(inputs=ipts, type="square_error", name=name, **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.COST, parents=parents) + return LayerOutput(name, LayerType.COST, parents=parents, size=1) @wrap_name_default("cost") @@ -2944,7 +2997,7 @@ def __add_evaluator__(e): for each_evaluator in evaluator: __add_evaluator__(each_evaluator) - return LayerOutput(name, LayerType.COST, parents=parents) + return LayerOutput(name, LayerType.COST, parents=parents, size=1) def conv_operator(img, filter, filter_size, num_filters, @@ -3326,13 +3379,14 @@ def sampling_id_layer(input, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - Layer( + l = Layer( name=name, type=LayerType.SAMPLING_ID_LAYER, inputs=[Input(input.name)], **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.SAMPLING_ID_LAYER, input) + return LayerOutput(name, LayerType.SAMPLING_ID_LAYER, input, + size=l.config.size) @wrap_name_default() @@ -3373,7 +3427,8 @@ def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0, inputs=[Input(input.name)], **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.SLOPE_INTERCEPT_LAYER, input) + return LayerOutput(name, LayerType.SLOPE_INTERCEPT_LAYER, input, + size=input.size) @wrap_name_default() @@ -3512,7 +3567,7 @@ def block_expand_layer(input, if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters - Layer(name=name, + l = Layer(name=name, inputs=Input(input.name, block_expand=BlockExpand(channels=num_channels, block_x=block_x, @@ -3525,7 +3580,8 @@ def block_expand_layer(input, **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.BLOCK_EXPAND, parents=[input]) + return LayerOutput(name, LayerType.BLOCK_EXPAND, parents=[input], + size=l.config.size) @wrap_name_default() @@ -3586,13 +3642,14 @@ def maxout_layer(input, assert input.num_filters is not None num_channels = input.num_filters assert num_channels % groups == 0 - Layer(name=name, + l = Layer(name=name, inputs=Input(input.name, maxout=MaxOut(channels=num_channels, groups=groups)), type=LayerType.MAXOUT, **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.MAXOUT, parents=[input]) + return LayerOutput(name, LayerType.MAXOUT, parents=[input], + size=l.config.size) @wrap_name_default() @@ -3718,7 +3775,10 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, parents = [input, label] if weight is not None: parents.append(weight) - return LayerOutput(name, LayerType.CRF_LAYER, parents, size=size) + # The size for LayerOutput means the dimension of the output. + # It's different from the meaning of crf layer, which is the number of + # classes. + return LayerOutput(name, LayerType.CRF_LAYER, parents, size=1) @wrap_name_default() @@ -3766,7 +3826,10 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None, parents = [input] if label is not None: parents.append(label) - return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=size) + # The size for LayerOutput means the dimension of the output. + # It's different from the meaning of crf layer, which is the number of + # classes. + return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=1) @wrap_bias_attr_default(has_bias=True) @wrap_name_default() @@ -3834,7 +3897,7 @@ def nce_layer(input, label, num_classes, weight=None, ipts_for_layer.append(weight.name) parents.append(weight) - Layer( + l = Layer( name=name, type=LayerType.NCE_LAYER, num_classes=num_classes, @@ -3844,7 +3907,8 @@ def nce_layer(input, label, num_classes, weight=None, bias=ParamAttr.to_bias(bias_attr), **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.NCE_LAYER, parents=parents) + return LayerOutput(name, LayerType.NCE_LAYER, parents=parents, + size=l.config.size) """ following are cost Layers. @@ -3919,7 +3983,7 @@ def rank_cost(left, right, label, weight=None, name=None, coeff=1.0, layer_attr= **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.RANK_COST, parents=parents) + return LayerOutput(name, LayerType.RANK_COST, parents=parents, size=1) @wrap_name_default() @@ -3971,7 +4035,8 @@ def lambda_cost(input, score, name, NDCG_num=5, max_sort_size=-1, layer_attr=Non **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.LAMBDA_COST, parents=[input, score]) + return LayerOutput(name, LayerType.LAMBDA_COST, parents=[input, score], + size=1) @wrap_name_default() @@ -4006,7 +4071,8 @@ def cross_entropy(input, label, name=None, coeff=1.0, layer_attr=None): coeff=coeff, **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.CROSS_ENTROPY, parents=[input, label]) + return LayerOutput(name, LayerType.CROSS_ENTROPY, parents=[input, label], + size=1) @wrap_name_default() @@ -4048,7 +4114,7 @@ def cross_entropy_with_selfnorm(input, label, name=None, coeff=1.0, return LayerOutput(name, LayerType.CROSS_ENTROPY_WITH_SELFNORM, - parents=[input, label]) + parents=[input, label], size=1) @wrap_name_default() @@ -4083,7 +4149,7 @@ def huber_cost(input, label, name=None, coeff=1.0, layer_attr=None): coeff=coeff, **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.HUBER, parents=[input, label]) + return LayerOutput(name, LayerType.HUBER, parents=[input, label], size=1) @wrap_name_default() @@ -4126,4 +4192,4 @@ def multi_binary_label_cross_entropy(input, label, name=None, coeff=1.0, **ExtraLayerAttribute.to_kwargs(layer_attr) ) return LayerOutput(name, LayerType.MULTI_BIN_LABEL_CROSS_ENTROPY, - parents=[input, label]) + parents=[input, label], size=1) diff --git a/python/paddle/trainer_config_helpers/math.py b/python/paddle/trainer_config_helpers/math.py index e35849b77ac53..7d7bb2914859f 100644 --- a/python/paddle/trainer_config_helpers/math.py +++ b/python/paddle/trainer_config_helpers/math.py @@ -13,10 +13,11 @@ # limitations under the License. from .layers import LayerOutput, mixed_layer, identity_projection, \ - slope_intercept_layer + slope_intercept_layer, scaling_layer, repeat_layer from .attrs import is_compatible_with from .default_decorators import * import activations as act +from paddle.trainer.config_parser import logger __all__ = [] @@ -40,7 +41,21 @@ def op(input, name=None): def add(layeroutput, other): if is_compatible_with(other, float): return slope_intercept_layer(input=layeroutput, intercept=other) - assert isinstance(other, LayerOutput) + if not isinstance(other, LayerOutput): + logger.fatal("LayerOutput can only be added with" + " another LayerOutput or a number") + if layeroutput.size == other.size: + return mixed_layer(input=[identity_projection(input=layeroutput), + identity_projection(input=other)]) + if other.size != 1 and layeroutput.size != 1: + logger.fatal("Two LayerOutput can be added only if they have equal size" + " or one of their sizes is 1. sizes are %s and %s" % + (layeroutput.size, other.size)) + elif layeroutput.size == 1: + tmp = layeroutput + layeroutput = other + other = tmp + other = repeat_layer(other, layeroutput.size) return mixed_layer(input=[identity_projection(input=layeroutput), identity_projection(input=other)]) @@ -50,10 +65,11 @@ def add(layeroutput, other): def sub(layeroutput, other): if is_compatible_with(other, float): return slope_intercept_layer(input=layeroutput, intercept=other) - assert isinstance(other, LayerOutput) + if not isinstance(other, LayerOutput): + logger.fatal("LayerOutput can only be subtracted with" + " another Layeroutput or a number") neg = slope_intercept_layer(input=other, slope=-1.0) - return mixed_layer(input=[identity_projection(input=layeroutput), - identity_projection(input=neg)]) + return add(layeroutput, neg) LayerOutput.__sub__ = sub @@ -62,3 +78,20 @@ def rsub(layeroutput, other): return add(neg, other) LayerOutput.__rsub__ = rsub + +def mul(layeroutput, other): + if is_compatible_with(other, float): + return slope_intercept_layer(input=layeroutput, slope=other) + if not isinstance(other, LayerOutput): + logger.fatal("LayerOutput can only be multiplied with" + " another Layeroutput or a number") + elif layeroutput.size == 1: + return scaling_layer(input=other, weight=layeroutput) + elif other.size == 1: + return scaling_layer(input=layeroutput, weight=other) + else: + logger.fatal("At least one of the operand of '*' must be a number" + " or a LayerOutput with size=1") + +LayerOutput.__mul__ = mul +LayerOutput.__rmul__ = mul diff --git a/python/paddle/trainer_config_helpers/tests/configs/math_ops.py b/python/paddle/trainer_config_helpers/tests/configs/math_ops.py index fe515b7029336..7c2770c616dc1 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/math_ops.py +++ b/python/paddle/trainer_config_helpers/tests/configs/math_ops.py @@ -19,6 +19,12 @@ y = y - x y = y - 2 y = 2 - y - +y = 2 * y +y = y * 3 +z= data_layer(name='data_2', size=1) +y = y * z +y = z * y +y = y + z +y = z + y outputs(y) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr index 1767445c44bf5..da8da1b541f37 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/math_ops.protostr @@ -209,8 +209,129 @@ layers { slope: 1.0 intercept: 2 } +layers { + name: "__slope_intercept_layer_6__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "__slope_intercept_layer_5__" + } + slope: 2 + intercept: 0.0 +} +layers { + name: "__slope_intercept_layer_7__" + type: "slope_intercept" + size: 100 + active_type: "" + inputs { + input_layer_name: "__slope_intercept_layer_6__" + } + slope: 3 + intercept: 0.0 +} +layers { + name: "data_2" + type: "data" + size: 1 + active_type: "" +} +layers { + name: "__scaling_layer_0__" + type: "scaling" + size: 100 + active_type: "" + inputs { + input_layer_name: "data_2" + } + inputs { + input_layer_name: "__slope_intercept_layer_7__" + } +} +layers { + name: "__scaling_layer_1__" + type: "scaling" + size: 100 + active_type: "" + inputs { + input_layer_name: "data_2" + } + inputs { + input_layer_name: "__scaling_layer_0__" + } +} +layers { + name: "__repeat_layer_0__" + type: "featmap_expand" + size: 100 + active_type: "" + inputs { + input_layer_name: "data_2" + } + num_filters: 100 +} +layers { + name: "__mixed_2__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__scaling_layer_1__" + proj_conf { + type: "identity" + name: "___mixed_2__.w0" + input_size: 100 + output_size: 100 + } + } + inputs { + input_layer_name: "__repeat_layer_0__" + proj_conf { + type: "identity" + name: "___mixed_2__.w1" + input_size: 100 + output_size: 100 + } + } +} +layers { + name: "__repeat_layer_1__" + type: "featmap_expand" + size: 100 + active_type: "" + inputs { + input_layer_name: "data_2" + } + num_filters: 100 +} +layers { + name: "__mixed_3__" + type: "mixed" + size: 100 + active_type: "" + inputs { + input_layer_name: "__mixed_2__" + proj_conf { + type: "identity" + name: "___mixed_3__.w0" + input_size: 100 + output_size: 100 + } + } + inputs { + input_layer_name: "__repeat_layer_1__" + proj_conf { + type: "identity" + name: "___mixed_3__.w1" + input_size: 100 + output_size: 100 + } + } +} +input_layer_names: "data_2" input_layer_names: "data" -output_layer_names: "__slope_intercept_layer_5__" +output_layer_names: "__mixed_3__" sub_models { name: "root" layer_names: "data" @@ -228,8 +349,18 @@ sub_models { layer_names: "__slope_intercept_layer_3__" layer_names: "__slope_intercept_layer_4__" layer_names: "__slope_intercept_layer_5__" + layer_names: "__slope_intercept_layer_6__" + layer_names: "__slope_intercept_layer_7__" + layer_names: "data_2" + layer_names: "__scaling_layer_0__" + layer_names: "__scaling_layer_1__" + layer_names: "__repeat_layer_0__" + layer_names: "__mixed_2__" + layer_names: "__repeat_layer_1__" + layer_names: "__mixed_3__" + input_layer_names: "data_2" input_layer_names: "data" - output_layer_names: "__slope_intercept_layer_5__" + output_layer_names: "__mixed_3__" is_recurrent_layer_group: false } From 45f6e1abee60f0bb14c84b455ffe47a903c3c806 Mon Sep 17 00:00:00 2001 From: Haonan Date: Thu, 10 Nov 2016 16:29:23 -0800 Subject: [PATCH 287/324] change hlactivetype instead of act.name --- paddle/utils/Util.cpp | 2 +- python/paddle/trainer_config_helpers/activations.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/utils/Util.cpp b/paddle/utils/Util.cpp index 2cdff9d1aca92..b16d4314654ff 100644 --- a/paddle/utils/Util.cpp +++ b/paddle/utils/Util.cpp @@ -378,7 +378,7 @@ hl_activation_mode_t hlActiveType(const std::string& type) { return HL_ACTIVATION_RELU; } else if (type == "tanh") { return HL_ACTIVATION_TANH; - } else if (type == "linear") { + } else if (type == "linear" || type == "") { return HL_ACTIVATION_LINEAR; } else { LOG(FATAL) << "Do not support activation type " << type; diff --git a/python/paddle/trainer_config_helpers/activations.py b/python/paddle/trainer_config_helpers/activations.py index 29b5437446d78..2202d0bf96976 100644 --- a/python/paddle/trainer_config_helpers/activations.py +++ b/python/paddle/trainer_config_helpers/activations.py @@ -104,7 +104,7 @@ class IdentityActivation(BaseActivation): Just do nothing for output both forward/backward. """ - def __init__(self): BaseActivation.__init__(self, 'linear', False) + def __init__(self): BaseActivation.__init__(self, '', False) LinearActivation = IdentityActivation From aa560dbb97d110086b9bdbd361fed8db95d93a89 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 11 Nov 2016 12:10:10 +0800 Subject: [PATCH 288/324] fix bug in sum_cost --- python/paddle/trainer_config_helpers/layers.py | 18 ++++++++++++------ .../tests/configs/test_cost_layers.py | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 10737c90ccfa3..92e09b51eb652 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -4048,7 +4048,8 @@ def cross_entropy(input, label, name=None, coeff=1.0, layer_attr=None): .. code-block:: python - cost = cross_entropy(input, label) + cost = cross_entropy(input=input_layer, + label=label_layer) :param input: The first input layer. :type input: LayerOutput. @@ -4084,7 +4085,8 @@ def cross_entropy_with_selfnorm(input, label, name=None, coeff=1.0, .. code-block:: python - cost = cross_entropy_with_selfnorm(input, label) + cost = cross_entropy_with_selfnorm(input=input_layer, + label=label_layer) :param input: The first input layer. :type input: LayerOutput. @@ -4122,7 +4124,7 @@ def sum_cost(input, name=None, layer_attr=None): .. code-block:: python - cost = sum_cost(input) + cost = sum_cost(input=input_layer) :param input: The first input layer. :type input: LayerOutput. @@ -4133,6 +4135,7 @@ def sum_cost(input, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput. """ + assert isinstance(input, LayerOutput) Layer(name=name, type=LayerType.SUM_COST, inputs=[input.name], @@ -4141,7 +4144,8 @@ def sum_cost(input, name=None, layer_attr=None): return LayerOutput(name, LayerType.SUM_COST, - parents=[input]) + parents=[input], + size=1) @wrap_name_default() @@ -4152,7 +4156,8 @@ def huber_cost(input, label, name=None, coeff=1.0, layer_attr=None): .. code-block:: python - cost = huber_cost(input, label) + cost = huber_cost(input=input_layer, + label=label_layer) :param input: The first input layer. :type input: LayerOutput. @@ -4188,7 +4193,8 @@ def multi_binary_label_cross_entropy(input, label, name=None, coeff=1.0, .. code-block:: python - cost = multi_binary_label_cross_entropy(input, label) + cost = multi_binary_label_cross_entropy(input=input_layer, + label=label_layer) :param input: The first input layer. :type input: LayerOutput diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py index f1b3365f84e3e..cfaf2da001106 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_cost_layers.py @@ -25,4 +25,4 @@ huber_cost(input=data_layer(name='huber_probs', size=1), label=data_layer(name='huber_label', size=1)), multi_binary_label_cross_entropy(input=probs, label=xe_label), - sum_cost(hidden)) + sum_cost(input=hidden)) From 62c5389bad0191c8574d7be58fc1564567b913ef Mon Sep 17 00:00:00 2001 From: qijun Date: Fri, 11 Nov 2016 10:50:24 +0000 Subject: [PATCH 289/324] fix test_layerHelpers unittest error --- python/paddle/trainer_config_helpers/layers.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 82c57e7f90ad5..a0a367f2d50df 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -1827,7 +1827,6 @@ def img_pool_layer(input, pool_size, name=None, @layer_support() def spp_layer(input, name=None, num_channels=None, pool_type=None, pyramid_height=None, img_width=None, layer_attr=None): - pass """ Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition. The details please refer to @@ -1864,7 +1863,7 @@ def spp_layer(input, name=None, num_channels=None, pool_type=None, if (isinstance(pool_type, AvgPooling) or isinstance(pool_type, MaxPooling)): type_name += '-projection' - Layer( + l = Layer( name=name, type=LayerType.SPP_LAYER, inputs=Input(input.name, @@ -1875,8 +1874,8 @@ def spp_layer(input, name=None, num_channels=None, pool_type=None, ), **ExtraLayerAttribute.to_kwargs(layer_attr) ) - return LayerOutput(name, LayerType.SPP_LAYER, parents=[input], - num_filters=num_channels) + return LayerOutput(name, layer_type=LayerType.SPP_LAYER, parents=[input], + num_filters=num_channels, size=l.config.size) def __img_norm_layer__(name, input, size, norm_type, scale, power, From b3bcc52f8f3f45bb459afcfdcd624a8817351b71 Mon Sep 17 00:00:00 2001 From: qijun Date: Fri, 11 Nov 2016 11:04:54 +0000 Subject: [PATCH 290/324] change python code style to pep8 --- .style.yapf | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.style.yapf b/.style.yapf index c9cdc2e790a40..4741fb4f3bbc6 100644 --- a/.style.yapf +++ b/.style.yapf @@ -1,5 +1,3 @@ [style] -based_on_style = google -indent_width = 4 -spaces_before_comment = 4 -split_before_logical_operator = true +based_on_style = pep8 +column_limit = 80 From 33b81648a37c282f6128548ae7eea47faf77b6d7 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 11 Nov 2016 19:11:36 +0800 Subject: [PATCH 291/324] Fix bug in multple objects in define_py_sources --- python/paddle/trainer_config_helpers/data_sources.py | 2 +- .../tests/configs/generate_protostr.sh | 3 ++- .../tests/configs/test_split_datasource.py | 12 ++++++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_split_datasource.py diff --git a/python/paddle/trainer_config_helpers/data_sources.py b/python/paddle/trainer_config_helpers/data_sources.py index f51140656d0dc..283a45df30844 100644 --- a/python/paddle/trainer_config_helpers/data_sources.py +++ b/python/paddle/trainer_config_helpers/data_sources.py @@ -139,7 +139,7 @@ def __is_splitable__(o): test_obj = obj train_obj = obj if __is_splitable__(obj): - train_module, test_module = module + train_obj, test_obj = obj if args is None: args = "" diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index 9f614e3983ffa..cafc2142f25c7 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -11,7 +11,8 @@ test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers img_trans_layers util_layers simple_rnn_layers unused_layers test_cost_layers test_rnn_group shared_fc shared_lstm test_cost_layers_with_weight -test_bilinear_interp test_maxout test_bi_grumemory math_ops) +test_bilinear_interp test_maxout test_bi_grumemory math_ops +test_spilit_datasource) for conf in ${configs[*]} diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_split_datasource.py b/python/paddle/trainer_config_helpers/tests/configs/test_split_datasource.py new file mode 100644 index 0000000000000..c8dcb1bd8a47b --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_split_datasource.py @@ -0,0 +1,12 @@ +from paddle.trainer_config_helpers import * + +define_py_data_sources2(train_list="train.list", + test_list="test.list", + module=["a", "b"], + obj=("c", "d")) +settings( + learning_rate=1e-3, + batch_size=1000 +) + +outputs(data_layer(name="a", size=10)) From 4607d517bfba481cbb02b2beb1dfa3773eadfded Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 11 Nov 2016 19:31:28 +0800 Subject: [PATCH 292/324] Add unittest for split datasource * Fix #436 --- .../tests/configs/generate_protostr.sh | 10 ++- .../protostr/test_split_datasource.protostr | 72 +++++++++++++++++++ python/paddle/utils/dump_config.py | 10 ++- 3 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_split_datasource.protostr diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index e84e2a4b7f36a..bb594ac2c245d 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -11,12 +11,18 @@ test_sequence_pooling test_lstmemory_layer test_grumemory_layer last_first_seq test_expand_layer test_ntm_layers test_hsigmoid img_layers img_trans_layers util_layers simple_rnn_layers unused_layers test_cost_layers test_rnn_group shared_fc shared_lstm test_cost_layers_with_weight -test_spp_layer test_bilinear_interp test_maxout test_bi_grumemory math_ops -test_split_datasource) +test_spp_layer test_bilinear_interp test_maxout test_bi_grumemory math_ops) +whole_configs=(test_split_datasource) for conf in ${configs[*]} do echo "Generating " $conf python -m paddle.utils.dump_config $conf.py > $protostr/$conf.protostr.unitest done + +for conf in ${whole_configs[*]} +do + echo "Generating " $conf + python -m paddle.utils.dump_config $conf.py "" --whole > $protostr/$conf.protostr.unitest +done diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_split_datasource.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_split_datasource.protostr new file mode 100644 index 0000000000000..1cfb92255aa92 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_split_datasource.protostr @@ -0,0 +1,72 @@ +model_config { + type: "nn" + layers { + name: "a" + type: "data" + size: 10 + active_type: "" + } + input_layer_names: "a" + output_layer_names: "a" + sub_models { + name: "root" + layer_names: "a" + input_layer_names: "a" + output_layer_names: "a" + is_recurrent_layer_group: false + } +} +data_config { + type: "py2" + files: "train.list" + async_load_data: true + for_test: false + load_data_module: "a" + load_data_object: "c" + load_data_args: "" + data_ratio: 1 + is_main_data: true + usage_ratio: 1.0 +} +opt_config { + batch_size: 1000 + algorithm: "sgd" + learning_rate: 0.001 + learning_rate_decay_a: 0.0 + learning_rate_decay_b: 0.0 + l1weight: 0.1 + l2weight: 0.0 + c1: 0.0001 + backoff: 0.5 + owlqn_steps: 10 + max_backoff: 5 + l2weight_zero_iter: 0 + average_window: 0 + learning_method: "momentum" + ada_epsilon: 1e-06 + do_average_in_cpu: false + ada_rou: 0.95 + learning_rate_schedule: "poly" + delta_add_rate: 1.0 + shrink_parameter_value: 0 + adam_beta1: 0.9 + adam_beta2: 0.999 + adam_epsilon: 1e-08 + learning_rate_args: "" + async_lagged_grad_discard_ratio: 1.5 +} +test_data_config { + type: "py2" + files: "test.list" + async_load_data: true + for_test: true + load_data_module: "b" + load_data_object: "d" + load_data_args: "" + data_ratio: 1 + is_main_data: true + usage_ratio: 1.0 +} +save_dir: "./output/model" +start_pass: 0 + diff --git a/python/paddle/utils/dump_config.py b/python/paddle/utils/dump_config.py index d8a2722575d53..c5ce5c8d9a084 100644 --- a/python/paddle/utils/dump_config.py +++ b/python/paddle/utils/dump_config.py @@ -19,13 +19,21 @@ __all__ = [] if __name__ == '__main__': + whole_conf = False if len(sys.argv) == 2: conf = parse_config(sys.argv[1], '') elif len(sys.argv) == 3: conf = parse_config(sys.argv[1], sys.argv[2]) + elif len(sys.argv) == 4: + conf = parse_config(sys.argv[1], sys.argv[2]) + if sys.argv[3] == '--whole': + whole_conf = True else: raise RuntimeError() assert isinstance(conf, TrainerConfig_pb2.TrainerConfig) - print conf.model_config + if whole_conf: + print conf + else: + print conf.model_config From 069d0004dc1334987e40d151b6e8521ebc80f661 Mon Sep 17 00:00:00 2001 From: Haonan Date: Fri, 4 Nov 2016 11:46:07 -0700 Subject: [PATCH 293/324] multi_binary_cross_entropy when ids vector is provided --- paddle/cuda/include/hl_matrix.h | 30 +++++++++ paddle/cuda/include/stub/hl_matrix_stub.h | 12 ++++ paddle/cuda/src/hl_cuda_matrix.cu | 78 +++++++++++++++++++++++ paddle/gserver/layers/CostLayer.cpp | 4 ++ paddle/gserver/tests/test_LayerGrad.cpp | 7 +- paddle/math/Matrix.cpp | 36 +++++++++++ paddle/math/Matrix.h | 4 ++ paddle/math/tests/test_matrixCompare.cpp | 66 ++++++++++++++++++- paddle/parameter/Argument.cpp | 22 +++++++ paddle/parameter/Argument.h | 8 +++ 10 files changed, 263 insertions(+), 4 deletions(-) diff --git a/paddle/cuda/include/hl_matrix.h b/paddle/cuda/include/hl_matrix.h index 71e8f8e3a60c9..6195e30b9974d 100644 --- a/paddle/cuda/include/hl_matrix.h +++ b/paddle/cuda/include/hl_matrix.h @@ -126,6 +126,36 @@ extern void hl_matrix_cross_entropy_bp(real* grad_d, int dimM, int dimN); +/** + * @brief Matrix multi-binary label cross entropy + * + * @param[in] output input matrix (M x N). + * @param[out] entropy output matrix (M x 1). + * @param[in] mat input sparse matrix. + * @param[in] dimM matrix height. + * @param[in] dimN matrix width. + */ +extern void hl_matrix_multi_binary_cross_entropy(real* output, + real* entropy, + hl_sparse_matrix_s mat, + int dimM, + int dimN); + +/** + * @brief Matrix multi-binary label cross entropy backprop + * + * @param[in] output input matrix (M x N). + * @param[out] grad output matrix (M x N). + * @param[in] mat input sparse matrix. + * @param[in] dimM matrix height. + * @param[in] dimN matrix width. + */ +extern void hl_matrix_multi_binary_cross_entropy_bp(real* output, + real* grad, + hl_sparse_matrix_s mat, + int dimM, + int dimN); + /** * @brief Matrix zero memory. * diff --git a/paddle/cuda/include/stub/hl_matrix_stub.h b/paddle/cuda/include/stub/hl_matrix_stub.h index e37b1275432ca..76cac2e577693 100644 --- a/paddle/cuda/include/stub/hl_matrix_stub.h +++ b/paddle/cuda/include/stub/hl_matrix_stub.h @@ -57,6 +57,18 @@ inline void hl_matrix_cross_entropy_bp(real* grad_d, int dimM, int dimN) {} +inline void hl_matrix_multi_binary_cross_entropy(real* output, + real* entropy, + hl_sparse_matrix_s mat, + int dimM, + int dimN) {} + +inline void hl_matrix_multi_binary_cross_entropy_bp(real* output, + real* grad, + hl_sparse_matrix_s mat, + int dimM, + int dimN) {} + inline void hl_matrix_zero_mem(real* data, int num) {} inline void hl_param_relu_forward(real* output, diff --git a/paddle/cuda/src/hl_cuda_matrix.cu b/paddle/cuda/src/hl_cuda_matrix.cu index 3df9f63f9e4b7..001b62a6b94d6 100644 --- a/paddle/cuda/src/hl_cuda_matrix.cu +++ b/paddle/cuda/src/hl_cuda_matrix.cu @@ -18,6 +18,7 @@ limitations under the License. */ #include "hl_matrix_ops.cuh" #include "hl_matrix_apply.cuh" #include "hl_sequence.h" +#include "hl_sparse.ph" #include "paddle/utils/Logging.h" #include "hl_device_functions.cuh" #include "hl_gpu_matrix_kernel.cuh" @@ -317,6 +318,83 @@ void hl_matrix_classification_error(real* A_d, CHECK_SYNC("hl_matrix_classification_error"); } +__global__ void KeMatrixMultiBinaryCrossEntropy(real* output, + real* entropy, + int* row, + int* col, + int dimM, + int dimN) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < dimM) { + for (int i = 0; i < dimN; i ++) { + entropy[index] -= log(1 - output[index * dimN + i]); + } + int *row_col = col + row[index]; + int col_num = row[index + 1] - row[index]; + for (int i = 0; i < col_num; i ++) { + real o = output[index * dimN + row_col[i]]; + entropy[index] -= log(o / (1 - o)); + } + } +} + +void hl_matrix_multi_binary_cross_entropy(real* output, + real* entropy, + hl_sparse_matrix_s csr_mat, + int dimM, + int dimN) { + CHECK_NOTNULL(output); + CHECK_NOTNULL(entropy); + CHECK_NOTNULL(csr_mat); + int n_threads = 1024; + int blocks = (dimM + n_threads - 1) / n_threads; + dim3 threads(n_threads); + dim3 grid(blocks); + hl_csr_matrix mat = (hl_csr_matrix)(csr_mat->matrix); + KeMatrixMultiBinaryCrossEntropy<<< grid, threads, 0, STREAM_DEFAULT >>> + (output, entropy, mat->csr_row, mat->csr_col, dimM, dimN); + CHECK_SYNC("hl_matrix_multi_binary_cross_entropy failed"); +} + +__global__ void KeMatrixMultiBinaryCrossEntropyBp(real* output, + real* grad, + int* row, + int* col, + int dimM, + int dimN) { + int row_idx = blockIdx.x * blockDim.x + threadIdx.x; + if (row_idx < dimM) { + for (int i = 0; i < dimN; i ++) { + int index = row_idx * dimN + i; + grad[index] += 1.0 / (1 - output[index]); + } + int col_num = row[row_idx + 1] - row[row_idx]; + int *row_col = col + row[row_idx]; + for (int i = 0; i < col_num; i ++) { + int index = row_idx * dimN + row_col[i]; + grad[index] -= 1.0 / (output[index] * (1 - output[index])); + } + } +} + +void hl_matrix_multi_binary_cross_entropy_bp(real* output, + real* grad, + hl_sparse_matrix_s csr_mat, + int dimM, + int dimN) { + CHECK_NOTNULL(output); + CHECK_NOTNULL(grad); + CHECK_NOTNULL(csr_mat); + int n_threads = 1024; + int blocks = (dimM + n_threads - 1) / n_threads; + dim3 threads(n_threads); + dim3 grid(blocks); + hl_csr_matrix mat = (hl_csr_matrix)(csr_mat->matrix); + KeMatrixMultiBinaryCrossEntropyBp<<< grid, threads, 0, STREAM_DEFAULT >>> + (output, grad, mat->csr_row, mat->csr_col, dimM, dimN); + CHECK_SYNC("hl_matrix_multi_binary_cross_entropy_bp failed"); +} + __global__ void KeMatrixCrossEntropy(real* O, real* E, int* label, diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index 949788be49787..c86e562d0e445 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -462,6 +462,8 @@ bool MultiBinaryLabelCrossEntropy::init(const LayerMap& layerMap, void MultiBinaryLabelCrossEntropy::forwardImp(Matrix& output, Argument& label, Matrix& target) { + label.idsToSparseMatrix(output.getWidth(), useGpu_); + if (dynamic_cast(label.value.get()) || dynamic_cast(label.value.get())) { target.multiBinaryLabelCrossEntropy(output, *label.value); @@ -476,6 +478,8 @@ void MultiBinaryLabelCrossEntropy::forwardImp(Matrix& output, Argument& label, void MultiBinaryLabelCrossEntropy::backwardImp( Matrix& output, Argument& label, Matrix& outputG) { + label.idsToSparseMatrix(output.getWidth(), useGpu_); + if (dynamic_cast(label.value.get()) || dynamic_cast(label.value.get())) { outputG.multiBinaryLabelCrossEntropyBp(output, *label.value); diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index e7e07e9e69dc7..f19c14f56925a 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -538,9 +538,10 @@ TEST(Layer, multi_binary_label) { config.layerConfig.add_inputs(); config.layerConfig.add_inputs(); - // Not support GPU now - testLayerGrad(config, "multi_binary_label_cross_entropy", 100, - /* trans */ false, /* useGpu */ false); + for (auto useGpu : {false, true}) { + testLayerGrad(config, "multi_binary_label_cross_entropy", 100, + /* trans */ false, useGpu); + } } TEST(Layer, multi_cross_with_selfnorm) { diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 950c3bb6cca28..9acc6005532fc 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -1268,6 +1268,42 @@ void GpuMatrix::bilinearBackward(const Matrix& out, } } +void GpuMatrix::multiBinaryLabelCrossEntropy(Matrix& output, Matrix& label) { + GpuMatrix* output_ptr = dynamic_cast(&output); + auto label_ptr = dynamic_cast(&label); + + CHECK(output_ptr && label_ptr) << "Invalid argument pointer"; + CHECK(label_ptr->format_ == SPARSE_CSR) << "Matrix format not supported"; + CHECK(height_ == output_ptr->height_ && width_ == 1 + && output_ptr->width_ == label_ptr->getWidth() + && output_ptr->height_ == label_ptr->getHeight()) + << "Matrix dimensions are not equal"; + + real* output_d = output_ptr->data_; + real* entropy_d = data_; + hl_sparse_matrix_s mat_d = label_ptr->sMatrix_.get(); + hl_matrix_multi_binary_cross_entropy( + output_d, entropy_d, mat_d, height_, output_ptr->width_); +} + +void GpuMatrix::multiBinaryLabelCrossEntropyBp(Matrix &output, Matrix &label) { + GpuMatrix* output_ptr = dynamic_cast(&output); + auto label_ptr = dynamic_cast(&label); + + CHECK(output_ptr && label_ptr) << "Invalid argument pointer"; + CHECK(label_ptr->format_ == SPARSE_CSR) << "Matrix format not supported"; + CHECK(height_ == output_ptr->height_ && width_ == output_ptr->width_ + && output_ptr->width_ == label_ptr->getWidth() + && output_ptr->height_ == label_ptr->getHeight()) + << "Matrix dimensions are not equal"; + + real* output_d = output_ptr->data_; + real* grad_d = data_; + hl_sparse_matrix_s mat_d = label_ptr->sMatrix_.get(); + hl_matrix_multi_binary_cross_entropy_bp( + output_d, grad_d, mat_d, height_, width_); +} + /** * CpuMatrix */ diff --git a/paddle/math/Matrix.h b/paddle/math/Matrix.h index 700be7590240c..6c3c4804d2fc6 100644 --- a/paddle/math/Matrix.h +++ b/paddle/math/Matrix.h @@ -1303,6 +1303,10 @@ class GpuMatrix : public Matrix { const size_t numChannels, const real ratioH, const real ratioW); + + void multiBinaryLabelCrossEntropy(Matrix& output, Matrix& label); + + void multiBinaryLabelCrossEntropyBp(Matrix& output, Matrix& label); }; class CpuMatrix : public Matrix { diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index b3ee4bc34995a..a41e21903f560 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -2208,7 +2208,6 @@ void testCollectSharedBias(int numSamples, int dim, int channel) { MatrixCheckErr(*cpuBias, *check); } - TEST(Matrix, sharedBias) { for (auto numSamples : {1, 100, 520}) { for (auto dim : {100 * 16, 100 * 32}) { @@ -2222,6 +2221,71 @@ TEST(Matrix, sharedBias) { } } +void testMultiBinaryLabelCrossEntropy(int numSamples, int dim) { + MatrixPtr output = std::make_shared(numSamples, dim); + MatrixPtr cpuOutput = std::make_shared(numSamples, dim); + MatrixPtr gpuOutput = std::make_shared(numSamples, dim); + + MatrixPtr cpuEntropy = std::make_shared(numSamples, 1); + MatrixPtr gpuEntropy = std::make_shared(numSamples, 1); + + MatrixPtr cpuGrad = std::make_shared(numSamples, dim); + MatrixPtr gpuGrad = std::make_shared(numSamples, dim); + + auto cpuRows = IVector::create(numSamples + 1, false); + auto cpuCols = IVector::create(numSamples, false); + auto gpuRows = IVector::create(numSamples + 1, true); + auto gpuCols = IVector::create(numSamples, true); + cpuRows->setElement(0, 0); + gpuRows->setElement(0, 0); + for (int i = 0; i < numSamples; i ++) { + int id = rand() % dim; // NOLINT + cpuRows->setElement(i + 1, i + 1); + gpuRows->setElement(i + 1, i + 1); + cpuCols->setElement(i, id); + gpuCols->setElement(i, id); + } + + MatrixPtr cpuLabel = std::make_shared + (nullptr, cpuRows->getData(), cpuCols->getData(), + numSamples, dim, numSamples, NO_VALUE, SPARSE_CSR, false); + MatrixPtr gpuLabel = std::make_shared + (nullptr, gpuRows->getData(), gpuCols->getData(), + numSamples, dim, numSamples, NO_VALUE, SPARSE_CSR, false); + + output->randomizeUniform(); + cpuOutput->zeroMem(); + output->softmax(*cpuOutput); + gpuOutput->copyFrom(*cpuOutput); + + cpuEntropy->zeroMem(); + gpuEntropy->zeroMem(); + cpuEntropy->multiBinaryLabelCrossEntropy(*cpuOutput, *cpuLabel); + gpuEntropy->multiBinaryLabelCrossEntropy(*gpuOutput, *gpuLabel); + + MatrixPtr check1 = std::make_shared(numSamples, 1); + check1->copyFrom(*gpuEntropy); + MatrixCheckErr(*cpuEntropy, *check1); + + cpuGrad->zeroMem(); + gpuGrad->zeroMem(); + cpuGrad->multiBinaryLabelCrossEntropyBp(*cpuOutput, *cpuLabel); + gpuGrad->multiBinaryLabelCrossEntropyBp(*gpuOutput, *gpuLabel); + + MatrixPtr check2 = std::make_shared(numSamples, dim); + check2->copyFrom(*gpuGrad); + MatrixCheckErr(*cpuGrad, *check2); +} + +TEST(Matrix, multiBinaryCrossEntropy) { + for (auto numSamples : {1, 100, 500}) { + for (auto dim : {1000, 10000, 100000}) { + VLOG(3) << " numSamples=" << numSamples << " dim=" << dim; + testMultiBinaryLabelCrossEntropy(numSamples, dim); + } + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/paddle/parameter/Argument.cpp b/paddle/parameter/Argument.cpp index 42c74661d2b2c..a5a96742e4cad 100644 --- a/paddle/parameter/Argument.cpp +++ b/paddle/parameter/Argument.cpp @@ -572,4 +572,26 @@ void Argument::subArgFrom(const Argument& input, size_t offset, size_t height, } } +void Argument::idsToSparseMatrix(int width, bool useGpu) { + if (ids) { + CHECK(!value); + int height = ids->getSize(); + int nnz = height; + auto rows = IVector::create(height + 1, useGpu); + auto cols = IVector::create(nnz, useGpu); + rows->setElement(0, 0); + for (int i = 0; i < height; i ++) { + int id = ids->getElement(i); + CHECK_LT(id, width); + rows->setElement(i + 1, i + 1); + cols->setElement(i, id); + } + value = Matrix::createSparseMatrix( + nullptr, rows->getData(), cols->getData(), + height, width, nnz, NO_VALUE, SPARSE_CSR, false, useGpu); + } else { + CHECK(value); + } +} + } // namespace paddle diff --git a/paddle/parameter/Argument.h b/paddle/parameter/Argument.h index 81ff9029bc4c8..48e1551258fe7 100644 --- a/paddle/parameter/Argument.h +++ b/paddle/parameter/Argument.h @@ -286,6 +286,14 @@ struct Argument { sequence has sub-sequence degrades to a sequence. */ void degradeSequence(const Argument& input, bool useGpu); + + /* + @brief convert the ids vector to value as a sparse matrix + the ids vector keeps valid + @param the matrix width (id range) + @useGpu + */ + void idsToSparseMatrix(int width, bool useGpu); }; } // namespace paddle From 728defbec90a162ee5d7f8521106ded7797e72fa Mon Sep 17 00:00:00 2001 From: Haonan Date: Fri, 11 Nov 2016 16:31:56 -0800 Subject: [PATCH 294/324] copy the data when createSparseMatrix --- paddle/gserver/layers/CostLayer.cpp | 40 ++++++++++++++++------ paddle/parameter/Argument.cpp | 52 +++++++++++++++++++---------- paddle/parameter/Argument.h | 6 ++-- 3 files changed, 66 insertions(+), 32 deletions(-) diff --git a/paddle/gserver/layers/CostLayer.cpp b/paddle/gserver/layers/CostLayer.cpp index c86e562d0e445..900981d1e7d36 100644 --- a/paddle/gserver/layers/CostLayer.cpp +++ b/paddle/gserver/layers/CostLayer.cpp @@ -462,29 +462,49 @@ bool MultiBinaryLabelCrossEntropy::init(const LayerMap& layerMap, void MultiBinaryLabelCrossEntropy::forwardImp(Matrix& output, Argument& label, Matrix& target) { - label.idsToSparseMatrix(output.getWidth(), useGpu_); + MatrixPtr value = nullptr; + if (label.ids) { + CHECK(!label.value); + value = Matrix::createSparseMatrix( + label.ids->getSize(), output.getWidth(), label.ids->getSize(), + NO_VALUE, SPARSE_CSR, false, useGpu_); + label.idsToSparseMatrix(value); + } else { + CHECK(label.value); + value = label.value; + } - if (dynamic_cast(label.value.get()) || - dynamic_cast(label.value.get())) { - target.multiBinaryLabelCrossEntropy(output, *label.value); + if (dynamic_cast(value.get()) || + dynamic_cast(value.get())) { + target.multiBinaryLabelCrossEntropy(output, *value); } else { Matrix::resizeOrCreate(targetPerDim_, output.getHeight(), output.getWidth(), false, useGpu_); - targetPerDim_->binaryLabelCrossEntropy(output, *label.value); + targetPerDim_->binaryLabelCrossEntropy(output, *value); targetPerDim_->rowSum(target); } } void MultiBinaryLabelCrossEntropy::backwardImp( Matrix& output, Argument& label, Matrix& outputG) { - label.idsToSparseMatrix(output.getWidth(), useGpu_); + MatrixPtr value = nullptr; + if (label.ids) { + CHECK(!value); + value = Matrix::createSparseMatrix( + label.ids->getSize(), output.getWidth(), label.ids->getSize(), + NO_VALUE, SPARSE_CSR, false, useGpu_); + label.idsToSparseMatrix(value); + } else { + CHECK(label.value); + value = label.value; + } - if (dynamic_cast(label.value.get()) || - dynamic_cast(label.value.get())) { - outputG.multiBinaryLabelCrossEntropyBp(output, *label.value); + if (dynamic_cast(value.get()) || + dynamic_cast(value.get())) { + outputG.multiBinaryLabelCrossEntropyBp(output, *value); } else { - outputG.binaryLabelCrossEntropyBp(output, *label.value); + outputG.binaryLabelCrossEntropyBp(output, *value); } } diff --git a/paddle/parameter/Argument.cpp b/paddle/parameter/Argument.cpp index a5a96742e4cad..354d0ead071b3 100644 --- a/paddle/parameter/Argument.cpp +++ b/paddle/parameter/Argument.cpp @@ -572,25 +572,41 @@ void Argument::subArgFrom(const Argument& input, size_t offset, size_t height, } } -void Argument::idsToSparseMatrix(int width, bool useGpu) { - if (ids) { - CHECK(!value); - int height = ids->getSize(); - int nnz = height; - auto rows = IVector::create(height + 1, useGpu); - auto cols = IVector::create(nnz, useGpu); - rows->setElement(0, 0); - for (int i = 0; i < height; i ++) { - int id = ids->getElement(i); - CHECK_LT(id, width); - rows->setElement(i + 1, i + 1); - cols->setElement(i, id); - } - value = Matrix::createSparseMatrix( - nullptr, rows->getData(), cols->getData(), - height, width, nnz, NO_VALUE, SPARSE_CSR, false, useGpu); +void Argument::idsToSparseMatrix(MatrixPtr sparse_mat) { + int height = ids->getSize(); + int width = sparse_mat->getWidth(); + + CpuIVector cpu_ids(height); + cpu_ids.copyFrom(*ids); + int *id_data = cpu_ids.getData(); + + int *rows = nullptr; + int *cols = nullptr; + if (sparse_mat->useGpu()) { + auto gpu_sparse_mat = + dynamic_cast(sparse_mat.get()); + rows = gpu_sparse_mat->rows_; + cols = gpu_sparse_mat->cols_; } else { - CHECK(value); + rows = sparse_mat->getRows(); + cols = sparse_mat->getCols(); + } + + rows[0] = 0; + for (int i = 0; i < height; i ++) { + int id = id_data[i]; + CHECK_LT(id, width); + rows[i + 1] = i + 1; + cols[i] = id; + } + + if (sparse_mat->useGpu()) { + auto gpu_sparse_mat = + dynamic_cast(sparse_mat.get()); + hl_memcpy_csr_matrix(gpu_sparse_mat->sMatrix_.get(), + nullptr, rows, cols, + HPPL_STREAM_DEFAULT); + hl_stream_synchronize(HPPL_STREAM_DEFAULT); } } diff --git a/paddle/parameter/Argument.h b/paddle/parameter/Argument.h index 48e1551258fe7..695033138b545 100644 --- a/paddle/parameter/Argument.h +++ b/paddle/parameter/Argument.h @@ -289,11 +289,9 @@ struct Argument { /* @brief convert the ids vector to value as a sparse matrix - the ids vector keeps valid - @param the matrix width (id range) - @useGpu + @param[out] the output sparse_mat (already allocated) */ - void idsToSparseMatrix(int width, bool useGpu); + void idsToSparseMatrix(MatrixPtr sparse_mat); }; } // namespace paddle From 319742c641d32d64113655c0b0129f10b7ca42b6 Mon Sep 17 00:00:00 2001 From: qijun Date: Sat, 12 Nov 2016 02:23:15 +0000 Subject: [PATCH 295/324] format python code in demo, doc, doc_cn and paddle directories --- .../data/process_cifar.py | 30 +- demo/image_classification/image_provider.py | 12 +- demo/image_classification/image_util.py | 76 +++-- demo/image_classification/prediction.py | 59 ++-- demo/image_classification/preprocess.py | 48 ++-- demo/image_classification/vgg_16_cifar.py | 50 ++-- demo/introduction/dataprovider.py | 6 +- demo/introduction/evaluate_model.py | 7 +- demo/introduction/trainer_config.py | 19 +- demo/mnist/data/generate_list.py | 6 +- demo/mnist/mnist_provider.py | 7 +- demo/mnist/vgg_16_mnist.py | 29 +- demo/model_zoo/embedding/extract_para.py | 45 ++- demo/model_zoo/embedding/paraconvert.py | 32 ++- demo/model_zoo/resnet/classify.py | 133 +++++---- demo/model_zoo/resnet/example/__init__.py | 1 - .../resnet/example/image_list_provider.py | 15 +- demo/model_zoo/resnet/load_feature.py | 12 +- demo/model_zoo/resnet/resnet.py | 265 +++++++++--------- demo/quick_start/api_train.py | 42 +-- demo/quick_start/dataprovider_bow.py | 10 +- demo/quick_start/dataprovider_emb.py | 6 +- demo/quick_start/preprocess.py | 4 +- demo/quick_start/trainer_config.bidi-lstm.py | 21 +- demo/quick_start/trainer_config.cnn.py | 14 +- demo/quick_start/trainer_config.db-lstm.py | 29 +- demo/quick_start/trainer_config.emb.py | 16 +- demo/quick_start/trainer_config.lr.py | 14 +- demo/quick_start/trainer_config.lstm.py | 22 +- demo/recommendation/common_utils.py | 7 +- demo/recommendation/data/config_generator.py | 17 +- demo/recommendation/data/meta_generator.py | 30 +- demo/recommendation/data/split.py | 1 - demo/recommendation/dataprovider.py | 2 + demo/recommendation/prediction.py | 10 +- demo/recommendation/trainer_config.py | 42 +-- demo/semantic_role_labeling/dataprovider.py | 6 +- demo/semantic_role_labeling/db_lstm.py | 3 +- demo/semantic_role_labeling/predict.py | 20 +- demo/sentiment/dataprovider.py | 9 +- demo/sentiment/predict.py | 63 +++-- demo/sentiment/preprocess.py | 93 +++--- demo/sentiment/sentiment_net.py | 48 ++-- demo/sentiment/trainer_config.py | 17 +- demo/seqToseq/dataprovider.py | 9 +- demo/seqToseq/preprocess.py | 69 +++-- demo/seqToseq/seqToseq_net.py | 106 +++---- demo/sequence_tagging/dataprovider.py | 106 +++---- demo/sequence_tagging/linear_crf.py | 42 ++- demo/sequence_tagging/rnn_crf.py | 80 +++--- doc/ui/predict/predict_sample.py | 173 +++++++----- doc_cn/concepts/trainer_config.py | 28 +- doc_cn/faq/word2vec_config.py | 14 +- doc_cn/faq/word2vec_dataprovider.py | 14 +- doc_cn/ui/data_provider/mnist_config.py | 9 +- .../ui/data_provider/mnist_provider.dict.py | 7 +- doc_cn/ui/data_provider/mnist_provider.py | 5 +- doc_cn/ui/data_provider/sentimental_config.py | 15 +- .../ui/data_provider/sentimental_provider.py | 3 +- paddle/api/__init__.py | 1 - paddle/api/paddle_ld_flags.py | 34 ++- paddle/api/test/testArguments.py | 2 +- paddle/api/test/testGradientMachine.py | 6 +- paddle/api/test/testMatrix.py | 3 +- paddle/api/test/testTrain.py | 3 +- paddle/api/test/testTrainConfig.py | 5 +- paddle/api/test/testTrainer.py | 6 +- paddle/api/test/testVector.py | 3 +- paddle/gserver/tests/__init__.py | 1 - paddle/gserver/tests/pyDataProvider.py | 97 ++++--- paddle/gserver/tests/rnn_data_provider.py | 59 ++-- paddle/gserver/tests/sequenceGen.py | 22 +- .../gserver/tests/sequence_layer_group.conf | 38 +-- .../tests/sequence_nest_layer_group.conf | 64 +++-- paddle/gserver/tests/test_PyDataProvider2.py | 38 +-- paddle/py_paddle/__init__.py | 11 +- paddle/py_paddle/dataprovider_converter.py | 29 +- paddle/py_paddle/util.py | 82 +++--- paddle/scripts/cluster_train/conf.py | 11 +- paddle/scripts/cluster_train/paddle.py | 63 +++-- paddle/trainer/tests/__init__.py | 1 - paddle/trainer/tests/config_parser_test.py | 2 +- paddle/trainer/tests/gen_proto_data.py | 127 ++++----- paddle/trainer/tests/testPyDataWrapper.py | 49 +++- paddle/utils/enable_virtualenv.py | 12 +- 85 files changed, 1580 insertions(+), 1267 deletions(-) diff --git a/demo/image_classification/data/process_cifar.py b/demo/image_classification/data/process_cifar.py index b766118eb0073..b235010e4ece3 100644 --- a/demo/image_classification/data/process_cifar.py +++ b/demo/image_classification/data/process_cifar.py @@ -16,7 +16,6 @@ import sys import os import PIL.Image as Image - """ Usage: python process_cifar input_dir output_dir """ @@ -30,6 +29,7 @@ def mkdir_not_exist(path): if not os.path.exists(path): os.mkdir(path) + def create_dir_structure(output_dir): """ Create the directory structure for the directory. @@ -39,8 +39,8 @@ def create_dir_structure(output_dir): mkdir_not_exist(os.path.join(output_dir, "train")) mkdir_not_exist(os.path.join(output_dir, "test")) -def convert_batch(batch_path, label_set, label_map, - output_dir, data_split): + +def convert_batch(batch_path, label_set, label_map, output_dir, data_split): """ Convert CIFAR batch to the structure of Paddle format. batch_path: the batch to be converted. @@ -67,11 +67,23 @@ def convert_batch(batch_path, label_set, label_map, output_dir = sys.argv[2] num_batch = 5 create_dir_structure(output_dir) - label_map = {0: "airplane", 1: "automobile", 2: "bird", 3: "cat", 4: "deer", - 5: "dog", 6: "frog", 7: "horse", 8: "ship", 9: "truck"} + label_map = { + 0: "airplane", + 1: "automobile", + 2: "bird", + 3: "cat", + 4: "deer", + 5: "dog", + 6: "frog", + 7: "horse", + 8: "ship", + 9: "truck" + } labels = {} for i in range(1, num_batch + 1): - convert_batch(os.path.join(input_dir, "data_batch_%d" % i), labels, - label_map, output_dir, "train") - convert_batch(os.path.join(input_dir, "test_batch"), {}, - label_map, output_dir, "test") \ No newline at end of file + convert_batch( + os.path.join(input_dir, "data_batch_%d" % i), labels, label_map, + output_dir, "train") + convert_batch( + os.path.join(input_dir, "test_batch"), {}, label_map, output_dir, + "test") diff --git a/demo/image_classification/image_provider.py b/demo/image_classification/image_provider.py index 305efbcdc6bb1..28bf1bb02c1f0 100644 --- a/demo/image_classification/image_provider.py +++ b/demo/image_classification/image_provider.py @@ -46,14 +46,14 @@ def hook(settings, img_size, mean_img_size, num_classes, color, meta, use_jpeg, settings.img_mean = image_util.load_meta(settings.meta_path, settings.mean_img_size, - settings.img_size, - settings.color) + settings.img_size, settings.color) settings.logger.info('Image size: %s', settings.img_size) settings.logger.info('Meta path: %s', settings.meta_path) settings.input_types = [ dense_vector(settings.img_raw_size), # image feature - integer_value(settings.num_classes)] # labels + integer_value(settings.num_classes) + ] # labels settings.logger.info('DataProvider Initialization finished') @@ -79,8 +79,8 @@ def processData(settings, file_list): img = image_util.decode_jpeg(data['images'][i]) else: img = data['images'][i] - img_feat = image_util.preprocess_img(img, settings.img_mean, - settings.img_size, settings.is_train, - settings.color) + img_feat = image_util.preprocess_img( + img, settings.img_mean, settings.img_size, + settings.is_train, settings.color) label = data['labels'][i] yield img_feat.astype('float32'), int(label) diff --git a/demo/image_classification/image_util.py b/demo/image_classification/image_util.py index c545d16aafbc7..b5c6431c06f77 100644 --- a/demo/image_classification/image_util.py +++ b/demo/image_classification/image_util.py @@ -16,17 +16,20 @@ from PIL import Image from cStringIO import StringIO + def resize_image(img, target_size): """ Resize an image so that the shorter edge has length target_size. img: the input image to be resized. target_size: the target resized image size. """ - percent = (target_size/float(min(img.size[0], img.size[1]))) - resized_size = int(round(img.size[0] * percent)), int(round(img.size[1] * percent)) + percent = (target_size / float(min(img.size[0], img.size[1]))) + resized_size = int(round(img.size[0] * percent)), int( + round(img.size[1] * percent)) img = img.resize(resized_size, Image.ANTIALIAS) return img + def flip(im): """ Return the flipped image. @@ -38,6 +41,7 @@ def flip(im): else: return im[:, ::-1] + def crop_img(im, inner_size, color=True, test=True): """ Return cropped image. @@ -50,20 +54,22 @@ def crop_img(im, inner_size, color=True, test=True): If True, crop the center of images. """ if color: - height, width = max(inner_size, im.shape[1]), max(inner_size, im.shape[2]) + height, width = max(inner_size, im.shape[1]), max(inner_size, + im.shape[2]) padded_im = np.zeros((3, height, width)) startY = (height - im.shape[1]) / 2 startX = (width - im.shape[2]) / 2 endY, endX = startY + im.shape[1], startX + im.shape[2] - padded_im[:, startY: endY, startX: endX] = im + padded_im[:, startY:endY, startX:endX] = im else: im = im.astype('float32') - height, width = max(inner_size, im.shape[0]), max(inner_size, im.shape[1]) + height, width = max(inner_size, im.shape[0]), max(inner_size, + im.shape[1]) padded_im = np.zeros((height, width)) startY = (height - im.shape[0]) / 2 startX = (width - im.shape[1]) / 2 endY, endX = startY + im.shape[0], startX + im.shape[1] - padded_im[startY: endY, startX: endX] = im + padded_im[startY:endY, startX:endX] = im if test: startY = (height - inner_size) / 2 startX = (width - inner_size) / 2 @@ -72,19 +78,21 @@ def crop_img(im, inner_size, color=True, test=True): startX = np.random.randint(0, width - inner_size + 1) endY, endX = startY + inner_size, startX + inner_size if color: - pic = padded_im[:, startY: endY, startX: endX] + pic = padded_im[:, startY:endY, startX:endX] else: - pic = padded_im[startY: endY, startX: endX] + pic = padded_im[startY:endY, startX:endX] if (not test) and (np.random.randint(2) == 0): pic = flip(pic) return pic + def decode_jpeg(jpeg_string): np_array = np.array(Image.open(StringIO(jpeg_string))) if len(np_array.shape) == 3: np_array = np.transpose(np_array, (2, 0, 1)) return np_array + def preprocess_img(im, img_mean, crop_size, is_train, color=True): """ Does data augmentation for images. @@ -99,6 +107,7 @@ def preprocess_img(im, img_mean, crop_size, is_train, color=True): pic -= img_mean return pic.flatten() + def load_meta(meta_path, mean_img_size, crop_size, color=True): """ Return the loaded meta file. @@ -109,17 +118,18 @@ def load_meta(meta_path, mean_img_size, crop_size, color=True): mean = np.load(meta_path)['data_mean'] border = (mean_img_size - crop_size) / 2 if color: - assert(mean_img_size * mean_img_size * 3 == mean.shape[0]) + assert (mean_img_size * mean_img_size * 3 == mean.shape[0]) mean = mean.reshape(3, mean_img_size, mean_img_size) - mean = mean[:, border: border + crop_size, - border: border + crop_size].astype('float32') + mean = mean[:, border:border + crop_size, border:border + + crop_size].astype('float32') else: - assert(mean_img_size * mean_img_size == mean.shape[0]) + assert (mean_img_size * mean_img_size == mean.shape[0]) mean = mean.reshape(mean_img_size, mean_img_size) - mean = mean[border: border + crop_size, - border: border + crop_size].astype('float32') + mean = mean[border:border + crop_size, border:border + + crop_size].astype('float32') return mean + def load_image(img_path, is_color=True): """ Load image and return. @@ -130,6 +140,7 @@ def load_image(img_path, is_color=True): img.load() return img + def oversample(img, crop_dims): """ image : iterable of (H x W x K) ndarrays @@ -152,50 +163,53 @@ def oversample(img, crop_dims): for j in w_indices: crops_ix[curr] = (i, j, i + crop_dims[0], j + crop_dims[1]) curr += 1 - crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate([ - -crop_dims / 2.0, - crop_dims / 2.0 - ]) + crops_ix[4] = np.tile(im_center, (1, 2)) + np.concatenate( + [-crop_dims / 2.0, crop_dims / 2.0]) crops_ix = np.tile(crops_ix, (2, 1)) # Extract crops - crops = np.empty((10 * len(img), crop_dims[0], crop_dims[1], - im_shape[-1]), dtype=np.float32) + crops = np.empty( + (10 * len(img), crop_dims[0], crop_dims[1], im_shape[-1]), + dtype=np.float32) ix = 0 for im in img: for crop in crops_ix: crops[ix] = im[crop[0]:crop[2], crop[1]:crop[3], :] ix += 1 - crops[ix-5:ix] = crops[ix-5:ix, :, ::-1, :] # flip for mirrors + crops[ix - 5:ix] = crops[ix - 5:ix, :, ::-1, :] # flip for mirrors return crops + class ImageTransformer: - def __init__(self, transpose = None, - channel_swap = None, mean = None, is_color = True): + def __init__(self, + transpose=None, + channel_swap=None, + mean=None, + is_color=True): self.transpose = transpose self.channel_swap = None self.mean = None - self.is_color = is_color + self.is_color = is_color - def set_transpose(self, order): + def set_transpose(self, order): if self.is_color: - assert 3 == len(order) + assert 3 == len(order) self.transpose = order - def set_channel_swap(self, order): + def set_channel_swap(self, order): if self.is_color: - assert 3 == len(order) + assert 3 == len(order) self.channel_swap = order def set_mean(self, mean): # mean value, may be one value per channel if mean.ndim == 1: - mean = mean[:, np.newaxis, np.newaxis] - else: + mean = mean[:, np.newaxis, np.newaxis] + else: # elementwise mean if self.is_color: assert len(mean.shape) == 3 - self.mean = mean + self.mean = mean def transformer(self, data): if self.transpose is not None: diff --git a/demo/image_classification/prediction.py b/demo/image_classification/prediction.py index 5d9e932658673..6a47bd5851c99 100755 --- a/demo/image_classification/prediction.py +++ b/demo/image_classification/prediction.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os,sys +import os, sys import numpy as np import logging from PIL import Image @@ -24,9 +24,11 @@ from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config -logging.basicConfig(format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') logging.getLogger().setLevel(logging.INFO) + class ImageClassifier(): def __init__(self, train_conf, @@ -58,18 +60,19 @@ def __init__(self, self.oversample = oversample self.is_color = is_color - self.transformer = image_util.ImageTransformer(is_color = is_color) - self.transformer.set_transpose((2,0,1)) + self.transformer = image_util.ImageTransformer(is_color=is_color) + self.transformer.set_transpose((2, 0, 1)) self.mean_file = mean_file mean = np.load(self.mean_file)['data_mean'] mean = mean.reshape(3, self.crop_dims[0], self.crop_dims[1]) - self.transformer.set_mean(mean) # mean pixel + self.transformer.set_mean(mean) # mean pixel gpu = 1 if use_gpu else 0 conf_args = "is_test=1,use_gpu=%d,is_predict=1" % (gpu) conf = parse_config(train_conf, conf_args) swig_paddle.initPaddle("--use_gpu=%d" % (gpu)) - self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + self.network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(self.network, swig_paddle.GradientMachine) self.network.loadParameters(self.model_dir) @@ -90,14 +93,14 @@ def get_data(self, img_path): # image_util.resize_image: short side is self.resize_dim image = image_util.resize_image(image, self.resize_dim) image = np.array(image) - input = np.zeros((1, image.shape[0], image.shape[1], 3), - dtype=np.float32) + input = np.zeros( + (1, image.shape[0], image.shape[1], 3), dtype=np.float32) input[0] = image.astype(np.float32) input = image_util.oversample(input, self.crop_dims) else: image = image.resize(self.crop_dims, Image.ANTIALIAS) - input = np.zeros((1, self.crop_dims[0], self.crop_dims[1], 3), - dtype=np.float32) + input = np.zeros( + (1, self.crop_dims[0], self.crop_dims[1], 3), dtype=np.float32) input[0] = np.array(image).astype(np.float32) data_in = [] @@ -133,22 +136,24 @@ def predict(self, image=None, output_layer=None): lab = np.argsort(-prob) logging.info("Label of %s is: %d", image, lab[0]) + if __name__ == '__main__': - image_size=32 - crop_size=32 - multi_crop=True - config="vgg_16_cifar.py" - output_layer="__fc_layer_1__" - mean_path="data/cifar-out/batches/batches.meta" - model_path=sys.argv[1] - image=sys.argv[2] - use_gpu=bool(int(sys.argv[3])) - - obj = ImageClassifier(train_conf=config, - model_dir=model_path, - resize_dim=image_size, - crop_dim=crop_size, - mean_file=mean_path, - use_gpu=use_gpu, - oversample=multi_crop) + image_size = 32 + crop_size = 32 + multi_crop = True + config = "vgg_16_cifar.py" + output_layer = "__fc_layer_1__" + mean_path = "data/cifar-out/batches/batches.meta" + model_path = sys.argv[1] + image = sys.argv[2] + use_gpu = bool(int(sys.argv[3])) + + obj = ImageClassifier( + train_conf=config, + model_dir=model_path, + resize_dim=image_size, + crop_dim=crop_size, + mean_file=mean_path, + use_gpu=use_gpu, + oversample=multi_crop) obj.predict(image, output_layer) diff --git a/demo/image_classification/preprocess.py b/demo/image_classification/preprocess.py index fe7ea19bf0277..10b9c1691b5e5 100755 --- a/demo/image_classification/preprocess.py +++ b/demo/image_classification/preprocess.py @@ -19,24 +19,36 @@ def option_parser(): parser = OptionParser(usage="usage: python preprcoess.py "\ "-i data_dir [options]") - parser.add_option("-i", "--input", action="store", - dest="input", help="Input data directory.") - parser.add_option("-s", "--size", action="store", - dest="size", help="Processed image size.") - parser.add_option("-c", "--color", action="store", - dest="color", help="whether to use color images.") + parser.add_option( + "-i", + "--input", + action="store", + dest="input", + help="Input data directory.") + parser.add_option( + "-s", + "--size", + action="store", + dest="size", + help="Processed image size.") + parser.add_option( + "-c", + "--color", + action="store", + dest="color", + help="whether to use color images.") return parser.parse_args() + if __name__ == '__main__': - options, args = option_parser() - data_dir = options.input - processed_image_size = int(options.size) - color = options.color == "1" - data_creator = ImageClassificationDatasetCreater(data_dir, - processed_image_size, - color) - data_creator.train_list_name = "train.txt" - data_creator.test_list_name = "test.txt" - data_creator.num_per_batch = 1000 - data_creator.overwrite = True - data_creator.create_batches() + options, args = option_parser() + data_dir = options.input + processed_image_size = int(options.size) + color = options.color == "1" + data_creator = ImageClassificationDatasetCreater( + data_dir, processed_image_size, color) + data_creator.train_list_name = "train.txt" + data_creator.test_list_name = "test.txt" + data_creator.num_per_batch = 1000 + data_creator.overwrite = True + data_creator.create_batches() diff --git a/demo/image_classification/vgg_16_cifar.py b/demo/image_classification/vgg_16_cifar.py index edd6988c48acd..58ceff5fc2f46 100755 --- a/demo/image_classification/vgg_16_cifar.py +++ b/demo/image_classification/vgg_16_cifar.py @@ -18,36 +18,38 @@ ####################Data Configuration ################## if not is_predict: - data_dir='data/cifar-out/batches/' - meta_path=data_dir+'batches.meta' - - args = {'meta':meta_path,'mean_img_size': 32, - 'img_size': 32,'num_classes': 10, - 'use_jpeg': 1,'color': "color"} - - define_py_data_sources2(train_list="train.list", - test_list="train.list", - module='image_provider', - obj='processData', - args=args) + data_dir = 'data/cifar-out/batches/' + meta_path = data_dir + 'batches.meta' + + args = { + 'meta': meta_path, + 'mean_img_size': 32, + 'img_size': 32, + 'num_classes': 10, + 'use_jpeg': 1, + 'color': "color" + } + + define_py_data_sources2( + train_list="train.list", + test_list="train.list", + module='image_provider', + obj='processData', + args=args) ######################Algorithm Configuration ############# settings( - batch_size = 128, - learning_rate = 0.1 / 128.0, - learning_method = MomentumOptimizer(0.9), - regularization = L2Regularization(0.0005 * 128) -) + batch_size=128, + learning_rate=0.1 / 128.0, + learning_method=MomentumOptimizer(0.9), + regularization=L2Regularization(0.0005 * 128)) #######################Network Configuration ############# -data_size=3*32*32 -label_size=10 -img = data_layer(name='image', - size=data_size) +data_size = 3 * 32 * 32 +label_size = 10 +img = data_layer(name='image', size=data_size) # small_vgg is predefined in trainer_config_helpers.networks -predict = small_vgg(input_image=img, - num_channels=3, - num_classes=label_size) +predict = small_vgg(input_image=img, num_channels=3, num_classes=label_size) if not is_predict: lbl = data_layer(name="label", size=label_size) diff --git a/demo/introduction/dataprovider.py b/demo/introduction/dataprovider.py index be8c0bc89156c..8515022e18dc6 100644 --- a/demo/introduction/dataprovider.py +++ b/demo/introduction/dataprovider.py @@ -15,10 +15,10 @@ from paddle.trainer.PyDataProvider2 import * import random + # define data types of input: 2 real numbers -@provider(input_types=[dense_vector(1), dense_vector(1)],use_seq=False) +@provider(input_types=[dense_vector(1), dense_vector(1)], use_seq=False) def process(settings, input_file): for i in xrange(2000): x = random.random() - yield [x], [2*x+0.3] - + yield [x], [2 * x + 0.3] diff --git a/demo/introduction/evaluate_model.py b/demo/introduction/evaluate_model.py index 8cfb843c42105..ca4a1872731ab 100755 --- a/demo/introduction/evaluate_model.py +++ b/demo/introduction/evaluate_model.py @@ -23,14 +23,17 @@ import numpy as np import os + def load(file_name): with open(file_name, 'rb') as f: - f.read(16) # skip header for float type. + f.read(16) # skip header for float type. return np.fromfile(f, dtype=np.float32) + def main(): print 'w=%.6f, b=%.6f from pass 29' % (load('output/pass-00029/w'), - load('output/pass-00029/b')) + load('output/pass-00029/b')) + if __name__ == '__main__': main() diff --git a/demo/introduction/trainer_config.py b/demo/introduction/trainer_config.py index 3e3df5583282a..7c838c1a8f5b3 100644 --- a/demo/introduction/trainer_config.py +++ b/demo/introduction/trainer_config.py @@ -16,9 +16,14 @@ # 1. read data. Suppose you saved above python code as dataprovider.py data_file = 'empty.list' -with open(data_file, 'w') as f: f.writelines(' ') -define_py_data_sources2(train_list=data_file, test_list=None, - module='dataprovider', obj='process',args={}) +with open(data_file, 'w') as f: + f.writelines(' ') +define_py_data_sources2( + train_list=data_file, + test_list=None, + module='dataprovider', + obj='process', + args={}) # 2. learning algorithm settings(batch_size=12, learning_rate=1e-3, learning_method=MomentumOptimizer()) @@ -26,7 +31,11 @@ # 3. Network configuration x = data_layer(name='x', size=1) y = data_layer(name='y', size=1) -y_predict = fc_layer(input=x, param_attr=ParamAttr(name='w'), size=1, act=LinearActivation(), bias_attr=ParamAttr(name='b')) +y_predict = fc_layer( + input=x, + param_attr=ParamAttr(name='w'), + size=1, + act=LinearActivation(), + bias_attr=ParamAttr(name='b')) cost = regression_cost(input=y_predict, label=y) outputs(cost) - diff --git a/demo/mnist/data/generate_list.py b/demo/mnist/data/generate_list.py index 1b929048b4d82..d880721f94c68 100644 --- a/demo/mnist/data/generate_list.py +++ b/demo/mnist/data/generate_list.py @@ -13,9 +13,9 @@ # limitations under the License. o = open("./" + "train.list", "w") -o.write("./data/raw_data/train" +"\n") +o.write("./data/raw_data/train" + "\n") o.close() o = open("./" + "test.list", "w") -o.write("./data/raw_data/t10k" +"\n") -o.close() \ No newline at end of file +o.write("./data/raw_data/t10k" + "\n") +o.close() diff --git a/demo/mnist/mnist_provider.py b/demo/mnist/mnist_provider.py index 32af29730a736..6df4676da3bdc 100644 --- a/demo/mnist/mnist_provider.py +++ b/demo/mnist/mnist_provider.py @@ -2,10 +2,9 @@ # Define a py data provider -@provider(input_types={ - 'pixel': dense_vector(28 * 28), - 'label': integer_value(10) -}) +@provider( + input_types={'pixel': dense_vector(28 * 28), + 'label': integer_value(10)}) def process(settings, filename): # settings is not used currently. imgf = filename + "-images-idx3-ubyte" labelf = filename + "-labels-idx1-ubyte" diff --git a/demo/mnist/vgg_16_mnist.py b/demo/mnist/vgg_16_mnist.py index 45a45bb061aa7..f9e89bc588aba 100644 --- a/demo/mnist/vgg_16_mnist.py +++ b/demo/mnist/vgg_16_mnist.py @@ -18,32 +18,29 @@ ####################Data Configuration ################## - if not is_predict: - data_dir='./data/' - define_py_data_sources2(train_list= data_dir + 'train.list', - test_list= data_dir + 'test.list', - module='mnist_provider', - obj='process') + data_dir = './data/' + define_py_data_sources2( + train_list=data_dir + 'train.list', + test_list=data_dir + 'test.list', + module='mnist_provider', + obj='process') ######################Algorithm Configuration ############# settings( - batch_size = 128, - learning_rate = 0.1 / 128.0, - learning_method = MomentumOptimizer(0.9), - regularization = L2Regularization(0.0005 * 128) -) + batch_size=128, + learning_rate=0.1 / 128.0, + learning_method=MomentumOptimizer(0.9), + regularization=L2Regularization(0.0005 * 128)) #######################Network Configuration ############# -data_size=1*28*28 -label_size=10 +data_size = 1 * 28 * 28 +label_size = 10 img = data_layer(name='pixel', size=data_size) # small_vgg is predined in trainer_config_helpers.network -predict = small_vgg(input_image=img, - num_channels=1, - num_classes=label_size) +predict = small_vgg(input_image=img, num_channels=1, num_classes=label_size) if not is_predict: lbl = data_layer(name="label", size=label_size) diff --git a/demo/model_zoo/embedding/extract_para.py b/demo/model_zoo/embedding/extract_para.py index 17067792fc38d..47e06fae9caa9 100755 --- a/demo/model_zoo/embedding/extract_para.py +++ b/demo/model_zoo/embedding/extract_para.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Example: python extract_para.py --preModel PREMODEL --preDict PREDICT \ @@ -29,6 +28,7 @@ from optparse import OptionParser import struct + def get_row_index(preDict, usrDict): """ Get the row positions for all words in user dictionary from pre-trained dictionary. @@ -47,7 +47,9 @@ def get_row_index(preDict, usrDict): pos.append(index[word]) return pos -def extract_parameters_by_usrDict(preModel, preDict, usrModel, usrDict, paraDim): + +def extract_parameters_by_usrDict(preModel, preDict, usrModel, usrDict, + paraDim): """ Extract desired parameters from a pretrained embedding model based on user dictionary """ @@ -70,6 +72,7 @@ def extract_parameters_by_usrDict(preModel, preDict, usrModel, usrDict, paraDim) print "extract parameters finish, total", len(rowIndex), "lines" fi.close() + def main(): """ Main entry for running paraconvert.py @@ -78,19 +81,33 @@ def main(): "python %prog --preModel PREMODEL --preDict PREDICT" \ " --usrModel USRMODEL --usrDict USRDICT -d DIM" parser = OptionParser(usage) - parser.add_option("--preModel", action="store", dest="preModel", - help="the name of pretrained embedding model") - parser.add_option("--preDict", action="store", dest="preDict", - help="the name of pretrained dictionary") - parser.add_option("--usrModel", action="store", dest="usrModel", - help="the name of output usr embedding model") - parser.add_option("--usrDict", action="store", dest="usrDict", - help="the name of user specified dictionary") - parser.add_option("-d", action="store", dest="dim", - help="dimension of parameter") + parser.add_option( + "--preModel", + action="store", + dest="preModel", + help="the name of pretrained embedding model") + parser.add_option( + "--preDict", + action="store", + dest="preDict", + help="the name of pretrained dictionary") + parser.add_option( + "--usrModel", + action="store", + dest="usrModel", + help="the name of output usr embedding model") + parser.add_option( + "--usrDict", + action="store", + dest="usrDict", + help="the name of user specified dictionary") + parser.add_option( + "-d", action="store", dest="dim", help="dimension of parameter") (options, args) = parser.parse_args() - extract_parameters_by_usrDict(options.preModel, options.preDict, - options.usrModel, options.usrDict, int(options.dim)) + extract_parameters_by_usrDict(options.preModel, options.preDict, + options.usrModel, options.usrDict, + int(options.dim)) + if __name__ == '__main__': main() diff --git a/demo/model_zoo/embedding/paraconvert.py b/demo/model_zoo/embedding/paraconvert.py index 523412303617a..54155eff8e26b 100755 --- a/demo/model_zoo/embedding/paraconvert.py +++ b/demo/model_zoo/embedding/paraconvert.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Example: python paraconvert.py --b2t -i INPUT -o OUTPUT -d DIM @@ -29,6 +28,7 @@ from optparse import OptionParser import struct + def binary2text(input, output, paraDim): """ Convert a binary parameter file of embedding model to be a text file. @@ -76,12 +76,13 @@ def binary2text(input, output, paraDim): fo.close() print "binary2text finish, total", line, "lines" + def get_para_count(input): """ Compute the total number of embedding parameters in input text file. input: the name of input text file """ - numRows = 1 + numRows = 1 paraDim = 0 with open(input) as f: line = f.readline() @@ -90,6 +91,7 @@ def get_para_count(input): numRows += 1 return numRows * paraDim + def text2binary(input, output, paddle_head=True): """ Convert a text parameter file of embedding model to be a binary file. @@ -123,6 +125,7 @@ def text2binary(input, output, paddle_head=True): fo.close() print "text2binary finish, total", count, "lines" + def main(): """ Main entry for running paraconvert.py @@ -131,21 +134,26 @@ def main(): "python %prog --b2t -i INPUT -o OUTPUT -d DIM \n" \ "python %prog --t2b -i INPUT -o OUTPUT" parser = OptionParser(usage) - parser.add_option("--b2t", action="store_true", - help="convert parameter file of embedding model from binary to text") - parser.add_option("--t2b", action="store_true", - help="convert parameter file of embedding model from text to binary") - parser.add_option("-i", action="store", dest="input", - help="input parameter file name") - parser.add_option("-o", action="store", dest="output", - help="output parameter file name") - parser.add_option("-d", action="store", dest="dim", - help="dimension of parameter") + parser.add_option( + "--b2t", + action="store_true", + help="convert parameter file of embedding model from binary to text") + parser.add_option( + "--t2b", + action="store_true", + help="convert parameter file of embedding model from text to binary") + parser.add_option( + "-i", action="store", dest="input", help="input parameter file name") + parser.add_option( + "-o", action="store", dest="output", help="output parameter file name") + parser.add_option( + "-d", action="store", dest="dim", help="dimension of parameter") (options, args) = parser.parse_args() if options.b2t: binary2text(options.input, options.output, options.dim) if options.t2b: text2binary(options.input, options.output) + if __name__ == '__main__': main() diff --git a/demo/model_zoo/resnet/classify.py b/demo/model_zoo/resnet/classify.py index 06d471722f805..7855126edcfec 100755 --- a/demo/model_zoo/resnet/classify.py +++ b/demo/model_zoo/resnet/classify.py @@ -26,16 +26,22 @@ from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config -logging.basicConfig(format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') logging.getLogger().setLevel(logging.INFO) + class ImageClassifier(): - def __init__(self, train_conf, model_dir=None, - resize_dim=256, crop_dim=224, + def __init__(self, + train_conf, + model_dir=None, + resize_dim=256, + crop_dim=224, use_gpu=True, mean_file=None, output_layer=None, - oversample=False, is_color=True): + oversample=False, + is_color=True): """ train_conf: network configure. model_dir: string, directory of model. @@ -62,24 +68,25 @@ def __init__(self, train_conf, model_dir=None, assert isinstance(self.output_layer, basestring) self.output_layer = self.output_layer.split(",") - self.transformer = image_util.ImageTransformer(is_color = is_color) - self.transformer.set_transpose((2,0,1)) - self.transformer.set_channel_swap((2,1,0)) + self.transformer = image_util.ImageTransformer(is_color=is_color) + self.transformer.set_transpose((2, 0, 1)) + self.transformer.set_channel_swap((2, 1, 0)) self.mean_file = mean_file if self.mean_file is not None: mean = np.load(self.mean_file)['data_mean'] mean = mean.reshape(3, self.crop_dims[0], self.crop_dims[1]) - self.transformer.set_mean(mean) # mean pixel + self.transformer.set_mean(mean) # mean pixel else: # if you use three mean value, set like: # this three mean value is calculated from ImageNet. - self.transformer.set_mean(np.array([103.939,116.779,123.68])) + self.transformer.set_mean(np.array([103.939, 116.779, 123.68])) conf_args = "is_test=1,use_gpu=%d,is_predict=1" % (int(use_gpu)) conf = parse_config(train_conf, conf_args) swig_paddle.initPaddle("--use_gpu=%d" % (int(use_gpu))) - self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + self.network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(self.network, swig_paddle.GradientMachine) self.network.loadParameters(self.model_dir) @@ -105,14 +112,14 @@ def get_data(self, img_path): # image_util.resize_image: short side is self.resize_dim image = image_util.resize_image(image, self.resize_dim) image = np.array(image) - input = np.zeros((1, image.shape[0], image.shape[1], 3), - dtype=np.float32) + input = np.zeros( + (1, image.shape[0], image.shape[1], 3), dtype=np.float32) input[0] = image.astype(np.float32) input = image_util.oversample(input, self.crop_dims) else: image = image.resize(self.crop_dims, Image.ANTIALIAS) - input = np.zeros((1, self.crop_dims[0], self.crop_dims[1], 3), - dtype=np.float32) + input = np.zeros( + (1, self.crop_dims[0], self.crop_dims[1], 3), dtype=np.float32) input[0] = np.array(image).astype(np.float32) data_in = [] @@ -172,7 +179,7 @@ def predict(self, data_file): logging.info("Label of %s is: %d", image, lab[0]) return results - def extract(self, data_file, output_dir, batch_size = 10000): + def extract(self, data_file, output_dir, batch_size=10000): """ extract and save features of output layers, which are specify in Outputs() in network configure. @@ -197,7 +204,7 @@ def extract(self, data_file, output_dir, batch_size = 10000): image_feature[file_name] = feature sample_num += 1 if sample_num == batch_size: - batch_name = os.path.join(output_dir, 'batch_%d' %(batch_num)) + batch_name = os.path.join(output_dir, 'batch_%d' % (batch_num)) self.save_file(image_feature, batch_name) logging.info('Finish batch %d', batch_num) batch_num += 1 @@ -206,7 +213,7 @@ def extract(self, data_file, output_dir, batch_size = 10000): if idx % 1000 == 0: logging.info('%d/%d, %s', idx, len(image_files), file_name) if sample_num > 0: - batch_name = os.path.join(output_dir, 'batch_%d' %(batch_num)) + batch_name = os.path.join(output_dir, 'batch_%d' % (batch_num)) self.save_file(image_feature, batch_name) logging.info('Finish batch %d', batch_num) logging.info('Done: make image feature batch') @@ -215,38 +222,64 @@ def save_file(self, data, file): of = open(file, 'wb') cPickle.dump(data, of, protocol=cPickle.HIGHEST_PROTOCOL) + def option_parser(): """ Main entry for predciting """ usage = "%prog -c config -i data_list -w model_dir [options]" parser = OptionParser(usage="usage: %s" % usage) - parser.add_option("-j", "--job", - action="store", dest="job_type", - help="job type: predict, extract\ + parser.add_option( + "-j", + "--job", + action="store", + dest="job_type", + help="job type: predict, extract\ predict: predicting,\ extract: extract features") - parser.add_option("-c", "--conf", - action="store", dest="train_conf", - help="network config") - parser.add_option("-i", "--data", - action="store", dest="data_file", - help="image list") - parser.add_option("-w", "--model", - action="store", dest="model_path", - default=None, help="model path") - parser.add_option("-g", "--use_gpu", action="store", - dest="use_gpu", default=True, - help="Whether to use gpu mode.") - parser.add_option("-o", "--output_dir", - action="store", dest="output_dir", - default="output", help="output path") - parser.add_option("-m", "--mean", action="store", - dest="mean", default=None, - help="mean file.") - parser.add_option("-p", "--multi_crop", action="store_true", - dest="multi_crop", default=False, - help="Wether to use multiple crops on image.") + parser.add_option( + "-c", + "--conf", + action="store", + dest="train_conf", + help="network config") + parser.add_option( + "-i", "--data", action="store", dest="data_file", help="image list") + parser.add_option( + "-w", + "--model", + action="store", + dest="model_path", + default=None, + help="model path") + parser.add_option( + "-g", + "--use_gpu", + action="store", + dest="use_gpu", + default=True, + help="Whether to use gpu mode.") + parser.add_option( + "-o", + "--output_dir", + action="store", + dest="output_dir", + default="output", + help="output path") + parser.add_option( + "-m", + "--mean", + action="store", + dest="mean", + default=None, + help="mean file.") + parser.add_option( + "-p", + "--multi_crop", + action="store_true", + dest="multi_crop", + default=False, + help="Wether to use multiple crops on image.") parser.add_option("-l", "--output_layer", action="store", dest="output_layer", default=None, help="--job=extract, specify layers to extract "\ @@ -254,24 +287,26 @@ def option_parser(): "classification probability, output in resnet.py.") return parser.parse_args() + def main(): """ 1. parse input arguments. 2. predicting or extract features according job type. """ options, args = option_parser() - obj = ImageClassifier(options.train_conf, - options.model_path, - use_gpu=options.use_gpu, - mean_file=options.mean, - output_layer=options.output_layer, - oversample=options.multi_crop) + obj = ImageClassifier( + options.train_conf, + options.model_path, + use_gpu=options.use_gpu, + mean_file=options.mean, + output_layer=options.output_layer, + oversample=options.multi_crop) if options.job_type == "predict": obj.predict(options.data_file) elif options.job_type == "extract": - obj.extract(options.data_file, - options.output_dir) + obj.extract(options.data_file, options.output_dir) + if __name__ == '__main__': main() diff --git a/demo/model_zoo/resnet/example/__init__.py b/demo/model_zoo/resnet/example/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/demo/model_zoo/resnet/example/__init__.py +++ b/demo/model_zoo/resnet/example/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/demo/model_zoo/resnet/example/image_list_provider.py b/demo/model_zoo/resnet/example/image_list_provider.py index ee457e1fffc7e..9e415f76a5332 100644 --- a/demo/model_zoo/resnet/example/image_list_provider.py +++ b/demo/model_zoo/resnet/example/image_list_provider.py @@ -16,8 +16,7 @@ from paddle.trainer.PyDataProvider2 import * -def hook(settings, image_size, crop_size, color, file_list, - is_train, **kwargs): +def hook(settings, image_size, crop_size, color, file_list, is_train, **kwargs): """ Description: Init with a list of data file file_list is the name list of input files. @@ -58,7 +57,7 @@ def hook(settings, image_size, crop_size, color, file_list, sz = settings.crop_size * settings.crop_size settings.img_mean = np.zeros(sz * 3, dtype=np.single) for idx, value in enumerate(settings.mean_value): - settings.img_mean[idx * sz: (idx + 1) * sz] = value + settings.img_mean[idx * sz:(idx + 1) * sz] = value settings.img_mean = settings.img_mean.reshape(3, settings.crop_size, settings.crop_size) @@ -69,7 +68,8 @@ def hook(settings, image_size, crop_size, color, file_list, settings.input_types = [ dense_vector(settings.img_input_size), # image feature - integer_value(1)] # labels + integer_value(1) + ] # labels settings.logger.info('Image short side: %s', settings.img_size) settings.logger.info('Crop size: %s', settings.crop_size) @@ -97,9 +97,6 @@ def processData(settings, file_list): # swap channel if settings.is_swap_channel: img = img[settings.swap_channel, :, :] - img_feat = preprocess_img(img, - settings.img_mean, - settings.crop_size, - settings.is_train, - settings.color) + img_feat = preprocess_img(img, settings.img_mean, settings.crop_size, + settings.is_train, settings.color) yield img_feat.tolist(), int(lab.strip()) diff --git a/demo/model_zoo/resnet/load_feature.py b/demo/model_zoo/resnet/load_feature.py index ee4930b7a17f7..b0948b75fd0ac 100644 --- a/demo/model_zoo/resnet/load_feature.py +++ b/demo/model_zoo/resnet/load_feature.py @@ -17,9 +17,11 @@ import cPickle import logging -logging.basicConfig(format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') +logging.basicConfig( + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s') logging.getLogger().setLevel(logging.INFO) + def load_feature_c(file): """ Load feature extracted by C++ interface. @@ -30,14 +32,15 @@ def load_feature_c(file): f = open(file, 'r') for line in f: sample = [] - for slot in line.strip().split(";"): - fea = [float(val) for val in slot.strip().split()] + for slot in line.strip().split(";"): + fea = [float(val) for val in slot.strip().split()] if fea: sample.append(fea) features.append(sample) f.close() return features + def load_feature_py(feature_dir): """ Load feature extracted by python interface. @@ -54,6 +57,7 @@ def load_feature_py(feature_dir): logging.info('Load feature file %s', file_name) return features + if __name__ == '__main__': - print load_feature_py(sys.argv[1]) + print load_feature_py(sys.argv[1]) #print load_feature_c(sys.argv[1]) diff --git a/demo/model_zoo/resnet/resnet.py b/demo/model_zoo/resnet/resnet.py index 483e308ac804e..015b74cd48459 100644 --- a/demo/model_zoo/resnet/resnet.py +++ b/demo/model_zoo/resnet/resnet.py @@ -13,7 +13,6 @@ # limitations under the License. from paddle.trainer_config_helpers import * - """ paper: https://arxiv.org/abs/1512.03385 """ @@ -28,15 +27,19 @@ # mean.meta size : 3 x 224 x 224. # If you use three mean value, set like: # "mean_value:103.939,116.779,123.68;" - args={ + args = { 'mean_meta': "model/mean_meta_224/mean.meta", - 'image_size': 224, 'crop_size': 224, - 'color': True,'swap_channel:': [2, 1, 0]} - define_py_data_sources2(train_list, - 'example/test.list', - module="example.image_list_provider", - obj="processData", - args=args) + 'image_size': 224, + 'crop_size': 224, + 'color': True, + 'swap_channel:': [2, 1, 0] + } + define_py_data_sources2( + train_list, + 'example/test.list', + module="example.image_list_provider", + obj="processData", + args=args) batch_size = 1 learning_rate = 0.1 / batch_size @@ -54,12 +57,16 @@ learning_method='momentum', learning_rate_decay_a=0.5, learning_rate_decay_b=1200000 * 10, - learning_rate_schedule="discexp", -) + learning_rate_schedule="discexp", ) -def conv_bn_layer(name, input, filter_size, num_filters, - stride, padding, channels=None, +def conv_bn_layer(name, + input, + filter_size, + num_filters, + stride, + padding, + channels=None, active_type=ReluActivation()): """ A wrapper for conv layer with batch normalization layers. @@ -67,19 +74,18 @@ def conv_bn_layer(name, input, filter_size, num_filters, conv layer has no activation. """ - tmp = img_conv_layer(name=name + "_conv", - input=input, - filter_size=filter_size, - num_channels=channels, - num_filters=num_filters, - stride=stride, - padding=padding, - act=LinearActivation(), - bias_attr=False) - return batch_norm_layer(name=name + "_bn", - input=tmp, - act=active_type, - use_global_stats=is_test) + tmp = img_conv_layer( + name=name + "_conv", + input=input, + filter_size=filter_size, + num_channels=channels, + num_filters=num_filters, + stride=stride, + padding=padding, + act=LinearActivation(), + bias_attr=False) + return batch_norm_layer( + name=name + "_bn", input=tmp, act=active_type, use_global_stats=is_test) def bottleneck_block(name, input, num_filters1, num_filters2): @@ -88,29 +94,31 @@ def bottleneck_block(name, input, num_filters1, num_filters2): Last conv_bn_layer has no activation. Addto layer has activation of relu. """ - last_name = conv_bn_layer(name=name + '_branch2a', - input=input, - filter_size=1, - num_filters=num_filters1, - stride=1, - padding=0) - last_name = conv_bn_layer(name=name + '_branch2b', - input=last_name, - filter_size=3, - num_filters=num_filters1, - stride=1, - padding=1) - last_name = conv_bn_layer(name=name + '_branch2c', - input=last_name, - filter_size=1, - num_filters=num_filters2, - stride=1, - padding=0, - active_type=LinearActivation()) - - return addto_layer(name=name + "_addto", - input=[input, last_name], - act=ReluActivation()) + last_name = conv_bn_layer( + name=name + '_branch2a', + input=input, + filter_size=1, + num_filters=num_filters1, + stride=1, + padding=0) + last_name = conv_bn_layer( + name=name + '_branch2b', + input=last_name, + filter_size=3, + num_filters=num_filters1, + stride=1, + padding=1) + last_name = conv_bn_layer( + name=name + '_branch2c', + input=last_name, + filter_size=1, + num_filters=num_filters2, + stride=1, + padding=0, + active_type=LinearActivation()) + + return addto_layer( + name=name + "_addto", input=[input, last_name], act=ReluActivation()) def mid_projection(name, input, num_filters1, num_filters2, stride=2): @@ -123,38 +131,41 @@ def mid_projection(name, input, num_filters1, num_filters2, stride=2): branch2x: bottleneck building block, shortcuts are identity. """ # stride = 2 - branch1 = conv_bn_layer(name=name + '_branch1', - input=input, - filter_size=1, - num_filters=num_filters2, - stride=stride, - padding=0, - active_type=LinearActivation()) - - last_name = conv_bn_layer(name=name + '_branch2a', - input=input, - filter_size=1, - num_filters=num_filters1, - stride=stride, - padding=0) - last_name = conv_bn_layer(name=name + '_branch2b', - input=last_name, - filter_size=3, - num_filters=num_filters1, - stride=1, - padding=1) - - last_name = conv_bn_layer(name=name + '_branch2c', - input=last_name, - filter_size=1, - num_filters=num_filters2, - stride=1, - padding=0, - active_type=LinearActivation()) - - return addto_layer(name=name + "_addto", - input=[branch1, last_name], - act=ReluActivation()) + branch1 = conv_bn_layer( + name=name + '_branch1', + input=input, + filter_size=1, + num_filters=num_filters2, + stride=stride, + padding=0, + active_type=LinearActivation()) + + last_name = conv_bn_layer( + name=name + '_branch2a', + input=input, + filter_size=1, + num_filters=num_filters1, + stride=stride, + padding=0) + last_name = conv_bn_layer( + name=name + '_branch2b', + input=last_name, + filter_size=3, + num_filters=num_filters1, + stride=1, + padding=1) + + last_name = conv_bn_layer( + name=name + '_branch2c', + input=last_name, + filter_size=1, + num_filters=num_filters2, + stride=1, + padding=0, + active_type=LinearActivation()) + + return addto_layer( + name=name + "_addto", input=[branch1, last_name], act=ReluActivation()) def deep_res_net(res2_num=3, res3_num=4, res4_num=6, res5_num=3): @@ -168,67 +179,67 @@ def deep_res_net(res2_num=3, res3_num=4, res4_num=6, res5_num=3): # For ImageNet # conv1: 112x112 img = data_layer(name='input', size=224 * 224 * 3) - tmp = conv_bn_layer("conv1", img, - filter_size=7, - channels=3, - num_filters=64, - stride=2, - padding=3) + tmp = conv_bn_layer( + "conv1", + img, + filter_size=7, + channels=3, + num_filters=64, + stride=2, + padding=3) tmp = img_pool_layer(name="pool1", input=tmp, pool_size=3, stride=2) # conv2_x: 56x56 - tmp = mid_projection(name="res2_1", - input=tmp, - num_filters1=64, - num_filters2=256, - stride=1) + tmp = mid_projection( + name="res2_1", input=tmp, num_filters1=64, num_filters2=256, stride=1) for i in xrange(2, res2_num + 1, 1): - tmp = bottleneck_block(name="res2_" + str(i), - input=tmp, - num_filters1=64, - num_filters2=256) + tmp = bottleneck_block( + name="res2_" + str(i), input=tmp, num_filters1=64, num_filters2=256) # conv3_x: 28x28 - tmp = mid_projection(name="res3_1", - input=tmp, - num_filters1=128, - num_filters2=512) + tmp = mid_projection( + name="res3_1", input=tmp, num_filters1=128, num_filters2=512) for i in xrange(2, res3_num + 1, 1): - tmp = bottleneck_block(name="res3_" + str(i), - input=tmp, num_filters1=128, - num_filters2=512) + tmp = bottleneck_block( + name="res3_" + str(i), + input=tmp, + num_filters1=128, + num_filters2=512) # conv4_x: 14x14 - tmp = mid_projection(name="res4_1", input=tmp, - num_filters1=256, num_filters2=1024) + tmp = mid_projection( + name="res4_1", input=tmp, num_filters1=256, num_filters2=1024) for i in xrange(2, res4_num + 1, 1): - tmp = bottleneck_block(name="res4_" + str(i), - input=tmp, - num_filters1=256, - num_filters2=1024) + tmp = bottleneck_block( + name="res4_" + str(i), + input=tmp, + num_filters1=256, + num_filters2=1024) # conv5_x: 7x7 - tmp = mid_projection(name="res5_1", input=tmp, - num_filters1=512, num_filters2=2048) + tmp = mid_projection( + name="res5_1", input=tmp, num_filters1=512, num_filters2=2048) for i in xrange(2, res5_num + 1, 1): - tmp = bottleneck_block(name="res5_" + str(i), - input=tmp, num_filters1=512, - num_filters2=2048) - - tmp = img_pool_layer(name='avgpool', - input=tmp, - pool_size=7, - stride=1, - pool_type=AvgPooling()) - - output = fc_layer(name='output', - input=tmp, - size=1000, - act=SoftmaxActivation()) + tmp = bottleneck_block( + name="res5_" + str(i), + input=tmp, + num_filters1=512, + num_filters2=2048) + + tmp = img_pool_layer( + name='avgpool', + input=tmp, + pool_size=7, + stride=1, + pool_type=AvgPooling()) + + output = fc_layer( + name='output', input=tmp, size=1000, act=SoftmaxActivation()) if not is_predict: - classification_cost(input=output, label=data_layer(name='label', - size=1)) + classification_cost( + input=output, label=data_layer( + name='label', size=1)) def res_net_50(): diff --git a/demo/quick_start/api_train.py b/demo/quick_start/api_train.py index 5ae19b8d26534..66cbb856484d2 100644 --- a/demo/quick_start/api_train.py +++ b/demo/quick_start/api_train.py @@ -22,27 +22,32 @@ from paddle.trainer.PyDataProvider2 \ import integer_value, integer_value_sequence, sparse_binary_vector + def parse_arguments(): parser = argparse.ArgumentParser() - parser.add_argument("--train_data", - type=str, required=False, help="train data file") + parser.add_argument( + "--train_data", type=str, required=False, help="train data file") parser.add_argument("--test_data", type=str, help="test data file") - parser.add_argument("--config", - type=str, required=True, help="config file name") + parser.add_argument( + "--config", type=str, required=True, help="config file name") parser.add_argument("--dict_file", required=True, help="dictionary file") - parser.add_argument("--seq", - default=1, type=int, - help="whether use sequence training") - parser.add_argument("--use_gpu", default=0, type=int, - help="whether use GPU for training") - parser.add_argument("--trainer_count", default=1, type=int, - help="Number of threads for training") - parser.add_argument("--num_passes", default=5, type=int, - help="Number of training passes") + parser.add_argument( + "--seq", default=1, type=int, help="whether use sequence training") + parser.add_argument( + "--use_gpu", default=0, type=int, help="whether use GPU for training") + parser.add_argument( + "--trainer_count", + default=1, + type=int, + help="Number of threads for training") + parser.add_argument( + "--num_passes", default=5, type=int, help="Number of training passes") return parser.parse_args() + UNK_IDX = 0 + def load_data(file_name, word_dict): with open(file_name, 'r') as f: for line in f: @@ -51,6 +56,7 @@ def load_data(file_name, word_dict): word_slot = [word_dict.get(w, UNK_IDX) for w in words] yield word_slot, int(label) + def load_dict(dict_file): word_dict = dict() with open(dict_file, 'r') as f: @@ -59,6 +65,7 @@ def load_dict(dict_file): word_dict[w] = i return word_dict + def main(): options = parse_arguments() api.initPaddle("--use_gpu=%s" % options.use_gpu, @@ -86,9 +93,9 @@ def main(): # create a data converter which converts data to PaddlePaddle # internal format input_types = [ - integer_value_sequence(len(word_dict)) if options.seq - else sparse_binary_vector(len(word_dict)), - integer_value(2)] + integer_value_sequence(len(word_dict)) if options.seq else + sparse_binary_vector(len(word_dict)), integer_value(2) + ] converter = DataProviderConverter(input_types) batch_size = trainer_config.opt_config.batch_size @@ -102,7 +109,7 @@ def main(): trainer.trainOneDataBatch(size, converter(batch)) trainer.finishTrainPass() if test_dataset: - trainer.startTestPeriod(); + trainer.startTestPeriod() for pos in xrange(0, len(test_dataset), batch_size): batch = itertools.islice(test_dataset, pos, pos + batch_size) size = min(batch_size, len(test_dataset) - pos) @@ -110,5 +117,6 @@ def main(): trainer.finishTestPeriod() trainer.finishTrain() + if __name__ == '__main__': main() diff --git a/demo/quick_start/dataprovider_bow.py b/demo/quick_start/dataprovider_bow.py index f8cde189cf87d..a5156a2d40cc0 100644 --- a/demo/quick_start/dataprovider_bow.py +++ b/demo/quick_start/dataprovider_bow.py @@ -17,6 +17,7 @@ # id of the word not in dictionary UNK_IDX = 0 + # initializer is called by the framework during initialization. # It allows the user to describe the data types and setup the # necessary data structure for later use. @@ -38,7 +39,9 @@ def initializer(settings, dictionary, **kwargs): # The second input is an integer. It represents the category id of the # sample. 2 means there are two labels in the dataset. # (1 for positive and 0 for negative) - integer_value(2)] + integer_value(2) + ] + # Delaring a data provider. It has an initializer 'data_initialzer'. # It will cache the generated data of the first pass in memory, so that @@ -69,9 +72,8 @@ def process(settings, file_name): def predict_initializer(settings, dictionary, **kwargs): settings.word_dict = dictionary - settings.input_types = [ - sparse_binary_vector(len(dictionary)) - ] + settings.input_types = [sparse_binary_vector(len(dictionary))] + # Declaring a data provider for prediction. The difference with process # is that label is not generated. diff --git a/demo/quick_start/dataprovider_emb.py b/demo/quick_start/dataprovider_emb.py index f5632d5f3f8bd..286f3f5c82081 100755 --- a/demo/quick_start/dataprovider_emb.py +++ b/demo/quick_start/dataprovider_emb.py @@ -24,7 +24,8 @@ def initializer(settings, dictionary, **kwargs): # The value of the integers range from 0 to len(dictrionary)-1 integer_value_sequence(len(dictionary)), # Define the second input for label id - integer_value(2)] + integer_value(2) + ] @provider(init_hook=initializer, cache=CacheType.CACHE_PASS_IN_MEM) @@ -40,7 +41,8 @@ def process(settings, file_name): def predict_initializer(settings, dictionary, **kwargs): settings.word_dict = dictionary settings.input_types = [ - integer_value(len(dictionary), seq_type=SequenceType.SEQUENCE) + integer_value( + len(dictionary), seq_type=SequenceType.SEQUENCE) ] diff --git a/demo/quick_start/preprocess.py b/demo/quick_start/preprocess.py index 69fdbe44b5245..d87fad632a742 100755 --- a/demo/quick_start/preprocess.py +++ b/demo/quick_start/preprocess.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ 1. (remove HTML before or not)tokensizing 2. pos sample : rating score 5; neg sample: rating score 1-2. @@ -35,7 +34,8 @@ batch_size = 5000 word_count = {} -num_tokenize = max(1, multiprocessing.cpu_count() - 2) # parse + tokenize + save +num_tokenize = max(1, + multiprocessing.cpu_count() - 2) # parse + tokenize + save max_queue_size = 8 parse_queue = Queue(maxsize=max_queue_size + num_tokenize) tokenize_queue = Queue(maxsize=max_queue_size + num_tokenize) diff --git a/demo/quick_start/trainer_config.bidi-lstm.py b/demo/quick_start/trainer_config.bidi-lstm.py index 3be3d37342271..51deaf31f9468 100644 --- a/demo/quick_start/trainer_config.bidi-lstm.py +++ b/demo/quick_start/trainer_config.bidi-lstm.py @@ -27,11 +27,12 @@ trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -39,19 +40,17 @@ learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + gradient_clipping_threshold=25) -bias_attr = ParamAttr(initial_std=0.,l2_rate=0.) +bias_attr = ParamAttr(initial_std=0., l2_rate=0.) data = data_layer(name="word", size=len(word_dict)) emb = embedding_layer(input=data, size=128) bi_lstm = bidirectional_lstm(input=emb, size=128) dropout = dropout_layer(input=bi_lstm, dropout_rate=0.5) -output = fc_layer(input=dropout, size=2, - bias_attr=bias_attr, - act=SoftmaxActivation()) +output = fc_layer( + input=dropout, size=2, bias_attr=bias_attr, act=SoftmaxActivation()) if is_predict: maxid = maxid_layer(output) diff --git a/demo/quick_start/trainer_config.cnn.py b/demo/quick_start/trainer_config.cnn.py index 253ec0aee26cf..388efa75f903e 100644 --- a/demo/quick_start/trainer_config.cnn.py +++ b/demo/quick_start/trainer_config.cnn.py @@ -27,11 +27,12 @@ trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -39,8 +40,7 @@ learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + gradient_clipping_threshold=25) data = data_layer(name="word", size=len(word_dict)) embedding = embedding_layer(input=data, size=128) diff --git a/demo/quick_start/trainer_config.db-lstm.py b/demo/quick_start/trainer_config.db-lstm.py index b35bdf5a61b47..02bc898d881ef 100644 --- a/demo/quick_start/trainer_config.db-lstm.py +++ b/demo/quick_start/trainer_config.db-lstm.py @@ -27,11 +27,12 @@ trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -39,10 +40,9 @@ learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + gradient_clipping_threshold=25) -bias_attr = ParamAttr(initial_std=0.,l2_rate=0.) +bias_attr = ParamAttr(initial_std=0., l2_rate=0.) data = data_layer(name="word", size=len(word_dict)) emb = embedding_layer(input=data, size=128) @@ -52,17 +52,18 @@ input_layers = [hidden_0, lstm_0] -for i in range(1,8): +for i in range(1, 8): fc = fc_layer(input=input_layers, size=128) - lstm = lstmemory(input=fc, layer_attr=ExtraAttr(drop_rate=0.1), - reverse=(i % 2) == 1,) + lstm = lstmemory( + input=fc, + layer_attr=ExtraAttr(drop_rate=0.1), + reverse=(i % 2) == 1, ) input_layers = [fc, lstm] lstm_last = pooling_layer(input=lstm, pooling_type=MaxPooling()) -output = fc_layer(input=lstm_last, size=2, - bias_attr=bias_attr, - act=SoftmaxActivation()) +output = fc_layer( + input=lstm_last, size=2, bias_attr=bias_attr, act=SoftmaxActivation()) if is_predict: maxid = maxid_layer(output) diff --git a/demo/quick_start/trainer_config.emb.py b/demo/quick_start/trainer_config.emb.py index 34dd7b96f2f14..8fd18a7aac704 100644 --- a/demo/quick_start/trainer_config.emb.py +++ b/demo/quick_start/trainer_config.emb.py @@ -27,18 +27,16 @@ trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( - batch_size=batch_size, - learning_rate=2e-3, - learning_method=AdamOptimizer() -) + batch_size=batch_size, learning_rate=2e-3, learning_method=AdamOptimizer()) data = data_layer(name="word", size=len(word_dict)) embedding = embedding_layer(input=data, size=128) diff --git a/demo/quick_start/trainer_config.lr.py b/demo/quick_start/trainer_config.lr.py index c6059947f30b3..b9c9441baac28 100644 --- a/demo/quick_start/trainer_config.lr.py +++ b/demo/quick_start/trainer_config.lr.py @@ -32,11 +32,12 @@ # We need to use different process for training and prediction. # For training, the input data includes both word IDs and labels. # For prediction, the input data only includs word Ids. -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_bow", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_bow", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -44,8 +45,7 @@ learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + gradient_clipping_threshold=25) # Define the data for text features. The size of the data layer is the number # of words in the dictionary. diff --git a/demo/quick_start/trainer_config.lstm.py b/demo/quick_start/trainer_config.lstm.py index b412a9cbd914d..8821e02d9bd4a 100644 --- a/demo/quick_start/trainer_config.lstm.py +++ b/demo/quick_start/trainer_config.lstm.py @@ -27,11 +27,12 @@ trn = 'data/train.list' if not is_predict else None tst = 'data/test.list' if not is_predict else 'data/pred.list' process = 'process' if not is_predict else 'process_predict' -define_py_data_sources2(train_list=trn, - test_list=tst, - module="dataprovider_emb", - obj=process, - args={"dictionary": word_dict}) +define_py_data_sources2( + train_list=trn, + test_list=tst, + module="dataprovider_emb", + obj=process, + args={"dictionary": word_dict}) batch_size = 128 if not is_predict else 1 settings( @@ -39,17 +40,14 @@ learning_rate=2e-3, learning_method=AdamOptimizer(), regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) - + gradient_clipping_threshold=25) data = data_layer(name="word", size=len(word_dict)) emb = embedding_layer(input=data, size=128) -lstm = simple_lstm(input=emb, size=128, - lstm_cell_attr=ExtraAttr(drop_rate=0.25)) +lstm = simple_lstm( + input=emb, size=128, lstm_cell_attr=ExtraAttr(drop_rate=0.25)) lstm_max = pooling_layer(input=lstm, pooling_type=MaxPooling()) -output = fc_layer(input=lstm_max, size=2, - act=SoftmaxActivation()) +output = fc_layer(input=lstm_max, size=2, act=SoftmaxActivation()) if is_predict: maxid = maxid_layer(output) outputs([maxid, output]) diff --git a/demo/recommendation/common_utils.py b/demo/recommendation/common_utils.py index a5f00b3ef9ca0..613e36b496e47 100755 --- a/demo/recommendation/common_utils.py +++ b/demo/recommendation/common_utils.py @@ -21,8 +21,9 @@ def meta_to_header(meta, name): yield integer_value(each_meta['max']) elif each_meta['type'] == 'embedding': is_seq = each_meta['seq'] == 'sequence' - yield integer_value(len(each_meta['dict']), - seq_type=SequenceType.SEQUENCE if is_seq - else SequenceType.NO_SEQUENCE) + yield integer_value( + len(each_meta['dict']), + seq_type=SequenceType.SEQUENCE + if is_seq else SequenceType.NO_SEQUENCE) elif each_meta['type'] == 'one_hot_dense': yield dense_vector(len(each_meta['dict'])) diff --git a/demo/recommendation/data/config_generator.py b/demo/recommendation/data/config_generator.py index 29f38082693ad..fa605458300f8 100644 --- a/demo/recommendation/data/config_generator.py +++ b/demo/recommendation/data/config_generator.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ config_generator.py @@ -29,10 +28,7 @@ import docopt import copy -DEFAULT_FILE = { - "type": "split", - "delimiter": "," -} +DEFAULT_FILE = {"type": "split", "delimiter": ","} DEFAULT_FIELD = { "id": { @@ -107,19 +103,16 @@ def main(filename, fmt): field = copy.deepcopy(DEFAULT_FIELD[field_key]) field['pos'] = pos fields.append(field) - obj[k] = { - "file": file_dict, - "fields": fields - } - meta = { - "meta": obj - } + obj[k] = {"file": file_dict, "fields": fields} + meta = {"meta": obj} # print meta if fmt == 'json': + def formatter(x): import json return json.dumps(x, indent=2) elif fmt == 'yaml': + def formatter(x): import yaml return yaml.safe_dump(x, default_flow_style=False) diff --git a/demo/recommendation/data/meta_generator.py b/demo/recommendation/data/meta_generator.py index 8d1a33d02aea1..593c863670d5e 100644 --- a/demo/recommendation/data/meta_generator.py +++ b/demo/recommendation/data/meta_generator.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Preprocess Movielens dataset, to get movie/user object. @@ -66,8 +65,8 @@ def scan(self, key): self.__key_set__.add(key) def finish_scan(self, compare=None, key=None, reverse=False): - self.__key_set__ = sorted(list(self.__key_set__), cmp=compare, - key=key, reverse=reverse) + self.__key_set__ = sorted( + list(self.__key_set__), cmp=compare, key=key, reverse=reverse) self.dict = dict() for idx, each_key in enumerate(self.__key_set__): self.dict[each_key] = idx @@ -207,11 +206,10 @@ def __init__(self, config): self.dict = EmbeddingFieldParser.CharBasedEmbeddingDict( self.seq_type == EmbeddingFieldParser.SEQUENCE) elif config['dict']['type'] == 'split': - self.dict = SplitEmbeddingDict( - config['dict'].get('delimiter', ',')) + self.dict = SplitEmbeddingDict(config['dict'].get('delimiter', ',')) elif config['dict']['type'] == 'whole_content': - self.dict = EmbeddingFieldParser.WholeContentDict( - config['dict']['sort']) + self.dict = EmbeddingFieldParser.WholeContentDict(config['dict'][ + 'sort']) else: print config assert False @@ -333,8 +331,8 @@ def create(config): return PositionContentExtractor(config['pos']) else: extra_args = config['regex'] - return RegexPositionContentExtractor(pos=config['pos'], - **extra_args) + return RegexPositionContentExtractor( + pos=config['pos'], **extra_args) class MetaFile(object): @@ -364,9 +362,10 @@ def parse(self, config): metas = map(lambda x: x.meta_field(), field_parsers) # print metas - key_index = filter(lambda x: x is not None, map( - lambda (idx, meta): idx if 'is_key' in meta and meta['is_key'] - else None, enumerate(metas)))[0] + key_index = filter( + lambda x: x is not None, + map(lambda (idx, meta): idx if 'is_key' in meta and meta['is_key'] else None, + enumerate(metas)))[0] key_map = [] for i in range(min(key_index, len(metas))): @@ -374,12 +373,7 @@ def parse(self, config): for i in range(key_index + 1, len(metas)): key_map.append(i) - obj = { - '__meta__': { - 'raw_meta': metas, - 'feature_map': key_map - } - } + obj = {'__meta__': {'raw_meta': metas, 'feature_map': key_map}} for each_block in reader.read(): idx = field_parsers[key_index].parse(each_block) diff --git a/demo/recommendation/data/split.py b/demo/recommendation/data/split.py index ff1f7fab7befd..8dd0cbd32af60 100644 --- a/demo/recommendation/data/split.py +++ b/demo/recommendation/data/split.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Separate movielens 1m dataset to train/test file. diff --git a/demo/recommendation/dataprovider.py b/demo/recommendation/dataprovider.py index 454467f40b44b..ff3932be03f1e 100755 --- a/demo/recommendation/dataprovider.py +++ b/demo/recommendation/dataprovider.py @@ -15,6 +15,7 @@ from paddle.trainer.PyDataProvider2 import * import common_utils # parse + def hook(settings, meta, **kwargs): """ Init hook is invoked before process data. It will set obj.slots and store @@ -41,6 +42,7 @@ def hook(settings, meta, **kwargs): settings.input_types = headers settings.meta = meta + @provider(init_hook=hook, cache=CacheType.CACHE_PASS_IN_MEM) def process(settings, filename): with open(filename, 'r') as f: diff --git a/demo/recommendation/prediction.py b/demo/recommendation/prediction.py index f8044a3195ec2..e2a202cfd1a47 100755 --- a/demo/recommendation/prediction.py +++ b/demo/recommendation/prediction.py @@ -28,7 +28,8 @@ model_path = sys.argv[1] swig_paddle.initPaddle('--use_gpu=0') conf = parse_config("trainer_config.py", "is_predict=1") - network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) network.loadParameters(model_path) with open('./data/meta.bin', 'rb') as f: @@ -39,11 +40,12 @@ while True: movie_id = int(raw_input("Input movie_id: ")) user_id = int(raw_input("Input user_id: ")) - movie_meta = meta['movie'][movie_id] # Query Data From Meta. + movie_meta = meta['movie'][movie_id] # Query Data From Meta. user_meta = meta['user'][user_id] data = [movie_id - 1] data.extend(movie_meta) data.append(user_id - 1) data.extend(user_meta) - print "Prediction Score is %.2f" % ((network.forwardTest( - cvt.convert([data]))[0]['value'][0][0] + 5) / 2) + print "Prediction Score is %.2f" % ( + (network.forwardTest(cvt.convert([data]))[0]['value'][0][0] + 5) + / 2) diff --git a/demo/recommendation/trainer_config.py b/demo/recommendation/trainer_config.py index 624c22ec969dc..cec340b0b65a8 100755 --- a/demo/recommendation/trainer_config.py +++ b/demo/recommendation/trainer_config.py @@ -27,8 +27,8 @@ # load meta file meta = pickle.load(f) -settings(batch_size=1600, learning_rate=1e-3, - learning_method=RMSPropOptimizer()) +settings( + batch_size=1600, learning_rate=1e-3, learning_method=RMSPropOptimizer()) def construct_feature(name): @@ -59,11 +59,10 @@ def construct_feature(name): slot_name = each_meta.get('name', '%s_id' % name) if type_name == 'id': slot_dim = each_meta['max'] - embedding = embedding_layer(input=data_layer(slot_name, - size=slot_dim), - size=256) - fusion.append(fc_layer(input=embedding, - size=256)) + embedding = embedding_layer( + input=data_layer( + slot_name, size=slot_dim), size=256) + fusion.append(fc_layer(input=embedding, size=256)) elif type_name == 'embedding': is_seq = each_meta['seq'] == 'sequence' slot_dim = len(each_meta['dict']) @@ -71,17 +70,14 @@ def construct_feature(name): embedding = embedding_layer(input=din, size=256) if is_seq: fusion.append( - text_conv_pool(input=embedding, context_len=5, - hidden_size=256)) + text_conv_pool( + input=embedding, context_len=5, hidden_size=256)) else: - fusion.append(fc_layer(input=embedding, - size=256)) + fusion.append(fc_layer(input=embedding, size=256)) elif type_name == 'one_hot_dense': slot_dim = len(each_meta['dict']) - hidden = fc_layer(input=data_layer(slot_name, slot_dim), - size=256) - fusion.append(fc_layer(input=hidden, - size=256)) + hidden = fc_layer(input=data_layer(slot_name, slot_dim), size=256) + fusion.append(fc_layer(input=hidden, size=256)) return fc_layer(name="%s_fusion" % name, input=fusion, size=256) @@ -90,10 +86,16 @@ def construct_feature(name): user_feature = construct_feature("user") similarity = cos_sim(a=movie_feature, b=user_feature) if not is_predict: - outputs(regression_cost(input=similarity, - label=data_layer('rating', size=1))) - - define_py_data_sources2('data/train.list', 'data/test.list', module='dataprovider', - obj='process', args={'meta': meta}) + outputs( + regression_cost( + input=similarity, label=data_layer( + 'rating', size=1))) + + define_py_data_sources2( + 'data/train.list', + 'data/test.list', + module='dataprovider', + obj='process', + args={'meta': meta}) else: outputs(similarity) diff --git a/demo/semantic_role_labeling/dataprovider.py b/demo/semantic_role_labeling/dataprovider.py index 2ef25c42c1794..5c003584a52d4 100644 --- a/demo/semantic_role_labeling/dataprovider.py +++ b/demo/semantic_role_labeling/dataprovider.py @@ -26,9 +26,9 @@ def hook(settings, word_dict, label_dict, **kwargs): integer_value_sequence(len(word_dict)), integer_value_sequence(len(word_dict)), integer_value_sequence(len(word_dict)), - integer_value_sequence(len(word_dict)), - integer_value_sequence(2), - integer_value_sequence(len(label_dict))] + integer_value_sequence(len(word_dict)), integer_value_sequence(2), + integer_value_sequence(len(label_dict)) + ] @provider(init_hook=hook) diff --git a/demo/semantic_role_labeling/db_lstm.py b/demo/semantic_role_labeling/db_lstm.py index 364460afbe31c..e3f6edad69721 100644 --- a/demo/semantic_role_labeling/db_lstm.py +++ b/demo/semantic_role_labeling/db_lstm.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - import math import os import sys @@ -42,7 +41,7 @@ label_dict[w] = i if is_test: - train_list_file = None + train_list_file = None #define data provider define_py_data_sources2( diff --git a/demo/semantic_role_labeling/predict.py b/demo/semantic_role_labeling/predict.py index 9a27112828e44..f051d4175cf6f 100644 --- a/demo/semantic_role_labeling/predict.py +++ b/demo/semantic_role_labeling/predict.py @@ -41,22 +41,16 @@ def __init__(self, train_conf, dict_file, model_dir, label_file): len_dict = len(self.dict) len_label = len(self.labels) - conf = parse_config( - train_conf, - 'dict_len=' + str(len_dict) + - ',label_len=' + str(len_label) + - ',is_predict=True') + conf = parse_config(train_conf, 'dict_len=' + str(len_dict) + + ',label_len=' + str(len_label) + ',is_predict=True') self.network = swig_paddle.GradientMachine.createFromConfigProto( conf.model_config) self.network.loadParameters(model_dir) slots = [ - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(len_dict), - integer_value_sequence(2) + integer_value_sequence(len_dict), integer_value_sequence(len_dict), + integer_value_sequence(len_dict), integer_value_sequence(len_dict), + integer_value_sequence(len_dict), integer_value_sequence(2) ] self.converter = DataProviderConverter(slots) @@ -110,8 +104,8 @@ def predict(self, data_file): len_sen = len(sen.split()) line_labels = lab[index:index + len_sen] index += len_sen - fout.write(sen + '\t' + ' '.join([self.labels_reverse[ - i] for i in line_labels]) + '\n') + fout.write(sen + '\t' + ' '.join( + [self.labels_reverse[i] for i in line_labels]) + '\n') def option_parser(): diff --git a/demo/sentiment/dataprovider.py b/demo/sentiment/dataprovider.py index 9a9fd81f030cb..53e3d1d20df92 100755 --- a/demo/sentiment/dataprovider.py +++ b/demo/sentiment/dataprovider.py @@ -17,8 +17,8 @@ def hook(settings, dictionary, **kwargs): settings.word_dict = dictionary settings.input_types = [ - integer_value_sequence(len(settings.word_dict)), - integer_value(2)] + integer_value_sequence(len(settings.word_dict)), integer_value(2) + ] settings.logger.info('dict len : %d' % (len(settings.word_dict))) @@ -29,6 +29,7 @@ def process(settings, file_name): label, comment = line.strip().split('\t\t') label = int(label) words = comment.split() - word_slot = [settings.word_dict[w] for w in words if w in - settings.word_dict] + word_slot = [ + settings.word_dict[w] for w in words if w in settings.word_dict + ] yield word_slot, label diff --git a/demo/sentiment/predict.py b/demo/sentiment/predict.py index 7d0baeabbba68..bc0f6f3126429 100755 --- a/demo/sentiment/predict.py +++ b/demo/sentiment/predict.py @@ -18,14 +18,14 @@ from py_paddle import swig_paddle, DataProviderConverter from paddle.trainer.PyDataProvider2 import integer_value_sequence from paddle.trainer.config_parser import parse_config - """ Usage: run following command to show help message. python predict.py -h """ + class SentimentPrediction(): - def __init__(self, train_conf, dict_file, model_dir=None, label_file = None): + def __init__(self, train_conf, dict_file, model_dir=None, label_file=None): """ train_conf: trainer configure. dict_file: word dictionary file name. @@ -44,7 +44,8 @@ def __init__(self, train_conf, dict_file, model_dir=None, label_file = None): self.load_label(label_file) conf = parse_config(train_conf, "is_predict=1") - self.network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + self.network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) self.network.loadParameters(self.model_dir) input_types = [integer_value_sequence(self.dict_dim)] self.converter = DataProviderConverter(input_types) @@ -61,7 +62,7 @@ def load_label(self, label_file): """ Load label. """ - self.label={} + self.label = {} for v in open(label_file, 'r'): self.label[int(v.split('\t')[1])] = v.split('\t')[0] @@ -72,7 +73,9 @@ def get_data(self, data_file): with open(data_file, 'r') as fdata: for line in fdata: words = line.strip().split() - word_slot = [self.word_dict[w] for w in words if w in self.word_dict] + word_slot = [ + self.word_dict[w] for w in words if w in self.word_dict + ] if not word_slot: print "all words are not in dictionary: %s", line continue @@ -89,25 +92,48 @@ def predict(self, data_file): if self.label is None: print("%s: predicting label is %d" % (data_file, lab[0][0])) else: - print("%s: predicting label is %s" % (data_file, self.label[lab[0][0]])) + print("%s: predicting label is %s" % + (data_file, self.label[lab[0][0]])) + def option_parser(): usage = "python predict.py -n config -w model_dir -d dictionary -i input_file " parser = OptionParser(usage="usage: %s [options]" % usage) - parser.add_option("-n", "--tconf", action="store", - dest="train_conf", help="network config") - parser.add_option("-d", "--dict", action="store", - dest="dict_file",help="dictionary file") - parser.add_option("-b", "--label", action="store", - dest="label", default=None, - help="dictionary file") - parser.add_option("-i", "--data", action="store", - dest="data", help="data file to predict") - parser.add_option("-w", "--model", action="store", - dest="model_path", default=None, - help="model path") + parser.add_option( + "-n", + "--tconf", + action="store", + dest="train_conf", + help="network config") + parser.add_option( + "-d", + "--dict", + action="store", + dest="dict_file", + help="dictionary file") + parser.add_option( + "-b", + "--label", + action="store", + dest="label", + default=None, + help="dictionary file") + parser.add_option( + "-i", + "--data", + action="store", + dest="data", + help="data file to predict") + parser.add_option( + "-w", + "--model", + action="store", + dest="model_path", + default=None, + help="model path") return parser.parse_args() + def main(): options, args = option_parser() train_conf = options.train_conf @@ -119,5 +145,6 @@ def main(): predict = SentimentPrediction(train_conf, dict_file, model_path, label) predict.predict(data) + if __name__ == '__main__': main() diff --git a/demo/sentiment/preprocess.py b/demo/sentiment/preprocess.py index 49b53d500a1bf..7146e95d751c4 100755 --- a/demo/sentiment/preprocess.py +++ b/demo/sentiment/preprocess.py @@ -22,13 +22,13 @@ from optparse import OptionParser from paddle.utils.preprocess_util import * - """ Usage: run following command to show help message. python preprocess.py -h """ -def save_dict(dict, filename, is_reverse = True): + +def save_dict(dict, filename, is_reverse=True): """ Save dictionary into file. dict: input dictionary. @@ -39,9 +39,10 @@ def save_dict(dict, filename, is_reverse = True): f = open(filename, 'w') for k, v in sorted(dict.items(), key=operator.itemgetter(1),\ reverse=is_reverse): - f.write('%s\t%s\n'%(k, v)) + f.write('%s\t%s\n' % (k, v)) f.close() + def tokenize(sentences): """ Use tokenizer.perl to tokenize input sentences. @@ -58,6 +59,7 @@ def tokenize(sentences): toks = tok_text.split('\n')[:-1] return toks + def read_lines(path): """ path: String, file path. @@ -71,12 +73,17 @@ def read_lines(path): seqs.append(line) return seqs + class SentimentDataSetCreate(): """ A class to process data for sentiment analysis task. """ - def __init__(self, data_path, output_path, - use_okenizer = True, multi_lines = False): + + def __init__(self, + data_path, + output_path, + use_okenizer=True, + multi_lines=False): """ data_path: string, traing and testing dataset path output_path: string, output path, store processed dataset @@ -164,23 +171,17 @@ def create_dataset(self): # Preprocess train data. train_data, train_lab_set = self.data_list(self.train_dir) print "processing train set..." - file_lists = self.save_data(train_data, - "train", - self.batch_size, - True, - True) + file_lists = self.save_data(train_data, "train", self.batch_size, True, + True) save_list(file_lists, self.train_list) # If have test data path, preprocess test data. if os.path.exists(self.test_dir): test_data, test_lab_set = self.data_list(self.test_dir) - assert(train_lab_set == test_lab_set) + assert (train_lab_set == test_lab_set) print "processing test set..." - file_lists = self.save_data(test_data, - "test", - self.batch_size, - False, - self.dict_with_test) + file_lists = self.save_data(test_data, "test", self.batch_size, + False, self.dict_with_test) save_list(file_lists, self.test_list) # save labels set. @@ -191,7 +192,9 @@ def create_dataset(self): save_dict(self.word_count, self.dict_file, True) self.dict_size = len(self.word_count) - def save_data(self, data, prefix = "", + def save_data(self, + data, + prefix="", batch_size=50000, is_shuffle=False, build_dict=False): @@ -205,7 +208,8 @@ def save_data(self, data, prefix = "", return: list of batch names """ if is_shuffle and self.multi_lines: - return self.save_data_multi_lines(data, prefix, batch_size, build_dict) + return self.save_data_multi_lines(data, prefix, batch_size, + build_dict) if is_shuffle: random.shuffle(data) @@ -213,7 +217,7 @@ def save_data(self, data, prefix = "", batch_names = [] for i in range(num_batches): batch_name = join_path(self.output_path, - "%s_part_%03d" %(prefix, i)) + "%s_part_%03d" % (prefix, i)) begin = i * batch_size end = min((i + 1) * batch_size, len(data)) # read a batch of data @@ -246,7 +250,9 @@ def get_data_list(self, begin, end, data): data_list = tokenize(data_list) return label_list, data_list - def save_data_multi_lines(self, data, prefix = "", + def save_data_multi_lines(self, + data, + prefix="", batch_size=50000, build_dict=False): """ @@ -274,14 +280,14 @@ def save_data_multi_lines(self, data, prefix = "", self.create_dict(data_list) length = len(label_list) - perm_list = np.array([ i for i in xrange(length) ]) + perm_list = np.array([i for i in xrange(length)]) random.shuffle(perm_list) num_batches = int(math.ceil(length / float(batch_size))) batch_names = [] for i in range(num_batches): batch_name = join_path(self.output_path, - "%s_part_%03d" %(prefix, i)) + "%s_part_%03d" % (prefix, i)) begin = i * batch_size end = min((i + 1) * batch_size, length) sub_label = [label_list[perm_list[i]] for i in range(begin, end)] @@ -304,35 +310,50 @@ def save_file(self, label_list, data_list, filename): f.write('%s\t\t%s\n' % (lab, seq)) f.close() + def option_parser(): parser = OptionParser(usage="usage: python preprcoess.py "\ "-i data_dir [options]") - parser.add_option("-i", "--data", action="store", - dest="input", help="Input data directory.") - parser.add_option("-o", "--output", action="store", - dest="output", default=None, - help="Output directory.") - parser.add_option("-t", "--tokenizer", action="store", - dest="use_tokenizer", default=True, - help="Whether to use tokenizer.") + parser.add_option( + "-i", + "--data", + action="store", + dest="input", + help="Input data directory.") + parser.add_option( + "-o", + "--output", + action="store", + dest="output", + default=None, + help="Output directory.") + parser.add_option( + "-t", + "--tokenizer", + action="store", + dest="use_tokenizer", + default=True, + help="Whether to use tokenizer.") parser.add_option("-m", "--multi_lines", action="store", dest="multi_lines", default=False, help="If input text files have multi lines and they "\ "need to be shuffled, you should set -m True,") return parser.parse_args() + def main(): options, args = option_parser() - data_dir=options.input - output_dir=options.output - use_tokenizer=options.use_tokenizer - multi_lines=options.multi_lines + data_dir = options.input + output_dir = options.output + use_tokenizer = options.use_tokenizer + multi_lines = options.multi_lines if output_dir is None: outname = os.path.basename(options.input) output_dir = join_path(os.path.dirname(data_dir), 'pre-' + outname) - data_creator = SentimentDataSetCreate(data_dir, output_dir, - use_tokenizer, multi_lines) + data_creator = SentimentDataSetCreate(data_dir, output_dir, use_tokenizer, + multi_lines) data_creator.create_dataset() + if __name__ == '__main__': main() diff --git a/demo/sentiment/sentiment_net.py b/demo/sentiment/sentiment_net.py index 31e585edcaa11..ff6a3624a404c 100644 --- a/demo/sentiment/sentiment_net.py +++ b/demo/sentiment/sentiment_net.py @@ -47,10 +47,12 @@ def sentiment_data(data_dir=None, for i, line in enumerate(open(dict_file, 'r')): word_dict[line.split('\t')[0]] = i - define_py_data_sources2(train_list, test_list, - module="dataprovider", - obj="process", - args={'dictionary': word_dict}) + define_py_data_sources2( + train_list, + test_list, + module="dataprovider", + obj="process", + args={'dictionary': word_dict}) return dict_dim, class_dim @@ -64,8 +66,7 @@ def bidirectional_lstm_net(input_dim, emb = embedding_layer(input=data, size=emb_dim) bi_lstm = bidirectional_lstm(input=emb, size=lstm_dim) dropout = dropout_layer(input=bi_lstm, dropout_rate=0.5) - output = fc_layer(input=dropout, size=class_dim, - act=SoftmaxActivation()) + output = fc_layer(input=dropout, size=class_dim, act=SoftmaxActivation()) if not is_predict: lbl = data_layer("label", 1) @@ -109,27 +110,36 @@ def stacked_lstm_net(input_dim, data = data_layer("word", input_dim) emb = embedding_layer(input=data, size=emb_dim) - fc1 = fc_layer(input=emb, size=hid_dim, act=linear, - bias_attr=bias_attr) - lstm1 = lstmemory(input=fc1, act=relu, bias_attr=bias_attr, - layer_attr=layer_attr) + fc1 = fc_layer(input=emb, size=hid_dim, act=linear, bias_attr=bias_attr) + lstm1 = lstmemory( + input=fc1, act=relu, bias_attr=bias_attr, layer_attr=layer_attr) inputs = [fc1, lstm1] for i in range(2, stacked_num + 1): - fc = fc_layer(input=inputs, size=hid_dim, act=linear, - param_attr=para_attr, bias_attr=bias_attr) - lstm = lstmemory(input=fc, reverse=(i % 2) == 0, act=relu, - bias_attr=bias_attr, layer_attr=layer_attr) + fc = fc_layer( + input=inputs, + size=hid_dim, + act=linear, + param_attr=para_attr, + bias_attr=bias_attr) + lstm = lstmemory( + input=fc, + reverse=(i % 2) == 0, + act=relu, + bias_attr=bias_attr, + layer_attr=layer_attr) inputs = [fc, lstm] fc_last = pooling_layer(input=inputs[0], pooling_type=MaxPooling()) lstm_last = pooling_layer(input=inputs[1], pooling_type=MaxPooling()) - output = fc_layer(input=[fc_last, lstm_last], size=class_dim, - act=SoftmaxActivation(), - bias_attr=bias_attr, param_attr=para_attr) + output = fc_layer( + input=[fc_last, lstm_last], + size=class_dim, + act=SoftmaxActivation(), + bias_attr=bias_attr, + param_attr=para_attr) if is_predict: outputs(output) else: - outputs( - classification_cost(input=output, label=data_layer('label', 1))) + outputs(classification_cost(input=output, label=data_layer('label', 1))) diff --git a/demo/sentiment/trainer_config.py b/demo/sentiment/trainer_config.py index db24182a8d735..894070e7c97dc 100644 --- a/demo/sentiment/trainer_config.py +++ b/demo/sentiment/trainer_config.py @@ -20,20 +20,19 @@ # whether this config is used for prediction is_predict = get_config_arg('is_predict', bool, False) -data_dir = "./data/pre-imdb" +data_dir = "./data/pre-imdb" dict_dim, class_dim = sentiment_data(data_dir, is_test, is_predict) ################## Algorithm Config ##################### settings( - batch_size=128, - learning_rate=2e-3, - learning_method=AdamOptimizer(), - regularization=L2Regularization(8e-4), - gradient_clipping_threshold=25 -) + batch_size=128, + learning_rate=2e-3, + learning_method=AdamOptimizer(), + regularization=L2Regularization(8e-4), + gradient_clipping_threshold=25) #################### Network Config ###################### -stacked_lstm_net(dict_dim, class_dim=class_dim, - stacked_num=3, is_predict=is_predict) +stacked_lstm_net( + dict_dim, class_dim=class_dim, stacked_num=3, is_predict=is_predict) # bidirectional_lstm_net(dict_dim, class_dim=class_dim, is_predict=is_predict) diff --git a/demo/seqToseq/dataprovider.py b/demo/seqToseq/dataprovider.py index df19db109ed22..c5da1b7685f47 100755 --- a/demo/seqToseq/dataprovider.py +++ b/demo/seqToseq/dataprovider.py @@ -30,14 +30,14 @@ def hook(settings, src_dict, trg_dict, file_list, **kwargs): if settings.job_mode: settings.trg_dict = trg_dict settings.slots = [ - integer_value_sequence(len(settings.src_dict)), - integer_value_sequence(len(settings.trg_dict)), + integer_value_sequence(len(settings.src_dict)), + integer_value_sequence(len(settings.trg_dict)), integer_value_sequence(len(settings.trg_dict)) ] settings.logger.info("trg dict len : %d" % (len(settings.trg_dict))) else: settings.slots = [ - integer_value_sequence(len(settings.src_dict)), + integer_value_sequence(len(settings.src_dict)), integer_value_sequence(len(open(file_list[0], "r").readlines())) ] @@ -62,8 +62,7 @@ def process(settings, file_name): if settings.job_mode: trg_seq = line_split[1] # one target sequence trg_words = trg_seq.split() - trg_ids = [settings.trg_dict.get(w, UNK_IDX) - for w in trg_words] + trg_ids = [settings.trg_dict.get(w, UNK_IDX) for w in trg_words] # remove sequence whose length > 80 in training mode if len(src_ids) > 80 or len(trg_ids) > 80: diff --git a/demo/seqToseq/preprocess.py b/demo/seqToseq/preprocess.py index 5efb17a664b9a..bd1c51b1514b7 100755 --- a/demo/seqToseq/preprocess.py +++ b/demo/seqToseq/preprocess.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Example: python preprocess.py -i INPUT [-d DICTSIZE] [-m] @@ -24,12 +23,13 @@ -m --mergeDict merge source and target dictionary """ import os -import sys +import sys import string from optparse import OptionParser from paddle.utils.preprocess_util import save_list, DatasetCreater + class SeqToSeqDatasetCreater(DatasetCreater): """ A class to process data for sequence to sequence application. @@ -75,7 +75,7 @@ def cat_file(self, dir_path, suffix, output_path, output): if not os.path.exists(output): os.system(cmd + '> ' + output) - def build_dict(self, file_path, dict_path, dict_size = -1): + def build_dict(self, file_path, dict_path, dict_size=-1): """ Create the dictionary for the file, Note that 1. Valid characters include all printable characters @@ -99,20 +99,23 @@ def build_dict(self, file_path, dict_path, dict_size = -1): for word in words: if word not in dictory: dictory[word] = 1 - else: + else: dictory[word] += 1 output = open(dict_path, "w+") output.write('\n\n\n') count = 3 - for key, value in sorted(dictory.items(), key = lambda d:d[1], reverse = True): + for key, value in sorted( + dictory.items(), key=lambda d: d[1], reverse=True): output.write(key + "\n") count += 1 if count == dict_size: break self.dict_size = count - - def create_dataset(self, dict_size = -1, mergeDict = False, - suffixes = ['.src', '.trg']): + + def create_dataset(self, + dict_size=-1, + mergeDict=False, + suffixes=['.src', '.trg']): """ Create seqToseq dataset """ @@ -135,13 +138,14 @@ def create_dataset(self, dict_size = -1, mergeDict = False, # checkout dataset should be parallel corpora suffix_len = len(suffixes[0]) for dataset in dataset_list: - file_list = os.listdir(dataset) - if len(file_list) % 2 == 1: - raise RuntimeError("dataset should be parallel corpora") - file_list.sort() - for i in range(0, len(file_list), 2): - if file_list[i][:-suffix_len] != file_list[i + 1][:-suffix_len]: - raise RuntimeError("source and target file name should be equal") + file_list = os.listdir(dataset) + if len(file_list) % 2 == 1: + raise RuntimeError("dataset should be parallel corpora") + file_list.sort() + for i in range(0, len(file_list), 2): + if file_list[i][:-suffix_len] != file_list[i + 1][:-suffix_len]: + raise RuntimeError( + "source and target file name should be equal") # cat all the files with the same suffix in dataset for suffix in suffixes: @@ -155,16 +159,18 @@ def create_dataset(self, dict_size = -1, mergeDict = False, list = ['train.list', 'test.list', 'gen.list'] for dataset in dataset_list: outname = os.path.basename(dataset) - self.concat_file(dataset, outname + suffixes[0], + self.concat_file(dataset, outname + suffixes[0], outname + suffixes[1], dir_list[id], outname) - save_list([os.path.join(dir_list[id], outname)], + save_list([os.path.join(dir_list[id], outname)], os.path.join(self.output_path, list[id])) id += 1 # build dictionary for train data dict = ['src.dict', 'trg.dict'] - dict_path = [os.path.join(self.output_path, dict[0]), - os.path.join(self.output_path, dict[1])] + dict_path = [ + os.path.join(self.output_path, dict[0]), + os.path.join(self.output_path, dict[1]) + ] if mergeDict: outname = os.path.join(train_dir, train_dataset.split('/')[-1]) print 'build src dictionary for train data' @@ -173,22 +179,30 @@ def create_dataset(self, dict_size = -1, mergeDict = False, os.system('cp ' + dict_path[0] + ' ' + dict_path[1]) else: outname = os.path.join(train_dataset, self.train_dir_name) - for id in range(0,2): + for id in range(0, 2): suffix = suffixes[id] print 'build ' + suffix[1:] + ' dictionary for train data' self.build_dict(outname + suffix, dict_path[id], dict_size) print 'dictionary size is', self.dict_size + def main(): usage = "usage: \n" \ "python %prog -i INPUT [-d DICTSIZE] [-m]" parser = OptionParser(usage) - parser.add_option("-i", action="store", dest="input", - help="input original dataset path") - parser.add_option("-d", action="store", dest="dictsize", - help="specified word count of dictionary") - parser.add_option("-m", "--mergeDict", action="store_true", dest="mergeDict", - help="merge source and target dictionary") + parser.add_option( + "-i", action="store", dest="input", help="input original dataset path") + parser.add_option( + "-d", + action="store", + dest="dictsize", + help="specified word count of dictionary") + parser.add_option( + "-m", + "--mergeDict", + action="store_true", + dest="mergeDict", + help="merge source and target dictionary") (options, args) = parser.parse_args() if options.input[-1] == os.path.sep: options.input = options.input[:-1] @@ -200,5 +214,6 @@ def main(): data_creator = SeqToSeqDatasetCreater(options.input, output_path) data_creator.create_dataset(dictsize, options.mergeDict) + if __name__ == "__main__": - main(); + main() diff --git a/demo/seqToseq/seqToseq_net.py b/demo/seqToseq/seqToseq_net.py index edd6ad3f739b6..ad5e3339c1461 100644 --- a/demo/seqToseq/seqToseq_net.py +++ b/demo/seqToseq/seqToseq_net.py @@ -50,16 +50,21 @@ def seq_to_seq_data(data_dir, trg_dict = None else: train_list = os.path.join(data_dir, train_list) - test_list = os.path.join(data_dir,test_list) + test_list = os.path.join(data_dir, test_list) - define_py_data_sources2(train_list, test_list, - module = "dataprovider", - obj = "process", - args = {"src_dict": src_dict, - "trg_dict": trg_dict}) + define_py_data_sources2( + train_list, + test_list, + module="dataprovider", + obj="process", + args={"src_dict": src_dict, + "trg_dict": trg_dict}) - return {"src_dict_path": src_lang_dict, "trg_dict_path": trg_lang_dict, - "gen_result": gen_result} + return { + "src_dict_path": src_lang_dict, + "trg_dict_path": trg_lang_dict, + "gen_result": gen_result + } def gru_encoder_decoder(data_conf, @@ -90,51 +95,55 @@ def gru_encoder_decoder(data_conf, size=word_vector_dim, param_attr=ParamAttr(name='_source_language_embedding')) src_forward = simple_gru(input=src_embedding, size=encoder_size) - src_backward = simple_gru(input=src_embedding, - size=encoder_size, - reverse=True) + src_backward = simple_gru( + input=src_embedding, size=encoder_size, reverse=True) encoded_vector = concat_layer(input=[src_forward, src_backward]) with mixed_layer(size=decoder_size) as encoded_proj: encoded_proj += full_matrix_projection(input=encoded_vector) backward_first = first_seq(input=src_backward) - with mixed_layer(size=decoder_size, - act=TanhActivation(), ) as decoder_boot: + with mixed_layer( + size=decoder_size, + act=TanhActivation(), ) as decoder_boot: decoder_boot += full_matrix_projection(input=backward_first) def gru_decoder_with_attention(enc_vec, enc_proj, current_word): - decoder_mem = memory(name='gru_decoder', - size=decoder_size, - boot_layer=decoder_boot) + decoder_mem = memory( + name='gru_decoder', size=decoder_size, boot_layer=decoder_boot) - context = simple_attention(encoded_sequence=enc_vec, - encoded_proj=enc_proj, - decoder_state=decoder_mem, ) + context = simple_attention( + encoded_sequence=enc_vec, + encoded_proj=enc_proj, + decoder_state=decoder_mem, ) with mixed_layer(size=decoder_size * 3) as decoder_inputs: decoder_inputs += full_matrix_projection(input=context) decoder_inputs += full_matrix_projection(input=current_word) - gru_step = gru_step_layer(name='gru_decoder', - input=decoder_inputs, - output_mem=decoder_mem, - size=decoder_size) + gru_step = gru_step_layer( + name='gru_decoder', + input=decoder_inputs, + output_mem=decoder_mem, + size=decoder_size) - with mixed_layer(size=target_dict_dim, - bias_attr=True, - act=SoftmaxActivation()) as out: + with mixed_layer( + size=target_dict_dim, bias_attr=True, + act=SoftmaxActivation()) as out: out += full_matrix_projection(input=gru_step) return out decoder_group_name = "decoder_group" - group_inputs=[StaticInput(input=encoded_vector,is_seq=True), - StaticInput(input=encoded_proj,is_seq=True)] + group_inputs = [ + StaticInput( + input=encoded_vector, is_seq=True), StaticInput( + input=encoded_proj, is_seq=True) + ] if not is_generating: trg_embedding = embedding_layer( - input=data_layer(name='target_language_word', - size=target_dict_dim), + input=data_layer( + name='target_language_word', size=target_dict_dim), size=word_vector_dim, param_attr=ParamAttr(name='_target_language_embedding')) group_inputs.append(trg_embedding) @@ -144,12 +153,12 @@ def gru_decoder_with_attention(enc_vec, enc_proj, current_word): # while encoded source sequence is accessed to as an unbounded memory. # Here, the StaticInput defines a read-only memory # for the recurrent_group. - decoder = recurrent_group(name=decoder_group_name, - step=gru_decoder_with_attention, - input=group_inputs) + decoder = recurrent_group( + name=decoder_group_name, + step=gru_decoder_with_attention, + input=group_inputs) - lbl = data_layer(name='target_language_next_word', - size=target_dict_dim) + lbl = data_layer(name='target_language_next_word', size=target_dict_dim) cost = classification_cost(input=decoder, label=lbl) outputs(cost) else: @@ -168,16 +177,19 @@ def gru_decoder_with_attention(enc_vec, enc_proj, current_word): embedding_size=word_vector_dim) group_inputs.append(trg_embedding) - beam_gen = beam_search(name=decoder_group_name, - step=gru_decoder_with_attention, - input=group_inputs, - bos_id=0, - eos_id=1, - beam_size=beam_size, - max_length=max_length) - - seqtext_printer_evaluator(input=beam_gen, - id_input=data_layer(name="sent_id", size=1), - dict_file=trg_dict_path, - result_file=gen_trans_file) + beam_gen = beam_search( + name=decoder_group_name, + step=gru_decoder_with_attention, + input=group_inputs, + bos_id=0, + eos_id=1, + beam_size=beam_size, + max_length=max_length) + + seqtext_printer_evaluator( + input=beam_gen, + id_input=data_layer( + name="sent_id", size=1), + dict_file=trg_dict_path, + result_file=gen_trans_file) outputs(beam_gen) diff --git a/demo/sequence_tagging/dataprovider.py b/demo/sequence_tagging/dataprovider.py index 6f412d6834be6..37dcb7aa17c0a 100644 --- a/demo/sequence_tagging/dataprovider.py +++ b/demo/sequence_tagging/dataprovider.py @@ -17,8 +17,7 @@ import logging logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', -) + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', ) logger = logging.getLogger('paddle') logger.setLevel(logging.INFO) @@ -32,59 +31,58 @@ # [[-1,0], [0,0]] means previous token at column 0 and current token at # column 0 are combined as one feature. patterns = [ - [[-2,0]], - [[-1,0]], - [[0,0]], - [[1,0]], - [[2,0]], - - [[-1,0], [0,0]], - [[0,0], [1,0]], - - [[-2,1]], - [[-1,1]], - [[0,1]], - [[1,1]], - [[2,1]], - [[-2,1], [-1,1]], - [[-1,1], [0,1]], - [[0,1], [1,1]], - [[1,1], [2,1]], - - [[-2,1], [-1,1], [0,1]], - [[-1,1], [0,1], [1,1]], - [[0,1], [1,1], [2,1]], + [[-2, 0]], + [[-1, 0]], + [[0, 0]], + [[1, 0]], + [[2, 0]], + [[-1, 0], [0, 0]], + [[0, 0], [1, 0]], + [[-2, 1]], + [[-1, 1]], + [[0, 1]], + [[1, 1]], + [[2, 1]], + [[-2, 1], [-1, 1]], + [[-1, 1], [0, 1]], + [[0, 1], [1, 1]], + [[1, 1], [2, 1]], + [[-2, 1], [-1, 1], [0, 1]], + [[-1, 1], [0, 1], [1, 1]], + [[0, 1], [1, 1], [2, 1]], ] dict_label = { - 'B-ADJP': 0, - 'I-ADJP': 1, - 'B-ADVP': 2, - 'I-ADVP': 3, - 'B-CONJP': 4, - 'I-CONJP': 5, - 'B-INTJ': 6, - 'I-INTJ': 7, - 'B-LST': 8, - 'I-LST': 9, - 'B-NP': 10, - 'I-NP': 11, - 'B-PP': 12, - 'I-PP': 13, - 'B-PRT': 14, - 'I-PRT': 15, - 'B-SBAR': 16, - 'I-SBAR': 17, - 'B-UCP': 18, - 'I-UCP': 19, - 'B-VP': 20, - 'I-VP': 21, - 'O': 22 + 'B-ADJP': 0, + 'I-ADJP': 1, + 'B-ADVP': 2, + 'I-ADVP': 3, + 'B-CONJP': 4, + 'I-CONJP': 5, + 'B-INTJ': 6, + 'I-INTJ': 7, + 'B-LST': 8, + 'I-LST': 9, + 'B-NP': 10, + 'I-NP': 11, + 'B-PP': 12, + 'I-PP': 13, + 'B-PRT': 14, + 'I-PRT': 15, + 'B-SBAR': 16, + 'I-SBAR': 17, + 'B-UCP': 18, + 'I-UCP': 19, + 'B-VP': 20, + 'I-VP': 21, + 'O': 22 } + def make_features(sequence): length = len(sequence) num_features = len(sequence[0]) + def get_features(pos): if pos < 0: return ['#B%s' % -pos] * num_features @@ -94,9 +92,10 @@ def get_features(pos): for i in xrange(length): for pattern in patterns: - fname = '/'.join([get_features(i+pos)[f] for pos, f in pattern]) + fname = '/'.join([get_features(i + pos)[f] for pos, f in pattern]) sequence[i].append(fname) + ''' Source file format: Each line is for one timestep. The features are separated by space. @@ -109,6 +108,8 @@ def get_features(pos): return a list of dict for each column ''' + + def create_dictionaries(filename, cutoff, oov_policy): def add_to_dict(sequence, dicts): num_features = len(dicts) @@ -140,7 +141,6 @@ def add_to_dict(sequence, dicts): features = line.split(' ') sequence.append(features) - for i in xrange(num_features): dct = dicts[i] n = 1 if oov_policy[i] == OOV_POLICY_USE else 0 @@ -151,7 +151,7 @@ def add_to_dict(sequence, dicts): else: dct[k] = n n += 1 - + if oov_policy[i] == OOV_POLICY_USE: # placeholder so that len(dct) will be the number of features # including OOV @@ -187,12 +187,15 @@ def initializer(settings, **xargs): logger.info("feature size=%s" % dim) settings.input_types = input_types + ''' if oov_policy[i] == OOV_POLICY_USE, features in i-th column which are not existed in dicts[i] will be assigned to id 0. if oov_policy[i] == OOV_POLICY_ERROR, all features in i-th column MUST exist in dicts[i]. ''' + + @provider(init_hook=initializer, cache=CacheType.CACHE_PASS_IN_MEM) def process(settings, filename): input_file = filename @@ -231,7 +234,7 @@ def gen_sample(sequence): logger.fatal("Unknown token: %s" % features[i]) else: vec.ids.append(dim + 0) - + dim += len(dicts[i]) sample[-1].append(vec) return sample @@ -255,4 +258,3 @@ def gen_sample(sequence): f.close() logger.info("num_sequences=%s" % num_sequences) - diff --git a/demo/sequence_tagging/linear_crf.py b/demo/sequence_tagging/linear_crf.py index 2bd1a20bc52fc..64895742e1b8c 100644 --- a/demo/sequence_tagging/linear_crf.py +++ b/demo/sequence_tagging/linear_crf.py @@ -16,11 +16,11 @@ import math -define_py_data_sources2(train_list="data/train.list", - test_list="data/test.list", - module="dataprovider", - obj="process") - +define_py_data_sources2( + train_list="data/train.list", + test_list="data/test.list", + module="dataprovider", + obj="process") batch_size = 1 settings( @@ -30,14 +30,15 @@ average_window=0.5, learning_rate=1e-1, learning_rate_decay_a=1e-5, - learning_rate_decay_b=0.25, -) + learning_rate_decay_b=0.25, ) + +num_label_types = 23 -num_label_types=23 def get_simd_size(size): return int(math.ceil(float(size) / 8)) * 8 + # Currently, in order to use sparse_update=True, # the size has to be aligned. num_label_types = get_simd_size(num_label_types) @@ -45,40 +46,37 @@ def get_simd_size(size): features = data_layer(name="features", size=76328) word = data_layer(name="word", size=6778) pos = data_layer(name="pos", size=44) -chunk = data_layer(name="chunk", - size=num_label_types) +chunk = data_layer(name="chunk", size=num_label_types) crf_input = fc_layer( input=features, size=num_label_types, act=LinearActivation(), bias_attr=False, - param_attr=ParamAttr(initial_std=0, sparse_update=True)) + param_attr=ParamAttr( + initial_std=0, sparse_update=True)) -crf=crf_layer( +crf = crf_layer( input=crf_input, label=chunk, - param_attr=ParamAttr(name="crfw", initial_std=0), -) + param_attr=ParamAttr( + name="crfw", initial_std=0), ) -crf_decoding=crf_decoding_layer( +crf_decoding = crf_decoding_layer( size=num_label_types, input=crf_input, label=chunk, - param_attr=ParamAttr(name="crfw"), -) + param_attr=ParamAttr(name="crfw"), ) sum_evaluator( name="error", - input=crf_decoding, -) + input=crf_decoding, ) chunk_evaluator( name="chunk_f1", - input =[crf_decoding, chunk], + input=[crf_decoding, chunk], chunk_scheme="IOB", - num_chunk_types=11, -) + num_chunk_types=11, ) inputs(word, pos, chunk, features) outputs(crf) diff --git a/demo/sequence_tagging/rnn_crf.py b/demo/sequence_tagging/rnn_crf.py index fb157bf3ea719..90d4bbdddfdb4 100644 --- a/demo/sequence_tagging/rnn_crf.py +++ b/demo/sequence_tagging/rnn_crf.py @@ -16,10 +16,11 @@ import math -define_py_data_sources2(train_list="data/train.list", - test_list="data/test.list", - module="dataprovider", - obj="process") +define_py_data_sources2( + train_list="data/train.list", + test_list="data/test.list", + module="dataprovider", + obj="process") batch_size = 16 settings( @@ -27,29 +28,27 @@ batch_size=batch_size, regularization=L2Regularization(batch_size * 1e-5), average_window=0.5, - learning_rate = 2e-3, - learning_rate_decay_a = 5e-7, - learning_rate_decay_b = 0.5, -) + learning_rate=2e-3, + learning_rate_decay_a=5e-7, + learning_rate_decay_b=0.5, ) -word_dim=128 +word_dim = 128 hidden_dim = 128 with_rnn = True -initial_std=1/math.sqrt(hidden_dim) -param_attr=ParamAttr(initial_std=initial_std) -cpu_layer_attr=ExtraLayerAttribute(device=-1) +initial_std = 1 / math.sqrt(hidden_dim) +param_attr = ParamAttr(initial_std=initial_std) +cpu_layer_attr = ExtraLayerAttribute(device=-1) default_device(0) -num_label_types=23 +num_label_types = 23 features = data_layer(name="features", size=76328) word = data_layer(name="word", size=6778) pos = data_layer(name="pos", size=44) -chunk = data_layer(name="chunk", - size=num_label_types, - layer_attr=cpu_layer_attr) +chunk = data_layer( + name="chunk", size=num_label_types, layer_attr=cpu_layer_attr) emb = embedding_layer( input=word, size=word_dim, param_attr=ParamAttr(initial_std=0)) @@ -58,73 +57,64 @@ size=hidden_dim, act=STanhActivation(), bias_attr=True, - input=[full_matrix_projection(emb), - table_projection(pos, param_attr=param_attr)] -) + input=[ + full_matrix_projection(emb), table_projection( + pos, param_attr=param_attr) + ]) if with_rnn: rnn1 = recurrent_layer( act=ReluActivation(), bias_attr=True, input=hidden1, - param_attr=ParamAttr(initial_std=0), - ) + param_attr=ParamAttr(initial_std=0), ) hidden2 = mixed_layer( size=hidden_dim, act=STanhActivation(), bias_attr=True, - input=[full_matrix_projection(hidden1) - ] + ([ - full_matrix_projection(rnn1, param_attr=ParamAttr(initial_std=0)) - ] if with_rnn else []), -) + input=[full_matrix_projection(hidden1)] + + ([full_matrix_projection( + rnn1, param_attr=ParamAttr(initial_std=0))] if with_rnn else []), ) if with_rnn: - rnn2=recurrent_layer( + rnn2 = recurrent_layer( reverse=True, act=ReluActivation(), bias_attr=True, input=hidden2, - param_attr=ParamAttr(initial_std=0), - ) + param_attr=ParamAttr(initial_std=0), ) crf_input = mixed_layer( size=num_label_types, bias_attr=False, - input=[ - full_matrix_projection(hidden2), - ] + ([ - full_matrix_projection(rnn2, param_attr=ParamAttr(initial_std=0)) - ] if with_rnn else []), -) + input=[full_matrix_projection(hidden2), ] + + ([full_matrix_projection( + rnn2, param_attr=ParamAttr(initial_std=0))] if with_rnn else []), ) crf = crf_layer( input=crf_input, label=chunk, - param_attr=ParamAttr(name="crfw", initial_std=0), - layer_attr=cpu_layer_attr, -) + param_attr=ParamAttr( + name="crfw", initial_std=0), + layer_attr=cpu_layer_attr, ) crf_decoding = crf_decoding_layer( size=num_label_types, input=crf_input, label=chunk, param_attr=ParamAttr(name="crfw"), - layer_attr=cpu_layer_attr, -) + layer_attr=cpu_layer_attr, ) sum_evaluator( name="error", - input=crf_decoding, -) + input=crf_decoding, ) chunk_evaluator( name="chunk_f1", - input =[crf_decoding, chunk], + input=[crf_decoding, chunk], chunk_scheme="IOB", - num_chunk_types=11, -) + num_chunk_types=11, ) inputs(word, pos, chunk, features) outputs(crf) diff --git a/doc/ui/predict/predict_sample.py b/doc/ui/predict/predict_sample.py index d55d2c730dece..63e8b36d26057 100644 --- a/doc/ui/predict/predict_sample.py +++ b/doc/ui/predict/predict_sample.py @@ -16,82 +16,113 @@ from paddle.trainer.PyDataProvider2 import dense_vector from paddle.trainer.config_parser import parse_config -TEST_DATA = [[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.215686, - 0.533333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.67451, - 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.070588, 0.886275, - 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.192157, 0.070588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.670588, 0.992157, 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.117647, 0.933333, 0.858824, 0.313725, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0.090196, 0.858824, 0.992157, 0.831373, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.141176, - 0.992157, 0.992157, 0.611765, 0.054902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.258824, 0.992157, 0.992157, - 0.529412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.368627, 0.992157, 0.992157, 0.419608, 0.003922, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0.094118, 0.835294, 0.992157, 0.992157, 0.517647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.603922, 0.992157, - 0.992157, 0.992157, 0.603922, 0.545098, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0.447059, 0.992157, 0.992157, - 0.956863, 0.062745, 0, 0, 0, 0, 0, 0, 0, 0, 0.011765, 0.666667, 0.992157, 0.992157, 0.992157, 0.992157, - 0.992157, 0.745098, 0.137255, 0, 0, 0, 0, 0, 0.152941, 0.866667, 0.992157, 0.992157, 0.521569, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0.070588, 0.992157, 0.992157, 0.992157, 0.803922, 0.352941, 0.745098, 0.992157, - 0.945098, 0.317647, 0, 0, 0, 0, 0.580392, 0.992157, 0.992157, 0.764706, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0.070588, 0.992157, 0.992157, 0.776471, 0.043137, 0, 0.007843, 0.27451, 0.882353, 0.941176, 0.176471, - 0, 0, 0.180392, 0.898039, 0.992157, 0.992157, 0.313725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.070588, 0.992157, - 0.992157, 0.713725, 0, 0, 0, 0, 0.627451, 0.992157, 0.729412, 0.062745, 0, 0.509804, 0.992157, 0.992157, - 0.776471, 0.035294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.494118, 0.992157, 0.992157, 0.968627, 0.168627, 0, 0, - 0, 0.423529, 0.992157, 0.992157, 0.364706, 0, 0.717647, 0.992157, 0.992157, 0.317647, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0.533333, 0.992157, 0.984314, 0.945098, 0.603922, 0, 0, 0, 0.003922, 0.466667, 0.992157, - 0.988235, 0.976471, 0.992157, 0.992157, 0.788235, 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.686275, - 0.882353, 0.364706, 0, 0, 0, 0, 0, 0, 0.098039, 0.588235, 0.992157, 0.992157, 0.992157, 0.980392, - 0.305882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.101961, 0.67451, 0.321569, 0, 0, 0, 0, 0, 0, 0, 0.105882, - 0.733333, 0.976471, 0.811765, 0.713725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.65098, 0.992157, - 0.321569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.25098, 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, - 0.94902, 0.219608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.968627, - 0.764706, 0.152941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.498039, - 0.25098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], [ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.298039, 0.333333, 0.333333, 0.333333, 0.337255, 0.333333, - 0.333333, 0.109804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.027451, 0.223529, 0.776471, - 0.964706, 0.988235, 0.988235, 0.988235, 0.992157, 0.988235, 0.988235, 0.780392, 0.098039, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.14902, 0.698039, 0.988235, 0.992157, 0.988235, 0.901961, 0.87451, - 0.568627, 0.882353, 0.976471, 0.988235, 0.988235, 0.501961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.188235, 0.647059, 0.988235, 0.988235, 0.745098, 0.439216, 0.098039, 0, 0, 0, 0.572549, 0.988235, - 0.988235, 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.2, 0.933333, 0.992157, 0.941176, - 0.247059, 0, 0, 0, 0, 0, 0, 0.188235, 0.898039, 0.992157, 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.039216, 0.639216, 0.933333, 0.988235, 0.913725, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0.113725, 0.843137, - 0.988235, 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.235294, 0.988235, 0.992157, 0.988235, 0.815686, - 0.07451, 0, 0, 0, 0, 0, 0, 0, 0.333333, 0.988235, 0.988235, 0.552941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.211765, 0.878431, 0.988235, 0.992157, 0.701961, 0.329412, 0.109804, 0, 0, 0, 0, 0, 0, 0, 0.698039, - 0.988235, 0.913725, 0.145098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.188235, 0.890196, 0.988235, 0.988235, - 0.745098, 0.047059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.882353, 0.988235, 0.568627, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0.2, 0.933333, 0.992157, 0.992157, 0.992157, 0.447059, 0.294118, 0, 0, 0, 0, 0, 0, 0, 0, 0.447059, - 0.992157, 0.768627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.623529, 0.988235, 0.988235, 0.988235, 0.988235, - 0.992157, 0.47451, 0, 0, 0, 0, 0, 0, 0, 0.188235, 0.933333, 0.87451, 0.509804, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0.992157, 0.988235, 0.937255, 0.792157, 0.988235, 0.894118, 0.082353, 0, 0, 0, 0, 0, 0, - 0.027451, 0.647059, 0.992157, 0.654902, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.623529, 0.988235, 0.913725, - 0.329412, 0.376471, 0.184314, 0, 0, 0, 0, 0, 0, 0.027451, 0.513725, 0.988235, 0.635294, 0.219608, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.196078, 0.929412, 0.988235, 0.988235, 0.741176, 0.309804, 0, 0, 0, 0, - 0, 0, 0.529412, 0.988235, 0.678431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.223529, 0.992157, - 0.992157, 1, 0.992157, 0.992157, 0.992157, 0.992157, 1, 0.992157, 0.992157, 0.882353, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.023529, 0.478431, 0.654902, 0.658824, 0.952941, 0.988235, 0.988235, - 0.988235, 0.992157, 0.988235, 0.729412, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0.196078, 0.647059, 0.764706, 0.764706, 0.768627, 0.580392, 0.047059, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0]]] +TEST_DATA = [[[ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.215686, 0.533333, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.67451, 0.992157, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.070588, 0.886275, 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.192157, + 0.070588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.670588, 0.992157, + 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.117647, 0.933333, 0.858824, 0.313725, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.090196, 0.858824, 0.992157, 0.831373, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.141176, 0.992157, 0.992157, 0.611765, 0.054902, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.258824, 0.992157, 0.992157, 0.529412, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.368627, 0.992157, 0.992157, 0.419608, 0.003922, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.094118, 0.835294, 0.992157, 0.992157, 0.517647, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.603922, 0.992157, 0.992157, 0.992157, 0.603922, + 0.545098, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0.447059, 0.992157, 0.992157, + 0.956863, 0.062745, 0, 0, 0, 0, 0, 0, 0, 0, 0.011765, 0.666667, 0.992157, + 0.992157, 0.992157, 0.992157, 0.992157, 0.745098, 0.137255, 0, 0, 0, 0, 0, + 0.152941, 0.866667, 0.992157, 0.992157, 0.521569, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.070588, 0.992157, 0.992157, 0.992157, 0.803922, 0.352941, 0.745098, + 0.992157, 0.945098, 0.317647, 0, 0, 0, 0, 0.580392, 0.992157, 0.992157, + 0.764706, 0.043137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.070588, 0.992157, 0.992157, + 0.776471, 0.043137, 0, 0.007843, 0.27451, 0.882353, 0.941176, 0.176471, 0, + 0, 0.180392, 0.898039, 0.992157, 0.992157, 0.313725, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0.070588, 0.992157, 0.992157, 0.713725, 0, 0, 0, 0, 0.627451, + 0.992157, 0.729412, 0.062745, 0, 0.509804, 0.992157, 0.992157, 0.776471, + 0.035294, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.494118, 0.992157, 0.992157, + 0.968627, 0.168627, 0, 0, 0, 0.423529, 0.992157, 0.992157, 0.364706, 0, + 0.717647, 0.992157, 0.992157, 0.317647, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.533333, 0.992157, 0.984314, 0.945098, 0.603922, 0, 0, 0, 0.003922, + 0.466667, 0.992157, 0.988235, 0.976471, 0.992157, 0.992157, 0.788235, + 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.686275, 0.882353, 0.364706, 0, + 0, 0, 0, 0, 0, 0.098039, 0.588235, 0.992157, 0.992157, 0.992157, 0.980392, + 0.305882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.101961, 0.67451, 0.321569, + 0, 0, 0, 0, 0, 0, 0, 0.105882, 0.733333, 0.976471, 0.811765, 0.713725, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.65098, 0.992157, 0.321569, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0.25098, 0.007843, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, + 0.94902, 0.219608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.968627, 0.764706, 0.152941, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.498039, 0.25098, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0 +]], [[ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.298039, 0.333333, 0.333333, 0.333333, 0.337255, + 0.333333, 0.333333, 0.109804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0.027451, 0.223529, 0.776471, 0.964706, 0.988235, 0.988235, 0.988235, + 0.992157, 0.988235, 0.988235, 0.780392, 0.098039, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0.14902, 0.698039, 0.988235, 0.992157, 0.988235, 0.901961, + 0.87451, 0.568627, 0.882353, 0.976471, 0.988235, 0.988235, 0.501961, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.188235, 0.647059, 0.988235, 0.988235, + 0.745098, 0.439216, 0.098039, 0, 0, 0, 0.572549, 0.988235, 0.988235, + 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.2, 0.933333, 0.992157, + 0.941176, 0.247059, 0, 0, 0, 0, 0, 0, 0.188235, 0.898039, 0.992157, + 0.992157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.039216, 0.639216, 0.933333, + 0.988235, 0.913725, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0.113725, 0.843137, + 0.988235, 0.988235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.235294, 0.988235, + 0.992157, 0.988235, 0.815686, 0.07451, 0, 0, 0, 0, 0, 0, 0, 0.333333, + 0.988235, 0.988235, 0.552941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.211765, + 0.878431, 0.988235, 0.992157, 0.701961, 0.329412, 0.109804, 0, 0, 0, 0, 0, + 0, 0, 0.698039, 0.988235, 0.913725, 0.145098, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.188235, 0.890196, 0.988235, 0.988235, 0.745098, 0.047059, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0.882353, 0.988235, 0.568627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.2, + 0.933333, 0.992157, 0.992157, 0.992157, 0.447059, 0.294118, 0, 0, 0, 0, 0, + 0, 0, 0, 0.447059, 0.992157, 0.768627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.623529, 0.988235, 0.988235, 0.988235, 0.988235, 0.992157, 0.47451, 0, 0, + 0, 0, 0, 0, 0, 0.188235, 0.933333, 0.87451, 0.509804, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0.992157, 0.988235, 0.937255, 0.792157, 0.988235, 0.894118, + 0.082353, 0, 0, 0, 0, 0, 0, 0.027451, 0.647059, 0.992157, 0.654902, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.623529, 0.988235, 0.913725, 0.329412, 0.376471, + 0.184314, 0, 0, 0, 0, 0, 0, 0.027451, 0.513725, 0.988235, 0.635294, + 0.219608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.196078, 0.929412, 0.988235, + 0.988235, 0.741176, 0.309804, 0, 0, 0, 0, 0, 0, 0.529412, 0.988235, + 0.678431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.223529, 0.992157, + 0.992157, 1, 0.992157, 0.992157, 0.992157, 0.992157, 1, 0.992157, 0.992157, + 0.882353, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.023529, + 0.478431, 0.654902, 0.658824, 0.952941, 0.988235, 0.988235, 0.988235, + 0.992157, 0.988235, 0.729412, 0.278431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0.196078, 0.647059, 0.764706, 0.764706, 0.768627, + 0.580392, 0.047059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0 +]]] def main(): conf = parse_config("./mnist_model/trainer_config.py", "") print conf.data_config.load_data_args - network = swig_paddle.GradientMachine.createFromConfigProto(conf.model_config) + network = swig_paddle.GradientMachine.createFromConfigProto( + conf.model_config) assert isinstance(network, swig_paddle.GradientMachine) # For code hint. network.loadParameters("./mnist_model/") converter = DataProviderConverter([dense_vector(784)]) diff --git a/doc_cn/concepts/trainer_config.py b/doc_cn/concepts/trainer_config.py index 8d8c79fb39e0c..3eccbd7bc11f4 100644 --- a/doc_cn/concepts/trainer_config.py +++ b/doc_cn/concepts/trainer_config.py @@ -1,23 +1,29 @@ from paddle.trainer_config_helpers import * -define_py_data_sources2(train_list='train.list', - test_list='test.list', - module='provider', - obj='process') +define_py_data_sources2( + train_list='train.list', + test_list='test.list', + module='provider', + obj='process') settings( batch_size=128, learning_rate=1e-3, learning_method=AdamOptimizer(), - regularization=L2Regularization(0.5) -) + regularization=L2Regularization(0.5)) img = data_layer(name='pixel', size=28 * 28) -hidden1 = simple_img_conv_pool(input=img, filter_size=3, num_filters=32, pool_size=3, - num_channel=1) +hidden1 = simple_img_conv_pool( + input=img, filter_size=3, num_filters=32, pool_size=3, num_channel=1) -hidden2 = fc_layer(input=hidden1, size=200, act=TanhActivation(), - layer_attr=ExtraAttr(drop_rate=0.5)) +hidden2 = fc_layer( + input=hidden1, + size=200, + act=TanhActivation(), + layer_attr=ExtraAttr(drop_rate=0.5)) predict = fc_layer(input=hidden2, size=10, act=SoftmaxActivation()) -outputs(classification_cost(input=predict, label=data_layer(name='label', size=10))) +outputs( + classification_cost( + input=predict, label=data_layer( + name='label', size=10))) diff --git a/doc_cn/faq/word2vec_config.py b/doc_cn/faq/word2vec_config.py index e347252476eab..866b40c3d4c96 100644 --- a/doc_cn/faq/word2vec_config.py +++ b/doc_cn/faq/word2vec_config.py @@ -1,8 +1,12 @@ -... # the settings and define data provider is omitted. -DICT_DIM=3000 # dictionary dimension. -word_ids=data_layer('word_ids', size=DICT_DIM) +... # the settings and define data provider is omitted. +DICT_DIM = 3000 # dictionary dimension. +word_ids = data_layer('word_ids', size=DICT_DIM) -emb = embedding_layer(input=word_ids, size=256, param_attr=ParamAttr(sparse_update=True)) +emb = embedding_layer( + input=word_ids, size=256, param_attr=ParamAttr(sparse_update=True)) emb_sum = pooling_layer(input=emb, pooling_type=SumPooling()) predict = fc_layer(input=emb_sum, size=DICT_DIM, act=Softmax()) -outputs(classification_cost(input=predict, label=data_layer('label', size=DICT_DIM))) \ No newline at end of file +outputs( + classification_cost( + input=predict, label=data_layer( + 'label', size=DICT_DIM))) diff --git a/doc_cn/faq/word2vec_dataprovider.py b/doc_cn/faq/word2vec_dataprovider.py index a0a39080cece9..ec2753a7d01d7 100644 --- a/doc_cn/faq/word2vec_dataprovider.py +++ b/doc_cn/faq/word2vec_dataprovider.py @@ -1,8 +1,10 @@ -DICT_DIM=3000 +DICT_DIM = 3000 + + @provider(input_types=[integer_sequence(DICT_DIM), integer_value(DICT_DIM)]) def process(settings, filename): - with open(filename) as f: - # yield word ids to predict inner word id - # such as [28, 29, 10, 4], 4 - # It means the sentance is 28, 29, 4, 10, 4. - yield read_next_from_file(f) \ No newline at end of file + with open(filename) as f: + # yield word ids to predict inner word id + # such as [28, 29, 10, 4], 4 + # It means the sentance is 28, 29, 4, 10, 4. + yield read_next_from_file(f) diff --git a/doc_cn/ui/data_provider/mnist_config.py b/doc_cn/ui/data_provider/mnist_config.py index 7ba344338c374..39becff03b08f 100644 --- a/doc_cn/ui/data_provider/mnist_config.py +++ b/doc_cn/ui/data_provider/mnist_config.py @@ -1,8 +1,9 @@ from paddle.trainer_config_helpers import * -define_py_data_sources2(train_list='train.list', - test_list=None, - module='mnist_provider', - obj='process') +define_py_data_sources2( + train_list='train.list', + test_list=None, + module='mnist_provider', + obj='process') img = data_layer(name='pixel', size=784) label = data_layer(name='label', size=10) diff --git a/doc_cn/ui/data_provider/mnist_provider.dict.py b/doc_cn/ui/data_provider/mnist_provider.dict.py index bf13b56372b56..2ba0b126a0d62 100644 --- a/doc_cn/ui/data_provider/mnist_provider.dict.py +++ b/doc_cn/ui/data_provider/mnist_provider.dict.py @@ -2,10 +2,9 @@ # Define a py data provider -@provider(input_types={ - 'pixel': dense_vector(28 * 28), - 'label': integer_value(10) -}) +@provider( + input_types={'pixel': dense_vector(28 * 28), + 'label': integer_value(10)}) def process(settings, filename): # settings is not used currently. f = open(filename, 'r') # open one of training file diff --git a/doc_cn/ui/data_provider/mnist_provider.py b/doc_cn/ui/data_provider/mnist_provider.py index 92f1915c10725..8b828641d5573 100644 --- a/doc_cn/ui/data_provider/mnist_provider.py +++ b/doc_cn/ui/data_provider/mnist_provider.py @@ -2,10 +2,7 @@ # Define a py data provider -@provider(input_types=[ - dense_vector(28 * 28), - integer_value(10) -]) +@provider(input_types=[dense_vector(28 * 28), integer_value(10)]) def process(settings, filename): # settings is not used currently. f = open(filename, 'r') # open one of training file diff --git a/doc_cn/ui/data_provider/sentimental_config.py b/doc_cn/ui/data_provider/sentimental_config.py index 051f75e32b5c0..7ce71608a2372 100644 --- a/doc_cn/ui/data_provider/sentimental_config.py +++ b/doc_cn/ui/data_provider/sentimental_config.py @@ -3,9 +3,12 @@ dictionary = dict() ... # read dictionary from outside -define_py_data_sources2(train_list='train.list', test_list=None, - module='sentimental_provider', obj='process', - # above codes same as mnist sample. - args={ # pass to provider. - 'dictionary': dictionary - }) +define_py_data_sources2( + train_list='train.list', + test_list=None, + module='sentimental_provider', + obj='process', + # above codes same as mnist sample. + args={ # pass to provider. + 'dictionary': dictionary + }) diff --git a/doc_cn/ui/data_provider/sentimental_provider.py b/doc_cn/ui/data_provider/sentimental_provider.py index bda37d7722a0b..0fb0bb88e95a2 100644 --- a/doc_cn/ui/data_provider/sentimental_provider.py +++ b/doc_cn/ui/data_provider/sentimental_provider.py @@ -12,7 +12,8 @@ def on_init(settings, dictionary, **kwargs): # The text is a sequence of integer values, and each value is a word id. # The whole sequence is the sentences that we want to predict its # sentimental. - integer_value(len(dictionary), seq_type=SequenceType), # text input + integer_value( + len(dictionary), seq_type=SequenceType), # text input # label positive/negative integer_value(2) diff --git a/paddle/api/__init__.py b/paddle/api/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/paddle/api/__init__.py +++ b/paddle/api/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/paddle/api/paddle_ld_flags.py b/paddle/api/paddle_ld_flags.py index 05d741f8859ba..ebe00798e8b71 100644 --- a/paddle/api/paddle_ld_flags.py +++ b/paddle/api/paddle_ld_flags.py @@ -29,7 +29,10 @@ whole_start = "" whole_end = "" - LIB_DIRS = ["math", 'utils', 'parameter', "gserver", "api", "cuda", "pserver", "trainer"] + LIB_DIRS = [ + "math", 'utils', 'parameter', "gserver", "api", "cuda", "pserver", + "trainer" + ] PARENT_LIB_DIRS = ['proto'] class PaddleLDFlag(object): @@ -55,19 +58,20 @@ def __init__(self): self.curt = CUDA_LIBRARIES def ldflag_str(self): - return " ".join([self.libs_dir_str(), - self.parent_dir_str(), - self.libs_str()]) + return " ".join( + [self.libs_dir_str(), self.parent_dir_str(), self.libs_str()]) def libs_dir_str(self): libdirs = LIB_DIRS - return " ".join(map(lambda x: "-L" + os.path.join(self.paddle_build_dir, x), - libdirs)) + return " ".join( + map(lambda x: "-L" + os.path.join(self.paddle_build_dir, x), + libdirs)) def parent_dir_str(self): libdirs = PARENT_LIB_DIRS - return " ".join(map(lambda x: "-L" + os.path.join(self.paddle_build_dir, '..', x), - libdirs)) + return " ".join( + map(lambda x: "-L" + os.path.join(self.paddle_build_dir, '..', x), + libdirs)) def libs_str(self): libs = [ @@ -113,10 +117,10 @@ def normalize_flag(self, cmake_flag): return cmake_flag elif cmake_flag.startswith("-l"): # normal link command return cmake_flag - elif cmake_flag in ["gflags-shared", - "gflags-static", - "gflags_nothreads-shared", - "gflags_nothreads-static"]: # special for gflags + elif cmake_flag in [ + "gflags-shared", "gflags-static", "gflags_nothreads-shared", + "gflags_nothreads-static" + ]: # special for gflags assert PaddleLDFlag.cmake_bool(self.gflags_location) return self.gflags_location elif len(cmake_flag) != 0: @@ -132,18 +136,22 @@ def cmake_bool(cmake_str): :type cmake_str: str :rtype: bool """ - if cmake_str in ["FALSE", "OFF", "NO"] or cmake_str.endswith("-NOTFOUND"): + if cmake_str in ["FALSE", "OFF", "NO"] or cmake_str.endswith( + "-NOTFOUND"): return False else: return True + def c_flag(self): if self.with_coverage: return ["-fprofile-arcs", "-ftest-coverage", "-O0", "-g"] else: return None except ImportError: + class PaddleLDFlag(object): def ldflag_str(self): pass + def c_flag(self): pass diff --git a/paddle/api/test/testArguments.py b/paddle/api/test/testArguments.py index daedd2409effc..70fb169fd5c43 100644 --- a/paddle/api/test/testArguments.py +++ b/paddle/api/test/testArguments.py @@ -32,7 +32,7 @@ def test_load_arguments(self): iv = args.getSlotIds(0) assert isinstance(iv, swig_paddle.IVector) np_arr = iv.toNumpyArrayInplace() - self.assertEqual(np_arr.shape, (6,)) + self.assertEqual(np_arr.shape, (6, )) if __name__ == '__main__': diff --git a/paddle/api/test/testGradientMachine.py b/paddle/api/test/testGradientMachine.py index 59b36a012a239..e12613fbb8a66 100644 --- a/paddle/api/test/testGradientMachine.py +++ b/paddle/api/test/testGradientMachine.py @@ -30,8 +30,8 @@ def test_create_gradient_machine(self): self.assertIsNotNone(model_config) machine = swig_paddle.GradientMachine.createByModelConfig( model_config, swig_paddle.CREATE_MODE_NORMAL, - swig_paddle.ParameterOptimizer.create( - opt_config).getParameterTypes()) + swig_paddle.ParameterOptimizer.create(opt_config).getParameterTypes( + )) self.assertIsNotNone(machine) ipt, _ = util.loadMNISTTrainData() output = swig_paddle.Arguments.createArguments(0) @@ -43,7 +43,7 @@ def test_create_gradient_machine(self): assert isinstance(param, swig_paddle.Parameter) val = param.getBuf(swig_paddle.PARAMETER_VALUE) assert isinstance(val, swig_paddle.Vector) - arr = numpy.full((len(val),), 0.1, dtype="float32") + arr = numpy.full((len(val), ), 0.1, dtype="float32") val.copyFromNumpyArray(arr) param_config = param.getConfig().toProto() assert isinstance(param_config, diff --git a/paddle/api/test/testMatrix.py b/paddle/api/test/testMatrix.py index 2216ef30a58b0..11035a9281656 100644 --- a/paddle/api/test/testMatrix.py +++ b/paddle/api/test/testMatrix.py @@ -69,7 +69,8 @@ def test_createDenseMat(self): def test_numpy(self): numpy_mat = np.matrix([[1, 2], [3, 4], [5, 6]], dtype="float32") m = swig_paddle.Matrix.createCpuDenseFromNumpy(numpy_mat) - self.assertEqual((int(m.getHeight()), int(m.getWidth())), numpy_mat.shape) + self.assertEqual( + (int(m.getHeight()), int(m.getWidth())), numpy_mat.shape) # the numpy matrix and paddle matrix shared the same memory. numpy_mat[0, 1] = 342.23 diff --git a/paddle/api/test/testTrain.py b/paddle/api/test/testTrain.py index 7759118a3d9d1..a3ba4eaaa69b3 100644 --- a/paddle/api/test/testTrain.py +++ b/paddle/api/test/testTrain.py @@ -98,7 +98,8 @@ def update_callback(param): cost_vec = outArgs.getSlotValue(0) assert isinstance(cost_vec, swig_paddle.Matrix) cost_vec = cost_vec.copyToNumpyMat() - print 'Finish Batch', batch_id, 'with cost ', cost_vec.sum() / batch_size + print 'Finish Batch', batch_id, 'with cost ', cost_vec.sum( + ) / batch_size batch_id += 1 for optimizer in optimizers: diff --git a/paddle/api/test/testTrainConfig.py b/paddle/api/test/testTrainConfig.py index 22148e31915da..77e0cd37d566d 100644 --- a/paddle/api/test/testTrainConfig.py +++ b/paddle/api/test/testTrainConfig.py @@ -1,9 +1,6 @@ from paddle.trainer_config_helpers import * -settings( - batch_size=100, - learning_method=AdamOptimizer() -) +settings(batch_size=100, learning_method=AdamOptimizer()) din = data_layer(name='input', size=784) diff --git a/paddle/api/test/testTrainer.py b/paddle/api/test/testTrainer.py index da69a60f84f4d..edd5a2da5785c 100644 --- a/paddle/api/test/testTrainer.py +++ b/paddle/api/test/testTrainer.py @@ -17,9 +17,9 @@ from py_paddle import swig_paddle import util + def main(): - trainer_config = parse_config( - "./testTrainConfig.py", "") + trainer_config = parse_config("./testTrainConfig.py", "") model = swig_paddle.GradientMachine.createFromConfigProto( trainer_config.model_config) trainer = swig_paddle.Trainer.create(trainer_config, model) @@ -56,7 +56,7 @@ def main(): logger.info('test cost=%f' % (cost / num)) trainer.finishTrain() - + if __name__ == '__main__': swig_paddle.initPaddle("--use_gpu=0", "--trainer_count=1") diff --git a/paddle/api/test/testVector.py b/paddle/api/test/testVector.py index f5b5d0e32e420..5226df79eea3b 100644 --- a/paddle/api/test/testVector.py +++ b/paddle/api/test/testVector.py @@ -112,5 +112,6 @@ def testCopyFromNumpy(self): if __name__ == '__main__': - swig_paddle.initPaddle("--use_gpu=1" if swig_paddle.isGpuVersion() else "--use_gpu=0") + swig_paddle.initPaddle("--use_gpu=1" + if swig_paddle.isGpuVersion() else "--use_gpu=0") unittest.main() diff --git a/paddle/gserver/tests/__init__.py b/paddle/gserver/tests/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/paddle/gserver/tests/__init__.py +++ b/paddle/gserver/tests/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/paddle/gserver/tests/pyDataProvider.py b/paddle/gserver/tests/pyDataProvider.py index c3155e7adea04..91863b4175b1a 100644 --- a/paddle/gserver/tests/pyDataProvider.py +++ b/paddle/gserver/tests/pyDataProvider.py @@ -16,72 +16,79 @@ import struct import traceback + def header_creator(): ret = "" - ret += struct.pack('i', 3) # slot num - ret += struct.pack('i', 1) # sequence flag - ret += struct.pack('i', 0) # slot0 dense type - ret += struct.pack('i', 3) # slot0 dim - ret += struct.pack('i', 1) # slot1 sparse non value type - ret += struct.pack('i', 7) # slot1 dim - ret += struct.pack('i', 3) # slot2 index type - ret += struct.pack('i', 2) # slot2 dim + ret += struct.pack('i', 3) # slot num + ret += struct.pack('i', 1) # sequence flag + ret += struct.pack('i', 0) # slot0 dense type + ret += struct.pack('i', 3) # slot0 dim + ret += struct.pack('i', 1) # slot1 sparse non value type + ret += struct.pack('i', 7) # slot1 dim + ret += struct.pack('i', 3) # slot2 index type + ret += struct.pack('i', 2) # slot2 dim return ret + def dense_value_creator(sample_num): ret = "" - ret += struct.pack('i', sample_num) # slot0 sample num - for i in range(sample_num): # slot0 value + ret += struct.pack('i', sample_num) # slot0 sample num + for i in range(sample_num): # slot0 value ret += struct.pack('f', 1.0) ret += struct.pack('f', 2.0) ret += struct.pack('f', 3.0) return ret + def sparse_value_creator(sample_num): ret = "" - ret += struct.pack('i', sample_num) # slot1 sample num - for i in range(sample_num): # slot1 index + ret += struct.pack('i', sample_num) # slot1 sample num + for i in range(sample_num): # slot1 index ret += struct.pack('i', i * 2) - ret += struct.pack('i', sample_num * 2) #slot1 length - for i in range(sample_num): # slot1 value + ret += struct.pack('i', sample_num * 2) #slot1 length + for i in range(sample_num): # slot1 value ret += struct.pack('i', 1) ret += struct.pack('i', 2) return ret + def index_value_creator(sample_num): ret = "" - ret += struct.pack('i', sample_num) # slot2 sample num - for i in range(sample_num): # slot2 value + ret += struct.pack('i', sample_num) # slot2 sample num + for i in range(sample_num): # slot2 value ret += struct.pack('i', 0) return ret + def sequenceStartPositions_creator(): ret = "" - ret += struct.pack('i', 2) # slot0 sequence num - ret += struct.pack('i', 0) # slot0 sequence value1 - ret += struct.pack('i', 1) # slot0 sequence value2 - ret += struct.pack('i', 1) # slot1 sequence num - ret += struct.pack('i', 0) # slot1 sequence value1 - ret += struct.pack('i', 2) # slot2 sequence num - ret += struct.pack('i', 0) # slot2 sequence value1 - ret += struct.pack('i', 1) # slot2 sequence value2 + ret += struct.pack('i', 2) # slot0 sequence num + ret += struct.pack('i', 0) # slot0 sequence value1 + ret += struct.pack('i', 1) # slot0 sequence value2 + ret += struct.pack('i', 1) # slot1 sequence num + ret += struct.pack('i', 0) # slot1 sequence value1 + ret += struct.pack('i', 2) # slot2 sequence num + ret += struct.pack('i', 0) # slot2 sequence value1 + ret += struct.pack('i', 1) # slot2 sequence value2 return ret + def subSequenceStartPositions_creator(): ret = "" - ret += struct.pack('i', 3) # slot0 subsequence num - ret += struct.pack('i', 0) # slot0 subsequence value1 - ret += struct.pack('i', 1) # slot0 subsequence value2 - ret += struct.pack('i', 2) # slot0 subsequence value3 - ret += struct.pack('i', 2) # slot1 subsequence num - ret += struct.pack('i', 0) # slot1 subsequence value1 - ret += struct.pack('i', 1) # slot1 subsequence value2 - ret += struct.pack('i', 3) # slot2 subsequence num - ret += struct.pack('i', 0) # slot2 subsequence value1 - ret += struct.pack('i', 1) # slot2 subsequence value2 - ret += struct.pack('i', 2) # slot2 subsequence value3 + ret += struct.pack('i', 3) # slot0 subsequence num + ret += struct.pack('i', 0) # slot0 subsequence value1 + ret += struct.pack('i', 1) # slot0 subsequence value2 + ret += struct.pack('i', 2) # slot0 subsequence value3 + ret += struct.pack('i', 2) # slot1 subsequence num + ret += struct.pack('i', 0) # slot1 subsequence value1 + ret += struct.pack('i', 1) # slot1 subsequence value2 + ret += struct.pack('i', 3) # slot2 subsequence num + ret += struct.pack('i', 0) # slot2 subsequence value1 + ret += struct.pack('i', 1) # slot2 subsequence value2 + ret += struct.pack('i', 2) # slot2 subsequence value3 return ret + class SimpleDataProvider: def __init__(self, *file_list): self.file_list = file_list @@ -93,17 +100,18 @@ def reset(self): pass def getHeader(self): - return header_creator() + return header_creator() def getNextBatch(self, batch_size): ret = "" - ret += struct.pack('i', 2) # batch size - ret += dense_value_creator(2) # slot0 - ret += sparse_value_creator(2) # slot1 - ret += index_value_creator(2) # slot2 + ret += struct.pack('i', 2) # batch size + ret += dense_value_creator(2) # slot0 + ret += sparse_value_creator(2) # slot1 + ret += index_value_creator(2) # slot2 ret += sequenceStartPositions_creator() return ret + class SimpleNestDataProvider: def __init__(self, *file_list): self.file_list = file_list @@ -119,14 +127,15 @@ def getHeader(self): def getNextBatch(self, batch_size): ret = "" - ret += struct.pack('i', 2) # batch size - ret += dense_value_creator(4) # slot0 - ret += sparse_value_creator(4) # slot1 - ret += index_value_creator(4) # slot2 + ret += struct.pack('i', 2) # batch size + ret += dense_value_creator(4) # slot0 + ret += sparse_value_creator(4) # slot1 + ret += index_value_creator(4) # slot2 ret += sequenceStartPositions_creator() ret += subSequenceStartPositions_creator() return ret + if __name__ == "__main__": # test code data_provider = SimpleDataProvider('./test_batch') diff --git a/paddle/gserver/tests/rnn_data_provider.py b/paddle/gserver/tests/rnn_data_provider.py index 321c78cb1741b..715ac08a42d05 100644 --- a/paddle/gserver/tests/rnn_data_provider.py +++ b/paddle/gserver/tests/rnn_data_provider.py @@ -22,18 +22,20 @@ [[[0, 2], [2, 5], [0, 1, 2]], 1], ] + # Used for sequence_nest_rnn.conf -@provider(input_types=[integer_value_sub_sequence(10), - integer_value(3)], - should_shuffle=False) +@provider( + input_types=[integer_value_sub_sequence(10), integer_value(3)], + should_shuffle=False) def process_subseq(settings, file_name): for d in data: yield d + # Used for sequence_rnn.conf -@provider(input_types=[integer_value_sequence(10), - integer_value(3)], - should_shuffle=False) +@provider( + input_types=[integer_value_sequence(10), integer_value(3)], + should_shuffle=False) def process_seq(settings, file_name): for d in data: seq = [] @@ -41,18 +43,20 @@ def process_seq(settings, file_name): seq += subseq yield seq, d[1] + # Used for sequence_nest_rnn_multi_input.conf -@provider(input_types=[integer_value_sub_sequence(10), - integer_value(3)], - should_shuffle=False) +@provider( + input_types=[integer_value_sub_sequence(10), integer_value(3)], + should_shuffle=False) def process_subseq2(settings, file_name): for d in data: yield d + # Used for sequence_rnn_multi_input.conf -@provider(input_types=[integer_value_sequence(10), - integer_value(3)], - should_shuffle=False) +@provider( + input_types=[integer_value_sequence(10), integer_value(3)], + should_shuffle=False) def process_seq2(settings, file_name): for d in data: seq = [] @@ -60,31 +64,34 @@ def process_seq2(settings, file_name): seq += subseq yield seq, d[1] + ########################################################### data2 = [ - [[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]] ,0], - [[[0, 2], [2, 5], [0, 1, 2]],[[1, 5], [4], [2, 3, 6, 1]], 1], + [[[1, 2], [4, 5, 2]], [[5, 4, 1], [3, 1]], 0], + [[[0, 2], [2, 5], [0, 1, 2]], [[1, 5], [4], [2, 3, 6, 1]], 1], ] + # Used for sequence_nest_rnn_multi_unequalength_inputs.conf -@provider(input_types=[integer_value_sub_sequence(10), - integer_value_sub_sequence(10), - integer_value(2)], - should_shuffle=False) +@provider( + input_types=[ + integer_value_sub_sequence(10), integer_value_sub_sequence(10), + integer_value(2) + ], + should_shuffle=False) def process_unequalength_subseq(settings, file_name): for d in data2: yield d # Used for sequence_rnn_multi_unequalength_inputs.conf -@provider(input_types=[integer_value_sequence(10), - integer_value_sequence(10), - integer_value(2)], - should_shuffle=False) +@provider( + input_types=[ + integer_value_sequence(10), integer_value_sequence(10), integer_value(2) + ], + should_shuffle=False) def process_unequalength_seq(settings, file_name): for d in data2: - words1=reduce(lambda x,y: x+y, d[0]) - words2=reduce(lambda x,y: x+y, d[1]) + words1 = reduce(lambda x, y: x + y, d[0]) + words2 = reduce(lambda x, y: x + y, d[1]) yield words1, words2, d[2] - - diff --git a/paddle/gserver/tests/sequenceGen.py b/paddle/gserver/tests/sequenceGen.py index b166e778d7a33..fab876fd30da0 100644 --- a/paddle/gserver/tests/sequenceGen.py +++ b/paddle/gserver/tests/sequenceGen.py @@ -20,8 +20,9 @@ def hook(settings, dict_file, **kwargs): settings.word_dict = dict_file - settings.input_types = [integer_value_sequence(len(settings.word_dict)), - integer_value(3)] + settings.input_types = [ + integer_value_sequence(len(settings.word_dict)), integer_value(3) + ] settings.logger.info('dict len : %d' % (len(settings.word_dict))) @@ -32,16 +33,19 @@ def process(settings, file_name): label, comment = line.strip().split('\t') label = int(''.join(label.split())) words = comment.split() - word_slot = [settings.word_dict[w] for w in words if - w in settings.word_dict] + word_slot = [ + settings.word_dict[w] for w in words if w in settings.word_dict + ] yield word_slot, label ## for hierarchical sequence network def hook2(settings, dict_file, **kwargs): settings.word_dict = dict_file - settings.input_types = [integer_value_sub_sequence(len(settings.word_dict)), - integer_value_sequence(3)] + settings.input_types = [ + integer_value_sub_sequence(len(settings.word_dict)), + integer_value_sequence(3) + ] settings.logger.info('dict len : %d' % (len(settings.word_dict))) @@ -55,8 +59,10 @@ def process2(settings, file_name): label, comment = line.strip().split('\t') label = int(''.join(label.split())) words = comment.split() - word_slot = [settings.word_dict[w] for w in words if - w in settings.word_dict] + word_slot = [ + settings.word_dict[w] for w in words + if w in settings.word_dict + ] label_list.append(label) word_slot_list.append(word_slot) else: diff --git a/paddle/gserver/tests/sequence_layer_group.conf b/paddle/gserver/tests/sequence_layer_group.conf index ac031b31280df..087aa96ccb5a7 100644 --- a/paddle/gserver/tests/sequence_layer_group.conf +++ b/paddle/gserver/tests/sequence_layer_group.conf @@ -21,15 +21,16 @@ dict_file = dict() for line_count, line in enumerate(open(dict_path, "r")): dict_file[line.strip()] = line_count -define_py_data_sources2(train_list='gserver/tests/Sequence/train.list', - test_list=None, - module='sequenceGen', - obj='process', - args={"dict_file":dict_file}) +define_py_data_sources2( + train_list='gserver/tests/Sequence/train.list', + test_list=None, + module='sequenceGen', + obj='process', + args={"dict_file": dict_file}) settings(batch_size=5) ######################## network configure ################################ -dict_dim = len(open(dict_path,'r').readlines()) +dict_dim = len(open(dict_path, 'r').readlines()) word_dim = 128 hidden_dim = 256 label_dim = 3 @@ -39,21 +40,24 @@ data = data_layer(name="word", size=dict_dim) emb = embedding_layer(input=data, size=word_dim) # (lstm_input + lstm) is equal to lstmemory -with mixed_layer(size=hidden_dim*4) as lstm_input: +with mixed_layer(size=hidden_dim * 4) as lstm_input: lstm_input += full_matrix_projection(input=emb) -lstm = lstmemory_group(input=lstm_input, - size=hidden_dim, - act=TanhActivation(), - gate_act=SigmoidActivation(), - state_act=TanhActivation(), - lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) +lstm = lstmemory_group( + input=lstm_input, + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) lstm_last = last_seq(input=lstm) -with mixed_layer(size=label_dim, - act=SoftmaxActivation(), - bias_attr=True) as output: +with mixed_layer( + size=label_dim, act=SoftmaxActivation(), bias_attr=True) as output: output += full_matrix_projection(input=lstm_last) -outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) +outputs( + classification_cost( + input=output, label=data_layer( + name="label", size=1))) diff --git a/paddle/gserver/tests/sequence_nest_layer_group.conf b/paddle/gserver/tests/sequence_nest_layer_group.conf index 38c60b657b969..93a0f6da7905c 100644 --- a/paddle/gserver/tests/sequence_nest_layer_group.conf +++ b/paddle/gserver/tests/sequence_nest_layer_group.conf @@ -21,15 +21,16 @@ dict_file = dict() for line_count, line in enumerate(open(dict_path, "r")): dict_file[line.strip()] = line_count -define_py_data_sources2(train_list='gserver/tests/Sequence/train.list.nest', - test_list=None, - module='sequenceGen', - obj='process2', - args={"dict_file":dict_file}) +define_py_data_sources2( + train_list='gserver/tests/Sequence/train.list.nest', + test_list=None, + module='sequenceGen', + obj='process2', + args={"dict_file": dict_file}) settings(batch_size=2) ######################## network configure ################################ -dict_dim = len(open(dict_path,'r').readlines()) +dict_dim = len(open(dict_path, 'r').readlines()) word_dim = 128 hidden_dim = 256 label_dim = 3 @@ -38,37 +39,46 @@ data = data_layer(name="word", size=dict_dim) emb_group = embedding_layer(input=data, size=word_dim) + # (lstm_input + lstm) is equal to lstmemory def lstm_group(lstm_group_input): - with mixed_layer(size=hidden_dim*4) as group_input: - group_input += full_matrix_projection(input=lstm_group_input) + with mixed_layer(size=hidden_dim * 4) as group_input: + group_input += full_matrix_projection(input=lstm_group_input) - lstm_output = lstmemory_group(input=group_input, - name="lstm_group", - size=hidden_dim, - act=TanhActivation(), - gate_act=SigmoidActivation(), - state_act=TanhActivation(), - lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) + lstm_output = lstmemory_group( + input=group_input, + name="lstm_group", + size=hidden_dim, + act=TanhActivation(), + gate_act=SigmoidActivation(), + state_act=TanhActivation(), + lstm_layer_attr=ExtraLayerAttribute(error_clipping_threshold=50)) return lstm_output -lstm_nest_group = recurrent_group(input=SubsequenceInput(emb_group), - step=lstm_group, - name="lstm_nest_group") + +lstm_nest_group = recurrent_group( + input=SubsequenceInput(emb_group), step=lstm_group, name="lstm_nest_group") # hasSubseq ->(seqlastins) seq -lstm_last = last_seq(input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE) +lstm_last = last_seq( + input=lstm_nest_group, agg_level=AggregateLevel.EACH_SEQUENCE) # seq ->(expand) hasSubseq -lstm_expand = expand_layer(input=lstm_last, expand_as=emb_group, expand_level=ExpandLevel.FROM_SEQUENCE) +lstm_expand = expand_layer( + input=lstm_last, + expand_as=emb_group, + expand_level=ExpandLevel.FROM_SEQUENCE) # hasSubseq ->(average) seq -lstm_average = pooling_layer(input=lstm_expand, - pooling_type=AvgPooling(), - agg_level=AggregateLevel.EACH_SEQUENCE) +lstm_average = pooling_layer( + input=lstm_expand, + pooling_type=AvgPooling(), + agg_level=AggregateLevel.EACH_SEQUENCE) -with mixed_layer(size=label_dim, - act=SoftmaxActivation(), - bias_attr=True) as output: +with mixed_layer( + size=label_dim, act=SoftmaxActivation(), bias_attr=True) as output: output += full_matrix_projection(input=lstm_average) -outputs(classification_cost(input=output, label=data_layer(name="label", size=1))) +outputs( + classification_cost( + input=output, label=data_layer( + name="label", size=1))) diff --git a/paddle/gserver/tests/test_PyDataProvider2.py b/paddle/gserver/tests/test_PyDataProvider2.py index 71c3335231e52..7ca30198fb1d0 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.py +++ b/paddle/gserver/tests/test_PyDataProvider2.py @@ -33,16 +33,19 @@ def test_init_hooker(setting, value, **kwargs): setting.value = value -@provider(input_types=[dense_vector(20, seq_type=SequenceType.NO_SEQUENCE)], - init_hook=test_init_hooker) +@provider( + input_types=[dense_vector( + 20, seq_type=SequenceType.NO_SEQUENCE)], + init_hook=test_init_hooker) def test_init_hook(setting, filename): for i in xrange(200): yield setting.value -@provider( - input_types=[ - sparse_binary_vector(30000, seq_type=SequenceType.NO_SEQUENCE)]) +@provider(input_types=[ + sparse_binary_vector( + 30000, seq_type=SequenceType.NO_SEQUENCE) +]) def test_sparse_non_value_no_seq(setting, filename): for i in xrange(200): yield [(i + 1) * (j + 1) for j in xrange(10)] @@ -77,28 +80,28 @@ def test_min_pool_size(setting, filename): yield random.randint(0, 100 - 1) -@provider(input_types=[index_slot(100, seq_type=SequenceType.SEQUENCE)], - can_over_batch_size=False, - calc_batch_size=lambda x: len(x[0])) +@provider( + input_types=[index_slot( + 100, seq_type=SequenceType.SEQUENCE)], + can_over_batch_size=False, + calc_batch_size=lambda x: len(x[0])) def test_can_over_batch_size(setting, filename): for _ in xrange(1 << 10): seq_len = random.randint(0, 99) yield [random.randint(0, 100 - 1) for _ in xrange(seq_len)] -@provider(input_types={'input1':index_slot(10), 'input2': index_slot(10)}) +@provider(input_types={'input1': index_slot(10), 'input2': index_slot(10)}) def test_input_order(setting, filename): for _ in xrange(1000): - yield { - 'input1': 0, - 'input2': 1 - } + yield {'input1': 0, 'input2': 1} -@provider(input_types=[index_slot(10)], - check=True, - check_fail_continue=True, - should_shuffle="123") # also test should shuffle +@provider( + input_types=[index_slot(10)], + check=True, + check_fail_continue=True, + should_shuffle="123") # also test should shuffle def test_check(settings, filename): yield_good_value = False @@ -108,4 +111,3 @@ def test_check(settings, filename): if i < 10: yield_good_value = True yield i - diff --git a/paddle/py_paddle/__init__.py b/paddle/py_paddle/__init__.py index f372068942ea3..f8399f9c63d81 100644 --- a/paddle/py_paddle/__init__.py +++ b/paddle/py_paddle/__init__.py @@ -15,9 +15,10 @@ from util import DataProviderWrapperConverter from dataprovider_converter import DataProviderConverter -__all__ = ['paddle', - 'DataProviderConverter', - 'DataProviderWrapperConverter', # for deprecated usage. - 'loadParameterFile'] +__all__ = [ + 'paddle', + 'DataProviderConverter', + 'DataProviderWrapperConverter', # for deprecated usage. + 'loadParameterFile' +] util.monkeypatches() - diff --git a/paddle/py_paddle/dataprovider_converter.py b/paddle/py_paddle/dataprovider_converter.py index dd2e146d112c0..d64c7b20cb65a 100644 --- a/paddle/py_paddle/dataprovider_converter.py +++ b/paddle/py_paddle/dataprovider_converter.py @@ -45,10 +45,8 @@ def scan(self, dat): def finish_scan(self, argument): assert isinstance(argument, swig_paddle.Arguments) assert isinstance(self.input_type, dp2.InputType) - m = swig_paddle.Matrix.createDense(self.__mat__, - self.__height__, - self.input_type.dim, - False) + m = swig_paddle.Matrix.createDense(self.__mat__, self.__height__, + self.input_type.dim, False) argument.setSlotValue(self.pos, m) @@ -141,8 +139,10 @@ def convert(self, dat, argument=None): assert isinstance(argument, swig_paddle.Arguments) argument.resize(len(self.input_types)) - scanners = [DataProviderConverter.create_scanner(i, each_type) - for i, each_type in enumerate(self.input_types)] + scanners = [ + DataProviderConverter.create_scanner(i, each_type) + for i, each_type in enumerate(self.input_types) + ] for each_sample in dat: for each_step, scanner in zip(each_sample, scanners): @@ -171,11 +171,14 @@ def create_scanner(i, each): assert retv is not None if each.seq_type == dp2.SequenceType.SUB_SEQUENCE: - retv = SequenceScanner(each, i, retv, lambda a, p, seq: - a.setSlotSubSequenceStartPositions(p, seq)) - - if each.seq_type in [dp2.SequenceType.SUB_SEQUENCE, - dp2.SequenceType.SEQUENCE]: - retv = SequenceScanner(each, i, retv, lambda a, p, seq: - a.setSlotSequenceStartPositions(p, seq)) + retv = SequenceScanner( + each, i, retv, + lambda a, p, seq: a.setSlotSubSequenceStartPositions(p, seq)) + + if each.seq_type in [ + dp2.SequenceType.SUB_SEQUENCE, dp2.SequenceType.SEQUENCE + ]: + retv = SequenceScanner( + each, i, retv, + lambda a, p, seq: a.setSlotSequenceStartPositions(p, seq)) return retv diff --git a/paddle/py_paddle/util.py b/paddle/py_paddle/util.py index 53f67a861e7d9..8ebcb346100c2 100644 --- a/paddle/py_paddle/util.py +++ b/paddle/py_paddle/util.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Some Useful method for py_paddle. """ @@ -79,6 +78,7 @@ def wrap(callback): else: return __ParameterCallbackWrapper__(callback).__disown__() + def __arguments_to_numpy__(i, arg): assert isinstance(arg, swig_paddle.Arguments) value = arg.getSlotValue(i) @@ -89,10 +89,8 @@ def __arguments_to_numpy__(i, arg): if ids is not None: assert isinstance(ids, swig_paddle.IVector) ids = ids.copyToNumpyArray() - return { - "value": value, - "id": ids - } + return {"value": value, "id": ids} + def __monkeypatch_gradient_machine__(): """ @@ -102,7 +100,6 @@ def __monkeypatch_gradient_machine__(): swig_paddle.GradientMachine.loadFromConfigFile = \ staticmethod(loadGradientMachine) - def __matrix_to_numpy__(m): if isinstance(m, swig_paddle.Matrix): return m.copyToNumpyMat() @@ -113,9 +110,11 @@ def __matrix_to_numpy__(m): def createFromConfigProto(protoObj, createMode=swig_paddle.CREATE_MODE_NORMAL, - paramTypes=[swig_paddle.PARAMETER_VALUE, - swig_paddle.PARAMETER_GRADIENT, - swig_paddle.PARAMETER_MOMENTUM]): + paramTypes=[ + swig_paddle.PARAMETER_VALUE, + swig_paddle.PARAMETER_GRADIENT, + swig_paddle.PARAMETER_MOMENTUM + ]): """ Create Gradient Machine From Proto object. :param protoObj: Model config @@ -145,8 +144,10 @@ def forwardTest(self, inArgs): """ outArgs = swig_paddle.Arguments.createArguments(0) self.forward(inArgs, outArgs, swig_paddle.PASS_TEST) - return [__arguments_to_numpy__(i, outArgs) for i in xrange( - outArgs.getSlotNum())] + return [ + __arguments_to_numpy__(i, outArgs) + for i in xrange(outArgs.getSlotNum()) + ] swig_paddle.GradientMachine.forwardTest = forwardTest @@ -167,7 +168,10 @@ def backward(self, callback): swig_paddle.GradientMachine.__forwardBackward__ = \ swig_paddle.GradientMachine.forwardBackward - def forwardBackward(self, inArgs, outArgs, passType, + def forwardBackward(self, + inArgs, + outArgs, + passType, callback=swig_paddle.UpdateCallback()): """ GradientMachine forward backward. @@ -315,9 +319,8 @@ def append(self, other): self.cols += other def __call__(self, slot_idx, arg): - mat = swig_paddle.Matrix.createSparse(len(self.indices) - 1, - self.dim, - len(self.cols), True) + mat = swig_paddle.Matrix.createSparse( + len(self.indices) - 1, self.dim, len(self.cols), True) assert isinstance(mat, swig_paddle.Matrix) mat.sparseCopyFrom(self.indices, self.cols) self.putIntoArg(slot_idx, arg, mat) @@ -341,9 +344,8 @@ def append(self, other): self.values += map(lambda x: x[1], other) def __call__(self, slot_idx, arg): - mat = swig_paddle.Matrix.createSparse(len(self.indices) - 1, - self.dim, - len(self.cols), False) + mat = swig_paddle.Matrix.createSparse( + len(self.indices) - 1, self.dim, len(self.cols), False) assert isinstance(mat, swig_paddle.Matrix) mat.sparseCopyFrom(self.indices, self.cols, self.values) self.putIntoArg(slot_idx, arg, mat) @@ -352,8 +354,9 @@ def __call__(self, slot_idx, arg): paddle.trainer.PyDataProviderWrapper.DenseSlot: DenseValueConverter, paddle.trainer.PyDataProviderWrapper.IndexSlot: IdValueConverter, paddle.trainer.PyDataProviderWrapper.SparseNonValueSlot: - SparseNonValueConverter, - paddle.trainer.PyDataProviderWrapper.SparseValueSlot: SparseValueConverter + SparseNonValueConverter, + paddle.trainer.PyDataProviderWrapper.SparseValueSlot: + SparseValueConverter } def __init__(self, use_seq, header): @@ -381,10 +384,9 @@ def convert(self, wrapper_data, argument=None): assert isinstance(argument, swig_paddle.Arguments) argument.resize(len(self.__header__)) - values = map(lambda x: - DataProviderWrapperConverter.__SLOT_VALUE_CONVERTER_MAP__[ - x.__class__](x), - self.__header__) + values = map( + lambda x: DataProviderWrapperConverter.__SLOT_VALUE_CONVERTER_MAP__[x.__class__](x), + self.__header__) if self.__use_seq__: seq_dim = [[] for _ in xrange(self.__header__.__len__())] @@ -394,14 +396,13 @@ def convert(self, wrapper_data, argument=None): for slot_idx, sequence in enumerate(each_sample): for raw_data in sequence: values[slot_idx].append(raw_data) - seq_start_pos[slot_idx].append( - seq_start_pos[slot_idx][-1] + len(sequence)) + seq_start_pos[slot_idx].append(seq_start_pos[slot_idx][-1] + + len(sequence)) seq_dim[slot_idx].append(len(sequence)) for slot_idx in xrange(len(self.__header__)): - argument.setSlotSequenceDim(slot_idx, - swig_paddle.IVector.create( - seq_dim[slot_idx])) + argument.setSlotSequenceDim( + slot_idx, swig_paddle.IVector.create(seq_dim[slot_idx])) argument.setSlotSequenceStartPositions( slot_idx, swig_paddle.IVector.create(seq_start_pos[slot_idx])) @@ -422,7 +423,6 @@ def __call__(self, wrapper_data, argument=None): return self.convert(wrapper_data, argument) - def __monkey_patch_protobuf_objects__(): def ParameterConfig_toProto(self): """ @@ -459,8 +459,7 @@ def OptimizationConfig_createFromProto(protoObj): :return: paddle.OptimizationConfig """ - assert isinstance(protoObj, - paddle.proto.OptimizationConfig) + assert isinstance(protoObj, paddle.proto.OptimizationConfig) return swig_paddle.OptimizationConfig.createFromProtoString( protoObj.SerializeToString()) @@ -475,8 +474,7 @@ def TrainerConfig_createFromProto(protoObj): :param protoObj: proto.TrainerConfig :return: paddle.TrainerConfig """ - assert isinstance(protoObj, - paddle.proto.TrainerConfig) + assert isinstance(protoObj, paddle.proto.TrainerConfig) return swig_paddle.TrainerConfig.createFromProtoString( protoObj.SerializeToString()) @@ -537,6 +535,7 @@ def Trainer_create(config, model=None): assert isinstance(model, swig_paddle.GradientMachine) return swig_paddle.Trainer.__create__( swig_paddle.TrainerConfig.createFromProto(config), model) + swig_paddle.Trainer.create = staticmethod(Trainer_create) swig_paddle.Trainer.__getForwardOutput__ = \ @@ -551,14 +550,19 @@ def getForwardOutput(self): numpy.ndarray. """ outArgs = self.__getForwardOutput__() - return [__arguments_to_numpy__(i, outArgs) for i in xrange( - outArgs.getSlotNum())] + return [ + __arguments_to_numpy__(i, outArgs) + for i in xrange(outArgs.getSlotNum()) + ] swig_paddle.Trainer.getForwardOutput = getForwardOutput + def monkeypatches(): - patches = [__monkeypatch_init_paddle__, __monkeypatch_gradient_machine__, - __monkey_patch_protobuf_objects__, - __monkey_patch_parameter__, __monkey_patch_trainer__] + patches = [ + __monkeypatch_init_paddle__, __monkeypatch_gradient_machine__, + __monkey_patch_protobuf_objects__, __monkey_patch_parameter__, + __monkey_patch_trainer__ + ] for patch in patches: patch() diff --git a/paddle/scripts/cluster_train/conf.py b/paddle/scripts/cluster_train/conf.py index c8fd360e7552e..f1114a59201b9 100644 --- a/paddle/scripts/cluster_train/conf.py +++ b/paddle/scripts/cluster_train/conf.py @@ -13,17 +13,14 @@ # limitations under the License. HOSTS = [ - "root@192.168.100.17", - "root@192.168.100.18", - ] - + "root@192.168.100.17", + "root@192.168.100.18", +] ''' workspace configuration ''' #root dir for workspace, can be set as any director with real user account ROOT_DIR = "/home/paddle" - - ''' network configuration ''' @@ -37,4 +34,4 @@ PADDLE_PORTS_NUM_FOR_SPARSE = 2 #environments setting for all processes in cluster job -LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/lib64" +LD_LIBRARY_PATH = "/usr/local/cuda/lib64:/usr/lib64" diff --git a/paddle/scripts/cluster_train/paddle.py b/paddle/scripts/cluster_train/paddle.py index 79698c72e619f..7343a600c1bf5 100644 --- a/paddle/scripts/cluster_train/paddle.py +++ b/paddle/scripts/cluster_train/paddle.py @@ -12,8 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - """ module for launching cluster job """ import os @@ -23,13 +21,13 @@ import time import signal - from fabric.api import run, put, settings, env, prefix from fabric.tasks import execute #configuration for cluster import conf + def refine_unknown_args(cmd_args): ''' refine unknown parameters to handle some special parameters @@ -37,7 +35,7 @@ def refine_unknown_args(cmd_args): new_args = [] for arg in cmd_args: if arg.startswith("--") and arg.find("=") != -1: - equal_pos = arg.find("=") #find first = pos + equal_pos = arg.find("=") #find first = pos arglist = list(arg) arglist[equal_pos] = " " arg = "".join(arglist) @@ -50,6 +48,7 @@ def refine_unknown_args(cmd_args): new_args.append(arg) return new_args + def kill_process(): ''' kill comments threads @@ -60,6 +59,7 @@ def kill_process(): | awk '{print $2}' \ | xargs kill > /dev/null 2>&1") + def job_prepare(jobdir, data=None): ''' prepare job related workspace data @@ -70,6 +70,7 @@ def job_prepare(jobdir, data=None): This function just prepare all related model and other resources needed at runtime. ''' + def job_create_workspace(jobdir, data=None): ''' prepare job workspace, common file, etc. @@ -94,7 +95,8 @@ def set_nodefile(nodeid): execute(set_nodefile, i, hosts=conf.HOSTS[i]) #clean rubbish caused by exception with settings(warn_only=True): - execute(kill_process, hosts=conf.HOSTS) + execute(kill_process, hosts=conf.HOSTS) + def job_pserver(jobdir, pids=None): ''' @@ -124,9 +126,8 @@ def start_pserver(jobdir, pargs): execute(start_pserver, jobdir, pargs, hosts=conf.HOSTS) -def job_trainer(jobdir, - train_args_dict, - pids=None): + +def job_trainer(jobdir, train_args_dict, pids=None): ''' start paddle trainer ''' @@ -171,9 +172,8 @@ def start_trainer(jobdir, args): train_args += " --trainer_id=" + str(i) execute(start_trainer, jobdir, train_args, hosts=conf.HOSTS[i]) -def job_all(job_package, - jobdir=None, - train_args_dict=None): + +def job_all(job_package, jobdir=None, train_args_dict=None): ''' param job_package param train_args_dict @@ -183,41 +183,52 @@ def job_all(job_package, jobdir = conf.ROOT_DIR + "/JOB" + timestamp job_prepare(jobdir, job_package) job_pserver(jobdir) - time.sleep(5) #wait until pservers completely start + time.sleep(5) #wait until pservers completely start job_trainer(jobdir, train_args_dict) job_clean() + def job_clean(): ''' if starting job failed from paddle internal, the framework always is launched successfully since these process are daemon processes. so this job_clean can alway clean job rubbish process with ctrl+c. ''' + def signal_handler(signal, frame): ''' SIGINT handler ''' + def kill_process(): - run("ps aux \ + run("ps aux \ | grep paddle_process_by_paddle \ | grep -v grep \ | awk '{print $2}' \ | xargs kill > /dev/null 2>&1") + with settings(warn_only=True): - execute(kill_process, hosts=conf.HOSTS) + execute(kill_process, hosts=conf.HOSTS) signal.signal(signal.SIGINT, signal_handler) signal.pause() + if __name__ == '__main__': - parser = argparse.ArgumentParser(prog="paddle.py", - description='simple tool for cluster training') - parser.add_argument('-j', '--job_workspace', - required=False, default=None, - help='job workspace') - parser.add_argument('-p', '--job_dispatch_package', - required=False, default=None, - help='job package for dispatching to all other nodes') + parser = argparse.ArgumentParser( + prog="paddle.py", description='simple tool for cluster training') + parser.add_argument( + '-j', + '--job_workspace', + required=False, + default=None, + help='job workspace') + parser.add_argument( + '-p', + '--job_dispatch_package', + required=False, + default=None, + help='job package for dispatching to all other nodes') args, train_args_list = parser.parse_known_args() train_args = refine_unknown_args(train_args_list) @@ -227,14 +238,10 @@ def kill_process(): #if assigned workspace, do not need to dispatch data, #so job_local_package should be None assert args.job_dispatch_package is None - job_all(None, - args.job_workspace, - train_args_dict) + job_all(None, args.job_workspace, train_args_dict) elif args.job_dispatch_package is not None: assert args.job_workspace is None assert os.path.isdir(args.job_dispatch_package) - job_all(args.job_dispatch_package, - None, - train_args_dict) + job_all(args.job_dispatch_package, None, train_args_dict) else: print "--job_workspace or --job_dispatch_package should be set" diff --git a/paddle/trainer/tests/__init__.py b/paddle/trainer/tests/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/paddle/trainer/tests/__init__.py +++ b/paddle/trainer/tests/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/paddle/trainer/tests/config_parser_test.py b/paddle/trainer/tests/config_parser_test.py index 5ca874cec7914..c5ec315d6b01b 100644 --- a/paddle/trainer/tests/config_parser_test.py +++ b/paddle/trainer/tests/config_parser_test.py @@ -17,6 +17,6 @@ if __name__ == '__main__': parse_config_and_serialize('trainer/tests/test_config.conf', '') parse_config_and_serialize( - 'trainer/tests/sample_trainer_config.conf', + 'trainer/tests/sample_trainer_config.conf', 'extension_module_name=paddle.trainer.config_parser_extension') parse_config_and_serialize('gserver/tests/pyDataProvider/trainer.conf', '') diff --git a/paddle/trainer/tests/gen_proto_data.py b/paddle/trainer/tests/gen_proto_data.py index c818a94bee7c2..a3dbc10c886e1 100644 --- a/paddle/trainer/tests/gen_proto_data.py +++ b/paddle/trainer/tests/gen_proto_data.py @@ -21,8 +21,7 @@ import pprint logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', -) + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', ) logger = logging.getLogger('paddle') logger.setLevel(logging.INFO) @@ -36,33 +35,32 @@ # [[-1,0], [0,0]] means previous token at column 0 and current token at # column 0 are combined as one feature. patterns = [ - [[-2,0]], - [[-1,0]], - [[0,0]], - [[1,0]], - [[2,0]], - - [[-1,0], [0,0]], - [[0,0], [1,0]], - - [[-2,1]], - [[-1,1]], - [[0,1]], - [[1,1]], - [[2,1]], - [[-2,1], [-1,1]], - [[-1,1], [0,1]], - [[0,1], [1,1]], - [[1,1], [2,1]], - - [[-2,1], [-1,1], [0,1]], - [[-1,1], [0,1], [1,1]], - [[0,1], [1,1], [2,1]], + [[-2, 0]], + [[-1, 0]], + [[0, 0]], + [[1, 0]], + [[2, 0]], + [[-1, 0], [0, 0]], + [[0, 0], [1, 0]], + [[-2, 1]], + [[-1, 1]], + [[0, 1]], + [[1, 1]], + [[2, 1]], + [[-2, 1], [-1, 1]], + [[-1, 1], [0, 1]], + [[0, 1], [1, 1]], + [[1, 1], [2, 1]], + [[-2, 1], [-1, 1], [0, 1]], + [[-1, 1], [0, 1], [1, 1]], + [[0, 1], [1, 1], [2, 1]], ] + def make_features(sequence): length = len(sequence) num_features = len(sequence[0]) + def get_features(pos): if pos < 0: return ['#B%s' % -pos] * num_features @@ -72,9 +70,10 @@ def get_features(pos): for i in xrange(length): for pattern in patterns: - fname = '/'.join([get_features(i+pos)[f] for pos, f in pattern]) + fname = '/'.join([get_features(i + pos)[f] for pos, f in pattern]) sequence[i].append(fname) + ''' Source file format: Each line is for one timestep. The features are separated by space. @@ -87,6 +86,8 @@ def get_features(pos): return a list of dict for each column ''' + + def create_dictionaries(filename, cutoff, oov_policy): def add_to_dict(sequence, dicts): num_features = len(dicts) @@ -118,7 +119,6 @@ def add_to_dict(sequence, dicts): features = line.split(' ') sequence.append(features) - for i in xrange(num_features): dct = dicts[i] n = 1 if oov_policy[i] == OOV_POLICY_USE else 0 @@ -161,12 +161,9 @@ def write_proto(file, message): if oov_policy[i] == OOV_POLICY_ERROR, all features in i-th column MUST exist in dicts[i]. ''' -def gen_proto_file( - input_file, - dicts, - oov_policy, - output_file): + +def gen_proto_file(input_file, dicts, oov_policy, output_file): def write_sequence(out, sequence): num_features = len(dicts) is_beginning = True @@ -213,8 +210,8 @@ def write_sequence(out, sequence): if patterns: slot_def = header.slot_defs.add() slot_def.type = DataFormat.SlotDef.VECTOR_SPARSE_NON_VALUE - slot_def.dim = sum([len(dicts[i]) - for i in xrange(num_original_columns, len(dicts))]) + slot_def.dim = sum( + [len(dicts[i]) for i in xrange(num_original_columns, len(dicts))]) logger.info("feature_dim=%s" % slot_def.dim) for i in xrange(num_original_columns): @@ -242,30 +239,31 @@ def write_sequence(out, sequence): logger.info("num_sequences=%s" % num_sequences) + dict2 = { - 'B-ADJP': 0, - 'I-ADJP': 1, - 'B-ADVP': 2, - 'I-ADVP': 3, - 'B-CONJP': 4, - 'I-CONJP': 5, - 'B-INTJ': 6, - 'I-INTJ': 7, - 'B-LST': 8, - 'I-LST': 9, - 'B-NP': 10, - 'I-NP': 11, - 'B-PP': 12, - 'I-PP': 13, - 'B-PRT': 14, - 'I-PRT': 15, - 'B-SBAR': 16, - 'I-SBAR': 17, - 'B-UCP': 18, - 'I-UCP': 19, - 'B-VP': 20, - 'I-VP': 21, - 'O': 22 + 'B-ADJP': 0, + 'I-ADJP': 1, + 'B-ADVP': 2, + 'I-ADVP': 3, + 'B-CONJP': 4, + 'I-CONJP': 5, + 'B-INTJ': 6, + 'I-INTJ': 7, + 'B-LST': 8, + 'I-LST': 9, + 'B-NP': 10, + 'I-NP': 11, + 'B-PP': 12, + 'I-PP': 13, + 'B-PRT': 14, + 'I-PRT': 15, + 'B-SBAR': 16, + 'I-SBAR': 17, + 'B-UCP': 18, + 'I-UCP': 19, + 'B-VP': 20, + 'I-VP': 21, + 'O': 22 } if __name__ == '__main__': @@ -273,16 +271,9 @@ def write_sequence(out, sequence): cutoff += [3] * len(patterns) oov_policy = [OOV_POLICY_IGNORE, OOV_POLICY_ERROR, OOV_POLICY_ERROR] oov_policy += [OOV_POLICY_IGNORE] * len(patterns) - dicts = create_dictionaries( - 'trainer/tests/train.txt', cutoff, oov_policy) + dicts = create_dictionaries('trainer/tests/train.txt', cutoff, oov_policy) dicts[2] = dict2 - gen_proto_file( - 'trainer/tests/train.txt', - dicts, - oov_policy, - 'trainer/tests/train_proto.bin') - gen_proto_file( - 'trainer/tests/test.txt', - dicts, - oov_policy, - 'trainer/tests/test_proto.bin') + gen_proto_file('trainer/tests/train.txt', dicts, oov_policy, + 'trainer/tests/train_proto.bin') + gen_proto_file('trainer/tests/test.txt', dicts, oov_policy, + 'trainer/tests/test_proto.bin') diff --git a/paddle/trainer/tests/testPyDataWrapper.py b/paddle/trainer/tests/testPyDataWrapper.py index 49bd760f4e20e..4607bec24e1fe 100644 --- a/paddle/trainer/tests/testPyDataWrapper.py +++ b/paddle/trainer/tests/testPyDataWrapper.py @@ -21,7 +21,10 @@ import string -@provider(slots=[SparseNonValueSlot(10), DenseSlot(2), SparseValueSlot(10), StringSlot(1), IndexSlot(3)]) +@provider(slots=[ + SparseNonValueSlot(10), DenseSlot(2), SparseValueSlot(10), StringSlot(1), + IndexSlot(3) +]) def processNonSequenceData(obj, filename): with open(filename, "rb") as f: for line in f: @@ -50,6 +53,7 @@ def __values_mapper__(s): seq_count_randomer = lambda: random.randrange(1, SEQUENCE_LIMIT) str_count_randomer = lambda: random.randrange(1, STRING_LIMIT) + class IDRandomer(): # A random generator, return unique id def __init__(self): self.id_set = set() @@ -61,38 +65,57 @@ def __call__(self): return idx else: return self.__call__() + + # SparseValueSlot def sparse_value_creator(_): rand = IDRandomer() return [(rand(), val_randomer()) for _ in xrange(sparse_count_randomer())] + + sparse_value = map(sparse_value_creator, range(seq_count_randomer())) + # DenseSlot def dense_creator(_): return [val_randomer() for _ in xrange(SPARSE_ID_LIMIT)] + + dense = map(dense_creator, range(seq_count_randomer())) + # SparseNonValueSlot def sparse_creator(_): rand = IDRandomer() return [rand() for _ in xrange(sparse_count_randomer())] + + sparse_nonvalue = map(sparse_creator, range(seq_count_randomer())) # IndexSlot ids = [sparse_id_randomer() for _ in range(seq_count_randomer())] + # StringSlot -def random_str(size = 8, chars=string.ascii_letters + string.digits): +def random_str(size=8, chars=string.ascii_letters + string.digits): return ''.join(random.choice(chars) for _ in range(size)) + + strs = [random_str(str_count_randomer()) for _ in range(seq_count_randomer())] + def processSeqAndGenerateDataInit(obj, *args, **kwargs): obj.json_filename = kwargs.get("load_data_args", "test_data.json") -@provider(slots=[SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), - SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), - StringSlot(SPARSE_ID_LIMIT)], - use_seq=True, init_hook=processSeqAndGenerateDataInit) + +@provider( + slots=[ + SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), + SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), + StringSlot(SPARSE_ID_LIMIT) + ], + use_seq=True, + init_hook=processSeqAndGenerateDataInit) def processSeqAndGenerateData(obj, name): retv = [sparse_value, dense, sparse_nonvalue, ids, strs] # Write to protoseq. @@ -104,10 +127,15 @@ def processSeqAndGenerateData(obj, name): def processSubSeqAndGenerateDataInit(obj, *args, **kwargs): obj.json_filename = kwargs.get("load_data_args", "test_data.json") -@provider(slots=[SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), - SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), - StringSlot(SPARSE_ID_LIMIT)], - use_seq=True, init_hook=processSubSeqAndGenerateDataInit) + +@provider( + slots=[ + SparseValueSlot(SPARSE_ID_LIMIT), DenseSlot(SPARSE_ID_LIMIT), + SparseNonValueSlot(SPARSE_ID_LIMIT), IndexSlot(SPARSE_ID_LIMIT), + StringSlot(SPARSE_ID_LIMIT) + ], + use_seq=True, + init_hook=processSubSeqAndGenerateDataInit) def processSubSeqAndGenerateData(obj, name): retv_json = [sparse_value, dense, sparse_nonvalue, ids, strs] retv_wrapper = [[sparse_value], [dense], [sparse_nonvalue], [ids], [strs]] @@ -116,6 +144,7 @@ def processSubSeqAndGenerateData(obj, name): json.dump(retv_json, f) yield retv_wrapper + if __name__ == "__main__": pvd = processNonSequenceData("test.txt") print pvd.getNextBatch(100) diff --git a/paddle/utils/enable_virtualenv.py b/paddle/utils/enable_virtualenv.py index 99d822a4145cc..ccfaa7c147b2c 100644 --- a/paddle/utils/enable_virtualenv.py +++ b/paddle/utils/enable_virtualenv.py @@ -1,10 +1,12 @@ import os + def __activate_virtual_env__(): - __path__ = os.getenv('VIRTUAL_ENV') - if __path__ is None: - return - __script__ = os.path.join(__path__, 'bin', 'activate_this.py') - execfile(__script__, {'__file__': __script__}) + __path__ = os.getenv('VIRTUAL_ENV') + if __path__ is None: + return + __script__ = os.path.join(__path__, 'bin', 'activate_this.py') + execfile(__script__, {'__file__': __script__}) + __activate_virtual_env__() From a1ba3f442fd80382969ed2c434a66985be1e2c1f Mon Sep 17 00:00:00 2001 From: qijun Date: Sat, 12 Nov 2016 02:26:18 +0000 Subject: [PATCH 296/324] format python code in python directory --- python/paddle/__init__.py | 1 - python/paddle/trainer/PyDataProvider2.py | 35 +- .../paddle/trainer/PyDataProviderWrapper.py | 35 +- python/paddle/trainer/__init__.py | 1 - python/paddle/trainer/config_parser.py | 2008 ++++++++--------- .../paddle/trainer/config_parser_extension.py | 10 +- python/paddle/trainer/recurrent_units.py | 489 ++-- .../trainer_config_helpers/activations.py | 51 +- python/paddle/trainer_config_helpers/attrs.py | 37 +- .../trainer_config_helpers/data_sources.py | 55 +- .../default_decorators.py | 19 +- .../trainer_config_helpers/evaluators.py | 241 +- .../paddle/trainer_config_helpers/layers.py | 1433 +++++++----- python/paddle/trainer_config_helpers/math.py | 27 +- .../paddle/trainer_config_helpers/networks.py | 772 ++++--- .../trainer_config_helpers/optimizers.py | 51 +- .../paddle/trainer_config_helpers/poolings.py | 23 +- .../tests/configs/img_layers.py | 19 +- .../tests/configs/img_trans_layers.py | 20 +- .../tests/configs/last_first_seq.py | 17 +- .../tests/configs/layer_activations.py | 16 +- .../tests/configs/math_ops.py | 8 +- .../tests/configs/projections.py | 29 +- .../tests/configs/shared_fc.py | 27 +- .../tests/configs/shared_lstm.py | 28 +- .../tests/configs/simple_rnn_layers.py | 31 +- .../tests/configs/test_bi_grumemory.py | 5 +- .../tests/configs/test_bilinear_interp.py | 37 +- .../tests/configs/test_cost_layers.py | 49 +- .../configs/test_cost_layers_with_weight.py | 12 +- .../tests/configs/test_expand_layer.py | 14 +- .../tests/configs/test_fc.py | 12 +- .../tests/configs/test_grumemory_layer.py | 14 +- .../tests/configs/test_hsigmoid.py | 7 +- .../tests/configs/test_lstmemory_layer.py | 14 +- .../tests/configs/test_maxout.py | 68 +- .../tests/configs/test_ntm_layers.py | 35 +- .../tests/configs/test_print_layer.py | 5 +- .../tests/configs/test_rnn_group.py | 25 +- .../tests/configs/test_sequence_pooling.py | 21 +- .../tests/configs/test_split_datasource.py | 14 +- .../tests/configs/test_spp_layer.py | 16 +- .../tests/configs/unused_layers.py | 7 +- .../tests/configs/util_layers.py | 8 +- .../tests/layers_test_config.py | 65 +- python/paddle/trainer_config_helpers/utils.py | 4 +- python/paddle/utils/image_util.py | 76 +- python/paddle/utils/make_model_diagram.py | 17 +- python/paddle/utils/plotcurve.py | 40 +- python/paddle/utils/predefined_net.py | 240 +- python/paddle/utils/preprocess_img.py | 37 +- python/paddle/utils/preprocess_util.py | 65 +- python/paddle/utils/show_pb.py | 9 +- python/paddle/utils/torch2paddle.py | 25 +- 54 files changed, 3498 insertions(+), 2926 deletions(-) diff --git a/python/paddle/__init__.py b/python/paddle/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/python/paddle/__init__.py +++ b/python/paddle/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/python/paddle/trainer/PyDataProvider2.py b/python/paddle/trainer/PyDataProvider2.py index 53409b746d811..0c577ec657bc6 100644 --- a/python/paddle/trainer/PyDataProvider2.py +++ b/python/paddle/trainer/PyDataProvider2.py @@ -18,9 +18,8 @@ import functools import itertools -logging.basicConfig( - format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" - " %(message)s") +logging.basicConfig(format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" + " %(message)s") class SequenceType(object): @@ -132,8 +131,10 @@ def __init__(self, generator, input_order): def __call__(self, obj, filename): for item in self.generator(obj, filename): if isinstance(item, dict): - yield [item.get(input_name, None) for input_name in - self.input_order] + yield [ + item.get(input_name, None) + for input_name in self.input_order + ] else: yield item @@ -162,8 +163,8 @@ def __call__(self, obj, filename): yield items except AssertionError as e: self.logger.warning( - "Item (%s) is not fit the input type with error %s" - % (repr(item), repr(e))) + "Item (%s) is not fit the input type with error %s" % + (repr(item), repr(e))) if self.check_fail_continue: continue @@ -202,13 +203,17 @@ def loop_check(callback, item): callback(each) -def provider(input_types=None, should_shuffle=None, pool_size=-1, +def provider(input_types=None, + should_shuffle=None, + pool_size=-1, min_pool_size=-1, can_over_batch_size=True, calc_batch_size=None, cache=CacheType.NO_CACHE, - check=False, check_fail_continue=False, - init_hook=None, **kwargs): + check=False, + check_fail_continue=False, + init_hook=None, + **kwargs): """ Provider decorator. Use it to make a function into PyDataProvider2 object. In this function, user only need to get each sample for some train/test @@ -318,9 +323,9 @@ def __init__(self, file_list, **kwargs): "Could not recognize should_shuffle (%s), " "just use default value of should_shuffle." " Please set should_shuffle to bool value or " - "something in %s" % ( - repr(self.should_shuffle), - repr(true_table + false_table))) + "something in %s" % + (repr(self.should_shuffle), + repr(true_table + false_table))) self.should_shuffle = None self.pool_size = pool_size @@ -351,8 +356,7 @@ def __init__(self, file_list, **kwargs): self.generator = InputOrderWrapper(self.generator, self.input_order) if self.check: - self.generator = CheckWrapper(self.generator, - self.slots, + self.generator = CheckWrapper(self.generator, self.slots, check_fail_continue, self.logger) @@ -368,4 +372,3 @@ def deserialize_args(args): :return: """ return cPickle.loads(args) - diff --git a/python/paddle/trainer/PyDataProviderWrapper.py b/python/paddle/trainer/PyDataProviderWrapper.py index c4b907af54699..90b684a000017 100644 --- a/python/paddle/trainer/PyDataProviderWrapper.py +++ b/python/paddle/trainer/PyDataProviderWrapper.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ This module provide a wrapper(decorator) to wrap a data process method into a PyDataProvider. Some examples are shown `here `_. @@ -47,6 +46,7 @@ import io + class SlotType(object): # Just a hint for user. pass @@ -83,6 +83,7 @@ class SparseNonValueSlot(SlotType): - **SubSeq**: [[[int, int, ...], [int, ....], ...] , \ [[int, int, ...], [int, ....], ...] , ...] """ + def __init__(self, dim): """ :param dim: slot dimension @@ -294,8 +295,9 @@ def reset(self): fn = "%s_%d" % (self.profile_filename, self.profile_count) sortby = "cumulative" with open(fn, "w") as f: - pstats.Stats(self.profiler, stream=f).sort_stats( - sortby).print_stats() + pstats.Stats( + self.profiler, + stream=f).sort_stats(sortby).print_stats() self.logger.info("saving profile to file %s" % fn) self.profile_count += 1 self.logger.info("resetting profile") @@ -453,9 +455,10 @@ def writeDataStream(dat, data_callback): seq_stream.flush() subseq_stream.flush() - return "".join([self.int_packer.pack(current_batch_size), - data_bytes.getvalue(), - seq_bytes.getvalue(), subseq_bytes.getvalue()]) + return "".join([ + self.int_packer.pack(current_batch_size), data_bytes.getvalue(), + seq_bytes.getvalue(), subseq_bytes.getvalue() + ]) finally: data_stream.close() @@ -516,7 +519,7 @@ def __prepareData(self, batch_size, ret_list): self.data_pool[idx]) idx -= 1 - ret_list += self.data_pool[self.data_pool_idx: idx + 1] + ret_list += self.data_pool[self.data_pool_idx:idx + 1] # for speed reason, just shift left index, not delete data actually. self.data_pool_idx = idx + 1 @@ -537,8 +540,8 @@ def fillPool(self): if self.max_pool_size == 0: for i in xrange(min(self.file_count, len(self.generators))): self.data_pool += list(self.generators[i]) - self.generators = self.generators[ - min(self.file_count, len(self.generators)):] + self.generators = self.generators[min(self.file_count, + len(self.generators)):] self.max_pool_size = len(self.data_pool) else: while len(self.data_pool) < self.max_pool_size and len( @@ -562,9 +565,15 @@ def default_init_hook(cls, *args, **kwargs): del cls, args, kwargs -def provider(slots=None, use_seq=False, should_shuffle=True, pool_size=1, - can_over_batch_size=True, calc_batch_size=lambda data: 1, - debug=False, init_hook=default_init_hook, profile_filename=None): +def provider(slots=None, + use_seq=False, + should_shuffle=True, + pool_size=1, + can_over_batch_size=True, + calc_batch_size=lambda data: 1, + debug=False, + init_hook=default_init_hook, + profile_filename=None): """ The decorator for PyDataProvider. User should use this to create Provider class. User should only concern how to read sample from file. @@ -663,7 +672,7 @@ class Cls(GeneralPyDataProvider): def __init__(self, *file_list, **kwargs): logging.basicConfig( format="[%(levelname)s %(asctime)s %(filename)s:%(lineno)s]" - " %(message)s") + " %(message)s") self.logger = logging.getLogger("") if debug: diff --git a/python/paddle/trainer/__init__.py b/python/paddle/trainer/__init__.py index 7f9e87eee6037..c90af2ee000d4 100644 --- a/python/paddle/trainer/__init__.py +++ b/python/paddle/trainer/__init__.py @@ -11,4 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index eec978e1faf48..881f0b821491b 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -13,7 +13,6 @@ # limitations under the License. from __future__ import print_function - ''' The following functions are available in the config file: @@ -101,50 +100,45 @@ raise logging.basicConfig( - format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', -) + format='[%(levelname)s %(asctime)s %(filename)s:%(lineno)s] %(message)s', ) logger = logging.getLogger('paddle') logger.setLevel(logging.INFO) __real_print__ = print -print=logger.info +print = logger.info # from layer type name to layer class g_layer_type_map = {} + # Initialize global variables. We use this function so that we can # call parse_config() multiple times def init_config_environment( - g_default_momentum = None, - g_default_decay_rate = None, - g_default_initial_mean = 0., - g_default_initial_std = 0.01, - g_default_num_batches_regularization = None, - g_default_initial_strategy = 0, - g_default_initial_smart = False, - g_default_gradient_clipping_threshold = None, - g_default_device = None, - g_default_update_hooks = None, - g_default_compact_func = None, - - g_config = TrainerConfig(), - g_layer_map = {}, - g_parameter_map = {}, - - g_extended_config_funcs = {}, + g_default_momentum=None, + g_default_decay_rate=None, + g_default_initial_mean=0., + g_default_initial_std=0.01, + g_default_num_batches_regularization=None, + g_default_initial_strategy=0, + g_default_initial_smart=False, + g_default_gradient_clipping_threshold=None, + g_default_device=None, + g_default_update_hooks=None, + g_default_compact_func=None, + g_config=TrainerConfig(), + g_layer_map={}, + g_parameter_map={}, + g_extended_config_funcs={}, # store command args of paddle_trainer - g_command_config_args = {}, + g_command_config_args={}, # Used for PyDataProvider to avoid duplicate module name - g_py_module_name_list = [], - - g_current_submodel = None, - g_root_submodel = None, - g_submodel_map = {}, - g_submodel_stack = [], - - g_add_submodel_suffix = False, - ): + g_py_module_name_list=[], + g_current_submodel=None, + g_root_submodel=None, + g_submodel_map={}, + g_submodel_stack=[], + g_add_submodel_suffix=False, ): for k, v in locals().iteritems(): globals()[k] = copy.deepcopy(v) @@ -161,43 +155,54 @@ def config_assert(b, msg): if not b: logger.fatal(msg) + g_config_funcs = {} + # decorator for indicating a function which can be used in config file def config_func(func): g_config_funcs[func.func_name] = func return func + # decorator for indicating a class which can be used in config file def config_class(cls): g_config_funcs[cls.__name__] = cls return cls + # decorator for indicating a class for a layer type def config_layer(layer_type): def wrap(cls): g_config_funcs[cls.__name__] = cls g_layer_type_map[layer_type] = cls return cls + return wrap + def gen_parameter_name(layer_name, input_index): return '_%s.w%d' % (layer_name, input_index) + def gen_bias_parameter_name(layer_name): return '_%s.wbias' % layer_name + def default(x, default_value): return default_value if x is None else x + class Cfg(object): def add_keys(self, locals): for k, v in locals.iteritems(): if not k.startswith('_'): self.__setattr__(k, v) + # functions available in config file + # Define the name of the input layers of the NeuralNetwork. # The type of these layers must be "data". # These layers will be provided with the DataBatch obtained @@ -216,6 +221,7 @@ def Inputs(*args): if g_current_submodel is g_root_submodel: g_config.model_config.input_layer_names.append(name) + @config_func def HasInputsSet(): return len(g_current_submodel.input_layer_names) != 0 @@ -244,7 +250,7 @@ def SubModelBegin(name): global g_current_submodel, g_root_submodel, g_submodel_stack g_submodel_stack.append(g_current_submodel) - name = MakeLayerNameInParentSubmodel(name) #rename in nested submodel + name = MakeLayerNameInParentSubmodel(name) #rename in nested submodel config_assert(name not in g_submodel_map, 'Duplicated submodel name: %s' % name) @@ -254,36 +260,42 @@ def SubModelBegin(name): g_submodel_map[name] = sub_model g_current_submodel = sub_model + @config_func -def SubModelEnd(name = None): +def SubModelEnd(name=None): global g_current_submodel, g_root_submodel, g_submodel_stack - config_assert(g_current_submodel is not g_root_submodel, "submodel not begin") + config_assert(g_current_submodel is not g_root_submodel, + "submodel not begin") if name is not None: - config_assert(g_current_submodel.name == MakeLayerNameInParentSubmodel(name), - "submodel name error") + config_assert( + g_current_submodel.name == MakeLayerNameInParentSubmodel(name), + "submodel name error") g_current_submodel = g_submodel_stack.pop() + def MakeLayerNameInParentSubmodel(name): suffix = "" if len(g_submodel_stack) > 1: suffix = "@" + g_submodel_stack[-1].name return name + suffix + def GetLayerBaseName(name): return name.split('@')[0] -def MakeLayerNameInSubmodel(name, submodel_name = None): + +def MakeLayerNameInSubmodel(name, submodel_name=None): global g_current_submodel global g_add_submodel_suffix - if (submodel_name is None - and not g_add_submodel_suffix - and not g_current_submodel.is_recurrent_layer_group): + if (submodel_name is None and not g_add_submodel_suffix and + not g_current_submodel.is_recurrent_layer_group): return name if submodel_name is None: submodel_name = g_current_submodel.name return name + "@" + submodel_name + # Define a recurrent layer group begin with RecurrentLayerGroupBegin # and end with RecurrentLayerGroupEnd. # A recurrent layer group forward/backward one frame after previous frame @@ -332,8 +344,10 @@ def RecurrentLayerGroupWithoutOutLinksBegin(name, if in_links_count == 0: in_links_has_subseq = has_subseq else: - config_assert(in_links_has_subseq == has_subseq, - "The sequence type of in_links should be the same in RecurrentLayerGroup") + config_assert( + in_links_has_subseq == has_subseq, + "The sequence type of in_links should be the same in RecurrentLayerGroup" + ) in_links_count += 1 layer_name = MakeLayerNameInParentSubmodel(name) layer = g_layer_map[layer_name] @@ -347,6 +361,7 @@ def RecurrentLayerGroupWithoutOutLinksBegin(name, pair.link_name = MakeLayerNameInSubmodel(name) pair.has_subseq = has_subseq + @config_func def RecurrentLayerGroupSetOutLink(link): if isinstance(link, basestring): @@ -363,8 +378,7 @@ def RecurrentLayerGroupSetOutLink(link): def RecurrentLayerGroupSetGenerator(generator=None): - generator.eos_layer_name = MakeLayerNameInSubmodel( - generator.eos_layer_name) + generator.eos_layer_name = MakeLayerNameInSubmodel(generator.eos_layer_name) g_current_submodel.generator.CopyFrom(generator) @@ -375,21 +389,18 @@ def RecurrentLayerGroupBegin(name, generator=None, target_inlinkname="", seq_reversed=False): - RecurrentLayerGroupWithoutOutLinksBegin(name, - in_links, - seq_reversed, + RecurrentLayerGroupWithoutOutLinksBegin(name, in_links, seq_reversed, target_inlinkname) for link in out_links: RecurrentLayerGroupSetOutLink(link) - if generator is not None: RecurrentLayerGroupSetGenerator(generator) - config_assert(len(in_links) == 0, - "no in_links should be passed to generator") - config_assert(len(out_links) >= 1, - "one or more than one out_links should be passed to generator") - + config_assert( + len(in_links) == 0, "no in_links should be passed to generator") + config_assert( + len(out_links) >= 1, + "one or more than one out_links should be passed to generator") @config_func @@ -397,9 +408,10 @@ def RecurrentLayerGroupEnd(name): global g_current_submodel config_assert(g_current_submodel.is_recurrent_layer_group, "RecurrentLayerGroup not begin") - for pair in g_current_submodel.memories: #check exist + for pair in g_current_submodel.memories: #check exist layer = g_layer_map[pair.layer_name] - config_assert(layer is not None, "memory declare wrong name:%s" % pair.layer_name) + config_assert(layer is not None, "memory declare wrong name:%s" % + pair.layer_name) memory_link = g_layer_map[pair.link_name] config_assert(layer.size == memory_link.size, "memory declare wrong size:%d" % memory_link.size) @@ -418,12 +430,14 @@ def RecurrentLayerGroupEnd(name): else: GatherAgentLayer(name=agent_name, size=layer.size) + # Define the model type # currently, the paddle supports "nn", "recurrent_nn", "recursive_nn" and "multi_nn" @config_func def model_type(name): g_config.model_config.type = name + @config_class class Bias(Cfg): def __init__( @@ -441,10 +455,10 @@ def __init__( sparse_remote_update=None, gradient_clipping_threshold=None, is_static=None, - is_shared=None, - ): + is_shared=None, ): self.add_keys(locals()) + # Define one input for a layer @config_class class Input(Cfg): @@ -477,19 +491,20 @@ def __init__( is_static=None, is_shared=None, update_hooks=None, - input_layer_argument=None, - ): + input_layer_argument=None, ): self.add_keys(locals()) self.input_layer_name = MakeLayerNameInSubmodel(input_layer_name) + # Define a projection for iexed layer @config_class class Projection(Input): - type = None # subclass should set it correctly + type = None # subclass should set it correctly + def __init__( self, input_layer_name, - size = 0, # projection output size + size=0, # projection output size parameter_name=None, learning_rate=None, momentum=None, @@ -509,8 +524,7 @@ def __init__( is_static=None, is_shared=None, update_hooks=None, - input_layer_argument=None, - ): + input_layer_argument=None, ): self.add_keys(locals()) self.input_layer_name = MakeLayerNameInSubmodel(input_layer_name) @@ -524,8 +538,10 @@ def __init__( # to indicate using the size from Layer config def calc_output_size(self, input_layer_config): return self.size + def calc_parameter_size(self, input_size, output_size): raise NotimplementedError + def calc_parameter_dims(self, input_size, output_size): raise NotimplementedError @@ -536,31 +552,32 @@ class IdentityProjection(Projection): def calc_output_size(self, input_layer_config): return input_layer_config.size + def calc_parameter_size(self, input_size, output_size): return 0 + def calc_parameter_dims(self, input_size, output_size): return [] + # Like IdentityProjection, but layer size may smaller than input size, # the projection select dimesions [offset, offset+layer_size) from input @config_class class IdentityOffsetProjection(Projection): type = 'identity_offset' - def __init__( - self, - input_layer_name, - offset, - **xargs): - super(IdentityOffsetProjection, self).__init__( - input_layer_name, **xargs) + def __init__(self, input_layer_name, offset, **xargs): + super(IdentityOffsetProjection, self).__init__(input_layer_name, + **xargs) self.proj_conf.offset = offset def calc_parameter_size(self, input_size, output_size): return 0 + def calc_parameter_dims(self, input_size, output_size): return [] + # DotMulProjection performs element-wise multiplication with weight @config_class class DotMulProjection(Projection): @@ -568,49 +585,53 @@ class DotMulProjection(Projection): def calc_output_size(self, input_layer_config): return input_layer_config.size + def calc_parameter_size(self, input_size, output_size): return output_size + def calc_parameter_dims(self, input_size, output_size): return [1, output_size] + @config_class class TableProjection(Projection): type = 'table' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [input_size, output_size] + @config_class class FullMatrixProjection(Projection): type = 'fc' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [input_size, output_size] + @config_class class TransposedFullMatrixProjection(Projection): type = 'trans_fc' def calc_parameter_size(self, input_size, output_size): return input_size * output_size + def calc_parameter_dims(self, input_size, output_size): return [output_size, input_size] + @config_class class ContextProjection(Projection): type = 'context' - def __init__( - self, - input_layer_name, - context_start, - context_length, - trainable_padding, - **xargs): + def __init__(self, input_layer_name, context_start, context_length, + trainable_padding, **xargs): super(ContextProjection, self).__init__(input_layer_name, **xargs) self.proj_conf.context_start = context_start self.proj_conf.context_length = context_length @@ -638,23 +659,21 @@ def calc_parameter_dims(self, input_size, output_size): class ConvProjection(Projection): type = 'conv' - def __init__( - self, - input_layer_name, - num_filters=None, - conv_conf=None, - **xargs): + def __init__(self, + input_layer_name, + num_filters=None, + conv_conf=None, + **xargs): super(ConvProjection, self).__init__(input_layer_name, **xargs) if num_filters is not None: self.proj_conf.num_filters = num_filters - parse_conv(conv_conf, - input_layer_name, - self.proj_conf.conv_conf, + parse_conv(conv_conf, input_layer_name, self.proj_conf.conv_conf, num_filters) # TODO: support rectangle input - self.proj_conf.output_size = (self.proj_conf.conv_conf.output_x ** 2) * num_filters + self.proj_conf.output_size = (self.proj_conf.conv_conf.output_x** + 2) * num_filters def calc_output_size(self, input_layer_config): return self.proj_conf.output_size @@ -672,14 +691,15 @@ def calc_bias_size(self): def calc_parameter_dims(self, input_size, output_size): return None + # Define a operator for mixed layer @config_class class Operator(Cfg): - type = None # subclass should set it correctly + type = None # subclass should set it correctly + def __init__( self, - input_layer_names, - ): + input_layer_names, ): self.add_keys(locals()) self.operator_conf = OperatorConfig() self.operator_conf.type = self.type @@ -690,16 +710,13 @@ def check_dims(self): def calc_output_size(self, input_sizes): return 0 + @config_class class DotMulOperator(Operator): type = 'dot_mul' - def __init__( - self, - input_layer_names, - scale=None, - **xargs): - super(DotMulOperator, self).__init__( - input_layer_names, **xargs) + + def __init__(self, input_layer_names, scale=None, **xargs): + super(DotMulOperator, self).__init__(input_layer_names, **xargs) if scale is not None: self.operator_conf.dotmul_scale = scale @@ -715,26 +732,24 @@ def calc_output_size(self, input_sizes): return input_sizes[0] - @config_class class ConvOperator(Operator): type = 'conv' - def __init__( - self, - input_layer_names, - num_filters=None, - conv_conf=None, - **xargs): - super(ConvOperator, self).__init__( - input_layer_names, **xargs) + + def __init__(self, + input_layer_names, + num_filters=None, + conv_conf=None, + **xargs): + super(ConvOperator, self).__init__(input_layer_names, **xargs) if num_filters is not None: self.operator_conf.num_filters = num_filters parse_conv(conv_conf, MakeLayerNameInSubmodel(input_layer_names[0]), - self.operator_conf.conv_conf, - num_filters) - self.operator_conf.output_size = (self.operator_conf.conv_conf.output_x ** 2) * num_filters + self.operator_conf.conv_conf, num_filters) + self.operator_conf.output_size = (self.operator_conf.conv_conf.output_x + **2) * num_filters config_assert(len(input_layer_names) == 2, "Conv is binary operator") @@ -745,119 +760,106 @@ def calc_output_size(self, input_sizes): # please refer to the comments in proto/ModelConfig.proto @config_class class Conv(Cfg): - def __init__( - self, - filter_size, - channels, - padding = None, - stride = None, - groups = None, - filter_channels = None, - output_x = None, - img_size = None, - caffe_mode = True, - filter_size_y = None, - padding_y = None, - stride_y = None): + def __init__(self, + filter_size, + channels, + padding=None, + stride=None, + groups=None, + filter_channels=None, + output_x=None, + img_size=None, + caffe_mode=True, + filter_size_y=None, + padding_y=None, + stride_y=None): self.add_keys(locals()) if filter_size_y is None: - self.filter_size_y = filter_size + self.filter_size_y = filter_size if padding_y is None: - self.padding_y = padding + self.padding_y = padding if stride_y is None: - self.stride_y = stride + self.stride_y = stride if output_x is not None: - config_assert(output_x <= 0) + config_assert(output_x <= 0) + # please refer to the comments in proto/ModelConfig.proto @config_class class BilinearInterp(Cfg): - def __init__( - self, - out_size_x = None, - out_size_y = None, - num_channels = None): + def __init__(self, out_size_x=None, out_size_y=None, num_channels=None): self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Pool(Cfg): - def __init__( - self, - pool_type, - channels, - size_x, - size_y = None, - img_width = None, - start = None, - stride = None, - stride_y = None, - padding = None, - padding_y = None): + def __init__(self, + pool_type, + channels, + size_x, + size_y=None, + img_width=None, + start=None, + stride=None, + stride_y=None, + padding=None, + padding_y=None): self.add_keys(locals()) - + + # please refer to the comments in proto/ModelConfig.proto @config_class class SpatialPyramidPool(Cfg): - def __init__( - self, - pool_type, - pyramid_height, - channels, - img_width = None): + def __init__(self, pool_type, pyramid_height, channels, img_width=None): self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Norm(Cfg): - def __init__( - self, - norm_type, - channels, - size, - scale, - pow, - output_x = None, - img_size = None, - blocked = None): + def __init__(self, + norm_type, + channels, + size, + scale, + pow, + output_x=None, + img_size=None, + blocked=None): self.add_keys(locals()) + # please refer to the comments in proto/ModelConfig.proto @config_class class Image(Cfg): - def __init__( - self, - channels, - img_size = None): + def __init__(self, channels, img_size=None): self.add_keys(locals()) + @config_class class BlockExpand(Cfg): - def __init__( - self, - channels, - padding_x = 0, - padding_y = 0, - stride_x = 0, - stride_y = 0, - block_x = 0, - block_y = 0, - img_size_x = 0, - img_size_y = 0, - output_x = 0, - output_y = 0): + def __init__(self, + channels, + padding_x=0, + padding_y=0, + stride_x=0, + stride_y=0, + block_x=0, + block_y=0, + img_size_x=0, + img_size_y=0, + output_x=0, + output_y=0): self.add_keys(locals()) + @config_class class MaxOut(Cfg): - def __init__( - self, - channels, - groups, - img_size_x = 0, - img_size_y = 0): + def __init__(self, channels, groups, img_size_x=0, img_size_y=0): self.add_keys(locals()) + def DataBase(async_load_data=False, constant_slots=None, data_ratio=1, @@ -871,23 +873,23 @@ def DataBase(async_load_data=False, if constant_slots: data_config.constant_slots.extend(constant_slots) - data_config.data_ratio=data_ratio - data_config.is_main_data=is_main_data + data_config.data_ratio = data_ratio + data_config.is_main_data = is_main_data - usage_ratio=default(usage_ratio, settings_deprecated["usage_ratio"]) + usage_ratio = default(usage_ratio, settings_deprecated["usage_ratio"]) config_assert(usage_ratio >= 0 and usage_ratio <= 1, "The range of usage_ratio is [0, 1]") data_config.usage_ratio = usage_ratio return data_config + @config_func -def SimpleData( - files=None, - feat_dim=None, - context_len=None, - buffer_capacity=None, - **xargs): +def SimpleData(files=None, + feat_dim=None, + context_len=None, + buffer_capacity=None, + **xargs): data_config = DataBase(**xargs) data_config.type = 'simple' data_config.files = files @@ -898,31 +900,36 @@ def SimpleData( data_config.buffer_capacity = buffer_capacity return data_config + @config_func -def PyData( - files=None, - type=None, - file_group_queue_capacity=None, - load_data_module=None, - load_data_object=None, - load_data_args="", - load_file_count=None, - constant_slots=None, - load_thread_num=None, - **xargs): +def PyData(files=None, + type=None, + file_group_queue_capacity=None, + load_data_module=None, + load_data_object=None, + load_data_args="", + load_file_count=None, + constant_slots=None, + load_thread_num=None, + **xargs): data_config = DataBase(**xargs) data_config.type = 'py' if load_data_module in g_py_module_name_list: + def get_path(module): m = __import__(load_data_module) return os.path.split(os.path.realpath(m.__file__))[0] + # python C-api is not thread safe, one module can only be import once, # so here we nedd to copy the module with different names if it has to be # imported several times. - module_new_name = "%s_copy_%d" % (load_data_module, len(g_py_module_name_list)) + module_new_name = "%s_copy_%d" % (load_data_module, + len(g_py_module_name_list)) g_py_module_name_list.append(module_new_name) - module_path = "%s/%s.py" % (get_path(load_data_module), load_data_module) - new_module_path = "%s/%s.py" % (get_path(load_data_module), module_new_name) + module_path = "%s/%s.py" % (get_path(load_data_module), + load_data_module) + new_module_path = "%s/%s.py" % (get_path(load_data_module), + module_new_name) if os.path.isfile(module_path) == False: raise Exception("File %s is not exist." % module_path) shutil.copy2(module_path, new_module_path) @@ -947,15 +954,15 @@ def get_path(module): data_config.constant_slots.extend(constant_slots) return data_config + @config_func -def ProtoData( - files=None, - type=None, - file_group_queue_capacity=None, - load_file_count=None, - constant_slots=None, - load_thread_num=None, - **xargs): +def ProtoData(files=None, + type=None, + file_group_queue_capacity=None, + load_file_count=None, + constant_slots=None, + load_thread_num=None, + **xargs): data_config = DataBase(**xargs) if type is None: data_config.type = 'proto' @@ -976,25 +983,24 @@ def ProtoData( data_config.constant_slots.extend(constant_slots) return data_config + #real data for training is actually provided by "sub_data" data providers. @config_func -def MultiData( - sub_data=[] - ): +def MultiData(sub_data=[]): data_config = DataConfig() data_config.type = 'multi' data_config.sub_data_configs.extend(sub_data) return data_config + @config_func -def Data( - type, - files=None, - feat_dim=None, - slot_dims=None, - context_len=None, - buffer_capacity=None, - **xargs): +def Data(type, + files=None, + feat_dim=None, + slot_dims=None, + context_len=None, + buffer_capacity=None, + **xargs): data_config = DataBase(**xargs) data_config.type = type @@ -1030,15 +1036,19 @@ def TestData(data_config, async_load_data=None): " Data definition") g_config.test_data_config.async_load_data = async_load_data + def parse_bilinear(bilinear, input_layer_name, bilinear_conf): - bilinear_conf.out_size_x = bilinear.out_size_x; - bilinear_conf.out_size_y = bilinear.out_size_y; - bilinear_conf.num_channels = bilinear.num_channels; + bilinear_conf.out_size_x = bilinear.out_size_x + bilinear_conf.out_size_y = bilinear.out_size_y + bilinear_conf.num_channels = bilinear.num_channels + ''' caffe_mode: compute the output size using floor instead of ceil, which is consistent of caffe and CuDNN's convention. ''' + + def cnn_output_size(img_size, filter_size, padding, stride, caffe_mode): output = (2 * padding + img_size - filter_size) / float(stride) if caffe_mode: @@ -1046,81 +1056,89 @@ def cnn_output_size(img_size, filter_size, padding, stride, caffe_mode): else: return 1 + int(math.ceil(output)) + ''' calcualte image_size based on output_size for convolution. It is the reverse function of cnn_output_size ''' + + def cnn_image_size(output_size, filter_size, padding, stride, caffe_mode): if caffe_mode: img_size = (output_size - 1) * stride + filter_size - 2 * padding else: - img_size = (output_size - 2) * stride + filter_size - 2 * padding + 1 + img_size = (output_size - 2) * stride + filter_size - 2 * padding + 1 return img_size + def parse_pool(pool, input_layer_name, pool_conf): pool_conf.pool_type = pool.pool_type - config_assert(pool.pool_type in ['max-projection', 'avg-projection', - 'cudnn-max-pool', 'cudnn-avg-pool'], - "pool-type %s is not in " + config_assert(pool.pool_type in [ + 'max-projection', 'avg-projection', 'cudnn-max-pool', 'cudnn-avg-pool' + ], "pool-type %s is not in " "['max-projection', 'avg-projection', " - "'cudnn-max-pool', 'cudnn-avg-pool']" - % pool.pool_type) + "'cudnn-max-pool', 'cudnn-avg-pool']" % pool.pool_type) pool_conf.channels = pool.channels pool_conf.size_x = pool.size_x pool_conf.stride = pool.stride pool_conf.size_y = default(pool.size_y, pool_conf.size_x) - pool_conf.stride_y = default(pool.stride_y, pool_conf.stride); + pool_conf.stride_y = default(pool.stride_y, pool_conf.stride) img_pixels = g_layer_map[input_layer_name].size / pool.channels # the img_width may be removed, # and it can be calculated automatically later. - pool_conf.img_size = default(pool.img_width, int(img_pixels ** 0.5)) + pool_conf.img_size = default(pool.img_width, int(img_pixels**0.5)) pool_conf.img_size_y = img_pixels / pool_conf.img_size config_assert(pool_conf.img_size * pool_conf.img_size_y == img_pixels, - "Incorrect input image size %d for input image pixels %d" - % (pool_conf.img_size, img_pixels)) + "Incorrect input image size %d for input image pixels %d" % + (pool_conf.img_size, img_pixels)) config_assert(not pool.start, "start is deprecated in pooling.") if pool.padding is not None: pool_conf.padding = pool.padding pool_conf.padding_y = default(pool.padding_y, pool_conf.padding) - pool_conf.output_x = cnn_output_size(pool_conf.img_size, pool_conf.size_x, - pool_conf.padding, pool_conf.stride, False) - pool_conf.output_y = cnn_output_size(pool_conf.img_size_y, pool_conf.size_y, - pool_conf.padding_y, pool_conf.stride_y, False) + pool_conf.output_x = cnn_output_size( + pool_conf.img_size, pool_conf.size_x, pool_conf.padding, + pool_conf.stride, False) + pool_conf.output_y = cnn_output_size( + pool_conf.img_size_y, pool_conf.size_y, pool_conf.padding_y, + pool_conf.stride_y, False) + def parse_spp(spp, input_layer_name, spp_conf): spp_conf.pool_type = spp.pool_type config_assert(spp.pool_type in ['max-projection', 'avg-projection'], - "pool-type %s is not in " "['max-projection', 'avg-projection']" - % spp.pool_type) + "pool-type %s is not in " + "['max-projection', 'avg-projection']" % spp.pool_type) spp_conf.pyramid_height = spp.pyramid_height spp_conf.channels = spp.channels img_pixels = g_layer_map[input_layer_name].size / spp_conf.channels - spp_conf.img_size = default(spp.img_width, int(img_pixels ** 0.5)) + spp_conf.img_size = default(spp.img_width, int(img_pixels**0.5)) spp_conf.img_size_y = img_pixels / spp_conf.img_size config_assert(spp_conf.img_size * spp_conf.img_size_y == img_pixels, - "Incorrect input image size %d for input image pixels %d" - % (spp_conf.img_size, img_pixels)) + "Incorrect input image size %d for input image pixels %d" % + (spp_conf.img_size, img_pixels)) + def parse_image(image, input_layer_name, image_conf): image_conf.channels = image.channels image_pixels = g_layer_map[input_layer_name].size / image_conf.channels - image_conf.img_size = int(image_pixels ** 0.5) - config_assert((image_conf.img_size ** 2) == image_pixels, - "Incorrect input image size %d for input image pixels %d" - % (image_conf.img_size, image_pixels)) + image_conf.img_size = int(image_pixels**0.5) + config_assert((image_conf.img_size**2) == image_pixels, + "Incorrect input image size %d for input image pixels %d" % + (image_conf.img_size, image_pixels)) + def parse_norm(norm, input_layer_name, norm_conf): norm_conf.norm_type = norm.norm_type config_assert(norm.norm_type in ['rnorm', 'cmrnorm-projection'], - "norm-type %s is not in [rnorm, 'cmrnorm-projection']" - % norm.norm_type) + "norm-type %s is not in [rnorm, 'cmrnorm-projection']" % + norm.norm_type) norm_conf.channels = norm.channels norm_conf.size = norm.size norm_conf.scale = norm.scale @@ -1128,20 +1146,23 @@ def parse_norm(norm, input_layer_name, norm_conf): norm_conf.blocked = norm.blocked img_pixels = g_layer_map[input_layer_name].size / norm.channels - norm_conf.img_size = int(img_pixels ** 0.5) - config_assert((norm_conf.img_size ** 2) == img_pixels, - "Incorrect input image size %d for input image pixels %d" - % (norm_conf.img_size, img_pixels)) + norm_conf.img_size = int(img_pixels**0.5) + config_assert((norm_conf.img_size**2) == img_pixels, + "Incorrect input image size %d for input image pixels %d" % + (norm_conf.img_size, img_pixels)) norm_conf.output_x = norm_conf.img_size if norm.norm_type in ['cmrnorm-projection']: norm_conf.scale /= norm.size else: - norm_conf.scale /= norm.size ** 2 + norm_conf.scale /= norm.size**2 + ''' caffe_mode: compute the output size using floor instead of ceil, which is consistent of caffe and CuDNN's convention. ''' + + def parse_conv(conv, input_layer_name, conv_conf, num_filters, trans=False): conv_conf.filter_size = conv.filter_size conv_conf.filter_size_y = conv.filter_size_y @@ -1152,36 +1173,37 @@ def parse_conv(conv, input_layer_name, conv_conf, num_filters, trans=False): conv_conf.stride_y = conv.stride_y conv_conf.groups = conv.groups conv_conf.caffe_mode = conv.caffe_mode - + if not trans: conv_conf.filter_channels = conv.channels / conv.groups img_pixels = g_layer_map[input_layer_name].size / conv.channels - print('channels=%d size=%d'%(conv.channels, - g_layer_map[input_layer_name].size)) - conv_conf.img_size = int(img_pixels ** 0.5) - config_assert((conv_conf.img_size ** 2) == img_pixels, - ("Input layer %s: Incorrect input image size %d for input " - + "image pixels %d") - % (input_layer_name, conv_conf.img_size, img_pixels)) - + print('channels=%d size=%d' % (conv.channels, + g_layer_map[input_layer_name].size)) + conv_conf.img_size = int(img_pixels**0.5) + config_assert((conv_conf.img_size**2) == img_pixels, ( + "Input layer %s: Incorrect input image size %d for input " + + "image pixels %d") % + (input_layer_name, conv_conf.img_size, img_pixels)) + conv_conf.output_x = cnn_output_size( - conv_conf.img_size, conv_conf.filter_size, - conv_conf.padding, conv_conf.stride, conv_conf.caffe_mode) + conv_conf.img_size, conv_conf.filter_size, conv_conf.padding, + conv_conf.stride, conv_conf.caffe_mode) else: conv_conf.filter_channels = num_filters / conv.groups - + outputSize = g_layer_map[input_layer_name].size / conv.channels - print('channels=%d size=%d'%(conv.channels, - g_layer_map[input_layer_name].size)) - conv_conf.output_x = int(outputSize ** 0.5) - config_assert((conv_conf.output_x ** 2) == outputSize, - ("Input layer %s: Incorrect input image size %d for input " - + "image pixels %d") - % (input_layer_name, conv_conf.output_x, outputSize)) + print('channels=%d size=%d' % (conv.channels, + g_layer_map[input_layer_name].size)) + conv_conf.output_x = int(outputSize**0.5) + config_assert((conv_conf.output_x**2) == outputSize, ( + "Input layer %s: Incorrect input image size %d for input " + + "image pixels %d") % + (input_layer_name, conv_conf.output_x, outputSize)) conv_conf.img_size = cnn_image_size( - conv_conf.output_x, conv_conf.filter_size, - conv_conf.padding, conv_conf.stride, conv_conf.caffe_mode) + conv_conf.output_x, conv_conf.filter_size, conv_conf.padding, + conv_conf.stride, conv_conf.caffe_mode) + def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand_conf.channels = block_expand.channels @@ -1207,27 +1229,28 @@ def parse_block_expand(block_expand, input_layer_name, block_expand_conf): block_expand.img_size_y, block_expand.block_y, block_expand.padding_y, block_expand.stride_y, False) + def parse_maxout(maxout, input_layer_name, maxout_conf): maxout_conf.channels = maxout.channels maxout_conf.groups = maxout.groups maxout_conf.img_size_x = maxout.img_size_x maxout_conf.img_size_y = maxout.img_size_y + # Define an evaluator @config_func def Evaluator( name, type, inputs, - chunk_scheme = None, - num_chunk_types = None, - classification_threshold = None, - positive_label = None, - dict_file = None, - result_file = None, - num_results = None, - delimited = None, - ): + chunk_scheme=None, + num_chunk_types=None, + classification_threshold=None, + positive_label=None, + dict_file=None, + result_file=None, + num_results=None, + delimited=None, ): evaluator = g_config.model_config.evaluators.add() evaluator.type = type evaluator.name = MakeLayerNameInSubmodel(name) @@ -1256,19 +1279,20 @@ def Evaluator( if delimited is not None: evaluator.delimited = delimited + class LayerBase(object): def __init__( self, name, type, - size, # size can be 0. In this case, subclass should set it. + size, # size can be 0. In this case, subclass should set it. inputs, device=None, active_type="", drop_rate=0., coeff=None): config_assert('@' not in name, - "layer name: %s contain special character @" % name) + "layer name: %s contain special character @" % name) global g_current_submodel name = MakeLayerNameInSubmodel(name) @@ -1307,8 +1331,8 @@ def __init__( if type_of(input) == str: input_layer_name = input input_config = Input( - input_layer_name = input, - parameter_name = gen_parameter_name(name, input_index)) + input_layer_name=input, + parameter_name=gen_parameter_name(name, input_index)) input_layer_name = input_config.input_layer_name elif isinstance(input, Input): input_layer_name = input.input_layer_name @@ -1317,16 +1341,15 @@ def __init__( input_config.parameter_name = \ gen_parameter_name(name, input_index) elif isinstance(input, Operator): - self.operators.append(input); + self.operators.append(input) input.operator_conf.input_indices.append(input_index) input_config = Input(input.input_layer_names[0]) input_layer_name = input_config.input_layer_name else: - raise ValueError( - 'Wrong type for inputs: %s' % type_of(input)) + raise ValueError('Wrong type for inputs: %s' % type_of(input)) config_assert(input_layer_name in g_layer_map, - "Unknown input layer '%s' for layer %s" - % (input_layer_name, name)) + "Unknown input layer '%s' for layer %s" % + (input_layer_name, name)) self.inputs[input_index] = input_config layer_input = self.config.inputs.add() layer_input.input_layer_name = input_config.input_layer_name @@ -1338,26 +1361,26 @@ def __init__( g_current_submodel.layer_names.append(self.config.name) - def get_input_layer(self, input_index): return g_layer_map[self.config.inputs[input_index].input_layer_name] # will return the bias created if not *for_self* def create_bias_parameter( self, - bias, # True/False or BiasCfg + bias, # True/False or BiasCfg size, - dims = None, - for_self = True, # whether create bias for layer self - ): + dims=None, + for_self=True, # whether create bias for layer self + ): if size == 0: return if dims is None: dims = [1, size] - config_assert(type_of(bias) == bool or type_of(bias) == Bias, - 'Incorrect type for bias: %s' % type_of(bias)) + config_assert( + type_of(bias) == bool or type_of(bias) == Bias, + 'Incorrect type for bias: %s' % type_of(bias)) if type_of(bias) == bool: if bias: @@ -1372,7 +1395,8 @@ def create_bias_parameter( Parameter( bias.parameter_name, size, - self.config.device if self.config.HasField('device') else None, + self.config.device + if self.config.HasField('device') else None, dims, bias.learning_rate, bias.momentum, @@ -1384,22 +1408,21 @@ def create_bias_parameter( initial_smart=bias.initial_smart, num_batches_regularization=bias.num_batches_regularization, sparse_remote_update=bias.sparse_remote_update, - gradient_clipping_threshold=bias.gradient_clipping_threshold, + gradient_clipping_threshold=bias. + gradient_clipping_threshold, is_static=bias.is_static, - is_shared=bias.is_shared, - ) + is_shared=bias.is_shared, ) if for_self: self.config.bias_parameter_name = bias.parameter_name else: return bias.parameter_name - def create_input_parameter( - self, - input_index, - size, - dims=None, - sparse = None, - format = None): + def create_input_parameter(self, + input_index, + size, + dims=None, + sparse=None, + format=None): if dims is None: # TODO(yuyang18): print warning and callstack here! dims = list() @@ -1414,12 +1437,12 @@ def create_input_parameter( if input_config.parameter_name in g_parameter_map: para = g_parameter_map[input_config.parameter_name] - config_assert(size == para.size, ('Shared parameter "%s" does not ' - + 'have same size: %s vs. %s') + config_assert(size == para.size, ( + 'Shared parameter "%s" does not ' + 'have same size: %s vs. %s') % (input_config.parameter_name, para.size, size)) - config_assert(dims == para.dims, ('Shared parameter "%s" does not ' - + 'have same dims: %s vs. %s') + config_assert(dims == para.dims, ( + 'Shared parameter "%s" does not ' + 'have same dims: %s vs. %s') % (input_config.parameter_name, para.dims, dims)) return @@ -1439,13 +1462,13 @@ def create_input_parameter( num_batches_regularization=input_config.num_batches_regularization, sparse_remote_update=input_config.sparse_remote_update, sparse_update=input_config.sparse_update, - gradient_clipping_threshold=input_config.gradient_clipping_threshold, + gradient_clipping_threshold=input_config. + gradient_clipping_threshold, sparse=sparse, format=format, is_static=input_config.is_static, is_shared=input_config.is_shared, - update_hooks=input_config.update_hooks - ) + update_hooks=input_config.update_hooks) def set_layer_size(self, size): if self.config.size == 0: @@ -1455,27 +1478,18 @@ def set_layer_size(self, size): 'Different inputs result in' + 'different layer size at layer %s' % self.config.name) + @config_layer('multi_class_cross_entropy_with_selfnorm') class MultiClassCrossEntropySelfNormCostLayer(LayerBase): - def __init__( - self, - name, - inputs, - softmax_selfnorm_alpha=0.1, - **xargs): - super(MultiClassCrossEntropySelfNormCostLayer, self).__init__(name, - 'multi_class_cross_entropy_with_selfnorm', 0, inputs, **xargs) + def __init__(self, name, inputs, softmax_selfnorm_alpha=0.1, **xargs): + super(MultiClassCrossEntropySelfNormCostLayer, self).__init__( + name, 'multi_class_cross_entropy_with_selfnorm', 0, inputs, **xargs) self.config.softmax_selfnorm_alpha = softmax_selfnorm_alpha + @config_layer('fc') class FCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - bias=True, - **xargs): + def __init__(self, name, size, inputs, bias=True, **xargs): super(FCLayer, self).__init__(name, 'fc', size, inputs=inputs, **xargs) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) @@ -1489,22 +1503,23 @@ def __init__( else: sparse = None - self.create_input_parameter(input_index, psize, dims, sparse, format) + self.create_input_parameter(input_index, psize, dims, sparse, + format) self.create_bias_parameter(bias, self.config.size) + @config_layer('selective_fc') class SelectiveFCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - bias=True, - selective_fc_pass_generation=False, - has_selected_colums=True, - selective_fc_full_mul_ratio=0.02, - selective_fc_parallel_plain_mul_thread_num=None, - **xargs): + def __init__(self, + name, + size, + inputs, + bias=True, + selective_fc_pass_generation=False, + has_selected_colums=True, + selective_fc_full_mul_ratio=0.02, + selective_fc_parallel_plain_mul_thread_num=None, + **xargs): super(SelectiveFCLayer, self).__init__( name, 'selective_fc', size, inputs=inputs, **xargs) # user MUST know if selctive fc is used in training, @@ -1525,8 +1540,8 @@ def __init__( input_num = len(self.inputs) if has_selected_colums: config_assert(input_num >= 2, - ("if indices of selected columns are not specified, " - "selective_fc Layer has at least two inputs")) + ("if indices of selected columns are not specified, " + "selective_fc Layer has at least two inputs")) input_num -= 1 for input_index in xrange(input_num): @@ -1539,26 +1554,23 @@ def __init__( if sparse: psize = self.inputs[input_index].nnz - self.create_input_parameter( - input_index, psize, dims, sparse, format) + self.create_input_parameter(input_index, psize, dims, sparse, + format) self.create_bias_parameter(bias, self.config.size) + @config_layer('print') class PrintLayer(LayerBase): - def __init__( - self, - name, - inputs): + def __init__(self, name, inputs): super(PrintLayer, self).__init__(name, 'print', 0, inputs) + @config_layer('data') class DataLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): - super(DataLayer, self).__init__(name, 'data' , size, inputs=[], device=device) + def __init__(self, name, size, device=None): + super(DataLayer, self).__init__( + name, 'data', size, inputs=[], device=device) + ''' DataNormLayer: A layer for data normalization @@ -1586,14 +1598,11 @@ def __init__( min-max: y = (x-min)/(max-min) decimal-scaling: y = x/10^j, where j is the smallest integer such that max(|y|)<1 ''' + + @config_layer('data_norm') class DataNormLayer(LayerBase): - def __init__( - self, - name, - inputs, - data_norm_strategy="z-score", - device=None): + def __init__(self, name, inputs, data_norm_strategy="z-score", device=None): super(DataNormLayer, self).__init__( name, 'data_norm', 0, inputs=inputs, device=device) self.config.data_norm_strategy = data_norm_strategy @@ -1605,15 +1614,12 @@ def __init__( self.inputs[0].is_static = True self.create_input_parameter(0, para_size, para_dims) + @config_layer('prelu') class ParameterReluLayer(LayerBase): layer_type = 'prelu' - def __init__( - self, - name, - inputs, - partial_sum = 1, - **args): + + def __init__(self, name, inputs, partial_sum=1, **args): super(ParameterReluLayer, self).__init__( name, self.layer_type, 0, inputs=inputs, **args) config_assert(len(self.inputs) == 1) @@ -1622,17 +1628,18 @@ def __init__( self.set_layer_size(input_layer.size) self.create_input_parameter(0, input_layer.size / partial_sum) + @config_layer('conv') class ConvLayerBase(LayerBase): layer_type = 'conv' - def __init__( - self, - name, - inputs=[], - bias=True, - num_filters=None, - shared_biases=False, - **xargs): + + def __init__(self, + name, + inputs=[], + bias=True, + num_filters=None, + shared_biases=False, + **xargs): super(ConvLayerBase, self).__init__( name, self.layer_type, 0, inputs=inputs, **xargs) @@ -1649,7 +1656,7 @@ def __init__( config_assert(use_gpu, "cudnn_conv only support GPU") if (use_gpu == 1 and self.layer_type != "exconv" and - (parallel_nn == 0 or self.config.device > -1)): + (parallel_nn == 0 or self.config.device > -1)): self.layer_type = "cudnn_conv" else: self.layer_type = "exconv" @@ -1661,17 +1668,14 @@ def __init__( for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_conv( - self.inputs[input_index].conv, - input_layer.name, - self.config.inputs[input_index].conv_conf, - num_filters) + parse_conv(self.inputs[input_index].conv, input_layer.name, + self.config.inputs[input_index].conv_conf, num_filters) conv_conf = self.config.inputs[input_index].conv_conf psize = self.calc_parameter_size(conv_conf) print("output size for %s is %d " % (name, conv_conf.output_x)) self.create_input_parameter(input_index, psize) self.set_layer_size( - (conv_conf.output_x ** 2) * self.config.num_filters) + (conv_conf.output_x**2) * self.config.num_filters) psize = self.config.size if shared_biases: @@ -1682,10 +1686,12 @@ def calc_parameter_size(self, conv_conf): return self.config.num_filters * conv_conf.filter_channels \ * (conv_conf.filter_size * conv_conf.filter_size_y) + @config_layer('exconv') class ConvLayer(ConvLayerBase): layer_type = 'exconv' + @config_layer('cudnn_conv') class ConvLayer(ConvLayerBase): layer_type = 'cudnn_conv' @@ -1694,14 +1700,14 @@ class ConvLayer(ConvLayerBase): @config_layer('convt') class ConvTransLayerBase(LayerBase): layer_type = 'convt' - def __init__( - self, - name, - inputs=[], - bias=True, - num_filters=None, - shared_biases=False, - **xargs): + + def __init__(self, + name, + inputs=[], + bias=True, + num_filters=None, + shared_biases=False, + **xargs): super(ConvTransLayerBase, self).__init__( name, self.layer_type, 0, inputs=inputs, **xargs) @@ -1732,7 +1738,7 @@ def __init__( print("output size for %s is %d " % (name, conv_conf.output_x)) self.create_input_parameter(input_index, psize) self.set_layer_size( - (conv_conf.img_size ** 2) * self.config.num_filters) + (conv_conf.img_size**2) * self.config.num_filters) psize = self.config.size if shared_biases: @@ -1743,85 +1749,76 @@ def calc_parameter_size(self, conv_conf): return conv_conf.channels * conv_conf.filter_channels \ * (conv_conf.filter_size * conv_conf.filter_size_y) + @config_layer('exconvt') class ConvTransLayer(ConvTransLayerBase): layer_type = 'exconvt' + @config_layer('norm') class NormLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(NormLayer, self).__init__(name, 'norm', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(NormLayer, self).__init__( + name, 'norm', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_norm( - self.inputs[input_index].norm, - input_layer.name, - self.config.inputs[input_index].norm_conf) + parse_norm(self.inputs[input_index].norm, input_layer.name, + self.config.inputs[input_index].norm_conf) norm_conf = self.config.inputs[input_index].norm_conf - self.set_layer_size((norm_conf.output_x ** 2) * norm_conf.channels) + self.set_layer_size((norm_conf.output_x**2) * norm_conf.channels) + @config_layer('pool') class PoolLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(PoolLayer, self).__init__(name, 'pool', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(PoolLayer, self).__init__( + name, 'pool', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_pool( - self.inputs[input_index].pool, - input_layer.name, - self.config.inputs[input_index].pool_conf) + parse_pool(self.inputs[input_index].pool, input_layer.name, + self.config.inputs[input_index].pool_conf) pool_conf = self.config.inputs[input_index].pool_conf - print("output size for %s is %d*%d " % ( - name, pool_conf.output_y, pool_conf.output_x)) - self.set_layer_size((pool_conf.output_x * pool_conf.output_y) * pool_conf.channels) + print("output size for %s is %d*%d " % (name, pool_conf.output_y, + pool_conf.output_x)) + self.set_layer_size( + (pool_conf.output_x * pool_conf.output_y) * pool_conf.channels) + @config_layer('spp') class SpatialPyramidPoolLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(SpatialPyramidPoolLayer, self).__init__(name, 'spp', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(SpatialPyramidPoolLayer, self).__init__( + name, 'spp', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_spp( - self.inputs[input_index].spp, - input_layer.name, - self.config.inputs[input_index].spp_conf) + parse_spp(self.inputs[input_index].spp, input_layer.name, + self.config.inputs[input_index].spp_conf) spp_conf = self.config.inputs[input_index].spp_conf output_size = (pow(4, spp_conf.pyramid_height) - 1) / (4 - 1) print("output size for %s is %d " % (name, output_size)) self.set_layer_size(output_size * spp_conf.channels) + @config_layer('batch_norm') class BatchNormLayer(LayerBase): layer_type = 'batch_norm' - def __init__( - self, - name, - inputs, - active_type="linear", - bias=True, - device=None, - use_global_stats=True, - moving_average_fraction=0.9, - batch_norm_type=None, - **xargs): + + def __init__(self, + name, + inputs, + active_type="linear", + bias=True, + device=None, + use_global_stats=True, + moving_average_fraction=0.9, + batch_norm_type=None, + **xargs): if inputs is None: inputs = [] elif not isinstance(inputs, list): inputs = [inputs] - config_assert(len(inputs) == 1, - "BatchNormLayer must have one and only one input") + config_assert( + len(inputs) == 1, "BatchNormLayer must have one and only one input") # Create Input for moving mean and std, # in batch normalization layer. # These paras no need to update, so set is_static is true. @@ -1830,12 +1827,13 @@ def __init__( use_gpu = bool(int(g_command_config_args.get("use_gpu", 0))) is_shared = True if not use_gpu else False for i in xrange(2): - inputs.append(Input(inputs[0].input_layer_name, - initial_std=0.0, - initial_mean=0.0, - is_static=True, - is_shared=is_shared, - )) + inputs.append( + Input( + inputs[0].input_layer_name, + initial_std=0.0, + initial_mean=0.0, + is_static=True, + is_shared=is_shared, )) parallel_nn = bool(int(g_command_config_args.get("parallel_nn", 0))) cudnn_version = int(g_command_config_args.get("cudnn_version", 0)) @@ -1845,21 +1843,25 @@ def __init__( ((not parallel_nn) or self.config.device > -1) and \ cudnn_version >= 4007 self.layer_type = "cudnn_batch_norm" if use_cudnn else "batch_norm" - super(BatchNormLayer, self).__init__(name, self.layer_type, 0, - active_type=active_type, - inputs=inputs, device=device, **xargs) + super(BatchNormLayer, self).__init__( + name, + self.layer_type, + 0, + active_type=active_type, + inputs=inputs, + device=device, + **xargs) if use_global_stats is not None: self.config.use_global_stats = use_global_stats if moving_average_fraction is not None: self.config.moving_average_fraction = moving_average_fraction - input_layer= self.get_input_layer(0) - parse_image(self.inputs[0].image, - input_layer.name, + input_layer = self.get_input_layer(0) + parse_image(self.inputs[0].image, input_layer.name, self.config.inputs[0].image_conf) image_conf = self.config.inputs[0].image_conf - self.set_layer_size((image_conf.img_size ** 2) * image_conf.channels) + self.set_layer_size((image_conf.img_size**2) * image_conf.channels) psize = self.calc_parameter_size(image_conf) dims = [1, psize] @@ -1872,75 +1874,74 @@ def __init__( def calc_parameter_size(self, image_conf): return image_conf.channels + @config_layer('trans') class TransLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(TransLayer, self).__init__(name, 'trans', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'TransLayer must have one and only one input') + def __init__(self, name, inputs, device=None): + super(TransLayer, self).__init__( + name, 'trans', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, + 'TransLayer must have one and only one input') self.set_layer_size(self.get_input_layer(0).size) + @config_layer('resize') class ResizeLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): - super(ResizeLayer, self).__init__(name, 'resize', size=size, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'ResizeLayer must have one and only one input') + def __init__(self, name, size, inputs, device=None): + super(ResizeLayer, self).__init__( + name, 'resize', size=size, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, + 'ResizeLayer must have one and only one input') + @config_layer('blockexpand') class BlockExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(BlockExpandLayer, self).__init__(name, 'blockexpand', 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(BlockExpandLayer, self).__init__( + name, 'blockexpand', 0, inputs=inputs, device=device) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) - parse_block_expand(self.inputs[input_index].block_expand, - input_layer.name, + parse_block_expand( + self.inputs[input_index].block_expand, input_layer.name, self.config.inputs[input_index].block_expand_conf) - block_expand_conf = self.config.inputs[input_index].block_expand_conf - self.set_layer_size(block_expand_conf.block_x * block_expand_conf.block_y - * block_expand_conf.channels) + block_expand_conf = self.config.inputs[ + input_index].block_expand_conf + self.set_layer_size(block_expand_conf.block_x * + block_expand_conf.block_y * + block_expand_conf.channels) + @config_layer('maxout') class MaxOutLayer(LayerBase): - def __init__( - self, - name, - inputs, - **xargs): - super(MaxOutLayer, self).__init__(name, 'maxout', 0, inputs=inputs, **xargs) + def __init__(self, name, inputs, **xargs): + super(MaxOutLayer, self).__init__( + name, 'maxout', 0, inputs=inputs, **xargs) input_layer = self.get_input_layer(0) - parse_maxout(self.inputs[0].maxout, - input_layer.name, + parse_maxout(self.inputs[0].maxout, input_layer.name, self.config.inputs[0].maxout_conf) maxout_conf = self.config.inputs[0].maxout_conf - self.set_layer_size(g_layer_map[input_layer.name].size / maxout_conf.groups) + self.set_layer_size(g_layer_map[input_layer.name].size / + maxout_conf.groups) + # key: cost type # value: cost class g_cost_map = {} + # define a cost layer without any parameters def define_cost(class_name, cost_type): def init(cls, name, inputs, device=None, coeff=1.): - super(type(cls), cls).__init__(name, cost_type, 1, inputs, device=device, coeff=coeff) + super(type(cls), cls).__init__( + name, cost_type, 1, inputs, device=device, coeff=coeff) - cls = type(class_name, (LayerBase,), dict(__init__=init)) + cls = type(class_name, (LayerBase, ), dict(__init__=init)) global g_cost_map g_cost_map[cost_type] = cls + define_cost('MultiClassCrossEntropy', 'multi-class-cross-entropy') define_cost('RankingCost', 'rank-cost') define_cost('AucValidation', 'auc-validation') @@ -1951,19 +1952,15 @@ def init(cls, name, inputs, device=None, coeff=1.): define_cost('HuberTwoClass', 'huber') define_cost('SumCost', 'sum_cost') + @config_layer('hsigmoid') class HierarchicalSigmoidLayer(LayerBase): - def __init__( - self, - name, - num_classes, - inputs, - device=None, - bias=True): + def __init__(self, name, num_classes, inputs, device=None, bias=True): super(HierarchicalSigmoidLayer, self).__init__( name, 'hsigmoid', 1, inputs=inputs, device=device) - config_assert(len(self.inputs) >= 2, - 'HierarchicalSigmoidLayer must have at least 2 inputs') + config_assert( + len(self.inputs) >= 2, + 'HierarchicalSigmoidLayer must have at least 2 inputs') self.config.num_classes = num_classes for input_index in xrange(len(self.inputs) - 1): input_layer = self.get_input_layer(input_index) @@ -1972,6 +1969,7 @@ def __init__( self.create_input_parameter(input_index, psize, dims) self.create_bias_parameter(bias, num_classes - 1) + ''' lambdaCost for lambdaRank LTR approach @@ -1996,59 +1994,57 @@ def __init__( max_sort_size can be greater than the size of a list, in which case the algorithm will sort the entire list to get gradient. ''' + + @config_layer('lambda_cost') class LambdaCost(LayerBase): - def __init__( - self, - name, - inputs, - NDCG_num = 5, - max_sort_size = -1, - device=None): + def __init__(self, name, inputs, NDCG_num=5, max_sort_size=-1, device=None): super(LambdaCost, self).__init__( name, 'lambda_cost', 1, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'lambdaCost must have 2 inputs') + config_assert(len(self.inputs) == 2, 'lambdaCost must have 2 inputs') self.config.NDCG_num = NDCG_num if max_sort_size != -1: - config_assert(NDCG_num <= max_sort_size, - 'NDCG_num must be less than or equal to max_sort_size') + config_assert( + NDCG_num <= max_sort_size, + 'NDCG_num must be less than or equal to max_sort_size') self.config.max_sort_size = max_sort_size + @config_layer('nce') class NCELayer(LayerBase): - def __init__( - self, - name, - num_classes, - inputs, - num_neg_samples=10, - neg_sampling_dist=None, - bias=True, - **xargs): + def __init__(self, + name, + num_classes, + inputs, + num_neg_samples=10, + neg_sampling_dist=None, + bias=True, + **xargs): super(NCELayer, self).__init__(name, 'nce', 1, inputs=inputs, **xargs) - config_assert(len(self.inputs) >= 2, - 'NCELayer must have at least 2 inputs') + config_assert( + len(self.inputs) >= 2, 'NCELayer must have at least 2 inputs') self.config.num_classes = num_classes if neg_sampling_dist is not None: - config_assert(len(neg_sampling_dist) == num_classes, - 'len(neg_sampling_dist)(%s) is not same as num_classes (%s)' - % (len(neg_sampling_dist), num_classes)) + config_assert( + len(neg_sampling_dist) == num_classes, + 'len(neg_sampling_dist)(%s) is not same as num_classes (%s)' % + (len(neg_sampling_dist), num_classes)) s = sum(neg_sampling_dist) - config_assert(abs(s - 1) < 1e-5, - 'The sum of neg_sampling_dist (%s) is not 1' % s) + config_assert( + abs(s - 1) < 1e-5, + 'The sum of neg_sampling_dist (%s) is not 1' % s) self.config.neg_sampling_dist.extend(neg_sampling_dist) self.config.num_neg_samples = num_neg_samples num_real_inputs = len(self.inputs) - 1 - input_layer = self.get_input_layer(num_real_inputs) + input_layer = self.get_input_layer(num_real_inputs) config_assert(input_layer.type == 'data', 'Expecting the last input layer of an nce layer to be ' 'a data layer') - if (num_real_inputs > 1 and input_layer.size == 1 - and self.get_input_layer(num_real_inputs - 1).type == 'data'): + if (num_real_inputs > 1 and input_layer.size == 1 and + self.get_input_layer(num_real_inputs - 1).type == 'data'): # This input layer is assumed to be a sample weight layer num_real_inputs -= 1 @@ -2062,105 +2058,82 @@ def __init__( @config_layer('addto') class AddToLayer(LayerBase): - def __init__( - self, - name, - inputs, - bias=True, - **xargs): + def __init__(self, name, inputs, bias=True, **xargs): super(AddToLayer, self).__init__( name, 'addto', 0, inputs=inputs, **xargs) - config_assert(len(inputs) > 0, - 'inputs cannot be empty for AddToLayer') + config_assert(len(inputs) > 0, 'inputs cannot be empty for AddToLayer') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('agent') class AgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): - super(AgentLayer, self).__init__(name, 'agent', size, inputs=[], device=device) + def __init__(self, name, size, device=None): + super(AgentLayer, self).__init__( + name, 'agent', size, inputs=[], device=device) + @config_layer('sequence_agent') class SequenceAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceAgentLayer, self).__init__( name, 'sequence_agent', size, inputs=[], device=device) + @config_layer('gather_agent') class GatherAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(GatherAgentLayer, self).__init__( name, 'gather_agent', size, inputs=[], device=device) + @config_layer('scatter_agent') class ScatterAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(ScatterAgentLayer, self).__init__( name, 'scatter_agent', size, inputs=[], device=device) + @config_layer('sequence_gather_agent') class SequenceGatherAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceGatherAgentLayer, self).__init__( - name, 'sequence_gather_agent', size, inputs=[], device=device) + name, 'sequence_gather_agent', size, inputs=[], device=device) + @config_layer('sequence_scatter_agent') class SequenceScatterAgentLayer(LayerBase): - def __init__( - self, - name, - size, - device=None): + def __init__(self, name, size, device=None): super(SequenceScatterAgentLayer, self).__init__( - name, 'sequence_scatter_agent', size, inputs=[], device=device) + name, 'sequence_scatter_agent', size, inputs=[], device=device) + @config_layer('multiplex') class MultiplexLayer(LayerBase): - def __init__( - self, - name, - inputs, - size, - device=None): - super(MultiplexLayer, self).__init__(name, 'multiplex', size, inputs=inputs, device=device) - config_assert(len(inputs) > 2, - 'MultiplexLayer should have more than 2 inputs.') + def __init__(self, name, inputs, size, device=None): + super(MultiplexLayer, self).__init__( + name, 'multiplex', size, inputs=inputs, device=device) + config_assert( + len(inputs) > 2, 'MultiplexLayer should have more than 2 inputs.') for i in range(1, len(inputs)): - config_assert(self.get_input_layer(i).size == size, - "All the input layers except the first one should" - "have the same size as the MultiplexLayer.") + config_assert( + self.get_input_layer(i).size == size, + "All the input layers except the first one should" + "have the same size as the MultiplexLayer.") + @config_func -def Link(name, - has_subseq=False, - ): +def Link( + name, + has_subseq=False, ): link_config = LinkConfig() link_config.link_name = name link_config.has_subseq = has_subseq return link_config + # memory for recurrent layer group. # *name* and *size* are actual layer's name and size. # will return name of the memory, @@ -2175,43 +2148,46 @@ def Link(name, # can only be initailized by a *boot_layer* which is a sequence. # @config_func -def Memory(name, - size, - is_sequence=False, - boot_layer=None, - boot_bias=False, - boot_bias_active_type="", - boot_with_const_id=None, - ): +def Memory( + name, + size, + is_sequence=False, + boot_layer=None, + boot_bias=False, + boot_bias_active_type="", + boot_with_const_id=None, ): agent_name = name + "+delay1" if is_sequence: agent_layer = SequenceAgentLayer(agent_name, size) else: agent_layer = AgentLayer(agent_name, size) config_assert(g_current_submodel.is_recurrent_layer_group, - 'Memory should be used in recurrent layer group only') + 'Memory should be used in recurrent layer group only') memory = g_current_submodel.memories.add() memory.layer_name = MakeLayerNameInSubmodel(name) memory.link_name = MakeLayerNameInSubmodel(agent_name) memory.is_sequence = is_sequence - options = sum((boot_layer is not None, - bool(boot_bias), + options = sum((boot_layer is not None, bool(boot_bias), boot_with_const_id is not None)) - config_assert(options <= 1, - 'take one option at most from boot_layer, boot_bias, or boot_with_const_id') + config_assert( + options <= 1, + 'take one option at most from boot_layer, boot_bias, or boot_with_const_id' + ) if boot_layer is not None: boot_layer = MakeLayerNameInParentSubmodel(boot_layer) config_assert(boot_layer in g_layer_map, - 'boot_layer "%s" does not correspond to a layer name' % boot_layer) + 'boot_layer "%s" does not correspond to a layer name' % + boot_layer) memory.boot_layer_name = boot_layer elif boot_bias: memory.boot_bias_parameter_name = agent_layer.create_bias_parameter( - boot_bias, size, for_self = False) + boot_bias, size, for_self=False) memory.boot_bias_active_type = boot_bias_active_type elif boot_with_const_id is not None: memory.boot_with_const_id = boot_with_const_id return agent_name + # Generator for recurrent layer group, to use it: # 1. define a id layer as output of layer group # 2. define a memory of this id layer, and assign a boot id(begin of sequence) @@ -2223,11 +2199,10 @@ def Memory(name, @config_func def Generator( max_num_frames, - eos_layer_name = "eos_check", - num_results_per_sample = 1, - beam_size = 1, - log_prob = None, - ): + eos_layer_name="eos_check", + num_results_per_sample=1, + beam_size=1, + log_prob=None, ): generator_config = GeneratorConfig() generator_config.max_num_frames = max_num_frames generator_config.eos_layer_name = eos_layer_name @@ -2237,60 +2212,55 @@ def Generator( generator_config.log_prob = log_prob return generator_config + @config_layer('expand') class ExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - trans_type='non-seq', - device=None, - bias=False): - super(ExpandLayer, self).__init__( - name, 'expand', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'ExpandLayer takes 2 and only 2 inputs') - self.config.trans_type = trans_type - for input_index in xrange(len(self.inputs)): - input_layer = self.get_input_layer(input_index) - self.set_layer_size(self.get_input_layer(0).size) - self.create_bias_parameter(bias, self.config.size) + def __init__(self, + name, + inputs, + trans_type='non-seq', + device=None, + bias=False): + super(ExpandLayer, self).__init__( + name, 'expand', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 2, 'ExpandLayer takes 2 and only 2 inputs') + self.config.trans_type = trans_type + for input_index in xrange(len(self.inputs)): + input_layer = self.get_input_layer(input_index) + self.set_layer_size(self.get_input_layer(0).size) + self.create_bias_parameter(bias, self.config.size) + @config_layer('featmap_expand') class FeatMapExpandLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None, - num_filters=None, - bias=False): - super(FeatMapExpandLayer, self).__init__( - name, 'featmap_expand', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'ExpandLayer takes 1 and only 1 inputs') - if num_filters is not None: + def __init__(self, name, inputs, device=None, num_filters=None, bias=False): + super(FeatMapExpandLayer, self).__init__( + name, 'featmap_expand', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, 'ExpandLayer takes 1 and only 1 inputs') + if num_filters is not None: self.config.num_filters = num_filters - else: + else: logger.fatal("FeatMapExpandLayer must specify num_filters.") - self.set_layer_size(self.get_input_layer(0).size * num_filters) + self.set_layer_size(self.get_input_layer(0).size * num_filters) @config_layer('max') class MaxLayer(LayerBase): - def __init__( - self, - name, - inputs, - trans_type='non-seq', - active_type='linear', - device=None, - bias=False, - output_max_index=None): - super(MaxLayer, self).__init__(name, 'max', 0, inputs=inputs, device=device) + def __init__(self, + name, + inputs, + trans_type='non-seq', + active_type='linear', + device=None, + bias=False, + output_max_index=None): + super(MaxLayer, self).__init__( + name, 'max', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'MaxLayer must have 1 input') - self.config.trans_type = trans_type - self.config.active_type = active_type + self.config.trans_type = trans_type + self.config.active_type = active_type for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) @@ -2301,12 +2271,7 @@ def __init__( @config_layer('maxid') class MaxIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - beam_size=None, - device=None): + def __init__(self, name, inputs, beam_size=None, device=None): super(MaxIdLayer, self).__init__( name, 'maxid', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'MaxIdLayer must have 1 input') @@ -2324,37 +2289,39 @@ def __init__( @config_layer('eos_id') class EosIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - eos_id, - device=None): + def __init__(self, name, inputs, eos_id, device=None): super(EosIdLayer, self).__init__( name, 'eos_id', 0, inputs=inputs, device=device) config_assert(len(self.inputs) == 1, 'EosIdLayer must have 1 input') - self.set_layer_size(2) # boolean output + self.set_layer_size(2) # boolean output self.config.eos_id = eos_id + @config_layer('seqlastins') class SequenceLastInstanceLayer(LayerBase): - def __init__( - self, - name, - inputs, - active_type='linear', - trans_type='non-seq', - device=None, - bias=False): - super(SequenceLastInstanceLayer, self).__init__(name, 'seqlastins', - 0, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 1, 'SequenceLastInstanceLayer must have 1 input') - self.config.trans_type = trans_type + def __init__(self, + name, + inputs, + active_type='linear', + trans_type='non-seq', + device=None, + bias=False): + super(SequenceLastInstanceLayer, self).__init__( + name, + 'seqlastins', + 0, + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 1, 'SequenceLastInstanceLayer must have 1 input') + self.config.trans_type = trans_type for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('seqfirstins') class SequenceFirstInstanceLayer(SequenceLastInstanceLayer): def __init__( @@ -2364,167 +2331,163 @@ def __init__( active_type='linear', trans_type='non-seq', device=None, - bias=False, - ): - super(SequenceFirstInstanceLayer, self).__init__(name, - inputs=inputs, active_type=active_type, device=device, bias=bias) - self.config.trans_type = trans_type + bias=False, ): + super(SequenceFirstInstanceLayer, self).__init__( + name, + inputs=inputs, + active_type=active_type, + device=device, + bias=bias) + self.config.trans_type = trans_type self.config.select_first = True + @config_layer('seqconcat') class SequenceConcatLayer(LayerBase): - def __init__( - self, - name, - inputs, - active_type='linear', - device=None, - bias=False): - super(SequenceConcatLayer, self).__init__(name, 'seqconcat', - 0, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 2, 'SequenceConcatLayer must have 2 inputs') + def __init__(self, + name, + inputs, + active_type='linear', + device=None, + bias=False): + super(SequenceConcatLayer, self).__init__( + name, + 'seqconcat', + 0, + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 2, 'SequenceConcatLayer must have 2 inputs') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('seqreshape') class SequenceReshapeLayer(LayerBase): - def __init__( - self, - name, + def __init__(self, + name, + size, + inputs, + active_type='linear', + device=None, + bias=False): + super(SequenceReshapeLayer, self).__init__( + name, + 'seqreshape', size, - inputs, - active_type='linear', - device=None, - bias=False): - super(SequenceReshapeLayer, self).__init__(name, 'seqreshape', - size, inputs=inputs, device=device, active_type=active_type) - config_assert(len(inputs) == 1, 'SequenceReshapeLayer must have 1 inputs') + inputs=inputs, + device=device, + active_type=active_type) + config_assert( + len(inputs) == 1, 'SequenceReshapeLayer must have 1 inputs') self.set_layer_size(size) self.create_bias_parameter(bias, size) + @config_layer('subseq') class SubSequenceLayer(LayerBase): - def __init__( - self, - name, - inputs, - active_type='linear', - device=None, - bias=False): - super(SubSequenceLayer, self).__init__(name, 'subseq', - 0, inputs=inputs, device=device, active_type=active_type) + def __init__(self, + name, + inputs, + active_type='linear', + device=None, + bias=False): + super(SubSequenceLayer, self).__init__( + name, + 'subseq', + 0, + inputs=inputs, + device=device, + active_type=active_type) config_assert(len(inputs) == 3, 'SubSequenceLayer must have 3 inputs') input_layer0 = self.get_input_layer(0) size = input_layer0.size self.set_layer_size(size) self.create_bias_parameter(bias, size) + @config_layer('out_prod') class OuterProdLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(OuterProdLayer, self).__init__(name, 'out_prod', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(OuterProdLayer, self).__init__( + name, 'out_prod', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'OuterProdLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer0.size * input_layer1.size) + @config_layer('power') class PowerLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(PowerLayer, self).__init__(name, 'power', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(PowerLayer, self).__init__( + name, 'power', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'PowerLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) input_layer0 = self.get_input_layer(0) - config_assert(1==input_layer0.size, - 'The left input is the exponent and should be of size 1') + config_assert(1 == input_layer0.size, + 'The left input is the exponent and should be of size 1') + @config_layer('slope_intercept') class SlopeInterceptLayer(LayerBase): - def __init__( - self, - name, - inputs, - slope=1.0, - intercept=0.0, - device=None): - super(SlopeInterceptLayer, self).__init__(name, 'slope_intercept', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, slope=1.0, intercept=0.0, device=None): + super(SlopeInterceptLayer, self).__init__( + name, 'slope_intercept', 0, inputs=inputs, device=device) self.config.slope = slope self.config.intercept = intercept config_assert(len(inputs) == 1, 'SlopeInterceptLayer must have 1 input') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('scaling') class ScalingLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(ScalingLayer, self).__init__(name, 'scaling', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(ScalingLayer, self).__init__( + name, 'scaling', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'ScalingLayer must have 2 inputs') input_layer1 = self.get_input_layer(1) self.set_layer_size(input_layer1.size) input_layer0 = self.get_input_layer(0) - config_assert(1==input_layer0.size, - 'The left input should be of size 1') + config_assert(1 == input_layer0.size, + 'The left input should be of size 1') + @config_layer('conv_shift') class ConvShiftLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): - super(ConvShiftLayer, self).__init__(name, 'conv_shift', - 0, inputs=inputs, device=device) + def __init__(self, name, inputs, device=None): + super(ConvShiftLayer, self).__init__( + name, 'conv_shift', 0, inputs=inputs, device=device) config_assert(len(inputs) == 2, 'ConvShiftLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('convex_comb') class ConvexCombinationLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): + def __init__(self, name, size, inputs, device=None): super(ConvexCombinationLayer, self).__init__( - name, 'convex_comb', size, inputs=inputs, device=device) - config_assert(len(self.inputs) == 2, - 'ConvexCombinationLayer must have 2 inputs') + name, 'convex_comb', size, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 2, 'ConvexCombinationLayer must have 2 inputs') config_assert( size * self.get_input_layer(0).size == self.get_input_layer(1).size, 'Wrong input size for ConvexCombinationLayer') self.set_layer_size(size) + @config_layer('interpolation') class InterpolationLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(InterpolationLayer, self).__init__( name, 'interpolation', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 3, - 'InterpolationLayer must have 3 inputs') + config_assert( + len(self.inputs) == 3, 'InterpolationLayer must have 3 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) input_layer2 = self.get_input_layer(2) @@ -2533,64 +2496,51 @@ def __init__( config_assert(input_layer1.size == input_layer2.size, 'the two vector inputs should be of the same size') + @config_layer('bilinear_interp') class BilinearInterpLayer(LayerBase): - def __init__( - self, - name, - inputs, - **xargs): + def __init__(self, name, inputs, **xargs): super(BilinearInterpLayer, self).__init__( name, 'bilinear_interp', 0, inputs=inputs, **xargs) input_layer = self.get_input_layer(0) - parse_bilinear(self.inputs[0].bilinear_interp, - input_layer.name, - self.config.inputs[0].bilinear_interp_conf); + parse_bilinear(self.inputs[0].bilinear_interp, input_layer.name, + self.config.inputs[0].bilinear_interp_conf) conf = self.inputs[0].bilinear_interp - self.set_layer_size(conf.out_size_x * conf.out_size_y * conf.num_channels) + self.set_layer_size(conf.out_size_x * conf.out_size_y * + conf.num_channels) + @config_layer('sum_to_one_norm') class SumToOneNormLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(SumToOneNormLayer, self).__init__( - name, 'sum_to_one_norm', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, - 'SumToOneNormLayer must have 1 input') + name, 'sum_to_one_norm', 0, inputs=inputs, device=device) + config_assert( + len(self.inputs) == 1, 'SumToOneNormLayer must have 1 input') input_layer0 = self.get_input_layer(0) self.set_layer_size(input_layer0.size) + @config_layer('cos_vm') class CosSimVecMatLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - cos_scale=1.0, - device=None): + def __init__(self, name, size, inputs, cos_scale=1.0, device=None): super(CosSimVecMatLayer, self).__init__( - name, 'cos_vm', size, inputs=inputs, device=device) + name, 'cos_vm', size, inputs=inputs, device=device) self.config.cos_scale = cos_scale - config_assert(len(self.inputs) == 2, - 'CosSimVecMatLayer must have 2 inputs') + config_assert( + len(self.inputs) == 2, 'CosSimVecMatLayer must have 2 inputs') config_assert( size * self.get_input_layer(0).size == self.get_input_layer(1).size, 'Wrong input size for CosSimVecMatLayer') + @config_layer('sampling_id') class SamplingIdLayer(LayerBase): - def __init__( - self, - name, - inputs, - device=None): + def __init__(self, name, inputs, device=None): super(SamplingIdLayer, self).__init__( name, 'sampling_id', 0, inputs=inputs, device=device) - config_assert(len(self.inputs) == 1, 'SamplingIdLayer must have 1 input') + config_assert( + len(self.inputs) == 1, 'SamplingIdLayer must have 1 input') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) @@ -2603,33 +2553,33 @@ def __init__( # 'squarerootn': sum each sample, but divide by sqrt(sample_num). @config_layer('average') class AverageLayer(LayerBase): - def __init__( - self, - name, - inputs, - average_strategy='average', - trans_type='non-seq', - active_type='linear', - device=None, - bias=False): - super(AverageLayer, self).__init__(name, 'average', 0, inputs=inputs, - device=device, active_type=active_type) + def __init__(self, + name, + inputs, + average_strategy='average', + trans_type='non-seq', + active_type='linear', + device=None, + bias=False): + super(AverageLayer, self).__init__( + name, + 'average', + 0, + inputs=inputs, + device=device, + active_type=active_type) self.config.average_strategy = average_strategy - self.config.trans_type = trans_type + self.config.trans_type = trans_type config_assert(len(inputs) == 1, 'AverageLayer must have 1 input') for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) self.set_layer_size(input_layer.size) self.create_bias_parameter(bias, self.config.size) + @config_layer('cos') class CosSimLayer(LayerBase): - def __init__( - self, - name, - inputs, - cos_scale=5, - device=None): + def __init__(self, name, inputs, cos_scale=5, device=None): super(CosSimLayer, self).__init__( name, 'cos', 1, inputs=inputs, device=device) config_assert(len(self.inputs) == 2, 'CosSimLayer must have 2 inputs') @@ -2641,18 +2591,13 @@ def __init__( @config_layer('tensor') class TensorLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None, - bias=True, - **xargs): - super(TensorLayer, self).__init__(name, 'tensor', size, inputs=inputs, device=device, **xargs) + def __init__(self, name, size, inputs, device=None, bias=True, **xargs): + super(TensorLayer, self).__init__( + name, 'tensor', size, inputs=inputs, device=device, **xargs) config_assert(len(self.inputs) == 2, 'TensorLayer must have 2 inputs') config_assert(size > 0, 'size must be positive') - config_assert(inputs[1].parameter_name == None, 'second parameter should be None.') + config_assert(inputs[1].parameter_name == None, + 'second parameter should be None.') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) psize = size * input_layer0.size * input_layer1.size @@ -2663,14 +2608,13 @@ def __init__( @config_layer('mixed') class MixedLayer(LayerBase): - def __init__( - self, - name, - inputs, - size=0, - bias=True, - error_clipping_threshold=None, - **xargs): + def __init__(self, + name, + inputs, + size=0, + bias=True, + error_clipping_threshold=None, + **xargs): config_assert(inputs, 'inputs cannot be empty') super(MixedLayer, self).__init__( name, 'mixed', size, inputs=inputs, **xargs) @@ -2695,24 +2639,28 @@ def __init__( else: sz = operator.calc_output_size(operator_conf.input_sizes) if sz != 0: - config_assert(sz == self.config.size, - "different inputs have different size: %s vs. %s" % - (sz, self.config.size)) + config_assert( + sz == self.config.size, + "different inputs have different size: %s vs. %s" % + (sz, self.config.size)) for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) input = self.inputs[input_index] if input_index not in operator_input_index: - config_assert(isinstance(input, Projection), "input should be projection or operation") + config_assert( + isinstance(input, Projection), + "input should be projection or operation") if self.config.size == 0 and isinstance(input, Projection): size = input.calc_output_size(input_layer) if size != 0: self.set_layer_size(size) elif isinstance(input, Projection): - sz = input.calc_output_size(input_layer) - if sz != 0: - config_assert(sz == self.config.size, - "different inputs have different size: %s vs. %s" % - (sz, self.config.size)) + sz = input.calc_output_size(input_layer) + if sz != 0: + config_assert( + sz == self.config.size, + "different inputs have different size: %s vs. %s" % + (sz, self.config.size)) config_assert(size != 0, "size is not set") for input_index in xrange(len(self.inputs)): @@ -2724,7 +2672,8 @@ def __init__( input_config = self.config.inputs[input_index] input_config.proj_conf.CopyFrom(input.proj_conf) - input_config.proj_conf.name = gen_parameter_name(name, input_index) + input_config.proj_conf.name = gen_parameter_name(name, + input_index) psize = input.calc_parameter_size(input_layer.size, size) dims = input.calc_parameter_dims(input_layer.size, size) self.create_input_parameter(input_index, psize, dims) @@ -2750,21 +2699,16 @@ def __init__( if error_clipping_threshold is not None: self.config.error_clipping_threshold = error_clipping_threshold + # like MixedLayer, but no bias parameter @config_func -def ExpressionLayer(name, - inputs, - **xargs): +def ExpressionLayer(name, inputs, **xargs): MixedLayer(name, inputs, bias=False, **xargs) + @config_layer('concat') class ConcatenateLayer(LayerBase): - def __init__( - self, - name, - inputs, - bias=False, - **xargs): + def __init__(self, name, inputs, bias=False, **xargs): config_assert(inputs, 'inputs cannot be empty') config_assert(not bias, 'ConcatenateLayer cannot support bias.') super(ConcatenateLayer, self).__init__( @@ -2773,30 +2717,27 @@ def __init__( for input_index in xrange(len(self.inputs)): input_layer = self.get_input_layer(input_index) input = self.inputs[input_index] - if self.config.size == 0: + if self.config.size == 0: size += input_layer.size self.set_layer_size(size) + # like concat layer, but each input layer was processed by a Projection. @config_layer('concat2') class ConcatenateLayer2(LayerBase): - def __init__( - self, - name, - inputs, - bias=False, - **xargs): + def __init__(self, name, inputs, bias=False, **xargs): config_assert(inputs, 'inputs cannot be empty') super(ConcatenateLayer2, self).__init__( name, 'concat2', 0, inputs=inputs, **xargs) if isinstance(self.inputs[0], ConvProjection): - for input_index in xrange(len(self.inputs) - 1): - input = self.inputs[input_index + 1] - config_assert(isinstance(input, ConvProjection), - "The first input of ConcatenateLayer2 is ConvProjection, " - "the other inputs should also be ConvProjection.") + for input_index in xrange(len(self.inputs) - 1): + input = self.inputs[input_index + 1] + config_assert( + isinstance(input, ConvProjection), + "The first input of ConcatenateLayer2 is ConvProjection, " + "the other inputs should also be ConvProjection.") size = 0 for input_index in xrange(len(self.inputs)): @@ -2818,9 +2759,9 @@ def __init__( input_config.proj_conf.CopyFrom(input.proj_conf) input_config.proj_conf.name = gen_parameter_name(name, input_index) psize = input.calc_parameter_size(input.proj_conf.input_size, - input.proj_conf.output_size) + input.proj_conf.output_size) dims = input.calc_parameter_dims(input.proj_conf.input_size, - input.proj_conf.output_size) + input.proj_conf.output_size) self.create_input_parameter(input_index, psize, dims) psize = self.config.size @@ -2834,16 +2775,12 @@ def __init__( self.config.bias_size = psize self.create_bias_parameter(bias, psize) + @config_layer('recurrent') class RecurrentLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - bias=True, - **xargs): - super(RecurrentLayer, self).__init__(name, 'recurrent', 0, inputs, **xargs) + def __init__(self, name, inputs, reversed=False, bias=True, **xargs): + super(RecurrentLayer, self).__init__(name, 'recurrent', 0, inputs, ** + xargs) config_assert(len(self.inputs) == 1, 'RecurrentLayer must have 1 input') input_layer = self.get_input_layer(0) size = input_layer.size @@ -2853,17 +2790,17 @@ def __init__( self.create_input_parameter(0, size * size, dims) self.create_bias_parameter(bias, self.config.size) + @config_layer('lstmemory') class LstmLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): + def __init__(self, + name, + inputs, + reversed=False, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): super(LstmLayer, self).__init__(name, 'lstmemory', 0, inputs, **xargs) config_assert(len(self.inputs) == 1, 'LstmLayer must have 1 input') input_layer = self.get_input_layer(0) @@ -2872,117 +2809,126 @@ def __init__( size = input_layer.size / 4 self.set_layer_size(size) self.config.reversed = reversed - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type self.create_input_parameter(0, size * size * 4, [size, size, 4]) #bias includes 3 kinds of peephole, 4 + 3 = 7 self.create_bias_parameter(bias, size * 7) + @config_layer('lstm_step') class LstmStepLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): - super(LstmStepLayer, self).__init__(name, 'lstm_step', - size, inputs, **xargs) + def __init__(self, + name, + size, + inputs, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): + super(LstmStepLayer, self).__init__(name, 'lstm_step', size, inputs, + **xargs) config_assert(len(inputs) == 2, 'LstmStepLayer must have 2 inputs') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) - config_assert(input_layer0.size == 4 * size, 'input_layer0.size != 4 * layer.size') - config_assert(input_layer1.size == size, 'input_layer1.size != layer.size') - self.config.active_gate_type = active_gate_type + config_assert(input_layer0.size == 4 * size, + 'input_layer0.size != 4 * layer.size') + config_assert(input_layer1.size == size, + 'input_layer1.size != layer.size') + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type self.create_bias_parameter(bias, size * 3) + # get the specific output from the input layer. @config_layer('get_output') class GetOutputLayer(LayerBase): - def __init__( - self, - name, - size, - inputs): - super(GetOutputLayer, self).__init__(name, 'get_output' , size, inputs) - config_assert(len(self.inputs) == 1, 'GetOutputLayer must have 1 inputs') + def __init__(self, name, size, inputs): + super(GetOutputLayer, self).__init__(name, 'get_output', size, inputs) + config_assert( + len(self.inputs) == 1, 'GetOutputLayer must have 1 inputs') inputs = self.inputs[0] config_assert(inputs.input_layer_argument, 'input_layer_argument cannot be empty') + @config_layer('mdlstmemory') class MDLstmLayer(LayerBase): - def __init__( - self, - name, - inputs, - directions=True, - active_gate_type="sigmoid", - active_state_type="sigmoid", - bias=True, - **xargs): - super(MDLstmLayer, self).__init__(name, 'mdlstmemory', 0, inputs, **xargs) + def __init__(self, + name, + inputs, + directions=True, + active_gate_type="sigmoid", + active_state_type="sigmoid", + bias=True, + **xargs): + super(MDLstmLayer, self).__init__(name, 'mdlstmemory', 0, inputs, ** + xargs) config_assert(len(self.inputs) == 1, 'MDLstmLayer must have 1 input') input_layer = self.get_input_layer(0) dim_num = len(directions) #check input_layer.size is divided by (3+dim_num) - config_assert(input_layer.size % (3+dim_num) == 0, "size % (dim_num) should be 0!") - size = input_layer.size / (3+dim_num) + config_assert(input_layer.size % + (3 + dim_num) == 0, "size % (dim_num) should be 0!") + size = input_layer.size / (3 + dim_num) self.set_layer_size(size) - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.config.active_state_type = active_state_type for i in xrange(len(directions)): self.config.directions.append(int(directions[i])) - self.create_input_parameter(0, size * size * (3+dim_num), [size, size, 3+dim_num]) + self.create_input_parameter(0, size * size * + (3 + dim_num), [size, size, 3 + dim_num]) #bias includes 3 kinds of peephole, 3+dim_num+2+dim_num - self.create_bias_parameter(bias, size * (5+2*dim_num)) + self.create_bias_parameter(bias, size * (5 + 2 * dim_num)) + @config_layer('gated_recurrent') class GatedRecurrentLayer(LayerBase): - def __init__( - self, - name, - inputs, - reversed=False, - active_gate_type="sigmoid", - bias=True, - **xargs): - super(GatedRecurrentLayer, self).__init__(name, 'gated_recurrent', 0, inputs, **xargs) - config_assert(len(self.inputs) == 1, 'GatedRecurrentLayer must have 1 input') + def __init__(self, + name, + inputs, + reversed=False, + active_gate_type="sigmoid", + bias=True, + **xargs): + super(GatedRecurrentLayer, self).__init__(name, 'gated_recurrent', 0, + inputs, **xargs) + config_assert( + len(self.inputs) == 1, 'GatedRecurrentLayer must have 1 input') input_layer = self.get_input_layer(0) #check input_layer.size is divided by 3 config_assert(input_layer.size % 3 == 0, "size % 3 should be 0!") size = input_layer.size / 3 self.set_layer_size(size) self.config.reversed = reversed - self.config.active_gate_type = active_gate_type + self.config.active_gate_type = active_gate_type self.create_input_parameter(0, size * size * 3, [size, size * 3]) self.create_bias_parameter(bias, size * 3) + @config_layer('gru_step') class GruStepLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - active_gate_type="sigmoid", - bias=True, - **xargs): - super(GruStepLayer, self).__init__(name, 'gru_step', size, inputs, **xargs) + def __init__(self, + name, + size, + inputs, + active_gate_type="sigmoid", + bias=True, + **xargs): + super(GruStepLayer, self).__init__(name, 'gru_step', size, inputs, ** + xargs) config_assert(len(self.inputs) == 2, 'GruStepLayer must have 2 input') input_layer0 = self.get_input_layer(0) input_layer1 = self.get_input_layer(1) - config_assert(input_layer0.size == 3 * size, 'input_layer0.size != 3 * layer.size') - config_assert(input_layer1.size == size, 'input_layer1.size != layer.size') - self.config.active_gate_type = active_gate_type + config_assert(input_layer0.size == 3 * size, + 'input_layer0.size != 3 * layer.size') + config_assert(input_layer1.size == size, + 'input_layer1.size != layer.size') + self.config.active_gate_type = active_gate_type self.create_input_parameter(0, size * size * 3, [size, size * 3]) self.create_bias_parameter(bias, size * 3) + ''' A layer for calculating the cost of sequential conditional random field model. Example: CRFLayer(name="crf_cost", size=label_num, @@ -2990,20 +2936,18 @@ def __init__( where "weight" is optional, one weight for each sequence @param coeff: weight of the layer ''' + + @config_layer('crf') class CRFLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - coeff=1.0, - device=None): + def __init__(self, name, size, inputs, coeff=1.0, device=None): super(CRFLayer, self).__init__(name, 'crf', size, inputs, device=device) - config_assert(2 <= len(self.inputs) <= 3, 'CRFLayer must have 2 or 3 inputs') + config_assert(2 <= len(self.inputs) <= 3, + 'CRFLayer must have 2 or 3 inputs') self.create_input_parameter(0, size * (size + 2), [size, size + 2]) self.config.coeff = coeff + ''' A layer for calculating the decoding sequence of sequential conditional random field model. @@ -3012,14 +2956,11 @@ def __init__( this layer will also calculate error, output_.value[i] is 1 for incorrect decoding or 0 for correct decoding ''' + + @config_layer('crf_decoding') class CRFDecodingLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - device=None): + def __init__(self, name, size, inputs, device=None): super(CRFDecodingLayer, self).__init__( name, 'crf_decoding', size, inputs, device=device) config_assert( @@ -3027,47 +2968,35 @@ def __init__( 'CRFDecodingLayer cannot have more than 2 inputs') self.create_input_parameter(0, size * (size + 2), [size, size + 2]) + @config_layer('ctc') class CTCLayer(LayerBase): - def __init__( - self, - name, - size, - inputs, - norm_by_times = False, - device=None): + def __init__(self, name, size, inputs, norm_by_times=False, device=None): super(CTCLayer, self).__init__(name, 'ctc', size, inputs, device=device) self.config.norm_by_times = norm_by_times config_assert(len(self.inputs) == 2, 'CTCLayer must have 2 inputs') + @config_layer('recurrent_layer_group') class RecurrentLayerGroup(LayerBase): - def __init__( - self, - name, - device=None): + def __init__(self, name, device=None): super(RecurrentLayerGroup, self).__init__( name, 'recurrent_layer_group', 0, inputs=[], device=device) # Deprecated, use a new layer specific class instead @config_func -def Layer( - name, - type, - **xargs): +def Layer(name, type, **xargs): layers = {} layers.update(g_cost_map) layers.update(g_layer_type_map) layer_func = layers.get(type) - config_assert(layer_func, - "layer type '%s' not supported." % type) + config_assert(layer_func, "layer type '%s' not supported." % type) return layer_func(name, **xargs) + @config_func -def ParameterHook( - type, - **kwargs): +def ParameterHook(type, **kwargs): if type == 'pruning': mask_filename = kwargs.get('mask_filename', None) assert mask_filename is not None @@ -3080,30 +3009,28 @@ def ParameterHook( @config_func -def Parameter( - name, - size, - device, - dims, - learning_rate=None, - momentum=None, - decay_rate=None, - decay_rate_l1=None, - initial_mean=None, - initial_std=None, - initial_strategy=None, - initial_smart=None, - num_batches_regularization=None, - sparse_remote_update=None, - sparse_update=None, - gradient_clipping_threshold=None, - sparse=None, - format=None, - need_compact=None, - is_static=None, - is_shared=None, - update_hooks=None - ): +def Parameter(name, + size, + device, + dims, + learning_rate=None, + momentum=None, + decay_rate=None, + decay_rate_l1=None, + initial_mean=None, + initial_std=None, + initial_strategy=None, + initial_smart=None, + num_batches_regularization=None, + sparse_remote_update=None, + sparse_update=None, + gradient_clipping_threshold=None, + sparse=None, + format=None, + need_compact=None, + is_static=None, + is_shared=None, + update_hooks=None): config_assert(name not in g_parameter_map, 'Duplicated parameter name: ' + name) @@ -3134,8 +3061,8 @@ def Parameter( para.initial_std = default(initial_std, g_default_initial_std) para.initial_mean = default(initial_mean, g_default_initial_mean) - num_batches_regularization = default( - num_batches_regularization, g_default_num_batches_regularization) + num_batches_regularization = default(num_batches_regularization, + g_default_num_batches_regularization) if num_batches_regularization is not None: para.num_batches_regularization = int(num_batches_regularization) @@ -3145,18 +3072,21 @@ def Parameter( g_config.opt_config.use_sparse_remote_updater = True if sparse_update is not None: para.sparse_update = sparse_update - gradient_clipping_threshold = default( - gradient_clipping_threshold, g_default_gradient_clipping_threshold) + gradient_clipping_threshold = default(gradient_clipping_threshold, + g_default_gradient_clipping_threshold) if gradient_clipping_threshold is not None: para.gradient_clipping_threshold = gradient_clipping_threshold - para.initial_strategy = default(initial_strategy, g_default_initial_strategy) + para.initial_strategy = default(initial_strategy, + g_default_initial_strategy) para.initial_smart = default(initial_smart, g_default_initial_smart) if para.initial_smart: para.initial_mean = 0. if len(para.dims) != 0: para.initial_std = 1. / math.sqrt(para.dims[0]) else: - print("Use initial_smart, but dims not set. Initial_smart may not be used in this layer") + print( + "Use initial_smart, but dims not set. Initial_smart may not be used in this layer" + ) traceback.print_exc() para.initial_std = 1. / math.sqrt(para.size) if g_default_compact_func is not None: @@ -3195,64 +3125,78 @@ def default_initial_std(val): global g_default_initial_std g_default_initial_std = val + @config_func def default_initial_mean(val): global g_default_initial_mean g_default_initial_mean = val + @config_func def default_initial_strategy(val): global g_default_initial_strategy g_default_initial_strategy = val + @config_func def default_initial_smart(val): global g_default_initial_smart g_default_initial_smart = val + @config_func def default_momentum(val): global g_default_momentum g_default_momentum = val + @config_func def default_decay_rate(val): global g_default_decay_rate g_default_decay_rate = val + @config_func def default_num_batches_regularization(val): global g_default_num_batches_regularization g_default_num_batches_regularization = val + @config_func def default_gradient_clipping_threshold(val): global g_default_gradient_clipping_threshold g_default_gradient_clipping_threshold = val + @config_func def default_device(val): global g_default_device g_default_device = val + @config_func def default_update_hooks(val): global g_default_update_hooks g_default_update_hooks = val + @config_func def default_compact_func(val): global g_default_compact_func g_default_compact_func = val + def make_importer(config_dir, config_args): def Import(config_file, local_args={}): if not config_file.startswith('/'): config_file = config_dir + '/' + config_file g_config.config_files.append(config_file) - execfile(config_file, make_config_environment(config_file, config_args), local_args) + execfile(config_file, + make_config_environment(config_file, config_args), local_args) + return Import + settings = dict( batch_size=None, mini_batch_size=None, @@ -3281,26 +3225,24 @@ def Import(config_file, local_args={}): ada_rou=0.95, delta_add_rate=1.0, shrink_parameter_value=0, - adam_beta1 = 0.9, - adam_beta2 = 0.999, - adam_epsilon = 1e-8, -) + adam_beta1=0.9, + adam_beta2=0.999, + adam_epsilon=1e-8, ) -settings_deprecated = dict( - usage_ratio=1., -) +settings_deprecated = dict(usage_ratio=1., ) trainer_settings = dict( save_dir="./output/model", init_model_path=None, - start_pass=0, -) + start_pass=0, ) + @config_func def Settings(**args): for k, v in args.iteritems(): if k == "usage_ratio": - logger.warning("Deprecated: define usage_ratio in DataConfig instead") + logger.warning( + "Deprecated: define usage_ratio in DataConfig instead") if g_config.HasField("data_config"): g_config.data_config.__setattr__(k, v) settings_deprecated[k] = v @@ -3312,10 +3254,12 @@ def Settings(**args): else: logger.fatal('Unkown setting: %s' % k) + @config_func def cluster_config(**args): pass + @config_func def EnableSubmodelSuffix(flag=True): """ @@ -3325,10 +3269,12 @@ def EnableSubmodelSuffix(flag=True): global g_add_submodel_suffix g_add_submodel_suffix = flag + def make_config_environment(config_file, config_args): def make_setter(k): def setter(v): logger.fatal("Obsolete: use Settings(%s=%s, ...) instead" % (k, v)) + return setter funcs = {} @@ -3344,13 +3290,13 @@ def setter(v): funcs.update( Import=make_importer(config_dir, config_args), - get_config_arg=make_get_config_arg(config_args), - ) + get_config_arg=make_get_config_arg(config_args), ) funcs.update(g_extended_config_funcs) return funcs + def make_get_config_arg(config_args): def get_config_arg(name, type, default=None): if type == bool: @@ -3367,6 +3313,7 @@ def get_config_arg(name, type, default=None): return get_config_arg + def importlib(name): __import__(name) return sys.modules[name] @@ -3379,10 +3326,12 @@ def find_caller(): return s[0], s[1], s[2] return "(unknown file)", 0, "(unknown function)" + def my_fatal(s): logger.critical(s) raise Exception() + def parse_config(config_file, config_arg_str): ''' @param config_arg_str: a string of the form var1=val1,var2=val2. It will be @@ -3420,7 +3369,7 @@ def parse_config(config_file, config_arg_str): for k, v in settings.iteritems(): if v is None: continue - g_config.opt_config.__setattr__(k, v); + g_config.opt_config.__setattr__(k, v) for k, v in trainer_settings.iteritems(): if v is None: @@ -3447,6 +3396,7 @@ def parse_config_and_serialize(config_file, config_arg_str): traceback.print_exc() raise + if __name__ == '__main__': try: config = parse_config(sys.argv[1], '') diff --git a/python/paddle/trainer/config_parser_extension.py b/python/paddle/trainer/config_parser_extension.py index 3445076274b0a..ba4c79efdc10e 100644 --- a/python/paddle/trainer/config_parser_extension.py +++ b/python/paddle/trainer/config_parser_extension.py @@ -17,11 +17,10 @@ g_config = None -def SimpleData( - files=None, - feat_dim=None, - context_len=None, - buffer_capacity=None): +def SimpleData(files=None, + feat_dim=None, + context_len=None, + buffer_capacity=None): data_config = DataConfig() data_config.type = 'simple' @@ -33,6 +32,7 @@ def SimpleData( data_config.buffer_capacity = buffer_capacity return data_config + def get_config_funcs(trainer_config): global g_config g_config = trainer_config diff --git a/python/paddle/trainer/recurrent_units.py b/python/paddle/trainer/recurrent_units.py index 7d51de78b0d79..a80ad13d1ed52 100644 --- a/python/paddle/trainer/recurrent_units.py +++ b/python/paddle/trainer/recurrent_units.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - + # recurrent_units.py # Version 2.0 # @@ -22,161 +22,175 @@ from paddle.trainer.config_parser import * + # long short term memory, can be used in recurrent machine # *inputs* must be a list of Projections, for example: # inputs = [FullMatrixProjection("input_layer_name")], # *para_prefix* defines parameter names, if the *para_prefix* of # two LstmRecurrentUnit is same, they share same parameters # *out_memory* can be defined outside if it's used outside -def LstmRecurrentUnit(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): +def LstmRecurrentUnit(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): - if para_prefix is None: + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) + + state_memory = Memory(name=name + "_" + "state", size=size) - state_memory = Memory(name = name + "_" + "state", size = size) - Layer( - name = name + "_" + "input_recurrent", - type = "mixed", - size = size * 4, #(input_s, input_gate, forget_gate, output_gate) - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, - parameter_name = para_prefix + "_input_recurrent.b"), - inputs = inputs + [ - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_input_recurrent.w"), - ], - ) + name=name + "_" + "input_recurrent", + type="mixed", + size=size * 4, #(input_s, input_gate, forget_gate, output_gate) + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_input_recurrent.b"), + inputs=inputs + [ + FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_input_recurrent.w"), + ], ) LstmStepLayer( - name = name, - size = size, - bias = Bias(parameter_name = para_prefix + "_check.b"), - inputs = [name + "_" + "input_recurrent", state_memory], - active_type = active_type, - active_gate_type = gate_active_type, - active_state_type = state_active_type, - ) + name=name, + size=size, + bias=Bias(parameter_name=para_prefix + "_check.b"), + inputs=[name + "_" + "input_recurrent", state_memory], + active_type=active_type, + active_gate_type=gate_active_type, + active_state_type=state_active_type, ) GetOutputLayer( - name = name + "_" + "state", - size = size, - inputs = Input(name, input_layer_argument = "state"), - ) - -def LstmRecurrentUnitNaive(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - - if para_prefix is None: + name=name + "_" + "state", + size=size, + inputs=Input( + name, input_layer_argument="state"), ) + + +def LstmRecurrentUnitNaive(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) + + state_memory = Memory(name=name + "_" + "state", size=size) - state_memory = Memory(name = name + "_" + "state", size = size) - Layer( - name = name + "_" + "input_recurrent", - type = "mixed", - size = size * 4, #(input_s, input_gate, forget_gate, output_gate) - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, - parameter_name = para_prefix + "_input_recurrent.b"), - inputs = inputs + [ - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_input_recurrent.w"), - ], - ) + name=name + "_" + "input_recurrent", + type="mixed", + size=size * 4, #(input_s, input_gate, forget_gate, output_gate) + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_input_recurrent.b"), + inputs=inputs + [ + FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_input_recurrent.w"), + ], ) ExpressionLayer( - name = name + "_" + "input_s", - size = size, - active_type = active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=0)], - ) + name=name + "_" + "input_s", + size=size, + active_type=active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=0) + ], ) ExpressionLayer( - name = name + "_" + "input_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size), - DotMulProjection(state_memory, - parameter_name = para_prefix + "_input_check.w")], - ) + name=name + "_" + "input_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size), DotMulProjection( + state_memory, parameter_name=para_prefix + "_input_check.w") + ], ) ExpressionLayer( - name = name + "_" + "forget_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size*2), - DotMulProjection(state_memory, - parameter_name = para_prefix + "_forget_check.w")], - ) + name=name + "_" + "forget_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size * 2), + DotMulProjection( + state_memory, parameter_name=para_prefix + "_forget_check.w") + ], ) ExpressionLayer( - name = name + "_" + "state", - inputs = [DotMulOperator([name + "_" + "input_s", - name + "_" + "input_gate"]), - DotMulOperator([state_memory, - name + "_" + "forget_gate"]), - ], - ) + name=name + "_" + "state", + inputs=[ + DotMulOperator([name + "_" + "input_s", name + "_" + "input_gate"]), + DotMulOperator([state_memory, name + "_" + "forget_gate"]), + ], ) ExpressionLayer( - name = name + "_" + "output_gate", - active_type = gate_active_type, - inputs = [IdentityOffsetProjection(name + "_" + "input_recurrent", offset=size*3), - DotMulProjection(name + "_" + "state", - parameter_name = para_prefix + "_output_check.w")], - ) + name=name + "_" + "output_gate", + active_type=gate_active_type, + inputs=[ + IdentityOffsetProjection( + name + "_" + "input_recurrent", offset=size * 3), + DotMulProjection( + name + "_" + "state", + parameter_name=para_prefix + "_output_check.w") + ], ) ExpressionLayer( - name = name + "_" + "state_atv", - active_type = state_active_type, - inputs = IdentityProjection(name + "_" + "state"), - ) + name=name + "_" + "state_atv", + active_type=state_active_type, + inputs=IdentityProjection(name + "_" + "state"), ) ExpressionLayer( - name = name, - inputs = DotMulOperator([name + "_" + "state_atv", - name + "_" + "output_gate"]), - ) + name=name, + inputs=DotMulOperator( + [name + "_" + "state_atv", name + "_" + "output_gate"]), ) + # like LstmRecurrentUnit, but it's a layer group. # it is equivalent to LstmLayer -def LstmRecurrentLayerGroup(name, size, - active_type, state_active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - seq_reversed = False): +def LstmRecurrentLayerGroup(name, + size, + active_type, + state_active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + seq_reversed=False): input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 4, - active_type = "", - bias = False, - inputs = inputs, - ) - - RecurrentLayerGroupBegin(name + "_layer_group", - in_links = [input_layer_name], - out_links = [name], - seq_reversed = seq_reversed) + name=input_layer_name, + type="mixed", + size=size * 4, + active_type="", + bias=False, + inputs=inputs, ) + + RecurrentLayerGroupBegin( + name + "_layer_group", + in_links=[input_layer_name], + out_links=[name], + seq_reversed=seq_reversed) LstmRecurrentUnit( - name = name, - size = size, - active_type = active_type, - state_active_type = state_active_type, - gate_active_type = gate_active_type, - inputs = [IdentityProjection(input_layer_name)], - para_prefix = para_prefix, - error_clipping_threshold = error_clipping_threshold, - ) + name=name, + size=size, + active_type=active_type, + state_active_type=state_active_type, + gate_active_type=gate_active_type, + inputs=[IdentityProjection(input_layer_name)], + para_prefix=para_prefix, + error_clipping_threshold=error_clipping_threshold, ) RecurrentLayerGroupEnd(name + "_layer_group") - # gated recurrent unit, can be used in recurrent machine # *inputs* should be a list of Projections, for example: # inputs = [FullMatrixProjection("input_layer_name")], @@ -184,142 +198,157 @@ def LstmRecurrentLayerGroup(name, size, # two GatedRecurrentUnit is same, they share same parameters # *out_memory* can be defined outside if it's used outside -def GatedRecurrentUnit(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup + +def GatedRecurrentUnit(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup input_layer_name = inputs else: input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - if para_prefix is None: + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) GruStepLayer( - name = name, - size = size, - bias = Bias(parameter_name = para_prefix + "_gate.b"), - inputs = [input_layer_name, - Input(out_memory, parameter_name = para_prefix + "_gate.w")], - active_type = active_type, - active_gate_type = gate_active_type, - ) - -def GatedRecurrentUnitNaive(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - out_memory = None): - - if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup + name=name, + size=size, + bias=Bias(parameter_name=para_prefix + "_gate.b"), + inputs=[ + input_layer_name, Input( + out_memory, parameter_name=para_prefix + "_gate.w") + ], + active_type=active_type, + active_gate_type=gate_active_type, ) + + +def GatedRecurrentUnitNaive(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + out_memory=None): + + if type_of(inputs) == str: #only used by GatedRecurrentLayerGroup input_layer_name = inputs else: input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - if para_prefix is None: + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + if para_prefix is None: para_prefix = name if out_memory is None: - out_memory = Memory(name = name, size = size) + out_memory = Memory(name=name, size=size) Layer( - name = name + "_" + "update_gate", - type = "mixed", - size = size, - active_type = gate_active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_update_gate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=0), - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_update_gate.w")], - ) + name=name + "_" + "update_gate", + type="mixed", + size=size, + active_type=gate_active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_update_gate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=0), FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_update_gate.w") + ], ) Layer( - name = name + "_" + "reset_gate", - type = "mixed", - size = size, - active_type = gate_active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_reset_gate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=size), - FullMatrixProjection(out_memory, - parameter_name = para_prefix + "_reset_gate.w")], - ) + name=name + "_" + "reset_gate", + type="mixed", + size=size, + active_type=gate_active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_reset_gate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=size), FullMatrixProjection( + out_memory, parameter_name=para_prefix + "_reset_gate.w") + ], ) ExpressionLayer( - name = name + "_" + "reset_output", - inputs = DotMulOperator([out_memory, name + "_" + "reset_gate"]), - ) + name=name + "_" + "reset_output", + inputs=DotMulOperator([out_memory, name + "_" + "reset_gate"]), ) Layer( - name = name + "_" + "output_candidate", - type = "mixed", - size = size, - active_type = active_type, - error_clipping_threshold = error_clipping_threshold, - bias = Bias(initial_std = 0, parameter_name = para_prefix + "_output_candidate.b"), - inputs = [IdentityOffsetProjection(input_layer_name, offset=size*2), - FullMatrixProjection(name + "_" + "reset_output", - parameter_name = para_prefix + "_output_candidate.w")], - ) - ExpressionLayer( #element-wise interpolation - name = name, - inputs = [IdentityProjection(out_memory), - DotMulOperator([out_memory, - name + "_" + "update_gate"], scale=-1.0), - DotMulOperator([name + "_" + "output_candidate", - name + "_" + "update_gate"]), - ], - ) + name=name + "_" + "output_candidate", + type="mixed", + size=size, + active_type=active_type, + error_clipping_threshold=error_clipping_threshold, + bias=Bias( + initial_std=0, parameter_name=para_prefix + "_output_candidate.b"), + inputs=[ + IdentityOffsetProjection( + input_layer_name, offset=size * 2), FullMatrixProjection( + name + "_" + "reset_output", + parameter_name=para_prefix + "_output_candidate.w") + ], ) + ExpressionLayer( #element-wise interpolation + name=name, + inputs=[ + IdentityProjection(out_memory), + DotMulOperator( + [out_memory, name + "_" + "update_gate"], scale=-1.0), + DotMulOperator( + [name + "_" + "output_candidate", name + "_" + "update_gate"]), + ], ) + # like GatedRecurrentUnit, but it's a layer group. # it is equivalent to GatedRecurrentLayer. -def GatedRecurrentLayerGroup(name, size, - active_type, gate_active_type, - inputs, para_prefix = None, - error_clipping_threshold = 0, - seq_reversed = False): +def GatedRecurrentLayerGroup(name, + size, + active_type, + gate_active_type, + inputs, + para_prefix=None, + error_clipping_threshold=0, + seq_reversed=False): input_layer_name = name + "_" + "transform_input" Layer( - name = input_layer_name, - type = "mixed", - size = size * 3, - active_type = "", - bias = False, - inputs = inputs, - ) - - RecurrentLayerGroupBegin(name + "_layer_group", - in_links = [input_layer_name], - out_links = [name], - seq_reversed = seq_reversed) + name=input_layer_name, + type="mixed", + size=size * 3, + active_type="", + bias=False, + inputs=inputs, ) + + RecurrentLayerGroupBegin( + name + "_layer_group", + in_links=[input_layer_name], + out_links=[name], + seq_reversed=seq_reversed) GatedRecurrentUnit( - name = name, - size = size, - active_type = active_type, - gate_active_type = gate_active_type, - inputs = input_layer_name, #transform outside - para_prefix = para_prefix, - error_clipping_threshold = error_clipping_threshold, - ) + name=name, + size=size, + active_type=active_type, + gate_active_type=gate_active_type, + inputs=input_layer_name, #transform outside + para_prefix=para_prefix, + error_clipping_threshold=error_clipping_threshold, ) RecurrentLayerGroupEnd(name + "_layer_group") - diff --git a/python/paddle/trainer_config_helpers/activations.py b/python/paddle/trainer_config_helpers/activations.py index 2202d0bf96976..6261934e1bc8e 100644 --- a/python/paddle/trainer_config_helpers/activations.py +++ b/python/paddle/trainer_config_helpers/activations.py @@ -12,13 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -__all__ = ["TanhActivation", "SigmoidActivation", - "SoftmaxActivation", "IdentityActivation", "LinearActivation", - 'SequenceSoftmaxActivation', 'ExpActivation', - "ReluActivation", "BReluActivation", "SoftReluActivation", - "STanhActivation", - "AbsActivation", "SquareActivation", - "BaseActivation"] +__all__ = [ + "TanhActivation", "SigmoidActivation", "SoftmaxActivation", + "IdentityActivation", "LinearActivation", 'SequenceSoftmaxActivation', + 'ExpActivation', "ReluActivation", "BReluActivation", "SoftReluActivation", + "STanhActivation", "AbsActivation", "SquareActivation", "BaseActivation" +] class BaseActivation(object): @@ -51,7 +50,8 @@ class TanhActivation(BaseActivation): f(z)=tanh(z)=\\frac{e^z-e^{-z}}{e^z+e^{-z}} """ - def __init__(self): BaseActivation.__init__(self, 'tanh', True) + def __init__(self): + BaseActivation.__init__(self, 'tanh', True) class SigmoidActivation(BaseActivation): @@ -63,7 +63,8 @@ class SigmoidActivation(BaseActivation): f(z) = \\frac{1}{1+exp(-z)} """ - def __init__(self): BaseActivation.__init__(self, 'sigmoid', True) + def __init__(self): + BaseActivation.__init__(self, 'sigmoid', True) class SoftmaxActivation(BaseActivation): @@ -104,7 +105,8 @@ class IdentityActivation(BaseActivation): Just do nothing for output both forward/backward. """ - def __init__(self): BaseActivation.__init__(self, '', False) + def __init__(self): + BaseActivation.__init__(self, '', False) LinearActivation = IdentityActivation @@ -124,7 +126,8 @@ class ReluActivation(BaseActivation): 0 &\\quad\\mathrm{otherwize} """ - def __init__(self): BaseActivation.__init__(self, 'relu', True) + def __init__(self): + BaseActivation.__init__(self, 'relu', True) class BReluActivation(BaseActivation): @@ -141,7 +144,8 @@ class BReluActivation(BaseActivation): 0 &\\quad \\mathrm{otherwise} """ - def __init__(self): BaseActivation.__init__(self, 'brelu', False) + def __init__(self): + BaseActivation.__init__(self, 'brelu', False) class SoftReluActivation(BaseActivation): @@ -149,7 +153,9 @@ class SoftReluActivation(BaseActivation): SoftRelu Activation. """ - def __init__(self): BaseActivation.__init__(self, 'softrelu', False) + def __init__(self): + BaseActivation.__init__(self, 'softrelu', False) + class STanhActivation(BaseActivation): """ @@ -160,7 +166,8 @@ class STanhActivation(BaseActivation): f(z) = 1.7159 * tanh(2/3*z) """ - def __init__(self): BaseActivation.__init__(self, 'stanh', False) + def __init__(self): + BaseActivation.__init__(self, 'stanh', False) class AbsActivation(BaseActivation): @@ -178,7 +185,8 @@ class AbsActivation(BaseActivation): 0 &\\quad if \\quad z = 0 """ - def __init__(self): BaseActivation.__init__(self, 'abs', False) + def __init__(self): + BaseActivation.__init__(self, 'abs', False) class SquareActivation(BaseActivation): @@ -189,7 +197,9 @@ class SquareActivation(BaseActivation): f(z) = z^2. """ - def __init__(self): BaseActivation.__init__(self, 'square', False) + def __init__(self): + BaseActivation.__init__(self, 'square', False) + class ExpActivation(BaseActivation): """ @@ -198,7 +208,10 @@ class ExpActivation(BaseActivation): .. math:: f(z) = e^z. """ - def __init__(self): BaseActivation.__init__(self, 'exponential', False) + + def __init__(self): + BaseActivation.__init__(self, 'exponential', False) + class LogActivation(BaseActivation): """ @@ -207,4 +220,6 @@ class LogActivation(BaseActivation): .. math:: f(z) = log(z) """ - def __init__(self): BaseActivation.__init__(self, 'log', False) + + def __init__(self): + BaseActivation.__init__(self, 'log', False) diff --git a/python/paddle/trainer_config_helpers/attrs.py b/python/paddle/trainer_config_helpers/attrs.py index d263441247332..54169f382f164 100644 --- a/python/paddle/trainer_config_helpers/attrs.py +++ b/python/paddle/trainer_config_helpers/attrs.py @@ -13,8 +13,9 @@ # limitations under the License. from paddle.trainer.config_parser import * -__all__ = ['ParamAttr', 'ExtraAttr', 'ParameterAttribute', - 'ExtraLayerAttribute'] +__all__ = [ + 'ParamAttr', 'ExtraAttr', 'ParameterAttribute', 'ExtraLayerAttribute' +] def convert_and_compare(x, Type): @@ -25,7 +26,8 @@ def convert_and_compare(x, Type): :param Type: target type to check x over """ - return type(x)(Type(x))==x + return type(x)(Type(x)) == x + def is_compatible_with(x, Type): """ @@ -38,9 +40,9 @@ def is_compatible_with(x, Type): return True try: if float == Type or int == Type: - # avoid those types that can be converted to float/int but not very - # meaningful and could potentially lead to error - # i.e., str and bool typed value should not be used for initializing float/int variable + # avoid those types that can be converted to float/int but not very + # meaningful and could potentially lead to error + # i.e., str and bool typed value should not be used for initializing float/int variable if not isinstance(x, str) and not isinstance(x, bool): return convert_and_compare(x, Type) elif bool == Type: @@ -91,9 +93,17 @@ class ParameterAttribute(object): :type sparse_update: bool """ - def __init__(self, name=None, is_static=False, initial_std=None, - initial_mean=None, initial_max=None, initial_min=None, - l1_rate=None, l2_rate=None, learning_rate=None, momentum=None, + def __init__(self, + name=None, + is_static=False, + initial_std=None, + initial_mean=None, + initial_max=None, + initial_min=None, + l1_rate=None, + l2_rate=None, + learning_rate=None, + momentum=None, sparse_update=False): # initialize strategy. if is_static: @@ -183,7 +193,10 @@ class ExtraLayerAttribute(object): :type device: int """ - def __init__(self, error_clipping_threshold=None, drop_rate=None, device=None): + def __init__(self, + error_clipping_threshold=None, + drop_rate=None, + device=None): self.attr = dict() if isinstance(error_clipping_threshold, float): assert error_clipping_threshold > 0 @@ -200,8 +213,8 @@ def check(self, layer_name): for key in self.attr: if not hasattr(self, 'can_%s' % key) or \ not getattr(self, 'can_%s' % key): - raise NotImplementedError( - "Layer %s cannot support %s" % (layer_name, key)) + raise NotImplementedError("Layer %s cannot support %s" % + (layer_name, key)) @staticmethod def to_kwargs(attr): diff --git a/python/paddle/trainer_config_helpers/data_sources.py b/python/paddle/trainer_config_helpers/data_sources.py index 283a45df30844..b41097953dad8 100644 --- a/python/paddle/trainer_config_helpers/data_sources.py +++ b/python/paddle/trainer_config_helpers/data_sources.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Data Sources are helpers to define paddle training data or testing data. """ @@ -26,8 +25,12 @@ __all__ = ['define_py_data_sources2'] -def define_py_data_source(file_list, cls, module, - obj, args=None, async=False, +def define_py_data_source(file_list, + cls, + module, + obj, + args=None, + async=False, data_cls=PyData): """ Define a python data source. @@ -76,8 +79,9 @@ def define_py_data_source(file_list, cls, module, args = pickle.dumps(args, 0) if data_cls is None: + def py_data2(files, load_data_module, load_data_object, load_data_args, - **kwargs): + **kwargs): data = DataBase() data.type = 'py2' data.files = files @@ -86,17 +90,25 @@ def py_data2(files, load_data_module, load_data_object, load_data_args, data.load_data_args = load_data_args data.async_load_data = True return data - data_cls = py_data2 - - cls(data_cls(files=file_list, - load_data_module=module, - load_data_object=obj, - load_data_args=args, - async_load_data=async)) + data_cls = py_data2 -def define_py_data_sources(train_list, test_list, module, obj, args=None, - train_async=False, data_cls=PyData): + cls( + data_cls( + files=file_list, + load_data_module=module, + load_data_object=obj, + load_data_args=args, + async_load_data=async)) + + +def define_py_data_sources(train_list, + test_list, + module, + obj, + args=None, + train_async=False, + data_cls=PyData): """ The annotation is almost the same as define_py_data_sources2, except that it can specific train_async and data_cls. @@ -125,8 +137,8 @@ def define_py_data_sources(train_list, test_list, module, obj, args=None, """ def __is_splitable__(o): - return (isinstance(o, list) or isinstance(o, tuple) - ) and hasattr(o, '__len__') and len(o) == 2 + return (isinstance(o, list) or + isinstance(o, tuple)) and hasattr(o, '__len__') and len(o) == 2 assert train_list is not None or test_list is not None assert module is not None and obj is not None @@ -196,9 +208,10 @@ def define_py_data_sources2(train_list, test_list, module, obj, args=None): :return: None :rtype: None """ - define_py_data_sources(train_list=train_list, - test_list=test_list, - module=module, - obj=obj, - args=args, - data_cls=None) + define_py_data_sources( + train_list=train_list, + test_list=test_list, + module=module, + obj=obj, + args=args, + data_cls=None) diff --git a/python/paddle/trainer_config_helpers/default_decorators.py b/python/paddle/trainer_config_helpers/default_decorators.py index be00f48b457c1..c01050e338d59 100644 --- a/python/paddle/trainer_config_helpers/default_decorators.py +++ b/python/paddle/trainer_config_helpers/default_decorators.py @@ -18,16 +18,18 @@ from .activations import TanhActivation from paddle.trainer.config_parser import * -__all__ = ['wrap_name_default', 'wrap_param_attr_default', - 'wrap_bias_attr_default', 'wrap_act_default', - 'wrap_param_default'] +__all__ = [ + 'wrap_name_default', 'wrap_param_attr_default', 'wrap_bias_attr_default', + 'wrap_act_default', 'wrap_param_default' +] def __default_not_set_callback__(kwargs, name): return name not in kwargs or kwargs[name] is None -def wrap_param_default(param_names=None, default_factory=None, +def wrap_param_default(param_names=None, + default_factory=None, not_set_callback=__default_not_set_callback__): assert param_names is not None assert isinstance(param_names, list) or isinstance(param_names, tuple) @@ -43,7 +45,8 @@ def __wrapper__(*args, **kwargs): if argspec.defaults: num_positional -= len(argspec.defaults) if not argspec.varargs and len(args) > num_positional: - logger.fatal("Must use keyword arguments for non-positional args") + logger.fatal( + "Must use keyword arguments for non-positional args") for name in param_names: if not_set_callback(kwargs, name): # Not set kwargs[name] = default_factory(func) @@ -112,13 +115,13 @@ def wrap_param_attr_default(param_names=None, default_factory=None): return wrap_param_default(param_names, default_factory) -def wrap_bias_attr_default(param_names=None, default_factory=None, +def wrap_bias_attr_default(param_names=None, + default_factory=None, has_bias=True): if param_names is None: param_names = ['bias_attr'] if default_factory is None: - default_factory = lambda _: ParamAttr(initial_std=0., - initial_mean=0.) + default_factory = lambda _: ParamAttr(initial_std=0., initial_mean=0.) def __bias_attr_not_set__(kwargs, name): if has_bias: diff --git a/python/paddle/trainer_config_helpers/evaluators.py b/python/paddle/trainer_config_helpers/evaluators.py index ded124a5c8ca4..dc6a36392f9c6 100644 --- a/python/paddle/trainer_config_helpers/evaluators.py +++ b/python/paddle/trainer_config_helpers/evaluators.py @@ -15,13 +15,14 @@ from paddle.trainer.config_parser import * from default_decorators import * -__all__ = ["evaluator_base","classification_error_evaluator", "auc_evaluator", - "pnpair_evaluator", "precision_recall_evaluator", - "ctc_error_evaluator", "chunk_evaluator", "sum_evaluator", - "column_sum_evaluator", "value_printer_evaluator", - "gradient_printer_evaluator", "maxid_printer_evaluator", - "maxframe_printer_evaluator", "seqtext_printer_evaluator", - "classification_error_printer_evaluator"] +__all__ = [ + "evaluator_base", "classification_error_evaluator", "auc_evaluator", + "pnpair_evaluator", "precision_recall_evaluator", "ctc_error_evaluator", + "chunk_evaluator", "sum_evaluator", "column_sum_evaluator", + "value_printer_evaluator", "gradient_printer_evaluator", + "maxid_printer_evaluator", "maxframe_printer_evaluator", + "seqtext_printer_evaluator", "classification_error_printer_evaluator" +] class EvaluatorAttribute(object): @@ -32,10 +33,7 @@ class EvaluatorAttribute(object): FOR_UTILS = 1 << 4 KEYS = [ - "for_classification", - "for_regression", - "for_rank", - "for_print", + "for_classification", "for_regression", "for_rank", "for_print", "for_utils" ] @@ -55,22 +53,23 @@ def impl(method): setattr(method, EvaluatorAttribute.to_key(attr), True) method.is_evaluator = True return method + return impl -def evaluator_base( - input, - type, - label=None, - weight=None, - name=None, - chunk_scheme=None, - num_chunk_types=None, - classification_threshold=None, - positive_label=None, - dict_file=None, - result_file=None, - num_results=None, - delimited=None): + +def evaluator_base(input, + type, + label=None, + weight=None, + name=None, + chunk_scheme=None, + num_chunk_types=None, + classification_threshold=None, + positive_label=None, + dict_file=None, + result_file=None, + num_results=None, + delimited=None): """ Evaluator will evaluate the network status while training/testing. @@ -130,14 +129,14 @@ def evaluator_base( result_file=result_file, delimited=delimited) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() -def classification_error_evaluator( - input, - label, - name=None, - weight=None, - threshold=None): +def classification_error_evaluator(input, + label, + name=None, + weight=None, + threshold=None): """ Classification Error Evaluator. It will print error rate for classification. @@ -170,13 +169,14 @@ def classification_error_evaluator( :return: None. """ - evaluator_base(name=name, - type="classification_error", - input=input, - label=label, - weight=weight, - classification_threshold=threshold, - ) + evaluator_base( + name=name, + type="classification_error", + input=input, + label=label, + weight=weight, + classification_threshold=threshold, ) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -184,8 +184,7 @@ def auc_evaluator( input, label, name=None, - weight=None, - ): + weight=None, ): """ Auc Evaluator which adapts to binary classification. @@ -205,11 +204,13 @@ def auc_evaluator( [sample_num, 1]. :type weight: LayerOutput """ - evaluator_base(name=name, - type="last-column-auc", - input=input, - label=label, - weight=weight) + evaluator_base( + name=name, + type="last-column-auc", + input=input, + label=label, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_RANK) @wrap_name_default() @@ -218,8 +219,7 @@ def pnpair_evaluator( label, info, name=None, - weight=None, - ): + weight=None, ): """ Positive-negative pair rate Evaluator which adapts to rank task like learning to rank. This evaluator must contain at least three layers. @@ -242,12 +242,14 @@ def pnpair_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="pnpair", - input=input, - label=label, - info=info, - weight=weight) + evaluator_base( + name=name, + type="pnpair", + input=input, + label=label, + info=info, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -256,8 +258,7 @@ def precision_recall_evaluator( label, positive_label=None, weight=None, - name=None, - ): + name=None, ): """ An Evaluator to calculate precision and recall, F1-score. It is adapt to the task with multiple labels. @@ -286,20 +287,21 @@ def precision_recall_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="precision_recall", - input=input, - label=label, - positive_label=positive_label, - weight=weight) + evaluator_base( + name=name, + type="precision_recall", + input=input, + label=label, + positive_label=positive_label, + weight=weight) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() def ctc_error_evaluator( input, label, - name=None, - ): + name=None, ): """ This evaluator is to calculate sequence-to-sequence edit distance. @@ -317,10 +319,9 @@ def ctc_error_evaluator( label for ctc_layer :type label: LayerOutput """ - evaluator_base(name=name, - type="ctc_edit_distance", - input=input, - label=label) + evaluator_base( + name=name, type="ctc_edit_distance", input=input, label=label) + @evaluator(EvaluatorAttribute.FOR_CLASSIFICATION) @wrap_name_default() @@ -328,8 +329,7 @@ def chunk_evaluator( input, name=None, chunk_scheme=None, - num_chunk_types=None, - ): + num_chunk_types=None, ): """ Chunk evaluator is used to evaluate segment labelling accuracy for a sequence. It calculates the chunk detection F1 score. @@ -375,19 +375,20 @@ def chunk_evaluator( :type chunk_scheme: basestring :param num_chunk_types: number of chunk types other than "other" """ - evaluator_base(name=name, - type="chunk", - input=input, - chunk_scheme=chunk_scheme, - num_chunk_types=num_chunk_types) + evaluator_base( + name=name, + type="chunk", + input=input, + chunk_scheme=chunk_scheme, + num_chunk_types=num_chunk_types) + @evaluator(EvaluatorAttribute.FOR_UTILS) @wrap_name_default() def sum_evaluator( input, name=None, - weight=None, - ): + weight=None, ): """ An Evaluator to sum the result of input. @@ -405,18 +406,15 @@ def sum_evaluator( [sample_num, 1]. (TODO, explaination) :type weight: LayerOutput """ - evaluator_base(name=name, - type="sum", - input=input, - weight=weight) + evaluator_base(name=name, type="sum", input=input, weight=weight) + @evaluator(EvaluatorAttribute.FOR_UTILS) @wrap_name_default() def column_sum_evaluator( input, name=None, - weight=None, - ): + weight=None, ): """ This Evaluator is used to sum the last column of input. @@ -431,22 +429,22 @@ def column_sum_evaluator( :param input: Input Layer name. :type input: LayerOutput """ - evaluator_base(name=name, - type="last-column-sum", - input=input, - weight=weight) + evaluator_base( + name=name, type="last-column-sum", input=input, weight=weight) + """ The following are printer Evaluators which are usually used to print the result, like value or gradient of input layers, the results generated in machine translation, the classification error etc. """ + + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def value_printer_evaluator( input, - name=None, - ): + name=None, ): """ This Evaluator is used to print the values of input layers. It contains one or more input layers. @@ -462,16 +460,14 @@ def value_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="value_printer", - input=input) + evaluator_base(name=name, type="value_printer", input=input) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def gradient_printer_evaluator( input, - name=None, - ): + name=None, ): """ This Evaluator is used to print the gradient of input layers. It contains one or more input layers. @@ -487,17 +483,15 @@ def gradient_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="gradient_printer", - input=input) + evaluator_base(name=name, type="gradient_printer", input=input) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def maxid_printer_evaluator( input, num_results=None, - name=None, - ): + name=None, ): """ This Evaluator is used to print maximum top k values and their indexes of each row of input layers. It contains one or more input layers. @@ -517,18 +511,16 @@ def maxid_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="max_id_printer", - input=input, - num_results=num_results) + evaluator_base( + name=name, type="max_id_printer", input=input, num_results=num_results) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() def maxframe_printer_evaluator( input, num_results=None, - name=None, - ): + name=None, ): """ This Evaluator is used to print the top k frames of each input layers. The input layers should contain sequences info or sequences type. @@ -549,10 +541,12 @@ def maxframe_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="max_frame_printer", - input=input, - num_results=num_results) + evaluator_base( + name=name, + type="max_frame_printer", + input=input, + num_results=num_results) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() @@ -562,8 +556,7 @@ def seqtext_printer_evaluator( id_input=None, dict_file=None, delimited=None, - name=None, - ): + name=None, ): """ Sequence text printer will print text according to index matrix and a dictionary. There can be multiple input to this layer: @@ -636,12 +629,14 @@ def seqtext_printer_evaluator( inputs = [id_input, input] input.parents.append(id_input) - evaluator_base(name=name, - type="seq_text_printer", - input=inputs, - dict_file=dict_file, - result_file=result_file, - delimited=delimited) + evaluator_base( + name=name, + type="seq_text_printer", + input=inputs, + dict_file=dict_file, + result_file=result_file, + delimited=delimited) + @evaluator(EvaluatorAttribute.FOR_PRINT) @wrap_name_default() @@ -649,8 +644,7 @@ def classification_error_printer_evaluator( input, label, threshold=0.5, - name=None, - ): + name=None, ): """ This Evaluator is used to print the classification error of each sample. @@ -667,8 +661,9 @@ def classification_error_printer_evaluator( :param name: Evaluator name. :type name: None|basestring """ - evaluator_base(name=name, - type="classification_error_printer", - input=input, - label=label, - classification_threshold=threshold) + evaluator_base( + name=name, + type="classification_error_printer", + input=input, + label=label, + classification_threshold=threshold) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index a0a367f2d50df..796121a64136e 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -29,36 +29,83 @@ import pickle import copy -__all__ = ["full_matrix_projection", "AggregateLevel", "ExpandLevel", - "identity_projection", "dotmul_projection", "dotmul_operator", - "repeat_layer", - "table_projection", "mixed_layer", "data_layer", - "embedding_layer", "fc_layer", "grumemory", - "pooling_layer", "lstmemory", "last_seq", "first_seq", - "cos_sim", "hsigmoid", "conv_projection", - "regression_cost", 'classification_cost', "LayerOutput", - 'img_conv_layer', 'img_pool_layer', 'batch_norm_layer', - 'img_cmrnorm_layer', 'addto_layer', - 'concat_layer', 'lstm_step_layer', 'recurrent_group', - 'memory', 'StaticInput', 'expand_layer', 'scaling_layer', - 'power_layer', 'interpolation_layer', 'bilinear_interp_layer', - 'trans_layer', 'sum_to_one_norm_layer', - 'get_output_layer', 'LayerType', 'context_projection', - 'beam_search', 'maxid_layer', 'GeneratedInput', 'SubsequenceInput', - 'gru_step_layer', 'recurrent_layer', - 'BaseGeneratedInput', 'conv_operator', 'conv_shift_layer', - 'tensor_layer', 'selective_fc_layer', 'sampling_id_layer', - 'slope_intercept_layer', 'trans_full_matrix_projection', - 'linear_comb_layer', - 'convex_comb_layer', 'ctc_layer', 'crf_layer', 'crf_decoding_layer', - 'nce_layer', - 'cross_entropy_with_selfnorm', 'cross_entropy', - 'multi_binary_label_cross_entropy', 'sum_cost', - 'rank_cost', 'lambda_cost', 'huber_cost', - 'block_expand_layer', - 'maxout_layer', 'out_prod_layer', 'print_layer', - 'spp_layer', - ] +__all__ = [ + "full_matrix_projection", + "AggregateLevel", + "ExpandLevel", + "identity_projection", + "dotmul_projection", + "dotmul_operator", + "repeat_layer", + "table_projection", + "mixed_layer", + "data_layer", + "embedding_layer", + "fc_layer", + "grumemory", + "pooling_layer", + "lstmemory", + "last_seq", + "first_seq", + "cos_sim", + "hsigmoid", + "conv_projection", + "regression_cost", + 'classification_cost', + "LayerOutput", + 'img_conv_layer', + 'img_pool_layer', + 'batch_norm_layer', + 'img_cmrnorm_layer', + 'addto_layer', + 'concat_layer', + 'lstm_step_layer', + 'recurrent_group', + 'memory', + 'StaticInput', + 'expand_layer', + 'scaling_layer', + 'power_layer', + 'interpolation_layer', + 'bilinear_interp_layer', + 'trans_layer', + 'sum_to_one_norm_layer', + 'get_output_layer', + 'LayerType', + 'context_projection', + 'beam_search', + 'maxid_layer', + 'GeneratedInput', + 'SubsequenceInput', + 'gru_step_layer', + 'recurrent_layer', + 'BaseGeneratedInput', + 'conv_operator', + 'conv_shift_layer', + 'tensor_layer', + 'selective_fc_layer', + 'sampling_id_layer', + 'slope_intercept_layer', + 'trans_full_matrix_projection', + 'linear_comb_layer', + 'convex_comb_layer', + 'ctc_layer', + 'crf_layer', + 'crf_decoding_layer', + 'nce_layer', + 'cross_entropy_with_selfnorm', + 'cross_entropy', + 'multi_binary_label_cross_entropy', + 'sum_cost', + 'rank_cost', + 'lambda_cost', + 'huber_cost', + 'block_expand_layer', + 'maxout_layer', + 'out_prod_layer', + 'print_layer', + 'spp_layer', +] class LayerType(object): @@ -181,8 +228,15 @@ class LayerOutput(object): :type parents: list|tuple|collections.Sequence """ - def __init__(self, name, layer_type, parents=None, activation=None, - num_filters=None, img_norm_type=None, size=None, outputs=None, + def __init__(self, + name, + layer_type, + parents=None, + activation=None, + num_filters=None, + img_norm_type=None, + size=None, + outputs=None, reverse=None): assert isinstance(name, basestring) assert isinstance(layer_type, basestring) @@ -223,6 +277,7 @@ def __str__(self): def layer_support(*attrs): attrs_list = list(attrs) attrs_list.append(DEVICE) + def decorator(method): @functools.wraps(method) def wrapper(*args, **kwargs): @@ -282,9 +337,8 @@ def full_matrix_projection(input, size=0, param_attr=None): :return: A FullMatrixProjection Object. :rtype: FullMatrixProjection """ - proj = FullMatrixProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = FullMatrixProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -319,9 +373,8 @@ def trans_full_matrix_projection(input, size=0, param_attr=None): :return: A TransposedFullMatrixProjection Object. :rtype: TransposedFullMatrixProjection """ - proj = TransposedFullMatrixProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = TransposedFullMatrixProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -365,9 +418,8 @@ def table_projection(input, size=0, param_attr=None): :return: A TableProjection Object. :rtype: TableProjection """ - proj = TableProjection(input_layer_name=input.name, - size=size, - **param_attr.attr) + proj = TableProjection( + input_layer_name=input.name, size=size, **param_attr.attr) proj.origin = input return proj @@ -413,8 +465,8 @@ def identity_projection(input, offset=None): proj = IdentityProjection(input_layer_name=input.name) proj.origin = input else: - proj = IdentityOffsetProjection(input_layer_name=input.name, - offset=offset) + proj = IdentityOffsetProjection( + input_layer_name=input.name, offset=offset) proj.origin = input return proj @@ -443,9 +495,8 @@ def dotmul_projection(input, param_attr=None): :return: A DotMulProjection Object. :rtype: DotMulProjection """ - proj = DotMulProjection(input_layer_name=input.name, - size=input.size, - **param_attr.attr) + proj = DotMulProjection( + input_layer_name=input.name, size=input.size, **param_attr.attr) proj.origin = input return proj @@ -478,21 +529,22 @@ def dotmul_operator(a=None, b=None, scale=1, **kwargs): if 'x' in kwargs or 'y' in kwargs: logger.warning('x and y arguments for dotmul_operator is deprecated. ' 'Please use a and b as parameter.') - a = kwargs.get('x', a) # For Backward capacity. + a = kwargs.get('x', a) # For Backward capacity. b = kwargs.get('y', b) assert isinstance(a, LayerOutput) assert isinstance(b, LayerOutput) if a.size is not None and b.size is not None: assert a.size == b.size - op = DotMulOperator(input_layer_names=[a.name, b.name], - scale=scale) + op = DotMulOperator(input_layer_names=[a.name, b.name], scale=scale) op.origin = [a, b] return op @wrap_bias_attr_default(['padding_attr']) -def context_projection(input, context_len, context_start=None, +def context_projection(input, + context_len, + context_start=None, padding_attr=False): """ Context Projection. @@ -529,11 +581,12 @@ def context_projection(input, context_len, context_start=None, if trainable: extra_dict = padding_attr.attr - proj = ContextProjection(input_layer_name=input.name, - context_length=context_len, - context_start=context_start, - trainable_padding=trainable, - **extra_dict) + proj = ContextProjection( + input_layer_name=input.name, + context_length=context_len, + context_start=context_start, + trainable_padding=trainable, + **extra_dict) proj.origin = input return proj @@ -547,8 +600,7 @@ class AddToSealedMixedLayerException(Exception): def __init__(self): Exception.__init__(self) - def __init__(self, name, size, act, bias_attr, layer_attr, - parents=None): + def __init__(self, name, size, act, bias_attr, layer_attr, parents=None): """ Ctor. :param name: layer name. @@ -565,8 +617,13 @@ def __init__(self, name, size, act, bias_attr, layer_attr, :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute or None """ - LayerOutput.__init__(self, name, LayerType.MIXED_LAYER, parents, - size=size, activation=act) + LayerOutput.__init__( + self, + name, + LayerType.MIXED_LAYER, + parents, + size=size, + activation=act) self.bias_attr = bias_attr self.layer_attr = layer_attr self.inputs = [] @@ -604,8 +661,7 @@ def __exit__(self, *args, **kwargs): active_type=self.activation.name, bias=ParamAttr.to_bias(self.bias_attr), inputs=self.inputs, - **ExtraLayerAttribute.to_kwargs(self.layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(self.layer_attr)) # update the size which might be computed inside MixedLayer # according to the operator's output size self.size = ml.config.size @@ -615,7 +671,11 @@ def __exit__(self, *args, **kwargs): @wrap_act_default(act=LinearActivation()) @wrap_bias_attr_default(has_bias=False) @layer_support(ERROR_CLIPPING, DROPOUT) -def mixed_layer(size=0, input=None, name=None, act=None, bias_attr=False, +def mixed_layer(size=0, + input=None, + name=None, + act=None, + bias_attr=False, layer_attr=None): """ Mixed Layer. A mixed layer will add all inputs together, then activate. @@ -660,8 +720,12 @@ def mixed_layer(size=0, input=None, name=None, act=None, bias_attr=False, if input is None: return MixedLayerType(name, size, act, bias_attr, layer_attr) else: - with mixed_layer(name=name, size=size, act=act, bias_attr=bias_attr, - layer_attr=layer_attr) as m: + with mixed_layer( + name=name, + size=size, + act=act, + bias_attr=bias_attr, + layer_attr=layer_attr) as m: if isinstance(input, collections.Sequence): for each in input: m += each @@ -691,8 +755,11 @@ def data_layer(name, size, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - Layer(type=LayerType.DATA, name=name, size=size, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + type=LayerType.DATA, + name=name, + size=size, + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.DATA, size=size) @@ -718,9 +785,12 @@ def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - with mixed_layer(name=name, size=size, act=LinearActivation(), - bias_attr=False, - layer_attr=layer_attr) as mix: + with mixed_layer( + name=name, + size=size, + act=LinearActivation(), + bias_attr=False, + layer_attr=layer_attr) as mix: mix += table_projection(input=input, size=size, param_attr=param_attr) return mix @@ -730,8 +800,13 @@ def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None): @wrap_bias_attr_default() @wrap_act_default() @layer_support(ERROR_CLIPPING, DROPOUT) -def fc_layer(input, size, act=None, name=None, - param_attr=None, bias_attr=None, layer_attr=None): +def fc_layer(input, + size, + act=None, + name=None, + param_attr=None, + bias_attr=None, + layer_attr=None): """ Helper for declare fully connected layer. @@ -783,17 +858,17 @@ def fc_layer(input, size, act=None, name=None, assert isinstance(input, collections.Sequence) Layer( - inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( - input, param_attr)], + inputs=[ + Input(ipt.name, **attr.attr) for ipt, attr in zip(input, param_attr) + ], name=name, type=LayerType.FC_LAYER, size=size, bias=ParamAttr.to_bias(bias_attr), active_type=act.name, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.FC_LAYER, input, activation=act, - size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.FC_LAYER, input, activation=act, size=size) @wrap_name_default("print") @@ -816,8 +891,7 @@ def print_layer(input, name=None): Layer( name=name, type=LayerType.PRINT_LAYER, - inputs=[l.name for l in input], - ) + inputs=[l.name for l in input], ) # this layer don't return anything, can not be input of other layer. @@ -825,7 +899,10 @@ def print_layer(input, name=None): @wrap_bias_attr_default(has_bias=False) @wrap_param_default(['pooling_type'], default_factory=lambda _: MaxPooling()) @layer_support() -def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, +def pooling_layer(input, + pooling_type=None, + name=None, + bias_attr=None, agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ @@ -872,24 +949,27 @@ def pooling_layer(input, pooling_type=None, name=None, bias_attr=None, inputs=[Input(input.name)], bias=ParamAttr.to_bias(bias_attr), trans_type=agg_level, - **extra_dict - ) - - return LayerOutput(name, pooling_type.name, parents=[input], - size=input.size) + **extra_dict) + return LayerOutput( + name, pooling_type.name, parents=[input], size=input.size) @wrap_bias_attr_default() @wrap_param_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(param_names=["act", 'state_act'], act=TanhActivation()) @wrap_name_default("lstmemory") @layer_support(DROPOUT) -def lstmemory(input, name=None, reverse=False, act=None, - gate_act=None, size=None, - state_act=None, bias_attr=None, param_attr=None, +def lstmemory(input, + name=None, + reverse=False, + act=None, + gate_act=None, + size=None, + state_act=None, + bias_attr=None, + param_attr=None, layer_attr=None): """ Long Short-term Memory Cell. @@ -964,30 +1044,38 @@ def lstmemory(input, name=None, reverse=False, act=None, "layer. The lstm size should be equal with input layer size/4. The" " size which is set explicitly will be ignored." % name) - Layer(name=name, - type=LayerType.LSTMEMORY, - active_type=act.name, - active_state_type=state_act.name, - active_gate_type=gate_act.name, - reversed=reverse, - bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(input.name, **param_attr.attr)], - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + name=name, + type=LayerType.LSTMEMORY, + active_type=act.name, + active_state_type=state_act.name, + active_gate_type=gate_act.name, + reversed=reverse, + bias=ParamAttr.to_bias(bias_attr), + inputs=[Input(input.name, **param_attr.attr)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.LSTMEMORY, [input], size=input.size / 4, - reverse=reverse) + return LayerOutput( + name, + LayerType.LSTMEMORY, [input], + size=input.size / 4, + reverse=reverse) @wrap_bias_attr_default() @wrap_param_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(param_names=["act"], act=TanhActivation()) @wrap_name_default("gru") @layer_support(DROPOUT) -def grumemory(input, name=None, reverse=False, act=None, - gate_act=None, size=None, - bias_attr=None, param_attr=None, +def grumemory(input, + name=None, + reverse=False, + act=None, + gate_act=None, + size=None, + bias_attr=None, + param_attr=None, layer_attr=None): """ Gate Recurrent Unit Layer. @@ -1078,23 +1166,28 @@ def grumemory(input, name=None, reverse=False, act=None, " and should be input size / 3. Set size explicitly will be " "ignored.") - Layer(name=name, - type=LayerType.GRUMEMORY, - active_type=act.name, - active_gate_type=gate_act.name, - reversed=reverse, - bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(input.name, **param_attr.attr)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + Layer( + name=name, + type=LayerType.GRUMEMORY, + active_type=act.name, + active_gate_type=gate_act.name, + reversed=reverse, + bias=ParamAttr.to_bias(bias_attr), + inputs=[Input(input.name, **param_attr.attr)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.GRUMEMORY, [input], size=input.size / 3, - reverse=reverse) + return LayerOutput( + name, + LayerType.GRUMEMORY, [input], + size=input.size / 3, + reverse=reverse) @wrap_name_default() @layer_support() -def last_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, +def last_seq(input, + name=None, + agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ Get Last Timestamp Activation of a sequence. @@ -1120,15 +1213,19 @@ def last_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, type=LayerType.SEQUENCE_LAST_INSTANCE, inputs=[input.name], trans_type=agg_level, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEQUENCE_LAST_INSTANCE, parents=[input], - size=input.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEQUENCE_LAST_INSTANCE, + parents=[input], + size=input.size) @wrap_name_default() @layer_support() -def first_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, +def first_seq(input, + name=None, + agg_level=AggregateLevel.EACH_TIMESTEP, layer_attr=None): """ Get First Timestamp Activation of a sequence. @@ -1155,10 +1252,12 @@ def first_seq(input, name=None, agg_level=AggregateLevel.EACH_TIMESTEP, type=LayerType.SEQUENCE_FIRST_INSTANCE, inputs=[input.name], trans_type=agg_level, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEQUENCE_FIRST_INSTANCE, - parents=[input], size=input.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEQUENCE_FIRST_INSTANCE, + parents=[input], + size=input.size) class ExpandLevel(object): @@ -1168,7 +1267,8 @@ class ExpandLevel(object): @wrap_name_default() @layer_support() -def expand_layer(input, expand_as, +def expand_layer(input, + expand_as, name=None, bias_attr=False, expand_level=ExpandLevel.FROM_TIMESTEP, @@ -1208,19 +1308,17 @@ def expand_layer(input, expand_as, bias=ParamAttr.to_bias(bias_attr=bias_attr), type=LayerType.EXPAND_LAYER, trans_type=expand_level, - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name=name, - size=input.size, - layer_type=LayerType.EXPAND_LAYER, - parents=[input, expand_as]) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + size=input.size, + layer_type=LayerType.EXPAND_LAYER, + parents=[input, expand_as]) @wrap_name_default() @layer_support() -def repeat_layer(input, num_repeats, - name=None, - layer_attr=None): +def repeat_layer(input, num_repeats, name=None, layer_attr=None): """ A layer for repeating the input for num_repeats times. This is equivalent to apply concat_layer() with num_repeats same input. @@ -1251,12 +1349,13 @@ def repeat_layer(input, num_repeats, name=name, num_filters=num_repeats, type=LayerType.FEATURE_MAP_EXPAND_LAYER, - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name=name, - size=l.config.size, - layer_type=LayerType.FEATURE_MAP_EXPAND_LAYER, - parents=[input]) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + size=l.config.size, + layer_type=LayerType.FEATURE_MAP_EXPAND_LAYER, + parents=[input]) + @wrap_name_default() @layer_support() @@ -1302,11 +1401,12 @@ def interpolation_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.INTERPOLATION_LAYER, inputs=[weight.name, input[0].name, input[1].name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.INTERPOLATION_LAYER, - parents=[weight, input[0], input[1]], - size=input[0].size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.INTERPOLATION_LAYER, + parents=[weight, input[0], input[1]], + size=input[0].size) @wrap_name_default() @@ -1345,15 +1445,23 @@ def bilinear_interp_layer(input, assert out_size_x > 0 and out_size_y > 0 assert input.num_filters is not None num_channels = input.num_filters - l = Layer(name=name, - inputs=Input(input.name, - bilinear_interp=BilinearInterp(out_size_x=out_size_x, - out_size_y=out_size_y, - num_channels=num_channels)), - type=LayerType.BILINEAR_INTERP_LAYER, - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.BILINEAR_INTERP_LAYER, parents=[input], - num_filters=num_channels, size=l.config.size) + l = Layer( + name=name, + inputs=Input( + input.name, + bilinear_interp=BilinearInterp( + out_size_x=out_size_x, + out_size_y=out_size_y, + num_channels=num_channels)), + type=LayerType.BILINEAR_INTERP_LAYER, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.BILINEAR_INTERP_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) + @wrap_name_default() @layer_support() @@ -1392,10 +1500,9 @@ def power_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.POWER_LAYER, inputs=[weight.name, input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.POWER_LAYER, - parents=[input, weight], size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.POWER_LAYER, parents=[input, weight], size=input.size) @wrap_name_default() @@ -1437,10 +1544,9 @@ def scaling_layer(input, weight, name=None, layer_attr=None): name=name, type=LayerType.SCALING_LAYER, inputs=[weight.name, input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SCALING_LAYER, parents=[weight, input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SCALING_LAYER, parents=[weight, input], size=input.size) @wrap_name_default() @@ -1473,10 +1579,9 @@ def trans_layer(input, name=None, layer_attr=None): name=name, type=LayerType.TRANS_LAYER, inputs=[input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.TRANS_LAYER, parents=[input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.TRANS_LAYER, parents=[input], size=input.size) @wrap_name_default() @@ -1518,8 +1623,7 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): type=LayerType.COSINE_SIM, cos_scale=scale, inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) else: if a.size is not None and b.size is not None: assert size == b.size / a.size @@ -1529,8 +1633,7 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): size=size, cos_scale=scale, inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.COSINE_SIM, parents=[a, b], size=size) @@ -1538,8 +1641,13 @@ def cos_sim(a, b, scale=5, size=1, name=None, layer_attr=None): @wrap_bias_attr_default(has_bias=True) @wrap_param_attr_default() @layer_support() -def hsigmoid(input, label, num_classes, name=None, bias_attr=None, - param_attr=None, layer_attr=None): +def hsigmoid(input, + label, + num_classes, + name=None, + bias_attr=None, + param_attr=None, + layer_attr=None): """ Organize the classes into a binary tree. At each node, a sigmoid function is used to calculate the probability of belonging to the right branch. @@ -1600,10 +1708,9 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, num_classes=num_classes, bias=ParamAttr.to_bias(bias_attr), inputs=ipts_for_layer, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.HSIGMOID, parents=parents, - size=l.config.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.HSIGMOID, parents=parents, size=l.config.size) @wrap_name_default("conv") @@ -1611,11 +1718,22 @@ def hsigmoid(input, label, num_classes, name=None, bias_attr=None, @wrap_bias_attr_default() @wrap_act_default(act=ReluActivation()) @layer_support(DROPOUT) -def img_conv_layer(input, filter_size, num_filters, - name=None, num_channels=None, - act=None, groups=1, stride=1, padding=0, bias_attr=None, - param_attr=None, shared_biases=True, layer_attr=None, - filter_size_y=None, stride_y=None, padding_y=None, +def img_conv_layer(input, + filter_size, + num_filters, + name=None, + num_channels=None, + act=None, + groups=1, + stride=1, + padding=0, + bias_attr=None, + param_attr=None, + shared_biases=True, + layer_attr=None, + filter_size_y=None, + stride_y=None, + padding_y=None, trans=False): """ Convolution layer for image. Paddle only support square input currently and @@ -1713,40 +1831,56 @@ def img_conv_layer(input, filter_size, num_filters, if param_attr.attr.get('initial_smart'): # special initial for conv layers. - init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 + init_w = (2.0 / (filter_size**2 * num_channels))**0.5 param_attr.attr["initial_mean"] = 0.0 param_attr.attr["initial_std"] = init_w param_attr.attr["initial_strategy"] = 0 param_attr.attr["initial_smart"] = False - + lt = LayerType.CONVTRANS_LAYER if trans else LayerType.CONV_LAYER - + l = Layer( name=name, - inputs=Input(input.name, conv=Conv( - filter_size=filter_size, padding=padding, stride=stride, - channels=num_channels, groups=groups, - filter_size_y=filter_size_y, padding_y=padding_y, - stride_y=stride_y), - **param_attr.attr), + inputs=Input( + input.name, + conv=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + groups=groups, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y), + **param_attr.attr), active_type=act.name, num_filters=num_filters, bias=ParamAttr.to_bias(bias_attr), shared_biases=shared_biases, type=lt, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, lt, parents=[input], - activation=act, num_filters=num_filters, - size=l.config.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + lt, + parents=[input], + activation=act, + num_filters=num_filters, + size=l.config.size) @wrap_name_default("pool") @layer_support() -def img_pool_layer(input, pool_size, name=None, - num_channels=None, pool_type=None, - stride=1, padding=0, layer_attr=None, - pool_size_y=None, stride_y=None, padding_y=None, +def img_pool_layer(input, + pool_size, + name=None, + num_channels=None, + pool_type=None, + stride=1, + padding=0, + layer_attr=None, + pool_size_y=None, + stride_y=None, + padding_y=None, img_width=None): """ Image pooling Layer. @@ -1804,29 +1938,39 @@ def img_pool_layer(input, pool_size, name=None, l = Layer( name=name, type=LayerType.POOL_LAYER, - inputs=[Input(input.name, - pool=Pool( - pool_type=type_name, - channels=num_channels, - size_x=pool_size, - start=None, - stride=stride, - padding=padding, - size_y=pool_size_y, - stride_y=stride_y, - padding_y=padding_y, - img_width=img_width - ))], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.POOL_LAYER, parents=[input], - num_filters=num_channels, size=l.config.size) + inputs=[ + Input( + input.name, + pool=Pool( + pool_type=type_name, + channels=num_channels, + size_x=pool_size, + start=None, + stride=stride, + padding=padding, + size_y=pool_size_y, + stride_y=stride_y, + padding_y=padding_y, + img_width=img_width)) + ], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.POOL_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) @wrap_name_default("spp") @layer_support() -def spp_layer(input, name=None, num_channels=None, pool_type=None, - pyramid_height=None, img_width=None, layer_attr=None): +def spp_layer(input, + name=None, + num_channels=None, + pool_type=None, + pyramid_height=None, + img_width=None, + layer_attr=None): """ Spatial Pyramid Pooling in Deep Convolutional Networks for Visual Recognition. The details please refer to @@ -1866,42 +2010,58 @@ def spp_layer(input, name=None, num_channels=None, pool_type=None, l = Layer( name=name, type=LayerType.SPP_LAYER, - inputs=Input(input.name, - spp=SpatialPyramidPool(pool_type=type_name, - channels=num_channels, - pyramid_height=pyramid_height, - img_width=img_width) - ), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, layer_type=LayerType.SPP_LAYER, parents=[input], - num_filters=num_channels, size=l.config.size) - - -def __img_norm_layer__(name, input, size, norm_type, scale, power, - num_channels, blocked, layer_attr): + inputs=Input( + input.name, + spp=SpatialPyramidPool( + pool_type=type_name, + channels=num_channels, + pyramid_height=pyramid_height, + img_width=img_width)), + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + layer_type=LayerType.SPP_LAYER, + parents=[input], + num_filters=num_channels, + size=l.config.size) + + +def __img_norm_layer__(name, input, size, norm_type, scale, power, num_channels, + blocked, layer_attr): if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters l = Layer( - name=name, type=LayerType.NORM_LAYER, inputs=Input( - input.name, norm=Norm(norm_type=norm_type, - channels=num_channels, size=size, - scale=scale, - pow=power, blocked=blocked) - ), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, layer_type=LayerType.NORM_LAYER, parents=[input], - num_filters=num_channels, img_norm_type=norm_type, - size=l.config.size) + name=name, + type=LayerType.NORM_LAYER, + inputs=Input( + input.name, + norm=Norm( + norm_type=norm_type, + channels=num_channels, + size=size, + scale=scale, + pow=power, + blocked=blocked)), + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + layer_type=LayerType.NORM_LAYER, + parents=[input], + num_filters=num_channels, + img_norm_type=norm_type, + size=l.config.size) @wrap_name_default("crmnorm") @layer_support() -def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, - name=None, num_channels=None, +def img_cmrnorm_layer(input, + size, + scale=0.0128, + power=0.75, + name=None, + num_channels=None, layer_attr=None): """ Response normalization across feature maps. @@ -1935,8 +2095,13 @@ def img_cmrnorm_layer(input, size, scale=0.0128, power=0.75, @wrap_act_default(act=ReluActivation()) @wrap_name_default("batch_norm") @layer_support(DROPOUT) -def batch_norm_layer(input, act=None, name=None, num_channels=None, - bias_attr=None, param_attr=None, layer_attr=None, +def batch_norm_layer(input, + act=None, + name=None, + num_channels=None, + bias_attr=None, + param_attr=None, + layer_attr=None, batch_norm_type=None, moving_average_fraction=0.9, use_global_stats=None): @@ -2022,22 +2187,23 @@ def batch_norm_layer(input, act=None, name=None, num_channels=None, (batch_norm_type == "cudnn_batch_norm") l = Layer( name=name, - inputs=Input(input.name, - image=Image(channels=num_channels), - **param_attr.attr), + inputs=Input( + input.name, image=Image(channels=num_channels), **param_attr.attr), active_type=act.name, type=LayerType.BATCH_NORM_LAYER, batch_norm_type=batch_norm_type, bias=ParamAttr.to_bias(bias_attr), moving_average_fraction=moving_average_fraction, use_global_stats=use_global_stats, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.BATCH_NORM_LAYER, - parents=[input], activation=act, - num_filters=num_channels, - size=l.config.size) + return LayerOutput( + name=name, + layer_type=LayerType.BATCH_NORM_LAYER, + parents=[input], + activation=act, + num_filters=num_channels, + size=l.config.size) @wrap_name_default() @@ -2072,18 +2238,16 @@ def sum_to_one_norm_layer(input, name=None, layer_attr=None): name=name, type=LayerType.SUM_TO_ONE_NORM_LAYER, inputs=[input.name], - **ExtraAttr.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SUM_TO_ONE_NORM_LAYER, parents=[input], - size=input.size) + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SUM_TO_ONE_NORM_LAYER, parents=[input], size=input.size) @wrap_name_default("addto") @wrap_act_default(act=LinearActivation()) @wrap_bias_attr_default(has_bias=False) @layer_support(DROPOUT) -def addto_layer(input, act=None, name=None, bias_attr=None, - layer_attr=None): +def addto_layer(input, act=None, name=None, bias_attr=None, layer_attr=None): """ AddtoLayer. @@ -2143,15 +2307,20 @@ def addto_layer(input, act=None, name=None, bias_attr=None, num_filters = each_input.num_filters l = Layer( - name=name, type=LayerType.ADDTO_LAYER, inputs=ipts_for_layer, + name=name, + type=LayerType.ADDTO_LAYER, + inputs=ipts_for_layer, bias=ParamAttr.to_bias(bias_attr), active_type=act.name, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.ADDTO_LAYER, parents=input, - activation=act, num_filters=num_filters, - size=l.config.size) + return LayerOutput( + name, + LayerType.ADDTO_LAYER, + parents=input, + activation=act, + num_filters=num_filters, + size=l.config.size) @wrap_act_default(act=IdentityActivation()) @@ -2210,22 +2379,22 @@ def __reduce_concat_type__(a, b): LayerOutput) return a - is_concat_layer = __is_type__(reduce(__reduce_concat_type__, - map(type, input)), LayerOutput) + is_concat_layer = __is_type__( + reduce(__reduce_concat_type__, map(type, input)), LayerOutput) - layer_type = (LayerType.CONCAT_LAYER if is_concat_layer - else LayerType.CONCAT_PROJ_LAYER) + layer_type = (LayerType.CONCAT_LAYER + if is_concat_layer else LayerType.CONCAT_PROJ_LAYER) if layer_type == LayerType.CONCAT_LAYER: assert not bias_attr Layer( - name=name, type=layer_type, + name=name, + type=layer_type, inputs=[x.name for x in input] if is_concat_layer else input, active_type=act.name, bias=ParamAttr.to_bias(bias_attr), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) sz = 0 for each_input in input: @@ -2235,14 +2404,20 @@ def __reduce_concat_type__(a, b): sz = None break - return LayerOutput(name, layer_type=layer_type, - parents=input if is_concat_layer else [ - x.origin for x in input], - activation=act, size=sz) - - -def memory(name, size, is_seq=False, boot_layer=None, - boot_bias=None, boot_bias_active_type=None, + return LayerOutput( + name, + layer_type=layer_type, + parents=input if is_concat_layer else [x.origin for x in input], + activation=act, + size=sz) + + +def memory(name, + size, + is_seq=False, + boot_layer=None, + boot_bias=None, + boot_bias_active_type=None, boot_with_const_id=None): """ The memory layers is a layer cross each time step. Reference this output @@ -2290,30 +2465,33 @@ def memory(name, size, is_seq=False, boot_layer=None, assert boot_layer is None or isinstance(boot_layer, LayerOutput) - agent_name = Memory(name, size, - is_seq, - boot_layer.name if boot_layer is not None else None, - boot_bias, - boot_bias_active_type.name, - boot_with_const_id) - - lout = LayerOutput(name=agent_name, size=size, - layer_type=LayerType.MEMORY, - parents=[boot_layer] if boot_layer is not None - else None) + agent_name = Memory(name, size, is_seq, boot_layer.name + if boot_layer is not None else None, boot_bias, + boot_bias_active_type.name, boot_with_const_id) + + lout = LayerOutput( + name=agent_name, + size=size, + layer_type=LayerType.MEMORY, + parents=[boot_layer] if boot_layer is not None else None) return lout @wrap_bias_attr_default() -@wrap_act_default(param_names=['gate_act', - 'state_act'], - act=SigmoidActivation()) +@wrap_act_default( + param_names=['gate_act', 'state_act'], act=SigmoidActivation()) @wrap_act_default(act=TanhActivation()) @wrap_name_default('lstm_step') @layer_support() -def lstm_step_layer(input, state, size, act=None, - name=None, gate_act=None, state_act=None, - bias_attr=None, layer_attr=None): +def lstm_step_layer(input, + state, + size, + act=None, + name=None, + gate_act=None, + state_act=None, + bias_attr=None, + layer_attr=None): """ LSTM Step Layer. It used in recurrent_group. The lstm equations are shown as follow. @@ -2380,24 +2558,32 @@ def lstm_step_layer(input, state, size, act=None, active_gate_type=gate_act.name, active_state_type=state_act.name, bias=ParamAttr.to_bias(bias_attr), - size=size, inputs=[input.name, state.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + size=size, + inputs=[input.name, state.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.LSTM_STEP_LAYER, - parents=[input, state], activation=act, - size=size, outputs=['default', 'state']) + return LayerOutput( + name=name, + layer_type=LayerType.LSTM_STEP_LAYER, + parents=[input, state], + activation=act, + size=size, + outputs=['default', 'state']) @wrap_bias_attr_default() -@wrap_act_default(param_names=['gate_act'], - act=SigmoidActivation()) +@wrap_act_default(param_names=['gate_act'], act=SigmoidActivation()) @wrap_act_default(act=TanhActivation()) @wrap_name_default('gru_step') @layer_support() -def gru_step_layer(input, output_mem, size=None, act=None, - name=None, gate_act=None, - bias_attr=None, layer_attr=None): +def gru_step_layer(input, + output_mem, + size=None, + act=None, + name=None, + gate_act=None, + bias_attr=None, + layer_attr=None): """ :param input: @@ -2418,20 +2604,18 @@ def gru_step_layer(input, output_mem, size=None, act=None, Layer( name=name, type=LayerType.GRU_STEP_LAYER, - inputs=[ - input.name, - output_mem.name - ], + inputs=[input.name, output_mem.name], bias=ParamAttr.to_bias(bias_attr), size=size, active_type=act.name, active_gate_type=gate_act.name, - **ExtraAttr.to_kwargs(layer_attr) - ) + **ExtraAttr.to_kwargs(layer_attr)) return LayerOutput( - name=name, layer_type=LayerType.GRU_STEP_LAYER, + name=name, + layer_type=LayerType.GRU_STEP_LAYER, parents=[input, output_mem], - size=size, activation=act) + size=size, + activation=act) @wrap_name_default() @@ -2459,13 +2643,19 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): ' The get output name is %s, which not' \ ' in %s' % ( arg_name, ",".join(input.outputs)) - Layer(name=name, type=LayerType.GET_OUTPUT_LAYER, - inputs=[Input(input.name, input_layer_argument=arg_name)], - size=input.size, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + name=name, + type=LayerType.GET_OUTPUT_LAYER, + inputs=[Input( + input.name, input_layer_argument=arg_name)], + size=input.size, + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.GET_OUTPUT_LAYER, - parents=[input], size=input.size) + return LayerOutput( + name=name, + layer_type=LayerType.GET_OUTPUT_LAYER, + parents=[input], + size=input.size) @wrap_name_default() @@ -2473,8 +2663,13 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): @wrap_bias_attr_default() @wrap_param_attr_default() @layer_support() -def recurrent_layer(input, act=None, bias_attr=None, - param_attr=None, name=None, reverse=False, layer_attr=None): +def recurrent_layer(input, + act=None, + bias_attr=None, + param_attr=None, + name=None, + reverse=False, + layer_attr=None): """ Simple recurrent unit layer. It is just a fully connect layer through both time and neural network. @@ -2509,16 +2704,21 @@ def recurrent_layer(input, act=None, bias_attr=None, :return: LayerOutput object. :rtype: LayerOutput """ - Layer(name=name, - type=LayerType.RECURRENT_LAYER, - inputs=Input(input.name, **param_attr.attr), - active_type=act.name, - bias=ParamAttr.to_bias(bias_attr), - reversed=reverse, - **ExtraAttr.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.RECURRENT_LAYER, - parents=[input], size=input.size, activation=act, - reverse=reverse) + Layer( + name=name, + type=LayerType.RECURRENT_LAYER, + inputs=Input(input.name, **param_attr.attr), + active_type=act.name, + bias=ParamAttr.to_bias(bias_attr), + reversed=reverse, + **ExtraAttr.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.RECURRENT_LAYER, + parents=[input], + size=input.size, + activation=act, + reverse=reverse) class StaticInput(object): @@ -2646,7 +2846,7 @@ def targetInlink_in_inlinks(): return True return False - assert(targetInlink == None or targetInlink_in_inlinks()) + assert (targetInlink == None or targetInlink_in_inlinks()) targetInlinkName = None if targetInlink == None \ else targetInlink.name if isinstance(targetInlink, LayerOutput) \ else targetInlink.input.name @@ -2661,7 +2861,8 @@ def map_in_links(x): return x.name RecurrentLayerGroupWithoutOutLinksBegin( - name=name, in_links=map(map_in_links, in_links), + name=name, + in_links=map(map_in_links, in_links), seq_reversed=reverse, target_inlinkname=targetInlinkName) in_args = [] @@ -2673,12 +2874,15 @@ def map_in_links(x): in_args.append(each_input.input) else: mem_name = "__%s_memory__" % each_input.input.name - mem = memory(name=mem_name, - is_seq=each_input.is_seq, - size=each_input.input.size, - boot_layer=each_input.input) - with mixed_layer(name=mem_name, size=each_input.input.size, - act=IdentityActivation()) as mix: + mem = memory( + name=mem_name, + is_seq=each_input.is_seq, + size=each_input.input.size, + boot_layer=each_input.input) + with mixed_layer( + name=mem_name, + size=each_input.input.size, + act=IdentityActivation()) as mix: mix += identity_projection(mem) in_args.append(mem) @@ -2720,14 +2924,15 @@ def after_real_step(self, input): return maxid_layer(input=input, name='__beam_search_predict__') def before_real_step(self): - predict_id = memory(name='__beam_search_predict__', - size=self.size, - boot_with_const_id=self.bos_id) - - trg_emb = embedding_layer(input=predict_id, - size=self.embedding_size, - param_attr=ParamAttr( - name=self.embedding_name)) + predict_id = memory( + name='__beam_search_predict__', + size=self.size, + boot_with_const_id=self.bos_id) + + trg_emb = embedding_layer( + input=predict_id, + size=self.embedding_size, + param_attr=ParamAttr(name=self.embedding_name)) return trg_emb def __init__(self, size, embedding_name, embedding_size): @@ -2760,14 +2965,16 @@ def maxid_layer(input, name=None, layer_attr=None): """ assert isinstance(input, LayerOutput) - l = Layer(name=name, - type='maxid', - inputs=[input.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, - layer_type=LayerType.MAXID_LAYER, - parents=[input], - size=l.config.size) + l = Layer( + name=name, + type='maxid', + inputs=[input.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.MAXID_LAYER, + parents=[input], + size=l.config.size) @wrap_name_default() @@ -2796,14 +3003,16 @@ def out_prod_layer(input1, input2, name=None, layer_attr=None): assert isinstance(input1, LayerOutput) assert isinstance(input2, LayerOutput) - l = Layer(name=name, - type=LayerType.OUT_PROD_LAYER, - inputs=[input1.name, input2.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, - layer_type=LayerType.OUT_PROD_LAYER, - parents=[input1, input2], - size=l.config.size) + l = Layer( + name=name, + type=LayerType.OUT_PROD_LAYER, + inputs=[input1.name, input2.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.OUT_PROD_LAYER, + parents=[input1, input2], + size=l.config.size) @wrap_name_default() @@ -2832,19 +3041,27 @@ def eos_layer(input, eos_id, name=None, layer_attr=None): :return: LayerOutput object. :rtype: LayerOutput """ - l = Layer(name=name, - type=LayerType.EOSID_LAYER, - eos_id=eos_id, - inputs=[input.name], - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name=name, layer_type=LayerType.EOSID_LAYER, - parents=[input], - size=l.config.size) + l = Layer( + name=name, + type=LayerType.EOSID_LAYER, + eos_id=eos_id, + inputs=[input.name], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name=name, + layer_type=LayerType.EOSID_LAYER, + parents=[input], + size=l.config.size) @wrap_name_default() -def beam_search(step, input, bos_id, eos_id, beam_size, - max_length=500, name=None, +def beam_search(step, + input, + bos_id, + eos_id, + beam_size, + max_length=500, + name=None, num_results_per_sample=None): """ Beam search is a heuristic search algorithm used in sequence generation. @@ -2918,8 +3135,7 @@ def rnn_step(input): if num_results_per_sample > beam_size: logger.warning("num_results_per_sample should be less than beam_size") - if isinstance(input, StaticInput) or isinstance(input, - BaseGeneratedInput): + if isinstance(input, StaticInput) or isinstance(input, BaseGeneratedInput): input = [input] generated_input_index = -1 @@ -2944,11 +3160,12 @@ def rnn_step(input): def __real_step__(*args): eos_name = "__%s_eos_layer__" % name - RecurrentLayerGroupSetGenerator(Generator( - eos_layer_name=eos_name, - max_num_frames=max_length, - beam_size=beam_size, - num_results_per_sample=num_results_per_sample)) + RecurrentLayerGroupSetGenerator( + Generator( + eos_layer_name=eos_name, + max_num_frames=max_length, + beam_size=beam_size, + num_results_per_sample=num_results_per_sample)) args = list(args) args.insert(generated_input_index, gipt.before_real_step()) @@ -2959,11 +3176,12 @@ def __real_step__(*args): return predict - tmp = recurrent_group(step=__real_step__, input=real_input, reverse=False, - name=name) + tmp = recurrent_group( + step=__real_step__, input=real_input, reverse=False, name=name) return tmp + def __cost_input__(input, label, weight=None): """ inputs and parents for cost layers. @@ -2979,8 +3197,7 @@ def __cost_input__(input, label, weight=None): @wrap_name_default() @layer_support() -def regression_cost(input, label, weight=None, name=None, - layer_attr=None): +def regression_cost(input, label, weight=None, name=None, layer_attr=None): """ Regression Layer. @@ -3002,14 +3219,20 @@ def regression_cost(input, label, weight=None, name=None, """ ipts, parents = __cost_input__(input, label, weight) - Layer(inputs=ipts, type="square_error", name=name, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + inputs=ipts, + type="square_error", + name=name, + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.COST, parents=parents, size=1) @wrap_name_default("cost") @layer_support() -def classification_cost(input, label, weight=None, name=None, +def classification_cost(input, + label, + weight=None, + name=None, evaluator=classification_error_evaluator, layer_attr=None): """ @@ -3036,8 +3259,11 @@ def classification_cost(input, label, weight=None, name=None, ipts, parents = __cost_input__(input, label, weight) - Layer(name=name, type="multi-class-cross-entropy", inputs=ipts, - **ExtraLayerAttribute.to_kwargs(layer_attr)) + Layer( + name=name, + type="multi-class-cross-entropy", + inputs=ipts, + **ExtraLayerAttribute.to_kwargs(layer_attr)) def __add_evaluator__(e): assert callable(e) @@ -3059,9 +3285,16 @@ def __add_evaluator__(e): return LayerOutput(name, LayerType.COST, parents=parents, size=1) -def conv_operator(img, filter, filter_size, num_filters, - num_channels=None, stride=1, padding=0, - filter_size_y=None, stride_y=None, padding_y=None): +def conv_operator(img, + filter, + filter_size, + num_filters, + num_channels=None, + stride=1, + padding=0, + filter_size_y=None, + stride_y=None, + padding_y=None): """ Different from img_conv_layer, conv_op is an Operator, which can be used in mixed_layer. And conv_op takes two inputs to perform convolution. @@ -3117,24 +3350,34 @@ def conv_operator(img, filter, filter_size, num_filters, if filter.size is not None: filter.size = filter_size * filter_size_y * num_filters * num_channels - op = ConvOperator(input_layer_names=[img.name, filter.name], - num_filters=num_filters, - conv_conf=Conv(filter_size=filter_size, - padding=padding, - stride=stride, - channels=num_channels, - filter_size_y=filter_size_y, - padding_y=padding_y, - stride_y=stride_y, - groups=1)) + op = ConvOperator( + input_layer_names=[img.name, filter.name], + num_filters=num_filters, + conv_conf=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y, + groups=1)) op.origin = [img, filter] return op + @wrap_param_attr_default() -def conv_projection(input, filter_size, num_filters, - num_channels=None, stride=1, padding=0, - filter_size_y=None, stride_y=None, padding_y=None, - groups=1, param_attr=None): +def conv_projection(input, + filter_size, + num_filters, + num_channels=None, + stride=1, + padding=0, + filter_size_y=None, + stride_y=None, + padding_y=None, + groups=1, + param_attr=None): """ ConvProjection with a layer as input. It performs element-wise multiplication with weight. @@ -3206,23 +3449,25 @@ def conv_projection(input, filter_size, num_filters, if param_attr.attr.get('initial_smart'): # special initial for conv layers. - init_w = (2.0 / (filter_size ** 2 * num_channels)) ** 0.5 + init_w = (2.0 / (filter_size**2 * num_channels))**0.5 param_attr.attr["initial_mean"] = 0.0 param_attr.attr["initial_std"] = init_w param_attr.attr["initial_strategy"] = 0 param_attr.attr["initial_smart"] = False - proj = ConvProjection(input_layer_name=input.name, - num_filters=num_filters, - conv_conf=Conv(filter_size=filter_size, - padding=padding, - stride=stride, - channels=num_channels, - filter_size_y=filter_size_y, - padding_y=padding_y, - stride_y=stride_y, - groups=groups), - **param_attr.attr) + proj = ConvProjection( + input_layer_name=input.name, + num_filters=num_filters, + conv_conf=Conv( + filter_size=filter_size, + padding=padding, + stride=stride, + channels=num_channels, + filter_size_y=filter_size_y, + padding_y=padding_y, + stride_y=stride_y, + groups=groups), + **param_attr.attr) proj.origin = input return proj @@ -3270,11 +3515,10 @@ def conv_shift_layer(a, b, name=None, layer_attr=None): name=name, type=LayerType.CONV_SHIFT_LAYER, inputs=[a.name, b.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], - size=a.size) + return LayerOutput( + name, LayerType.CONV_SHIFT_LAYER, parents=[a, b], size=a.size) @wrap_name_default() @@ -3282,8 +3526,14 @@ def conv_shift_layer(a, b, name=None, layer_attr=None): @wrap_bias_attr_default() @wrap_act_default(act=LinearActivation()) @layer_support(ERROR_CLIPPING, DROPOUT) -def tensor_layer(a, b, size, act=None, name=None, - param_attr=None, bias_attr=None, layer_attr=None): +def tensor_layer(a, + b, + size, + act=None, + name=None, + param_attr=None, + bias_attr=None, + layer_attr=None): """ This layer performs tensor operation for two input. For example, each sample: @@ -3332,12 +3582,10 @@ def tensor_layer(a, b, size, act=None, name=None, type=LayerType.TENSOR_LAYER, active_type=act.name, bias=ParamAttr.to_bias(bias_attr), - inputs=[Input(a.name, **param_attr.attr), - Input(b.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.TENSOR_LAYER, parents=[a, b], - activation=act, size=size) + inputs=[Input(a.name, **param_attr.attr), Input(b.name)], + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.TENSOR_LAYER, parents=[a, b], activation=act, size=size) @wrap_name_default() @@ -3345,11 +3593,17 @@ def tensor_layer(a, b, size, act=None, name=None, @wrap_bias_attr_default() @wrap_act_default() @layer_support() -def selective_fc_layer(input, select, size, act=None, name=None, +def selective_fc_layer(input, + select, + size, + act=None, + name=None, pass_generation=False, has_selected_colums=True, mul_ratio=0.02, - param_attr=None, bias_attr=None, layer_attr=None): + param_attr=None, + bias_attr=None, + layer_attr=None): """ Selectived fully connected layer. Different from fc_layer, the output of this layer maybe sparse. It requires an additional input to indicate @@ -3399,8 +3653,9 @@ def selective_fc_layer(input, select, size, act=None, name=None, if select.size is not None: assert select.size == size Layer( - inputs=[Input(ipt.name, **attr.attr) for ipt, attr in zip( - input, param_attr)] + [select.name], + inputs=[ + Input(ipt.name, **attr.attr) for ipt, attr in zip(input, param_attr) + ] + [select.name], name=name, type=LayerType.SEL_FC_LAYER, size=size, @@ -3409,11 +3664,13 @@ def selective_fc_layer(input, select, size, act=None, name=None, selective_fc_pass_generation=pass_generation, has_selected_colums=has_selected_colums, selective_fc_full_mul_ratio=mul_ratio, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SEL_FC_LAYER, list(input) + [select], - activation=act, - size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, + LayerType.SEL_FC_LAYER, + list(input) + [select], + activation=act, + size=size) @wrap_name_default() @@ -3442,15 +3699,17 @@ def sampling_id_layer(input, name=None, layer_attr=None): name=name, type=LayerType.SAMPLING_ID_LAYER, inputs=[Input(input.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SAMPLING_ID_LAYER, input, - size=l.config.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SAMPLING_ID_LAYER, input, size=l.config.size) @wrap_name_default() @layer_support() -def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0, +def slope_intercept_layer(input, + name=None, + slope=1.0, + intercept=0.0, layer_attr=None): """ This layer for applying a slope and an intercept to the input @@ -3484,16 +3743,14 @@ def slope_intercept_layer(input, name=None, slope=1.0, intercept=0.0, slope=slope, intercept=intercept, inputs=[Input(input.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.SLOPE_INTERCEPT_LAYER, input, - size=input.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.SLOPE_INTERCEPT_LAYER, input, size=input.size) @wrap_name_default() @layer_support() -def linear_comb_layer(weights, vectors, size=None, name=None, - layer_attr=None): +def linear_comb_layer(weights, vectors, size=None, name=None, layer_attr=None): """ A layer for weighted sum of vectors takes two inputs. - Input: size of weights is M @@ -3543,7 +3800,7 @@ def linear_comb_layer(weights, vectors, size=None, name=None, if vectors.size is not None and weights.size is not None: assert vectors.size % weights.size == 0 if size is None: - size = vectors.size / weights.size + size = vectors.size / weights.size else: assert size == vectors.size / weights.size Layer( @@ -3551,10 +3808,9 @@ def linear_comb_layer(weights, vectors, size=None, name=None, type=LayerType.LINEAR_COMBINATION_LAYER, size=size, inputs=[Input(weights.name), Input(vectors.name)], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.LINEAR_COMBINATION_LAYER, - [weights, vectors], size=size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.LINEAR_COMBINATION_LAYER, [weights, vectors], size=size) convex_comb_layer = linear_comb_layer @@ -3626,21 +3882,23 @@ def block_expand_layer(input, if num_channels is None: assert input.num_filters is not None num_channels = input.num_filters - l = Layer(name=name, - inputs=Input(input.name, - block_expand=BlockExpand(channels=num_channels, - block_x=block_x, - block_y=block_y, - stride_x=stride_x, - stride_y=stride_y, - padding_x=padding_x, - padding_y=padding_y)), - type=LayerType.BLOCK_EXPAND, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - - return LayerOutput(name, LayerType.BLOCK_EXPAND, parents=[input], - size=l.config.size) + l = Layer( + name=name, + inputs=Input( + input.name, + block_expand=BlockExpand( + channels=num_channels, + block_x=block_x, + block_y=block_y, + stride_x=stride_x, + stride_y=stride_y, + padding_x=padding_x, + padding_y=padding_y)), + type=LayerType.BLOCK_EXPAND, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + + return LayerOutput( + name, LayerType.BLOCK_EXPAND, parents=[input], size=l.config.size) @wrap_name_default() @@ -3701,19 +3959,24 @@ def maxout_layer(input, assert input.num_filters is not None num_channels = input.num_filters assert num_channels % groups == 0 - l = Layer(name=name, - inputs=Input(input.name, - maxout=MaxOut(channels=num_channels, - groups=groups)), - type=LayerType.MAXOUT, - **ExtraLayerAttribute.to_kwargs(layer_attr)) - return LayerOutput(name, LayerType.MAXOUT, parents=[input], - size=l.config.size) + l = Layer( + name=name, + inputs=Input( + input.name, maxout=MaxOut( + channels=num_channels, groups=groups)), + type=LayerType.MAXOUT, + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.MAXOUT, parents=[input], size=l.config.size) @wrap_name_default() @layer_support() -def ctc_layer(input, label, size=None, name=None, norm_by_times=False, +def ctc_layer(input, + label, + size=None, + name=None, + norm_by_times=False, layer_attr=None): """ Connectionist Temporal Classification (CTC) is designed for temporal @@ -3769,15 +4032,19 @@ def ctc_layer(input, label, size=None, name=None, norm_by_times=False, size=size, norm_by_times=norm_by_times, inputs=[input.name, label.name], - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) return LayerOutput(name, LayerType.CTC_LAYER, [input, label], size=size) @wrap_name_default() @wrap_param_attr_default() @layer_support() -def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, +def crf_layer(input, + label, + size=None, + weight=None, + param_attr=None, + name=None, layer_attr=None): """ A layer for calculating the cost of sequential conditional random @@ -3819,8 +4086,7 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, else: assert size == input.size - ipts = [Input(input.name, **param_attr.attr), - Input(label.name)] + ipts = [Input(input.name, **param_attr.attr), Input(label.name)] if weight is not None: ipts.append(Input(weight.name)) @@ -3829,8 +4095,7 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, type=LayerType.CRF_LAYER, size=size, inputs=ipts, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) parents = [input, label] if weight is not None: parents.append(weight) @@ -3843,7 +4108,11 @@ def crf_layer(input, label, size=None, weight=None, param_attr=None, name=None, @wrap_name_default() @wrap_param_attr_default() @layer_support() -def crf_decoding_layer(input, size, label=None, param_attr=None, name=None, +def crf_decoding_layer(input, + size, + label=None, + param_attr=None, + name=None, layer_attr=None): """ A layer for calculating the decoding sequence of sequential conditional @@ -3880,8 +4149,7 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None, type=LayerType.CRF_DECODING_LAYER, size=size, inputs=ipts, - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) + **ExtraLayerAttribute.to_kwargs(layer_attr)) parents = [input] if label is not None: parents.append(label) @@ -3890,12 +4158,19 @@ def crf_decoding_layer(input, size, label=None, param_attr=None, name=None, # classes. return LayerOutput(name, LayerType.CRF_DECODING_LAYER, parents, size=1) + @wrap_bias_attr_default(has_bias=True) @wrap_name_default() @layer_support() -def nce_layer(input, label, num_classes, weight=None, - num_neg_samples=10, neg_distribution=None, - name=None, bias_attr=None, layer_attr=None): +def nce_layer(input, + label, + num_classes, + weight=None, + num_neg_samples=10, + neg_distribution=None, + name=None, + bias_attr=None, + layer_attr=None): """ Noise-contrastive estimation. Implements the method in the following paper: @@ -3964,10 +4239,10 @@ def nce_layer(input, label, num_classes, weight=None, num_neg_samples=num_neg_samples, inputs=ipts_for_layer, bias=ParamAttr.to_bias(bias_attr), - **ExtraLayerAttribute.to_kwargs(layer_attr) - ) - return LayerOutput(name, LayerType.NCE_LAYER, parents=parents, - size=l.config.size) + **ExtraLayerAttribute.to_kwargs(layer_attr)) + return LayerOutput( + name, LayerType.NCE_LAYER, parents=parents, size=l.config.size) + """ following are cost Layers. @@ -3976,7 +4251,13 @@ def nce_layer(input, label, num_classes, weight=None, @wrap_name_default() @layer_support() -def rank_cost(left, right, label, weight=None, name=None, coeff=1.0, layer_attr=None): +def rank_cost(left, + right, + label, + weight=None, + name=None, + coeff=1.0, + layer_attr=None): """ A cost Layer for learning to rank using gradient descent. Details can refer to `papers