From 1d5492e5ef51b2d7eb7066e35d8a82934ca390e2 Mon Sep 17 00:00:00 2001 From: Julia Beliaeva Date: Mon, 15 Nov 2021 01:07:55 +0300 Subject: [PATCH] Add transposed convolution layers --- .../kotlinx/dl/api/core/layer/Layer.kt | 8 + .../core/layer/convolutional/AbstractConv.kt | 2 +- .../dl/api/core/layer/convolutional/Conv1D.kt | 72 +++++-- .../layer/convolutional/Conv1DTranspose.kt | 103 ++++++++++ .../layer/convolutional/Conv2DTranspose.kt | 126 ++++++++++++ .../layer/convolutional/Conv3DTranspose.kt | 88 +++++++++ .../core/layer/convolutional/ConvTranspose.kt | 92 +++++++++ .../kotlinx/dl/api/core/shape/ConvUtil.kt | 52 ++++- .../dl/api/core/util/nameConventions.kt | 8 +- .../dl/api/inference/keras/KerasConstants.kt | 5 +- .../dl/api/inference/keras/ModelLoader.kt | 73 +++++++ .../dl/api/inference/keras/ModelSaver.kt | 72 ++++++- .../dl/api/inference/keras/WeightMappings.kt | 32 +++- .../api/inference/keras/config/LayerConfig.kt | 4 +- .../kotlinx/dl/api/core/layer/Conv1DTest.kt | 30 +-- .../dl/api/core/layer/Conv1DTransposeTest.kt | 144 ++++++++++++++ .../kotlinx/dl/api/core/layer/Conv2DTest.kt | 38 ++-- .../dl/api/core/layer/Conv2DTransposeTest.kt | 179 ++++++++++++++++++ .../kotlinx/dl/api/core/layer/Conv3DTest.kt | 47 +++-- .../dl/api/core/layer/Conv3DTransposeTest.kt | 161 ++++++++++++++++ .../dl/api/core/layer/ConvLayerTest.kt | 3 +- .../keras/ConvTransposePersistenceTest.kt | 121 ++++++++++++ 22 files changed, 1386 insertions(+), 74 deletions(-) create mode 100644 api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv1DTranspose.kt create mode 100644 api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv2DTranspose.kt create mode 100644 api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv3DTranspose.kt create mode 100644 api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/ConvTranspose.kt create mode 100644 api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv1DTransposeTest.kt create mode 100644 api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv2DTransposeTest.kt create mode 100644 api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv3DTransposeTest.kt create mode 100644 api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ConvTransposePersistenceTest.kt diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Layer.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Layer.kt index 64af49fa0..49b4e54a3 100644 --- a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Layer.kt +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Layer.kt @@ -159,4 +159,12 @@ internal fun IntArray.toLongArray(): LongArray { 1 -> longArrayOf(this[0].toLong()) else -> LongArray(size) { this[it].toLong() } } +} + +internal fun LongArray.toIntArray(): IntArray { + return when (size) { + 0 -> intArrayOf() + 1 -> intArrayOf(this[0].toInt()) + else -> IntArray(size) { this[it].toInt() } + } } \ No newline at end of file diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/AbstractConv.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/AbstractConv.kt index 7ae2c967f..4d5e1f9eb 100644 --- a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/AbstractConv.kt +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/AbstractConv.kt @@ -59,7 +59,7 @@ public abstract class AbstractConv( protected abstract val biasRegularizer: Regularizer? protected abstract val activityRegularizer: Regularizer? protected abstract val padding: ConvPadding - protected abstract val useBias: Boolean + internal abstract val useBias: Boolean /** Tensor with kernel weights */ protected lateinit var kernel: KVariable diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv1D.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv1D.kt index c6c7b46ad..4c267dcaf 100644 --- a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv1D.kt +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv1D.kt @@ -12,6 +12,7 @@ import org.jetbrains.kotlinx.dl.api.core.initializer.Initializer import org.jetbrains.kotlinx.dl.api.core.layer.requireArraySize import org.jetbrains.kotlinx.dl.api.core.layer.toLongList import org.jetbrains.kotlinx.dl.api.core.regularizer.Regularizer +import org.jetbrains.kotlinx.dl.api.core.shape.TensorShape import org.jetbrains.kotlinx.dl.api.core.shape.convOutputLength import org.jetbrains.kotlinx.dl.api.core.util.convBiasVarName import org.jetbrains.kotlinx.dl.api.core.util.convKernelVarName @@ -21,8 +22,6 @@ import org.tensorflow.op.Ops import org.tensorflow.op.core.Squeeze import org.tensorflow.op.nn.Conv2d -private const val EXTRA_DIM = 1L - /** * 1D convolution layer (e.g. convolution over audio data). * @@ -79,10 +78,6 @@ public class Conv1D( override val kernelSize: IntArray = intArrayOf(kernelLength) - /** Axis of height for which the extra dimension is added (unsqueezed) before actual - * convolution operation and the output from actual implementation are squeezed. */ - private val squeezeAxis = Squeeze.axis(listOf(EXTRA_DIM)) - override fun kernelVarName(name: String): String = convKernelVarName(name, dim = 1) override fun biasVarName(name: String): String = convBiasVarName(name, dim = 1) @@ -91,15 +86,13 @@ public class Conv1D( tf: Ops, input: Operand ): Operand { - val expandedInput = tf.expandDims(input, tf.constant(EXTRA_DIM)) - val expandedKernel = tf.expandDims(kernel.variable, tf.constant(EXTRA_DIM - 1)) - val expandedStrides = intArrayOf(strides[0], 1, strides[1], strides[2]) - val expandedDilations = intArrayOf(dilations[0], 1, dilations[1], dilations[2]) - val options = Conv2d.dilations(expandedDilations.toLongList()).dataFormat("NHWC") - val result = tf.nn.conv2d( - expandedInput, expandedKernel, expandedStrides.toLongList(), padding.paddingName, options - ) - return tf.squeeze(result, squeezeAxis) + return tf.withExpandedDimensions(input) { expandedInput -> + val options = Conv2d.dilations(expand(dilations).toLongList()).dataFormat("NHWC") + return@withExpandedDimensions tf.nn.conv2d( + expandedInput, tf.expandKernel(kernel.variable), expand(strides).toLongList(), + padding.paddingName, options + ) + } } protected override fun defineOutputShape(inputShape: Shape): Shape { @@ -122,4 +115,53 @@ public class Conv1D( "dilation=${dilations.contentToString()}, activation=$activation, kernelInitializer=$kernelInitializer, " + "biasInitializer=$biasInitializer, kernelShape=${kernel.shape}, biasShape=${bias?.shape}, padding=$padding, " + "biasRegularizer=$biasRegularizer, kernelRegularizer=$kernelRegularizer, activityRegularizer=$activityRegularizer)" + + internal companion object { + internal const val EXTRA_DIM = 1 + + /** Axis of height for which the extra dimension is added (unsqueezed) before actual + * convolution operation and the output from actual implementation are squeezed. */ + private val squeezeAxis = Squeeze.axis(listOf(EXTRA_DIM.toLong())) + + internal fun expandKernel(kernel: IntArray): IntArray { + return kernel.withAdded(EXTRA_DIM - 1, 1) + } + + internal fun Ops.expandKernel(kernel: Operand): Operand { + return expandDims(kernel, constant(EXTRA_DIM - 1)) + } + + internal fun TensorShape.expand(): TensorShape { + return TensorShape(dims().withAdded(EXTRA_DIM, 1)) + } + + internal fun expand(array: IntArray): IntArray { + return array.withAdded(EXTRA_DIM, 1) + } + + /** + * Adds an extra dimension to the input, performs the provided operation + * and squeezes the result by removing the dimension added previously. + * This allows to perform 2D operations on 1D inputs. + */ + internal fun Ops.withExpandedDimensions(input: Operand, + operation: (Operand) -> Operand + ): Operand { + val expandedInput = expandDims(input, constant(EXTRA_DIM)) + val expandedOutput = operation(expandedInput) + return squeeze(expandedOutput, squeezeAxis) + } + + internal fun LongArray.withAdded(position: Int, element: Long): LongArray { + return toMutableList().apply { add(position, element) }.toLongArray() + } + + internal fun IntArray.withAdded(position: Int, element: Int): IntArray { + return toMutableList().apply { add(position, element) }.toIntArray() + } + + internal fun IntArray.withAdded(position: Int, elements: List): IntArray { + return toMutableList().apply { addAll(position, elements) }.toIntArray() + } + } } diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv1DTranspose.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv1DTranspose.kt new file mode 100644 index 000000000..f0cf14cc7 --- /dev/null +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv1DTranspose.kt @@ -0,0 +1,103 @@ +/* + * Copyright 2021-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. + */ + +package org.jetbrains.kotlinx.dl.api.core.layer.convolutional + +import org.jetbrains.kotlinx.dl.api.core.activation.Activations +import org.jetbrains.kotlinx.dl.api.core.initializer.HeNormal +import org.jetbrains.kotlinx.dl.api.core.initializer.HeUniform +import org.jetbrains.kotlinx.dl.api.core.initializer.Initializer +import org.jetbrains.kotlinx.dl.api.core.layer.NoGradients +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv1D.Companion.EXTRA_DIM +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv1D.Companion.expand +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv1D.Companion.expandKernel +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv1D.Companion.withAdded +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv1D.Companion.withExpandedDimensions +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv2DTranspose.Companion.withStandardPadding +import org.jetbrains.kotlinx.dl.api.core.layer.requireArraySize +import org.jetbrains.kotlinx.dl.api.core.layer.toLongList +import org.jetbrains.kotlinx.dl.api.core.regularizer.Regularizer +import org.tensorflow.Operand +import org.tensorflow.op.Ops + +/** + * 1D convolution transpose layer. + * + * This is an operation going in the opposite direction of a normal convolution: + * it transforms a tensor shaped like an output of some convolution into tensor that has the shape of the input. + * + * This layer expects input data of size `(N, L, C)` where + * ``` + * N - batch size + * L - length of signal sequence + * C - number of channels + * ``` + * + * Note: dilation values greater than 1 are not supported on cpu + * (see https://github.com/tensorflow/tensorflow/issues/28264). + * + * @property [filters] dimensionality of the output space (i.e. the number of filters in the convolution) + * @property [kernelLength] size of the convolutional kernel (one number) + * @property [strides] strides of the convolution for each dimension of the input tensor (three numbers) + * @property [dilations] dilations of the convolution for each dimension of the input tensor (three numbers). + * Currently, dilation values greater than 1 are not supported on cpu. + * @property [activation] activation function + * @property [kernelInitializer] initializer for the kernel + * @property [biasInitializer] initializer for the bias + * @property [kernelRegularizer] regularizer for the kernel + * @property [biasRegularizer] regularizer for the bias + * @property [activityRegularizer] regularizer function applied to the output of the layer + * @property [padding] type of padding to use + * @property [outputPadding] the amount of explicit padding to use (six numbers: two for each dimension). + * @property [useBias] a flag that specifies if the bias should be used + * @param [name] custom layer name + */ +public class Conv1DTranspose( + public override val filters: Int = 3, + public val kernelLength: Int = 3, + public override val strides: IntArray = intArrayOf(1, 1, 1), + public override val dilations: IntArray = intArrayOf(1, 1, 1), + public override val activation: Activations = Activations.Relu, + public override val kernelInitializer: Initializer = HeNormal(), + public override val biasInitializer: Initializer = HeUniform(), + public override val kernelRegularizer: Regularizer? = null, + public override val biasRegularizer: Regularizer? = null, + public override val activityRegularizer: Regularizer? = null, + public override val padding: ConvPadding = ConvPadding.SAME, + public override val outputPadding: IntArray? = null, + public override val useBias: Boolean = true, + name: String = "" +) : ConvTranspose(dimensions = 1, name), NoGradients { + + init { + requireArraySize(strides, dimensions + 2, "strides") + requireArraySize(dilations, dimensions + 2, "dilations") + if (outputPadding != null) requireArraySize(outputPadding, 2 * (dimensions + 2), "outputPadding") + isTrainable = false + } + + override val kernelSize: IntArray = intArrayOf(kernelLength) + + override fun convImplementation(tf: Ops, input: Operand): Operand { + return tf.withExpandedDimensions(input) { expandedInput -> + val expandedOutputPadding = outputPadding?.withAdded(EXTRA_DIM * 2, listOf(0, 0)) + return@withExpandedDimensions tf.nn.conv2dBackpropInput( + tf.shapeWithDynamicBatchSize(outputShape.expand(), input), + tf.expandKernel(kernel.variable), + expandedInput, + expand(strides).toLongList(), + if (outputPadding != null) Conv2DTranspose.EXPLICIT else padding.paddingName, + *Conv2DTranspose.buildOptions( + expand(dilations), + expandedOutputPadding?.withStandardPadding( + padding, + expandKernel(kernelSize), + expand(dilations) + ) + ) + ) + } + } +} \ No newline at end of file diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv2DTranspose.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv2DTranspose.kt new file mode 100644 index 000000000..3297d9297 --- /dev/null +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv2DTranspose.kt @@ -0,0 +1,126 @@ +/* + * Copyright 2021-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. + */ + +package org.jetbrains.kotlinx.dl.api.core.layer.convolutional + +import org.jetbrains.kotlinx.dl.api.core.activation.Activations +import org.jetbrains.kotlinx.dl.api.core.initializer.HeNormal +import org.jetbrains.kotlinx.dl.api.core.initializer.HeUniform +import org.jetbrains.kotlinx.dl.api.core.initializer.Initializer +import org.jetbrains.kotlinx.dl.api.core.layer.NoGradients +import org.jetbrains.kotlinx.dl.api.core.layer.requireArraySize +import org.jetbrains.kotlinx.dl.api.core.layer.toLongList +import org.jetbrains.kotlinx.dl.api.core.regularizer.Regularizer +import org.jetbrains.kotlinx.dl.api.core.shape.convTransposeSingleSidePadding +import org.tensorflow.Operand +import org.tensorflow.op.Ops +import org.tensorflow.op.nn.Conv2dBackpropInput + +/** + * 2D convolution transpose layer. + * + * This is an operation going in the opposite direction of a normal convolution: + * it transforms a tensor shaped like an output of some convolution into tensor that has the shape of the input. + * + * This layer expects input data of size `(N, H, W, C)` where + * ``` + * N - batch size + * H - height + * W - width + * C - number of channels + * ``` + * + * Note: dilation values greater than 1 are not supported on cpu + * (see https://github.com/tensorflow/tensorflow/issues/28264). + * + * @property [filters] dimensionality of the output space (i.e. the number of filters in the convolution) + * @property [kernelSize] size of the convolutional kernel (two numbers) + * @property [strides] strides of the convolution for each dimension of the input tensor (four numbers) + * @property [dilations] dilations of the convolution for each dimension of the input tensor (four numbers). + * Currently, dilation values greater than 1 are not supported on cpu. + * @property [activation] activation function + * @property [kernelInitializer] initializer for the kernel + * @property [biasInitializer] initializer for the bias + * @property [kernelRegularizer] regularizer for the kernel + * @property [biasRegularizer] regularizer for the bias + * @property [activityRegularizer] regularizer function applied to the output of the layer + * @property [padding] type of padding to use + * @property [outputPadding] the amount of explicit padding to use (eight numbers: two for each dimension). + * @property [useBias] a flag that specifies if the bias should be used + * @param [name] custom layer name + */ +public class Conv2DTranspose( + public override val filters: Int = 3, + public override val kernelSize: IntArray = intArrayOf(3, 3), + public override val strides: IntArray = intArrayOf(1, 1, 1, 1), + public override val dilations: IntArray = intArrayOf(1, 1, 1, 1), + public override val activation: Activations = Activations.Relu, + public override val kernelInitializer: Initializer = HeNormal(), + public override val biasInitializer: Initializer = HeUniform(), + public override val kernelRegularizer: Regularizer? = null, + public override val biasRegularizer: Regularizer? = null, + public override val activityRegularizer: Regularizer? = null, + public override val padding: ConvPadding = ConvPadding.SAME, + public override val outputPadding: IntArray? = null, + public override val useBias: Boolean = true, + name: String = "" +) : ConvTranspose(dimensions = 2, name), NoGradients { + + init { + requireArraySize(kernelSize, dimensions, "kernelSize") + requireArraySize(strides, dimensions + 2, "strides") + requireArraySize(dilations, dimensions + 2, "dilations") + if (outputPadding != null) requireArraySize(outputPadding, 2 * (dimensions + 2), "outputPadding") + isTrainable = false + } + + override fun convImplementation(tf: Ops, input: Operand): Operand { + return tf.nn.conv2dBackpropInput( + tf.shapeWithDynamicBatchSize(outputShape, input), + kernel.variable, + input, + strides.toLongList(), + if (outputPadding != null) EXPLICIT else padding.paddingName, + *buildOptions( + dilations, + outputPadding?.withStandardPadding( + padding, + kernelSize, + dilations + ) + ) + ) + } + + internal companion object { + internal const val EXPLICIT = "EXPLICIT" + + /** + * Combines explicitly provided padding value with the standard padding from the provided padding method. + * This is needed since [org.tensorflow.op.NnOps.conv2dBackpropInput] function does not support specifying + * both padding method and explicit output padding at the same time. + */ + internal fun IntArray.withStandardPadding(padding: ConvPadding, + kernelSize: IntArray, + dilations: IntArray + ): IntArray { + val withStandardPadding = kernelSize.indices.flatMap { dim -> + listOf( + convTransposeSingleSidePadding(padding, this[2 * dim], kernelSize[dim], dilations[dim + 1]), + convTransposeSingleSidePadding(padding, this[2 * dim + 1], kernelSize[dim], dilations[dim + 1]) + ) + } + return intArrayOf(0, 0, *(withStandardPadding.toIntArray()), 0, 0) + } + + internal fun buildOptions(dilations: IntArray, outputPadding: IntArray?): Array { + val options = mutableListOf(Conv2dBackpropInput.dilations(dilations.toLongList())) + if (outputPadding != null) { + options.add(Conv2dBackpropInput.explicitPaddings(outputPadding.toLongList())) + } + return options.map { it.dataFormat("NHWC") }.toTypedArray() + } + } +} \ No newline at end of file diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv3DTranspose.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv3DTranspose.kt new file mode 100644 index 000000000..2caba0f29 --- /dev/null +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/Conv3DTranspose.kt @@ -0,0 +1,88 @@ +/* + * Copyright 2021-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. + */ + +package org.jetbrains.kotlinx.dl.api.core.layer.convolutional + +import org.jetbrains.kotlinx.dl.api.core.activation.Activations +import org.jetbrains.kotlinx.dl.api.core.initializer.HeNormal +import org.jetbrains.kotlinx.dl.api.core.initializer.HeUniform +import org.jetbrains.kotlinx.dl.api.core.initializer.Initializer +import org.jetbrains.kotlinx.dl.api.core.layer.NoGradients +import org.jetbrains.kotlinx.dl.api.core.layer.requireArraySize +import org.jetbrains.kotlinx.dl.api.core.layer.toLongList +import org.jetbrains.kotlinx.dl.api.core.regularizer.Regularizer +import org.tensorflow.Operand +import org.tensorflow.op.Ops +import org.tensorflow.op.nn.Conv3dBackpropInput + +/** + * 3D convolution transpose layer. + * + * This is an operation going in the opposite direction of a normal convolution: + * it transforms a tensor shaped like an output of some convolution into tensor that has the shape of the input. + * + * This layer expects input data of size `(N, D, H, W, C)` where + * ``` + * N - batch size + * D - depth + * H - height + * W - width + * C - number of channels + * ``` + * + * Note: providing explicit output padding is currently not supported. + * Dilation values greater than 1 are not supported on cpu. + * + * @param [filters] dimensionality of the output space (i.e. the number of filters in the convolution) + * @property [kernelSize] size of the convolutional kernel (three numbers) + * @property [strides] strides of the convolution for each dimension of the input tensor (five numbers) + * @property [dilations] dilations of the convolution for each dimension of the input tensor (five numbers). + * Currently, dilation values greater than 1 are not supported on cpu. + * @param [activation] activation function + * @param [kernelInitializer] initializer for the kernel + * @param [biasInitializer] initializer for the bias + * @param [kernelRegularizer] regularizer for the kernel + * @param [biasRegularizer] regularizer for the bias + * @param [activityRegularizer] regularizer function applied to the output of the layer + * @param [padding] type of padding to use + * @param [useBias] a flag that specifies if the bias should be used + * @param [name] custom layer name + */ +public class Conv3DTranspose( + public override val filters: Int = 3, + public override val kernelSize: IntArray = intArrayOf(3, 3, 3), + public override val strides: IntArray = intArrayOf(1, 1, 1, 1, 1), + public override val dilations: IntArray = intArrayOf(1, 1, 1, 1, 1), + public override val activation: Activations = Activations.Relu, + public override val kernelInitializer: Initializer = HeNormal(), + public override val biasInitializer: Initializer = HeUniform(), + public override val kernelRegularizer: Regularizer? = null, + public override val biasRegularizer: Regularizer? = null, + public override val activityRegularizer: Regularizer? = null, + public override val padding: ConvPadding = ConvPadding.SAME, + public override val useBias: Boolean = true, + name: String = "" +) : ConvTranspose(dimensions = 3, name), NoGradients { + init { + requireArraySize(kernelSize, dimensions, "kernelSize") + requireArraySize(strides, dimensions + 2, "strides") + requireArraySize(dilations, dimensions + 2, "dilations") + isTrainable = false + } + + override val outputPadding: IntArray? get() = null + + override fun convImplementation(tf: Ops, input: Operand): Operand { + val options = Conv3dBackpropInput.dilations(dilations.toLongList()).dataFormat("NDHWC") + return tf.nn.conv3dBackpropInput( + tf.shapeWithDynamicBatchSize(outputShape, input), + kernel.variable, + input, + strides.toLongList(), + padding.paddingName, + options + ) + } +} \ No newline at end of file diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/ConvTranspose.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/ConvTranspose.kt new file mode 100644 index 000000000..cabda32b9 --- /dev/null +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/convolutional/ConvTranspose.kt @@ -0,0 +1,92 @@ +/* + * Copyright 2021-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. + */ + +package org.jetbrains.kotlinx.dl.api.core.layer.convolutional + +import org.jetbrains.kotlinx.dl.api.core.layer.toLongArray +import org.jetbrains.kotlinx.dl.api.core.shape.TensorShape +import org.jetbrains.kotlinx.dl.api.core.shape.convTransposeOutputLength +import org.jetbrains.kotlinx.dl.api.core.shape.shapeFromDims +import org.jetbrains.kotlinx.dl.api.core.util.convTransposeBiasVarName +import org.jetbrains.kotlinx.dl.api.core.util.convTransposeKernelVarName +import org.tensorflow.Operand +import org.tensorflow.Shape +import org.tensorflow.op.Ops + +/** + * A base class for defining transposed convolution layers (sometimes called deconvolution) of different dimensions. + * + * This is an operation going in the opposite direction of a normal convolution: + * it transforms a tensor shaped like an output of some convolution into tensor that has the shape of the input. + * + * @property [dimensions] dimensionality of this convolution operation + * @property [outputPadding] the amount of padding to use for each dimension of the input tensor + * @param [name] custom layer name + */ +public abstract class ConvTranspose( + public val dimensions: Int, + name: String = "" +) : AbstractConv(name = name) { + + protected abstract val outputPadding: IntArray? + + override fun defineOutputShape(inputShape: Shape): Shape { + val shapes = (kernelSize.indices).map { + convTransposeOutputLength( + inputShape.size(it + 1), + kernelSize[it], + padding, + outputPadding?.get(2 * (it + 1)), + outputPadding?.get(2 * (it + 1) + 1), + strides[it + 1], + dilations[it + 1] + ) + } + return Shape.make(inputShape.size(0), *(shapes + filters.toLong()).toLongArray()) + } + + override fun kernelVarName(name: String): String = convTransposeKernelVarName(name, dimensions) + override fun biasVarName(name: String): String = convTransposeBiasVarName(name, dimensions) + + protected override fun computeKernelShape(numberOfChannels: Long): Shape { + return shapeFromDims(*kernelSize.toLongArray(), filters.toLong(), numberOfChannels) + } + + override fun toString(): String { + return "Conv${dimensions}DTranspose(" + + "filters=$filters, " + + "kernelSize=${kernelSize.contentToString()}, " + + "kernelShape=${kernel.shape}, " + + "biasShape=${bias?.shape}, " + + "strides=${strides.contentToString()}, " + + "dilations=${dilations.contentToString()}, " + + "activation=$activation, " + + "kernelInitializer=$kernelInitializer, " + + "biasInitializer=$biasInitializer, " + + "biasRegularizer=$biasRegularizer, " + + "kernelRegularizer=$kernelRegularizer, " + + "activityRegularizer=$activityRegularizer, " + + "padding=$padding, " + + "outputPadding=${outputPadding?.contentToString()} " + + ")" + } + + internal companion object { + /** + * Creates an integer vector with the contents of [tensorShape], except for the first dimension (batch size): + * batch size from the [input] is used instead. + * This is needed as [org.tensorflow.op.NnOps.conv2dBackpropInput] and [org.tensorflow.op.NnOps.conv3dBackpropInput] + * need to have an exact shape provided, including batch size. + * Typically, when the layer is built, batch size is not known and [tensorShape] contains a "-1" instead. + * This why here a first value of the [input] shape is used, which is going to be known at runtime. + * See also [https://github.com/tensorflow/tensorflow/issues/833](https://github.com/tensorflow/tensorflow/issues/833) + */ + internal fun Ops.shapeWithDynamicBatchSize(tensorShape: TensorShape, input: Operand): Operand { + val batchSize = squeeze(slice(shape(input), constant(intArrayOf(0)), constant(intArrayOf(1)))) + val otherDims = tensorShape.dims().toList().drop(1).map { constant(it.toInt()) } + return stack(listOf(batchSize) + otherDims) + } + } +} \ No newline at end of file diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/shape/ConvUtil.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/shape/ConvUtil.kt index 6eb7e09eb..67920e488 100644 --- a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/shape/ConvUtil.kt +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/shape/ConvUtil.kt @@ -1,14 +1,27 @@ /* - * Copyright 2020 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Copyright 2020-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. */ package org.jetbrains.kotlinx.dl.api.core.shape import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.ConvPadding +import java.lang.Integer.max + +private fun dilatedFilterSize(filterSize: Int, dilation: Int): Int { + return filterSize + (filterSize - 1) * (dilation - 1) +} + +private fun ConvPadding.value(filterSize: Int): Int { + return when (this) { + ConvPadding.VALID -> 0 + ConvPadding.SAME -> filterSize - 1 + ConvPadding.FULL -> 2 * filterSize - 1 + } +} /** - * Calculates output length. + * Calculates output length after applying convolution operation on a single axis. */ internal fun convOutputLength( inputLength: Long, @@ -17,11 +30,34 @@ internal fun convOutputLength( stride: Int, dilation: Int = 1 ): Long { - val dilatedFilterSize = filterSize + (filterSize - 1) * (dilation - 1) - val outputLength = when (padding) { - ConvPadding.SAME -> inputLength - ConvPadding.VALID -> inputLength - dilatedFilterSize + 1 - ConvPadding.FULL -> inputLength + dilatedFilterSize - 1 - } + val dilatedFilterSize = dilatedFilterSize(filterSize, dilation) + val outputLength = inputLength - filterSize + 1 + padding.value(dilatedFilterSize) return ((outputLength + stride - 1).toFloat() / stride).toLong() } + +/** + * Calculates output length after applying transposed convolution operation on a single axis. + */ +internal fun convTransposeOutputLength( + inputLength: Long, + filterSize: Int, + padding: ConvPadding, + outputPaddingStart: Int?, + outputPaddingEnd: Int?, + stride: Int, + dilation: Int +): Long { + val dilatedFilterSize = dilatedFilterSize(filterSize, dilation) + val totalPadding = (outputPaddingStart ?: 0) + (outputPaddingEnd ?: 0) - padding.value(dilatedFilterSize) + return (inputLength - 1) * stride + dilatedFilterSize + totalPadding +} + +internal fun convTransposeSingleSidePadding(padding: ConvPadding, + outputPadding: Int, + filterSize: Int, + dilation: Int +): Int { + val dilatedKernelSize = dilatedFilterSize(filterSize, dilation) + val automaticPadding = padding.value(dilatedKernelSize) / 2 + return max(0, outputPadding - automaticPadding) +} \ No newline at end of file diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/util/nameConventions.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/util/nameConventions.kt index 12fecc410..919b216ce 100644 --- a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/util/nameConventions.kt +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/core/util/nameConventions.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Copyright 2020-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. */ @@ -30,6 +30,12 @@ internal fun convBiasVarName(name: String, dim: Int): String = layerVarName(name /** Default Conv kernel variable name in TensorFlow graph, based on layer's name. */ internal fun convKernelVarName(name: String, dim: Int): String = layerVarName(name, "conv${dim}d_kernel") +/** Default Conv transpose bias variable name in TensorFlow graph, based on layer's name. */ +internal fun convTransposeBiasVarName(name: String, dim: Int): String = layerVarName(name, "conv${dim}d_transpose_bias") + +/** Default Conv transpose kernel variable name in TensorFlow graph, based on layer's name. */ +internal fun convTransposeKernelVarName(name: String, dim: Int): String = layerVarName(name, "conv${dim}d_transpose_kernel") + /** Default DepthwiseConv2d bias variable name in TensorFlow graph, based on layer's name. */ internal fun depthwiseConv2dBiasVarName(name: String): String = layerVarName(name, "depthwise_conv2d_bias") diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/KerasConstants.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/KerasConstants.kt index 8e6432f41..fffdc390c 100644 --- a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/KerasConstants.kt +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/KerasConstants.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Copyright 2020-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. */ @@ -15,6 +15,9 @@ internal const val LAYER_PERMUTE: String = "Permute" internal const val LAYER_CONV1D: String = "Conv1D" internal const val LAYER_CONV2D: String = "Conv2D" internal const val LAYER_CONV3D: String = "Conv3D" +internal const val LAYER_CONV1D_TRANSPOSE: String = "Conv1DTranspose" +internal const val LAYER_CONV2D_TRANSPOSE: String = "Conv2DTranspose" +internal const val LAYER_CONV3D_TRANSPOSE: String = "Conv3DTranspose" internal const val LAYER_DEPTHWISE_CONV2D: String = "DepthwiseConv2D" internal const val LAYER_SEPARABLE_CONV2D: String = "SeparableConv2D" // Pooling layers diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ModelLoader.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ModelLoader.kt index 461a490a2..551de877c 100644 --- a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ModelLoader.kt +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ModelLoader.kt @@ -81,6 +81,9 @@ private fun convertToLayer( LAYER_CONV1D -> createConv1DLayer(kerasLayer.config!!, kerasLayer.config.name!!) LAYER_CONV2D -> createConv2DLayer(kerasLayer.config!!, kerasLayer.config.name!!) LAYER_CONV3D -> createConv3DLayer(kerasLayer.config!!, kerasLayer.config.name!!) + LAYER_CONV1D_TRANSPOSE -> createConv1DTransposeLayer(kerasLayer.config!!, kerasLayer.config.name!!) + LAYER_CONV2D_TRANSPOSE -> createConv2DTransposeLayer(kerasLayer.config!!, kerasLayer.config.name!!) + LAYER_CONV3D_TRANSPOSE -> createConv3DTransposeLayer(kerasLayer.config!!, kerasLayer.config.name!!) LAYER_DEPTHWISE_CONV2D -> createDepthwiseConv2DLayer(kerasLayer.config!!, kerasLayer.config.name!!) LAYER_SEPARABLE_CONV2D -> createSeparableConv2DLayer(kerasLayer.config!!, kerasLayer.config.name!!) // Pooling layers @@ -727,6 +730,76 @@ private fun createConv3DLayer(config: LayerConfig, name: String): Layer { ) } +private fun createConv1DTransposeLayer(config: LayerConfig, name: String): Layer { + return Conv1DTranspose( + filters = config.filters!!, + kernelLength = config.kernel_size!![0], + strides = intArrayOf(1, config.strides!![0], 1), + dilations = intArrayOf(1, config.dilation_rate!![0], 1), + activation = convertToActivation(config.activation!!), + kernelInitializer = convertToInitializer(config.kernel_initializer!!), + biasInitializer = convertToInitializer(config.bias_initializer!!), + kernelRegularizer = convertToRegularizer(config.kernel_regularizer), + biasRegularizer = convertToRegularizer(config.bias_regularizer), + activityRegularizer = convertToRegularizer(config.activity_regularizer), + padding = convertPadding(config.padding!!), + outputPadding = config.output_padding?.convertToOutputPadding(), + useBias = config.use_bias!!, + name = name + ) +} + +private fun createConv2DTransposeLayer(config: LayerConfig, name: String): Layer { + val kernelSize = config.kernel_size!!.toIntArray() + val strides = config.strides!!.toIntArray() + val dilation = config.dilation_rate!!.toIntArray() + return Conv2DTranspose( + filters = config.filters!!, + kernelSize = kernelSize, + strides = intArrayOf(1, *strides, 1), + dilations = intArrayOf(1, *dilation, 1), + activation = convertToActivation(config.activation!!), + kernelInitializer = convertToInitializer(config.kernel_initializer!!), + biasInitializer = convertToInitializer(config.bias_initializer!!), + kernelRegularizer = convertToRegularizer(config.kernel_regularizer), + biasRegularizer = convertToRegularizer(config.bias_regularizer), + activityRegularizer = convertToRegularizer(config.activity_regularizer), + padding = convertPadding(config.padding!!), + outputPadding = config.output_padding?.convertToOutputPadding(), + useBias = config.use_bias!!, + name = name + ) +} + +private fun createConv3DTransposeLayer(config: LayerConfig, name: String): Layer { + val kernelSize = config.kernel_size!!.toIntArray() + val strides = config.strides!!.toIntArray() + val dilation = config.dilation_rate!!.toIntArray() + return Conv3DTranspose( + filters = config.filters!!, + kernelSize = kernelSize, + strides = intArrayOf(1, *strides, 1), + dilations = intArrayOf(1, *dilation, 1), + activation = convertToActivation(config.activation!!), + kernelInitializer = convertToInitializer(config.kernel_initializer!!), + biasInitializer = convertToInitializer(config.bias_initializer!!), + kernelRegularizer = convertToRegularizer(config.kernel_regularizer), + biasRegularizer = convertToRegularizer(config.bias_regularizer), + activityRegularizer = convertToRegularizer(config.activity_regularizer), + padding = convertPadding(config.padding!!), + useBias = config.use_bias!!, + name = name + ) +} + +private fun List.convertToOutputPadding(): IntArray { + return intArrayOf( + 0, 0, + *flatMap { padding -> listOf(padding / 2, padding - padding / 2) }.toIntArray(), + 0, 0 + ) +} + private fun createDepthwiseConv2DLayer(config: LayerConfig, name: String): Layer { val kernelSize = config.kernel_size!!.toIntArray() val strides = config.strides!!.toIntArray() diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ModelSaver.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ModelSaver.kt index 52cb0d2c7..f5497d767 100644 --- a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ModelSaver.kt +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ModelSaver.kt @@ -60,6 +60,9 @@ private fun convertToKerasLayer(layer: Layer, isKerasFullyCompatible: Boolean, i is Conv1D -> createKerasConv1DLayer(layer, isKerasFullyCompatible) is Conv2D -> createKerasConv2DLayer(layer, isKerasFullyCompatible) is Conv3D -> createKerasConv3DLayer(layer, isKerasFullyCompatible) + is Conv1DTranspose -> createKerasConv1DTransposeLayer(layer, isKerasFullyCompatible) + is Conv2DTranspose -> createKerasConv2DTransposeLayer(layer, isKerasFullyCompatible) + is Conv3DTranspose -> createKerasConv3DTransposeLayer(layer, isKerasFullyCompatible) is DepthwiseConv2D -> createKerasDepthwiseConv2DLayer(layer, isKerasFullyCompatible) is SeparableConv2D -> createKerasSeparableConv2DLayer(layer, isKerasFullyCompatible) // Pooling layers @@ -626,7 +629,7 @@ private fun createKerasConcatenateLayer(layer: Concatenate): KerasLayer { return KerasLayer(class_name = LAYER_CONCATENATE, config = configX) } -private fun createKerasDotLayer(layer: Dot):KerasLayer{ +private fun createKerasDotLayer(layer: Dot): KerasLayer { val configX = LayerConfig( dtype = DATATYPE_FLOAT32, axis = layer.axis, @@ -695,6 +698,73 @@ private fun createKerasConv3DLayer(layer: Conv3D, isKerasFullyCompatible: Boolea return KerasLayer(class_name = LAYER_CONV3D, config = configX) } +private fun createKerasConv1DTransposeLayer(layer: Conv1DTranspose, isKerasFullyCompatible: Boolean): KerasLayer { + val configX = LayerConfig( + filters = layer.filters, + kernel_size = listOf(layer.kernelLength), + strides = listOf(layer.strides[1]), + dilation_rate = listOf(layer.dilations[1]), + activation = convertToKerasActivation(layer.activation), + kernel_initializer = convertToKerasInitializer(layer.kernelInitializer, isKerasFullyCompatible), + bias_initializer = convertToKerasInitializer(layer.biasInitializer, isKerasFullyCompatible), + kernel_regularizer = convertToKerasRegularizer(layer.kernelRegularizer), + bias_regularizer = convertToKerasRegularizer(layer.biasRegularizer), + activity_regularizer = convertToKerasRegularizer(layer.activityRegularizer), + padding = convertToKerasPadding(layer.padding), + output_padding = layer.outputPadding?.convertToKerasOutputPadding(), + trainable = layer.isTrainable, + use_bias = layer.useBias, + name = layer.name + ) + return KerasLayer(class_name = LAYER_CONV1D_TRANSPOSE, config = configX) +} + +private fun createKerasConv2DTransposeLayer(layer: Conv2DTranspose, isKerasFullyCompatible: Boolean): KerasLayer { + val configX = LayerConfig( + filters = layer.filters, + kernel_size = layer.kernelSize.toList(), + strides = listOf(layer.strides[1], layer.strides[2]), + dilation_rate = listOf(layer.dilations[1], layer.dilations[2]), + activation = convertToKerasActivation(layer.activation), + kernel_initializer = convertToKerasInitializer(layer.kernelInitializer, isKerasFullyCompatible), + bias_initializer = convertToKerasInitializer(layer.biasInitializer, isKerasFullyCompatible), + kernel_regularizer = convertToKerasRegularizer(layer.kernelRegularizer), + bias_regularizer = convertToKerasRegularizer(layer.biasRegularizer), + activity_regularizer = convertToKerasRegularizer(layer.activityRegularizer), + padding = convertToKerasPadding(layer.padding), + output_padding = layer.outputPadding?.convertToKerasOutputPadding(), + trainable = layer.isTrainable, + use_bias = layer.useBias, + name = layer.name + ) + return KerasLayer(class_name = LAYER_CONV2D_TRANSPOSE, config = configX) +} + +private fun createKerasConv3DTransposeLayer(layer: Conv3DTranspose, isKerasFullyCompatible: Boolean): KerasLayer { + val configX = LayerConfig( + filters = layer.filters, + kernel_size = layer.kernelSize.toList(), + strides = listOf(layer.strides[1], layer.strides[2], layer.strides[3]), + dilation_rate = listOf(layer.dilations[1], layer.dilations[2], layer.dilations[3]), + activation = convertToKerasActivation(layer.activation), + kernel_initializer = convertToKerasInitializer(layer.kernelInitializer, isKerasFullyCompatible), + bias_initializer = convertToKerasInitializer(layer.biasInitializer, isKerasFullyCompatible), + kernel_regularizer = convertToKerasRegularizer(layer.kernelRegularizer), + bias_regularizer = convertToKerasRegularizer(layer.biasRegularizer), + activity_regularizer = convertToKerasRegularizer(layer.activityRegularizer), + padding = convertToKerasPadding(layer.padding), + name = layer.name, + use_bias = layer.useBias + ) + return KerasLayer(class_name = LAYER_CONV3D_TRANSPOSE, config = configX) +} + +private fun IntArray.convertToKerasOutputPadding(): List { + return (0 until (size - 4) / 2).map { dimension -> + get(2 * (dimension + 1)) + get(2 * (dimension + 1) + 1) + } +} + private fun createKerasDepthwiseConv2DLayer(layer: DepthwiseConv2D, isKerasFullyCompatible: Boolean): KerasLayer { val configX = LayerConfig( kernel_size = layer.kernelSize.toList(), diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/WeightMappings.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/WeightMappings.kt index b344ec24c..cbca2549e 100644 --- a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/WeightMappings.kt +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/WeightMappings.kt @@ -1,5 +1,5 @@ /* - * Copyright 2021 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Copyright 2021-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. */ @@ -7,6 +7,7 @@ package org.jetbrains.kotlinx.dl.api.inference.keras import org.jetbrains.kotlinx.dl.api.core.layer.Layer import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv2D +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.ConvTranspose import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.DepthwiseConv2D import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.SeparableConv2D import org.jetbrains.kotlinx.dl.api.core.layer.core.Dense @@ -30,6 +31,7 @@ internal object WeightMappings { return when (layer) { is Dense -> getDenseVariables(layer) is Conv2D -> getConv2DVariables(layer) + is ConvTranspose -> getConvTransposeVariables(layer) is DepthwiseConv2D -> getDepthwiseConv2DVariables(layer) is SeparableConv2D -> getSeparableConv2DVariables(layer) is BatchNorm -> getBatchNormVariables(layer) @@ -45,6 +47,7 @@ internal object WeightMappings { return when (layer) { is Dense -> getDenseVariablesPathTemplates(layer, layerPaths) is Conv2D -> getConv2DVariablePathTemplates(layer, layerPaths) + is ConvTranspose -> getConvTransposeVariablePathTemplates(layer, layerPaths) is DepthwiseConv2D -> getDepthwiseConv2DVariablePathTemplates(layer, layerPaths) is SeparableConv2D -> getSeparableConv2DVariablePathTemplates(layer, layerPaths) is BatchNorm -> getBatchNormVariablePathTemplates(layer, layerPaths) @@ -74,6 +77,28 @@ internal object WeightMappings { return variables } + private fun getConvTransposeVariables(layer: ConvTranspose): Map> { + val variables = mutableMapOf( + Pair("kernel:0", Pair(convTransposeKernelVarName(layer.name, layer.dimensions), layer.kernelShapeArray)) + ) + if (layer.useBias) { + variables["bias:0"] = Pair(convTransposeBiasVarName(layer.name, layer.dimensions), layer.biasShapeArray!!) + } + return variables + } + + private fun getConvTransposeVariablePathTemplates(layer: ConvTranspose, layerPaths: LayerPaths?): Map { + val layerConvOrDensePaths = layerPaths as? LayerConvOrDensePaths + ?: LayerConvOrDensePaths(layer.name, KERNEL_DATA_PATH_TEMPLATE, BIAS_DATA_PATH_TEMPLATE) + val variables = mutableMapOf( + Pair(convTransposeKernelVarName(layer.name, layer.dimensions), layerConvOrDensePaths.kernelPath) + ) + if (layer.useBias) { + variables[convTransposeBiasVarName(layer.name, layer.dimensions)] = layerConvOrDensePaths.biasPath + } + return variables + } + private fun getDepthwiseConv2DVariables(layer: DepthwiseConv2D): Map> { val variables = mutableMapOf( Pair("depthwise_kernel:0", Pair(depthwiseConv2dKernelVarName(layer.name), layer.kernelShapeArray)) @@ -119,8 +144,9 @@ internal object WeightMappings { return variables } - private fun getSeparableConv2DVariablePathTemplates(layer: SeparableConv2D, - layerPaths: LayerPaths? + private fun getSeparableConv2DVariablePathTemplates( + layer: SeparableConv2D, + layerPaths: LayerPaths? ): Map { val layerSeparableConv2DPaths = layerPaths as? LayerSeparableConv2DPaths ?: LayerSeparableConv2DPaths( diff --git a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/config/LayerConfig.kt b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/config/LayerConfig.kt index c6d2ad382..3d1a764ce 100644 --- a/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/config/LayerConfig.kt +++ b/api/src/main/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/config/LayerConfig.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Copyright 2020-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. */ @@ -95,6 +95,8 @@ internal data class LayerConfig( @Json(serializeNull = false) val padding: KerasPadding? = null, @Json(serializeNull = false) + val output_padding: List? = null, + @Json(serializeNull = false) val pointwise_initializer: KerasInitializer? = null, @Json(serializeNull = false) val pointwise_regularizer: KerasRegularizer? = null, diff --git a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv1DTest.kt b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv1DTest.kt index 1f00ed584..1e107f764 100644 --- a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv1DTest.kt +++ b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv1DTest.kt @@ -12,17 +12,10 @@ import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.ConvPadding import org.junit.jupiter.api.Test internal class Conv1DTest : ConvLayerTest() { - private fun createFloatConv1DTensor( - batchSize: Int, - size: Int, - channels: Int, - initValue: Float - ) = Array(batchSize) { Array(size) { FloatArray(channels) { initValue } } } - @Test fun zeroedInputTensorWithDefaultValues() { - val input = createFloatConv1DTensor(batchSize = 1, size = 3, channels = 1, initValue = 0.0f) - val expected = createFloatConv1DTensor(batchSize = 1, size = 3, channels = 32, initValue = 0.0f) + val input = create1DTensor(batchSize = 1, size = 3, channels = 1, initValue = 0.0f) + val expected = create1DTensor(batchSize = 1, size = 3, channels = 32, initValue = 0.0f) assertTensorsEquals( Conv1D( @@ -36,8 +29,8 @@ internal class Conv1DTest : ConvLayerTest() { @Test fun constantInputTensorWithValidPadding() { - val input = createFloatConv1DTensor(batchSize = 1, size = 3, channels = 1, initValue = 1.0f) - val expected = createFloatConv1DTensor(batchSize = 1, size = 2, channels = 16, initValue = 2.0f) + val input = create1DTensor(batchSize = 1, size = 3, channels = 1, initValue = 1.0f) + val expected = create1DTensor(batchSize = 1, size = 2, channels = 16, initValue = 2.0f) assertTensorsEquals( Conv1D( @@ -77,4 +70,19 @@ internal class Conv1DTest : ConvLayerTest() { expected ) } + + companion object { + internal fun create1DTensor( + batchSize: Int, + size: Int, + channels: Int, + initValue: Float + ) = Array(batchSize) { Array(size) { FloatArray(channels) { initValue } } } + + internal fun create1DTensor( + batchSize: Int, + channels: Int, + sequence: FloatArray, + ) = Array(batchSize) { Array(sequence.size) { idx -> FloatArray(channels) { sequence[idx] } } } + } } diff --git a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv1DTransposeTest.kt b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv1DTransposeTest.kt new file mode 100644 index 000000000..aba268300 --- /dev/null +++ b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv1DTransposeTest.kt @@ -0,0 +1,144 @@ +/* + * Copyright 2021-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. + */ + +package org.jetbrains.kotlinx.dl.api.core.layer + +import org.jetbrains.kotlinx.dl.api.core.initializer.Constant +import org.jetbrains.kotlinx.dl.api.core.initializer.Zeros +import org.jetbrains.kotlinx.dl.api.core.layer.Conv1DTest.Companion.create1DTensor +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv1DTranspose +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.ConvPadding +import org.junit.jupiter.api.Test + +class Conv1DTransposeTest : ConvLayerTest() { + @Test + fun zeroInput() { + val input = create1DTensor(batchSize = 1, size = 3, channels = 32, initValue = 0f) + val expected = create1DTensor(batchSize = 1, size = 3, channels = 3, initValue = 0f) + + assertTensorsEquals( + Conv1DTranspose( + name = "TestConv1DTranspose_zeroInput", + filters = 3, + biasInitializer = Zeros() + ), + input, + expected + ) + } + + @Test + fun noPadding() { + val input = create1DTensor(batchSize = 1, size = 3, channels = 32, initValue = 1f) + val expected = create1DTensor(batchSize = 1, channels = 3, floatArrayOf(32f, 64f, 96f, 64f, 32f)) + + assertTensorsEquals( + Conv1DTranspose( + name = "TestConv1DTranspose_noPadding", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.VALID + ), + input, + expected + ) + } + + @Test + fun samePadding() { + val input = create1DTensor(batchSize = 1, size = 3, channels = 32, initValue = 1f) + val expected = create1DTensor(batchSize = 1, channels = 3, floatArrayOf(64f, 96f, 64f)) + + assertTensorsEquals( + Conv1DTranspose( + name = "TestConv1DTranspose_samePadding", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.SAME + ), + input, + expected + ) + } + + @Test + fun outputPadding() { + val input = create1DTensor(batchSize = 1, size = 3, channels = 32, initValue = 1f) + val expected = create1DTensor(batchSize = 1, channels = 3, floatArrayOf(32f, 64f, 96f, 64f, 32f)) + + assertTensorsEquals( + Conv1DTranspose( + name = "TestConv1DTranspose_outputPadding", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.SAME, + outputPadding = intArrayOf(0, 0, 1, 1, 0, 0) + ), + input, + expected + ) + } + + @Test + fun noPaddingStrides() { + val input = create1DTensor(batchSize = 1, size = 3, channels = 32, initValue = 1f) + val expected = create1DTensor(batchSize = 1, channels = 3, floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f)) + + assertTensorsEquals( + Conv1DTranspose( + name = "TestConv1DTranspose_noPadding_strides", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.VALID, + strides = intArrayOf(1, 2, 1) + ), + input, + expected + ) + } + + @Test + fun samePaddingStrides() { + val input = create1DTensor(batchSize = 1, size = 3, channels = 32, initValue = 1f) + val expected = create1DTensor(batchSize = 1, channels = 3, floatArrayOf(32f, 64f, 32f, 64f, 32f)) + + assertTensorsEquals( + Conv1DTranspose( + name = "TestConv1DTranspose_samePadding_strides", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.SAME, + strides = intArrayOf(1, 2, 1) + ), + input, + expected + ) + } + + @Test + fun outputPaddingStrides() { + val input = create1DTensor(batchSize = 1, size = 3, channels = 32, initValue = 1f) + val expected = create1DTensor(batchSize = 1, channels = 3, floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f)) + + assertTensorsEquals( + Conv1DTranspose( + name = "TestConv1DTranspose_outputPadding_strides", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.SAME, + outputPadding = intArrayOf(0, 0, 1, 1, 0, 0), + strides = intArrayOf(1, 2, 1) + ), + input, + expected + ) + } +} \ No newline at end of file diff --git a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv2DTest.kt b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv2DTest.kt index 2a07cf99f..8d767f844 100644 --- a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv2DTest.kt +++ b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv2DTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Copyright 2020-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. */ @@ -12,18 +12,10 @@ import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.ConvPadding import org.junit.jupiter.api.Test internal class Conv2DTest : ConvLayerTest() { - private fun createFloatConv2DTensor( - batchSize: Int, - height: Int, - width: Int, - channels: Int, - initValue: Float - ) = Array(batchSize) { Array(height) { Array(width) { FloatArray(channels) { initValue } } } } - @Test fun zeroedInputTensorWithDefaultValues() { - val input = createFloatConv2DTensor(batchSize = 1, height = 3, width = 3, channels = 1, initValue = 0.0f) - val expected = createFloatConv2DTensor(batchSize = 1, height = 3, width = 3, channels = 32, initValue = 0.0f) + val input = create2DTensor(batchSize = 1, height = 3, width = 3, channels = 1, initValue = 0.0f) + val expected = create2DTensor(batchSize = 1, height = 3, width = 3, channels = 32, initValue = 0.0f) assertTensorsEquals( Conv2D( @@ -37,8 +29,8 @@ internal class Conv2DTest : ConvLayerTest() { @Test fun constantInputTensorWithValidPadding() { - val input = createFloatConv2DTensor(batchSize = 1, height = 3, width = 3, channels = 1, initValue = 1.0f) - val expected = createFloatConv2DTensor(batchSize = 1, height = 2, width = 2, channels = 16, initValue = 4.0f) + val input = create2DTensor(batchSize = 1, height = 3, width = 3, channels = 1, initValue = 1.0f) + val expected = create2DTensor(batchSize = 1, height = 2, width = 2, channels = 16, initValue = 4.0f) assertTensorsEquals( Conv2D( @@ -84,4 +76,24 @@ internal class Conv2DTest : ConvLayerTest() { expected ) } + + internal companion object { + internal fun create2DTensor( + batchSize: Int, + height: Int, + width: Int, + channels: Int, + initValue: Float + ) = Array(batchSize) { Array(height) { Array(width) { FloatArray(channels) { initValue } } } } + + internal fun create2DTensor( + batchSize: Int, + channels: Int, + vararg rows: FloatArray + ) = Array(batchSize) { + Array(rows.size) { rowIdx -> + Array(rows[rowIdx].size) { columnIdx -> FloatArray(channels) { rows[rowIdx][columnIdx] } } + } + } + } } diff --git a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv2DTransposeTest.kt b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv2DTransposeTest.kt new file mode 100644 index 000000000..7cab6bf47 --- /dev/null +++ b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv2DTransposeTest.kt @@ -0,0 +1,179 @@ +/* + * Copyright 2020-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. + */ + +package org.jetbrains.kotlinx.dl.api.core.layer + +import org.jetbrains.kotlinx.dl.api.core.initializer.Constant +import org.jetbrains.kotlinx.dl.api.core.initializer.Zeros +import org.jetbrains.kotlinx.dl.api.core.layer.Conv2DTest.Companion.create2DTensor +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv2DTranspose +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.ConvPadding +import org.junit.jupiter.api.Test + +class Conv2DTransposeTest : ConvLayerTest() { + @Test + fun testZeroInput() { + val input = create2DTensor(batchSize = 1, height = 2, width = 3, channels = 32, initValue = 0.0f) + val expected = create2DTensor(batchSize = 1, height = 2, width = 3, channels = 3, initValue = 0.0f) + + assertTensorsEquals( + Conv2DTranspose( + name = "TestConv2DTranspose_zeroInput", + biasInitializer = Zeros() + ), + input, + expected + ) + } + + + @Test + fun noPadding() { + val input = create2DTensor(batchSize = 1, height = 2, width = 3, channels = 32, initValue = 1f) + val expected = create2DTensor( + batchSize = 1, channels = 3, + floatArrayOf(32f, 64f, 96f, 64f, 32f), + floatArrayOf(64f, 128f, 192f, 128f, 64f), + floatArrayOf(64f, 128f, 192f, 128f, 64f), + floatArrayOf(32f, 64f, 96f, 64f, 32f) + ) + + assertTensorsEquals( + Conv2DTranspose( + name = "TestConv2DTranspose_noPadding", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.VALID + ), + input, + expected + ) + } + + @Test + fun samePadding() { + val input = create2DTensor(batchSize = 1, height = 2, width = 3, channels = 32, initValue = 1f) + val expected = create2DTensor( + batchSize = 1, channels = 3, + floatArrayOf(128f, 192f, 128f), + floatArrayOf(128f, 192f, 128f) + ) + + assertTensorsEquals( + Conv2DTranspose( + name = "TestConv2DTranspose_samePadding", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.SAME + ), + input, + expected + ) + } + + @Test + fun outputPadding() { + val input = create2DTensor(batchSize = 1, height = 2, width = 3, channels = 32, initValue = 1f) + val expected = create2DTensor( + batchSize = 1, channels = 3, + floatArrayOf(32f, 64f, 96f, 64f, 32f), + floatArrayOf(64f, 128f, 192f, 128f, 64f), + floatArrayOf(64f, 128f, 192f, 128f, 64f), + floatArrayOf(32f, 64f, 96f, 64f, 32f) + ) + + assertTensorsEquals( + Conv2DTranspose( + name = "TestConv2DTranspose_outputPadding", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.SAME, + outputPadding = intArrayOf(0, 0, 1, 1, 1, 1, 0, 0) + ), + input, + expected + ) + } + + @Test + fun noPaddingStrides() { + val input = create2DTensor(batchSize = 1, height = 2, width = 3, channels = 32, initValue = 1f) + val expected = create2DTensor( + batchSize = 1, channels = 3, + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(64f, 64f, 128f, 64f, 128f, 64f, 64f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f) + ) + + assertTensorsEquals( + Conv2DTranspose( + name = "TestConv2DTranspose_noPadding_strides", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.VALID, + strides = intArrayOf(1, 2, 2, 1) + ), + input, + expected + ) + } + + @Test + fun samePaddingStrides() { + val input = create2DTensor(batchSize = 1, height = 2, width = 3, channels = 32, initValue = 1f) + val expected = create2DTensor( + batchSize = 1, channels = 3, + floatArrayOf(32f, 64f, 32f, 64f, 32f), + floatArrayOf(64f, 128f, 64f, 128f, 64f), + floatArrayOf(32f, 64f, 32f, 64f, 32f) + ) + + assertTensorsEquals( + Conv2DTranspose( + name = "TestConv2DTranspose_samePadding_strides", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.SAME, + strides = intArrayOf(1, 2, 2, 1) + ), + input, + expected + ) + } + + @Test + fun outputPaddingStrides() { + val input = create2DTensor(batchSize = 1, height = 2, width = 3, channels = 32, initValue = 1f) + val expected = create2DTensor( + batchSize = 1, channels = 3, + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(64f, 64f, 128f, 64f, 128f, 64f, 64f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f) + ) + + assertTensorsEquals( + Conv2DTranspose( + name = "TestConv2DTranspose_outputPadding_strides", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.SAME, + outputPadding = intArrayOf(0, 0, 1, 1, 1, 1, 0, 0), + strides = intArrayOf(1, 2, 2, 1) + ), + input, + expected + ) + } +} \ No newline at end of file diff --git a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv3DTest.kt b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv3DTest.kt index af8b2ad3a..1bd85d485 100644 --- a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv3DTest.kt +++ b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv3DTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Copyright 2020-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. */ @@ -12,21 +12,10 @@ import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.ConvPadding import org.junit.jupiter.api.Test internal class Conv3DTest : ConvLayerTest() { - private fun createFloatConv3DTensor( - batchSize: Int, - depth: Int, - height: Int, - width: Int, - channels: Int, - initValue: Float - ) = Array(batchSize) { Array(depth) { Array(height) { Array(width) { FloatArray(channels) { initValue } } } } } - @Test fun zeroedInputTensorWithDefaultValues() { - val input = - createFloatConv3DTensor(batchSize = 1, depth = 3, height = 3, width = 3, channels = 1, initValue = 0.0f) - val expected = - createFloatConv3DTensor(batchSize = 1, depth = 3, height = 3, width = 3, channels = 32, initValue = 0.0f) + val input = create3DTensor(batchSize = 1, depth = 3, height = 3, width = 3, channels = 1, initValue = 0.0f) + val expected = create3DTensor(batchSize = 1, depth = 3, height = 3, width = 3, channels = 32, initValue = 0.0f) assertTensorsEquals( Conv3D( @@ -40,10 +29,8 @@ internal class Conv3DTest : ConvLayerTest() { @Test fun constantInputTensorWithValidPadding() { - val input = - createFloatConv3DTensor(batchSize = 1, depth = 3, height = 3, width = 3, channels = 1, initValue = 1.0f) - val expected = - createFloatConv3DTensor(batchSize = 1, depth = 2, height = 2, width = 2, channels = 16, initValue = 8.0f) + val input = create3DTensor(batchSize = 1, depth = 3, height = 3, width = 3, channels = 1, initValue = 1.0f) + val expected = create3DTensor(batchSize = 1, depth = 2, height = 2, width = 2, channels = 16, initValue = 8.0f) assertTensorsEquals( Conv3D( @@ -100,4 +87,28 @@ internal class Conv3DTest : ConvLayerTest() { expected ) } + + internal companion object { + internal fun create3DTensor( + batchSize: Int, + depth: Int, + height: Int, + width: Int, + channels: Int, + initValue: Float + ) = Array(batchSize) { Array(depth) { Array(height) { Array(width) { FloatArray(channels) { initValue } } } } } + + internal fun create3DTensor( + batchSize: Int, + channels: Int, + vararg frames: Array + ) = Array(batchSize) { + Array(frames.size) { frameIdx -> + val rows = frames[frameIdx] + Array(rows.size) { rowIdx -> + Array(rows[rowIdx].size) { columnIdx -> FloatArray(channels) { rows[rowIdx][columnIdx] } } + } + } + } + } } diff --git a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv3DTransposeTest.kt b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv3DTransposeTest.kt new file mode 100644 index 000000000..a42e5095f --- /dev/null +++ b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/Conv3DTransposeTest.kt @@ -0,0 +1,161 @@ +/* + * Copyright 2020-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. + */ + +package org.jetbrains.kotlinx.dl.api.core.layer + +import org.jetbrains.kotlinx.dl.api.core.initializer.Constant +import org.jetbrains.kotlinx.dl.api.core.initializer.Zeros +import org.jetbrains.kotlinx.dl.api.core.layer.Conv3DTest.Companion.create3DTensor +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv3DTranspose +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.ConvPadding +import org.junit.jupiter.api.Test + +class Conv3DTransposeTest : ConvLayerTest() { + @Test + fun testZeroInput() { + val input = create3DTensor(batchSize = 1, depth = 1, height = 2, width = 3, channels = 32, initValue = 0.0f) + val expected = create3DTensor(batchSize = 1, depth = 1, height = 2, width = 3, channels = 3, initValue = 0.0f) + + assertTensorsEquals( + Conv3DTranspose( + name = "TestConv3DTranspose_zeroInput", + biasInitializer = Zeros() + ), + input, + expected + ) + } + + + @Test + fun noPadding() { + val input = create3DTensor(batchSize = 1, depth = 1, height = 2, width = 3, channels = 32, initValue = 1f) + val expected = create3DTensor( + batchSize = 1, channels = 3, + arrayOf( + floatArrayOf(32f, 64f, 96f, 64f, 32f), + floatArrayOf(64f, 128f, 192f, 128f, 64f), + floatArrayOf(64f, 128f, 192f, 128f, 64f), + floatArrayOf(32f, 64f, 96f, 64f, 32f) + ), + arrayOf( + floatArrayOf(32f, 64f, 96f, 64f, 32f), + floatArrayOf(64f, 128f, 192f, 128f, 64f), + floatArrayOf(64f, 128f, 192f, 128f, 64f), + floatArrayOf(32f, 64f, 96f, 64f, 32f) + ), + arrayOf( + floatArrayOf(32f, 64f, 96f, 64f, 32f), + floatArrayOf(64f, 128f, 192f, 128f, 64f), + floatArrayOf(64f, 128f, 192f, 128f, 64f), + floatArrayOf(32f, 64f, 96f, 64f, 32f) + ) + ) + + assertTensorsEquals( + Conv3DTranspose( + name = "TestConv3DTranspose_noPadding", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.VALID + ), + input, + expected + ) + } + + @Test + fun samePadding() { + val input = create3DTensor(batchSize = 1, depth = 1, height = 2, width = 3, channels = 32, initValue = 1f) + val expected = create3DTensor( + batchSize = 1, channels = 3, + arrayOf( + floatArrayOf(128f, 192f, 128f), + floatArrayOf(128f, 192f, 128f) + ) + ) + + assertTensorsEquals( + Conv3DTranspose( + name = "TestConv3DTranspose_samePadding", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.SAME + ), + input, + expected + ) + } + + @Test + fun noPaddingStrides() { + val input = create3DTensor(batchSize = 1, depth = 1, height = 2, width = 3, channels = 32, initValue = 1f) + val expected = create3DTensor( + batchSize = 1, channels = 3, + arrayOf( + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(64f, 64f, 128f, 64f, 128f, 64f, 64f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f) + ), + arrayOf( + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(64f, 64f, 128f, 64f, 128f, 64f, 64f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f) + ), + arrayOf( + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(64f, 64f, 128f, 64f, 128f, 64f, 64f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f), + floatArrayOf(32f, 32f, 64f, 32f, 64f, 32f, 32f) + ) + ) + + assertTensorsEquals( + Conv3DTranspose( + name = "TestConv3DTranspose_noPadding_strides", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.VALID, + strides = intArrayOf(1, 2, 2, 2, 1) + ), + input, + expected + ) + } + + @Test + fun samePaddingStrides() { + val input = create3DTensor(batchSize = 1, depth = 1, height = 2, width = 3, channels = 32, initValue = 1f) + val expected = create3DTensor( + batchSize = 1, channels = 3, + arrayOf( + floatArrayOf(32f, 64f, 32f, 64f, 32f), + floatArrayOf(64f, 128f, 64f, 128f, 64f), + floatArrayOf(32f, 64f, 32f, 64f, 32f) + ) + ) + + assertTensorsEquals( + Conv3DTranspose( + name = "TestConv3DTranspose_samePadding_strides", + filters = 3, + kernelInitializer = Constant(1.0f), + biasInitializer = Zeros(), + padding = ConvPadding.SAME, + strides = intArrayOf(1, 2, 2, 2, 1) + ), + input, + expected + ) + } +} \ No newline at end of file diff --git a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/ConvLayerTest.kt b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/ConvLayerTest.kt index 5f3cac575..d19f4f2c4 100644 --- a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/ConvLayerTest.kt +++ b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/core/layer/ConvLayerTest.kt @@ -1,5 +1,5 @@ /* - * Copyright 2020 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Copyright 2020-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. */ @@ -32,6 +32,7 @@ open class ConvLayerTest { val numberOfLosses = tf.constant(1.0f) layer.build(tf, kGraph, input.shape) + layer.computeOutputShape(input.shape) val output = layer.forward(tf, inputOp, isTraining, numberOfLosses).asOutput() kGraph.initializeGraphVariables(session) session.runner().fetch(output).run().first().use { outputTensor -> diff --git a/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ConvTransposePersistenceTest.kt b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ConvTransposePersistenceTest.kt new file mode 100644 index 000000000..aa9db94dd --- /dev/null +++ b/api/src/test/kotlin/org/jetbrains/kotlinx/dl/api/inference/keras/ConvTransposePersistenceTest.kt @@ -0,0 +1,121 @@ +/* + * Copyright 2021-2022 JetBrains s.r.o. and Kotlin Deep Learning project contributors. All Rights Reserved. + * Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE.txt file. + */ + +package org.jetbrains.kotlinx.dl.api.inference.keras + +import org.jetbrains.kotlinx.dl.api.core.GraphTrainableModel +import org.jetbrains.kotlinx.dl.api.core.Sequential +import org.jetbrains.kotlinx.dl.api.core.activation.Activations +import org.jetbrains.kotlinx.dl.api.core.initializer.HeUniform +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv1DTranspose +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv2DTranspose +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.Conv3DTranspose +import org.jetbrains.kotlinx.dl.api.core.layer.convolutional.ConvPadding +import org.jetbrains.kotlinx.dl.api.core.layer.core.Input +import org.jetbrains.kotlinx.dl.api.core.loss.Losses +import org.jetbrains.kotlinx.dl.api.core.metric.Metrics +import org.jetbrains.kotlinx.dl.api.core.optimizer.Adam +import org.jetbrains.kotlinx.dl.api.core.regularizer.L2 +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import java.io.File + +class ConvTransposePersistenceTest { + private lateinit var tempFile: File + + @BeforeEach + fun createTempFile() { + tempFile = File.createTempFile("model", ".json") + } + + @AfterEach + fun deleteTempFile() { + tempFile.delete() + } + + @Test + fun conv1DTranspose() { + testSequentialModel( + Sequential.of( + Input(dims = longArrayOf(3)), + Conv1DTranspose( + filters = 5, + kernelLength = 5, + strides = intArrayOf(1, 2, 1), + activation = Activations.Tanh, + dilations = intArrayOf(1, 2, 1), + kernelInitializer = HeUniform(), + biasInitializer = HeUniform(), + kernelRegularizer = L2(), + biasRegularizer = L2(), + activityRegularizer = L2(), + padding = ConvPadding.VALID, + outputPadding = intArrayOf(0, 0, 1, 1, 0, 0) + ) + ) + ) + } + + @Test + fun conv2DTranspose() { + testSequentialModel( + Sequential.of( + Input(dims = longArrayOf(3, 3)), + Conv2DTranspose( + filters = 5, + kernelSize = intArrayOf(5, 5), + strides = intArrayOf(1, 2, 4, 1), + activation = Activations.Tanh, + dilations = intArrayOf(1, 2, 4, 1), + kernelInitializer = HeUniform(), + biasInitializer = HeUniform(), + kernelRegularizer = L2(), + biasRegularizer = L2(), + activityRegularizer = L2(), + padding = ConvPadding.VALID, + outputPadding = intArrayOf(0, 0, 1, 1, 2, 2, 0, 0) + ) + ) + ) + } + + @Test + fun conv3DTranspose() { + testSequentialModel( + Sequential.of( + Input(dims = longArrayOf(3, 3, 3)), + Conv3DTranspose( + filters = 5, + kernelSize = intArrayOf(5, 5, 5), + strides = intArrayOf(1, 2, 4, 2, 1), + activation = Activations.Tanh, + dilations = intArrayOf(1, 2, 4, 2, 1), + kernelInitializer = HeUniform(), + biasInitializer = HeUniform(), + kernelRegularizer = L2(), + biasRegularizer = L2(), + activityRegularizer = L2(), + padding = ConvPadding.VALID, + ) + ) + ) + } + + private fun testSequentialModel(originalModel: Sequential) { + originalModel.saveModelConfiguration(tempFile) + val restoredModel = Sequential.loadModelConfiguration(tempFile) + assertSameModel(originalModel, restoredModel) + } + + private fun assertSameModel(expectedModel: GraphTrainableModel, actualModel: GraphTrainableModel) { + listOf(expectedModel, actualModel).forEach { + it.compile(Adam(), Losses.MSE, Metrics.ACCURACY) + } + Assertions.assertEquals(expectedModel.layers.joinToString("\n") { it.toString() }, + actualModel.layers.joinToString("\n") { it.toString() }) + } +} \ No newline at end of file