Merge pull request #23 from Evizero/imagetuple

Add image tuple support

NEWS.md

@@ -1,3 +1,27 @@
+# v0.5.0
+
+New functionality:
+
+- Add support for multiple images per observation (represented as
+  tuples), in which case all images of an observation are
+  processed the same way (even when randomness is involved).
+  See #23 for more information.
+
+# v0.4.0
+
+New operations:
+
+- `MapFun`: Maps the given function over all individual array
+  elements.
+
+- `AggregateThenMapFun`: Compute some aggregated value of the
+  current image using some given function `aggfun`, and map that
+  value over the current image using some given function `mapfun`.
+
+Online documentation:
+
+- Replace tensorflow tutorial with Knet tutorial
+
 # v0.3.1
 
 Small fixes:

docs/src/operations.md

@@ -27,7 +27,7 @@ save("assets/tiny_CropRatio.png", augment(pattern, CropRatio(1)))
 srand(1337)
 save("assets/tiny_RCropRatio.png", augment(pattern, RCropRatio(1)))
 # Conversion
-save("assets/tiny_ConvertEltype.png", augment(pattern, ConvertEltype(GrayA)))
+save("assets/tiny_ConvertEltype.png", augment(pattern, ConvertEltype(GrayA{N0f8})))
 nothing;
 ```
 

docs/src/operations/converteltype.md

@@ -6,5 +6,5 @@ ConvertEltype
 
 ```@eval
 include("optable.jl")
-@optable ConvertEltype(GrayA)
+@optable ConvertEltype(GrayA{N0f8})
 ```

src/Augmentor.jl

@@ -76,6 +76,7 @@ include("types.jl")
 include("operation.jl")
 
 include("operations/channels.jl")
+include("operations/dims.jl")
 include("operations/convert.jl")
 include("operations/mapfun.jl")
 

src/augment.jl

@@ -1,8 +1,30 @@
 """
-    augment([img], pipeline) -> imga
+    augment([img], pipeline) -> out
 
-Apply the operations of the given `pipeline` to the image `img`
-and return the resulting image `imga`.
+Apply the operations of the given `pipeline` sequentially to the
+given image `img` and return the resulting image `out`.
+
+```julia-repl
+julia> img = testpattern();
+
+julia> out = augment(img, FlipX() |> FlipY())
+3×2 Array{Gray{N0f8},2}:
+[...]
+```
+
+The parameter `img` can either be a single image, or a tuple of
+multiple images. In case `img` is a tuple of images, its elements
+will be assumed to be conceptually connected. Consequently, all
+images in the tuple will take the exact same path through the
+pipeline; even when randomness is involved. This is useful for the
+purpose of image segmentation, for which the input and output are
+both images that need to be transformed exactly the same way.
+
+```julia
+img1 = testpattern()
+img2 = Gray.(testpattern())
+out1, out2 = augment((img1, img2), FlipX() |> FlipY())
+```
 
 The parameter `pipeline` can be a `Augmentor.Pipeline`, a tuple
 of `Augmentor.Operation`, or a single `Augmentor.Operation`.
@@ -51,8 +73,27 @@ end
 """
     augment!(out, img, pipeline) -> out
 
-Apply the operations of the given `pipeline` to the image `img`
-and write the resulting image into `out`.
+Apply the operations of the given `pipeline` sequentially to the
+image `img` and write the resulting image into the preallocated
+parameter `out`. For convenience `out` is also the function's
+return-value.
+
+```julia
+img = testpattern()
+out = similar(img)
+augment!(out, img, FlipX() |> FlipY())
+```
+
+The parameter `img` can either be a single image, or a tuple of
+multiple images. In case `img` is a tuple of images, the
+parameter `out` has to be a tuple of the same length and
+ordering. See [`augment`](@ref) for more information.
+
+```julia
+imgs = (testpattern(), Gray.(testpattern()))
+outs = (similar(imgs[1]), similar(imgs[2]))
+augment!(outs, imgs, FlipX() |> FlipY())
+```
 
 The parameter `pipeline` can be a `Augmentor.Pipeline`, a tuple
 of `Augmentor.Operation`, or a single `Augmentor.Operation`.
@@ -67,12 +108,20 @@ augment!(out, img, FlipX())
 """
 augment!(out, img, op::Operation) = augment!(out, img, (op,))
 
-function augment!(out, img, pipeline::AbstractPipeline)
+function augment!(out::AbstractArray, img::AbstractArray, pipeline::AbstractPipeline)
     out_lazy = _augment_avoid_eager(img, pipeline)
     copy!(match_idx(out, indices(out_lazy)), out_lazy)
     out
 end
 
+function augment!(outs::NTuple{N,AbstractArray}, imgs::NTuple{N,AbstractArray}, pipeline::AbstractPipeline) where N
+    outs_lazy = _augment_avoid_eager(imgs, pipeline)
+    map(outs, outs_lazy) do out, out_lazy
+        copy!(match_idx(out, indices(out_lazy)), out_lazy)
+    end
+    outs
+end
+
 @inline function _augment_avoid_eager(img, pipeline::AbstractPipeline)
     _augment_avoid_eager(img, operations(pipeline)...)
 end

src/augmentbatch.jl

@@ -1,7 +1,9 @@
 _berror() = throw(ArgumentError("Number of output images must be equal to the number of input images"))
 
 imagesvector(imgs::AbstractArray, args...) = obsview(imgs, args...)
+imagesvector(imgs::Tuple{Vararg{AbstractArray}}, args...) = obsview(imgs, args...)
 @inline imagesvector(imgs::AbstractVector{<:AbstractArray}, args...) = imgs
+@inline imagesvector(imgs::AbstractVector{<:Tuple{Vararg{AbstractArray}}}, args...) = imgs
 
 # --------------------------------------------------------------------
 
@@ -13,35 +15,80 @@ Apply the operations of the given `pipeline` to the images in
 
 Both `outs` and `imgs` have to contain the same number of images.
 Each of these two variables can either be in the form of a higher
-dimensional array, or alternatively in the form of a vector of
-arrays, for which each vector element denotes an image.
+dimensional array, in the form of a vector of arrays for which
+each vector element denotes an image.
+
+```julia
+# create five example observations of size 3x3
+imgs = rand(3,3,5)
+# create output arrays of appropriate shape
+outs = similar(imgs)
+# transform the batch of images
+augmentbatch!(outs, imgs, FlipX() |> FlipY())
+```
 
 If one (or both) of the two parameters `outs` and `imgs` is a
 higher dimensional array, then the optional parameter `obsdim`
 can be used specify which dimension denotes the observations
 (defaults to `ObsDim.Last()`),
 
+```julia
+# create five example observations of size 3x3
+imgs = rand(5,3,3)
+# create output arrays of appropriate shape
+outs = similar(imgs)
+# transform the batch of images
+augmentbatch!(outs, imgs, FlipX() |> FlipY(), ObsDim.First())
+```
+
+Similar to [`augment!`](@ref), it is also allowed for `outs` and
+`imgs` to both be tuples of the same length. If that is the case,
+then each tuple element can be in any of the forms listed above.
+This is useful for tasks such as image segmentation, where each
+observations is made up of more than one image.
+
+```julia
+# create five example observations where each observation is
+# made up of two conceptually linked 3x3 arrays
+imgs = (rand(3,3,5), rand(3,3,5))
+# create output arrays of appropriate shape
+outs = similar.(imgs)
+# transform the batch of images
+augmentbatch!(outs, imgs, FlipX() |> FlipY())
+```
+
 The parameter `pipeline` can be a `Augmentor.Pipeline`, a tuple
 of `Augmentor.Operation`, or a single `Augmentor.Operation`.
 
+```julia
+augmentbatch!(outs, imgs, FlipX() |> FlipY())
+augmentbatch!(outs, imgs, (FlipX(), FlipY()))
+augmentbatch!(outs, imgs, FlipX())
+```
+
 The optional first parameter `resource` can either be `CPU1()`
 (default) or `CPUThreads()`. In the later case the images will be
 augmented in parallel. For this to make sense make sure that the
 environment variable `JULIA_NUM_THREADS` is set to a reasonable
 number so that `Threads.nthreads()` is greater than 1.
+
+```julia
+# transform the batch of images in parallel using multithreading
+augmentbatch!(CPUThreads(), outs, imgs, FlipX() |> FlipY())
+```
 """
 function augmentbatch!(
-        outs::AbstractArray,
-        imgs::AbstractArray,
+        outs::Union{Tuple, AbstractArray},
+        imgs::Union{Tuple, AbstractArray},
         pipeline,
         args...)
     augmentbatch!(CPU1(), outs, imgs, pipeline, args...)
 end
 
 function augmentbatch!(
         r::AbstractResource,
-        outs::AbstractArray,
-        imgs::AbstractArray,
+        outs::Union{Tuple, AbstractArray},
+        imgs::Union{Tuple, AbstractArray},
         pipeline,
         obsdim = MLDataPattern.default_obsdim(outs))
     augmentbatch!(r, imagesvector(outs, obsdim), imagesvector(imgs, obsdim), pipeline)
@@ -50,8 +97,8 @@ end
 
 function augmentbatch!(
         ::CPU1,
-        outs::AbstractVector{<:AbstractArray},
-        imgs::AbstractVector{<:AbstractArray},
+        outs::AbstractVector{<:Union{Tuple, AbstractArray}},
+        imgs::AbstractVector{<:Union{Tuple, AbstractArray}},
         pipeline)
     length(outs) == length(imgs) || _berror()
     for i in 1:length(outs)
@@ -62,8 +109,8 @@ end
 
 function augmentbatch!(
         ::CPUThreads,
-        outs::AbstractVector{<:AbstractArray},
-        imgs::AbstractVector{<:AbstractArray},
+        outs::AbstractVector{<:Union{Tuple, AbstractArray}},
+        imgs::AbstractVector{<:Union{Tuple, AbstractArray}},
         pipeline)
     length(outs) == length(imgs) || _berror()
     Threads.@threads for i in 1:length(outs)

src/operation.jl

@@ -35,54 +35,75 @@ prepareaffine(img) = invwarpedview(img, toaffinemap(NoOp(), img), Flat())
 prepareaffine(img::AbstractExtrapolation) = invwarpedview(img, toaffinemap(NoOp(), img))
 @inline prepareaffine(img::SubArray{T,N,<:InvWarpedView}) where {T,N} = img
 @inline prepareaffine(img::InvWarpedView) = img
+prepareaffine(imgs::Tuple) = map(prepareaffine, imgs)
 
 # currently unused
 @inline preparelazy(img) = img
 
 # --------------------------------------------------------------------
 # Operation and AffineOperation fallbacks
 
-function applyeager(op::Operation, img)
-    plain_array(applylazy(op, img))
+# discard param if operations doesn't need it
+@inline randparam(op::Operation, img) = nothing
+@inline toaffinemap(op::Operation, img, param) = toaffinemap(op, img)
+
+for FUN in (:applyeager, :applylazy, :applypermute,
+            :applyaffine, :applyaffineview,
+            :applyaffine_common, :applyaffineview_common,
+            :applyview, :applystepview)
+    @eval begin
+        function ($FUN)(op::Operation, imgs::Tuple)
+            param = randparam(op, imgs)
+            map(img -> ($FUN)(op, img, param), imgs)
+        end
+        @inline function ($FUN)(op::Operation, img::AbstractArray)
+            ($FUN)(op, img, randparam(op, img))
+        end
+    end
 end
 
-function applyaffine(op::AffineOperation, img)
-    invwarpedview(img, toaffinemap(op, img))
+function applyeager(op::Operation, img::AbstractArray, param)
+    maybe_copy(applylazy(op, img, param))
 end
 
-function applyaffineview(op::Operation, img)
-    wv = applyaffine(op, img)
+function applyaffineview(op::Operation, img::AbstractArray, param)
+    wv = applyaffine(op, img, param)
     direct_view(wv, indices(wv))
 end
 
+function applyaffine(op::AffineOperation, img::AbstractArray, param)
+    invwarpedview(img, toaffinemap(op, img, param))
+end
+
 # Allow affine operations to omit specifying a custom
 # "applylazy". On the other hand this also makes sure that a
 # custom implementation of "applylazy" is preferred over
 # "applylazy_fallback" which by default just calls "applyaffine".
-function applylazy(op::AffineOperation, img)
-    _applylazy(op, img)
+function applylazy(op::AffineOperation, img::AbstractArray, param)
+    _applylazy(op, img, param)
 end
 
 # The purpose of having a separate "_applylazy" is to not
-# force "applylazy" implementations to specify the type of "img".
-function _applylazy(op::AffineOperation, img::InvWarpedView)
-    applyaffine(op, img)
+# force "applylazy" implementations to specify the type of "img",
+# when typeof(img) <: AbstractArray.
+function _applylazy(op::AffineOperation, img::InvWarpedView, param)
+    applyaffine(op, img, param)
 end
 
-function _applylazy(op::AffineOperation, img::SubArray{T,N,<:InvWarpedView}) where {T,N}
-    applyaffine(op, img)
+function _applylazy(op::AffineOperation, img::SubArray{T,N,<:InvWarpedView}, param) where {T,N}
+    applyaffine(op, img, param)
 end
 
-function _applylazy(op::AffineOperation, img)
-    applylazy_fallback(op, img)
+function _applylazy(op::AffineOperation, img, param)
+    applylazy_fallback(op, img, param)
 end
 
 # Defining "applylazy_fallback" instead of "applylazy" will
 # make sure that the custom implementation is only used if
 # "img" is not already an "InvWarpedView", in which case
 # "applyaffine" would be called instead of "applylazy_fallback".
-function applylazy_fallback(op::AffineOperation, img)
-    applyaffine(op, prepareaffine(img))
+function applylazy_fallback(op::AffineOperation, img, param)
+    applyaffine(op, prepareaffine(img), param)
 end
 
 # --------------------------------------------------------------------
@@ -92,24 +113,26 @@ end
 # we need a way to force a common "AffineMap" type using only
 # "SArray" internally (i.e. no "RotMatrix" or other special types).
 
-@generated function toaffinemap_common(op::AffineOperation, img::AbstractArray{T,N}) where {T,N}
+@generated function toaffinemap_common(op::AffineOperation, img::AbstractArray{T,N}, param) where {T,N}
     quote
-        tfm = toaffinemap(op, img)
+        tfm = toaffinemap(op, img, param)
         AffineMap(SMatrix(tfm.m), SVector(tfm.v))::AffineMap{SArray{Tuple{$N,$N},Float64,$N,$(N*N)},SVector{$N,Float64}}
     end
 end
 
-function applyaffine_common(op::AffineOperation, img)
-    invwarpedview(img, toaffinemap_common(op, img))
+function applyaffine_common(op::AffineOperation, img::AbstractArray, param)
+    invwarpedview(img, toaffinemap_common(op, img, param))
 end
 
-function applyaffineview_common(op::AffineOperation, img)
-    wv = applyaffine_common(op, img)
+function applyaffineview_common(op::AffineOperation, img::AbstractArray, param)
+    wv = applyaffine_common(op, img, param)
     direct_view(wv, indices(wv))
 end
 
 # We trust that non-affine operations use SArray-only AffineMap.
-applyaffineview_common(op::Operation, img) = applyaffineview(op, img)
+function applyaffineview_common(op::Operation, img::AbstractArray, param)
+    applyaffineview(op, img, param)
+end
 
 # --------------------------------------------------------------------
 # Functions to unroll sequences of Operation. These are called by