From d098cf8b71e8cb4e731c72b6e37c0b2c5b3cd099 Mon Sep 17 00:00:00 2001 From: Christof Stocker Date: Wed, 4 Apr 2018 11:31:45 +0200 Subject: [PATCH 1/2] outline MapFun --- src/Augmentor.jl | 2 ++ src/operations/mapfun.jl | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 src/operations/mapfun.jl diff --git a/src/Augmentor.jl b/src/Augmentor.jl index 613f1907..7f84a29c 100644 --- a/src/Augmentor.jl +++ b/src/Augmentor.jl @@ -34,6 +34,7 @@ export Reshape, ConvertEltype, + MapFun, Rotate90, Rotate180, @@ -75,6 +76,7 @@ include("operation.jl") include("operations/channels.jl") include("operations/convert.jl") +include("operations/mapfun.jl") include("operations/noop.jl") include("operations/cache.jl") diff --git a/src/operations/mapfun.jl b/src/operations/mapfun.jl new file mode 100644 index 00000000..e4e2d412 --- /dev/null +++ b/src/operations/mapfun.jl @@ -0,0 +1,26 @@ +struct MapFun{T} <: Operation + fun::T +end + +@inline supports_lazy(::Type{<:MapFun}) = true + +function applyeager(op::MapFun{T}, img::AbstractArray) where T + map(op.fun, img) +end + +function applylazy(op::MapFun{T}, img::AbstractArray) where T + mappedarray(op.fun, img) +end + +function showconstruction(io::IO, op::MapFun) + print(io, typeof(op).name.name, '(', op.fun, ')') +end + +function Base.show(io::IO, op::MapFun) + if get(io, :compact, false) + print(io, "Map function \"", op.fun, "\" over image") + else + print(io, "Augmentor.") + showconstruction(io, op) + end +end From 1836ea0e6c70de18f9e7d3fc5a78034d70d1816d Mon Sep 17 00:00:00 2001 From: Christof Stocker Date: Sun, 8 Apr 2018 17:33:54 +0200 Subject: [PATCH 2/2] add MapFun and AggregateThenMapFun --- README.md | 2 + docs/make.jl | 2 + docs/src/operations.md | 3 +- docs/src/operations/aggmapfun.md | 5 + docs/src/operations/mapfun.md | 5 + src/Augmentor.jl | 1 + src/operations/mapfun.jl | 132 ++++++++++++++++++++- test/operations/tst_mapfun.jl | 117 ++++++++++++++++++ test/reference/rot45_crop_zoom_convert.txt | 13 ++ test/runtests.jl | 1 + test/tst_augment.jl | 11 ++ 11 files changed, 288 insertions(+), 4 deletions(-) create mode 100644 docs/src/operations/aggmapfun.md create mode 100644 docs/src/operations/mapfun.md create mode 100644 test/operations/tst_mapfun.jl create mode 100644 test/reference/rot45_crop_zoom_convert.txt diff --git a/README.md b/README.md index 05f1d08a..61193790 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,8 @@ look at the corresponding section of the | | [`CropRatio`](https://evizero.github.io/Augmentor.jl/operations/cropratio) | ![](https://evizero.github.io/Augmentor.jl/assets/tiny_CropRatio.png) | Crop to specified aspect ratio. | | [`RCropRatio`](https://evizero.github.io/Augmentor.jl/operations/rcropratio) | ![](https://evizero.github.io/Augmentor.jl/assets/tiny_RCropRatio.png) | Crop random window of specified aspect ratio. | *Conversion:* | [`ConvertEltype`](https://evizero.github.io/Augmentor.jl/operations/converteltype) | ![](https://evizero.github.io/Augmentor.jl/assets/tiny_ConvertEltype.png) | Convert the array elements to the given type. +| *Mapping:* | [`MapFun`](https://evizero.github.io/Augmentor.jl/operations/mapfun) | - | Map custom function over image +| | [`AggregateThenMapFun`](https://evizero.github.io/Augmentor.jl/operations/aggmapfun) | - | Map aggregated value over image | *Layout:* | [`SplitChannels`](https://evizero.github.io/Augmentor.jl/operations/splitchannels) | - | Separate the color channels into a dedicated array dimension. | | [`CombineChannels`](https://evizero.github.io/Augmentor.jl/operations/combinechannels) | - | Collapse the first dimension into a specific colorant. | | [`PermuteDims`](https://evizero.github.io/Augmentor.jl/operations/permutedims) | - | Reorganize the array dimensions into a specific order. diff --git a/docs/make.jl b/docs/make.jl index eadd43b9..6bc70b53 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -27,6 +27,8 @@ op_fnames = [ "rcropratio", "resize", "converteltype", + "mapfun", + "aggmapfun", "splitchannels", "combinechannels", "permutedims", diff --git a/docs/src/operations.md b/docs/src/operations.md index 17ce7374..9e6b5627 100644 --- a/docs/src/operations.md +++ b/docs/src/operations.md @@ -102,7 +102,7 @@ in order to preserve type-stability. |:---------:|:--:|:------------------:|:------------------------:|:----------------------:|:-----------------------:|:------------------------:| | ![](assets/tiny_pattern.png) | → | [![](assets/tiny_Crop.png)](@ref Crop) | [![](assets/tiny_CropNative.png)](@ref CropNative) | [![](assets/tiny_CropSize.png)](@ref CropSize) | [![](assets/tiny_CropRatio.png)](@ref CropRatio) | [![](assets/tiny_RCropRatio.png)](@ref RCropRatio) | -## Conversion and Layout +## Element-wise Transformations and Layout It is not uncommon that machine learning frameworks require the data in a specific form and layout. For example many deep @@ -115,6 +115,7 @@ or end of a augmentation pipeline. Category | Available Operations ----------------------|----------------------------------------------- Conversion | [`ConvertEltype`](@ref ConvertEltype) (e.g. convert to grayscale) +Mapping | [`MapFun`](@ref MapFun), [`AggregateThenMapFun`](@ref AggregateThenMapFun) Information Layout | [`SplitChannels`](@ref SplitChannels), [`CombineChannels`](@ref CombineChannels), [`PermuteDims`](@ref PermuteDims), [`Reshape`](@ref Reshape) ## Utility Operations diff --git a/docs/src/operations/aggmapfun.md b/docs/src/operations/aggmapfun.md new file mode 100644 index 00000000..b6608764 --- /dev/null +++ b/docs/src/operations/aggmapfun.md @@ -0,0 +1,5 @@ +# [AggregateThenMapFun: Aggregate and Map over Image](@id AggregateThenMapFun) + +```@docs +AggregateThenMapFun +``` diff --git a/docs/src/operations/mapfun.md b/docs/src/operations/mapfun.md new file mode 100644 index 00000000..4a66e415 --- /dev/null +++ b/docs/src/operations/mapfun.md @@ -0,0 +1,5 @@ +# [MapFun: Map function over Image](@id MapFun) + +```@docs +MapFun +``` diff --git a/src/Augmentor.jl b/src/Augmentor.jl index 7f84a29c..54d1b51e 100644 --- a/src/Augmentor.jl +++ b/src/Augmentor.jl @@ -35,6 +35,7 @@ export ConvertEltype, MapFun, + AggregateThenMapFun, Rotate90, Rotate180, diff --git a/src/operations/mapfun.jl b/src/operations/mapfun.jl index e4e2d412..d559d6c4 100644 --- a/src/operations/mapfun.jl +++ b/src/operations/mapfun.jl @@ -1,14 +1,60 @@ +""" + MapFun <: Augmentor.Operation + +Description +-------------- + +Maps the given function over all individual array elements. + +This means that the given function is called with an individual +elements and is expected to return a transformed element that +should take the original's place. This further implies that the +function is expected to be unary. It is encouraged that the +function should be consistent with its return type and +type-stable. + +Usage +-------------- + + MapFun(fun) + +Arguments +-------------- + +- **`fun`** : The unary function that should be mapped over all + individual array elements. + +See also +-------------- + +[`AggregateThenMapFun`](@ref), [`ConvertEltype`](@ref), [`augment`](@ref) + +Examples +-------------- + +```julia +using Augmentor, ColorTypes +img = testpattern() + +# subtract the constant RGBA value from each pixel +augment(img, MapFun(px -> px - RGBA(0.5, 0.3, 0.7, 0.0))) + +# separate channels to scale each numeric element by a constant value +pl = SplitChannels() |> MapFun(el -> el * 0.5) |> CombineChannels(RGBA) +augment(img, pl) +``` +""" struct MapFun{T} <: Operation fun::T end @inline supports_lazy(::Type{<:MapFun}) = true -function applyeager(op::MapFun{T}, img::AbstractArray) where T - map(op.fun, img) +function applyeager(op::MapFun, img::AbstractArray) + plain_array(map(op.fun, img)) end -function applylazy(op::MapFun{T}, img::AbstractArray) where T +function applylazy(op::MapFun, img::AbstractArray) mappedarray(op.fun, img) end @@ -24,3 +70,83 @@ function Base.show(io::IO, op::MapFun) showconstruction(io, op) end end + +# -------------------------------------------------------------------- + +""" + AggregateThenMapFun <: Augmentor.Operation + +Description +-------------- + +Compute some aggregated value of the current image using the +given function `aggfun`, and map that value over the current +image using the given function `mapfun`. + +This is particularly useful for achieving effects such as +per-image normalization. + +Usage +-------------- + + AggregateThenMapFun(aggfun, mapfun) + +Arguments +-------------- + +- **`aggfun`** : A function that takes the whole current image as + input and which result will also be passed to `mapfun`. It + should have a signature of `img -> agg`, where `img` will the + the current image. What type and value `agg` should be is up + to the user. + +- **`mapfun`** : The binary function that should be mapped over + all individual array elements. It should have a signature of + `(px, agg) -> new_px` where `px` is a single element of the + current image, and `agg` is the output of `aggfun`. + +See also +-------------- + +[`MapFun`](@ref), [`ConvertEltype`](@ref), [`augment`](@ref) + +Examples +-------------- + +```julia +using Augmentor +img = testpattern() + +# subtract the average RGB value of the current image +augment(img, AggregateThenMapFun(img -> mean(img), (px, agg) -> px - agg)) +``` +""" +struct AggregateThenMapFun{A,M} <: Operation + aggfun::A + mapfun::M +end + +@inline supports_lazy(::Type{<:AggregateThenMapFun}) = true + +function applyeager(op::AggregateThenMapFun, img::AbstractArray) + agg = op.aggfun(img) + plain_array(map(x -> op.mapfun(x, agg), img)) +end + +function applylazy(op::AggregateThenMapFun, img::AbstractArray) + agg = op.aggfun(img) + mappedarray(x -> op.mapfun(x, agg), img) +end + +function showconstruction(io::IO, op::AggregateThenMapFun) + print(io, typeof(op).name.name, '(', op.aggfun, ", ", op.mapfun, ')') +end + +function Base.show(io::IO, op::AggregateThenMapFun) + if get(io, :compact, false) + print(io, "Map result of \"", op.aggfun, "\" using \"", op.mapfun, "\" over image") + else + print(io, "Augmentor.") + showconstruction(io, op) + end +end diff --git a/test/operations/tst_mapfun.jl b/test/operations/tst_mapfun.jl new file mode 100644 index 00000000..a72e481f --- /dev/null +++ b/test/operations/tst_mapfun.jl @@ -0,0 +1,117 @@ +@testset "MapFun" begin + @test (MapFun <: Augmentor.ImageOperation) == false + @test (MapFun <: Augmentor.Operation) == true + @testset "constructor" begin + @test_throws MethodError MapFun() + @test typeof(@inferred(MapFun((x)->x))) <: MapFun <: Augmentor.Operation + @test typeof(@inferred(MapFun(identity))) <: MapFun <: Augmentor.Operation + @test str_show(MapFun(identity)) == "Augmentor.MapFun(identity)" + @test str_showconst(MapFun(identity)) == "MapFun(identity)" + @test str_showcompact(MapFun(identity)) == "Map function \"identity\" over image" + end + @testset "eager" begin + @test_throws MethodError Augmentor.applyeager(MapFun(identity), nothing) + @test Augmentor.supports_eager(MapFun) === true + for img in (Augmentor.prepareaffine(rect), rect, view(rect, IdentityRange(1:2), IdentityRange(1:3))) + res = @inferred(Augmentor.applyeager(MapFun(identity), img)) + @test res == rect + @test typeof(res) <: Array{eltype(img)} + res = @inferred(Augmentor.applyeager(MapFun(x->x .- Gray(0.1)), img)) + @test res ≈ rect .- 0.1 + @test typeof(res) <: Array{Gray{Float64}} + end + img = OffsetArray(rect, -2, -1) + res = @inferred(Augmentor.applyeager(MapFun(identity), img)) + @test res == rect + @test typeof(res) <: Array{eltype(img)} + img = OffsetArray(rgb_rect, -2, -1) + res = @inferred(Augmentor.applyeager(MapFun(x -> x - RGB(.1,.1,.1)), img)) + @test res == rgb_rect .- RGB(.1,.1,.1) + @test typeof(res) <: Array{RGB{Float64}} + end + @testset "affine" begin + @test Augmentor.supports_affine(MapFun) === false + end + @testset "affineview" begin + @test Augmentor.supports_affineview(MapFun) === false + end + @testset "lazy" begin + @test Augmentor.supports_lazy(MapFun) === true + res = @inferred(Augmentor.applylazy(MapFun(identity), rect)) + @test res === mappedarray(identity, rect) + res = @inferred(Augmentor.applylazy(MapFun(identity), rgb_rect)) + @test res === mappedarray(identity, rgb_rect) + res = @inferred(Augmentor.applylazy(MapFun(x->x-RGB(.1,.1,.1)), rgb_rect)) + @test res == mappedarray(x->x-RGB(.1,.1,.1), rgb_rect) + @test typeof(res) <: MappedArrays.ReadonlyMappedArray{ColorTypes.RGB{Float64}} + end + @testset "view" begin + @test Augmentor.supports_view(MapFun) === false + end + @testset "stepview" begin + @test Augmentor.supports_stepview(MapFun) === false + end + @testset "permute" begin + @test Augmentor.supports_permute(MapFun) === false + end +end + +# -------------------------------------------------------------------- + +@testset "AggregateThenMapFun" begin + @test (AggregateThenMapFun <: Augmentor.ImageOperation) == false + @test (AggregateThenMapFun <: Augmentor.Operation) == true + @testset "constructor" begin + @test_throws MethodError AggregateThenMapFun() + @test_throws MethodError AggregateThenMapFun(x->x) + @test typeof(@inferred(AggregateThenMapFun(x->x, x->x))) <: AggregateThenMapFun <: Augmentor.Operation + @test typeof(@inferred(AggregateThenMapFun(mean, identity))) <: AggregateThenMapFun <: Augmentor.Operation + @test str_show(AggregateThenMapFun(mean, identity)) == "Augmentor.AggregateThenMapFun(mean, identity)" + @test str_showconst(AggregateThenMapFun(mean, identity)) == "AggregateThenMapFun(mean, identity)" + @test str_showcompact(AggregateThenMapFun(mean, identity)) == "Map result of \"mean\" using \"identity\" over image" + end + @testset "eager" begin + @test_throws MethodError Augmentor.applyeager(AggregateThenMapFun(mean, identity), nothing) + @test Augmentor.supports_eager(AggregateThenMapFun) === true + for img in (Augmentor.prepareaffine(rect), rect, view(rect, IdentityRange(1:2), IdentityRange(1:3))) + res = @inferred(Augmentor.applyeager(AggregateThenMapFun(mean, (x,a)->x), img)) + @test res == rect + @test typeof(res) <: Array{eltype(img)} + res = @inferred(Augmentor.applyeager(AggregateThenMapFun(mean, (x,a)->x-a), img)) + @test res ≈ rect .- mean(rect) + @test typeof(res) <: Array{Gray{Float64}} + end + img = OffsetArray(rect, -2, -1) + res = @inferred(Augmentor.applyeager(AggregateThenMapFun(mean, (x,a)->x), img)) + @test res == rect + @test typeof(res) <: Array{eltype(img)} + img = OffsetArray(rgb_rect, -2, -1) + res = @inferred(Augmentor.applyeager(AggregateThenMapFun(mean, (x,a)->x-a), img)) + @test res == rgb_rect .- mean(rgb_rect) + @test typeof(res) <: Array{RGB{Float64}} + end + @testset "affine" begin + @test Augmentor.supports_affine(AggregateThenMapFun) === false + end + @testset "affineview" begin + @test Augmentor.supports_affineview(AggregateThenMapFun) === false + end + @testset "lazy" begin + @test Augmentor.supports_lazy(AggregateThenMapFun) === true + res = @inferred(Augmentor.applylazy(AggregateThenMapFun(mean, (x,a)->x), rect)) + @test res == rect + @test res isa ReadonlyMappedArray + res = @inferred(Augmentor.applylazy(AggregateThenMapFun(mean, (x,a)->x-a), rgb_rect)) + @test res == mappedarray(x->x-mean(rgb_rect), rgb_rect) + @test typeof(res) <: MappedArrays.ReadonlyMappedArray{ColorTypes.RGB{Float64}} + end + @testset "view" begin + @test Augmentor.supports_view(AggregateThenMapFun) === false + end + @testset "stepview" begin + @test Augmentor.supports_stepview(AggregateThenMapFun) === false + end + @testset "permute" begin + @test Augmentor.supports_permute(AggregateThenMapFun) === false + end +end diff --git a/test/reference/rot45_crop_zoom_convert.txt b/test/reference/rot45_crop_zoom_convert.txt new file mode 100644 index 00000000..5285a4e3 --- /dev/null +++ b/test/reference/rot45_crop_zoom_convert.txt @@ -0,0 +1,13 @@ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ +▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀ \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 3697a8eb..bbcc2ce2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -46,6 +46,7 @@ tests = [ "tst_utils.jl", "operations/tst_channels.jl", "operations/tst_convert.jl", + "operations/tst_mapfun.jl", "operations/tst_noop.jl", "operations/tst_cache.jl", "operations/tst_rotation.jl", diff --git a/test/tst_augment.jl b/test/tst_augment.jl index a222b852..f03740a8 100644 --- a/test/tst_augment.jl +++ b/test/tst_augment.jl @@ -287,5 +287,16 @@ ops = (Rotate(45),Zoom(2)) @test_reference "reference/rot45_zoom.txt" img end +ops = (Rotate(45),CropSize(200,200),Zoom(1.1),ConvertEltype(RGB{Float64}),SplitChannels()) +@testset "$(str_showcompact(ops))" begin + wv1 = @inferred Augmentor._augment(camera, ops[1:3]) + wv2 = @inferred Augmentor._augment(camera, ops[1:4]) + wv3 = @inferred Augmentor._augment(camera, ops) + img = colorview(RGB{Float64}, wv3) + @test RGB{Float64}.(collect(wv1)) ≈ wv2 + @test wv1 ≈ img + @test_reference "reference/rot45_crop_zoom_convert.txt" wv2 +end + # just for code coverage @test typeof(@inferred(Augmentor.augment_impl(Rotate90()))) <: Expr