From 7c7f6ef82c9603162ffb4a740047c3e79c0d71b8 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 1 Oct 2017 06:53:23 -0500 Subject: [PATCH 1/3] Rework the broadcast API and document it (fixes #20740) --- NEWS.md | 6 + base/broadcast.jl | 529 ++++++++++++++++++++++++---------- base/cartesian.jl | 2 +- base/deprecated.jl | 11 + base/exports.jl | 1 + base/linalg/uniformscaling.jl | 12 +- base/sparse/higherorderfns.jl | 183 +++++------- doc/src/manual/interfaces.md | 201 +++++++++++++ doc/src/stdlib/arrays.md | 11 + test/broadcast.jl | 122 ++++++-- test/sparse/higherorderfns.jl | 18 +- 11 files changed, 786 insertions(+), 310 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1cee9e536ba0f..bc5018c3f1c46 100644 --- a/NEWS.md +++ b/NEWS.md @@ -290,6 +290,11 @@ This section lists changes that do not have deprecation warnings. Its return value has been removed. Use the `process_running` function to determine if a process has already exited. + * Broadcasting has been redesigned with an extensible public interface. The new API is + documented at https://docs.julialang.org/en/latest/manual/interfaces/#Interfaces-1. + `AbstractArray` types that specialized broadcasting using the old internal API will + need to switch to the new API. ([#20740]) + Library improvements -------------------- @@ -1490,6 +1495,7 @@ Command-line option changes [#20549]: https://github.com/JuliaLang/julia/issues/20549 [#20575]: https://github.com/JuliaLang/julia/issues/20575 [#20609]: https://github.com/JuliaLang/julia/issues/20609 +[#20740]: https://github.com/JuliaLang/julia/issues/20740 [#20816]: https://github.com/JuliaLang/julia/issues/20816 [#20889]: https://github.com/JuliaLang/julia/issues/20889 [#20912]: https://github.com/JuliaLang/julia/issues/20912 diff --git a/base/broadcast.jl b/base/broadcast.jl index ffa047e94c367..7a1fac61a8755 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -3,63 +3,271 @@ module Broadcast using Base.Cartesian -using Base: linearindices, tail, OneTo, to_shape, +using Base: Indices, OneTo, linearindices, tail, to_shape, _msk_end, unsafe_bitgetindex, bitcache_chunks, bitcache_size, dumpbitcache, nullable_returntype, null_safe_op, hasvalue, isoperator import Base: broadcast, broadcast! -export broadcast_getindex, broadcast_setindex!, dotview, @__dot__ +export BroadcastStyle, broadcast_indices, broadcast_similar, + broadcast_getindex, broadcast_setindex!, dotview, @__dot__ -const ScalarType = Union{Type{Any}, Type{Nullable}} +### Objects with customized broadcasting behavior should declare a BroadcastStyle + +""" +`BroadcastStyle` is an abstract type and trait-function used to determine behavior of +objects under broadcasting. `BroadcastStyle(typeof(x))` returns the style associated +with `x`. To customize the broadcasting behavior of a type, one can declare a style +by defining a type/method pair + + struct MyContainerStyle <: BroadcastStyle end + Base.BroadcastStyle(::Type{<:MyContainer}) = MyContainerStyle() + +One then writes method(s) (at least [`broadcast_similar`](@ref)) operating on +`MyContainerStyle`. There are also several pre-defined subtypes of `BroadcastStyle` +that you may be able to leverage; see the +[Interfaces chapter](@ref man-interfaces-broadcasting) for more information. +""" +abstract type BroadcastStyle end + +""" +`Broadcast.Style{C}()` defines a [`BroadcastStyle`](@ref) signaling through the type +parameter `C`. You can use this as an alternative to creating custom subtypes of `BroadcastStyle`, +for example + + Base.BroadcastStyle(::Type{<:MyContainer}) = Broadcast.Style{MyContainer}() + +There is a pre-defined [`broadcast_similar`](@ref) method + + broadcast_similar(f, ::Style{C}, ::Type{ElType}, inds, args...) = + similar(C, ElType, inds) + +Naturally you can specialize this for your particular `C` (e.g., `MyContainer`). +""" +struct Style{T} <: BroadcastStyle end + +BroadcastStyle(::Type{<:Tuple}) = Style{Tuple}() +BroadcastStyle(::Type{<:Nullable}) = Style{Nullable}() + +struct Unknown <: BroadcastStyle end +BroadcastStyle(::Type{Union{}}) = Unknown() # ambiguity resolution + +""" +`Broadcast.Scalar()` is a [`BroadcastStyle`](@ref) indicating that an object is not +treated as a container for the purposes of broadcasting. This is the default for objects +that have not customized `BroadcastStyle`. +""" +struct Scalar <: BroadcastStyle end +BroadcastStyle(::Type) = Scalar() +BroadcastStyle(::Type{<:Ptr}) = Scalar() + +""" +`Broadcast.AbstractArrayStyle{N} <: BroadcastStyle` is the abstract supertype for any style +associated with an `AbstractArray` type. +The `N` parameter is the dimensionality, which can be handy for AbstractArray types +that only support specific dimensionalities: + + struct SparseMatrixStyle <: Broadcast.AbstractArrayStyle{2} end + Base.BroadcastStyle(::Type{<:SparseMatrixCSC}) = SparseMatrixStyle() + +For AbstractArray types that support arbitrary dimensionality, `N` can be set to `Any`: + + struct MyArrayStyle <: Broadcast.AbstractArrayStyle{Any} end + Base.BroadcastStyle(::Type{<:MyArray}) = MyArrayStyle() + +In cases where you want to be able to mix multiple `AbstractArrayStyle`s and keep track +of dimensionality, your style needs to support a `Val` constructor: + + struct MyArrayStyleDim{N} <: Broadcast.AbstractArrayStyle{N} end + (::Type{<:MyArrayStyleDim})(::Val{N}) where N = MyArrayStyleDim{N}() + +Note that if two or more `AbstractArrayStyle` subtypes conflict, broadcasting machinery +will fall back to producing `Array`s. If this is undesirable, you may need to +define binary [`BroadcastStyle`](@ref) rules to control the output type. + +See also [`Broadcast.DefaultArrayStyle`](@ref). +""" +abstract type AbstractArrayStyle{N} <: BroadcastStyle end + +""" +`Broadcast.ArrayStyle{MyArrayType}()` is a [`BroadcastStyle`](@ref) indicating that an object +behaves as an array for broadcasting. It presents a simple way to construct +[`Broadcast.AbstractArrayStyle`](@ref)s for specific `AbstractArray` container types. +Broadcast styles created this way lose track of dimensionality; if keeping track is important +for your type, you should create your own custom [`Broadcast.AbstractArrayStyle`](@ref). +""" +struct ArrayStyle{A<:AbstractArray} <: AbstractArrayStyle{Any} end +(::Type{<:ArrayStyle{A}})(::Val) where A = A() + +""" +`Broadcast.DefaultArrayStyle{N}()` is a [`BroadcastStyle`](@ref) indicating that an object +behaves as an `N`-dimensional array for broadcasting. Specifically, `DefaultArrayStyle` is +used for any +AbstractArray type that hasn't defined a specialized style, and in the absence of +overrides from other `broadcast` arguments the resulting output type is `Array`. +When there are multiple inputs to `broadcast`, `DefaultArrayStyle` "wins" over [`Broadcast.Scalar`](@ref) +but "loses" to any other [`Broadcast.ArrayStyle`](@ref). +""" +struct DefaultArrayStyle{N} <: AbstractArrayStyle{N} end +(::Type{<:DefaultArrayStyle})(::Val{N}) where N = DefaultArrayStyle{N}() +const DefaultVectorStyle = DefaultArrayStyle{1} +const DefaultMatrixStyle = DefaultArrayStyle{2} +BroadcastStyle(::Type{<:AbstractArray{T,N}}) where {T,N} = DefaultArrayStyle{N}() +BroadcastStyle(::Type{<:Ref}) = DefaultArrayStyle{0}() + +# `ArrayConflict` is an internal type signaling that two or more different `AbstractArrayStyle` +# objects were supplied as arguments, and that no rule was defined for resolving the +# conflict. The resulting output is `Array`. While this is the same output type +# produced by `DefaultArrayStyle`, `ArrayConflict` "poisons" the BroadcastStyle so that +# 3 or more arguments still return an `ArrayConflict`. +struct ArrayConflict <: AbstractArrayStyle{Any} end + +### Binary BroadcastStyle rules +""" + BroadcastStyle(::Style1, ::Style2) = Style3() + +Indicate how to resolve different `BroadcastStyle`s. For example, + + Broadcast.rule(::Primary, ::Secondary) = Primary() + +would indicate that style `Primary` has precedence over `Secondary`. +You do not have to (and generally should not) define both argument orders. +The result does not have to be one of the input arguments, it could be a third type. + +Please see the [Interfaces chapter](@ref man-interfaces-broadcasting) of the manual for +more information. +""" +BroadcastStyle(::S, ::S) where S<:BroadcastStyle = S() # homogeneous types preserved +# Fall back to Unknown. This is necessary to implement argument-swapping +BroadcastStyle(::BroadcastStyle, ::BroadcastStyle) = Unknown() +# Unknown loses to everything +BroadcastStyle(::Unknown, ::Unknown) = Unknown() +BroadcastStyle(::S, ::Unknown) where S<:BroadcastStyle = S() +# Precedence rules +BroadcastStyle(::Style{Nullable}, ::Scalar) = Style{Nullable}() +BroadcastStyle(::Style{Tuple}, ::Scalar) = Style{Tuple}() +BroadcastStyle(a::AbstractArrayStyle{0}, ::Style{Tuple}) = typeof(a)(Val(1)) +BroadcastStyle(a::AbstractArrayStyle, ::Style{Tuple}) = a +BroadcastStyle(a::AbstractArrayStyle, ::Scalar) = a +BroadcastStyle(::A, ::A) where A<:ArrayStyle = A() +BroadcastStyle(::ArrayStyle, ::ArrayStyle) = Unknown() +BroadcastStyle(::A, ::A) where A<:AbstractArrayStyle = A() +Base.@pure function BroadcastStyle(a::A, b::B) where {A<:AbstractArrayStyle{M},B<:AbstractArrayStyle{N}} where {M,N} + if Base.typename(A).wrapper == Base.typename(B).wrapper + return A(_max(Val(M),Val(N))) + end + Unknown() +end +# Any specific array type beats DefaultArrayStyle +BroadcastStyle(a::AbstractArrayStyle{Any}, ::DefaultArrayStyle) = a +BroadcastStyle(a::AbstractArrayStyle{N}, ::DefaultArrayStyle{N}) where N = a +BroadcastStyle(a::AbstractArrayStyle{M}, ::DefaultArrayStyle{N}) where {M,N} = + typeof(a)(_max(Val(M),Val(N))) + +## Allocating the output container +""" + broadcast_similar(f, ::BroadcastStyle, ::Type{ElType}, inds, As...) + +Allocate an output object for [`broadcast`](@ref), appropriate for the indicated +[`Broadcast.BroadcastStyle`](@ref). `ElType` and `inds` specify the desired element type and indices of the +container. +`f` is the broadcast operation, and `As...` are the arguments supplied to `broadcast`. +""" +broadcast_similar(f, ::DefaultArrayStyle{N}, ::Type{ElType}, inds::Indices{N}, As...) where {N,ElType} = + similar(Array{ElType}, inds) +broadcast_similar(f, ::DefaultArrayStyle{N}, ::Type{Bool}, inds::Indices{N}, As...) where N = + similar(BitArray, inds) +# In cases of conflict we fall back on Array +broadcast_similar(f, ::ArrayConflict, ::Type{ElType}, inds::Indices, As...) where ElType = + similar(Array{ElType}, inds) +broadcast_similar(f, ::ArrayConflict, ::Type{Bool}, inds::Indices, As...) = + similar(BitArray, inds) + +## Computing the result's indices. Most types probably won't need to specialize this. +broadcast_indices() = () +broadcast_indices(::Type{T}) where T = () +broadcast_indices(A) = broadcast_indices(combine_styles(A), A) +broadcast_indices(::Scalar, A) = () +broadcast_indices(::Style{Nullable}, A) = () +broadcast_indices(::Style{Tuple}, A) = (OneTo(length(A)),) +broadcast_indices(::DefaultArrayStyle{0}, A::Ref) = () +broadcast_indices(::AbstractArrayStyle, A) = Base.indices(A) +""" + Base.broadcast_indices(::SrcStyle, A) + +Compute the indices for objects `A` with [`BroadcastStyle`](@ref) `SrcStyle`. +If needed, you can specialize this method for your styles. +You should only need to provide a custom implementation for non-AbstractArrayStyles. +""" +broadcast_indices + +### End of methods that users will typically have to specialize ### ## Broadcasting utilities ## -# fallbacks for some special cases -@inline broadcast(f, x::Number...) = f(x...) +# special cases defined for performance +broadcast(f, x::Number...) = f(x...) @inline broadcast(f, t::NTuple{N,Any}, ts::Vararg{NTuple{N,Any}}) where {N} = map(f, t, ts...) -broadcast!(::typeof(identity), x::Array{T,N}, y::Array{S,N}) where {T,S,N} = - size(x) == size(y) ? copy!(x, y) : broadcast_c!(identity, Array, Array, x, y) +@inline broadcast!(::typeof(identity), x::AbstractArray{T,N}, y::AbstractArray{S,N}) where {T,S,N} = + Base.indices(x) == Base.indices(y) ? copy!(x, y) : _broadcast!(identity, x, y) # special cases for "X .= ..." (broadcast!) assignments broadcast!(::typeof(identity), X::AbstractArray, x::Number) = fill!(X, x) broadcast!(f, X::AbstractArray, x::Number...) = (@inbounds for I in eachindex(X); X[I] = f(x...); end; X) -# logic for deciding the resulting container type -_containertype(::Type) = Any -_containertype(::Type{<:Ptr}) = Any -_containertype(::Type{<:Tuple}) = Tuple -_containertype(::Type{<:Ref}) = Array -_containertype(::Type{<:AbstractArray}) = Array -_containertype(::Type{<:Nullable}) = Nullable -containertype(x) = _containertype(typeof(x)) -containertype(ct1, ct2) = promote_containertype(containertype(ct1), containertype(ct2)) -@inline containertype(ct1, ct2, cts...) = promote_containertype(containertype(ct1), containertype(ct2, cts...)) - -promote_containertype(::Type{Array}, ::Type{Array}) = Array -promote_containertype(::Type{Array}, ct) = Array -promote_containertype(ct, ::Type{Array}) = Array -promote_containertype(::Type{Tuple}, ::ScalarType) = Tuple -promote_containertype(::ScalarType, ::Type{Tuple}) = Tuple -promote_containertype(::Type{Any}, ::Type{Nullable}) = Nullable -promote_containertype(::Type{Nullable}, ::Type{Any}) = Nullable -promote_containertype(::Type{T}, ::Type{T}) where {T} = T - -## Calculate the broadcast indices of the arguments, or error if incompatible -# array inputs -broadcast_indices() = () -broadcast_indices(A) = broadcast_indices(containertype(A), A) -@inline broadcast_indices(A, B...) = broadcast_shape(broadcast_indices(A), broadcast_indices(B...)) -broadcast_indices(::ScalarType, A) = () -broadcast_indices(::Type{Tuple}, A) = (OneTo(length(A)),) -broadcast_indices(::Type{Array}, A::Ref) = () -broadcast_indices(::Type{Array}, A) = indices(A) +## logic for deciding the BroadcastStyle +# Dimensionality: computing max(M,N) in the type domain so we preserve inferrability +_max(V1::Val{Any}, V2::Val{Any}) = Val(Any) +_max(V1::Val{Any}, V2::Val{N}) where N = Val(Any) +_max(V1::Val{N}, V2::Val{Any}) where N = Val(Any) +_max(V1::Val, V2::Val) = __max(longest(ntuple(identity, V1), ntuple(identity, V2))) +__max(::NTuple{N,Bool}) where N = Val(N) +longest(t1::Tuple, t2::Tuple) = (true, longest(Base.tail(t1), Base.tail(t2))...) +longest(::Tuple{}, t2::Tuple) = (true, longest((), Base.tail(t2))...) +longest(t1::Tuple, ::Tuple{}) = (true, longest(Base.tail(t1), ())...) +longest(::Tuple{}, ::Tuple{}) = () + +# combine_styles operates on values (arbitrarily many) +combine_styles(c) = result_style(BroadcastStyle(typeof(c))) +combine_styles(c1, c2) = result_style(combine_styles(c1), combine_styles(c2)) +combine_styles(c1, c2, cs...) = result_style(combine_styles(c1), combine_styles(c2, cs...)) + +# result_style works on types (singletons and pairs), and leverages `BroadcastStyle` +result_style(s::BroadcastStyle) = s +result_style(s1::S, s2::S) where S<:BroadcastStyle = S() +# Test both orders so users typically only have to declare one order +result_style(s1, s2) = result_join(s1, s2, BroadcastStyle(s1, s2), BroadcastStyle(s2, s1)) + +# result_join is the final arbiter. Because `BroadcastStyle` for undeclared pairs results in Unknown, +# we defer to any case where the result of `BroadcastStyle` is known. +result_join(::Any, ::Any, ::Unknown, ::Unknown) = Unknown() +result_join(::Any, ::Any, ::Unknown, s::BroadcastStyle) = s +result_join(::Any, ::Any, s::BroadcastStyle, ::Unknown) = s +# For AbstractArray types with specialized broadcasting and undefined precedence rules, +# we have to signal conflict. Because ArrayConflict is a subtype of AbstractArray, +# this will "poison" any future operations (if we instead returned `DefaultArrayStyle`, then for +# 3-array broadcasting the returned type would depend on argument order). +result_join(::AbstractArrayStyle, ::AbstractArrayStyle, ::Unknown, ::Unknown) = + ArrayConflict() +# Fallbacks in case users define `rule` for both argument-orders (not recommended) +result_join(::Any, ::Any, ::S, ::S) where S<:BroadcastStyle = S() +@noinline function result_join(::S, ::T, ::U, ::V) where {S,T,U,V} + error(""" +conflicting broadcast rules defined + Broadcast.BroadcastStyle(::$S, ::$T) = $U() + Broadcast.BroadcastStyle(::$T, ::$S) = $V() +One of these should be undefined (and thus return Broadcast.Unknown).""") +end + +# Indices utilities +combine_indices(A, B...) = broadcast_shape(broadcast_indices(A), combine_indices(B...)) +combine_indices(A) = broadcast_indices(A) # shape (i.e., tuple-of-indices) inputs broadcast_shape(shape::Tuple) = shape -@inline broadcast_shape(shape::Tuple, shape1::Tuple, shapes::Tuple...) = broadcast_shape(_bcs(shape, shape1), shapes...) +broadcast_shape(shape::Tuple, shape1::Tuple, shapes::Tuple...) = broadcast_shape(_bcs(shape, shape1), shapes...) # _bcs consolidates two shapes into a single output shape _bcs(::Tuple{}, ::Tuple{}) = () -@inline _bcs(::Tuple{}, newshape::Tuple) = (newshape[1], _bcs((), tail(newshape))...) -@inline _bcs(shape::Tuple, ::Tuple{}) = (shape[1], _bcs(tail(shape), ())...) -@inline function _bcs(shape::Tuple, newshape::Tuple) +_bcs(::Tuple{}, newshape::Tuple) = (newshape[1], _bcs((), tail(newshape))...) +_bcs(shape::Tuple, ::Tuple{}) = (shape[1], _bcs(tail(shape), ())...) +function _bcs(shape::Tuple, newshape::Tuple) return (_bcs1(shape[1], newshape[1]), _bcs(tail(shape), tail(newshape))...) end # _bcs1 handles the logic for a single dimension @@ -95,6 +303,7 @@ end # is appropriate for a particular broadcast array/scalar. `keep` is a # NTuple{N,Bool}, where keep[d] == true means that one should preserve # I[d]; if false, replace it with Idefault[d]. +# If dot-broadcasting were already defined, this would be `ifelse.(keep, I, Idefault)`. @inline newindex(I::CartesianIndex, keep, Idefault) = CartesianIndex(_newindex(I.I, keep, Idefault)) @inline _newindex(I, keep, Idefault) = (ifelse(keep[1], I[1], Idefault[1]), _newindex(tail(I), tail(keep), tail(Idefault))...) @@ -102,7 +311,7 @@ end # newindexer(shape, A) generates `keep` and `Idefault` (for use by # `newindex` above) for a particular array `A`, given the -# broadcast_indices `shape` +# broadcast indices `shape` # `keep` is equivalent to map(==, indices(A), shape) (but see #17126) @inline newindexer(shape, A) = shapeindexer(shape, broadcast_indices(A)) @inline shapeindexer(shape, indsA::Tuple{}) = (), () @@ -126,12 +335,12 @@ end (keep, keeps...), (Idefault, Idefaults...) end -Base.@propagate_inbounds _broadcast_getindex(A, I) = _broadcast_getindex(containertype(A), A, I) -# `(x,)`, where `x` is a scalar, broadcasts the same way as `[x]` or `x` -Base.@propagate_inbounds _broadcast_getindex(::Type{Tuple}, A::Tuple{Any}, I) = A[1] -Base.@propagate_inbounds _broadcast_getindex(::Type{Array}, A::Ref, I) = A[] -Base.@propagate_inbounds _broadcast_getindex(::ScalarType, A, I) = A +Base.@propagate_inbounds _broadcast_getindex(::Type{T}, I) where T = T +Base.@propagate_inbounds _broadcast_getindex(A, I) = _broadcast_getindex(combine_styles(A), A, I) +Base.@propagate_inbounds _broadcast_getindex(::DefaultArrayStyle{0}, A::Ref, I) = A[] +Base.@propagate_inbounds _broadcast_getindex(::Union{Unknown,Scalar,Style{Nullable}}, A, I) = A Base.@propagate_inbounds _broadcast_getindex(::Any, A, I) = A[I] +Base.@propagate_inbounds _broadcast_getindex(::Style{Tuple}, A::Tuple{Any}, I) = A[1] ## Broadcasting core # nargs encodes the number of As arguments (which matches the number @@ -203,9 +412,12 @@ arguments to `f` unless it is also listed in the `As`, as in `broadcast!(f, A, A, B)` to perform `A[:] = broadcast(f, A, B)`. """ @inline broadcast!(f, C::AbstractArray, A, Bs::Vararg{Any,N}) where {N} = - broadcast_c!(f, containertype(C), containertype(A, Bs...), C, A, Bs...) -@inline function broadcast_c!(f, ::Type, ::Type, C, A, Bs::Vararg{Any,N}) where N - shape = indices(C) + _broadcast!(f, C, A, Bs...) + +# This indirection allows size-dependent implementations (e.g., see the copying `identity` +# specialization above) +@inline function _broadcast!(f, C, A, Bs::Vararg{Any,N}) where N + shape = broadcast_indices(C) @boundscheck check_broadcast_indices(shape, A, Bs...) keeps, Idefaults = map_newindexer(shape, A, Bs) iter = CartesianRange(shape) @@ -213,7 +425,9 @@ as in `broadcast!(f, A, A, B)` to perform `A[:] = broadcast(f, A, B)`. return C end -# broadcast with computed element type +# broadcast with element type adjusted on-the-fly. This widens the element type of +# B as needed (allocating a new container and copying previously-computed values) to +# accommodate any incompatible new elements. @generated function _broadcast!(f, B::AbstractArray, keeps::K, Idefaults::ID, As::AT, ::Val{nargs}, iter, st, count) where {K,ID,AT,nargs} quote $(Expr(:meta, :noinline)) @@ -234,13 +448,14 @@ end if S <: eltype(B) @inbounds B[I] = V else - R = typejoin(eltype(B), S) - new = similar(B, R) + # This element type doesn't fit in B. Allocate a new B with wider eltype, + # copy over old values, and continue + newB = Base.similar(B, typejoin(eltype(B), S)) for II in Iterators.take(iter, count) - new[II] = B[II] + newB[II] = B[II] end - new[I] = V - return _broadcast!(f, new, keeps, Idefaults, As, Val(nargs), iter, st, count+1) + newB[I] = V + return _broadcast!(f, newB, keeps, Idefaults, As, Val(nargs), iter, st, count+1) end count += 1 end @@ -248,123 +463,31 @@ end end end -# broadcast methods that dispatch on the type found by inference -function broadcast_t(f, ::Type{Any}, shape, iter, As...) - nargs = length(As) - keeps, Idefaults = map_newindexer(shape, As) - st = start(iter) - I, st = next(iter, st) - val = f([ _broadcast_getindex(As[i], newindex(I, keeps[i], Idefaults[i])) for i=1:nargs ]...) - if val isa Bool - B = similar(BitArray, shape) - else - B = similar(Array{typeof(val)}, shape) - end - B[I] = val - return _broadcast!(f, B, keeps, Idefaults, As, Val(nargs), iter, st, 1) -end -@inline function broadcast_t(f, T, shape, iter, A, Bs::Vararg{Any,N}) where N - C = similar(Array{T}, shape) - keeps, Idefaults = map_newindexer(shape, A, Bs) - _broadcast!(f, C, keeps, Idefaults, A, Bs, Val(N), iter) - return C -end - -# default to BitArray for broadcast operations producing Bool, to save 8x space -# in the common case where this is used for logical array indexing; in -# performance-critical cases where Array{Bool} is desired, one can always -# use broadcast! instead. -@inline function broadcast_t(f, ::Type{Bool}, shape, iter, A, Bs::Vararg{Any,N}) where N - C = similar(BitArray, shape) - keeps, Idefaults = map_newindexer(shape, A, Bs) - _broadcast!(f, C, keeps, Idefaults, A, Bs, Val(N), iter) - return C -end - maptoTuple(f) = Tuple{} maptoTuple(f, a, b...) = Tuple{f(a), maptoTuple(f, b...).types...} # An element type satisfying for all A: # broadcast_getindex( -# containertype(A), +# combine_styles(A), # A, broadcast_indices(A) # )::_broadcast_getindex_eltype(A) -_broadcast_getindex_eltype(A) = _broadcast_getindex_eltype(containertype(A), A) -_broadcast_getindex_eltype(::ScalarType, T::Type) = Type{T} -_broadcast_getindex_eltype(::ScalarType, A) = typeof(A) -_broadcast_getindex_eltype(::Any, A) = eltype(A) # Tuple, Array, etc. +_broadcast_getindex_eltype(A) = _broadcast_getindex_eltype(combine_styles(A), A) +_broadcast_getindex_eltype(::Scalar, ::Type{T}) where T = Type{T} +_broadcast_getindex_eltype(::Union{Unknown,Scalar,Style{Nullable}}, A) = typeof(A) +_broadcast_getindex_eltype(::BroadcastStyle, A) = eltype(A) # Tuple, Array, etc. # An element type satisfying for all A: # unsafe_get(A)::unsafe_get_eltype(A) _unsafe_get_eltype(x::Nullable) = eltype(x) -_unsafe_get_eltype(T::Type) = Type{T} +_unsafe_get_eltype(::Type{T}) where T = Type{T} _unsafe_get_eltype(x) = typeof(x) # Inferred eltype of result of broadcast(f, xs...) -_broadcast_eltype(f, A, As...) = +combine_eltypes(f, A, As...) = Base._return_type(f, maptoTuple(_broadcast_getindex_eltype, A, As...)) _nullable_eltype(f, A, As...) = Base._return_type(f, maptoTuple(_unsafe_get_eltype, A, As...)) -# broadcast methods that dispatch on the type of the final container -@inline function broadcast_c(f, ::Type{Array}, A, Bs...) - T = _broadcast_eltype(f, A, Bs...) - shape = broadcast_indices(A, Bs...) - iter = CartesianRange(shape) - if Base._isleaftype(T) - return broadcast_t(f, T, shape, iter, A, Bs...) - end - if isempty(iter) - return similar(Array{T}, shape) - end - return broadcast_t(f, Any, shape, iter, A, Bs...) -end -@inline function broadcast_c(f, ::Type{Nullable}, a...) - nonnull = all(hasvalue, a) - S = _nullable_eltype(f, a...) - if Base._isleaftype(S) && null_safe_op(f, maptoTuple(_unsafe_get_eltype, - a...).types...) - Nullable{S}(f(map(unsafe_get, a)...), nonnull) - else - if nonnull - Nullable(f(map(unsafe_get, a)...)) - else - Nullable{nullable_returntype(S)}() - end - end -end -@inline broadcast_c(f, ::Type{Any}, a...) = f(a...) -@inline broadcast_c(f, ::Type{Tuple}, A, Bs...) = - tuplebroadcast(f, tuplebroadcast_maxtuple(A, Bs...), A, Bs...) -@inline tuplebroadcast(f, ::NTuple{N,Any}, As...) where {N} = - ntuple(k -> f(tuplebroadcast_getargs(As, k)...), Val(N)) -@inline tuplebroadcast(f, ::NTuple{N,Any}, ::Type{T}, As...) where {N,T} = - ntuple(k -> f(T, tuplebroadcast_getargs(As, k)...), Val(N)) -# When the result of broadcast is a tuple it can only come from mixing n-tuples -# of the same length with scalars and 1-tuples. So, in order to have a -# type-stable broadcast, we need to find a tuple of maximum length (except when -# there are only scalars, empty tuples and 1-tuples, in which case the -# returned value will be an empty tuple). -# The following methods compare broadcast arguments pairwise to determine the -# length of the final tuple. -tuplebroadcast_maxtuple(A, B) = - _tuplebroadcast_maxtuple(containertype(A), containertype(B), A, B) -@inline tuplebroadcast_maxtuple(A, Bs...) = - tuplebroadcast_maxtuple(A, tuplebroadcast_maxtuple(Bs...)) -tuplebroadcast_maxtuple(A::NTuple{N,Any}, ::NTuple{N,Any}...) where {N} = A -# Here we use the containertype trait to easier disambiguate between methods -_tuplebroadcast_maxtuple(::Type{Any}, ::Type{Any}, A, B) = (nothing,) -_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Any}, A, B) = A -_tuplebroadcast_maxtuple(::Type{Any}, ::Type{Tuple}, A, B) = B -_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A, B::Tuple{Any}) = A -_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A::Tuple{Any}, B) = B -_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A::Tuple{Any}, ::Tuple{Any}) = A -_tuplebroadcast_maxtuple(::Type{Tuple}, ::Type{Tuple}, A, B) = - throw(DimensionMismatch("tuples could not be broadcast to a common size")) -tuplebroadcast_getargs(::Tuple{}, k) = () -@inline tuplebroadcast_getargs(As, k) = - (_broadcast_getindex(first(As), k), tuplebroadcast_getargs(tail(As), k)...) - """ broadcast(f, As...) @@ -452,7 +575,89 @@ julia> (1 + im) ./ Nullable{Int}() Nullable{Complex{Float64}}() ``` """ -@inline broadcast(f, A, Bs...) = broadcast_c(f, containertype(A, Bs...), A, Bs...) +@inline broadcast(f, A, Bs...) = + broadcast(f, combine_styles(A, Bs...), nothing, nothing, A, Bs...) + +@inline broadcast(f, s::BroadcastStyle, ::Void, ::Void, A, Bs...) = + broadcast(f, s, combine_eltypes(f, A, Bs...), combine_indices(A, Bs...), + A, Bs...) + +const NonleafHandlingTypes = Union{DefaultArrayStyle,ArrayConflict} + +@inline function broadcast(f, s::NonleafHandlingTypes, ::Type{ElType}, inds::Indices, As...) where ElType + if !Base._isleaftype(ElType) + return broadcast_nonleaf(f, s, ElType, inds, As...) + end + dest = broadcast_similar(f, s, ElType, inds, As...) + broadcast!(f, dest, As...) +end + +@inline function broadcast(f, s::BroadcastStyle, ::Type{ElType}, inds::Indices, As...) where ElType + dest = broadcast_similar(f, s, ElType, inds, As...) + broadcast!(f, dest, As...) +end + +# When ElType is not concrete, use narrowing. Use the first element of each input to determine +# the starting output eltype; the _broadcast! method will widen `dest` as needed to +# accommodate later values. +function broadcast_nonleaf(f, s::NonleafHandlingTypes, ::Type{ElType}, shape::Indices, As...) where ElType + nargs = length(As) + iter = CartesianRange(shape) + if isempty(iter) + return Base.similar(Array{ElType}, shape) + end + keeps, Idefaults = map_newindexer(shape, As) + st = start(iter) + I, st = next(iter, st) + val = f([ _broadcast_getindex(As[i], newindex(I, keeps[i], Idefaults[i])) for i=1:nargs ]...) + if val isa Bool + dest = Base.similar(BitArray, shape) + else + dest = Base.similar(Array{typeof(val)}, shape) + end + dest[I] = val + return _broadcast!(f, dest, keeps, Idefaults, As, Val(nargs), iter, st, 1) +end + +@inline function broadcast(f, ::Style{Nullable}, ::Void, ::Void, a...) + nonnull = all(hasvalue, a) + S = _nullable_eltype(f, a...) + if Base._isleaftype(S) && null_safe_op(f, maptoTuple(_unsafe_get_eltype, + a...).types...) + Nullable{S}(f(map(unsafe_get, a)...), nonnull) + else + if nonnull + Nullable(f(map(unsafe_get, a)...)) + else + Nullable{nullable_returntype(S)}() + end + end +end + +broadcast(f, ::Union{Scalar,Unknown}, ::Void, ::Void, a...) = f(a...) + +@inline broadcast(f, ::Style{Tuple}, ::Void, ::Void, A, Bs...) = + tuplebroadcast(f, longest_tuple(A, Bs...), A, Bs...) +@inline tuplebroadcast(f, ::NTuple{N,Any}, As...) where {N} = + ntuple(k -> f(tuplebroadcast_getargs(As, k)...), Val(N)) +@inline tuplebroadcast(f, ::NTuple{N,Any}, ::Type{T}, As...) where {N,T} = + ntuple(k -> f(T, tuplebroadcast_getargs(As, k)...), Val(N)) +longest_tuple(A::Tuple, B::Tuple, Bs...) = longest_tuple(_longest_tuple(A, B), Bs...) +longest_tuple(A, B::Tuple, Bs...) = longest_tuple(B, Bs...) +longest_tuple(A::Tuple, B, Bs...) = longest_tuple(A, Bs...) +longest_tuple(A::Tuple) = A +# Support only 1-tuples and N-tuples where there are no conflicts in N +_longest_tuple(A::Tuple{Any}, B::Tuple{Any}) = A +_longest_tuple(A::NTuple{N,Any}, B::NTuple{N,Any}) where N = A +_longest_tuple(A::NTuple{N,Any}, B::Tuple{Any}) where N = A +_longest_tuple(A::Tuple{Any}, B::NTuple{N,Any}) where N = B +@noinline _longest_tuple(A, B) = + throw(DimensionMismatch("tuples $A and $B could not be broadcast to a common size")) + +tuplebroadcast_getargs(::Tuple{}, k) = () +@inline tuplebroadcast_getargs(As, k) = + (_broadcast_getindex(first(As), k), tuplebroadcast_getargs(tail(As), k)...) + """ broadcast_getindex(A, inds...) @@ -494,15 +699,19 @@ julia> broadcast_getindex(C,[1,2,10]) 15 ``` """ -broadcast_getindex(src::AbstractArray, I::AbstractArray...) = broadcast_getindex!(similar(Array{eltype(src)}, broadcast_indices(I...)), src, I...) +broadcast_getindex(src::AbstractArray, I::AbstractArray...) = + broadcast_getindex!(Base.similar(Array{eltype(src)}, combine_indices(I...)), + src, + I...) + @generated function broadcast_getindex!(dest::AbstractArray, src::AbstractArray, I::AbstractArray...) N = length(I) Isplat = Expr[:(I[$d]) for d = 1:N] quote @nexprs $N d->(I_d = I[d]) - check_broadcast_indices(indices(dest), $(Isplat...)) # unnecessary if this function is never called directly + check_broadcast_indices(Base.indices(dest), $(Isplat...)) # unnecessary if this function is never called directly checkbounds(src, $(Isplat...)) - @nexprs $N d->(@nexprs $N k->(Ibcast_d_k = indices(I_k, d) == OneTo(1))) + @nexprs $N d->(@nexprs $N k->(Ibcast_d_k = Base.indices(I_k, d) == OneTo(1))) @nloops $N i dest d->(@nexprs $N k->(j_d_k = Ibcast_d_k ? 1 : i_d)) begin @nexprs $N k->(@inbounds J_k = @nref $N I_k d->j_d_k) @inbounds (@nref $N dest i) = (@nref $N src J) @@ -523,9 +732,9 @@ position in `X` at the indices in `A` given by the same positions in `inds`. quote @nexprs $N d->(I_d = I[d]) checkbounds(A, $(Isplat...)) - shape = broadcast_indices($(Isplat...)) + shape = combine_indices($(Isplat...)) @nextract $N shape d->(length(shape) < d ? OneTo(1) : shape[d]) - @nexprs $N d->(@nexprs $N k->(Ibcast_d_k = indices(I_k, d) == 1:1)) + @nexprs $N d->(@nexprs $N k->(Ibcast_d_k = Base.indices(I_k, d) == 1:1)) if !isa(x, AbstractArray) xA = convert(eltype(A), x) @nloops $N i d->shape_d d->(@nexprs $N k->(j_d_k = Ibcast_d_k ? 1 : i_d)) begin diff --git a/base/cartesian.jl b/base/cartesian.jl index 77af57775ea85..f0fbd28be75db 100644 --- a/base/cartesian.jl +++ b/base/cartesian.jl @@ -41,7 +41,7 @@ end function _nloops(N::Int, itersym::Symbol, arraysym::Symbol, args::Expr...) @gensym d - _nloops(N, itersym, :($d->indices($arraysym, $d)), args...) + _nloops(N, itersym, :($d->Base.indices($arraysym, $d)), args...) end function _nloops(N::Int, itersym::Symbol, rangeexpr::Expr, args::Expr...) diff --git a/base/deprecated.jl b/base/deprecated.jl index e8cc748edfe32..af51f1de55f9b 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -2119,6 +2119,17 @@ end finalizer(f::Ptr{Void}, o::Ptr{Void}) = invoke(finalizer, Tuple{Ptr{Void}, Any}, f, o) finalizer(f::Ptr{Void}, o::Function) = invoke(finalizer, Tuple{Ptr{Void}, Any}, f, o) +# Broadcast extension API (#23939) +@eval Broadcast begin + Base.@deprecate_binding containertype combine_styles false + Base.@deprecate_binding _containertype BroadcastStyle false + Base.@deprecate_binding promote_containertype BroadcastStyle false + Base.@deprecate_binding broadcast_c! broadcast! false ", broadcast_c!(f, ::Type, ::Type, C, As...) should become broadcast!(f, C, As...) (see the manual chapter Interfaces)" + Base.@deprecate_binding broadcast_c broadcast false ", `broadcast_c(f, ::Type{C}, As...)` should become `broadcast(f, C, nothing, nothing, As...))` (see the manual chapter Interfaces)" + Base.@deprecate_binding broadcast_t broadcast false ", broadcast_t(f, ::Type{ElType}, shape, iter, As...)` should become `broadcast(f, Broadcast.DefaultArrayStyle{N}(), ElType, shape, As...))` (see the manual chapter Interfaces)" +end + + # END 0.7 deprecations # BEGIN 1.0 deprecations diff --git a/base/exports.jl b/base/exports.jl index 275fc68be1c17..21c5c51c0982d 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -18,6 +18,7 @@ export Threads, Iterators, Distributed, + Broadcast, # Types AbstractChannel, diff --git a/base/linalg/uniformscaling.jl b/base/linalg/uniformscaling.jl index 229427da856f1..d87e42b4a2e78 100644 --- a/base/linalg/uniformscaling.jl +++ b/base/linalg/uniformscaling.jl @@ -99,7 +99,7 @@ for (t1, t2) in ((:UnitUpperTriangular, :UpperTriangular), ($op)(UL::$t2, J::UniformScaling) = ($t2)(($op)(UL.data, J)) function ($op)(UL::$t1, J::UniformScaling) - ULnew = copy_oftype(UL.data, Base.Broadcast._broadcast_eltype($op, UL, J)) + ULnew = copy_oftype(UL.data, Broadcast.combine_eltypes($op, UL, J)) for i = 1:size(ULnew, 1) ULnew[i,i] = ($op)(1, J.λ) end @@ -110,7 +110,7 @@ for (t1, t2) in ((:UnitUpperTriangular, :UpperTriangular), end function (-)(J::UniformScaling, UL::Union{UpperTriangular,UnitUpperTriangular}) - ULnew = similar(parent(UL), Base.Broadcast._broadcast_eltype(-, J, UL)) + ULnew = similar(parent(UL), Broadcast.combine_eltypes(-, J, UL)) n = size(ULnew, 1) ULold = UL.data for j = 1:n @@ -126,7 +126,7 @@ function (-)(J::UniformScaling, UL::Union{UpperTriangular,UnitUpperTriangular}) return UpperTriangular(ULnew) end function (-)(J::UniformScaling, UL::Union{LowerTriangular,UnitLowerTriangular}) - ULnew = similar(parent(UL), Base.Broadcast._broadcast_eltype(-, J, UL)) + ULnew = similar(parent(UL), Broadcast.combine_eltypes(-, J, UL)) n = size(ULnew, 1) ULold = UL.data for j = 1:n @@ -144,7 +144,7 @@ end function (+)(A::AbstractMatrix, J::UniformScaling) n = checksquare(A) - B = similar(A, Base.Broadcast._broadcast_eltype(+, A, J)) + B = similar(A, Broadcast.combine_eltypes(+, A, J)) copy!(B,A) @inbounds for i = 1:n B[i,i] += J.λ @@ -154,7 +154,7 @@ end function (-)(A::AbstractMatrix, J::UniformScaling) n = checksquare(A) - B = similar(A, Base.Broadcast._broadcast_eltype(-, A, J)) + B = similar(A, Broadcast.combine_eltypes(-, A, J)) copy!(B, A) @inbounds for i = 1:n B[i,i] -= J.λ @@ -163,7 +163,7 @@ function (-)(A::AbstractMatrix, J::UniformScaling) end function (-)(J::UniformScaling, A::AbstractMatrix) n = checksquare(A) - B = convert(AbstractMatrix{Base.Broadcast._broadcast_eltype(-, J, A)}, -A) + B = convert(AbstractMatrix{Broadcast.combine_eltypes(-, J, A)}, -A) @inbounds for j = 1:n B[j,j] += J.λ end diff --git a/base/sparse/higherorderfns.jl b/base/sparse/higherorderfns.jl index 843036167ab5a..1c551fe446aba 100644 --- a/base/sparse/higherorderfns.jl +++ b/base/sparse/higherorderfns.jl @@ -5,12 +5,11 @@ module HigherOrderFns # This module provides higher order functions specialized for sparse arrays, # particularly map[!]/broadcast[!] for SparseVectors and SparseMatrixCSCs at present. import Base: map, map!, broadcast, broadcast! -import Base.Broadcast: _containertype, promote_containertype, - broadcast_indices, broadcast_c, broadcast_c! using Base: front, tail, to_shape using ..SparseArrays: SparseVector, SparseMatrixCSC, AbstractSparseVector, AbstractSparseMatrix, AbstractSparseArray, indtype, nnz, nzrange +using Base.Broadcast: BroadcastStyle # This module is organized as follows: # (1) Define a common interface to SparseVectors and SparseMatrixCSCs sufficient for @@ -23,10 +22,10 @@ using ..SparseArrays: SparseVector, SparseMatrixCSC, AbstractSparseVector, # (7) Define _broadcast_[not]zeropres! specialized for a single (input) sparse vector/matrix. # (8) Define _broadcast_[not]zeropres! specialized for a pair of (input) sparse vectors/matrices. # (9) Define general _broadcast_[not]zeropres! capable of handling >2 (input) sparse vectors/matrices. -# (10) Define (broadcast[!]) methods handling combinations of broadcast scalars and sparse vectors/matrices. -# (11) Define (broadcast[!]) methods handling combinations of scalars, sparse vectors/matrices, +# (10) Define broadcast methods handling combinations of broadcast scalars and sparse vectors/matrices. +# (11) Define broadcast[!] methods handling combinations of scalars, sparse vectors/matrices, # structured matrices, and one- and two-dimensional Arrays. -# (12) Define (map[!]) methods handling combinations of sparse and structured matrices. +# (12) Define map[!] methods handling combinations of sparse and structured matrices. # (1) The definitions below provide a common interface to sparse vectors and matrices @@ -85,7 +84,7 @@ function _noshapecheck_map(f::Tf, A::SparseVecOrMat, Bs::Vararg{SparseVecOrMat,N fofzeros = f(_zeros_eltypes(A, Bs...)...) fpreszeros = _iszero(fofzeros) maxnnzC = fpreszeros ? min(length(A), _sumnnzs(A, Bs...)) : length(A) - entrytypeC = Base.Broadcast._broadcast_eltype(f, A, Bs...) + entrytypeC = Base.Broadcast.combine_eltypes(f, A, Bs...) indextypeC = _promote_indtype(A, Bs...) C = _allocres(size(A), indextypeC, entrytypeC, maxnnzC) return fpreszeros ? _map_zeropres!(f, C, A, Bs...) : @@ -126,8 +125,8 @@ function _diffshape_broadcast(f::Tf, A::SparseVecOrMat, Bs::Vararg{SparseVecOrMa fofzeros = f(_zeros_eltypes(A, Bs...)...) fpreszeros = _iszero(fofzeros) indextypeC = _promote_indtype(A, Bs...) - entrytypeC = Base.Broadcast._broadcast_eltype(f, A, Bs...) - shapeC = to_shape(Base.Broadcast.broadcast_indices(A, Bs...)) + entrytypeC = Base.Broadcast.combine_eltypes(f, A, Bs...) + shapeC = to_shape(Base.Broadcast.combine_indices(A, Bs...)) maxnnzC = fpreszeros ? _checked_maxnnzbcres(shapeC, A, Bs...) : _densennz(shapeC) C = _allocres(shapeC, indextypeC, entrytypeC, maxnnzC) return fpreszeros ? _broadcast_zeropres!(f, C, A, Bs...) : @@ -897,29 +896,40 @@ end end -# (10) broadcast[!] over combinations of broadcast scalars and sparse vectors/matrices +# (10) broadcast over combinations of broadcast scalars and sparse vectors/matrices -# broadcast shape promotion for combinations of sparse arrays and other types -broadcast_indices(::Type{AbstractSparseArray}, A) = indices(A) # broadcast container type promotion for combinations of sparse arrays and other types -_containertype(::Type{<:SparseVecOrMat}) = AbstractSparseArray -# combinations of sparse arrays with broadcast scalars should yield sparse arrays -promote_containertype(::Type{Any}, ::Type{AbstractSparseArray}) = AbstractSparseArray -promote_containertype(::Type{AbstractSparseArray}, ::Type{Any}) = AbstractSparseArray -# combinations of sparse arrays with tuples should divert to the generic AbstractArray broadcast code -# (we handle combinations involving dense vectors/matrices below) -promote_containertype(::Type{Tuple}, ::Type{AbstractSparseArray}) = Array -promote_containertype(::Type{AbstractSparseArray}, ::Type{Tuple}) = Array +struct SparseVecStyle <: Broadcast.AbstractArrayStyle{1} end +struct SparseMatStyle <: Broadcast.AbstractArrayStyle{2} end +Broadcast.BroadcastStyle(::Type{<:SparseVector}) = SparseVecStyle() +Broadcast.BroadcastStyle(::Type{<:SparseMatrixCSC}) = SparseMatStyle() +const SPVM = Union{SparseVecStyle,SparseMatStyle} -# broadcast[!] entry points for combinations of sparse arrays and other (scalar) types -@inline function broadcast_c(f, ::Type{AbstractSparseArray}, mixedargs::Vararg{Any,N}) where N +# SparseVecStyle handles 0-1 dimensions, SparseMatStyle 0-2 dimensions. +# SparseVecStyle promotes to SparseMatStyle for 2 dimensions. +# Fall back to DefaultArrayStyle for higher dimensionality. +SparseVecStyle(::Val{0}) = SparseVecStyle() +SparseVecStyle(::Val{1}) = SparseVecStyle() +SparseVecStyle(::Val{2}) = SparseMatStyle() +SparseVecStyle(::Val{N}) where N = Broadcast.DefaultArrayStyle{N}() +SparseMatStyle(::Val{0}) = SparseMatStyle() +SparseMatStyle(::Val{1}) = SparseMatStyle() +SparseMatStyle(::Val{2}) = SparseMatStyle() +SparseMatStyle(::Val{N}) where N = Broadcast.DefaultArrayStyle{N}() + +Broadcast.BroadcastStyle(::SparseMatStyle, ::SparseVecStyle) = SparseMatStyle() + +# Tuples promote to dense +Broadcast.BroadcastStyle(::SparseVecStyle, ::Broadcast.Style{Tuple}) = Broadcast.DefaultArrayStyle{1}() +Broadcast.BroadcastStyle(::SparseMatStyle, ::Broadcast.Style{Tuple}) = Broadcast.DefaultArrayStyle{2}() + +# broadcast entry points for combinations of sparse arrays and other (scalar) types +function broadcast(f, ::SPVM, ::Void, ::Void, mixedargs::Vararg{Any,N}) where N parevalf, passedargstup = capturescalars(f, mixedargs) return broadcast(parevalf, passedargstup...) end -@inline function broadcast_c!(f, ::Type{AbstractSparseArray}, dest::SparseVecOrMat, mixedsrcargs::Vararg{Any,N}) where N - parevalf, passedsrcargstup = capturescalars(f, mixedsrcargs) - return broadcast!(parevalf, dest, passedsrcargstup...) -end +# for broadcast! see (11) + # capturescalars takes a function (f) and a tuple of mixed sparse vectors/matrices and # broadcast scalar arguments (mixedargs), and returns a function (parevalf, i.e. partially # evaluated f) and a reduced argument tuple (passedargstup) containing only the sparse @@ -966,99 +976,60 @@ broadcast(f::Tf, A::SparseMatrixCSC, ::Type{T}) where {Tf,T} = broadcast(x -> f( # for combinations involving only scalars, sparse arrays, structured matrices, and dense # vectors/matrices, promote all structured matrices and dense vectors/matrices to sparse # and rebroadcast. otherwise, divert to generic AbstractArray broadcast code. -# -# this requires three steps: segregate combinations to promote to sparse via Broadcast's -# containertype promotion and dispatch layer (broadcast_c[!], containertype, -# promote_containertype), separate ambiguous cases from the preceding dispatch -# layer in sparse broadcast's internal containertype promotion and dispatch layer -# (spbroadcast_c[!], spcontainertype, promote_spcontainertype), and then promote -# arguments to sparse as appropriate and rebroadcast. - -# first (Broadcast containertype) dispatch layer's promotion logic -struct PromoteToSparse end - -# broadcast containertype definitions for structured matrices +struct PromoteToSparse <: Broadcast.AbstractArrayStyle{2} end StructuredMatrix = Union{Diagonal,Bidiagonal,Tridiagonal,SymTridiagonal} -_containertype(::Type{<:StructuredMatrix}) = PromoteToSparse -broadcast_indices(::Type{PromoteToSparse}, A) = indices(A) +Broadcast.BroadcastStyle(::Type{<:StructuredMatrix}) = PromoteToSparse() -# combinations explicitly involving Tuples and PromoteToSparse collections -# divert to the generic AbstractArray broadcast code -promote_containertype(::Type{PromoteToSparse}, ::Type{Tuple}) = Array -promote_containertype(::Type{Tuple}, ::Type{PromoteToSparse}) = Array -# combinations involving scalars and PromoteToSparse collections continue in the promote-to-sparse funnel -promote_containertype(::Type{PromoteToSparse}, ::Type{Any}) = PromoteToSparse -promote_containertype(::Type{Any}, ::Type{PromoteToSparse}) = PromoteToSparse -# combinations involving sparse arrays and PromoteToSparse collections continue in the promote-to-sparse funnel -promote_containertype(::Type{PromoteToSparse}, ::Type{AbstractSparseArray}) = PromoteToSparse -promote_containertype(::Type{AbstractSparseArray}, ::Type{PromoteToSparse}) = PromoteToSparse -# combinations involving Arrays and PromoteToSparse collections continue in the promote-to-sparse funnel -promote_containertype(::Type{PromoteToSparse}, ::Type{Array}) = PromoteToSparse -promote_containertype(::Type{Array}, ::Type{PromoteToSparse}) = PromoteToSparse -# combinations involving Arrays and sparse arrays continue in the promote-to-sparse funnel -promote_containertype(::Type{AbstractSparseArray}, ::Type{Array}) = PromoteToSparse -promote_containertype(::Type{Array}, ::Type{AbstractSparseArray}) = PromoteToSparse +PromoteToSparse(::Val{0}) = PromoteToSparse() +PromoteToSparse(::Val{1}) = PromoteToSparse() +PromoteToSparse(::Val{2}) = PromoteToSparse() +PromoteToSparse(::Val{N}) where N = Broadcast.DefaultArrayStyle{N}() -# second (internal sparse broadcast containertype) dispatch layer's promotion logic -# mostly just disambiguates Array from the main containertype promotion mechanism -# AbstractArray serves as a marker to shunt to the generic AbstractArray broadcast code -_spcontainertype(x) = _containertype(x) -_spcontainertype(::Type{<:Vector}) = Vector -_spcontainertype(::Type{<:Matrix}) = Matrix -_spcontainertype(::Type{<:RowVector}) = Matrix -_spcontainertype(::Type{<:Ref}) = AbstractArray -_spcontainertype(::Type{<:AbstractArray}) = AbstractArray -# need the following two methods to override the immediately preceding method -_spcontainertype(::Type{<:StructuredMatrix}) = PromoteToSparse -_spcontainertype(::Type{<:SparseVecOrMat}) = AbstractSparseArray -spcontainertype(x) = _spcontainertype(typeof(x)) -spcontainertype(ct1, ct2) = promote_spcontainertype(spcontainertype(ct1), spcontainertype(ct2)) -@inline spcontainertype(ct1, ct2, cts...) = promote_spcontainertype(spcontainertype(ct1), spcontainertype(ct2, cts...)) +Broadcast.BroadcastStyle(::PromoteToSparse, ::SPVM) = PromoteToSparse() +Broadcast.BroadcastStyle(::PromoteToSparse, ::Broadcast.Style{Tuple}) = Broadcast.DefaultArrayStyle{2}() -promote_spcontainertype(::Type{T}, ::Type{T}) where {T} = T -# combinations involving AbstractArrays and/or Tuples divert to the generic AbstractArray broadcast code -DivertToAbsArrayBC = Union{Type{AbstractArray},Type{Tuple}} -promote_spcontainertype(::DivertToAbsArrayBC, ct) = AbstractArray -promote_spcontainertype(ct, ::DivertToAbsArrayBC) = AbstractArray -promote_spcontainertype(::DivertToAbsArrayBC, ::DivertToAbsArrayBC) = AbstractArray -# combinations involving scalars, sparse arrays, structured matrices (PromoteToSparse), -# dense vectors/matrices, and PromoteToSparse collections continue in the promote-to-sparse funnel -FunnelToSparseBC = Union{Type{Any},Type{Vector},Type{Matrix},Type{PromoteToSparse},Type{AbstractSparseArray}} -promote_spcontainertype(::FunnelToSparseBC, ::FunnelToSparseBC) = PromoteToSparse +Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{0}) = PromoteToSparse() +Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{1}) = PromoteToSparse() +Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{2}) = PromoteToSparse() +broadcast(f, ::PromoteToSparse, ::Void, ::Void, As::Vararg{Any,N}) where {N} = + broadcast(f, map(_sparsifystructured, As)...) -# first (Broadcast containertype) dispatch layer -# (broadcast_c[!], containertype, promote_containertype) -@inline broadcast_c(f, ::Type{PromoteToSparse}, As::Vararg{Any,N}) where {N} = - spbroadcast_c(f, spcontainertype(As...), As...) -@inline broadcast_c!(f, ::Type{AbstractSparseArray}, ::Type{PromoteToSparse}, C, B, As::Vararg{Any,N}) where {N} = - spbroadcast_c!(f, AbstractSparseArray, spcontainertype(B, As...), C, B, As...) -# where destination C is not an AbstractSparseArray, divert to generic AbstractArray broadcast code -@inline broadcast_c!(f, CT::Type, ::Type{PromoteToSparse}, C, B, As::Vararg{Any,N}) where {N} = - broadcast_c!(f, CT, Array, C, B, As...) +# ambiguity resolution +broadcast!(::typeof(identity), dest::SparseVecOrMat, x::Number) = + fill!(dest, x) +broadcast!(f, dest::SparseVecOrMat, x::Number...) = + spbroadcast_args!(f, dest, SPVM, x...) -# second (internal sparse broadcast containertype) dispatch layer -# (spbroadcast_c[!], spcontainertype, promote_spcontainertype) -@inline spbroadcast_c(f, ::Type{PromoteToSparse}, As::Vararg{Any,N}) where {N} = - broadcast(f, map(_sparsifystructured, As)...) -@inline spbroadcast_c(f, ::Type{AbstractArray}, As::Vararg{Any,N}) where {N} = - broadcast_c(f, Array, As...) -@inline spbroadcast_c!(f, ::Type{AbstractSparseArray}, ::Type{PromoteToSparse}, C, B, As::Vararg{Any,N}) where {N} = - broadcast!(f, C, _sparsifystructured(B), map(_sparsifystructured, As)...) -@inline spbroadcast_c!(f, ::Type{AbstractSparseArray}, ::Type{AbstractArray}, C, B, As::Vararg{Any,N}) where {N} = - broadcast_c!(f, Array, Array, C, B, As...) +# For broadcast! with ::Any inputs, we need a layer of indirection to determine whether +# the inputs can be promoted to SparseVecOrMat. If it's just SparseVecOrMat and scalars, +# we can handle it here, otherwise see below for the promotion machinery. +broadcast!(f, dest::SparseVecOrMat, mixedsrcargs::Vararg{Any,N}) where N = + spbroadcast_args!(f, dest, Broadcast.combine_styles(mixedsrcargs...), mixedsrcargs...) +function spbroadcast_args!(f, dest, ::Type{SPVM}, mixedsrcargs::Vararg{Any,N}) where N + # mixedsrcargs contains nothing but SparseVecOrMat and scalars + parevalf, passedsrcargstup = capturescalars(f, mixedsrcargs) + return broadcast!(parevalf, dest, passedsrcargstup...) +end +function spbroadcast_args!(f, dest, ::PromoteToSparse, mixedsrcargs::Vararg{Any,N}) where N + broadcast!(f, dest, map(_sparsifystructured, mixedsrcargs)...) +end +function spbroadcast_args!(f, dest, ::Any, mixedsrcargs::Vararg{Any,N}) where N + # Fallback. From a performance perspective would it be best to densify? + Broadcast._broadcast!(f, dest, mixedsrcargs...) +end -@inline _sparsifystructured(M::AbstractMatrix) = SparseMatrixCSC(M) -@inline _sparsifystructured(V::AbstractVector) = SparseVector(V) -@inline _sparsifystructured(M::AbstractSparseMatrix) = SparseMatrixCSC(M) -@inline _sparsifystructured(V::AbstractSparseVector) = SparseVector(V) -@inline _sparsifystructured(S::SparseVecOrMat) = S -@inline _sparsifystructured(x) = x +_sparsifystructured(M::AbstractMatrix) = SparseMatrixCSC(M) +_sparsifystructured(V::AbstractVector) = SparseVector(V) +_sparsifystructured(P::AbstractArray{<:Any,0}) = SparseVector(reshape(P, 1)) +_sparsifystructured(M::AbstractSparseMatrix) = SparseMatrixCSC(M) +_sparsifystructured(V::AbstractSparseVector) = SparseVector(V) +_sparsifystructured(S::SparseVecOrMat) = S +_sparsifystructured(x) = x # (12) map[!] over combinations of sparse and structured matrices -StructuredMatrix = Union{Diagonal,Bidiagonal,Tridiagonal,SymTridiagonal} SparseOrStructuredMatrix = Union{SparseMatrixCSC,StructuredMatrix} map(f::Tf, A::StructuredMatrix) where {Tf} = _noshapecheck_map(f, _sparsifystructured(A)) map(f::Tf, A::SparseOrStructuredMatrix, Bs::Vararg{SparseOrStructuredMatrix,N}) where {Tf,N} = diff --git a/doc/src/manual/interfaces.md b/doc/src/manual/interfaces.md index 7e19c255df667..59a6cfd433d3f 100644 --- a/doc/src/manual/interfaces.md +++ b/doc/src/manual/interfaces.md @@ -390,3 +390,204 @@ If you are defining an array type that allows non-traditional indexing (indices something other than 1), you should specialize `indices`. You should also specialize [`similar`](@ref) so that the `dims` argument (ordinarily a `Dims` size-tuple) can accept `AbstractUnitRange` objects, perhaps range-types `Ind` of your own design. For more information, see [Arrays with custom indices](@ref). + +## [Broadcasting](@id man-interfaces-broadcasting) + +| Methods to implement | Brief description | +|:-------------------- |:----------------- | +| `Base.BroadcastStyle(::Type{SrcType}) = SrcStyle()` | Broadcasting behavior of `SrcType` | +| `Base.broadcast_similar(f, ::DestStyle, ::Type{ElType}, inds, As...)` | Allocation of output container | +| **Optional methods** | | | +| `Base.BroadcastStyle(::Style1, ::Style2) = Style12()` | Precedence rules for mixing styles | +| `Base.broadcast_indices(::StyleA, A)` | Declaration of the indices of `A` for broadcasting purposes (for AbstractArrays, defaults to `indices(A)`) | +| **Bypassing default machinery** | | +| `broadcast(f, As...)` | Complete bypass of broadcasting machinery | +| `broadcast(f, ::DestStyle, ::Void, ::Void, As...)` | Bypass after container type is computed | +| `broadcast(f, ::DestStyle, ::Type{ElType}, inds::Tuple, As...)` | Bypass after container type, eltype, and indices are computed | + +[Broadcasting](@ref) is triggered by an explicit call to `broadcast` or `broadcast!`, or implicitly by +"dot" operations like `A .+ b`. Any `AbstractArray` type supports broadcasting, +but the default result (output) type is `Array`. To specialize the result for specific input type(s), +the main task is the allocation of an appropriate result object. +(This is not an issue for `broadcast!`, where +the result object is passed as an argument.) This process is split into two stages: computation +of the behavior and type from the arguments ([`Base.BroadcastStyle`](@ref)), and allocation of the object +given the resulting type with [`Base.broadcast_similar`](@ref). + +`Base.BroadcastStyle` is an abstract type from which all styles are +derived. When used as a function it has two possible forms, +unary (single-argument) and binary. +The unary variant states that you intend to +implement specific broadcasting behavior and/or output type, +and do not wish to rely on the default fallback ([`Broadcast.Scalar`](@ref) or [`Broadcast.DefaultArrayStyle`](@ref)). +To achieve this, you can define a custom `BroadcastStyle` for your object: + +```julia +struct MyStyle <: Broadcast.BroadcastStyle end +Base.BroadcastStyle(::Type{<:MyType}) = MyStyle() +``` + +In some cases it might be convenient not to have to define `MyStyle`, in which case you can +leverage one of the general broadcast wrappers: + + - `Base.BroadcastStyle(::Type{<:MyType}) = Broadcast.Style{MyType}()` can be + used for arbitrary types. + - `Base.BroadcastStyle(::Type{<:MyType}) = Broadcast.ArrayStyle{MyType}()` is preferred + if `MyType` is an `AbstractArray`. + - For `AbstractArrays` that only support a certain dimensionality, create a subtype of `Broadcast.AbstractArrayStyle{N}` (see below). + +When your broadcast operation involves several arguments, individual argument styles get +combined to determine a single `DestStyle` that controls the type of the output container. +For more detail, see [below](@ref writing-binary-broadcasting-rules). + +The actual allocation of the result array is handled by `Base.broadcast_similar`: + +```julia +Base.broadcast_similar(f, ::DestStyle, ::Type{ElType}, inds, As...) +``` + +`f` is the operation being performed and `DestStyle` signals the final result from +combining the input styles. +`As...` is the list of input objects. You may not need to use `f` or `As...` +unless they help you build the appropriate object; the fallback definition is + +```julia +broadcast_similar(f, ::DefaultArrayStyle{N}, ::Type{ElType}, inds::Indices{N}, As...) where {N,ElType} = + similar(Array{ElType}, inds) +``` + +However, if needed you can specialize on any or all of these arguments. + +For a complete example, let's say you have created a type, `ArrayAndChar`, that stores an +array and a single character: + +```jldoctest +struct ArrayAndChar{T,N} <: AbstractArray{T,N} + data::Array{T,N} + char::Char +end +Base.size(A::ArrayAndChar) = size(A.data) +Base.getindex(A::ArrayAndChar{T,N}, inds::Vararg{Int,N}) where {T,N} = A.data[inds...] +Base.setindex!(A::ArrayAndChar{T,N}, val, inds::Vararg{Int,N}) where {T,N} = A.data[inds...] = val +Base.showarg(io::IO, A::ArrayAndChar, toplevel) = print(io, typeof(A), " with char '", A.char, "'") +``` + +You might want broadcasting to preserve the `char` "metadata." First we define + +```jldoctest +Base.BroadcastStyle(::Type{<:ArrayAndChar}) = Broadcast.ArrayStyle{ArrayAndChar}() +``` + +This forces us to also define a `broadcast_similar` method: +```jldoctest +function Base.broadcast_similar(f, ::Broadcast.ArrayStyle{ArrayAndChar}, ::Type{ElType}, inds, As...) where ElType + # Scan the inputs for the ArrayAndChar: + A = find_aac(As...) + # Use the char field of A to create the output + ArrayAndChar(similar(Array{ElType}, inds), A.char) +end + +"`A = find_aac(As...)` returns the first ArrayAndChar among the arguments." +find_aac(A::ArrayAndChar, B...) = A +find_aac(A, B...) = find_aac(B...) +``` + +From these definitions, one obtains the following behavior: +```jldoctest +julia> a = ArrayAndChar([1 2; 3 4], 'x') +2×2 ArrayAndChar{Int64,2} with char 'x': + 1 2 + 3 4 + +julia> a .+ 1 +2×2 ArrayAndChar{Int64,2} with char 'x': + 2 3 + 4 5 + +julia> a .+ [5,10] +2×2 ArrayAndChar{Int64,2} with char 'x': + 6 7 + 13 14 +``` + +Finally, it's worth noting that sometimes it's easier simply to bypass the machinery for +computing result types and container sizes, and just do everything manually. For example, +you can convert a `UnitRange{Int}` `r` to a `UnitRange{BigInt}` with `big.(r)`; the definition +of this method is approximately + +```julia +Broadcast.broadcast(::typeof(big), r::UnitRange) = big(first(r)):big(last(r)) +``` + +This exploits Julia's ability to dispatch on a particular function type. (This kind of +explicit definition can indeed be necessary if the output container does not support `setindex!`.) +You can optionally choose to implement the actual broadcasting yourself, but allow +the internal machinery to compute the container type, element type, and indices by specializing + +```julia +Broadcast.broadcast(::typeof(somefunction), ::MyStyle, ::Type{ElType}, inds, As...) +``` + +### [Writing binary broadcasting rules](@id writing-binary-broadcasting-rules) + +The precedence rules are defined by binary `BroadcastStyle` calls: + +```julia +Base.BroadcastStyle(::Style1, ::Style2) = Style12() +``` + +where `Style12` is the `BroadcastStyle` you want to choose for outputs involving +arguments of `Style1` and `Style2`. For example, + +```julia +Base.BroadcastStyle(::Broadcast.Style{Tuple}, ::Broadcast.Scalar) = Broadcast.Style{Tuple}() +``` + +indicates that `Tuple` "wins" over scalars (the output container will be a tuple). +It is worth noting that you do not need to (and should not) define both argument orders +of this call; defining one is sufficient no matter what order the user supplies the arguments in. + +For `AbstractArray` types, defining a `BroadcastStyle` supersedes the fallback choice, +[`Broadcast.DefaultArrayStyle`](@ref). `DefaultArrayStyle` and the abstract supertype, `AbstractArrayStyle`, store the dimensionality as a type parameter to support specialized +array types that have fixed dimensionality requirements. + +`DefaultArrayStyle` "loses" to any other +`AbstractArrayStyle` that has been defined because of the following methods: + +```julia +BroadcastStyle(a::AbstractArrayStyle{Any}, ::DefaultArrayStyle) = a +BroadcastStyle(a::AbstractArrayStyle{N}, ::DefaultArrayStyle{N}) where N = a +BroadcastStyle(a::AbstractArrayStyle{M}, ::DefaultArrayStyle{N}) where {M,N} = + typeof(a)(_max(Val(M),Val(N))) +``` + +You do not need to write binary `BroadcastStyle` +rules unless you want to establish precedence for +two or more non-`DefaultArrayStyle` types. + +If your array type does have fixed dimensionality requirements, then you should +subtype `AbstractArrayStyle`. For example, the sparse array code has the following definitions: + +```julia +struct SparseVecStyle <: Broadcast.AbstractArrayStyle{1} end +struct SparseMatStyle <: Broadcast.AbstractArrayStyle{2} end +Base.BroadcastStyle(::Type{<:SparseVector}) = SparseVecStyle() +Base.BroadcastStyle(::Type{<:SparseMatrixCSC}) = SparseMatStyle() +``` + +Whenever you subtype `AbstractArrayStyle`, you also need to define rules for combining +dimensionalities, by creating a constructor for your style that takes a `Val(N)` argument. +For example: + +```julia +SparseVecStyle(::Val{0}) = SparseVecStyle() +SparseVecStyle(::Val{1}) = SparseVecStyle() +SparseVecStyle(::Val{2}) = SparseMatStyle() +SparseVecStyle(::Val{N}) where N = Broadcast.DefaultArrayStyle{N}() +``` + +These rules indicate that the combination of a `SparseVecStyle` with 0- or 1-dimensional arrays +yields another `SparseVecStyle`, that its combination with a 2-dimensional array +yields a `SparseMatStyle`, and anything of higher dimensionality falls back to the dense arbitrary-dimensional framework. +These rules allow broadcasting to keep the sparse representation for operations that result +in one or two dimensional outputs, but produce an `Array` for any other dimensionality. diff --git a/doc/src/stdlib/arrays.md b/doc/src/stdlib/arrays.md index 383b886ccb895..d9516659e8944 100644 --- a/doc/src/stdlib/arrays.md +++ b/doc/src/stdlib/arrays.md @@ -66,6 +66,17 @@ Base.Broadcast.broadcast_getindex Base.Broadcast.broadcast_setindex! ``` +For specializing broadcast on custom types, see +```@docs +Base.BroadcastStyle +Base.broadcast_similar +Base.broadcast_indices +Base.Broadcast.Scalar +Base.Broadcast.AbstractArrayStyle +Base.Broadcast.ArrayStyle +Base.Broadcast.DefaultArrayStyle +``` + ## Indexing and assignment ```@docs diff --git a/test/broadcast.jl b/test/broadcast.jl index 26f6258e8f3bd..feb44df004e55 100644 --- a/test/broadcast.jl +++ b/test/broadcast.jl @@ -2,8 +2,7 @@ module TestBroadcastInternals -using Base.Broadcast: broadcast_indices, check_broadcast_indices, - check_broadcast_shape, newindex, _bcs +using Base.Broadcast: check_broadcast_indices, check_broadcast_shape, newindex, _bcs using Base: OneTo using Test @@ -20,10 +19,10 @@ using Test @test_throws DimensionMismatch _bcs((-1:1, 2:6), (-1:1, 2:5)) @test_throws DimensionMismatch _bcs((-1:1, 2:5), (2, 2:5)) -@test @inferred(broadcast_indices(zeros(3,4), zeros(3,4))) == (OneTo(3),OneTo(4)) -@test @inferred(broadcast_indices(zeros(3,4), zeros(3))) == (OneTo(3),OneTo(4)) -@test @inferred(broadcast_indices(zeros(3), zeros(3,4))) == (OneTo(3),OneTo(4)) -@test @inferred(broadcast_indices(zeros(3), zeros(1,4), zeros(1))) == (OneTo(3),OneTo(4)) +@test @inferred(Broadcast.combine_indices(zeros(3,4), zeros(3,4))) == (OneTo(3),OneTo(4)) +@test @inferred(Broadcast.combine_indices(zeros(3,4), zeros(3))) == (OneTo(3),OneTo(4)) +@test @inferred(Broadcast.combine_indices(zeros(3), zeros(3,4))) == (OneTo(3),OneTo(4)) +@test @inferred(Broadcast.combine_indices(zeros(3), zeros(1,4), zeros(1))) == (OneTo(3),OneTo(4)) check_broadcast_indices((OneTo(3),OneTo(5)), zeros(3,5)) check_broadcast_indices((OneTo(3),OneTo(5)), zeros(3,1)) @@ -404,7 +403,7 @@ StrangeType18623(x,y) = (x,y) let f(A, n) = broadcast(x -> +(x, n), A) @test @inferred(f([1.0], 1)) == [2.0] - g() = (a = 1; Base.Broadcast._broadcast_eltype(x -> x + a, 1.0)) + g() = (a = 1; Broadcast.combine_eltypes(x -> x + a, 1.0)) @test @inferred(g()) === Float64 end @@ -419,34 +418,67 @@ end @test let z = 1; A = broadcast!(() -> z += 1, zeros(2)); A[1] != A[2]; end @test let z = 1; A = broadcast!(x -> z += x, zeros(2), 1); A[1] != A[2]; end -# broadcasting for custom AbstractArray -struct Array19745{T,N} <: AbstractArray{T,N} +## broadcasting for custom AbstractArray +abstract type ArrayData{T,N} <: AbstractArray{T,N} end +Base.getindex(A::ArrayData, i::Integer...) = A.data[i...] +Base.setindex!(A::ArrayData, v::Any, i::Integer...) = setindex!(A.data, v, i...) +Base.size(A::ArrayData) = size(A.data) +Base.broadcast_similar(f, ::Broadcast.ArrayStyle{A}, ::Type{T}, inds::Tuple, As...) where {A,T} = + A(Array{T}(uninitialized, length.(inds))) + +struct Array19745{T,N} <: ArrayData{T,N} data::Array{T,N} end -Base.getindex(A::Array19745, i::Integer...) = A.data[i...] -Base.setindex!(A::Array19745, v::Any, i::Integer...) = setindex!(A.data, v, i...) -Base.size(A::Array19745) = size(A.data) +Base.BroadcastStyle(::Type{T}) where {T<:Array19745} = Broadcast.ArrayStyle{Array19745}() -Base.Broadcast._containertype(::Type{T}) where {T<:Array19745} = Array19745 +# Two specialized broadcast rules with no declared precedence +struct AD1{T,N} <: ArrayData{T,N} + data::Array{T,N} +end +Base.BroadcastStyle(::Type{T}) where {T<:AD1} = Broadcast.ArrayStyle{AD1}() +struct AD2{T,N} <: ArrayData{T,N} + data::Array{T,N} +end +Base.BroadcastStyle(::Type{T}) where {T<:AD2} = Broadcast.ArrayStyle{AD2}() -Base.Broadcast.promote_containertype(::Type{Array19745}, ::Type{Array19745}) = Array19745 -Base.Broadcast.promote_containertype(::Type{Array19745}, ::Type{Array}) = Array19745 -Base.Broadcast.promote_containertype(::Type{Array19745}, ct) = Array19745 -Base.Broadcast.promote_containertype(::Type{Array}, ::Type{Array19745}) = Array19745 -Base.Broadcast.promote_containertype(ct, ::Type{Array19745}) = Array19745 +# Two specialized broadcast rules with explicit precedence +struct AD1P{T,N} <: ArrayData{T,N} + data::Array{T,N} +end +Base.BroadcastStyle(::Type{T}) where {T<:AD1P} = Broadcast.ArrayStyle{AD1P}() +struct AD2P{T,N} <: ArrayData{T,N} + data::Array{T,N} +end +Base.BroadcastStyle(::Type{T}) where {T<:AD2P} = Broadcast.ArrayStyle{AD2P}() -Base.Broadcast.broadcast_indices(::Type{Array19745}, A) = indices(A) -Base.Broadcast.broadcast_indices(::Type{Array19745}, A::Ref) = () +Base.BroadcastStyle(a1::Broadcast.ArrayStyle{AD1P}, ::Broadcast.ArrayStyle{AD2P}) = a1 -getfield19745(x::Array19745) = x.data -getfield19745(x) = x +# Two specialized broadcast rules where users unnecessarily +# define `BroadcastStyle` for both argument orders (but do so consistently) +struct AD1B{T,N} <: ArrayData{T,N} + data::Array{T,N} +end +Base.BroadcastStyle(::Type{T}) where {T<:AD1B} = Broadcast.ArrayStyle{AD1B}() +struct AD2B{T,N} <: ArrayData{T,N} + data::Array{T,N} +end +Base.BroadcastStyle(::Type{T}) where {T<:AD2B} = Broadcast.ArrayStyle{AD2B}() -function Base.Broadcast.broadcast_c(f, ::Type{Array19745}, A, Bs...) - T = Base.Broadcast._broadcast_eltype(f, A, Bs...) - shape = Base.Broadcast.broadcast_indices(A, Bs...) - dest = Array19745(Array{T}(uninitialized, Base.index_lengths(shape...))) - return broadcast!(f, dest, A, Bs...) +Base.BroadcastStyle(a1::Broadcast.ArrayStyle{AD1B}, a2::Broadcast.ArrayStyle{AD2B}) = a1 +Base.BroadcastStyle(a2::Broadcast.ArrayStyle{AD2B}, a1::Broadcast.ArrayStyle{AD1B}) = a1 + +# Two specialized broadcast rules with conflicting precedence +struct AD1C{T,N} <: ArrayData{T,N} + data::Array{T,N} end +Base.BroadcastStyle(::Type{T}) where {T<:AD1C} = Broadcast.ArrayStyle{AD1C}() +struct AD2C{T,N} <: ArrayData{T,N} + data::Array{T,N} +end +Base.BroadcastStyle(::Type{T}) where {T<:AD2C} = Broadcast.ArrayStyle{AD2C}() + +Base.BroadcastStyle(a1::Broadcast.ArrayStyle{AD1C}, a2::Broadcast.ArrayStyle{AD2C}) = a1 +Base.BroadcastStyle(a2::Broadcast.ArrayStyle{AD2C}, a1::Broadcast.ArrayStyle{AD1C}) = a2 @testset "broadcasting for custom AbstractArray" begin a = randn(10) @@ -455,6 +487,35 @@ end @test a .* a' == @inferred(aa .* aa') @test isa(aa .+ 1, Array19745) @test isa(aa .* aa', Array19745) + a1 = AD1(rand(2,3)) + a2 = AD2(rand(2)) + @test a1 .+ 1 isa AD1 + @test a2 .+ 1 isa AD2 + @test a1 .+ a2 isa Array + @test a2 .+ a1 isa Array + @test a1 .+ a2 .+ a1 isa Array + @test a1 .+ a2 .+ a2 isa Array + a1 = AD1P(rand(2,3)) + a2 = AD2P(rand(2)) + @test a1 .+ 1 isa AD1P + @test a2 .+ 1 isa AD2P + @test a1 .+ a2 isa AD1P + @test a2 .+ a1 isa AD1P + @test a1 .+ a2 .+ a1 isa AD1P + @test a1 .+ a2 .+ a2 isa AD1P + a1 = AD1B(rand(2,3)) + a2 = AD2B(rand(2)) + @test a1 .+ 1 isa AD1B + @test a2 .+ 1 isa AD2B + @test a1 .+ a2 isa AD1B + @test a2 .+ a1 isa AD1B + @test a1 .+ a2 .+ a1 isa AD1B + @test a1 .+ a2 .+ a2 isa AD1B + a1 = AD1C(rand(2,3)) + a2 = AD2C(rand(2)) + @test a1 .+ 1 isa AD1C + @test a2 .+ 1 isa AD2C + @test_throws ErrorException a1 .+ a2 end # broadcast should only "peel off" one container layer @@ -466,7 +527,7 @@ end # Test that broadcast's promotion mechanism handles closures accepting more than one argument. # (See issue #19641 and referenced issues and pull requests.) -let f() = (a = 1; Base.Broadcast._broadcast_eltype((x, y) -> x + y + a, 1.0, 1.0)) +let f() = (a = 1; Broadcast.combine_eltypes((x, y) -> x + y + a, 1.0, 1.0)) @test @inferred(f()) == Float64 end @@ -485,7 +546,7 @@ end # Test that broadcast treats type arguments as scalars, i.e. containertype yields Any, # even for subtypes of abstract array. (https://github.com/JuliaStats/DataArrays.jl/issues/229) @testset "treat type arguments as scalars, DataArrays issue 229" begin - @test Base.Broadcast.containertype(AbstractArray) == Any + @test Broadcast.combine_styles(AbstractArray) == Broadcast.Scalar() @test broadcast(==, [1], AbstractArray) == BitArray([false]) @test broadcast(==, 1, AbstractArray) == false end @@ -546,3 +607,6 @@ end # @test A[1] == [0,0,0] # @test_throws ErrorException A[2] .= 0 # end + +# Issue #22180 +@test isequal(convert.(Nullable, [1,2]), [Nullable(1), Nullable(2)]) diff --git a/test/sparse/higherorderfns.jl b/test/sparse/higherorderfns.jl index f5a753025e020..b8638ad1b2ba6 100644 --- a/test/sparse/higherorderfns.jl +++ b/test/sparse/higherorderfns.jl @@ -118,7 +118,7 @@ end @test broadcast!(cos, Z, X) == sparse(broadcast!(cos, fZ, fX)) # --> test shape checks for broadcast! entry point # TODO strengthen this test, avoiding dependence on checking whether - # broadcast_indices throws to determine whether sparse broadcast should throw + # check_broadcast_indices throws to determine whether sparse broadcast should throw try Base.Broadcast.check_broadcast_indices(indices(Z), spzeros((shapeX .- 1)...)) catch @@ -142,7 +142,7 @@ end @test broadcast!(cos, V, X) == sparse(broadcast!(cos, fV, fX)) # --> test shape checks for broadcast! entry point # TODO strengthen this test, avoiding dependence on checking whether - # broadcast_indices throws to determine whether sparse broadcast should throw + # check_broadcast_indices throws to determine whether sparse broadcast should throw try Base.Broadcast.check_broadcast_indices(indices(V), spzeros((shapeX .- 1)...)) catch @@ -177,9 +177,9 @@ end @test broadcast(*, X, Y) == sparse(broadcast(*, fX, fY)) @test broadcast(f, X, Y) == sparse(broadcast(f, fX, fY)) # TODO strengthen this test, avoiding dependence on checking whether - # broadcast_indices throws to determine whether sparse broadcast should throw + # check_broadcast_indices throws to determine whether sparse broadcast should throw try - Base.Broadcast.broadcast_indices(spzeros((shapeX .- 1)...), Y) + Base.Broadcast.combine_indices(spzeros((shapeX .- 1)...), Y) catch @test_throws DimensionMismatch broadcast(+, spzeros((shapeX .- 1)...), Y) end @@ -200,7 +200,7 @@ end @test broadcast!(f, Z, X, Y) == sparse(broadcast!(f, fZ, fX, fY)) # --> test shape checks for both broadcast and broadcast! entry points # TODO strengthen this test, avoiding dependence on checking whether - # broadcast_indices throws to determine whether sparse broadcast should throw + # check_broadcast_indices throws to determine whether sparse broadcast should throw try Base.Broadcast.check_broadcast_indices(indices(Z), spzeros((shapeX .- 1)...), Y) catch @@ -227,9 +227,9 @@ end @test broadcast(*, X, Y, Z) == sparse(broadcast(*, fX, fY, fZ)) @test broadcast(f, X, Y, Z) == sparse(broadcast(f, fX, fY, fZ)) # TODO strengthen this test, avoiding dependence on checking whether - # broadcast_indices throws to determine whether sparse broadcast should throw + # check_broadcast_indices throws to determine whether sparse broadcast should throw try - Base.Broadcast.broadcast_indices(spzeros((shapeX .- 1)...), Y, Z) + Base.Broadcast.combine_indices(spzeros((shapeX .- 1)...), Y, Z) catch @test_throws DimensionMismatch broadcast(+, spzeros((shapeX .- 1)...), Y, Z) end @@ -257,7 +257,7 @@ end @test broadcast!(f, Q, X, Y, Z) == sparse(broadcast!(f, fQ, fX, fY, fZ)) # --> test shape checks for both broadcast and broadcast! entry points # TODO strengthen this test, avoiding dependence on checking whether - # broadcast_indices throws to determine whether sparse broadcast should throw + # check_broadcast_indices throws to determine whether sparse broadcast should throw try Base.Broadcast.check_broadcast_indices(indices(Q), spzeros((shapeX .- 1)...), Y, Z) catch @@ -418,6 +418,8 @@ end @test broadcast(+, A, X')::SparseMatrixCSC == sparse(broadcast(+, fA, X')) @test broadcast(*, V, X')::SparseMatrixCSC == sparse(broadcast(*, fV, X')) end + @test V .+ ntuple(identity, N) isa Vector + @test A .+ ntuple(identity, N) isa Matrix end @testset "broadcast! where the destination is a structured matrix" begin From 7f487b67769e41452adedc8c9792483645554352 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sun, 1 Oct 2017 07:28:51 -0500 Subject: [PATCH 2/3] Improve the docstrings for broadcast_getindex and broadcast_setindex! --- base/broadcast.jl | 83 ++++++++++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index 7a1fac61a8755..6e2809daa2a48 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -662,41 +662,56 @@ tuplebroadcast_getargs(::Tuple{}, k) = () """ broadcast_getindex(A, inds...) -Broadcasts the `inds` arrays to a common size like [`broadcast`](@ref) -and returns an array of the results `A[ks...]`, -where `ks` goes over the positions in the broadcast result `A`. +Equivalent to [`broadcast`](@ref)ing the `inds` arrays to a common size +and returning an array `[A[ks...] for ks in zip(indsb...)]` (where `indsb` +would be the broadcast `inds`). The shape of the output is equal to the shape of each +element of `indsb`. # Examples ```jldoctest -julia> A = [1, 2, 3, 4, 5] -5-element Array{Int64,1}: - 1 - 2 - 3 - 4 - 5 +julia> A = [11 12; 21 22] +2×2 Array{Int64,2}: + 11 12 + 21 22 -julia> B = [1 2; 3 4; 5 6; 7 8; 9 10] -5×2 Array{Int64,2}: - 1 2 - 3 4 - 5 6 - 7 8 - 9 10 +julia> A[1:2, 1:2] +2×2 Array{Int64,2}: + 11 12 + 21 22 -julia> C = broadcast(+,A,B) -5×2 Array{Int64,2}: - 2 3 - 5 6 - 8 9 +julia> broadcast_getindex(A, 1:2, 1:2) +2-element Array{Int64,1}: + 11 + 22 + +julia> A[1:2, 2:-1:1] +2×2 Array{Int64,2}: + 12 11 + 22 21 + +julia> broadcast_getindex(A, 1:2, 2:-1:1) +2-element Array{Int64,1}: + 12 + 21 + ``` +Because the indices are all vectors, these calls are like `[A[i[k], j[k]] for k = 1:2]` +where `i` and `j` are the two index vectors. + +```jldoctest +julia> broadcast_getindex(A, 1:2, (1:2)') +2×2 Array{Int64,2}: 11 12 - 14 15 + 21 22 + +julia> broadcast_getindex(A, (1:2)', 1:2) +2×2 Array{Int64,2}: + 11 21 + 12 22 -julia> broadcast_getindex(C,[1,2,10]) -3-element Array{Int64,1}: - 2 - 5 - 15 +julia> broadcast_getindex(A, [1 2 1; 1 2 2], [1, 2]) +2×3 Array{Int64,2}: + 11 21 11 + 12 22 22 ``` """ broadcast_getindex(src::AbstractArray, I::AbstractArray...) = @@ -723,8 +738,16 @@ end """ broadcast_setindex!(A, X, inds...) -Broadcasts the `X` and `inds` arrays to a common size and stores the value from each -position in `X` at the indices in `A` given by the same positions in `inds`. +Efficient element-by-element setting of the values of `A` in a pattern established by `inds`. +Equivalent to broadcasting the `X` and `inds` arrays to a common size, and then executing + + for (is, js) in zip(zip(indsb), eachindex(Xb)) + A[is...] = Xb[js...] + end + +where `Xb` and `indsb` are the broadcast `X` and `inds`. + +See [`broadcast_getindex`](@ref) for examples of the treatment of `inds`. """ @generated function broadcast_setindex!(A::AbstractArray, x, I::AbstractArray...) N = length(I) From 776139b453c7f4b23f14412ef32c3d3c8170e1d8 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Fri, 3 Nov 2017 09:51:46 -0500 Subject: [PATCH 3/3] Restrict sparse broadcast promotion to Array This should be reverted someday --- base/broadcast.jl | 39 ++++++++++++++++++++++++++++++++++- base/sparse/higherorderfns.jl | 15 +++++++++++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/base/broadcast.jl b/base/broadcast.jl index 6e2809daa2a48..e07ccab2443bc 100644 --- a/base/broadcast.jl +++ b/base/broadcast.jl @@ -162,6 +162,32 @@ BroadcastStyle(a::AbstractArrayStyle{N}, ::DefaultArrayStyle{N}) where N = a BroadcastStyle(a::AbstractArrayStyle{M}, ::DefaultArrayStyle{N}) where {M,N} = typeof(a)(_max(Val(M),Val(N))) +# FIXME +# The following definitions are necessary to limit SparseArray broadcasting to "plain Arrays" +# (see https://github.com/JuliaLang/julia/pull/23939#pullrequestreview-72075382). +# They should be deleted once the sparse broadcast infrastucture is capable of handling +# arbitrary AbstractArrays. +struct VectorStyle <: AbstractArrayStyle{1} end +struct MatrixStyle <: AbstractArrayStyle{2} end +const VMStyle = Union{VectorStyle,MatrixStyle} +# These lose to DefaultArrayStyle +VectorStyle(::Val{N}) where N = DefaultArrayStyle{N}() +MatrixStyle(::Val{N}) where N = DefaultArrayStyle{N}() + +BroadcastStyle(::Type{<:Vector}) = VectorStyle() +BroadcastStyle(::Type{<:Matrix}) = MatrixStyle() + +BroadcastStyle(::MatrixStyle, ::VectorStyle) = MatrixStyle() +BroadcastStyle(a::AbstractArrayStyle{Any}, ::VectorStyle) = a +BroadcastStyle(a::AbstractArrayStyle{Any}, ::MatrixStyle) = a +BroadcastStyle(a::AbstractArrayStyle{N}, ::VectorStyle) where N = typeof(a)(_max(Val(N), Val(1))) +BroadcastStyle(a::AbstractArrayStyle{N}, ::MatrixStyle) where N = typeof(a)(_max(Val(N), Val(2))) +BroadcastStyle(::VectorStyle, ::DefaultArrayStyle{N}) where N = DefaultArrayStyle(_max(Val(N), Val(1))) +BroadcastStyle(::MatrixStyle, ::DefaultArrayStyle{N}) where N = DefaultArrayStyle(_max(Val(N), Val(2))) +# to avoid the VectorStyle(::Val) constructor we also need the following +BroadcastStyle(::VectorStyle, ::MatrixStyle) = MatrixStyle() +# end FIXME + ## Allocating the output container """ broadcast_similar(f, ::BroadcastStyle, ::Type{ElType}, inds, As...) @@ -181,6 +207,17 @@ broadcast_similar(f, ::ArrayConflict, ::Type{ElType}, inds::Indices, As...) wher broadcast_similar(f, ::ArrayConflict, ::Type{Bool}, inds::Indices, As...) = similar(BitArray, inds) +# FIXME: delete when we get rid of VectorStyle and MatrixStyle +broadcast_similar(f, ::VectorStyle, ::Type{ElType}, inds::Indices{1}, As...) where ElType = + similar(Vector{ElType}, inds) +broadcast_similar(f, ::MatrixStyle, ::Type{ElType}, inds::Indices{2}, As...) where ElType = + similar(Matrix{ElType}, inds) +broadcast_similar(f, ::VectorStyle, ::Type{Bool}, inds::Indices{1}, As...) = + similar(BitArray, inds) +broadcast_similar(f, ::MatrixStyle, ::Type{Bool}, inds::Indices{2}, As...) = + similar(BitArray, inds) +# end FIXME + ## Computing the result's indices. Most types probably won't need to specialize this. broadcast_indices() = () broadcast_indices(::Type{T}) where T = () @@ -582,7 +619,7 @@ Nullable{Complex{Float64}}() broadcast(f, s, combine_eltypes(f, A, Bs...), combine_indices(A, Bs...), A, Bs...) -const NonleafHandlingTypes = Union{DefaultArrayStyle,ArrayConflict} +const NonleafHandlingTypes = Union{DefaultArrayStyle,ArrayConflict,VectorStyle,MatrixStyle} @inline function broadcast(f, s::NonleafHandlingTypes, ::Type{ElType}, inds::Indices, As...) where ElType if !Base._isleaftype(ElType) diff --git a/base/sparse/higherorderfns.jl b/base/sparse/higherorderfns.jl index 1c551fe446aba..447d247746838 100644 --- a/base/sparse/higherorderfns.jl +++ b/base/sparse/higherorderfns.jl @@ -989,9 +989,18 @@ PromoteToSparse(::Val{N}) where N = Broadcast.DefaultArrayStyle{N}() Broadcast.BroadcastStyle(::PromoteToSparse, ::SPVM) = PromoteToSparse() Broadcast.BroadcastStyle(::PromoteToSparse, ::Broadcast.Style{Tuple}) = Broadcast.DefaultArrayStyle{2}() -Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{0}) = PromoteToSparse() -Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{1}) = PromoteToSparse() -Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{2}) = PromoteToSparse() +# FIXME: switch to DefaultArrayStyle once we can delete VectorStyle and MatrixStyle +# Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{0}) = PromoteToSparse() +# Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{1}) = PromoteToSparse() +# Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{2}) = PromoteToSparse() +BroadcastStyle(::Type{<:Base.RowVector{T,<:Vector}}) where T = Broadcast.MatrixStyle() # RowVector not yet defined when broadcast.jl loaded +Broadcast.BroadcastStyle(::SPVM, ::Broadcast.VectorStyle) = PromoteToSparse() +Broadcast.BroadcastStyle(::SPVM, ::Broadcast.MatrixStyle) = PromoteToSparse() +Broadcast.BroadcastStyle(::SparseVecStyle, ::Broadcast.DefaultArrayStyle{N}) where N = + Broadcast.DefaultArrayStyle(Broadcast._max(Val(N), Val(1))) +Broadcast.BroadcastStyle(::SparseMatStyle, ::Broadcast.DefaultArrayStyle{N}) where N = + Broadcast.DefaultArrayStyle(Broadcast._max(Val(N), Val(2))) +# end FIXME broadcast(f, ::PromoteToSparse, ::Void, ::Void, As::Vararg{Any,N}) where {N} = broadcast(f, map(_sparsifystructured, As)...)