Deprecate defaulting to scalar in broadcast (take 2) (#26435)

* fixup Pkg3 broadcasting over iterables

* Default to iterating over broadcast arguments

Broadcast now calls a special helper function, `broadcastable`, on each argument. It ensures that the arguments support indexing and have a shape, or otherwise have defined a custom `BroadcastStyle`.

* Add a deprecation and comment out future behavior. Add NEWS and docs.

NEWS.md

@@ -914,6 +914,12 @@ Deprecated or removed
   * `map` on dictionaries previously operated on `key=>value` pairs. This behavior is deprecated,
     and in the future `map` will operate only on values ([#5794]).
 
+  * Previously, broadcast defaulted to treating its arguments as scalars if they were not
+    arrays. This behavior is deprecated, and in the future `broadcast` will default to
+    iterating over all its arguments. Wrap arguments you wish to be treated as scalars with
+    `Ref()` or a 1-tuple. Package developers can choose to allow a non-iterable type `T` to
+    always behave as a scalar by implementing `broadcastable(x::T) = Ref(x)` ([#26212]).
+
   * Automatically broadcasted `+` and `-` for `array + scalar`, `scalar - array`, and so-on have
     been deprecated due to inconsistency with linear algebra. Use `.+` and `.-` for these operations
     instead ([#22880], [#22932]).

base/broadcast.jl

@@ -48,15 +48,7 @@ BroadcastStyle(::Type{<:Tuple}) = Style{Tuple}()
 
 struct Unknown <: BroadcastStyle end
 BroadcastStyle(::Type{Union{}}) = Unknown()  # ambiguity resolution
-
-"""
-`Broadcast.Scalar()` is a [`BroadcastStyle`](@ref) indicating that an object is not
-treated as a container for the purposes of broadcasting. This is the default for objects
-that have not customized `BroadcastStyle`.
-"""
-struct Scalar <: BroadcastStyle end
-BroadcastStyle(::Type) = Scalar()
-BroadcastStyle(::Type{<:Ptr}) = Scalar()
+BroadcastStyle(::Type) = Unknown()
 
 """
 `Broadcast.AbstractArrayStyle{N} <: BroadcastStyle` is the abstract supertype for any style
@@ -102,15 +94,14 @@ behaves as an `N`-dimensional array for broadcasting. Specifically, `DefaultArra
 used for any
 AbstractArray type that hasn't defined a specialized style, and in the absence of
 overrides from other `broadcast` arguments the resulting output type is `Array`.
-When there are multiple inputs to `broadcast`, `DefaultArrayStyle` "wins" over [`Broadcast.Scalar`](@ref)
-but "loses" to any other [`Broadcast.ArrayStyle`](@ref).
+When there are multiple inputs to `broadcast`, `DefaultArrayStyle` "loses" to any other [`Broadcast.ArrayStyle`](@ref).
 """
 struct DefaultArrayStyle{N} <: AbstractArrayStyle{N} end
 (::Type{<:DefaultArrayStyle})(::Val{N}) where N = DefaultArrayStyle{N}()
 const DefaultVectorStyle = DefaultArrayStyle{1}
 const DefaultMatrixStyle = DefaultArrayStyle{2}
 BroadcastStyle(::Type{<:AbstractArray{T,N}}) where {T,N} = DefaultArrayStyle{N}()
-BroadcastStyle(::Type{<:Ref}) = DefaultArrayStyle{0}()
+BroadcastStyle(::Type{<:Union{Ref,Number}}) = DefaultArrayStyle{0}()
 
 # `ArrayConflict` is an internal type signaling that two or more different `AbstractArrayStyle`
 # objects were supplied as arguments, and that no rule was defined for resolving the
@@ -141,10 +132,8 @@ BroadcastStyle(::BroadcastStyle, ::BroadcastStyle) = Unknown()
 BroadcastStyle(::Unknown, ::Unknown) = Unknown()
 BroadcastStyle(::S, ::Unknown) where S<:BroadcastStyle = S()
 # Precedence rules
-BroadcastStyle(::Style{Tuple}, ::Scalar)          = Style{Tuple}()
-BroadcastStyle(a::AbstractArrayStyle{0}, ::Style{Tuple}) = typeof(a)(Val(1))
+BroadcastStyle(a::AbstractArrayStyle{0}, b::Style{Tuple}) = b
 BroadcastStyle(a::AbstractArrayStyle, ::Style{Tuple})    = a
-BroadcastStyle(a::AbstractArrayStyle, ::Scalar)          = a
 BroadcastStyle(::A, ::A) where A<:ArrayStyle             = A()
 BroadcastStyle(::ArrayStyle, ::ArrayStyle)               = Unknown()
 BroadcastStyle(::A, ::A) where A<:AbstractArrayStyle     = A()
@@ -183,10 +172,9 @@ broadcast_similar(f, ::ArrayConflict, ::Type{Bool}, inds::Indices, As...) =
 broadcast_indices() = ()
 broadcast_indices(::Type{T}) where T = ()
 broadcast_indices(A) = broadcast_indices(combine_styles(A), A)
-broadcast_indices(::Scalar, A) = ()
 broadcast_indices(::Style{Tuple}, A) = (OneTo(length(A)),)
 broadcast_indices(::DefaultArrayStyle{0}, A::Ref) = ()
-broadcast_indices(::AbstractArrayStyle, A) = Base.axes(A)
+broadcast_indices(::BroadcastStyle, A) = Base.axes(A)
 """
     Base.broadcast_indices(::SrcStyle, A)
 
@@ -328,8 +316,7 @@ end
 
 Base.@propagate_inbounds _broadcast_getindex(::Type{T}, I) where T = T
 Base.@propagate_inbounds _broadcast_getindex(A, I) = _broadcast_getindex(combine_styles(A), A, I)
-Base.@propagate_inbounds _broadcast_getindex(::DefaultArrayStyle{0}, A::Ref, I) = A[]
-Base.@propagate_inbounds _broadcast_getindex(::Union{Unknown,Scalar}, A, I) = A
+Base.@propagate_inbounds _broadcast_getindex(::DefaultArrayStyle{0}, A, I) = A[]
 Base.@propagate_inbounds _broadcast_getindex(::Any, A, I) = A[I]
 Base.@propagate_inbounds _broadcast_getindex(::Style{Tuple}, A::Tuple{Any}, I) = A[1]
 
@@ -395,6 +382,36 @@ end
     end
 end
 
+"""
+    broadcastable(x)
+
+Return either `x` or an object like `x` such that it supports `axes` and indexing.
+
+If `x` supports iteration, the returned value should have the same `axes` and indexing behaviors as [`collect(x)`](@ref).
+
+# Examples
+```jldoctest
+julia> broadcastable([1,2,3]) # like `identity` since arrays already support axes and indexing
+3-element Array{Int64,1}:
+ 1
+ 2
+ 3
+
+julia> broadcastable(Int) # Types don't support axes, indexing, or iteration but are commonly used as scalars
+Base.RefValue{Type{Int64}}(Int64)
+
+julia> broadcastable("hello") # Strings break convention of matching iteration and act like a scalar instead
+Base.RefValue{String}("hello")
+```
+"""
+broadcastable(x::Union{Symbol,AbstractString,Function,UndefInitializer,Nothing,RoundingMode,Missing}) = Ref(x)
+broadcastable(x::Ptr) = Ref{Ptr}(x) # Cannot use Ref(::Ptr) until ambiguous deprecation goes through
+broadcastable(::Type{T}) where {T} = Ref{Type{T}}(T)
+broadcastable(x::AbstractArray) = x
+# In the future, default to collecting arguments. TODO: uncomment once deprecations are removed
+# broadcastable(x) = BroadcastStyle(typeof(x)) isa Unknown ? collect(x) : x
+# broadcastable(::Union{AbstractDict, NamedTuple}) = error("intentionally unimplemented to allow development in 1.x")
+
 """
     broadcast!(f, dest, As...)
 
@@ -404,7 +421,10 @@ Note that `dest` is only used to store the result, and does not supply
 arguments to `f` unless it is also listed in the `As`,
 as in `broadcast!(f, A, A, B)` to perform `A[:] = broadcast(f, A, B)`.
 """
-@inline broadcast!(f::Tf, dest, As::Vararg{Any,N}) where {Tf,N} = broadcast!(f, dest, combine_styles(As...), As...)
+@inline function broadcast!(f::Tf, dest, As::Vararg{Any,N}) where {Tf,N}
+    As′ = map(broadcastable, As)
+    broadcast!(f, dest, combine_styles(As′...), As′...)
+end
 @inline broadcast!(f::Tf, dest, ::BroadcastStyle, As::Vararg{Any,N}) where {Tf,N} = broadcast!(f, dest, nothing, As...)
 
 # Default behavior (separated out so that it can be called by users who want to extend broadcast!).
@@ -419,14 +439,14 @@ as in `broadcast!(f, A, A, B)` to perform `A[:] = broadcast(f, A, B)`.
     return dest
 end
 
-# Optimization for the all-Scalar case.
-@inline function broadcast!(f, dest, ::Scalar, As::Vararg{Any, N}) where N
+# Optimization for the case where all arguments are 0-dimensional
+@inline function broadcast!(f, dest, ::AbstractArrayStyle{0}, As::Vararg{Any, N}) where N
     if dest isa AbstractArray
         if f isa typeof(identity) && N == 1
-            return fill!(dest, As[1])
+            return fill!(dest, As[1][])
         else
             @inbounds for I in eachindex(dest)
-                dest[I] = f(As...)
+                dest[I] = f(map(getindex, As)...)
             end
             return dest
         end
@@ -501,9 +521,8 @@ maptoTuple(f, a, b...) = Tuple{f(a), maptoTuple(f, b...).types...}
 #     A, broadcast_indices(A)
 # )::_broadcast_getindex_eltype(A)
 _broadcast_getindex_eltype(A) = _broadcast_getindex_eltype(combine_styles(A), A)
-_broadcast_getindex_eltype(::Scalar, ::Type{T}) where T = Type{T}
-_broadcast_getindex_eltype(::Union{Unknown,Scalar}, A) = typeof(A)
 _broadcast_getindex_eltype(::BroadcastStyle, A) = eltype(A)  # Tuple, Array, etc.
+_broadcast_getindex_eltype(::DefaultArrayStyle{0}, ::Ref{T}) where {T} = T
 
 # Inferred eltype of result of broadcast(f, xs...)
 combine_eltypes(f, A, As...) =
@@ -512,17 +531,23 @@ combine_eltypes(f, A, As...) =
 """
     broadcast(f, As...)
 
-Broadcasts the arrays, tuples, `Ref`s and/or scalars `As` to a
-container of the appropriate type and dimensions. In this context, anything
-that is not a subtype of `AbstractArray`, `Ref` (except for `Ptr`s) or `Tuple`
-is considered a scalar. The resulting container is established by
-the following rules:
+Broadcast the function `f` over the arrays, tuples, collections, `Ref`s and/or scalars `As`.
 
- - If all the arguments are scalars, it returns a scalar.
- - If the arguments are tuples and zero or more scalars, it returns a tuple.
- - If the arguments contain at least one array or `Ref`, it returns an array
-   (expanding singleton dimensions), and treats `Ref`s as 0-dimensional arrays,
-   and tuples as 1-dimensional arrays.
+Broadcasting applies the function `f` over the elements of the container arguments and the
+scalars themselves in `As`. Singleton and missing dimensions are expanded to match the
+extents of the other arguments by virtually repeating the value. By default, only a limited
+number of types are considered scalars, including `Number`s, `String`s, `Symbol`s, `Type`s,
+`Function`s and some common singletons like `missing` and `nothing`. All other arguments are
+iterated over or indexed into elementwise.
+
+The resulting container type is established by the following rules:
+
+ - If all the arguments are scalars or zero-dimensional arrays, it returns an unwrapped scalar.
+ - If at least one argument is a tuple and all others are scalars or zero-dimensional arrays,
+   it returns a tuple.
+ - All other combinations of arguments default to returning an `Array`, but
+   custom container types can define their own implementation and promotion-like
+   rules to customize the result when they appear as arguments.
 
 A special syntax exists for broadcasting: `f.(args...)` is equivalent to
 `broadcast(f, args...)`, and nested `f.(g.(args...))` calls are fused into a
@@ -584,12 +609,17 @@ julia> string.(("one","two","three","four"), ": ", 1:4)
 
 ```
 """
-@inline broadcast(f, A, Bs...) =
-    broadcast(f, combine_styles(A, Bs...), nothing, nothing, A, Bs...)
+@inline function broadcast(f, A, Bs...)
+    A′ = broadcastable(A)
+    Bs′ = map(broadcastable, Bs)
+    broadcast(f, combine_styles(A′, Bs′...), nothing, nothing, A′, Bs′...)
+end
+
+# In the scalar case we unwrap the arguments and just call `f`
+@inline broadcast(f, ::AbstractArrayStyle{0}, ::Nothing, ::Nothing, A, Bs...) = f(A[], map(getindex, Bs)...)
 
 @inline broadcast(f, s::BroadcastStyle, ::Nothing, ::Nothing, A, Bs...) =
-    broadcast(f, s, combine_eltypes(f, A, Bs...), combine_indices(A, Bs...),
-              A, Bs...)
+    broadcast(f, s, combine_eltypes(f, A, Bs...), combine_indices(A, Bs...), A, Bs...)
 
 const NonleafHandlingTypes = Union{DefaultArrayStyle,ArrayConflict}
 
@@ -628,13 +658,11 @@ function broadcast_nonleaf(f, s::NonleafHandlingTypes, ::Type{ElType}, shape::In
     _broadcast!(f, dest, keeps, Idefaults, As, Val(nargs), iter, st, 1)
 end
 
-broadcast(f, ::Union{Scalar,Unknown}, ::Nothing, ::Nothing, a...) = f(a...)
-
 @inline broadcast(f, ::Style{Tuple}, ::Nothing, ::Nothing, A, Bs...) =
     tuplebroadcast(f, longest_tuple(A, Bs...), A, Bs...)
 @inline tuplebroadcast(f, ::NTuple{N,Any}, As...) where {N} =
     ntuple(k -> f(tuplebroadcast_getargs(As, k)...), Val(N))
-@inline tuplebroadcast(f, ::NTuple{N,Any}, ::Type{T}, As...) where {N,T} =
+@inline tuplebroadcast(f, ::NTuple{N,Any}, ::Ref{Type{T}}, As...) where {N,T} =
     ntuple(k -> f(T, tuplebroadcast_getargs(As, k)...), Val(N))
 longest_tuple(A::Tuple, B::Tuple, Bs...) = longest_tuple(_longest_tuple(A, B), Bs...)
 longest_tuple(A, B::Tuple, Bs...) = longest_tuple(B, Bs...)

base/deprecated.jl

@@ -687,6 +687,21 @@ end
 # After deprecation is removed, enable the @testset "indexing by Bool values" in test/arrayops.jl
 # Also un-comment the new definition in base/indices.jl
 
+# Broadcast no longer defaults to treating its arguments as scalar (#)
+@noinline function Broadcast.broadcastable(x)
+    if Base.Broadcast.BroadcastStyle(typeof(x)) isa Broadcast.Unknown
+        depwarn("""
+            broadcast will default to iterating over its arguments in the future. Wrap arguments of
+            type `x::$(typeof(x))` with `Ref(x)` to ensure they broadcast as "scalar" elements.
+            """, (:broadcast, :broadcast!))
+        return Ref{typeof(x)}(x)
+    else
+        return x
+    end
+end
+@eval Base.Broadcast Base.@deprecate_binding Scalar DefaultArrayStyle{0} false
+# After deprecation is removed, enable the fallback broadcastable definitions in base/broadcast.jl
+
 # deprecate BitArray{...}(shape...) constructors to BitArray{...}(undef, shape...) equivalents
 @deprecate BitArray{N}(dims::Vararg{Int,N}) where {N}   BitArray{N}(undef, dims)
 @deprecate BitArray(dims::NTuple{N,Int}) where {N}      BitArray(undef, dims...)

base/range.jl

@@ -514,7 +514,7 @@ function _getindex_hiprec(r::StepRangeLen, i::Integer)  # without rounding by T
 end
 
 function unsafe_getindex(r::LinRange, i::Integer)
-    lerpi.(i-1, r.lendiv, r.start, r.stop)
+    lerpi(i-1, r.lendiv, r.start, r.stop)
 end
 
 function lerpi(j::Integer, d::Integer, a::T, b::T) where T

base/version.jl

@@ -67,6 +67,8 @@ function print(io::IO, v::VersionNumber)
 end
 show(io::IO, v::VersionNumber) = print(io, "v\"", v, "\"")
 
+Broadcast.broadcastable(v::VersionNumber) = Ref(v)
+
 const VERSION_REGEX = r"^
     v?                                      # prefix        (optional)
     (\d+)                                   # major         (required)

stdlib/Dates/src/arithmetic.jl

@@ -97,3 +97,6 @@ end
 # AbstractArray{TimeType}, AbstractArray{TimeType}
 (-)(x::OrdinalRange{T}, y::OrdinalRange{T}) where {T<:TimeType} = Vector(x) - Vector(y)
 (-)(x::AbstractRange{T}, y::AbstractRange{T}) where {T<:TimeType} = Vector(x) - Vector(y)
+
+# Allow dates and times to broadcast as unwrapped scalars
+Base.Broadcast.broadcastable(x::AbstractTime) = Ref(x)

stdlib/Dates/src/io.jl

@@ -376,6 +376,7 @@ function Base.show(io::IO, df::DateFormat)
     end
     print(io, '"')
 end
+Base.Broadcast.broadcastable(x::DateFormat) = Ref(x)
 
 """
     dateformat"Y-m-d H:M:S"

stdlib/Pkg3/src/Operations.jl

@@ -657,7 +657,7 @@ function with_dependencies_loadable_at_toplevel(f, mainctx::Context, pkg::Packag
     need_to_resolve = false
 
     if Types.is_project(localctx.env, pkg)
-        delete!.(localctx.env.project, ["name", "uuid", "version"])
+        foreach(k->delete!(localctx.env.project, k), ("name", "uuid", "version"))
         localctx.env.project["deps"][pkg.name] = string(pkg.uuid)
         localctx.env.manifest[pkg.name] = [Dict(
             "deps" => mainctx.env.project["deps"],

stdlib/Pkg3/src/Types.jl

@@ -482,7 +482,7 @@ mutable struct EnvCache
         git = ispath(joinpath(project_dir, ".git")) ? LibGit2.GitRepo(project_dir) : nothing
 
         project = read_project(project_file)
-        if any(haskey.(project, ["name", "uuid", "version"]))
+        if any(k->haskey(project, k), ("name", "uuid", "version"))
             project_package = PackageSpec(
                 get(project, "name", ""),
                 UUID(get(project, "uuid", 0)),

stdlib/Pkg3/src/resolve/FieldValues.jl

@@ -105,4 +105,7 @@ function secondmax(f::Field, msk::BitVector = trues(length(f)))
     return m2 - m
 end
 
+# Support broadcasting like a scalar by default
+Base.Broadcast.broadcastable(a::FieldValue) = Ref(a)
+
 end

stdlib/SparseArrays/src/higherorderfns.jl

@@ -933,6 +933,8 @@ end
 
 nonscalararg(::SparseVecOrMat) = true
 nonscalararg(::Any) = false
+scalarwrappedarg(::Union{AbstractArray{<:Any,0},Ref}) = true
+scalarwrappedarg(::Any) = false
 
 @inline function _capturescalars()
     return (), () -> ()
@@ -941,6 +943,8 @@ end
     let (rest, f) = _capturescalars(mixedargs...)
         if nonscalararg(arg)
             return (arg, rest...), (head, tail...) -> (head, f(tail...)...) # pass-through to broadcast
+        elseif scalarwrappedarg(arg)
+            return rest, (tail...) -> (arg[], f(tail...)...) # unwrap and add back scalararg after (in makeargs)
         else
             return rest, (tail...) -> (arg, f(tail...)...) # add back scalararg after (in makeargs)
         end
@@ -949,6 +953,8 @@ end
 @inline function _capturescalars(arg) # this definition is just an optimization (to bottom out the recursion slightly sooner)
     if nonscalararg(arg)
         return (arg,), (head,) -> (head,) # pass-through
+    elseif scalarwrappedarg(arg)
+        return (), () -> (arg[],) # unwrap
     else
         return (), () -> (arg,) # add scalararg
     end
@@ -977,9 +983,10 @@ Broadcast.BroadcastStyle(::Type{<:StructuredMatrix}) = PromoteToSparse()
 Broadcast.BroadcastStyle(::Type{<:Adjoint{T,<:Union{SparseVector,SparseMatrixCSC}} where T}) = PromoteToSparse()
 Broadcast.BroadcastStyle(::Type{<:Transpose{T,<:Union{SparseVector,SparseMatrixCSC}} where T}) = PromoteToSparse()
 
-Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{0}) = PromoteToSparse()
+Broadcast.BroadcastStyle(s::SPVM, ::Broadcast.AbstractArrayStyle{0}) = s
 Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{1}) = PromoteToSparse()
 Broadcast.BroadcastStyle(::SPVM, ::Broadcast.DefaultArrayStyle{2}) = PromoteToSparse()
+
 Broadcast.BroadcastStyle(::PromoteToSparse, ::SPVM) = PromoteToSparse()
 Broadcast.BroadcastStyle(::PromoteToSparse, ::Broadcast.Style{Tuple}) = Broadcast.DefaultArrayStyle{2}()
 
@@ -992,7 +999,8 @@ is_supported_sparse_broadcast(::AbstractSparseArray, rest...) = is_supported_spa
 is_supported_sparse_broadcast(::StructuredMatrix, rest...) = is_supported_sparse_broadcast(rest...)
 is_supported_sparse_broadcast(::Array, rest...) = is_supported_sparse_broadcast(rest...)
 is_supported_sparse_broadcast(t::Union{Transpose, Adjoint}, rest...) = is_supported_sparse_broadcast(t.parent, rest...)
-is_supported_sparse_broadcast(x, rest...) = BroadcastStyle(typeof(x)) === Broadcast.Scalar() && is_supported_sparse_broadcast(rest...)
+is_supported_sparse_broadcast(x, rest...) = axes(x) === () && is_supported_sparse_broadcast(rest...)
+is_supported_sparse_broadcast(x::Ref, rest...) = is_supported_sparse_broadcast(rest...)
 function broadcast(f, s::PromoteToSparse, ::Nothing, ::Nothing, As::Vararg{Any,N}) where {N}
     if is_supported_sparse_broadcast(As...)
         return broadcast(f, map(_sparsifystructured, As)...)

test/broadcast.jl

@@ -408,8 +408,8 @@ end
 
 # Ref as 0-dimensional array for broadcast
 @test (-).(C_NULL, C_NULL)::UInt == 0
-@test (+).(1, Ref(2)) == fill(3)
-@test (+).(Ref(1), Ref(2)) == fill(3)
+@test (+).(1, Ref(2)) == 3
+@test (+).(Ref(1), Ref(2)) == 3
 @test (+).([[0,2], [1,3]], Ref{Vector{Int}}([1,-1])) == [[1,1], [2,2]]
 
 # Check that broadcast!(f, A) populates A via independent calls to f (#12277, #19722),
@@ -545,7 +545,7 @@ end
 # Test that broadcast treats type arguments as scalars, i.e. containertype yields Any,
 # even for subtypes of abstract array. (https://github.com/JuliaStats/DataArrays.jl/issues/229)
 @testset "treat type arguments as scalars, DataArrays issue 229" begin
-    @test Broadcast.combine_styles(AbstractArray) == Broadcast.Scalar()
+    @test Broadcast.combine_styles(AbstractArray) == Broadcast.Unknown()
     @test broadcast(==, [1], AbstractArray) == BitArray([false])
     @test broadcast(==, 1, AbstractArray) == false
 end
@@ -574,7 +574,7 @@ end
     @test broadcast(foo, "x", [1, 2, 3]) == ["hello", "hello", "hello"]
 
     @test isequal(
-        [Set([1]), Set([2])] .∪ Set([3]),
+        [Set([1]), Set([2])] .∪ Ref(Set([3])),
         [Set([1, 3]), Set([2, 3])])
 end