Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add skipnothing function which returns an iterator without nothing #30549

Closed
wants to merge 12 commits into from
Closed

Add skipnothing function which returns an iterator without nothing #30549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
b0f2fc5
Add skipnothing function which returns an iterator without `nothing`
galenlynch Jan 1, 2019
0f1ace6
Add PR number for skipnothing to NEWS.md
galenlynch Jan 1, 2019
b9d109a
Combine skipmissing and skipnothing into skipoftype
galenlynch Jan 2, 2019
8a6439b
Allow union_poptype to handle partially-overlapping unions
galenlynch Jan 17, 2019
1f4bfc3
Rename union_poptype to typesubtract
galenlynch Mar 14, 2019
d801ea6
Use Core.Compiler.typesubtract instead of reinventing it poorly
galenlynch Mar 14, 2019
e76e3e9
Adapt indexing functions from SkipMissing to SkipOfType
galenlynch Mar 14, 2019
92072a9
Change skipoftype to skip
galenlynch Mar 30, 2019
68fa222
Remove skipnothing
galenlynch Mar 30, 2019
6cec3ac
Make filter(f, itr::Skip) return a vector
galenlynch Mar 30, 2019
051ed30
Fix doctests
galenlynch Mar 30, 2019
6272728
Undo preallocation
galenlynch Mar 31, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions NEWS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ New library functions
* Added `Base.hasproperty` and `Base.hasfield` ([#28850]).
* One argument `!=(x)`, `>(x)`, `>=(x)`, `<(x)`, `<=(x)` has been added for currying,
similar to the existing `==(x)` and `isequal(x)` methods ([#30915]).
* `skip(T, itr)` function returns an iterator skipping values of type T ([#30549]).

Standard library changes
------------------------
Expand Down
3 changes: 3 additions & 0 deletions base/Base.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,9 @@ end
include("multimedia.jl")
using .Multimedia

# SkipOfType type
include("skip.jl")

# Some type
include("some.jl")

Expand Down
1 change: 1 addition & 0 deletions base/exports.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -671,6 +671,7 @@ export
coalesce,
ismissing,
missing,
skip,
skipmissing,
something,
isnothing,
Expand Down
158 changes: 3 additions & 155 deletions base/missing.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ of the input.
# Examples
```jldoctest
julia> x = skipmissing([1, missing, 2])
Base.SkipMissing{Array{Union{Missing, Int64},1}}(Union{Missing, Int64}[1, missing, 2])
Base.Skip{Missing,Array{Union{Missing, Int64},1}}(Union{Missing, Int64}[1, missing, 2])

julia> sum(x)
3
Expand All @@ -184,7 +184,7 @@ julia> x[1]
1

julia> x[2]
ERROR: MissingException: the value at index (2,) is missing
ERROR: MissingException: the value at index (2,) is of type Missing
[...]

julia> argmax(x)
Expand All @@ -206,159 +206,7 @@ julia> collect(skipmissing([1 missing; 2 missing]))
2
```
"""
skipmissing(itr) = SkipMissing(itr)

struct SkipMissing{T}
x::T
end
IteratorSize(::Type{<:SkipMissing}) = SizeUnknown()
IteratorEltype(::Type{SkipMissing{T}}) where {T} = IteratorEltype(T)
eltype(::Type{SkipMissing{T}}) where {T} = nonmissingtype(eltype(T))

function iterate(itr::SkipMissing, state...)
y = iterate(itr.x, state...)
y === nothing && return nothing
item, state = y
while item === missing
y = iterate(itr.x, state)
y === nothing && return nothing
item, state = y
end
item, state
end

IndexStyle(::Type{<:SkipMissing{T}}) where {T} = IndexStyle(T)
eachindex(itr::SkipMissing) =
Iterators.filter(i -> @inbounds(itr.x[i]) !== missing, eachindex(itr.x))
keys(itr::SkipMissing) =
Iterators.filter(i -> @inbounds(itr.x[i]) !== missing, keys(itr.x))
@propagate_inbounds function getindex(itr::SkipMissing, I...)
v = itr.x[I...]
v === missing && throw(MissingException("the value at index $I is missing"))
v
end

# Optimized mapreduce implementation
# The generic method is faster when !(eltype(A) >: Missing) since it does not need
# additional loops to identify the two first non-missing values of each block
mapreduce(f, op, itr::SkipMissing{<:AbstractArray}) =
_mapreduce(f, op, IndexStyle(itr.x), eltype(itr.x) >: Missing ? itr : itr.x)

function _mapreduce(f, op, ::IndexLinear, itr::SkipMissing{<:AbstractArray})
A = itr.x
local ai
inds = LinearIndices(A)
i = first(inds)
ilast = last(inds)
while i <= ilast
@inbounds ai = A[i]
ai === missing || break
i += 1
end
i > ilast && return mapreduce_empty(f, op, eltype(itr))
a1 = ai
i += 1
while i <= ilast
@inbounds ai = A[i]
ai === missing || break
i += 1
end
i > ilast && return mapreduce_first(f, op, a1)
# We know A contains at least two non-missing entries: the result cannot be nothing
something(mapreduce_impl(f, op, itr, first(inds), last(inds)))
end

_mapreduce(f, op, ::IndexCartesian, itr::SkipMissing) = mapfoldl(f, op, itr)

mapreduce_impl(f, op, A::SkipMissing, ifirst::Integer, ilast::Integer) =
mapreduce_impl(f, op, A, ifirst, ilast, pairwise_blocksize(f, op))

# Returns nothing when the input contains only missing values, and Some(x) otherwise
@noinline function mapreduce_impl(f, op, itr::SkipMissing{<:AbstractArray},
ifirst::Integer, ilast::Integer, blksize::Int)
A = itr.x
if ifirst == ilast
@inbounds a1 = A[ifirst]
if a1 === missing
return nothing
else
return Some(mapreduce_first(f, op, a1))
end
elseif ifirst + blksize > ilast
# sequential portion
local ai
i = ifirst
while i <= ilast
@inbounds ai = A[i]
ai === missing || break
i += 1
end
i > ilast && return nothing
a1 = ai::eltype(itr)
i += 1
while i <= ilast
@inbounds ai = A[i]
ai === missing || break
i += 1
end
i > ilast && return Some(mapreduce_first(f, op, a1))
a2 = ai::eltype(itr)
i += 1
v = op(f(a1), f(a2))
@simd for i = i:ilast
@inbounds ai = A[i]
if ai !== missing
v = op(v, f(ai))
end
end
return Some(v)
else
# pairwise portion
imid = (ifirst + ilast) >> 1
v1 = mapreduce_impl(f, op, itr, ifirst, imid, blksize)
v2 = mapreduce_impl(f, op, itr, imid+1, ilast, blksize)
if v1 === nothing && v2 === nothing
return nothing
elseif v1 === nothing
return v2
elseif v2 === nothing
return v1
else
return Some(op(something(v1), something(v2)))
end
end
end

"""
filter(f, itr::SkipMissing{<:AbstractArray})

Return a vector similar to the array wrapped by the given `SkipMissing` iterator
but with all missing elements and those for which `f` returns `false` removed.

!!! compat "Julia 1.2"
This method requires Julia 1.2 or later.

# Examples
```jldoctest
julia> x = [1 2; missing 4]
2×2 Array{Union{Missing, Int64},2}:
1 2
missing 4

julia> filter(isodd, skipmissing(x))
1-element Array{Int64,1}:
1
```
"""
function filter(f, itr::SkipMissing{<:AbstractArray})
y = similar(itr.x, eltype(itr), 0)
for xi in itr.x
if xi !== missing && f(xi)
push!(y, xi)
end
end
y
end
skipmissing(itr) = skip(Missing, itr)

"""
coalesce(x, y...)
Expand Down
Loading