add keep option to readuntil; change default to exclude delimiter

NEWS.md

@@ -196,6 +196,9 @@ Breaking changes
 
 This section lists changes that do not have deprecation warnings.
 
+  * `readuntil` now does *not* include the delimiter in its result, matching the
+    behavior of `readline`. Pass `keep=true` to get the old behavior ([#25633]).
+
   * `getindex(s::String, r::UnitRange{Int})` now throws `UnicodeError` if `last(r)`
     is not a valid index into `s` ([#22572]).
 

base/io.jl

@@ -228,8 +228,10 @@ flush(io::AbstractPipe) = flush(pipe_writer(io))
 read(io::AbstractPipe, byte::Type{UInt8}) = read(pipe_reader(io), byte)
 unsafe_read(io::AbstractPipe, p::Ptr{UInt8}, nb::UInt) = unsafe_read(pipe_reader(io), p, nb)
 read(io::AbstractPipe) = read(pipe_reader(io))
-readuntil(io::AbstractPipe, arg::UInt8) = readuntil(pipe_reader(io), arg)
-readuntil(io::AbstractPipe, arg::Char) = readuntil(pipe_reader(io), arg)
+readuntil(io::AbstractPipe, arg::UInt8; kw...) = readuntil(pipe_reader(io), arg; kw...)
+readuntil(io::AbstractPipe, arg::Char; kw...) = readuntil(pipe_reader(io), arg; kw...)
+readuntil(io::AbstractPipe, arg::AbstractString; kw...) = readuntil(pipe_reader(io), arg; kw...)
+readuntil(io::AbstractPipe, arg::AbstractVector; kw...) = readuntil(pipe_reader(io), arg; kw...)
 readuntil_indexable(io::AbstractPipe, target#=::Indexable{T}=#, out) = readuntil_indexable(pipe_reader(io), target, out)
 
 readavailable(io::AbstractPipe) = readavailable(pipe_reader(io))
@@ -297,10 +299,12 @@ function read! end
 read!(filename::AbstractString, a) = open(io->read!(io, a), filename)
 
 """
-    readuntil(stream::IO, delim)
-    readuntil(filename::AbstractString, delim)
+    readuntil(stream::IO, delim; keep::Bool = false)
+    readuntil(filename::AbstractString, delim; keep::Bool = false)
 
-Read a string from an I/O stream or a file, up to and including the given delimiter byte.
+Read a string from an I/O stream or a file, up to the given delimiter.
+The delimiter can be a `UInt8`, `Char`, string, or vector.
+Keyword argument `keep` controls whether the delimiter is included in the result.
 The text is assumed to be encoded in UTF-8.
 
 # Examples
@@ -319,7 +323,7 @@ julia> readuntil("my_file.txt", '.')
 julia> rm("my_file.txt")
 ```
 """
-readuntil(filename::AbstractString, args...) = open(io->readuntil(io, args...), filename)
+readuntil(filename::AbstractString, args...; kw...) = open(io->readuntil(io, args...; kw...), filename)
 
 """
     readline(io::IO=STDIN; keep::Bool=false)
@@ -363,7 +367,7 @@ function readline(s::IO=STDIN; chomp=nothing, keep::Bool=false)
         keep = !chomp
         depwarn("The `chomp=$chomp` argument to `readline` is deprecated in favor of `keep=$keep`.", :readline)
     end
-    line = readuntil(s, 0x0a)
+    line = readuntil(s, 0x0a, keep=true)
     i = length(line)
     if keep || i == 0 || line[i] != 0x0a
         return String(line)
@@ -626,41 +630,44 @@ end
 
 # readuntil_string is useful below since it has
 # an optimized method for s::IOStream
-readuntil_string(s::IO, delim::UInt8) = String(readuntil(s, delim))
+readuntil_string(s::IO, delim::UInt8, keep::Bool) = String(readuntil(s, delim, keep=keep))
 
-function readuntil(s::IO, delim::Char)
+function readuntil(s::IO, delim::Char; keep::Bool=false)
     if delim ≤ '\x7f'
-        return readuntil_string(s, delim % UInt8)
+        return readuntil_string(s, delim % UInt8, keep)
     end
     out = IOBuffer()
     while !eof(s)
         c = read(s, Char)
-        write(out, c)
         if c == delim
+            keep && write(out, c)
             break
         end
+        write(out, c)
     end
     return String(take!(out))
 end
 
-function readuntil(s::IO, delim::T) where T
+function readuntil(s::IO, delim::T; keep::Bool=false) where T
     out = (T === UInt8 ? StringVector(0) : Vector{T}())
     while !eof(s)
         c = read(s, T)
-        push!(out, c)
         if c == delim
+            keep && push!(out, c)
             break
         end
+        push!(out, c)
     end
     return out
 end
 
 # requires that indices for target are small ordered integers bounded by start and endof
+# returns whether the delimiter was matched
 function readuntil_indexable(io::IO, target#=::Indexable{T}=#, out)
     T = eltype(target)
     first = start(target)
     if done(target, first)
-        return
+        return true
     end
     len = endof(target)
     local cache # will be lazy initialized when needed
@@ -701,39 +708,45 @@ function readuntil_indexable(io::IO, target#=::Indexable{T}=#, out)
                 pos = cache[pos] + first
             end
         end
-        done(target, pos) && break
+        done(target, pos) && return true
     end
+    return false
 end
 
-function readuntil(io::IO, target::AbstractString)
+function readuntil(io::IO, target::AbstractString; keep::Bool=false)
     # small-string target optimizations
     i = start(target)
     done(target, i) && return ""
     c, i = next(target, start(target))
     if done(target, i) && c <= '\x7f'
-        return readuntil_string(io, c % UInt8)
+        return readuntil_string(io, c % UInt8, keep)
     end
-    # decide how we can index target
-    if target isa String
-        # convert String to a utf8-byte-iterator
+    # convert String to a utf8-byte-iterator
+    if target isa String || target isa SubString{String}
         target = codeunits(target)
-    #elseif applicable(codeunit, target)
-    #   TODO: a more general version of above optimization
-    #         would be to permit accessing any string via codeunit
-    #   target = CodeUnitVector(target)
-    elseif !(target isa SubString{String})
-        # type with unknown indexing behavior: convert to array
-        target = collect(target)
-    end
-    out = (eltype(target) === UInt8 ? StringVector(0) : IOBuffer())
-    readuntil_indexable(io, target, out)
-    out = isa(out, IO) ? take!(out) : out
+    else
+        target = codeunits(String(target))
+    end
+    out = StringVector(0)
+    found = readuntil_indexable(io, target, out)
+    if !keep && found
+        lo, lt = length(out), length(target)
+        if lt <= lo
+            resize!(out, lo - lt)
+        end
+    end
     return String(out)
 end
 
-function readuntil(io::IO, target::AbstractVector{T}) where T
+function readuntil(io::IO, target::AbstractVector{T}; keep::Bool=false) where T
     out = (T === UInt8 ? StringVector(0) : Vector{T}())
-    readuntil_indexable(io, target, out)
+    found = readuntil_indexable(io, target, out)
+    if !keep && found
+        lo, lt = length(out), length(target)
+        if lt <= lo
+            resize!(out, lo - lt)
+        end
+    end
     return out
 end
 

base/iobuffer.jl

@@ -444,26 +444,30 @@ function findfirst(delim::EqualTo{UInt8}, buf::GenericIOBuffer)
     return nothing
 end
 
-function readuntil(io::GenericIOBuffer, delim::UInt8)
+function readuntil(io::GenericIOBuffer, delim::UInt8; keep::Bool=false)
     lb = 70
     A = StringVector(lb)
-    n = 0
+    nread = 0
+    nout = 0
     data = io.data
     for i = io.ptr : io.size
-        n += 1
-        if n > lb
-            lb = n*2
-            resize!(A, lb)
-        end
         @inbounds b = data[i]
-        @inbounds A[n] = b
+        nread += 1
+        if keep || b != delim
+            nout += 1
+            if nout > lb
+                lb = nout*2
+                resize!(A, lb)
+            end
+            @inbounds A[nout] = b
+        end
         if b == delim
             break
         end
     end
-    io.ptr += n
-    if lb != n
-        resize!(A, n)
+    io.ptr += nread
+    if lb != nout
+        resize!(A, nout)
     end
     A
 end

base/iostream.jl

@@ -357,21 +357,21 @@ end
 take!(s::IOStream) =
     ccall(:jl_take_buffer, Vector{UInt8}, (Ptr{Cvoid},), s.ios)
 
-function readuntil(s::IOStream, delim::UInt8)
-    ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, delim, 0, 0)
+function readuntil(s::IOStream, delim::UInt8; keep::Bool=false)
+    ccall(:jl_readuntil, Array{UInt8,1}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, delim, 0, !keep)
 end
 
 # like readuntil, above, but returns a String without requiring a copy
-function readuntil_string(s::IOStream, delim::UInt8)
-    ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, delim, 1, false)
+function readuntil_string(s::IOStream, delim::UInt8, keep::Bool)
+    ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, delim, 1, !keep)
 end
 
 function readline(s::IOStream; chomp=nothing, keep::Bool=false)
     if chomp !== nothing
         keep = !chomp
         depwarn("The `chomp=$chomp` argument to `readline` is deprecated in favor of `keep=$keep`.", :readline)
     end
-    ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, '\n', 1, !keep)
+    ccall(:jl_readuntil, Ref{String}, (Ptr{Cvoid}, UInt8, UInt8, UInt8), s.ios, '\n', 1, keep ? 0 : 2)
 end
 
 function readbytes_all!(s::IOStream, b::Array{UInt8}, nb)

base/loading.jl

@@ -1057,7 +1057,7 @@ function create_expr_cache(input::String, output::String, concrete_deps::typeof(
     rm(output, force=true)   # Remove file if it exists
     code_object = """
         while !eof(STDIN)
-            code = chop(readuntil(STDIN, '\\0'))
+            code = readuntil(STDIN, '\\0')
             eval(Main, Meta.parse(code))
         end
         """

base/pkg/entry.jl

@@ -634,8 +634,8 @@ function build!(pkgs::Vector, errs::Dict, seen::Set=Set())
     mktemp() do errfile, f
         build!(pkgs, seen, errfile)
         while !eof(f)
-            pkg = chop(readuntil(f, '\0'))
-            err = chop(readuntil(f, '\0'))
+            pkg = readuntil(f, '\0')
+            err = readuntil(f, '\0')
             errs[pkg] = err
         end
     end

base/repl/LineEdit.jl

@@ -1815,8 +1815,7 @@ function setup_search_keymap(hp)
         # Bracketed paste mode
         "\e[200~" => (s,data,c)-> begin
             ps = state(s, mode(s))
-            str = readuntil(ps.terminal, "\e[201~")
-            input = str[1:prevind(str, end-5)]
+            input = readuntil(ps.terminal, "\e[201~", keep=true)
             edit_insert(data.query_buffer, input); update_display_buffer(s, data)
         end,
         "*"       => (s,data,c)->(edit_insert(data.query_buffer, c); update_display_buffer(s, data))
@@ -1880,8 +1879,7 @@ end
 
 function bracketed_paste(s; tabwidth=options(s).tabwidth)
     ps = state(s, mode(s))
-    str = readuntil(ps.terminal, "\e[201~")
-    input = str[1:prevind(str, end-5)]
+    input = readuntil(ps.terminal, "\e[201~")
     input = replace(input, '\r' => '\n')
     if position(buffer(s)) == 0
         indent = Base.indentation(input; tabwidth=tabwidth)[1]

base/stream.jl

@@ -840,11 +840,11 @@ function readavailable(this::LibuvStream)
     return take!(buf)
 end
 
-function readuntil(this::LibuvStream, c::UInt8)
+function readuntil(this::LibuvStream, c::UInt8; keep::Bool=false)
     wait_readbyte(this, c)
     buf = this.buffer
     @assert buf.seekable == false
-    return readuntil(buf, c)
+    return readuntil(buf, c, keep=keep)
 end
 
 uv_write(s::LibuvStream, p::Vector{UInt8}) = uv_write(s, pointer(p), UInt(sizeof(p)))

src/sys.c

@@ -252,24 +252,29 @@ JL_DLLEXPORT jl_array_t *jl_take_buffer(ios_t *s)
     return a;
 }
 
+// str: if 1 return a string, otherwise return a Vector{UInt8}
+// chomp:
+//   0 - keep delimiter
+//   1 - remove 1 byte delimiter
+//   2 - remove 2 bytes \r\n if present
 JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint8_t chomp)
 {
     jl_array_t *a;
     // manually inlined common case
     char *pd = (char*)memchr(s->buf + s->bpos, delim, (size_t)(s->size - s->bpos));
     if (pd) {
         size_t n = pd - (s->buf + s->bpos) + 1;
+        size_t nchomp = 0;
+        if (chomp) {
+            nchomp = chomp == 2 ? ios_nchomp(s, n) : 1;
+        }
         if (str) {
-            size_t nchomp = 0;
-            if (chomp) {
-                nchomp = ios_nchomp(s, n);
-            }
             jl_value_t *str = jl_pchar_to_string(s->buf + s->bpos, n - nchomp);
             s->bpos += n;
             return str;
         }
-        a = jl_alloc_array_1d(jl_array_uint8_type, n);
-        memcpy(jl_array_data(a), s->buf + s->bpos, n);
+        a = jl_alloc_array_1d(jl_array_uint8_type, n - nchomp);
+        memcpy(jl_array_data(a), s->buf + s->bpos, n - nchomp);
         s->bpos += n;
     }
     else {
@@ -278,9 +283,9 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint
         ios_mem(&dest, 0);
         ios_setbuf(&dest, (char*)a->data, 80, 0);
         size_t n = ios_copyuntil(&dest, s, delim);
-        if (chomp && n > 0 && dest.buf[n - 1] == '\n') {
+        if (chomp && n > 0 && dest.buf[n - 1] == delim) {
             n--;
-            if (n > 0 && dest.buf[n - 1] == '\r') {
+            if (chomp == 2 && n > 0 && dest.buf[n - 1] == '\r') {
                 n--;
             }
             int truncret = ios_trunc(&dest, n); // it should always be possible to truncate dest

test/TestHelpers.jl

@@ -105,7 +105,7 @@ function challenge_prompt(cmd::Cmd, challenges; timeout::Integer=10, debug::Bool
 
         try
             for (challenge, response) in challenges
-                write(out, readuntil(master, challenge))
+                write(out, readuntil(master, challenge, keep=true))
                 if !isopen(master)
                     error("Could not locate challenge: \"$challenge\". ",
                           format_output(out))

test/perf/shootout/revcomp.jl

@@ -43,7 +43,7 @@ function revcomp(infile="revcomp-input.txt")
     input = open(infile, "r")
     buff = UInt8[]
     while true
-        line = readuntil(input, UInt8('\n'))
+        line = readuntil(input, UInt8('\n'), keep=true)
         if isempty(line)
 #            print_buff(buff)
             return