From c703642e5ed17ce5769a8695f96275f5961dde9a Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 27 Oct 2017 13:23:24 -0400 Subject: [PATCH] replace Base.Base64 with stdlib Base64 module --- base/base64.jl | 290 ----------------------------------- base/deprecated.jl | 5 + base/exports.jl | 4 - base/multimedia.jl | 4 +- base/stream.jl | 2 - base/sysimg.jl | 45 +++--- doc/src/stdlib/io-network.md | 4 - stdlib/Base64/src/Base64.jl | 2 - test/base64.jl | 46 ------ test/choosetests.jl | 2 +- test/compile.jl | 2 +- 11 files changed, 34 insertions(+), 372 deletions(-) delete mode 100644 base/base64.jl delete mode 100644 test/base64.jl diff --git a/base/base64.jl b/base/base64.jl deleted file mode 100644 index b06c2fbd7a939..0000000000000 --- a/base/base64.jl +++ /dev/null @@ -1,290 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -module Base64 -import Base: read, write, close, eof, empty! -export Base64EncodePipe, Base64DecodePipe, base64encode, base64decode - -# Base64EncodePipe is a pipe-like IO object, which converts into base64 data sent -# to a stream. (You must close the pipe to complete the encode, separate from -# closing the target stream). We also have a function base64encode(f, -# args...) which works like sprint except that it produces -# base64-encoded data, along with base64encode(args...) which is equivalent -# to base64encode(write, args...), to return base64 strings. -# A Base64DecodePipe object can be used to decode base64-encoded data read from a stream -# , while function base64decode is useful for decoding strings -############################################################################# - -""" - Base64EncodePipe(ostream) - -Returns a new write-only I/O stream, which converts any bytes written to it into -base64-encoded ASCII bytes written to `ostream`. -Calling [`close`](@ref) on the `Base64EncodePipe` stream -is necessary to complete the encoding (but does not close `ostream`). - -# Examples -```jldoctest -julia> io = IOBuffer(); - -julia> iob64_encode = Base64EncodePipe(io); - -julia> write(iob64_encode, "Hello!") -6 - -julia> close(iob64_encode); - -julia> str = String(take!(io)) -"SGVsbG8h" - -julia> String(base64decode(str)) -"Hello!" -``` -""" -mutable struct Base64EncodePipe <: IO - io::IO - # writing works in groups of 3, so we need to cache last two bytes written - b0::UInt8 - b1::UInt8 - nb::UInt8 # number of bytes in cache: 0, 1, or 2 - - function Base64EncodePipe(io::IO) - b = new(io,0,0,0) - finalizer(b, close) - return b - end -end - -############################################################################# - -# Based on code by Stefan Karpinski from https://github.com/hackerschool/WebSockets.jl (distributed under the same MIT license as Julia) - -const b64chars = ['A':'Z';'a':'z';'0':'9';'+';'/'] - -const base64_pad = UInt8('=') - -function b64(x::UInt8, y::UInt8, z::UInt8) - n = Int(x)<<16 | Int(y)<<8 | Int(z) - b64chars[(n >> 18) + 1], - b64chars[(n >> 12) & 0b111111 + 1], - b64chars[(n >> 6) & 0b111111 + 1], - b64chars[(n ) & 0b111111 + 1] -end - -function b64(x::UInt8, y::UInt8) - a, b, c = b64(x, y, 0x0) - a, b, c, base64_pad -end - -function b64(x::UInt8) - a, b = b64(x, 0x0, 0x0) - a, b, base64_pad, base64_pad -end - -const sentinel = typemax(UInt8) -const revb64chars = fill(sentinel, 256) -# Fill revb64chars -for (val, ch) in enumerate(b64chars) - revb64chars[UInt8(ch)] = UInt8(val - 1) -end - -# Decode a block of at least 2 and at most 4 bytes, received in encvec -# Returns the first decoded byte and stores up to two more in cache -function b64decode!(encvec::Vector{UInt8}, cache::Vector{UInt8}) - if length(encvec) < 2 - throw(ArgumentError("incorrect base64 format, block must be at least 2 and at most 4 bytes")) - end - @inbounds u = revb64chars[encvec[1]] - @inbounds v = revb64chars[encvec[2]] - empty!(cache) - res = (u << 2) | (v >> 4) - if length(encvec) > 2 - @inbounds w = revb64chars[encvec[3]] - push!(cache, (v << 4) | (w >> 2)) - end - if length(encvec) > 3 - @inbounds z = revb64chars[encvec[4]] - push!(cache, (w << 6) | z) - end - res -end - - -############################################################################# - -function unsafe_write(b::Base64EncodePipe, x::Ptr{UInt8}, n::UInt) - s = 1 # starting index - # finish any cached data to write: - if b.nb == 1 - if n >= 2 - write(b.io, b64(b.b0, unsafe_load(x, 1), unsafe_load(x, 2))...) - s = 3 - elseif n == 1 - b.b1 = unsafe_load(x, 1) - b.nb = 2 - return - else - return - end - elseif b.nb == 2 - if n >= 1 - write(b.io, b64(b.b0, b.b1, unsafe_load(x, 1))...) - s = 2 - else - return - end - end - # write all groups of three bytes: - while s + 2 <= n - write(b.io, b64(unsafe_load(x, s), unsafe_load(x, s + 1), unsafe_load(x, s + 2))...) - s += 3 - end - # cache any leftover bytes: - if s + 1 == n - b.b0 = unsafe_load(x, s) - b.b1 = unsafe_load(x, s + 1) - b.nb = 2 - elseif s == n - b.b0 = unsafe_load(x, s) - b.nb = 1 - else - b.nb = 0 - end - n -end - -function write(b::Base64EncodePipe, x::UInt8) - if b.nb == 0 - b.b0 = x - b.nb = 1 - elseif b.nb == 1 - b.b1 = x - b.nb = 2 - else - write(b.io, b64(b.b0,b.b1,x)...) - b.nb = 0 - end - 1 -end - -function close(b::Base64EncodePipe) - if b.nb > 0 - # write leftover bytes + padding - if b.nb == 1 - write(b.io, b64(b.b0)...) - else # b.nb == 2 - write(b.io, b64(b.b0, b.b1)...) - end - b.nb = 0 - end - nothing -end - -# like sprint, but returns base64 string -""" - base64encode(writefunc, args...) - base64encode(args...) - -Given a [`write`](@ref)-like function `writefunc`, which takes an I/O stream as its first argument, -`base64encode(writefunc, args...)` calls `writefunc` to write `args...` to a base64-encoded -string, and returns the string. `base64encode(args...)` is equivalent to `base64encode(write, args...)`: -it converts its arguments into bytes using the standard [`write`](@ref) functions and returns the -base64-encoded string. - -See also [`base64decode`](@ref). -""" -function base64encode(f::Function, args...) - s = IOBuffer() - b = Base64EncodePipe(s) - f(b, args...) - close(b) - String(take!(s)) -end -base64encode(x...) = base64encode(write, x...) - -############################################################################# - -""" - Base64DecodePipe(istream) - -Returns a new read-only I/O stream, which decodes base64-encoded data read from `istream`. - -# Examples -```jldoctest -julia> io = IOBuffer(); - -julia> iob64_decode = Base64DecodePipe(io); - -julia> write(io, "SGVsbG8h") -8 - -julia> seekstart(io); - -julia> String(read(iob64_decode)) -"Hello!" -``` -""" -mutable struct Base64DecodePipe <: IO - io::IO - # reading works in blocks of 4 characters that are decoded into 3 bytes and 2 of them cached - cache::Vector{UInt8} - encvec::Vector{UInt8} - - function Base64DecodePipe(io::IO) - b = new(io,[],[]) - finalizer(b, close) - return b - end -end - -function read(b::Base64DecodePipe, t::Type{UInt8}) - if !isempty(b.cache) - return shift!(b.cache) - else - empty!(b.encvec) - while !eof(b.io) && length(b.encvec) < 4 - c::UInt8 = read(b.io, t) - @inbounds if revb64chars[c] != sentinel - push!(b.encvec, c) - end - end - return b64decode!(b.encvec,b.cache) - end -end - -eof(b::Base64DecodePipe) = isempty(b.cache) && eof(b.io) -close(b::Base64DecodePipe) = nothing - -# Decodes a base64-encoded string - -""" - base64decode(string) - -Decodes the base64-encoded `string` and returns a `Vector{UInt8}` of the decoded bytes. - -See also [`base64encode`](@ref) - -# Examples -```jldoctest -julia> b = base64decode("SGVsbG8h") -6-element Array{UInt8,1}: - 0x48 - 0x65 - 0x6c - 0x6c - 0x6f - 0x21 - -julia> String(b) -"Hello!" -``` -""" -function base64decode(s) - b = IOBuffer(s) - try - return read(Base64DecodePipe(b)) - finally - close(b) - end -end - -end # module diff --git a/base/deprecated.jl b/base/deprecated.jl index 50d73124cd142..87ce94dad9586 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -1356,6 +1356,11 @@ export conv, conv2, deconv, filt, filt!, xcorr @deprecate_binding Profile nothing true ", run `using Profile` instead" @eval @deprecate_moved $(Symbol("@profile")) "Profile" true true +@deprecate_moved base64encode "Base64" true true +@deprecate_moved base64decode "Base64" true true +@deprecate_moved Base64EncodePipe "Base64" true true +@deprecate_moved Base64DecodePipe "Base64" true true + # PR #21709 @deprecate cov(x::AbstractVector, corrected::Bool) cov(x, corrected=corrected) @deprecate cov(x::AbstractMatrix, vardim::Int, corrected::Bool) cov(x, vardim, corrected=corrected) diff --git a/base/exports.jl b/base/exports.jl index 973aaf81840eb..1bf92e86fb631 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -715,10 +715,6 @@ export # strings and text output ascii, base, - base64encode, - base64decode, - Base64EncodePipe, - Base64DecodePipe, startswith, bin, bitstring, diff --git a/base/multimedia.jl b/base/multimedia.jl index 19aa7e438c2d7..a6ec0651fff33 100644 --- a/base/multimedia.jl +++ b/base/multimedia.jl @@ -122,8 +122,8 @@ that binary data is base64-encoded as an ASCII string. """ stringmime(m::MIME, x) = istextmime(m) ? reprmime(m, x) : _binstringmime(m, x) -_binstringmime(m::MIME, x) = base64encode(verbose_show, m, x) -_binstringmime(m::MIME, x::Vector{UInt8}) = base64encode(write, x) +_binstringmime(m::MIME, x) = Base64.base64encode(verbose_show, m, x) +_binstringmime(m::MIME, x::Vector{UInt8}) = Base64.base64encode(write, x) """ istextmime(m::MIME) diff --git a/base/stream.jl b/base/stream.jl index 7eb93ad07fdfe..15d050dc0ee60 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -16,8 +16,6 @@ abstract type LibuvStream <: IO end # . +- Pipe # . +- Process (not exported) # . +- ProcessChain (not exported) -# +- Base64DecodePipe -# +- Base64EncodePipe # +- BufferStream # +- DevNullStream (not exported) # +- Filesystem.File diff --git a/base/sysimg.jl b/base/sysimg.jl index c4c96f88f2435..a0ec603619702 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -260,10 +260,6 @@ end end end -# base64 conversions (need broadcast) -include("base64.jl") -using .Base64 - # version include("version.jl") @@ -289,8 +285,6 @@ include("socket.jl") include("filesystem.jl") using .Filesystem include("process.jl") -include("multimedia.jl") -using .Multimedia include("grisu/grisu.jl") import .Grisu.print_shortest include("methodshow.jl") @@ -378,18 +372,13 @@ include("replutil.jl") include("i18n.jl") using .I18n -# frontend -include("initdefs.jl") -include("repl/Terminals.jl") -include("repl/LineEdit.jl") -include("repl/REPLCompletions.jl") -include("repl/REPL.jl") -include("client.jl") - # Stack frames and traces include("stacktraces.jl") using .StackTraces +include("initdefs.jl") +include("client.jl") + # misc useful functions & macros include("util.jl") @@ -421,11 +410,29 @@ include("asyncmap.jl") include("distributed/Distributed.jl") using .Distributed +# worker threads +include("threadcall.jl") + # code loading include("loading.jl") -# worker threads -include("threadcall.jl") +# set up load path to be able to find stdlib packages +init_load_path(ccall(:jl_get_julia_home, Any, ())) + +INCLUDE_STATE = 3 # include = include_relative + +import Base64 + +INCLUDE_STATE = 2 + +include("multimedia.jl") +using .Multimedia + +# frontend +include("repl/Terminals.jl") +include("repl/LineEdit.jl") +include("repl/REPLCompletions.jl") +include("repl/REPL.jl") # deprecated functions include("deprecated.jl") @@ -449,8 +456,9 @@ function __init__() init_threadcall() end +include("precompile.jl") + INCLUDE_STATE = 3 # include = include_relative -include(Base, "precompile.jl") end # baremodule Base @@ -459,9 +467,6 @@ using Base # Ensure this file is also tracked unshift!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "sysimg.jl"))) -# set up load path to be able to find stdlib packages -Base.init_load_path(ccall(:jl_get_julia_home, Any, ())) - # load some stdlib packages but don't put their names in Main Base.require(:DelimitedFiles) Base.require(:Test) diff --git a/doc/src/stdlib/io-network.md b/doc/src/stdlib/io-network.md index 482ceb3d7015c..7401410a8dbc3 100644 --- a/doc/src/stdlib/io-network.md +++ b/doc/src/stdlib/io-network.md @@ -77,10 +77,6 @@ Base.readline Base.readuntil Base.readlines Base.eachline -Base.Base64.Base64EncodePipe -Base.Base64.Base64DecodePipe -Base.Base64.base64encode -Base.Base64.base64decode Base.displaysize ``` diff --git a/stdlib/Base64/src/Base64.jl b/stdlib/Base64/src/Base64.jl index b065011ae36b0..d6176e6e47e90 100644 --- a/stdlib/Base64/src/Base64.jl +++ b/stdlib/Base64/src/Base64.jl @@ -2,13 +2,11 @@ module Base64 -#= export Base64EncodePipe, base64encode, Base64DecodePipe, base64decode -=# # Base64EncodePipe is a pipe-like IO object, which converts into base64 data # sent to a stream. (You must close the pipe to complete the encode, separate diff --git a/test/base64.jl b/test/base64.jl deleted file mode 100644 index e36fbae6bb491..0000000000000 --- a/test/base64.jl +++ /dev/null @@ -1,46 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -const inputText = "Man is distinguished, not only by his reason, but by this singular passion from other animals, which is a lust of the mind, that by a perseverance of delight in the continued and indefatigable generation of knowledge, exceeds the short vehemence of any carnal pleasure." -const encodedMaxLine76 = -"""TWFuIGlzIGRpc3Rpbmd1aXNoZWQsIG5vdCBvbmx5IGJ5IGhpcyByZWFzb24sIGJ1dCBieSB0aGlz -IHNpbmd1bGFyIHBhc3Npb24gZnJvbSBvdGhlciBhbmltYWxzLCB3aGljaCBpcyBhIGx1c3Qgb2Yg -dGhlIG1pbmQsIHRoYXQgYnkgYSBwZXJzZXZlcmFuY2Ugb2YgZGVsaWdodCBpbiB0aGUgY29udGlu -dWVkIGFuZCBpbmRlZmF0aWdhYmxlIGdlbmVyYXRpb24gb2Yga25vd2xlZGdlLCBleGNlZWRzIHRo -ZSBzaG9ydCB2ZWhlbWVuY2Ugb2YgYW55IGNhcm5hbCBwbGVhc3VyZS4=""" - -# Encode and decode -fname = tempname() -open(fname, "w") do f - opipe = Base64EncodePipe(f) - write(opipe,inputText) - @test close(opipe) === nothing -end - -open(fname, "r") do f - ipipe = Base64DecodePipe(f) - @test read(ipipe, String) == inputText - @test close(ipipe) === nothing -end -rm(fname) - -# Encode to string and decode -@test String(base64decode(base64encode(inputText))) == inputText - -# Decode with max line chars = 76 and padding -ipipe = Base64DecodePipe(IOBuffer(encodedMaxLine76)) -@test read(ipipe, String) == inputText - -# Decode with max line chars = 76 and no padding -ipipe = Base64DecodePipe(IOBuffer(encodedMaxLine76[1:end-1])) -@test read(ipipe, String) == inputText - -# Decode with two padding characters ("==") -ipipe = Base64DecodePipe(IOBuffer(string(encodedMaxLine76[1:end-2],"=="))) -@test read(ipipe, String) == inputText[1:end-1] - -# Test incorrect format -ipipe = Base64DecodePipe(IOBuffer(encodedMaxLine76[1:end-3])) -@test_throws ArgumentError read(ipipe, String) - -# issue #21314 -@test base64decode(chomp("test")) == base64decode("test") diff --git a/test/choosetests.jl b/test/choosetests.jl index f4d8f478766ce..d3aa1a7b8aa4c 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -44,7 +44,7 @@ function choosetests(choices = []) "euler", "show", "lineedit", "replcompletions", "repl", "replutil", "sets", "goto", "llvmcall", "llvmcall2", "grisu", "nullable", "meta", "stacktraces", "libgit2", "docs", - "markdown", "base64", "serialize", "misc", "threads", + "markdown", "serialize", "misc", "threads", "enums", "cmdlineargs", "i18n", "workspace", "libdl", "int", "checked", "bitset", "floatfuncs", "compile", "distributed", "inline", "boundscheck", "error", "ambiguous", "cartesian", "asmvariant", "osutils", diff --git a/test/compile.jl b/test/compile.jl index eae916a096284..8a736f1cf7cd3 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -218,7 +218,7 @@ try [:Base, :Core, Foo2_module, FooBase_module, :Main, :Test]), # plus modules included in the system image Dict(s => Base.module_uuid(Base.root_module(s)) for s in - [:DelimitedFiles,:Mmap])) + [:DelimitedFiles,:Mmap,:Base64])) @test discard_module.(deps) == deps1 @test current_task()(0x01, 0x4000, 0x30031234) == 2