From d42df6333eb0e5b0ef9a2ea89259f14272d42d3a Mon Sep 17 00:00:00 2001 From: Andrew Burrows Date: Mon, 17 Mar 2014 22:39:13 +0000 Subject: [PATCH 01/47] Add the ability to install test dependencies and run tests to Pkg `Pkg.test("PackageName")` will install a package's test dependencies (marked with `@test` in the REQUIRES file) then run its tests. If the tests fail or there is no run_tests.jl provided then an error is raised. --- base/pkg.jl | 6 +++- base/pkg/entry.jl | 42 +++++++++++++++++++++++++++ base/pkg/reqs.jl | 16 ++++++++--- test/pkg.jl | 72 +++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 123 insertions(+), 13 deletions(-) diff --git a/base/pkg.jl b/base/pkg.jl index 80334f7dda710..491abe1d3f18d 100644 --- a/base/pkg.jl +++ b/base/pkg.jl @@ -2,7 +2,7 @@ module Pkg export Git, Dir, GitHub, Types, Reqs, Cache, Read, Query, Resolve, Write, Generate, Entry export dir, init, rm, add, available, installed, status, clone, checkout, - release, fix, update, resolve, register, tag, publish, generate + release, fix, update, resolve, register, tag, publish, generate, test const DEFAULT_META = "git://github.com/JuliaLang/METADATA.jl" const META_BRANCH = "metadata-v2" @@ -62,6 +62,10 @@ build(pkgs::String...) = cd(Entry.build,[pkgs...]) generate(pkg::String, license::String; force::Bool=false) = cd(Generate.package,pkg,license,force=force) + +test() = cd(Entry.test) +test(pkgs::String...) = cd(Entry.test,String[pkgs...]) + @deprecate release free @deprecate fixup build @deprecate fix pin diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 67da7c1d84e93..dece29ee3fcdd 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -663,4 +663,46 @@ function updatehook(pkgs::Vector) """) end +@windows_only const JULIA = joinpath(JULIA_HOME, ENV["JULIA_EXE"]) +@unix_only const JULIA = joinpath(JULIA_HOME, "julia-readline") + +function test!(pkg::String, errs::Vector{String}, notests::Vector{String}) + const tests_require = Reqs.parse("$pkg/REQUIRE",true) + if (!isempty(tests_require)) + info("Computing test dependencies for $pkg...") + resolve(tests_require) + end + const path = abspath(pkg,"run_tests.jl") + if isfile(path) + info("Testing $pkg") + cd(dirname(path)) do + try + run(`$JULIA $path`) + info("$pkg tests passed") + catch err + warnbanner(err, label="[ ERROR: $pkg ]") + push!(errs,pkg) + end + end + else + push!(notests,pkg) + end +end + +function test(pkgs::Vector{String}) + errs = String[] + notests = String[] + for pkg in pkgs + test!(pkg,errs,notests) + end + if !isempty(errs) || !isempty(notests) + messages = String[] + isempty(errs) || push!(messages, "$(join(errs,", "," and ")) had test errors") + isempty(notests) || push!(messages, "$(join(notests,", "," and ")) did not provide a run_test.jl file") + error(join(messages, "and")) + end +end + +test() = test(sort!(String[keys(installed())...])) + end # module diff --git a/base/pkg/reqs.jl b/base/pkg/reqs.jl index bf7fe90202e15..a80db0ef943cd 100644 --- a/base/pkg/reqs.jl +++ b/base/pkg/reqs.jl @@ -13,12 +13,19 @@ immutable Requirement <: Line package::String versions::VersionSet system::Vector{String} + test::Bool function Requirement(content::String) + test = false fields = split(replace(content, r"#.*$", "")) system = String[] while !isempty(fields) && fields[1][1] == '@' - push!(system,shift!(fields)[2:end]) + sys = shift!(fields)[2:end] + if sys=="test" + test = true + else + push!(system, sys) + end end isempty(fields) && error("invalid requires entry: $content") package = shift!(fields) @@ -26,7 +33,7 @@ immutable Requirement <: Line error("invalid requires entry for $package: $content") versions = [ convert(VersionNumber, field) for field in fields ] issorted(versions) || error("invalid requires entry for $package: $content") - new(content, package, VersionSet(versions), system) + new(content, package, VersionSet(versions), system, test) end function Requirement(package::String, versions::VersionSet, system::Vector{String}=String[]) content = "" @@ -72,7 +79,7 @@ function write(io::IO, reqs::Requires) end write(file::String, r::Union(Vector{Line},Requires)) = open(io->write(io,r), file, "w") -function parse(lines::Vector{Line}) +function parse(lines::Vector{Line}, test::Bool=false) reqs = Requires() for line in lines if isa(line,Requirement) @@ -88,13 +95,14 @@ function parse(lines::Vector{Line}) @linux_only applies &= !("!linux" in line.system) applies || continue end + (test==line.test) || continue reqs[line.package] = haskey(reqs, line.package) ? intersect(reqs[line.package], line.versions) : line.versions end end return reqs end -parse(x) = parse(read(x)) +parse(x, test::Bool=false) = parse(read(x),test) # add & rm – edit the content a requires file diff --git a/test/pkg.jl b/test/pkg.jl index 428502675a5e1..09b33c754d8d2 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -1,15 +1,71 @@ -ENV["JULIA_PKGDIR"] = string("tmp.",randstring()) -@test !isdir(Pkg.dir()) -try # ensure directory removal - Pkg.init() - @test isdir(Pkg.dir()) - Pkg.resolve() +function temp_pkg_dir(fn::Function) + # Used in tests below to setup and teardown a sandboxed package directory + const tmpdir = ENV["JULIA_PKGDIR"] = string("tmp.",randstring()) + @test !isdir(Pkg.dir()) + try + Pkg.init() + @test isdir(Pkg.dir()) + Pkg.resolve() + fn() + finally + run(`rm -rf $tmpdir`) + end +end + +# Test adding a removing a package +temp_pkg_dir() do @test isempty(Pkg.installed()) Pkg.add("Example") @test [keys(Pkg.installed())...] == ["Example"] Pkg.rm("Example") @test isempty(Pkg.installed()) -finally - run(`rm -rf $(Pkg.dir())`) end + +# testing a package with @test dependencies causes them to be installed +temp_pkg_dir() do + Pkg.generate("PackageWithTestDependencies", "MIT") + @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] + + open(Pkg.dir("PackageWithTestDependencies","REQUIRE"),"a") do f + println(f,"@test Example") + end + + open(Pkg.dir("PackageWithTestDependencies","run_tests.jl"),"w") do f + println(f,"") + end + + Pkg.resolve() + @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] + + Pkg.test("PackageWithTestDependencies") + @test sort([keys(Pkg.installed())...]) == sort(["PackageWithTestDependencies", "Example"]) +end + +# testing a package with no run_test.jl errors +temp_pkg_dir() do + Pkg.generate("PackageWithNoTests", "MIT") + + try + Pkg.test("PackageWithNoTests") + catch err + @test err.msg == "PackageWithNoTests did not provide a run_test.jl file" + end +end + +# testing a package with failing tests errors +temp_pkg_dir() do + Pkg.generate("PackageWithFailingTests", "MIT") + + open(Pkg.dir("PackageWithFailingTests","run_tests.jl"),"w") do f + println(f,"using Base.Test") + println(f,"@test false") + end + + try + Pkg.test("PackageWithFailingTests") + catch err + @test err.msg == "PackageWithFailingTests had test errors" + end +end + From e06ad49004d4ed8f821807f31d9f02d81b360515 Mon Sep 17 00:00:00 2001 From: Andrew Burrows Date: Tue, 18 Mar 2014 13:06:13 +0000 Subject: [PATCH 02/47] make test a named argument in Reqs.parse --- base/pkg/entry.jl | 2 +- base/pkg/reqs.jl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index dece29ee3fcdd..8e81ba2022ceb 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -667,7 +667,7 @@ end @unix_only const JULIA = joinpath(JULIA_HOME, "julia-readline") function test!(pkg::String, errs::Vector{String}, notests::Vector{String}) - const tests_require = Reqs.parse("$pkg/REQUIRE",true) + const tests_require = Reqs.parse("$pkg/REQUIRE",test=true) if (!isempty(tests_require)) info("Computing test dependencies for $pkg...") resolve(tests_require) diff --git a/base/pkg/reqs.jl b/base/pkg/reqs.jl index a80db0ef943cd..f496fa5f07496 100644 --- a/base/pkg/reqs.jl +++ b/base/pkg/reqs.jl @@ -79,7 +79,7 @@ function write(io::IO, reqs::Requires) end write(file::String, r::Union(Vector{Line},Requires)) = open(io->write(io,r), file, "w") -function parse(lines::Vector{Line}, test::Bool=false) +function parse(lines::Vector{Line}; test::Bool=false) reqs = Requires() for line in lines if isa(line,Requirement) @@ -102,7 +102,7 @@ function parse(lines::Vector{Line}, test::Bool=false) end return reqs end -parse(x, test::Bool=false) = parse(read(x),test) +parse(x; test::Bool=false) = parse(read(x);test=test) # add & rm – edit the content a requires file From 7ba8a0072fb4ef8040a5a607a383da40e4dc06d0 Mon Sep 17 00:00:00 2001 From: Dahua Lin Date: Tue, 18 Mar 2014 12:11:18 -0500 Subject: [PATCH 03/47] Inplace reduction functions Inplace reduction functions (including sum!, prod!, maximum!, minimum!, all!, and any!) have been within the Base for a while. This PR exports them with a modified interface that is safe & flexible. --- base/exports.jl | 6 ++++++ base/reducedim.jl | 44 +++++++++++++++++++++++++++++++------------- test/reducedim.jl | 16 +++++++++++----- 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index 689c37fc09f78..614e2cd1d1014 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -529,8 +529,10 @@ export logspace, mapslices, max, + maximum!, maximum, min, + minimum!, minimum, minmax, nans, @@ -548,6 +550,7 @@ export permute!, permutedims, permutedims!, + prod!, prod, promote_shape, randcycle, @@ -581,6 +584,7 @@ export strides, sub, sub2ind, + sum!, sum, sum_kbn, vcat, @@ -699,7 +703,9 @@ export unshift!, # collections + all!, all, + any!, any, collect, complement!, diff --git a/base/reducedim.jl b/base/reducedim.jl index 81025bcda3ac0..f63be2d6ba398 100644 --- a/base/reducedim.jl +++ b/base/reducedim.jl @@ -116,20 +116,38 @@ reduction_init{T}(A::AbstractArray, region, initial::T) = fill!(similar(A,T,redu ### Pre-generated cases # For performance, these bypass reducedim_cache -all(A::AbstractArray{Bool}, region) = all!(reduction_init(A,region,true), A) -eval(ngenerate(:N, :(typeof(R)), :(all!{N}(R::AbstractArray, A::AbstractArray{Bool,N})), N->gen_reduction_body(N, &))) -any(A::AbstractArray{Bool}, region) = any!(reduction_init(A,region,false), A) -eval(ngenerate(:N, :(typeof(R)), :(any!{N}(R::AbstractArray, A::AbstractArray{Bool,N})), N->gen_reduction_body(N, |))) +function initarray!{T}(a::AbstractArray{T}, v::T, init::Bool) + if init + fill!(a, v) + end + return a +end + +eval(ngenerate(:N, :(typeof(R)), :(_all!{N}(R::AbstractArray, A::AbstractArray{Bool,N})), N->gen_reduction_body(N, &))) +all!(r::AbstractArray, A::AbstractArray{Bool}; init::Bool=true) = _all!(initarray!(r, true, init), A) +all(A::AbstractArray{Bool}, region) = _all!(reduction_init(A, region, true), A) + +eval(ngenerate(:N, :(typeof(R)), :(_any!{N}(R::AbstractArray, A::AbstractArray{Bool,N})), N->gen_reduction_body(N, |))) +any!(r::AbstractArray, A::AbstractArray{Bool}; init::Bool=true) = _any!(initarray!(r, false, init), A) +any(A::AbstractArray{Bool}, region) = any!(reduction_init(A, region, false), A) + +eval(ngenerate(:N, :(typeof(R)), :(_maximum!{T,N}(R::AbstractArray, A::AbstractArray{T,N})), N->gen_reduction_body(N, scalarmax))) +maximum!{R}(r::AbstractArray{R}, A::AbstractArray; init::Bool=true) = _maximum!(initarray!(r, typemin(R), init), A) maximum{T}(A::AbstractArray{T}, region) = - isempty(A) ? similar(A,reduced_dims0(A,region)) : maximum!(reduction_init(A,region,typemin(T)), A) -eval(ngenerate(:N, :(typeof(R)), :(maximum!{T,N}(R::AbstractArray, A::AbstractArray{T,N})), N->gen_reduction_body(N, scalarmax))) + isempty(A) ? similar(A,reduced_dims0(A,region)) : _maximum!(reduction_init(A, region, typemin(T)), A) + +eval(ngenerate(:N, :(typeof(R)), :(_minimum!{T,N}(R::AbstractArray, A::AbstractArray{T,N})), N->gen_reduction_body(N, scalarmin))) +minimum!{R}(r::AbstractArray{R}, A::AbstractArray; init::Bool=true) = _minimum!(initarray!(r, typemax(R), init), A) minimum{T}(A::AbstractArray{T}, region) = - isempty(A) ? similar(A,reduced_dims0(A,region)) : minimum!(reduction_init(A,region,typemax(T)), A) -eval(ngenerate(:N, :(typeof(R)), :(minimum!{T,N}(R::AbstractArray, A::AbstractArray{T,N})), N->gen_reduction_body(N, scalarmin))) -sum{T}(A::AbstractArray{T}, region) = sum!(reduction_init(A,region,zero(T)), A) -sum(A::AbstractArray{Bool}, region) = sum!(reduction_init(A,region,0), A) -eval(ngenerate(:N, :(typeof(R)), :(sum!{T,N}(R::AbstractArray, A::AbstractArray{T,N})), N->gen_reduction_body(N, +))) -prod{T}(A::AbstractArray{T}, region) = prod!(reduction_init(A,region,one(T)), A) -eval(ngenerate(:N, :(typeof(R)), :(prod!{T,N}(R::AbstractArray, A::AbstractArray{T,N})), N->gen_reduction_body(N, *))) + isempty(A) ? similar(A, reduced_dims0(A, region)) : _minimum!(reduction_init(A, region, typemax(T)), A) + +eval(ngenerate(:N, :(typeof(R)), :(_sum!{T,N}(R::AbstractArray, A::AbstractArray{T,N})), N->gen_reduction_body(N, +))) +sum!{R}(r::AbstractArray{R}, A::AbstractArray; init::Bool=true) = _sum!(initarray!(r, zero(R), init), A) +sum{T}(A::AbstractArray{T}, region) = _sum!(reduction_init(A, region, zero(T)+zero(T)), A) + +eval(ngenerate(:N, :(typeof(R)), :(_prod!{T,N}(R::AbstractArray, A::AbstractArray{T,N})), N->gen_reduction_body(N, *))) +prod!{R}(r::AbstractArray{R}, A::AbstractArray; init::Bool=true) = _prod!(initarray!(r, one(R), init), A) +prod{T}(A::AbstractArray{T}, region) = _prod!(reduction_init(A, region, one(T)*one(T)), A) prod(A::AbstractArray{Bool}, region) = error("use all() instead of prod() for boolean arrays") + diff --git a/test/reducedim.jl b/test/reducedim.jl index 944e68e556b3a..3a1130e171e9f 100644 --- a/test/reducedim.jl +++ b/test/reducedim.jl @@ -12,12 +12,18 @@ safe_minimum{T}(A::Array{T}, region) = safe_mapslices(minimum, A, region) Areduc = rand(3, 4, 5, 6) for region in { 1, 2, 3, 4, 5, (1, 2), (1, 3), (1, 4), (2, 3), (2, 4), (3, 4), - (1, 2, 3), (1, 3, 4), (2, 3, 4), (1, 2, 3, 4)} + (1, 2, 3), (1, 3, 4), (2, 3, 4), (1, 2, 3, 4)} - @test_approx_eq sum(Areduc, region) safe_sum(Areduc, region) - @test_approx_eq prod(Areduc, region) safe_prod(Areduc, region) - @test_approx_eq maximum(Areduc, region) safe_maximum(Areduc, region) - @test_approx_eq minimum(Areduc, region) safe_minimum(Areduc, region) + r = fill(NaN, Base.reduced_dims(size(Areduc), region)) + @test_approx_eq sum!(r, Areduc) safe_sum(Areduc, region) + @test_approx_eq prod!(r, Areduc) safe_prod(Areduc, region) + @test_approx_eq maximum!(r, Areduc) safe_maximum(Areduc, region) + @test_approx_eq minimum!(r, Areduc) safe_minimum(Areduc, region) + + @test_approx_eq sum(Areduc, region) safe_sum(Areduc, region) + @test_approx_eq prod(Areduc, region) safe_prod(Areduc, region) + @test_approx_eq maximum(Areduc, region) safe_maximum(Areduc, region) + @test_approx_eq minimum(Areduc, region) safe_minimum(Areduc, region) end @test reducedim((a,b) -> a|b, [true false; false false], 1, false) == [true false] From a6c0f618da39c9d9cbcaf3411ca9124eed16c107 Mon Sep 17 00:00:00 2001 From: Dahua Lin Date: Tue, 18 Mar 2014 14:38:57 -0500 Subject: [PATCH 04/47] add documentation for sum! and friends --- doc/stdlib/base.rst | 48 +++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 727cf551ddd6d..1c279365d3d65 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -554,19 +554,27 @@ Iterable Collections .. function:: maximum(itr) - Returns the largest element in a collection + Returns the largest element in a collection. .. function:: maximum(A, dims) - Compute the maximum value of an array over the given dimensions + Compute the maximum value of an array over the given dimensions. + +.. function:: maximum!(r, A) + + Compute the maximum value of ``A`` over the singleton dimensions of ``r``, and write results to ``r``. .. function:: minimum(itr) - Returns the smallest element in a collection + Returns the smallest element in a collection. .. function:: minimum(A, dims) - Compute the minimum value of an array over the given dimensions + Compute the minimum value of an array over the given dimensions. + +.. function:: minimum!(r, A) + + Compute the minimum value of ``A`` over the singleton dimensions of ``r``, and write results to ``r``. .. function:: extrema(itr) @@ -575,56 +583,72 @@ Iterable Collections .. function:: indmax(itr) -> Integer - Returns the index of the maximum element in a collection + Returns the index of the maximum element in a collection. .. function:: indmin(itr) -> Integer - Returns the index of the minimum element in a collection + Returns the index of the minimum element in a collection. .. function:: findmax(itr) -> (x, index) - Returns the maximum element and its index + Returns the maximum element and its index. .. function:: findmin(itr) -> (x, index) - Returns the minimum element and its index + Returns the minimum element and its index. .. function:: sum(itr) - Returns the sum of all elements in a collection + Returns the sum of all elements in a collection. .. function:: sum(A, dims) Sum elements of an array over the given dimensions. +.. function:: sum!(r, A) + + Sum elements of ``A`` over the singleton dimensions of ``r``, and write results to ``r``. + .. function:: sum(f, itr) Sum the results of calling function ``f`` on each element of ``itr``. .. function:: prod(itr) - Returns the product of all elements of a collection + Returns the product of all elements of a collection. .. function:: prod(A, dims) Multiply elements of an array over the given dimensions. +.. function:: prod!(r, A) + + Multiply elements of ``A`` over the singleton dimensions of ``r``, and write results to ``r``. + .. function:: any(itr) -> Bool - Test whether any elements of a boolean collection are true + Test whether any elements of a boolean collection are true. .. function:: any(A, dims) Test whether any values along the given dimensions of an array are true. +.. function:: any!(r, A) + + Test whether any values in ``A`` along the singleton dimensions of ``r`` are true, and write results to ``r``. + .. function:: all(itr) -> Bool - Test whether all elements of a boolean collection are true + Test whether all elements of a boolean collection are true. .. function:: all(A, dims) Test whether all values along the given dimensions of an array are true. +.. function:: all!(r, A) + + Test whether all values in ``A`` along the singleton dimensions of ``r`` are true, and write results to ``r``. + .. function:: count(p, itr) -> Integer Count the number of elements in ``itr`` for which predicate ``p`` returns true. From b2e25f97efb3a6293c0025faed9b7c70a596cb24 Mon Sep 17 00:00:00 2001 From: Dahua Lin Date: Tue, 18 Mar 2014 14:41:42 -0500 Subject: [PATCH 05/47] add an entry to NEWS.md about the reduction functions that allow pre-allocated output --- NEWS.md | 3 +++ doc/stdlib/base.rst | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/NEWS.md b/NEWS.md index b14c2616b70eb..339e8cab60040 100644 --- a/NEWS.md +++ b/NEWS.md @@ -256,6 +256,9 @@ Deprecated or removed * `read` methods that modify a passed array are now called `read!` ([#5970]) + * Reduction functions that accept a pre-allocated output array, including + `sum!`, `prod!`, `maximum!`, `minimum!`, `all!`, `any!` ([#6197]) + [#4042]: https://github.com/JuliaLang/julia/issues/4042 [#5164]: https://github.com/JuliaLang/julia/issues/5164 [#4026]: https://github.com/JuliaLang/julia/issues/4026 diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 1c279365d3d65..d54b839843d83 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -562,7 +562,8 @@ Iterable Collections .. function:: maximum!(r, A) - Compute the maximum value of ``A`` over the singleton dimensions of ``r``, and write results to ``r``. + Compute the maximum value of ``A`` over the singleton dimensions of ``r``, + and write results to ``r``. .. function:: minimum(itr) @@ -574,7 +575,8 @@ Iterable Collections .. function:: minimum!(r, A) - Compute the minimum value of ``A`` over the singleton dimensions of ``r``, and write results to ``r``. + Compute the minimum value of ``A`` over the singleton dimensions of ``r``, + and write results to ``r``. .. function:: extrema(itr) @@ -607,7 +609,8 @@ Iterable Collections .. function:: sum!(r, A) - Sum elements of ``A`` over the singleton dimensions of ``r``, and write results to ``r``. + Sum elements of ``A`` over the singleton dimensions of ``r``, + and write results to ``r``. .. function:: sum(f, itr) @@ -623,7 +626,8 @@ Iterable Collections .. function:: prod!(r, A) - Multiply elements of ``A`` over the singleton dimensions of ``r``, and write results to ``r``. + Multiply elements of ``A`` over the singleton dimensions of ``r``, + and write results to ``r``. .. function:: any(itr) -> Bool @@ -635,7 +639,8 @@ Iterable Collections .. function:: any!(r, A) - Test whether any values in ``A`` along the singleton dimensions of ``r`` are true, and write results to ``r``. + Test whether any values in ``A`` along the singleton dimensions of ``r`` are true, + and write results to ``r``. .. function:: all(itr) -> Bool @@ -647,7 +652,8 @@ Iterable Collections .. function:: all!(r, A) - Test whether all values in ``A`` along the singleton dimensions of ``r`` are true, and write results to ``r``. + Test whether all values in ``A`` along the singleton dimensions of ``r`` are true, + and write results to ``r``. .. function:: count(p, itr) -> Integer From 548854c04eceb754034d63afdb2c249eef1b4c30 Mon Sep 17 00:00:00 2001 From: Dahua Lin Date: Tue, 18 Mar 2014 14:43:26 -0500 Subject: [PATCH 06/47] add the link to 6197 in news --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 339e8cab60040..1b2c90759705b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -332,6 +332,7 @@ Deprecated or removed [#5778]: https://github.com/JuliaLang/julia/issues/5778 [#6169]: https://github.com/JuliaLang/julia/issues/6169 [#5970]: https://github.com/JuliaLang/julia/issues/5970 +[#6197]: https://github.com/JuliaLang/julia/pull/6197 Julia v0.2.0 Release Notes ========================== From ad12c49528c3c9de39c60372f83d805b357d2356 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Tue, 18 Mar 2014 15:11:01 -0400 Subject: [PATCH 07/47] Fix macro name for @sprintf errors Also fix triggering condition for "format must be a plain static string" error --- base/printf.jl | 53 +++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/base/printf.jl b/base/printf.jl index cfe1f4bd13925..a5ed4db32a574 100644 --- a/base/printf.jl +++ b/base/printf.jl @@ -5,7 +5,7 @@ export @printf, @sprintf ### printf formatter generation ### function gen(s::String) - args = {:(out::IO)} + args = {} blk = Expr(:block, :(local neg, pt, len, exp)) for x in parse(s) if isa(x,String) @@ -748,43 +748,42 @@ end ### external printf interface ### is_str_expr(ex) = - isa(ex,Expr) && ex.head==:macrocall && isa(ex.args[1],Symbol) && - (ex.args[1] == :str || endswith(string(ex.args[1]),"_str")) + isa(ex,Expr) && (ex.head == :string || (ex.head == :macrocall && isa(ex.args[1],Symbol) && + endswith(string(ex.args[1]),"str"))) -macro printf(args...) - if length(args) == 0 - error("@printf: called with zero arguments") - end - if !isa(args[1],String) && !(length(args) > 1 && isa(args[2],String)) - if is_str_expr(args[1]) || length(args) > 1 && is_str_expr(args[2]) - error("format must be a plain static string (no interpolation or prefix)") +function _printf(macroname, io, args) + if isempty(args) || !isa(args[1], String) + if !isempty(args) && is_str_expr(args[1]) + error("$macroname: format must be a plain static string (no interpolation or prefix)") + else + error("$macroname: first or second argument must be a format string") end - error("first or second argument must be a format string") - end - local io, fmt - if isa(args[1],String) - io = :(Base.STDOUT) - fmt = args[1] - args = args[2:end] - else - io = args[1] - fmt = args[2] - args = args[3:end] end - args = {io,args...} + fmt = args[1] sym_args, blk = gen(fmt) - if length(sym_args) != length(args) - error("@printf: wrong number of arguments") + if length(sym_args) != length(args)-1 + error("$macroname: wrong number of arguments") end - for i = length(args):-1:1 - var = sym_args[i].args[1] + for i = length(args):-1:2 + var = sym_args[i-1].args[1] unshift!(blk.args, :($var = $(esc(args[i])))) end + unshift!(blk.args, :(out = $io)) blk end +macro printf(args...) + !isempty(args) || error("@printf: called with zero arguments") + if isa(args[1], String) || is_str_expr(args[1]) + _printf("@printf", :STDOUT, args) + else + _printf("@printf", esc(args[1]), args[2:end]) + end +end + macro sprintf(args...) - :(sprint(io->@printf(io,$(map(esc,args)...)))) + !isempty(args) || error("@sprintf: called with zero arguments") + :(sprint(io->$(_printf("@sprintf", :io, args)))) end end # module From b1b584447af841aaae6e10d33b7d8fc9c8a6f446 Mon Sep 17 00:00:00 2001 From: Andrew Burrows Date: Tue, 18 Mar 2014 21:31:47 +0000 Subject: [PATCH 08/47] use test/REQUIRE and test/runtests.jl as discussed in pull request --- base/pkg/entry.jl | 21 ++++++++++++--------- base/pkg/reqs.jl | 16 ++++------------ test/pkg.jl | 28 ++++++++++++++++------------ 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 8e81ba2022ceb..21ad007d63973 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -667,17 +667,20 @@ end @unix_only const JULIA = joinpath(JULIA_HOME, "julia-readline") function test!(pkg::String, errs::Vector{String}, notests::Vector{String}) - const tests_require = Reqs.parse("$pkg/REQUIRE",test=true) - if (!isempty(tests_require)) - info("Computing test dependencies for $pkg...") - resolve(tests_require) + const reqs_path = abspath(pkg,"test","REQUIRE") + if isfile(reqs_path) + const tests_require = Reqs.parse(reqs_path) + if (!isempty(tests_require)) + info("Computing test dependencies for $pkg...") + resolve(tests_require) + end end - const path = abspath(pkg,"run_tests.jl") - if isfile(path) + const test_path = abspath(pkg,"test","runtests.jl") + if isfile(test_path) info("Testing $pkg") - cd(dirname(path)) do + cd(dirname(test_path)) do try - run(`$JULIA $path`) + run(`$JULIA $test_path`) info("$pkg tests passed") catch err warnbanner(err, label="[ ERROR: $pkg ]") @@ -698,7 +701,7 @@ function test(pkgs::Vector{String}) if !isempty(errs) || !isempty(notests) messages = String[] isempty(errs) || push!(messages, "$(join(errs,", "," and ")) had test errors") - isempty(notests) || push!(messages, "$(join(notests,", "," and ")) did not provide a run_test.jl file") + isempty(notests) || push!(messages, "$(join(notests,", "," and ")) did not provide a test/runtests.jl file") error(join(messages, "and")) end end diff --git a/base/pkg/reqs.jl b/base/pkg/reqs.jl index f496fa5f07496..bf7fe90202e15 100644 --- a/base/pkg/reqs.jl +++ b/base/pkg/reqs.jl @@ -13,19 +13,12 @@ immutable Requirement <: Line package::String versions::VersionSet system::Vector{String} - test::Bool function Requirement(content::String) - test = false fields = split(replace(content, r"#.*$", "")) system = String[] while !isempty(fields) && fields[1][1] == '@' - sys = shift!(fields)[2:end] - if sys=="test" - test = true - else - push!(system, sys) - end + push!(system,shift!(fields)[2:end]) end isempty(fields) && error("invalid requires entry: $content") package = shift!(fields) @@ -33,7 +26,7 @@ immutable Requirement <: Line error("invalid requires entry for $package: $content") versions = [ convert(VersionNumber, field) for field in fields ] issorted(versions) || error("invalid requires entry for $package: $content") - new(content, package, VersionSet(versions), system, test) + new(content, package, VersionSet(versions), system) end function Requirement(package::String, versions::VersionSet, system::Vector{String}=String[]) content = "" @@ -79,7 +72,7 @@ function write(io::IO, reqs::Requires) end write(file::String, r::Union(Vector{Line},Requires)) = open(io->write(io,r), file, "w") -function parse(lines::Vector{Line}; test::Bool=false) +function parse(lines::Vector{Line}) reqs = Requires() for line in lines if isa(line,Requirement) @@ -95,14 +88,13 @@ function parse(lines::Vector{Line}; test::Bool=false) @linux_only applies &= !("!linux" in line.system) applies || continue end - (test==line.test) || continue reqs[line.package] = haskey(reqs, line.package) ? intersect(reqs[line.package], line.versions) : line.versions end end return reqs end -parse(x; test::Bool=false) = parse(read(x);test=test) +parse(x) = parse(read(x)) # add & rm – edit the content a requires file diff --git a/test/pkg.jl b/test/pkg.jl index 09b33c754d8d2..c211d6a3b662c 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -1,3 +1,5 @@ +using Base.Test + function temp_pkg_dir(fn::Function) # Used in tests below to setup and teardown a sandboxed package directory const tmpdir = ENV["JULIA_PKGDIR"] = string("tmp.",randstring()) @@ -26,18 +28,19 @@ end temp_pkg_dir() do Pkg.generate("PackageWithTestDependencies", "MIT") @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] - - open(Pkg.dir("PackageWithTestDependencies","REQUIRE"),"a") do f - println(f,"@test Example") + + mkdir(Pkg.dir("PackageWithTestDependencies","test")) + open(Pkg.dir("PackageWithTestDependencies","test","REQUIRE"),"a") do f + println(f,"Example") end - - open(Pkg.dir("PackageWithTestDependencies","run_tests.jl"),"w") do f + + open(Pkg.dir("PackageWithTestDependencies","test","runtests.jl"),"w") do f println(f,"") end - - Pkg.resolve() + + Pkg.resolve() @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] - + Pkg.test("PackageWithTestDependencies") @test sort([keys(Pkg.installed())...]) == sort(["PackageWithTestDependencies", "Example"]) end @@ -45,19 +48,20 @@ end # testing a package with no run_test.jl errors temp_pkg_dir() do Pkg.generate("PackageWithNoTests", "MIT") - + try Pkg.test("PackageWithNoTests") catch err - @test err.msg == "PackageWithNoTests did not provide a run_test.jl file" + @test err.msg == "PackageWithNoTests did not provide a test/runtests.jl file" end end # testing a package with failing tests errors temp_pkg_dir() do Pkg.generate("PackageWithFailingTests", "MIT") - - open(Pkg.dir("PackageWithFailingTests","run_tests.jl"),"w") do f + + mkdir(Pkg.dir("PackageWithFailingTests","test")) + open(Pkg.dir("PackageWithFailingTests", "test", "runtests.jl"),"w") do f println(f,"using Base.Test") println(f,"@test false") end From 5c7e8c2e10e9775a909c101b8bbe3e7bcf6614d4 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 18 Mar 2014 17:41:51 -0400 Subject: [PATCH 09/47] replace unbox warning with an assertion this code is fine and the warning is not needed. --- src/intrinsics.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 6d0729b558b64..8930843f16c7e 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -244,9 +244,7 @@ static Value *emit_unbox(Type *to, Value *x, jl_value_t *jt) // this can happen when a branch yielding a different type ends // up being dead code, and type inference knows that the other // branch's type is the only one that matters. -#ifdef DEBUG - JL_PRINTF(JL_STDERR, "warning: unbox: T != typeof(x)\n"); -#endif + assert(ty == T_void); return UndefValue::get(to); } return x; From c6046e7104a4ae87edc022607f120c7b77968826 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Tue, 18 Mar 2014 18:08:13 -0400 Subject: [PATCH 10/47] remove jl_get_exception_str. fixes #6199 a better name for this function would be jl_cause_segfault(). --- doc/manual/embedding.rst | 2 +- src/julia.h | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/manual/embedding.rst b/doc/manual/embedding.rst index e966301eee41f..65d206d652c30 100644 --- a/doc/manual/embedding.rst +++ b/doc/manual/embedding.rst @@ -195,7 +195,7 @@ Julia code can throw exceptions. For example, consider:: This call will appear to do nothing. However, it is possible to check whether an exception was thrown:: if (jl_exception_occurred()) - printf("%s \n", jl_get_exception_str(jl_exception_occurred())); + printf("%s \n", jl_typeof_str(jl_exception_occurred())); If you are using the Julia C API from a language that supports exceptions (e.g. Python, C#, C++), it makes sense to wrap each call into libjulia with a function that checks whether an exception was thrown, and then rethrows the exception in the host language. diff --git a/src/julia.h b/src/julia.h index bd190e5c5a825..b9dd27d76e578 100644 --- a/src/julia.h +++ b/src/julia.h @@ -833,10 +833,6 @@ DLLEXPORT void jl_undefined_var_error(jl_sym_t *var); void jl_check_type_tuple(jl_tuple_t *t, jl_sym_t *name, const char *ctx); DLLEXPORT jl_value_t *jl_exception_occurred(void); DLLEXPORT void jl_exception_clear(void); -STATIC_INLINE char *jl_get_exception_str(jl_value_t *exception) -{ - return jl_string_data(jl_fieldref(exception, 0)); -} #define JL_NARGS(fname, min, max) \ if (nargs < min) jl_too_few_args(#fname, min); \ From 30483fecb0bb0da74a3f445722a8bc8e33160674 Mon Sep 17 00:00:00 2001 From: Iain Dunning Date: Tue, 18 Mar 2014 16:12:14 -0700 Subject: [PATCH 11/47] Update package generation for changes in #6191 --- base/pkg/generate.jl | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index aa0fa2dd5494e..3e5f1ec9f8ca9 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -50,6 +50,7 @@ function package( Generate.license(pkg,license,years,authors,force=force) Generate.readme(pkg,user,force=force) Generate.entrypoint(pkg,force=force) + Generate.tests(pkg,force=force) Generate.travis(pkg,force=force) msg = """ @@ -113,6 +114,17 @@ function readme(pkg::String, user::String=""; force::Bool=false) end end +function tests(pkg::String; force::Bool=false) + genfile(pkg,"test/runtests.jl",force) do io + print(io, """ + using $pkg + using Base.Test + + # write your own tests here + @test 1 == 1 + """) + end + function travis(pkg::String; force::Bool=false) genfile(pkg,".travis.yml",force) do io print(io, """ @@ -131,8 +143,7 @@ function travis(pkg::String; force::Bool=false) - sudo apt-get update -qq -y - sudo apt-get install libpcre3-dev julia -y script: - - julia -e 'Pkg.init(); run(`ln -s \$(pwd()) \$(Pkg.dir("$pkg"))`); Pkg.pin("$pkg"); Pkg.resolve()' - - julia -e 'using $pkg; @assert isdefined(:$pkg); @assert typeof($pkg) === Module' + - julia -e 'Pkg.init(); Pkg.clone(pwd()); Pkg.test("$pkg")' """) end end From dad1a2a2c4bc85bbd132f14ffe41c4e3bf5dbb88 Mon Sep 17 00:00:00 2001 From: catawbasam Date: Tue, 18 Mar 2014 19:45:11 -0400 Subject: [PATCH 12/47] fix typos --- doc/packages/packagelist.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/packages/packagelist.rst b/doc/packages/packagelist.rst index 325d809e56299..9055058e9597c 100644 --- a/doc/packages/packagelist.rst +++ b/doc/packages/packagelist.rst @@ -3649,7 +3649,7 @@ _________________________________________________________ :alt: Theodore Papamarkou :target: https://github.com/scidom - Juila package for performing forward mode automatic differentiation + Julia package for performing forward mode automatic differentiation Current Version: ``0.0.0`` (updated: 2014-02-06) @@ -5421,7 +5421,7 @@ __________________________________________________________ :alt: The Julia Language :target: https://github.com/JuliaLang - Provides types and helper functions for dealing with the HTTP protocl in Julia + Provides types and helper functions for dealing with the HTTP protocol in Julia Current Version: ``0.0.5`` (updated: 2013-11-19) From 52353f7d9bb7b88a4faeaa65f475858351caef03 Mon Sep 17 00:00:00 2001 From: Iain Dunning Date: Tue, 18 Mar 2014 16:54:35 -0700 Subject: [PATCH 13/47] Typo --- base/pkg/generate.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/pkg/generate.jl b/base/pkg/generate.jl index 3e5f1ec9f8ca9..9a6fb541fde51 100644 --- a/base/pkg/generate.jl +++ b/base/pkg/generate.jl @@ -124,6 +124,7 @@ function tests(pkg::String; force::Bool=false) @test 1 == 1 """) end +end function travis(pkg::String; force::Bool=false) genfile(pkg,".travis.yml",force) do io From 90d4036a79fb1d70f62366cbd542c5425a1a2599 Mon Sep 17 00:00:00 2001 From: Andrew Burrows Date: Tue, 18 Mar 2014 23:56:05 +0000 Subject: [PATCH 14/47] Update docs and clean up test dependencies after running tests It turns out that doing a Pkg.resolve() after a Pkg.test() cleans up the test dependencies. May aswell do this at the end of test() as it is going to happen at somepoint anyway. Also updated the stdlib docs to cover Pkg.test() --- base/pkg/entry.jl | 1 + doc/stdlib/pkg.rst | 8 ++++++++ test/pkg.jl | 10 ++++++---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 21ad007d63973..ffa1339557f9a 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -698,6 +698,7 @@ function test(pkgs::Vector{String}) for pkg in pkgs test!(pkg,errs,notests) end + resolve() if !isempty(errs) || !isempty(notests) messages = String[] isempty(errs) || push!(messages, "$(join(errs,", "," and ")) had test errors") diff --git a/doc/stdlib/pkg.rst b/doc/stdlib/pkg.rst index 47463cce32565..0a56858559f14 100644 --- a/doc/stdlib/pkg.rst +++ b/doc/stdlib/pkg.rst @@ -131,3 +131,11 @@ to use them, you'll need to prefix each function call with an explicit ``Pkg.``, For each new package version tagged in ``METADATA`` not already published, make sure that the tagged package commits have been pushed to the repo at the registered URL for the package and if they all have, push ``METADATA``. +.. function:: test() + + Run the tests for all installed packages ensuring that each package's test dependencies are installed for the duration of the test. A package is tested by running its ``test/runtests.jl`` file and test dependencies are specified in ``test/REQUIRE``. + +.. function:: test(pkgs...) + + Run the tests for each package in ``pkgs`` ensuring that each package's test dependencies are installed for the duration of the test. A package is tested by running its ``test/runtests.jl`` file and test dependencies are specified in ``test/REQUIRE``. + diff --git a/test/pkg.jl b/test/pkg.jl index c211d6a3b662c..008fcd175e137 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -2,7 +2,7 @@ using Base.Test function temp_pkg_dir(fn::Function) # Used in tests below to setup and teardown a sandboxed package directory - const tmpdir = ENV["JULIA_PKGDIR"] = string("tmp.",randstring()) + const tmpdir = ENV["JULIA_PKGDIR"] = abspath(string("tmp.",randstring())) @test !isdir(Pkg.dir()) try Pkg.init() @@ -24,7 +24,7 @@ temp_pkg_dir() do @test isempty(Pkg.installed()) end -# testing a package with @test dependencies causes them to be installed +# testing a package with test dependencies causes them to be installed for the duration of the test temp_pkg_dir() do Pkg.generate("PackageWithTestDependencies", "MIT") @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] @@ -35,14 +35,16 @@ temp_pkg_dir() do end open(Pkg.dir("PackageWithTestDependencies","test","runtests.jl"),"w") do f - println(f,"") + println(f,"using Base.Test") + println(f,"@test haskey(Pkg.installed(), \"Example\")") end Pkg.resolve() @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] Pkg.test("PackageWithTestDependencies") - @test sort([keys(Pkg.installed())...]) == sort(["PackageWithTestDependencies", "Example"]) + + @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] end # testing a package with no run_test.jl errors From fdb909bfc7ffc2782fdd6d50372f6793ad81be12 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Tue, 18 Mar 2014 20:07:46 -0400 Subject: [PATCH 15/47] fix 'parse("immutable X end")' show bug --- base/show.jl | 3 ++- test/show.jl | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/base/show.jl b/base/show.jl index 7c3997247f437..499d260a26b6d 100644 --- a/base/show.jl +++ b/base/show.jl @@ -445,7 +445,8 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) # type declaration elseif is(head, :type) && nargs==3 - show_block(io, head, args[2], args[3], indent); print(io, "end") + show_block(io, args[1] ? :type : :immutable, args[2], args[3], indent) + print(io, "end") # empty return (i.e. "function f() return end") elseif is(head, :return) && nargs == 1 && is(args[1], nothing) diff --git a/test/show.jl b/test/show.jl index adb2b8cf7c470..9c0d6f22fae7c 100644 --- a/test/show.jl +++ b/test/show.jl @@ -9,6 +9,9 @@ immutable T5589 end @test replstr(T5589(Array(UTF8String,100))) == "T5589(UTF8String[#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef … #undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef,#undef])" +@test replstr(parse("type X end")) == ":(type X\n end)" +@test replstr(parse("immutable X end")) == ":(immutable X\n end)" + # expression printing macro test_repr(x) From 8d4deface21bdb0223d1c7a13ab50458c3958c61 Mon Sep 17 00:00:00 2001 From: Dahua Lin Date: Tue, 18 Mar 2014 19:20:12 -0500 Subject: [PATCH 16/47] add mean!, which allows pre-allocated output --- base/exports.jl | 1 + base/statistics.jl | 17 ++++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index 614e2cd1d1014..b5f946965d305 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -877,6 +877,7 @@ export hist, hist2d, histrange, + mean!, mean, median!, median, diff --git a/base/statistics.jl b/base/statistics.jl index 077331f26585c..b1cd166c44a1a 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -15,18 +15,21 @@ function mean(iterable) end mean(v::AbstractArray) = sum(v) / length(v) -function mean(v::AbstractArray, region) - rs = regionsize(v, region) - dst = reduction_init(v, region, zero((v[1]+v[1])/rs)) - sum!(dst, v) +function mean!{T}(r::AbstractArray{T}, v::AbstractArray) + sum!(r, v; init=true) + rs = convert(T, length(v) / length(r)) if rs != 1 - for i = 1 : length(dst) - @inbounds dst[i] /= rs + for i = 1:length(r) + @inbounds r[i] /= rs end end - return dst + return r end +meantype{T}(::Type{T}) = typeof((zero(T) + zero(T)) / 2) +mean{T}(v::AbstractArray{T}, region) = + mean!(Array(meantype(T), reduced_dims(size(v), region)), v) + function median!{T<:Real}(v::AbstractVector{T}; checknan::Bool=true) isempty(v) && error("median of an empty array is undefined") checknan && any(isnan,v) && error("median of an array with NaNs is undefined") From 530e17ad684e11776b0360ae7930ecdb3db97db1 Mon Sep 17 00:00:00 2001 From: Andrew Burrows Date: Wed, 19 Mar 2014 00:22:47 +0000 Subject: [PATCH 17/47] Make sure tests will work after #6205 Pkg.generate() will start creating a test/runtests.jl so need the tests to cope with that --- test/pkg.jl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index 008fcd175e137..8b02c2538b479 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -29,8 +29,8 @@ temp_pkg_dir() do Pkg.generate("PackageWithTestDependencies", "MIT") @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] - mkdir(Pkg.dir("PackageWithTestDependencies","test")) - open(Pkg.dir("PackageWithTestDependencies","test","REQUIRE"),"a") do f + isdir(Pkg.dir("PackageWithTestDependencies","test")) || mkdir(Pkg.dir("PackageWithTestDependencies","test")) + open(Pkg.dir("PackageWithTestDependencies","test","REQUIRE"),"w") do f println(f,"Example") end @@ -47,10 +47,14 @@ temp_pkg_dir() do @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] end -# testing a package with no run_test.jl errors +# testing a package with no runtests.jl errors temp_pkg_dir() do Pkg.generate("PackageWithNoTests", "MIT") + if isfile(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) + rm(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) + end + try Pkg.test("PackageWithNoTests") catch err @@ -62,7 +66,7 @@ end temp_pkg_dir() do Pkg.generate("PackageWithFailingTests", "MIT") - mkdir(Pkg.dir("PackageWithFailingTests","test")) + isdir(Pkg.dir("PackageWithFailingTests","test")) || mkdir(Pkg.dir("PackageWithFailingTests","test")) open(Pkg.dir("PackageWithFailingTests", "test", "runtests.jl"),"w") do f println(f,"using Base.Test") println(f,"@test false") From 7efc8fc5bfca39bf988578da269e0be3a21127f4 Mon Sep 17 00:00:00 2001 From: Dahua Lin Date: Tue, 18 Mar 2014 19:23:04 -0500 Subject: [PATCH 18/47] add documentation for mean! --- doc/stdlib/base.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index d54b839843d83..b8a5bc9c4ee27 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -4016,6 +4016,10 @@ Statistics For applications requiring the handling of missing data, the ``DataArray`` package is recommended. +.. function:: mean!(r, v) + + Compute the mean of ``v`` over the singleton dimensions of ``r``, and write results to ``r``. + .. function:: std(v[, region]) Compute the sample standard deviation of a vector or array ``v``, optionally along dimensions in ``region``. The algorithm returns an estimator of the generative distribution's standard deviation under the assumption that each entry of ``v`` is an IID drawn from that generative distribution. This computation is equivalent to calculating ``sqrt(sum((v - mean(v)).^2) / (length(v) - 1))``. From 667263eb3bd39d7ab6c98e7a6f396c982a6f4af7 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Tue, 18 Mar 2014 21:26:00 -0400 Subject: [PATCH 19/47] improve printing of address op exprs (e.g., :(&x)) --- base/show.jl | 4 ++++ test/show.jl | 2 ++ 2 files changed, 6 insertions(+) diff --git a/base/show.jl b/base/show.jl index 499d260a26b6d..da060e3c22fff 100644 --- a/base/show.jl +++ b/base/show.jl @@ -515,6 +515,10 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) end print(io, '"', a..., '"') + elseif is(head, :&) && length(args) == 1 + print(io, '&') + show_unquoted(io, args[1]) + # print anything else as "Expr(head, args...)" else print(io, "\$(Expr(") diff --git a/test/show.jl b/test/show.jl index 9c0d6f22fae7c..67a76b21b102a 100644 --- a/test/show.jl +++ b/test/show.jl @@ -11,6 +11,8 @@ end @test replstr(parse("type X end")) == ":(type X\n end)" @test replstr(parse("immutable X end")) == ":(immutable X\n end)" +s = "ccall(:f,Int,(Ptr{Void},),&x)" +@test replstr(parse(s)) == ":($s)" # expression printing From da8e71cb92f0534ab28e7983cf72b241cc5fbfd0 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 18 Mar 2014 21:41:21 -0400 Subject: [PATCH 20/47] make sjlj compatible with msvc compiler --- deps/openlibm | 2 +- src/support/_longjmp.win32.S | 20 ++++++++-------- src/support/_longjmp.win64.S | 40 ++++++++++++++++---------------- src/support/_setjmp.win32.S | 16 ++++++------- src/support/_setjmp.win64.S | 44 ++++++++++++++++++------------------ 5 files changed, 61 insertions(+), 61 deletions(-) diff --git a/deps/openlibm b/deps/openlibm index e9c0ba7ad6d19..995c70eaf70cd 160000 --- a/deps/openlibm +++ b/deps/openlibm @@ -1 +1 @@ -Subproject commit e9c0ba7ad6d19fd0e2faf065f428512bf343869d +Subproject commit 995c70eaf70cd1a0e9cf47cee0c57ff5e9fff525 diff --git a/src/support/_longjmp.win32.S b/src/support/_longjmp.win32.S index 148a43f62d085..9382812692ff4 100644 --- a/src/support/_longjmp.win32.S +++ b/src/support/_longjmp.win32.S @@ -53,17 +53,17 @@ .intel_syntax ENTRY(jl_longjmp) - mov edx,DWORD PTR [esp+0x04] - mov eax,DWORD PTR [esp+0x08] - mov ebp,DWORD PTR [edx+0x00] - mov ebx,DWORD PTR [edx+0x04] - mov edi,DWORD PTR [edx+0x08] - mov esi,DWORD PTR [edx+0x0c] - mov esp,DWORD PTR [edx+0x10] - mov ecx,DWORD PTR [edx+0x14] + mov edx,DWORD PTR [esp+4] + mov eax,DWORD PTR [esp+8] + mov ebp,DWORD PTR [edx+0] + mov ebx,DWORD PTR [edx+4] + mov edi,DWORD PTR [edx+8] + mov esi,DWORD PTR [edx+12] + mov esp,DWORD PTR [edx+16] + mov ecx,DWORD PTR [edx+20] test eax,eax - jne 1f + jne a inc eax -1: mov DWORD PTR [esp],ecx +a: mov DWORD PTR [esp],ecx ret END(jl_longjmp) diff --git a/src/support/_longjmp.win64.S b/src/support/_longjmp.win64.S index e09cdb30241af..e43ba99595a68 100644 --- a/src/support/_longjmp.win64.S +++ b/src/support/_longjmp.win64.S @@ -2,26 +2,26 @@ .intel_syntax noprefix ENTRY(jl_longjmp) - mov rbx,QWORD PTR [rcx+0x08] - mov rsp,QWORD PTR [rcx+0x10] - mov rbp,QWORD PTR [rcx+0x18] - mov rsi,QWORD PTR [rcx+0x20] - mov rdi,QWORD PTR [rcx+0x28] - mov r12,QWORD PTR [rcx+0x30] - mov r13,QWORD PTR [rcx+0x38] - mov r14,QWORD PTR [rcx+0x40] - mov r15,QWORD PTR [rcx+0x48] - mov r8, QWORD PTR [rcx+0x50] - movaps xmm6,XMMWORD PTR [rcx+0x60] - movaps xmm7,XMMWORD PTR [rcx+0x70] - movaps xmm8,XMMWORD PTR [rcx+0x80] - movaps xmm9,XMMWORD PTR [rcx+0x90] - movaps xmm10,XMMWORD PTR [rcx+0xa0] - movaps xmm11,XMMWORD PTR [rcx+0xb0] - movaps xmm12,XMMWORD PTR [rcx+0xc0] - movaps xmm13,XMMWORD PTR [rcx+0xd0] - movaps xmm14,XMMWORD PTR [rcx+0xe0] - movaps xmm15,XMMWORD PTR [rcx+0xf0] + mov rbx,QWORD PTR [rcx+8] + mov rsp,QWORD PTR [rcx+16] + mov rbp,QWORD PTR [rcx+24] + mov rsi,QWORD PTR [rcx+32] + mov rdi,QWORD PTR [rcx+40] + mov r12,QWORD PTR [rcx+48] + mov r13,QWORD PTR [rcx+56] + mov r14,QWORD PTR [rcx+64] + mov r15,QWORD PTR [rcx+72] + mov r8, QWORD PTR [rcx+80] + movaps xmm6,XMMWORD PTR [rcx+96] + movaps xmm7,XMMWORD PTR [rcx+120] + movaps xmm8,XMMWORD PTR [rcx+128] + movaps xmm9,XMMWORD PTR [rcx+130] + movaps xmm10,XMMWORD PTR [rcx+160] + movaps xmm11,XMMWORD PTR [rcx+176] + movaps xmm12,XMMWORD PTR [rcx+192] + movaps xmm13,XMMWORD PTR [rcx+208] + movaps xmm14,XMMWORD PTR [rcx+224] + movaps xmm15,XMMWORD PTR [rcx+240] mov eax,edx test eax,eax jne 1f diff --git a/src/support/_setjmp.win32.S b/src/support/_setjmp.win32.S index 3b0b04a96794f..f9cc6cd38cd55 100644 --- a/src/support/_setjmp.win32.S +++ b/src/support/_setjmp.win32.S @@ -53,14 +53,14 @@ .intel_syntax ENTRY(jl_setjmp) - mov eax,DWORD PTR [esp+0x04] - mov edx,DWORD PTR [esp+0x00] - mov DWORD PTR [eax+0x00],ebp /* rta */ - mov DWORD PTR [eax+0x04],ebx - mov DWORD PTR [eax+0x08],edi - mov DWORD PTR [eax+0x0c],esi - mov DWORD PTR [eax+0x10],esp - mov DWORD PTR [eax+0x14],edx + mov eax,DWORD PTR [esp+4] + mov edx,DWORD PTR [esp+0] + mov DWORD PTR [eax+0],ebp /* rta */ + mov DWORD PTR [eax+4],ebx + mov DWORD PTR [eax+8],edi + mov DWORD PTR [eax+12],esi + mov DWORD PTR [eax+16],esp + mov DWORD PTR [eax+20],edx xor eax,eax ret END(jl_setjmp) diff --git a/src/support/_setjmp.win64.S b/src/support/_setjmp.win64.S index 4d572b06342e8..a50c6a8e83257 100644 --- a/src/support/_setjmp.win64.S +++ b/src/support/_setjmp.win64.S @@ -3,28 +3,28 @@ .intel_syntax noprefix ENTRY(jl_setjmp) mov rdx,QWORD PTR [rsp] - mov QWORD PTR [rcx],0x0 - mov QWORD PTR [rcx+0x08],rbx - mov QWORD PTR [rcx+0x10],rsp - mov QWORD PTR [rcx+0x18],rbp - mov QWORD PTR [rcx+0x20],rsi - mov QWORD PTR [rcx+0x28],rdi - mov QWORD PTR [rcx+0x30],r12 - mov QWORD PTR [rcx+0x38],r13 - mov QWORD PTR [rcx+0x40],r14 - mov QWORD PTR [rcx+0x48],r15 - mov QWORD PTR [rcx+0x50],rdx - mov QWORD PTR [rcx+0x58],0x0 - movaps XMMWORD PTR [rcx+0x60],xmm6 - movaps XMMWORD PTR [rcx+0x70],xmm7 - movaps XMMWORD PTR [rcx+0x80],xmm8 - movaps XMMWORD PTR [rcx+0x90],xmm9 - movaps XMMWORD PTR [rcx+0xa0],xmm10 - movaps XMMWORD PTR [rcx+0xb0],xmm11 - movaps XMMWORD PTR [rcx+0xc0],xmm12 - movaps XMMWORD PTR [rcx+0xd0],xmm13 - movaps XMMWORD PTR [rcx+0xe0],xmm14 - movaps XMMWORD PTR [rcx+0xf0],xmm15 + mov QWORD PTR [rcx],0 + mov QWORD PTR [rcx+8],rbx + mov QWORD PTR [rcx+16],rsp + mov QWORD PTR [rcx+24],rbp + mov QWORD PTR [rcx+32],rsi + mov QWORD PTR [rcx+40],rdi + mov QWORD PTR [rcx+48],r12 + mov QWORD PTR [rcx+56],r13 + mov QWORD PTR [rcx+64],r14 + mov QWORD PTR [rcx+72],r15 + mov QWORD PTR [rcx+80],rdx + mov QWORD PTR [rcx+88],0x0 + movaps XMMWORD PTR [rcx+96],xmm6 + movaps XMMWORD PTR [rcx+120],xmm7 + movaps XMMWORD PTR [rcx+128],xmm8 + movaps XMMWORD PTR [rcx+130],xmm9 + movaps XMMWORD PTR [rcx+160],xmm10 + movaps XMMWORD PTR [rcx+176],xmm11 + movaps XMMWORD PTR [rcx+192],xmm12 + movaps XMMWORD PTR [rcx+208],xmm13 + movaps XMMWORD PTR [rcx+224],xmm14 + movaps XMMWORD PTR [rcx+240],xmm15 xor rax,rax ret END(jl_setjmp) From 29288f243af9bde53b501fa14b125d979312ca29 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 18 Mar 2014 22:04:47 -0400 Subject: [PATCH 21/47] work around msvc bug where .model must be on its own line or it forgets about it --- deps/openlibm | 2 +- src/support/_longjmp.win32.S | 4 ++++ src/support/_longjmp.win64.S | 6 ++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/deps/openlibm b/deps/openlibm index 995c70eaf70cd..5f4979e5d1ac9 160000 --- a/deps/openlibm +++ b/deps/openlibm @@ -1 +1 @@ -Subproject commit 995c70eaf70cd1a0e9cf47cee0c57ff5e9fff525 +Subproject commit 5f4979e5d1ac985a5081a5de873ce42537c5a6ce diff --git a/src/support/_longjmp.win32.S b/src/support/_longjmp.win32.S index 9382812692ff4..d0fe08ef72abd 100644 --- a/src/support/_longjmp.win32.S +++ b/src/support/_longjmp.win32.S @@ -51,7 +51,11 @@ * The previous signal state is NOT restored. */ +#ifndef _MSC_VER .intel_syntax +#else +.model small,C +#endif ENTRY(jl_longjmp) mov edx,DWORD PTR [esp+4] mov eax,DWORD PTR [esp+8] diff --git a/src/support/_longjmp.win64.S b/src/support/_longjmp.win64.S index e43ba99595a68..79639e07762ad 100644 --- a/src/support/_longjmp.win64.S +++ b/src/support/_longjmp.win64.S @@ -1,6 +1,8 @@ #include "../../deps/openlibm/amd64/bsd_asm.h" +#ifndef _MSC_VER .intel_syntax noprefix +#endif ENTRY(jl_longjmp) mov rbx,QWORD PTR [rcx+8] mov rsp,QWORD PTR [rcx+16] @@ -24,8 +26,8 @@ ENTRY(jl_longjmp) movaps xmm15,XMMWORD PTR [rcx+240] mov eax,edx test eax,eax - jne 1f + jne a inc eax -1: mov QWORD PTR [rsp],r8 +a: mov QWORD PTR [rsp],r8 ret END(jl_longjmp) From 3171e29c438230fd82eee0113219d6053a7eef26 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 18 Mar 2014 22:08:37 -0400 Subject: [PATCH 22/47] forgot the same update as previous for setjmp --- src/support/_setjmp.win32.S | 4 ++++ src/support/_setjmp.win64.S | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/support/_setjmp.win32.S b/src/support/_setjmp.win32.S index f9cc6cd38cd55..26e7f4e02fe80 100644 --- a/src/support/_setjmp.win32.S +++ b/src/support/_setjmp.win32.S @@ -51,7 +51,11 @@ * The previous signal state is NOT restored. */ +#ifndef _MSC_VER .intel_syntax +#else +.model small,C +#endif ENTRY(jl_setjmp) mov eax,DWORD PTR [esp+4] mov edx,DWORD PTR [esp+0] diff --git a/src/support/_setjmp.win64.S b/src/support/_setjmp.win64.S index a50c6a8e83257..08fda5eec1a01 100644 --- a/src/support/_setjmp.win64.S +++ b/src/support/_setjmp.win64.S @@ -1,6 +1,8 @@ #include "../../deps/openlibm/amd64/bsd_asm.h" +#ifndef _MSC_VER .intel_syntax noprefix +#endif ENTRY(jl_setjmp) mov rdx,QWORD PTR [rsp] mov QWORD PTR [rcx],0 From a9758ba7a82001f3026d13a1553e3a12d23fab1e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 18 Mar 2014 22:10:44 -0400 Subject: [PATCH 23/47] fix a hex digit in win64 setjmp that i missed in previous commit --- src/support/_setjmp.win64.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/support/_setjmp.win64.S b/src/support/_setjmp.win64.S index 08fda5eec1a01..f4b7b5c34d9c6 100644 --- a/src/support/_setjmp.win64.S +++ b/src/support/_setjmp.win64.S @@ -16,7 +16,7 @@ ENTRY(jl_setjmp) mov QWORD PTR [rcx+64],r14 mov QWORD PTR [rcx+72],r15 mov QWORD PTR [rcx+80],rdx - mov QWORD PTR [rcx+88],0x0 + mov QWORD PTR [rcx+88],0 movaps XMMWORD PTR [rcx+96],xmm6 movaps XMMWORD PTR [rcx+120],xmm7 movaps XMMWORD PTR [rcx+128],xmm8 From 7ccd12b4eebe5a31b0116006603958a30138a3d0 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 18 Mar 2014 23:57:00 -0400 Subject: [PATCH 24/47] fix hex conversion typo in win64 sjlj code --- src/support/_longjmp.win64.S | 4 ++-- src/support/_setjmp.win64.S | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/support/_longjmp.win64.S b/src/support/_longjmp.win64.S index 79639e07762ad..bda7f280482e9 100644 --- a/src/support/_longjmp.win64.S +++ b/src/support/_longjmp.win64.S @@ -15,9 +15,9 @@ ENTRY(jl_longjmp) mov r15,QWORD PTR [rcx+72] mov r8, QWORD PTR [rcx+80] movaps xmm6,XMMWORD PTR [rcx+96] - movaps xmm7,XMMWORD PTR [rcx+120] + movaps xmm7,XMMWORD PTR [rcx+112] movaps xmm8,XMMWORD PTR [rcx+128] - movaps xmm9,XMMWORD PTR [rcx+130] + movaps xmm9,XMMWORD PTR [rcx+144] movaps xmm10,XMMWORD PTR [rcx+160] movaps xmm11,XMMWORD PTR [rcx+176] movaps xmm12,XMMWORD PTR [rcx+192] diff --git a/src/support/_setjmp.win64.S b/src/support/_setjmp.win64.S index f4b7b5c34d9c6..5b98b9cc22747 100644 --- a/src/support/_setjmp.win64.S +++ b/src/support/_setjmp.win64.S @@ -18,9 +18,9 @@ ENTRY(jl_setjmp) mov QWORD PTR [rcx+80],rdx mov QWORD PTR [rcx+88],0 movaps XMMWORD PTR [rcx+96],xmm6 - movaps XMMWORD PTR [rcx+120],xmm7 + movaps XMMWORD PTR [rcx+112],xmm7 movaps XMMWORD PTR [rcx+128],xmm8 - movaps XMMWORD PTR [rcx+130],xmm9 + movaps XMMWORD PTR [rcx+144],xmm9 movaps XMMWORD PTR [rcx+160],xmm10 movaps XMMWORD PTR [rcx+176],xmm11 movaps XMMWORD PTR [rcx+192],xmm12 From cbc95cb536cb49ff3f8127ccfdc93d875c4899be Mon Sep 17 00:00:00 2001 From: Kenta Sato Date: Wed, 19 Mar 2014 13:30:21 +0900 Subject: [PATCH 25/47] export singleton boolean values This commit exports two boolean values: * jl_true * jl_false These values are useful for API users. --- src/julia.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/julia.h b/src/julia.h index b9dd27d76e578..f584069f1be19 100644 --- a/src/julia.h +++ b/src/julia.h @@ -386,8 +386,8 @@ extern DLLEXPORT jl_datatype_t *jl_task_type; extern DLLEXPORT jl_tuple_t *jl_null; #define JL_NULL ((void*)jl_null) -extern jl_value_t *jl_true; -extern jl_value_t *jl_false; +DLLEXPORT extern jl_value_t *jl_true; +DLLEXPORT extern jl_value_t *jl_false; DLLEXPORT extern jl_value_t *jl_nothing; DLLEXPORT extern uv_lib_t *jl_dl_handle; From 80cb5ee7903a8fe3fbe2e82f77f4301f647b4b0b Mon Sep 17 00:00:00 2001 From: Andrew Burrows Date: Wed, 19 Mar 2014 07:33:11 +0000 Subject: [PATCH 26/47] replace tabs with spaces --- test/pkg.jl | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index 8b02c2538b479..203ba70b99c59 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -17,17 +17,17 @@ end # Test adding a removing a package temp_pkg_dir() do - @test isempty(Pkg.installed()) - Pkg.add("Example") - @test [keys(Pkg.installed())...] == ["Example"] - Pkg.rm("Example") - @test isempty(Pkg.installed()) + @test isempty(Pkg.installed()) + Pkg.add("Example") + @test [keys(Pkg.installed())...] == ["Example"] + Pkg.rm("Example") + @test isempty(Pkg.installed()) end # testing a package with test dependencies causes them to be installed for the duration of the test temp_pkg_dir() do - Pkg.generate("PackageWithTestDependencies", "MIT") - @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] + Pkg.generate("PackageWithTestDependencies", "MIT") + @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] isdir(Pkg.dir("PackageWithTestDependencies","test")) || mkdir(Pkg.dir("PackageWithTestDependencies","test")) open(Pkg.dir("PackageWithTestDependencies","test","REQUIRE"),"w") do f @@ -49,13 +49,13 @@ end # testing a package with no runtests.jl errors temp_pkg_dir() do - Pkg.generate("PackageWithNoTests", "MIT") + Pkg.generate("PackageWithNoTests", "MIT") if isfile(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) rm(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) end - try + try Pkg.test("PackageWithNoTests") catch err @test err.msg == "PackageWithNoTests did not provide a test/runtests.jl file" @@ -64,15 +64,15 @@ end # testing a package with failing tests errors temp_pkg_dir() do - Pkg.generate("PackageWithFailingTests", "MIT") + Pkg.generate("PackageWithFailingTests", "MIT") isdir(Pkg.dir("PackageWithFailingTests","test")) || mkdir(Pkg.dir("PackageWithFailingTests","test")) open(Pkg.dir("PackageWithFailingTests", "test", "runtests.jl"),"w") do f println(f,"using Base.Test") println(f,"@test false") end - - try + + try Pkg.test("PackageWithFailingTests") catch err @test err.msg == "PackageWithFailingTests had test errors" From 21588c7d52f9f7b3b2b4ffd4db787f3185abde8a Mon Sep 17 00:00:00 2001 From: Andrew Burrows Date: Wed, 19 Mar 2014 08:19:37 +0000 Subject: [PATCH 27/47] remove using Base.Test --- test/pkg.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/pkg.jl b/test/pkg.jl index 203ba70b99c59..4b8316dee0d6d 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -1,5 +1,3 @@ -using Base.Test - function temp_pkg_dir(fn::Function) # Used in tests below to setup and teardown a sandboxed package directory const tmpdir = ENV["JULIA_PKGDIR"] = abspath(string("tmp.",randstring())) From 7e6671d4f16b476faea690ced7f44d84a66a1f05 Mon Sep 17 00:00:00 2001 From: Andrew Burrows Date: Wed, 19 Mar 2014 18:55:58 +0000 Subject: [PATCH 28/47] move clean-up resolve() into test! --- base/pkg/entry.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index ffa1339557f9a..3c930462a6ae8 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -690,6 +690,7 @@ function test!(pkg::String, errs::Vector{String}, notests::Vector{String}) else push!(notests,pkg) end + resolve() end function test(pkgs::Vector{String}) @@ -698,7 +699,6 @@ function test(pkgs::Vector{String}) for pkg in pkgs test!(pkg,errs,notests) end - resolve() if !isempty(errs) || !isempty(notests) messages = String[] isempty(errs) || push!(messages, "$(join(errs,", "," and ")) had test errors") From a95f79c307c31dcbd7cb0764ef9354f8d07be5a3 Mon Sep 17 00:00:00 2001 From: Dahua Lin Date: Wed, 19 Mar 2014 14:30:21 -0500 Subject: [PATCH 29/47] add hist!, which accepts a pre-allocated array to store the output --- base/statistics.jl | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/base/statistics.jl b/base/statistics.jl index b1cd166c44a1a..16bd57c307932 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -147,29 +147,38 @@ function sturges(n) # Sturges' formula iceil(log2(n))+1 end -hist(v::AbstractVector, n::Integer) = hist(v,histrange(v,n)) -hist(v::AbstractVector) = hist(v,sturges(length(v))) - -function hist(v::AbstractVector, edg::AbstractVector) - n = length(edg)-1 - h = zeros(Int, n) +function hist!{HT}(h::StoredArray{HT}, v::AbstractVector, edg::AbstractVector; init::Bool=true) + n = length(edg) - 1 + length(h) == n || error("length(h) must equal length(edg) - 1.") + if init + fill!(h, zero(HT)) + end for x in v i = searchsortedfirst(edg, x)-1 if 1 <= i <= n h[i] += 1 end end - edg,h + edg, h end -function hist(A::AbstractMatrix, edg::AbstractVector) +hist(v::AbstractVector, edg::AbstractVector) = hist!(Array(Int, length(edg)-1), v, edg) +hist(v::AbstractVector, n::Integer) = hist(v,histrange(v,n)) +hist(v::AbstractVector) = hist(v,sturges(length(v))) + +function hist!{HT}(H::StoredArray{HT, 2}, A::AbstractMatrix, edg::AbstractVector; init::Bool=true) m, n = size(A) - H = Array(Int, length(edg)-1, n) + size(H) == (length(edg)-1, n) || error("Incorrect size of H.") + if init + fill!(H, zero(HT)) + end for j = 1:n - _,H[:,j] = hist(sub(A, 1:m, j), edg) + hist!(sub(H(H, :, j), sub(A, :, j), edg)) end - edg,H + edg, H end + +hist(A::AbstractMatrix, edg::AbstractVector) = hist!(Array(Int, length(edg-1), size(A,2)), A, edg) hist(A::AbstractMatrix, n::Integer) = hist(A,histrange(A,n)) hist(A::AbstractMatrix) = hist(A,sturges(size(A,1))) From bc882c5f9a3dedf1e6f780ee065060cea225ee5b Mon Sep 17 00:00:00 2001 From: Dahua Lin Date: Wed, 19 Mar 2014 15:03:56 -0500 Subject: [PATCH 30/47] add hist2d! that accepts a pre-allocated result array --- base/statistics.jl | 42 +++++++++++++++++++++++------------------- test/statistics.jl | 1 + 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/base/statistics.jl b/base/statistics.jl index 16bd57c307932..21ab84ba02414 100644 --- a/base/statistics.jl +++ b/base/statistics.jl @@ -166,7 +166,7 @@ hist(v::AbstractVector, edg::AbstractVector) = hist!(Array(Int, length(edg)-1), hist(v::AbstractVector, n::Integer) = hist(v,histrange(v,n)) hist(v::AbstractVector) = hist(v,sturges(length(v))) -function hist!{HT}(H::StoredArray{HT, 2}, A::AbstractMatrix, edg::AbstractVector; init::Bool=true) +function hist!{HT}(H::StoredArray{HT,2}, A::AbstractMatrix, edg::AbstractVector; init::Bool=true) m, n = size(A) size(H) == (length(edg)-1, n) || error("Incorrect size of H.") if init @@ -182,31 +182,35 @@ hist(A::AbstractMatrix, edg::AbstractVector) = hist!(Array(Int, length(edg-1), s hist(A::AbstractMatrix, n::Integer) = hist(A,histrange(A,n)) hist(A::AbstractMatrix) = hist(A,sturges(size(A,1))) -function hist2d(v::AbstractMatrix, edg1::AbstractVector, edg2::AbstractVector) - if size(v,2) != 2 - error("hist2d requires an Nx2 matrix") + +## hist2d +function hist2d!{HT}(H::StoredArray{HT,2}, v::AbstractMatrix, + edg1::AbstractVector, edg2::AbstractVector; init::Bool=true) + size(v,2) == 2 || error("hist2d requires an Nx2 matrix.") + n = length(edg1) - 1 + m = length(edg2) - 1 + size(H) == (n, m) || error("Incorrect size of H.") + if init + fill!(H, zero(HT)) end - n = length(edg1)-1 - m = length(edg2)-1 - h = zeros(Int, n, m) for i = 1:size(v,1) - x = searchsortedfirst(edg1, v[i, 1])-1 - y = searchsortedfirst(edg2, v[i, 2])-1 + x = searchsortedfirst(edg1, v[i,1]) - 1 + y = searchsortedfirst(edg2, v[i,2]) - 1 if 1 <= x <= n && 1 <= y <= m - h[x,y] += 1 + @inbounds H[x,y] += 1 end end - edg1,edg2,h + edg1, edg2, H end + +hist2d(v::AbstractMatrix, edg1::AbstractVector, edg2::AbstractVector) = + hist2d!(Array(Int, length(edg1)-1, length(edg2)-1), v, edg1, edg2) + hist2d(v::AbstractMatrix, edg::AbstractVector) = hist2d(v, edg, edg) -function hist2d(v::AbstractMatrix, n::Integer) - m = size(v,1) - hist2d(v, histrange(sub(v, 1:m, 1),n), histrange(sub(v, 1:m, 2),n)) -end -function hist2d(v::AbstractMatrix, n1::Integer, n2::Integer) - m = size(v,1) - hist2d(v, histrange(sub(v, 1:m,1),n1), histrange(sub(v, 1:m,2),n2)) -end + +hist2d(v::AbstractMatrix, n1::Integer, n2::Integer) = + hist2d(v, histrange(sub(v,:,1),n1), histrange(sub(v,:,2),n2)) +hist2d(v::AbstractMatrix, n::Integer) = hist2d(v, n, n) hist2d(v::AbstractMatrix) = hist2d(v, sturges(size(v,1))) ## pearson covariance functions ## diff --git a/test/statistics.jl b/test/statistics.jl index 9026fb1db1146..e39e4ba02c466 100644 --- a/test/statistics.jl +++ b/test/statistics.jl @@ -32,6 +32,7 @@ @test hist([1,2,3],0:2:4) == (0:2:4,[2,1]) @test all(hist([1:100]/100,0.0:0.01:1.0)[2] .==1) @test hist([1,1,1,1,1])[2][1] == 5 +@test sum(hist2d(rand(100, 2))[3]) == 100 A = Complex128[exp(i*im) for i in 1:10^4] @test_approx_eq varm(A,0.) sum(map(abs2,A))/(length(A)-1) From 21e1b295533a6832f975b6f04771647915c0f7c8 Mon Sep 17 00:00:00 2001 From: Dahua Lin Date: Wed, 19 Mar 2014 15:07:51 -0500 Subject: [PATCH 31/47] add documentation for hist! and hist2d! --- doc/stdlib/base.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index b8a5bc9c4ee27..50b7d0863e154 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -4071,6 +4071,11 @@ Statistics element at location ``i`` satisfies ``sum(e[i] .< v .<= e[i+1])``. Note: Julia does not ignore ``NaN`` values in the computation. +.. function:: hist!(counts, v, e) -> e, counts + + Compute the histogram of ``v``, using a vector/range ``e`` as the edges for the bins. + This function writes the resultant counts to a pre-allocated array ``counts``. + .. function:: hist2d(M, e1, e2) -> (edge1, edge2, counts) Compute a "2d histogram" of a set of N points specified by N-by-2 matrix ``M``. @@ -4081,6 +4086,12 @@ Statistics ``(length(edge1)-1, length(edge2)-1)``. Note: Julia does not ignore ``NaN`` values in the computation. +.. function:: hist2d!(counts, M, e1, e2) -> (e1, e2, counts) + + Compute a "2d histogram" with respect to the bins delimited by the edges given + in ``e1`` and ``e2``. This function writes the results to a pre-allocated + array ``counts``. + .. function:: histrange(v, n) Compute *nice* bin ranges for the edges of a histogram of ``v``, using From 1639c0691dade01f527267ec1917bd7fad53eaa4 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Wed, 19 Mar 2014 17:13:22 -0400 Subject: [PATCH 32/47] deprecate infs and nans these functions are silly. we could also have fill, ones, and zeros return BitArrys for Bool elements, and then we wouldn't need trues and falses. --- NEWS.md | 2 ++ base/array.jl | 5 +---- base/deprecated.jl | 7 +++++++ base/exports.jl | 2 -- doc/stdlib/base.rst | 8 -------- 5 files changed, 10 insertions(+), 14 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1b2c90759705b..f07af3289ebed 100644 --- a/NEWS.md +++ b/NEWS.md @@ -259,6 +259,8 @@ Deprecated or removed * Reduction functions that accept a pre-allocated output array, including `sum!`, `prod!`, `maximum!`, `minimum!`, `all!`, `any!` ([#6197]) + * `infs` and `nans` are deprecated in favor of the more general `fill`. + [#4042]: https://github.com/JuliaLang/julia/issues/4042 [#5164]: https://github.com/JuliaLang/julia/issues/5164 [#4026]: https://github.com/JuliaLang/julia/issues/4026 diff --git a/base/array.jl b/base/array.jl index 51d3f1183d73f..6a5c0f0763f96 100644 --- a/base/array.jl +++ b/base/array.jl @@ -201,10 +201,7 @@ end fill(v, dims::Dims) = fill!(Array(typeof(v), dims), v) fill(v, dims::Integer...) = fill!(Array(typeof(v), dims...), v) -for (fname, felt) in ((:zeros,:zero), - (:ones,:one), - (:infs,:inf), - (:nans,:nan)) +for (fname, felt) in ((:zeros,:zero), (:ones,:one)) @eval begin ($fname){T}(::Type{T}, dims...) = fill!(Array(T, dims...), ($felt)(T)) ($fname)(dims...) = fill!(Array(Float64, dims...), ($felt)(Float64)) diff --git a/base/deprecated.jl b/base/deprecated.jl index df8c8b62f05a2..22c0a71fa0aa5 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -428,6 +428,13 @@ Set{T<:Number}(xs::T...) = Set{T}(xs) @deprecate read(s::IO, a::Array) read!(s, a) @deprecate read(s::IO, B::BitArray) read!(s, B) +@deprecate nans{T}(::Type{T}, dims...) fill(convert(T,NaN), dims) +@deprecate nans(dims...) fill(NaN, dims) +@deprecate nans{T}(x::AbstractArray{T}) fill(convert(T,NaN), size(x)) +@deprecate infs{T}(::Type{T}, dims...) fill(convert(T,Inf), dims) +@deprecate infs(dims...) fill(Inf, dims) +@deprecate infs{T}(x::AbstractArray{T}) fill(convert(T,Inf), size(x)) + # 0.3 discontinued functions function nnz(X) diff --git a/base/exports.jl b/base/exports.jl index b5f946965d305..de4ae777ce91e 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -518,7 +518,6 @@ export indexin, indmax, indmin, - infs, invperm, ipermute!, ipermutedims, @@ -535,7 +534,6 @@ export minimum!, minimum, minmax, - nans, ndims, nfilled, nonzeros, diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index b8a5bc9c4ee27..11be043a52bf3 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -3564,14 +3564,6 @@ Constructors Create an array of all ones of specified type -.. function:: infs(type, dims) - - Create an array where every element is infinite and of the specified type - -.. function:: nans(type, dims) - - Create an array where every element is NaN of the specified type - .. function:: trues(dims) Create a ``BitArray`` with all values set to true From c90c0afd6ebc9bd89f6f2e128fa6dee08b6e7a77 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 19 Mar 2014 18:57:17 -0400 Subject: [PATCH 33/47] Fix #5459 This combines the .o emission code from #5787 with some error checking code for cpuid mismatch. When releasing a binary, set JULIA_CPU_TARGET to "core2" (we discussed that this is a reasonable minimum) in your Make.user and everything should work just fine. --- Make.inc | 27 ++++++++++++----- Makefile | 9 ++---- src/Makefile | 1 + src/cgutils.cpp | 28 +++++++++++++++++ src/codegen.cpp | 72 ++++++++++++++++++++++++++++++++++++++++---- src/dump.c | 22 +++++++++++++- src/init.c | 10 +++--- src/julia_internal.h | 1 + ui/Makefile | 1 + 9 files changed, 146 insertions(+), 25 deletions(-) diff --git a/Make.inc b/Make.inc index 8864e5f43bed8..7f69d3a9c4419 100644 --- a/Make.inc +++ b/Make.inc @@ -42,6 +42,10 @@ USE_INTEL_JITEVENTS = 0 # libc++ is standard on OS X 10.9, but not for earlier releases USE_LIBCPP = 0 +# Select the cpu architecture to target +# Current available options are "native" and "core2" +JULIA_CPU_TARGET = native + # we include twice to pickup user definitions better ifeq (exists, $(shell [ -e $(JULIAHOME)/Make.user ] && echo exists )) include $(JULIAHOME)/Make.user @@ -207,7 +211,8 @@ $(error Address Sanitizer only supported with clang. Try setting SANITIZE=0) endif CC = $(CROSS_COMPILE)gcc CXX = $(CROSS_COMPILE)g++ -JCFLAGS = -std=gnu99 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 +JCFLAGS = -std=gnu99 -pipe $(fPIC) -fno-strict-aliasing +JCPPFLAGS = -D_FILE_OFFSET_BITS=64 JCXXFLAGS = -pipe $(fPIC) -fno-rtti DEBUGFLAGS = -O0 -ggdb3 -DDEBUG -fstack-protector-all SHIPFLAGS = -O3 -falign-functions @@ -216,7 +221,8 @@ endif ifeq ($(USECLANG),1) CC = $(CROSS_COMPILE)clang CXX = $(CROSS_COMPILE)clang++ -JCFLAGS = -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 +JCFLAGS = -pipe $(fPIC) -fno-strict-aliasing +JCPPFLAGS = -D_FILE_OFFSET_BITS=64 JCXXFLAGS = -pipe $(fPIC) -fno-rtti DEBUGFLAGS = -O0 -g -DDEBUG -fstack-protector-all SHIPFLAGS = -O3 @@ -229,7 +235,7 @@ else CC += $(STDLIBCPP_FLAG) -mmacosx-version-min=10.6 CXX += $(STDLIBCPP_FLAG) -mmacosx-version-min=10.6 endif -JCFLAGS += -D_LARGEFILE_SOURCE -D_DARWIN_USE_64_BIT_INODE=1 +JCPPFLAGS += -D_LARGEFILE_SOURCE -D_DARWIN_USE_64_BIT_INODE=1 endif endif @@ -241,6 +247,14 @@ AS := $(CROSS_COMPILE)as LD := $(CROSS_COMPILE)ld RANLIB := $(CROSS_COMPILE)ranlib +ifeq ($(JULIA_CPU_TARGET),native) + JCPPFLAGS += -DJULIA_TARGET_NATIVE +else ifeq ($(JULIA_CPU_TARGET),core2) + JCPPFLAGS += -DJULIA_TARGET_CORE2 +else + $(error Unknown cpu target architecture) +endif + # Calculate relative paths to libdir and private_libdir build_libdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(build_bindir) $(build_libdir)) @@ -250,7 +264,7 @@ build_private_libdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(build private_libdir_rel = $(shell $(JULIAHOME)/contrib/relative_path.sh $(bindir) $(private_libdir)) # if not absolute, then relative to the directory of the julia executable -JCFLAGS += "-DJL_SYSTEM_IMAGE_PATH=\"$(build_private_libdir_rel)/sys.ji\"" +JCPPFLAGS += "-DJL_SYSTEM_IMAGE_PATH=\"$(build_private_libdir_rel)/sys.ji\"" # On Windows, we want shared library files to end up in $(build_bindir), instead of $(build_libdir) ifeq ($(OS),WINNT) @@ -321,7 +335,7 @@ endif else ifeq ($(OS),Darwin) LIBUNWIND=$(build_libdir)/libosxunwind.a -JCFLAGS+=-DLIBOSXUNWIND +JCPPFLAGS+=-DLIBOSXUNWIND else LIBUNWIND=$(build_libdir)/libunwind-generic.a $(build_libdir)/libunwind.a endif @@ -485,8 +499,7 @@ endif ifeq ($(OS), WINNT) OSLIBS += -Wl,--export-all-symbols -Wl,--version-script=$(JULIAHOME)/src/julia.expmap \ $(NO_WHOLE_ARCHIVE) -lpsapi -lkernel32 -lws2_32 -liphlpapi -lwinmm -ldbghelp -lssp -JCFLAGS += -D_WIN32_WINNT=0x0600 -JCXXFLAGS += -D_WIN32_WINNT=0x0600 +JCPPFLAGS += -D_WIN32_WINNT=0x0600 JLDFLAGS = -Wl,--stack,8388608 ifeq ($(ARCH),i686) JLDFLAGS += -Wl,--large-address-aware diff --git a/Makefile b/Makefile index d1e83195b70a4..09afb82b4d461 100644 --- a/Makefile +++ b/Makefile @@ -65,10 +65,7 @@ $(build_sysconfdir)/julia/juliarc.jl: contrib/windows/juliarc.jl endif # use sys.ji if it exists, otherwise run two stages -$(build_private_libdir)/sys%ji: $(build_private_libdir)/sys%bc - -$(build_private_libdir)/sys%o: $(build_private_libdir)/sys%bc - $(call spawn,$(LLVM_LLC)) -filetype=obj -relocation-model=pic -mattr=-bmi2,-avx2 -o $(call cygpath_w,$@) $(call cygpath_w,$<) +$(build_private_libdir)/sys%ji: $(build_private_libdir)/sys%o .PRECIOUS: $(build_private_libdir)/sys%o @@ -77,11 +74,11 @@ $(build_private_libdir)/sys%$(SHLIB_EXT): $(build_private_libdir)/sys%o $$([ $(OS) = Darwin ] && echo -Wl,-undefined,dynamic_lookup || echo -Wl,--unresolved-symbols,ignore-all ) \ $$([ $(OS) = WINNT ] && echo -ljulia -lssp) -$(build_private_libdir)/sys0.bc: +$(build_private_libdir)/sys0.o: @$(QUIET_JULIA) cd base && \ $(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys0) sysimg.jl -$(build_private_libdir)/sys.bc: VERSION base/*.jl base/pkg/*.jl base/linalg/*.jl base/sparse/*.jl $(build_datarootdir)/julia/helpdb.jl $(build_datarootdir)/man/man1/julia.1 $(build_private_libdir)/sys0.$(SHLIB_EXT) +$(build_private_libdir)/sys.o: VERSION base/*.jl base/pkg/*.jl base/linalg/*.jl base/sparse/*.jl $(build_datarootdir)/julia/helpdb.jl $(build_datarootdir)/man/man1/julia.1 $(build_private_libdir)/sys0.$(SHLIB_EXT) @$(QUIET_JULIA) cd base && \ $(call spawn,$(JULIA_EXECUTABLE)) --build $(call cygpath_w,$(build_private_libdir)/sys) \ -J$(call cygpath_w,$(build_private_libdir))/$$([ -e $(build_private_libdir)/sys.ji ] && echo sys.ji || echo sys0.ji) -f sysimg.jl \ diff --git a/src/Makefile b/src/Makefile index f2969f394d7f1..ef2f92bf7d041 100644 --- a/src/Makefile +++ b/src/Makefile @@ -4,6 +4,7 @@ include $(JULIAHOME)/deps/Versions.make override CFLAGS += $(JCFLAGS) override CXXFLAGS += $(JCXXFLAGS) +override CPPFLAGS += $(JCPPFLAGS) SRCS = \ jltypes gf ast builtins module codegen interpreter \ diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 88e56795f0e43..2b330f7a7a5f5 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -215,6 +215,11 @@ extern "C" int32_t jl_get_llvm_gv(jl_value_t *p) return 0; return it->second.index; } + +extern "C" { + extern void cpuid(int32_t CPUInfo[4], int32_t InfoType); +} + static void jl_gen_llvm_gv_array() { // emit the variable table into the code image (can only call this once) @@ -234,6 +239,29 @@ static void jl_gen_llvm_gv_array() GlobalVariable::ExternalLinkage, ConstantInt::get(T_size,globalUnique+1), "jl_globalUnique"); + + Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext,jl_cpu_string); + new GlobalVariable( + *jl_Module, + feature_string->getType(), + true, + GlobalVariable::ExternalLinkage, + feature_string, + "jl_sysimg_cpu_target"); + + // For native also store the cpuid + if(strcmp(jl_cpu_string,"native") == 0) { + uint32_t info[4]; + + cpuid((int32_t*)info, 1); + new GlobalVariable( + *jl_Module, + T_int64, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_int64,((uint64_t)info[2])|(((uint64_t)info[3])<<32)), + "jl_sysimg_cpu_cpuid"); + } } static int32_t jl_assign_functionID(Function *functionObject) diff --git a/src/codegen.cpp b/src/codegen.cpp index e6b895d194c4b..08cbee719ab2e 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -30,11 +30,14 @@ #include "llvm/ExecutionEngine/JITEventListener.h" #include "llvm/ExecutionEngine/JITMemoryManager.h" #include "llvm/PassManager.h" +#include "llvm/Target/TargetLibraryInfo.h" +#include "llvm/Support/TargetRegistry.h" #include "llvm/Analysis/Passes.h" #include "llvm/Bitcode/ReaderWriter.h" #if defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 5 #define LLVM35 1 #include "llvm/IR/Verifier.h" +#include "llvm/Object/ObjectFile.h" #include "llvm/IR/DIBuilder.h" #include "llvm/AsmParser/Parser.h" #else @@ -80,6 +83,7 @@ #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Instrumentation.h" #if defined(LLVM_VERSION_MAJOR) && LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR >= 1 #include "llvm/Transforms/Vectorize.h" #endif @@ -127,6 +131,7 @@ static LLVMContext &jl_LLVMContext = getGlobalContext(); static IRBuilder<> builder(getGlobalContext()); static bool nested_compile=false; static ExecutionEngine *jl_ExecutionEngine; +static TargetMachine *jl_TargetMachine; #ifdef USE_MCJIT static Module *shadow_module; static RTDyldMemoryManager *jl_mcjmm; @@ -302,6 +307,47 @@ void jl_dump_bitcode(char* fname) #endif } +extern "C" +void jl_dump_objfile(char* fname, int jit_model) +{ + std::string err; + raw_fd_ostream OS(fname, err); + formatted_raw_ostream FOS(OS); + jl_gen_llvm_gv_array(); + + // We don't want to use MCJIT's target machine because + // it uses the large code model and we may potentially + // want less optimizations there. + OwningPtr + TM(jl_TargetMachine->getTarget().createTargetMachine( + jl_TargetMachine->getTargetTriple(), + jl_TargetMachine->getTargetCPU(), + jl_TargetMachine->getTargetFeatureString(), + jl_TargetMachine->Options, +#ifdef _OS_LINUX_ + Reloc::PIC_, +#else + jit_model ? Reloc::PIC_ : Reloc::Default, +#endif + jit_model ? CodeModel::JITDefault : CodeModel::Default, + CodeGenOpt::Aggressive // -O3 + )); + + PassManager PM; + PM.add(new TargetLibraryInfo(Triple(jl_TargetMachine->getTargetTriple()))); + PM.add(new DataLayout(*jl_ExecutionEngine->getDataLayout())); + if (TM->addPassesToEmitFile(PM, FOS, + TargetMachine::CGFT_ObjectFile, false)) { + jl_error("Could not generate obj file for this target"); + } + +#ifdef USE_MCJIT + PM.run(*shadow_module); +#else + PM.run(*jl_Module); +#endif +} + // aggregate of array metadata typedef struct { Value *dataptr; @@ -366,13 +412,20 @@ static Type *NoopType; // --- utilities --- +#if defined(JULIA_TARGET_CORE2) +const char *jl_cpu_string = "core2"; +#elif defined(JULIA_TARGET_NATIVE) +const char *jl_cpu_string = "native"; +#else +#error "Must select julia cpu target" +#endif + extern "C" { int globalUnique = 0; } #include "cgutils.cpp" - static void jl_rethrow_with_add(const char *fmt, ...) { if (jl_typeis(jl_exception_in_transit, jl_errorexception_type)) { @@ -4082,23 +4135,30 @@ extern "C" void jl_init_codegen(void) #endif #ifdef USE_MCJIT jl_mcjmm = new SectionMemoryManager(); -#endif +#else // Temporarily disable Haswell BMI2 features due to LLVM bug. const char *mattr[] = {"-bmi2", "-avx2"}; +#endif std::vector attrvec (mattr, mattr+2); - jl_ExecutionEngine = EngineBuilder(engine_module) + EngineBuilder eb = EngineBuilder(engine_module) .setEngineKind(EngineKind::JIT) #if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) .setJITMemoryManager(new JITMemoryManagerWin()) #endif .setTargetOptions(options) +#if defined(JULIA_TARGET_NATIVE) + .setMCPU("") +#else + .setMCPU(jl_cpu_string) +#endif #ifdef USE_MCJIT .setUseMCJIT(true) - .setMAttrs(attrvec) + .setMAttrs(attrvec); #else - .setMAttrs(attrvec) + .setMAttrs(attrvec); #endif - .create(); + jl_TargetMachine = eb.selectTarget(); + jl_ExecutionEngine = eb.create(jl_TargetMachine); #endif // LLVM VERSION jl_ExecutionEngine->DisableLazyCompilation(); diff --git a/src/dump.c b/src/dump.c index 4a3d7f13ae18a..8fd591cb7c697 100644 --- a/src/dump.c +++ b/src/dump.c @@ -95,9 +95,11 @@ static void write_as_tag(ios_t *s, uint8_t tag) static void jl_serialize_value_(ios_t *s, jl_value_t *v); static jl_value_t *jl_deserialize_value(ios_t *s); static jl_value_t *jl_deserialize_value_internal(ios_t *s); -static jl_value_t ***sysimg_gvars = NULL; +jl_value_t ***sysimg_gvars = NULL; extern int globalUnique; +extern void cpuid(int32_t CPUInfo[4], int32_t InfoType); +extern const char *jl_cpu_string; static void jl_load_sysimg_so(char *fname) { @@ -108,6 +110,24 @@ static void jl_load_sysimg_so(char *fname) if (sysimg_handle != 0) { sysimg_gvars = (jl_value_t***)jl_dlsym(sysimg_handle, "jl_sysimg_gvars"); globalUnique = *(size_t*)jl_dlsym(sysimg_handle, "jl_globalUnique"); + const char *cpu_target = (const char*)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_target"); + if (strcmp(cpu_target,jl_cpu_string) != 0) + jl_error("Julia and the system image were compiled for different architectures."); + uint32_t info[4]; + cpuid((int32_t*)info, 1); + if (strcmp(cpu_target,"native") == 0) + { + uint64_t saved_cpuid = *(uint64_t*)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_cpuid"); + if (saved_cpuid != (((uint64_t)info[2])|(((uint64_t)info[3])<<32))) + jl_error("Target architecture mismatch. Please regenerate sys.{so,dll,dylib}."); + } else if(strcmp(cpu_target,"core2") == 0) { + int HasSSSE3 = (info[3] & 1<<9); + if (!HasSSSE3) + jl_error("The current host does not support SSSE3, but the system image was compiled for Core2.\n" + "Please delete or regenerate sys.{so,dll,dylib}."); + } else { + jl_error("System image has unknown target cpu architecture."); + } } else { sysimg_gvars = 0; diff --git a/src/init.c b/src/init.c index 9ef8565bd42f6..c0970aea20aa8 100644 --- a/src/init.c +++ b/src/init.c @@ -867,13 +867,13 @@ DLLEXPORT int julia_trampoline(int argc, char **argv, int (*pmain)(int ac,char * if (asprintf(&build_ji, "%s.ji",build_path) > 0) { jl_save_system_image(build_ji); free(build_ji); - char *build_bc; - if (asprintf(&build_bc, "%s.bc",build_path) > 0) { - jl_dump_bitcode(build_bc); - free(build_bc); + char *build_o; + if (asprintf(&build_o, "%s.o",build_path) > 0) { + jl_dump_objfile(build_o,0); + free(build_o); } else { - ios_printf(ios_stderr,"FATAL: failed to create string for .bc build path"); + ios_printf(ios_stderr,"FATAL: failed to create string for .o build path"); } } else { diff --git a/src/julia_internal.h b/src/julia_internal.h index 6729da8cae53a..feb379ff86469 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -87,6 +87,7 @@ void jl_init_tasks(void *stack, size_t ssize); void jl_init_serializer(void); void jl_dump_bitcode(char *fname); +void jl_dump_objfile(char *fname, int jit_model); int32_t jl_get_llvm_gv(jl_value_t *p); #ifdef _OS_LINUX_ diff --git a/ui/Makefile b/ui/Makefile index 76cb85d8954ee..0d556869187f2 100644 --- a/ui/Makefile +++ b/ui/Makefile @@ -3,6 +3,7 @@ include $(JULIAHOME)/Make.inc override CFLAGS += $(JCFLAGS) override CXXFLAGS += $(JCXXFLAGS) +override CPPFLAGS += $(JCPPFLAGS) FLAGS = -Wall -Wno-strict-aliasing -fno-omit-frame-pointer \ -I$(JULIAHOME)/src -I$(JULIAHOME)/src/support -I$(build_includedir) From 509b4199fe220ce4fe7e09ca845715dcf7666cef Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 19 Mar 2014 23:16:22 -0400 Subject: [PATCH 34/47] Add Please delete or regenerate sys.{so,dll,dylib}'' to all the error messages --- src/dump.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/dump.c b/src/dump.c index 8fd591cb7c697..b5b29ee3e7e2e 100644 --- a/src/dump.c +++ b/src/dump.c @@ -112,21 +112,23 @@ static void jl_load_sysimg_so(char *fname) globalUnique = *(size_t*)jl_dlsym(sysimg_handle, "jl_globalUnique"); const char *cpu_target = (const char*)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_target"); if (strcmp(cpu_target,jl_cpu_string) != 0) - jl_error("Julia and the system image were compiled for different architectures."); + jl_error("Julia and the system image were compiled for different architectures." + "Please delete or regenerate sys.{so,dll,dylib}."); uint32_t info[4]; cpuid((int32_t*)info, 1); if (strcmp(cpu_target,"native") == 0) { uint64_t saved_cpuid = *(uint64_t*)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_cpuid"); if (saved_cpuid != (((uint64_t)info[2])|(((uint64_t)info[3])<<32))) - jl_error("Target architecture mismatch. Please regenerate sys.{so,dll,dylib}."); + jl_error("Target architecture mismatch. Please delete or regenerate sys.{so,dll,dylib}."); } else if(strcmp(cpu_target,"core2") == 0) { int HasSSSE3 = (info[3] & 1<<9); if (!HasSSSE3) jl_error("The current host does not support SSSE3, but the system image was compiled for Core2.\n" "Please delete or regenerate sys.{so,dll,dylib}."); } else { - jl_error("System image has unknown target cpu architecture."); + jl_error("System image has unknown target cpu architecture." + "Please delete or regenerate sys.{so,dll,dylib}."); } } else { From 571cb290114bf78d32fc079456d12ccdb2c3b3a2 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 19 Mar 2014 23:22:40 -0400 Subject: [PATCH 35/47] Move -D_FILE_OFFSET_BITS back to CFLAGS These should really go in CPPFLAGS, but apparently on mingw, there's defines in the stdio headers that rename the function (rather than doing it on a symbol lelve), so we can't use this when we include LLVM headers. Anyway, as long as we don't call the IO functions from C++ we should be fine. --- Make.inc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Make.inc b/Make.inc index 7f69d3a9c4419..421ef097d9aca 100644 --- a/Make.inc +++ b/Make.inc @@ -211,8 +211,8 @@ $(error Address Sanitizer only supported with clang. Try setting SANITIZE=0) endif CC = $(CROSS_COMPILE)gcc CXX = $(CROSS_COMPILE)g++ -JCFLAGS = -std=gnu99 -pipe $(fPIC) -fno-strict-aliasing -JCPPFLAGS = -D_FILE_OFFSET_BITS=64 +JCFLAGS = -std=gnu99 -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 +JCPPFLAGS = JCXXFLAGS = -pipe $(fPIC) -fno-rtti DEBUGFLAGS = -O0 -ggdb3 -DDEBUG -fstack-protector-all SHIPFLAGS = -O3 -falign-functions @@ -221,8 +221,8 @@ endif ifeq ($(USECLANG),1) CC = $(CROSS_COMPILE)clang CXX = $(CROSS_COMPILE)clang++ -JCFLAGS = -pipe $(fPIC) -fno-strict-aliasing -JCPPFLAGS = -D_FILE_OFFSET_BITS=64 +JCFLAGS = -pipe $(fPIC) -fno-strict-aliasing -D_FILE_OFFSET_BITS=64 +JCPPFLAGS = JCXXFLAGS = -pipe $(fPIC) -fno-rtti DEBUGFLAGS = -O0 -g -DDEBUG -fstack-protector-all SHIPFLAGS = -O3 From a6476e05fc33ff2678a77068ec7d8ec76b2a1206 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 19 Mar 2014 23:25:42 -0400 Subject: [PATCH 36/47] Add newlines to error messages --- src/dump.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dump.c b/src/dump.c index b5b29ee3e7e2e..68bcbb69dd571 100644 --- a/src/dump.c +++ b/src/dump.c @@ -112,7 +112,7 @@ static void jl_load_sysimg_so(char *fname) globalUnique = *(size_t*)jl_dlsym(sysimg_handle, "jl_globalUnique"); const char *cpu_target = (const char*)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_target"); if (strcmp(cpu_target,jl_cpu_string) != 0) - jl_error("Julia and the system image were compiled for different architectures." + jl_error("Julia and the system image were compiled for different architectures.\n" "Please delete or regenerate sys.{so,dll,dylib}."); uint32_t info[4]; cpuid((int32_t*)info, 1); @@ -127,7 +127,7 @@ static void jl_load_sysimg_so(char *fname) jl_error("The current host does not support SSSE3, but the system image was compiled for Core2.\n" "Please delete or regenerate sys.{so,dll,dylib}."); } else { - jl_error("System image has unknown target cpu architecture." + jl_error("System image has unknown target cpu architecture.\n" "Please delete or regenerate sys.{so,dll,dylib}."); } } From aa76e5863e90bf8db5af2586edf15438cf1be8c0 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 19 Mar 2014 23:32:18 -0400 Subject: [PATCH 37/47] cpuid -> jl_cpuid --- src/cgutils.cpp | 4 ++-- src/dump.c | 4 ++-- src/sys.c | 12 +++--------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 2b330f7a7a5f5..9647de6c78b9d 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -217,7 +217,7 @@ extern "C" int32_t jl_get_llvm_gv(jl_value_t *p) } extern "C" { - extern void cpuid(int32_t CPUInfo[4], int32_t InfoType); + extern void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType); } static void jl_gen_llvm_gv_array() @@ -253,7 +253,7 @@ static void jl_gen_llvm_gv_array() if(strcmp(jl_cpu_string,"native") == 0) { uint32_t info[4]; - cpuid((int32_t*)info, 1); + jl_cpuid((int32_t*)info, 1); new GlobalVariable( *jl_Module, T_int64, diff --git a/src/dump.c b/src/dump.c index 68bcbb69dd571..ab8c5568bc606 100644 --- a/src/dump.c +++ b/src/dump.c @@ -98,7 +98,7 @@ static jl_value_t *jl_deserialize_value_internal(ios_t *s); jl_value_t ***sysimg_gvars = NULL; extern int globalUnique; -extern void cpuid(int32_t CPUInfo[4], int32_t InfoType); +extern void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType); extern const char *jl_cpu_string; static void jl_load_sysimg_so(char *fname) @@ -115,7 +115,7 @@ static void jl_load_sysimg_so(char *fname) jl_error("Julia and the system image were compiled for different architectures.\n" "Please delete or regenerate sys.{so,dll,dylib}."); uint32_t info[4]; - cpuid((int32_t*)info, 1); + jl_cpuid((int32_t*)info, 1); if (strcmp(cpu_target,"native") == 0) { uint64_t saved_cpuid = *(uint64_t*)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_cpuid"); diff --git a/src/sys.c b/src/sys.c index e7200fa98bf3d..7442406b117f8 100644 --- a/src/sys.c +++ b/src/sys.c @@ -413,11 +413,7 @@ JL_STREAM *jl_stderr_stream(void) { return (JL_STREAM*)JL_STDERR; } #ifdef __SSE__ -#ifdef _OS_WINDOWS_ -#define cpuid __cpuid -#else - -void cpuid(int32_t CPUInfo[4], int32_t InfoType) +void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType) { __asm__ __volatile__ ( #if defined(__i386__) && defined(__PIC__) @@ -436,16 +432,14 @@ void cpuid(int32_t CPUInfo[4], int32_t InfoType) ); } -#endif - DLLEXPORT uint8_t jl_zero_subnormals(uint8_t isZero) { uint32_t flags = 0x00000000; int32_t info[4]; - cpuid(info, 0); + jl_cpuid(info, 0); if (info[0] >= 1) { - cpuid(info, 0x00000001); + jl_cpuid(info, 0x00000001); if ((info[3] & ((int)1 << 26)) != 0) { // SSE2 supports both FZ and DAZ flags = 0x00008040; From 88b1e32372c66920f24e460886e142bc63121ac0 Mon Sep 17 00:00:00 2001 From: Dahua Lin Date: Thu, 20 Mar 2014 11:29:40 -0500 Subject: [PATCH 38/47] introduce AbstractSparseArray. Close #6213. --- base/exports.jl | 2 ++ base/sparse/abstractsparse.jl | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index de4ae777ce91e..5e50c21823f6c 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -14,7 +14,9 @@ export # Types AbstractMatrix, + AbstractSparseArray, AbstractSparseMatrix, + AbstractSparseVector, AbstractVector, AbstractVecOrMat, Array, diff --git a/base/sparse/abstractsparse.jl b/base/sparse/abstractsparse.jl index 7e99625e5226d..6c6726d678b4e 100644 --- a/base/sparse/abstractsparse.jl +++ b/base/sparse/abstractsparse.jl @@ -1,6 +1,9 @@ -abstract AbstractSparseMatrix{Tv,Ti} <: StoredArray{Tv,2} +abstract AbstractSparseArray{Tv,Ti,N} <: AbstractArray{Tv,N} + +typealias AbstractSparseVector{Tv,Ti} AbstractSparseArray{Tv,Ti,1} +typealias AbstractSparseMatrix{Tv,Ti} AbstractSparseArray{Tv,Ti,2} issparse(A::AbstractArray) = false -issparse(S::AbstractSparseMatrix) = true +issparse(S::AbstractSparseArray) = true -indtype{Tv,Ti}(S::AbstractSparseMatrix{Tv,Ti}) = Ti +indtype{Tv,Ti}(S::AbstractSparseArray{Tv,Ti}) = Ti From 11e6757aab4b7618edcc9e80a1dc245a1da8e7e2 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 20 Mar 2014 12:50:14 -0400 Subject: [PATCH 39/47] clarify docs on unsafe load and store and pointer arithmetic --- doc/manual/calling-c-and-fortran-code.rst | 6 ++++++ doc/stdlib/base.rst | 8 +++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/manual/calling-c-and-fortran-code.rst b/doc/manual/calling-c-and-fortran-code.rst index 77664853f96f6..3deabfb5fda29 100644 --- a/doc/manual/calling-c-and-fortran-code.rst +++ b/doc/manual/calling-c-and-fortran-code.rst @@ -349,6 +349,12 @@ buffer and call ``free(ptr)`` when the returned ``Array`` object is finalized. If the ``own`` parameter is omitted or false, the caller must ensure the buffer remains in existence until all access is complete. +Arithmetic on the ``Ptr`` type in Julia (e.g. using ``+``) does not behave the +same as C's pointer arithmetic. Adding an integer to a ``Ptr`` in Julia always +moves the pointer by some number of *bytes*, not elements. This way, the +address values obtained from pointer arithmetic do not depend on the +element types of pointers. + Passing Pointers for Modifying Inputs ------------------------------------- diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst index 54f6988d3c582..22f6d7bb4320c 100644 --- a/doc/stdlib/base.rst +++ b/doc/stdlib/base.rst @@ -5068,11 +5068,13 @@ C Interface .. function:: unsafe_load(p::Ptr{T},i::Integer) - Dereference the pointer ``p[i]`` or ``*p``, returning a copy of type T. + Load a value of type ``T`` from the address of the ith element (1-indexed) + starting at ``p``. This is equivalent to the C expression ``p[i-1]``. .. function:: unsafe_store!(p::Ptr{T},x,i::Integer) - Assign to the pointer ``p[i] = x`` or ``*p = x``, making a copy of object x into the memory at p. + Store a value of type ``T`` to the address of the ith element (1-indexed) + starting at ``p``. This is equivalent to the C expression ``p[i-1] = x``. .. function:: unsafe_copy!(dest::Ptr{T}, src::Ptr{T}, N) @@ -5082,7 +5084,7 @@ C Interface .. function:: unsafe_copy!(dest::Array, do, src::Array, so, N) Copy ``N`` elements from a source array to a destination, starting at offset ``so`` - in the source and ``do`` in the destination. + in the source and ``do`` in the destination (1-indexed). .. function:: copy!(dest, src) From 9776fd273772b5e46af80333fb32f79f527d3938 Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Thu, 20 Mar 2014 20:33:55 -0400 Subject: [PATCH 40/47] don't hardcode bash path On FreeBSD, bash is in /usr/local/bin/. --- contrib/relative_path.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/relative_path.sh b/contrib/relative_path.sh index ebfa7d987a224..9ca6bb4c2bd0e 100755 --- a/contrib/relative_path.sh +++ b/contrib/relative_path.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # both $1 and $2 are absolute paths beginning with / # returns relative path to $2/$target from $1/$source From 5f4611f244b1ee732a79dcd6b61511f191116c45 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 20 Mar 2014 20:35:20 -0400 Subject: [PATCH 41/47] Move jl_cpuid --- src/sys.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sys.c b/src/sys.c index 7442406b117f8..0fda9bf5b30a6 100644 --- a/src/sys.c +++ b/src/sys.c @@ -409,11 +409,9 @@ JL_STREAM *jl_stdin_stream(void) { return (JL_STREAM*)JL_STDIN; } JL_STREAM *jl_stdout_stream(void) { return (JL_STREAM*)JL_STDOUT; } JL_STREAM *jl_stderr_stream(void) { return (JL_STREAM*)JL_STDERR; } -// -- set/clear the FZ/DAZ flags on x86 & x86-64 -- - -#ifdef __SSE__ +// CPUID -void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType) +DLLEXPORT void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType) { __asm__ __volatile__ ( #if defined(__i386__) && defined(__PIC__) @@ -432,6 +430,9 @@ void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType) ); } +// -- set/clear the FZ/DAZ flags on x86 & x86-64 -- +#ifdef __SSE__ + DLLEXPORT uint8_t jl_zero_subnormals(uint8_t isZero) { uint32_t flags = 0x00000000; From 000a0209bf07d6f54256afc3a6dc6c876cb3b29a Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 20 Mar 2014 22:45:38 -0400 Subject: [PATCH 42/47] WINDOWS: fix redirect_stdio to TTY --- base/stream.jl | 20 +++++++++++++++----- src/jl_uv.c | 4 ++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/base/stream.jl b/base/stream.jl index e18b0ff36d746..3b8a530faf008 100644 --- a/base/stream.jl +++ b/base/stream.jl @@ -880,18 +880,28 @@ end connect(path::String) = connect(Pipe(),path) dup(x::RawFD) = RawFD(ccall((@windows? :_dup : :dup),Int32,(Int32,),x.fd)) -dup(src::RawFD,target::RawFD) = systemerror("dup",ccall((@windows? :_dup2 : :dup2),Int32,(Int32,Int32),src.fd,target.fd) == -1) +dup(src::RawFD,target::RawFD) = systemerror("dup",-1== + ccall((@windows? :_dup2 : :dup2),Int32, + (Int32,Int32),src.fd,target.fd)) -@unix_only _fd(x::AsyncStream) = RawFD(ccall(:jl_uv_pipe_fd,Int32,(Ptr{Void},),x.handle)) -@windows_only _fd(x::Pipe) = WindowsRawSocket(ccall(:jl_uv_pipe_handle,Ptr{Void},(Ptr{Void},),x.handle)) +@unix_only _fd(x::AsyncStream) = RawFD( + ccall(:jl_uv_pipe_fd,Int32,(Ptr{Void},),x.handle)) +@windows_only _fd(x::Pipe) = WindowsRawSocket( + ccall(:jl_uv_pipe_handle,Ptr{Void},(Ptr{Void},),x.handle)) +@windows_only _fd(x::TTY) = WindowsRawSocket( + ccall(:jl_uv_tty_handle,Ptr{Void},(Ptr{Void},),x.handle)) for (x,writable,unix_fd,c_symbol) in ((:STDIN,false,0,:jl_uv_stdin),(:STDOUT,true,1,:jl_uv_stdout),(:STDERR,true,2,:jl_uv_stderr)) f = symbol("redirect_"*lowercase(string(x))) @eval begin function ($f)(handle::AsyncStream) global $x - @windows? ccall(:SetStdHandle,stdcall,Int32,(Uint32,Ptr{Void}),$(-10-unix_fd),_fd(handle).handle) : dup(_fd(handle), RawFD($unix_fd)) - unsafe_store!(cglobal($(Expr(:quote,c_symbol)),Ptr{Void}),handle.handle) + @windows? ( + ccall(:SetStdHandle,stdcall,Int32,(Uint32,Ptr{Void}), + $(-10-unix_fd),_get_osfhandle(_fd(handle)).handle) : + dup(_fd(handle), RawFD($unix_fd)) ) + unsafe_store!(cglobal($(Expr(:quote,c_symbol)),Ptr{Void}), + handle.handle) $x = handle end function ($f)() diff --git a/src/jl_uv.c b/src/jl_uv.c index 2b7bd58d585d2..59f5323a3da93 100644 --- a/src/jl_uv.c +++ b/src/jl_uv.c @@ -959,6 +959,10 @@ DLLEXPORT HANDLE jl_uv_pipe_handle(uv_pipe_t *handle) { return handle->handle; } +DLLEXPORT HANDLE jl_uv_tty_handle(uv_tty_t *handle) +{ + return handle->handle; +} #endif #ifdef __cplusplus } From fbd4c76da32166977bb1259185787574d6370478 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 20 Mar 2014 23:07:30 -0400 Subject: [PATCH 43/47] formatting fixes --- src/cgutils.cpp | 30 ++++++++++++++---------------- src/codegen.cpp | 3 +-- src/dump.c | 9 +++++---- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 9647de6c78b9d..a2d5576072565 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -240,27 +240,25 @@ static void jl_gen_llvm_gv_array() ConstantInt::get(T_size,globalUnique+1), "jl_globalUnique"); - Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext,jl_cpu_string); - new GlobalVariable( - *jl_Module, - feature_string->getType(), - true, - GlobalVariable::ExternalLinkage, - feature_string, - "jl_sysimg_cpu_target"); + Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_cpu_string); + new GlobalVariable(*jl_Module, + feature_string->getType(), + true, + GlobalVariable::ExternalLinkage, + feature_string, + "jl_sysimg_cpu_target"); // For native also store the cpuid - if(strcmp(jl_cpu_string,"native") == 0) { + if (strcmp(jl_cpu_string,"native") == 0) { uint32_t info[4]; jl_cpuid((int32_t*)info, 1); - new GlobalVariable( - *jl_Module, - T_int64, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_int64,((uint64_t)info[2])|(((uint64_t)info[3])<<32)), - "jl_sysimg_cpu_cpuid"); + new GlobalVariable(*jl_Module, + T_int64, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_int64,((uint64_t)info[2])|(((uint64_t)info[3])<<32)), + "jl_sysimg_cpu_cpuid"); } } diff --git a/src/codegen.cpp b/src/codegen.cpp index 08cbee719ab2e..8f7c1ab051c21 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -336,8 +336,7 @@ void jl_dump_objfile(char* fname, int jit_model) PassManager PM; PM.add(new TargetLibraryInfo(Triple(jl_TargetMachine->getTargetTriple()))); PM.add(new DataLayout(*jl_ExecutionEngine->getDataLayout())); - if (TM->addPassesToEmitFile(PM, FOS, - TargetMachine::CGFT_ObjectFile, false)) { + if (TM->addPassesToEmitFile(PM, FOS, TargetMachine::CGFT_ObjectFile, false)) { jl_error("Could not generate obj file for this target"); } diff --git a/src/dump.c b/src/dump.c index ab8c5568bc606..b616ddda4895d 100644 --- a/src/dump.c +++ b/src/dump.c @@ -116,17 +116,18 @@ static void jl_load_sysimg_so(char *fname) "Please delete or regenerate sys.{so,dll,dylib}."); uint32_t info[4]; jl_cpuid((int32_t*)info, 1); - if (strcmp(cpu_target,"native") == 0) - { + if (strcmp(cpu_target, "native") == 0) { uint64_t saved_cpuid = *(uint64_t*)jl_dlsym(sysimg_handle, "jl_sysimg_cpu_cpuid"); if (saved_cpuid != (((uint64_t)info[2])|(((uint64_t)info[3])<<32))) jl_error("Target architecture mismatch. Please delete or regenerate sys.{so,dll,dylib}."); - } else if(strcmp(cpu_target,"core2") == 0) { + } + else if(strcmp(cpu_target,"core2") == 0) { int HasSSSE3 = (info[3] & 1<<9); if (!HasSSSE3) jl_error("The current host does not support SSSE3, but the system image was compiled for Core2.\n" "Please delete or regenerate sys.{so,dll,dylib}."); - } else { + } + else { jl_error("System image has unknown target cpu architecture.\n" "Please delete or regenerate sys.{so,dll,dylib}."); } From 5b21c49a70f9e1161fb27a5d075a0fb7e985dadb Mon Sep 17 00:00:00 2001 From: Mike Nolta Date: Fri, 21 Mar 2014 00:14:36 -0400 Subject: [PATCH 44/47] fix #6202 (strtod.c error on centos5) --- src/support/strtod.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/support/strtod.c b/src/support/strtod.c index a99a919ddc159..ec6e77b161510 100644 --- a/src/support/strtod.c +++ b/src/support/strtod.c @@ -1,17 +1,15 @@ -#include "libsupport.h" #define _GNU_SOURCE +#include "libsupport.h" #include #include -#if !defined(_OS_WINDOWS_) -// This code path should be used for systems that support the strtod_l function - +#if defined(__APPLE__) || defined(__FreeBSD__) #include -#if defined(_OS_LINUX_) -extern double strtod_l(const char *nptr, char **endptr, locale_t loc); -extern float strtof_l(const char *nptr, char **endptr, locale_t loc); #endif +#if !defined(_OS_WINDOWS_) +// This code path should be used for systems that support the strtod_l function + // Cache locale object static int c_locale_initialized = 0; static locale_t c_locale; From 7f5f214041d63a8e3ff122c3f9a3c6c8aa4420d1 Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Fri, 21 Mar 2014 19:21:18 -0400 Subject: [PATCH 45/47] Only first argument for @sprintf is allowed to be a format string --- base/printf.jl | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/base/printf.jl b/base/printf.jl index a5ed4db32a574..b12eef368c149 100644 --- a/base/printf.jl +++ b/base/printf.jl @@ -751,21 +751,14 @@ is_str_expr(ex) = isa(ex,Expr) && (ex.head == :string || (ex.head == :macrocall && isa(ex.args[1],Symbol) && endswith(string(ex.args[1]),"str"))) -function _printf(macroname, io, args) - if isempty(args) || !isa(args[1], String) - if !isempty(args) && is_str_expr(args[1]) - error("$macroname: format must be a plain static string (no interpolation or prefix)") - else - error("$macroname: first or second argument must be a format string") - end - end - fmt = args[1] +function _printf(macroname, io, fmt, args) + isa(fmt, String) || error("$macroname: format must be a plain static string (no interpolation or prefix)") sym_args, blk = gen(fmt) - if length(sym_args) != length(args)-1 + if length(sym_args) != length(args) error("$macroname: wrong number of arguments") end - for i = length(args):-1:2 - var = sym_args[i-1].args[1] + for i = length(args):-1:1 + var = sym_args[i].args[1] unshift!(blk.args, :($var = $(esc(args[i])))) end unshift!(blk.args, :(out = $io)) @@ -775,15 +768,19 @@ end macro printf(args...) !isempty(args) || error("@printf: called with zero arguments") if isa(args[1], String) || is_str_expr(args[1]) - _printf("@printf", :STDOUT, args) + _printf("@printf", :STDOUT, args[1], args[2:end]) else - _printf("@printf", esc(args[1]), args[2:end]) + (length(args) >= 2 && (isa(args[2], String) || is_str_expr(args[2]))) || + error("@printf: first or second argument must be a format string") + _printf("@printf", esc(args[1]), args[2], args[3:end]) end end macro sprintf(args...) !isempty(args) || error("@sprintf: called with zero arguments") - :(sprint(io->$(_printf("@sprintf", :io, args)))) + isa(args[1], String) || is_str_expr(args[1]) || + error("@sprintf: first argument must be a format string") + :(sprint(io->$(_printf("@sprintf", :io, args[1], args[2:end])))) end end # module From a9949c604b77b7a42458528b79aefd203fd25f7a Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Sat, 22 Mar 2014 00:04:17 -0400 Subject: [PATCH 46/47] Bump Rmath, fixes #6232 --- deps/Rmath | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/Rmath b/deps/Rmath index e61abe6ff3e2f..1864cf7d1fd77 160000 --- a/deps/Rmath +++ b/deps/Rmath @@ -1 +1 @@ -Subproject commit e61abe6ff3e2f6a2f13e204d68caea7309253915 +Subproject commit 1864cf7d1fd77e21a9bf8b3a50db6a816f255a0f From d988b011cbea655937fcd90e62363a657da72825 Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Sat, 22 Mar 2014 18:15:32 -0400 Subject: [PATCH 47/47] More updates for llvm-svn --- src/codegen.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 8f7c1ab051c21..bab7602724a2a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -40,6 +40,8 @@ #include "llvm/Object/ObjectFile.h" #include "llvm/IR/DIBuilder.h" #include "llvm/AsmParser/Parser.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/ADT/OwningPtr.h" #else #include "llvm/Analysis/Verifier.h" #endif @@ -311,7 +313,11 @@ extern "C" void jl_dump_objfile(char* fname, int jit_model) { std::string err; +#ifdef LLVM35 + raw_fd_ostream OS(fname, err, sys::fs::F_None); +#else raw_fd_ostream OS(fname, err); +#endif formatted_raw_ostream FOS(OS); jl_gen_llvm_gv_array(); @@ -335,7 +341,11 @@ void jl_dump_objfile(char* fname, int jit_model) PassManager PM; PM.add(new TargetLibraryInfo(Triple(jl_TargetMachine->getTargetTriple()))); +#ifdef LLVM35 + PM.add(new DataLayoutPass(*jl_ExecutionEngine->getDataLayout())); +#else PM.add(new DataLayout(*jl_ExecutionEngine->getDataLayout())); +#endif if (TM->addPassesToEmitFile(PM, FOS, TargetMachine::CGFT_ObjectFile, false)) { jl_error("Could not generate obj file for this target"); } @@ -4134,11 +4144,12 @@ extern "C" void jl_init_codegen(void) #endif #ifdef USE_MCJIT jl_mcjmm = new SectionMemoryManager(); + std::vector attrvec; #else // Temporarily disable Haswell BMI2 features due to LLVM bug. const char *mattr[] = {"-bmi2", "-avx2"}; -#endif std::vector attrvec (mattr, mattr+2); +#endif EngineBuilder eb = EngineBuilder(engine_module) .setEngineKind(EngineKind::JIT) #if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_)