From f1ea56cc4b1c48d72371932fbc69979c81520833 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sun, 26 Nov 2017 11:59:02 -0800 Subject: [PATCH] start implementing test (#33) --- src/API.jl | 15 ++++++++++++ src/Operations.jl | 60 +++++++++++++++++++++++++++++++++++++++++++++++ src/Pkg3.jl | 6 +++-- src/REPLMode.jl | 44 +++++++++++++++++++++++++++++++++- test/runtests.jl | 36 ++++++++++++++++++---------- 5 files changed, 146 insertions(+), 15 deletions(-) diff --git a/src/API.jl b/src/API.jl index f35af66a3c659..c87a005908fad 100644 --- a/src/API.jl +++ b/src/API.jl @@ -61,4 +61,19 @@ function up(env::EnvCache, pkgs::Vector{PackageSpec}; Pkg3.Operations.up(env, pkgs) end +test(;kwargs...) = test(PackageSpec[], kwargs...) +test(pkg::String; kwargs...) = test([pkg]; kwargs...) +test(pkgs::Vector{String}; kwargs...) = test([PackageSpec(pkg) for pkg in pkgs]; kwargs...) +test(pkgs::Vector{PackageSpec}; kwargs...) = test(EnvCache(), pkgs; kwargs...) + +function test(env::EnvCache, pkgs::Vector{PackageSpec}; coverage=false, preview=env.preview[]) + env.preview[] = preview + preview && previewmode_info() + project_resolve!(env, pkgs) + manifest_resolve!(env, pkgs) + ensure_resolved(env, pkgs) + Pkg3.Operations.test(env, pkgs; coverage=coverage) end + +end # module + diff --git a/src/Operations.jl b/src/Operations.jl index 14ea84452c27c..17fe445a2a5d6 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -606,5 +606,65 @@ function up(env::EnvCache, pkgs::Vector{PackageSpec}) build_versions(env, new) end +function test(env::EnvCache, pkgs::Vector{PackageSpec}; coverage=false) + # See if we can find the test files for all packages + missing_runtests = String[] + testfiles = String[] + version_paths = String[] + for pkg in pkgs + info = manifest_info(env, pkg.uuid) + haskey(info, "hash-sha1") || cmderror("Could not find hash-sha for package $(pkg.name)") + version_path = find_installed(pkg.uuid, SHA1(info["hash-sha1"])) + testfile = joinpath(version_path, "test", "runtests.jl") + if !isfile(testfile) + push!(missing_runtests, pkg.name) + end + push!(version_paths, version_path) + push!(testfiles, testfile) + end + if !isempty(missing_runtests) + cmderror(length(missing_runtests) == 1 ? "Package " : "Packages ", + join(missing_runtests, ", "), + " did not provide a `test/runtests.jl` file") + end + + pkgs_errored = [] + for (pkg, testfile, version_path) in zip(pkgs, testfiles, version_paths) + info("Testing $(pkg.name) located at $version_path") + if env.preview[] + info("In preview mode, skipping tests for $(pkg.name)") + continue + end + # TODO, cd to test folder (need to be careful with getting the same EnvCache + # as for this session in that case + compilemod_opt, compilemod_val = VERSION < v"0.7.0-DEV.1735" ? + ("compilecache" , Base.JLOptions().use_compilecache) : + ("compiled-modules", Base.JLOptions().use_compiled_modules) + + testcmd = `"import Pkg3; include(\"$testfile\")"` + cmd = ``` + $(Base.julia_cmd()) + --code-coverage=$(coverage ? "user" : "none") + --color=$(Base.have_color ? "yes" : "no") + --$compilemod_opt=$(Bool(compilemod_val) ? "yes" : "no") + --check-bounds=yes + --startup-file=$(Base.JLOptions().startupfile != 2 ? "yes" : "no") + $testfile + ``` + try + run(cmd) + info("$(pkg.name) tests passed") + catch err + push!(pkgs_errored, pkg.name) + end + + end + + if !isempty(pkgs_errored) + cmderror(length(pkgs_errored) == 1 ? "Package " : "Packages ", + join(pkgs_errored, ", "), + " errored during testing") + end +end end # module diff --git a/src/Pkg3.jl b/src/Pkg3.jl index 96ca5dd768f5c..0c4fe34e2f4cd 100644 --- a/src/Pkg3.jl +++ b/src/Pkg3.jl @@ -33,6 +33,9 @@ include("Operations.jl") include("REPLMode.jl") include("API.jl") +import .API: add, rm, up, test +const update = up + @enum LoadErrorChoice LOAD_ERROR_QUERY LOAD_ERROR_INSTALL LOAD_ERROR_ERROR Base.@kwdef mutable struct GlobalSettings @@ -56,7 +59,7 @@ if VERSION < v"0.7.0-DEV.2303" Base.find_in_path(name::String, wd::Void) = _find_package(name) Base.find_in_path(name::String, wd::String) = _find_package(name) else - Base.find_package(name::String) = _find_package(name, ) + Base.find_package(name::String) = _find_package(name) end function _find_package(name::String) @@ -67,7 +70,6 @@ function _find_package(name::String) else name = string(base, ".jl") end - info = Pkg3.Operations.package_env_info(base, verb = "use") info == nothing && @goto find_global haskey(info, "uuid") || @goto find_global diff --git a/src/REPLMode.jl b/src/REPLMode.jl index c68fd1634204a..b2df6528e720d 100644 --- a/src/REPLMode.jl +++ b/src/REPLMode.jl @@ -44,6 +44,7 @@ const opts = Dict( "minor" => :minor, "patch" => :patch, "fixed" => :fixed, + "coverage" => :coverage, ) function parse_option(word::AbstractString) @@ -143,6 +144,7 @@ function do_cmd!(env, tokens, repl) cmd == :add ? do_add!(env, tokens) : cmd == :up ? do_up!(env, tokens) : cmd == :status ? do_status!(env, tokens) : + cmd == :test ? do_test!(env, tokens) : cmderror("`$cmd` command not yet implemented") end @@ -184,6 +186,8 @@ const help = Base.Markdown.parse(""" `up`: update packages in manifest `preview`: previews a subsequent command without affecting the current state + + `test`: run tests for packages """) const helps = Dict( @@ -260,7 +264,16 @@ const helps = Dict( Runs the command `cmd` in preview mode. This is defined such that no side effects will take place i.e. no packages are downloaded and neither the project nor manifest is modified. - """ + """, :test => md""" + + test [opts] pkg[=uuid] ... + + opts: --coverage + + Run the tests for package `pkg`. This is done by running the file `test/runtests.jl` + in the package directory. The option `--coverage` can be used to run the tests with + coverage enabled. + """, ) function do_help!( @@ -392,6 +405,35 @@ function do_status!(env::EnvCache, tokens::Vector{Tuple{Symbol,Vararg{Any}}}) Pkg3.Display.status(env, mode) end +# TODO , test recursive dependencies as on option. +function do_test!(env::EnvCache, tokens::Vector{Tuple{Symbol,Vararg{Any}}}) + pkgs = PackageSpec[] + coverage = false + while !isempty(tokens) + token = shift!(tokens) + if token[1] == :pkg + if length(token) == 2 + pkg = PackageSpec(token[2]) + pkg.mode = :manifest + push!(pkgs, pkg) + else + cmderror("`test` only takes a set of packages to test") + end + elseif token[1] == :opt + if token[2] == :coverage + coverage = true + else + cmderror("invalid option for `test`: --$(token[2])") + end + else + # TODO: Better error message + cmderror("invalid usage for `test`") + end + end + isempty(pkgs) && cmderror("`test` takes a set of packages") + Pkg3.API.test(env, pkgs; coverage = coverage) +end + function create_mode(repl, main) pkg_mode = LineEdit.Prompt("pkg> "; prompt_prefix = Base.text_colors[:blue], diff --git a/test/runtests.jl b/test/runtests.jl index e533b875bc86f..c41892f5c12af 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -19,25 +19,37 @@ function temp_pkg_dir(fn::Function) end end +# Tests for Example.jl fail on master, +# so let's use another small package +# in the meantime +const TEST_PKG = "Crayons" + temp_pkg_dir() do - Pkg3.API.add("Example"; preview = true) + Pkg3.add(TEST_PKG; preview = true) @test_warn "not in project" Pkg3.API.rm("Example") - Pkg3.API.add("Example") - @eval import Example - Pkg3.API.up() - Pkg3.API.rm("Example"; preview = true) - # TODO: Check Example is still considered install - Pkg3.API.rm("Example") + Pkg3.add(TEST_PKG) + @eval import $(Symbol(TEST_PKG)) + Pkg3.up() + Pkg3.rm(TEST_PKG; preview = true) + + # TODO: Check coverage kwargs + # TODO: Check that preview = true doesn't actually execute the test + # by creating a package with a test file that fails. + Pkg3.test(TEST_PKG) + Pkg3.test(TEST_PKG; preview = true) + + Pkg3.rm(TEST_PKG) try - Pkg3.API.add([PackageSpec("Example", VersionSpec(v"55"))]) + Pkg3.add([PackageSpec(TEST_PKG, VersionSpec(v"55"))]) catch e - @test contains(sprint(showerror, e), "Example") + @test contains(sprint(showerror, e), TEST_PKG) end + nonexisting_pkg = randstring(14) - @test_throws CommandError Pkg3.API.add(nonexisting_pkg) - @test_throws CommandError Pkg3.API.up(nonexisting_pkg) - @test_warn "not in project" Pkg3.API.rm(nonexisting_pkg) + @test_throws CommandError Pkg3.add(nonexisting_pkg) + @test_throws CommandError Pkg3.up(nonexisting_pkg) + @test_warn "not in project" Pkg3.rm(nonexisting_pkg) end