start implementing test (#33)

src/API.jl

@@ -61,4 +61,19 @@ function up(env::EnvCache, pkgs::Vector{PackageSpec};
     Pkg3.Operations.up(env, pkgs)
 end
 
+test(;kwargs...)                           = test(PackageSpec[], kwargs...)
+test(pkg::String; kwargs...)               = test([pkg]; kwargs...)
+test(pkgs::Vector{String}; kwargs...)      = test([PackageSpec(pkg) for pkg in pkgs]; kwargs...)
+test(pkgs::Vector{PackageSpec}; kwargs...) = test(EnvCache(), pkgs; kwargs...)
+
+function test(env::EnvCache, pkgs::Vector{PackageSpec}; coverage=false, preview=env.preview[])
+    env.preview[] = preview
+    preview && previewmode_info()
+    project_resolve!(env, pkgs)
+    manifest_resolve!(env, pkgs)
+    ensure_resolved(env, pkgs)
+    Pkg3.Operations.test(env, pkgs; coverage=coverage)
 end
+
+end # module
+

src/Operations.jl

@@ -606,5 +606,65 @@ function up(env::EnvCache, pkgs::Vector{PackageSpec})
     build_versions(env, new)
 end
 
+function test(env::EnvCache, pkgs::Vector{PackageSpec}; coverage=false)
+    # See if we can find the test files for all packages
+    missing_runtests = String[]
+    testfiles        = String[]
+    version_paths    = String[]
+    for pkg in pkgs
+        info = manifest_info(env, pkg.uuid)
+        haskey(info, "hash-sha1") || cmderror("Could not find hash-sha for package $(pkg.name)")
+        version_path = find_installed(pkg.uuid, SHA1(info["hash-sha1"]))
+        testfile = joinpath(version_path, "test", "runtests.jl")
+        if !isfile(testfile)
+            push!(missing_runtests, pkg.name)
+        end
+        push!(version_paths, version_path)
+        push!(testfiles, testfile)
+    end
+    if !isempty(missing_runtests)
+        cmderror(length(missing_runtests) == 1 ? "Package " : "Packages ",
+                join(missing_runtests, ", "),
+                " did not provide a `test/runtests.jl` file")
+    end
+
+    pkgs_errored = []
+    for (pkg, testfile, version_path) in zip(pkgs, testfiles, version_paths)
+        info("Testing $(pkg.name) located at $version_path")
+        if env.preview[]
+            info("In preview mode, skipping tests for $(pkg.name)")
+            continue
+        end
+        # TODO, cd to test folder (need to be careful with getting the same EnvCache
+        # as for this session in that case
+        compilemod_opt, compilemod_val = VERSION < v"0.7.0-DEV.1735" ?
+                ("compilecache" ,     Base.JLOptions().use_compilecache) :
+                ("compiled-modules",  Base.JLOptions().use_compiled_modules)
+
+        testcmd = `"import Pkg3; include(\"$testfile\")"`
+        cmd = ```
+            $(Base.julia_cmd())
+            --code-coverage=$(coverage ? "user" : "none")
+            --color=$(Base.have_color ? "yes" : "no")
+            --$compilemod_opt=$(Bool(compilemod_val) ? "yes" : "no")
+            --check-bounds=yes
+            --startup-file=$(Base.JLOptions().startupfile != 2 ? "yes" : "no")
+            $testfile
+        ```
+        try
+            run(cmd)
+            info("$(pkg.name) tests passed")
+        catch err
+            push!(pkgs_errored, pkg.name)
+        end
+
+    end
+
+    if !isempty(pkgs_errored)
+        cmderror(length(pkgs_errored) == 1 ? "Package " : "Packages ",
+                 join(pkgs_errored, ", "),
+                 " errored during testing")
+    end
+end
 end # module
 

src/Pkg3.jl

@@ -33,6 +33,9 @@ include("Operations.jl")
 include("REPLMode.jl")
 include("API.jl")
 
+import .API: add, rm, up, test
+const update = up
+
 @enum LoadErrorChoice LOAD_ERROR_QUERY LOAD_ERROR_INSTALL LOAD_ERROR_ERROR
 
 Base.@kwdef mutable struct GlobalSettings
@@ -56,7 +59,7 @@ if VERSION < v"0.7.0-DEV.2303"
     Base.find_in_path(name::String, wd::Void)   = _find_package(name)
     Base.find_in_path(name::String, wd::String) = _find_package(name)
 else
-    Base.find_package(name::String) = _find_package(name, )
+    Base.find_package(name::String) = _find_package(name)
 end
 
 function _find_package(name::String)
@@ -67,7 +70,6 @@ function _find_package(name::String)
     else
         name = string(base, ".jl")
     end
-
     info = Pkg3.Operations.package_env_info(base, verb = "use")
     info == nothing && @goto find_global
     haskey(info, "uuid") || @goto find_global

src/REPLMode.jl

@@ -44,6 +44,7 @@ const opts = Dict(
     "minor"    => :minor,
     "patch"    => :patch,
     "fixed"    => :fixed,
+    "coverage" => :coverage,
 )
 
 function parse_option(word::AbstractString)
@@ -143,6 +144,7 @@ function do_cmd!(env, tokens, repl)
     cmd == :add     ?     do_add!(env, tokens) :
     cmd == :up      ?      do_up!(env, tokens) :
     cmd == :status  ?  do_status!(env, tokens) :
+    cmd == :test   ?   do_test!(env, tokens) :
         cmderror("`$cmd` command not yet implemented")
 end
 
@@ -184,6 +186,8 @@ const help = Base.Markdown.parse("""
     `up`: update packages in manifest
 
     `preview`: previews a subsequent command without affecting the current state
+
+    `test`: run tests for packages
     """)
 
 const helps = Dict(
@@ -260,7 +264,16 @@ const helps = Dict(
     Runs the command `cmd` in preview mode. This is defined such that no side effects
     will take place i.e. no packages are downloaded and neither the project nor manifest
     is modified.
-    """
+    """, :test => md"""
+
+        test [opts] pkg[=uuid] ...
+
+        opts: --coverage
+
+    Run the tests for package `pkg`. This is done by running the file `test/runtests.jl`
+    in the package directory. The option `--coverage` can be used to run the tests with
+    coverage enabled.
+    """,
 )
 
 function do_help!(
@@ -392,6 +405,35 @@ function do_status!(env::EnvCache, tokens::Vector{Tuple{Symbol,Vararg{Any}}})
     Pkg3.Display.status(env, mode)
 end
 
+# TODO , test recursive dependencies as on option.
+function do_test!(env::EnvCache, tokens::Vector{Tuple{Symbol,Vararg{Any}}})
+    pkgs = PackageSpec[]
+    coverage = false
+    while !isempty(tokens)
+        token = shift!(tokens)
+        if token[1] == :pkg
+            if length(token) == 2
+                pkg = PackageSpec(token[2])
+                pkg.mode = :manifest
+                push!(pkgs, pkg)
+            else
+                cmderror("`test` only takes a set of packages to test")
+            end
+        elseif token[1] == :opt
+            if token[2] == :coverage
+                coverage = true
+            else
+                cmderror("invalid option for `test`: --$(token[2])")
+            end
+        else
+            # TODO: Better error message
+            cmderror("invalid usage for `test`")
+        end
+    end
+    isempty(pkgs) && cmderror("`test` takes a set of packages")
+    Pkg3.API.test(env, pkgs; coverage = coverage)
+end
+
 function create_mode(repl, main)
     pkg_mode = LineEdit.Prompt("pkg> ";
         prompt_prefix = Base.text_colors[:blue],

test/runtests.jl

@@ -19,25 +19,37 @@ function temp_pkg_dir(fn::Function)
     end
 end
 
+# Tests for Example.jl fail on master,
+# so let's use another small package
+# in the meantime
+const TEST_PKG = "Crayons"
+
 temp_pkg_dir() do
-    Pkg3.API.add("Example"; preview = true)
+    Pkg3.add(TEST_PKG; preview = true)
     @test_warn "not in project" Pkg3.API.rm("Example")
-    Pkg3.API.add("Example")
-    @eval import Example
-    Pkg3.API.up()
-    Pkg3.API.rm("Example"; preview = true)
-    # TODO: Check Example is still considered install
-    Pkg3.API.rm("Example")
+    Pkg3.add(TEST_PKG)
+    @eval import $(Symbol(TEST_PKG))
+    Pkg3.up()
+    Pkg3.rm(TEST_PKG; preview = true)
+
+    # TODO: Check coverage kwargs
+    # TODO: Check that preview = true doesn't actually execute the test
+    # by creating a package with a test file that fails.
+    Pkg3.test(TEST_PKG)
+    Pkg3.test(TEST_PKG; preview = true)
+
+    Pkg3.rm(TEST_PKG)
 
     try
-        Pkg3.API.add([PackageSpec("Example", VersionSpec(v"55"))])
+        Pkg3.add([PackageSpec(TEST_PKG, VersionSpec(v"55"))])
     catch e
-        @test contains(sprint(showerror, e), "Example")
+        @test contains(sprint(showerror, e), TEST_PKG)
     end
 
+
     nonexisting_pkg = randstring(14)
-    @test_throws CommandError Pkg3.API.add(nonexisting_pkg)
-    @test_throws CommandError Pkg3.API.up(nonexisting_pkg)
-    @test_warn "not in project" Pkg3.API.rm(nonexisting_pkg)
+    @test_throws CommandError Pkg3.add(nonexisting_pkg)
+    @test_throws CommandError Pkg3.up(nonexisting_pkg)
+    @test_warn "not in project" Pkg3.rm(nonexisting_pkg)
 end