Merge pull request #154 from JuliaCI/tb/package_mode

Support for invoking Nanosoldier's `runtests` from package repositories,
and various other improvements.

Manifest.toml

@@ -1,8 +1,8 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.8.3"
+julia_version = "1.8.4"
 manifest_format = "2.0"
-project_hash = "2fcc11f38f8a02864b6890127960604119a99d66"
+project_hash = "ec4b234d8a35990664a89d221f3915e2ca33ee34"
 
 [[deps.AWS]]
 deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MbedTLS", "Mocking", "OrderedCollections", "Random", "Sockets", "URIs", "UUIDs", "XMLDict"]
@@ -23,6 +23,11 @@ git-tree-sha1 = "b132f9aeb209b8790dcc286c857f300369219d8d"
 uuid = "1fd713ca-387f-5abc-8002-d8b8b1623b73"
 version = "2.5.1+0"
 
+[[deps.AutoHashEquals]]
+git-tree-sha1 = "45bb6705d93be619b81451bb2006b7ee5d4e4453"
+uuid = "15f4f7f2-30c1-5605-9d31-71845cf9641f"
+version = "0.2.0"
+
 [[deps.Base64]]
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
@@ -81,7 +86,7 @@ version = "4.4.0"
 [[deps.CompilerSupportLibraries_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
-version = "0.5.2+0"
+version = "1.0.1+0"
 
 [[deps.Crayons]]
 git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
@@ -270,9 +275,9 @@ uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
 [[deps.Libiconv_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778"
+git-tree-sha1 = "c7cb1f5d892775ba13767a87c7ada0b980ea0a71"
 uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531"
-version = "1.16.1+1"
+version = "1.16.1+2"
 
 [[deps.LinearAlgebra]]
 deps = ["Libdl", "libblastrampoline_jll"]
@@ -428,6 +433,12 @@ git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
 uuid = "189a3867-3050-52da-a836-e630ba90ab69"
 version = "1.2.2"
 
+[[deps.RegistryTools]]
+deps = ["AutoHashEquals", "LibGit2", "Pkg", "SHA", "UUIDs"]
+git-tree-sha1 = "8737d3632216c6aaaf8adbfeced16bbcc1208bfd"
+uuid = "d1eb7eb1-105f-429d-abf5-b0f65cb9e2c4"
+version = "2.0.0"
+
 [[deps.Requires]]
 deps = ["UUIDs"]
 git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"

Project.toml

@@ -22,6 +22,7 @@ Pidfile = "fa939f87-e72e-5be4-a000-7fc836dbe307"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PkgEval = "9f2e2246-6dce-11e8-3d98-4b291446da6e"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+RegistryTools = "d1eb7eb1-105f-429d-abf5-b0f65cb9e2c4"
 Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

README.md

@@ -7,19 +7,31 @@ This package contains the infrastructure powering the @nanosoldier CI bot used b
 
 ## Quick start
 
-If you're a collaborator in the JuliaLang/julia repository, you can submit CI jobs to the Julia Lab's Nanosoldier cluster at MIT by commenting on commits or pull requests. The @nanosoldier bot looks for a special "trigger phrase" in your comment, and if the trigger phrase is found, it is parsed by the bot to configure and submit a CI job.
+If you're a collaborator in the JuliaLang/julia repository, you can submit CI jobs by
+commenting on commits or pull requests. The @nanosoldier bot looks for a special "trigger
+phrase" in your comment, and if the trigger phrase is found, it is parsed by the bot to
+configure and submit a CI job.
 
 The trigger phrase syntax is:
 
 ```
 @nanosoldier `command(args..., kwargs...)`
 ```
 
-Backticks are mandatory. There are two kinds of jobs you can invoke: **benchmark jobs**, which run the [BaseBenchmarks.jl](https://github.com/JuliaCI/BaseBenchmarks.jl) suite, and **package test jobs** which rely on [PkgEval.jl](https://github.com/JuliaCI/PkgEval.jl) to run the test suite of all registered packages.
+Backticks are mandatory. If the job is accepted, a status update will be pushed to the
+commit you commented on (look for a yellow dot, green check or red cross). Once the job
+finishes, @nanosoldier will reply with a comment, and upload results to the
+[NanosoldierReports](https://github.com/JuliaCI/NanosoldierReports) repository.
+
+There are two kinds of jobs you can invoke: **benchmark jobs**, which run the
+[BaseBenchmarks.jl](https://github.com/JuliaCI/BaseBenchmarks.jl) suite, and **package test
+jobs** which rely on [PkgEval.jl](https://github.com/JuliaCI/PkgEval.jl) to run the test
+suite of all registered packages.
 
 **Note that only one job can be triggered per comment.**
 
-One of the most common invocations runs all benchmarks on your PR, comparing against the current Julia master branch:
+One of the most common invocations runs all benchmarks on your PR, comparing against the
+current Julia master branch:
 
 ```
 @nanosoldier `runbenchmarks()`
@@ -31,31 +43,40 @@ Similarly, you can run all package tests, e.g. if you suspect your PR might be b
 @nanosoldier `runtests()`
 ```
 
-Both operations take a long time, so it might be wise to restrict which benchmarks you want to run, or which packages you want to test:
+Both operations take a long time, so it might be wise to restrict which benchmarks you want
+to run, or which packages you want to test:
 
 ```
 @nanosoldier `runbenchmarks("linalg")`
 
 @nanosoldier `runtests(["JSON", "Crayons"])`
 ```
 
-When a job is completed, @nanosoldier will reply to your comment to tell you how the job went and link you to any relevant results.
+When a job is completed, @nanosoldier will reply to your comment to tell you how the job
+went and link you to any relevant results.
 
 
 ## Available job types
 
-CI jobs are implemented in this package as subtypes of `Nanosoldier.AbstractJob`. See [here](https://github.com/JuliaCI/Nanosoldier.jl/blob/master/src/jobs/jobs.jl) for a description of the interface new job types need to implement.
+CI jobs are implemented in this package as subtypes of `Nanosoldier.AbstractJob`. See
+[here](https://github.com/JuliaCI/Nanosoldier.jl/blob/master/src/jobs/jobs.jl) for a
+description of the interface new job types need to implement.
 
 ### `BenchmarkJob`
 
 #### Execution Cycle
 
 A `BenchmarkJob` has the following execution cycle:
 
-1. Pull in the JuliaLang/julia repository and build the commit specified by the context of the trigger phrase.
-2. Using the new Julia build, fetch the `nanosoldier` branch of the [BaseBenchmarks](https://github.com/JuliaCI/BaseBenchmarks.jl) repository and run the benchmarks specified by the trigger phrase.
-3. If the trigger phrase specifies a commit to compare against, build that version of Julia and perform step 2 using the comparison build.
-4. Upload a markdown report to the [NanosoldierReports](https://github.com/JuliaCI/NanosoldierReports) repository.
+1. Pull in the JuliaLang/julia repository and build the commit specified by the context of
+   the trigger phrase.
+2. Using the new Julia build, fetch the `nanosoldier` branch of the
+   [BaseBenchmarks](https://github.com/JuliaCI/BaseBenchmarks.jl) repository and run the
+   benchmarks specified by the trigger phrase.
+3. If the trigger phrase specifies a commit to compare against, build that version of Julia
+   and perform step 2 using the comparison build.
+4. Upload a markdown report to the
+   [NanosoldierReports](https://github.com/JuliaCI/NanosoldierReports) repository.
 
 #### Trigger Syntax
 
@@ -65,11 +86,19 @@ A `BenchmarkJob` is triggered with the following syntax:
 @nanosoldier `runbenchmarks(tag_predicate, vs = "ref")`
 ```
 
-The `vs` keyword argument is optional; if invoked from a pull request, it will be derived automatically from the merge base. In other cases, the comparison step (step 3 above) will be skipped.
+The `vs` keyword argument is optional; if invoked from a pull request, it will be derived
+automatically from the merge base. In other cases, the comparison step (step 3 above) will
+be skipped.
 
-The tag predicate is used to decide which benchmarks to run, and supports the syntax defined by the [tagging system](https://github.com/JuliaCI/BenchmarkTools.jl/blob/master/doc/manual.md#indexing-into-a-benchmarkgroup-using-tagged) implemented in the [BenchmarkTools](https://github.com/JuliaCI/BenchmarkTools.jl) package. Additionally, you can run all benchmarks by using the keyword `ALL`, e.g. `runbenchmarks(ALL)`, which is the same as specifying no predicate at all.
+The tag predicate is used to decide which benchmarks to run, and supports the syntax defined
+by the [tagging
+system](https://github.com/JuliaCI/BenchmarkTools.jl/blob/master/doc/manual.md#indexing-into-a-benchmarkgroup-using-tagged)
+implemented in the [BenchmarkTools](https://github.com/JuliaCI/BenchmarkTools.jl) package.
+Additionally, you can run all benchmarks by using the keyword `ALL`, e.g.
+`runbenchmarks(ALL)`, which is the same as specifying no predicate at all.
 
-The `vs` keyword argument takes a reference string which can points to a Julia commit to compare against. The following syntax is supported for reference string:
+The `vs` keyword argument takes a reference string which can points to a Julia commit to
+compare against. The following syntax is supported for reference string:
 
 - `":branch"`: the head commit of the branch named `branch` in the current repository (`JuliaLang/julia`)
 - `"@sha"`: the commit specified by `sha` in the current repository (`JuliaLang/julia`)
@@ -140,10 +169,15 @@ against the head commit of my fork's branch.
 
 A `PkgEvalJob` has the following execution cycle:
 
-1. Pull in the JuliaLang/julia repository and build the commit specified by the context of the trigger phrase.
-2. Using the new Julia build, test the packages from the [General](https://github.com/JuliaRegistries/General) registry as specified by the trigger phrase.
-3. If the trigger phrase specifies a commit to compare against, build that version of Julia and perform step 2 using the comparison build.
-4. Upload a markdown report to the [NanosoldierReports](https://github.com/JuliaCI/NanosoldierReports) repository.
+1. Pull in the JuliaLang/julia repository and build the commit specified by the context of
+   the trigger phrase.
+2. Using the new Julia build, test the packages from the
+   [General](https://github.com/JuliaRegistries/General) registry as specified by the
+   trigger phrase.
+3. If the trigger phrase specifies a commit to compare against, build that version of Julia
+   and perform step 2 using the comparison build.
+4. Upload a markdown report to the
+   [NanosoldierReports](https://github.com/JuliaCI/NanosoldierReports) repository.
 
 #### Trigger Syntax
 
@@ -153,11 +187,17 @@ A `PkgEvalJob` is triggered with the following syntax:
 @nanosoldier `runtests(package_selection, vs = "ref")`
 ```
 
-The package selection argument is used to decide which packages to test. It should be a list of package names, e.g. `["Example"]`, that will be looked up in the registry. Additionally, you can test all packages in the registry by using the keyword `ALL`, e.g. `runtests(ALL)`, which is the same as not providing a package selection argument at all.
+The package selection argument is used to decide which packages to test. It should be a list
+of package names, e.g. `["Example"]`, that will be looked up in the registry. Additionally,
+you can test all packages in the registry by using the keyword `ALL`, e.g. `runtests(ALL)`,
+which is the same as not providing a package selection argument at all.
 
-The `vs` keyword argument is again optional. Its syntax and behavior is identical to the `BenchmarkJob` `vs` keyword argument.
+The `vs` keyword argument is again optional. Its syntax and behavior is identical to the
+`BenchmarkJob` `vs` keyword argument.
 
-Both sides of the comparison can be further configured by using respectively the `configuration` and `vs_configuration` arguments. These options expect a named tuple where the elements correspond to fields of the `PkgEval.Configuration` type.
+Both sides of the comparison can be further configured by using respectively the
+`configuration` and `vs_configuration` arguments. These options expect a named tuple where
+the elements correspond to fields of the `PkgEval.Configuration` type.
 
 For example, a common configuration is to include buildflags that enable assertions:
 
@@ -171,79 +211,27 @@ Another useful example makes PkgEval run under rr and use a Julia debug build fo
 @nanosoldier `runtests(configuration = (buildflags=["JULIA_BUILD_MODE=debug"], julia_binary="julia-debug", rr=true))`
 ```
 
-If no configuration arguments are specified, the defaults as specified by the `PkgEval.Configuration` constructor are used.
-
-#### Results
-
-Once a `PkgEvalJob` is complete, the results are uploaded to the
-[NanosoldierReports](https://github.com/JuliaCI/NanosoldierReports) repository. Each job
-has its own directory for results. This directory contains the following items:
-
-- `report.md` is a markdown report that summarizes the job results
-- `data.tar.xz` contains raw test data as Feather files encoding a DataFrame. To untar this file, run
-`tar -xvf data.tar.xz`.
-
-In addition, a rendered version of the report as well as the logs for each package are uploaded to AWS, and will be posted as a reply on GitHub where the job was invoked.
-
-## Initial Setup for BenchmarksJob
-
-On all computers:
-```
-echo "if this is a shared machine, you must use a password to secure this:"
-[ -f ~/.ssh/id_rsa ] || ssh-keygen -f ~/.ssh/id_rsa
-echo "add to https://github.com/settings/keys:"
-cat ~/.ssh/id_rsa.pub
-EDITOR=vim git config --global --edit
-sudo mkdir /nanosoldier
-sudo chown `whoami` /nanosoldier
-cd /nanosoldier
-git clone <URL>
-cd ./Nanosoldier.jl
-git checkout <branch>
-./provision-<worker|server>.sh
-```
-
-On main server:
-```
-scp ~nanosoldier/.ssh/id_rsa ~nanosoldier/.ssh/id_rsa.pub <workers>:
-ssh -t <workers> sudo chown nanosoldier:nanosoldier id_rsa id_rsa.pub
-ssh -t <workers> sudo mv id_rsa id_rsa.pub ~nanosoldier/.ssh
-ssh -t <workers> sudo -u nanosoldier cat .ssh/id_rsa.pub >> .ssh/authorized_keys
-ssh -t <workers> sudo -u nanosoldier "bash -c 'cat ~nanosoldier/.ssh/id_rsa.pub >> ~nanosoldier/.ssh/authorized_keys'"
-sudo -u nanosoldier ssh <workers> exit
-# repeat above for every worker, then:
-sudo -u nanosoldier scp ~nanosoldier/.ssh/known_hosts <workers>:.ssh
-```
+If no configuration arguments are specified, the defaults as specified by the
+`PkgEval.Configuration` constructor are used.
 
-To run:
+#### Reverse-CI for packages
 
-```
-cd /nanosoldier/Nanosoldier.jl
-byobu
-./run_base_ci
-```
+Nanosoldier.jl also supports testing for regression introduced by *package changes*. This
+feature is currently only enabled on select repositories (contact @maleadt if you think this
+is valuable for your package).
 
-## Upgrading for BenchmarksJob
+The interface for testing package changes is identical to testing Julia changes: just invoke
+Nanosoldier by commenting with an appropriate trigger phrase on a commit, issue or pull
+request on a package repository. The execution cycle is slightly different:
 
-# on server
-```
-cd /nanosoldier/Nanosoldier.jl
-git pull
-chmod 666 *.toml
-sudo -u nanosoldier ../julia-1.6.6/bin/julia --project=. -e 'using Pkg; Pkg.update()'
-chmod 664 *.toml
-./provision-server.sh
-git add -u
-git commit
-git push
-```
-
-# on each worker
-```
-cd /nanosoldier/Nanosoldier.jl
-git pull
-./provision-worker.sh
-```
+- The Julia version will be the same for both sides of the comparison, defaulting to
+  `stable` (which can be customized by setting the `julia` argument of the respective
+  `configuration`, e.g., to `"1.8"`)
+- If no package selection is made, or the set of `ALL` packages is requested, Nanosoldier
+  will look up the direct dependents of the package and test those.
+- Tests will be run after registering the current state of the package in a temporary
+  registry (implying that your `Project.toml` should contain a version bump). The `vs` side
+  of the comparison will use an unmodified version of the registry.
 
 
 ## Acknowledgements

bin/run_base_ci.jl

@@ -9,7 +9,6 @@ secret = GITHUB_SECRET
 port = Int(0xffff)
 
 config = Nanosoldier.Config("nanosoldier-worker", nodes, auth, secret;
-                            trackrepo = "JuliaLang/julia",
                             reportrepo = "JuliaCI/NanosoldierReports",
                             testmode = false)
 

bin/run_base_pkgeval.jl

@@ -8,7 +8,6 @@ auth = GitHub.authenticate(ENV["GITHUB_AUTH"])
 secret = ENV["GITHUB_SECRET"]
 
 config = Nanosoldier.Config(ENV["USER"], nodes, auth, secret;
-                            trackrepo = "JuliaLang/julia",
                             reportrepo = "JuliaCI/NanosoldierReports",
                             trigger = r"\@nanosoldier\s*`runtests\(.*?\)`",
                             admin = "maleadt",

bin/setup_test_ci.jl

@@ -8,7 +8,6 @@ auth = GitHub.authenticate(ENV["GITHUB_AUTH"])
 secret = ENV["GITHUB_SECRET"]
 
 config = Nanosoldier.Config("nanosoldier-worker", nodes, auth, secret;
-                            trackrepo = "vtjnash/julia",
                             reportrepo = "vtjnash/NanosoldierReports",
                             admin = "vtjnash",
                             testmode = true)