Skip to content

Commit

Permalink
test: more parallel testing based on the machine
Browse files Browse the repository at this point in the history
  • Loading branch information
@avik-pal
avik-pal committed Jul 26, 2024
1 parent e1594bf commit 6a4453d
Show file tree
Hide file tree
Showing 5 changed files with 15 additions and 29 deletions.
13 changes: 0 additions & 13 deletions .buildkite/testing.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,6 @@ steps:
agents:
queue: "juliagpu"
cuda: "*"
env:
RETESTITEMS_NWORKERS: 2
if: build.message !~ /\[skip tests\]/ && build.message !~ /\[skip downstream\]/ && build.message !~ /\[skip ci\]/ && build.pull_request.labels includes "run downstream test"
timeout_in_minutes: 60
matrix:
Expand All @@ -64,11 +62,7 @@ steps:
- src
- ext
env:
JULIA_AMDGPU_CORE_MUST_LOAD: "1"
JULIA_AMDGPU_HIP_MUST_LOAD: "1"
JULIA_AMDGPU_DISABLE_ARTIFACTS: "1"
BACKEND_GROUP: "AMDGPU"
RETESTITEMS_NWORKERS: 2
agents:
queue: "juliagpu"
rocm: "*"
Expand Down Expand Up @@ -107,12 +101,5 @@ steps:
repo:
- "Boltz"


env:
JULIA_AMDGPU_LOGGING_ENABLED: true
RETESTITEMS_NWORKERS: 8
RETESTITEMS_NWORKER_THREADS: 2
RETESTITEMS_TESTITEM_TIMEOUT: 3600
JULIA_PKG_SERVER: ""
JULIA_NUM_THREADS: 4
SECRET_CODECOV_TOKEN: "jQ0BMTQgyZx7QGyU0Q2Ec7qB9mtE2q/tDu0FsfxvEG7/zOAGvXkyXrzIFFOQxvDoFcP+K2+hYZKMxicYdNqzr5wcxu505aNGN2GM3wyegAr+hO6q12bCFYx6qXzU9FLCCdeqINqn9gUSSOlGtWNFrbAlrTyz/D4Yo66TqBDzvaLL63FMnhCLaXW/zJt3hNuEAJaPY2O6Ze1rX2WZ3Y+i+s3uQ8aLImtoCJhPe8CRx+OhuYiTzGhynFfGntZ0738/1RN4gNM0S/hTC4gLE7XMVBanJpGh32rFaiDwW4zAyXKBrDkL3QA3MS1RvLTJxGJ085S16hCk0C4ddAhZCvIM9Q==;U2FsdGVkX1+bXdFeKMs5G79catOCyby2n07A2fg0FjVAvrjQLZ0yfvDS4paJiFikLkodho0khz2YALKb2Y0K6w=="
5 changes: 0 additions & 5 deletions .github/workflows/CI.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,10 +157,7 @@ jobs:
- uses: julia-actions/julia-buildpkg@v1
- uses: julia-actions/julia-runtest@v1
env:
BACKEND_GROUP: "CPU"
LUX_TEST_GROUP: ${{ matrix.test_group }}
RETESTITEMS_NWORKERS: 4
RETESTITEMS_NWORKER_THREADS: 2
- uses: julia-actions/julia-processcoverage@v1
with:
directories: src,ext
Expand Down Expand Up @@ -202,5 +199,3 @@ jobs:

env:
BACKEND_GROUP: "CPU"
RETESTITEMS_NWORKERS: 4
RETESTITEMS_NWORKER_THREADS: 2
6 changes: 1 addition & 5 deletions .github/workflows/CIPreRelease.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ jobs:
- uses: julia-actions/julia-runtest@v1
env:
LUX_TEST_GROUP: ${{ matrix.test_group }}
BACKEND_GROUP: "CPU"
- uses: julia-actions/julia-processcoverage@v1
with:
directories: src,ext
Expand All @@ -69,8 +70,3 @@ jobs:
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true
fail_ci_if_error: true

env:
BACKEND_GROUP: "CPU"
RETESTITEMS_NWORKERS: 4
RETESTITEMS_NWORKER_THREADS: 2
6 changes: 5 additions & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ ForwardDiff = "0.10.36"
FunctionWrappers = "1.1.3"
Functors = "0.4.10"
GPUArraysCore = "0.1.6"
Hwloc = "3.2.0"
InteractiveUtils = "<0.0.1, 1"
LinearAlgebra = "1.10"
Logging = "1.10"
LossFunctions = "0.11.1"
Expand Down Expand Up @@ -128,6 +130,8 @@ DynamicExpressions = "a40a106e-89c9-4ca8-8020-a735e8728b6b"
Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9"
ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7"
FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000"
Hwloc = "0e44f5e4-bd66-52a0-8798-143a42290a1d"
InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
LuxTestUtils = "ac9de150-d08f-4546-94fb-7472b5760531"
MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54"
Expand All @@ -143,4 +147,4 @@ Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c"
Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"

[targets]
test = ["Aqua", "ComponentArrays", "Documenter", "DynamicExpressions", "Enzyme", "ExplicitImports", "FiniteDifferences", "Logging", "LuxTestUtils", "MLUtils", "OneHotArrays", "Pkg", "ReTestItems", "ReverseDiff", "SimpleChains", "StableRNGs", "Statistics", "Test", "Tracker", "Zygote"]
test = ["Aqua", "ComponentArrays", "Documenter", "DynamicExpressions", "Enzyme", "ExplicitImports", "FiniteDifferences", "Hwloc", "InteractiveUtils", "Logging", "LuxTestUtils", "MLUtils", "OneHotArrays", "Pkg", "ReTestItems", "ReverseDiff", "SimpleChains", "StableRNGs", "Statistics", "Test", "Tracker", "Zygote"]
14 changes: 9 additions & 5 deletions test/runtests.jl
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
using ReTestItems, Pkg, Preferences, Test
using InteractiveUtils, Hwloc

@info sprint(io -> versioninfo(io; verbose=true))

const BACKEND_GROUP = lowercase(get(ENV, "BACKEND_GROUP", "all"))
const ALL_LUX_TEST_GROUPS = ["core_layers", "contrib", "helpers", "distributed",
Expand Down Expand Up @@ -65,15 +68,16 @@ using Lux
end
end

const RETESTITEMS_NWORKERS = parse(
Int, get(ENV, "RETESTITEMS_NWORKERS", string(min(Hwloc.num_physical_cores(), 16))))

@testset "ReTestItem Tests" begin
for (i, tag) in enumerate(LUX_TEST_GROUP)
(tag == "distributed" || tag == "eltype_match") && continue
@info "Running tests for group: [$(i)/$(length(LUX_TEST_GROUP))] $tag"
if tag == "all"
ReTestItems.runtests(@__DIR__)
else
ReTestItems.runtests(@__DIR__; tags=[Symbol(tag)])
end

ReTestItems.runtests(@__DIR__; tags=(tag == "all" ? nothing : [Symbol(tag)]),
nworkers=RETESTITEMS_NWORKERS, testitem_timeout=3600)
end
end

Expand Down

1 comment on commit 6a4453d

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Benchmark Results

Benchmark suite Current: 6a4453d Previous: e1594bf Ratio
Dense(2 => 2)/cpu/reverse/ReverseDiff (compiled)/(2, 128) 3733.6666666666665 ns 3693.125 ns 1.01
Dense(2 => 2)/cpu/reverse/Zygote/(2, 128) 7376.714285714285 ns 7382.428571428572 ns 1.00
Dense(2 => 2)/cpu/reverse/Tracker/(2, 128) 20980 ns 20759 ns 1.01
Dense(2 => 2)/cpu/reverse/ReverseDiff/(2, 128) 9760 ns 9812.4 ns 0.99
Dense(2 => 2)/cpu/reverse/Flux/(2, 128) 8970.7 ns 9094.5 ns 0.99
Dense(2 => 2)/cpu/reverse/SimpleChains/(2, 128) 4422.666666666667 ns 4493.375 ns 0.98
Dense(2 => 2)/cpu/reverse/Enzyme/(2, 128) 4676.25 ns 4562.25 ns 1.02
Dense(2 => 2)/cpu/forward/NamedTuple/(2, 128) 1107.1069182389938 ns 1109.9668874172185 ns 1.00
Dense(2 => 2)/cpu/forward/ComponentArray/(2, 128) 1164.4285714285713 ns 1181.937062937063 ns 0.99
Dense(2 => 2)/cpu/forward/Flux/(2, 128) 1789.4313725490197 ns 1793.1929824561403 ns 1.00
Dense(2 => 2)/cpu/forward/SimpleChains/(2, 128) 179.6459802538787 ns 179.68289290681503 ns 1.00
Dense(20 => 20)/cpu/reverse/ReverseDiff (compiled)/(20, 128) 17112 ns 17342 ns 0.99
Dense(20 => 20)/cpu/reverse/Zygote/(20, 128) 16811 ns 16932 ns 0.99
Dense(20 => 20)/cpu/reverse/Tracker/(20, 128) 37100 ns 37520 ns 0.99
Dense(20 => 20)/cpu/reverse/ReverseDiff/(20, 128) 28213 ns 28584 ns 0.99
Dense(20 => 20)/cpu/reverse/Flux/(20, 128) 19947 ns 20278 ns 0.98
Dense(20 => 20)/cpu/reverse/SimpleChains/(20, 128) 17192 ns 17392.5 ns 0.99
Dense(20 => 20)/cpu/reverse/Enzyme/(20, 128) 25488 ns 25578 ns 1.00
Dense(20 => 20)/cpu/forward/NamedTuple/(20, 128) 3822.25 ns 3938.625 ns 0.97
Dense(20 => 20)/cpu/forward/ComponentArray/(20, 128) 3913.625 ns 3991.25 ns 0.98
Dense(20 => 20)/cpu/forward/Flux/(20, 128) 4784.714285714285 ns 5029.428571428572 ns 0.95
Dense(20 => 20)/cpu/forward/SimpleChains/(20, 128) 1651.1 ns 1652.1 ns 1.00
Conv((3, 3), 3 => 3)/cpu/reverse/ReverseDiff (compiled)/(64, 64, 3, 128) 39070284 ns 39521781 ns 0.99
Conv((3, 3), 3 => 3)/cpu/reverse/Zygote/(64, 64, 3, 128) 58211891 ns 58585547 ns 0.99
Conv((3, 3), 3 => 3)/cpu/reverse/Tracker/(64, 64, 3, 128) 77724823 ns 78099068 ns 1.00
Conv((3, 3), 3 => 3)/cpu/reverse/ReverseDiff/(64, 64, 3, 128) 89555817 ns 90769018 ns 0.99
Conv((3, 3), 3 => 3)/cpu/reverse/Flux/(64, 64, 3, 128) 88370701 ns 72690086 ns 1.22
Conv((3, 3), 3 => 3)/cpu/reverse/SimpleChains/(64, 64, 3, 128) 11594550 ns 12054967 ns 0.96
Conv((3, 3), 3 => 3)/cpu/reverse/Enzyme/(64, 64, 3, 128) 91959934 ns 83400155.5 ns 1.10
Conv((3, 3), 3 => 3)/cpu/forward/NamedTuple/(64, 64, 3, 128) 7684647 ns 7721292 ns 1.00
Conv((3, 3), 3 => 3)/cpu/forward/ComponentArray/(64, 64, 3, 128) 7572126.5 ns 7581495.5 ns 1.00
Conv((3, 3), 3 => 3)/cpu/forward/Flux/(64, 64, 3, 128) 9887620.5 ns 9955016 ns 0.99
Conv((3, 3), 3 => 3)/cpu/forward/SimpleChains/(64, 64, 3, 128) 6379262 ns 6388090.5 ns 1.00
vgg16/cpu/reverse/Zygote/(32, 32, 3, 16) 680001467 ns 725603932 ns 0.94
vgg16/cpu/reverse/Zygote/(32, 32, 3, 64) 2574834317 ns 2550320476 ns 1.01
vgg16/cpu/reverse/Zygote/(32, 32, 3, 2) 133556588.5 ns 138974187 ns 0.96
vgg16/cpu/reverse/Tracker/(32, 32, 3, 16) 832059382 ns 803301267 ns 1.04
vgg16/cpu/reverse/Tracker/(32, 32, 3, 64) 2940015627 ns 2816236020 ns 1.04
vgg16/cpu/reverse/Tracker/(32, 32, 3, 2) 219247861 ns 235878850.5 ns 0.93
vgg16/cpu/reverse/Flux/(32, 32, 3, 16) 712943058.5 ns 650791127 ns 1.10
vgg16/cpu/reverse/Flux/(32, 32, 3, 64) 2615778063 ns 2603380595.5 ns 1.00
vgg16/cpu/reverse/Flux/(32, 32, 3, 2) 129342095 ns 135543456 ns 0.95
vgg16/cpu/forward/NamedTuple/(32, 32, 3, 16) 175664907 ns 175275149 ns 1.00
vgg16/cpu/forward/NamedTuple/(32, 32, 3, 64) 664305664.5 ns 658937990.5 ns 1.01
vgg16/cpu/forward/NamedTuple/(32, 32, 3, 2) 45501101 ns 35195315 ns 1.29
vgg16/cpu/forward/ComponentArray/(32, 32, 3, 16) 165819611 ns 166640478.5 ns 1.00
vgg16/cpu/forward/ComponentArray/(32, 32, 3, 64) 651918836 ns 647602016 ns 1.01
vgg16/cpu/forward/ComponentArray/(32, 32, 3, 2) 30062079 ns 30477927 ns 0.99
vgg16/cpu/forward/Flux/(32, 32, 3, 16) 186124645.5 ns 186029467 ns 1.00
vgg16/cpu/forward/Flux/(32, 32, 3, 64) 724968189.5 ns 718031730.5 ns 1.01
vgg16/cpu/forward/Flux/(32, 32, 3, 2) 36094692 ns 38445179.5 ns 0.94
Conv((3, 3), 64 => 64)/cpu/reverse/ReverseDiff (compiled)/(64, 64, 64, 128) 1304162516.5 ns 1233631654 ns 1.06
Conv((3, 3), 64 => 64)/cpu/reverse/Zygote/(64, 64, 64, 128) 1868331598 ns 1887704692.5 ns 0.99
Conv((3, 3), 64 => 64)/cpu/reverse/Tracker/(64, 64, 64, 128) 2245902418 ns 2231708206 ns 1.01
Conv((3, 3), 64 => 64)/cpu/reverse/ReverseDiff/(64, 64, 64, 128) 2495872396 ns 2471164221 ns 1.01
Conv((3, 3), 64 => 64)/cpu/reverse/Flux/(64, 64, 64, 128) 1950101995.5 ns 1848957394.5 ns 1.05
Conv((3, 3), 64 => 64)/cpu/reverse/Enzyme/(64, 64, 64, 128) 2157665806 ns 1990760702 ns 1.08
Conv((3, 3), 64 => 64)/cpu/forward/NamedTuple/(64, 64, 64, 128) 330073872 ns 333383541 ns 0.99
Conv((3, 3), 64 => 64)/cpu/forward/ComponentArray/(64, 64, 64, 128) 327700026 ns 330228531 ns 0.99
Conv((3, 3), 64 => 64)/cpu/forward/Flux/(64, 64, 64, 128) 458706730 ns 365192205 ns 1.26
Conv((3, 3), 1 => 1)/cpu/reverse/ReverseDiff (compiled)/(64, 64, 1, 128) 11936295 ns 11909407 ns 1.00
Conv((3, 3), 1 => 1)/cpu/reverse/Zygote/(64, 64, 1, 128) 17958576 ns 18028312 ns 1.00
Conv((3, 3), 1 => 1)/cpu/reverse/Tracker/(64, 64, 1, 128) 19018839.5 ns 19050490 ns 1.00
Conv((3, 3), 1 => 1)/cpu/reverse/ReverseDiff/(64, 64, 1, 128) 23737989 ns 23803362 ns 1.00
Conv((3, 3), 1 => 1)/cpu/reverse/Flux/(64, 64, 1, 128) 17752292.5 ns 17829963 ns 1.00
Conv((3, 3), 1 => 1)/cpu/reverse/SimpleChains/(64, 64, 1, 128) 1158352 ns 1163076.5 ns 1.00
Conv((3, 3), 1 => 1)/cpu/reverse/Enzyme/(64, 64, 1, 128) 22921221 ns 22925657 ns 1.00
Conv((3, 3), 1 => 1)/cpu/forward/NamedTuple/(64, 64, 1, 128) 2438307 ns 2329735 ns 1.05
Conv((3, 3), 1 => 1)/cpu/forward/ComponentArray/(64, 64, 1, 128) 2207289 ns 2215453 ns 1.00
Conv((3, 3), 1 => 1)/cpu/forward/Flux/(64, 64, 1, 128) 2059552 ns 2068279 ns 1.00
Conv((3, 3), 1 => 1)/cpu/forward/SimpleChains/(64, 64, 1, 128) 193042 ns 200525.5 ns 0.96
Dense(200 => 200)/cpu/reverse/ReverseDiff (compiled)/(200, 128) 290375 ns 293690.5 ns 0.99
Dense(200 => 200)/cpu/reverse/Zygote/(200, 128) 264375 ns 267291 ns 0.99
Dense(200 => 200)/cpu/reverse/Tracker/(200, 128) 365440 ns 369002 ns 0.99
Dense(200 => 200)/cpu/reverse/ReverseDiff/(200, 128) 409918 ns 409669 ns 1.00
Dense(200 => 200)/cpu/reverse/Flux/(200, 128) 280316 ns 275557 ns 1.02
Dense(200 => 200)/cpu/reverse/SimpleChains/(200, 128) 405440 ns 407905 ns 0.99
Dense(200 => 200)/cpu/reverse/Enzyme/(200, 128) 396112 ns 397495 ns 1.00
Dense(200 => 200)/cpu/forward/NamedTuple/(200, 128) 81703 ns 81824 ns 1.00
Dense(200 => 200)/cpu/forward/ComponentArray/(200, 128) 83426 ns 82175 ns 1.02
Dense(200 => 200)/cpu/forward/Flux/(200, 128) 88967 ns 87213 ns 1.02
Dense(200 => 200)/cpu/forward/SimpleChains/(200, 128) 104306 ns 104706 ns 1.00
Conv((3, 3), 16 => 16)/cpu/reverse/ReverseDiff (compiled)/(64, 64, 16, 128) 199911623 ns 193496448 ns 1.03
Conv((3, 3), 16 => 16)/cpu/reverse/Zygote/(64, 64, 16, 128) 330582216.5 ns 330361771 ns 1.00
Conv((3, 3), 16 => 16)/cpu/reverse/Tracker/(64, 64, 16, 128) 431440040 ns 425098445.5 ns 1.01
Conv((3, 3), 16 => 16)/cpu/reverse/ReverseDiff/(64, 64, 16, 128) 483202494 ns 494104527 ns 0.98
Conv((3, 3), 16 => 16)/cpu/reverse/Flux/(64, 64, 16, 128) 390600061 ns 375779602 ns 1.04
Conv((3, 3), 16 => 16)/cpu/reverse/SimpleChains/(64, 64, 16, 128) 329500710.5 ns 317049031.5 ns 1.04
Conv((3, 3), 16 => 16)/cpu/reverse/Enzyme/(64, 64, 16, 128) 476877423.5 ns 471078717.5 ns 1.01
Conv((3, 3), 16 => 16)/cpu/forward/NamedTuple/(64, 64, 16, 128) 47470381 ns 47539749 ns 1.00
Conv((3, 3), 16 => 16)/cpu/forward/ComponentArray/(64, 64, 16, 128) 46946569 ns 46905764 ns 1.00
Conv((3, 3), 16 => 16)/cpu/forward/Flux/(64, 64, 16, 128) 60012110 ns 56265522 ns 1.07
Conv((3, 3), 16 => 16)/cpu/forward/SimpleChains/(64, 64, 16, 128) 27667373.5 ns 28878315 ns 0.96
Dense(2000 => 2000)/cpu/reverse/ReverseDiff (compiled)/(2000, 128) 19236897 ns 19552999 ns 0.98
Dense(2000 => 2000)/cpu/reverse/Zygote/(2000, 128) 19653638 ns 19606842 ns 1.00
Dense(2000 => 2000)/cpu/reverse/Tracker/(2000, 128) 23866747 ns 23577960 ns 1.01
Dense(2000 => 2000)/cpu/reverse/ReverseDiff/(2000, 128) 24431304 ns 24212297 ns 1.01
Dense(2000 => 2000)/cpu/reverse/Flux/(2000, 128) 19723207.5 ns 19666577 ns 1.00
Dense(2000 => 2000)/cpu/reverse/Enzyme/(2000, 128) 21004501 ns 21011715 ns 1.00
Dense(2000 => 2000)/cpu/forward/NamedTuple/(2000, 128) 6534005 ns 6537708 ns 1.00
Dense(2000 => 2000)/cpu/forward/ComponentArray/(2000, 128) 6514002 ns 6511419 ns 1.00
Dense(2000 => 2000)/cpu/forward/Flux/(2000, 128) 6501944.5 ns 6527700 ns 1.00

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.