From 90a029a1c92414b9b15687ae83ea467921d75b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Fri, 7 Jun 2024 18:27:08 +0200 Subject: [PATCH 1/8] Try to make CI faster --- .github/workflows/CI.yml | 2 ++ test/runtests.jl | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8eafd93..5e578d2 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -8,6 +8,8 @@ on: jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} + env: + JULIA_NUM_THREADS: 2 runs-on: ${{ matrix.os }} strategy: fail-fast: false diff --git a/test/runtests.jl b/test/runtests.jl index a33e5b1..e4dac24 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -23,7 +23,7 @@ const url = "http://127.0.0.1:$port" workload() = @async begin for _ in 1:1000 if done[] return end - InteractiveUtils.peakflops() + InteractiveUtils.peakflops(1024, ntrials=1) yield() # yield to allow the tests to run end end From 3bc3fe4d3b346e063f637328b65362432667120d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Fri, 7 Jun 2024 18:36:20 +0200 Subject: [PATCH 2/8] . --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index e4dac24..c8a68a0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -23,7 +23,7 @@ const url = "http://127.0.0.1:$port" workload() = @async begin for _ in 1:1000 if done[] return end - InteractiveUtils.peakflops(1024, ntrials=1) + InteractiveUtils.peakflops(1024) yield() # yield to allow the tests to run end end From cfb5841be7dcd5e6e4315cc9a40cb59b8ac113fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Fri, 7 Jun 2024 18:50:26 +0200 Subject: [PATCH 3/8] . --- test/runtests.jl | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index c8a68a0..cc5a9b2 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -21,8 +21,7 @@ const url = "http://127.0.0.1:$port" done = Threads.Atomic{Bool}(false) # Schedule some work that's known to be expensive, to profile it workload() = @async begin - for _ in 1:1000 - if done[] return end + while !done[] InteractiveUtils.peakflops(1024) yield() # yield to allow the tests to run end @@ -86,6 +85,7 @@ const url = "http://127.0.0.1:$port" fname = read(IOBuffer(req.body), String) @info "filename: $fname" @test isfile(fname) + done[] = true end @testset "debug endpoint cpu profile start/end" begin @@ -216,8 +216,7 @@ const url = "http://127.0.0.1:$port" done = Threads.Atomic{Bool}(false) # Schedule some work that's known to be expensive, to profile it workload() = @async begin - for _ in 1:200 - if done[] return end + while !done[] global a = [[] for i in 1:1000] yield() # yield to allow the tests to run end From 10e15063030954581cd1dba11ac10200db677c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Fri, 7 Jun 2024 19:56:24 +0200 Subject: [PATCH 4/8] . --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5e578d2..ce6c62b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -21,7 +21,7 @@ jobs: - "nightly" os: - ubuntu-latest - - macOS-latest + - macos-latest - windows-latest arch: - x64 From 3b3f61737b5ef95a21ad3db8ae8c1f893421f5c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Sat, 22 Jun 2024 11:00:08 +0200 Subject: [PATCH 5/8] PR feedback --- .github/workflows/CI.yml | 1 - test/runtests.jl | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ce6c62b..2d3f2c2 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -17,7 +17,6 @@ jobs: version: - "1.6" # LTS - "1" # Latest - - "~1.9.0-0" # To test heap snapshot; remove when v1.9 is latest Julia release. - "nightly" os: - ubuntu-latest diff --git a/test/runtests.jl b/test/runtests.jl index cc5a9b2..af6f9b1 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,7 +20,7 @@ const url = "http://127.0.0.1:$port" @testset "CPU profiling" begin done = Threads.Atomic{Bool}(false) # Schedule some work that's known to be expensive, to profile it - workload() = @async begin + workload() = Threads.@spawn begin while !done[] InteractiveUtils.peakflops(1024) yield() # yield to allow the tests to run @@ -215,7 +215,7 @@ const url = "http://127.0.0.1:$port" @testset "Allocation profiling" begin done = Threads.Atomic{Bool}(false) # Schedule some work that's known to be expensive, to profile it - workload() = @async begin + workload() = Threads.@spawn begin while !done[] global a = [[] for i in 1:1000] yield() # yield to allow the tests to run From 2f7d1e7f4848eac743d29424f6a10febcfa378e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Sun, 23 Jun 2024 10:01:55 +0200 Subject: [PATCH 6/8] Use ReTestItems --- Project.toml | 3 +- test/Project.toml | 2 + test/all-tests.jl | 245 +++++++++++++++++++++++++++++++++++ test/runtests.jl | 317 +--------------------------------------------- 4 files changed, 252 insertions(+), 315 deletions(-) create mode 100644 test/all-tests.jl diff --git a/Project.toml b/Project.toml index 4e88e93..e9f407a 100644 --- a/Project.toml +++ b/Project.toml @@ -17,7 +17,8 @@ PProf = "2, 3" julia = "1.6" [extras] +ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test"] +test = ["ReTestItems", "Test"] diff --git a/test/Project.toml b/test/Project.toml index 896cbca..235adb4 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -4,5 +4,7 @@ InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" PProf = "e4faabce-9ead-11e9-39d9-4379958e3056" Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" +ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823" +Reexport = "189a3867-3050-52da-a836-e630ba90ab69" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/all-tests.jl b/test/all-tests.jl new file mode 100644 index 0000000..31b8885 --- /dev/null +++ b/test/all-tests.jl @@ -0,0 +1,245 @@ +@testsetup module TestSetup + using Reexport + @reexport using ProfileEndpoints + @reexport using Serialization + @reexport using Test + + @reexport import InteractiveUtils + @reexport import HTTP + @reexport import JSON3 + @reexport import Profile + @reexport import PProf + using Random + + export port, server, url, workload + + const port = 13423 + const server = ProfileEndpoints.serve_profiling_server(;port=port) + const url = "http://127.0.0.1:$port" + + # Schedule some work that's known to be expensive, to profile it + function workload() + done = Ref(false) + a = 0 + t = Threads.@spawn begin + while !done[] + InteractiveUtils.peakflops(1024) + global a = [[] for i in 1:1000] + yield() + end + end + return t, done + end +end + +@testitem "CPU profiling" setup=[TestSetup] begin + @testset "profile endpoint" begin + t, done = workload() + req = HTTP.get("$url/profile?duration=3&pprof=false") + @test req.status == 200 + @test length(req.body) > 0 + + data, lidict = deserialize(IOBuffer(req.body)) + # Test that the profile seems like valid profile data + @test data isa Vector{UInt64} + @test lidict isa Dict{UInt64, Vector{Base.StackTraces.StackFrame}} + + @info "Finished `profile` tests, waiting for peakflops workload to finish." + done[] = true + wait(t) # handle errors + end + + @testset "profile_start/stop endpoints" begin + t, done = workload() + req = HTTP.get("$url/profile_start") + @test req.status == 200 + @test String(req.body) == "CPU profiling started." + + sleep(3) # Allow workload to run a while before we stop profiling. + + req = HTTP.get("$url/profile_stop?pprof=false") + @test req.status == 200 + data, lidict = deserialize(IOBuffer(req.body)) + # Test that the profile seems like valid profile data + @test data isa Vector{UInt64} + @test lidict isa Dict{UInt64, Vector{Base.StackTraces.StackFrame}} + + @info "Finished `profile_start/stop` tests, waiting for peakflops workload to finish." + done[] = true + wait(t) # handle errors + + # We retrive data via PProf directly if `pprof=true`; make sure that path's tested. + # This second call to `profile_stop` should still return the profile, even though + # the profiler is already stopped, as it's `profile_start` that calls `clear()`. + req = HTTP.get("$url/profile_stop?pprof=true") + @test req.status == 200 + # Test that there's something here + # TODO: actually parse the profile + data = read(IOBuffer(req.body), String) + @test length(data) > 100 + end + + @testset "debug endpoint cpu profile" begin + t, done = workload() + headers = ["Content-Type" => "application/json"] + payload = JSON3.write(Dict("profile_type" => "cpu_profile")) + req = HTTP.post("$url/debug_engine", headers, payload) + @test req.status == 200 + fname = read(IOBuffer(req.body), String) + @info "filename: $fname" + @test isfile(fname) + done[] = true + end + + @testset "debug endpoint cpu profile start/end" begin + t, done = workload() + # JSON payload should contain profile_type + headers = ["Content-Type" => "application/json"] + payload = JSON3.write(Dict("profile_type" => "cpu_profile_start")) + req = HTTP.post("$url/debug_engine", headers, payload) + @test req.status == 200 + @test String(req.body) == "CPU profiling started." + + sleep(3) # Allow workload to run a while before we stop profiling. + + payload = JSON3.write(Dict("profile_type" => "cpu_profile_stop")) + req = HTTP.post("$url/debug_engine", headers, payload) + @test req.status == 200 + fname = read(IOBuffer(req.body), String) + @info "filename: $fname" + @test isfile(fname) + + @info "Finished `debug profile_start/stop` tests, waiting for peakflops workload to finish." + done[] = true + wait(t) # handle errors + + # We retrive data via PProf directly if `pprof=true`; make sure that path's tested. + # This second call to `profile_stop` should still return the profile, even though + # the profiler is already stopped, as it's `profile_start` that calls `clear()`. + payload = JSON3.write(Dict("profile_type" => "cpu_profile_stop", "pprof" => "true")) + req = HTTP.post("$url/debug_engine", headers, payload) + @test req.status == 200 + # Test that there's something here + # TODO: actually parse the profile + fname = read(IOBuffer(req.body), String) + @info "filename: $fname" + @test isfile(fname) + end + + @testset "Debug endpoint task backtraces" begin + @static if VERSION >= v"1.10.0-DEV.0" + headers = ["Content-Type" => "application/json"] + payload = JSON3.write(Dict("profile_type" => "task_backtraces")) + req = HTTP.post("$url/debug_engine", headers, payload) + @test req.status == 200 + fname = read(IOBuffer(req.body), String) + @info "filename: $fname" + @test isfile(fname) + end + end +end + +@testitem "Heap snapshot" setup=[TestSetup] begin + @testset "Heap snapshot $query" for query in ("", "?all_one=true") + req = HTTP.get("$url/heap_snapshot$query", retry=false, status_exception=false) + if !isdefined(Profile, :take_heap_snapshot) + # Assert the version is before https://github.com/JuliaLang/julia/pull/46862 + # Although we actually also need https://github.com/JuliaLang/julia/pull/47300 + @assert VERSION < v"1.9.0-DEV.1643" + @test req.status == 501 # not implemented + else + @test req.status == 200 + data = read(IOBuffer(req.body), String) + # Test that there's something here + # TODO: actually parse the profile + @test length(data) > 100 + end + end +end + +@testitem "Allocation profiling" setup=[TestSetup] begin + @testset "allocs_profile endpoint" begin + t, done = workload() + req = HTTP.get("$url/allocs_profile?duration=3", retry=false, status_exception=false) + if !(isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs)) + @assert VERSION < v"1.8.0-DEV.1346" + @test req.status == 501 # not implemented + else + @test req.status == 200 + @test length(req.body) > 0 + + data = read(IOBuffer(req.body), String) + # Test that there's something here + # TODO: actually parse the profile + @test length(data) > 100 + end + @info "Finished `allocs_profile` tests, waiting for workload to finish." + done[] = true + wait(t) # handle errors + end + + @testset "allocs_profile_start/stop endpoints" begin + t, done = workload() + req = HTTP.get("$url/allocs_profile_start", retry=false, status_exception=false) + if !(isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs)) + @assert VERSION < v"1.8.0-DEV.1346" + @test req.status == 501 # not implemented + else + @test req.status == 200 + @test String(req.body) == "Allocation profiling started." + end + + sleep(3) # Allow workload to run a while before we stop profiling. + + req = HTTP.get("$url/allocs_profile_stop", retry=false, status_exception=false) + if !(isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs)) + @assert VERSION < v"1.8.0-DEV.1346" + @test req.status == 501 # not implemented + else + @test req.status == 200 + data = read(IOBuffer(req.body), String) + # Test that there's something here + # TODO: actually parse the profile + @test length(data) > 100 + end + @info "Finished `allocs_profile_stop` tests, waiting for workload to finish." + done[] = true + wait(t) # handle errors + end +end + +@testitem "task backtraces" setup=[TestSetup] begin + @testset "task_backtraces endpoint" begin + @static if VERSION >= v"1.10.0-DEV.0" + req = HTTP.get("$url/task_backtraces", retry=false, status_exception=false) + @test req.status == 200 + @test length(req.body) > 0 + + # Test whether the profile returned a valid file + data = read(IOBuffer(req.body), String) + @test isfile(data) + end + end +end + +@testitem "error handling" setup=[TestSetup] begin + let res = HTTP.get("$url/profile", status_exception=false) + @test 400 <= res.status < 500 + @test res.status != 404 + # Make sure we describe how to use the endpoint + body = String(res.body) + @test occursin("duration", body) + @test occursin("delay", body) + end + + if (isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs)) + let res = HTTP.get("$url/allocs_profile", status_exception=false) + @test 400 <= res.status < 500 + @test res.status != 404 + # Make sure we describe how to use the endpoint + body = String(res.body) + @test occursin("duration", body) + @test occursin("sample_rate", body) + end + end +end diff --git a/test/runtests.jl b/test/runtests.jl index af6f9b1..39442de 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,316 +1,5 @@ -module ProfileEndpointsTests +using ReTestItems, ProfileEndpoints -using ProfileEndpoints -using Serialization -using Test - -import InteractiveUtils -import HTTP -import JSON3 -import Profile -import PProf - -const port = 13423 -const stage_path = tempdir() -const server = ProfileEndpoints.serve_profiling_server(;port=port, stage_path=stage_path) -const url = "http://127.0.0.1:$port" - -@testset "ProfileEndpoints.jl" begin - - @testset "CPU profiling" begin - done = Threads.Atomic{Bool}(false) - # Schedule some work that's known to be expensive, to profile it - workload() = Threads.@spawn begin - while !done[] - InteractiveUtils.peakflops(1024) - yield() # yield to allow the tests to run - end - end - - @testset "profile endpoint" begin - done[] = false - t = workload() - req = HTTP.get("$url/profile?duration=3&pprof=false") - @test req.status == 200 - @test length(req.body) > 0 - - data, lidict = deserialize(IOBuffer(req.body)) - # Test that the profile seems like valid profile data - @test data isa Vector{UInt64} - @test lidict isa Dict{UInt64, Vector{Base.StackTraces.StackFrame}} - - @info "Finished `profile` tests, waiting for peakflops workload to finish." - done[] = true - wait(t) # handle errors - end - - @testset "profile_start/stop endpoints" begin - done[] = false - t = workload() - req = HTTP.get("$url/profile_start") - @test req.status == 200 - @test String(req.body) == "CPU profiling started." - - sleep(3) # Allow workload to run a while before we stop profiling. - - req = HTTP.get("$url/profile_stop?pprof=false") - @test req.status == 200 - data, lidict = deserialize(IOBuffer(req.body)) - # Test that the profile seems like valid profile data - @test data isa Vector{UInt64} - @test lidict isa Dict{UInt64, Vector{Base.StackTraces.StackFrame}} - - @info "Finished `profile_start/stop` tests, waiting for peakflops workload to finish." - done[] = true - wait(t) # handle errors - - # We retrive data via PProf directly if `pprof=true`; make sure that path's tested. - # This second call to `profile_stop` should still return the profile, even though - # the profiler is already stopped, as it's `profile_start` that calls `clear()`. - req = HTTP.get("$url/profile_stop?pprof=true") - @test req.status == 200 - # Test that there's something here - # TODO: actually parse the profile - data = read(IOBuffer(req.body), String) - @test length(data) > 100 - end - - @testset "debug endpoint cpu profile" begin - done[] = false - t = workload() - headers = ["Content-Type" => "application/json"] - payload = JSON3.write(Dict("profile_type" => "cpu_profile")) - req = HTTP.post("$url/debug_engine", headers, payload) - @test req.status == 200 - fname = read(IOBuffer(req.body), String) - @info "filename: $fname" - @test isfile(fname) - done[] = true - end - - @testset "debug endpoint cpu profile start/end" begin - done[] = false - t = workload() - # JSON payload should contain profile_type - headers = ["Content-Type" => "application/json"] - payload = JSON3.write(Dict("profile_type" => "cpu_profile_start")) - req = HTTP.post("$url/debug_engine", headers, payload) - @test req.status == 200 - @test String(req.body) == "CPU profiling started." - - sleep(3) # Allow workload to run a while before we stop profiling. - - payload = JSON3.write(Dict("profile_type" => "cpu_profile_stop")) - req = HTTP.post("$url/debug_engine", headers, payload) - @test req.status == 200 - fname = read(IOBuffer(req.body), String) - @info "filename: $fname" - @test isfile(fname) - - @info "Finished `debug profile_start/stop` tests, waiting for peakflops workload to finish." - done[] = true - wait(t) # handle errors - - # We retrive data via PProf directly if `pprof=true`; make sure that path's tested. - # This second call to `profile_stop` should still return the profile, even though - # the profiler is already stopped, as it's `profile_start` that calls `clear()`. - payload = JSON3.write(Dict("profile_type" => "cpu_profile_stop", "pprof" => "true")) - req = HTTP.post("$url/debug_engine", headers, payload) - @test req.status == 200 - # Test that there's something here - # TODO: actually parse the profile - fname = read(IOBuffer(req.body), String) - @info "filename: $fname" - @test isfile(fname) - end - - @testset "Debug endpoint heap snapshot" begin - @static if isdefined(Profile, :take_heap_snapshot) - headers = ["Content-Type" => "application/json"] - payload = JSON3.write(Dict("profile_type" => "heap_snapshot")) - req = HTTP.post("$url/debug_engine", headers, payload) - @test req.status == 200 - fname = read(IOBuffer(req.body), String) - @info "filename: $fname" - @test isfile(fname) - end - end - - @testset "Debug endpoint allocation profile" begin - @static if isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs) - headers = ["Content-Type" => "application/json"] - payload = JSON3.write(Dict("profile_type" => "allocs_profile")) - req = HTTP.post("$url/debug_engine", headers, payload) - @test req.status == 200 - fname = read(IOBuffer(req.body), String) - @info "filename: $fname" - @test isfile(fname) - end - end - - @testset "Debug endpoint allocation profile start/stop" begin - @static if isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs) - headers = ["Content-Type" => "application/json"] - payload = JSON3.write(Dict("profile_type" => "allocs_profile_start")) - req = HTTP.post("$url/debug_engine", headers, payload) - @test req.status == 200 - @test String(req.body) == "Allocation profiling started." - - sleep(3) # Allow workload to run a while before we stop profiling. - - payload = JSON3.write(Dict("profile_type" => "allocs_profile_stop")) - req = HTTP.post("$url/debug_engine", headers, payload) - @test req.status == 200 - fname = read(IOBuffer(req.body), String) - @info "filename: $fname" - @test isfile(fname) - end - end - - @testset "Debug endpoint task backtraces" begin - @static if VERSION >= v"1.10.0-DEV.0" - headers = ["Content-Type" => "application/json"] - payload = JSON3.write(Dict("profile_type" => "task_backtraces")) - req = HTTP.post("$url/debug_engine", headers, payload) - @test req.status == 200 - fname = read(IOBuffer(req.body), String) - @info "filename: $fname" - @test isfile(fname) - end - end - - @testset "Debug endpoint task backtraces with subdir" begin - @static if VERSION >= v"1.10.0-DEV.0" - headers = ["Content-Type" => "application/json"] - payload = JSON3.write(Dict("profile_type" => "task_backtraces")) - if !isdir(joinpath(stage_path, "foo")) - mkdir(joinpath(stage_path, "foo")) - end - req = HTTP.post("$url/debug_engine?subdir=foo", headers, payload) - @test req.status == 200 - fname = read(IOBuffer(req.body), String) - @info "filename: $fname" - @test isfile(fname) - @test occursin("foo", fname) - end - end - end - - @testset "Heap snapshot $query" for query in ("", "?all_one=true") - req = HTTP.get("$url/heap_snapshot$query", retry=false, status_exception=false) - if !isdefined(Profile, :take_heap_snapshot) - # Assert the version is before https://github.com/JuliaLang/julia/pull/46862 - # Although we actually also need https://github.com/JuliaLang/julia/pull/47300 - @assert VERSION < v"1.9.0-DEV.1643" - @test req.status == 501 # not implemented - else - @test req.status == 200 - data = read(IOBuffer(req.body), String) - # Test that there's something here - # TODO: actually parse the profile - @test length(data) > 100 - end - end - - @testset "Allocation profiling" begin - done = Threads.Atomic{Bool}(false) - # Schedule some work that's known to be expensive, to profile it - workload() = Threads.@spawn begin - while !done[] - global a = [[] for i in 1:1000] - yield() # yield to allow the tests to run - end - end - - @testset "allocs_profile endpoint" begin - done[] = false - t = workload() - req = HTTP.get("$url/allocs_profile?duration=3", retry=false, status_exception=false) - if !(isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs)) - @assert VERSION < v"1.8.0-DEV.1346" - @test req.status == 501 # not implemented - else - @test req.status == 200 - @test length(req.body) > 0 - - data = read(IOBuffer(req.body), String) - # Test that there's something here - # TODO: actually parse the profile - @test length(data) > 100 - end - @info "Finished `allocs_profile` tests, waiting for workload to finish." - done[] = true - wait(t) # handle errors - end - - @testset "allocs_profile_start/stop endpoints" begin - done[] = false - t = workload() - req = HTTP.get("$url/allocs_profile_start", retry=false, status_exception=false) - if !(isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs)) - @assert VERSION < v"1.8.0-DEV.1346" - @test req.status == 501 # not implemented - else - @test req.status == 200 - @test String(req.body) == "Allocation profiling started." - end - - sleep(3) # Allow workload to run a while before we stop profiling. - - req = HTTP.get("$url/allocs_profile_stop", retry=false, status_exception=false) - if !(isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs)) - @assert VERSION < v"1.8.0-DEV.1346" - @test req.status == 501 # not implemented - else - @test req.status == 200 - data = read(IOBuffer(req.body), String) - # Test that there's something here - # TODO: actually parse the profile - @test length(data) > 100 - end - @info "Finished `allocs_profile_stop` tests, waiting for workload to finish." - done[] = true - wait(t) # handle errors - end - end - - @testset "task backtraces" begin - @testset "task_backtraces endpoint" begin - @static if VERSION >= v"1.10.0-DEV.0" - req = HTTP.get("$url/task_backtraces", retry=false, status_exception=false) - @test req.status == 200 - @test length(req.body) > 0 - - # Test whether the profile returned a valid file - data = read(IOBuffer(req.body), String) - @test isfile(data) - end - end - end - - @testset "error handling" begin - let res = HTTP.get("$url/profile", status_exception=false) - @test 400 <= res.status < 500 - @test res.status != 404 - # Make sure we describe how to use the endpoint - body = String(res.body) - @test occursin("duration", body) - @test occursin("delay", body) - end - - if (isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs)) - let res = HTTP.get("$url/allocs_profile", status_exception=false) - @test 400 <= res.status < 500 - @test res.status != 404 - # Make sure we describe how to use the endpoint - body = String(res.body) - @test occursin("duration", body) - @test occursin("sample_rate", body) - end - end - end +withenv("OPENBLAS_NUM_THREADS" => "1") do + runtests(ProfileEndpoints, nworker_threads=2, nworkers=1, testitem_timeout=300) end - -close(server) - -end # module ProfileEndpointsTests From 394c75de7c6c7de58fc19c063fde8bfb09d73a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Sun, 23 Jun 2024 10:38:17 +0200 Subject: [PATCH 7/8] Sync with main --- test/Project.toml | 1 - test/all-tests.jl | 110 +++++++++++++++++++++++++++++++++++----------- test/runtests.jl | 4 +- 3 files changed, 85 insertions(+), 30 deletions(-) diff --git a/test/Project.toml b/test/Project.toml index 235adb4..a009c05 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,6 +1,5 @@ [deps] HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" -InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" PProf = "e4faabce-9ead-11e9-39d9-4379958e3056" Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" diff --git a/test/all-tests.jl b/test/all-tests.jl index 31b8885..195714e 100644 --- a/test/all-tests.jl +++ b/test/all-tests.jl @@ -4,37 +4,34 @@ @reexport using Serialization @reexport using Test - @reexport import InteractiveUtils @reexport import HTTP @reexport import JSON3 @reexport import Profile @reexport import PProf - using Random - export port, server, url, workload + export port, stage_path, server, url, workload const port = 13423 - const server = ProfileEndpoints.serve_profiling_server(;port=port) + const stage_path = tempdir() + const server = ProfileEndpoints.serve_profiling_server(;port=port, stage_path=stage_path) const url = "http://127.0.0.1:$port" # Schedule some work that's known to be expensive, to profile it function workload() - done = Ref(false) - a = 0 + shouldstop = Ref(false) t = Threads.@spawn begin - while !done[] - InteractiveUtils.peakflops(1024) + while !shouldstop[] global a = [[] for i in 1:1000] yield() end end - return t, done + return t, shouldstop end end @testitem "CPU profiling" setup=[TestSetup] begin - @testset "profile endpoint" begin - t, done = workload() + @time @testset "profile endpoint" begin + t, shouldstop = workload() req = HTTP.get("$url/profile?duration=3&pprof=false") @test req.status == 200 @test length(req.body) > 0 @@ -45,12 +42,12 @@ end @test lidict isa Dict{UInt64, Vector{Base.StackTraces.StackFrame}} @info "Finished `profile` tests, waiting for peakflops workload to finish." - done[] = true + shouldstop[] = true wait(t) # handle errors end - @testset "profile_start/stop endpoints" begin - t, done = workload() + @time @testset "profile_start/stop endpoints" begin + t, shouldstop = workload() req = HTTP.get("$url/profile_start") @test req.status == 200 @test String(req.body) == "CPU profiling started." @@ -65,7 +62,7 @@ end @test lidict isa Dict{UInt64, Vector{Base.StackTraces.StackFrame}} @info "Finished `profile_start/stop` tests, waiting for peakflops workload to finish." - done[] = true + shouldstop[] = true wait(t) # handle errors # We retrive data via PProf directly if `pprof=true`; make sure that path's tested. @@ -79,8 +76,8 @@ end @test length(data) > 100 end - @testset "debug endpoint cpu profile" begin - t, done = workload() + @time @testset "debug endpoint cpu profile" begin + t, shouldstop = workload() headers = ["Content-Type" => "application/json"] payload = JSON3.write(Dict("profile_type" => "cpu_profile")) req = HTTP.post("$url/debug_engine", headers, payload) @@ -88,11 +85,13 @@ end fname = read(IOBuffer(req.body), String) @info "filename: $fname" @test isfile(fname) - done[] = true + @info "Finished `debug profile` tests, waiting for peakflops workload to finish." + shouldstop[] = true + wait(t) # handle errors end - @testset "debug endpoint cpu profile start/end" begin - t, done = workload() + @time @testset "debug endpoint cpu profile start/end" begin + t, shouldstop = workload() # JSON payload should contain profile_type headers = ["Content-Type" => "application/json"] payload = JSON3.write(Dict("profile_type" => "cpu_profile_start")) @@ -110,7 +109,7 @@ end @test isfile(fname) @info "Finished `debug profile_start/stop` tests, waiting for peakflops workload to finish." - done[] = true + shouldstop[] = true wait(t) # handle errors # We retrive data via PProf directly if `pprof=true`; make sure that path's tested. @@ -126,7 +125,50 @@ end @test isfile(fname) end - @testset "Debug endpoint task backtraces" begin + @time @testset "Debug endpoint heap snapshot" begin + @static if isdefined(Profile, :take_heap_snapshot) + headers = ["Content-Type" => "application/json"] + payload = JSON3.write(Dict("profile_type" => "heap_snapshot")) + req = HTTP.post("$url/debug_engine", headers, payload) + @test req.status == 200 + fname = read(IOBuffer(req.body), String) + @info "filename: $fname" + @test isfile(fname) + end + end + + @time @testset "Debug endpoint allocation profile" begin + @static if isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs) + headers = ["Content-Type" => "application/json"] + payload = JSON3.write(Dict("profile_type" => "allocs_profile")) + req = HTTP.post("$url/debug_engine", headers, payload) + @test req.status == 200 + fname = read(IOBuffer(req.body), String) + @info "filename: $fname" + @test isfile(fname) + end + end + + @time @testset "Debug endpoint allocation profile start/stop" begin + @static if isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs) + headers = ["Content-Type" => "application/json"] + payload = JSON3.write(Dict("profile_type" => "allocs_profile_start")) + req = HTTP.post("$url/debug_engine", headers, payload) + @test req.status == 200 + @test String(req.body) == "Allocation profiling started." + + sleep(3) # Allow workload to run a while before we stop profiling. + + payload = JSON3.write(Dict("profile_type" => "allocs_profile_stop")) + req = HTTP.post("$url/debug_engine", headers, payload) + @test req.status == 200 + fname = read(IOBuffer(req.body), String) + @info "filename: $fname" + @test isfile(fname) + end + end + + @time @testset "Debug endpoint task backtraces" begin @static if VERSION >= v"1.10.0-DEV.0" headers = ["Content-Type" => "application/json"] payload = JSON3.write(Dict("profile_type" => "task_backtraces")) @@ -137,6 +179,22 @@ end @test isfile(fname) end end + + @time @testset "Debug endpoint task backtraces with subdir" begin + @static if VERSION >= v"1.10.0-DEV.0" + headers = ["Content-Type" => "application/json"] + payload = JSON3.write(Dict("profile_type" => "task_backtraces")) + if !isdir(joinpath(stage_path, "foo")) + mkdir(joinpath(stage_path, "foo")) + end + req = HTTP.post("$url/debug_engine?subdir=foo", headers, payload) + @test req.status == 200 + fname = read(IOBuffer(req.body), String) + @info "filename: $fname" + @test isfile(fname) + @test occursin("foo", fname) + end + end end @testitem "Heap snapshot" setup=[TestSetup] begin @@ -159,7 +217,7 @@ end @testitem "Allocation profiling" setup=[TestSetup] begin @testset "allocs_profile endpoint" begin - t, done = workload() + t, shouldstop = workload() req = HTTP.get("$url/allocs_profile?duration=3", retry=false, status_exception=false) if !(isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs)) @assert VERSION < v"1.8.0-DEV.1346" @@ -174,12 +232,12 @@ end @test length(data) > 100 end @info "Finished `allocs_profile` tests, waiting for workload to finish." - done[] = true + shouldstop[] = true wait(t) # handle errors end @testset "allocs_profile_start/stop endpoints" begin - t, done = workload() + t, shouldstop = workload() req = HTTP.get("$url/allocs_profile_start", retry=false, status_exception=false) if !(isdefined(Profile, :Allocs) && isdefined(PProf, :Allocs)) @assert VERSION < v"1.8.0-DEV.1346" @@ -203,7 +261,7 @@ end @test length(data) > 100 end @info "Finished `allocs_profile_stop` tests, waiting for workload to finish." - done[] = true + shouldstop[] = true wait(t) # handle errors end end diff --git a/test/runtests.jl b/test/runtests.jl index 39442de..e2bc518 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,3 @@ using ReTestItems, ProfileEndpoints -withenv("OPENBLAS_NUM_THREADS" => "1") do - runtests(ProfileEndpoints, nworker_threads=2, nworkers=1, testitem_timeout=300) -end +runtests(ProfileEndpoints, nworker_threads=2, nworkers=1, testitem_timeout=300) From 84e92c6cd27f49a388f8fc5379b9d9874e011984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Sun, 23 Jun 2024 10:46:56 +0200 Subject: [PATCH 8/8] Bump min Julia version --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2d3f2c2..d056fc1 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -15,7 +15,7 @@ jobs: fail-fast: false matrix: version: - - "1.6" # LTS + - "1.8" - "1" # Latest - "nightly" os: