From b9546c8b487364cd998b3ad87d873dc3ef35ac43 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 29 Oct 2019 11:24:41 -0400 Subject: [PATCH] make allocated macro more reliable Changes testset to avoid compiler heuristics (copyast) that disables inference. And changes the allocated macro to rely less on inference to elid allocations for the machinary itself. --- base/util.jl | 34 +++++++++++----------------------- src/gc.c | 17 +++++++++++++---- src/julia.h | 3 --- src/julia_internal.h | 1 + stdlib/Test/src/Test.jl | 4 ++-- test/core.jl | 8 +++++--- test/offsetarray.jl | 8 ++++---- test/ranges.jl | 2 -- test/syntax.jl | 3 +-- 9 files changed, 37 insertions(+), 43 deletions(-) diff --git a/base/util.jl b/base/util.jl index 986a7e55b2e25..fedd25fe516c1 100644 --- a/base/util.jl +++ b/base/util.jl @@ -60,9 +60,6 @@ end # total time spend in garbage collection, in nanoseconds gc_time_ns() = ccall(:jl_gc_total_hrtime, UInt64, ()) -# total number of bytes allocated so far -gc_bytes() = ccall(:jl_gc_total_bytes, Int64, ()) - # print elapsed time, return expression value const _mem_units = ["byte", "KiB", "MiB", "GiB", "TiB", "PiB"] const _cnt_units = ["", " k", " M", " G", " T", " P"] @@ -223,21 +220,14 @@ macro elapsed(ex) end end -# measure bytes allocated without *most* contamination from compilation -# Note: This reports a different value from the @time macros, because -# it wraps the call in a function, however, this means that things -# like: @allocated y = foo() -# will not work correctly, because it will set y in the context of -# the local function made by the macro, not the current function +# total number of bytes allocated so far +gc_bytes(b::Ref{Int64}) = ccall(:jl_gc_get_total_bytes, Cvoid, (Ptr{Int64},), b) + """ @allocated A macro to evaluate an expression, discarding the resulting value, instead returning the -total number of bytes allocated during evaluation of the expression. Note: the expression is -evaluated inside a local function, instead of the current context, in order to eliminate the -effects of compilation, however, there still may be some allocations due to JIT compilation. -This also makes the results inconsistent with the `@time` macros, which do not try to adjust -for the effects of compilation. +total number of bytes allocated during evaluation of the expression. See also [`@time`](@ref), [`@timev`](@ref), [`@timed`](@ref), and [`@elapsed`](@ref). @@ -249,15 +239,13 @@ julia> @allocated rand(10^6) """ macro allocated(ex) quote - let - local f - function f() - b0 = gc_bytes() - $(esc(ex)) - gc_bytes() - b0 - end - f() - end + while false; end # compiler heuristic: compile this block (alter this if the heuristic changes) + local b0 = Ref{Int64}(0) + local b1 = Ref{Int64}(0) + gc_bytes(b0) + $(esc(ex)) + gc_bytes(b1) + b1[] - b0[] end end diff --git a/src/gc.c b/src/gc.c index 4f28af0e12734..083e38cf49c3e 100644 --- a/src/gc.c +++ b/src/gc.c @@ -2625,23 +2625,26 @@ JL_DLLEXPORT int jl_gc_enable(int on) } return prev; } + JL_DLLEXPORT int jl_gc_is_enabled(void) { jl_ptls_t ptls = jl_get_ptls_states(); return !ptls->disable_gc; } -JL_DLLEXPORT int64_t jl_gc_total_bytes(void) +JL_DLLEXPORT void jl_gc_get_total_bytes(int64_t *bytes) { jl_gc_num_t num = gc_num; combine_thread_gc_counts(&num); // Sync this logic with `base/util.jl:GC_Diff` - return (num.total_allocd + num.deferred_alloc + num.allocd); + *bytes = (num.total_allocd + num.deferred_alloc + num.allocd); } + JL_DLLEXPORT uint64_t jl_gc_total_hrtime(void) { return gc_num.total_time; } + JL_DLLEXPORT jl_gc_num_t jl_gc_num(void) { jl_gc_num_t num = gc_num; @@ -2649,14 +2652,20 @@ JL_DLLEXPORT jl_gc_num_t jl_gc_num(void) return num; } +// TODO: these were supposed to be thread local JL_DLLEXPORT int64_t jl_gc_diff_total_bytes(void) { int64_t oldtb = last_gc_total_bytes; - int64_t newtb = jl_gc_total_bytes(); + int64_t newtb; + jl_gc_get_total_bytes(&newtb); last_gc_total_bytes = newtb; return newtb - oldtb; } -void jl_gc_sync_total_bytes(void) {last_gc_total_bytes = jl_gc_total_bytes();} + +void jl_gc_sync_total_bytes(void) +{ + jl_gc_get_total_bytes(&last_gc_total_bytes); +} static void jl_gc_premark(jl_ptls_t ptls2) { diff --git a/src/julia.h b/src/julia.h index 64c2d5456a668..1ef3e4f0aeb45 100644 --- a/src/julia.h +++ b/src/julia.h @@ -749,9 +749,6 @@ extern void JL_GC_POP() JL_NOTSAFEPOINT; JL_DLLEXPORT int jl_gc_enable(int on); JL_DLLEXPORT int jl_gc_is_enabled(void); -JL_DLLEXPORT int64_t jl_gc_total_bytes(void); -JL_DLLEXPORT uint64_t jl_gc_total_hrtime(void); -JL_DLLEXPORT int64_t jl_gc_diff_total_bytes(void); typedef enum { JL_GC_AUTO = 0, // use heuristics to determine the collection type diff --git a/src/julia_internal.h b/src/julia_internal.h index 2c914761c6b8a..5fe519467b1ee 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -319,6 +319,7 @@ JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz); JL_DLLEXPORT void JL_NORETURN jl_throw_out_of_memory_error(void); +JL_DLLEXPORT int64_t jl_gc_diff_total_bytes(void); void jl_gc_sync_total_bytes(void); void jl_gc_track_malloced_array(jl_ptls_t ptls, jl_array_t *a) JL_NOTSAFEPOINT; void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT; diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 98c0e8c983e8d..b313950f81b8c 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -1109,7 +1109,7 @@ function testset_beginend(args, tests, source) err isa InterruptException && rethrow() # something in the test block threw an error. Count that as an # error in this test set - record(ts, Error(:nontest_error, :(), err, Base.catch_stack(), $(QuoteNode(source)))) + record(ts, Error(:nontest_error, Expr(:tuple), err, Base.catch_stack(), $(QuoteNode(source)))) finally copy!(RNG, oldrng) end @@ -1182,7 +1182,7 @@ function testset_forloop(args, testloop, source) err isa InterruptException && rethrow() # Something in the test block threw an error. Count that as an # error in this test set - record(ts, Error(:nontest_error, :(), err, Base.catch_stack(), $(QuoteNode(source)))) + record(ts, Error(:nontest_error, Expr(:tuple), err, Base.catch_stack(), $(QuoteNode(source)))) end end quote diff --git a/test/core.jl b/test/core.jl index cefb450ff3853..e92cb2b96eb60 100644 --- a/test/core.jl +++ b/test/core.jl @@ -5019,13 +5019,15 @@ end # when calculating total allocation size. @noinline function f17255(n) GC.enable(false) - b0 = Base.gc_bytes() + b0 = Ref{Int64}(0) + b1 = Ref{Int64}(0) + Base.gc_bytes(b0) local a for i in 1:n a, t, allocd = @timed [Ref(1) for i in 1:1000] @test allocd > 0 - b1 = Base.gc_bytes() - if b1 < b0 + Base.gc_bytes(b1) + if b1[] < b0[] return false, a end end diff --git a/test/offsetarray.jl b/test/offsetarray.jl index 5db5ee24062ff..480bedecf9ed8 100644 --- a/test/offsetarray.jl +++ b/test/offsetarray.jl @@ -533,15 +533,15 @@ end B = OffsetArray(reshape(1:24, 4, 3, 2), -5, 6, -7) for R in (fill(0, -4:-1), fill(0, -4:-1, 7:7), fill(0, -4:-1, 7:7, -6:-6)) @test @inferred(maximum!(R, B)) == reshape(maximum(B, dims=(2,3)), axes(R)) == reshape(21:24, axes(R)) - @test @allocated(maximum!(R, B)) <= 400 + @test @allocated(maximum!(R, B)) <= 800 @test @inferred(minimum!(R, B)) == reshape(minimum(B, dims=(2,3)), axes(R)) == reshape(1:4, axes(R)) - @test @allocated(minimum!(R, B)) <= 400 + @test @allocated(minimum!(R, B)) <= 800 end for R in (fill(0, -4:-4, 7:9), fill(0, -4:-4, 7:9, -6:-6)) @test @inferred(maximum!(R, B)) == reshape(maximum(B, dims=(1,3)), axes(R)) == reshape(16:4:24, axes(R)) - @test @allocated(maximum!(R, B)) <= 400 + @test @allocated(maximum!(R, B)) <= 800 @test @inferred(minimum!(R, B)) == reshape(minimum(B, dims=(1,3)), axes(R)) == reshape(1:4:9, axes(R)) - @test @allocated(minimum!(R, B)) <= 400 + @test @allocated(minimum!(R, B)) <= 800 end @test_throws DimensionMismatch maximum!(fill(0, -4:-1, 7:7, -6:-6, 1:1), B) @test_throws DimensionMismatch minimum!(fill(0, -4:-1, 7:7, -6:-6, 1:1), B) diff --git a/test/ranges.jl b/test/ranges.jl index c765900f6d78b..a2ff3b78da9e7 100644 --- a/test/ranges.jl +++ b/test/ranges.jl @@ -1490,8 +1490,6 @@ end end @testset "allocation of TwicePrecision call" begin - 0:286.493442:360 - 0:286:360 @test @allocated(0:286.493442:360) == 0 @test @allocated(0:286:360) == 0 end diff --git a/test/syntax.jl b/test/syntax.jl index 3b56947627474..3c0ae650b265a 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -1845,8 +1845,7 @@ end @testset "closure conversion in testsets" begin p = (2, 3, 4) @test p == (2, 3, 4) - identity(p) - allocs = @allocated identity(p) + allocs = (() -> @allocated identity(p))() @test allocs == 0 end