From edf0798086f8a31d8934c703f3da558804c11658 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Fri, 6 Oct 2023 21:37:16 +0200 Subject: [PATCH 1/3] Fix `@spawn_or_run_task` with interactive threads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Julia ≥ 1.9 is started with `-tX,Y`, by default tasks use the pool of interactive threads instead of the default pool. Fix this by updating `spawn_or_run_task` to match the code used by `@spawn` in Julia master. --- src/other/utils.jl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/other/utils.jl b/src/other/utils.jl index 455c406f46..4d264340b7 100644 --- a/src/other/utils.jl +++ b/src/other/utils.jl @@ -221,16 +221,22 @@ end Equivalent to `Threads.@spawn` if `threads === true`, otherwise run `expr` and return a `Task` that returns its value. """ -macro spawn_or_run_task(threads, expr) - letargs = Base._lift_one_interp!(expr) +macro spawn_or_run_task(threads, ex) + letargs = Base._lift_one_interp!(ex) - thunk = esc(:(()->($expr))) + thunk = :(()->($(esc(ex)))) + @static if VERSION >= v"1.10.0-DEV" + Base.replace_linenums!(thunk, __source__) + end var = esc(Base.sync_varname) quote let $(letargs...) if $(esc(threads)) local task = Task($thunk) task.sticky = false + if VERSION >= v"1.9.0" + Base.Threads._spawn_set_thrpool(task, :default) + end else # Run expr immediately res = $thunk() From 767276abd42940d271714f90f343a445204edf0c Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Sat, 7 Oct 2023 21:53:26 +0200 Subject: [PATCH 2/3] Fixes --- .github/workflows/ci.yml | 2 +- docs/src/lib/functions.md | 3 ++- src/other/utils.jl | 20 ++++++++++++++------ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4222741a09..5bbb7f0772 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 env: - JULIA_NUM_THREADS: 4 + JULIA_NUM_THREADS: 4,1 - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v1 with: diff --git a/docs/src/lib/functions.md b/docs/src/lib/functions.md index 625cf60665..1cb8dfd66c 100644 --- a/docs/src/lib/functions.md +++ b/docs/src/lib/functions.md @@ -7,7 +7,8 @@ CurrentModule = DataFrames ## Multithreading support By default, selected operations in DataFrames.jl automatically use multiple threads -when available. It is task-based and implemented using the `@spawn` macro from Julia Base. +when available. Multi-threading is task-based and implemented using the `@spawn` +macro from Julia Base. Tasks are therefore scheduled on the `:default` threadpool. Functions that take user-defined functions and may run it in parallel accept a `threads` keyword argument which allows disabling multithreading when the provided function requires serial execution or is not thread-safe. diff --git a/src/other/utils.jl b/src/other/utils.jl index 4d264340b7..78f91785b1 100644 --- a/src/other/utils.jl +++ b/src/other/utils.jl @@ -229,14 +229,15 @@ macro spawn_or_run_task(threads, ex) Base.replace_linenums!(thunk, __source__) end var = esc(Base.sync_varname) + spawn_set_thrpool = VERSION >= v"1.9.0" ? + :(Base.Threads._spawn_set_thrpool(task, :default)) : + :() quote let $(letargs...) if $(esc(threads)) local task = Task($thunk) task.sticky = false - if VERSION >= v"1.9.0" - Base.Threads._spawn_set_thrpool(task, :default) - end + $(spawn_set_thrpool) else # Run expr immediately res = $thunk() @@ -259,16 +260,23 @@ end Equivalent to `Threads.@spawn` if `threads === true`, otherwise run `expr`. """ -macro spawn_or_run(threads, expr) - letargs = Base._lift_one_interp!(expr) +macro spawn_or_run(threads, ex) + letargs = Base._lift_one_interp!(ex) - thunk = esc(:(()->($expr))) + thunk = :(()->($(esc(ex)))) + if VERSION >= v"1.10.0-DEV" + Base.replace_linenums!(thunk, __source__) + end var = esc(Base.sync_varname) + spawn_set_thrpool = VERSION >= v"1.9.0" ? + :(Base.Threads._spawn_set_thrpool(task, :default)) : + :() quote let $(letargs...) if $(esc(threads)) local task = Task($thunk) task.sticky = false + $(spawn_set_thrpool) if $(Expr(:islocal, var)) put!($var, task) end From 2b299c9a223f24c63dbdddb44c4bb96c4e0f196d Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Tue, 10 Oct 2023 11:15:50 +0200 Subject: [PATCH 3/3] Update NEWS.md --- NEWS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NEWS.md b/NEWS.md index 468c3656e7..8a46730dac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -10,6 +10,13 @@ keyword argument ([#3380](https://github.com/JuliaData/DataFrames.jl/pull/3380)) +## Bug fixes + +* Always use the default thread pool for multithreaded operations, + instead of using the interactive thread pool when Julia was started + with `-tM,N` with N > 0 + ([#3385](https://github.com/JuliaData/DataFrames.jl/pull/3385)) + # DataFrames.jl v1.6.1 Release Notes ## Bug fixes