Skip to content

Commit

Permalink
Try #641:
Browse files Browse the repository at this point in the history
  • Loading branch information
bors[bot] authored Mar 31, 2023
2 parents 5ff20cc + 6d114b6 commit 34ed14d
Show file tree
Hide file tree
Showing 55 changed files with 154 additions and 160 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ jobs:
-DPIKA_WITH_PRECOMPILED_HEADERS=Off \
-DPIKA_WITH_MALLOC=system \
-DPIKA_WITH_MPI=ON \
-DPIKA_WITH_P2300_REFERENCE_IMPLEMENTATION=ON \
-DPIKA_WITH_STDEXEC=ON \
-DPIKA_WITH_EXAMPLES=On \
-DPIKA_WITH_TESTS=On \
-DPIKA_WITH_TESTS_HEADERS=On \
Expand Down
2 changes: 1 addition & 1 deletion .jenkins/cscs-ault/env-hipcc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,4 @@ configure_extra_options+=" -DCMAKE_HIP_ARCHITECTURES=gfx906:xnack-"
configure_extra_options+=" -DPIKA_WITH_HIP=ON"
configure_extra_options+=" -DPIKA_WITH_CXX_STANDARD=${cxx_std}"
configure_extra_options+=" -DPIKA_WITH_MALLOC=system"
configure_extra_options+=" -DPIKA_WITH_P2300_REFERENCE_IMPLEMENTATION=ON"
configure_extra_options+=" -DPIKA_WITH_STDEXEC=ON"
4 changes: 2 additions & 2 deletions .jenkins/cscs/env-clang-13.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ spack_compiler="clang@${clang_version}"
spack_arch="cray-cnl7-broadwell"
stdexec_version="6510f5bd69cc03b24668f26eda3dd3cca7e81bb2"

spack_spec="pika@main arch=${spack_arch} %${spack_compiler} cxxflags=-stdlib=libc++ malloc=system cxxstd=${cxx_std} +p2300 ^boost@${boost_version} ^hwloc@${hwloc_version} ^fmt cxxflags=-stdlib=libc++ ^stdexec@${stdexec_version}"
spack_spec="pika@main arch=${spack_arch} %${spack_compiler} cxxflags=-stdlib=libc++ malloc=system cxxstd=${cxx_std} +stdexec ^boost@${boost_version} ^hwloc@${hwloc_version} ^fmt cxxflags=-stdlib=libc++ ^stdexec@${stdexec_version}"

configure_extra_options+=" -DCMAKE_CXX_FLAGS=-stdlib=libc++"
configure_extra_options+=" -DPIKA_WITH_CXX_STANDARD=${cxx_std}"
configure_extra_options+=" -DPIKA_WITH_MALLOC=system"
configure_extra_options+=" -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON"
configure_extra_options+=" -DPIKA_WITH_P2300_REFERENCE_IMPLEMENTATION=ON"
configure_extra_options+=" -DPIKA_WITH_STDEXEC=ON"
4 changes: 2 additions & 2 deletions .jenkins/cscs/env-clang-14-cuda.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@ spack_compiler="clang@${clang_version}"
spack_arch="cray-cnl7-haswell"
stdexec_version="6510f5bd69cc03b24668f26eda3dd3cca7e81bb2"

spack_spec="pika@main arch=${spack_arch} %${spack_compiler} +cuda malloc=system cxxstd=${cxx_std} +p2300 ^boost@${boost_version} ^cuda@${cuda_version} +allow-unsupported-compilers ^hwloc@${hwloc_version} ^stdexec@${stdexec_version}"
spack_spec="pika@main arch=${spack_arch} %${spack_compiler} +cuda malloc=system cxxstd=${cxx_std} +stdexec ^boost@${boost_version} ^cuda@${cuda_version} +allow-unsupported-compilers ^hwloc@${hwloc_version} ^stdexec@${stdexec_version}"

configure_extra_options+=" -DPIKA_WITH_CXX_STANDARD=${cxx_std}"
configure_extra_options+=" -DPIKA_WITH_CUDA=ON"
configure_extra_options+=" -DPIKA_WITH_MALLOC=system"
configure_extra_options+=" -DPIKA_WITH_SPINLOCK_DEADLOCK_DETECTION=ON"
configure_extra_options+=" -DPIKA_WITH_P2300_REFERENCE_IMPLEMENTATION=ON"
configure_extra_options+=" -DPIKA_WITH_STDEXEC=ON"
configure_extra_options+=" -DCMAKE_CUDA_COMPILER=c++"
configure_extra_options+=" -DCMAKE_CUDA_ARCHITECTURES=60"

Expand Down
2 changes: 1 addition & 1 deletion .jenkins/cscs/env-nvhpc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ configure_extra_options+=" -DPIKA_WITH_CXX_STANDARD=${cxx_std}"
configure_extra_options+=" -DPIKA_WITH_MALLOC=system"
configure_extra_options+=" -DPIKA_WITH_CUDA=ON"
configure_extra_options+=" -DCMAKE_CUDA_ARCHITECTURES=60"
configure_extra_options+=" -DPIKA_WITH_P2300_REFERENCE_IMPLEMENTATION=ON"
configure_extra_options+=" -DPIKA_WITH_STDEXEC=ON"
13 changes: 6 additions & 7 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -991,12 +991,11 @@ if(PIKA_WITH_THREAD_DEBUG_INFO)
endif()

pika_option(
PIKA_WITH_P2300_REFERENCE_IMPLEMENTATION BOOL
"Use the P2300 reference implementation for sender/receiver functionality."
OFF CATEGORY "Advanced"
PIKA_WITH_STDEXEC BOOL "Use stdexec for sender/receiver functionality." OFF
CATEGORY "Advanced"
)
if(PIKA_WITH_P2300_REFERENCE_IMPLEMENTATION)
pika_add_config_define(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
if(PIKA_WITH_STDEXEC)
pika_add_config_define(PIKA_HAVE_STDEXEC)
endif()

# Developer tools
Expand Down Expand Up @@ -1310,8 +1309,8 @@ if(PIKA_WITH_COMPILER_WARNINGS)
endif()
endif()

# GCC gives a bogus warning in the P2300 reference implementation
if(PIKA_WITH_P2300_REFERENCE_IMPLEMENTATION AND CMAKE_COMPILER_IS_GNUCXX)
# GCC gives a bogus warning in stdexec
if(PIKA_WITH_STDEXEC AND CMAKE_COMPILER_IS_GNUCXX)
pika_add_compile_flag_if_available(-Wno-non-template-friend)
endif()

Expand Down
6 changes: 3 additions & 3 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ pika optionally requires:
* `HIP <https://rocmdocs.amd.com/en/latest/index.html>`_ 5.2.0 or greater
* `MPI <https://www.mpi-forum.org/>`_
* `Boost.Context, Thread, and Chrono <https://boost.org>`_ on macOS
* `stdexec <https://github.com/NVIDIA/stdexec>`_ when
``PIKA_WITH_P2300_REFERENCE_IMPLEMENTATION`` is enabled (currently tested with
commit `6510f5bd69cc03b24668f26eda3dd3cca7e81bb2
* `stdexec <https://github.com/NVIDIA/stdexec>`_ when ``PIKA_WITH_STDEXEC`` is
enabled (currently tested with commit
`6510f5bd69cc03b24668f26eda3dd3cca7e81bb2
<https://github.com/NVIDIA/stdexec/commit/6510f5bd69cc03b24668f26eda3dd3cca7e81bb2>`_).
The integration is experimental.

Expand Down
4 changes: 2 additions & 2 deletions cmake/pika_setup_stdexec.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
# Distributed under the Boost Software License, Version 1.0. (See accompanying
# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

if(PIKA_WITH_P2300_REFERENCE_IMPLEMENTATION)
if(PIKA_WITH_STDEXEC)

if(PIKA_WITH_CXX_STANDARD LESS 20)

pika_error(
"PIKA_WITH_P2300_REFERENCE_IMPLEMENTATION requires at least C++20 (PIKA_WITH_CXX_STANDARD is currently ${PIKA_WITH_CXX_STANDARD})"
"PIKA_WITH_STDEXEC requires at least C++20 (PIKA_WITH_CXX_STANDARD is currently ${PIKA_WITH_CXX_STANDARD})"
)

endif()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ namespace pika::cuda::experimental {
cuda_scheduler_sender(cuda_scheduler_sender const&) = delete;
cuda_scheduler_sender& operator=(cuda_scheduler_sender const&) = delete;

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
using completion_signatures = pika::execution::experimental::completion_signatures<
pika::execution::experimental::set_value_t()>;
#else
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ namespace pika::cuda::experimental {
then_on_host_sender_type(then_on_host_sender_type const&) = default;
then_on_host_sender_type& operator=(then_on_host_sender_type const&) = default;

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
template <typename... Ts>
requires std::is_invocable_v<F, Ts...>
using invoke_result_helper =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ namespace pika::cuda::experimental::then_with_stream_detail {
then_with_cuda_stream_sender_type& operator=(
then_with_cuda_stream_sender_type const&) = default;

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
template <typename... Ts>
requires std::is_invocable_v<F, cuda_stream const&,
std::add_lvalue_reference_t<std::decay_t<Ts>>...>
Expand Down Expand Up @@ -457,7 +457,7 @@ namespace pika::cuda::experimental::then_with_stream_detail {
using type = pika::util::detail::transform_t<Tuple, std::decay>;
};

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
using ts_type = pika::util::detail::prepend_t<
pika::util::detail::transform_t<
pika::execution::experimental::value_types_of_t<std::decay_t<Sender>,
Expand Down Expand Up @@ -502,7 +502,7 @@ namespace pika::cuda::experimental::then_with_stream_detail {
{
using type = pika::detail::monostate;
};
#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
using result_type = pika::util::detail::change_pack_t<pika::detail::variant,
pika::util::detail::unique_t<pika::util::detail::prepend_t<
pika::util::detail::transform_t<
Expand Down
2 changes: 1 addition & 1 deletion libs/pika/async_cuda/tests/performance/synchronize.cu
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ int pika_main(pika::program_options::variables_map& vm)

// The compilation of this loop unrolling with llvm-amdgpu's
// Clang 5.3.3 is hanging.
#if !(defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION) || defined(PIKA_HAVE_HIP))
#if !(defined(PIKA_HAVE_STDEXEC) || defined(PIKA_HAVE_HIP))
{
cu::enable_user_polling poll("default");

Expand Down
8 changes: 4 additions & 4 deletions libs/pika/async_cuda/tests/unit/cublas_matmul.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -188,9 +188,9 @@ void matrixMultiply(pika::cuda::experimental::cuda_scheduler& cuda_sched, sMatri

// See https://github.com/brycelelbach/wg21_p2300_std_execution/issues/466
// for details.
#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
std::cout << "skipping remainder of test because the P2300 reference implementation of split "
"does not yet support move-only senders"
#if defined(PIKA_HAVE_STDEXEC)
std::cout << "skipping remainder of test because the stdexec implementation of split does not "
"yet support move-only senders"
<< std::endl;
#else
pika::chrono::detail::high_resolution_timer t2;
Expand Down Expand Up @@ -353,7 +353,7 @@ int main(int argc, char** argv)
pika::init_params init_args;
init_args.desc_cmdline = cmdline;

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
PIKA_TEST(true);
#endif

Expand Down
2 changes: 1 addition & 1 deletion libs/pika/async_cuda/tests/unit/cuda_bulk.cu
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ int pika_main()
}

// The reference implementation does not support ranges as the shape
#if !defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if !defined(PIKA_HAVE_STDEXEC)
// Range
for (element_type n : {1, 42, 10007})
{
Expand Down
6 changes: 3 additions & 3 deletions libs/pika/async_mpi/include/pika/async_mpi/transform_mpi.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ namespace pika::mpi::experimental {
std::decay_t<F> f;
stream_type stream;

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
template <typename... Ts>
requires is_mpi_request_invocable_v<F, Ts...>
using invoke_result_helper = pika::execution::experimental::completion_signatures<
Expand Down Expand Up @@ -250,7 +250,7 @@ namespace pika::mpi::experimental {
using type = pika::util::detail::transform_t<Tuple, std::decay>;
};

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
using ts_type = pika::util::detail::prepend_t<
pika::util::detail::transform_t<
pika::execution::experimental::value_types_of_t<std::decay_t<Sender>,
Expand Down Expand Up @@ -297,7 +297,7 @@ namespace pika::mpi::experimental {
{
using type = pika::detail::monostate;
};
#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
using result_type = pika::util::detail::change_pack_t<pika::detail::variant,
pika::util::detail::unique_t<pika::util::detail::prepend_t<
pika::util::detail::transform_t<
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@

#include <pika/config.hpp>

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
# include <pika/execution_base/p2300_forward.hpp>
#if defined(PIKA_HAVE_STDEXEC)
# include <pika/execution_base/stdexec_forward.hpp>
#else
# include <pika/concepts/concepts.hpp>
# include <pika/datastructures/variant.hpp>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ namespace pika::drop_value_detail {

PIKA_NO_UNIQUE_ADDRESS std::decay_t<Sender> sender;

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
template <class...>
using empty_set_value = pika::execution::experimental::completion_signatures<
pika::execution::experimental::set_value_t()>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

#include <pika/config.hpp>

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
# include <pika/execution_base/p2300_forward.hpp>
#if defined(PIKA_HAVE_STDEXEC)
# include <pika/execution_base/stdexec_forward.hpp>
#else
# include <pika/allocator_support/allocator_deleter.hpp>
# include <pika/allocator_support/internal_allocator.hpp>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

#include <pika/config.hpp>

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
# include <pika/execution_base/p2300_forward.hpp>
#if defined(PIKA_HAVE_STDEXEC)
# include <pika/execution_base/stdexec_forward.hpp>
#else
# include <pika/execution/algorithms/start_detached.hpp>
# include <pika/execution/algorithms/then.hpp>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

#include <pika/config.hpp>

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
# include <pika/execution_base/p2300_forward.hpp>
#if defined(PIKA_HAVE_STDEXEC)
# include <pika/execution_base/stdexec_forward.hpp>
#else
# include <pika/datastructures/member_pack.hpp>
# include <pika/errors/try_catch_exception_ptr.hpp>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

#include <pika/config.hpp>

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
# include <pika/execution_base/p2300_forward.hpp>
#if defined(PIKA_HAVE_STDEXEC)
# include <pika/execution_base/stdexec_forward.hpp>
#else
# include <pika/assert.hpp>
# include <pika/concepts/concepts.hpp>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

#include <pika/config.hpp>

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
# include <pika/execution_base/p2300_forward.hpp>
#if defined(PIKA_HAVE_STDEXEC)
# include <pika/execution_base/stdexec_forward.hpp>
#else
# include <pika/assert.hpp>
# include <pika/concepts/concepts.hpp>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

#include <pika/config.hpp>

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
# include <pika/execution_base/p2300_forward.hpp>
#if defined(PIKA_HAVE_STDEXEC)
# include <pika/execution_base/stdexec_forward.hpp>
#endif

#include <pika/allocator_support/allocator_deleter.hpp>
Expand Down Expand Up @@ -185,7 +185,7 @@ namespace pika::make_future_detail {
{
using allocator_type = Allocator;

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
using value_types =
typename pika::execution::experimental::value_types_of_t<std::decay_t<Sender>,
pika::execution::experimental::empty_env, pika::util::detail::pack,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

#include <pika/config.hpp>

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
# include <pika/execution_base/p2300_forward.hpp>
#if defined(PIKA_HAVE_STDEXEC)
# include <pika/execution_base/stdexec_forward.hpp>
#else
# include <pika/concepts/concepts.hpp>
# include <pika/datastructures/variant.hpp>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

#include <pika/config.hpp>

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
# include <pika/execution_base/p2300_forward.hpp>
#if defined(PIKA_HAVE_STDEXEC)
# include <pika/execution_base/stdexec_forward.hpp>
#else
# include <pika/allocator_support/allocator_deleter.hpp>
# include <pika/allocator_support/internal_allocator.hpp>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ namespace pika::split_tuple_detail {
std::optional<operation_state_type> os;

// nvcc does not like decay_t, so this uses decay<>::type instead.
#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
using value_type = typename std::decay<
pika::execution::experimental::detail::single_result_t<typename pika::execution::
experimental::value_types_of_t<Sender, pika::execution::experimental::empty_env,
Expand All @@ -86,7 +86,7 @@ namespace pika::split_tuple_detail {
pika::util::detail::pack, pika::util::detail::pack>>>::type;
#endif

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
using error_type = pika::util::detail::unique_t<pika::util::detail::prepend_t<
pika::util::detail::transform_t<
typename pika::execution::experimental::error_types_of_t<Sender,
Expand Down Expand Up @@ -132,7 +132,7 @@ namespace pika::split_tuple_detail {
// This typedef is duplicated from the parent struct. The
// parent typedef is not instantiated early enough for use
// here.
#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
using value_type =
typename std::decay<pika::execution::experimental::detail::single_result_t<
typename pika::execution::experimental::value_types_of_t<Sender,
Expand Down Expand Up @@ -197,7 +197,7 @@ namespace pika::split_tuple_detail {
void operator()(pika::execution::detail::stopped_type)
{
constexpr bool sends_stopped =
#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
pika::execution::experimental::sends_stopped<Sender,
pika::execution::experimental::empty_env>
#else
Expand Down Expand Up @@ -388,7 +388,7 @@ namespace pika::split_tuple_detail {
using shared_state_type = shared_state<Sender, Allocator>;

// nvcc does not like decay_t, so this uses decay<>::type instead.
#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
using value_type = typename std::decay<
pika::execution::experimental::detail::single_result_t<typename pika::execution::
experimental::value_types_of_t<Sender, pika::execution::experimental::empty_env,
Expand All @@ -409,7 +409,7 @@ namespace pika::split_tuple_detail {
using type = std::add_lvalue_reference_t<std::add_const_t<T>>;
};

#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
template <typename...>
using set_value_helper = pika::execution::experimental::completion_signatures<
pika::execution::experimental::set_value_t(split_tuple_sender_value_type)>;
Expand Down Expand Up @@ -523,7 +523,7 @@ namespace pika::split_tuple_detail {
pika::intrusive_ptr<shared_state_type> state = p.release();

// nvcc does not like decay_t, so this uses decay<>::type instead.
#if defined(PIKA_HAVE_P2300_REFERENCE_IMPLEMENTATION)
#if defined(PIKA_HAVE_STDEXEC)
using value_type = typename std::decay<
pika::execution::experimental::detail::single_result_t<typename pika::execution::
experimental::value_types_of_t<Sender, pika::execution::experimental::empty_env,
Expand Down
Loading

0 comments on commit 34ed14d

Please sign in to comment.