From 4233fef583b4f8cbf9f781311717600feaaa0694 Mon Sep 17 00:00:00 2001 From: tbbdev Date: Thu, 23 May 2019 13:35:29 +0300 Subject: [PATCH] Committing TBB 2019 Update 7 source code --- .gitignore | 1 + CHANGES | 36 + README.md | 4 +- build/AIX.inc | 2 +- build/FreeBSD.gcc.inc | 2 +- build/SunOS.inc | 2 +- build/build.py | 49 +- build/detect.js | 2 +- build/linux.gcc.inc | 10 +- build/linux.inc | 2 +- build/macos.gcc.inc | 6 +- cmake/README.rst | 8 +- cmake/TBBInstallConfig.cmake | 40 +- cmake/tbb_config_installer.cmake | 10 +- cmake/templates/TBBConfig.cmake.in | 4 +- examples/common/examples-common.inc | 2 +- examples/graph/index.html | 20 +- examples/test_all/fibonacci/CMakeLists.txt | 25 + include/tbb/concurrent_map.h | 383 +++++ include/tbb/concurrent_set.h | 297 ++++ include/tbb/concurrent_unordered_map.h | 4 +- include/tbb/concurrent_unordered_set.h | 4 +- include/tbb/flow_graph.h | 121 +- .../tbb/internal/_concurrent_skip_list_impl.h | 1043 ++++++++++++ .../tbb/internal/_concurrent_unordered_impl.h | 131 +- include/tbb/internal/_flow_graph_body_impl.h | 76 +- include/tbb/internal/_flow_graph_cache_impl.h | 34 + include/tbb/internal/_flow_graph_impl.h | 68 +- include/tbb/internal/_flow_graph_join_impl.h | 5 - include/tbb/internal/_flow_graph_types_impl.h | 16 +- include/tbb/internal/_node_handle_impl.h | 168 ++ include/tbb/internal/_template_helpers.h | 22 + include/tbb/iterators.h | 2 + include/tbb/scalable_allocator.h | 7 +- include/tbb/task.h | 2 +- include/tbb/tbb.h | 4 + include/tbb/tbb_config.h | 11 +- include/tbb/tbb_stddef.h | 2 +- index.html | 2 - python/Makefile | 7 +- python/index.html | 2 +- python/rml/ipc_server.cpp | 8 +- python/tbb/__init__.py | 12 +- src/perf/time_async_return.cpp | 2 + src/tbbmalloc/backend.cpp | 9 +- src/tbbmalloc/backend.h | 13 +- src/tbbmalloc/frontend.cpp | 55 +- src/tbbmalloc/large_objects.cpp | 175 +- src/tbbmalloc/large_objects.h | 368 ++++ src/tbbmalloc/shared_utils.h | 19 + src/tbbmalloc/tbbmalloc_internal.h | 243 +-- src/test/harness_allocator.h | 106 ++ src/test/harness_defs.h | 3 + src/test/test_async_node.cpp | 104 +- src/test/test_concurrent_associative_common.h | 1488 +++++++++++++++++ src/test/test_concurrent_hash_map.cpp | 121 +- src/test/test_concurrent_map.cpp | 267 +++ src/test/test_concurrent_ordered_common.h | 349 ++++ src/test/test_concurrent_priority_queue.cpp | 1 - src/test/test_concurrent_set.cpp | 255 +++ src/test/test_concurrent_unordered_common.h | 1229 +------------- src/test/test_concurrent_unordered_map.cpp | 305 +--- src/test/test_concurrent_unordered_set.cpp | 144 +- src/test/test_flow_graph_priorities.cpp | 5 +- src/test/test_flow_graph_whitebox.cpp | 4 + src/test/test_initializer_list.h | 3 + src/test/test_iterators.cpp | 7 + src/test/test_limiter_node.cpp | 110 +- src/test/test_malloc_whitebox.cpp | 195 ++- src/test/test_range_based_for.h | 15 + src/test/test_runtime_loader.cpp | 4 +- src/test/test_tbb_header.cpp | 7 + src/test/test_tbb_version.cpp | 2 +- 73 files changed, 5965 insertions(+), 2299 deletions(-) create mode 100644 examples/test_all/fibonacci/CMakeLists.txt create mode 100644 include/tbb/concurrent_map.h create mode 100644 include/tbb/concurrent_set.h create mode 100644 include/tbb/internal/_concurrent_skip_list_impl.h create mode 100644 include/tbb/internal/_node_handle_impl.h create mode 100644 src/tbbmalloc/large_objects.h create mode 100644 src/test/test_concurrent_associative_common.h create mode 100644 src/test/test_concurrent_map.cpp create mode 100644 src/test/test_concurrent_ordered_common.h create mode 100644 src/test/test_concurrent_set.cpp diff --git a/.gitignore b/.gitignore index 0182b1411b..13b5c5a11f 100644 --- a/.gitignore +++ b/.gitignore @@ -71,6 +71,7 @@ Thumbs.db /rules.ninja *~ .emacs.desktop +.tags # Build system generated files # ################################ diff --git a/CHANGES b/CHANGES index d357ef5375..a681c98433 100644 --- a/CHANGES +++ b/CHANGES @@ -2,6 +2,42 @@ The list of most significant changes made over time in Intel(R) Threading Building Blocks (Intel(R) TBB). +Intel TBB 2019 Update 7 +TBB_INTERFACE_VERSION == 11007 + +Changes (w.r.t. Intel TBB 2019 Update 6): + +- Added TBBMALLOC_SET_HUGE_SIZE_THRESHOLD parameter to set the lower + bound for allocations that are not released back to OS unless + a cleanup is explicitly requested. +- Added zip_iterator::base() method to get the tuple of underlying + iterators. +- Improved async_node to never block a thread that sends a message + through its gateway. +- Extended decrement port of the tbb::flow::limiter_node to accept + messages of integral types. +- Added support of Windows* to the CMake module TBBInstallConfig. +- Added packaging of CMake configuration files to TBB packages built + using build/build.py script + (https://github.com/intel/tbb/issues/141). + +Changes affecting backward compatibility: + +- Removed the number_of_decrement_predecessors parameter from the + constructor of flow::limiter_node. To allow its usage, set + TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR macro to 1. + +Preview Features: + +- Added ordered associative containers: + concurrent_{map,multimap,set,multiset} (requires C++11). + +Open-source contributions integrated: + +- Fixed makefiles to properly obtain the GCC version for GCC 7 + and later (https://github.com/intel/tbb/pull/147) by Timmmm. + +------------------------------------------------------------------------ Intel TBB 2019 Update 6 TBB_INTERFACE_VERSION == 11006 diff --git a/README.md b/README.md index d2f5bbc381..41150d23f0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -# Threading Building Blocks 2019 Update 5 -[![Stable release](https://img.shields.io/badge/version-2019_U6-green.svg)](https://github.com/01org/tbb/releases/tag/2019_U6) +# Threading Building Blocks 2019 Update 7 +[![Stable release](https://img.shields.io/badge/version-2019_U7-green.svg)](https://github.com/01org/tbb/releases/tag/2019_U7) [![Apache License Version 2.0](https://img.shields.io/badge/license-Apache_2.0-green.svg)](LICENSE) Threading Building Blocks (TBB) lets you easily write parallel C++ programs that take diff --git a/build/AIX.inc b/build/AIX.inc index 5d081c672b..45e3651cf2 100644 --- a/build/AIX.inc +++ b/build/AIX.inc @@ -18,7 +18,7 @@ ifndef arch endif ifndef runtime - gcc_version:=$(shell gcc -dumpversion) + gcc_version:=$(shell gcc -dumpfullversion -dumpversion) os_version:=$(shell uname -r) os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') export runtime:=cc$(gcc_version)_kernel$(os_kernel_version) diff --git a/build/FreeBSD.gcc.inc b/build/FreeBSD.gcc.inc index 190522894a..653d4a3ed1 100644 --- a/build/FreeBSD.gcc.inc +++ b/build/FreeBSD.gcc.inc @@ -32,7 +32,7 @@ C_FLAGS = $(CPLUS_FLAGS) # gcc 6.0 and later have -flifetime-dse option that controls # elimination of stores done outside the object lifetime -ifneq (,$(shell gcc -dumpversion | egrep "^([6-9])")) +ifneq (,$(shell gcc -dumpfullversion -dumpversion | egrep "^([6-9])")) # keep pre-contruction stores for zero initialization DSE_KEY = -flifetime-dse=1 endif diff --git a/build/SunOS.inc b/build/SunOS.inc index 0d8044e978..569d3214a2 100644 --- a/build/SunOS.inc +++ b/build/SunOS.inc @@ -27,7 +27,7 @@ ifndef arch endif ifndef runtime - gcc_version:=$(shell gcc -dumpversion) + gcc_version:=$(shell gcc -dumpfullversion -dumpversion) os_version:=$(shell uname -r) os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') export runtime:=cc$(gcc_version)_kernel$(os_kernel_version) diff --git a/build/build.py b/build/build.py index 35a86cb76e..aecc9885a4 100644 --- a/build/build.py +++ b/build/build.py @@ -36,19 +36,20 @@ default_prefix = jp(default_prefix, 'Library') # conda-specific by default on Windows parser = argparse.ArgumentParser() -parser.add_argument('--tbbroot', default='.', help='Take Intel TBB from here') -parser.add_argument('--prefix', default=default_prefix, help='Prefix') -parser.add_argument('--prebuilt', default=[], action='append', help='Directories to find prebuilt files') -parser.add_argument('--no-rebuild', default=False, action='store_true', help='do not rebuild') -parser.add_argument('--install', default=False, action='store_true', help='install all') -parser.add_argument('--install-libs', default=False, action='store_true', help='install libs') -parser.add_argument('--install-devel', default=False, action='store_true', help='install devel') -parser.add_argument('--install-docs', default=False, action='store_true', help='install docs') -parser.add_argument('--install-python',default=False, action='store_true', help='install python module') -parser.add_argument('--make-tool', default='make', help='Use different make command instead') -parser.add_argument('--copy-tool', default=None, help='Use this command for copying ($ tool file dest-dir)') -parser.add_argument('--build-args', default="", help='specify extra build args') -parser.add_argument('--build-prefix', default='local', help='build dir prefix') +parser.add_argument('--tbbroot', default='.', help='Take Intel TBB from here') +parser.add_argument('--prefix', default=default_prefix, help='Prefix') +parser.add_argument('--prebuilt', default=[], action='append', help='Directories to find prebuilt files') +parser.add_argument('--no-rebuild', default=False, action='store_true', help='do not rebuild') +parser.add_argument('--install', default=False, action='store_true', help='install all') +parser.add_argument('--install-libs', default=False, action='store_true', help='install libs') +parser.add_argument('--install-devel', default=False, action='store_true', help='install devel') +parser.add_argument('--install-docs', default=False, action='store_true', help='install docs') +parser.add_argument('--install-python', default=False, action='store_true', help='install python module') +parser.add_argument('--make-tool', default='make', help='Use different make command instead') +parser.add_argument('--copy-tool', default=None, help='Use this command for copying ($ tool file dest-dir)') +parser.add_argument('--build-args', default="", help='specify extra build args') +parser.add_argument('--build-prefix', default='local', help='build dir prefix') +parser.add_argument('--cmake-dir', help='directory to install CMake configuraion files. Default: /lib/cmake/tbb') if is_win: parser.add_argument('--msbuild', default=False, action='store_true', help='Use msbuild') parser.add_argument('--vs', default="2012", help='select VS version for build') @@ -71,10 +72,12 @@ def custom_cp(src, dst): else: install_cp = shutil.copy -bin_dir = jp(args.prefix, "bin") -lib_dir = jp(args.prefix, "lib") -inc_dir = jp(args.prefix, 'include') -doc_dir = jp(args.prefix, 'share', 'doc', 'tbb') +bin_dir = jp(args.prefix, "bin") +lib_dir = jp(args.prefix, "lib") +inc_dir = jp(args.prefix, 'include') +doc_dir = jp(args.prefix, 'share', 'doc', 'tbb') +cmake_dir = jp(args.prefix, "lib", "cmake", "tbb") if args.cmake_dir is None else args.cmake_dir + if is_win: os.environ["OS"] = "Windows_NT" # make sure TBB will interpret it corretly libext = '.dll' @@ -151,6 +154,18 @@ def append_files(names, dst, paths=release_dirs): files = [f for f in filenames if not '.html' in f] append_files(files, jp(inc_dir, rootdir.split('include')[1][1:]), paths=(rootdir,)) + # Preparing CMake configuration files + cmake_build_dir = jp(args.tbbroot, 'build', args.build_prefix+'_release', 'cmake_configs') + assert system('cmake -DINSTALL_DIR=%s -DSYSTEM_NAME=%s -DTBB_VERSION_FILE=%s -DINC_REL_PATH=%s -DLIB_REL_PATH=%s -DBIN_REL_PATH=%s -P %s' % \ + (cmake_build_dir, + platform.system(), + jp(args.tbbroot, 'include', 'tbb', 'tbb_stddef.h'), + os.path.relpath(inc_dir, cmake_dir), + os.path.relpath(lib_dir, cmake_dir), + os.path.relpath(bin_dir, cmake_dir), + jp(args.tbbroot, 'cmake', 'tbb_config_installer.cmake'))) == 0 + append_files(['TBBConfig.cmake', 'TBBConfigVersion.cmake'], cmake_dir, paths=[cmake_build_dir]) + if args.install_python: # RML part irml_dir = jp(args.tbbroot, 'build', args.build_prefix+'_release') run_make('-C src tbb_build_prefix=%s %s python_rml'% (args.build_prefix, args.build_args)) diff --git a/build/detect.js b/build/detect.js index ddc83ffe8b..ccbe98634b 100644 --- a/build/detect.js +++ b/build/detect.js @@ -51,7 +51,7 @@ function doWork() { WScript.Echo("unknown"); } } else { - tmpExec = WshShell.Exec(compilerPath + " -dumpversion"); + tmpExec = WshShell.Exec(compilerPath + " -dumpfullversion -dumpversion"); var gccVersion = tmpExec.StdOut.ReadLine(); if (WScript.Arguments(0) == "/runtime") { WScript.Echo("mingw" + gccVersion); diff --git a/build/linux.gcc.inc b/build/linux.gcc.inc index aef54f5461..b8986768a1 100644 --- a/build/linux.gcc.inc +++ b/build/linux.gcc.inc @@ -36,22 +36,22 @@ LINK_FLAGS = -Wl,-rpath-link=. -rdynamic C_FLAGS = $(CPLUS_FLAGS) # gcc 4.2 and higher support OpenMP -ifneq (,$(shell $(CONLY) -dumpversion | egrep "^(4\.[2-9]|[5-9])")) +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^(4\.[2-9]|[5-9])")) OPENMP_FLAG = -fopenmp endif # gcc 4.8 and later support RTM intrinsics, but require command line switch to enable them -ifneq (,$(shell $(CONLY) -dumpversion | egrep "^(4\.[8-9]|[5-9])")) +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^(4\.[8-9]|[5-9])")) RTM_KEY = -mrtm endif # gcc 4.0 and later have -Wextra that is used by some our customers. -ifneq (,$(shell $(CONLY) -dumpversion | egrep "^([4-9])")) +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^([4-9])")) WARNING_KEY += -Wextra endif # gcc 5.0 and later have -Wsuggest-override and -Wno-sized-deallocation options -ifneq (,$(shell $(CONLY) -dumpversion | egrep "^([5-9])")) +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^([5-9])")) # enable -Wsuggest-override via a pre-included header in order to limit to C++11 and above INCLUDE_TEST_HEADERS = -include $(tbb_root)/src/test/harness_preload.h WARNING_SUPPRESS += -Wno-sized-deallocation @@ -59,7 +59,7 @@ endif # gcc 6.0 and later have -flifetime-dse option that controls # elimination of stores done outside the object lifetime -ifneq (,$(shell $(CONLY) -dumpversion | egrep "^([6-9])")) +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^([6-9])")) # keep pre-contruction stores for zero initialization DSE_KEY = -flifetime-dse=1 endif diff --git a/build/linux.inc b/build/linux.inc index e7a264f17c..4d59aaacf8 100644 --- a/build/linux.inc +++ b/build/linux.inc @@ -55,7 +55,7 @@ ifndef arch endif ifndef runtime - export gcc_version:=$(shell gcc -dumpversion) + export gcc_version:=$(shell gcc -dumpfullversion -dumpversion) os_version:=$(shell uname -r) os_kernel_version:=$(shell uname -r | sed -e 's/-.*$$//') export os_glibc_version_full:=$(shell getconf GNU_LIBC_VERSION | grep glibc | sed -e 's/^glibc //') diff --git a/build/macos.gcc.inc b/build/macos.gcc.inc index 4df8a5db38..b03dc9cba1 100644 --- a/build/macos.gcc.inc +++ b/build/macos.gcc.inc @@ -35,19 +35,19 @@ LIB_LINK_FLAGS = -dynamiclib -install_name @rpath/$(BUILDING_LIBRARY) C_FLAGS = $(CPLUS_FLAGS) # gcc 4.8 and later support RTM intrinsics, but require command line switch to enable them -ifneq (,$(shell $(CONLY) -dumpversion | egrep "^(4\.[8-9]|[5-9])")) +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^(4\.[8-9]|[5-9])")) RTM_KEY = -mrtm endif # gcc 5.0 and later have -Wsuggest-override option # enable it via a pre-included header in order to limit to C++11 and above -ifneq (,$(shell $(CONLY) -dumpversion | egrep "^([5-9])")) +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^([5-9])")) INCLUDE_TEST_HEADERS = -include $(tbb_root)/src/test/harness_preload.h endif # gcc 6.0 and later have -flifetime-dse option that controls # elimination of stores done outside the object lifetime -ifneq (,$(shell $(CONLY) -dumpversion | egrep "^([6-9])")) +ifneq (,$(shell $(CONLY) -dumpfullversion -dumpversion | egrep "^([6-9])")) # keep pre-contruction stores for zero initialization DSE_KEY = -flifetime-dse=1 endif diff --git a/cmake/README.rst b/cmake/README.rst index 68c30815d4..40e2830958 100644 --- a/cmake/README.rst +++ b/cmake/README.rst @@ -218,7 +218,7 @@ Provides the following functions: .. code:: cmake - tbb_install_config(INSTALL_DIR SYSTEM_NAME Linux|Darwin + tbb_install_config(INSTALL_DIR SYSTEM_NAME Linux|Darwin|Windows [TBB_VERSION ..|TBB_VERSION_FILE ] [LIB_REL_PATH INC_REL_PATH ] [LIB_PATH INC_PATH ])`` @@ -243,7 +243,8 @@ The use case is applicable for package maintainers who create own TBB packages a write it to TBBConfigVersion.cmake ``TBB_VERSION ..`` Directly specified TBB version; alternative to ``TBB_VERSION_FILE`` parameter -``LIB_REL_PATH `` Relative path to TBB binaries, default: ``../..`` +``LIB_REL_PATH `` Relative path to TBB binaries (.lib files on Windows), default: ``../../../lib`` +``BIN_REL_PATH `` Relative path to TBB DLLs, default: ``../../../bin`` (applicable for Windows only) ``INC_REL_PATH `` Relative path to TBB headers, default: ``../../../include`` =========================================== =========================================================== @@ -268,7 +269,8 @@ The use case is applicable for users who have installed TBB, but do not have (or ============================ ============================================== ``INSTALL_DIR `` Directory to install CMake configuration files ``SYSTEM_NAME Linux|Darwin`` OS name to generate config files for -``LIB_PATH `` Path to installed TBB binaries +``LIB_PATH `` Path to installed TBB binaries (.lib files on Windows) +``BIN_PATH `` Path to installed TBB DLLs (applicable for Windows only) ``INC_PATH `` Path to installed TBB headers ============================ ============================================== diff --git a/cmake/TBBInstallConfig.cmake b/cmake/TBBInstallConfig.cmake index 8128816f8e..b6ed34b0bc 100644 --- a/cmake/TBBInstallConfig.cmake +++ b/cmake/TBBInstallConfig.cmake @@ -21,8 +21,8 @@ set(_tbb_cmake_module_path ${CMAKE_CURRENT_LIST_DIR}) function(tbb_install_config) set(oneValueArgs INSTALL_DIR SYSTEM_NAME - LIB_REL_PATH INC_REL_PATH TBB_VERSION TBB_VERSION_FILE - LIB_PATH INC_PATH) # If TBB is installed on the system + LIB_REL_PATH INC_REL_PATH BIN_REL_PATH TBB_VERSION TBB_VERSION_FILE + LIB_PATH BIN_PATH INC_PATH) # If TBB is installed on the system cmake_parse_arguments(tbb_IC "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) @@ -30,10 +30,10 @@ function(tbb_install_config) file(MAKE_DIRECTORY ${config_install_dir}) # --- TBB_LIB_REL_PATH handling --- - set(TBB_LIB_REL_PATH "../..") + set(TBB_LIB_REL_PATH "../../../lib") if (tbb_IC_LIB_REL_PATH) - set(TBB_LIB_REL_PATH ${tbb_IC_LIB_REL_PATH}) + file(TO_CMAKE_PATH ${tbb_IC_LIB_REL_PATH} TBB_LIB_REL_PATH) endif() if (tbb_IC_LIB_PATH) @@ -43,11 +43,25 @@ function(tbb_install_config) endif() # ------ + # --- TBB_BIN_REL_PATH handling --- + set(TBB_BIN_REL_PATH "../../../bin") + + if (tbb_IC_BIN_REL_PATH) + file(TO_CMAKE_PATH ${tbb_IC_BIN_REL_PATH} TBB_BIN_REL_PATH) + endif() + + if (tbb_IC_BIN_PATH) + get_filename_component(bin_abs_path ${tbb_IC_BIN_PATH} ABSOLUTE) + file(RELATIVE_PATH TBB_BIN_REL_PATH ${config_install_dir} ${bin_abs_path}) + unset(bin_abs_path) + endif() + # ------ + # --- TBB_INC_REL_PATH handling --- set(TBB_INC_REL_PATH "../../../include") if (tbb_IC_INC_REL_PATH) - set(TBB_INC_REL_PATH ${tbb_IC_INC_REL_PATH}) + file(TO_CMAKE_PATH ${tbb_IC_INC_REL_PATH} TBB_INC_REL_PATH) endif() if (tbb_IC_INC_PATH) @@ -82,9 +96,25 @@ function(tbb_install_config) if (tbb_system_name STREQUAL "Linux") set(TBB_LIB_PREFIX "lib") set(TBB_LIB_EXT "so.2") + set(TBB_IMPLIB_RELEASE "") + set(TBB_IMPLIB_DEBUG "") elseif (tbb_system_name STREQUAL "Darwin") set(TBB_LIB_PREFIX "lib") set(TBB_LIB_EXT "dylib") + set(TBB_IMPLIB_RELEASE "") + set(TBB_IMPLIB_DEBUG "") + elseif (tbb_system_name STREQUAL "Windows") + set(TBB_LIB_PREFIX "") + set(TBB_LIB_EXT "dll") + # .lib files installed to TBB_LIB_REL_PATH (e.g. /lib); + # .dll files installed to TBB_BIN_REL_PATH (e.g. /bin); + # Expand TBB_LIB_REL_PATH here in IMPORTED_IMPLIB property and + # redefine it with TBB_BIN_REL_PATH value to properly fill IMPORTED_LOCATION property in TBBConfig.cmake.in template. + set(TBB_IMPLIB_RELEASE " + IMPORTED_IMPLIB_RELEASE \"\${CMAKE_CURRENT_LIST_DIR}/${TBB_LIB_REL_PATH}/\${_tbb_component}.lib\"") + set(TBB_IMPLIB_DEBUG " + IMPORTED_IMPLIB_DEBUG \"\${CMAKE_CURRENT_LIST_DIR}/${TBB_LIB_REL_PATH}/\${_tbb_component}_debug.lib\"") + set(TBB_LIB_REL_PATH ${TBB_BIN_REL_PATH}) else() message(FATAL_ERROR "Unsupported OS name: ${tbb_system_name}") endif() diff --git a/cmake/tbb_config_installer.cmake b/cmake/tbb_config_installer.cmake index ec6abebd2f..fa165e8e13 100644 --- a/cmake/tbb_config_installer.cmake +++ b/cmake/tbb_config_installer.cmake @@ -13,17 +13,19 @@ # limitations under the License. function(tbb_conf_gen_print_help) - message("Usage: cmake -DINSTALL_DIR= -DSYSTEM_NAME=Linux|Darwin -P tbb_config_generator.cmake + message("Usage: cmake -DINSTALL_DIR= -DSYSTEM_NAME=Linux|Darwin|Windows -P tbb_config_generator.cmake Parameters: For custom TBB package: -DTBB_VERSION_FILE= -DTBB_VERSION=.. (alternative to TBB_VERSION_FILE) - -DLIB_REL_PATH= -DINC_REL_PATH= + -DLIB_REL_PATH= + -DBIN_REL_PATH= (only for Windows) For installed TBB: - -DLIB_PATH= -DINC_PATH= + -DLIB_PATH= + -DBIN_PATH= (only for Windows) ") endfunction() @@ -37,7 +39,7 @@ if (NOT DEFINED SYSTEM_NAME) message(FATAL_ERROR "Required parameter SYSTEM_NAME is not defined") endif() -foreach (arg TBB_VERSION LIB_REL_PATH INC_REL_PATH TBB_VERSION_FILE LIB_PATH INC_PATH) +foreach (arg TBB_VERSION INC_REL_PATH LIB_REL_PATH BIN_REL_PATH TBB_VERSION_FILE INC_PATH LIB_PATH BIN_PATH) set(optional_args ${optional_args} ${arg} ${${arg}}) endforeach() diff --git a/cmake/templates/TBBConfig.cmake.in b/cmake/templates/TBBConfig.cmake.in index 5c2b91dadd..84e25399f8 100644 --- a/cmake/templates/TBBConfig.cmake.in +++ b/cmake/templates/TBBConfig.cmake.in @@ -65,13 +65,13 @@ foreach (_tbb_component ${TBB_FIND_COMPONENTS}) if (EXISTS "${_tbb_release_lib}") set_target_properties(TBB::${_tbb_component} PROPERTIES - IMPORTED_LOCATION_RELEASE "${_tbb_release_lib}") + IMPORTED_LOCATION_RELEASE "${_tbb_release_lib}"@TBB_IMPLIB_RELEASE@) set_property(TARGET TBB::${_tbb_component} APPEND PROPERTY IMPORTED_CONFIGURATIONS RELEASE) endif() if (EXISTS "${_tbb_debug_lib}") set_target_properties(TBB::${_tbb_component} PROPERTIES - IMPORTED_LOCATION_DEBUG "${_tbb_debug_lib}") + IMPORTED_LOCATION_DEBUG "${_tbb_debug_lib}"@TBB_IMPLIB_DEBUG@) set_property(TARGET TBB::${_tbb_component} APPEND PROPERTY IMPORTED_CONFIGURATIONS DEBUG) endif() diff --git a/examples/common/examples-common.inc b/examples/common/examples-common.inc index fb72cfb928..c0257a7316 100644 --- a/examples/common/examples-common.inc +++ b/examples/common/examples-common.inc @@ -25,7 +25,7 @@ else CXX0XFLAGS ?= -std=c++0x else # support of lambda started GCC 4.5 - ifneq (, $(strip $(shell g++ -dumpversion | egrep "^(4\.[5-9]|[5-9])"))) + ifneq (, $(strip $(shell g++ -dumpfullversion -dumpversion | egrep "^(4\.[5-9]|[5-9])"))) CXX0XFLAGS ?= -std=c++0x endif endif diff --git a/examples/graph/index.html b/examples/graph/index.html index 1e0d5bb67d..ba5601af28 100644 --- a/examples/graph/index.html +++ b/examples/graph/index.html @@ -119,11 +119,11 @@ .circ { list-style-type:circle } - + .single { padding: 0 0.5em; } - + /* ------------------------------------------------- */ /* Table styles */ table{ @@ -150,9 +150,9 @@ } th{ border:1px #dddddd solid; - padding-top:2px; + padding-top:2px; padding-bottom:0px; - padding-right:3px; + padding-right:3px; padding-left:3px; } td{ @@ -225,7 +225,7 @@ padding-right:5px; vertical-align:top; } - + .specs { border-collapse:collapse; } @@ -242,7 +242,7 @@ padding: 0 0.2em 0.2em; text-align: center; } - .specs td tr:last-child td, + .specs td tr:last-child td, .specs td tr:last-child th { padding: 0 0.2em; } @@ -266,7 +266,7 @@ Intel® Threading Building Blocks (Intel® TBB). Samples on Intel® TBB Flow Graph feature - + - +

This directory has examples of the Intel TBB Flow Graph feature.

@@ -334,8 +334,6 @@

Intel® Threading Building Blocks (Intel® TBB).
Samp
Several versions of Cholesky Factorization algorithm implementation.
stereo
An implementation of stereo image creation from two images (anaglyph effect). -
matmult -
Matrix multiplication Gen kernel implementation with the flow graph interface. @@ -348,7 +346,7 @@

Intel® Threading Building Blocks (Intel® TBB).
Samp

Intel and the Intel logo are trademarks of Intel Corporation in the U.S. and/or other countries. -
* Other names and brands may be claimed as the property of others. +
* Other names and brands may be claimed as the property of others.
© 2019, Intel Corporation

diff --git a/examples/test_all/fibonacci/CMakeLists.txt b/examples/test_all/fibonacci/CMakeLists.txt new file mode 100644 index 0000000000..0a6d6f53a0 --- /dev/null +++ b/examples/test_all/fibonacci/CMakeLists.txt @@ -0,0 +1,25 @@ +# Copyright (c) 2019 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.0.0 FATAL_ERROR) + +project(fibonacci CXX) +add_executable(fibonacci Fibonacci.cpp) + +# find_package will search for available TBBConfig using variables CMAKE_PREFIX_PATH and TBB_DIR. +find_package(TBB REQUIRED tbb) + +target_link_libraries(fibonacci + ${TBB_IMPORTED_TARGETS} # Link TBB imported targets to the executable; "TBB::tbb" can be used instead of "${TBB_IMPORTED_TARGETS}". + $<$:rt>) # Link "rt" library on Linux diff --git a/include/tbb/concurrent_map.h b/include/tbb/concurrent_map.h new file mode 100644 index 0000000000..d022d88031 --- /dev/null +++ b/include/tbb/concurrent_map.h @@ -0,0 +1,383 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_map_H +#define __TBB_concurrent_map_H + +#if !TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS +#error Set TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS to include concurrent_map.h +#endif + +#include "tbb_config.h" + +// concurrent_map requires C++11 support +#if __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT + +#include "internal/_concurrent_skip_list_impl.h" + +namespace tbb { + +namespace interface10 { + +template +class map_traits { +public: + static constexpr size_t MAX_LEVEL = MAX_LEVELS; + using random_level_generator_type = RandomGenerator; + using key_type = Key; + using mapped_type = Value; + using compare_type = KeyCompare; + using value_type = std::pair; + using reference = value_type&; + using const_reference = const value_type&; + using allocator_type = Allocator; + using mutex_type = tbb::spin_mutex; + using node_type = tbb::internal::node_handle, allocator_type>; + + static const bool allow_multimapping = AllowMultimapping; + + class value_compare { + public: + // TODO: these member types are deprecated in C++17, do we need to let them + using result_type = bool; + using first_argument_type = value_type; + using second_argument_type = value_type; + + bool operator()(const value_type& lhs, const value_type& rhs) const { + return comp(lhs.first, rhs.first); + } + + protected: + value_compare(compare_type c) : comp(c) {} + + friend class map_traits; + + compare_type comp; + }; + + static value_compare value_comp(compare_type comp) { return value_compare(comp); } + + static const key_type& get_key(const_reference val) { + return val.first; + } +}; // class map_traits + +template +class concurrent_multimap; + +template , typename Allocator = tbb_allocator>> +class concurrent_map + : public internal::concurrent_skip_list, 64, Allocator, false>> { + using traits_type = map_traits, 64, Allocator, false>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using mapped_type = Value; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::end; + using base_type::find; + using base_type::emplace; + using base_type::insert; + + concurrent_map() = default; + + explicit concurrent_map(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_map(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_map(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(first, last, comp, alloc) {} + + template< class InputIt > + concurrent_map(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} + + /** Copy constructor */ + concurrent_map(const concurrent_map&) = default; + + concurrent_map(const concurrent_map& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_map(concurrent_map&&) = default; + + concurrent_map(concurrent_map&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_map(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_map(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_map& operator=(const concurrent_map& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_map& operator=(concurrent_map&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + mapped_type& at(const key_type& key) { + iterator it = find(key); + + if (it == end()) { + tbb::internal::throw_exception(tbb::internal::eid_invalid_key); + } + + return it->second; + } + + const mapped_type& at(const key_type& key) const { + const_iterator it = find(key); + + if (it == end()) { + tbb::internal::throw_exception(tbb::internal::eid_invalid_key); + } + + return it->second; + } + + mapped_type& operator[](const key_type& key) { + iterator it = find(key); + + if (it == end()) { + it = emplace(std::piecewise_construct, std::forward_as_tuple(key), std::tuple<>()).first; + } + + return it->second; + } + + mapped_type& operator[](key_type&& key) { + iterator it = find(key); + + if (it == end()) { + it = emplace(std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::tuple<>()).first; + } + + return it->second; + } + + template::value>::type> + std::pair insert(P&& value) { + return emplace(std::forward

(value)); + } + + template::value>::type> + iterator insert(const_iterator hint, P&& value) { + return emplace_hint(hint, std::forward

(value)); + return end(); + } + + template + void merge(concurrent_map& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_map&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multimap& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multimap&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_map + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace internal { + +using namespace tbb::internal; + +template typename Map, typename Key, typename T, typename... Args> +using c_map_t = Map 0) && !is_allocator_v >, + pack_element_t<0, Args...>, std::less >, + std::conditional_t< (sizeof...(Args) > 0) && is_allocator_v >, + pack_element_t, tbb_allocator > > >; +} // namespace internal + +template +concurrent_map(It, It, Args...) +-> internal::c_map_t, internal::iterator_mapped_t, Args...>; + +template +concurrent_map(std::initializer_list>, Args...) +-> internal::c_map_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template , typename Allocator = tbb_allocator>> +class concurrent_multimap + : public internal::concurrent_skip_list, 64, Allocator, true>> { + using traits_type = map_traits, 64, Allocator, true>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using mapped_type = Value; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::end; + using base_type::find; + using base_type::emplace; + using base_type::insert; + + concurrent_multimap() = default; + + explicit concurrent_multimap(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_multimap(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_multimap(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(first, last, comp, alloc) {} + + template< class InputIt > + concurrent_multimap(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} + + /** Copy constructor */ + concurrent_multimap(const concurrent_multimap&) = default; + + concurrent_multimap(const concurrent_multimap& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_multimap(concurrent_multimap&&) = default; + + concurrent_multimap(concurrent_multimap&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_multimap(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_multimap(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_multimap& operator=(const concurrent_multimap& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_multimap& operator=(concurrent_multimap&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + template::value>::type> + std::pair insert(P&& value) { + return emplace(std::forward

(value)); + } + + template::value>::type> + iterator insert(const_iterator hint, P&& value) { + return emplace_hint(hint, std::forward

(value)); + return end(); + } + + template + void merge(concurrent_multimap& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multimap&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_map& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_map&& source) { + this->internal_merge(std::move(source)); + } + +}; // class concurrent_multimap + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template +concurrent_multimap(It, It, Args...) +-> internal::c_map_t, internal::iterator_mapped_t, Args...>; + +template +concurrent_multimap(std::initializer_list>, Args...) +-> internal::c_map_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +} // namespace interface10 + +using interface10::concurrent_map; +using interface10::concurrent_multimap; + +} // namespace tbb + +#endif // __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT +#endif // __TBB_concurrent_map_H diff --git a/include/tbb/concurrent_set.h b/include/tbb/concurrent_set.h new file mode 100644 index 0000000000..70269947e1 --- /dev/null +++ b/include/tbb/concurrent_set.h @@ -0,0 +1,297 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_set_H +#define __TBB_concurrent_set_H + +#if !TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS +#error Set TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS to include concurrent_set.h +#endif + +#include "tbb/tbb_config.h" + +// concurrent_set requires C++11 support +#if __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT + +#include "internal/_concurrent_skip_list_impl.h" + +namespace tbb { +namespace interface10 { + +// TODO: test this class +template +class set_traits { +public: + static constexpr size_t MAX_LEVEL = MAX_LEVELS; + using random_level_generator_type = RandomGenerator; + using key_type = Key; + using value_type = key_type; + using compare_type = KeyCompare; + using value_compare = compare_type; + using reference = value_type & ; + using const_reference = const value_type&; + using allocator_type = Allocator; + using mutex_type = tbb::spin_mutex; + using node_type = tbb::internal::node_handle, allocator_type>; + + static const bool allow_multimapping = AllowMultimapping; + + static const key_type& get_key(const_reference val) { + return val; + } + + static value_compare value_comp(compare_type comp) { return comp; } +}; + +template +class concurrent_multiset; + +template , typename Allocator = tbb_allocator> +class concurrent_set + : public internal::concurrent_skip_list, 64, Allocator, false>> { + using traits_type = set_traits, 64, Allocator, false>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::insert; + + concurrent_set() = default; + + explicit concurrent_set(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_set(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_set(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(first, last, comp, alloc) {} + + template< class InputIt > + concurrent_set(InputIt first, InputIt last, const allocator_type& alloc) : base_type(first, last, key_compare(), alloc) {} + + /** Copy constructor */ + concurrent_set(const concurrent_set&) = default; + + concurrent_set(const concurrent_set& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_set(concurrent_set&&) = default; + + concurrent_set(concurrent_set&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_set(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_set(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_set& operator=(const concurrent_set& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_set& operator=(concurrent_set&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + template + void merge(concurrent_set& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_set&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multiset& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multiset&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_set + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +namespace internal { + +using namespace tbb::internal; + +template typename Set, typename Key, typename... Args> +using c_set_t = Set 0) && !is_allocator_v >, + pack_element_t<0, Args...>, std::less >, + std::conditional_t< (sizeof...(Args) > 0) && is_allocator_v >, + pack_element_t, tbb_allocator > >; +} // namespace internal + +template +concurrent_set(It, It, Args...) +-> internal::c_set_t, Args...>; + +template +concurrent_set(std::initializer_list, Args...) +-> internal::c_set_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +template , typename Allocator = tbb_allocator> +class concurrent_multiset + : public internal::concurrent_skip_list, 64, Allocator, true>> { + using traits_type = set_traits, 64, Allocator, true>; + using base_type = internal::concurrent_skip_list; +#if __TBB_EXTRA_DEBUG +public: +#endif + using base_type::allow_multimapping; +public: + using key_type = Key; + using value_type = typename traits_type::value_type; + using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; + using key_compare = Comp; + using value_compare = typename base_type::value_compare; + using allocator_type = Allocator; + + using reference = typename base_type::reference; + using const_reference = typename base_type::const_reference; + using pointer = typename base_type::pointer; + using const_pointer = typename base_type::pointer; + + using iterator = typename base_type::iterator; + using const_iterator = typename base_type::const_iterator; + using reverse_iterator = typename base_type::reverse_iterator; + using const_reverse_iterator = typename base_type::const_reverse_iterator; + + using node_type = typename base_type::node_type; + + using base_type::insert; + + concurrent_multiset() = default; + + explicit concurrent_multiset(const key_compare& comp, const allocator_type& alloc = allocator_type()) : base_type(comp, alloc) {} + + explicit concurrent_multiset(const allocator_type& alloc) : base_type(key_compare(), alloc) {} + + template< class InputIt > + concurrent_multiset(InputIt first, InputIt last, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(first, last); + } + + template< class InputIt > + concurrent_multiset(InputIt first, InputIt last, const allocator_type& alloc) : base_type(key_compare(), alloc) { + insert(first, last); + } + + /** Copy constructor */ + concurrent_multiset(const concurrent_multiset&) = default; + + concurrent_multiset(const concurrent_multiset& other, const allocator_type& alloc) : base_type(other, alloc) {} + + concurrent_multiset(concurrent_multiset&&) = default; + + concurrent_multiset(concurrent_multiset&& other, const allocator_type& alloc) : base_type(std::move(other), alloc) {} + + concurrent_multiset(std::initializer_list init, const key_compare& comp = Comp(), const allocator_type& alloc = allocator_type()) + : base_type(comp, alloc) { + insert(init); + } + + concurrent_multiset(std::initializer_list init, const allocator_type& alloc) + : base_type(key_compare(), alloc) { + insert(init); + } + + concurrent_multiset& operator=(const concurrent_multiset& other) { + return static_cast(base_type::operator=(other)); + } + + concurrent_multiset& operator=(concurrent_multiset&& other) { + return static_cast(base_type::operator=(std::move(other))); + } + + template + void merge(concurrent_set& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_set&& source) { + this->internal_merge(std::move(source)); + } + + template + void merge(concurrent_multiset& source) { + this->internal_merge(source); + } + + template + void merge(concurrent_multiset&& source) { + this->internal_merge(std::move(source)); + } +}; // class concurrent_multiset + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + + +template +concurrent_multiset(It, It, Args...) +-> internal::c_set_t, Args...>; + +template +concurrent_multiset(std::initializer_list, Args...) +-> internal::c_set_t; + +#endif // __TBB_CPP17_DEDUCTION_GUIDES_PRESENT + +} // namespace interface10 + +using interface10::concurrent_set; +using interface10::concurrent_multiset; + +} // namespace tbb + +#endif // __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT +#endif // __TBB_concurrent_set_H diff --git a/include/tbb/concurrent_unordered_map.h b/include/tbb/concurrent_unordered_map.h index 94c0edbfec..cc73dad78e 100644 --- a/include/tbb/concurrent_unordered_map.h +++ b/include/tbb/concurrent_unordered_map.h @@ -37,7 +37,9 @@ class concurrent_unordered_map_traits typedef Hash_compare hash_compare; typedef typename tbb::internal::allocator_rebind::type allocator_type; #if __TBB_UNORDERED_NODE_HANDLE_PRESENT - typedef internal::node_handle node_type; + typedef tbb::internal::node_handle::node, + allocator_type> node_type; #endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT enum { allow_multimapping = Allow_multimapping }; diff --git a/include/tbb/concurrent_unordered_set.h b/include/tbb/concurrent_unordered_set.h index 07161f5e04..93a772ba5e 100644 --- a/include/tbb/concurrent_unordered_set.h +++ b/include/tbb/concurrent_unordered_set.h @@ -37,7 +37,9 @@ class concurrent_unordered_set_traits typedef Hash_compare hash_compare; typedef typename tbb::internal::allocator_rebind::type allocator_type; #if __TBB_UNORDERED_NODE_HANDLE_PRESENT - typedef internal::node_handle node_type; + typedef tbb::internal::node_handle::node, + allocator_type> node_type; #endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT enum { allow_multimapping = Allow_multimapping }; diff --git a/include/tbb/flow_graph.h b/include/tbb/flow_graph.h index a52d1180ae..cbdad30650 100644 --- a/include/tbb/flow_graph.h +++ b/include/tbb/flow_graph.h @@ -95,7 +95,11 @@ class continue_msg {}; template< typename T > class sender; template< typename T > class receiver; class continue_receiver; -template< typename T > class limiter_node; // needed for resetting decrementer +} // namespaceX +namespace interface11 { +template< typename T, typename U > class limiter_node; // needed for resetting decrementer +} +namespace interface10 { template< typename R, typename B > class run_and_put_task; namespace internal { @@ -325,7 +329,6 @@ class untyped_sender { class untyped_receiver { template< typename, typename > friend class run_and_put_task; - template< typename > friend class limiter_node; template< typename, typename > friend class internal::broadcast_cache; template< typename, typename > friend class internal::round_robin_cache; @@ -551,7 +554,6 @@ class receiver { protected: //! put receiver back in initial state - template friend class limiter_node; virtual void reset_receiver(reset_flags f = rf_reset_protocol) = 0; template friend class internal::successor_cache; @@ -663,7 +665,7 @@ class continue_receiver : public receiver< continue_msg > { __TBB_FLOW_GRAPH_PRIORITY_EXPR( node_priority_t my_priority; ) // the friend declaration in the base class did not eliminate the "protected class" // error in gcc 4.1.2 - template friend class limiter_node; + template friend class tbb::flow::interface11::limiter_node; void reset_receiver( reset_flags f ) __TBB_override { my_current_count = 0; @@ -2392,11 +2394,18 @@ class priority_queue_node : public buffer_node { } }; // priority_queue_node +} // interfaceX + +namespace interface11 { + +using namespace interface10; +namespace internal = interface10::internal; + //! Forwards messages only if the threshold has not been reached /** This node forwards items until its threshold is reached. It contains no buffering. If the downstream node rejects, the message is dropped. */ -template< typename T > +template< typename T, typename DecrementType=continue_msg > class limiter_node : public graph_node, public receiver< T >, public sender< T > { public: typedef T input_type; @@ -2418,12 +2427,12 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > internal::reservable_predecessor_cache< T, spin_mutex > my_predecessors; spin_mutex my_mutex; internal::broadcast_cache< T > my_successors; - int init_decrement_predecessors; + __TBB_DEPRECATED_LIMITER_EXPR( int init_decrement_predecessors; ) - friend class internal::forward_task_bypass< limiter_node >; + friend class internal::forward_task_bypass< limiter_node >; // Let decrementer call decrement_counter() - friend class internal::decrementer< limiter_node >; + friend class internal::decrementer< limiter_node, DecrementType >; bool check_conditions() { // always called under lock return ( my_count + my_tries < my_threshold && !my_predecessors.empty() && !my_successors.empty() ); @@ -2456,7 +2465,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > if ( check_conditions() ) { if ( internal::is_graph_active(this->my_graph) ) { task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); + internal::forward_task_bypass< limiter_node >( *this ); internal::spawn_in_graph_arena(graph_reference(), *rtask); } } @@ -2474,7 +2483,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > if ( check_conditions() ) { if ( internal::is_graph_active(this->my_graph) ) { task *rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); + internal::forward_task_bypass< limiter_node >( *this ); __TBB_ASSERT(!rval, "Have two tasks to handle"); return rtask; } @@ -2488,45 +2497,59 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > return; } - task * decrement_counter() { + task* decrement_counter( long long delta ) { { spin_mutex::scoped_lock lock(my_mutex); - if(my_count) --my_count; + if( delta > 0 && size_t(delta) > my_count ) + my_count = 0; + else if( delta < 0 && size_t(delta) > my_threshold - my_count ) + my_count = my_threshold; + else + my_count -= size_t(delta); // absolute value of delta is sufficiently small } return forward_task(); } -public: - //! The internal receiver< continue_msg > that decrements the count - internal::decrementer< limiter_node > decrement; - - //! Constructor - limiter_node(graph &g, size_t threshold, int num_decrement_predecessors=0) : - graph_node(g), my_threshold(threshold), my_count(0), my_tries(0), - init_decrement_predecessors(num_decrement_predecessors), - decrement(num_decrement_predecessors) - { + void initialize() { my_predecessors.set_owner(this); my_successors.set_owner(this); decrement.set_owner(this); - tbb::internal::fgt_node( tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(&decrement), - static_cast *>(this) ); + tbb::internal::fgt_node( + tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, + static_cast *>(this), static_cast *>(&decrement), + static_cast *>(this) + ); + } +public: + //! The internal receiver< DecrementType > that decrements the count + internal::decrementer< limiter_node, DecrementType > decrement; + +#if TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR + __TBB_STATIC_ASSERT( (tbb::internal::is_same_type::value), + "Deprecated interface of the limiter node can be used only in conjunction " + "with continue_msg as the type of DecrementType template parameter." ); +#endif // Check for incompatible interface + + //! Constructor + limiter_node(graph &g, + __TBB_DEPRECATED_LIMITER_ARG2(size_t threshold, int num_decrement_predecessors=0)) + : graph_node(g), my_threshold(threshold), my_count(0), + __TBB_DEPRECATED_LIMITER_ARG4( + my_tries(0), decrement(), + init_decrement_predecessors(num_decrement_predecessors), + decrement(num_decrement_predecessors)) { + initialize(); } //! Copy constructor limiter_node( const limiter_node& src ) : graph_node(src.my_graph), receiver(), sender(), - my_threshold(src.my_threshold), my_count(0), my_tries(0), - init_decrement_predecessors(src.init_decrement_predecessors), - decrement(src.init_decrement_predecessors) - { - my_predecessors.set_owner(this); - my_successors.set_owner(this); - decrement.set_owner(this); - tbb::internal::fgt_node( tbb::internal::FLOW_LIMITER_NODE, &this->my_graph, - static_cast *>(this), static_cast *>(&decrement), - static_cast *>(this) ); + my_threshold(src.my_threshold), my_count(0), + __TBB_DEPRECATED_LIMITER_ARG4( + my_tries(0), decrement(), + init_decrement_predecessors(src.init_decrement_predecessors), + decrement(src.init_decrement_predecessors)) { + initialize(); } #if TBB_PREVIEW_FLOW_GRAPH_TRACE @@ -2544,7 +2567,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > if ( was_empty && !my_predecessors.empty() && my_count + my_tries < my_threshold ) { if ( internal::is_graph_active(this->my_graph) ) { task* task = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ); + internal::forward_task_bypass < limiter_node >( *this ); internal::spawn_in_graph_arena(graph_reference(), *task); } } @@ -2605,7 +2628,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > my_predecessors.add( src ); if ( my_count + my_tries < my_threshold && !my_successors.empty() && internal::is_graph_active(this->my_graph) ) { task* task = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass < limiter_node >( *this ); + internal::forward_task_bypass < limiter_node >( *this ); internal::spawn_in_graph_arena(graph_reference(), *task); } return true; @@ -2639,7 +2662,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > --my_tries; if (check_conditions() && internal::is_graph_active(this->my_graph)) { rtask = new ( task::allocate_additional_child_of( *(this->my_graph.root_task()) ) ) - internal::forward_task_bypass< limiter_node >( *this ); + internal::forward_task_bypass< limiter_node >( *this ); } } else { @@ -2650,9 +2673,7 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > return rtask; } - graph& graph_reference() __TBB_override { - return my_graph; - } + graph& graph_reference() __TBB_override { return my_graph; } void reset_receiver(reset_flags /*f*/) __TBB_override { __TBB_ASSERT(false,NULL); // should never be called @@ -2671,6 +2692,9 @@ class limiter_node : public graph_node, public receiver< T >, public sender< T > decrement.reset_receiver(f); } }; // limiter_node +} // namespace interfaceX + +namespace interface10 { #include "internal/_flow_graph_join_impl.h" @@ -3541,11 +3565,18 @@ class async_node : public multifunction_node< Input, tuple< Output >, Policy, Al //! Implements gateway_type::try_put for an external activity to submit a message to FG bool try_put_impl(const Output &i) { internal::multifunction_output &port_0 = internal::output_port<0>(*this); + internal::broadcast_cache& port_successors = port_0.successors(); tbb::internal::fgt_async_try_put_begin(this, &port_0); - try_put_functor tpf(port_0, i); - internal::execute_in_graph_arena(this->my_graph, tpf); + task_list tasks; + bool is_at_least_one_put_successful = port_successors.gather_successful_try_puts(i, tasks); + __TBB_ASSERT( is_at_least_one_put_successful || tasks.empty(), + "Return status is inconsistent with the method operation." ); + + while( !tasks.empty() ) { + internal::enqueue_in_graph_arena(this->my_graph, tasks.pop_front()); + } tbb::internal::fgt_async_try_put_end(this, &port_0); - return tpf.result; + return is_at_least_one_put_successful; } public: @@ -3919,7 +3950,7 @@ class write_once_node : public overwrite_node { using interface10::queue_node; using interface10::sequencer_node; using interface10::priority_queue_node; - using interface10::limiter_node; + using interface11::limiter_node; using namespace interface10::internal::graph_policy_namespace; using interface10::join_node; using interface10::input_port; diff --git a/include/tbb/internal/_concurrent_skip_list_impl.h b/include/tbb/internal/_concurrent_skip_list_impl.h new file mode 100644 index 0000000000..685c45ae01 --- /dev/null +++ b/include/tbb/internal/_concurrent_skip_list_impl.h @@ -0,0 +1,1043 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_concurrent_skip_list_H +#define __TBB_concurrent_skip_list_H + +#if !defined(__TBB_concurrent_map_H) && !defined(__TBB_concurrent_set_H) +#error Do not #include this internal file directly; use public TBB headers instead. +#endif + +#include "../tbb_config.h" +#include "../tbb_stddef.h" +#include "../tbb_allocator.h" +#include "../spin_mutex.h" +#include "../tbb_exception.h" +#include "../enumerable_thread_specific.h" +#include "_allocator_traits.h" +#include "_template_helpers.h" +#include "_node_handle_impl.h" +#include // Need std::pair +#include +#include +#include // Need std::allocator_traits +#include +#include +#include +#include +#include +#include +#include +#include + +#if _MSC_VER +#pragma warning(disable: 4189) // warning 4189 -- local variable is initialized but not referenced +#pragma warning(disable: 4127) // warning 4127 -- while (true) has a constant expression in it +#endif + +namespace tbb { +namespace interface10 { +namespace internal { + +template +class skip_list_node { + +public: + using value_type = Value; + using size_type = std::size_t; + using reference = value_type & ; + using const_reference = const value_type & ; + using pointer = value_type * ; + using const_pointer = const value_type *; + using node_pointer = skip_list_node * ; + using atomic_node_pointer = std::atomic; + + using mutex_type = Mutex; + using lock_type = std::unique_lock; + + skip_list_node(size_type levels) : my_height(levels), my_fullyLinked(false) { + for (size_type lev = 0; lev < my_height; ++lev) + new(&my_next(lev)) atomic_node_pointer(nullptr); + __TBB_ASSERT(height() == levels, "Wrong node height"); + } + + ~skip_list_node() { + for(size_type lev = 0; lev < my_height; ++lev) + my_next(lev).~atomic(); + } + + skip_list_node(const skip_list_node&) = delete; + + skip_list_node(skip_list_node&&) = delete; + + skip_list_node& operator=(const skip_list_node&) = delete; + + pointer storage() { + return reinterpret_cast(&my_val); + } + + reference value() { + return *storage(); + } + + node_pointer next(size_type level) const { + __TBB_ASSERT(level < height(), "Cannot get next on the level greater than height"); + return my_next(level).load(std::memory_order_acquire); + } + + void set_next(size_type level, node_pointer next) { + __TBB_ASSERT(level < height(), "Cannot set next on the level greater than height"); + + my_next(level).store(next, std::memory_order_release); + } + + /** @return number of layers */ + size_type height() const { + return my_height; + } + + bool fully_linked() const { + return my_fullyLinked.load(std::memory_order_acquire); + } + + void mark_linked() { + my_fullyLinked.store(true, std::memory_order_release); + } + + lock_type acquire() { + return lock_type(my_mutex); + } + +private: + using aligned_storage_type = typename std::aligned_storage::type; + + atomic_node_pointer& my_next(size_type level) { + atomic_node_pointer* arr = reinterpret_cast(this + 1); + return arr[level]; + } + + const atomic_node_pointer& my_next(size_type level) const { + const atomic_node_pointer* arr = reinterpret_cast(this + 1); + return arr[level]; + } + + mutex_type my_mutex; + aligned_storage_type my_val; + size_type my_height; + std::atomic_bool my_fullyLinked; +}; + +template +class skip_list_iterator { + using node_type = NodeType; + using node_ptr = node_type*; +public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename node_type::value_type; + using difference_type = std::ptrdiff_t; + using pointer = typename std::conditional::type; + using reference = typename std::conditional::type; + + skip_list_iterator() : my_node_ptr(nullptr) {} + + // TODO: the code above does not compile in VS2015 (seems like a bug) - consider enabling it for all other platforms + // template ::type> + // skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + // skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + template ::type> + skip_list_iterator(const skip_list_iterator& other) : my_node_ptr(other.my_node_ptr) {} + + reference operator*() const { return *(my_node_ptr->storage()); } + pointer operator->() const { return &**this; } + + skip_list_iterator& operator++() { + __TBB_ASSERT(my_node_ptr != nullptr, NULL); + my_node_ptr = my_node_ptr->next(0); + return *this; + } + + skip_list_iterator operator++(int) { + skip_list_iterator tmp = *this; + ++*this; + return tmp; + } + +private: + skip_list_iterator(node_type* n) : my_node_ptr(n) {} + + node_ptr my_node_ptr; + + template + friend class concurrent_skip_list; + + friend class skip_list_iterator; + + friend class const_range; + friend class range; + + template + friend bool operator==(const skip_list_iterator&, const skip_list_iterator&); + + template + friend bool operator!=(const skip_list_iterator&, const skip_list_iterator&); +}; + +template +bool operator==(const skip_list_iterator& lhs, const skip_list_iterator& rhs) { + return lhs.my_node_ptr == rhs.my_node_ptr; +} + +template +bool operator!=(const skip_list_iterator& lhs, const skip_list_iterator& rhs) { + return lhs.my_node_ptr != rhs.my_node_ptr; +} + +template +class concurrent_skip_list { +protected: + using traits_type = Traits; + using allocator_type = typename traits_type::allocator_type; + using allocator_traits_type = std::allocator_traits; + using key_compare = typename traits_type::compare_type; + using value_compare = typename traits_type::value_compare; + using key_type = typename traits_type::key_type; + using value_type = typename traits_type::value_type; + using node_type = typename traits_type::node_type; + using list_node_type = skip_list_node; + + using iterator = skip_list_iterator; + using const_iterator = skip_list_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename allocator_traits_type::pointer; + using const_pointer = typename allocator_traits_type::const_pointer; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + + using random_level_generator_type = typename traits_type::random_level_generator_type; + using node_allocator_type = typename std::allocator_traits::template rebind_alloc; + using node_allocator_traits = typename std::allocator_traits::template rebind_traits; + using node_ptr = list_node_type*; + + static constexpr size_type MAX_LEVEL = traits_type::MAX_LEVEL; + + using array_type = std::array; + using lock_array = std::array; + +public: + static bool const allow_multimapping = traits_type::allow_multimapping; + + /** + * Default constructor. Construct empty skip list. + */ + concurrent_skip_list() : my_size(0) { + create_dummy_head(); + } + + explicit concurrent_skip_list(const key_compare& comp, const allocator_type& alloc = allocator_type()) + : my_node_allocator(alloc), my_compare(comp), my_size(0) + { + create_dummy_head(); + } + + template + concurrent_skip_list(InputIt first, InputIt last, const key_compare& comp = key_compare(), + const allocator_type& alloc = allocator_type()) + : my_node_allocator(alloc), my_compare(comp), my_size(0) + { + create_dummy_head(); + internal_copy(first, last); + } + + /** Copy constructor */ + concurrent_skip_list(const concurrent_skip_list& other) + : my_node_allocator(node_allocator_traits::select_on_container_copy_construction(other.get_allocator())), + my_compare(other.my_compare), my_rnd_generator(other.my_rnd_generator), my_size(0) + { + create_dummy_head(); + internal_copy(other); + __TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container"); + } + + concurrent_skip_list(const concurrent_skip_list& other, const allocator_type& alloc) + : my_node_allocator(alloc), my_compare(other.my_compare), + my_rnd_generator(other.my_rnd_generator), my_size(0) + { + create_dummy_head(); + internal_copy(other); + __TBB_ASSERT(my_size == other.my_size, "Wrong size of copy-constructed container"); + } + + concurrent_skip_list(concurrent_skip_list&& other) + : my_node_allocator(std::move(other.my_node_allocator)), my_compare(other.my_compare), + my_rnd_generator(other.my_rnd_generator) + { + internal_move(std::move(other)); + } + + concurrent_skip_list(concurrent_skip_list&& other, const allocator_type& alloc) + : my_node_allocator(alloc), my_compare(other.my_compare), + my_rnd_generator(other.my_rnd_generator) + { + if (alloc == other.get_allocator()) { + internal_move(std::move(other)); + } else { + my_size = 0; + create_dummy_head(); + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + } + } + + ~concurrent_skip_list() { + clear(); + delete_dummy_head(); + } + + concurrent_skip_list& operator=(const concurrent_skip_list& other) { + if (this != &other) { + using pocca_type = typename node_allocator_traits::propagate_on_container_copy_assignment; + clear(); + tbb::internal::allocator_copy_assignment(my_node_allocator, other.my_node_allocator, pocca_type()); + my_compare = other.my_compare; + my_rnd_generator = other.my_rnd_generator; + internal_copy(other); + } + return *this; + } + + concurrent_skip_list& operator=(concurrent_skip_list&& other) { + if (this != &other) { + using pocma_type = typename node_allocator_traits::propagate_on_container_move_assignment; + clear(); + my_compare = other.my_compare; + my_rnd_generator = other.my_rnd_generator; + internal_move_assign(std::move(other), pocma_type()); + } + return *this; + } + + concurrent_skip_list& operator=(std::initializer_list il) + { + clear(); + insert(il.begin(),il.end()); + return *this; + } + + std::pair insert(const value_type& value) { + return internal_insert(value); + } + + std::pair insert(value_type&& value) { + return internal_insert(std::move(value)); + } + + iterator insert(const_iterator, const_reference value) { + // Ignore hint + return insert(value).first; + } + + iterator insert(const_iterator, value_type&& value) { + // Ignore hint + return insert(std::move(value)).first; + } + + template + void insert(InputIterator first, InputIterator last) { + for (InputIterator it = first; it != last; ++it) + insert(*it); + } + + void insert(std::initializer_list init) { + insert(init.begin(), init.end()); + } + + std::pair insert(node_type&& nh) { + if(!nh.empty()) { + std::pair insert_result = internal_insert_node(nh.my_node); + if(insert_result.second) { + nh.deactivate(); + } + return insert_result; + } + return std::pair(end(), false); + } + + iterator insert(const_iterator, node_type&& nh) { + // Ignore hint + return insert(std::move(nh)).first; + } + + template + std::pair emplace(Args&&... args) { + return internal_insert(std::forward(args)...); + } + + template + iterator emplace_hint(const_iterator, Args&&... args) { + // Ignore hint + return emplace(std::forward(args)...).first; + } + + iterator unsafe_erase(iterator pos) { + std::pair extract_result = internal_extract(pos); + if(extract_result.first) { // node was extracted + delete_node(extract_result.first); + return extract_result.second; + } + return end(); + } + + iterator unsafe_erase(const_iterator first, const_iterator last) { + while(first != last) { + first = unsafe_erase(get_iterator(first)); + } + return get_iterator(first); + } + + size_type unsafe_erase(const key_type& key) { + std::pair range = equal_range(key); + size_type sz = std::distance(range.first, range.second); + unsafe_erase(range.first, range.second); + return sz; + } + + node_type unsafe_extract(const_iterator pos) { + std::pair extract_result = internal_extract(pos); + return extract_result.first ? node_type(extract_result.first) : node_type(); + } + + node_type unsafe_extract(const key_type& key) { + return unsafe_extract(find(key)); + } + + iterator lower_bound(const key_type& key) { + return internal_get_bound(key, my_compare); + } + + const_iterator lower_bound(const key_type& key) const { + return internal_get_bound(key, my_compare); + } + + template::value, K>::type> + iterator lower_bound(const K& key) { + return internal_get_bound(key, my_compare); + } + + template::value, K>::type> + const_iterator lower_bound(const K& key) const { + return internal_get_bound(key, my_compare); + } + + iterator upper_bound(const key_type& key) { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + const_iterator upper_bound(const key_type& key) const { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + template::value, K>::type> + iterator upper_bound(const K& key) { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + template::value, K>::type> + const_iterator upper_bound(const K& key) const { + return internal_get_bound(key, not_greater_compare(my_compare)); + } + + iterator find(const key_type& key) { + return internal_find(key); + } + + const_iterator find(const key_type& key) const { + return internal_find(key); + } + + template::value, K>::type> + iterator find(const K& key) { + return internal_find(key); + } + + template::value, K>::type> + const_iterator find(const K& key) const { + return internal_find(key); + } + + size_type count( const key_type& key ) const { + return internal_count(key); + } + + template::value, K>::type> + size_type count(const K& key) const { + return internal_count(key); + } + + bool contains(const key_type& key) const { + return find(key) != end(); + } + + template::value, K>::type> + bool contains(const K& key) const { + return find(key) != end(); + } + + void clear() noexcept { + __TBB_ASSERT(dummy_head->height() > 0, NULL); + + node_ptr current = dummy_head->next(0); + while (current) { + __TBB_ASSERT(current->height() > 0, NULL); + node_ptr next = current->next(0); + delete_node(current); + current = next; + } + + my_size = 0; + for (size_type i = 0; i < dummy_head->height(); ++i) { + dummy_head->set_next(i, nullptr); + } + } + + iterator begin() { + return iterator(dummy_head->next(0)); + } + + const_iterator begin() const { + return const_iterator(dummy_head->next(0)); + } + + const_iterator cbegin() const { + return const_iterator(dummy_head->next(0)); + } + + iterator end() { + return iterator(nullptr); + } + + const_iterator end() const { + return const_iterator(nullptr); + } + + const_iterator cend() const { + return const_iterator(nullptr); + } + + size_type size() const { + return my_size.load(std::memory_order_relaxed); + } + + size_type max_size() const { + return my_node_allocator.max_size(); + } + + bool empty() const { + return 0 == size(); + } + + allocator_type get_allocator() const { + return my_node_allocator; + } + + void swap(concurrent_skip_list& other) { + using std::swap; + using pocs_type = typename node_allocator_traits::propagate_on_container_swap; + tbb::internal::allocator_swap(my_node_allocator, other.my_node_allocator, pocs_type()); + swap(my_compare, other.my_compare); + swap(my_rnd_generator, other.my_rnd_generator); + swap(dummy_head, other.dummy_head); + + size_type tmp = my_size; + my_size.store(other.my_size); + other.my_size.store(tmp); + } + + std::pair equal_range(const key_type& key) { + return std::pair(lower_bound(key), upper_bound(key)); + } + + std::pair equal_range(const key_type& key) const { + return std::pair(lower_bound(key), upper_bound(key)); + } + + template::value, K>::type> + std::pair equal_range(const K& key) { + return std::pair(lower_bound(key), upper_bound(key)); + } + + template::value, K>::type> + std::pair equal_range(const K& key) const { + return std::pair(lower_bound(key), upper_bound(key)); + } + + key_compare key_comp() const { return my_compare; } + + value_compare value_comp() const { return traits_type::value_comp(my_compare); } + + class const_range_type : tbb::internal::no_assign { + public: + using size_type = typename concurrent_skip_list::size_type; + using value_type = typename concurrent_skip_list::value_type; + using iterator = typename concurrent_skip_list::const_iterator; + private: + const_iterator my_end; + const_iterator my_begin; + size_type my_level; + + public: + + bool empty() const { + return my_begin.my_node_ptr->next(0) == my_end.my_node_ptr; + } + + bool is_divisible() const { + return my_level != 0 ? my_begin.my_node_ptr->next(my_level - 1) != my_end.my_node_ptr : false; + } + + size_type size() const { return std::distance(my_begin, my_end);} + + const_range_type( const_range_type& r, split) + : my_end(r.my_end) { + my_begin = iterator(r.my_begin.my_node_ptr->next(r.my_level - 1)); + my_level = my_begin.my_node_ptr->height(); + r.my_end = my_begin; + } + + const_range_type( const concurrent_skip_list& l) + : my_end(l.end()), my_begin(l.begin()), my_level(my_begin.my_node_ptr->height() ) {} + + iterator begin() const { return my_begin; } + iterator end() const { return my_end; } + size_t grainsize() const { return 1; } + + }; // class const_range_type + + class range_type : public const_range_type { + public: + using iterator = typename concurrent_skip_list::iterator; + + range_type(range_type& r, split) : const_range_type(r, split()) {} + range_type(const concurrent_skip_list& l) : const_range_type(l) {} + + iterator begin() const { + node_ptr node = const_range_type::begin().my_node_ptr; + return iterator(node); + } + + iterator end() const { + node_ptr node = const_range_type::end().my_node_ptr; + return iterator(node); } + }; // class range_type + + range_type range() { return range_type(*this); } + const_range_type range() const { return const_range_type(*this); } + +private: + void internal_move(concurrent_skip_list&& other) { + dummy_head = other.dummy_head; + other.dummy_head = nullptr; + other.create_dummy_head(); + + my_size = other.my_size.load(); + other.my_size = 0; + } + + static const key_type& get_key(node_ptr n) { + __TBB_ASSERT(n, NULL); + return traits_type::get_key(n->value()); + } + + template + iterator internal_find(const K& key) { + iterator it = lower_bound(key); + return (it == end() || my_compare(key, traits_type::get_key(*it))) ? end() : it; + } + + template + const_iterator internal_find(const K& key) const { + const_iterator it = lower_bound(key); + return (it == end() || my_compare(key, traits_type::get_key(*it))) ? end() : it; + } + + template + size_type internal_count( const K& key ) const { + if (allow_multimapping) { + std::pair range = equal_range(key); + return std::distance(range.first, range.second); + } + return (find(key) == end()) ? size_type(0) : size_type(1); + } + + /** + * Finds position on the @param level using @param cmp + * @param level - on which level search prev node + * @param prev - pointer to the start node to search + * @param key - key to search + * @param cmp - callable object to compare two objects + * (my_compare member is default comparator) + * @returns pointer to the node which is not satisfy the comparison with @param key + */ + template + pointer_type internal_find_position( size_type level, pointer_type& prev, const K& key, + const comparator& cmp) const { + __TBB_ASSERT(level < prev->height(), "Wrong level to find position"); + pointer_type curr = prev->next(level); + + while (curr && cmp(get_key(curr), key)) { + prev = curr; + __TBB_ASSERT(level < prev->height(), NULL); + curr = prev->next(level); + } + + return curr; + } + + template + void fill_prev_next_arrays(array_type& prev_nodes, array_type& next_nodes, node_ptr prev, const key_type& key, + const comparator& cmp) { + prev_nodes.fill(dummy_head); + next_nodes.fill(nullptr); + + for (size_type h = prev->height(); h > 0; --h) { + node_ptr next = internal_find_position(h - 1, prev, key, cmp); + prev_nodes[h - 1] = prev; + next_nodes[h - 1] = next; + } + } + + template + std::pair internal_insert(Args&&... args) { + node_ptr new_node = create_node(std::forward(args)...); + std::pair insert_result = internal_insert_node(new_node); + if(!insert_result.second) { + delete_node(new_node); + } + return insert_result; + } + + std::pair internal_insert_node(node_ptr new_node) { + array_type prev_nodes; + array_type next_nodes; + __TBB_ASSERT(dummy_head->height() >= new_node->height(), "Wrong height for new node"); + + do { + if (allow_multimapping) { + fill_prev_next_arrays(prev_nodes, next_nodes, dummy_head, get_key(new_node), + not_greater_compare(my_compare)); + } else { + fill_prev_next_arrays(prev_nodes, next_nodes, dummy_head, get_key(new_node), my_compare); + } + + node_ptr next = next_nodes[0]; + if (next && !allow_multimapping && !my_compare(get_key(new_node), get_key(next))) { + // TODO: do we really need to wait? + while (!next->fully_linked()) { + // TODO: atomic backoff + } + + return std::pair(iterator(next), false); + } + __TBB_ASSERT(allow_multimapping || !next || my_compare(get_key(new_node), get_key(next)), + "Wrong elements order"); + + } while (!try_insert_node(new_node, prev_nodes, next_nodes)); + + __TBB_ASSERT(new_node, NULL); + return std::pair(iterator(new_node), true); + } + + bool try_insert_node(node_ptr new_node, array_type& prev_nodes, array_type& next_nodes) { + __TBB_ASSERT(dummy_head->height() >= new_node->height(), NULL); + + lock_array locks; + + if (!try_lock_nodes(new_node->height(), prev_nodes, next_nodes, locks)) { + return false; + } + + __TBB_ASSERT(allow_multimapping || + ((prev_nodes[0] == dummy_head || + my_compare(get_key(prev_nodes[0]), get_key(new_node))) && + (next_nodes[0] == nullptr || my_compare(get_key(new_node), get_key(next_nodes[0])))), + "Wrong elements order"); + + for (size_type level = 0; level < new_node->height(); ++level) { + __TBB_ASSERT(prev_nodes[level]->height() > level, NULL); + __TBB_ASSERT(prev_nodes[level]->next(level) == next_nodes[level], NULL); + new_node->set_next(level, next_nodes[level]); + prev_nodes[level]->set_next(level, new_node); + } + new_node->mark_linked(); + + ++my_size; + + return true; + } + + bool try_lock_nodes(size_type height, array_type& prevs, array_type& next_nodes, lock_array& locks) { + for (size_type l = 0; l < height; ++l) { + if (l == 0 || prevs[l] != prevs[l - 1]) + locks[l] = prevs[l]->acquire(); + + node_ptr next = prevs[l]->next(l); + if ( next != next_nodes[l]) return false; + } + + return true; + } + + template + const_iterator internal_get_bound(const K& key, const comparator& cmp) const { + node_ptr prev = dummy_head; + __TBB_ASSERT(dummy_head->height() > 0, NULL); + node_ptr next = nullptr; + + for (size_type h = prev->height(); h > 0; --h) { + next = internal_find_position(h - 1, prev, key, cmp); + } + + return const_iterator(next); + } + + template + iterator internal_get_bound(const K& key, const comparator& cmp){ + node_ptr prev = dummy_head; + __TBB_ASSERT(dummy_head->height() > 0, NULL); + node_ptr next = nullptr; + + for (size_type h = prev->height(); h > 0; --h) { + next = internal_find_position(h - 1, prev, key, cmp); + } + + return iterator(next); + } + + // Returns node_ptr to the extracted node and node_ptr to the next node after the extracted + std::pair internal_extract(const_iterator it) { + if ( it != end() ) { + key_type key = traits_type::get_key(*it); + node_ptr prev = dummy_head; + __TBB_ASSERT(dummy_head->height() > 0, NULL); + + array_type prev_nodes; + array_type next_nodes; + + fill_prev_next_arrays(prev_nodes, next_nodes, prev, key, my_compare); + + node_ptr erase_node = next_nodes[0]; + node_ptr next_node = erase_node->next(0); + + if (erase_node && !my_compare(key, get_key(erase_node))) { + for(size_type level = 0; level < erase_node->height(); ++level) { + __TBB_ASSERT(prev_nodes[level]->height() > level, NULL); + __TBB_ASSERT(next_nodes[level] == erase_node, NULL); + prev_nodes[level]->set_next(level, erase_node->next(level)); + } + --my_size; + return std::pair(erase_node, next_node); + } + } + return std::pair(nullptr, nullptr); + } + +protected: + template + void internal_merge(SourceType&& source) { + using source_type = typename std::decay::type; + using source_iterator = typename source_type::iterator; + __TBB_STATIC_ASSERT((std::is_same::value), "Incompatible containers cannot be merged"); + + for(source_iterator it = source.begin(); it != source.end();) { + source_iterator where = it++; + if (allow_multimapping || !contains(traits_type::get_key(*where))) { + std::pair extract_result = source.internal_extract(where); + + //If the insertion fails - return the node into source + node_type handle(extract_result.first); + __TBB_ASSERT(!handle.empty(), "Extracted handle in merge is empty"); + + if (!insert(std::move(handle)).second) { + source.insert(std::move(handle)); + } + handle.deactivate(); + } + } + } + +private: + void internal_copy(const concurrent_skip_list& other) { + internal_copy(other.begin(), other.end()); + } + + template + void internal_copy(Iterator first, Iterator last) { + clear(); + try { + for (auto it = first; it != last; ++it) + insert(*it); + } + catch (...) { + clear(); + delete_dummy_head(); + throw; + } + } + + /** Generate random level */ + size_type random_level() { + return my_rnd_generator(); + } + + static size_type calc_node_size(size_type height) { + return sizeof(list_node_type) + height*sizeof(typename list_node_type::atomic_node_pointer); + } + + /** Creates new node */ + template + node_ptr create_node(Args&&... args) { + size_type levels = random_level(); + + size_type sz = calc_node_size(levels); + + node_ptr node = reinterpret_cast(node_allocator_traits::allocate(my_node_allocator, sz)); + + try { + node_allocator_traits::construct(my_node_allocator, node, levels); + + } + catch(...) { + deallocate_node(node, sz); + throw; + } + + try { + node_allocator_traits::construct(my_node_allocator, node->storage(), std::forward(args)...); + } + catch (...) { + node_allocator_traits::destroy(my_node_allocator, node); + deallocate_node(node, sz); + throw; + } + + return node; + } + + void create_dummy_head() { + size_type sz = calc_node_size(MAX_LEVEL); + + dummy_head = reinterpret_cast(node_allocator_traits::allocate(my_node_allocator, sz)); + // TODO: investigate linkage fail in debug without this workaround + auto max_level = MAX_LEVEL; + + try { + node_allocator_traits::construct(my_node_allocator, dummy_head, max_level); + } + catch(...) { + deallocate_node(dummy_head, sz); + throw; + } + } + + template + void delete_node(node_ptr node) { + size_type sz = calc_node_size(node->height()); + // Destroy value + if (!is_dummy) node_allocator_traits::destroy(my_node_allocator, node->storage()); + // Destroy node + node_allocator_traits::destroy(my_node_allocator, node); + // Deallocate memory + deallocate_node(node, sz); + } + + void deallocate_node(node_ptr node, size_type sz) { + node_allocator_traits::deallocate(my_node_allocator, reinterpret_cast(node), sz); + } + + void delete_dummy_head() { + delete_node(dummy_head); + } + + static iterator get_iterator(const_iterator it) { + return iterator(it.my_node_ptr); + } + + void internal_move_assign(concurrent_skip_list&& other, /*POCMA=*/std::true_type) { + delete_dummy_head(); + tbb::internal::allocator_move_assignment(my_node_allocator, other.my_node_allocator, std::true_type()); + internal_move(std::move(other)); + } + + void internal_move_assign(concurrent_skip_list&& other, /*POCMA=*/std::false_type) { + if (my_node_allocator == other.my_node_allocator) { + delete_dummy_head(); + internal_move(std::move(other)); + } else { + internal_copy(std::make_move_iterator(other.begin()), std::make_move_iterator(other.end())); + } + } + + struct not_greater_compare { + const key_compare& my_less_compare; + + not_greater_compare(const key_compare& less_compare) : my_less_compare(less_compare) {} + + template + bool operator()(const K1& first, const K2& second) const { + return !my_less_compare(second, first); + } + }; + + node_allocator_type my_node_allocator; + key_compare my_compare; + random_level_generator_type my_rnd_generator; + node_ptr dummy_head; + + template + friend class concurrent_skip_list; + + std::atomic my_size; +}; // class concurrent_skip_list + +template +class concurrent_geometric_level_generator { +public: + static constexpr size_t max_level = MAX_LEVEL; + + concurrent_geometric_level_generator() : engines(time(NULL)) {} + + size_t operator()() { + return (distribution(engines.local()) % MAX_LEVEL) + 1; + } + +private: + tbb::enumerable_thread_specific engines; + std::geometric_distribution distribution; +}; + +} // namespace internal +} // namespace interface10 +} // namespace tbb + +#endif // __TBB_concurrent_skip_list_H diff --git a/include/tbb/internal/_concurrent_unordered_impl.h b/include/tbb/internal/_concurrent_unordered_impl.h index b998072351..3998c3fa45 100644 --- a/include/tbb/internal/_concurrent_unordered_impl.h +++ b/include/tbb/internal/_concurrent_unordered_impl.h @@ -48,6 +48,10 @@ #include "_tbb_hash_compare_impl.h" #include "_template_helpers.h" +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT +#include "_node_handle_impl.h" +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT + namespace tbb { namespace interface5 { //! @cond INTERNAL @@ -227,6 +231,15 @@ class split_ordered_list return my_order_key; } + // get() and value() is a common interface for getting access to node`s element (required by node_handle) + value_type* storage() { + return reinterpret_cast(&my_element); + } + + value_type& value() { + return *storage(); + } + // Inserts the new element in the list in an atomic fashion nodeptr_t atomic_set_next(nodeptr_t new_node, nodeptr_t current_node) { @@ -1647,124 +1660,6 @@ class concurrent_unordered_base : public Traits #pragma warning(pop) // warning 4127 is back #endif -#if __TBB_UNORDERED_NODE_HANDLE_PRESENT - -template -class node_handle_base { -public: - typedef Allocator allocator_type; -protected: - typedef typename split_ordered_list::node node; -public: - - node_handle_base() : my_node(NULL), my_allocator() {} - node_handle_base(node_handle_base&& nh) : my_node(nh.my_node), - my_allocator(std::move(nh.my_allocator)) { - nh.my_node = NULL; - } - - bool empty() const { return my_node == NULL; } - explicit operator bool() const { return my_node != NULL; } - - ~node_handle_base() { internal_destroy(); } - - node_handle_base& operator=(node_handle_base&& nh) { - internal_destroy(); - my_node = nh.my_node; - typedef typename tbb::internal::allocator_traits:: - propagate_on_container_move_assignment pocma_type; - tbb::internal::allocator_move_assignment(my_allocator, nh.my_allocator, pocma_type()); - nh.deactivate(); - return *this; - } - - void swap(node_handle_base& nh) { - std::swap(my_node, nh.my_node); - typedef typename tbb::internal::allocator_traits:: - propagate_on_container_swap pocs_type; - tbb::internal::allocator_swap(my_allocator, nh.my_allocator, pocs_type()); - } - - allocator_type get_allocator() const { - return my_allocator; - } - -protected: - node_handle_base(node* n) : my_node(n) {} - - void internal_destroy() { - if(my_node) { - my_allocator.destroy(&(my_node->my_element)); - // TODO: Consider using node_allocator from the container - typename tbb::internal::allocator_rebind::type node_allocator; - node_allocator.deallocate(my_node, 1); - } - } - - void deactivate() { my_node = NULL; } - - node* my_node; - allocator_type my_allocator; -}; - -// node handle for concurrent_unordered maps -template -class node_handle : public node_handle_base { - typedef node_handle_base base_type; -public: - typedef Key key_type; - typedef typename Value::second_type mapped_type; - typedef typename base_type::allocator_type allocator_type; - - node_handle() : base_type() {} - - key_type& key() const { - __TBB_ASSERT(!this->empty(), "Cannot get key from the empty node_type object"); - return *const_cast(&(this->my_node->my_element.first)); - } - - mapped_type& mapped() const { - __TBB_ASSERT(!this->empty(), "Cannot get mapped value from the empty node_type object"); - return this->my_node->my_element.second; - } - -private: - template - friend class split_ordered_list; - - template - friend class concurrent_unordered_base; - - node_handle(typename base_type::node* n) : base_type(n) {} -}; - -// node handle for concurrent_unordered sets -template -class node_handle : public node_handle_base { - typedef node_handle_base base_type; -public: - typedef Key value_type; - typedef typename base_type::allocator_type allocator_type; - - node_handle() : base_type() {} - - value_type& value() const { - __TBB_ASSERT(!this->empty(), "Cannot get value from the empty node_type object"); - return *const_cast(&(this->my_node->my_element)); - } - -private: - template - friend class split_ordered_list; - - template - friend class concurrent_unordered_base; - - node_handle(typename base_type::node* n) : base_type(n) {} -}; - -#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT - } // namespace internal //! @endcond } // namespace interface5 diff --git a/include/tbb/internal/_flow_graph_body_impl.h b/include/tbb/internal/_flow_graph_body_impl.h index 25be5ecb97..57264fd300 100644 --- a/include/tbb/internal/_flow_graph_body_impl.h +++ b/include/tbb/internal/_flow_graph_body_impl.h @@ -340,13 +340,85 @@ struct empty_body { Output operator()( const Input & ) const { return Output(); } }; +template +class decrementer; + +template +class decrementer::value, void>::type + > : public receiver, tbb::internal::no_copy { + T* my_node; +protected: + + task* try_put_task( const DecrementType& value ) __TBB_override { + task* result = my_node->decrement_counter( value ); + if( !result ) + result = SUCCESSFULLY_ENQUEUED; + return result; + } + + graph& graph_reference() __TBB_override { + return my_node->my_graph; + } + + template friend class tbb::flow::interface11::limiter_node; + void reset_receiver( reset_flags f ) __TBB_override { +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + if (f & rf_clear_edges) + my_built_predecessors.clear(); +#else + tbb::internal::suppress_unused_warning( f ); +#endif + } + +public: + // Since decrementer does not make use of possibly unconstructed owner inside its + // constructor, my_node can be directly initialized with 'this' pointer passed from the + // owner, hence making method 'set_owner' needless. + decrementer() : my_node(NULL) {} + void set_owner( T *node ) { my_node = node; } + +#if TBB_DEPRECATED_FLOW_NODE_EXTRACTION + spin_mutex my_mutex; + //! The predecessor type for this node + typedef typename receiver::predecessor_type predecessor_type; + + typedef internal::edge_container built_predecessors_type; + typedef typename built_predecessors_type::edge_list_type predecessor_list_type; + built_predecessors_type &built_predecessors() __TBB_override { return my_built_predecessors; } + + void internal_add_built_predecessor( predecessor_type &s) __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + my_built_predecessors.add_edge( s ); + } + + void internal_delete_built_predecessor( predecessor_type &s) __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + my_built_predecessors.delete_edge(s); + } + + void copy_predecessors( predecessor_list_type &v) __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + my_built_predecessors.copy_edges(v); + } + + size_t predecessor_count() __TBB_override { + spin_mutex::scoped_lock l(my_mutex); + return my_built_predecessors.edge_count(); + } +protected: + built_predecessors_type my_built_predecessors; +#endif /* TBB_DEPRECATED_FLOW_NODE_EXTRACTION */ +}; + template -class decrementer : public continue_receiver, tbb::internal::no_copy { +class decrementer : public continue_receiver, tbb::internal::no_copy { T *my_node; task *execute() __TBB_override { - return my_node->decrement_counter(); + return my_node->decrement_counter( 1 ); } protected: diff --git a/include/tbb/internal/_flow_graph_cache_impl.h b/include/tbb/internal/_flow_graph_cache_impl.h index 42a8051124..f3081b6f98 100644 --- a/include/tbb/internal/_flow_graph_cache_impl.h +++ b/include/tbb/internal/_flow_graph_cache_impl.h @@ -504,6 +504,40 @@ class broadcast_cache : public successor_cache { return last_task; } + // call try_put_task and return list of received tasks +#if __TBB_PREVIEW_ASYNC_MSG + template + bool gather_successful_try_puts( const X &t, task_list &tasks ) { +#else + bool gather_successful_try_puts( const T &t, task_list &tasks ) { +#endif // __TBB_PREVIEW_ASYNC_MSG + bool upgraded = true; + bool is_at_least_one_put_successful = false; + typename mutex_type::scoped_lock l(this->my_mutex, upgraded); + typename successors_type::iterator i = this->my_successors.begin(); + while ( i != this->my_successors.end() ) { + task * new_task = (*i)->try_put_task(t); + if(new_task) { + ++i; + if(new_task != SUCCESSFULLY_ENQUEUED) { + tasks.push_back(*new_task); + } + is_at_least_one_put_successful = true; + } + else { // failed + if ( (*i)->register_predecessor(*this->my_owner) ) { + if (!upgraded) { + l.upgrade_to_writer(); + upgraded = true; + } + i = this->my_successors.erase(i); + } else { + ++i; + } + } + } + return is_at_least_one_put_successful; + } }; //! A cache of successors that are put in a round-robin fashion diff --git a/include/tbb/internal/_flow_graph_impl.h b/include/tbb/internal/_flow_graph_impl.h index 2dc5e4c4cf..a4b96d102c 100644 --- a/include/tbb/internal/_flow_graph_impl.h +++ b/include/tbb/internal/_flow_graph_impl.h @@ -44,6 +44,16 @@ #define __TBB_FLOW_GRAPH_PRIORITY_ARG1( arg1, priority ) arg1 #endif // __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +#if TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR +#define __TBB_DEPRECATED_LIMITER_EXPR( expr ) expr +#define __TBB_DEPRECATED_LIMITER_ARG2( arg1, arg2 ) arg1, arg2 +#define __TBB_DEPRECATED_LIMITER_ARG4( arg1, arg2, arg3, arg4 ) arg1, arg3, arg4 +#else +#define __TBB_DEPRECATED_LIMITER_EXPR( expr ) +#define __TBB_DEPRECATED_LIMITER_ARG2( arg1, arg2 ) arg1 +#define __TBB_DEPRECATED_LIMITER_ARG4( arg1, arg2, arg3, arg4 ) arg1, arg2 +#endif // TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR + namespace tbb { namespace flow { @@ -154,9 +164,10 @@ namespace internal { void activate_graph(graph& g); void deactivate_graph(graph& g); bool is_graph_active(graph& g); +tbb::task& prioritize_task(graph& g, tbb::task& arena_task); void spawn_in_graph_arena(graph& g, tbb::task& arena_task); +void enqueue_in_graph_arena(graph &g, tbb::task& arena_task); void add_task_to_graph_reset_list(graph& g, tbb::task *tp); -template void execute_in_graph_arena(graph& g, F& f); #if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES struct graph_task_comparator { @@ -407,13 +418,10 @@ class graph : tbb::internal::no_copy, public tbb::flow::graph_proxy { friend void internal::activate_graph(graph& g); friend void internal::deactivate_graph(graph& g); friend bool internal::is_graph_active(graph& g); + friend tbb::task& internal::prioritize_task(graph& g, tbb::task& arena_task); friend void internal::spawn_in_graph_arena(graph& g, tbb::task& arena_task); + friend void internal::enqueue_in_graph_arena(graph &g, tbb::task& arena_task); friend void internal::add_task_to_graph_reset_list(graph& g, tbb::task *tp); - template friend void internal::execute_in_graph_arena(graph& g, F& f); -#if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES - template - friend class async_node; -#endif friend class tbb::interface7::internal::task_arena_base; @@ -459,33 +467,43 @@ inline bool is_graph_active(graph& g) { return g.my_is_active; } -//! Executes custom functor inside graph arena -template -inline void execute_in_graph_arena(graph& g, F& f) { - if (is_graph_active(g)) { - __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), NULL); - g.my_task_arena->execute(f); - } -} - -//! Spawns a task inside graph arena -inline void spawn_in_graph_arena(graph& g, tbb::task& arena_task) { - task* task_to_spawn = &arena_task; #if __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES +inline tbb::task& prioritize_task(graph& g, tbb::task& t) { + task* critical_task = &t; // TODO: change flow graph's interfaces to work with graph_task type instead of tbb::task. - graph_task* t = static_cast(&arena_task); - if( t->priority != no_priority ) { + graph_task* gt = static_cast(&t); + if( gt->priority != no_priority ) { //! Non-preemptive priority pattern. The original task is submitted as a work item to the //! priority queue, and a new critical task is created to take and execute a work item with //! the highest known priority. The reference counting responsibility is transferred (via //! allocate_continuation) to the new task. - task_to_spawn = new( t->allocate_continuation() ) priority_task_selector(g.my_priority_queue); - tbb::internal::make_critical( *task_to_spawn ); - g.my_priority_queue.push(t); + critical_task = new( gt->allocate_continuation() ) priority_task_selector(g.my_priority_queue); + tbb::internal::make_critical( *critical_task ); + g.my_priority_queue.push(gt); } + return *critical_task; +} +#else +inline tbb::task& prioritize_task(graph&, tbb::task& t) { + return t; +} #endif /* __TBB_PREVIEW_FLOW_GRAPH_PRIORITIES */ - graph::spawn_functor s_fn(*task_to_spawn); - execute_in_graph_arena(g, s_fn); + +//! Spawns a task inside graph arena +inline void spawn_in_graph_arena(graph& g, tbb::task& arena_task) { + if (is_graph_active(g)) { + graph::spawn_functor s_fn(prioritize_task(g, arena_task)); + __TBB_ASSERT(g.my_task_arena && g.my_task_arena->is_active(), NULL); + g.my_task_arena->execute(s_fn); + } +} + +//! Enqueues a task inside graph arena +inline void enqueue_in_graph_arena(graph &g, tbb::task& arena_task) { + if (is_graph_active(g)) { + __TBB_ASSERT( g.my_task_arena && g.my_task_arena->is_active(), "Is graph's arena initialized and active?" ); + task::enqueue(prioritize_task(g, arena_task), *g.my_task_arena); + } } inline void add_task_to_graph_reset_list(graph& g, tbb::task *tp) { diff --git a/include/tbb/internal/_flow_graph_join_impl.h b/include/tbb/internal/_flow_graph_join_impl.h index f7d141247d..4ccaef9f88 100644 --- a/include/tbb/internal/_flow_graph_join_impl.h +++ b/include/tbb/internal/_flow_graph_join_impl.h @@ -221,7 +221,6 @@ namespace internal { , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; typedef reserving_port class_type; class reserving_port_operation : public aggregated_operation { @@ -441,7 +440,6 @@ namespace internal { , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; class queueing_port_operation : public aggregated_operation { public: @@ -675,7 +673,6 @@ namespace internal { , add_blt_pred, del_blt_pred, blt_pred_cnt, blt_pred_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; class key_matching_port_operation : public aggregated_operation { public: @@ -1063,7 +1060,6 @@ namespace internal { // and the output_buffer_type base class private: enum op_type { res_count, inc_count, may_succeed, try_make }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; typedef join_node_FE, InputTuple, OutputTuple> class_type; class key_matching_FE_operation : public aggregated_operation { @@ -1282,7 +1278,6 @@ namespace internal { , add_blt_succ, del_blt_succ, blt_succ_cnt, blt_succ_cpy #endif }; - enum op_stat {WAIT=0, SUCCEEDED, FAILED}; typedef join_node_base class_type; class join_node_base_operation : public aggregated_operation { diff --git a/include/tbb/internal/_flow_graph_types_impl.h b/include/tbb/internal/_flow_graph_types_impl.h index 01e8914c71..e223dae6cb 100644 --- a/include/tbb/internal/_flow_graph_types_impl.h +++ b/include/tbb/internal/_flow_graph_types_impl.h @@ -44,7 +44,7 @@ namespace internal { typedef KHashp KHash; }; -// wrap each element of a tuple in a template, and make a tuple of the result. + // wrap each element of a tuple in a template, and make a tuple of the result. template class PT, typename TypeTuple> struct wrap_tuple_elements; @@ -53,6 +53,19 @@ namespace internal { template class PT, typename KeyTraits, typename TypeTuple> struct wrap_key_tuple_elements; +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_VARIADIC_TUPLE_PRESENT + template class PT, typename... Args> + struct wrap_tuple_elements >{ + typedef typename tbb::flow::tuple... > type; + }; + + template class PT, typename KeyTraits, typename... Args> + struct wrap_key_tuple_elements > { + typedef typename KeyTraits::key_type K; + typedef typename KeyTraits::hash_compare_type KHash; + typedef typename tbb::flow::tuple >... > type; + }; +#else template class PT, typename TypeTuple> struct wrap_tuple_elements<1, PT, TypeTuple> { typedef typename tbb::flow::tuple< @@ -310,6 +323,7 @@ namespace internal { PT > type; }; #endif +#endif /* __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_VARIADIC_TUPLE_PRESENT */ #if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT template< int... S > class sequence {}; diff --git a/include/tbb/internal/_node_handle_impl.h b/include/tbb/internal/_node_handle_impl.h new file mode 100644 index 0000000000..a910b5fa5c --- /dev/null +++ b/include/tbb/internal/_node_handle_impl.h @@ -0,0 +1,168 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_node_handle_H +#define __TBB_node_handle_H + +#include "_allocator_traits.h" +#include "../tbb_config.h" + + +namespace tbb { + +// This classes must be declared here for correct friendly relationship +// TODO: Consider creation some internal class to access node_handle private fields without any friendly classes +namespace interface5 { +namespace internal { + template + class split_ordered_list; + template + class concurrent_unordered_base; +} +} + +namespace interface10{ +namespace internal { + template + class concurrent_skip_list; +} +} + +namespace internal { + +template +class node_handle_base { +public: + typedef Allocator allocator_type; +protected: + typedef Node node; + typedef tbb::internal::allocator_traits traits_type; +public: + + node_handle_base() : my_node(NULL), my_allocator() {} + node_handle_base(node_handle_base&& nh) : my_node(nh.my_node), + my_allocator(std::move(nh.my_allocator)) { + nh.my_node = NULL; + } + + bool empty() const { return my_node == NULL; } + explicit operator bool() const { return my_node != NULL; } + + ~node_handle_base() { internal_destroy(); } + + node_handle_base& operator=(node_handle_base&& nh) { + internal_destroy(); + my_node = nh.my_node; + typedef typename traits_type::propagate_on_container_move_assignment pocma_type; + tbb::internal::allocator_move_assignment(my_allocator, nh.my_allocator, pocma_type()); + nh.deactivate(); + return *this; + } + + void swap(node_handle_base& nh) { + std::swap(my_node, nh.my_node); + typedef typename traits_type::propagate_on_container_swap pocs_type; + tbb::internal::allocator_swap(my_allocator, nh.my_allocator, pocs_type()); + } + + allocator_type get_allocator() const { + return my_allocator; + } + +protected: + node_handle_base(node* n) : my_node(n) {} + + void internal_destroy() { + if(my_node) { + traits_type::destroy(my_allocator, my_node->storage()); + typename tbb::internal::allocator_rebind::type node_allocator; + node_allocator.deallocate(my_node, 1); + } + } + + void deactivate() { my_node = NULL; } + + node* my_node; + allocator_type my_allocator; +}; + +// node handle for maps +template +class node_handle : public node_handle_base { + typedef node_handle_base base_type; +public: + typedef Key key_type; + typedef typename Value::second_type mapped_type; + typedef typename base_type::allocator_type allocator_type; + + node_handle() : base_type() {} + + key_type& key() const { + __TBB_ASSERT(!this->empty(), "Cannot get key from the empty node_type object"); + return *const_cast(&(this->my_node->value().first)); + } + + mapped_type& mapped() const { + __TBB_ASSERT(!this->empty(), "Cannot get mapped value from the empty node_type object"); + return this->my_node->value().second; + } + +private: + template + friend class tbb::interface5::internal::split_ordered_list; + + template + friend class tbb::interface5::internal::concurrent_unordered_base; + + template + friend class tbb::interface10::internal::concurrent_skip_list; + + node_handle(typename base_type::node* n) : base_type(n) {} +}; + +// node handle for sets +template +class node_handle : public node_handle_base { + typedef node_handle_base base_type; +public: + typedef Key value_type; + typedef typename base_type::allocator_type allocator_type; + + node_handle() : base_type() {} + + value_type& value() const { + __TBB_ASSERT(!this->empty(), "Cannot get value from the empty node_type object"); + return *const_cast(&(this->my_node->value())); + } + +private: + template + friend class tbb::interface5::internal::split_ordered_list; + + template + friend class tbb::interface5::internal::concurrent_unordered_base; + + template + friend class tbb::interface10::internal::concurrent_skip_list; + + node_handle(typename base_type::node* n) : base_type(n) {} +}; + + +}// namespace internal +}// namespace tbb + +#endif /*__TBB_node_handle_H*/ diff --git a/include/tbb/internal/_template_helpers.h b/include/tbb/internal/_template_helpers.h index ea425a1409..87c3efbeff 100644 --- a/include/tbb/internal/_template_helpers.h +++ b/include/tbb/internal/_template_helpers.h @@ -64,6 +64,23 @@ template struct is_same_type { static const bool value = template struct is_ref { static const bool value = false; }; template struct is_ref { static const bool value = true; }; +//! Partial support for std::is_integral +template struct is_integral_impl { static const bool value = false; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +#if __TBB_CPP11_PRESENT +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +#endif +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; +template<> struct is_integral_impl { static const bool value = true; }; + +template +struct is_integral : is_integral_impl::type> {}; + #if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT //! std::void_t internal implementation (to avoid GCC < 4.7 "template aliases" absence) template struct void_t { typedef void type; }; @@ -238,6 +255,11 @@ struct pack_element<0, T, Args...> { template< std::size_t N, typename... Args > using pack_element_t = typename pack_element::type; +template using is_transparent = typename Comp::is_transparent; + +template +using has_is_transparent = supports; + #endif /* __TBB_CPP11_PRESENT */ } } // namespace internal, namespace tbb diff --git a/include/tbb/iterators.h b/include/tbb/iterators.h index e834b48075..7a3c927318 100644 --- a/include/tbb/iterators.h +++ b/include/tbb/iterators.h @@ -221,6 +221,8 @@ class zip_iterator { bool operator==(const zip_iterator& it) const { return *this - it == 0; } + it_types base() const { return my_it; } + bool operator!=(const zip_iterator& it) const { return !(*this == it); } bool operator<(const zip_iterator& it) const { return *this - it < 0; } bool operator>(const zip_iterator& it) const { return it < *this; } diff --git a/include/tbb/scalable_allocator.h b/include/tbb/scalable_allocator.h index d1cf711810..a077e00c14 100644 --- a/include/tbb/scalable_allocator.h +++ b/include/tbb/scalable_allocator.h @@ -91,9 +91,12 @@ typedef enum { TBBMALLOC_USE_HUGE_PAGES, /* value turns using huge pages on and off */ /* deprecated, kept for backward compatibility only */ USE_HUGE_PAGES = TBBMALLOC_USE_HUGE_PAGES, - /* try to limit memory consumption value Bytes, clean internal buffers + /* try to limit memory consumption value (Bytes), clean internal buffers if limit is exceeded, but not prevents from requesting memory from OS */ - TBBMALLOC_SET_SOFT_HEAP_LIMIT + TBBMALLOC_SET_SOFT_HEAP_LIMIT, + /* Lower bound for the size (Bytes), that is interpreted as huge + * and not released during regular cleanup operations. */ + TBBMALLOC_SET_HUGE_SIZE_THRESHOLD } AllocationModeParam; /** Set TBB allocator-specific allocation modes. diff --git a/include/tbb/task.h b/include/tbb/task.h index 8498b20977..b062717f8c 100644 --- a/include/tbb/task.h +++ b/include/tbb/task.h @@ -1005,7 +1005,7 @@ class task_list: internal::no_copy { //! Destroys the list, but does not destroy the task objects. ~task_list() {} - //! True if list if empty; false otherwise. + //! True if list is empty; false otherwise. bool empty() const {return !first;} //! Push task onto back of list. diff --git a/include/tbb/tbb.h b/include/tbb/tbb.h index 5a91808a5f..ba4b112250 100644 --- a/include/tbb/tbb.h +++ b/include/tbb/tbb.h @@ -46,6 +46,10 @@ #include "concurrent_queue.h" #include "concurrent_unordered_map.h" #include "concurrent_unordered_set.h" +#if TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS +#include "concurrent_map.h" +#include "concurrent_set.h" +#endif #include "concurrent_vector.h" #include "critical_section.h" #include "enumerable_thread_specific.h" diff --git a/include/tbb/tbb_config.h b/include/tbb/tbb_config.h index 6d83c65ffa..dbfc9fff94 100644 --- a/include/tbb/tbb_config.h +++ b/include/tbb/tbb_config.h @@ -204,7 +204,7 @@ #define __TBB_CPP11_TEMPLATE_ALIASES_PRESENT (__INTEL_CXX11_MODE__ && __INTEL_COMPILER >= 1210) #define __TBB_CPP14_INTEGER_SEQUENCE_PRESENT (__cplusplus >= 201402L) #define __TBB_CPP14_VARIABLE_TEMPLATES_PRESENT (__cplusplus >= 201402L) - #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__INTEL_COMPILER > 1901) // a future version + #define __TBB_CPP17_DEDUCTION_GUIDES_PRESENT (__INTEL_COMPILER > 1910) // a future version #define __TBB_CPP17_INVOKE_RESULT_PRESENT (__cplusplus >= 201703L) #elif __clang__ /** TODO: these options need to be rechecked **/ @@ -509,6 +509,15 @@ There are four cases that are supported: /** Internal TBB features & modes **/ +/** __TBB_CONCURRENT_ORDERED_CONTAINERS indicates that all conditions of use + * concurrent_map and concurrent_set are met. **/ +// TODO: Add cpp11 random generation macro +#ifndef __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT + #define __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT ( __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT \ + && __TBB_IMPLICIT_MOVE_PRESENT && __TBB_CPP11_AUTO_PRESENT && __TBB_CPP11_LAMBDAS_PRESENT && __TBB_CPP11_ARRAY_PRESENT \ + && __TBB_INITIALIZER_LISTS_PRESENT ) +#endif + /** __TBB_WEAK_SYMBOLS_PRESENT denotes that the system supports the weak symbol mechanism **/ #ifndef __TBB_WEAK_SYMBOLS_PRESENT #define __TBB_WEAK_SYMBOLS_PRESENT ( !_WIN32 && !__APPLE__ && !__sun && (__TBB_GCC_VERSION >= 40000 || __INTEL_COMPILER ) ) diff --git a/include/tbb/tbb_stddef.h b/include/tbb/tbb_stddef.h index ba74e4f77c..3112f73385 100644 --- a/include/tbb/tbb_stddef.h +++ b/include/tbb/tbb_stddef.h @@ -22,7 +22,7 @@ #define TBB_VERSION_MINOR 0 // Engineering-focused interface version -#define TBB_INTERFACE_VERSION 11006 +#define TBB_INTERFACE_VERSION 11007 #define TBB_INTERFACE_VERSION_MAJOR TBB_INTERFACE_VERSION/1000 // The oldest major interface version still supported diff --git a/index.html b/index.html index f18a1a97c2..d8d3eeafa0 100644 --- a/index.html +++ b/index.html @@ -31,8 +31,6 @@

Directories

Intel TBB binary package

Directories

-
doc -
Documentation for the library.
bin
Start-up scripts for sourcing library for Linux* OS and macOS*. For Windows* OS: start-up scripts and dynamic-link libraries.
lib diff --git a/python/Makefile b/python/Makefile index 48e20e3593..ca7beb9ccf 100644 --- a/python/Makefile +++ b/python/Makefile @@ -13,6 +13,7 @@ # limitations under the License. tbb_root?=.. +BUILDING_PHASE:=0 include $(tbb_root)/build/common.inc .PHONY: all release test install test-install @@ -23,15 +24,15 @@ PY_SETUP=python $(tbb_root)/python/setup.py all: install test clean: - $(PY_SETUP) clean -b$(CURDIR) + $(PY_SETUP) clean -b$(work_dir)_release release: CC=$(compiler) release: $(SRC) rml - $(PY_SETUP) build -b$(CURDIR) -f check + $(PY_SETUP) build -b$(work_dir)_release -f check install: CC=$(compiler) install: $(SRC) rml - $(PY_SETUP) build -b$(CURDIR) install + $(PY_SETUP) build -b$(work_dir)_release build_ext -f -I$(tbb_root)/include -L$(work_dir)_release install -f test: python -m tbb test diff --git a/python/index.html b/python/index.html index a475388997..b84461907a 100644 --- a/python/index.html +++ b/python/index.html @@ -33,7 +33,7 @@

Files

Alternative entry point for Python module.
-

Build and install

 +

Build and install (source package only)

 For accessing targets defined in python/Makefile, please use src/Makefile instead and build runtime libraries before working with Python. diff --git a/python/rml/ipc_server.cpp b/python/rml/ipc_server.cpp index bbe3a3e175..50824894d0 100644 --- a/python/rml/ipc_server.cpp +++ b/python/rml/ipc_server.cpp @@ -33,7 +33,7 @@ static const char* IPC_ENABLE_VAR_NAME = "IPC_ENABLE"; typedef versioned_object::version_type version_type; -extern "C" factory::status_type __RML_open_factory(factory& f, version_type& server_version, version_type client_version) { +extern "C" factory::status_type __RML_open_factory(factory& f, version_type& /*server_version*/, version_type /*client_version*/) { if( !tbb::internal::rml::get_enable_flag( IPC_ENABLE_VAR_NAME ) ) { return factory::st_incompatible; } @@ -53,7 +53,7 @@ extern "C" factory::status_type __RML_open_factory(factory& f, version_type& ser return factory::st_success; } -extern "C" void __RML_close_factory(factory& f) { +extern "C" void __RML_close_factory(factory& /*f*/) { } class ipc_thread_monitor : public thread_monitor { @@ -1092,7 +1092,7 @@ void rml_atfork_child() { #endif /* USE_PTHREAD */ -extern "C" tbb_factory::status_type __TBB_make_rml_server(tbb_factory& f, tbb_server*& server, tbb_client& client) { +extern "C" tbb_factory::status_type __TBB_make_rml_server(tbb_factory& /*f*/, tbb_server*& server, tbb_client& client) { server = new( tbb::cache_aligned_allocator().allocate(1) ) ipc_server(client); #if USE_PTHREAD my_global_client = &client; @@ -1106,7 +1106,7 @@ extern "C" tbb_factory::status_type __TBB_make_rml_server(tbb_factory& f, tbb_se return tbb_factory::st_success; } -extern "C" void __TBB_call_with_my_server_info(::rml::server_info_callback_t cb, void* arg) { +extern "C" void __TBB_call_with_my_server_info(::rml::server_info_callback_t /*cb*/, void* /*arg*/) { } } // namespace rml diff --git a/python/tbb/__init__.py b/python/tbb/__init__.py index 728082f567..0fb79fede1 100644 --- a/python/tbb/__init__.py +++ b/python/tbb/__init__.py @@ -190,8 +190,10 @@ def __enter__(self): global is_active assert is_active == False, "tbb.Monkey does not support nesting yet" is_active = True - self.env = os.getenv('MKL_THREADING_LAYER') + self.env_mkl = os.getenv('MKL_THREADING_LAYER') os.environ['MKL_THREADING_LAYER'] = 'TBB' + self.env_numba = os.getenv('NUMBA_THREADING_LAYER') + os.environ['NUMBA_THREADING_LAYER'] = 'TBB' if ipc_enabled: if sys.version_info.major == 2 and sys.version_info.minor >= 7: @@ -205,10 +207,14 @@ def __exit__(self, exc_type, exc_value, traceback): global is_active assert is_active == True, "modified?" is_active = False - if self.env is None: + if self.env_mkl is None: del os.environ['MKL_THREADING_LAYER'] else: - os.environ['MKL_THREADING_LAYER'] = self.env + os.environ['MKL_THREADING_LAYER'] = self.env_mkl + if self.env_numba is None: + del os.environ['NUMBA_THREADING_LAYER'] + else: + os.environ['NUMBA_THREADING_LAYER'] = self.env_numba for name in self._items.keys(): setattr(self._modules[name], name, self._items[name]) diff --git a/src/perf/time_async_return.cpp b/src/perf/time_async_return.cpp index 8764bc3b02..6ffe7ff77c 100644 --- a/src/perf/time_async_return.cpp +++ b/src/perf/time_async_return.cpp @@ -63,6 +63,8 @@ The parameters are chosen so that CPU and ASYNC work take approximately the same time. */ +#define TBB_PREVIEW_FLOW_GRAPH_FEATURES __TBB_CPF_BUILD + #include "tbb/task_scheduler_init.h" #include "tbb/parallel_for.h" #include "tbb/concurrent_queue.h" diff --git a/src/tbbmalloc/backend.cpp b/src/tbbmalloc/backend.cpp index d9a37e35d6..b5d2e391b0 100644 --- a/src/tbbmalloc/backend.cpp +++ b/src/tbbmalloc/backend.cpp @@ -964,9 +964,10 @@ void *Backend::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment if (oldRegion->type != MEMREG_ONE_BLOCK) return NULL; // we are not single in the region const size_t userOffset = (uintptr_t)ptr - (uintptr_t)oldRegion; + const size_t alignedSize = LargeObjectCache::alignToBin(newSize); const size_t requestSize = - alignUp(userOffset + newSize + sizeof(LastFreeBlock), extMemPool->granularity); - if (requestSize < newSize) // is wrapped around? + alignUp(userOffset + alignedSize + sizeof(LastFreeBlock), extMemPool->granularity); + if (requestSize < alignedSize) // is wrapped around? return NULL; regionList.remove(oldRegion); @@ -978,12 +979,10 @@ void *Backend::remap(void *ptr, size_t oldSize, size_t newSize, size_t alignment MemRegion *region = (MemRegion*)ret; MALLOC_ASSERT(region->type == MEMREG_ONE_BLOCK, ASSERT_TEXT); region->allocSz = requestSize; + region->blockSz = alignedSize; FreeBlock *fBlock = (FreeBlock *)alignUp((uintptr_t)region + sizeof(MemRegion), largeObjectAlignment); - // put LastFreeBlock at the very end of region - const uintptr_t fBlockEnd = (uintptr_t)region + requestSize - sizeof(LastFreeBlock); - region->blockSz = fBlockEnd - (uintptr_t)fBlock; regionList.add(region); startUseBlock(region, fBlock, /*addToBin=*/false); diff --git a/src/tbbmalloc/backend.h b/src/tbbmalloc/backend.h index ed509157f3..2acf9a0982 100644 --- a/src/tbbmalloc/backend.h +++ b/src/tbbmalloc/backend.h @@ -14,6 +14,10 @@ limitations under the License. */ +#ifndef __TBB_tbbmalloc_internal_H + #error tbbmalloc_internal.h must be included at this point +#endif + #ifndef __TBB_backend_H #define __TBB_backend_H @@ -120,8 +124,9 @@ class Backend { VALID_BLOCK_IN_BIN = 1 // valid block added to bin, not returned as result }; public: - static const int freeBinsNum = - (maxBinned_HugePage-minBinnedSize)/LargeObjectCache::largeBlockCacheStep + 1; + // Backend bins step is the same as CacheStep for large object cache + static const size_t freeBinsStep = LargeObjectCache::LargeBSProps::CacheStep; + static const unsigned freeBinsNum = (maxBinned_HugePage-minBinnedSize)/freeBinsStep + 1; // if previous access missed per-thread slabs pool, // allocate numOfSlabAllocOnMiss blocks in advance @@ -314,7 +319,7 @@ class Backend { else if (size < minBinnedSize) return NO_BIN; - int bin = (size - minBinnedSize)/LargeObjectCache::largeBlockCacheStep; + int bin = (size - minBinnedSize)/freeBinsStep; MALLOC_ASSERT(bin < HUGE_BIN, "Invalid size."); return bin; @@ -372,7 +377,7 @@ class Backend { static size_t binToSize(int bin) { MALLOC_ASSERT(bin <= HUGE_BIN, "Invalid bin."); - return bin*LargeObjectCache::largeBlockCacheStep + minBinnedSize; + return bin*freeBinsStep + minBinnedSize; } #endif }; diff --git a/src/tbbmalloc/frontend.cpp b/src/tbbmalloc/frontend.cpp index 7d3995a785..7d18ed73e4 100644 --- a/src/tbbmalloc/frontend.cpp +++ b/src/tbbmalloc/frontend.cpp @@ -362,7 +362,7 @@ class Block : public LocalBlockFields, inline FreeObject* allocate(); inline FreeObject *allocateFromFreeList(); - inline void adjustFullness(); + inline bool adjustFullness(); void adjustPositionInBin(Bin* bin = NULL); bool freeListNonNull() { return freeList; } @@ -1314,35 +1314,34 @@ bool Bin::cleanPublicFreeLists() return released; } -void Block::adjustFullness() +bool Block::adjustFullness() { - const float threshold = (slabSize - sizeof(Block)) * (1 - emptyEnoughRatio); - if (bumpPtr) { /* If we are still using a bump ptr for this block it is empty enough to use. */ STAT_increment(getThreadId(), getIndex(objectSize), examineEmptyEnough); isFull = false; - return; - } - - /* allocatedCount shows how many objects in the block are in use; however it still counts - * blocks freed by other threads; so prior call to privatizePublicFreeList() is recommended */ - isFull = (allocatedCount*objectSize > threshold) ? true : false; + } else { + const float threshold = (slabSize - sizeof(Block)) * (1 - emptyEnoughRatio); + /* allocatedCount shows how many objects in the block are in use; however it still counts + * blocks freed by other threads; so prior call to privatizePublicFreeList() is recommended */ + isFull = (allocatedCount*objectSize > threshold) ? true : false; #if COLLECT_STATISTICS - if (isFull) - STAT_increment(getThreadId(), getIndex(objectSize), examineNotEmpty); - else - STAT_increment(getThreadId(), getIndex(objectSize), examineEmptyEnough); + if (isFull) + STAT_increment(getThreadId(), getIndex(objectSize), examineNotEmpty); + else + STAT_increment(getThreadId(), getIndex(objectSize), examineEmptyEnough); #endif + } + return isFull; } // This method resides in class Block, and not in class Bin, in order to avoid // calling getAllocationBin on a reasonably hot path in Block::freeOwnObject void Block::adjustPositionInBin(Bin* bin/*=NULL*/) { - bool fullBefore = isFull; - adjustFullness(); - if (fullBefore && !isFull) { + // If the block were full, but became empty enough to use, + // move it to the front of the list + if (isFull && !adjustFullness()) { if (!bin) bin = tlsPtr->getAllocationBin(objectSize); bin->moveBlockToFront(this); @@ -1931,21 +1930,6 @@ static MallocMutex initMutex; delivers a clean result. */ static char VersionString[] = "\0" TBBMALLOC_VERSION_STRINGS; -void AllocControlledMode::initReadEnv(const char *envName, intptr_t defaultVal) -{ - if (!setDone) { -#if !__TBB_WIN8UI_SUPPORT - // TODO: use strtol to get the actual value of the envirable - const char *envVal = getenv(envName); - if (envVal && !strcmp(envVal, "1")) - val = 1; - else -#endif - val = defaultVal; - setDone = true; - } -} - #if USE_PTHREAD && (__TBB_SOURCE_DIRECTLY_INCLUDED || __TBB_USE_DLOPEN_REENTRANCY_WORKAROUND) /* Decrease race interval between dynamic library unloading and pthread key @@ -2002,7 +1986,7 @@ bool isMallocInitializedExt() { return isMallocInitialized(); } -/** Caller is responsible for ensuring this routine is called exactly once. */ +/* Caller is responsible for ensuring this routine is called exactly once. */ extern "C" void MallocInitializeITT() { #if DO_ITT_NOTIFY if (!usedBySrcIncluded) @@ -3279,6 +3263,9 @@ extern "C" int scalable_allocation_mode(int param, intptr_t value) return TBBMALLOC_INVALID_PARAM; } #endif + } else if (param == TBBMALLOC_SET_HUGE_SIZE_THRESHOLD) { + defaultMemPool->extMemPool.loc.setHugeSizeThreshold((size_t)value); + return TBBMALLOC_OK; } return TBBMALLOC_INVALID_PARAM; } @@ -3292,7 +3279,7 @@ extern "C" int scalable_allocation_command(int cmd, void *param) switch(cmd) { case TBBMALLOC_CLEAN_THREAD_BUFFERS: if (TLSData *tls = defaultMemPool->getTLS(/*create=*/false)) - released = tls->externalCleanup(/*cleanOnlyUsed*/false, /*cleanBins=*/true); + released = tls->externalCleanup(/*cleanOnlyUnused*/false, /*cleanBins=*/true); break; case TBBMALLOC_CLEAN_ALL_BUFFERS: released = defaultMemPool->extMemPool.hardCachesCleanup(); diff --git a/src/tbbmalloc/large_objects.cpp b/src/tbbmalloc/large_objects.cpp index 4fde01d705..055d430a31 100644 --- a/src/tbbmalloc/large_objects.cpp +++ b/src/tbbmalloc/large_objects.cpp @@ -15,13 +15,51 @@ */ #include "tbbmalloc_internal.h" +#include "tbb/tbb_environment.h" -/********* Allocation of large objects ************/ - +/******************************* Allocation of large objects *********************************************/ namespace rml { namespace internal { +/* ---------------------------- Large Object cache init section ---------------------------------------- */ + +void LargeObjectCache::init(ExtMemoryPool *memPool) +{ + extMemPool = memPool; + // scalable_allocation_mode can be called before allocator initialization, respect this manual request + if (hugeSizeThreshold == 0) { + // Huge size threshold initialization if environment variable was set + long requestedThreshold = tbb::internal::GetIntegralEnvironmentVariable("TBB_MALLOC_SET_HUGE_SIZE_THRESHOLD"); + // Read valid env or initialize by default with max possible values + if (requestedThreshold != -1) { + setHugeSizeThreshold(requestedThreshold); + } else { + setHugeSizeThreshold(maxHugeSize); + } + } +} + +/* ----------------------------- Huge size threshold settings ----------------------------------------- */ + +void LargeObjectCache::setHugeSizeThreshold(size_t value) +{ + // Valid in the huge cache range: [MaxLargeSize, MaxHugeSize]. + if (value <= maxHugeSize) { + hugeSizeThreshold = value >= maxLargeSize ? alignToBin(value) : maxLargeSize; + + // Calculate local indexes for the global threshold size (for fast search inside a regular cleanup) + largeCache.hugeSizeThresholdIdx = LargeCacheType::numBins; + hugeCache.hugeSizeThresholdIdx = HugeCacheType::sizeToIdx(hugeSizeThreshold); + } +} + +bool LargeObjectCache::sizeInCacheRange(size_t size) +{ + return size <= maxHugeSize && (size <= defaultMaxHugeSize || size >= hugeSizeThreshold); +} + +/* ----------------------------------------------------------------------------------------------------- */ /* The functor called by the aggregator for the operation list */ template @@ -97,23 +135,7 @@ class CacheBinFunctor { uintptr_t getCurrTime() const { return currTime; } }; -// ---------------- Cache Bin Aggregator Operation Helpers ---------------- // -// The list of possible operations. -enum CacheBinOperationType { - CBOP_INVALID = 0, - CBOP_GET, - CBOP_PUT_LIST, - CBOP_CLEAN_TO_THRESHOLD, - CBOP_CLEAN_ALL, - CBOP_UPDATE_USED_SIZE -}; - -// The operation status list. CBST_NOWAIT can be specified for non-blocking operations. -enum CacheBinOperationStatus { - CBST_WAIT = 0, - CBST_NOWAIT, - CBST_DONE -}; +/* ---------------- Cache Bin Aggregator Operation Helpers ---------------- */ // The list of structures which describe the operation data struct OpGet { @@ -183,7 +205,8 @@ template OpTypeData& opCast(CacheBinOperation &op) { return *reinterpret_cast(&op.data); } -// ------------------------------------------------------------------------ // + +/* ------------------------------------------------------------------------ */ #if __TBB_MALLOC_LOCACHE_STAT intptr_t mallocCalls, cacheHits; @@ -382,7 +405,7 @@ template void CacheBinFunctor::operator()(CacheBinOperati #if __TBB_MALLOC_WHITEBOX_TEST tbbmalloc_whitebox::locPutProcessed+=prep.putListNum; #endif - toRelease = bin->putList(prep.head, prep.tail, bitMask, idx, prep.putListNum); + toRelease = bin->putList(prep.head, prep.tail, bitMask, idx, prep.putListNum, extMemPool->loc.hugeSizeThreshold); } needCleanup = extMemPool->loc.isCleanupNeededOnRange(timeRange, startTime); currTime = endTime - 1; @@ -414,11 +437,13 @@ template void LargeObjectCacheImpl:: CacheBinFunctor func( this, extMemPool, bitMask, idx ); aggregator.execute( op, func, longLifeTime ); - if ( LargeMemoryBlock *toRelease = func.getToRelease() ) + if ( LargeMemoryBlock *toRelease = func.getToRelease()) { extMemPool->backend.returnLargeObject(toRelease); + } - if ( func.isCleanupNeeded() ) + if ( func.isCleanupNeeded() ) { extMemPool->loc.doCleanup( func.getCurrTime(), /*doThreshDecr=*/false); + } } template LargeMemoryBlock *LargeObjectCacheImpl:: @@ -488,22 +513,24 @@ template bool LargeObjectCacheImpl:: } template void LargeObjectCacheImpl:: - CacheBin::updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx) { + CacheBin::updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx) +{ OpUpdateUsedSize data = {size}; CacheBinOperation op(data); ExecuteOperation( &op, extMemPool, bitMask, idx ); } -/* ----------------------------------------------------------------------------------------------------- */ + /* ------------------------------ Unsafe methods used with the aggregator ------------------------------ */ + template LargeMemoryBlock *LargeObjectCacheImpl:: - CacheBin::putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, int idx, int num) + CacheBin::putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, int idx, int num, size_t hugeSizeThreshold) { size_t size = head->unalignedSize; usedSize -= num*size; MALLOC_ASSERT( !last || (last->age != 0 && last->age != -1U), ASSERT_TEXT ); MALLOC_ASSERT( (tail==head && num==1) || (tail!=head && num>1), ASSERT_TEXT ); LargeMemoryBlock *toRelease = NULL; - if (!lastCleanedAge) { + if (size < hugeSizeThreshold && !lastCleanedAge) { // 1st object of such size was released. // Not cache it, and remember when this occurs // to take into account during cache miss. @@ -556,7 +583,6 @@ template LargeMemoryBlock *LargeObjectCacheImpl:: return result; } -// forget the history for the bin if it was unused for long time template void LargeObjectCacheImpl:: CacheBin::forgetOutdatedState(uintptr_t currTime) { @@ -570,9 +596,9 @@ template void LargeObjectCacheImpl:: bool doCleanup = false; if (ageThreshold) - doCleanup = sinceLastGet > Props::LongWaitFactor*ageThreshold; + doCleanup = sinceLastGet > Props::LongWaitFactor * ageThreshold; else if (lastCleanedAge) - doCleanup = sinceLastGet > Props::LongWaitFactor*(lastCleanedAge - lastGet); + doCleanup = sinceLastGet > Props::LongWaitFactor * (lastCleanedAge - lastGet); if (doCleanup) { lastCleanedAge = 0; @@ -635,6 +661,7 @@ template LargeMemoryBlock *LargeObjectCacheImpl:: return toRelease; } + /* ----------------------------------------------------------------------------------------------------- */ template size_t LargeObjectCacheImpl:: @@ -652,34 +679,40 @@ template size_t LargeObjectCacheImpl:: return cachedSize; } -// release from cache blocks that are older than ageThreshold +// Release objects from cache blocks that are older than ageThreshold template bool LargeObjectCacheImpl::regularCleanup(ExtMemoryPool *extMemPool, uintptr_t currTime, bool doThreshDecr) { bool released = false; BinsSummary binsSummary; - for (int i = bitMask.getMaxTrue(numBins-1); i >= 0; - i = bitMask.getMaxTrue(i-1)) { + // Threshold settings is below this cache or starts from zero index + if (hugeSizeThresholdIdx == 0) return false; + + // Starting searching for bin that is less than huge size threshold (can be cleaned-up) + int startSearchIdx = hugeSizeThresholdIdx - 1; + + for (int i = bitMask.getMaxTrue(startSearchIdx); i >= 0; i = bitMask.getMaxTrue(i-1)) { bin[i].updateBinsSummary(&binsSummary); - if (!doThreshDecr && tooLargeLOC>2 && binsSummary.isLOCTooLarge()) { + if (!doThreshDecr && tooLargeLOC > 2 && binsSummary.isLOCTooLarge()) { // if LOC is too large for quite long time, decrease the threshold // based on bin hit statistics. // For this, redo cleanup from the beginning. // Note: on this iteration total usedSz can be not too large // in comparison to total cachedSz, as we calculated it only // partially. We are ok with it. - i = bitMask.getMaxTrue(numBins-1)+1; + i = bitMask.getMaxTrue(startSearchIdx)+1; doThreshDecr = true; binsSummary.reset(); continue; } if (doThreshDecr) bin[i].decreaseThreshold(); - if (bin[i].cleanToThreshold(extMemPool, &bitMask, currTime, i)) + + if (bin[i].cleanToThreshold(extMemPool, &bitMask, currTime, i)) { released = true; + } } - // We want to find if LOC was too large for some time continuously, // so OK with races between incrementing and zeroing, but incrementing // must be atomic. @@ -694,11 +727,20 @@ template bool LargeObjectCacheImpl::cleanAll(ExtMemoryPool *extMemPool) { bool released = false; - for (int i = numBins-1; i >= 0; i--) + for (int i = numBins-1; i >= 0; i--) { released |= bin[i].releaseAllToBackend(extMemPool, &bitMask, i); + } return released; } +template +void LargeObjectCacheImpl::reset() { + tooLargeLOC = 0; + for (int i = numBins-1; i >= 0; i--) + bin[i].init(); + bitMask.reset(); +} + #if __TBB_MALLOC_WHITEBOX_TEST template size_t LargeObjectCacheImpl::getLOCSize() const @@ -760,11 +802,16 @@ bool LargeObjectCache::cleanAll() return largeCache.cleanAll(extMemPool) | hugeCache.cleanAll(extMemPool); } +void LargeObjectCache::reset() +{ + largeCache.reset(); + hugeCache.reset(); +} + template LargeMemoryBlock *LargeObjectCacheImpl::get(ExtMemoryPool *extMemoryPool, size_t size) { - MALLOC_ASSERT( size%Props::CacheStep==0, ASSERT_TEXT ); - int idx = sizeToIdx(size); + int idx = Props::sizeToIdx(size); LargeMemoryBlock *lmb = bin[idx].get(extMemoryPool, size, &bitMask, idx); @@ -778,7 +825,7 @@ LargeMemoryBlock *LargeObjectCacheImpl::get(ExtMemoryPool *extMemoryPool, template void LargeObjectCacheImpl::updateCacheState(ExtMemoryPool *extMemPool, DecreaseOrIncrease op, size_t size) { - int idx = sizeToIdx(size); + int idx = Props::sizeToIdx(size); MALLOC_ASSERT(idx void LargeObjectCacheImpl::putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *toCache) { - int toBinIdx = sizeToIdx(toCache->unalignedSize); + int toBinIdx = Props::sizeToIdx(toCache->unalignedSize); MALLOC_ITT_SYNC_RELEASING(bin+toBinIdx); bin[toBinIdx].putList(extMemPool, toCache, &bitMask, toBinIdx); @@ -818,19 +865,33 @@ void LargeObjectCache::updateCacheState(DecreaseOrIncrease op, size_t size) hugeCache.updateCacheState(extMemPool, op, size); } +uintptr_t LargeObjectCache::getCurrTime() +{ + return (uintptr_t)AtomicIncrement((intptr_t&)cacheCurrTime); +} + +uintptr_t LargeObjectCache::getCurrTimeRange(uintptr_t range) +{ + return (uintptr_t)AtomicAdd((intptr_t&)cacheCurrTime, range) + 1; +} + void LargeObjectCache::registerRealloc(size_t oldSize, size_t newSize) { updateCacheState(decrease, oldSize); - updateCacheState(increase, newSize); + updateCacheState(increase, alignToBin(newSize)); +} + +size_t LargeObjectCache::alignToBin(size_t size) { + return size < maxLargeSize ? LargeCacheType::alignToBin(size) : HugeCacheType::alignToBin(size); } -// return artificial bin index, it's used only during sorting and never saved +// Used for internal purpose int LargeObjectCache::sizeToIdx(size_t size) { - MALLOC_ASSERT(size < maxHugeSize, ASSERT_TEXT); - return size < maxLargeSize? + MALLOC_ASSERT(size <= maxHugeSize, ASSERT_TEXT); + return size < maxLargeSize ? LargeCacheType::sizeToIdx(size) : - LargeCacheType::getNumBins()+HugeCacheType::sizeToIdx(size); + LargeCacheType::numBins + HugeCacheType::sizeToIdx(size); } void LargeObjectCache::putList(LargeMemoryBlock *list) @@ -840,7 +901,7 @@ void LargeObjectCache::putList(LargeMemoryBlock *list) for (LargeMemoryBlock *curr = list; curr; curr = toProcess) { LargeMemoryBlock *tail = curr; toProcess = curr->next; - if (curr->unalignedSize >= maxHugeSize) { + if (!sizeInCacheRange(curr->unalignedSize)) { extMemPool->backend.returnLargeObject(curr); continue; } @@ -873,23 +934,23 @@ void LargeObjectCache::putList(LargeMemoryBlock *list) void LargeObjectCache::put(LargeMemoryBlock *largeBlock) { - if (largeBlock->unalignedSize < maxHugeSize) { + size_t blockSize = largeBlock->unalignedSize; + if (sizeInCacheRange(blockSize)) { largeBlock->next = NULL; - if (largeBlock->unalignedSizebackend.returnLargeObject(largeBlock); + } } LargeMemoryBlock *LargeObjectCache::get(size_t size) { - MALLOC_ASSERT( size%largeBlockCacheStep==0, ASSERT_TEXT ); - MALLOC_ASSERT( size>=minLargeSize, ASSERT_TEXT ); - - if ( size < maxHugeSize) { - return size < maxLargeSize? + MALLOC_ASSERT( size >= minLargeSize, ASSERT_TEXT ); + if (sizeInCacheRange(size)) { + return size < maxLargeSize ? largeCache.get(extMemPool, size) : hugeCache.get(extMemPool, size); } return NULL; @@ -959,7 +1020,7 @@ void *ExtMemoryPool::remap(void *ptr, size_t oldSize, size_t newSize, size_t ali void *o = backend.remap(ptr, oldSize, newSize, alignment); if (o) { LargeMemoryBlock *lmb = ((LargeObjectHdr*)o - 1)->memoryBlock; - loc.registerRealloc(lmb->unalignedSize, oldUnalignedSize); + loc.registerRealloc(oldUnalignedSize, lmb->unalignedSize); } return o; } diff --git a/src/tbbmalloc/large_objects.h b/src/tbbmalloc/large_objects.h new file mode 100644 index 0000000000..1e93fb7544 --- /dev/null +++ b/src/tbbmalloc/large_objects.h @@ -0,0 +1,368 @@ +/* + Copyright (c) 2005-2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#ifndef __TBB_tbbmalloc_internal_H + #error tbbmalloc_internal.h must be included at this point +#endif + +#ifndef __TBB_large_objects_H +#define __TBB_large_objects_H + +//! The list of possible Cache Bin Aggregator operations. +/* Declared here to avoid Solaris Studio* 12.2 "multiple definitions" error */ +enum CacheBinOperationType { + CBOP_INVALID = 0, + CBOP_GET, + CBOP_PUT_LIST, + CBOP_CLEAN_TO_THRESHOLD, + CBOP_CLEAN_ALL, + CBOP_UPDATE_USED_SIZE +}; + +// The Cache Bin Aggregator operation status list. +// CBST_NOWAIT can be specified for non-blocking operations. +enum CacheBinOperationStatus { + CBST_WAIT = 0, + CBST_NOWAIT, + CBST_DONE +}; + +/* + * Bins that grow with arithmetic step + */ +template +struct LargeBinStructureProps { +public: + static const size_t MinSize = MIN_SIZE, MaxSize = MAX_SIZE; + static const size_t CacheStep = 8 * 1024; + static const unsigned NumBins = (MaxSize - MinSize) / CacheStep; + + static size_t alignToBin(size_t size) { + return alignUp(size, CacheStep); + } + + static int sizeToIdx(size_t size) { + MALLOC_ASSERT(MinSize <= size && size < MaxSize, ASSERT_TEXT); + MALLOC_ASSERT(size % CacheStep == 0, ASSERT_TEXT); + return (size - MinSize) / CacheStep; + } +}; + +/* + * Bins that grow with special geometric progression. + */ +template +struct HugeBinStructureProps { + +private: + // Sizes grow with the following formula: Size = MinSize * (2 ^ (Index / StepFactor)) + // There are StepFactor bins (8 be default) between each power of 2 bin + static const int MaxSizeExp = Log2::value; + static const int MinSizeExp = Log2::value; + static const int StepFactor = 8; + static const int StepFactorExp = Log2::value; + +public: + static const size_t MinSize = MIN_SIZE, MaxSize = MAX_SIZE; + static const unsigned NumBins = (MaxSizeExp - MinSizeExp) * StepFactor; + + static size_t alignToBin(size_t size) { + size_t minorStepExp = BitScanRev(size) - StepFactorExp; + return alignUp(size, 1ULL << minorStepExp); + } + + // Sizes between the power of 2 values are aproximated to StepFactor. + static int sizeToIdx(size_t size) { + MALLOC_ASSERT(MinSize <= size && size <= MaxSize, ASSERT_TEXT); + int sizeExp = (int)BitScanRev(size); // same as __TBB_Log2 + size_t majorStepSize = 1ULL << sizeExp; + int minorStepExp = sizeExp - StepFactorExp; + int minorIdx = (size - majorStepSize) >> minorStepExp; + MALLOC_ASSERT(size == majorStepSize + ((size_t)minorIdx << minorStepExp), + "Size is not aligned on the bin"); + return StepFactor * (sizeExp - MinSizeExp) + minorIdx; + } +}; + +/* + * Cache properties accessor + * + * TooLargeFactor -- when cache size treated "too large" in comparison to user data size + * OnMissFactor -- If cache miss occurred and cache was cleaned, + * set ageThreshold to OnMissFactor * the difference + * between current time and last time cache was cleaned. + * LongWaitFactor -- to detect rarely-used bins and forget about their usage history + */ +template +struct LargeObjectCacheProps : public StructureProps { + static const int TooLargeFactor = TOO_LARGE, OnMissFactor = ON_MISS, LongWaitFactor = LONG_WAIT; +}; + +template +class LargeObjectCacheImpl { +private: + + // Current sizes of used and cached objects. It's calculated while we are + // traversing bins, and used for isLOCTooLarge() check at the same time. + class BinsSummary { + size_t usedSz; + size_t cachedSz; + public: + BinsSummary() : usedSz(0), cachedSz(0) {} + // "too large" criteria + bool isLOCTooLarge() const { return cachedSz > Props::TooLargeFactor * usedSz; } + void update(size_t usedSize, size_t cachedSize) { + usedSz += usedSize; + cachedSz += cachedSize; + } + void reset() { usedSz = cachedSz = 0; } + }; + +public: + // The number of bins to cache large/huge objects. + static const uint32_t numBins = Props::NumBins; + + typedef BitMaskMax BinBitMask; + + // 2-linked list of same-size cached blocks ordered by age (oldest on top) + // TODO: are we really want the list to be 2-linked? This allows us + // reduce memory consumption and do less operations under lock. + // TODO: try to switch to 32-bit logical time to save space in CacheBin + // and move bins to different cache lines. + class CacheBin { + private: + LargeMemoryBlock *first, + *last; + /* age of an oldest block in the list; equal to last->age, if last defined, + used for quick cheching it without acquiring the lock. */ + uintptr_t oldest; + /* currAge when something was excluded out of list because of the age, + not because of cache hit */ + uintptr_t lastCleanedAge; + /* Current threshold value for the blocks of a particular size. + Set on cache miss. */ + intptr_t ageThreshold; + + /* total size of all objects corresponding to the bin and allocated by user */ + size_t usedSize, + /* total size of all objects cached in the bin */ + cachedSize; + /* mean time of presence of block in the bin before successful reuse */ + intptr_t meanHitRange; + /* time of last get called for the bin */ + uintptr_t lastGet; + + typename MallocAggregator::type aggregator; + + void ExecuteOperation(CacheBinOperation *op, ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx, bool longLifeTime = true); + + /* should be placed in zero-initialized memory, ctor not needed. */ + CacheBin(); + + public: + void init() { + memset(this, 0, sizeof(CacheBin)); + } + + /* ---------- Cache accessors ---------- */ + void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *head, BinBitMask *bitMask, int idx); + LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); + + /* ---------- Cleanup functions -------- */ + bool cleanToThreshold(ExtMemoryPool *extMemPool, BinBitMask *bitMask, uintptr_t currTime, int idx); + bool releaseAllToBackend(ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx); + /* ------------------------------------- */ + + void updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); + void decreaseThreshold() { + if (ageThreshold) + ageThreshold = (ageThreshold + meanHitRange) / 2; + } + void updateBinsSummary(BinsSummary *binsSummary) const { + binsSummary->update(usedSize, cachedSize); + } + size_t getSize() const { return cachedSize; } + size_t getUsedSize() const { return usedSize; } + size_t reportStat(int num, FILE *f); + + /* --------- Unsafe methods used with the aggregator ------- */ + void forgetOutdatedState(uintptr_t currTime); + LargeMemoryBlock *putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, + int idx, int num, size_t hugeObjectThreshold); + LargeMemoryBlock *get(); + LargeMemoryBlock *cleanToThreshold(uintptr_t currTime, BinBitMask *bitMask, int idx); + LargeMemoryBlock *cleanAll(BinBitMask *bitMask, int idx); + void updateUsedSize(size_t size, BinBitMask *bitMask, int idx) { + if (!usedSize) bitMask->set(idx, true); + usedSize += size; + if (!usedSize && !first) bitMask->set(idx, false); + } + void updateMeanHitRange( intptr_t hitRange ) { + hitRange = hitRange >= 0 ? hitRange : 0; + meanHitRange = meanHitRange ? (meanHitRange + hitRange) / 2 : hitRange; + } + void updateAgeThreshold( uintptr_t currTime ) { + if (lastCleanedAge) + ageThreshold = Props::OnMissFactor*(currTime - lastCleanedAge); + } + void updateCachedSize(size_t size) { + cachedSize += size; + } + void setLastGet( uintptr_t newLastGet ) { + lastGet = newLastGet; + } + /* -------------------------------------------------------- */ + }; + + // Huge bins index for fast regular cleanup searching in case of + // the "huge size threshold" setting defined + intptr_t hugeSizeThresholdIdx; + +private: + // How many times LOC was "too large" + intptr_t tooLargeLOC; + // for fast finding of used bins and bins with non-zero usedSize; + // indexed from the end, as we need largest 1st + BinBitMask bitMask; + // bins with lists of recently freed large blocks cached for re-use + CacheBin bin[numBins]; + +public: + /* ------------ CacheBin structure dependent stuff ------------ */ + static size_t alignToBin(size_t size) { + return Props::alignToBin(size); + } + static int sizeToIdx(size_t size) { + return Props::sizeToIdx(size); + } + + /* --------- Main cache functions (put, get object) ------------ */ + void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *largeBlock); + LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size); + + /* ------------------------ Cleanup ---------------------------- */ + bool regularCleanup(ExtMemoryPool *extMemPool, uintptr_t currAge, bool doThreshDecr); + bool cleanAll(ExtMemoryPool *extMemPool); + + /* -------------------------- Other ---------------------------- */ + void updateCacheState(ExtMemoryPool *extMemPool, DecreaseOrIncrease op, size_t size); + + void reset(); + void reportStat(FILE *f); +#if __TBB_MALLOC_WHITEBOX_TEST + size_t getLOCSize() const; + size_t getUsedSize() const; +#endif +}; + +class LargeObjectCache { +private: + // Large bins [minLargeSize, maxLargeSize) + // Huge bins [maxLargeSize, maxHugeSize) + static const size_t minLargeSize = 8 * 1024, + maxLargeSize = 8 * 1024 * 1024, + // Cache memory up to 1TB (or 2GB for 32-bit arch), but sieve objects from the special threshold + maxHugeSize = tbb::internal::select_size_t_constant<2147483648U, 1099511627776ULL>::value; + +public: + // Upper bound threshold for caching size. After that size all objects sieve through cache + // By default - 64MB, previous value was 129MB (needed by some Intel(R) Math Kernel Library (Intel(R) MKL) benchmarks) + static const size_t defaultMaxHugeSize = 64UL * 1024UL * 1024UL; + // After that size large object interpreted as huge and does not participate in regular cleanup. + // Can be changed during the program execution. + size_t hugeSizeThreshold; + +private: + // Large objects cache properties + typedef LargeBinStructureProps LargeBSProps; + typedef LargeObjectCacheProps LargeCacheTypeProps; + + // Huge objects cache properties + typedef HugeBinStructureProps HugeBSProps; + typedef LargeObjectCacheProps HugeCacheTypeProps; + + // Cache implementation type with properties + typedef LargeObjectCacheImpl< LargeCacheTypeProps > LargeCacheType; + typedef LargeObjectCacheImpl< HugeCacheTypeProps > HugeCacheType; + + // Beginning of largeCache is more actively used and smaller than hugeCache, + // so put hugeCache first to prevent false sharing + // with LargeObjectCache's predecessor + HugeCacheType hugeCache; + LargeCacheType largeCache; + + /* logical time, incremented on each put/get operation + To prevent starvation between pools, keep separately for each pool. + Overflow is OK, as we only want difference between + its current value and some recent. + + Both malloc and free should increment logical time, as in + a different case multiple cached blocks would have same age, + and accuracy of predictors suffers. + */ + uintptr_t cacheCurrTime; + + // Memory pool that owns this LargeObjectCache. + // strict 1:1 relation, never changed + ExtMemoryPool *extMemPool; + + // Returns artificial bin index, + // it's used only during sorting and never saved + static int sizeToIdx(size_t size); + + // Our friends + friend class Backend; + +public: + void init(ExtMemoryPool *memPool); + + // Item accessors + void put(LargeMemoryBlock *largeBlock); + void putList(LargeMemoryBlock *head); + LargeMemoryBlock *get(size_t size); + + void updateCacheState(DecreaseOrIncrease op, size_t size); + bool isCleanupNeededOnRange(uintptr_t range, uintptr_t currTime); + + // Cleanup operations + bool doCleanup(uintptr_t currTime, bool doThreshDecr); + bool decreasingCleanup(); + bool regularCleanup(); + bool cleanAll(); + void reset(); + + void reportStat(FILE *f); +#if __TBB_MALLOC_WHITEBOX_TEST + size_t getLOCSize() const; + size_t getUsedSize() const; +#endif + + // Cache deals with exact-fit sizes, so need to align each size + // to the specific bin when put object to cache + static size_t alignToBin(size_t size); + + void setHugeSizeThreshold(size_t value); + + // Check if we should cache or sieve this size + bool sizeInCacheRange(size_t size); + + uintptr_t getCurrTime(); + uintptr_t getCurrTimeRange(uintptr_t range); + void registerRealloc(size_t oldSize, size_t newSize); +}; + +#endif // __TBB_large_objects_H + diff --git a/src/tbbmalloc/shared_utils.h b/src/tbbmalloc/shared_utils.h index 396404cfef..7d8c4b0b40 100644 --- a/src/tbbmalloc/shared_utils.h +++ b/src/tbbmalloc/shared_utils.h @@ -56,6 +56,14 @@ inline size_t arrayLength(const T(&)[N]) { return N; } +/* + * Compile time Log2 calculation + */ +template +struct Log2 { static const int value = 1 + Log2<(NUM >> 1)>::value; }; +template <> +struct Log2<1> { static const int value = 0; }; + #if defined(min) #undef min #endif @@ -78,6 +86,13 @@ T min ( const T& val1, const T& val2 ) { #pragma warning(disable:4510 4512 4610) #endif +#if __SUNPRO_CC + // Suppress overzealous compiler warnings that a class with a reference member + // lacks a user-defined constructor, which can lead to errors + #pragma error_messages (off, refmemnoconstr) +#endif + +// TODO: add a constructor to remove warnings suppression struct parseFileItem { const char* format; unsigned long long& value; @@ -87,6 +102,10 @@ struct parseFileItem { #pragma warning(pop) #endif +#if __SUNPRO_CC + #pragma error_messages (on, refmemnoconstr) +#endif + template void parseFile(const char* file, const parseFileItem (&items)[N]) { // Tries to find all items in each line diff --git a/src/tbbmalloc/tbbmalloc_internal.h b/src/tbbmalloc/tbbmalloc_internal.h index e0464b1d84..12d5c6cb0d 100644 --- a/src/tbbmalloc/tbbmalloc_internal.h +++ b/src/tbbmalloc/tbbmalloc_internal.h @@ -15,7 +15,7 @@ */ #ifndef __TBB_tbbmalloc_internal_H -#define __TBB_tbbmalloc_internal_H 1 +#define __TBB_tbbmalloc_internal_H #include "TypeDefinitions.h" /* Also includes customization layer Customize.h */ @@ -280,220 +280,8 @@ class OrphanedBlocks { bool cleanup(Backend* backend); }; -/* cache blocks in range [MinSize; MaxSize) in bins with CacheStep - TooLargeFactor -- when cache size treated "too large" in comparison to user data size - OnMissFactor -- If cache miss occurred and cache was cleaned, - set ageThreshold to OnMissFactor * the difference - between current time and last time cache was cleaned. - LongWaitFactor -- to detect rarely-used bins and forget about their usage history -*/ -template -struct LargeObjectCacheProps { - static const size_t MinSize = MIN_SIZE, MaxSize = MAX_SIZE; - static const uint32_t CacheStep = CACHE_STEP; - static const int TooLargeFactor = TOO_LARGE, OnMissFactor = ON_MISS, - LongWaitFactor = LONG_WAIT; -}; - -template -class LargeObjectCacheImpl { -private: - // The number of bins to cache large objects. - static const uint32_t numBins = (Props::MaxSize-Props::MinSize)/Props::CacheStep; - // Current sizes of used and cached objects. It's calculated while we are - // traversing bins, and used for isLOCTooLarge() check at the same time. - class BinsSummary { - size_t usedSz; - size_t cachedSz; - public: - BinsSummary() : usedSz(0), cachedSz(0) {} - // "too large" criteria - bool isLOCTooLarge() const { return cachedSz > Props::TooLargeFactor*usedSz; } - void update(size_t usedSize, size_t cachedSize) { - usedSz += usedSize; - cachedSz += cachedSize; - } - void reset() { usedSz = cachedSz = 0; } - }; -public: - typedef BitMaskMax BinBitMask; - - // 2-linked list of same-size cached blocks ordered by age (oldest on top) - // TODO: are we really want the list to be 2-linked? This allows us - // reduce memory consumption and do less operations under lock. - // TODO: try to switch to 32-bit logical time to save space in CacheBin - // and move bins to different cache lines. - class CacheBin { - private: - LargeMemoryBlock *first, - *last; - /* age of an oldest block in the list; equal to last->age, if last defined, - used for quick cheching it without acquiring the lock. */ - uintptr_t oldest; - /* currAge when something was excluded out of list because of the age, - not because of cache hit */ - uintptr_t lastCleanedAge; - /* Current threshold value for the blocks of a particular size. - Set on cache miss. */ - intptr_t ageThreshold; - - /* total size of all objects corresponding to the bin and allocated by user */ - size_t usedSize, - /* total size of all objects cached in the bin */ - cachedSize; - /* mean time of presence of block in the bin before successful reuse */ - intptr_t meanHitRange; - /* time of last get called for the bin */ - uintptr_t lastGet; - - typename MallocAggregator::type aggregator; - - void ExecuteOperation(CacheBinOperation *op, ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx, bool longLifeTime = true); - /* should be placed in zero-initialized memory, ctor not needed. */ - CacheBin(); - public: - void init() { memset(this, 0, sizeof(CacheBin)); } - void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *head, BinBitMask *bitMask, int idx); - LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); - bool cleanToThreshold(ExtMemoryPool *extMemPool, BinBitMask *bitMask, uintptr_t currTime, int idx); - bool releaseAllToBackend(ExtMemoryPool *extMemPool, BinBitMask *bitMask, int idx); - void updateUsedSize(ExtMemoryPool *extMemPool, size_t size, BinBitMask *bitMask, int idx); - - void decreaseThreshold() { - if (ageThreshold) - ageThreshold = (ageThreshold + meanHitRange)/2; - } - void updateBinsSummary(BinsSummary *binsSummary) const { - binsSummary->update(usedSize, cachedSize); - } - size_t getSize() const { return cachedSize; } - size_t getUsedSize() const { return usedSize; } - size_t reportStat(int num, FILE *f); - /* ---------- unsafe methods used with the aggregator ---------- */ - void forgetOutdatedState(uintptr_t currTime); - LargeMemoryBlock *putList(LargeMemoryBlock *head, LargeMemoryBlock *tail, BinBitMask *bitMask, int idx, int num); - LargeMemoryBlock *get(); - LargeMemoryBlock *cleanToThreshold(uintptr_t currTime, BinBitMask *bitMask, int idx); - LargeMemoryBlock *cleanAll(BinBitMask *bitMask, int idx); - void updateUsedSize(size_t size, BinBitMask *bitMask, int idx) { - if (!usedSize) bitMask->set(idx, true); - usedSize += size; - if (!usedSize && !first) bitMask->set(idx, false); - } - void updateMeanHitRange( intptr_t hitRange ) { - hitRange = hitRange >= 0 ? hitRange : 0; - meanHitRange = meanHitRange ? (meanHitRange + hitRange)/2 : hitRange; - } - void updateAgeThreshold( uintptr_t currTime ) { - if (lastCleanedAge) - ageThreshold = Props::OnMissFactor*(currTime - lastCleanedAge); - } - void updateCachedSize(size_t size) { cachedSize += size; } - void setLastGet( uintptr_t newLastGet ) { lastGet = newLastGet; } - /* -------------------------------------------------------- */ - }; -private: - intptr_t tooLargeLOC; // how many times LOC was "too large" - // for fast finding of used bins and bins with non-zero usedSize; - // indexed from the end, as we need largest 1st - BinBitMask bitMask; - // bins with lists of recently freed large blocks cached for re-use - CacheBin bin[numBins]; - -public: - static int sizeToIdx(size_t size) { - MALLOC_ASSERT(Props::MinSize <= size && size < Props::MaxSize, ASSERT_TEXT); - return (size-Props::MinSize)/Props::CacheStep; - } - static int getNumBins() { return numBins; } - - void putList(ExtMemoryPool *extMemPool, LargeMemoryBlock *largeBlock); - LargeMemoryBlock *get(ExtMemoryPool *extMemPool, size_t size); - - void updateCacheState(ExtMemoryPool *extMemPool, DecreaseOrIncrease op, size_t size); - bool regularCleanup(ExtMemoryPool *extMemPool, uintptr_t currAge, bool doThreshDecr); - bool cleanAll(ExtMemoryPool *extMemPool); - void reset() { - tooLargeLOC = 0; - for (int i = numBins-1; i >= 0; i--) - bin[i].init(); - bitMask.reset(); - } - void reportStat(FILE *f); -#if __TBB_MALLOC_WHITEBOX_TEST - size_t getLOCSize() const; - size_t getUsedSize() const; -#endif -}; - -class LargeObjectCache { - static const size_t minLargeSize = 8*1024, - maxLargeSize = 8*1024*1024, - // There are benchmarks of interest that should work well with objects of this size - maxHugeSize = 129*1024*1024; -public: - // Difference between object sizes in large block bins - static const uint32_t largeBlockCacheStep = 8*1024, - hugeBlockCacheStep = 512*1024; -private: - typedef LargeObjectCacheProps LargeCacheTypeProps; - typedef LargeObjectCacheProps HugeCacheTypeProps; - typedef LargeObjectCacheImpl< LargeCacheTypeProps > LargeCacheType; - typedef LargeObjectCacheImpl< HugeCacheTypeProps > HugeCacheType; - - // beginning of largeCache is more actively used and smaller than hugeCache, - // so put hugeCache first to prevent false sharing - // with LargeObjectCache's predecessor - HugeCacheType hugeCache; - LargeCacheType largeCache; - - /* logical time, incremented on each put/get operation - To prevent starvation between pools, keep separately for each pool. - Overflow is OK, as we only want difference between - its current value and some recent. - - Both malloc and free should increment logical time, as in - a different case multiple cached blocks would have same age, - and accuracy of predictors suffers. - */ - uintptr_t cacheCurrTime; - - // memory pool that owns this LargeObjectCache, - ExtMemoryPool *extMemPool; // strict 1:1 relation, never changed - - static int sizeToIdx(size_t size); -public: - void init(ExtMemoryPool *memPool) { extMemPool = memPool; } - void put(LargeMemoryBlock *largeBlock); - void putList(LargeMemoryBlock *head); - LargeMemoryBlock *get(size_t size); - - void updateCacheState(DecreaseOrIncrease op, size_t size); - bool isCleanupNeededOnRange(uintptr_t range, uintptr_t currTime); - bool doCleanup(uintptr_t currTime, bool doThreshDecr); - - bool decreasingCleanup(); - bool regularCleanup(); - bool cleanAll(); - void reset() { - largeCache.reset(); - hugeCache.reset(); - } - void reportStat(FILE *f); -#if __TBB_MALLOC_WHITEBOX_TEST - size_t getLOCSize() const; - size_t getUsedSize() const; -#endif - static size_t alignToBin(size_t size) { - return size base_allocator; +typedef tbb::internal::traits_true_type true_type; +typedef tbb::internal::traits_false_type false_type; + +typedef propagating_allocator always_propagating_allocator; +typedef propagating_allocator never_propagating_allocator; +typedef propagating_allocator pocma_allocator; +typedef propagating_allocator pocca_allocator; +typedef propagating_allocator pocs_allocator; +} + template void swap(propagating_allocator& lhs, propagating_allocator&) { @@ -722,6 +739,95 @@ void swap(propagating_allocator& lhs, *lhs.propagated_on_swap = true; } +template +void test_allocator_traits_support() { + typedef typename ContainerType::allocator_type allocator_type; + typedef std::allocator_traits allocator_traits; + typedef typename allocator_traits::propagate_on_container_copy_assignment pocca_type; +#if __TBB_CPP11_RVALUE_REF_PRESENT + typedef typename allocator_traits::propagate_on_container_move_assignment pocma_type; +#endif + typedef typename allocator_traits::propagate_on_container_swap pocs_type; + + bool propagated_on_copy = false; + bool propagated_on_move = false; + bool propagated_on_swap = false; + bool selected_on_copy = false; + + allocator_type alloc(propagated_on_copy, propagated_on_move, propagated_on_swap, selected_on_copy); + + ContainerType c1(alloc), c2(c1); + ASSERT(selected_on_copy, "select_on_container_copy_construction function was not called"); + + c1 = c2; + ASSERT(propagated_on_copy == pocca_type::value, "Unexpected allocator propagation on copy assignment"); + +#if __TBB_CPP11_RVALUE_REF_PRESENT + c2 = std::move(c1); + ASSERT(propagated_on_move == pocma_type::value, "Unexpected allocator propagation on move assignment"); +#endif + + c1.swap(c2); + ASSERT(propagated_on_swap == pocs_type::value, "Unexpected allocator propagation on swap"); +} + +#if __TBB_CPP11_RVALUE_REF_PRESENT +class non_movable_object { + non_movable_object() {} +private: + non_movable_object(non_movable_object&&); + non_movable_object& operator=(non_movable_object&&); +}; + +template +void test_allocator_traits_with_non_movable_value_type() { + // Check, that if pocma is true, container allows move assignment without per-element move + typedef typename ContainerType::allocator_type allocator_type; + typedef std::allocator_traits allocator_traits; + typedef typename allocator_traits::propagate_on_container_move_assignment pocma_type; + ASSERT(pocma_type::value, "Allocator POCMA must be true for this test"); + allocator_type alloc; + ContainerType container1(alloc), container2(alloc); + container1 = std::move(container2); +} +#endif // __TBB_CPP11_RVALUE_REF_PRESENT + +#endif // __TBB_ALLOCATOR_TRAITS_PRESENT + +#if __TBB_CPP11_RVALUE_REF_PRESENT + +template +class allocator_aware_data { +public: + static bool assert_on_constructions; + typedef Allocator allocator_type; + + allocator_aware_data(const allocator_type& allocator = allocator_type()) + : my_allocator(allocator), my_value(0) {} + allocator_aware_data(int v, const allocator_type& allocator = allocator_type()) + : my_allocator(allocator), my_value(v) {} + allocator_aware_data(const allocator_aware_data&) { + ASSERT(!assert_on_constructions, "Allocator should propogate to the data during copy construction"); + } + allocator_aware_data(allocator_aware_data&&) { + ASSERT(!assert_on_constructions, "Allocator should propogate to the data during move construction"); + } + allocator_aware_data(const allocator_aware_data& rhs, const allocator_type& allocator) + : my_allocator(allocator), my_value(rhs.my_value) {} + allocator_aware_data(allocator_aware_data&& rhs, const allocator_type& allocator) + : my_allocator(allocator), my_value(rhs.my_value) {} + + int value() const { return my_value; } +private: + allocator_type my_allocator; + int my_value; +}; + +template +bool allocator_aware_data::assert_on_constructions = false; + +#endif // __TBB_CPP11_RVALUE_REF_PRESENT + #if defined(_MSC_VER) && !defined(__INTEL_COMPILER) // Workaround for overzealous compiler warnings #pragma warning (pop) diff --git a/src/test/harness_defs.h b/src/test/harness_defs.h index 9195c72b66..f7363dad5a 100644 --- a/src/test/harness_defs.h +++ b/src/test/harness_defs.h @@ -201,6 +201,9 @@ #ifndef TBB_PREVIEW_ALGORITHM_TRACE #define TBB_PREVIEW_ALGORITHM_TRACE 1 #endif + #ifndef TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR + #define TBB_DEPRECATED_LIMITER_NODE_CONSTRUCTOR 1 + #endif #endif namespace Harness { diff --git a/src/test/test_async_node.cpp b/src/test/test_async_node.cpp index 25662504aa..602f4f9a21 100644 --- a/src/test/test_async_node.cpp +++ b/src/test/test_async_node.cpp @@ -276,14 +276,19 @@ class async_activity : NoAssign { async_activity* my_activity; }; - async_activity(int expected_items, int sleep_time = 50) : my_expected_items(expected_items), my_sleep_time(sleep_time) { + async_activity(int expected_items, bool deferred = false, int sleep_time = 50) + : my_expected_items(expected_items), my_sleep_time(sleep_time) { + is_active = !deferred; my_quit = false; tbb::tbb_thread( ServiceThreadBody( this ) ).swap( my_service_thread ); } private: - async_activity( const async_activity& ) : my_expected_items(UNKNOWN_NUMBER_OF_ITEMS), my_sleep_time(0) { } + async_activity( const async_activity& ) + : my_expected_items(UNKNOWN_NUMBER_OF_ITEMS), my_sleep_time(0) { + is_active = true; + } public: ~async_activity() { @@ -299,14 +304,15 @@ class async_activity : NoAssign { void process() { do { work_type work; - if( my_work_queue.try_pop( work ) ) { + if( is_active && my_work_queue.try_pop( work ) ) { Harness::Sleep(my_sleep_time); ++async_activity_processed_msg_count; output_type output; wrapper_helper::copy_value(work.input, output); wrapper_helper::check(work.input, output); work.gateway->try_put(output); - if ( my_expected_items == UNKNOWN_NUMBER_OF_ITEMS || int(async_activity_processed_msg_count) == my_expected_items ) { + if ( my_expected_items == UNKNOWN_NUMBER_OF_ITEMS || + int(async_activity_processed_msg_count) == my_expected_items ) { work.gateway->release_wait(); } } @@ -317,6 +323,10 @@ class async_activity : NoAssign { my_quit = true; } + void activate() { + is_active = true; + } + bool should_reserve_each_time() { if ( my_expected_items == UNKNOWN_NUMBER_OF_ITEMS ) return true; @@ -328,6 +338,7 @@ class async_activity : NoAssign { const int my_expected_items; const int my_sleep_time; + tbb::atomic< bool > is_active; tbb::concurrent_queue< work_type > my_work_queue; @@ -538,7 +549,7 @@ struct spin_test { spin_test() {} static int run(int nthreads, int async_expected_items = UNKNOWN_NUMBER_OF_ITEMS) { - async_activity my_async_activity(async_expected_items, 0); + async_activity my_async_activity(async_expected_items, false, 0); Harness::SpinBarrier spin_barrier(nthreads); tbb::flow::graph g; tbb::flow::function_node< int, input_type > start_node( g, tbb::flow::unlimited, start_body_type() ); @@ -595,6 +606,88 @@ int run_tests() { return Harness::Done; } +#include "tbb/parallel_for.h" +template +class equeueing_on_inner_level { + typedef Input input_type; + typedef Output output_type; + typedef async_activity async_activity_type; + typedef tbb::flow::async_node async_node_type; + typedef typename async_node_type::gateway_type gateway_type; + + class start_body_type { + public: + input_type operator() ( int input ) { + return input_type( input); + } + }; + + class async_body_type { + public: + async_body_type( async_activity_type& activity ) : my_async_activity(&activity) {} + + void operator() ( const input_type &input, gateway_type& gateway ) { + gateway.reserve_wait(); + my_async_activity->submit( input, gateway ); + } + private: + async_activity_type* my_async_activity; + }; + + class end_body_type { + public: + void operator()( output_type ) {} + }; + + class body_graph_with_async { + public: + body_graph_with_async( Harness::SpinBarrier& barrier, async_activity_type& activity ) + : spin_barrier(&barrier), my_async_activity(&activity) {} + + void operator()(int) const { + tbb::flow::graph g; + tbb::flow::function_node< int, input_type > start_node( g, tbb::flow::unlimited, start_body_type() ); + + async_node_type offload_node( g, tbb::flow::unlimited, async_body_type( *my_async_activity ) ); + + tbb::flow::function_node< output_type > end_node( g, tbb::flow::unlimited, end_body_type() ); + + tbb::flow::make_edge( start_node, offload_node ); + tbb::flow::make_edge( offload_node, end_node ); + + start_node.try_put(1); + + spin_barrier->wait(); + + my_async_activity->activate(); + + g.wait_for_all(); + } + + private: + Harness::SpinBarrier* spin_barrier; + async_activity_type* my_async_activity; + }; + + +public: + static int run () + { + const int nthreads = tbb::this_task_arena::max_concurrency(); + Harness::SpinBarrier spin_barrier( nthreads ); + + async_activity_type my_async_activity( UNKNOWN_NUMBER_OF_ITEMS, true ); + + tbb::parallel_for( 0, nthreads, body_graph_with_async( spin_barrier, my_async_activity ) ); + return Harness::Done; + } +}; + +int run_test_equeueing_on_inner_level() { + equeueing_on_inner_level::run(); + return Harness::Done; +} + int TestMain() { tbb::task_scheduler_init init(4); run_tests(); @@ -606,6 +699,7 @@ int TestMain() { test_reset(); test_copy_ctor(); test_for_spin_avoidance(); + run_test_equeueing_on_inner_level(); return Harness::Done; } diff --git a/src/test/test_concurrent_associative_common.h b/src/test/test_concurrent_associative_common.h new file mode 100644 index 0000000000..382827e324 --- /dev/null +++ b/src/test/test_concurrent_associative_common.h @@ -0,0 +1,1488 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* Some tests in this source file are based on PPL tests provided by Microsoft. */ +#include "tbb/parallel_for.h" +#include "tbb/tick_count.h" +#include "harness.h" +#include "test_container_move_support.h" +// Test that unordered containers do not require keys have default constructors. +#define __HARNESS_CHECKTYPE_DEFAULT_CTOR 0 +#include "harness_checktype.h" +#undef __HARNESS_CHECKTYPE_DEFAULT_CTOR +#include "harness_allocator.h" + +#if _MSC_VER +#pragma warning(disable: 4189) // warning 4189 -- local variable is initialized but not referenced +#pragma warning(disable: 4127) // warning 4127 -- while (true) has a constant expression in it +#endif + +// TestInitListSupportWithoutAssign with an empty initializer list causes internal error in Intel Compiler. +#define __TBB_ICC_EMPTY_INIT_LIST_TESTS_BROKEN (__INTEL_COMPILER && __INTEL_COMPILER <= 1500) + +typedef local_counting_allocator,std::allocator> > MyAllocator; + +template +inline void CheckAllocator(typename Table::allocator_type& a, size_t expected_allocs, size_t expected_frees, + bool exact = true) { + if(exact) { + ASSERT( a.allocations == expected_allocs, NULL); ASSERT( a.frees == expected_frees, NULL); + } else { + ASSERT( a.allocations >= expected_allocs, NULL); ASSERT( a.frees >= expected_frees, NULL); + ASSERT( a.allocations - a.frees == expected_allocs - expected_frees, NULL ); + } +} + +// Check that only dummy node allocated if table is empty +// Specialize this function for custom container, if it node allocation size > 1 +#define CheckEmptyContainerAllocatorE(t,a,f) CheckEmptyContainerAllocator(t,a,f,true,__LINE__) +#define CheckEmptyContainerAllocatorA(t,a,f) CheckEmptyContainerAllocator(t,a,f,false,__LINE__) +template +inline void CheckEmptyContainerAllocator(MyTable &table, size_t expected_allocs, size_t expected_frees, bool exact = true, int line = 0); + +template +struct strip_const { typedef T type; }; + +template +struct strip_const { typedef T type; }; + +// value generator for map +template > +struct ValueFactory { + typedef typename strip_const::type Kstrip; + static V make(const K &value) { return V(value, value); } + static Kstrip key(const V &value) { return value.first; } + static Kstrip get(const V &value) { return (Kstrip)value.second; } + template< typename U > + static U convert(const V &value) { return U(value.second); } +}; + +// generator for set +template +struct ValueFactory { + static T make(const T &value) { return value; } + static T key(const T &value) { return value; } + static T get(const T &value) { return value; } + template< typename U > + static U convert(const T &value) { return U(value); } +}; + +template +struct Value : ValueFactory { + template + static bool compare( const typename T::iterator& it, U val ) { + return (Value::template convert(*it) == val); + } +}; + +template +void check_value_state(/* typename do_check_element_state =*/ tbb::internal::true_type, T const& t, const char* filename, int line ) +{ + ASSERT_CUSTOM(is_state_f()(t), "", filename, line); +} + +template +void check_value_state(/* typename do_check_element_state =*/ tbb::internal::false_type, T const&, const char* , int ) {/*do nothing*/} + +#define ASSERT_VALUE_STATE(do_check_element_state,state,value) check_value_state(do_check_element_state,value,__FILE__,__LINE__) + +#if __TBB_CPP11_RVALUE_REF_PRESENT +template +void test_rvalue_insert(V v1, V v2) +{ + typedef T container_t; + + container_t cont; + + std::pair ins = cont.insert(Value::make(v1)); + ASSERT(ins.second == true && Value::get(*(ins.first)) == v1, "Element 1 has not been inserted properly"); + ASSERT_VALUE_STATE(do_check_element_state(),Harness::StateTrackableBase::MoveInitialized,*ins.first); + + typename container_t::iterator it2 = cont.insert(ins.first, Value::make(v2)); + ASSERT(Value::get(*(it2)) == v2, "Element 2 has not been inserted properly"); + ASSERT_VALUE_STATE(do_check_element_state(),Harness::StateTrackableBase::MoveInitialized,*it2); + +} +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT +// The test does not use variadic templates, but emplace() does. + +namespace emplace_helpers { +template +std::pair call_emplace_impl(container_t& c, arg_t&& k, value_t *){ + // this is a set + return c.emplace(std::forward(k)); +} + +template +std::pair call_emplace_impl(container_t& c, arg_t&& k, std::pair *){ + // this is a map + return c.emplace(k, std::forward(k)); +} + +template +std::pair call_emplace(container_t& c, arg_t&& k){ + typename container_t::value_type * selector = NULL; + return call_emplace_impl(c, std::forward(k), selector); +} + +template +typename container_t::iterator call_emplace_hint_impl(container_t& c, typename container_t::const_iterator hint, arg_t&& k, value_t *){ + // this is a set + return c.emplace_hint(hint, std::forward(k)); +} + +template +typename container_t::iterator call_emplace_hint_impl(container_t& c, typename container_t::const_iterator hint, arg_t&& k, std::pair *){ + // this is a map + return c.emplace_hint(hint, k, std::forward(k)); +} + +template +typename container_t::iterator call_emplace_hint(container_t& c, typename container_t::const_iterator hint, arg_t&& k){ + typename container_t::value_type * selector = NULL; + return call_emplace_hint_impl(c, hint, std::forward(k), selector); +} +} +template +void test_emplace_insert(V v1, V v2){ + typedef T container_t; + container_t cont; + + std::pair ins = emplace_helpers::call_emplace(cont, v1); + ASSERT(ins.second == true && Value::compare(ins.first, v1), "Element 1 has not been inserted properly"); + ASSERT_VALUE_STATE(do_check_element_state(),Harness::StateTrackableBase::DirectInitialized,*ins.first); + + typename container_t::iterator it2 = emplace_helpers::call_emplace_hint(cont, ins.first, v2); + ASSERT(Value::compare(it2, v2), "Element 2 has not been inserted properly"); + ASSERT_VALUE_STATE(do_check_element_state(),Harness::StateTrackableBase::DirectInitialized,*it2); +} +#endif //__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT +#endif // __TBB_CPP11_RVALUE_REF_PRESENT + +template +std::pair CheckRecursiveRange(RangeType range) { + std::pair sum(0, 0); // count, sum + for( Iterator i = range.begin(), e = range.end(); i != e; ++i ) { + ++sum.first; sum.second += Value::get(*i); + } + if( range.is_divisible() ) { + RangeType range2( range, tbb::split() ); + std::pair sum1 = CheckRecursiveRange( range ); + std::pair sum2 = CheckRecursiveRange( range2 ); + sum1.first += sum2.first; sum1.second += sum2.second; + ASSERT( sum == sum1, "Mismatched ranges after division"); + } + return sum; +} + +template +void SpecialMapTests( const char *str ){ + Map cont; + const Map &ccont( cont ); + + // mapped_type& operator[](const key_type& k); + cont[1] = 2; + + // bool empty() const; + ASSERT( !ccont.empty( ), "Concurrent container empty after adding an element" ); + + // size_type size() const; + ASSERT( ccont.size( ) == 1, "Concurrent container size incorrect" ); + ASSERT( cont[1] == 2, "Concurrent container value incorrect" ); + + // mapped_type& at( const key_type& k ); + // const mapped_type& at(const key_type& k) const; + ASSERT( cont.at( 1 ) == 2, "Concurrent container value incorrect" ); + ASSERT( ccont.at( 1 ) == 2, "Concurrent container value incorrect" ); + + // iterator find(const key_type& k); + typename Map::iterator it = cont.find( 1 ); + ASSERT( it != cont.end( ) && Value::get( *(it) ) == 2, "Element with key 1 not properly found" ); + cont.unsafe_erase( it ); + + it = cont.find( 1 ); + ASSERT( it == cont.end( ), "Element with key 1 not properly erased" ); + REMARK( "passed -- specialized %s tests\n", str ); +} + +template +void CheckMultiMap(MultiMap &m, int *targets, int tcount, int key) { + std::vector vfound(tcount,false); + std::pair range = m.equal_range( key ); + for(typename MultiMap::iterator it = range.first; it != range.second; ++it) { + bool found = false; + for( int i = 0; i < tcount; ++i) { + if((*it).second == targets[i]) { + if(!vfound[i]) { // we can insert duplicate values + vfound[i] = found = true; + break; + } + } + } + // just in case an extra value in equal_range... + ASSERT(found, "extra value from equal range"); + } + for(int i = 0; i < tcount; ++i) ASSERT(vfound[i], "missing value"); +} + +template +void SpecialMultiMapTests( const char *str ){ + int one_values[] = { 7, 2, 13, 23, 13 }; + int zero_values[] = { 4, 9, 13, 29, 42, 111}; + int n_zero_values = sizeof(zero_values) / sizeof(int); + int n_one_values = sizeof(one_values) / sizeof(int); + MultiMap cont; + const MultiMap &ccont( cont ); + // mapped_type& operator[](const key_type& k); + cont.insert( std::make_pair( 1, one_values[0] ) ); + + // bool empty() const; + ASSERT( !ccont.empty( ), "Concurrent container empty after adding an element" ); + + // size_type size() const; + ASSERT( ccont.size( ) == 1, "Concurrent container size incorrect" ); + ASSERT( (*(cont.begin( ))).second == one_values[0], "Concurrent container value incorrect" ); + ASSERT( (*(cont.equal_range( 1 )).first).second == one_values[0], "Improper value from equal_range" ); + ASSERT( (cont.equal_range( 1 )).second == cont.end( ), "Improper iterator from equal_range" ); + + cont.insert( std::make_pair( 1, one_values[1] ) ); + + // bool empty() const; + ASSERT( !ccont.empty( ), "Concurrent container empty after adding an element" ); + + // size_type size() const; + ASSERT( ccont.size( ) == 2, "Concurrent container size incorrect" ); + CheckMultiMap(cont, one_values, 2, 1); + + // insert the other {1,x} values + for( int i = 2; i < n_one_values; ++i ) { + cont.insert( std::make_pair( 1, one_values[i] ) ); + } + + CheckMultiMap(cont, one_values, n_one_values, 1); + ASSERT( (cont.equal_range( 1 )).second == cont.end( ), "Improper iterator from equal_range" ); + + cont.insert( std::make_pair( 0, zero_values[0] ) ); + + // bool empty() const; + ASSERT( !ccont.empty( ), "Concurrent container empty after adding an element" ); + + // size_type size() const; + ASSERT( ccont.size( ) == (size_t)(n_one_values+1), "Concurrent container size incorrect" ); + CheckMultiMap(cont, one_values, n_one_values, 1); + CheckMultiMap(cont, zero_values, 1, 0); + ASSERT( (*(cont.begin( ))).second == zero_values[0], "Concurrent container value incorrect" ); + // insert the rest of the zero values + for( int i = 1; i < n_zero_values; ++i) { + cont.insert( std::make_pair( 0, zero_values[i] ) ); + } + CheckMultiMap(cont, one_values, n_one_values, 1); + CheckMultiMap(cont, zero_values, n_zero_values, 0); + + // clear, reinsert interleaved + cont.clear(); + int bigger_num = ( n_one_values > n_zero_values ) ? n_one_values : n_zero_values; + for( int i = 0; i < bigger_num; ++i ) { + if(i < n_one_values) cont.insert( std::make_pair( 1, one_values[i] ) ); + if(i < n_zero_values) cont.insert( std::make_pair( 0, zero_values[i] ) ); + } + CheckMultiMap(cont, one_values, n_one_values, 1); + CheckMultiMap(cont, zero_values, n_zero_values, 0); + + + REMARK( "passed -- specialized %s tests\n", str ); +} + +template +struct SpecialTests { + static void Test(const char *str) {REMARK("skipped -- specialized %s tests\n", str);} +}; + + + +#if __TBB_RANGE_BASED_FOR_PRESENT +#include "test_range_based_for.h" + +template +void TestRangeBasedFor() { + using namespace range_based_for_support_tests; + + REMARK( "testing range based for loop compatibility \n" ); + Container cont; + const int sequence_length = 100; + for ( int i = 1; i <= sequence_length; ++i ) { + cont.insert( Value::make(i) ); + } + + ASSERT( range_based_for_accumulate( cont, unified_summer(), 0 ) == + gauss_summ_of_int_sequence( sequence_length ), + "incorrect accumulated value generated via range based for ?" ); +} +#endif /* __TBB_RANGE_BASED_FOR_PRESENT */ + +#if __TBB_INITIALIZER_LISTS_PRESENT +// Required by test_initializer_list.h +template +bool equal_containers(container_type const& lhs, container_type const& rhs) { + if ( lhs.size() != rhs.size() ) { + return false; + } + return std::equal( lhs.begin(), lhs.end(), rhs.begin(), Harness::IsEqual() ); +} + +#include "test_initializer_list.h" + +template +void TestInitList( std::initializer_list il ) { + using namespace initializer_list_support_tests; + REMARK("testing initializer_list methods \n"); + + TestInitListSupportWithoutAssign(il); + TestInitListSupportWithoutAssign( il ); + +#if __TBB_ICC_EMPTY_INIT_LIST_TESTS_BROKEN + REPORT( "Known issue: TestInitListSupportWithoutAssign with an empty initializer list is skipped.\n"); +#else + TestInitListSupportWithoutAssign( {} ); + TestInitListSupportWithoutAssign( {} ); +#endif +} +#endif //if __TBB_INITIALIZER_LISTS_PRESENT + +template +void test_basic_common(const char * str, do_check_element_state) +{ + T cont; + const T &ccont(cont); + CheckEmptyContainerAllocatorE(cont, 1, 0); // one dummy is always allocated + // bool empty() const; + ASSERT(ccont.empty(), "Concurrent container is not empty after construction"); + + // size_type size() const; + ASSERT(ccont.size() == 0, "Concurrent container is not empty after construction"); + + // size_type max_size() const; + ASSERT(ccont.max_size() > 0, "Concurrent container max size is invalid"); + + //iterator begin(); + //iterator end(); + ASSERT(cont.begin() == cont.end(), "Concurrent container iterators are invalid after construction"); + ASSERT(ccont.begin() == ccont.end(), "Concurrent container iterators are invalid after construction"); + ASSERT(cont.cbegin() == cont.cend(), "Concurrent container iterators are invalid after construction"); + + //std::pair insert(const value_type& obj); + std::pair ins = cont.insert(Value::make(1)); + ASSERT(ins.second == true && Value::get(*(ins.first)) == 1, "Element 1 has not been inserted properly"); + +#if __TBB_CPP11_RVALUE_REF_PRESENT + test_rvalue_insert(1,2); +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + test_emplace_insert(1,2); +#endif // __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT +#endif // __TBB_CPP11_RVALUE_REF_PRESENT + + // bool empty() const; + ASSERT(!ccont.empty(), "Concurrent container is empty after adding an element"); + + // size_type size() const; + ASSERT(ccont.size() == 1, "Concurrent container size is incorrect"); + + std::pair ins2 = cont.insert(Value::make(1)); + + if (T::allow_multimapping) + { + // std::pair insert(const value_type& obj); + ASSERT(ins2.second == true && Value::get(*(ins2.first)) == 1, "Element 1 has not been inserted properly"); + + // size_type size() const; + ASSERT(ccont.size() == 2, "Concurrent container size is incorrect"); + + // size_type count(const key_type& k) const; + ASSERT(ccont.count(1) == 2, "Concurrent container count(1) is incorrect"); + // std::pair equal_range(const key_type& k); + std::pair range = cont.equal_range(1); + typename T::iterator it = range.first; + ASSERT(it != cont.end() && Value::get(*it) == 1, "Element 1 has not been found properly"); + unsigned int count = 0; + for (; it != range.second; it++) + { + count++; + ASSERT(Value::get(*it) == 1, "Element 1 has not been found properly"); + } + + ASSERT(count == 2, "Range doesn't have the right number of elements"); + } + else + { + // std::pair insert(const value_type& obj); + ASSERT(ins2.second == false && ins2.first == ins.first, "Element 1 should not be re-inserted"); + + // size_type size() const; + ASSERT(ccont.size() == 1, "Concurrent container size is incorrect"); + + // size_type count(const key_type& k) const; + ASSERT(ccont.count(1) == 1, "Concurrent container count(1) is incorrect"); + + // std::pair equal_range(const key_type& k) const; + // std::pair equal_range(const key_type& k); + std::pair range = cont.equal_range(1); + typename T::iterator it = range.first; + ASSERT(it != cont.end() && Value::get(*it) == 1, "Element 1 has not been found properly"); + ASSERT(++it == range.second, "Range doesn't have the right number of elements"); + } + + // const_iterator find(const key_type& k) const; + // iterator find(const key_type& k); + typename T::iterator it = cont.find(1); + ASSERT(it != cont.end() && Value::get(*(it)) == 1, "Element 1 has not been found properly"); + ASSERT(ccont.find(1) == it, "Element 1 has not been found properly"); + + // Will be implemented in unordered containers later +#if !__TBB_UNORDERED_TEST + //bool contains(const key_type&k) const + ASSERT(cont.contains(1), "contains() cannot detect existing element"); + ASSERT(!cont.contains(0), "contains() detect not existing element"); +#endif /*__TBB_UNORDERED_TEST*/ + + // iterator insert(const_iterator hint, const value_type& obj); + typename T::iterator it2 = cont.insert(ins.first, Value::make(2)); + ASSERT(Value::get(*it2) == 2, "Element 2 has not been inserted properly"); + + // T(const T& _Umap) + T newcont = ccont; + ASSERT(T::allow_multimapping ? (newcont.size() == 3) : (newcont.size() == 2), "Copy construction has not copied the elements properly"); + + // this functionality not implemented yet + // size_type unsafe_erase(const key_type& k); + typename T::size_type size = cont.unsafe_erase(1); + ASSERT(T::allow_multimapping ? (size == 2) : (size == 1), "Erase has not removed the right number of elements"); + + // iterator unsafe_erase(const_iterator position); + typename T::iterator it4 = cont.unsafe_erase(cont.find(2)); + ASSERT(it4 == cont.end() && cont.size() == 0, "Erase has not removed the last element properly"); + + // template void insert(InputIterator first, InputIterator last); + cont.insert(newcont.begin(), newcont.end()); + ASSERT(T::allow_multimapping ? (cont.size() == 3) : (cont.size() == 2), "Range insert has not copied the elements properly"); + + // this functionality not implemented yet + // iterator unsafe_erase(const_iterator first, const_iterator last); + std::pair range2 = newcont.equal_range(1); + newcont.unsafe_erase(range2.first, range2.second); + ASSERT(newcont.size() == 1, "Range erase has not erased the elements properly"); + + // void clear(); + newcont.clear(); + ASSERT(newcont.begin() == newcont.end() && newcont.size() == 0, "Clear has not cleared the container"); + +#if __TBB_INITIALIZER_LISTS_PRESENT +#if __TBB_CPP11_INIT_LIST_TEMP_OBJS_LIFETIME_BROKEN + REPORT("Known issue: the test for insert with initializer_list is skipped.\n"); +#else + // void insert(const std::initializer_list &il); + newcont.insert( { Value::make( 1 ), Value::make( 2 ), Value::make( 1 ) } ); + if (T::allow_multimapping) { + ASSERT(newcont.size() == 3, "Concurrent container size is incorrect"); + ASSERT(newcont.count(1) == 2, "Concurrent container count(1) is incorrect"); + ASSERT(newcont.count(2) == 1, "Concurrent container count(2) is incorrect"); + std::pair range = cont.equal_range(1); + it = range.first; + ASSERT(it != newcont.end() && Value::get(*it) == 1, "Element 1 has not been found properly"); + unsigned int count = 0; + for (; it != range.second; it++) { + count++; + ASSERT(Value::get(*it) == 1, "Element 1 has not been found properly"); + } + ASSERT(count == 2, "Range doesn't have the right number of elements"); + range = newcont.equal_range(2); it = range.first; + ASSERT(it != newcont.end() && Value::get(*it) == 2, "Element 2 has not been found properly"); + count = 0; + for (; it != range.second; it++) { + count++; + ASSERT(Value::get(*it) == 2, "Element 2 has not been found properly"); + } + ASSERT(count == 1, "Range doesn't have the right number of elements"); + } else { + ASSERT(newcont.size() == 2, "Concurrent container size is incorrect"); + ASSERT(newcont.count(1) == 1, "Concurrent container count(1) is incorrect"); + ASSERT(newcont.count(2) == 1, "Concurrent container count(2) is incorrect"); + std::pair range = newcont.equal_range(1); + it = range.first; + ASSERT(it != newcont.end() && Value::get(*it) == 1, "Element 1 has not been found properly"); + ASSERT(++it == range.second, "Range doesn't have the right number of elements"); + range = newcont.equal_range(2); it = range.first; + ASSERT(it != newcont.end() && Value::get(*it) == 2, "Element 2 has not been found properly"); + ASSERT(++it == range.second, "Range doesn't have the right number of elements"); + } +#endif /* __TBB_CPP11_INIT_LIST_TEMP_OBJS_COMPILATION_BROKEN */ +#endif /* __TBB_INITIALIZER_LISTS_PRESENT */ + + // T& operator=(const T& _Umap) + newcont = ccont; + ASSERT(T::allow_multimapping ? (newcont.size() == 3) : (newcont.size() == 2), "Assignment operator has not copied the elements properly"); + + REMARK("passed -- basic %s tests\n", str); + +#if defined (VERBOSE) + REMARK("container dump debug:\n"); + cont._Dump(); + REMARK("container dump release:\n"); + cont.dump(); + REMARK("\n"); +#endif + + cont.clear(); + CheckEmptyContainerAllocatorA(cont, 1, 0); // one dummy is always allocated + for (int i = 0; i < 256; i++) + { + std::pair ins3 = cont.insert(Value::make(i)); + ASSERT(ins3.second == true && Value::get(*(ins3.first)) == i, "Element 1 has not been inserted properly"); + } + ASSERT(cont.size() == 256, "Wrong number of elements have been inserted"); + ASSERT((256 == CheckRecursiveRange(cont.range()).first), NULL); + ASSERT((256 == CheckRecursiveRange(ccont.range()).first), NULL); + + // void swap(T&); + cont.swap(newcont); + ASSERT(newcont.size() == 256, "Wrong number of elements after swap"); + ASSERT(newcont.count(200) == 1, "Element with key 200 is not present after swap"); + ASSERT(newcont.count(16) == 1, "Element with key 16 is not present after swap"); + ASSERT(newcont.count(99) == 1, "Element with key 99 is not present after swap"); + ASSERT(T::allow_multimapping ? (cont.size() == 3) : (cont.size() == 2), "Assignment operator has not copied the elements properly"); + + // Need to be enabled + SpecialTests::Test(str); +} + +template +void test_basic_common(const char * str){ + test_basic_common(str, tbb::internal::false_type()); +} + +void test_machine() { + ASSERT(__TBB_ReverseByte(0)==0, NULL ); + ASSERT(__TBB_ReverseByte(1)==0x80, NULL ); + ASSERT(__TBB_ReverseByte(0xFE)==0x7F, NULL ); + ASSERT(__TBB_ReverseByte(0xFF)==0xFF, NULL ); +} + +template +class FillTable: NoAssign { + T &table; + const int items; + bool my_asymptotic; + typedef std::pair pairIB; +public: + FillTable(T &t, int i, bool asymptotic) : table(t), items(i), my_asymptotic(asymptotic) { + ASSERT( !(items&1) && items > 100, NULL); + } + void operator()(int threadn) const { + if( threadn == 0 ) { // Fill even keys forward (single thread) + bool last_inserted = true; + for( int i = 0; i < items; i+=2 ) { + pairIB pib = table.insert(Value::make(my_asymptotic?1:i)); + ASSERT(Value::get(*(pib.first)) == (my_asymptotic?1:i), "Element not properly inserted"); + ASSERT( last_inserted || !pib.second, "Previous key was not inserted but this is inserted" ); + last_inserted = pib.second; + } + } else if( threadn == 1 ) { // Fill even keys backward (single thread) + bool last_inserted = true; + for( int i = items-2; i >= 0; i-=2 ) { + pairIB pib = table.insert(Value::make(my_asymptotic?1:i)); + ASSERT(Value::get(*(pib.first)) == (my_asymptotic?1:i), "Element not properly inserted"); + ASSERT( last_inserted || !pib.second, "Previous key was not inserted but this is inserted" ); + last_inserted = pib.second; + } + } else if( !(threadn&1) ) { // Fill odd keys forward (multiple threads) + for( int i = 1; i < items; i+=2 ) +#if __TBB_INITIALIZER_LISTS_PRESENT && !__TBB_CPP11_INIT_LIST_TEMP_OBJS_LIFETIME_BROKEN + if ( i % 32 == 1 && i + 6 < items ) { + if (my_asymptotic) { + table.insert({ Value::make(1), Value::make(1), Value::make(1) }); + ASSERT(Value::get(*table.find(1)) == 1, "Element not properly inserted"); + } + else { + table.insert({ Value::make(i), Value::make(i + 2), Value::make(i + 4) }); + ASSERT(Value::get(*table.find(i)) == i, "Element not properly inserted"); + ASSERT(Value::get(*table.find(i + 2)) == i + 2, "Element not properly inserted"); + ASSERT(Value::get(*table.find(i + 4)) == i + 4, "Element not properly inserted"); + } + i += 4; + } else +#endif + { + pairIB pib = table.insert(Value::make(my_asymptotic ? 1 : i)); + ASSERT(Value::get(*(pib.first)) == (my_asymptotic ? 1 : i), "Element not properly inserted"); + } + } else { // Check odd keys backward (multiple threads) + if (!my_asymptotic) { + bool last_found = false; + for( int i = items-1; i >= 0; i-=2 ) { + typename T::iterator it = table.find(i); + if( it != table.end() ) { // found + ASSERT(Value::get(*it) == i, "Element not properly inserted"); + last_found = true; + } else { + ASSERT( !last_found, "Previous key was found but this is not" ); + } + } + } + } + } +}; + +typedef tbb::atomic AtomicByte; + +template +struct ParallelTraverseBody: NoAssign { + const int n; + AtomicByte* const array; + ParallelTraverseBody( AtomicByte an_array[], int a_n ) : + n(a_n), array(an_array) + {} + void operator()( const RangeType& range ) const { + for( typename RangeType::iterator i = range.begin(); i!=range.end(); ++i ) { + int k = static_cast(Value::key(*i)); + ASSERT( k == Value::get(*i), NULL ); + ASSERT( 0<=k && k +class CheckTable: NoAssign { + T &table; +public: + CheckTable(T &t) : NoAssign(), table(t) {} + void operator()(int i) const { + int c = (int)table.count( i ); + ASSERT( c, "must exist" ); + } +}; + +template +void test_concurrent_common(const char *tablename, bool asymptotic = false) { +#if TBB_USE_ASSERT + int items = 2000; +#else + int items = 20000; +#endif + int nItemsInserted = 0; + int nThreads = 0; +#if __TBB_UNORDERED_TEST + T table(items/1000); +#else + T table; +#endif + #if __bgp__ + nThreads = 6; + #else + nThreads = 16; + #endif + if(T::allow_multimapping) { + // even passes (threads 0 & 1) put N/2 items each + // odd passes (threads > 1) put N/2 if thread is odd, else checks if even. + items = 4*items / (nThreads + 2); // approximately same number of items inserted. + nItemsInserted = items + (nThreads-2) * items / 4; + } + else { + nItemsInserted = items; + } + REMARK("%s items == %d\n", tablename, items); + tbb::tick_count t0 = tbb::tick_count::now(); + NativeParallelFor( nThreads, FillTable(table, items, asymptotic) ); + tbb::tick_count t1 = tbb::tick_count::now(); + REMARK( "time for filling '%s' by %d items = %g\n", tablename, table.size(), (t1-t0).seconds() ); + ASSERT( int(table.size()) == nItemsInserted, NULL); + + if(!asymptotic) { + AtomicByte* array = new AtomicByte[items]; + memset( static_cast(array), 0, items*sizeof(AtomicByte) ); + + typename T::range_type r = table.range(); + std::pair p = CheckRecursiveRange(r); + ASSERT((nItemsInserted == p.first), NULL); + tbb::parallel_for( r, ParallelTraverseBody( array, items )); + CheckRange( array, items, T::allow_multimapping, (nThreads - 1)/2 ); + + const T &const_table = table; + memset( static_cast(array), 0, items*sizeof(AtomicByte) ); + typename T::const_range_type cr = const_table.range(); + ASSERT((nItemsInserted == CheckRecursiveRange(cr).first), NULL); + tbb::parallel_for( cr, ParallelTraverseBody( array, items )); + CheckRange( array, items, T::allow_multimapping, (nThreads - 1) / 2 ); + delete[] array; + + tbb::parallel_for( 0, items, CheckTable( table ) ); + } + + table.clear(); + CheckEmptyContainerAllocatorA(table, items+1, items); // one dummy is always allocated + +} + +#if __TBB_CPP11_RVALUE_REF_PRESENT +#include "test_container_move_support.h" + +template +void test_rvalue_ref_support(const char* container_name){ + TestMoveConstructor(); + TestMoveAssignOperator(); +#if TBB_USE_EXCEPTIONS + TestExceptionSafetyGuaranteesMoveConstructorWithUnEqualAllocatorMemoryFailure(); + TestExceptionSafetyGuaranteesMoveConstructorWithUnEqualAllocatorExceptionInElementCtor(); +#endif //TBB_USE_EXCEPTIONS + REMARK("passed -- %s move support tests\n", container_name); +} +#endif //__TBB_CPP11_RVALUE_REF_PRESENT + +namespace test_select_size_t_constant{ + __TBB_STATIC_ASSERT((tbb::internal::select_size_t_constant<1234,1234>::value == 1234),"select_size_t_constant::value is not compile time constant"); +// There will be two constant used in the test 32 bit and 64 bit one. +// The 64 bit constant should chosen so that it 32 bit halves adds up to the 32 bit one ( first constant used in the test). +// % ~0U is used to sum up 32bit halves of the 64 constant. ("% ~0U" essentially adds the 32-bit "digits", like "%9" adds +// the digits (modulo 9) of a number in base 10). +// So iff select_size_t_constant is correct result of the calculation below will be same on both 32bit and 64bit platforms. + __TBB_STATIC_ASSERT((tbb::internal::select_size_t_constant<0x12345678U,0x091A2B3C091A2B3CULL>::value % ~0U == 0x12345678U), + "select_size_t_constant have chosen the wrong constant"); +} + +#if __TBB_CPP11_SMART_POINTERS_PRESENT +// For the sake of simplified testing, make unique_ptr implicitly convertible to/from the pointer +namespace test { + template + class unique_ptr : public std::unique_ptr { + public: + typedef typename std::unique_ptr::pointer pointer; + unique_ptr( pointer p ) : std::unique_ptr(p) {} + operator pointer() const { return this->get(); } + }; +} +#endif /* __TBB_CPP11_SMART_POINTERS_PRESENT */ + +#include +#include +#include + +template +class TestRange : NoAssign { + const std::list &my_lst; + std::vector< tbb::atomic > &my_marks; +public: + TestRange( const std::list &lst, std::vector< tbb::atomic > &marks ) : my_lst( lst ), my_marks( marks ) { + std::fill( my_marks.begin(), my_marks.end(), false ); + } + template + void operator()( const Range &r ) const { doTestRange( r.begin(), r.end() ); } + template + void doTestRange( Iterator i, Iterator j ) const { + for ( Iterator it = i; it != j; ) { + Iterator prev_it = it++; + typename std::list::const_iterator it2 = std::search( my_lst.begin(), my_lst.end(), prev_it, it, Harness::IsEqual() ); + ASSERT( it2 != my_lst.end(), NULL ); + typename std::list::difference_type dist = std::distance( my_lst.begin( ), it2 ); + ASSERT( !my_marks[dist], NULL ); + my_marks[dist] = true; + } + } +}; + +// The helper to call a function only when a doCall == true. +template struct CallIf { + template void operator() ( FuncType func ) const { func(); } +}; +template <> struct CallIf { + template void operator()( FuncType ) const {} +}; + +template +class TestOperatorSquareBrackets : NoAssign { + typedef typename Table::value_type ValueType; + Table &my_c; + const ValueType &my_value; +public: + TestOperatorSquareBrackets( Table &c, const ValueType &value ) : my_c( c ), my_value( value ) {} + void operator()() const { + ASSERT( Harness::IsEqual()(my_c[my_value.first], my_value.second), NULL ); + } +}; + +template +void TestMapSpecificMethodsImpl(Table &c, const Value &value){ + CallIf()(TestOperatorSquareBrackets( c, value )); + ASSERT( Harness::IsEqual()(c.at( value.first ), value.second), NULL ); + const Table &constC = c; + ASSERT( Harness::IsEqual()(constC.at( value.first ), value.second), NULL ); +} + +// do nothing for common case +template +void TestMapSpecificMethods( Table&, const Value& ) {} + +template +class CheckValue : NoAssign { + Table &my_c; +public: + CheckValue( Table &c ) : my_c( c ) {} + void operator()( const typename Table::value_type &value ) { + typedef typename Table::iterator Iterator; + typedef typename Table::const_iterator ConstIterator; + const Table &constC = my_c; + ASSERT( my_c.count( Value
::key( value ) ) == 1, NULL ); + // find + ASSERT( Harness::IsEqual()(*my_c.find( Value
::key( value ) ), value), NULL ); + ASSERT( Harness::IsEqual()(*constC.find( Value
::key( value ) ), value), NULL ); + // erase + ASSERT( my_c.unsafe_erase( Value
::key( value ) ), NULL ); + ASSERT( my_c.count( Value
::key( value ) ) == 0, NULL ); + // insert + std::pair res = my_c.insert( value ); + ASSERT( Harness::IsEqual()(*res.first, value), NULL ); + ASSERT( res.second, NULL); + // erase + Iterator it = res.first; + it++; + ASSERT( my_c.unsafe_erase( res.first ) == it, NULL ); + // insert + ASSERT( Harness::IsEqual()(*my_c.insert( my_c.begin(), value ), value), NULL ); + // equal_range + std::pair r1 = my_c.equal_range( Value
::key( value ) ); + ASSERT( Harness::IsEqual()(*r1.first, value) && ++r1.first == r1.second, NULL ); + std::pair r2 = constC.equal_range( Value
::key( value ) ); + ASSERT( Harness::IsEqual()(*r2.first, value) && ++r2.first == r2.second, NULL ); + + TestMapSpecificMethods( my_c, value ); + } +}; + +#include "tbb/task_scheduler_init.h" + +template +void CommonExamine( Table c, const std::list lst) { + typedef typename Table::value_type ValueType; + + ASSERT( !c.empty() && c.size() == lst.size() && c.max_size() >= c.size(), NULL ); + + std::for_each( lst.begin(), lst.end(), CheckValue( c ) ); + + std::vector< tbb::atomic > marks( lst.size() ); + + TestRange( lst, marks ).doTestRange( c.begin(), c.end() ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + TestRange( lst, marks ).doTestRange( c.begin(), c.end() ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + const Table constC = c; + ASSERT( c.size() == constC.size(), NULL ); + + TestRange( lst, marks ).doTestRange( constC.cbegin(), constC.cend() ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + tbb::task_scheduler_init init; + + tbb::parallel_for( c.range(), TestRange( lst, marks ) ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + tbb::parallel_for( constC.range( ), TestRange( lst, marks ) ); + ASSERT( std::find( marks.begin(), marks.end(), false ) == marks.end(), NULL ); + + Table c2; + typename std::list::const_iterator begin5 = lst.begin(); + std::advance( begin5, 5 ); + c2.insert( lst.begin(), begin5 ); + std::for_each( lst.begin(), begin5, CheckValue( c2 ) ); + + c2.swap( c ); + ASSERT( c2.size() == lst.size(), NULL ); + ASSERT( c.size() == 5, NULL ); + std::for_each( lst.begin(), lst.end(), CheckValue( c2 ) ); + + c2.clear(); + ASSERT( c2.size() == 0, NULL ); + + typename Table::allocator_type a = c.get_allocator(); + ValueType *ptr = a.allocate( 1 ); + ASSERT( ptr, NULL ); + a.deallocate( ptr, 1 ); +} + +// overload for set and multiset +// second argument is needed just for right deduction +template +void TestSetCommonTypes() { + Checker CheckTypes; + const int NUMBER = 10; + + std::list arrInt; + for ( int i = 0; i( arrInt ); + + std::list< tbb::atomic > arrTbb(NUMBER); + int seq = 0; + for ( std::list< tbb::atomic >::iterator it = arrTbb.begin(); it != arrTbb.end(); ++it, ++seq ) *it = seq; + CheckTypes.template check( arrTbb ); + +#if __TBB_CPP11_REFERENCE_WRAPPER_PRESENT && !__TBB_REFERENCE_WRAPPER_COMPILATION_BROKEN + std::list< std::reference_wrapper > arrRef; + for ( std::list::iterator it = arrInt.begin( ); it != arrInt.end( ); ++it ) + arrRef.push_back( std::reference_wrapper(*it) ); + CheckTypes.template check( arrRef ); +#endif /* __TBB_CPP11_REFERENCE_WRAPPER_PRESENT && !__TBB_REFERENCE_WRAPPER_COMPILATION_BROKEN */ + +#if __TBB_CPP11_SMART_POINTERS_PRESENT + std::list< std::shared_ptr > arrShr; + for ( int i = 0; i( i ) ); + CheckTypes.template check( arrShr ); + + std::list< std::weak_ptr > arrWk; + std::copy( arrShr.begin( ), arrShr.end( ), std::back_inserter( arrWk ) ); + CheckTypes.template check( arrWk ); +#else + REPORT( "Known issue: C++11 smart pointer tests are skipped.\n" ); +#endif /* __TBB_CPP11_SMART_POINTERS_PRESENT */ +} + +template +void TestMapCommonTypes() { + Checker CheckTypes; + const int NUMBER = 10; + + std::list< std::pair > arrIntInt; + for ( int i = 0; i < NUMBER; ++i ) arrIntInt.push_back( std::make_pair( i, NUMBER - i ) ); + CheckTypes.template check( arrIntInt ); + + std::list< std::pair< const int, tbb::atomic > > arrIntTbb; + for ( int i = 0; i < NUMBER; ++i ) { + tbb::atomic b; + b = NUMBER - i; + arrIntTbb.push_back( std::make_pair( i, b ) ); + } + CheckTypes.template check( arrIntTbb ); + +#if __TBB_CPP11_REFERENCE_WRAPPER_PRESENT && !__TBB_REFERENCE_WRAPPER_COMPILATION_BROKEN + std::list< std::pair, int> > arrRefInt; + for ( std::list< std::pair >::iterator it = arrIntInt.begin(); it != arrIntInt.end(); ++it ) + arrRefInt.push_back( std::make_pair( std::reference_wrapper( it->first ), it->second ) ); + CheckTypes.template check( arrRefInt ); + + std::list< std::pair > > arrIntRef; + for ( std::list< std::pair >::iterator it = arrIntInt.begin(); it != arrIntInt.end(); ++it ) { + // Using std::make_pair below causes compilation issues with early implementations of std::reference_wrapper. + arrIntRef.push_back( std::pair >( it->first, std::reference_wrapper( it->second ) ) ); + } + CheckTypes.template check( arrIntRef ); +#endif /* __TBB_CPP11_REFERENCE_WRAPPER_PRESENT && !__TBB_REFERENCE_WRAPPER_COMPILATION_BROKEN */ + +#if __TBB_CPP11_SMART_POINTERS_PRESENT + std::list< std::pair< const std::shared_ptr, std::shared_ptr > > arrShrShr; + for ( int i = 0; i < NUMBER; ++i ) { + const int NUMBER_minus_i = NUMBER - i; + arrShrShr.push_back( std::make_pair( std::make_shared( i ), std::make_shared( NUMBER_minus_i ) ) ); + } + CheckTypes.template check( arrShrShr ); + + std::list< std::pair< const std::weak_ptr, std::weak_ptr > > arrWkWk; + std::copy( arrShrShr.begin(), arrShrShr.end(), std::back_inserter( arrWkWk ) ); + CheckTypes.template check( arrWkWk ); + +#else + REPORT( "Known issue: C++11 smart pointer tests are skipped.\n" ); +#endif /* __TBB_CPP11_SMART_POINTERS_PRESENT */ +} + + +#if __TBB_UNORDERED_NODE_HANDLE_PRESENT || __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT +namespace node_handling{ + template + bool compare_handle_getters( + const Handle& node, const std::pair& expected + ) { + return node.key() == expected.first && node.mapped() == expected.second; + } + + template + bool compare_handle_getters( const Handle& node, const typename Handle::value_type& value) { + return node.value() == value; + } + + template + void set_node_handle_value( + Handle& node, const std::pair& value + ) { + node.key() = value.first; + node.mapped() = value.second; + } + + template + void set_node_handle_value( Handle& node, const typename Handle::value_type& value) { + node.value() = value; + } + + template + void TestTraits() { + ASSERT( !std::is_copy_constructible::value, + "Node handle: Handle is copy constructable" ); + ASSERT( !std::is_copy_assignable::value, + "Node handle: Handle is copy assignable" ); + ASSERT( std::is_move_constructible::value, + "Node handle: Handle is not move constructable" ); + ASSERT( std::is_move_assignable::value, + "Node handle: Handle is not move constructable" ); + ASSERT( std::is_default_constructible::value, + "Node handle: Handle is not default constructable" ); + ASSERT( std::is_destructible::value, + "Node handle: Handle is not destructible" ); + } + + template + void TestHandle( Table test_table ) { + ASSERT( test_table.size()>1, "Node handle: Container must contains 2 or more elements" ); + // Initialization + using node_type = typename Table::node_type; + + TestTraits(); + + // Default Ctor and empty function + node_type nh; + ASSERT( nh.empty(), "Node handle: Node is not empty after initialization" ); + + // Move Assign + // key/mapped/value function + auto expected_value = *test_table.begin(); + + nh = test_table.unsafe_extract(test_table.begin()); + ASSERT( !nh.empty(), "Node handle: Node handle is empty after valid move assigning" ); + ASSERT( compare_handle_getters(nh,expected_value), + "Node handle: After valid move assigning " + "node handle does not contains expected value"); + + // Move Ctor + // key/mapped/value function + node_type nh2(std::move(nh)); + ASSERT( nh.empty(), "Node handle: After valid move construction node handle is empty" ); + ASSERT( !nh2.empty(), "Node handle: After valid move construction " + "argument hode handle was not moved" ); + ASSERT( compare_handle_getters(nh2,expected_value), + "Node handle: After valid move construction " + "node handle does not contains expected value" ); + + // Bool conversion + ASSERT( nh2, "Node hanlde: Wrong not handle bool conversion" ); + + // Change key/mapped/value of node handle + auto expected_value2 = *test_table.begin(); + set_node_handle_value(nh2, expected_value2); + ASSERT( compare_handle_getters(nh2, expected_value2), + "Node handle: Wrong node handle key/mapped/value changing behavior" ); + + // Member/non member swap check + node_type empty_node; + // We extract this element for nh2 and nh3 difference + test_table.unsafe_extract(test_table.begin()); + auto expected_value3 = *test_table.begin(); + node_type nh3(test_table.unsafe_extract(test_table.begin())); + + // Both of node handles are not empty + nh3.swap(nh2); + ASSERT( compare_handle_getters(nh3, expected_value2), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( compare_handle_getters(nh2, expected_value3), + "Node handle: Wrong node handle swap behavior" ); + + std::swap(nh2,nh3); + ASSERT( compare_handle_getters(nh3, expected_value3), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( compare_handle_getters(nh2, expected_value2), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( !nh2.empty(), "Node handle: Wrong node handle swap behavior" ); + ASSERT( !nh3.empty(), "Node handle: Wrong node handle swap behavior" ); + + // One of nodes is empty + nh3.swap(empty_node); + ASSERT( compare_handle_getters(std::move(empty_node), expected_value3), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( nh3.empty(), "Node handle: Wrong node handle swap behavior" ); + + std::swap(empty_node, nh3); + ASSERT( compare_handle_getters(std::move(nh3), expected_value3), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( empty_node.empty(), "Node handle: Wrong node handle swap behavior" ); + + empty_node.swap(nh3); + ASSERT( compare_handle_getters(std::move(empty_node), expected_value3), + "Node handle: Wrong node handle swap behavior" ); + ASSERT( nh3.empty(), "Node handle: Wrong node handle swap behavior" ); + } + + template + typename Table::node_type GenerateNodeHandle(const typename Table::value_type& value) { + Table temp_table; + temp_table.insert(value); + return temp_table.unsafe_extract(temp_table.cbegin()); + } + + template + void IteratorAssertion( const Table& table, + const typename Table::iterator& result, + const typename Table::value_type* node_value = nullptr ) { + if (node_value==nullptr) { + ASSERT( result==table.end(), "Insert: Result iterator does not " + "contains end pointer after empty node insertion" ); + } else { + if (!Table::allow_multimapping) { + ASSERT( result==table.find(Value
::key( *node_value )) && + result != table.end(), + "Insert: After node insertion result iterator" + " doesn't contains address to equal element in table" ); + } else { + ASSERT( *result==*node_value, "Insert: Result iterator contains" + "wrong content after successful insertion" ); + + for (auto it = table.begin(); it != table.end(); ++it) { + if (it == result) return; + } + ASSERT( false, "Insert: After successful insertion result " + "iterator contains address that is not in the table" ); + } + } + } + // overload for multitable or insertion with hint iterator + template + void InsertAssertion( const Table& table, + const typename Table::iterator& result, + bool, + const typename Table::value_type* node_value = nullptr ) { + IteratorAssertion(table, result, node_value); + } + + // Not multitable overload + template + void InsertAssertion( const Table& table, + const std::pair& result, + bool second_value, + const typename Table::value_type* node_value = nullptr ) { + IteratorAssertion(table, result.first, node_value); + + ASSERT( result.second == second_value || Table::allow_multimapping, + "Insert: Returned bool wrong value after node insertion" ); + } + +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + // Internal func for testing + // Can't delete ref from "Table" argument because hint must point to element of table + namespace { + template + void TestInsertOverloads( Table& table_to_insert, + const typename Table::value_type &value, const Hint&... hint ) { + // Insert empty element + typename Table::node_type nh; + + auto table_size = table_to_insert.size(); + auto result = table_to_insert.insert(hint..., std::move(nh)); + InsertAssertion(table_to_insert, result, /*second_value*/ false); + ASSERT( table_to_insert.size() == table_size, + "Insert: After empty node insertion table size changed" ); + + // Standart insertion + nh = GenerateNodeHandle
(value); + + result = table_to_insert.insert(hint..., std::move(nh)); + ASSERT( nh.empty(), "Insert: Not empty handle after successful insertion" ); + InsertAssertion(table_to_insert, result, /*second_value*/ true, &value); + + // Insert existing node + nh = GenerateNodeHandle
(value); + + result = table_to_insert.insert(hint..., std::move(nh)); + + InsertAssertion(table_to_insert, result, /*second_value*/ false, &value); + + if (Table::allow_multimapping){ + ASSERT( nh.empty(), "Insert: Failed insertion to multitable" ); + } else { + ASSERT( !nh.empty() , "Insert: Empty handle after failed insertion" ); + ASSERT( compare_handle_getters( std::move(nh), value ), + "Insert: Existing data does not equal to the one being inserted" ); + } + } + } + + template + void TestInsert( Table table, const typename Table::value_type & value) { + ASSERT( !table.empty(), "Insert: Map should contains 1 or more elements" ); + Table table_backup(table); + TestInsertOverloads(table, value); + TestInsertOverloads(table_backup, value, table_backup.begin()); + } +#endif /*__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT*/ + + template + void TestExtract( Table table_for_extract, typename Table::key_type new_key ) { + ASSERT( table_for_extract.size()>1, "Extract: Container must contains 2 or more element" ); + ASSERT( table_for_extract.find(new_key)==table_for_extract.end(), + "Extract: Table must not contains new element!"); + + // Extract new element + auto nh = table_for_extract.unsafe_extract(new_key); + ASSERT( nh.empty(), "Extract: Node handle is not empty after wrong key extraction" ); + + // Valid key extraction + auto expected_value = *table_for_extract.cbegin(); + auto key = Value
::key( expected_value ); + auto count = table_for_extract.count(key); + + nh = table_for_extract.unsafe_extract(key); + ASSERT( !nh.empty(), + "Extract: After successful extraction by key node handle is empty" ); + ASSERT( compare_handle_getters(std::move(nh), expected_value), + "Extract: After successful extraction by key node handle contains wrong value" ); + ASSERT( table_for_extract.count(key) == count - 1, + "Extract: After successful node extraction by key, table still contains this key" ); + + // Valid iterator overload + auto expected_value2 = *table_for_extract.cbegin(); + auto key2 = Value
::key( expected_value2 ); + auto count2 = table_for_extract.count(key2); + + nh = table_for_extract.unsafe_extract(table_for_extract.cbegin()); + ASSERT( !nh.empty(), + "Extract: After successful extraction by iterator node handle is empty" ); + ASSERT( compare_handle_getters(std::move(nh), expected_value2), + "Extract: After successful extraction by iterator node handle contains wrong value" ); + ASSERT( table_for_extract.count(key2) == count2 - 1, + "Extract: After successful extraction table also contains this element" ); + } + + // All test exclude merge + template + void NodeHandlingTests ( const Table& table, + const typename Table::value_type& new_value) { + TestHandle(table); +#if __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT + TestInsert(table, new_value); +#endif /*__TBB_CPP11_VARIADIC_TEMPLATES_PRESENT*/ + TestExtract(table, Value
::key( new_value )); + } + + template + void TestMerge( TableType1 table1, TableType2&& table2 ) { + using Table2PureType = typename std::decay::type; + // Initialization + TableType1 table1_backup = table1; + // For copying lvalue + Table2PureType table2_backup = table2; + + table1.merge(std::forward(table2)); + for (auto it: table2) { + ASSERT( table1.find( Value::key( it ) ) != table1.end(), + "Merge: Some key(s) was not merged" ); + } + + // After the following step table1 will contains only merged elements from table2 + for (auto it: table1_backup) { + table1.unsafe_extract(Value::key( it )); + } + // After the following step table2_backup will contains only merged elements from table2 + for (auto it: table2) { + table2_backup.unsafe_extract(Value::key( it )); + } + + ASSERT ( table1.size() == table2_backup.size(), "Merge: Size of tables is not equal" ); + for (auto it: table2_backup) { + ASSERT( table1.find( Value::key( it ) ) != table1.end(), + "Merge: Wrong merge behavior" ); + } + } + + // Testing of rvalue and lvalue overloads + template + void TestMergeOverloads( const TableType1& table1, TableType2 table2 ) { + TableType2 table_backup(table2); + TestMerge(table1, table2); + TestMerge(table1, std::move(table_backup)); + } + + template + void TestMergeTransposition( Table table1, Table table2, + MultiTable multitable1, MultiTable multitable2 ) { + Table empty_map; + MultiTable empty_multimap; + + // Map transpositions + node_handling::TestMergeOverloads(table1, table2); + node_handling::TestMergeOverloads(table1, empty_map); + node_handling::TestMergeOverloads(empty_map, table2); + + // Multimap transpositions + node_handling::TestMergeOverloads(multitable1, multitable2); + node_handling::TestMergeOverloads(multitable1, empty_multimap); + node_handling::TestMergeOverloads(empty_multimap, multitable2); + + // Map/Multimap transposition + node_handling::TestMergeOverloads(table1, multitable1); + node_handling::TestMergeOverloads(multitable2, table2); + } + + template + void AssertionConcurrentMerge ( Table start_data, Table src_table, std::vector
tables, + std::true_type) { + ASSERT( src_table.size() == start_data.size()*tables.size(), + "Merge: Incorrect merge for some elements" ); + + for(auto it: start_data) { + ASSERT( src_table.count( Value
::key( it ) ) == + start_data.count( Value
::key( it ) )*tables.size(), + "Merge: Incorrect merge for some element" ); + } + + for (size_t i = 0; i < tables.size(); i++) { + ASSERT( tables[i].empty(), "Merge: Some elements was not merged" ); + } + } + + template + void AssertionConcurrentMerge ( Table start_data, Table src_table, std::vector
tables, + std::false_type) { + Table expected_result; + for (auto table: tables) + for (auto it: start_data) { + // If we cannot find some element in some table, then it has been moved + if (table.find( Value
::key( it ) ) == table.end()){ + bool result = expected_result.insert( it ).second; + ASSERT( result, "Merge: Some element was merged twice or was not " + "returned to his owner after unsuccessful merge"); + } + } + + ASSERT( expected_result.size() == src_table.size() && start_data.size() == src_table.size(), + "Merge: wrong size of result table"); + for (auto it: expected_result) { + if ( src_table.find( Value
::key( it ) ) != src_table.end() && + start_data.find( Value
::key( it ) ) != start_data.end() ){ + src_table.unsafe_extract(Value
::key( it )); + start_data.unsafe_extract(Value
::key( it )); + } else { + ASSERT( false, "Merge: Incorrect merge for some element" ); + } + } + + ASSERT( src_table.empty()&&start_data.empty(), "Merge: Some elements were not merged" ); + } + + template + void TestConcurrentMerge (const Table& table_data) { + for (auto num_threads = MinThread + 1; num_threads <= MaxThread; num_threads++){ + std::vector
tables; + Table src_table; + + for (auto j = 0; j < num_threads; j++){ + tables.push_back(table_data); + } + + NativeParallelFor( num_threads, [&](size_t index){ src_table.merge(tables[index]); } ); + + AssertionConcurrentMerge( table_data, src_table, tables, + std::integral_constant{}); + } + } + + + template + void TestNodeHandling(){ + Table table; + + for (int i = 1; i < 5; i++) + table.insert(Value
::make(i)); + + if (Table::allow_multimapping) + table.insert(Value
::make(4)); + + node_handling::NodeHandlingTests(table, Value
::make(5)); + } + + template + void TestMerge(int size){ + TableType1 table1_1; + TableType1 table1_2; + int i = 1; + for (; i < 5; ++i) { + table1_1.insert(Value::make(i)); + table1_2.insert(Value::make(i*i)); + } + if (TableType1::allow_multimapping) { + table1_1.insert(Value::make(i)); + table1_2.insert(Value::make(i*i)); + } + + TableType2 table2_1; + TableType2 table2_2; + for (i = 3; i < 7; ++i) { + table1_1.insert(Value::make(i)); + table1_2.insert(Value::make(i*i)); + } + if (TableType2::allow_multimapping) { + table2_1.insert(Value::make(i)); + table2_2.insert(Value::make(i*i)); + } + + node_handling::TestMergeTransposition(table1_1, table1_2, + table2_1, table2_2); + + TableType1 table1_3; + for (i = 0; i::make(i)); + } + node_handling::TestConcurrentMerge(table1_3); + + TableType2 table2_3; + for (i = 0; i::make(i)); + } + node_handling::TestConcurrentMerge(table2_3); +} +} +#endif // __TBB_UNORDERED_NODE_HANDLE_PRESENT || __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT diff --git a/src/test/test_concurrent_hash_map.cpp b/src/test/test_concurrent_hash_map.cpp index 5fb9681bfb..d4d4a7a78e 100644 --- a/src/test/test_concurrent_hash_map.cpp +++ b/src/test/test_concurrent_hash_map.cpp @@ -1519,33 +1519,6 @@ void TestHashCompareConstructors() { #if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && !__TBB_SCOPED_ALLOCATOR_BROKEN #include -template -class allocator_aware_data { -public: - static bool assert_on_constructions; - typedef Allocator allocator_type; - - allocator_aware_data(const allocator_type& allocator = allocator_type()) - : my_allocator(allocator), my_value(0) {} - allocator_aware_data(int v, const allocator_type& allocator = allocator_type()) - : my_allocator(allocator), my_value(v) {} - allocator_aware_data(const allocator_aware_data&) { - ASSERT(!assert_on_constructions, "Allocator should propogate to the data during copy construction"); - } - allocator_aware_data(allocator_aware_data&&) { - ASSERT(!assert_on_constructions, "Allocator should propogate to the data during move construction"); - } - allocator_aware_data(const allocator_aware_data& rhs, const allocator_type& allocator) - : my_allocator(allocator), my_value(rhs.my_value) {} - allocator_aware_data(allocator_aware_data&& rhs, const allocator_type& allocator) - : my_allocator(allocator), my_value(rhs.my_value) {} - - int value() const { return my_value; } -private: - allocator_type my_allocator; - int my_value; -}; - struct custom_hash_compare { template static size_t hash(const allocator_aware_data& key) { @@ -1558,9 +1531,6 @@ struct custom_hash_compare { } }; -template -bool allocator_aware_data::assert_on_constructions = false; - void TestScopedAllocator() { typedef allocator_aware_data>> allocator_data_type; typedef std::scoped_allocator_adaptor> allocator_type; @@ -1607,86 +1577,29 @@ void TestScopedAllocator() { } #endif -// C++03 allocator doesn't have to be assignable or swappable, so -// tbb::internal::allocator_traits defines POCCA and POCS as false_type #if __TBB_ALLOCATOR_TRAITS_PRESENT - -template -void test_traits() { - typedef int key_type; - - typedef int mapped_type; - typedef tbb::tbb_hash_compare compare_type; - - typedef typename Allocator::propagate_on_container_copy_assignment pocca; - typedef typename Allocator::propagate_on_container_swap pocs; - - typedef tbb::concurrent_hash_map container_type; - bool propagated_on_copy_assign = false; - bool propagated_on_move = false; - bool propagated_on_swap = false; - bool selected_on_copy_construct = false; - - Allocator alloc(propagated_on_copy_assign, propagated_on_move, propagated_on_swap, selected_on_copy_construct); - - container_type c1(alloc), c2(c1); - ASSERT(selected_on_copy_construct, "select_on_container_copy_construction function was not called"); - - c1 = c2; - ASSERT(propagated_on_copy_assign == pocca::value, "Unexpected allocator propagation on copy assignment"); - -#if __TBB_CPP11_RVALUE_REF_PRESENT - typedef typename Allocator::propagate_on_container_move_assignment pocma; - c2 = std::move(c1); - ASSERT(propagated_on_move == pocma::value, "Unexpected allocator propagation on move assignment"); -#endif - - c1.swap(c2); - ASSERT(propagated_on_swap == pocs::value, "Unexpected allocator propagation on swap"); -} - -#if __TBB_CPP11_RVALUE_REF_PRESENT -class non_movable_object { - non_movable_object() {} -private: - non_movable_object(non_movable_object&&); - non_movable_object& operator=(non_movable_object&&); -}; - -void test_non_movable_value_type() { - // Check, that if pocma is true, concurrent_hash_map allows move assignment without per-element move - typedef propagating_allocator, /*POCMA=*/tbb::internal::traits_true_type> allocator_type; - typedef tbb::concurrent_hash_map, allocator_type> container_type; - allocator_type alloc; - container_type container1(alloc), container2(alloc); - container1 = std::move(container2); -} - -#endif // __TBB_CPP11_RVALUE_REF_PRESENT - void TestAllocatorTraits() { - typedef tbb::tbb_allocator base_allocator; - typedef tbb::internal::traits_true_type true_type; - typedef tbb::internal::traits_true_type false_type; - - typedef propagating_allocator - always_propagating_allocator; - typedef propagating_allocator never_propagating_allocator; - typedef propagating_allocator pocma_allocator; - typedef propagating_allocator pocca_allocator; - typedef propagating_allocator pocs_allocator; - - test_traits(); - test_traits(); - test_traits(); - test_traits(); - test_traits(); + using namespace propagating_allocators; + typedef int key; + typedef int mapped; + typedef tbb::tbb_hash_compare compare; + + typedef tbb::concurrent_hash_map always_propagating_map; + typedef tbb::concurrent_hash_map never_propagating_map; + typedef tbb::concurrent_hash_map pocma_map; + typedef tbb::concurrent_hash_map pocca_map; + typedef tbb::concurrent_hash_map pocs_map; + + test_allocator_traits_support(); + test_allocator_traits_support(); + test_allocator_traits_support(); + test_allocator_traits_support(); + test_allocator_traits_support(); #if __TBB_CPP11_RVALUE_REF_PRESENT - test_non_movable_value_type(); + test_allocator_traits_with_non_movable_value_type(); #endif } - #endif // __TBB_ALLOCATOR_TRAITS_PRESENT //------------------------------------------------------------------------ diff --git a/src/test/test_concurrent_map.cpp b/src/test/test_concurrent_map.cpp new file mode 100644 index 0000000000..8c05d1e04c --- /dev/null +++ b/src/test/test_concurrent_map.cpp @@ -0,0 +1,267 @@ +/* + Copyright (c) 2019 Intel Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#define __TBB_EXTRA_DEBUG 1 +#if _MSC_VER +#define _SCL_SECURE_NO_WARNINGS +#endif + +#include "tbb/tbb_config.h" +#include "harness.h" +#if __TBB_CONCURRENT_ORDERED_CONTAINERS_PRESENT + +#define TBB_PREVIEW_CONCURRENT_ORDERED_CONTAINERS 1 +#include "tbb/concurrent_map.h" +#if __TBB_INITIALIZER_LISTS_PRESENT +// These operator== are used implicitly in test_initializer_list.h. +// For some unknown reason clang is not able to find the if they a declared after the +// inclusion of test_initializer_list.h. +template +bool equal_containers( container_type const& lhs, container_type const& rhs ); +template +bool operator==( tbb::concurrent_map const& lhs, tbb::concurrent_map const& rhs ) { + return equal_containers( lhs, rhs ); +} +template +bool operator==( tbb::concurrent_multimap const& lhs, tbb::concurrent_multimap const& rhs ) { + return equal_containers( lhs, rhs ); +} +#endif /* __TBB_INITIALIZER_LISTS_PRESENT */ +#include "test_concurrent_ordered_common.h" + +typedef tbb::concurrent_map, MyAllocator> MyMap; +typedef tbb::concurrent_map, MyAllocator> MyGreaterMap; +typedef tbb::concurrent_map, std::less, MyAllocator> MyCheckedMap; +typedef tbb::concurrent_map, MyAllocator> MyCheckedStateMap; +typedef tbb::concurrent_multimap, MyAllocator> MyMultiMap; +typedef tbb::concurrent_multimap, MyAllocator> MyGreaterMultiMap; +typedef tbb::concurrent_multimap, std::less, MyAllocator> MyCheckedMultiMap; + +template <> +struct SpecialTests { + static void Test( const char *str ) { + SpecialMapTests(str); + } +}; + +template <> +struct SpecialTests { + static void Test( const char *str ) { + SpecialMultiMapTests(str); + } +}; + +struct co_map_type : ordered_move_traits_base { + template + struct apply { + typedef tbb::concurrent_map, allocator_type > type; + }; + + typedef FooPairIterator init_iterator_type; +}; + +struct co_multimap_type : ordered_move_traits_base { + template + struct apply { + typedef tbb::concurrent_multimap, allocator_type > type; + }; + + typedef FooPairIterator init_iterator_type; +}; + +template +void TestMapSpecificMethods( tbb::concurrent_map &c, + const typename tbb::concurrent_map::value_type &value ) { + TestMapSpecificMethodsImpl(c, value); + } + +struct OrderedMapTypesTester{ + template + void check( const std::list &lst ) { + typedef typename ValueType::first_type KeyType; + typedef typename ValueType::second_type ElemType; + TypeTester< defCtorPresent, tbb::concurrent_map< KeyType, ElemType>, + tbb::concurrent_map< KeyType, ElemType, std::less, debug_allocator > >( lst ); + TypeTester< defCtorPresent, tbb::concurrent_multimap< KeyType, ElemType>, + tbb::concurrent_multimap< KeyType, ElemType, std::less, debug_allocator > >( lst ); + } +}; + +void TestTypes() { + TestMapCommonTypes(); + + #if __TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_SMART_POINTERS_PRESENT + // Regression test for a problem with excessive requirements of emplace() + test_emplace_insert >, + tbb::internal::false_type>( new int, new int ); + test_emplace_insert >, + tbb::internal::false_type>( new int, new int ); + #endif /*__TBB_CPP11_RVALUE_REF_PRESENT && __TBB_CPP11_VARIADIC_TEMPLATES_PRESENT && __TBB_CPP11_SMART_POINTERS_PRESENT*/ +} + +#if __TBB_CPP17_DEDUCTION_GUIDES_PRESENT +template