From cc739d9b02ee515a4483a5cedfe18f78c16ad895 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 11 Aug 2023 10:05:26 +0100 Subject: [PATCH 1/6] Arnold CameraAlgo : Fix screen window for Lentil cameras --- Changes.md | 4 ++++ src/IECoreArnold/CameraAlgo.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index 947bbae98c3..df0a88de5d2 100644 --- a/Changes.md +++ b/Changes.md @@ -1,6 +1,10 @@ 1.2.10.x (relative to 1.2.10.1) ======== +Fixes +----- + +- Arnold : Fixed screen window export for Lentil cameras. 1.2.10.1 (relative to 1.2.10.0) ======== diff --git a/src/IECoreArnold/CameraAlgo.cpp b/src/IECoreArnold/CameraAlgo.cpp index 6289c2cca0b..5a1f4828cf4 100644 --- a/src/IECoreArnold/CameraAlgo.cpp +++ b/src/IECoreArnold/CameraAlgo.cpp @@ -190,7 +190,7 @@ Imath::Box2f screenWindow( const IECoreScene::Camera *camera ) { Imath::Box2f result = camera->frustum(); - if( camera->getProjection() == "perspective" ) + if( camera->getProjection() == "perspective" || camera->getProjection() == "lentil_camera" ) { // Normalise so that Arnold's NDC space goes from 0-1 across the aperture. // This is helpful when using Arnold `uv_remap` shaders. From 2aeb85d8975c1e83837430541530302fda53cc70 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Wed, 9 Aug 2023 11:46:17 +0100 Subject: [PATCH 2/6] Application : Clamp thread count at hardware concurrency. This fixes deadlocks caused by worker threads not being able to join a `tbb::task_arena`, because by default they are constructed to only allow the same number of workers as cores. Fixes #5403. Also added support for specifying negative thread counts, to reserve some cores for other applications. --- Changes.md | 1 + python/Gaffer/Application.py | 16 ++++++++++++---- python/GafferTest/ValuePlugTest.py | 29 +++++++---------------------- 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/Changes.md b/Changes.md index df0a88de5d2..900e4c0013c 100644 --- a/Changes.md +++ b/Changes.md @@ -5,6 +5,7 @@ Fixes ----- - Arnold : Fixed screen window export for Lentil cameras. +- Application : Fixed the `-threads` argument to clamp the number of threads to the number of available hardware cores (#5403). 1.2.10.1 (relative to 1.2.10.0) ======== diff --git a/python/Gaffer/Application.py b/python/Gaffer/Application.py index 5b4978751f9..bbbdb9730c9 100644 --- a/python/Gaffer/Application.py +++ b/python/Gaffer/Application.py @@ -64,10 +64,12 @@ def __init__( self, description="" ) : IECore.IntParameter( name = "threads", description = "The maximum number of threads used for computation. " - "The default value of zero causes the number of threads to " - " be chosen automatically based on the available hardware.", + "The default value of zero matches the number of threads to " + "the available hardware cores. Negative values specify a thread count " + "relative to the available cores, leaving some in reserve for other " + "applications. Positive values specify the thread count explicitly, " + "but are clamped so it does not exceed the available cores.", defaultValue = 0, - minValue = 0, ), IECore.FileNameParameter( @@ -131,11 +133,17 @@ def _executeStartupFiles( self, applicationName ) : def __run( self ) : + maxThreads = IECore.hardwareConcurrency() threads = self.parameters()["threads"].getTypedValue() + if threads <= 0 : + threads = max( maxThreads + threads, 1 ) + elif threads > maxThreads : + IECore.msg( IECore.Msg.Level.Warning, "Application", f"Clamping to `-threads {maxThreads}` to avoid oversubscription" ) + threads = maxThreads with IECore.tbb_global_control( IECore.tbb_global_control.parameter.max_allowed_parallelism, - IECore.hardwareConcurrency() if threads == 0 else threads + threads ) : self._executeStartupFiles( self.root().getName() ) diff --git a/python/GafferTest/ValuePlugTest.py b/python/GafferTest/ValuePlugTest.py index cb0bb0d7525..53162dc2281 100644 --- a/python/GafferTest/ValuePlugTest.py +++ b/python/GafferTest/ValuePlugTest.py @@ -41,6 +41,8 @@ import subprocess import threading import time +import unittest + import imath import IECore @@ -884,37 +886,20 @@ def testExceptionDuringParallelEval( self ) : with self.assertRaisesRegex( BaseException, "Foo" ): GafferTest.parallelGetValue( m["product"], 10000, "testVar" ) - def testCancellationOfSecondGetValueCall( self ) : - ## \todo Should just be checking `tbb.global_control.active_value( max_allowed_parallelism )` - # to get the true limit set by `-threads` argument. But IECore's Python - # binding of `global_control` doesn't expose that yet. - if IECore.hardwareConcurrency() < 3 and "VALUEPLUGTEST_SUBPROCESS" not in os.environ : + def testCancellationOfSecondGetValueCall( self ) : + if IECore.tbb_global_control.active_value( IECore.tbb_global_control.parameter.max_allowed_parallelism ) < 3 : # This test requires at least 3 TBB threads (including the main # thread), because we need the second enqueued BackgroundTask to # start execution before the first one has completed. If we have - # insufficient threads then we end up in deadlock, so to avoid this - # we relaunch in a subprocess with sufficient threads. + # insufficient threads then we end up in deadlock, so in this case + # we skip the test. # # Note : deadlock only ensues because the first task will never # return without cancellation. This is an artificial situation, not # one that would occur in practical usage of Gaffer itself. - - print( "Running in subprocess due to insufficient TBB threads" ) - - try : - env = os.environ.copy() - env["VALUEPLUGTEST_SUBPROCESS"] = "1" - subprocess.check_output( - [ str( Gaffer.executablePath() ), "test", "-threads", "3", "GafferTest.ValuePlugTest.testCancellationOfSecondGetValueCall" ], - stderr = subprocess.STDOUT, - env = env - ) - except subprocess.CalledProcessError as e : - self.fail( e.output ) - - return + self.skipTest( "Not enough worker threads" ) class InfiniteLoop( Gaffer.ComputeNode ) : From 57b933f6d37977fc6cc62b000df5266577a181d7 Mon Sep 17 00:00:00 2001 From: Murray Stevenson <50844517+murraystevenson@users.noreply.github.com> Date: Thu, 17 Aug 2023 14:32:39 -0700 Subject: [PATCH 3/6] GUI startup : Add USD presets for ShaderTweaks and ShaderQuery --- Changes.md | 1 + startup/gui/shaderPresets.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index fda94fccdfc..f680b92fde7 100644 --- a/Changes.md +++ b/Changes.md @@ -5,6 +5,7 @@ Improvements ------------ - Viewer : Added visualisation of light filters for USD lights. +- ShaderTweaks/ShaderQuery : Added presets for USD light and surface shaders. Fixes ----- diff --git a/startup/gui/shaderPresets.py b/startup/gui/shaderPresets.py index 8467f9e9ec9..df0e3c674a7 100644 --- a/startup/gui/shaderPresets.py +++ b/startup/gui/shaderPresets.py @@ -84,4 +84,10 @@ def __registerShaderPresets( presets ) : ] ) -__registerShaderPresets( [ ( "OpenGL Surface", "gl:surface" ) ] ) +__registerShaderPresets( [ + + ( "OpenGL Surface", "gl:surface" ), + ( "USD Surface", "surface" ), + ( "USD Light", "light" ), + + ] ) From e1e076bd65f8f9c183554fed7ad38b740ef46bb6 Mon Sep 17 00:00:00 2001 From: Murray Stevenson <50844517+murraystevenson@users.noreply.github.com> Date: Thu, 17 Aug 2023 11:13:17 -0700 Subject: [PATCH 4/6] SceneViewInspector : Support USD lights and shaders --- Changes.md | 4 +++- python/GafferSceneUI/_SceneViewInspector.py | 9 +++++++-- startup/GafferSceneUI/sceneViewInspector.py | 8 ++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Changes.md b/Changes.md index 29af425aa4f..46e1cb8afa9 100644 --- a/Changes.md +++ b/Changes.md @@ -4,7 +4,9 @@ Improvements ------------ -- Viewer : Added visualisation of light filters for USD lights. +- Viewer : + - Added visualisation of light filters for USD lights. + - Added support for USD lights and shaders in the floating inspector panel. - ShaderTweaks/ShaderQuery : Added presets for USD light and surface shaders. Fixes diff --git a/python/GafferSceneUI/_SceneViewInspector.py b/python/GafferSceneUI/_SceneViewInspector.py index 2e3683c6c78..502bb92ebbc 100644 --- a/python/GafferSceneUI/_SceneViewInspector.py +++ b/python/GafferSceneUI/_SceneViewInspector.py @@ -67,7 +67,8 @@ "dl" : "Delight", "gl" : "OpenGL", "osl" : "OSL", - "cycles" : "Cycles" + "cycles" : "Cycles", + "" : "USD", }; __registeredShaderParameters = OrderedDict() @@ -176,7 +177,11 @@ def __keyPress( self, gadget, event ) : @staticmethod def __attributeLabel( attribute ) : - prefix, name = attribute.split( ":", 1 ) + prefix, _, name = attribute.partition( ":" ) + if not name : + name = prefix + prefix = "" + prefix = _rendererAttributePrefixes.get( prefix, prefix ) name = " ".join( [ IECore.CamelCase.toSpaced( n ) for n in name.split( ":" ) ] ) return "{} {}".format( prefix, name ) diff --git a/startup/GafferSceneUI/sceneViewInspector.py b/startup/GafferSceneUI/sceneViewInspector.py index 6f7918424d1..f3c08a39a0c 100644 --- a/startup/GafferSceneUI/sceneViewInspector.py +++ b/startup/GafferSceneUI/sceneViewInspector.py @@ -60,3 +60,11 @@ for p in ["base_color", "subsurface_color", "metallic", "subsurface", "subsurface_radius", "specular", "roughness", "specular_tint" ] : GafferSceneUI._SceneViewInspector.registerShaderParameter( "cycles:surface", p ) + +# USD + +for p in [ "intensity", "exposure", "color", "enableColorTemperature", "colorTemperature", "width", "height", "radius", "angle", "shaping:cone:angle", "shaping:cone:softness" ] : + GafferSceneUI._SceneViewInspector.registerShaderParameter( "light", p ) + +for p in [ "diffuseColor", "emissiveColor", "useSpecularWorkflow", "specularColor", "metallic", "roughness", "clearcoat", "clearcoatRoughness" ] : + GafferSceneUI._SceneViewInspector.registerShaderParameter( "surface", p ) From 9ff2426b45e6fee1645f7100ff8692bf8a25d71e Mon Sep 17 00:00:00 2001 From: John Haddon Date: Wed, 16 Aug 2023 15:56:50 +0100 Subject: [PATCH 5/6] PerformanceMonitor : Remove todo We don't want to clutter up the public API with Process subclasses. --- src/Gaffer/PerformanceMonitor.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Gaffer/PerformanceMonitor.cpp b/src/Gaffer/PerformanceMonitor.cpp index 646ae1c38a6..754ee298daa 100644 --- a/src/Gaffer/PerformanceMonitor.cpp +++ b/src/Gaffer/PerformanceMonitor.cpp @@ -41,8 +41,6 @@ using namespace Gaffer; -/// \todo If we expose ValuePlug::HashProcess and ValuePlug::ComputeProcess -/// then we can use the types defined there directly. static IECore::InternedString g_hashType( "computeNode:hash" ); static IECore::InternedString g_computeType( "computeNode:compute" ); static PerformanceMonitor::Statistics g_emptyStatistics; From a5f94ac32bdf58d8b1dbddeed0246fe94c4aa317 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Wed, 16 Aug 2023 15:59:49 +0100 Subject: [PATCH 6/6] ThreadMonitor : Add class for monitoring threads used for processes --- Changes.md | 5 + include/Gaffer/ThreadMonitor.h | 108 +++++++++++++++++++ python/GafferTest/ThreadMonitorTest.py | 143 +++++++++++++++++++++++++ python/GafferTest/__init__.py | 1 + src/Gaffer/ThreadMonitor.cpp | 121 +++++++++++++++++++++ src/GafferModule/MonitorBinding.cpp | 57 ++++++++++ src/GafferTestModule/ValuePlugTest.cpp | 3 + 7 files changed, 438 insertions(+) create mode 100644 include/Gaffer/ThreadMonitor.h create mode 100644 python/GafferTest/ThreadMonitorTest.py create mode 100644 src/Gaffer/ThreadMonitor.cpp diff --git a/Changes.md b/Changes.md index 46e1cb8afa9..9a8dfe1d0f5 100644 --- a/Changes.md +++ b/Changes.md @@ -14,6 +14,11 @@ Fixes - Viewer : Fixed crash when visualising lights with a light filter intended for a different renderer. +API +--- + +- ThreadMonitor : Added new class for tracking the threads used to perform processes. + Documentation ------------- diff --git a/include/Gaffer/ThreadMonitor.h b/include/Gaffer/ThreadMonitor.h new file mode 100644 index 00000000000..289ce328476 --- /dev/null +++ b/include/Gaffer/ThreadMonitor.h @@ -0,0 +1,108 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2023, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "Gaffer/Monitor.h" + +#include "tbb/enumerable_thread_specific.h" + +#include + +namespace Gaffer +{ + +IE_CORE_FORWARDDECLARE( Plug ) + +/// A monitor which collects information about which threads +/// initiated processes on each plug. +class GAFFER_API ThreadMonitor : public Monitor +{ + + public : + + ThreadMonitor( const std::vector &processMask = { "computeNode:compute" } ); + ~ThreadMonitor() override; + + IE_CORE_DECLAREMEMBERPTR( ThreadMonitor ) + + /// Numeric identifier for a thread. Using our own identifier rather + /// than `std::thread::id` so that we can bind it to Python (and assign + /// human-readable contiguous values). + using ThreadId = int; + /// Returns the `ThreadId` for the calling thread. + static ThreadId thisThreadId(); + /// Maps from `ThreadId` to the number of times a process has been + /// invoked on that thread. + using ProcessesPerThread = std::unordered_map; + /// Stores per-thread process counts per-plug. + using PlugMap = std::unordered_map; + + /// Query functions. These are not thread-safe, and must be called + /// only when the Monitor is not active (as defined by `Monitor::Scope`). + const PlugMap &allStatistics() const; + const ProcessesPerThread &plugStatistics( const Plug *plug ) const; + const ProcessesPerThread &combinedStatistics() const; + + protected : + + void processStarted( const Process *process ) override; + void processFinished( const Process *process ) override; + + private : + + const std::vector m_processMask; + + // We collect statistics into a per-thread data structure to avoid contention. + struct ThreadData + { + ThreadData(); + using ProcessesPerPlug = std::unordered_map; + ThreadId id; + ProcessesPerPlug processesPerPlug; + }; + mutable tbb::enumerable_thread_specific m_threadData; + + // Then when we want to query it, we collate it into `m_statistics`. + void collate() const; + mutable PlugMap m_statistics; + mutable ProcessesPerThread m_combinedStatistics; + +}; + +IE_CORE_DECLAREPTR( ThreadMonitor ) + +} // namespace Gaffer diff --git a/python/GafferTest/ThreadMonitorTest.py b/python/GafferTest/ThreadMonitorTest.py new file mode 100644 index 00000000000..e449e11b39e --- /dev/null +++ b/python/GafferTest/ThreadMonitorTest.py @@ -0,0 +1,143 @@ +########################################################################## +# +# Copyright (c) 2023, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import threading +import unittest + +import IECore + +import Gaffer +import GafferTest + +class ThreadMonitorTest( GafferTest.TestCase ) : + + def testConstruction( self ) : + + monitor = Gaffer.ThreadMonitor() + self.assertEqual( monitor.allStatistics(), {} ) + self.assertEqual( monitor.plugStatistics( Gaffer.IntPlug() ), {} ) + self.assertEqual( monitor.combinedStatistics(), {} ) + + def testThisThreadId( self ) : + + id = Gaffer.ThreadMonitor.thisThreadId() + self.assertEqual( id, Gaffer.ThreadMonitor.thisThreadId() ) + + ids = { id } + lock = threading.Lock() + + def storeId() : + id = Gaffer.ThreadMonitor.thisThreadId() + self.assertEqual( id, Gaffer.ThreadMonitor.thisThreadId() ) + with lock : + ids.add( id ) + + threads = [] + for i in range( 0, 5 ) : + thread = threading.Thread( target = storeId ) + threads.append( thread ) + thread.start() + + for thread in threads : + thread.join() + + self.assertEqual( len( ids ), 6 ) + + def testMonitoring( self ) : + + random = Gaffer.Random() + monitor = Gaffer.ThreadMonitor() + + with monitor : + random["outFloat"].getValue() + + self.assertEqual( + monitor.allStatistics(), + { + random["outFloat"] : { + monitor.thisThreadId() : 1 + } + } + ) + self.assertEqual( + monitor.plugStatistics( random["outFloat"] ), + { monitor.thisThreadId() : 1 } + ) + self.assertEqual( + monitor.combinedStatistics(), + { monitor.thisThreadId() : 1 } + ) + + random["seedVariable"].setValue( "test" ) + with monitor : + GafferTest.parallelGetValue( random["outFloat"], 100000, "test" ) + + s = monitor.plugStatistics( random["outFloat"] ) + self.assertEqual( len( s ), IECore.tbb_global_control.active_value( IECore.tbb_global_control.parameter.max_allowed_parallelism ) ) + self.assertEqual( sum( s.values() ), 100001 ) + + self.assertEqual( monitor.allStatistics(), { random["outFloat"] : s } ) + self.assertEqual( monitor.combinedStatistics(), s ) + + def testProcessMask( self ) : + + for processType in [ "computeNode:hash", "computeNode:compute" ] : + + with self.subTest( processType = processType ) : + + Gaffer.ValuePlug.clearCache() + Gaffer.ValuePlug.clearHashCache() + + random = Gaffer.Random() + threadMonitor = Gaffer.ThreadMonitor( processMask = { processType } ) + performanceMonitor = Gaffer.PerformanceMonitor() + context = Gaffer.Context() + + with threadMonitor, performanceMonitor, context : + for i in range( 0, 5 ) : + context["i"] = i # Unique context to force hashing + random["outFloat"].getValue() + + self.assertEqual( performanceMonitor.plugStatistics( random["outFloat"] ).computeCount, 1 ) + self.assertEqual( performanceMonitor.plugStatistics( random["outFloat"] ).hashCount, 5 ) + + self.assertEqual( + sum( threadMonitor.plugStatistics( random["outFloat"] ).values() ), + 1 if processType == "computeNode:compute" else 5 + ) + +if __name__ == "__main__": + unittest.main() diff --git a/python/GafferTest/__init__.py b/python/GafferTest/__init__.py index df7ea2bccb5..fc78a62be9b 100644 --- a/python/GafferTest/__init__.py +++ b/python/GafferTest/__init__.py @@ -157,6 +157,7 @@ def inCI( platforms = set() ) : from .HiddenFilePathFilterTest import HiddenFilePathFilterTest from .ContextVariableTweaksTest import ContextVariableTweaksTest from .OptionalValuePlugTest import OptionalValuePlugTest +from .ThreadMonitorTest import ThreadMonitorTest from .IECorePreviewTest import * diff --git a/src/Gaffer/ThreadMonitor.cpp b/src/Gaffer/ThreadMonitor.cpp new file mode 100644 index 00000000000..8391fef4ac4 --- /dev/null +++ b/src/Gaffer/ThreadMonitor.cpp @@ -0,0 +1,121 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2023, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "Gaffer/ThreadMonitor.h" + +#include "Gaffer/Plug.h" +#include "Gaffer/Process.h" + +using namespace Gaffer; + +namespace +{ + +static std::atomic g_threadIdCounter = 0; +ThreadMonitor::ProcessesPerThread g_emptyStatistics; + +} // namespace + +ThreadMonitor::ThreadData::ThreadData() + : id( thisThreadId() ) +{ +} + +ThreadMonitor::ThreadMonitor( const std::vector &processMask ) + : m_processMask( processMask ) +{ +} + +ThreadMonitor::~ThreadMonitor() +{ +} + +ThreadMonitor::ThreadId ThreadMonitor::thisThreadId() +{ + thread_local int id = g_threadIdCounter++; + return id; +} + +const ThreadMonitor::PlugMap &ThreadMonitor::allStatistics() const +{ + collate(); + return m_statistics; +} + +const ThreadMonitor::ProcessesPerThread &ThreadMonitor::plugStatistics( const Plug *plug ) const +{ + collate(); + auto it = m_statistics.find( plug ); + if( it == m_statistics.end() ) + { + return g_emptyStatistics; + } + return it->second; +} + +const ThreadMonitor::ProcessesPerThread &ThreadMonitor::combinedStatistics() const +{ + collate(); + return m_combinedStatistics; +} + +void ThreadMonitor::processStarted( const Process *process ) +{ + if( std::find( m_processMask.begin(), m_processMask.end(), process->type() ) == m_processMask.end() ) + { + return; + } + + ThreadData &threadData = m_threadData.local(); + threadData.processesPerPlug[process->plug()]++; +} + +void ThreadMonitor::processFinished( const Process *process ) +{ +} + +void ThreadMonitor::collate() const +{ + for( auto &threadData : m_threadData ) + { + for( const auto &[plug, count] : threadData.processesPerPlug ) + { + m_statistics[plug][threadData.id] += count; + m_combinedStatistics[threadData.id] += count; + } + threadData.processesPerPlug.clear(); + } +} diff --git a/src/GafferModule/MonitorBinding.cpp b/src/GafferModule/MonitorBinding.cpp index 0663dea8dfa..0182e9455d7 100644 --- a/src/GafferModule/MonitorBinding.cpp +++ b/src/GafferModule/MonitorBinding.cpp @@ -44,11 +44,14 @@ #include "Gaffer/Node.h" #include "Gaffer/PerformanceMonitor.h" #include "Gaffer/Plug.h" +#include "Gaffer/ThreadMonitor.h" #include "Gaffer/VTuneMonitor.h" #include "IECorePython/RefCountedBinding.h" #include "IECorePython/ScopedGILRelease.h" +#include "boost/python/suite/indexing/container_utils.hpp" + #include "fmt/format.h" using namespace boost::python; @@ -149,6 +152,43 @@ void removeContextAnnotationsWrapper( Node &root ) MonitorAlgo::removeContextAnnotations( root ); } +ThreadMonitor::Ptr threadMonitorConstructor( boost::python::object pythonProcessMask ) +{ + std::vector processMask; + container_utils::extend_container( processMask, pythonProcessMask ); + return new ThreadMonitor( processMask ); +} + +dict processesPerThreadToPython( const ThreadMonitor::ProcessesPerThread &processesPerThread ) +{ + dict result; + for( auto [id, count] : processesPerThread ) + { + result[id] = count; + } + return result; +} + +dict threadMonitorAllStatisticsWrapper( const ThreadMonitor &monitor ) +{ + dict result; + for( const auto &[plug, processesPerThread] : monitor.allStatistics() ) + { + result[boost::const_pointer_cast( plug )] = processesPerThreadToPython( processesPerThread ); + } + return result; +} + +dict threadMonitorPlugStatisticsWrapper( const ThreadMonitor &monitor, const Plug &plug ) +{ + return processesPerThreadToPython( monitor.plugStatistics( &plug ) ); +} + +dict threadMonitorCombinedStatisticsWrapper( const ThreadMonitor &monitor ) +{ + return processesPerThreadToPython( monitor.combinedStatistics() ); +} + } // namespace void GafferModule::bindMonitor() @@ -264,6 +304,23 @@ void GafferModule::bindMonitor() ; } + { + scope s = IECorePython::RefCountedClass( "ThreadMonitor" ) + .def( + "__init__", + make_constructor( + threadMonitorConstructor, default_call_policies(), + arg( "processMask" ) = boost::python::make_tuple( "computeNode:compute" ) + ) + ) + .def( "thisThreadId", &ThreadMonitor::thisThreadId ) + .staticmethod( "thisThreadId" ) + .def( "allStatistics", &threadMonitorAllStatisticsWrapper ) + .def( "plugStatistics", &threadMonitorPlugStatisticsWrapper ) + .def( "combinedStatistics", &threadMonitorCombinedStatisticsWrapper ) + ; + } + #ifdef GAFFER_VTUNE { scope s = IECorePython::RefCountedClass( "VTuneMonitor" ) diff --git a/src/GafferTestModule/ValuePlugTest.cpp b/src/GafferTestModule/ValuePlugTest.cpp index a5bc7313565..ea4c28f8675 100644 --- a/src/GafferTestModule/ValuePlugTest.cpp +++ b/src/GafferTestModule/ValuePlugTest.cpp @@ -112,9 +112,12 @@ void parallelGetValueWithVar( const T *plug, int iterations, const IECore::Inter void GafferTestModule::bindValuePlugTest() { def( "repeatGetValue", &repeatGetValue ); + def( "repeatGetValue", &repeatGetValue ); def( "repeatGetValue", &repeatGetValue ); def( "parallelGetValue", ¶llelGetValue ); + def( "parallelGetValue", ¶llelGetValue ); def( "parallelGetValue", ¶llelGetValue ); def( "parallelGetValue", ¶llelGetValueWithVar ); + def( "parallelGetValue", ¶llelGetValueWithVar ); def( "parallelGetValue", ¶llelGetValueWithVar ); }