From b22e84dcb77fa8e20b7a1c0694f87dc1d9a6ab75 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Mon, 25 Nov 2024 16:22:56 -0800 Subject: [PATCH 1/9] (ci/cd) Drop support for cy2022 the ci-2022 image has been failing the pipelines for using an unsuported version of GLIBC (see https://github.com/AcademySoftwareFoundation/OpenCue/actions/runs/12016939050/job/33498210814) --- .github/workflows/testing-pipeline.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/testing-pipeline.yml b/.github/workflows/testing-pipeline.yml index fe3ade9a6..75d8c8111 100644 --- a/.github/workflows/testing-pipeline.yml +++ b/.github/workflows/testing-pipeline.yml @@ -7,20 +7,6 @@ on: branches: [ master ] jobs: - test_cuebot_2022: - name: Build Cuebot and Run Unit Tests (CY2022) - runs-on: ubuntu-22.04 - container: - image: aswf/ci-opencue:2022 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - steps: - - uses: actions/checkout@v3 - - name: Build with Gradle - run: | - chown -R aswfuser:aswfgroup . - su -c "cd cuebot && ./gradlew build --stacktrace --info" aswfuser - test_python_2023: name: Run Python Unit Tests (CY2023) runs-on: ubuntu-22.04 @@ -66,7 +52,7 @@ jobs: lint_python: name: Lint Python Code runs-on: ubuntu-22.04 - container: aswf/ci-opencue:2022 + container: aswf/ci-opencue:2024.1 env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: From 919f4f5ba63b22540eb8af5b238153bbba038c1b Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 27 Nov 2024 09:49:30 -0800 Subject: [PATCH 2/9] Fix lint errors --- ci/pylintrc_test | 1 - .../spcue/dispatcher/HostReportHandler.java | 892 ++++++++++++------ cuegui/cuegui/DependWizard.py | 14 + cuegui/cuegui/ShowsWidget.py | 2 +- cuegui/cuegui/SubscriptionGraphWidget.py | 4 +- cuegui/cuegui/ThreadPool.py | 1 + 6 files changed, 641 insertions(+), 273 deletions(-) diff --git a/ci/pylintrc_test b/ci/pylintrc_test index 434a1d754..d84fc406d 100644 --- a/ci/pylintrc_test +++ b/ci/pylintrc_test @@ -67,7 +67,6 @@ disable=arguments-differ, locally-disabled, missing-class-docstring, missing-function-docstring, - no-self-use, protected-access, raise-missing-from, too-many-arguments, diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java index 39423e80b..98a9df495 100644 --- a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java +++ b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java @@ -1,4 +1,3 @@ - /* * Copyright Contributors to the OpenCue Project * @@ -15,38 +14,20 @@ * limitations under the License. */ - - package com.imageworks.spcue.dispatcher; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import static com.imageworks.spcue.dispatcher.Dispatcher.*; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.imageworks.spcue.JobInterface; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.env.Environment; -import org.springframework.core.task.TaskRejectedException; -import org.springframework.dao.DataAccessException; -import org.springframework.dao.EmptyResultDataAccessException; - import com.imageworks.spcue.CommentDetail; import com.imageworks.spcue.DispatchHost; -import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.FrameDetail; +import com.imageworks.spcue.FrameInterface; import com.imageworks.spcue.JobEntity; -import com.imageworks.spcue.LayerEntity; +import com.imageworks.spcue.JobInterface; import com.imageworks.spcue.LayerDetail; +import com.imageworks.spcue.LayerEntity; import com.imageworks.spcue.LocalHostAssignment; import com.imageworks.spcue.PrometheusMetricsCollector; import com.imageworks.spcue.Source; @@ -72,12 +53,28 @@ import com.imageworks.spcue.service.HostManager; import com.imageworks.spcue.service.JobManager; import com.imageworks.spcue.util.CueUtil; - -import static com.imageworks.spcue.dispatcher.Dispatcher.*; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.env.Environment; +import org.springframework.core.task.TaskRejectedException; +import org.springframework.dao.DataAccessException; +import org.springframework.dao.EmptyResultDataAccessException; public class HostReportHandler { - private static final Logger logger = LogManager.getLogger(HostReportHandler.class); + private static final Logger logger = LogManager.getLogger( + HostReportHandler.class + ); private BookingManager bookingManager; private HostManager hostManager; @@ -91,16 +88,20 @@ public class HostReportHandler { private JobManager jobManager; private JobDao jobDao; private LayerDao layerDao; + @Autowired private Environment env; + @Autowired private CommentManager commentManager; + @Autowired private PrometheusMetricsCollector prometheusMetrics; // Comment constants - private static final String SUBJECT_COMMENT_FULL_TEMP_DIR = "Host set to REPAIR for not having enough storage " + - "space on the temporary directory (mcp)"; + private static final String SUBJECT_COMMENT_FULL_TEMP_DIR = + "Host set to REPAIR for not having enough storage " + + "space on the temporary directory (mcp)"; private static final String CUEBOT_COMMENT_USER = "cuebot"; private static final String WINDOWS_OS = "Windows"; @@ -108,8 +109,11 @@ public class HostReportHandler { // The cache expires after write to avoid growing unbounded. If a request for a host-frame doesn't appear // for a period of time, the entry will be removed. Cache killRequestCounterCache = CacheBuilder.newBuilder() - .expireAfterWrite(FRAME_KILL_CACHE_EXPIRE_AFTER_WRITE_MINUTES, TimeUnit.MINUTES) - .build(); + .expireAfterWrite( + FRAME_KILL_CACHE_EXPIRE_AFTER_WRITE_MINUTES, + TimeUnit.MINUTES + ) + .build(); /** * Boolean to toggle if this class is accepting data or not. @@ -141,8 +145,9 @@ public synchronized void shutdown() { public void queueBootReport(BootReport report) { if (isShutdown()) { throw new RqdRetryReportException( - "Error processing host report. Cuebot not " + - "accepting packets."); + "Error processing host report. Cuebot not " + + "accepting packets." + ); } reportQueue.execute(new DispatchHandleHostReport(report, this)); } @@ -155,8 +160,9 @@ public void queueBootReport(BootReport report) { public void queueHostReport(HostReport report) { if (isShutdown()) { throw new RqdRetryReportException( - "Error processing host report. Cuebot not " + - "accepting packets."); + "Error processing host report. Cuebot not " + + "accepting packets." + ); } reportQueue.execute(new DispatchHandleHostReport(report, this)); } @@ -164,31 +170,45 @@ public void queueHostReport(HostReport report) { public void handleHostReport(HostReport report, boolean isBoot) { long startTime = System.currentTimeMillis(); try { - long swapOut = 0; if (report.getHost().getAttributesMap().containsKey("swapout")) { - swapOut = Integer.parseInt(report.getHost().getAttributesMap().get("swapout")); - if (swapOut > 0) - logger.info(report.getHost().getName() + " swapout: " + - report.getHost().getAttributesMap().get("swapout")); + swapOut = Integer.parseInt( + report.getHost().getAttributesMap().get("swapout") + ); + if (swapOut > 0) logger.info( + report.getHost().getName() + + " swapout: " + + report.getHost().getAttributesMap().get("swapout") + ); } DispatchHost host; RenderHost rhost = report.getHost(); try { host = hostManager.findDispatchHost(rhost.getName()); - hostManager.setHostStatistics(host, - rhost.getTotalMem(), rhost.getFreeMem(), - rhost.getTotalSwap(), rhost.getFreeSwap(), - rhost.getTotalMcp(), rhost.getFreeMcp(), - rhost.getTotalGpuMem(), rhost.getFreeGpuMem(), - rhost.getLoad(), new Timestamp(rhost.getBootTime() * 1000l), - rhost.getAttributesMap().get("SP_OS")); + hostManager.setHostStatistics( + host, + rhost.getTotalMem(), + rhost.getFreeMem(), + rhost.getTotalSwap(), + rhost.getFreeSwap(), + rhost.getTotalMcp(), + rhost.getFreeMcp(), + rhost.getTotalGpuMem(), + rhost.getFreeGpuMem(), + rhost.getLoad(), + new Timestamp(rhost.getBootTime() * 1000l), + rhost.getAttributesMap().get("SP_OS") + ); // Both logics are conflicting, only change hardware state if // there was no need for a tempDirStorage state change if (!changeStateForTempDirStorage(host, report.getHost())) { - changeHardwareState(host, report.getHost().getState(), isBoot); + changeHardwareState( + host, + report.getHost().getState(), + isBoot + ); } changeNimbyState(host, report.getHost()); @@ -200,16 +220,22 @@ public void handleHostReport(HostReport report, boolean isBoot) { hostManager.setHostResources(host, report); } - dispatchSupport.determineIdleCores(host, report.getHost().getLoad()); - + dispatchSupport.determineIdleCores( + host, + report.getHost().getLoad() + ); } catch (DataAccessException dae) { - logger.info("Unable to find host " + rhost.getName() + "," - + dae + " , creating host."); + logger.info( + "Unable to find host " + + rhost.getName() + + "," + + dae + + " , creating host." + ); // TODO: Skip adding it if the host name is over 30 characters host = hostManager.createHost(report); - } - catch (Exception e) { + } catch (Exception e) { logger.warn("Error processing HostReport, " + e); return; } @@ -218,7 +244,9 @@ public void handleHostReport(HostReport report, boolean isBoot) { * Verify all the frames in the report are valid. * Frames that are not valid are removed. */ - List runningFrames = verifyRunningFrameInfo(report); + List runningFrames = verifyRunningFrameInfo( + report + ); /* * Updates memory usage for the proc, frames, @@ -245,7 +273,9 @@ public void handleHostReport(HostReport report, boolean isBoot) { */ String msg = null; boolean hasLocalJob = bookingManager.hasLocalHostAssignment(host); - int coresToReserve = host.handleNegativeCoresRequirement(Dispatcher.CORE_POINTS_RESERVED_MIN); + int coresToReserve = host.handleNegativeCoresRequirement( + Dispatcher.CORE_POINTS_RESERVED_MIN + ); if (hasLocalJob) { List lcas = @@ -255,38 +285,55 @@ public void handleHostReport(HostReport report, boolean isBoot) { } } long memReservedMin = env.getRequiredProperty( - "dispatcher.memory.mem_reserved_min", - Long.class); - - if (!isTempDirStorageEnough(report.getHost().getTotalMcp(), report.getHost().getFreeMcp(), host.getOs())) { + "dispatcher.memory.mem_reserved_min", + Long.class + ); + + if ( + !isTempDirStorageEnough( + report.getHost().getTotalMcp(), + report.getHost().getFreeMcp(), + host.getOs() + ) + ) { msg = String.format( "%s doesn't have enough free space in the temporary directory (mcp), %dMB", - host.name, (report.getHost().getFreeMcp()/1024)); - } - else if (coresToReserve <= 0 || host.idleCores < Dispatcher.CORE_POINTS_RESERVED_MIN) { - msg = String.format("%s doesn't have enough idle cores, %d needs %d", - host.name, host.idleCores, Dispatcher.CORE_POINTS_RESERVED_MIN); - } - else if (host.idleMemory < memReservedMin) { - msg = String.format("%s doesn't have enough idle memory, %d needs %d", - host.name, host.idleMemory, memReservedMin); - } - else if (report.getHost().getFreeMem() < CueUtil.MB512) { - msg = String.format("%s doesn't have enough free system mem, %d needs %d", - host.name, report.getHost().getFreeMem(), memReservedMin); - } - else if(!host.hardwareState.equals(HardwareState.UP)) { + host.name, + (report.getHost().getFreeMcp() / 1024) + ); + } else if ( + coresToReserve <= 0 || + host.idleCores < Dispatcher.CORE_POINTS_RESERVED_MIN + ) { + msg = String.format( + "%s doesn't have enough idle cores, %d needs %d", + host.name, + host.idleCores, + Dispatcher.CORE_POINTS_RESERVED_MIN + ); + } else if (host.idleMemory < memReservedMin) { + msg = String.format( + "%s doesn't have enough idle memory, %d needs %d", + host.name, + host.idleMemory, + memReservedMin + ); + } else if (report.getHost().getFreeMem() < CueUtil.MB512) { + msg = String.format( + "%s doesn't have enough free system mem, %d needs %d", + host.name, + report.getHost().getFreeMem(), + memReservedMin + ); + } else if (!host.hardwareState.equals(HardwareState.UP)) { msg = host + " is not in the Up state."; - } - else if (host.lockState.equals(LockState.LOCKED)) { + } else if (host.lockState.equals(LockState.LOCKED)) { msg = host + " is locked."; - } - else if (report.getHost().getNimbyLocked()) { + } else if (report.getHost().getNimbyLocked()) { if (!hasLocalJob) { msg = host + " is NIMBY locked."; } - } - else if (!dispatchSupport.isCueBookable(host)) { + } else if (!dispatchSupport.isCueBookable(host)) { msg = "The cue has no pending jobs"; } @@ -296,9 +343,7 @@ else if (!dispatchSupport.isCueBookable(host)) { */ if (msg != null) { logger.trace(msg); - } - else { - + } else { // check again. The dangling local host assignment could be removed. hasLocalJob = bookingManager.hasLocalHostAssignment(host); @@ -308,8 +353,8 @@ else if (!dispatchSupport.isCueBookable(host)) { if (hasLocalJob) { if (!bookingManager.hasResourceDeficit(host)) { bookingQueue.execute( - new DispatchBookHostLocal( - host, localDispatcher)); + new DispatchBookHostLocal(host, localDispatcher) + ); } return; } @@ -319,25 +364,38 @@ else if (!dispatchSupport.isCueBookable(host)) { * to that show first. */ if (hostManager.isPreferShow(host)) { - bookingQueue.execute(new DispatchBookHost( - host, hostManager.getPreferredShow(host), dispatcher, env)); + bookingQueue.execute( + new DispatchBookHost( + host, + hostManager.getPreferredShow(host), + dispatcher, + env + ) + ); return; } - bookingQueue.execute(new DispatchBookHost(host, dispatcher, env)); + bookingQueue.execute( + new DispatchBookHost(host, dispatcher, env) + ); } - } finally { - if (reportQueue.getQueue().size() > 0 || - System.currentTimeMillis() - startTime > 100) { + if ( + reportQueue.getQueue().size() > 0 || + System.currentTimeMillis() - startTime > 100 + ) { /* * Write a log if the host report takes a long time to process. */ - CueUtil.logDuration(startTime, "host report " + - report.getHost().getName() + " with " + - report.getFramesCount() + - " running frames, waiting: " + - reportQueue.getQueue().size()); + CueUtil.logDuration( + startTime, + "host report " + + report.getHost().getName() + + " with " + + report.getFramesCount() + + " running frames, waiting: " + + reportQueue.getQueue().size() + ); } } } @@ -354,16 +412,25 @@ else if (!dispatchSupport.isCueBookable(host)) { * @param hostOs Reported operational systems * @return */ - private boolean isTempDirStorageEnough(Long tempTotalStorage, Long tempFreeStorage, String[] hostOs) { + private boolean isTempDirStorageEnough( + Long tempTotalStorage, + Long tempFreeStorage, + String[] hostOs + ) { // The minimum amount of free space in the temporary directory to book a host int minAvailableTempPercentage = env.getRequiredProperty( - "dispatcher.min_available_temp_storage_percentage", Integer.class); + "dispatcher.min_available_temp_storage_percentage", + Integer.class + ); - return minAvailableTempPercentage == -1 + return ( + minAvailableTempPercentage == -1 || // It is safe to assume multiple OSs imply windows is not the base OS, // threfore Windows will always report a single hostOs - || (hostOs.length == 1 && hostOs[0].equalsIgnoreCase(WINDOWS_OS)) - || (((tempFreeStorage * 100.0) / tempTotalStorage) >= minAvailableTempPercentage); + (hostOs.length == 1 && hostOs[0].equalsIgnoreCase(WINDOWS_OS)) || + (((tempFreeStorage * 100.0) / tempTotalStorage) >= + minAvailableTempPercentage) + ); } /** @@ -379,7 +446,11 @@ private boolean isTempDirStorageEnough(Long tempTotalStorage, Long tempFreeStora * @param reportState * @param isBoot */ - private void changeHardwareState(DispatchHost host, HardwareState reportState, boolean isBoot) { + private void changeHardwareState( + DispatchHost host, + HardwareState reportState, + boolean isBoot + ) { // If the states are the same there is no reason to do this update. if (host.hardwareState.equals(reportState)) { return; @@ -405,7 +476,6 @@ private void changeHardwareState(DispatchHost host, HardwareState reportState, b hostManager.setHostState(host, reportState); host.hardwareState = reportState; break; - } } @@ -424,26 +494,41 @@ private void changeHardwareState(DispatchHost host, HardwareState reportState, b * @param reportHost * @return */ - private boolean changeStateForTempDirStorage(DispatchHost host, RenderHost reportHost) { + private boolean changeStateForTempDirStorage( + DispatchHost host, + RenderHost reportHost + ) { // The minimum amount of free space in the temporary directory to book a host int minAvailableTempPercentage = env.getRequiredProperty( - "dispatcher.min_available_temp_storage_percentage", Integer.class); + "dispatcher.min_available_temp_storage_percentage", + Integer.class + ); // Prevent cue frames from booking on hosts with full temporary directories boolean hasEnoughTempStorage = isTempDirStorageEnough( reportHost.getTotalMcp(), reportHost.getFreeMcp(), - host.getOs()); + host.getOs() + ); if (!hasEnoughTempStorage && host.hardwareState == HardwareState.UP) { // Insert a comment indicating that the Host status = Repair with reason = Full temporary directory CommentDetail c = new CommentDetail(); c.subject = SUBJECT_COMMENT_FULL_TEMP_DIR; c.user = CUEBOT_COMMENT_USER; c.timestamp = null; - long requiredTempMb = (long)((minAvailableTempPercentage / 100.0) * reportHost.getTotalMcp()/ 1024); - c.message = "Host " + host.getName() + " marked as REPAIR. The current amount of free space in the " + - "temporary directory (mcp) is " + (reportHost.getFreeMcp()/1024) + "MB. It must have at least " - + ((requiredTempMb)) + "MB of free space in temporary directory"; + long requiredTempMb = (long) (((minAvailableTempPercentage / + 100.0) * + reportHost.getTotalMcp()) / + 1024); + c.message = + "Host " + + host.getName() + + " marked as REPAIR. The current amount of free space in the " + + "temporary directory (mcp) is " + + (reportHost.getFreeMcp() / 1024) + + "MB. It must have at least " + + ((requiredTempMb)) + + "MB of free space in temporary directory"; commentManager.addComment(host, c); // Set the host state to REPAIR @@ -451,11 +536,17 @@ private boolean changeStateForTempDirStorage(DispatchHost host, RenderHost repor host.hardwareState = HardwareState.REPAIR; return true; - } else if (hasEnoughTempStorage && host.hardwareState == HardwareState.REPAIR) { + } else if ( + hasEnoughTempStorage && host.hardwareState == HardwareState.REPAIR + ) { // Check if the host with REPAIR status has comments with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and // user=CUEBOT_COMMENT_USER and delete the comments, if they exist - boolean commentsDeleted = commentManager.deleteCommentByHostUserAndSubject(host, - CUEBOT_COMMENT_USER, SUBJECT_COMMENT_FULL_TEMP_DIR); + boolean commentsDeleted = + commentManager.deleteCommentByHostUserAndSubject( + host, + CUEBOT_COMMENT_USER, + SUBJECT_COMMENT_FULL_TEMP_DIR + ); if (commentsDeleted) { // Set the host state to UP @@ -479,13 +570,20 @@ private void changeNimbyState(DispatchHost host, RenderHost rh) { if (rh.getNimbyLocked()) { if (host.lockState.equals(LockState.OPEN)) { host.lockState = LockState.NIMBY_LOCKED; - hostManager.setHostLock(host,LockState.NIMBY_LOCKED, new Source("NIMBY")); + hostManager.setHostLock( + host, + LockState.NIMBY_LOCKED, + new Source("NIMBY") + ); } - } - else { + } else { if (host.lockState.equals(LockState.NIMBY_LOCKED)) { host.lockState = LockState.OPEN; - hostManager.setHostLock(host,LockState.OPEN, new Source("NIMBY")); + hostManager.setHostLock( + host, + LockState.OPEN, + new Source("NIMBY") + ); } } } @@ -501,11 +599,19 @@ private void changeLockState(DispatchHost host, CoreDetail coreInfo) { if (host.lockState == LockState.LOCKED) { if (coreInfo.getLockedCores() < coreInfo.getTotalCores()) { host.lockState = LockState.OPEN; - hostManager.setHostLock(host, LockState.OPEN, new Source("cores")); + hostManager.setHostLock( + host, + LockState.OPEN, + new Source("cores") + ); } } else if (coreInfo.getLockedCores() >= coreInfo.getTotalCores()) { host.lockState = LockState.LOCKED; - hostManager.setHostLock(host, LockState.LOCKED, new Source("cores")); + hostManager.setHostLock( + host, + LockState.LOCKED, + new Source("cores") + ); } } @@ -525,53 +631,84 @@ private void changeLockState(DispatchHost host, CoreDetail coreInfo) { * @param dispatchHost * @param report */ - private void handleMemoryUsage(final DispatchHost dispatchHost, RenderHost renderHost, - List runningFrames) { + private void handleMemoryUsage( + final DispatchHost dispatchHost, + RenderHost renderHost, + List runningFrames + ) { // Don't keep memory balances on nimby hosts and hosts with invalid memory // information if (dispatchHost.isNimby || renderHost.getTotalMem() <= 0) { return; } - final double OOM_MAX_SAFE_USED_PHYSICAL_THRESHOLD = env - .getRequiredProperty("dispatcher.oom_max_safe_used_physical_memory_threshold", Double.class); - final double OOM_MAX_SAFE_USED_SWAP_THRESHOLD = env - .getRequiredProperty("dispatcher.oom_max_safe_used_swap_memory_threshold", Double.class); - final double OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD = env - .getRequiredProperty("dispatcher.oom_frame_overboard_allowed_threshold", Double.class); + final double OOM_MAX_SAFE_USED_PHYSICAL_THRESHOLD = + env.getRequiredProperty( + "dispatcher.oom_max_safe_used_physical_memory_threshold", + Double.class + ); + final double OOM_MAX_SAFE_USED_SWAP_THRESHOLD = env.getRequiredProperty( + "dispatcher.oom_max_safe_used_swap_memory_threshold", + Double.class + ); + final double OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD = + env.getRequiredProperty( + "dispatcher.oom_frame_overboard_allowed_threshold", + Double.class + ); - Double physMemoryUsageRatio = renderHost.getTotalMem() > 0 ? - 1.0 - renderHost.getFreeMem() / (double) renderHost.getTotalMem() : - 0.0; + Double physMemoryUsageRatio = renderHost.getTotalMem() > 0 + ? 1.0 - renderHost.getFreeMem() / (double) renderHost.getTotalMem() + : 0.0; - Double swapMemoryUsageRatio = renderHost.getTotalSwap() > 0 ? - 1.0 - renderHost.getFreeSwap() / (double) renderHost.getTotalSwap() : - 0.0; + Double swapMemoryUsageRatio = renderHost.getTotalSwap() > 0 + ? 1.0 - + renderHost.getFreeSwap() / (double) renderHost.getTotalSwap() + : 0.0; // If checking for the swap threshold has been disabled, only memory usage is // taken into consideration. // If checking for memory has been disabled, checking for swap isolated is not // safe, therefore disabled boolean memoryWarning = false; - if (OOM_MAX_SAFE_USED_PHYSICAL_THRESHOLD > 0.0 && OOM_MAX_SAFE_USED_SWAP_THRESHOLD > 0.0 && - !physMemoryUsageRatio.isNaN() && !swapMemoryUsageRatio.isNaN()) { - memoryWarning = physMemoryUsageRatio > OOM_MAX_SAFE_USED_PHYSICAL_THRESHOLD && + if ( + OOM_MAX_SAFE_USED_PHYSICAL_THRESHOLD > 0.0 && + OOM_MAX_SAFE_USED_SWAP_THRESHOLD > 0.0 && + !physMemoryUsageRatio.isNaN() && + !swapMemoryUsageRatio.isNaN() + ) { + memoryWarning = + physMemoryUsageRatio > OOM_MAX_SAFE_USED_PHYSICAL_THRESHOLD && swapMemoryUsageRatio > OOM_MAX_SAFE_USED_SWAP_THRESHOLD; - } else if (OOM_MAX_SAFE_USED_PHYSICAL_THRESHOLD > 0.0 && !physMemoryUsageRatio.isNaN()) { - memoryWarning = physMemoryUsageRatio > OOM_MAX_SAFE_USED_PHYSICAL_THRESHOLD; + } else if ( + OOM_MAX_SAFE_USED_PHYSICAL_THRESHOLD > 0.0 && + !physMemoryUsageRatio.isNaN() + ) { + memoryWarning = + physMemoryUsageRatio > OOM_MAX_SAFE_USED_PHYSICAL_THRESHOLD; } if (memoryWarning) { - logger.warn("Memory warning(" + renderHost.getName() + "): physMemoryRatio: " + - physMemoryUsageRatio + ", swapRatio: " + swapMemoryUsageRatio); + logger.warn( + "Memory warning(" + + renderHost.getName() + + "): physMemoryRatio: " + + physMemoryUsageRatio + + ", swapRatio: " + + swapMemoryUsageRatio + ); // Try to kill frames using swap memory as they are probably performing poorly - long swapUsed = renderHost.getTotalSwap() - renderHost.getFreeSwap(); - long maxSwapUsageAllowed = (long) (renderHost.getTotalSwap() - * OOM_MAX_SAFE_USED_SWAP_THRESHOLD); + long swapUsed = + renderHost.getTotalSwap() - renderHost.getFreeSwap(); + long maxSwapUsageAllowed = (long) (renderHost.getTotalSwap() * + OOM_MAX_SAFE_USED_SWAP_THRESHOLD); // Sort runningFrames bassed on how much swap they are using - runningFrames.sort(Comparator.comparingLong((RunningFrameInfo frame) -> - frame.getUsedSwapMemory()).reversed()); + runningFrames.sort( + Comparator.comparingLong((RunningFrameInfo frame) -> + frame.getUsedSwapMemory() + ).reversed() + ); int killAttemptsRemaining = 5; for (RunningFrameInfo frame : runningFrames) { @@ -579,16 +716,31 @@ private void handleMemoryUsage(final DispatchHost dispatchHost, RenderHost rende if (frame.getUsedSwapMemory() <= 0) { break; } - if (killProcForMemory(frame.getFrameId(), renderHost.getName(), - KillCause.HostUnderOom)) { + if ( + killProcForMemory( + frame.getFrameId(), + renderHost.getName(), + KillCause.HostUnderOom + ) + ) { swapUsed -= frame.getUsedSwapMemory(); - logger.info("Memory warning(" + renderHost.getName() + "): " + - "Killing frame on " + frame.getJobName() + "." + - frame.getFrameName() + ", using too much swap."); + logger.info( + "Memory warning(" + + renderHost.getName() + + "): " + + "Killing frame on " + + frame.getJobName() + + "." + + frame.getFrameName() + + ", using too much swap." + ); } killAttemptsRemaining -= 1; - if (killAttemptsRemaining <= 0 || swapUsed <= maxSwapUsageAllowed) { + if ( + killAttemptsRemaining <= 0 || + swapUsed <= maxSwapUsageAllowed + ) { break; } } @@ -597,10 +749,20 @@ private void handleMemoryUsage(final DispatchHost dispatchHost, RenderHost rende // if frames didn't go overboard, manage its reservations trying to increase // them accordingly for (final RunningFrameInfo frame : runningFrames) { - if (OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD > 0 && isFrameOverboard(frame)) { - if (!killFrameOverusingMemory(frame, dispatchHost.getName())) { - logger.warn("Frame " + frame.getJobName() + "." + frame.getFrameName() + - " is overboard but could not be killed"); + if ( + OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD > 0 && + isFrameOverboard(frame) + ) { + if ( + !killFrameOverusingMemory(frame, dispatchHost.getName()) + ) { + logger.warn( + "Frame " + + frame.getJobName() + + "." + + frame.getFrameName() + + " is overboard but could not be killed" + ); } } else { handleMemoryReservations(frame); @@ -615,29 +777,45 @@ public enum KillCause { FrameTimedOut("Frame timed out"), FrameLluTimedOut("Frame LLU timed out"), FrameVerificationFailure("Frame failed to be verified on the database"); + private final String message; private KillCause(String message) { this.message = message; } + @Override public String toString() { return message; } } - private boolean killFrameOverusingMemory(RunningFrameInfo frame, String hostname) { + private boolean killFrameOverusingMemory( + RunningFrameInfo frame, + String hostname + ) { try { - VirtualProc proc = hostManager.getVirtualProc(frame.getResourceId()); + VirtualProc proc = hostManager.getVirtualProc( + frame.getResourceId() + ); // Don't mess with localDispatch procs if (proc.isLocalDispatch) { return false; } - boolean killed = killProcForMemory(proc.frameId, hostname, KillCause.FrameOverboard); + boolean killed = killProcForMemory( + proc.frameId, + hostname, + KillCause.FrameOverboard + ); if (killed) { - logger.info("Killing frame on " + frame.getJobName() + "." + frame.getFrameName() + - ", using too much memory."); + logger.info( + "Killing frame on " + + frame.getJobName() + + "." + + frame.getFrameName() + + ", using too much memory." + ); } return killed; } catch (EmptyResultDataAccessException e) { @@ -647,8 +825,10 @@ private boolean killFrameOverusingMemory(RunningFrameInfo frame, String hostname private boolean getKillClearance(String hostname, String frameId) { String cacheKey = hostname + "-" + frameId; - final int FRAME_KILL_RETRY_LIMIT = - env.getRequiredProperty("dispatcher.frame_kill_retry_limit", Integer.class); + final int FRAME_KILL_RETRY_LIMIT = env.getRequiredProperty( + "dispatcher.frame_kill_retry_limit", + Integer.class + ); // Cache frame+host receiving a killRequest and count how many times the request is being retried // meaning rqd is probably failing at attempting to kill the related proc @@ -667,13 +847,18 @@ private boolean getKillClearance(String hostname, String frameId) { FrameInterface frame = jobManager.getFrame(frameId); JobInterface job = jobManager.getJob(frame.getJobId()); prometheusMetrics.incrementFrameKillFailureCounter( - hostname, - job.getName(), - frame.getName(), - frameId); + hostname, + job.getName(), + frame.getName(), + frameId + ); } catch (EmptyResultDataAccessException e) { logger.info( - "Trying to kill a frame that no longer exists: host=" + hostname + " frameId=" + frameId); + "Trying to kill a frame that no longer exists: host=" + + hostname + + " frameId=" + + frameId + ); } } return false; @@ -681,7 +866,11 @@ private boolean getKillClearance(String hostname, String frameId) { return true; } - private boolean killProcForMemory(String frameId, String hostname, KillCause killCause) { + private boolean killProcForMemory( + String frameId, + String hostname, + KillCause killCause + ) { if (!getKillClearance(hostname, frameId)) { return false; } @@ -689,15 +878,35 @@ private boolean killProcForMemory(String frameId, String hostname, KillCause kil FrameInterface frame = jobManager.getFrame(frameId); if (dispatcher.isTestMode()) { // Different threads don't share the same database state on the test environment - (new DispatchRqdKillFrameMemory(hostname, frame, killCause.toString(), rqdClient, - dispatchSupport, dispatcher.isTestMode())).run(); + (new DispatchRqdKillFrameMemory( + hostname, + frame, + killCause.toString(), + rqdClient, + dispatchSupport, + dispatcher.isTestMode() + )).run(); } else { try { - killQueue.execute(new DispatchRqdKillFrameMemory(hostname, frame, killCause.toString(), rqdClient, - dispatchSupport, dispatcher.isTestMode())); - prometheusMetrics.incrementFrameKilledCounter(hostname, killCause); + killQueue.execute( + new DispatchRqdKillFrameMemory( + hostname, + frame, + killCause.toString(), + rqdClient, + dispatchSupport, + dispatcher.isTestMode() + ) + ); + prometheusMetrics.incrementFrameKilledCounter( + hostname, + killCause + ); } catch (TaskRejectedException e) { - logger.warn("Unable to add a DispatchRqdKillFrame request, task rejected, " + e); + logger.warn( + "Unable to add a DispatchRqdKillFrame request, task rejected, " + + e + ); return false; } } @@ -705,23 +914,42 @@ private boolean killProcForMemory(String frameId, String hostname, KillCause kil return true; } - private boolean killFrame(String frameId, String hostname, KillCause killCause) { + private boolean killFrame( + String frameId, + String hostname, + KillCause killCause + ) { if (!getKillClearance(hostname, frameId)) { return false; } if (dispatcher.isTestMode()) { // Different threads don't share the same database state on the test environment - (new DispatchRqdKillFrame(hostname, frameId, killCause.toString(), rqdClient)).run(); + (new DispatchRqdKillFrame( + hostname, + frameId, + killCause.toString(), + rqdClient + )).run(); } else { try { - killQueue.execute(new DispatchRqdKillFrame(hostname, + killQueue.execute( + new DispatchRqdKillFrame( + hostname, frameId, killCause.toString(), - rqdClient)); - prometheusMetrics.incrementFrameKilledCounter(hostname, killCause); + rqdClient + ) + ); + prometheusMetrics.incrementFrameKilledCounter( + hostname, + killCause + ); } catch (TaskRejectedException e) { - logger.warn("Unable to add a DispatchRqdKillFrame request, task rejected, " + e); + logger.warn( + "Unable to add a DispatchRqdKillFrame request, task rejected, " + + e + ); } } DispatchSupport.killedOffenderProcs.incrementAndGet(); @@ -735,35 +963,62 @@ private boolean killFrame(String frameId, String hostname, KillCause killCause) */ private boolean isFrameOverboard(final RunningFrameInfo frame) { final double OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD = - env.getRequiredProperty("dispatcher.oom_frame_overboard_allowed_threshold", Double.class); + env.getRequiredProperty( + "dispatcher.oom_frame_overboard_allowed_threshold", + Double.class + ); if (OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD < 0) { return false; } - double rss = (double)frame.getRss(); - double maxRss = (double)frame.getMaxRss(); - final double MAX_RSS_OVERBOARD_THRESHOLD = OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD * 2; + double rss = (double) frame.getRss(); + double maxRss = (double) frame.getMaxRss(); + final double MAX_RSS_OVERBOARD_THRESHOLD = + OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD * 2; final double RSS_AVAILABLE_FOR_MAX_RSS_TRIGGER = 0.1; try { - VirtualProc proc = hostManager.getVirtualProc(frame.getResourceId()); - double reserved = (double)proc.memoryReserved; + VirtualProc proc = hostManager.getVirtualProc( + frame.getResourceId() + ); + double reserved = (double) proc.memoryReserved; // Last memory report is higher than the threshold - if (isOverboard(rss, reserved, OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD)) { + if ( + isOverboard( + rss, + reserved, + OOM_FRAME_OVERBOARD_ALLOWED_THRESHOLD + ) + ) { return true; } // If rss is not overboard, handle the situation where the frame might be going overboard from // time to time but the last report wasn't during a spike. For this case, consider a combination // of rss and maxRss. maxRss > 2 * threshold and rss > 0.9 else { - return isOverboard(maxRss, reserved, MAX_RSS_OVERBOARD_THRESHOLD) && - isOverboard(rss, reserved, -RSS_AVAILABLE_FOR_MAX_RSS_TRIGGER); + return ( + isOverboard( + maxRss, + reserved, + MAX_RSS_OVERBOARD_THRESHOLD + ) && + isOverboard( + rss, + reserved, + -RSS_AVAILABLE_FOR_MAX_RSS_TRIGGER + ) + ); } } catch (EmptyResultDataAccessException e) { - logger.info("HostReportHandler(isFrameOverboard): Virtual proc for frame " + - frame.getFrameName() + " on job " + frame.getJobName() + " doesn't exist on the database"); + logger.info( + "HostReportHandler(isFrameOverboard): Virtual proc for frame " + + frame.getFrameName() + + " on job " + + frame.getJobName() + + " doesn't exist on the database" + ); // Not able to mark the frame overboard is it couldn't be found on the db. // Proc accounting (verifyRunningProc) should take care of it return false; @@ -771,7 +1026,7 @@ private boolean isFrameOverboard(final RunningFrameInfo frame) { } private boolean isOverboard(double value, double total, double threshold) { - return value/total >= (1 + threshold); + return value / total >= (1 + threshold); } /** @@ -790,33 +1045,65 @@ private void handleMemoryReservations(final RunningFrameInfo frame) { if (dispatchSupport.increaseReservedMemory(proc, frame.getRss())) { proc.memoryReserved = frame.getRss(); - logger.info("frame " + frame.getFrameName() + " on job " + frame.getJobName() - + " increased its reserved memory to " + - CueUtil.KbToMb(frame.getRss())); + logger.info( + "frame " + + frame.getFrameName() + + " on job " + + frame.getJobName() + + " increased its reserved memory to " + + CueUtil.KbToMb(frame.getRss()) + ); } } catch (ResourceReservationFailureException e) { if (proc != null) { long memNeeded = frame.getRss() - proc.memoryReserved; - logger.info("frame " + frame.getFrameName() + " on job " + frame.getJobName() - + "was unable to reserve an additional " + CueUtil.KbToMb(memNeeded) - + "on proc " + proc.getName() + ", " + e); + logger.info( + "frame " + + frame.getFrameName() + + " on job " + + frame.getJobName() + + "was unable to reserve an additional " + + CueUtil.KbToMb(memNeeded) + + "on proc " + + proc.getName() + + ", " + + e + ); try { - if (dispatchSupport.balanceReservedMemory(proc, memNeeded)) { + if ( + dispatchSupport.balanceReservedMemory(proc, memNeeded) + ) { proc.memoryReserved = frame.getRss(); - logger.info("was able to balance host: " + proc.getName()); + logger.info( + "was able to balance host: " + proc.getName() + ); } else { - logger.info("failed to balance host: " + proc.getName()); + logger.info( + "failed to balance host: " + proc.getName() + ); } } catch (Exception ex) { - logger.warn("failed to balance host: " + proc.getName() + ", " + e); + logger.warn( + "failed to balance host: " + proc.getName() + ", " + e + ); } } else { - logger.info("frame " + frame.getFrameName() + " on job " + frame.getJobName() - + "was unable to reserve an additional memory. Proc could not be found"); + logger.info( + "frame " + + frame.getFrameName() + + " on job " + + frame.getJobName() + + "was unable to reserve an additional memory. Proc could not be found" + ); } } catch (EmptyResultDataAccessException e) { - logger.info("HostReportHandler: Memory reservations for frame " + frame.getFrameName() + - " on job " + frame.getJobName() + " proc could not be found"); + logger.info( + "HostReportHandler: Memory reservations for frame " + + frame.getFrameName() + + " on job " + + frame.getJobName() + + " proc could not be found" + ); } } @@ -825,22 +1112,39 @@ private void handleMemoryReservations(final RunningFrameInfo frame) { * * @param rFrames */ - private void killTimedOutFrames(List runningFrames, String hostname) { + private void killTimedOutFrames( + List runningFrames, + String hostname + ) { for (RunningFrameInfo frame : runningFrames) { String layerId = frame.getLayerId(); try { LayerDetail layer = layerDao.getLayerDetail(layerId); - long runtimeMinutes = ((System.currentTimeMillis() - frame.getStartTime()) / 1000l) / 60; + long runtimeMinutes = + ((System.currentTimeMillis() - frame.getStartTime()) / + 1000l) / + 60; if (layer.timeout != 0 && runtimeMinutes > layer.timeout) { - killFrame(frame.getFrameId(), hostname, KillCause.FrameTimedOut); + killFrame( + frame.getFrameId(), + hostname, + KillCause.FrameTimedOut + ); } else if (layer.timeout_llu != 0 && frame.getLluTime() != 0) { long r = System.currentTimeMillis() / 1000; long lastUpdate = (r - frame.getLluTime()) / 60; - if (layer.timeout_llu != 0 && lastUpdate > (layer.timeout_llu - 1)) { - killFrame(frame.getFrameId(), hostname, KillCause.FrameLluTimedOut); + if ( + layer.timeout_llu != 0 && + lastUpdate > (layer.timeout_llu - 1) + ) { + killFrame( + frame.getFrameId(), + hostname, + KillCause.FrameLluTimedOut + ); } } } catch (EmptyResultDataAccessException e) { @@ -855,18 +1159,27 @@ private void killTimedOutFrames(List runningFrames, String hos * @param rFrames */ private void updateMemoryUsageAndLluTime(List rFrames) { - - for (RunningFrameInfo rf: rFrames) { + for (RunningFrameInfo rf : rFrames) { FrameInterface frame = jobManager.getFrame(rf.getFrameId()); - dispatchSupport.updateFrameMemoryUsageAndLluTime(frame, - rf.getRss(), rf.getMaxRss(), rf.getLluTime()); - - dispatchSupport.updateProcMemoryUsage(frame, rf.getRss(), rf.getMaxRss(), - rf.getVsize(), rf.getMaxVsize(), rf.getUsedGpuMemory(), - rf.getMaxUsedGpuMemory(), rf.getUsedSwapMemory(), - rf.getChildren().toByteArray()); - + dispatchSupport.updateFrameMemoryUsageAndLluTime( + frame, + rf.getRss(), + rf.getMaxRss(), + rf.getLluTime() + ); + + dispatchSupport.updateProcMemoryUsage( + frame, + rf.getRss(), + rf.getMaxRss(), + rf.getVsize(), + rf.getMaxVsize(), + rf.getUsedGpuMemory(), + rf.getMaxUsedGpuMemory(), + rf.getUsedSwapMemory(), + rf.getChildren().toByteArray() + ); } updateJobMemoryUsage(rFrames); @@ -879,23 +1192,22 @@ private void updateMemoryUsageAndLluTime(List rFrames) { * @param frames */ private void updateJobMemoryUsage(List frames) { + final Map jobs = new HashMap( + frames.size() + ); - final Map jobs = - new HashMap(frames.size()); - - for (RunningFrameInfo frame: frames) { + for (RunningFrameInfo frame : frames) { JobEntity job = new JobEntity(frame.getJobId()); if (jobs.containsKey(job)) { if (jobs.get(job) < frame.getMaxRss()) { jobs.put(job, frame.getMaxRss()); } - } - else { + } else { jobs.put(job, frame.getMaxRss()); } } - for (Map.Entry set: jobs.entrySet()) { + for (Map.Entry set : jobs.entrySet()) { jobDao.updateMaxRSS(set.getKey(), set.getValue()); } } @@ -906,24 +1218,23 @@ private void updateJobMemoryUsage(List frames) { * @param frames */ private void updateLayerMemoryUsage(List frames) { + final Map layers = new HashMap( + frames.size() + ); - final Map layers = - new HashMap(frames.size()); - - for (RunningFrameInfo frame: frames) { + for (RunningFrameInfo frame : frames) { LayerEntity layer = new LayerEntity(frame.getLayerId()); if (layers.containsKey(layer)) { if (layers.get(layer) < frame.getMaxRss()) { layers.put(layer, frame.getMaxRss()); } - } - else { + } else { layers.put(layer, frame.getMaxRss()); } } /* Attempt to update the max RSS value for the job **/ - for (Map.Entry set: layers.entrySet()) { + for (Map.Entry set : layers.entrySet()) { layerDao.increaseLayerMinMemory(set.getKey(), set.getValue()); layerDao.updateLayerMaxRSS(set.getKey(), set.getValue(), false); } @@ -951,24 +1262,38 @@ private void updateLayerMemoryUsage(List frames) { * @param report */ public List verifyRunningFrameInfo(HostReport report) { - List runningFrames = new ArrayList(report.getFramesCount()); - - for (RunningFrameInfo runningFrame: report.getFramesList()) { + List runningFrames = new ArrayList( + report.getFramesCount() + ); - long runtimeSeconds = (System.currentTimeMillis() - - runningFrame.getStartTime()) / 1000l; + for (RunningFrameInfo runningFrame : report.getFramesList()) { + long runtimeSeconds = + (System.currentTimeMillis() - runningFrame.getStartTime()) / + 1000l; // Don't test frames that haven't been running long enough. if (runtimeSeconds < FRAME_VERIFICATION_GRACE_PERIOD_SECONDS) { - logger.info("verified " + runningFrame.getJobName() + - "/" + runningFrame.getFrameName() + " on " + - report.getHost().getName() + " by grace period " + - runtimeSeconds + " seconds."); + logger.info( + "verified " + + runningFrame.getJobName() + + "/" + + runningFrame.getFrameName() + + " on " + + report.getHost().getName() + + " by grace period " + + runtimeSeconds + + " seconds." + ); runningFrames.add(runningFrame); continue; } - if (hostManager.verifyRunningProc(runningFrame.getResourceId(), runningFrame.getFrameId())) { + if ( + hostManager.verifyRunningProc( + runningFrame.getResourceId(), + runningFrame.getFrameId() + ) + ) { runningFrames.add(runningFrame); continue; } @@ -984,11 +1309,14 @@ public List verifyRunningFrameInfo(HostReport report) { try { proc = hostManager.getVirtualProc(runningFrame.getResourceId()); - msg = "Virtual proc " + proc.getProcId() + - "is assigned to " + proc.getFrameId() + - " not " + runningFrame.getFrameId(); - } - catch (Exception e) { + msg = + "Virtual proc " + + proc.getProcId() + + "is assigned to " + + proc.getFrameId() + + " not " + + runningFrame.getFrameId(); + } catch (Exception e) { /* * This will happen if the host goes offline and then * comes back. In this case, we don't touch the frame @@ -1007,27 +1335,53 @@ public List verifyRunningFrameInfo(HostReport report) { if (proc == null) { // A frameCompleteReport might have been delivered before this report was // processed - FrameDetail frameLatestVersion = jobManager.getFrameDetail(runningFrame.getFrameId()); + FrameDetail frameLatestVersion = jobManager.getFrameDetail( + runningFrame.getFrameId() + ); if (frameLatestVersion.state != FrameState.RUNNING) { - logger.info("DelayedVerification, the proc " + - runningFrame.getResourceId() + " on host " + - report.getHost().getName() + " has already Completed " + - runningFrame.getJobName() + "/" + runningFrame.getFrameName()); - } else if (killFrame(runningFrame.getFrameId(), + logger.info( + "DelayedVerification, the proc " + + runningFrame.getResourceId() + + " on host " + + report.getHost().getName() + + " has already Completed " + + runningFrame.getJobName() + + "/" + + runningFrame.getFrameName() + ); + } else if ( + killFrame( + runningFrame.getFrameId(), report.getHost().getName(), - KillCause.FrameVerificationFailure)) { - logger.info("FrameVerificationError, the proc " + - runningFrame.getResourceId() + " on host " + - report.getHost().getName() + " was running for " + - (runtimeSeconds / 60.0f) + " minutes " + - runningFrame.getJobName() + "/" + runningFrame.getFrameName() + - " but the DB did not " + - "reflect this. " + - msg); + KillCause.FrameVerificationFailure + ) + ) { + logger.info( + "FrameVerificationError, the proc " + + runningFrame.getResourceId() + + " on host " + + report.getHost().getName() + + " was running for " + + (runtimeSeconds / 60.0f) + + " minutes " + + runningFrame.getJobName() + + "/" + + runningFrame.getFrameName() + + " but the DB did not " + + "reflect this. " + + msg + ); } else { - logger.warn("FrameStuckWarning: frameId=" + runningFrame.getFrameId() + - " render_node=" + report.getHost().getName() + " - " + - runningFrame.getJobName() + "/" + runningFrame.getFrameName()); + logger.warn( + "FrameStuckWarning: frameId=" + + runningFrame.getFrameId() + + " render_node=" + + report.getHost().getName() + + " - " + + runningFrame.getJobName() + + "/" + + runningFrame.getFrameName() + ); } } } @@ -1121,6 +1475,7 @@ public Dispatcher getLocalDispatcher() { public void setLocalDispatcher(Dispatcher localDispatcher) { this.localDispatcher = localDispatcher; } + public ThreadPoolExecutor getKillQueue() { return killQueue; } @@ -1129,4 +1484,3 @@ public void setKillQueue(ThreadPoolExecutor killQueue) { this.killQueue = killQueue; } } - diff --git a/cuegui/cuegui/DependWizard.py b/cuegui/cuegui/DependWizard.py index 25de89944..76b38c2d8 100644 --- a/cuegui/cuegui/DependWizard.py +++ b/cuegui/cuegui/DependWizard.py @@ -383,12 +383,14 @@ def __msg(self): if item[1]: return "this %s" % item[0] + # pylint: disable=missing-function-docstring def initializePage(self): self.setSubTitle("What type of dependency would you like %s to have?" % self.__msg()) # it is not respecting or providing my size hints otherwise self.wizard().setMinimumSize(500, 500) + # pylint: disable=missing-function-docstring def validatePage(self): # pylint: disable=consider-using-dict-items for option in self.__options: @@ -436,6 +438,7 @@ def __init__(self, parent): self.__layerList = self._addListWidget(2, 0, QtWidgets.QAbstractItemView.MultiSelection) + # pylint: disable=missing-function-docstring def initializePage(self): self.wizard().layerOptions = self.wizard().jobs[0].getLayers() @@ -448,6 +451,7 @@ def initializePage(self): self.__layerList.item(num).setSelected( str(self.__layerList.item(num).text()) in self._getNames(self.wizard().onLayer)) + # pylint: disable=missing-function-docstring def validatePage(self): self.wizard().layers = [] for num in range(self.__layerList.count()): @@ -486,9 +490,11 @@ def __init__(self, parent): self.__frame = self._addLineEdit(1, 9, "1") self.registerField("frame", self.__frame) + # pylint: disable=missing-function-docstring def initializePage(self): QtWidgets.QWizardPage.initializePage(self) + # pylint: disable=missing-function-docstring def validatePage(self): frames = str(self.field("frame")) if frames: @@ -541,6 +547,7 @@ def filterJobs(self, text): [job for job in self.wizard().onJobOptions if re.search(str(text), job, re.IGNORECASE) and job not in exclude]) + # pylint: disable=missing-function-docstring def initializePage(self): # If the filter edit box is empty, populate it with SHOW-SHOT-USER_ # based on the first job selected to receive the dependency @@ -558,6 +565,7 @@ def initializePage(self): QtWidgets.QWizardPage.initializePage(self) + # pylint: disable=missing-function-docstring def validatePage(self): self.wizard().onJob = [] for num in range(self.__jobList.count()): @@ -596,6 +604,7 @@ def __init__(self, parent): self._addLabel("Depend on Layer:", 0, 0) self.__onLayerList = self._addListWidget(1, 0) + # pylint: disable=missing-function-docstring def initializePage(self): QtWidgets.QWizardPage.initializePage(self) @@ -620,6 +629,7 @@ def initializePage(self): self.__onLayerList.item(num).setSelected( str(self.__onLayerList.item(num).text()) in self._getNames(self.wizard().onLayer)) + # pylint: disable=missing-function-docstring def validatePage(self): self.wizard().onLayer = [] for num in range(self.__onLayerList.count()): @@ -660,9 +670,11 @@ def __init__(self, parent): self.setField("onFrame", "") + # pylint: disable=missing-function-docstring def initializePage(self): QtWidgets.QWizardPage.initializePage(self) + # pylint: disable=missing-function-docstring def validatePage(self): frames = str(self.field("onFrame")) if frames: @@ -700,6 +712,7 @@ def __init__(self, parent, jobs, layers, frames): self.setTitle("Confirmation") self.setSubTitle("Are you sure?") + # pylint: disable=missing-function-docstring def initializePage(self): self._removeAllWidgets() @@ -724,6 +737,7 @@ def initializePage(self): self._displayItems("Frame", self.wizard().onFrame, 9) # pylint: disable=too-many-nested-blocks + # pylint: disable=missing-function-docstring def validatePage(self): # Just names: jobs = self._getNames(self.wizard().jobs) diff --git a/cuegui/cuegui/ShowsWidget.py b/cuegui/cuegui/ShowsWidget.py index 8d24b7ae7..db6fcda85 100644 --- a/cuegui/cuegui/ShowsWidget.py +++ b/cuegui/cuegui/ShowsWidget.py @@ -105,7 +105,7 @@ def contextMenuEvent(self, e): menu.addSeparator() self.__menuActions.shows().addAction(menu, "createSubscription") - menu.exec_(QtCore.QPoint(e.globalX(), e.globalY())) + menu.exec_(QtCore.QPoint(e.globalX(), e.globalY())) # pylint: disable=no-member def tick(self): pass diff --git a/cuegui/cuegui/SubscriptionGraphWidget.py b/cuegui/cuegui/SubscriptionGraphWidget.py index b17004d11..498e93a68 100644 --- a/cuegui/cuegui/SubscriptionGraphWidget.py +++ b/cuegui/cuegui/SubscriptionGraphWidget.py @@ -42,7 +42,7 @@ def __init__(self, parent): self.__shows = [] self.__showMenuActions = {} self.__subBars = [] - self.__timer = QtCore.QTimer(self) + self.__timer = QtCore.QTimer(self) # pylint: disable=no-member self.__timer.timeout.connect(self.update_data) # pylint: disable=no-member self.__timer.setInterval(1000 * 5) @@ -230,7 +230,7 @@ def contextMenuEvent(self, e): new_action = QtWidgets.QAction('Add new subscription', self) new_action.triggered.connect(self.createSubscription) # pylint: disable=no-member menu.addAction(new_action) - menu.exec_(QtCore.QPoint(e.globalX(),e.globalY())) + menu.exec_(QtCore.QPoint(e.globalX(),e.globalY())) # pylint: disable=no-member def createSubscription(self): """Shows a dialog for creating a new subscription.""" diff --git a/cuegui/cuegui/ThreadPool.py b/cuegui/cuegui/ThreadPool.py index 8a615ee6c..e1ffa8645 100644 --- a/cuegui/cuegui/ThreadPool.py +++ b/cuegui/cuegui/ThreadPool.py @@ -68,6 +68,7 @@ def systemCpuCount(): return 1 +# pylint: disable=no-member class ThreadPool(QtCore.QObject): """A general purpose work queue class.""" From 892cd4da7470acf59b85aa8395d95e386830a36a Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 27 Nov 2024 11:10:58 -0800 Subject: [PATCH 3/9] Fix lint errors --- cuegui/cuegui/AbstractDockWidget.py | 3 +++ cuegui/cuegui/CreatorDialog.py | 2 ++ cuegui/cuegui/CueJobMonitorTree.py | 8 +++++++- cuegui/cuegui/CueStateBarWidget.py | 6 +++--- cuegui/cuegui/FrameRangeSelection.py | 9 +++++++++ cuegui/cuegui/HostMonitor.py | 9 +++++++-- cuegui/cuegui/ItemDelegate.py | 4 +++- cuegui/cuegui/JobMonitorTree.py | 4 ++++ cuegui/cuegui/LayerDialog.py | 1 + cuegui/cuegui/Layout.py | 2 +- cuegui/cuegui/LimitsWidget.py | 2 +- cuegui/cuegui/PreviewWidget.py | 1 + cuegui/cuegui/ServiceDialog.py | 6 +++--- cuegui/cuegui/SubscriptionsWidget.py | 2 +- cuegui/cuegui/ThreadPool.py | 1 + cuegui/cuegui/UnbookDialog.py | 1 + 16 files changed, 48 insertions(+), 13 deletions(-) diff --git a/cuegui/cuegui/AbstractDockWidget.py b/cuegui/cuegui/AbstractDockWidget.py index 0697662e3..c5059b2c3 100644 --- a/cuegui/cuegui/AbstractDockWidget.py +++ b/cuegui/cuegui/AbstractDockWidget.py @@ -57,12 +57,15 @@ def __init__(self, parent, name, area = QtCore.Qt.LeftDockWidgetArea): self.widget().setLayout(self.__layout) def closeEvent(self, event): + """Delete event and emit close signal""" del event self.closed.emit(self) def showEvent(self, event): + """Delete event and emit enabled signal""" del event self.enabled.emit() def layout(self): + """Returns the layout""" return self.__layout diff --git a/cuegui/cuegui/CreatorDialog.py b/cuegui/cuegui/CreatorDialog.py index 5d2e5e450..2e86c6b4a 100644 --- a/cuegui/cuegui/CreatorDialog.py +++ b/cuegui/cuegui/CreatorDialog.py @@ -63,6 +63,7 @@ def __init__(self, show=None, parent=None): layout.addRow('Burst:', self.burstBox) def create(self): + """Create Subscription""" try: showMap = dict(list(zip([s.data.name for s in self.__shows], self.__shows))) allocMap = dict(list(zip([a.data.name for a in self.__allocs], self.__allocs))) @@ -103,5 +104,6 @@ def __init__(self, show=None, parent=None): # pylint: enable=no-member def create(self): + """Create subscription""" self.__creator.create() self.close() diff --git a/cuegui/cuegui/CueJobMonitorTree.py b/cuegui/cuegui/CueJobMonitorTree.py index 7b47b1902..f7e41dfd6 100644 --- a/cuegui/cuegui/CueJobMonitorTree.py +++ b/cuegui/cuegui/CueJobMonitorTree.py @@ -264,9 +264,11 @@ def startDrag(self, dropActions): cuegui.Utils.startDrag(self, dropActions, self.selectedObjects()) def dragEnterEvent(self, event): + """Called after a drag event begins""" cuegui.Utils.dragEnterEvent(event) def dragMoveEvent(self, event): + """Called on a drag move event""" cuegui.Utils.dragMoveEvent(event) # Causes the list to scroll when dragging is over the top or bottom 20% @@ -288,6 +290,7 @@ def dragMoveEvent(self, event): self.verticalScrollBar().setValue(self.verticalScrollBar().value() + move) def dropEvent(self, event): + """Drop event action""" item = self.itemAt(event.pos()) if item and item.type() in (cuegui.Constants.TYPE_ROOTGROUP, cuegui.Constants.TYPE_GROUP): @@ -307,7 +310,7 @@ def dropEvent(self, event): % item.rpcObject.data.name, event_item=item, items=body_content, - dist_groups={}, + dst_groups={}, parent=self) dialog.exec_() @@ -482,6 +485,7 @@ def __processUpdateHandleNested(self, parent, groups): "Failed to create tree item. RootView might be closed", exc_info=True) def mouseDoubleClickEvent(self, event): + """Event triggered by a mouse click""" del event objects = self.selectedObjects() if objects: @@ -861,12 +865,14 @@ def __init__(self, title, text, event_item, items, dst_groups, _hlayout.addWidget(_btn_cancel) _vlayout.addLayout(_hlayout) + # pylint: disable=no-member self.connect(_btn_accept, QtCore.SIGNAL("clicked()"), self.move_items) self.connect(_btn_cancel, QtCore.SIGNAL("clicked()"), self.reject) + # pylint: enable=no-member def move_items(self): """Reparent jobs to new group""" diff --git a/cuegui/cuegui/CueStateBarWidget.py b/cuegui/cuegui/CueStateBarWidget.py index 90fd7d652..8a61bb3bf 100644 --- a/cuegui/cuegui/CueStateBarWidget.py +++ b/cuegui/cuegui/CueStateBarWidget.py @@ -57,8 +57,8 @@ def __init__(self, sourceTree, parent=None): self.__sourceTree = weakref.proxy(sourceTree) self.__colors = [] self.__baseColor = self.app.palette().color(QtGui.QPalette.Base) - self.__colorsLock = QtCore.QReadWriteLock() - self.__timer = QtCore.QTimer(self) + self.__colorsLock = QtCore.QReadWriteLock() # pylint: disable=no-member + self.__timer = QtCore.QTimer(self) # pylint: disable=no-member self.__lastUpdate = 0 self.__timer.timeout.connect(self.updateColors) # pylint: disable=no-member @@ -173,7 +173,7 @@ def updateColors(self): def __updateColors(self): """Calls __processUpdateColors in 1 second""" self.__lastUpdate = time.time() - QtCore.QTimer.singleShot(1000, self.__processUpdateColors) + QtCore.QTimer.singleShot(1000, self.__processUpdateColors) # pylint: disable=no-member def __processUpdateColors(self): """Updates the list of colors to display diff --git a/cuegui/cuegui/FrameRangeSelection.py b/cuegui/cuegui/FrameRangeSelection.py index 003c4bc22..b074b50d0 100644 --- a/cuegui/cuegui/FrameRangeSelection.py +++ b/cuegui/cuegui/FrameRangeSelection.py @@ -157,6 +157,7 @@ def setSelectedFrameRange(self, selectedRange): # QT event handlers and implementation details below this line. def mousePressEvent(self, mouseEvent): + """Event triggered by a mouse click""" hitTime = self.__getTimeFromLocalPoint(mouseEvent.x()) if mouseEvent.buttons() & QtCore.Qt.LeftButton: @@ -164,6 +165,7 @@ def mousePressEvent(self, mouseEvent): self.__selectionRange = (hitTime, hitTime) def mouseMoveEvent(self, mouseEvent): + """Event triggered by a mouse movement""" self.__floatTime = None hitTime = self.__getTimeFromLocalPoint(mouseEvent.x()) @@ -181,6 +183,7 @@ def mouseMoveEvent(self, mouseEvent): self.update() def mouseReleaseEvent(self, mouseEvent): + """Event triggered when a mouse click is released""" if self.__double: self.__double = False self.setStartTime(self.__frameRange[0], True) @@ -198,10 +201,12 @@ def mouseReleaseEvent(self, mouseEvent): self.update() def mouseDoubleClickEvent(self, mouseEvent): + """Event triggered by a double click""" del mouseEvent self.__double = True def paintEvent(self, paintEvent): + """Paint event""" del paintEvent painter = QtGui.QPainter(self) self.__paintBackground(painter) @@ -217,15 +222,18 @@ def paintEvent(self, paintEvent): self.__paintEndTime(painter) def leaveEvent(self, event): + """Triggered at the end of any event""" del event self.__floatTime = None self.update() def __getTickAreaExtent(self): + """Return a QRect with the selectable area""" return QtCore.QRect( 10, -self.height() // 2, self.width() - self.__right_margin - 20, self.height()) def __getTickArea(self, time): + """Get a QRect with a tick area""" tickArea = self.__getTickAreaExtent() tickSpacing = ( float(self.__getTickAreaExtent().width()) / @@ -234,6 +242,7 @@ def __getTickArea(self, time): tickArea.top(), tickSpacing, tickArea.height()) def __getTimeFromLocalPoint(self, x): + """Get time from a local point""" tickSpacing = ( float(self.__getTickAreaExtent().width()) / max(1, (self.__frameRange[1] - self.__frameRange[0]))) diff --git a/cuegui/cuegui/HostMonitor.py b/cuegui/cuegui/HostMonitor.py index c5b84491a..334e612eb 100644 --- a/cuegui/cuegui/HostMonitor.py +++ b/cuegui/cuegui/HostMonitor.py @@ -113,6 +113,7 @@ def __filterByHostNameSetup(self, layout): btn.setFocusPolicy(QtCore.Qt.NoFocus) btn.setFixedWidth(24) layout.addWidget(btn) + # pylint: disable=no-member QtCore.QObject.connect(btn, QtCore.SIGNAL('clicked()'), self.__filterByHostNameClear) @@ -149,6 +150,7 @@ def __filterAllocationSetup(self, layout): menu = QtWidgets.QMenu(self) btn.setMenu(menu) + # pylint: disable=no-member QtCore.QObject.connect(menu, QtCore.SIGNAL("triggered(QAction*)"), self.__filterAllocationHandle) @@ -210,12 +212,13 @@ def __filterHardwareStateSetup(self, layout): btn.setContentsMargins(0, 0, 0, 0) btn.setFlat(True) + # pylint: disable=no-member menu = QtWidgets.QMenu(self) btn.setMenu(menu) QtCore.QObject.connect(menu, QtCore.SIGNAL("triggered(QAction*)"), self.__filterHardwareStateHandle) - + # pylint: enable=no-member for item in ["Clear", None] + self.__filterHardwareStateList: if item: a = QtWidgets.QAction(menu) @@ -266,6 +269,7 @@ def __filterHardwareStateHandle(self, action): # Checkbox to toggle auto-refresh # ============================================================================== def __refreshToggleCheckBoxSetup(self, layout): + # pylint: disable=no-member checkBox = QtWidgets.QCheckBox("Auto-refresh", self) layout.addWidget(checkBox) if self.hostMonitorTree.enableRefresh: @@ -274,6 +278,7 @@ def __refreshToggleCheckBoxSetup(self, layout): QtCore.SIGNAL('stateChanged(int)'), self.__refreshToggleCheckBoxHandle) __refreshToggleCheckBoxCheckBox = checkBox + # pylint: enable=no-member def __refreshToggleCheckBoxHandle(self, state): self.hostMonitorTree.enableRefresh = bool(state) @@ -300,7 +305,7 @@ def __refreshButtonEnableHandle(self): def __refreshButtonDisableHandle(self): """Called when the refresh button should be disabled""" self.btn_refresh.setEnabled(False) - QtCore.QTimer.singleShot(5000, self.__refreshButtonEnableHandle) + QtCore.QTimer.singleShot(5000, self.__refreshButtonEnableHandle) # pylint: disable=no-member # ============================================================================== # Button to clear all filters diff --git a/cuegui/cuegui/ItemDelegate.py b/cuegui/cuegui/ItemDelegate.py index a91f16b4f..48c828e66 100644 --- a/cuegui/cuegui/ItemDelegate.py +++ b/cuegui/cuegui/ItemDelegate.py @@ -68,6 +68,7 @@ def __init__(self, parent, *args): QtWidgets.QItemDelegate.__init__(self, parent, *args) def paint(self, painter, option, index): + """Paint QItem""" if option.state & QtWidgets.QStyle.State_Selected: # If selected cell self._paintSelected(painter, option, index) @@ -476,6 +477,7 @@ def paint(self, painter, option, index): AbstractDelegate.paint(self, painter, option, index) def sizeHint(self, option, index): + """Delete both option and index and return a new qsize""" del option del index - return QtCore.QSize(12, 12) + return QtCore.QSize(12, 12) # pylint: disable=no-member diff --git a/cuegui/cuegui/JobMonitorTree.py b/cuegui/cuegui/JobMonitorTree.py index 7a2946794..d520c6d52 100644 --- a/cuegui/cuegui/JobMonitorTree.py +++ b/cuegui/cuegui/JobMonitorTree.py @@ -249,15 +249,19 @@ def __itemSingleClickedComment(self, item, col): self.__menuActions.jobs().viewComments([job]) def startDrag(self, dropActions): + """Triggers a drag event""" cuegui.Utils.startDrag(self, dropActions, self.selectedObjects()) def dragEnterEvent(self, event): + """Enter Drag event""" cuegui.Utils.dragEnterEvent(event) def dragMoveEvent(self, event): + """Move Drag Event""" cuegui.Utils.dragMoveEvent(event) def dropEvent(self, event): + """Drop Drag Event""" for job_name in cuegui.Utils.dropEvent(event): self.addJob(job_name) diff --git a/cuegui/cuegui/LayerDialog.py b/cuegui/cuegui/LayerDialog.py index aed2f740c..8686ce17e 100644 --- a/cuegui/cuegui/LayerDialog.py +++ b/cuegui/cuegui/LayerDialog.py @@ -534,5 +534,6 @@ def __init__(self, layers, parent=None): # pylint: enable=no-member def accept(self): + """Accept action""" self._tags_widget.apply() self.close() diff --git a/cuegui/cuegui/Layout.py b/cuegui/cuegui/Layout.py index 9b26d23c8..b12de3c5f 100644 --- a/cuegui/cuegui/Layout.py +++ b/cuegui/cuegui/Layout.py @@ -41,7 +41,7 @@ def startup(app_name): """ # E.g. ~/.config/.cuecommander/config.ini config_path = "/.%s/config" % app_name.lower() - settings = QtCore.QSettings(QtCore.QSettings.IniFormat, QtCore.QSettings.UserScope, config_path) + settings = QtCore.QSettings(QtCore.QSettings.IniFormat, QtCore.QSettings.UserScope, config_path) # pylint: disable=no-member logger.info('Reading config file from %s', settings.fileName()) local = settings.fileName() diff --git a/cuegui/cuegui/LimitsWidget.py b/cuegui/cuegui/LimitsWidget.py index 82f236342..833f06a58 100644 --- a/cuegui/cuegui/LimitsWidget.py +++ b/cuegui/cuegui/LimitsWidget.py @@ -148,7 +148,7 @@ def contextMenuEvent(self, e): menu.addSeparator() self.__menuActions.limits().addAction(menu, "delete") self.__menuActions.limits().addAction(menu, "rename") - menu.exec_(QtCore.QPoint(e.globalX(), e.globalY())) + menu.exec_(QtCore.QPoint(e.globalX(), e.globalY())) # pylint: disable=no-member def tick(self): pass diff --git a/cuegui/cuegui/PreviewWidget.py b/cuegui/cuegui/PreviewWidget.py index 6e109bd01..a49b17de3 100644 --- a/cuegui/cuegui/PreviewWidget.py +++ b/cuegui/cuegui/PreviewWidget.py @@ -167,6 +167,7 @@ def __findHttpPort(self): raise Exception("This frame doesn't support previews. No Preview Server found.") +# pylint: disable=no-member class PreviewProcessorWatchThread(QtCore.QThread): """ Waits for preview files to appear and emits the progress every second. This diff --git a/cuegui/cuegui/ServiceDialog.py b/cuegui/cuegui/ServiceDialog.py index 5ca292262..20d959f70 100644 --- a/cuegui/cuegui/ServiceDialog.py +++ b/cuegui/cuegui/ServiceDialog.py @@ -232,10 +232,10 @@ def __init__(self, show, parent=None): self.__btn_del.clicked.connect(self.delService) self.__form.saved.connect(self.saved) self.__service_list.currentItemChanged.connect(self.selected) - # pylint: enable=no-member self.refresh() self.__service_list.setCurrentRow(0, QtCore.QItemSelectionModel.Select) + # pylint: enable=no-member def selected(self, item, old_item): """ @@ -287,7 +287,7 @@ def saved(self, service): item = self.__service_list.item(i) if item: if str(item.text()) == service.name(): - self.__service_list.setCurrentRow(i, QtCore.QItemSelectionModel.Select) + self.__service_list.setCurrentRow(i, QtCore.QItemSelectionModel.Select) # pylint: disable=no-member break def refresh(self): @@ -331,7 +331,7 @@ def delService(self): self.__selected.delete() row = self.currentRow() if row >= 1: - self.__service_list.setCurrentRow(row - 1, QtCore.QItemSelectionModel.Select) + self.__service_list.setCurrentRow(row - 1, QtCore.QItemSelectionModel.Select) # pylint: disable=no-member self.refresh() def currentRow(self): diff --git a/cuegui/cuegui/SubscriptionsWidget.py b/cuegui/cuegui/SubscriptionsWidget.py index eb474e074..58c2158e0 100644 --- a/cuegui/cuegui/SubscriptionsWidget.py +++ b/cuegui/cuegui/SubscriptionsWidget.py @@ -243,7 +243,7 @@ def contextMenuEvent(self, e): self.__menuActions.subscriptions().addAction(menu, "editBurst") menu.addSeparator() self.__menuActions.subscriptions().addAction(menu, "delete") - menu.exec_(QtCore.QPoint(e.globalX(),e.globalY())) + menu.exec_(QtCore.QPoint(e.globalX(),e.globalY())) # pylint: disable=no-member def tick(self): pass diff --git a/cuegui/cuegui/ThreadPool.py b/cuegui/cuegui/ThreadPool.py index e1ffa8645..c0352c1f9 100644 --- a/cuegui/cuegui/ThreadPool.py +++ b/cuegui/cuegui/ThreadPool.py @@ -141,6 +141,7 @@ def __init__(self, name, parent): self.__running = False # pylint: disable=protected-access + # pylint: disable=missing-function-docstring def run(self): self.__running = True while self.__running: diff --git a/cuegui/cuegui/UnbookDialog.py b/cuegui/cuegui/UnbookDialog.py index 334c43909..618427200 100644 --- a/cuegui/cuegui/UnbookDialog.py +++ b/cuegui/cuegui/UnbookDialog.py @@ -195,6 +195,7 @@ def _convert(val): return [result] def accept(self): + """Accept Unbook action""" if not self.__jobs: self.close() From f4a504cadf44ab6f9935e68b987ce2111e46d5f1 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 27 Nov 2024 11:19:35 -0800 Subject: [PATCH 4/9] Fix lint errors --- ci/run_python_lint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index ba98d2b6c..f93527d1e 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -37,7 +37,7 @@ cd .. echo "Running lint for cuegui/..." cd cuegui -PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cuegui --ignore=cuegui/images,cuegui/images/crystal +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cuegui --ignore=cuegui/images,cuegui/images/crystal --disable=no-member PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests cd .. From 31222dcac00efad6e6b5534ca704f79f49d0d9d8 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 27 Nov 2024 12:32:37 -0800 Subject: [PATCH 5/9] Fix lint errors --- cuegui/cuegui/AbstractTreeWidget.py | 1 + cuegui/cuegui/FilterDialog.py | 3 +++ cuegui/cuegui/GroupDialog.py | 2 +- cuegui/cuegui/HostMonitorTree.py | 2 ++ cuegui/cuegui/MainWindow.py | 1 + cuegui/cuegui/Redirect.py | 2 +- cuegui/cuegui/plugins/AllocationsPlugin.py | 4 ++++ cuegui/cuegui/plugins/LogViewPlugin.py | 2 ++ cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py | 3 +++ cuegui/cuegui/plugins/MonitorJobsPlugin.py | 1 + cuegui/cuegui/plugins/StuckFramePlugin.py | 5 +++++ 11 files changed, 24 insertions(+), 2 deletions(-) diff --git a/cuegui/cuegui/AbstractTreeWidget.py b/cuegui/cuegui/AbstractTreeWidget.py index eb748fa8a..6260ec669 100644 --- a/cuegui/cuegui/AbstractTreeWidget.py +++ b/cuegui/cuegui/AbstractTreeWidget.py @@ -117,6 +117,7 @@ def __init__(self, parent): self.setUpdateInterval(10) def closeEvent(self, event): + """Close Event""" if hasattr(self, '_timer'): self._timer.stop() del self._timer diff --git a/cuegui/cuegui/FilterDialog.py b/cuegui/cuegui/FilterDialog.py index ab4d4d25b..9fb5018f5 100644 --- a/cuegui/cuegui/FilterDialog.py +++ b/cuegui/cuegui/FilterDialog.py @@ -856,6 +856,7 @@ def __init__(self, parent): QtWidgets.QComboBox.__init__(self, parent) def wheelEvent(self, event): + """Handle wheel scroll event""" event.ignore() class NoWheelDoubleSpinBox(QtWidgets.QDoubleSpinBox): @@ -865,6 +866,7 @@ def __init__(self, parent): QtWidgets.QDoubleSpinBox.__init__(self, parent) def wheelEvent(self, event): + """Handle wheel scroll event""" event.ignore() class NoWheelSpinBox(QtWidgets.QSpinBox): @@ -874,4 +876,5 @@ def __init__(self, parent): QtWidgets.QSpinBox.__init__(self, parent) def wheelEvent(self, event): + """Handle wheel scroll event""" event.ignore() diff --git a/cuegui/cuegui/GroupDialog.py b/cuegui/cuegui/GroupDialog.py index d5f26ac18..4e0491c5c 100644 --- a/cuegui/cuegui/GroupDialog.py +++ b/cuegui/cuegui/GroupDialog.py @@ -1,4 +1,3 @@ - # Copyright Contributors to the OpenCue Project # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -155,6 +154,7 @@ def __createButtons(self, buttons, row, width): # pylint: enable=no-member def accept(self): + """Accept action""" __name = str(self._nameValue.text()) if not __name: return diff --git a/cuegui/cuegui/HostMonitorTree.py b/cuegui/cuegui/HostMonitorTree.py index a7c63147f..b504e8778 100644 --- a/cuegui/cuegui/HostMonitorTree.py +++ b/cuegui/cuegui/HostMonitorTree.py @@ -284,9 +284,11 @@ def startDrag(self, dropActions): cuegui.Utils.startDrag(self, dropActions, self.selectedObjects()) def dragEnterEvent(self, event): + """Enter drag event""" cuegui.Utils.dragEnterEvent(event, "application/x-host-ids") def dragMoveEvent(self, event): + """Move drag event""" cuegui.Utils.dragMoveEvent(event, "application/x-host-ids") diff --git a/cuegui/cuegui/MainWindow.py b/cuegui/cuegui/MainWindow.py index a27990cfd..14450aeeb 100644 --- a/cuegui/cuegui/MainWindow.py +++ b/cuegui/cuegui/MainWindow.py @@ -446,6 +446,7 @@ def __toggleFullscreen(self): ################################################################################ def keyPressEvent(self, event): + """Handle keys being pressed""" if event.key() == QtCore.Qt.Key_Space: self.app.request_update.emit() event.accept() diff --git a/cuegui/cuegui/Redirect.py b/cuegui/cuegui/Redirect.py index 1476b1ad2..3b8d4b936 100644 --- a/cuegui/cuegui/Redirect.py +++ b/cuegui/cuegui/Redirect.py @@ -892,7 +892,7 @@ class ProxyModel(QtCore.QSortFilterProxyModel): """Provides support for sorting data passed between the model and the tree view""" def lessThan(self, left, right): - + """Handle sorting comparison""" leftData = self.sourceModel().data(left) rightData = self.sourceModel().data(right) diff --git a/cuegui/cuegui/plugins/AllocationsPlugin.py b/cuegui/cuegui/plugins/AllocationsPlugin.py index cc8273c25..b88ab1b1a 100644 --- a/cuegui/cuegui/plugins/AllocationsPlugin.py +++ b/cuegui/cuegui/plugins/AllocationsPlugin.py @@ -161,12 +161,15 @@ def contextMenuEvent(self, e): """When right clicking on an item, this raises a context menu""" def dragEnterEvent(self, event): + """Drag enter event""" cuegui.Utils.dragEnterEvent(event, "application/x-host-ids") def dragMoveEvent(self, event): + """Drag move event""" cuegui.Utils.dragMoveEvent(event, "application/x-host-ids") def dropEvent(self, event): + """Drop event""" item = self.itemAt(event.pos()) if item.type() == cuegui.Constants.TYPE_ALLOC: @@ -181,6 +184,7 @@ def dropEvent(self, event): self.updateSoon() def tick(self): + """no-op""" # tick is unused in this widget pass diff --git a/cuegui/cuegui/plugins/LogViewPlugin.py b/cuegui/cuegui/plugins/LogViewPlugin.py index a59f20251..aecaa24fe 100644 --- a/cuegui/cuegui/plugins/LogViewPlugin.py +++ b/cuegui/cuegui/plugins/LogViewPlugin.py @@ -339,6 +339,7 @@ def __init__(self, fn, *args, **kwargs): @QtCore.Slot() def run(self): + """Thread run action""" # pylint: disable=bare-except try: content, log_mtime = self.fn(*self.args, **self.kwargs) @@ -1000,6 +1001,7 @@ def __init__(self, parent=None): self.completeFormat.setForeground(cuegui.Style.ColorTheme.LOG_COMPLETE) def highlightBlock(self, text): + """Handle blocks that should be highlighted""" if not self.on: return diff --git a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py index bb1b61d02..922a04140 100644 --- a/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobDetailsPlugin.py @@ -100,12 +100,15 @@ def __splitterMoved(self, pos, index): self.__monitorLayers.disableUpdate = not bool(pos) def dragEnterEvent(self, event): + """Enter drag event handler""" cuegui.Utils.dragEnterEvent(event) def dragMoveEvent(self, event): + """Move drag event handler""" cuegui.Utils.dragMoveEvent(event) def dropEvent(self, event): + """Drop Event handler""" for jobName in cuegui.Utils.dropEvent(event): self.__setJob(jobName) diff --git a/cuegui/cuegui/plugins/MonitorJobsPlugin.py b/cuegui/cuegui/plugins/MonitorJobsPlugin.py index 744603088..c504ba957 100644 --- a/cuegui/cuegui/plugins/MonitorJobsPlugin.py +++ b/cuegui/cuegui/plugins/MonitorJobsPlugin.py @@ -351,6 +351,7 @@ def __init__(self, parent): self.setToolTip(toolTip) def contextMenuEvent(self, e): + """Handle context menu events""" menu = QtWidgets.QMenu(self) menu.addAction(cuegui.Action.create(self, diff --git a/cuegui/cuegui/plugins/StuckFramePlugin.py b/cuegui/cuegui/plugins/StuckFramePlugin.py index aff41339d..94f886447 100644 --- a/cuegui/cuegui/plugins/StuckFramePlugin.py +++ b/cuegui/cuegui/plugins/StuckFramePlugin.py @@ -767,6 +767,7 @@ def __init__(self, parent): self.jobs_created = {} self.groups_created = {} self.currentHosts = [] + self.show = None # Bring Up a comment if it exists self._itemSingleClickedComment.connect(self.__itemSingleClickedComment) @@ -1447,15 +1448,19 @@ def RemoveJob(self): self._processUpdate(None, self.currentHosts) def startDrag(self, dropActions): + """Drag start action""" cuegui.Utils.startDrag(self, dropActions, self.selectedObjects()) def dragEnterEvent(self, event): + """Drag enter action""" cuegui.Utils.dragEnterEvent(event, "application/x-host-ids") def dragMoveEvent(self, event): + """Drag move event""" cuegui.Utils.dragMoveEvent(event, "application/x-host-ids") def updateFilters(self, filters, show): + """Update filters""" self.showData = show self.show = show.data.name self.filters = filters From a9e6c63ea99958e973cce42998276b1accf81a99 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 27 Nov 2024 12:46:10 -0800 Subject: [PATCH 6/9] Fix lint errors --- cuegui/cuegui/plugins/AllocationsPlugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cuegui/cuegui/plugins/AllocationsPlugin.py b/cuegui/cuegui/plugins/AllocationsPlugin.py index b88ab1b1a..1bf8ce3ec 100644 --- a/cuegui/cuegui/plugins/AllocationsPlugin.py +++ b/cuegui/cuegui/plugins/AllocationsPlugin.py @@ -185,8 +185,6 @@ def dropEvent(self, event): def tick(self): """no-op""" - # tick is unused in this widget - pass class AllocationWidgetItem(cuegui.AbstractWidgetItem.AbstractWidgetItem): From 8b736e85cc115a55e155b183646c31bdc804c324 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 27 Nov 2024 13:32:48 -0800 Subject: [PATCH 7/9] Fix lint errors --- ci/run_python_lint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index f93527d1e..57bfff18c 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -38,7 +38,7 @@ cd .. echo "Running lint for cuegui/..." cd cuegui PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_main cuegui --ignore=cuegui/images,cuegui/images/crystal --disable=no-member -PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests +PYTHONPATH=../pycue python -m pylint --rcfile=../ci/pylintrc_test tests --disable=no-member cd .. echo "Running lint for cuesubmit/..." From c4b1639ee5a96bc38360ef84ab8be671aa30ef44 Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 27 Nov 2024 13:38:53 -0800 Subject: [PATCH 8/9] Fix lint errors --- ci/run_python_lint.sh | 4 ++-- cuesubmit/cuesubmit/ui/Widgets.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index 57bfff18c..488b9c946 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -43,8 +43,8 @@ cd .. echo "Running lint for cuesubmit/..." cd cuesubmit -PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_main cuesubmit -PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_test tests +PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_main cuesubmit --disable=no-member +PYTHONPATH=../pycue:../pyoutline python -m pylint --rcfile=../ci/pylintrc_test tests --disable=no-member cd .. echo "Running lint for rqd/..." diff --git a/cuesubmit/cuesubmit/ui/Widgets.py b/cuesubmit/cuesubmit/ui/Widgets.py index fb16400a9..443096344 100644 --- a/cuesubmit/cuesubmit/ui/Widgets.py +++ b/cuesubmit/cuesubmit/ui/Widgets.py @@ -164,10 +164,12 @@ def __init__(self, defaultText=None, completerStrings=None, parent=None): self.setStyleSheet(Style.LINE_EDIT) def focusInEvent(self, e): + """Event when in focus""" super(CueLineEdit, self).focusInEvent(e) self.focusChange.emit(True) def focusOutEvent(self, e): + """Event when out of focus""" super(CueLineEdit, self).focusOutEvent(e) self.focusChange.emit(False) From e6d609194a2a390df215b2a3fb17b3c690f9e7fc Mon Sep 17 00:00:00 2001 From: Diego Tavares Date: Wed, 27 Nov 2024 15:12:04 -0800 Subject: [PATCH 9/9] Fix lint errors --- cuesubmit/cuesubmit/ui/Submit.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cuesubmit/cuesubmit/ui/Submit.py b/cuesubmit/cuesubmit/ui/Submit.py index cd2cbc39f..adbac9c4b 100644 --- a/cuesubmit/cuesubmit/ui/Submit.py +++ b/cuesubmit/cuesubmit/ui/Submit.py @@ -211,6 +211,7 @@ def __init__( self.jobDataChanged() def showEvent(self, event): + """Show Event""" del event if self.startupErrors: