Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/AcademySoftwareFoundation…
Browse files Browse the repository at this point in the history 
…/OpenCue into blender-addon-dev
  • Loading branch information
@n-jay
n-jay committed Jul 14, 2024
2 parents 8c13425 + e54987a commit 1d754d5
Show file tree
Hide file tree
Showing 80 changed files with 766 additions and 355 deletions.
12 changes: 10 additions & 2 deletions .github/workflows/testing-pipeline.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ jobs:
name: Run Python Unit Tests (CY2022)
runs-on: ubuntu-latest
container: aswf/ci-opencue:2022
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
steps:
- uses: actions/checkout@v3
- name: Run Python Tests
Expand All @@ -21,6 +23,8 @@ jobs:
runs-on: ubuntu-latest
container:
image: aswf/ci-opencue:2022
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
steps:
- uses: actions/checkout@v3
- name: Build with Gradle
Expand Down Expand Up @@ -53,6 +57,8 @@ jobs:
name: Run Python Unit Tests using Python2
runs-on: ubuntu-latest
container: aswf/ci-opencue:2019
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
steps:
- uses: actions/checkout@v3
- name: Run Python Tests
Expand All @@ -71,6 +77,8 @@ jobs:
name: Lint Python Code
runs-on: ubuntu-latest
container: aswf/ci-opencue:2022
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
steps:
- uses: actions/checkout@v3
- name: Lint Python Code
Expand All @@ -93,7 +101,7 @@ jobs:
- uses: actions/checkout@v3
- name: Get Changed Files
id: get_changed_files
uses: tj-actions/changed-files@v35
uses: tj-actions/changed-files@v41
- name: Check for Version Change
run: ci/check_changed_files.py ${{ steps.get_changed_files.outputs.modified_files }} ${{ steps.get_changed_files.outputs.deleted_files }}

Expand All @@ -112,6 +120,6 @@ jobs:
- uses: actions/checkout@v3
- name: Get Changed Files
id: get_changed_files
uses: tj-actions/changed-files@v35
uses: tj-actions/changed-files@v41
- name: Check for Version Change
run: ci/check_version_bump.py ${{ steps.get_changed_files.outputs.all_changed_and_modified_files }}
2 changes: 1 addition & 1 deletion VERSION.in
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.29
0.30
4 changes: 3 additions & 1 deletion cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,9 @@ public void insertLayerOutput(LayerInterface layer, String filespec) {
"FROM " +
"layer_output " +
"WHERE " +
"pk_layer = ?";
"pk_layer = ?" +
"ORDER BY " +
"ser_order";

private static final RowMapper<String> OUTPUT_MAPPER =
new RowMapper<String>() {
Expand Down
146 changes: 87 additions & 59 deletions cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ public class HostReportHandler {
private static final String SUBJECT_COMMENT_FULL_TEMP_DIR = "Host set to REPAIR for not having enough storage " +
"space on the temporary directory (mcp)";
private static final String CUEBOT_COMMENT_USER = "cuebot";
private static final String WINDOWS_OS = "Windows";

// A cache <hostname_frameId, count> to store kill requests and count the number of occurrences.
// The cache expires after write to avoid growing unbounded. If a request for a host-frame doesn't appear
Expand Down Expand Up @@ -182,7 +183,11 @@ public void handleHostReport(HostReport report, boolean isBoot) {
rhost.getLoad(), new Timestamp(rhost.getBootTime() * 1000l),
rhost.getAttributesMap().get("SP_OS"));

changeHardwareState(host, report.getHost().getState(), isBoot, report.getHost().getFreeMcp());
// Both logics are conflicting, only change hardware state if
// there was no need for a tempDirStorage state change
if (!changeStateForTempDirStorage(host, report.getHost())) {
changeHardwareState(host, report.getHost().getState(), isBoot);
}
changeNimbyState(host, report.getHost());

/**
Expand Down Expand Up @@ -247,12 +252,10 @@ public void handleHostReport(HostReport report, boolean isBoot) {
}
}

// The minimum amount of free space in the temporary directory to book a host
Long minBookableFreeTempDir = env.getRequiredProperty("dispatcher.min_bookable_free_temp_dir_kb", Long.class);

if (minBookableFreeTempDir != -1 && report.getHost().getFreeMcp() < minBookableFreeTempDir) {
msg = String.format("%s doens't have enough free space in the temporary directory (mcp), %dMB needs %dMB",
host.name, (report.getHost().getFreeMcp()/1024), (minBookableFreeTempDir/1024));
if (!isTempDirStorageEnough(report.getHost().getTotalMcp(), report.getHost().getFreeMcp(), host.os)) {
msg = String.format(
"%s doens't have enough free space in the temporary directory (mcp), %dMB",
host.name, (report.getHost().getFreeMcp()/1024));
}
else if (host.idleCores < Dispatcher.CORE_POINTS_RESERVED_MIN) {
msg = String.format("%s doesn't have enough idle cores, %d needs %d",
Expand Down Expand Up @@ -333,6 +336,27 @@ else if (!dispatchSupport.isCueBookable(host)) {
}
}

/**
* Check if a reported temp storage size and availability is enough for running a job
*
* Use dispatcher.min_available_temp_storage_percentage (opencue.properties) to
* define what's the accepted threshold. Providing hostOs is necessary as this feature
* is currently not available on Windows hosts
*
* @param tempTotalStorage Total storage on the temp directory
* @param tempFreeStorage Free storage on the temp directory
* @param hostOs Reported os
* @return
*/
private boolean isTempDirStorageEnough(Long tempTotalStorage, Long tempFreeStorage, String hostOs) {
// The minimum amount of free space in the temporary directory to book a host
int minAvailableTempPercentage = env.getRequiredProperty(
"dispatcher.min_available_temp_storage_percentage", Integer.class);

return minAvailableTempPercentage == -1 || hostOs.equalsIgnoreCase(WINDOWS_OS) ||
(((tempFreeStorage * 100.0) / tempTotalStorage) >= minAvailableTempPercentage);
}

/**
* Update the hardware state property.
*
Expand All @@ -342,62 +366,11 @@ else if (!dispatchSupport.isCueBookable(host)) {
* updated with a boot report. If the state is Repair, then state is
* never updated via RQD.
*
*
* Prevent cue frames from booking on hosts with full temporary directories.
*
* Change host state to REPAIR or UP according the amount of free space
* in the temporary directory:
* - Set the host state to REPAIR, when the amount of free space in the
* temporary directory is less than the minimum required. Add a comment with
* subject: SUBJECT_COMMENT_FULL_TEMP_DIR
* - Set the host state to UP, when the amount of free space in the temporary directory
* is greater or equals to the minimum required and the host has a comment with
* subject: SUBJECT_COMMENT_FULL_TEMP_DIR
*
* @param host
* @param reportState
* @param isBoot
* @param freeTempDir
*/
private void changeHardwareState(DispatchHost host, HardwareState reportState, boolean isBoot, long freeTempDir) {

// The minimum amount of free space in the temporary directory to book a host
Long minBookableFreeTempDir = env.getRequiredProperty("dispatcher.min_bookable_free_temp_dir_kb", Long.class);

// Prevent cue frames from booking on hosts with full temporary directories
if (minBookableFreeTempDir != -1 && !host.os.equalsIgnoreCase("Windows")) {
if (host.hardwareState == HardwareState.UP && freeTempDir < minBookableFreeTempDir) {

// Insert a comment indicating that the Host status = Repair with reason = Full temporary directory
CommentDetail c = new CommentDetail();
c.subject = SUBJECT_COMMENT_FULL_TEMP_DIR;
c.user = CUEBOT_COMMENT_USER;
c.timestamp = null;
c.message = "Host " + host.getName() + " marked as REPAIR. The current amount of free space in the " +
"temporary directory (mcp) is " + (freeTempDir/1024) + "MB. It must have at least "
+ (minBookableFreeTempDir/1024) + "MB of free space in temporary directory";
commentManager.addComment(host, c);

// Set the host state to REPAIR
hostManager.setHostState(host, HardwareState.REPAIR);
host.hardwareState = HardwareState.REPAIR;

return;
} else if (host.hardwareState == HardwareState.REPAIR && freeTempDir >= minBookableFreeTempDir) {
// Check if the host with REPAIR status has comments with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and
// user=CUEBOT_COMMENT_USER and delete the comments, if they exists
boolean commentsDeleted = commentManager.deleteCommentByHostUserAndSubject(host,
CUEBOT_COMMENT_USER, SUBJECT_COMMENT_FULL_TEMP_DIR);

if (commentsDeleted) {
// Set the host state to UP
hostManager.setHostState(host, HardwareState.UP);
host.hardwareState = HardwareState.UP;
return;
}
}
}

private void changeHardwareState(DispatchHost host, HardwareState reportState, boolean isBoot) {
// If the states are the same there is no reason to do this update.
if (host.hardwareState.equals(reportState)) {
return;
Expand Down Expand Up @@ -427,6 +400,61 @@ private void changeHardwareState(DispatchHost host, HardwareState reportState, b
}
}

/**
* Prevent cue frames from booking on hosts with full temporary directories.
*
* Change host state to REPAIR or UP according to the amount of free space
* in the temporary directory:
* - Set the host state to REPAIR, when the amount of free space in the
* temporary directory is less than the minimum required.
* - Set the host state to UP, when the amount of free space in the temporary directory
* is greater or equal to the minimum required and the host has a comment with
* subject: SUBJECT_COMMENT_FULL_TEMP_DIR
*
* @param host
* @param reportHost
* @return
*/
private boolean changeStateForTempDirStorage(DispatchHost host, RenderHost reportHost) {
// The minimum amount of free space in the temporary directory to book a host
int minAvailableTempPercentage = env.getRequiredProperty(
"dispatcher.min_available_temp_storage_percentage", Integer.class);

// Prevent cue frames from booking on hosts with full temporary directories
boolean hasEnoughTempStorage = isTempDirStorageEnough(reportHost.getTotalMcp(), reportHost.getFreeMcp(), host.os);
if (!hasEnoughTempStorage && host.hardwareState == HardwareState.UP) {
// Insert a comment indicating that the Host status = Repair with reason = Full temporary directory
CommentDetail c = new CommentDetail();
c.subject = SUBJECT_COMMENT_FULL_TEMP_DIR;
c.user = CUEBOT_COMMENT_USER;
c.timestamp = null;
long requiredTempMb = (long)((minAvailableTempPercentage / 100.0) * reportHost.getTotalMcp()/ 1024);
c.message = "Host " + host.getName() + " marked as REPAIR. The current amount of free space in the " +
"temporary directory (mcp) is " + (reportHost.getFreeMcp()/1024) + "MB. It must have at least "
+ ((requiredTempMb)) + "MB of free space in temporary directory";
commentManager.addComment(host, c);

// Set the host state to REPAIR
hostManager.setHostState(host, HardwareState.REPAIR);
host.hardwareState = HardwareState.REPAIR;

return true;
} else if (hasEnoughTempStorage && host.hardwareState == HardwareState.REPAIR) {
// Check if the host with REPAIR status has comments with subject=SUBJECT_COMMENT_FULL_TEMP_DIR and
// user=CUEBOT_COMMENT_USER and delete the comments, if they exist
boolean commentsDeleted = commentManager.deleteCommentByHostUserAndSubject(host,
CUEBOT_COMMENT_USER, SUBJECT_COMMENT_FULL_TEMP_DIR);

if (commentsDeleted) {
// Set the host state to UP
hostManager.setHostState(host, HardwareState.UP);
host.hardwareState = HardwareState.UP;
return true;
}
}
return false;
}

/**
* Changes the NIMBY lock state. If the DB indicates a NIMBY lock
* but RQD does not, then the host is unlocked. If the DB indicates
Expand Down
Loading

0 comments on commit 1d754d5

Please sign in to comment.