Feature - Selfish services (#1390)

A selfish service makes sure its frames have all the host available cores reserved, ensuring this job will not be competing resources with other frames. Selfish services can be configured on the opencue.properties file on `dispatcher.frame.selfish.services`.
AcademySoftwareFoundation · Jun 20, 2024 · 0523b2b · 0523b2b
1 parent b0da8e4
commit 0523b2b
Show file tree

Hide file tree

Showing 5 changed files with 58 additions and 8 deletions.
diff --git a/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java b/cuebot/src/main/java/com/imageworks/spcue/DispatchFrame.java
@@ -47,6 +47,7 @@ public class DispatchFrame extends FrameEntity implements FrameInterface {
     public int maxGpus;
     public long minGpuMemory;
 
+    // A comma separated list of services
     public String services;
 }
 
diff --git a/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java b/cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java
@@ -79,7 +79,7 @@ public String getName() {
      * @param frame
      * @return
      */
-    public static final VirtualProc build(DispatchHost host, DispatchFrame frame) {
+    public static final VirtualProc build(DispatchHost host, DispatchFrame frame, String... selfishServices) {
         VirtualProc proc = new VirtualProc();
         proc.allocationId = host.getAllocationId();
         proc.hostId = host.getHostId();
@@ -132,13 +132,19 @@ public static final VirtualProc build(DispatchHost host, DispatchFrame frame) {
                 proc.coresReserved = wholeCores * 100;
             } else {
                 if (frame.threadable) {
-                    if (host.idleMemory - frame.minMemory
-                            <= Dispatcher.MEM_STRANDED_THRESHHOLD) {
+                    if (selfishServices != null && 
+                        frame.services != null &&
+                        containsSelfishService(frame.services.split(","), selfishServices)){
                         proc.coresReserved = wholeCores * 100;
-                    } else {
-                        proc.coresReserved = getCoreSpan(host, frame.minMemory);
                     }
-
+                    else {
+                        if (host.idleMemory - frame.minMemory
+                                <= Dispatcher.MEM_STRANDED_THRESHHOLD) {
+                            proc.coresReserved = wholeCores * 100;
+                        } else {
+                            proc.coresReserved = getCoreSpan(host, frame.minMemory);
+                        }
+                    }
                     if (host.threadMode == ThreadMode.VARIABLE_VALUE
                             && proc.coresReserved <= 200) {
                         proc.coresReserved = 200;
@@ -194,6 +200,17 @@ public static final VirtualProc build(DispatchHost host, DispatchFrame frame) {
         return proc;
     }
 
+    private static final boolean containsSelfishService(String[] frameServices, String[] selfishServices) {
+        for (String frameService: frameServices){
+            for (String selfishService: selfishServices) {
+                if (frameService.equals(selfishService)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
     public static final VirtualProc build(DispatchHost host,
             DispatchFrame frame, LocalHostAssignment lja) {
 

diff --git a/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java b/cuebot/src/main/java/com/imageworks/spcue/dispatcher/CoreUnitDispatcher.java
@@ -259,9 +259,10 @@ public List<VirtualProc> dispatchHost(DispatchHost host, JobInterface job) {
                 host.getName() + " " + host.idleCores + "/" + host.idleMemory +
                 " on job " + job.getName());
 
+        String[] selfishServices = env.getProperty("dispatcher.frame.selfish.services", "").split(",");
         for (DispatchFrame frame: frames) {
 
-            VirtualProc proc =  VirtualProc.build(host, frame);
+            VirtualProc proc =  VirtualProc.build(host, frame, selfishServices);
 
             if (host.idleCores < frame.minCores ||
                     host.idleMemory < frame.minMemory ||

diff --git a/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java b/cuebot/src/test/java/com/imageworks/spcue/test/dao/postgres/ProcDaoTests.java
@@ -847,6 +847,32 @@ public void getProcsBySearch() {
         assertEquals(2, procDao.findVirtualProcs(r).size());
 
     }
+
+    @Test
+    @Transactional
+    @Rollback(true)
+    public void testVirtualProcWithSelfishService() {
+        DispatchHost host = createHost();
+        JobDetail job = launchJob();
+
+        FrameDetail frameDetail = frameDao.findFrameDetail(job, "0001-pass_1_preprocess");
+        DispatchFrame frame = frameDao.getDispatchFrame(frameDetail.id);
+        frame.minCores = 250;
+        frame.threadable = true;
+
+        // Frame from a non-selfish sevice
+        VirtualProc proc = VirtualProc.build(host, frame, "something-else");        
+        assertEquals(250, proc.coresReserved);
+
+        // When no selfish service config is provided
+        proc = VirtualProc.build(host, frame);        
+        assertEquals(250, proc.coresReserved);
+
+
+        // Frame with a selfish service        
+        proc = VirtualProc.build(host, frame, "shell", "something-else");        
+        assertEquals(800, proc.coresReserved);
+    }
 }
 
 
diff --git a/cuebot/src/test/resources/opencue.properties b/cuebot/src/test/resources/opencue.properties
@@ -68,4 +68,9 @@ dispatcher.min_bookable_free_temp_dir_kb=1048576
 dispatcher.min_bookable_free_mcp_kb=1048576
 dispatcher.oom_max_safe_used_memory_threshold=0.95
 dispatcher.oom_frame_overboard_allowed_threshold=0.6
-dispatcher.frame_kill_retry_limit=3
+dispatcher.frame_kill_retry_limit=3
+
+# A comma separated list of services that should have their frames considered 
+# selfish. A selfish frame will reserve all the available cores to avoid
+# having to share resources with other renders.
+dispatcher.frame.selfish.services=arnold,selfish-service