Merge branch 'master' into bug/windows_core_detection

ci/build_sphinx_docs.sh

@@ -10,7 +10,7 @@ python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 # Build the docs and treat warnings as errors
 ~/.local/bin/sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html

ci/fix_compiled_proto.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+"""Script that makes the imports in the generated compiled_proto python files relative.
+
+"""
+import os
+import re
+import sys
+import glob
+
+PYTHON_SCRIPT_PATH = sys.argv[1]
+
+if os.path.isdir(PYTHON_SCRIPT_PATH):
+    pattern = re.compile(r"^import \w+ as \w+_pb2")
+    for filepath in glob.glob(os.path.join(PYTHON_SCRIPT_PATH, "*_pb2*.py")):
+        filedata = []
+        with open(filepath) as f:
+            for line in f.readlines():
+                match = pattern.match(line)
+                if match is not None:
+                    line = f"from . {line}"
+                filedata.append(line.strip("\n"))
+        with open(filepath, "w") as f:
+            f.write("\n".join(filedata))
+else:
+    print("Argument is not a directory")

ci/python_coverage_report.sh

@@ -9,8 +9,8 @@ python -m pip install coverage pytest-xvfb
 # Protos need to have their Python code generated in order for tests to pass.
 python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
 python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
+python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
 
 # Run coverage for each component individually, but append it all into the same report.
 python -m coverage run --source=pycue/opencue/,pycue/FileSequence/ --omit=pycue/opencue/compiled_proto/* pycue/tests/test_suite.py

ci/run_python_lint.sh

@@ -13,8 +13,8 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
+python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
 
 echo "Running lint for pycue/..."
 cd pycue

ci/run_python_tests.sh

@@ -19,14 +19,14 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
-
-python3 -m unittest discover -s pycue/tests -t pycue -p "*.py"
-PYTHONPATH=pycue python3 -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py"
-PYTHONPATH=pycue python3 -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py"
-PYTHONPATH=pycue:pyoutline python3 -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py"
-python3 -m unittest discover -s rqd/tests -t rqd -p "*.py"
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
+python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
+
+python -m unittest discover -s pycue/tests -t pycue -p "*.py"
+PYTHONPATH=pycue python -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py"
+PYTHONPATH=pycue python -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py"
+PYTHONPATH=pycue:pyoutline python -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py"
+python -m unittest discover -s rqd/tests -t rqd -p "*.py"
 
 
 # Xvfb no longer supports Python 2.

connectors/prometheus_metrics/Dockerfile

@@ -48,7 +48,7 @@ RUN python -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 RUN cd pycue && python setup.py install
 

cueadmin/Dockerfile

@@ -24,7 +24,7 @@ RUN python3 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 COPY cueadmin/README.md ./cueadmin/
 COPY cueadmin/setup.py ./cueadmin/

cuebot/src/main/java/com/imageworks/spcue/ServiceEntity.java

@@ -54,12 +54,12 @@ public class ServiceEntity extends Entity {
     /**
      * Determines the default minimum memory per frame.
      */
-    public long minMemory = Dispatcher.MEM_RESERVED_DEFAULT;
+    public long minMemory = Dispatcher.MEM_SERVICE_RESERVED_DEFAULT;
 
     /**
      * Determines the default minimum gpu per frame.
      */
-    public long minGpuMemory = Dispatcher.MEM_GPU_RESERVED_DEFAULT;
+    public long minGpuMemory = Dispatcher.MEM_SERVICE_GPU_RESERVED_DEFAULT;
 
     /**
      * Determines the default tags.

cuebot/src/main/java/com/imageworks/spcue/VirtualProc.java

@@ -148,7 +148,7 @@ else if (proc.coresReserved >= 100) {
                 proc.coresReserved = wholeCores * 100;
             } else {
                 if (frame.threadable) {
-                    if (selfishServices != null && 
+                    if (selfishServices != null &&
                         frame.services != null &&
                         containsSelfishService(frame.services.split(","), selfishServices)){
                         proc.coresReserved = wholeCores * 100;

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/HostDaoJdbc.java

@@ -30,6 +30,8 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
 import org.springframework.dao.EmptyResultDataAccessException;
 import org.springframework.jdbc.core.CallableStatementCreator;
 import org.springframework.jdbc.core.RowMapper;
@@ -58,6 +60,9 @@
 
 public class HostDaoJdbc extends JdbcDaoSupport implements HostDao {
 
+    @Autowired
+    private Environment env;
+
     public static final RowMapper<HostEntity> HOST_DETAIL_MAPPER = new RowMapper<HostEntity>() {
         public HostEntity mapRow(ResultSet rs, int rowNum) throws SQLException {
             HostEntity host = new HostEntity();
@@ -324,9 +329,12 @@ public void insertRenderHost(RenderHost host, AllocationInterface a, boolean use
         }
 
         long memUnits = convertMemoryUnits(host);
-        if (memUnits < Dispatcher.MEM_RESERVED_MIN) {
+        long memReserverMin = env.getRequiredProperty(
+            "dispatcher.memory.mem_reserved_min",
+            Long.class);
+        if (memUnits < memReserverMin) {
             throw new EntityCreationError("could not create host " + host.getName() + ", " +
-                    " must have at least " + Dispatcher.MEM_RESERVED_MIN + " free memory.");
+                    " must have at least " + memReserverMin + " free memory.");
         }
 
         String fqdn;
@@ -727,18 +735,21 @@ private long convertMemoryUnits(RenderHost host) {
 
         long memUnits;
         if (host.getTagsList().contains("64bit")) {
-            memUnits = CueUtil.convertKbToFakeKb64bit(host.getTotalMem());
+            memUnits = CueUtil.convertKbToFakeKb64bit(env, host.getTotalMem());
         }
         else {
-            memUnits = CueUtil.convertKbToFakeKb32bit(host.getTotalMem());
+            memUnits = CueUtil.convertKbToFakeKb32bit(env, host.getTotalMem());
         }
 
         /*
          * If this is a desktop, we'll just cut the memory
          * so we don't annoy the user.
          */
         if (host.getNimbyEnabled()) {
-            memUnits = (long) (memUnits / 1.5) + Dispatcher.MEM_RESERVED_SYSTEM;
+            long memReservedSystem = env.getRequiredProperty(
+                "dispatcher.memory.mem_reserved_system",
+                Long.class);
+            memUnits = (long) (memUnits / 1.5) + memReservedSystem;
         }
 
         return memUnits;

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/LayerDaoJdbc.java

@@ -33,6 +33,8 @@
 import org.springframework.dao.EmptyResultDataAccessException;
 import org.springframework.jdbc.core.RowMapper;
 import org.springframework.jdbc.core.support.JdbcDaoSupport;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
 
 import com.imageworks.spcue.ExecutionSummary;
 import com.imageworks.spcue.FrameStateTotals;
@@ -41,7 +43,6 @@
 import com.imageworks.spcue.LayerEntity;
 import com.imageworks.spcue.LayerInterface;
 import com.imageworks.spcue.LimitEntity;
-import com.imageworks.spcue.LimitInterface;
 import com.imageworks.spcue.ResourceUsage;
 import com.imageworks.spcue.ThreadStats;
 import com.imageworks.spcue.dao.LayerDao;
@@ -56,6 +57,7 @@
 import org.apache.logging.log4j.LogManager;
 
 public class LayerDaoJdbc extends JdbcDaoSupport implements LayerDao {
+    private final long MEM_RESERVED_MIN;
     private static final Logger logger = LogManager.getLogger(LayerDaoJdbc.class);
     private static final String INSERT_OUTPUT_PATH =
         "INSERT INTO " +
@@ -67,6 +69,14 @@ public class LayerDaoJdbc extends JdbcDaoSupport implements LayerDao {
             "str_filespec " +
         ") VALUES (?,?,?,?)";
 
+    @Autowired
+    public LayerDaoJdbc(Environment env) {
+        this.MEM_RESERVED_MIN = env.getRequiredProperty(
+            "dispatcher.memory.mem_reserved_min",
+            Long.class
+        );
+    }
+
     @Override
     public void insertLayerOutput(LayerInterface layer, String filespec) {
         getJdbcTemplate().update(
@@ -341,8 +351,8 @@ public void insertLayerDetail(LayerDetail l) {
 
     @Override
     public void updateLayerMinMemory(LayerInterface layer, long val) {
-        if (val < Dispatcher.MEM_RESERVED_MIN) {
-            val = Dispatcher.MEM_RESERVED_MIN;
+        if (val < MEM_RESERVED_MIN) {
+            val = MEM_RESERVED_MIN;
         }
         getJdbcTemplate().update("UPDATE layer SET int_mem_min=? WHERE pk_layer=?",
                 val, layer.getLayerId());
@@ -380,8 +390,8 @@ public boolean balanceLayerMinMemory(LayerInterface layer, long frameMaxRss) {
         if (maxrss < frameMaxRss) {
             maxrss = frameMaxRss;
         }
-        if (maxrss < Dispatcher.MEM_RESERVED_MIN) {
-            maxrss = Dispatcher.MEM_RESERVED_MIN;
+        if (maxrss < MEM_RESERVED_MIN) {
+            maxrss = MEM_RESERVED_MIN;
         } else {
             maxrss = maxrss + CueUtil.MB256;
         }
@@ -603,11 +613,11 @@ public long findPastMaxRSS(JobInterface job, String name) {
         try {
             long maxRss = getJdbcTemplate().queryForObject(FIND_PAST_MAX_RSS,
                     Long.class, job.getJobId(), name);
-            if (maxRss >= Dispatcher.MEM_RESERVED_MIN) {
+            if (maxRss >= MEM_RESERVED_MIN) {
                 return maxRss;
             }
             else {
-                return Dispatcher.MEM_RESERVED_MIN;
+                return MEM_RESERVED_MIN;
             }
         } catch (EmptyResultDataAccessException e) {
             // Actually want to return 0 here, which means
@@ -625,8 +635,8 @@ public void updateTags(JobInterface job, String tags, LayerType type) {
 
     @Override
     public void updateMinMemory(JobInterface job, long mem, LayerType type) {
-        if (mem < Dispatcher.MEM_RESERVED_MIN) {
-            mem = Dispatcher.MEM_RESERVED_MIN;
+        if (mem < MEM_RESERVED_MIN) {
+            mem = MEM_RESERVED_MIN;
         }
         getJdbcTemplate().update(
                 "UPDATE layer SET int_mem_min=? WHERE pk_job=? AND str_type=?",

cuebot/src/main/java/com/imageworks/spcue/dao/postgres/ProcDaoJdbc.java

@@ -29,6 +29,8 @@
 import java.util.List;
 import java.util.Map;
 
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
 import org.springframework.dao.DataAccessException;
 import org.springframework.jdbc.core.PreparedStatementCreator;
 import org.springframework.jdbc.core.RowMapper;
@@ -45,14 +47,16 @@
 import com.imageworks.spcue.dao.ProcDao;
 import com.imageworks.spcue.dao.criteria.FrameSearchInterface;
 import com.imageworks.spcue.dao.criteria.ProcSearchInterface;
-import com.imageworks.spcue.dispatcher.Dispatcher;
 import com.imageworks.spcue.dispatcher.ResourceDuplicationFailureException;
 import com.imageworks.spcue.dispatcher.ResourceReservationFailureException;
 import com.imageworks.spcue.grpc.host.HardwareState;
 import com.imageworks.spcue.util.SqlUtil;
 
 public class ProcDaoJdbc extends JdbcDaoSupport implements ProcDao {
 
+    @Autowired
+    private Environment env;
+
     private static final String VERIFY_RUNNING_PROC =
         "SELECT " +
             "proc.pk_frame " +
@@ -121,15 +125,21 @@ public boolean deleteVirtualProc(VirtualProc proc) {
 
     public void insertVirtualProc(VirtualProc proc) {
         proc.id = SqlUtil.genKeyRandom();
+        long memReservedMin = env.getRequiredProperty(
+            "dispatcher.memory.mem_reserved_min",
+            Long.class);
+        long memGpuReservedMin = env.getRequiredProperty(
+            "dispatcher.memory.mem_gpu_reserved_min",
+            Long.class);
         int result = 0;
         try {
             result = getJdbcTemplate().update(INSERT_VIRTUAL_PROC,
                      proc.getProcId(), proc.getHostId(), proc.getShowId(),
                      proc.getLayerId(), proc.getJobId(), proc.getFrameId(),
                      proc.coresReserved, proc.memoryReserved,
-                     proc.memoryReserved, Dispatcher.MEM_RESERVED_MIN,
+                     proc.memoryReserved, memReservedMin,
                      proc.gpusReserved, proc.gpuMemoryReserved,
-                     proc.gpuMemoryReserved, Dispatcher.MEM_GPU_RESERVED_MIN,
+                     proc.gpuMemoryReserved, memGpuReservedMin,
                      proc.isLocalDispatch);
 
             // Update all of the resource counts
@@ -634,7 +644,10 @@ public boolean balanceUnderUtilizedProcs(ProcInterface targetProc, long targetMe
               for (Map<String,Object> map: result) {
                   String pk_proc = (String) map.get("pk_proc");
                   Long free_mem = (Long) map.get("free_mem");
-                  long available = free_mem - borrowMap.get(pk_proc) - Dispatcher.MEM_RESERVED_MIN;
+                  long memReservedMin = env.getRequiredProperty(
+                    "dispatcher.memory.mem_reserved_min",
+                    Long.class);
+                  long available = free_mem - borrowMap.get(pk_proc) - memReservedMin;
                   if (available > memPerFrame) {
                       borrowMap.put(pk_proc, borrowMap.get(pk_proc) + memPerFrame);
                       memBorrowedTotal = memBorrowedTotal + memPerFrame;