Webpublisher: timeout for PS studio processing

openpype/lib/remote_publish.py

@@ -9,6 +9,8 @@
 
 from openpype.client.mongo import OpenPypeMongoConnection
 from openpype.lib.plugin_tools import parse_json
+from openpype.lib.profiles_filtering import filter_profiles
+from openpype.api import get_project_settings
 
 ERROR_STATUS = "error"
 IN_PROGRESS_STATUS = "in_progress"
@@ -175,14 +177,12 @@ def publish_and_log(dbcon, _id, log, close_plugin_name=None, batch_id=None):
     )
 
 
-def fail_batch(_id, batches_in_progress, dbcon):
-    """Set current batch as failed as there are some stuck batches."""
-    running_batches = [str(batch["_id"])
-                       for batch in batches_in_progress
-                       if batch["_id"] != _id]
-    msg = "There are still running batches {}\n". \
-        format("\n".join(running_batches))
-    msg += "Ask admin to check them and reprocess current batch"
+def fail_batch(_id, dbcon, msg):
+    """Set current batch as failed as there is some problem.
+
+    Raises:
+        ValueError
+    """
     dbcon.update_one(
         {"_id": _id},
         {"$set":
@@ -259,3 +259,19 @@ def get_task_data(batch_dir):
             "Cannot parse batch meta in {} folder".format(task_data))
 
     return task_data
+
+
+def get_timeout(project_name, host_name, task_type):
+    """Returns timeout(seconds) from Setting profile."""
+    filter_data = {
+        "task_types": task_type,
+        "hosts": host_name
+    }
+    timeout_profiles = (get_project_settings(project_name)["webpublisher"]
+                                                          ["timeout_profiles"])
+    matching_item = filter_profiles(timeout_profiles, filter_data)
+    timeout = 3600
+    if matching_item:
+        timeout = matching_item["timeout"]
+
+    return timeout

openpype/pype_commands.py

@@ -15,6 +15,7 @@
     fail_batch,
     find_variant_key,
     get_task_data,
+    get_timeout,
     IN_PROGRESS_STATUS
 )
 
@@ -170,7 +171,7 @@ def publish(paths, targets=None, gui=False):
         log.info("Publish finished.")
 
     @staticmethod
-    def remotepublishfromapp(project, batch_path, host_name,
+    def remotepublishfromapp(project_name, batch_path, host_name,
                              user_email, targets=None):
         """Opens installed variant of 'host' and run remote publish there.
 
@@ -189,8 +190,8 @@ def remotepublishfromapp(project, batch_path, host_name,
         Runs publish process as user would, in automatic fashion.
 
         Args:
-            project (str): project to publish (only single context is expected
-                per call of remotepublish
+            project_name (str): project to publish (only single context is
+                expected per call of remotepublish
             batch_path (str): Path batch folder. Contains subfolders with
                 resources (workfile, another subfolder 'renders' etc.)
             host_name (str): 'photoshop'
@@ -222,19 +223,26 @@ def remotepublishfromapp(project, batch_path, host_name,
 
         batches_in_progress = list(dbcon.find({"status": IN_PROGRESS_STATUS}))
         if len(batches_in_progress) > 1:
-            fail_batch(_id, batches_in_progress, dbcon)
+            running_batches = [str(batch["_id"])
+                               for batch in batches_in_progress
+                               if batch["_id"] != _id]
+            msg = "There are still running batches {}\n". \
+                format("\n".join(running_batches))
+            msg += "Ask admin to check them and reprocess current batch"
+            fail_batch(_id, dbcon, msg)
             print("Another batch running, probably stuck, ask admin for help")
 
-        asset, task_name, _ = get_batch_asset_task_info(task_data["context"])
+        asset_name, task_name, task_type = get_batch_asset_task_info(
+            task_data["context"])
 
         application_manager = ApplicationManager()
         found_variant_key = find_variant_key(application_manager, host_name)
         app_name = "{}/{}".format(host_name, found_variant_key)
 
         # must have for proper launch of app
         env = get_app_environments_for_context(
-            project,
-            asset,
+            project_name,
+            asset_name,
             task_name,
             app_name
         )
@@ -262,15 +270,22 @@ def remotepublishfromapp(project, batch_path, host_name,
         data = {
             "last_workfile_path": workfile_path,
             "start_last_workfile": True,
-            "project_name": project,
-            "asset_name": asset,
+            "project_name": project_name,
+            "asset_name": asset_name,
             "task_name": task_name
         }
 
         launched_app = application_manager.launch(app_name, **data)
 
+        timeout = get_timeout(project_name, host_name, task_type)
+
+        time_start = time.time()
         while launched_app.poll() is None:
             time.sleep(0.5)
+            if time.time() - time_start > timeout:
+                launched_app.terminate()
+                msg = "Timeout reached"
+                fail_batch(_id, dbcon, msg)
 
     @staticmethod
     def remotepublish(project, batch_path, user_email, targets=None):

openpype/settings/defaults/project_settings/webpublisher.json

@@ -1,4 +1,13 @@
 {
+    "timeout_profiles": [
+        {
+            "hosts": [
+                "photoshop"
+            ],
+            "task_types": [],
+            "timeout": 600
+        }
+    ],
     "publish": {
         "CollectPublishedFiles": {
             "task_type_to_family": {

openpype/settings/entities/schemas/projects_schema/schema_project_webpublisher.json

@@ -5,6 +5,38 @@
     "label": "Web Publisher",
     "is_file": true,
     "children": [
+        {
+            "type": "list",
+            "collapsible": true,
+            "use_label_wrap": true,
+            "key": "timeout_profiles",
+            "label": "Timeout profiles",
+            "object_type": {
+                "type": "dict",
+                "children": [
+                    {
+                        "key": "hosts",
+                        "label": "Host names",
+                        "type": "hosts-enum",
+                        "multiselection": true
+                    },
+                    {
+                        "key": "task_types",
+                        "label": "Task types",
+                        "type": "task-types-enum",
+                        "multiselection": true
+                    },
+                    {
+                        "type": "separator"
+                    },
+                    {
+                        "type": "number",
+                        "key": "timeout",
+                        "label": "Timeout (sec)"
+                    }
+                ]
+            }
+        },
         {
             "type": "dict",
             "collapsible": true,