Merge pull request #2424 from pypeclub/feature/OP_2395_env_groups

General: Environment variables groups

openpype/cli.py

@@ -138,7 +138,10 @@ def webpublisherwebserver(debug, executable, upload_dir, host=None, port=None):
 @click.option("--asset", help="Asset name", default=None)
 @click.option("--task", help="Task name", default=None)
 @click.option("--app", help="Application name", default=None)
-def extractenvironments(output_json_path, project, asset, task, app):
+@click.option(
+    "--envgroup", help="Environment group (e.g. \"farm\")", default=None
+)
+def extractenvironments(output_json_path, project, asset, task, app, envgroup):
     """Extract environment variables for entered context to a json file.
 
     Entered output filepath will be created if does not exists.
@@ -149,7 +152,7 @@ def extractenvironments(output_json_path, project, asset, task, app):
     Context options are "project", "asset", "task", "app"
     """
     PypeCommands.extractenvironments(
-        output_json_path, project, asset, task, app
+        output_json_path, project, asset, task, app, envgroup
     )
 
 

openpype/hooks/pre_global_host_data.py

@@ -48,7 +48,7 @@ def execute(self):
             "log": self.log
         })
 
-        prepare_host_environments(temp_data)
+        prepare_host_environments(temp_data, self.launch_context.env_group)
         prepare_context_environments(temp_data)
 
         temp_data.pop("log")

openpype/lib/applications.py

@@ -41,6 +41,97 @@
 
 _logger = None
 
+PLATFORM_NAMES = {"windows", "linux", "darwin"}
+DEFAULT_ENV_SUBGROUP = "standard"
+
+
+def parse_environments(env_data, env_group=None, platform_name=None):
+    """Parse environment values from settings byt group and platfrom.
+
+    Data may contain up to 2 hierarchical levels of dictionaries. At the end
+    of the last level must be string or list. List is joined using platform
+    specific joiner (';' for windows and ':' for linux and mac).
+
+    Hierarchical levels can contain keys for subgroups and platform name.
+    Platform specific values must be always last level of dictionary. Platform
+    names are "windows" (MS Windows), "linux" (any linux distribution) and
+    "darwin" (any MacOS distribution).
+
+    Subgroups are helpers added mainly for standard and on farm usage. Farm
+    may require different environments for e.g. licence related values or
+    plugins. Default subgroup is "standard".
+
+    Examples:
+    ```
+    {
+        # Unchanged value
+        "ENV_KEY1": "value",
+        # Empty values are kept (unset environment variable)
+        "ENV_KEY2": "",
+
+        # Join list values with ':' or ';'
+        "ENV_KEY3": ["value1", "value2"],
+
+        # Environment groups
+        "ENV_KEY4": {
+            "standard": "DEMO_SERVER_URL",
+            "farm": "LICENCE_SERVER_URL"
+        },
+
+        # Platform specific (and only for windows and mac)
+        "ENV_KEY5": {
+            "windows": "windows value",
+            "darwin": ["value 1", "value 2"]
+        },
+
+        # Environment groups and platform combination
+        "ENV_KEY6": {
+            "farm": "FARM_VALUE",
+            "standard": {
+                "windows": ["value1", "value2"],
+                "linux": "value1",
+                "darwin": ""
+            }
+        }
+    }
+    ```
+    """
+    output = {}
+    if not env_data:
+        return output
+
+    if not env_group:
+        env_group = DEFAULT_ENV_SUBGROUP
+
+    if not platform_name:
+        platform_name = platform.system().lower()
+
+    for key, value in env_data.items():
+        if isinstance(value, dict):
+            # Look if any key is platform key
+            #   - expect that represents environment group if does not contain
+            #   platform keys
+            if not PLATFORM_NAMES.intersection(set(value.keys())):
+                # Skip the key if group is not available
+                if env_group not in value:
+                    continue
+                value = value[env_group]
+
+        # Check again if value is dictionary
+        #   - this time there should be only platform keys
+        if isinstance(value, dict):
+            value = value.get(platform_name)
+
+        # Check if value is list and join it's values
+        # QUESTION Should empty values be skipped?
+        if isinstance(value, (list, tuple)):
+            value = os.pathsep.join(value)
+
+        # Set key to output if value is string
+        if isinstance(value, six.string_types):
+            output[key] = value
+    return output
+
 
 def get_logger():
     """Global lib.applications logger getter."""
@@ -705,7 +796,7 @@ class ApplicationLaunchContext:
             preparation to store objects usable in multiple places.
     """
 
-    def __init__(self, application, executable, **data):
+    def __init__(self, application, executable, env_group=None, **data):
         from openpype.modules import ModulesManager
 
         # Application object
@@ -719,6 +810,11 @@ def __init__(self, application, executable, **data):
 
         self.executable = executable
 
+        if env_group is None:
+            env_group = DEFAULT_ENV_SUBGROUP
+
+        self.env_group = env_group
+
         self.data = dict(data)
 
         # subprocess.Popen launch arguments (first argument in constructor)
@@ -1052,7 +1148,7 @@ def __init__(self, data):
 
 
 def get_app_environments_for_context(
-    project_name, asset_name, task_name, app_name, env=None
+    project_name, asset_name, task_name, app_name, env_group=None, env=None
 ):
     """Prepare environment variables by context.
     Args:
@@ -1104,8 +1200,8 @@ def get_app_environments_for_context(
         "env": env
     })
 
-    prepare_host_environments(data)
-    prepare_context_environments(data)
+    prepare_host_environments(data, env_group)
+    prepare_context_environments(data, env_group)
 
     # Discard avalon connection
     dbcon.uninstall()
@@ -1125,7 +1221,7 @@ def _merge_env(env, current_env):
     return result
 
 
-def prepare_host_environments(data, implementation_envs=True):
+def prepare_host_environments(data, env_group=None, implementation_envs=True):
     """Modify launch environments based on launched app and context.
 
     Args:
@@ -1179,7 +1275,7 @@ def prepare_host_environments(data, implementation_envs=True):
             continue
 
         # Choose right platform
-        tool_env = acre.parse(_env_values)
+        tool_env = parse_environments(_env_values, env_group)
         # Merge dictionaries
         env_values = _merge_env(tool_env, env_values)
 
@@ -1211,7 +1307,9 @@ def prepare_host_environments(data, implementation_envs=True):
         data["env"].pop(key, None)
 
 
-def apply_project_environments_value(project_name, env, project_settings=None):
+def apply_project_environments_value(
+    project_name, env, project_settings=None, env_group=None
+):
     """Apply project specific environments on passed environments.
 
     The enviornments are applied on passed `env` argument value so it is not
@@ -1239,14 +1337,15 @@ def apply_project_environments_value(project_name, env, project_settings=None):
 
     env_value = project_settings["global"]["project_environments"]
     if env_value:
+        parsed_value = parse_environments(env_value, env_group)
         env.update(acre.compute(
-            _merge_env(acre.parse(env_value), env),
+            _merge_env(parsed_value, env),
             cleanup=False
         ))
     return env
 
 
-def prepare_context_environments(data):
+def prepare_context_environments(data, env_group=None):
     """Modify launch environemnts with context data for launched host.
 
     Args:
@@ -1276,7 +1375,7 @@ def prepare_context_environments(data):
     data["project_settings"] = project_settings
     # Apply project specific environments on current env value
     apply_project_environments_value(
-        project_name, data["env"], project_settings
+        project_name, data["env"], project_settings, env_group
     )
 
     app = data["app"]

openpype/pype_commands.py

@@ -305,13 +305,16 @@ def remotepublish(project, batch_path, user, targets=None):
         log.info("Publish finished.")
 
     @staticmethod
-    def extractenvironments(output_json_path, project, asset, task, app):
-        env = os.environ.copy()
+    def extractenvironments(
+        output_json_path, project, asset, task, app, env_group
+    ):
         if all((project, asset, task, app)):
             from openpype.api import get_app_environments_for_context
             env = get_app_environments_for_context(
-                project, asset, task, app, env
+                project, asset, task, app, env_group
             )
+        else:
+            env = os.environ.copy()
 
         output_dir = os.path.dirname(output_json_path)
         if not os.path.exists(output_dir):

vendor/deadline/custom/plugins/GlobalJobPreLoad.py

@@ -48,6 +48,7 @@ def inject_openpype_environment(deadlinePlugin):
         add_args['asset'] = job.GetJobEnvironmentKeyValue('AVALON_ASSET')
         add_args['task'] = job.GetJobEnvironmentKeyValue('AVALON_TASK')
         add_args['app'] = job.GetJobEnvironmentKeyValue('AVALON_APP_NAME')
+        add_args["envgroup"] = "farm"
 
         if all(add_args.values()):
             for key, value in add_args.items():