CumulusCI.yml linter based on Pydantic

cumulusci/core/config/project_config.py

@@ -2,6 +2,7 @@
 import json
 import os
 import re
+from io import StringIO
 from pathlib import Path
 from configparser import ConfigParser
 from itertools import chain
@@ -10,7 +11,6 @@
 API_VERSION_RE = re.compile(r"^\d\d+\.0$")
 
 import github3
-import yaml
 
 from cumulusci.core.utils import merge_config
 from cumulusci.core.config import BaseTaskFlowConfig
@@ -98,24 +98,26 @@ def _load_config(self):
             )
 
         # Load the project's yaml config file
-        with open(self.config_project_path, "r", encoding="utf-8") as f_config:
-            project_config = cci_safe_load(f_config)
+        project_config = cci_safe_load(self.config_project_path, logger=self.logger)
 
         if project_config:
             self.config_project.update(project_config)
 
         # Load the local project yaml config file if it exists
         if self.config_project_local_path:
-            with open(
-                self.config_project_local_path, "r", encoding="utf-8"
-            ) as f_local_config:
-                local_config = cci_safe_load(f_local_config)
+            local_config = cci_safe_load(
+                self.config_project_local_path, logger=self.logger
+            )
             if local_config:
                 self.config_project_local.update(local_config)
 
         # merge in any additional yaml that was passed along
         if self.additional_yaml:
-            additional_yaml_config = yaml.safe_load(self.additional_yaml)
+            additional_yaml_config = cci_safe_load(
+                StringIO(self.additional_yaml),
+                self.config_project_path,
+                logger=self.logger,
+            )
             if additional_yaml_config:
                 self.config_additional_yaml.update(additional_yaml_config)
 

cumulusci/core/config/universal_config.py

@@ -1,11 +1,10 @@
 import os
-
-import yaml
 from pathlib import Path
 
 from cumulusci.core.utils import merge_config
 from cumulusci.core.config.project_config import BaseProjectConfig
 from cumulusci.core.config import BaseTaskFlowConfig
+from cumulusci.utils.yaml.cumulusci_yml import cci_safe_load
 
 __location__ = os.path.dirname(os.path.realpath(__file__))
 
@@ -41,6 +40,7 @@ def default_cumulusci_dir():
 
     @property
     def config_global_path(self):
+        """The global config path. Usually ~/.cumulusci/cumulusci.yml"""
         directory = self.cumulusci_config_dir
         if not os.path.exists(directory):
             os.makedirs(directory)
@@ -63,15 +63,12 @@ def _load_config(self):
         if UniversalConfig.config is not None:
             return
 
-        # load the global config
-        with open(self.config_universal_path, "r", encoding="utf-8") as f_config:
-            config = yaml.safe_load(f_config)
-        UniversalConfig.config_universal = config
+        # load the universal config
+        UniversalConfig.config_universal = cci_safe_load(self.config_universal_path)
 
         # Load the local config
         if self.config_global_path:
-            with open(self.config_global_path, "r", encoding="utf-8") as f:
-                config = yaml.safe_load(f)
+            config = cci_safe_load(self.config_global_path)
         else:
             config = {}
         UniversalConfig.config_global = config

cumulusci/utils/fileutils.py

@@ -9,6 +9,8 @@
 
 """Utilities for working with files"""
 
+DataInput = Union[str, IO, Path, "FSResource"]
+
 
 def _get_path_from_stream(stream):
     "Try to infer a name from an open stream"
@@ -21,9 +23,7 @@ def _get_path_from_stream(stream):
 
 
 @contextmanager
-def load_from_source(
-    source: Union[str, IO, Path, "FSResource"]
-) -> ContextManager[Tuple[Text, IO[Text]]]:
+def load_from_source(source: DataInput) -> ContextManager[Tuple[IO[Text], Text]]:
     """Normalize potential data sources into uniform tuple
 
     Take as input a file-like, path-like, or URL-like
@@ -37,14 +37,14 @@ def load_from_source(
     For example:
 
     >>> from yaml import safe_load
-    >>> with load_from_source("cumulusci.yml") as (path, file):
+    >>> with load_from_source("cumulusci.yml") as (file, path):
     ...      print(path)
     ...      print(safe_load(file).keys())
     ...
     cumulusci.yml
     dict_keys(['project', 'tasks', 'flows', 'orgs'])
 
-    >>> with load_from_source('http://www.salesforce.com') as (path, file):
+    >>> with load_from_source('http://www.salesforce.com') as (file, path):
     ...     print(path)
     ...     print(file.read(10).strip())
     ...
@@ -53,7 +53,7 @@ def load_from_source(
 
     >>> from urllib.request import urlopen
     >>> with urlopen("https://www.salesforce.com") as f:
-    ...     with load_from_source(f) as (path, file):
+    ...     with load_from_source(f) as (file, path):
     ...         print(path)
     ...         print(file.read(10).strip())  #doctest: +ELLIPSIS
     ...
@@ -62,7 +62,7 @@ def load_from_source(
 
     >>> from pathlib import Path
     >>> p = Path(".") / "cumulusci.yml"
-    >>> with load_from_source(p) as (path, file):
+    >>> with load_from_source(p) as (file, path):
     ...     print(path)
     ...     print(file.readline().strip())
     ...
@@ -75,20 +75,20 @@ def load_from_source(
         path = _get_path_from_stream(source)
         if not hasattr(source, "encoding"):  # not decoded yet
             source = TextIOWrapper(source, "utf-8")
-        yield path, source
+        yield source, path
     elif hasattr(source, "open"):  # pathlib.Path-like
         with source.open("rt") as f:
             path = str(source)
-            yield path, f
+            yield f, path
     elif "://" in source:  # URL string-like
         url = source
         resp = requests.get(url)
         resp.raise_for_status()
-        yield url, StringIO(resp.text)
+        yield StringIO(resp.text), url
     else:  # path-string-like
         path = source
         with open(path, "rt") as f:
-            yield path, f
+            yield f, path
 
 
 def proxy(funcname):

cumulusci/utils/tests/test_fileutils.py

@@ -36,7 +36,7 @@ def test_docstrings(self, urlopen):
             raise
 
     def test_binary_becomes_string(self):
-        with load_from_source(BytesIO(b"foo")) as (path, file):
+        with load_from_source(BytesIO(b"foo")) as (file, path):
             data = file.read()
             assert isinstance(data, str)
             assert data == "foo"
@@ -45,37 +45,37 @@ def test_writable_file_throws(self):
         with temporary_dir():
             with open("writable", "wt") as t:
                 with pytest.raises(UnsupportedOperation):
-                    with load_from_source(t) as (filename, data):
+                    with load_from_source(t) as (data, filename):
                         pass
 
     @responses.activate
     def test_load_from_url(self):
         html = "<!DOCTYPE HTML ..."
         responses.add("GET", "http://www.salesforce.com", body=html)
-        with load_from_source("http://www.salesforce.com") as (filename, data):
+        with load_from_source("http://www.salesforce.com") as (data, filename):
             assert data.read() == html
 
     def test_load_from_Path(self):
         p = Path(cumulusci.__file__).parent / "cumulusci.yml"
-        with load_from_source(p) as (filename, data):
+        with load_from_source(p) as (data, filename):
             assert "tasks:" in data.read()
 
     def test_load_from_path_string(self):
         p = Path(cumulusci.__file__).parent / "cumulusci.yml"
-        with load_from_source(str(p)) as (filename, data):
+        with load_from_source(str(p)) as (data, filename):
             assert "tasks:" in data.read()
 
     def test_load_from_open_file(self):
         p = Path(cumulusci.__file__).parent / "cumulusci.yml"
         with open(p) as f:
-            with load_from_source(f) as (filename, data):
+            with load_from_source(f) as (data, filename):
                 assert "tasks:" in data.read()
                 assert str(p) == filename
 
     def test_load_from_fs_resource(self):
         p = Path(cumulusci.__file__).parent / "cumulusci.yml"
         with open_fs_resource(p) as p2:
-            with load_from_source(p2) as (filename, data):
+            with load_from_source(p2) as (data, filename):
                 assert "tasks:" in data.read()