Detect changes in environment-file and update as needed

HISTORY.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - The `prefix` configuration option to specify file system location for the environment.
 
 ### Fixed
+- Update conda env when conda file is updated
 - Fix `cov` script arg covering wrong package.
 
 ## [0.4.1] - 11/07/2023

hatch_conda/plugin.py

@@ -8,9 +8,10 @@
 from contextlib import contextmanager
 from pathlib import Path
 from types import FrameType
-from typing import Callable
+from typing import Any, Callable
 
 import pexpect
+import yaml
 from hatch.env.plugin.interface import EnvironmentInterface
 
 
@@ -47,6 +48,46 @@ def sigwinch_passthrough(sig: int, data: FrameType | None) -> None:
         self.environment.platform.exit_with_code(terminal.exitstatus)
 
 
+def normalize_conda_dict(config: dict[str, str | list | dict[str, Any]]) -> dict[str, str | dict[str, Any]]:
+    """Aims:
+    * remove duplicate entries in lists
+    * equality regardless of entry order in lists
+
+    Rationale:
+    * Removing and recreating environments is expensive
+    * Changing order of dependencies doesn't impact environment configuration
+    """
+    normalized_config: dict[str, str | dict[str, Any]] = {}
+    for key, value in config.items():
+        if isinstance(value, str):
+            normalized_config[key] = value
+        elif isinstance(value, dict):
+            normalized_config[key] = normalize_conda_dict(value)
+        elif isinstance(value, list):
+            list_config: dict[str, str | dict[str, Any]] = {}
+            for item in value:
+                if isinstance(item, str):
+                    if isinstance(list_config.get(item, None), dict):
+                        # we already have this as a key
+                        list_config[item][""] = ""  # type: ignore[index]
+                    else:
+                        list_config[item] = ""
+                elif isinstance(item, dict):
+                    new_dict = normalize_conda_dict(item)
+                    for k, v in new_dict.items():
+                        if isinstance(list_config.get(k, None), str):
+                            # we already have this as an entry
+                            list_config[k] = {"": "", **v}  # type: ignore[dict-item]
+                        else:
+                            list_config[k] = v
+                else:
+                    raise NotImplementedError("Unexpected list in a list for conda config")
+            normalized_config[key] = list_config
+        else:
+            raise NotImplementedError("Unexpected non-str, non-list, non-dict type in conda config")
+    return normalized_config
+
+
 class CondaEnvironment(EnvironmentInterface):
     PLUGIN_NAME = "conda"
 
@@ -60,6 +101,7 @@ def __init__(self, *args, **kwargs):
         self.__python_version = None
 
         self.conda_env_name = f"{self.metadata.core.name}_{self.name}_{self.python_version}"
+        self.conda_contents = {}
         self.project_path = "."
 
         self.shells = ShellManager(self)
@@ -142,19 +184,29 @@ def _get_conda_env_path(self, name: str):
     def find(self):
         return self._get_conda_env_path(self.conda_env_name)
 
-    def create(self):
+    def read_conda_file(self) -> dict[str, Any]:
+        env_file = Path(self.environment_file)
+        if not env_file.exists():
+            return {}
+        with env_file.open() as file:
+            contents = yaml.safe_load(file)
+        normalized_contents = normalize_conda_dict(contents)
+        return normalized_contents
+
+    def conda_env(self, command="create"):
         if not self.environment_file:
-            command = [self.config_command, "create", "-y"]
+            command = [self.config_command, command, "-y"]
             if self.config_conda_forge:
                 command += ["-c", "conda-forge", "--no-channel-priority"]
             command += [
                 f"python={self.python_version}",
                 "pip",
             ]
         elif self.config_command == "micromamba":
-            command = ["micromamba", "create", "-y", "--file", self.environment_file]
+            command = ["micromamba", command, "-y", "--file", self.environment_file]
         else:
-            command = [self.config_command, "env", "create", "--file", self.environment_file]
+            command = [self.config_command, "env", command, "-y", "--file", self.environment_file]
+
         if self.config_prefix is not None:
             command += ["--prefix", self.config_prefix]
         else:
@@ -165,6 +217,10 @@ def create(self):
         else:
             self.platform.check_command_output(command)
         self.apply_env_vars()
+        self.conda_contents = self.read_conda_file()
+
+    def create(self):
+        self.conda_env()
 
     def remove(self):
         command = [self.config_command, "env", "remove", "-y"]
@@ -208,6 +264,9 @@ def install_project_dev_mode(self):
             )
 
     def dependencies_in_sync(self):
+        new_contents = self.read_conda_file()
+        if self.conda_contents != new_contents:
+            return False
         if not self.dependencies:
             return True
         self.apply_env_vars()
@@ -219,6 +278,7 @@ def dependencies_in_sync(self):
             return not process.returncode
 
     def sync_dependencies(self):
+        self.conda_env("update")
         self.apply_env_vars()
         with self:
             self.platform.check_command(self.construct_pip_install_command(self.dependencies))

pyproject.toml

@@ -33,6 +33,7 @@ classifiers = [
 dependencies = [
   "hatch>=1.2.0",
   "pexpect~=4.8",
+  "pyyaml>=5.1.0",
 ]
 dynamic = ["version"]
 

tests/test_config.py

@@ -1,6 +1,11 @@
+import shutil
+from pathlib import Path
+
+import pytest
+import yaml
 from hatch.project.core import Project
 
-from hatch_conda.plugin import CondaEnvironment
+from hatch_conda.plugin import CondaEnvironment, normalize_conda_dict
 
 
 class TestPythonVersion:
@@ -73,3 +78,80 @@ def test_short(self, isolation, data_dir, platform):
         )
 
         assert environment.python_version == "3.10"
+
+
+class TestNormalizeConfig:
+    def test_simple(self):
+        config = {"a": "b", "c": {"d": "e"}}
+        assert normalize_conda_dict(config) == config
+
+    def test_list(self):
+        config = {"a": "b", "c": ["d", "e"]}
+        normalized_config = {"a": "b", "c": {"d": "", "e": ""}}
+        assert normalize_conda_dict(config) == normalized_config
+
+    def test_list_duplicate(self):
+        config = {"a": "b", "c": ["d", "d"]}
+        normalized_config = {"a": "b", "c": {"d": ""}}
+        assert normalize_conda_dict(config) == normalized_config
+
+    def test_nested_list(self):
+        config = {"a": ["b", {"c": "d"}, {"e": ["f", {"g": {"h": ["i"]}}]}]}
+        normalized_config = {"a": {"b": "", "c": "d", "e": {"f": "", "g": {"h": {"i": ""}}}}}
+        assert normalize_conda_dict(config) == normalized_config
+
+    def test_duplicate(self):
+        config = {"a": ["b", {"b": ["c"]}]}
+        normalized_config = {"a": {"b": {"c": "", "": ""}}}
+        assert normalize_conda_dict(config) == normalized_config
+
+
+@pytest.mark.skipif(
+    (shutil.which("conda") or shutil.which("micromamba")) is None,
+    reason="Need at least one of the commands to test env setup",
+)
+class TestDependencyUpdate:
+    @pytest.fixture
+    def environment(self, isolation, data_dir, platform) -> CondaEnvironment:
+        command = "micromamba" if shutil.which("micromamba") else "conda"
+        project = Project(
+            isolation,
+            config={
+                "project": {"name": "my_app", "version": "0.0.1"},
+                "tool": {"hatch": {"envs": {"default": {"environment-file": "test.yaml", "command": command}}}},
+            },
+        )
+        environment = CondaEnvironment(
+            root=isolation,
+            metadata=project.metadata,
+            name="default",
+            config=project.config.envs["default"],
+            matrix_variables={},
+            data_directory=data_dir,
+            isolated_data_directory=None,
+            platform=platform,
+            verbosity=0,
+        )
+
+        # write env file
+        deps = {"name": "test", "channels": ["conda-forge"], "dependencies": ["pip", {"pip": ["pyyaml"]}]}
+        env_file = Path(environment.environment_file)
+        with env_file.open("w") as file:
+            yaml.safe_dump(deps, file)
+
+        return environment
+
+    def test_environment_create(self, environment: CondaEnvironment):
+        environment.create()
+        assert "pyyaml" in environment.conda_contents["dependencies"]["pip"]
+        assert environment.dependencies_in_sync()
+
+        env_file = Path(environment.environment_file)
+        with env_file.open() as file:
+            config = yaml.safe_load(file)
+        pip_deps = [dep["pip"] for dep in config["dependencies"] if isinstance(dep, dict) and "pip" in dep][0]
+        pip_deps.append("hatch")
+        config["dependencies"] = ["pip", {"pip": pip_deps}]
+        with env_file.open("w") as file:
+            yaml.safe_dump(config, file)
+        assert not environment.dependencies_in_sync()