From 4bdedf55d88f4dfdb4ab5c3a541ef4db5b6a8c0e Mon Sep 17 00:00:00 2001 From: tianwei Date: Mon, 7 Aug 2023 18:22:07 +0800 Subject: [PATCH] runtime build does not dump pip and conda configs by default --- client/starwhale/core/runtime/cli.py | 38 ++++++++++++----------- client/starwhale/core/runtime/model.py | 42 ++++++++++++-------------- client/starwhale/core/runtime/view.py | 20 ++++++------ client/starwhale/utils/venv.py | 8 ++--- client/tests/core/test_runtime.py | 9 ++++-- 5 files changed, 60 insertions(+), 57 deletions(-) diff --git a/client/starwhale/core/runtime/cli.py b/client/starwhale/core/runtime/cli.py index 79d69cc0db..efe20ae4de 100644 --- a/client/starwhale/core/runtime/cli.py +++ b/client/starwhale/core/runtime/cli.py @@ -11,7 +11,6 @@ DEFAULT_PAGE_IDX, DEFAULT_PAGE_SIZE, ) -from starwhale.base.type import RuntimeLockFileType from starwhale.utils.cli import AliasedGroup from starwhale.base.uri.resource import Resource, ResourceType from starwhale.core.runtime.model import _SUPPORT_CUDA, _SUPPORT_CUDNN @@ -257,16 +256,18 @@ def _quickstart( hidden=True, ) @optgroup.option( # type: ignore[no-untyped-call] - "-epo", - "--emit-pip-options", + "-dpo", + "--dump-pip-options", is_flag=True, - help=f"Emit pip config options when the command dumps {RuntimeLockFileType.VENV}", + show_default=True, + help="Dump pip config options from the ~/.pip/pip.conf file.", ) @optgroup.option( # type: ignore[no-untyped-call] - "-ecc", - "--emit-condarc", + "-dcc", + "--dump-condarc", is_flag=True, - help="Emit to dump ~/.condarc from the local machine, default is False", + show_default=True, + help="Dump conda config from the ~/.condarc file", ) @optgroup.option( # type: ignore[no-untyped-call] "-ilw", @@ -299,8 +300,8 @@ def _build( shell: bool, yaml: str, docker: str, - emit_pip_options: bool, - emit_condarc: bool, + dump_pip_options: bool, + dump_condarc: bool, tags: t.List[str], ) -> None: """Create and build a relocated, shareable, packaged runtime bundle(aka `swrt` file). Support python and native libs. @@ -369,8 +370,8 @@ def _build( download_all_deps=download_all_deps, include_editable=include_editable, include_local_wheel=include_local_wheel, - emit_condarc=emit_condarc, - emit_pip_options=emit_pip_options, + dump_condarc=dump_condarc, + dump_pip_options=dump_pip_options, tags=tags, ) else: @@ -384,8 +385,8 @@ def _build( include_local_wheel=include_local_wheel, no_cache=no_cache, disable_env_lock=disable_env_lock, - emit_condarc=emit_condarc, - emit_pip_options=emit_pip_options, + dump_condarc=dump_condarc, + dump_pip_options=dump_pip_options, tags=tags, ) @@ -717,10 +718,11 @@ def _activate(uri: str, force_restore: bool) -> None: help="Include local wheel packages", ) @click.option( - "-epo", - "--emit-pip-options", + "-dpo", + "--dump-pip-options", is_flag=True, - help=f"Emit pip config options when the command dumps {RuntimeLockFileType.VENV}", + show_default=True, + help="Dump pip config options from the ~/.pip/pip.conf file.", ) @click.option( "-nc", @@ -738,7 +740,7 @@ def _lock( stdout: bool, include_editable: bool, include_local_wheel: bool, - emit_pip_options: bool, + dump_pip_options: bool, no_cache: bool, ) -> None: """ @@ -756,7 +758,7 @@ def _lock( stdout, include_editable, include_local_wheel, - emit_pip_options, + dump_pip_options, env_use_shell, ) diff --git a/client/starwhale/core/runtime/model.py b/client/starwhale/core/runtime/model.py index 812c4ca704..c21c18e513 100644 --- a/client/starwhale/core/runtime/model.py +++ b/client/starwhale/core/runtime/model.py @@ -759,7 +759,7 @@ def lock( stdout: bool = False, include_editable: bool = False, include_local_wheel: bool = False, - emit_pip_options: bool = False, + dump_pip_options: bool = False, env_use_shell: bool = False, ) -> t.Tuple[str, t.Optional[Path]]: return StandaloneRuntime.lock( @@ -771,7 +771,7 @@ def lock( stdout, include_editable, include_local_wheel, - emit_pip_options, + dump_pip_options, env_use_shell, ) @@ -807,8 +807,8 @@ def build_from_python_env( download_all_deps: bool = False, include_editable: bool = False, include_local_wheel: bool = False, - emit_pip_options: bool = False, - emit_condarc: bool = False, + dump_pip_options: bool = False, + dump_condarc: bool = False, tags: t.List[str] | None = None, ) -> None: raise NotImplementedError @@ -825,8 +825,8 @@ def build_from_runtime_yaml( env_prefix_path: str = "", env_name: str = "", env_use_shell: bool = False, - emit_pip_options: bool = False, - emit_condarc: bool = False, + dump_pip_options: bool = False, + dump_condarc: bool = False, tags: t.List[str] | None = None, ) -> None: raise NotImplementedError @@ -972,8 +972,8 @@ def build_from_python_env( download_all_deps: bool = False, include_editable: bool = False, include_local_wheel: bool = False, - emit_pip_options: bool = False, - emit_condarc: bool = False, + dump_pip_options: bool = False, + dump_condarc: bool = False, tags: t.List[str] | None = None, ) -> None: if conda_name or conda_prefix: @@ -1029,8 +1029,8 @@ def build_from_python_env( env_name=conda_name, env_prefix_path=conda_prefix or venv_prefix, env_use_shell=env_use_shell, - emit_pip_options=emit_pip_options, - emit_condarc=emit_condarc, + dump_pip_options=dump_pip_options, + dump_condarc=dump_condarc, tags=tags, ) finally: @@ -1048,8 +1048,8 @@ def build_from_runtime_yaml( env_prefix_path: str = "", env_name: str = "", env_use_shell: bool = False, - emit_pip_options: bool = False, - emit_condarc: bool = False, + dump_pip_options: bool = False, + dump_condarc: bool = False, tags: t.List[str] | None = None, ) -> None: StandaloneTag.check_tags_validation(tags) @@ -1074,7 +1074,7 @@ def build_from_runtime_yaml( env_name=env_name, env_prefix_path=env_prefix_path, env_use_shell=env_use_shell, - emit_pip_options=emit_pip_options, + dump_pip_options=dump_pip_options, ) _version_map = {} _setup_requires_libs = [] @@ -1118,7 +1118,7 @@ def build_from_runtime_yaml( self._dump_configs, 5, "dump configs", - dict(config=swrt_config, emit_condarc=emit_condarc), + dict(config=swrt_config, dump_condarc=dump_condarc), ), ( self._lock_environment, @@ -1180,15 +1180,11 @@ def _prepare_configs(workdir: Path, mode: str) -> None: # CONDARC env only works in the current process and subprocess by the starwhale.utils.process.check_call function os.environ["CONDARC"] = str(condarc_path) - def _dump_configs(self, config: RuntimeConfig, emit_condarc: bool = False) -> None: + def _dump_configs(self, config: RuntimeConfig, dump_condarc: bool = False) -> None: self._manifest["configs"] = config.configs.asdict() condarc_path = Path.home() / ".condarc" - if ( - config.mode == PythonRunEnv.CONDA - and not emit_condarc - and condarc_path.exists() - ): + if config.mode == PythonRunEnv.CONDA and dump_condarc and condarc_path.exists(): local_condarc = load_yaml(condarc_path) local_condarc.update(config.configs.conda.condarc) self._manifest["configs"]["conda"]["condarc"] = local_condarc @@ -1680,7 +1676,7 @@ def lock( stdout: bool = False, include_editable: bool = False, include_local_wheel: bool = False, - emit_pip_options: bool = False, + dump_pip_options: bool = False, env_use_shell: bool = False, ) -> t.Tuple[str, t.Optional[Path]]: """Install dependencies and save or print lock file @@ -1694,7 +1690,7 @@ def lock( stdout: just print the lock info into stdout without saving lock file include_editable: include the editable pkg (only for venv) include_local_wheel: include local wheel pkg (only for venv) - emit_pip_options: use user's pip configuration when freeze pkgs (only for venv) + dump_pip_options: use user's pip configuration when freeze pkgs (only for venv) env_use_shell: automatically detect env in current shell session Returns: @@ -1764,7 +1760,7 @@ def lock( py_bin=pybin, lock_fpath=temp_lock_path, include_editable=include_editable, - emit_options=emit_pip_options, + dump_options=dump_pip_options, include_local_wheel=include_local_wheel, ) else: diff --git a/client/starwhale/core/runtime/view.py b/client/starwhale/core/runtime/view.py index a32bd27604..eaf46b03f3 100644 --- a/client/starwhale/core/runtime/view.py +++ b/client/starwhale/core/runtime/view.py @@ -127,7 +127,7 @@ def lock( stdout: bool = False, include_editable: bool = False, include_local_wheel: bool = False, - emit_pip_options: bool = False, + dump_pip_options: bool = False, env_use_shell: bool = False, ) -> None: Runtime.lock( @@ -139,7 +139,7 @@ def lock( no_cache=no_cache, include_editable=include_editable, include_local_wheel=include_local_wheel, - emit_pip_options=emit_pip_options, + dump_pip_options=dump_pip_options, env_use_shell=env_use_shell, ) @@ -171,8 +171,8 @@ def build_from_python_env( download_all_deps: bool = False, include_editable: bool = False, include_local_wheel: bool = False, - emit_condarc: bool = False, - emit_pip_options: bool = False, + dump_condarc: bool = False, + dump_pip_options: bool = False, tags: t.List[str] | None = None, ) -> Resource: set_args = list(filter(bool, (conda_name, conda_prefix, venv_prefix))) @@ -215,8 +215,8 @@ def build_from_python_env( download_all_deps=download_all_deps, include_editable=include_editable, include_local_wheel=include_local_wheel, - emit_condarc=emit_condarc, - emit_pip_options=emit_pip_options, + dump_condarc=dump_condarc, + dump_pip_options=dump_pip_options, tags=tags, ) return runtime_uri @@ -234,8 +234,8 @@ def build_from_runtime_yaml( include_local_wheel: bool = False, no_cache: bool = False, disable_env_lock: bool = False, - emit_pip_options: bool = False, - emit_condarc: bool = False, + dump_pip_options: bool = False, + dump_condarc: bool = False, tags: t.List[str] | None = None, ) -> Resource: workdir = Path(workdir) @@ -267,8 +267,8 @@ def build_from_runtime_yaml( include_local_wheel=include_local_wheel, no_cache=no_cache, disable_env_lock=disable_env_lock, - emit_condarc=emit_condarc, - emit_pip_options=emit_pip_options, + dump_condarc=dump_condarc, + dump_pip_options=dump_pip_options, tags=tags, ) return _runtime_uri diff --git a/client/starwhale/utils/venv.py b/client/starwhale/utils/venv.py index 6f4ec452dc..3734b3106c 100644 --- a/client/starwhale/utils/venv.py +++ b/client/starwhale/utils/venv.py @@ -281,7 +281,7 @@ def pip_freeze_by_pybin( py_bin: str, lock_fpath: t.Union[str, Path], include_editable: bool = False, - emit_options: bool = False, + dump_options: bool = False, include_local_wheel: bool = False, ) -> None: lock_fpath = Path(lock_fpath) @@ -289,7 +289,7 @@ def pip_freeze_by_pybin( content = [f"# Generated by Starwhale({STARWHALE_VERSION}) Runtime Lock"] - if not emit_options: + if dump_options: _pip_config = get_user_pip_config_by_pybin(py_bin) if _pip_config.get("global.index-url"): content.append(f"--index-url {_pip_config['global.index-url']}") @@ -324,7 +324,7 @@ def pip_freeze( py_env: str, lock_fpath: t.Union[str, Path], include_editable: bool = False, - emit_options: bool = False, + dump_options: bool = False, include_local_wheel: bool = False, ) -> None: # TODO: add cmd timeout and error log @@ -333,7 +333,7 @@ def pip_freeze( py_bin=_py_bin, lock_fpath=lock_fpath, include_editable=include_editable, - emit_options=emit_options, + dump_options=dump_options, include_local_wheel=include_local_wheel, ) diff --git a/client/tests/core/test_runtime.py b/client/tests/core/test_runtime.py index fd58e5c081..1f3818987e 100644 --- a/client/tests/core/test_runtime.py +++ b/client/tests/core/test_runtime.py @@ -450,6 +450,8 @@ def test_build_from_conda_name( conda_name=conda_name, cuda="11.4", tags=["from-conda-v0"], + dump_condarc=True, + dump_pip_options=True, ) assert m_py_ver_bin.call_args[0][0] == os.path.join(conda_prefix, "bin/python3") @@ -2227,7 +2229,9 @@ def _patch_pip_freeze(*args: t.Any, **kwargs: t.Any) -> t.Any: os.environ[ENV_VENV] = venv_dir content = load_yaml(yaml_path) assert RuntimeLockFileType.VENV not in content.get("dependencies", []) - StandaloneRuntime.lock(target_dir, yaml_path, env_use_shell=True) + StandaloneRuntime.lock( + target_dir, yaml_path, env_use_shell=True, dump_pip_options=True + ) assert m_call.call_args_list[0][0][0] == [ f"{venv_dir}/bin/pip", @@ -2298,6 +2302,7 @@ def _patch_pip_freeze(*args: t.Any, **kwargs: t.Any) -> t.Any: env_prefix_path=venv_dir, include_editable=True, include_local_wheel=True, + dump_pip_options=True, ) assert m_call.call_args[0][0].startswith( " ".join( @@ -2329,7 +2334,7 @@ def _mock_cli_run(args: t.Any) -> t.Any: return MagicMock() m_venv.cli_run = _mock_cli_run - StandaloneRuntime.lock(target_dir, yaml_path) + StandaloneRuntime.lock(target_dir, yaml_path, dump_pip_options=True) assert m_call.call_count == 4 assert m_output.call_count == 2