Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(client): runtime build does not dump pip and conda configs by default #2601

Merged
merged 1 commit into from
Aug 7, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 20 additions & 18 deletions client/starwhale/core/runtime/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
DEFAULT_PAGE_IDX,
DEFAULT_PAGE_SIZE,
)
from starwhale.base.type import RuntimeLockFileType
from starwhale.utils.cli import AliasedGroup
from starwhale.base.uri.resource import Resource, ResourceType
from starwhale.core.runtime.model import _SUPPORT_CUDA, _SUPPORT_CUDNN
Expand Down Expand Up @@ -257,16 +256,18 @@ def _quickstart(
hidden=True,
)
@optgroup.option( # type: ignore[no-untyped-call]
"-epo",
"--emit-pip-options",
"-dpo",
"--dump-pip-options",
is_flag=True,
help=f"Emit pip config options when the command dumps {RuntimeLockFileType.VENV}",
show_default=True,
help="Dump pip config options from the ~/.pip/pip.conf file.",
)
@optgroup.option( # type: ignore[no-untyped-call]
"-ecc",
"--emit-condarc",
"-dcc",
"--dump-condarc",
is_flag=True,
help="Emit to dump ~/.condarc from the local machine, default is False",
show_default=True,
help="Dump conda config from the ~/.condarc file",
)
@optgroup.option( # type: ignore[no-untyped-call]
"-ilw",
Expand Down Expand Up @@ -299,8 +300,8 @@ def _build(
shell: bool,
yaml: str,
docker: str,
emit_pip_options: bool,
emit_condarc: bool,
dump_pip_options: bool,
dump_condarc: bool,
tags: t.List[str],
) -> None:
"""Create and build a relocated, shareable, packaged runtime bundle(aka `swrt` file). Support python and native libs.
Expand Down Expand Up @@ -369,8 +370,8 @@ def _build(
download_all_deps=download_all_deps,
include_editable=include_editable,
include_local_wheel=include_local_wheel,
emit_condarc=emit_condarc,
emit_pip_options=emit_pip_options,
dump_condarc=dump_condarc,
dump_pip_options=dump_pip_options,
tags=tags,
)
else:
Expand All @@ -384,8 +385,8 @@ def _build(
include_local_wheel=include_local_wheel,
no_cache=no_cache,
disable_env_lock=disable_env_lock,
emit_condarc=emit_condarc,
emit_pip_options=emit_pip_options,
dump_condarc=dump_condarc,
dump_pip_options=dump_pip_options,
tags=tags,
)

Expand Down Expand Up @@ -717,10 +718,11 @@ def _activate(uri: str, force_restore: bool) -> None:
help="Include local wheel packages",
)
@click.option(
"-epo",
"--emit-pip-options",
"-dpo",
"--dump-pip-options",
is_flag=True,
help=f"Emit pip config options when the command dumps {RuntimeLockFileType.VENV}",
show_default=True,
help="Dump pip config options from the ~/.pip/pip.conf file.",
)
@click.option(
"-nc",
Expand All @@ -738,7 +740,7 @@ def _lock(
stdout: bool,
include_editable: bool,
include_local_wheel: bool,
emit_pip_options: bool,
dump_pip_options: bool,
no_cache: bool,
) -> None:
"""
Expand All @@ -756,7 +758,7 @@ def _lock(
stdout,
include_editable,
include_local_wheel,
emit_pip_options,
dump_pip_options,
env_use_shell,
)

Expand Down
42 changes: 19 additions & 23 deletions client/starwhale/core/runtime/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -759,7 +759,7 @@ def lock(
stdout: bool = False,
include_editable: bool = False,
include_local_wheel: bool = False,
emit_pip_options: bool = False,
dump_pip_options: bool = False,
env_use_shell: bool = False,
) -> t.Tuple[str, t.Optional[Path]]:
return StandaloneRuntime.lock(
Expand All @@ -771,7 +771,7 @@ def lock(
stdout,
include_editable,
include_local_wheel,
emit_pip_options,
dump_pip_options,
env_use_shell,
)

Expand Down Expand Up @@ -807,8 +807,8 @@ def build_from_python_env(
download_all_deps: bool = False,
include_editable: bool = False,
include_local_wheel: bool = False,
emit_pip_options: bool = False,
emit_condarc: bool = False,
dump_pip_options: bool = False,
dump_condarc: bool = False,
tags: t.List[str] | None = None,
) -> None:
raise NotImplementedError
Expand All @@ -825,8 +825,8 @@ def build_from_runtime_yaml(
env_prefix_path: str = "",
env_name: str = "",
env_use_shell: bool = False,
emit_pip_options: bool = False,
emit_condarc: bool = False,
dump_pip_options: bool = False,
dump_condarc: bool = False,
tags: t.List[str] | None = None,
) -> None:
raise NotImplementedError
Expand Down Expand Up @@ -972,8 +972,8 @@ def build_from_python_env(
download_all_deps: bool = False,
include_editable: bool = False,
include_local_wheel: bool = False,
emit_pip_options: bool = False,
emit_condarc: bool = False,
dump_pip_options: bool = False,
dump_condarc: bool = False,
tags: t.List[str] | None = None,
) -> None:
if conda_name or conda_prefix:
Expand Down Expand Up @@ -1029,8 +1029,8 @@ def build_from_python_env(
env_name=conda_name,
env_prefix_path=conda_prefix or venv_prefix,
env_use_shell=env_use_shell,
emit_pip_options=emit_pip_options,
emit_condarc=emit_condarc,
dump_pip_options=dump_pip_options,
dump_condarc=dump_condarc,
tags=tags,
)
finally:
Expand All @@ -1048,8 +1048,8 @@ def build_from_runtime_yaml(
env_prefix_path: str = "",
env_name: str = "",
env_use_shell: bool = False,
emit_pip_options: bool = False,
emit_condarc: bool = False,
dump_pip_options: bool = False,
dump_condarc: bool = False,
tags: t.List[str] | None = None,
) -> None:
StandaloneTag.check_tags_validation(tags)
Expand All @@ -1074,7 +1074,7 @@ def build_from_runtime_yaml(
env_name=env_name,
env_prefix_path=env_prefix_path,
env_use_shell=env_use_shell,
emit_pip_options=emit_pip_options,
dump_pip_options=dump_pip_options,
)
_version_map = {}
_setup_requires_libs = []
Expand Down Expand Up @@ -1118,7 +1118,7 @@ def build_from_runtime_yaml(
self._dump_configs,
5,
"dump configs",
dict(config=swrt_config, emit_condarc=emit_condarc),
dict(config=swrt_config, dump_condarc=dump_condarc),
),
(
self._lock_environment,
Expand Down Expand Up @@ -1180,15 +1180,11 @@ def _prepare_configs(workdir: Path, mode: str) -> None:
# CONDARC env only works in the current process and subprocess by the starwhale.utils.process.check_call function
os.environ["CONDARC"] = str(condarc_path)

def _dump_configs(self, config: RuntimeConfig, emit_condarc: bool = False) -> None:
def _dump_configs(self, config: RuntimeConfig, dump_condarc: bool = False) -> None:
self._manifest["configs"] = config.configs.asdict()

condarc_path = Path.home() / ".condarc"
if (
config.mode == PythonRunEnv.CONDA
and not emit_condarc
and condarc_path.exists()
):
if config.mode == PythonRunEnv.CONDA and dump_condarc and condarc_path.exists():
local_condarc = load_yaml(condarc_path)
local_condarc.update(config.configs.conda.condarc)
self._manifest["configs"]["conda"]["condarc"] = local_condarc
Expand Down Expand Up @@ -1680,7 +1676,7 @@ def lock(
stdout: bool = False,
include_editable: bool = False,
include_local_wheel: bool = False,
emit_pip_options: bool = False,
dump_pip_options: bool = False,
env_use_shell: bool = False,
) -> t.Tuple[str, t.Optional[Path]]:
"""Install dependencies and save or print lock file
Expand All @@ -1694,7 +1690,7 @@ def lock(
stdout: just print the lock info into stdout without saving lock file
include_editable: include the editable pkg (only for venv)
include_local_wheel: include local wheel pkg (only for venv)
emit_pip_options: use user's pip configuration when freeze pkgs (only for venv)
dump_pip_options: use user's pip configuration when freeze pkgs (only for venv)
env_use_shell: automatically detect env in current shell session

Returns:
Expand Down Expand Up @@ -1764,7 +1760,7 @@ def lock(
py_bin=pybin,
lock_fpath=temp_lock_path,
include_editable=include_editable,
emit_options=emit_pip_options,
dump_options=dump_pip_options,
include_local_wheel=include_local_wheel,
)
else:
Expand Down
20 changes: 10 additions & 10 deletions client/starwhale/core/runtime/view.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ def lock(
stdout: bool = False,
include_editable: bool = False,
include_local_wheel: bool = False,
emit_pip_options: bool = False,
dump_pip_options: bool = False,
env_use_shell: bool = False,
) -> None:
Runtime.lock(
Expand All @@ -139,7 +139,7 @@ def lock(
no_cache=no_cache,
include_editable=include_editable,
include_local_wheel=include_local_wheel,
emit_pip_options=emit_pip_options,
dump_pip_options=dump_pip_options,
env_use_shell=env_use_shell,
)

Expand Down Expand Up @@ -171,8 +171,8 @@ def build_from_python_env(
download_all_deps: bool = False,
include_editable: bool = False,
include_local_wheel: bool = False,
emit_condarc: bool = False,
emit_pip_options: bool = False,
dump_condarc: bool = False,
dump_pip_options: bool = False,
tags: t.List[str] | None = None,
) -> Resource:
set_args = list(filter(bool, (conda_name, conda_prefix, venv_prefix)))
Expand Down Expand Up @@ -215,8 +215,8 @@ def build_from_python_env(
download_all_deps=download_all_deps,
include_editable=include_editable,
include_local_wheel=include_local_wheel,
emit_condarc=emit_condarc,
emit_pip_options=emit_pip_options,
dump_condarc=dump_condarc,
dump_pip_options=dump_pip_options,
tags=tags,
)
return runtime_uri
Expand All @@ -234,8 +234,8 @@ def build_from_runtime_yaml(
include_local_wheel: bool = False,
no_cache: bool = False,
disable_env_lock: bool = False,
emit_pip_options: bool = False,
emit_condarc: bool = False,
dump_pip_options: bool = False,
dump_condarc: bool = False,
tags: t.List[str] | None = None,
) -> Resource:
workdir = Path(workdir)
Expand Down Expand Up @@ -267,8 +267,8 @@ def build_from_runtime_yaml(
include_local_wheel=include_local_wheel,
no_cache=no_cache,
disable_env_lock=disable_env_lock,
emit_condarc=emit_condarc,
emit_pip_options=emit_pip_options,
dump_condarc=dump_condarc,
dump_pip_options=dump_pip_options,
tags=tags,
)
return _runtime_uri
Expand Down
8 changes: 4 additions & 4 deletions client/starwhale/utils/venv.py
Original file line number Diff line number Diff line change
Expand Up @@ -281,15 +281,15 @@ def pip_freeze_by_pybin(
py_bin: str,
lock_fpath: t.Union[str, Path],
include_editable: bool = False,
emit_options: bool = False,
dump_options: bool = False,
include_local_wheel: bool = False,
) -> None:
lock_fpath = Path(lock_fpath)
console.info(f"{py_bin}: pip freeze...")

content = [f"# Generated by Starwhale({STARWHALE_VERSION}) Runtime Lock"]

if not emit_options:
if dump_options:
_pip_config = get_user_pip_config_by_pybin(py_bin)
if _pip_config.get("global.index-url"):
content.append(f"--index-url {_pip_config['global.index-url']}")
Expand Down Expand Up @@ -324,7 +324,7 @@ def pip_freeze(
py_env: str,
lock_fpath: t.Union[str, Path],
include_editable: bool = False,
emit_options: bool = False,
dump_options: bool = False,
include_local_wheel: bool = False,
) -> None:
# TODO: add cmd timeout and error log
Expand All @@ -333,7 +333,7 @@ def pip_freeze(
py_bin=_py_bin,
lock_fpath=lock_fpath,
include_editable=include_editable,
emit_options=emit_options,
dump_options=dump_options,
include_local_wheel=include_local_wheel,
)

Expand Down
9 changes: 7 additions & 2 deletions client/tests/core/test_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,8 @@ def test_build_from_conda_name(
conda_name=conda_name,
cuda="11.4",
tags=["from-conda-v0"],
dump_condarc=True,
dump_pip_options=True,
)

assert m_py_ver_bin.call_args[0][0] == os.path.join(conda_prefix, "bin/python3")
Expand Down Expand Up @@ -2227,7 +2229,9 @@ def _patch_pip_freeze(*args: t.Any, **kwargs: t.Any) -> t.Any:
os.environ[ENV_VENV] = venv_dir
content = load_yaml(yaml_path)
assert RuntimeLockFileType.VENV not in content.get("dependencies", [])
StandaloneRuntime.lock(target_dir, yaml_path, env_use_shell=True)
StandaloneRuntime.lock(
target_dir, yaml_path, env_use_shell=True, dump_pip_options=True
)

assert m_call.call_args_list[0][0][0] == [
f"{venv_dir}/bin/pip",
Expand Down Expand Up @@ -2298,6 +2302,7 @@ def _patch_pip_freeze(*args: t.Any, **kwargs: t.Any) -> t.Any:
env_prefix_path=venv_dir,
include_editable=True,
include_local_wheel=True,
dump_pip_options=True,
)
assert m_call.call_args[0][0].startswith(
" ".join(
Expand Down Expand Up @@ -2329,7 +2334,7 @@ def _mock_cli_run(args: t.Any) -> t.Any:
return MagicMock()

m_venv.cli_run = _mock_cli_run
StandaloneRuntime.lock(target_dir, yaml_path)
StandaloneRuntime.lock(target_dir, yaml_path, dump_pip_options=True)

assert m_call.call_count == 4
assert m_output.call_count == 2
Expand Down
Loading