Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add command line option to specify the wanted package format #5209

Merged
merged 12 commits into from
Nov 11, 2024
5 changes: 3 additions & 2 deletions conda_build/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ def check(
return all(m[0].check_fields() for m in metadata)


# UP007 can be exception can be dropped when Python 3.10 is minimum version.
def build(
recipe_paths_or_metadata: str | os.PathLike | Path | MetaData,
post: bool | None = None,
Expand Down Expand Up @@ -230,8 +231,8 @@ def test(
"""Run tests on either packages (.tar.bz2 or extracted) or recipe folders

For a recipe folder, it renders the recipe enough to know what package to download, and obtains
it from your currently configuured channels."""
from .build import test
it from your currently configured channels."""
from conda_build.build import test

if hasattr(recipedir_or_package_or_metadata, "config"):
config = recipedir_or_package_or_metadata.config
Expand Down
19 changes: 10 additions & 9 deletions conda_build/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@

from . import __version__ as conda_build_version
from . import environ, noarch_python, source, tarcheck, utils
from .config import Config
from .config import CondaPkgFormat, Config
from .create_test import create_all_test_files
from .exceptions import (
BuildScriptException,
Expand Down Expand Up @@ -70,8 +70,6 @@
try_download,
)
from .utils import (
CONDA_PACKAGE_EXTENSION_V1,
CONDA_PACKAGE_EXTENSION_V2,
CONDA_PACKAGE_EXTENSIONS,
env_var,
glob,
Expand Down Expand Up @@ -1898,9 +1896,12 @@ def bundle_conda(
tmp_archives = []
final_outputs = []
cph_kwargs = {}
ext = CONDA_PACKAGE_EXTENSION_V1
if output.get("type") == "conda_v2" or metadata.config.conda_pkg_format == "2":
ext = CONDA_PACKAGE_EXTENSION_V2
ext = CondaPkgFormat.V1.ext
if (
output.get("type") == CondaPkgFormat.V2
or metadata.config.conda_pkg_format == CondaPkgFormat.V2
):
ext = CondaPkgFormat.V2.ext
cph_kwargs["compression_tuple"] = (
".tar.zst",
"zstd",
Expand All @@ -1918,7 +1919,7 @@ def bundle_conda(

# we're done building, perform some checks
for tmp_path in tmp_archives:
if tmp_path.endswith(CONDA_PACKAGE_EXTENSION_V1):
if tmp_path.endswith(CondaPkgFormat.V1.ext):
tarcheck.check_all(tmp_path, metadata.config)
output_filename = os.path.basename(tmp_path)

Expand Down Expand Up @@ -2060,8 +2061,8 @@ def scan_metadata(path):


bundlers = {
"conda": bundle_conda,
"conda_v2": bundle_conda,
CondaPkgFormat.V1: bundle_conda,
CondaPkgFormat.V2: bundle_conda,
"wheel": bundle_wheel,
}

Expand Down
7 changes: 7 additions & 0 deletions conda_build/cli/actions.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
# SPDX-License-Identifier: BSD-3-Clause
import argparse

from ..config import CondaPkgFormat


class KeyValueAction(argparse.Action):
def __call__(self, parser, namespace, items, option_string=None):
Expand All @@ -16,3 +18,8 @@ def __call__(self, parser, namespace, items, option_string=None):
"is already in use by conda-build."
)
getattr(namespace, self.dest)[key] = value


class PackageTypeNormalize(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
setattr(namespace, self.dest, CondaPkgFormat.normalize(values))
48 changes: 40 additions & 8 deletions conda_build/cli/main_build.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@

from .. import api, build, source, utils
from ..config import (
CondaPkgFormat,
conda_pkg_format_default,
get_channel_urls,
get_or_merge_config,
zstd_compression_level_default,
)
from ..utils import LoggingContext
from .actions import KeyValueAction
from .actions import KeyValueAction, PackageTypeNormalize
from .main_render import get_render_parser

try:
Expand All @@ -33,9 +35,12 @@
from conda.cli.conda_argparse import add_parser_channels

if TYPE_CHECKING:
import os
from argparse import ArgumentParser, Namespace
from collections.abc import Sequence

from ..config import Config


def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]:
parser = get_render_parser()
Expand Down Expand Up @@ -85,7 +90,7 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]:
"-t",
"--test",
action="store_true",
help="Test package (assumes package is already built). RECIPE_DIR argument must be a "
help="Test package (assumes package is already built). RECIPE_PATH argument must be a "
"path to built package .tar.bz2 file.",
beeankha marked this conversation as resolved.
Show resolved Hide resolved
)
parser.add_argument(
Expand Down Expand Up @@ -482,12 +487,23 @@ def parse_args(args: Sequence[str] | None) -> tuple[ArgumentParser, Namespace]:
"Do not display value of environment variables specified in build.script_env."
),
)

parser.add_argument(
"--package-format",
dest="conda_pkg_format",
choices=CondaPkgFormat.acceptable(),
action=PackageTypeNormalize,
default=CondaPkgFormat.normalize(
context.conda_build.get("pkg_format", conda_pkg_format_default)
),
help=(
"Choose which package type(s) are outputted. (Accepted inputs tar.bz2 or 1, .conda or 2)"
beeankha marked this conversation as resolved.
Show resolved Hide resolved
),
)
add_parser_channels(parser)

parsed = parser.parse_args(args)
check_recipe(parsed.recipe)
return parser, parsed
return parsed
jezdez marked this conversation as resolved.
Show resolved Hide resolved


def check_recipe(path_list):
Expand All @@ -508,25 +524,41 @@ def check_recipe(path_list):
)


def output_action(recipe, config):
def output_action(recipe: os.PathLike, config: Config):
"""Output the conda package filename which would have been created

:param recipe: Path to recipe or recipe folder
:param config: Config object used for various options
"""
with LoggingContext(logging.CRITICAL + 1):
config.verbose = False
config.debug = False
paths = api.get_output_file_paths(recipe, config=config)
print("\n".join(sorted(paths)))


def source_action(recipe, config):
def source_action(recipe: os.PathLike, config: Config):
"""Get source assets but don't build action.

:param recipe: Path to recipe or recipe folder
:param config: Config object used for various options
"""
metadata = api.render(recipe, config=config)[0][0]
source.provide(metadata)
print("Source tree in:", metadata.config.work_dir)


def test_action(recipe, config):
def test_action(recipe: os.PathLike, config: Config) -> bool:
"""Test a package action

:param recipe: Path to package
:param config: Config object used for various options
:return: True if tests succeed
"""
return api.test(recipe, move_broken=False, config=config)


def check_action(recipe, config):
def check_action(recipe: os.PathLike, config: Config):
return api.check(recipe, config=config)


Expand Down
48 changes: 45 additions & 3 deletions conda_build/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,14 @@
import shutil
import time
from collections import namedtuple
from enum import Enum
from os.path import abspath, expanduser, expandvars, join
from typing import TYPE_CHECKING

from conda.base.constants import (
CONDA_PACKAGE_EXTENSION_V1,
CONDA_PACKAGE_EXTENSION_V2, # noqa: F401
)
from conda.base.context import context
from conda.utils import url_path

Expand Down Expand Up @@ -46,6 +51,42 @@ def set_invocation_time():
set_invocation_time()


class CondaPkgFormat(Enum):
"""Conda Package Format class

Conda Package Version 1 => 'tar.bz2'
Conda Package Version 2 => '.conda'
"""

V1 = CONDA_PACKAGE_EXTENSION_V1
V2 = CONDA_PACKAGE_EXTENSION_V2

@classmethod
def normalize(cls, input) -> CondaPkgFormat:
if isinstance(input, cls):
return input
if not cls.is_acceptable(input):
raise ValueError(
f'{input} is not valid. Acceptable values [1, "1", ".tar.bz2", 2, "2", ".conda"]'
)
if input in (1, "1", "tar.bz2", ".tar.bz2", cls.V1):
return cls.V1
elif input in (2, "2", "conda", ".conda", cls.V2):
return cls.V2

@staticmethod
def acceptable():
return (1, "1", "tar.bz2", ".tar.bz2", 2, "2", "conda", ".conda")

@classmethod
def is_acceptable(cls, value):
return value in cls.acceptable()

@property
def ext(self):
return self.value


# Don't "save" an attribute of this module for later, like build_prefix =
# conda_build.config.config.build_prefix, as that won't reflect any mutated
# changes.
Expand All @@ -60,7 +101,7 @@ def set_invocation_time():
no_rewrite_stdout_env_default = "false"
ignore_verify_codes_default = []
exit_on_verify_error_default = False
conda_pkg_format_default = None
conda_pkg_format_default = CondaPkgFormat.V1
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
conda_pkg_format_default = CondaPkgFormat.V1
conda_pkg_format_default = CondaPkgFormat.V2

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this change should be postponed to 25.1.

zstd_compression_level_default = 19


Expand Down Expand Up @@ -232,10 +273,11 @@ def _get_default_settings():
"zstd_compression_level", zstd_compression_level_default
),
),
# this can be set to different values (currently only 2 means anything) to use package formats
Setting(
"conda_pkg_format",
context.conda_build.get("pkg_format", conda_pkg_format_default),
CondaPkgFormat.normalize(
context.conda_build.get("pkg_format", conda_pkg_format_default)
),
),
Setting("suppress_variables", False),
Setting("build_id_pat", context.conda_build.get("build_id_pat", "{n}_{t}")),
Expand Down
28 changes: 19 additions & 9 deletions conda_build/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
from frozendict import deepfreeze

from . import utils
from .config import Config, get_or_merge_config
from .config import CondaPkgFormat, Config, get_or_merge_config
from .exceptions import (
CondaBuildException,
CondaBuildUserError,
Expand Down Expand Up @@ -2195,9 +2195,9 @@ def extract_single_output_text(
out.get("name", self.name()),
out.get(
"type",
"conda_v2"
if self.config.conda_pkg_format == "2"
else "conda",
CondaPkgFormat.V2
if self.config.conda_pkg_format == CondaPkgFormat.V2
else CondaPkgFormat.V1,
),
)
for out in outputs
Expand Down Expand Up @@ -2277,7 +2277,11 @@ def copy(self: Self) -> MetaData:
new.config = self.config.copy()
new.meta = copy.deepcopy(self.meta)
new.type = getattr(
self, "type", "conda_v2" if self.config.conda_pkg_format == "2" else "conda"
self,
"type",
CondaPkgFormat.V2
if self.config.conda_pkg_format == CondaPkgFormat.V2
else CondaPkgFormat.V1,
)
return new

Expand Down Expand Up @@ -2367,7 +2371,10 @@ def get_output_metadata(self, output):
if output.get("name") == self.name():
output_metadata = self.copy()
output_metadata.type = output.get(
"type", "conda_v2" if self.config.conda_pkg_format == "2" else "conda"
"type",
CondaPkgFormat.V2
if self.config.conda_pkg_format == CondaPkgFormat.V2
else CondaPkgFormat.V1,
)

else:
Expand All @@ -2393,7 +2400,10 @@ def get_output_metadata(self, output):
self.reconcile_metadata_with_output_dict(output_metadata, output)

output_metadata.type = output.get(
"type", "conda_v2" if self.config.conda_pkg_format == "2" else "conda"
"type",
CondaPkgFormat.V2
if self.config.conda_pkg_format == CondaPkgFormat.V2
else CondaPkgFormat.V1,
)

if "name" in output:
Expand Down Expand Up @@ -2603,8 +2613,8 @@ def get_output_metadata_set(

for output_d, m in render_order:
if not output_d.get("type") or output_d["type"] in (
"conda",
"conda_v2",
CondaPkgFormat.V1,
CondaPkgFormat.V2,
):
conda_packages[
m.name(),
Expand Down
16 changes: 8 additions & 8 deletions conda_build/render.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,12 @@
from conda.models.version import VersionOrder

from . import environ, exceptions, source, utils
from .config import CondaPkgFormat
from .exceptions import DependencyNeedsBuildingError
from .index import get_build_index
from .metadata import MetaData, MetaDataTuple, combine_top_level_metadata_with_output
from .utils import (
CONDA_PACKAGE_EXTENSION_V1,
CONDA_PACKAGE_EXTENSION_V2,
package_record_to_requirement,
)
from .variants import (
Expand Down Expand Up @@ -71,21 +71,21 @@ def bldpkg_path(m: MetaData) -> str:
subdir = "noarch" if m.noarch or m.noarch_python else m.config.host_subdir

if not hasattr(m, "type"):
if m.config.conda_pkg_format == "2":
pkg_type = "conda_v2"
if m.config.conda_pkg_format == CondaPkgFormat.V2:
pkg_type = CondaPkgFormat.V2
else:
pkg_type = "conda"
pkg_type = CondaPkgFormat.V1
else:
pkg_type = m.type

# the default case will switch over to conda_v2 at some point
if pkg_type == "conda":
if pkg_type == CondaPkgFormat.V1:
path = join(
m.config.output_folder, subdir, f"{m.dist()}{CONDA_PACKAGE_EXTENSION_V1}"
m.config.output_folder, subdir, f"{m.dist()}{CondaPkgFormat.V1.ext}"
)
elif pkg_type == "conda_v2":
elif pkg_type == CondaPkgFormat.V2:
path = join(
m.config.output_folder, subdir, f"{m.dist()}{CONDA_PACKAGE_EXTENSION_V2}"
m.config.output_folder, subdir, f"{m.dist()}{CondaPkgFormat.V2.ext}"
)
else:
path = (
Expand Down
Loading
Loading