From a7bb9e2393f84f1b4e81506a6df96a9aee9743f4 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 12 Dec 2023 11:52:25 +0100 Subject: [PATCH] Include removed providers when preparing packages (#36160) As of recently, we have a possibility to schedule providers for removal. They should be removed once they got released the last time - with the removal note added to the documentation. We had a special step in the process to generate documentation and build packages, but this is pretty easy to forget it and miss such package when release is prepared. We modify the tooling slightly - removed providers will be included in release preparation even if they are techniclaly suspended and would be excluded (if they were not removed). --- dev/README_RELEASE_PROVIDER_PACKAGES.md | 83 +++---------- .../airflow_breeze/commands/common_options.py | 10 +- .../commands/developer_commands.py | 41 ++++--- .../commands/developer_commands_config.py | 1 + .../commands/release_management_commands.py | 116 ++++++++++-------- .../release_management_commands_config.py | 32 +++-- .../prepare_providers/provider_packages.py | 4 +- .../src/airflow_breeze/utils/packages.py | 33 +++-- images/breeze/output_build-docs.svg | 38 +++--- images/breeze/output_build-docs.txt | 2 +- ...release-management_add-back-references.svg | 42 ++++--- ...release-management_add-back-references.txt | 2 +- ...anagement_clean-old-provider-artifacts.svg | 4 +- ...anagement_clean-old-provider-artifacts.txt | 2 +- ...ement_generate-issue-content-providers.svg | 16 +-- ...ement_generate-issue-content-providers.txt | 2 +- ...agement_prepare-provider-documentation.svg | 56 +++++---- ...agement_prepare-provider-documentation.txt | 2 +- ...e-management_prepare-provider-packages.svg | 46 +++---- ...e-management_prepare-provider-packages.txt | 2 +- ...output_release-management_publish-docs.svg | 82 +++++++------ ...output_release-management_publish-docs.txt | 2 +- ...release-management_release-prod-images.svg | 4 +- ...release-management_release-prod-images.txt | 2 +- 24 files changed, 322 insertions(+), 302 deletions(-) diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md index 4cfbcfa9167a6..b8fe07dda4fb4 100644 --- a/dev/README_RELEASE_PROVIDER_PACKAGES.md +++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md @@ -161,17 +161,7 @@ Details about maintaining the SEMVER version are going to be discussed and imple [the related issue](https://github.com/apache/airflow/issues/11425) ```shell script -breeze release-management prepare-provider-documentation [packages] -``` - -NOTE! When you want to release a provider marked for removal (needed in order to prepare last release of the -provider), documentation for the provider will not be prepared when you prepare documentation for -all providers - you have to specifically use the provider name in a separate command. -For example to prepare documentation for `removed.provider` provider marked for removal you need to run -separately this command: - -```shell script -breeze release-management prepare-provider-documentation removed.provider +breeze release-management prepare-provider-documentation --include-removed-providers [packages] ``` This command will not only prepare documentation but will also help the release manager to review @@ -193,7 +183,7 @@ When you want to regenerate the changes before the release and make sure all cha are updated, run it in non-interactive mode: ```shell script -breeze release-management prepare-provider-documentation --answer yes [packages] +breeze release-management prepare-provider-documentation --include-removed-providers --answer yes [packages] ``` NOTE!! In case you prepare provider's documentation in a branch different than main, you need to manually @@ -202,7 +192,7 @@ For example if you try to build a `cncf.kubernetes` provider that is build from branch should be prepared like this: ```shell script -breeze release-management prepare-provider-documentation \ +breeze release-management prepare-provider-documentation --include-removed-providers \ --base-branch provider-cncf-kubernetes/v4-4 cncf.kubernetes ``` @@ -219,7 +209,7 @@ Regenerate the documentation templates by running the command with * Provider README file used when publishing package in PyPI ```shell script -breeze release-management prepare-provider-documentation --reapply-templates-only +breeze release-management prepare-provider-documentation --include-removed-providers --reapply-templates-only ``` ## Open PR with suggested version releases @@ -253,27 +243,16 @@ rm -rf ${AIRFLOW_REPO_ROOT}/dist/* * Release candidate packages: ```shell script -breeze release-management prepare-provider-packages --package-format both +breeze release-management prepare-provider-packages --include-removed-providers --package-format both ``` if you only build few packages, run: ```shell script -breeze release-management prepare-provider-packages --package-format both PACKAGE PACKAGE .... +breeze release-management prepare-provider-packages --include-removed-providers \ +--package-format both PACKAGE PACKAGE .... ``` - -NOTE! When you want to release a provider marked for removal (needed in order to prepare last release of the -provider), package for the provider will not be prepared when you prepare documentation for -all providers - you have to specifically use the provider name in a separate command. -For example to prepare documentation for `removed.provider` provider marked for removal you need to run -separately this command: - -```shell script -breeze release-management prepare-provider-packages --package-format both removed.provider -``` - - * Sign all your packages ```shell script @@ -334,26 +313,17 @@ this will clean up dist folder before generating the packages, so you will only ```shell script rm -rf ${AIRFLOW_REPO_ROOT}/dist/* -breeze release-management prepare-provider-packages --version-suffix-for-pypi rc1 --package-format both +breeze release-management prepare-provider-packages --include-removed-providers \ + --version-suffix-for-pypi rc1 --package-format both ``` if you only build few packages, run: ```shell script -breeze release-management prepare-provider-packages --version-suffix-for-pypi rc1 --package-format both PACKAGE PACKAGE .... -``` - -NOTE! When you want to release a provider marked for removal (needed in order to prepare last release of the -provider), package for the provider will not be prepared when you prepare documentation for -all providers - you have to specifically use the provider name in a separate command. -For example to prepare documentation for `removed.provider` provider marked for removal you need to run -separately this command: - -```shell script -breeze release-management prepare-provider-packages --package-format both removed.provider +breeze release-management prepare-provider-packages --include-removed-providers \ +--version-suffix-for-pypi rc1 --package-format both PACKAGE PACKAGE .... ``` - * Verify the artifacts that would be uploaded: ```shell script @@ -447,18 +417,6 @@ cd "${AIRFLOW_REPO_ROOT}" breeze build-docs apache-airflow-providers cncf.kubernetes sftp --clean-build ``` - -NOTE! When you want to release a provider marked for removal (needed in order to prepare last release of the -provider), doc for the provider will not be built when you prepare documentation for -all providers - you have to specifically use the provider name in a separate command. -For example to prepare documentation for `removed.provider` provider marked for removal you need to run -separately this command: - -```shell script -breeze build-docs removed.provider -``` - - - Now you can preview the documentation. ```shell script @@ -484,7 +442,8 @@ way faster on multi-cpu machines when you are publishing multiple providers: ```shell script cd "${AIRFLOW_REPO_ROOT}" -breeze release-management publish-docs apache-airflow-providers --package-filter 'apache-airflow-providers-*' \ +breeze release-management publish-docs apache-airflow-providers --include-removed-providers \ + --package-filter 'apache-airflow-providers-*' \ --override-versioned --run-in-parallel breeze release-management add-back-references all-providers @@ -501,18 +460,7 @@ If you have providers as list of provider ids because you just released them you ```shell script cd "${AIRFLOW_REPO_ROOT}" -breeze release-management publish-docs amazon apache.beam google .... -breeze release-management add-back-references all-providers -``` - -NOTE! When you want to release a provider marked for removal (needed in order to prepare last release of the -provider), docs for the provider will not be published when you prepare documentation for -all providers - you have to specifically use the provider name in a separate command. -For example to prepare documentation for `removed.provider` provider marked for removal you need to run -separately this command: - -```shell script -breeze release-management publish-docs removed.provider +breeze release-management publish-docs --include-removed-providers amazon apache.beam google .... breeze release-management add-back-references all-providers ``` @@ -712,7 +660,8 @@ rm -rf dist/* 4) Build the packages using checked out sources ```shell -breeze release-management prepare-provider-packages --package-format both +breeze release-management prepare-provider-packages --include-removed-providers \ +--package-format both --include-removed-providers ``` 5) Switch to the folder where you checked out the SVN dev files diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py b/dev/breeze/src/airflow_breeze/commands/common_options.py index 41652b7b12889..5cb8c771f81c0 100644 --- a/dev/breeze/src/airflow_breeze/commands/common_options.py +++ b/dev/breeze/src/airflow_breeze/commands/common_options.py @@ -73,7 +73,9 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, nargs=-1, required=False, type=NotVerifiedBetterChoice( - get_available_packages(include_non_provider_doc_packages=True, include_all_providers=True) + get_available_packages( + include_non_provider_doc_packages=True, include_all_providers=True, include_removed=True + ) ), ) option_airflow_extras = click.option( @@ -181,6 +183,12 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, envvar="PYTHON_VERSION", help="Python version to update sbom from. (defaults to all historical python versions)", ) +option_include_removed_providers = click.option( + "--include-removed-providers", + help="Whether to include providers that are removed.", + is_flag=True, + envvar="INCLUDE_REMOVED_PROVIDERS", +) option_include_success_outputs = click.option( "--include-success-outputs", help="Whether to include outputs of successful parallel runs (skipped by default).", diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 2e347f73c00c2..a30184ab68b0a 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -44,6 +44,7 @@ option_forward_credentials, option_github_repository, option_image_tag_for_running, + option_include_removed_providers, option_installation_package_format, option_integration, option_max_time, @@ -579,41 +580,43 @@ def start_airflow( @main.command(name="build-docs") -@click.option("-d", "--docs-only", help="Only build documentation.", is_flag=True) -@click.option("-s", "--spellcheck-only", help="Only run spell checking.", is_flag=True) @option_builder -@click.option( - "--package-filter", - help="List of packages to consider. You can use the full names like apache-airflow-providers-, " - "the short hand names or the glob pattern matching the full package name. " - "The list of short hand names can be found in --help output", - type=str, - multiple=True, -) @click.option( "--clean-build", help="Clean inventories of Inter-Sphinx documentation and generated APIs and sphinx artifacts " "before the build - useful for a clean build.", is_flag=True, ) +@click.option("-d", "--docs-only", help="Only build documentation.", is_flag=True) +@option_dry_run +@option_github_repository +@option_include_removed_providers @click.option( "--one-pass-only", help="Builds documentation in one pass only. This is useful for debugging sphinx errors.", is_flag=True, ) -@argument_doc_packages -@option_github_repository +@click.option( + "--package-filter", + help="List of packages to consider. You can use the full names like apache-airflow-providers-, " + "the short hand names or the glob pattern matching the full package name. " + "The list of short hand names can be found in --help output", + type=str, + multiple=True, +) +@click.option("-s", "--spellcheck-only", help="Only run spell checking.", is_flag=True) @option_verbose -@option_dry_run +@argument_doc_packages def build_docs( - doc_packages: tuple[str, ...], - docs_only: bool, - spellcheck_only: bool, builder: str, clean_build: bool, + docs_only: bool, + github_repository: str, + include_removed_providers: bool, one_pass_only: bool, package_filter: tuple[str, ...], - github_repository: str, + spellcheck_only: bool, + doc_packages: tuple[str, ...], ): """ Build documents. @@ -636,7 +639,9 @@ def build_docs( docs_only=docs_only, spellcheck_only=spellcheck_only, one_pass_only=one_pass_only, - short_doc_packages=expand_all_provider_packages(doc_packages), + short_doc_packages=expand_all_provider_packages( + doc_packages, include_removed=include_removed_providers + ), ) cmd = "/opt/airflow/scripts/in_container/run_docs_build.sh " + " ".join( [shlex.quote(arg) for arg in doc_builder.args_doc_builder] diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py index b14516be381f3..13f3d518c3daf 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py @@ -294,6 +294,7 @@ "--clean-build", "--one-pass-only", "--package-filter", + "--include-removed-providers", "--github-repository", "--builder", ], diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index b0eca736a1984..f527ee4b2725e 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -46,6 +46,7 @@ option_github_repository, option_historical_python_version, option_image_tag_for_running, + option_include_removed_providers, option_include_success_outputs, option_installation_package_format, option_mount_sources, @@ -152,7 +153,7 @@ "provider_packages", nargs=-1, required=False, - type=NotVerifiedBetterChoice(get_available_packages()), + type=NotVerifiedBetterChoice(get_available_packages(include_removed=False)), ) option_airflow_site_directory = click.option( "-a", @@ -342,6 +343,16 @@ def provider_action_summary(description: str, message_type: MessageType, package help="Base branch to use as diff for documentation generation (used for releasing from old branch)", ) @option_github_repository +@argument_provider_packages +@option_answer +@option_dry_run +@option_include_removed_providers +@click.option( + "--non-interactive", + is_flag=True, + help="Run in non-interactive mode. Provides random answers to the type of changes and confirms release" + "for providers prepared for release - useful to test the script in non-interactive mode in CI.", +) @click.option( "--only-min-version-update", is_flag=True, @@ -353,24 +364,16 @@ def provider_action_summary(description: str, message_type: MessageType, package help="Only reapply templates, do not bump version. Useful if templates were added" " and you need to regenerate documentation.", ) -@click.option( - "--non-interactive", - is_flag=True, - help="Run in non-interactive mode. Provides random answers to the type of changes and confirms release" - "for providers prepared for release - useful to test the script in non-interactive mode in CI.", -) -@argument_provider_packages @option_verbose -@option_dry_run -@option_answer def prepare_provider_documentation( - github_repository: str, - skip_git_fetch: bool, base_branch: str, - provider_packages: tuple[str], + github_repository: str, + include_removed_providers: bool, + non_interactive: bool, only_min_version_update: bool, + provider_packages: tuple[str], reapply_templates_only: bool, - non_interactive: bool, + skip_git_fetch: bool, ): from airflow_breeze.prepare_providers.provider_documentation import ( PrepareReleaseDocsChangesOnlyException, @@ -387,7 +390,7 @@ def prepare_provider_documentation( fix_ownership_using_docker() cleanup_python_generated_files() if not provider_packages: - provider_packages = get_available_packages() + provider_packages = get_available_packages(include_removed=include_removed_providers) if not skip_git_fetch: run_command(["git", "remote", "rm", "apache-https-for-providers"], check=False, stderr=DEVNULL) @@ -515,24 +518,28 @@ def basic_provider_checks(provider_package_id: str) -> dict[str, Any]: help="Clean dist directory before building packages. Useful when you want to build multiple packages " " in a clean environment", ) +@option_dry_run @option_github_repository +@option_include_removed_providers @argument_provider_packages @option_verbose -@option_dry_run def prepare_provider_packages( - package_format: str, - version_suffix_for_pypi: str, - package_list_file: IO | None, - skip_tag_check: bool, - skip_deleting_generated_files: bool, clean_dist: bool, github_repository: str, + include_removed_providers: bool, + package_format: str, + package_list_file: IO | None, provider_packages: tuple[str, ...], + skip_deleting_generated_files: bool, + skip_tag_check: bool, + version_suffix_for_pypi: str, ): perform_environment_checks() fix_ownership_using_docker() cleanup_python_generated_files() - packages_list = get_packages_list_to_act_on(package_list_file, provider_packages) + packages_list = get_packages_list_to_act_on( + package_list_file, provider_packages, include_removed_providers + ) if not skip_tag_check: run_command(["git", "remote", "rm", "apache-https-for-providers"], check=False, stderr=DEVNULL) make_sure_remote_apache_exists_and_fetch(github_repository=github_repository) @@ -1134,8 +1141,13 @@ def run_publish_docs_in_parallel( name="publish-docs", help="Command to publish generated documentation to airflow-site", ) -@click.option("-s", "--override-versioned", help="Overrides versioned directories.", is_flag=True) +@argument_doc_packages @option_airflow_site_directory +@option_debug_resources +@option_dry_run +@option_include_removed_providers +@option_include_success_outputs +@click.option("-s", "--override-versioned", help="Overrides versioned directories.", is_flag=True) @click.option( "--package-filter", help="List of packages to consider. You can use the full names like apache-airflow-providers-, " @@ -1144,23 +1156,20 @@ def run_publish_docs_in_parallel( type=str, multiple=True, ) -@option_run_in_parallel @option_parallelism -@option_debug_resources -@option_include_success_outputs +@option_run_in_parallel @option_skip_cleanup -@argument_doc_packages @option_verbose -@option_dry_run def publish_docs( - override_versioned: bool, airflow_site_directory: str, + debug_resources: bool, doc_packages: tuple[str, ...], + include_success_outputs: bool, + include_removed_providers: bool, + override_versioned: bool, package_filter: tuple[str, ...], - run_in_parallel: bool, parallelism: int, - debug_resources: bool, - include_success_outputs: bool, + run_in_parallel: bool, skip_cleanup: bool, ): """Publishes documentation to airflow-site.""" @@ -1171,7 +1180,8 @@ def publish_docs( ) current_packages = find_matching_long_package_names( - short_packages=expand_all_provider_packages(doc_packages), filters=package_filter + short_packages=expand_all_provider_packages(doc_packages, include_removed=include_removed_providers), + filters=package_filter, ) print(f"Publishing docs for {len(current_packages)} package(s)") for pkg in current_packages: @@ -1199,11 +1209,13 @@ def publish_docs( help="Command to add back references for documentation to make it backward compatible.", ) @option_airflow_site_directory +@option_include_removed_providers @argument_doc_packages -@option_verbose @option_dry_run +@option_verbose def add_back_references( airflow_site_directory: str, + include_removed_providers: bool, doc_packages: tuple[str, ...], ): """Adds back references for documentation generated by build-docs and publish-docs""" @@ -1219,7 +1231,9 @@ def add_back_references( "\n[error]You need to specify at least one package to generate back references for\n" ) sys.exit(1) - start_generating_back_references(site_path, list(expand_all_provider_packages(doc_packages))) + start_generating_back_references( + site_path, list(expand_all_provider_packages(doc_packages, include_removed=include_removed_providers)) + ) def _add_chicken_egg_providers_to_build_args( @@ -1239,8 +1253,8 @@ def _add_chicken_egg_providers_to_build_args( help="Cleans the old provider artifacts", ) @option_directory -@option_verbose @option_dry_run +@option_verbose def clean_old_provider_artifacts( directory: str, ): @@ -1288,17 +1302,14 @@ def clean_old_provider_artifacts( name="release-prod-images", help="Release production images to DockerHub (needs DockerHub permissions)." ) @click.option("--airflow-version", required=True, help="Airflow version to release (2.3.0, 2.3.0rc1 etc.)") +@option_dry_run +@option_chicken_egg_providers @click.option( "--dockerhub-repo", default=APACHE_AIRFLOW_GITHUB_REPOSITORY, show_default=True, help="DockerHub repository for the images", ) -@click.option( - "--slim-images", - is_flag=True, - help="Whether to prepare slim images instead of the regular ones.", -) @click.option( "--limit-python", type=BetterChoice(CURRENT_PYTHON_MAJOR_MINOR_VERSIONS), @@ -1312,6 +1323,7 @@ def clean_old_provider_artifacts( show_default=True, help="Specific platform to build images for (if not specified, multiplatform images will be built.", ) +@option_commit_sha @click.option( "--skip-latest", is_flag=True, @@ -1319,10 +1331,12 @@ def clean_old_provider_artifacts( "This should only be used if you release image for previous branches. Automatically set when " "rc/alpha/beta images are built.", ) -@option_commit_sha -@option_chicken_egg_providers +@click.option( + "--slim-images", + is_flag=True, + help="Whether to prepare slim images instead of the regular ones.", +) @option_verbose -@option_dry_run def release_prod_images( airflow_version: str, dockerhub_repo: str, @@ -1515,6 +1529,8 @@ def get_prs_for_package(provider_id: str) -> list[int]: @release_management.command( name="generate-issue-content-providers", help="Generates content for issue to test the release." ) +@click.option("--disable-progress", is_flag=True, help="Disable progress bar") +@click.option("--excluded-pr-list", type=str, help="Coma-separated list of PRs to exclude from the issue.") @click.option( "--github-token", envvar="GITHUB_TOKEN", @@ -1526,22 +1542,20 @@ def get_prs_for_package(provider_id: str) -> list[int]: https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status""" ), ) -@click.option("--suffix", default="rc1", help="Suffix to add to the version prepared") @click.option( "--only-available-in-dist", is_flag=True, help="Only consider package ids with packages prepared in the dist folder", ) -@click.option("--excluded-pr-list", type=str, help="Coma-separated list of PRs to exclude from the issue.") -@click.option("--disable-progress", is_flag=True, help="Disable progress bar") +@click.option("--suffix", default="rc1", help="Suffix to add to the version prepared") @argument_provider_packages def generate_issue_content_providers( - provider_packages: list[str], + disable_progress: bool, + excluded_pr_list: str, github_token: str, - suffix: str, only_available_in_dist: bool, - excluded_pr_list: str, - disable_progress: bool, + suffix: str, + provider_packages: list[str], ): import jinja2 import yaml diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index bf03085e992f8..1e240231596e8 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -130,13 +130,14 @@ { "name": "Package flags", "options": [ - "--package-format", - "--version-suffix-for-pypi", "--clean-dist", - "--skip-tag-check", - "--skip-deleting-generated-files", - "--package-list-file", "--github-repository", + "--include-removed-providers", + "--package-format", + "--package-list-file", + "--skip-deleting-generated-files", + "--skip-tag-check", + "--version-suffix-for-pypi", ], } ], @@ -144,12 +145,13 @@ { "name": "Provider documentation preparation flags", "options": [ - "--github-repository", - "--skip-git-fetch", "--base-branch", + "--github-repository", + "--include-removed-providers", + "--non-interactive", "--only-min-version-update", "--reapply-templates-only", - "--non-interactive", + "--skip-git-fetch", ], } ], @@ -194,9 +196,10 @@ { "name": "Publish Docs", "options": [ + "--airflow-site-directory", + "--include-removed-providers", "--override-versioned", "--package-filter", - "--airflow-site-directory", ], }, { @@ -213,18 +216,21 @@ "breeze release-management add-back-references": [ { "name": "Add Back References to Docs", - "options": ["--airflow-site-directory"], + "options": [ + "--airflow-site-directory", + "--include-removed-providers", + ], }, ], "breeze release-management generate-issue-content-providers": [ { "name": "Generate issue content flags", "options": [ + "--disable-progress", + "--excluded-pr-list", "--github-token", - "--suffix", "--only-available-in-dist", - "--excluded-pr-list", - "--disable-progress", + "--suffix", ], } ], diff --git a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py index 2f3b7eface50c..b9b79f3654b27 100644 --- a/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py +++ b/dev/breeze/src/airflow_breeze/prepare_providers/provider_packages.py @@ -228,7 +228,7 @@ def move_built_packages_and_cleanup( def get_packages_list_to_act_on( - package_list_file: IO | None, provider_packages: tuple[str, ...] + package_list_file: IO | None, provider_packages: tuple[str, ...], include_removed: bool = False ) -> list[str]: if package_list_file and provider_packages: get_console().print( @@ -244,4 +244,4 @@ def get_packages_list_to_act_on( ] elif provider_packages: return list(provider_packages) - return get_available_packages() + return get_available_packages(include_removed=include_removed) diff --git a/dev/breeze/src/airflow_breeze/utils/packages.py b/dev/breeze/src/airflow_breeze/utils/packages.py index 8e734d3d407a4..63c1f2572850b 100644 --- a/dev/breeze/src/airflow_breeze/utils/packages.py +++ b/dev/breeze/src/airflow_breeze/utils/packages.py @@ -132,16 +132,22 @@ def from_requirement(cls, requirement_string: str) -> PipRequirements: def refresh_provider_metadata_from_yaml_file(provider_yaml_path: Path): - import jsonschema import yaml schema = _load_schema() with open(provider_yaml_path) as yaml_file: provider = yaml.safe_load(yaml_file) try: - jsonschema.validate(provider, schema=schema) - except jsonschema.ValidationError: - raise Exception(f"Unable to parse: {provider_yaml_path}.") + import jsonschema + + try: + jsonschema.validate(provider, schema=schema) + except jsonschema.ValidationError: + raise Exception(f"Unable to parse: {provider_yaml_path}.") + except ImportError: + # we only validate the schema if jsonschema is available. This is needed for autocomplete + # to not fail with import error if jsonschema is not installed + pass PROVIDER_METADATA[get_short_package_name(provider["package-name"])] = provider @@ -229,12 +235,17 @@ def get_provider_requirements(provider_id: str) -> list[str]: @lru_cache def get_available_packages( - include_non_provider_doc_packages: bool = False, include_all_providers: bool = False + include_non_provider_doc_packages: bool = False, + include_all_providers: bool = False, + include_suspended: bool = False, + include_removed: bool = False, ) -> list[str]: """ Return provider ids for all packages that are available currently (not suspended). :rtype: object + :param include_suspended: whether the suspended packages should be included + :param include_removed: whether the removed packages should be included :param include_non_provider_doc_packages: whether the non-provider doc packages should be included (packages like apache-airflow, helm-chart, docker-stack) :param include_all_providers: whether "all-providers" should be included ni the list. @@ -247,14 +258,20 @@ def get_available_packages( if include_all_providers: available_packages.append("all-providers") available_packages.extend(provider_ids) - return available_packages + if include_suspended: + available_packages.extend(get_suspended_provider_ids()) + if include_removed: + available_packages.extend(get_removed_provider_ids()) + return sorted(set(available_packages)) -def expand_all_provider_packages(short_doc_packages: tuple[str, ...]) -> tuple[str, ...]: +def expand_all_provider_packages( + short_doc_packages: tuple[str, ...], include_removed: bool = False +) -> tuple[str, ...]: """In case there are "all-providers" in the list, expand the list with all providers.""" if "all-providers" in short_doc_packages: packages = [package for package in short_doc_packages if package != "all-providers"] - packages.extend(get_available_packages()) + packages.extend(get_available_packages(include_removed=include_removed)) short_doc_packages = tuple(set(packages)) return short_doc_packages diff --git a/images/breeze/output_build-docs.svg b/images/breeze/output_build-docs.svg index 8c5faa95ad8ca..9136e1258aa22 100644 --- a/images/breeze/output_build-docs.svg +++ b/images/breeze/output_build-docs.svg @@ -1,4 +1,4 @@ - +