From 86c4540dedef05f0fd751f79feac5efd86cfbec3 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 5 Jun 2022 11:51:04 +0200 Subject: [PATCH] Remove warnings when starting breeze (#24183) Breeze when started produced three warnings that were harmless, but we should fix them to remove "false positives". (cherry picked from commit ac8a790ab9dabfa747964b65aa9b93451b735ede) --- dev/breeze/src/airflow_breeze/breeze.py | 6 ++++- .../commands/developer_commands.py | 3 ++- .../airflow_breeze/commands/main_command.py | 2 -- .../airflow_breeze/params/build_ci_params.py | 1 + .../params/build_prod_params.py | 3 ++- .../src/airflow_breeze/params/shell_params.py | 5 ++++ .../utils/docker_command_utils.py | 16 +++-------- .../src/airflow_breeze/utils/path_utils.py | 14 ++++++++++ scripts/ci/docker-compose/local.yml | 7 +---- scripts/ci/docker-compose/mypy.yml | 27 +++++++++++++++++++ scripts/ci/pre_commit/pre_commit_flake8.py | 3 ++- .../pre_commit_migration_reference.py | 3 ++- scripts/ci/pre_commit/pre_commit_mypy.py | 3 ++- scripts/ci/pre_commit/pre_commit_ui_lint.py | 3 ++- scripts/ci/pre_commit/pre_commit_www_lint.py | 3 ++- 15 files changed, 71 insertions(+), 28 deletions(-) create mode 100644 scripts/ci/docker-compose/mypy.yml diff --git a/dev/breeze/src/airflow_breeze/breeze.py b/dev/breeze/src/airflow_breeze/breeze.py index a596c2be4967..608619ae8e48 100755 --- a/dev/breeze/src/airflow_breeze/breeze.py +++ b/dev/breeze/src/airflow_breeze/breeze.py @@ -17,9 +17,13 @@ # under the License. from airflow_breeze.configure_rich_click import click # isort: skip # noqa from airflow_breeze.commands.main_command import main -from airflow_breeze.utils.path_utils import find_airflow_sources_root_to_operate_on +from airflow_breeze.utils.path_utils import ( + create_directories_and_files, + find_airflow_sources_root_to_operate_on, +) find_airflow_sources_root_to_operate_on() +create_directories_and_files() if __name__ == '__main__': main() diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 1719bab5fc7a..e10e79d0a812 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -524,7 +524,7 @@ def stop(verbose: bool, dry_run: bool, preserve_volumes: bool): command_to_execute = ['docker-compose', 'down', "--remove-orphans"] if not preserve_volumes: command_to_execute.append("--volumes") - shell_params = ShellParams(verbose=verbose, backend="all") + shell_params = ShellParams(verbose=verbose, backend="all", include_mypy_volume=True) env_variables = get_env_variables_for_docker_commands(shell_params) run_command(command_to_execute, verbose=verbose, dry_run=dry_run, env=env_variables) @@ -579,6 +579,7 @@ def enter_shell(**kwargs) -> RunCommandResult: if read_from_cache_file('suppress_cheatsheet') is None: get_console().print(CHEATSHEET, style=CHEATSHEET_STYLE) enter_shell_params = ShellParams(**filter_out_none(**kwargs)) + enter_shell_params.include_mypy_volume = True rebuild_ci_image_if_needed(build_params=enter_shell_params, dry_run=dry_run, verbose=verbose) return run_shell(verbose, dry_run, enter_shell_params) diff --git a/dev/breeze/src/airflow_breeze/commands/main_command.py b/dev/breeze/src/airflow_breeze/commands/main_command.py index 8ed451b0e507..c62101a44f44 100644 --- a/dev/breeze/src/airflow_breeze/commands/main_command.py +++ b/dev/breeze/src/airflow_breeze/commands/main_command.py @@ -37,7 +37,6 @@ option_use_packages_from_dist, option_verbose, ) -from airflow_breeze.utils.path_utils import create_directories_and_files @click.group(invoke_without_command=True, context_settings={'help_option_names': ['-h', '--help']}) @@ -64,6 +63,5 @@ def main(ctx: click.Context, **kwargs): from airflow_breeze.commands.developer_commands import shell - create_directories_and_files() if not ctx.invoked_subcommand: ctx.forward(shell, extra_args={}) diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index aa9681fc955f..52c242b65a77 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -31,6 +31,7 @@ class BuildCiParams(CommonBuildParams): """ airflow_constraints_mode: str = "constraints-source-providers" + default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_extras: str = "devel_ci" airflow_pre_cached_pip_packages: bool = True diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index e81a5d6739ac..c37569f85f13 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -21,7 +21,7 @@ from dataclasses import dataclass from typing import List -from airflow_breeze.branch_defaults import AIRFLOW_BRANCH +from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import ( AIRFLOW_SOURCES_FROM, AIRFLOW_SOURCES_TO, @@ -41,6 +41,7 @@ class BuildProdParams(CommonBuildParams): """ airflow_constraints_mode: str = "constraints" + default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_constraints_reference: str = "" airflow_is_in_context: bool = False cleanup_context: bool = False diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 58107b9e2756..4ba402265152 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -48,6 +48,7 @@ class ShellParams: """ airflow_branch: str = AIRFLOW_BRANCH + default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_extras: str = "" answer: Optional[str] = None @@ -64,6 +65,7 @@ class ShellParams: github_repository: str = "apache/airflow" github_token: str = os.environ.get('GITHUB_TOKEN', "") image_tag: str = "latest" + include_mypy_volume: bool = False install_airflow_version: str = "" install_providers_from_sources: bool = True integration: Tuple[str, ...] = () @@ -189,6 +191,7 @@ def compose_files(self): local_all_sources_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/local-all-sources.yml" files_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/files.yml" remove_sources_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/remove-sources.yml" + mypy_docker_compose_file = f"{str(SCRIPTS_CI_DIR)}/docker-compose/mypy.yml" forward_credentials_docker_compose_file = ( f"{str(SCRIPTS_CI_DIR)}/docker-compose/forward-credentials.yml" ) @@ -213,6 +216,8 @@ def compose_files(self): compose_ci_file.append(forward_credentials_docker_compose_file) if self.use_airflow_version is not None: compose_ci_file.append(remove_sources_docker_compose_file) + if self.include_mypy_volume: + compose_ci_file.append(mypy_docker_compose_file) if "all" in self.integration: integrations = AVAILABLE_INTEGRATIONS else: diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 05bcc11d6cb8..1f30642edd6b 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -17,7 +17,6 @@ """Various utils to prepare docker and docker compose commands.""" import os import re -import subprocess import sys from copy import deepcopy from random import randint @@ -97,16 +96,6 @@ ] -def create_volume_if_missing(volume_name: str): - res_inspect = run_command(cmd=["docker", "inspect", volume_name], stdout=subprocess.DEVNULL, check=False) - if res_inspect.returncode != 0: - run_command(cmd=["docker", "volume", "create", volume_name], check=True) - - -def create_static_check_volumes(): - create_volume_if_missing("docker-compose_mypy-cache-volume") - - def get_extra_docker_flags(mount_sources: str) -> List[str]: """ Returns extra docker flags based on the type of mounting we want to do for sources. @@ -124,7 +113,7 @@ def get_extra_docker_flags(mount_sources: str) -> List[str]: ["--mount", f'type=bind,src={AIRFLOW_SOURCES_ROOT / src},dst={dst}'] ) extra_docker_flags.extend( - ['--mount', "type=volume,src=docker-compose_mypy-cache-volume,dst=/opt/airflow/.mypy_cache"] + ['--mount', "type=volume,src=mypy-cache-volume,dst=/opt/airflow/.mypy_cache"] ) else: # none extra_docker_flags.extend( @@ -512,6 +501,7 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None: :param env: environment variables to update with missing values if not set. """ set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_MODE', "constraints-source-providers") + set_value_to_default_if_not_set(env, 'AIRFLOW_CONSTRAINTS_REFERENCE', "constraints-source-providers") set_value_to_default_if_not_set(env, 'AIRFLOW_EXTRAS', "") set_value_to_default_if_not_set(env, 'ANSWER', "") set_value_to_default_if_not_set(env, 'BREEZE', "true") @@ -557,7 +547,9 @@ def update_expected_environment_variables(env: Dict[str, str]) -> None: "AIRFLOW_CI_IMAGE": "airflow_image_name", "AIRFLOW_CI_IMAGE_WITH_TAG": "airflow_image_name_with_tag", "AIRFLOW_EXTRAS": "airflow_extras", + "DEFAULT_CONSTRAINTS_BRANCH": "default-constraints-branch", "AIRFLOW_CONSTRAINTS_MODE": "airflow_constraints_mode", + "AIRFLOW_CONSTRAINTS_REFERENCE": "airflow_constraints_reference", "AIRFLOW_IMAGE_KUBERNETES": "airflow_image_kubernetes", "AIRFLOW_PROD_IMAGE": "airflow_image_name", "AIRFLOW_SOURCES": "airflow_sources", diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 759c3e09a7d6..e4e79d55202a 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -19,6 +19,7 @@ """ import hashlib import os +import subprocess import sys import tempfile from functools import lru_cache @@ -249,6 +250,18 @@ def find_airflow_sources_root_to_operate_on() -> Path: BREEZE_SOURCES_ROOT = AIRFLOW_SOURCES_ROOT / "dev" / "breeze" +def create_volume_if_missing(volume_name: str): + from airflow_breeze.utils.run_utils import run_command + + res_inspect = run_command(cmd=["docker", "inspect", volume_name], stdout=subprocess.DEVNULL, check=False) + if res_inspect.returncode != 0: + run_command(cmd=["docker", "volume", "create", volume_name], check=True) + + +def create_static_check_volumes(): + create_volume_if_missing("mypy-cache-volume") + + def create_directories_and_files() -> None: """ Creates all directories and files that are needed for Breeze to work via docker-compose. @@ -264,3 +277,4 @@ def create_directories_and_files() -> None: (AIRFLOW_SOURCES_ROOT / ".bash_aliases").touch() (AIRFLOW_SOURCES_ROOT / ".bash_history").touch() (AIRFLOW_SOURCES_ROOT / ".inputrc").touch() + create_static_check_volumes() diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml index e356cb873611..9e63ffa467d0 100644 --- a/scripts/ci/docker-compose/local.yml +++ b/scripts/ci/docker-compose/local.yml @@ -26,10 +26,7 @@ services: # or those that might be useful to see in the host as output of the # tests (such as logs) volumes: - - type: volume - source: mypy-cache-volume - target: /opt/airflow/.mypy_cache/ - # START automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py + # START automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py - type: bind source: ../../../.bash_aliases target: /root/.bash_aliases @@ -121,5 +118,3 @@ services: source: ../../../metastore_browser target: /opt/airflow/metastore_browser # END automatically generated volumes from NECESSARY_HOST_VOLUMES in docker_command_utils.py -volumes: - mypy-cache-volume: diff --git a/scripts/ci/docker-compose/mypy.yml b/scripts/ci/docker-compose/mypy.yml new file mode 100644 index 000000000000..728773869023 --- /dev/null +++ b/scripts/ci/docker-compose/mypy.yml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +version: "3.7" +services: + airflow: + volumes: + - type: volume + source: mypy-cache-volume + target: /opt/airflow/.mypy_cache/ +volumes: + mypy-cache-volume: + external: true diff --git a/scripts/ci/pre_commit/pre_commit_flake8.py b/scripts/ci/pre_commit/pre_commit_flake8.py index 59bd356d24df..22b09978935b 100755 --- a/scripts/ci/pre_commit/pre_commit_flake8.py +++ b/scripts/ci/pre_commit/pre_commit_flake8.py @@ -34,7 +34,8 @@ if __name__ == '__main__': sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags + from airflow_breeze.utils.path_utils import create_static_check_volumes from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) diff --git a/scripts/ci/pre_commit/pre_commit_migration_reference.py b/scripts/ci/pre_commit/pre_commit_migration_reference.py index 154a795f3e84..25ac796fdf4f 100755 --- a/scripts/ci/pre_commit/pre_commit_migration_reference.py +++ b/scripts/ci/pre_commit/pre_commit_migration_reference.py @@ -34,7 +34,8 @@ if __name__ == '__main__': sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags + from airflow_breeze.utils.path_utils import create_static_check_volumes from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py index 74d511a65b7e..965bed9a74d3 100755 --- a/scripts/ci/pre_commit/pre_commit_mypy.py +++ b/scripts/ci/pre_commit/pre_commit_mypy.py @@ -35,7 +35,8 @@ if __name__ == '__main__': sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags + from airflow_breeze.utils.path_utils import create_static_check_volumes from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) diff --git a/scripts/ci/pre_commit/pre_commit_ui_lint.py b/scripts/ci/pre_commit/pre_commit_ui_lint.py index 7755e3d204bd..5812344a4596 100755 --- a/scripts/ci/pre_commit/pre_commit_ui_lint.py +++ b/scripts/ci/pre_commit/pre_commit_ui_lint.py @@ -33,7 +33,8 @@ if __name__ == '__main__': sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags + from airflow_breeze.utils.path_utils import create_static_check_volumes from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN) diff --git a/scripts/ci/pre_commit/pre_commit_www_lint.py b/scripts/ci/pre_commit/pre_commit_www_lint.py index 65553c3d8134..b22750c56d07 100755 --- a/scripts/ci/pre_commit/pre_commit_www_lint.py +++ b/scripts/ci/pre_commit/pre_commit_www_lint.py @@ -34,7 +34,8 @@ if __name__ == '__main__': sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src")) from airflow_breeze.global_constants import MOUNT_SELECTED - from airflow_breeze.utils.docker_command_utils import create_static_check_volumes, get_extra_docker_flags + from airflow_breeze.utils.docker_command_utils import get_extra_docker_flags + from airflow_breeze.utils.path_utils import create_static_check_volumes from airflow_breeze.utils.run_utils import get_runnable_ci_image, run_command airflow_image = get_runnable_ci_image(verbose=VERBOSE, dry_run=DRY_RUN)