diff --git a/.changelog/4628.yml b/.changelog/4628.yml new file mode 100644 index 0000000000..04693124bb --- /dev/null +++ b/.changelog/4628.yml @@ -0,0 +1,4 @@ +changes: +- description: Reverted support for GAR DockerHub proxy when running in a Gitlab CI environment. + type: internal +pr_number: 4628 diff --git a/demisto_sdk/commands/common/docker/dockerhub_client.py b/demisto_sdk/commands/common/docker/dockerhub_client.py index 5c3fce9240..83f84c73fd 100644 --- a/demisto_sdk/commands/common/docker/dockerhub_client.py +++ b/demisto_sdk/commands/common/docker/dockerhub_client.py @@ -4,9 +4,7 @@ from typing import Any, Dict, List, Optional import dateparser -import google.auth import requests -from google.auth.transport.requests import Request from packaging.version import InvalidVersion, Version from requests.exceptions import ConnectionError, RequestException, Timeout @@ -18,8 +16,6 @@ DOCKERHUB_USER = "DOCKERHUB_USER" DOCKERHUB_PASSWORD = "DOCKERHUB_PASSWORD" DEFAULT_REPOSITORY = "demisto" -IS_CONTENT_GITLAB_CI = os.getenv("CONTENT_GITLAB_CI") -DOCKER_IO = os.getenv("DOCKER_IO", "") class DockerHubAuthScope(StrEnum): @@ -53,7 +49,7 @@ def __init__( password: str = "", verify_ssl: bool = False, ): - self.registry_api_url = get_registry_api_url(registry, self.DEFAULT_REGISTRY) + self.registry_api_url = registry or self.DEFAULT_REGISTRY self.docker_hub_api_url = docker_hub_api_url or self.DOCKER_HUB_API_BASE_URL self.username = username or os.getenv(DOCKERHUB_USER, "") self.password = password or os.getenv(DOCKERHUB_PASSWORD, "") @@ -81,18 +77,6 @@ def get_token( repo: the repository to retrieve the token for. scope: the scope needed for the repository """ - if IS_CONTENT_GITLAB_CI: - # If running in a Gitlab CI environment, try using the Google Cloud access token - logger.debug( - "Attempting to use Google Cloud access token for Docker Hub proxy authentication" - ) - try: - if gcloud_access_token := get_gcloud_access_token(): - logger.debug("returning gcloud_access_token") - return gcloud_access_token - except Exception as e: - logger.error(f"Failed to get gcloud access token: {e}") - if token_metadata := self._docker_hub_auth_tokens.get(f"{repo}:{scope}"): now = datetime.now() if expiration_time := dateparser.parse(token_metadata.get("issued_at")): @@ -387,10 +371,6 @@ def get_image_tag_metadata(self, docker_image: str, tag: str) -> Dict[str, Any]: tag: The tag of the docker image """ try: - if IS_CONTENT_GITLAB_CI: - image_digest = self.get_image_digest(docker_image, tag=tag) - response = self.get_image_blobs(docker_image, image_digest=image_digest) - return response return self.do_docker_hub_get_request( f"/repositories/{docker_image}/tags/{tag}" ) @@ -436,12 +416,9 @@ def get_docker_image_tag_creation_date( tag: The tag of the docker image """ response = self.get_image_tag_metadata(docker_image, tag=tag) - if creation_date := response.get("created"): - return datetime.strptime(creation_date, "%Y-%m-%dT%H:%M:%S.%fZ") - else: - return datetime.strptime( - response.get("last_updated", ""), "%Y-%m-%dT%H:%M:%S.%fZ" - ) + return datetime.strptime( + response.get("last_updated", ""), "%Y-%m-%dT%H:%M:%S.%fZ" + ) def get_latest_docker_image_tag(self, docker_image: str) -> Version: """ @@ -513,118 +490,3 @@ def get_repository_images_names(self, repo: str = DEFAULT_REPOSITORY) -> List[st for image_metadata in self.get_repository_images(repo) if image_metadata.get("name") ] - - -### Google Artifactory functions ### - - -def get_dockerhub_artifact_registry_url(base_path: str) -> str: - """ - Parses a DockerHub Google Artifact Registry internal base path into a base url for DockerHub proxy API calls. - See Confluence page: Shared-Services GCP Services - GAR. - - Args: - base_path (str): The base path of the Google Artifact Registry in the format 'region-domain/project/repository'. - - Returns: - str: The base URL for the DockerHub proxy API calls based on the provided Artifact Registry path. - Example: For base_path 'us-docker.pkg.dev/my-project/my-repo', - the returned URL would be 'https://region-domain-docker.pkg.dev/v2/my-project/my-repo' - - Raises: - ValueError: If the input base_path is not in the expected format. - """ - logger.debug( - f"Trying to parse a DockerHub Google Artifact Registry internal base path {base_path=}" - ) - # Split base path into region-domain, project, and repository - try: - region_domain, project, repository = base_path.split("/") - except ValueError: - raise ValueError( - "Invalid Artifact Registry path format. Expected format: 'region-domain/project/repository'" - ) - - # Construct and return the base URL for DockerHub proxy API calls - parsed_dockerhub_proxy_api_url = ( - f"https://{region_domain}/v2/{project}/{repository}" - ) - logger.debug( - f"Returning parsed DockerHub Google Artifact Registry API: {parsed_dockerhub_proxy_api_url=}" - ) - return parsed_dockerhub_proxy_api_url - - -def get_registry_api_url(registry: str, default_registry: str) -> str: - """ - Determine the appropriate registry API URL based on the environment and provided parameters. - - This function checks if the code is running in a GitLab CI environment and if a custom Docker.io URL is set. - If both conditions are true, it uses the custom Docker.io URL. Otherwise, it falls back to the provided - registry or the default registry. - - Args: - registry (str): The registry URL provided by the user. - default_registry (str): The default registry URL to use if no custom URL is provided. - - Returns: - str: The determined registry API URL to use for Docker operations. - """ - logger.debug(f"dockerhub_client | {IS_CONTENT_GITLAB_CI=}, {DOCKER_IO=}") - if IS_CONTENT_GITLAB_CI and DOCKER_IO: - try: - logger.debug( - "Running in a GitLab CI environment with custom DOCKER_IO environment variable, Trying to prase a DockerHub Google Artifact Registry from DOCKER_IO environment variable." - ) - if parsed_dockerhub_proxy_api_url := get_dockerhub_artifact_registry_url( - DOCKER_IO - ): - return parsed_dockerhub_proxy_api_url - else: - logger.warning( - "Could not parse a valid API URL from the DOCKER_IO environment variable." - ) - except Exception: - logger.exception( - f"Could not parse a valid API URL from the DOCKER_IO environment variable ({DOCKER_IO})" - ) - result = registry or default_registry - logger.debug(f"using registry {result}") - return result - - -def get_gcloud_access_token() -> Optional[str]: - """ - Retrieves a Google Cloud access token using environment credentials. - - This method attempts to obtain credentials from the environment and use them - to retrieve a valid Google Cloud access token. If successful, it returns the - token as a string. If unsuccessful, it returns None. - - Returns: - str | None: The Google Cloud access token if successful, None otherwise. - - Raises: - Exception: Any exception that occurs during the token retrieval process - is caught and logged, but not re-raised. - """ - try: - logger.debug("Trying to retrieve a Google Cloud access token.") - # Automatically obtain credentials from the environment - credentials, project_id = google.auth.default() - - # Refresh the token if needed (ensures the token is valid) - credentials.refresh(Request()) - # Extract the access token - access_token = credentials.token - if access_token: - logger.debug( - f"Successfully obtained Google Cloud access token, {project_id=}." - ) - return access_token - else: - logger.debug("Failed to obtain Google Cloud access token.") - return None - except Exception as e: - logger.debug(f"Failed to get access token: {str(e)}") - return None diff --git a/demisto_sdk/commands/common/docker_helper.py b/demisto_sdk/commands/common/docker_helper.py index e8be39f0b0..1a5aae8b5b 100644 --- a/demisto_sdk/commands/common/docker_helper.py +++ b/demisto_sdk/commands/common/docker_helper.py @@ -31,8 +31,6 @@ from demisto_sdk.commands.common.logger import logger from demisto_sdk.commands.common.tools import retry -IS_CONTENT_GITLAB_CI = os.getenv("CONTENT_GITLAB_CI") -DOCKER_IO = os.getenv("DOCKER_IO") DOCKER_CLIENT = None FILES_SRC_TARGET = List[Tuple[os.PathLike, str]] # this will be used to determine if the system supports mounts @@ -56,34 +54,6 @@ class DockerException(Exception): def init_global_docker_client(timeout: int = 60, log_prompt: str = ""): - """ - Initialize and return a global Docker client to access and use a local Docker Daemon. - - This function initializes a global Docker client if it doesn't exist, or returns the existing one. - It handles different environments, including GitLab CI, and attempts to log in to the Docker registry - if credentials are available. - - Args: - timeout (int, optional): The timeout for Docker client operations in seconds. Defaults to 60. - log_prompt (str, optional): A prefix for log messages. Defaults to an empty string. - - Returns: - docker.client.DockerClient: An initialized Docker client. - - Raises: - docker.errors.DockerException: If initialization fails, likely due to Docker daemon not running. - - Behavior: - 1. Checks if a global Docker client already exists. - 2. If in GitLab CI environment, attempts to create a client using the job environment. - 3. If not in GitLab CI or if connecting via Gitlab CI environment fails, - attempts to log in to the Docker registry if credentials are available. - 5. Logs various steps and outcomes of the initialization process. - - Note: - - The function uses environment variables for Docker credentials. - - It handles both standard and SSH-based Docker connections. - """ global DOCKER_CLIENT if DOCKER_CLIENT is None: if log_prompt: @@ -93,28 +63,6 @@ def init_global_docker_client(timeout: int = 60, log_prompt: str = ""): if ssh_client := os.getenv("DOCKER_SSH_CLIENT") is not None: logger.debug(f"{log_prompt} - Using ssh client setting: {ssh_client}") logger.debug(f"{log_prompt} - Using docker mounting: {CAN_MOUNT_FILES}") - try: - if IS_CONTENT_GITLAB_CI: - """In the case of running in Gitlab CI environment, try to init a docker client from the - job environment to utilize DockerHub API proxy requests (DOCKER_IO)""" - logger.info( - "Gitlab CI use case detected, trying to create docker client from Gitlab CI job environment." - ) - DOCKER_CLIENT = docker.from_env() - if DOCKER_CLIENT.ping(): - # see https://docker-py.readthedocs.io/en/stable/client.html#docker.client.DockerClient.ping for more information about ping(). - logger.info( - "Successfully initialized docker client from Gitlab CI job environment." - ) - return DOCKER_CLIENT - else: - logger.warning( - f"{log_prompt} - Failed to init docker client in Gitlab CI use case." - ) - except docker.errors.DockerException: - logger.warning( - f"{log_prompt} - Failed to init docker client in CONTENT_GITLAB_CI use case. " - ) try: DOCKER_CLIENT = docker.from_env(timeout=timeout, use_ssh_client=ssh_client) # type: ignore except docker.errors.DockerException: @@ -142,7 +90,8 @@ def init_global_docker_client(timeout: int = 60, log_prompt: str = ""): def is_custom_registry(): return ( - not IS_CONTENT_GITLAB_CI and DOCKER_REGISTRY_URL != DEFAULT_DOCKER_REGISTRY_URL + not os.getenv("CONTENT_GITLAB_CI") + and DOCKER_REGISTRY_URL != DEFAULT_DOCKER_REGISTRY_URL ) @@ -204,33 +153,6 @@ def get_pip_requirements_from_file(requirements_file: Path) -> List[str]: class DockerBase: - """ - Base class for Docker-related operations in the Demisto SDK. - - This class utilizes any environment where a Docker Daemon is initialized, - and provides core functionality for working with Docker containers and images. - - Attributes: - tmp_dir_name (tempfile.TemporaryDirectory): Temporary directory for Docker operations. - tmp_dir (Path): Path object for the temporary directory. - installation_scripts (dict): Mapping of container types to installation script paths. - changes (dict): Docker image changes for different container types. - requirements (Path): Path to the requirements.txt file. - _files_to_push_on_installation (List[Tuple[os.PathLike, str]]): Files to be pushed during installation. - - Methods: - version(): Get the Docker version. - installation_files(container_type): Get installation files for a specific container type. - pull_image(image): Pull a Docker image. - is_image_available(image): Check if a Docker image is available. - copy_files_container(container, files): Copy files to a Docker container. - create_container(image, command, files_to_push, environment, **kwargs): Create a Docker container. - push_image(image, log_prompt): Push a Docker image to the repository. - create_image(base_image, image, container_type, install_packages, push, log_prompt): Create a new Docker image. - get_image_registry(image): Get the full image name with registry. - get_or_create_test_image(base_image, container_type, python_version, additional_requirements, push, should_pull, log_prompt): Get or create a test Docker image. - """ - def __init__(self): self.tmp_dir_name = tempfile.TemporaryDirectory( prefix=os.path.join(os.getcwd(), "tmp") @@ -460,13 +382,13 @@ def create_image( container.commit( repository=repository, tag=tag, changes=self.changes[container_type] ) - if IS_CONTENT_GITLAB_CI: + if os.getenv("CONTENT_GITLAB_CI"): container.commit( repository=repository.replace(f"{DOCKER_REGISTRY_URL}/", ""), tag=tag, changes=self.changes[container_type], ) - if push and IS_CONTENT_GITLAB_CI: + if push and os.getenv("CONTENT_GITLAB_CI"): self.push_image(image, log_prompt=log_prompt) return image @@ -735,17 +657,6 @@ def get_python_version(image: Optional[str]) -> Optional[Version]: return python_version logger.debug(f"Could not get python version for {image=} from regex") - if IS_CONTENT_GITLAB_CI: - try: - logger.debug( - f"get python version for {image=} from available docker client" - ) - return _get_python_version_from_image_client(image) - except Exception: - logger.debug( - f"Could not get python version for {image=} from available docker client" - ) - try: logger.debug(f"get python version for {image=} from dockerhub api") return _get_python_version_from_dockerhub_api(image) @@ -797,7 +708,7 @@ def _get_python_version_from_dockerhub_api(image: str) -> Version: raise ValueError(f"Invalid docker image: {image}") else: repo, tag = image.split(":") - if IS_CONTENT_GITLAB_CI: + if os.getenv("CONTENT_GITLAB_CI"): # we need to remove the gitlab prefix, as we query the API repo = repo.replace(f"{DOCKER_REGISTRY_URL}/", "") try: