Skip to content

Commit

Permalink
Revert4528 (#4628)
Browse files Browse the repository at this point in the history 
* revert #4528

* add changelogs

* Update and rename 4627.yml to 4628.yml
  • Loading branch information
@sapirshuker
sapirshuker authored Oct 29, 2024
1 parent 7a36b00 commit ac66e46
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 236 deletions.
4 changes: 4 additions & 0 deletions .changelog/4628.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
changes:
- description: Reverted support for GAR DockerHub proxy when running in a Gitlab CI environment.
type: internal
pr_number: 4628
146 changes: 4 additions & 142 deletions demisto_sdk/commands/common/docker/dockerhub_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@
from typing import Any, Dict, List, Optional

import dateparser
import google.auth
import requests
from google.auth.transport.requests import Request
from packaging.version import InvalidVersion, Version
from requests.exceptions import ConnectionError, RequestException, Timeout

Expand All @@ -18,8 +16,6 @@
DOCKERHUB_USER = "DOCKERHUB_USER"
DOCKERHUB_PASSWORD = "DOCKERHUB_PASSWORD"
DEFAULT_REPOSITORY = "demisto"
IS_CONTENT_GITLAB_CI = os.getenv("CONTENT_GITLAB_CI")
DOCKER_IO = os.getenv("DOCKER_IO", "")


class DockerHubAuthScope(StrEnum):
Expand Down Expand Up @@ -53,7 +49,7 @@ def __init__(
password: str = "",
verify_ssl: bool = False,
):
self.registry_api_url = get_registry_api_url(registry, self.DEFAULT_REGISTRY)
self.registry_api_url = registry or self.DEFAULT_REGISTRY
self.docker_hub_api_url = docker_hub_api_url or self.DOCKER_HUB_API_BASE_URL
self.username = username or os.getenv(DOCKERHUB_USER, "")
self.password = password or os.getenv(DOCKERHUB_PASSWORD, "")
Expand Down Expand Up @@ -81,18 +77,6 @@ def get_token(
repo: the repository to retrieve the token for.
scope: the scope needed for the repository
"""
if IS_CONTENT_GITLAB_CI:
# If running in a Gitlab CI environment, try using the Google Cloud access token
logger.debug(
"Attempting to use Google Cloud access token for Docker Hub proxy authentication"
)
try:
if gcloud_access_token := get_gcloud_access_token():
logger.debug("returning gcloud_access_token")
return gcloud_access_token
except Exception as e:
logger.error(f"Failed to get gcloud access token: {e}")

if token_metadata := self._docker_hub_auth_tokens.get(f"{repo}:{scope}"):
now = datetime.now()
if expiration_time := dateparser.parse(token_metadata.get("issued_at")):
Expand Down Expand Up @@ -387,10 +371,6 @@ def get_image_tag_metadata(self, docker_image: str, tag: str) -> Dict[str, Any]:
tag: The tag of the docker image
"""
try:
if IS_CONTENT_GITLAB_CI:
image_digest = self.get_image_digest(docker_image, tag=tag)
response = self.get_image_blobs(docker_image, image_digest=image_digest)
return response
return self.do_docker_hub_get_request(
f"/repositories/{docker_image}/tags/{tag}"
)
Expand Down Expand Up @@ -436,12 +416,9 @@ def get_docker_image_tag_creation_date(
tag: The tag of the docker image
"""
response = self.get_image_tag_metadata(docker_image, tag=tag)
if creation_date := response.get("created"):
return datetime.strptime(creation_date, "%Y-%m-%dT%H:%M:%S.%fZ")
else:
return datetime.strptime(
response.get("last_updated", ""), "%Y-%m-%dT%H:%M:%S.%fZ"
)
return datetime.strptime(
response.get("last_updated", ""), "%Y-%m-%dT%H:%M:%S.%fZ"
)

def get_latest_docker_image_tag(self, docker_image: str) -> Version:
"""
Expand Down Expand Up @@ -513,118 +490,3 @@ def get_repository_images_names(self, repo: str = DEFAULT_REPOSITORY) -> List[st
for image_metadata in self.get_repository_images(repo)
if image_metadata.get("name")
]


### Google Artifactory functions ###


def get_dockerhub_artifact_registry_url(base_path: str) -> str:
"""
Parses a DockerHub Google Artifact Registry internal base path into a base url for DockerHub proxy API calls.
See Confluence page: Shared-Services GCP Services - GAR.
Args:
base_path (str): The base path of the Google Artifact Registry in the format 'region-domain/project/repository'.
Returns:
str: The base URL for the DockerHub proxy API calls based on the provided Artifact Registry path.
Example: For base_path 'us-docker.pkg.dev/my-project/my-repo',
the returned URL would be 'https://region-domain-docker.pkg.dev/v2/my-project/my-repo'
Raises:
ValueError: If the input base_path is not in the expected format.
"""
logger.debug(
f"Trying to parse a DockerHub Google Artifact Registry internal base path {base_path=}"
)
# Split base path into region-domain, project, and repository
try:
region_domain, project, repository = base_path.split("/")
except ValueError:
raise ValueError(
"Invalid Artifact Registry path format. Expected format: 'region-domain/project/repository'"
)

# Construct and return the base URL for DockerHub proxy API calls
parsed_dockerhub_proxy_api_url = (
f"https://{region_domain}/v2/{project}/{repository}"
)
logger.debug(
f"Returning parsed DockerHub Google Artifact Registry API: {parsed_dockerhub_proxy_api_url=}"
)
return parsed_dockerhub_proxy_api_url


def get_registry_api_url(registry: str, default_registry: str) -> str:
"""
Determine the appropriate registry API URL based on the environment and provided parameters.
This function checks if the code is running in a GitLab CI environment and if a custom Docker.io URL is set.
If both conditions are true, it uses the custom Docker.io URL. Otherwise, it falls back to the provided
registry or the default registry.
Args:
registry (str): The registry URL provided by the user.
default_registry (str): The default registry URL to use if no custom URL is provided.
Returns:
str: The determined registry API URL to use for Docker operations.
"""
logger.debug(f"dockerhub_client | {IS_CONTENT_GITLAB_CI=}, {DOCKER_IO=}")
if IS_CONTENT_GITLAB_CI and DOCKER_IO:
try:
logger.debug(
"Running in a GitLab CI environment with custom DOCKER_IO environment variable, Trying to prase a DockerHub Google Artifact Registry from DOCKER_IO environment variable."
)
if parsed_dockerhub_proxy_api_url := get_dockerhub_artifact_registry_url(
DOCKER_IO
):
return parsed_dockerhub_proxy_api_url
else:
logger.warning(
"Could not parse a valid API URL from the DOCKER_IO environment variable."
)
except Exception:
logger.exception(
f"Could not parse a valid API URL from the DOCKER_IO environment variable ({DOCKER_IO})"
)
result = registry or default_registry
logger.debug(f"using registry {result}")
return result


def get_gcloud_access_token() -> Optional[str]:
"""
Retrieves a Google Cloud access token using environment credentials.
This method attempts to obtain credentials from the environment and use them
to retrieve a valid Google Cloud access token. If successful, it returns the
token as a string. If unsuccessful, it returns None.
Returns:
str | None: The Google Cloud access token if successful, None otherwise.
Raises:
Exception: Any exception that occurs during the token retrieval process
is caught and logged, but not re-raised.
"""
try:
logger.debug("Trying to retrieve a Google Cloud access token.")
# Automatically obtain credentials from the environment
credentials, project_id = google.auth.default()

# Refresh the token if needed (ensures the token is valid)
credentials.refresh(Request())
# Extract the access token
access_token = credentials.token
if access_token:
logger.debug(
f"Successfully obtained Google Cloud access token, {project_id=}."
)
return access_token
else:
logger.debug("Failed to obtain Google Cloud access token.")
return None
except Exception as e:
logger.debug(f"Failed to get access token: {str(e)}")
return None
99 changes: 5 additions & 94 deletions demisto_sdk/commands/common/docker_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@
from demisto_sdk.commands.common.logger import logger
from demisto_sdk.commands.common.tools import retry

IS_CONTENT_GITLAB_CI = os.getenv("CONTENT_GITLAB_CI")
DOCKER_IO = os.getenv("DOCKER_IO")
DOCKER_CLIENT = None
FILES_SRC_TARGET = List[Tuple[os.PathLike, str]]
# this will be used to determine if the system supports mounts
Expand All @@ -56,34 +54,6 @@ class DockerException(Exception):


def init_global_docker_client(timeout: int = 60, log_prompt: str = ""):
"""
Initialize and return a global Docker client to access and use a local Docker Daemon.
This function initializes a global Docker client if it doesn't exist, or returns the existing one.
It handles different environments, including GitLab CI, and attempts to log in to the Docker registry
if credentials are available.
Args:
timeout (int, optional): The timeout for Docker client operations in seconds. Defaults to 60.
log_prompt (str, optional): A prefix for log messages. Defaults to an empty string.
Returns:
docker.client.DockerClient: An initialized Docker client.
Raises:
docker.errors.DockerException: If initialization fails, likely due to Docker daemon not running.
Behavior:
1. Checks if a global Docker client already exists.
2. If in GitLab CI environment, attempts to create a client using the job environment.
3. If not in GitLab CI or if connecting via Gitlab CI environment fails,
attempts to log in to the Docker registry if credentials are available.
5. Logs various steps and outcomes of the initialization process.
Note:
- The function uses environment variables for Docker credentials.
- It handles both standard and SSH-based Docker connections.
"""
global DOCKER_CLIENT
if DOCKER_CLIENT is None:
if log_prompt:
Expand All @@ -93,28 +63,6 @@ def init_global_docker_client(timeout: int = 60, log_prompt: str = ""):
if ssh_client := os.getenv("DOCKER_SSH_CLIENT") is not None:
logger.debug(f"{log_prompt} - Using ssh client setting: {ssh_client}")
logger.debug(f"{log_prompt} - Using docker mounting: {CAN_MOUNT_FILES}")
try:
if IS_CONTENT_GITLAB_CI:
"""In the case of running in Gitlab CI environment, try to init a docker client from the
job environment to utilize DockerHub API proxy requests (DOCKER_IO)"""
logger.info(
"Gitlab CI use case detected, trying to create docker client from Gitlab CI job environment."
)
DOCKER_CLIENT = docker.from_env()
if DOCKER_CLIENT.ping():
# see https://docker-py.readthedocs.io/en/stable/client.html#docker.client.DockerClient.ping for more information about ping().
logger.info(
"Successfully initialized docker client from Gitlab CI job environment."
)
return DOCKER_CLIENT
else:
logger.warning(
f"{log_prompt} - Failed to init docker client in Gitlab CI use case."
)
except docker.errors.DockerException:
logger.warning(
f"{log_prompt} - Failed to init docker client in CONTENT_GITLAB_CI use case. "
)
try:
DOCKER_CLIENT = docker.from_env(timeout=timeout, use_ssh_client=ssh_client) # type: ignore
except docker.errors.DockerException:
Expand Down Expand Up @@ -142,7 +90,8 @@ def init_global_docker_client(timeout: int = 60, log_prompt: str = ""):

def is_custom_registry():
return (
not IS_CONTENT_GITLAB_CI and DOCKER_REGISTRY_URL != DEFAULT_DOCKER_REGISTRY_URL
not os.getenv("CONTENT_GITLAB_CI")
and DOCKER_REGISTRY_URL != DEFAULT_DOCKER_REGISTRY_URL
)


Expand Down Expand Up @@ -204,33 +153,6 @@ def get_pip_requirements_from_file(requirements_file: Path) -> List[str]:


class DockerBase:
"""
Base class for Docker-related operations in the Demisto SDK.
This class utilizes any environment where a Docker Daemon is initialized,
and provides core functionality for working with Docker containers and images.
Attributes:
tmp_dir_name (tempfile.TemporaryDirectory): Temporary directory for Docker operations.
tmp_dir (Path): Path object for the temporary directory.
installation_scripts (dict): Mapping of container types to installation script paths.
changes (dict): Docker image changes for different container types.
requirements (Path): Path to the requirements.txt file.
_files_to_push_on_installation (List[Tuple[os.PathLike, str]]): Files to be pushed during installation.
Methods:
version(): Get the Docker version.
installation_files(container_type): Get installation files for a specific container type.
pull_image(image): Pull a Docker image.
is_image_available(image): Check if a Docker image is available.
copy_files_container(container, files): Copy files to a Docker container.
create_container(image, command, files_to_push, environment, **kwargs): Create a Docker container.
push_image(image, log_prompt): Push a Docker image to the repository.
create_image(base_image, image, container_type, install_packages, push, log_prompt): Create a new Docker image.
get_image_registry(image): Get the full image name with registry.
get_or_create_test_image(base_image, container_type, python_version, additional_requirements, push, should_pull, log_prompt): Get or create a test Docker image.
"""

def __init__(self):
self.tmp_dir_name = tempfile.TemporaryDirectory(
prefix=os.path.join(os.getcwd(), "tmp")
Expand Down Expand Up @@ -460,13 +382,13 @@ def create_image(
container.commit(
repository=repository, tag=tag, changes=self.changes[container_type]
)
if IS_CONTENT_GITLAB_CI:
if os.getenv("CONTENT_GITLAB_CI"):
container.commit(
repository=repository.replace(f"{DOCKER_REGISTRY_URL}/", ""),
tag=tag,
changes=self.changes[container_type],
)
if push and IS_CONTENT_GITLAB_CI:
if push and os.getenv("CONTENT_GITLAB_CI"):
self.push_image(image, log_prompt=log_prompt)
return image

Expand Down Expand Up @@ -735,17 +657,6 @@ def get_python_version(image: Optional[str]) -> Optional[Version]:
return python_version
logger.debug(f"Could not get python version for {image=} from regex")

if IS_CONTENT_GITLAB_CI:
try:
logger.debug(
f"get python version for {image=} from available docker client"
)
return _get_python_version_from_image_client(image)
except Exception:
logger.debug(
f"Could not get python version for {image=} from available docker client"
)

try:
logger.debug(f"get python version for {image=} from dockerhub api")
return _get_python_version_from_dockerhub_api(image)
Expand Down Expand Up @@ -797,7 +708,7 @@ def _get_python_version_from_dockerhub_api(image: str) -> Version:
raise ValueError(f"Invalid docker image: {image}")
else:
repo, tag = image.split(":")
if IS_CONTENT_GITLAB_CI:
if os.getenv("CONTENT_GITLAB_CI"):
# we need to remove the gitlab prefix, as we query the API
repo = repo.replace(f"{DOCKER_REGISTRY_URL}/", "")
try:
Expand Down

0 comments on commit ac66e46

Please sign in to comment.