#47: Make release timeout configurable

exasol_script_languages_container_ci_setup/cli/commands/start_ci_build.py

@@ -17,13 +17,20 @@
               help="""The project name. Must be same name as used for the AWS CodeBuild release stack creation.""")
 @click.option('--branch', type=str, required=True,
               help="""The branch of the repository which will be used(e.g. refs/heads/master).""")
+@click.option('--timeout-in-seconds', type=int, required=False, default=180,
+              help="""Time we wait for the build to finish.""")
 def start_ci_build(
         aws_profile: Optional[str],
         log_level: str,
         project: str,
-        branch: str):
+        branch: str,
+        timeout_in_seconds: int
+):
     """
     This command can be used to trigger the AWS CI CodeBuild locally (not by Github Webhook).
     """
     set_log_level(log_level)
-    run_start_ci_build(AwsAccess(aws_profile), project, branch)
+    run_start_ci_build(aws_access=AwsAccess(aws_profile),
+                       project=project,
+                       branch=branch,
+                       timeout_in_seconds=timeout_in_seconds)

exasol_script_languages_container_ci_setup/cli/commands/start_release_build.py

@@ -5,9 +5,9 @@
 
 from exasol_script_languages_container_ci_setup.cli.cli import cli
 from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
 from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess
-from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 from exasol_script_languages_container_ci_setup.lib.run_start_build import run_start_release_build
 
 
@@ -20,15 +20,28 @@
               help="""The URL of the Github release where artifacts will be stored.""")
 @click.option('--branch', type=str, required=True,
               help="""The branch of the repository which will be used.""")
+@click.option('--timeout-in-seconds', type=int, required=False,
+              help="""Time to wait for the release, anymore.""")
+@click.option('--config-file', type=click.Path(file_okay=True, dir_okay=False, exists=True), required=False,
+              help="Configuration file for build (project specific).")
 def start_release_build(
         aws_profile: Optional[str],
         log_level: str,
         project: str,
         upload_url: str,
-        branch: str):
+        branch: str,
+        timeout_in_seconds: Optional[int],
+        config_file: Optional[str]
+):
     """
     This command  triggers the AWS release Codebuild to upload the
     release artifacts onto the given Github release, indicated by parameter 'upload_url'.
     """
     set_log_level(log_level)
-    run_start_release_build(AwsAccess(aws_profile), project, upload_url, branch, os.getenv("GITHUB_TOKEN"))
+    run_start_release_build(aws_access=AwsAccess(aws_profile),
+                            project=project,
+                            upload_url=upload_url,
+                            branch=branch,
+                            gh_token=os.getenv("GITHUB_TOKEN"),
+                            timeout_in_seconds=timeout_in_seconds,
+                            config_file_path=config_file)

exasol_script_languages_container_ci_setup/cli/commands/start_test_release_build.py

@@ -5,9 +5,9 @@
 
 from exasol_script_languages_container_ci_setup.cli.cli import cli
 from exasol_script_languages_container_ci_setup.cli.common import add_options
+from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 from exasol_script_languages_container_ci_setup.cli.options.logging import logging_options, set_log_level
 from exasol_script_languages_container_ci_setup.lib.aws.aws_access import AwsAccess
-from exasol_script_languages_container_ci_setup.cli.options.aws_options import aws_options
 from exasol_script_languages_container_ci_setup.lib.github_draft_release_creator import GithubDraftReleaseCreator
 from exasol_script_languages_container_ci_setup.lib.run_start_build import run_start_test_release_build
 
@@ -24,18 +24,33 @@
               help="""The branch for which the test release should be created.""")
 @click.option('--release-title', type=str, required=True,
               help="""The title of the Github draft release which will be created.""")
+@click.option('--timeout-in-seconds', type=int, required=False,
+              help="""Time to wait for the release, anymore.""")
+@click.option('--config-file', type=click.Path(file_okay=True, dir_okay=False, exists=True),
+              help="Configuration file for build (project specific).")
 def start_test_release_build(
         aws_profile: Optional[str],
         log_level: str,
         repo_name: str,
         project: str,
         branch: str,
-        release_title: str
+        release_title: str,
+        timeout_in_seconds: Optional[str],
+        config_file: Optional[str]
 ):
     """
     This command creates a release draft on Github and triggers the AWS release Codebuild to upload the
     release artifacts onto the new Github release.
     """
     set_log_level(log_level)
-    run_start_test_release_build(AwsAccess(aws_profile), GithubDraftReleaseCreator(),
-                                 repo_name, project, branch, release_title, os.getenv("GITHUB_TOKEN"))
+    run_start_test_release_build(
+        aws_access=AwsAccess(aws_profile),
+        gh_release_creator=GithubDraftReleaseCreator(),
+        repo_name=repo_name,
+        project=project,
+        branch=branch,
+        release_title=release_title,
+        gh_token=os.getenv("GITHUB_TOKEN"),
+        timeout_in_seconds=timeout_in_seconds,
+        config_file_path=config_file
+    )

exasol_script_languages_container_ci_setup/lib/aws/aws_access.py

@@ -108,7 +108,9 @@ def start_codebuild(self,
                         project: PhysicalResourceId,
                         environment_variables_overrides: List[Dict[str, str]],
                         branch: str,
-                        poll_interval_seconds: int = 30) -> None:
+                        timeout_in_seconds: int,
+                        poll_interval_seconds: int = 30,
+                        sleep_function: Callable[[float], None] = time.sleep) -> None:
         """
         This functions uses Boto3 to start a batch build.
         It forwards all variables from parameter env_variables as environment variables to the CodeBuild project.
@@ -131,10 +133,8 @@ def wait_for(seconds: int, interval: int) -> Iterable[int]:
 
         build_id = build_batch.id
         logging.debug(f"Codebuild for project {project} with branch {branch} triggered. Id is {build_id}.")
-        interval = 30
-        timeout_time_in_seconds = 60 * 60 * 2  # We wait for maximal 2h + (something)
-        for seconds_to_wait in wait_for(seconds=timeout_time_in_seconds, interval=interval):
-            time.sleep(seconds_to_wait)
+        for seconds_to_wait in wait_for(seconds=timeout_in_seconds, interval=poll_interval_seconds):
+            sleep_function(seconds_to_wait)
             logging.debug(f"Checking status of codebuild id {build_id}.")
             build_batches = client.batch_get_build_batches(build_batch_ids=[build_id])
             logging.debug(f"Build response of codebuild id {build_id} is {build_batches}")

exasol_script_languages_container_ci_setup/lib/config/config_data_model.py

@@ -0,0 +1,38 @@
+# generated by datamodel-codegen:
+#   filename:  <stdin>
+
+from __future__ import annotations
+
+from typing import List
+
+from pydantic import BaseModel, Extra
+
+
+class Ignore(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    paths: List[str]
+
+
+class Build(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    ignore: Ignore
+    base_branch: str
+
+
+class Release(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    timeout_in_minutes: int
+
+
+class Config(BaseModel):
+    class Config:
+        extra = Extra.forbid
+
+    build: Build
+    release: Release

exasol_script_languages_container_ci_setup/lib/config/pydantic_model_generator.py

@@ -0,0 +1,51 @@
+import json
+import logging
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from datamodel_code_generator import generate, InputFileType
+
+from exasol_script_languages_container_ci_setup.lib.render_template import render_template
+
+logger = logging.getLogger(__name__)
+
+CONFIG_DATA_MODEL_FILE_NAME = "config_data_model.py"
+
+
+def get_config_data_model_pydantic_model_default_output_file() -> Path:
+    return Path(__file__).parent / CONFIG_DATA_MODEL_FILE_NAME
+
+
+def generate_config_data_model(output_file: Path) -> Path:
+    schema_dict = json.loads(render_template("config_schema.json"))
+    schema_json = json.dumps(schema_dict)
+    with TemporaryDirectory() as directory:
+        temp_output_file = Path(directory) / CONFIG_DATA_MODEL_FILE_NAME
+        generate(schema_json, input_file_type=InputFileType.JsonSchema, output=temp_output_file,
+                 class_name="Config", apply_default_values_for_required_fields=True)
+        with temp_output_file.open("rt") as temp_output_file_handle:
+            with output_file.open("wt") as output_file_handle:
+                lines = (line for line in temp_output_file_handle)
+                lines = filter(lambda line: "#   timestamp: " not in line, lines)
+                for line in lines:
+                    output_file_handle.write(line)
+    return output_file
+
+
+def regenerate_config_data_model(output_file: Path):
+    with TemporaryDirectory() as directory:
+        temp_output_file = Path(directory) / CONFIG_DATA_MODEL_FILE_NAME
+        generate_config_data_model(temp_output_file)
+        with temp_output_file.open("rt") as file:
+            new_model = file.read()
+        if output_file.is_file():
+            with output_file.open("rt") as file:
+                old_model = file.read()
+            if new_model != old_model:
+                logger.warning(f"Regenerating config pydantic model to {output_file}")
+                with output_file.open("wt") as file:
+                    file.write(new_model)
+        else:
+            logger.info(f"Generating config pydantic model to new file {output_file}")
+            with output_file.open("wt") as file:
+                file.write(new_model)

exasol_script_languages_container_ci_setup/lib/generate_buildspec_common.py

@@ -6,6 +6,7 @@
 
 import jsonschema
 
+from exasol_script_languages_container_ci_setup.lib.config.config_data_model import Config
 from exasol_script_languages_container_ci_setup.lib.render_template import render_template
 
 
@@ -25,22 +26,17 @@ def validate_config_file(config_file: Optional[str]):
     """
     Validates config file, path given by parameter config_file.
     :raises:
-
-        `jsonschema.exceptions.ValidationError` if the config file has invalid JSON format.
-        `jsonschema.exceptions.SchemaError` if the config file is not in accordance with the the schema.
+        `pydantic.ValidationError` if the config file has invalid JSON format.
         `ValueError` if the ignored path given in the config file does not exist.
     """
     if config_file is None:
         return
-    with open(config_file, "r") as config_file_:
-        config = json.load(config_file_)
-        config_schema = json.loads(render_template("config_schema.json"))
-        jsonschema.validate(instance=config, schema=config_schema)
-        ignored_paths = config["build_ignore"]["ignored_paths"]
-        for ignored_path in ignored_paths:
-            folder_path = Path(ignored_path)
-            if not folder_path.exists():
-                raise ValueError(f"Ignored folder '{ignored_path}' does not exist.")
+    config = Config.parse_file(config_file)
+    ignored_paths = config.build.ignore.paths
+    for ignored_path in ignored_paths:
+        folder_path = Path(ignored_path)
+        if not folder_path.exists():
+            raise ValueError(f"Ignored folder '{ignored_path}' does not exist.")
 
 
 def get_config_file_parameter(config_file: Optional[str]):