From e4a00ae0a468c467e0a21968b5b6dcc611c6e941 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:31:32 +0000 Subject: [PATCH 01/44] add gitea_external_server --- .../programs/sre/gitea_external_server.py | 349 ++++++++++++++++++ 1 file changed, 349 insertions(+) create mode 100644 data_safe_haven/infrastructure/programs/sre/gitea_external_server.py diff --git a/data_safe_haven/infrastructure/programs/sre/gitea_external_server.py b/data_safe_haven/infrastructure/programs/sre/gitea_external_server.py new file mode 100644 index 0000000000..138fe5d24a --- /dev/null +++ b/data_safe_haven/infrastructure/programs/sre/gitea_external_server.py @@ -0,0 +1,349 @@ +from collections.abc import Mapping + +from pulumi import ComponentResource, Input, Output, ResourceOptions +from pulumi_azure_native import containerinstance, storage + +from data_safe_haven.infrastructure.common import ( + DockerHubCredentials, + get_ip_address_from_container_group, +) +from data_safe_haven.infrastructure.components import ( + FileShareFile, + FileShareFileProps, + LocalDnsRecordComponent, + LocalDnsRecordProps, + PostgresqlDatabaseComponent, + PostgresqlDatabaseProps, +) +from data_safe_haven.resources import resources_path +from data_safe_haven.utility import FileReader + + +class SREGiteaExternalServerProps: + """Properties for SREGiteaExternalServerComponent""" + + def __init__( + self, + containers_subnet_id: Input[str], + database_password: Input[str], + database_subnet_id: Input[str], + dns_server_ip: Input[str], + dockerhub_credentials: DockerHubCredentials, + ldap_server_hostname: Input[str], + ldap_server_port: Input[int], + ldap_username_attribute: Input[str], + ldap_user_filter: Input[str], + ldap_user_search_base: Input[str], + location: Input[str], + resource_group_name: Input[str], + sre_fqdn: Input[str], + storage_account_key: Input[str], + storage_account_name: Input[str], + database_username: Input[str] | None = None, + ) -> None: + self.containers_subnet_id = containers_subnet_id + self.database_password = database_password + self.database_subnet_id = database_subnet_id + self.database_username = ( + database_username if database_username else "postgresadmin" + ) + self.dns_server_ip = dns_server_ip + self.dockerhub_credentials = dockerhub_credentials + self.ldap_server_hostname = ldap_server_hostname + self.ldap_server_port = ldap_server_port + self.ldap_username_attribute = ldap_username_attribute + self.ldap_user_filter = ldap_user_filter + self.ldap_user_search_base = ldap_user_search_base + self.location = location + self.resource_group_name = resource_group_name + self.sre_fqdn = sre_fqdn + self.storage_account_key = storage_account_key + self.storage_account_name = storage_account_name + + +class SREGiteaExternalServerComponent(ComponentResource): + """Deploy Gitea server with Pulumi""" + + def __init__( + self, + name: str, + stack_name: str, + props: SREGiteaExternalServerProps, + opts: ResourceOptions | None = None, + tags: Input[Mapping[str, Input[str]]] | None = None, + ) -> None: + super().__init__("dsh:sre:GiteaExternalServerComponent", name, {}, opts) + child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) + child_tags = tags if tags else {} + + # Define configuration file shares + file_share_gitea_caddy = storage.FileShare( + f"{self._name}_file_share_gitea_caddy", + access_tier=storage.ShareAccessTier.COOL, + account_name=props.storage_account_name, + resource_group_name=props.resource_group_name, + share_name="gitea-caddy", + share_quota=1, + signed_identifiers=[], + opts=child_opts, + ) + file_share_gitea_gitea = storage.FileShare( + f"{self._name}_file_share_gitea_gitea", + access_tier=storage.ShareAccessTier.COOL, + account_name=props.storage_account_name, + resource_group_name=props.resource_group_name, + share_name="gitea-gitea", + share_quota=1, + signed_identifiers=[], + opts=child_opts, + ) + + # Upload caddy file + caddy_caddyfile_reader = FileReader( + resources_path / "gitea" / "caddy" / "Caddyfile" + ) + file_share_gitea_caddy_caddyfile = FileShareFile( + f"{self._name}_file_share_gitea_caddy_caddyfile", + FileShareFileProps( + destination_path=caddy_caddyfile_reader.name, + share_name=file_share_gitea_caddy.name, + file_contents=Output.secret(caddy_caddyfile_reader.file_contents()), + storage_account_key=props.storage_account_key, + storage_account_name=props.storage_account_name, + ), + opts=ResourceOptions.merge( + child_opts, ResourceOptions(parent=file_share_gitea_caddy) + ), + ) + + # Upload Gitea configuration script + gitea_configure_sh_reader = FileReader( + resources_path / "gitea" / "gitea" / "configure.mustache.sh" + ) + gitea_configure_sh = Output.all( + admin_email="dshadmin@example.com", + admin_username="dshadmin", + ldap_username_attribute=props.ldap_username_attribute, + ldap_user_filter=props.ldap_user_filter, + ldap_server_hostname=props.ldap_server_hostname, + ldap_server_port=props.ldap_server_port, + ldap_user_search_base=props.ldap_user_search_base, + ).apply( + lambda mustache_values: gitea_configure_sh_reader.file_contents( + mustache_values + ) + ) + file_share_gitea_gitea_configure_sh = FileShareFile( + f"{self._name}_file_share_gitea_gitea_configure_sh", + FileShareFileProps( + destination_path=gitea_configure_sh_reader.name, + share_name=file_share_gitea_gitea.name, + file_contents=Output.secret(gitea_configure_sh), + storage_account_key=props.storage_account_key, + storage_account_name=props.storage_account_name, + ), + opts=ResourceOptions.merge( + child_opts, ResourceOptions(parent=file_share_gitea_gitea) + ), + ) + # Upload Gitea entrypoint script + gitea_entrypoint_sh_reader = FileReader( + resources_path / "gitea" / "gitea" / "entrypoint.sh" + ) + file_share_gitea_gitea_entrypoint_sh = FileShareFile( + f"{self._name}_file_share_gitea_gitea_entrypoint_sh", + FileShareFileProps( + destination_path=gitea_entrypoint_sh_reader.name, + share_name=file_share_gitea_gitea.name, + file_contents=Output.secret(gitea_entrypoint_sh_reader.file_contents()), + storage_account_key=props.storage_account_key, + storage_account_name=props.storage_account_name, + ), + opts=ResourceOptions.merge( + child_opts, ResourceOptions(parent=file_share_gitea_gitea) + ), + ) + + # Define a PostgreSQL server and default database + db_gitea_repository_name = "gitea" + db_server_gitea = PostgresqlDatabaseComponent( + f"{self._name}_db_gitea", + PostgresqlDatabaseProps( + database_names=[db_gitea_repository_name], + database_password=props.database_password, + database_resource_group_name=props.resource_group_name, + database_server_name=f"{stack_name}-db-server-gitea", + database_subnet_id=props.database_subnet_id, + database_username=props.database_username, + location=props.location, + ), + opts=child_opts, + tags=child_tags, + ) + + # Define the container group with guacd, guacamole and caddy + container_group = containerinstance.ContainerGroup( + f"{self._name}_container_group", + container_group_name=f"{stack_name}-container-group-gitea", + containers=[ + containerinstance.ContainerArgs( + image="caddy:2.8.4", + name="caddy"[:63], + ports=[ + containerinstance.ContainerPortArgs( + port=80, + protocol=containerinstance.ContainerGroupNetworkProtocol.TCP, + ), + ], + resources=containerinstance.ResourceRequirementsArgs( + requests=containerinstance.ResourceRequestsArgs( + cpu=0.5, + memory_in_gb=0.5, + ), + ), + volume_mounts=[ + containerinstance.VolumeMountArgs( + mount_path="/etc/caddy", + name="caddy-etc-caddy", + read_only=True, + ), + ], + ), + containerinstance.ContainerArgs( + image="gitea/gitea:1.22.1", + name="gitea"[:63], + command=["/app/custom/entrypoint.sh"], + environment_variables=[ + containerinstance.EnvironmentVariableArgs( + name="APP_NAME", value="Data Safe Haven Git server" + ), + containerinstance.EnvironmentVariableArgs( + name="RUN_MODE", value="dev" + ), + containerinstance.EnvironmentVariableArgs( + name="GITEA__database__DB_TYPE", value="postgres" + ), + containerinstance.EnvironmentVariableArgs( + name="GITEA__database__HOST", + value=db_server_gitea.private_ip_address, + ), + containerinstance.EnvironmentVariableArgs( + name="GITEA__database__NAME", value=db_gitea_repository_name + ), + containerinstance.EnvironmentVariableArgs( + name="GITEA__database__USER", + value=props.database_username, + ), + containerinstance.EnvironmentVariableArgs( + name="GITEA__database__PASSWD", + secure_value=props.database_password, + ), + containerinstance.EnvironmentVariableArgs( + name="GITEA__database__SSL_MODE", value="require" + ), + containerinstance.EnvironmentVariableArgs( + name="GITEA__log__LEVEL", + # Options are: "Trace", "Debug", "Info" [default], "Warn", "Error", "Critical" or "None". + value="Debug", + ), + containerinstance.EnvironmentVariableArgs( + name="GITEA__security__INSTALL_LOCK", value="true" + ), + ], + ports=[ + containerinstance.ContainerPortArgs( + port=22, + protocol=containerinstance.ContainerGroupNetworkProtocol.TCP, + ), + ], + resources=containerinstance.ResourceRequirementsArgs( + requests=containerinstance.ResourceRequestsArgs( + cpu=2, + memory_in_gb=2, + ), + ), + volume_mounts=[ + containerinstance.VolumeMountArgs( + mount_path="/app/custom", + name="gitea-app-custom", + read_only=True, + ), + ], + ), + ], + dns_config=containerinstance.DnsConfigurationArgs( + name_servers=[props.dns_server_ip], + ), + # Required due to DockerHub rate-limit: https://docs.docker.com/docker-hub/download-rate-limit/ + image_registry_credentials=[ + { + "password": Output.secret(props.dockerhub_credentials.access_token), + "server": props.dockerhub_credentials.server, + "username": props.dockerhub_credentials.username, + } + ], + ip_address=containerinstance.IpAddressArgs( + ports=[ + containerinstance.PortArgs( + port=80, + protocol=containerinstance.ContainerGroupNetworkProtocol.TCP, + ) + ], + type=containerinstance.ContainerGroupIpAddressType.PRIVATE, + ), + location=props.location, + os_type=containerinstance.OperatingSystemTypes.LINUX, + resource_group_name=props.resource_group_name, + restart_policy=containerinstance.ContainerGroupRestartPolicy.ALWAYS, + sku=containerinstance.ContainerGroupSku.STANDARD, + subnet_ids=[ + containerinstance.ContainerGroupSubnetIdArgs( + id=props.containers_subnet_id + ) + ], + volumes=[ + containerinstance.VolumeArgs( + azure_file=containerinstance.AzureFileVolumeArgs( + share_name=file_share_gitea_caddy.name, + storage_account_key=props.storage_account_key, + storage_account_name=props.storage_account_name, + ), + name="caddy-etc-caddy", + ), + containerinstance.VolumeArgs( + azure_file=containerinstance.AzureFileVolumeArgs( + share_name=file_share_gitea_gitea.name, + storage_account_key=props.storage_account_key, + storage_account_name=props.storage_account_name, + ), + name="gitea-app-custom", + ), + ], + opts=ResourceOptions.merge( + child_opts, + ResourceOptions( + delete_before_replace=True, + depends_on=[ + file_share_gitea_caddy_caddyfile, + file_share_gitea_gitea_configure_sh, + file_share_gitea_gitea_entrypoint_sh, + ], + replace_on_changes=["containers"], + ), + ), + tags=child_tags, + ) + + # Register the container group in the SRE DNS zone + LocalDnsRecordComponent( + f"{self._name}_gitea_dns_record_set", + LocalDnsRecordProps( + base_fqdn=props.sre_fqdn, + private_ip_address=get_ip_address_from_container_group(container_group), + record_name="gitea", + resource_group_name=props.resource_group_name, + ), + opts=ResourceOptions.merge( + child_opts, ResourceOptions(parent=container_group) + ), + ) From ce965cf2483eca966f7b4b0c92af4df554567eb3 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:31:58 +0000 Subject: [PATCH 02/44] add additional files for external gitea --- .../resources/gitea_external/caddy/Caddyfile | 14 ++++++++++++ .../gitea/configure.mustache.sh | 22 +++++++++++++++++++ .../gitea_external/gitea/entrypoint.sh | 9 ++++++++ 3 files changed, 45 insertions(+) create mode 100644 data_safe_haven/resources/gitea_external/caddy/Caddyfile create mode 100644 data_safe_haven/resources/gitea_external/gitea/configure.mustache.sh create mode 100644 data_safe_haven/resources/gitea_external/gitea/entrypoint.sh diff --git a/data_safe_haven/resources/gitea_external/caddy/Caddyfile b/data_safe_haven/resources/gitea_external/caddy/Caddyfile new file mode 100644 index 0000000000..0bef301196 --- /dev/null +++ b/data_safe_haven/resources/gitea_external/caddy/Caddyfile @@ -0,0 +1,14 @@ +# Refer to the Caddy docs for more information: +# https://caddyserver.com/docs/caddyfile +{ + log { + format console { + level_format upper + } + level DEBUG + } +} + +:80 { + reverse_proxy http://localhost:3000 +} diff --git a/data_safe_haven/resources/gitea_external/gitea/configure.mustache.sh b/data_safe_haven/resources/gitea_external/gitea/configure.mustache.sh new file mode 100644 index 0000000000..4108c5c9dd --- /dev/null +++ b/data_safe_haven/resources/gitea_external/gitea/configure.mustache.sh @@ -0,0 +1,22 @@ +#! /usr/bin/env bash + +# Ensure that default admin user exists +until su-exec "$USER" /usr/local/bin/gitea admin user list --admin | grep "{{admin_username}}" > /dev/null 2>&1; do + echo "$(date -Iseconds) Attempting to create default admin user '{{admin_username}}'..." | tee -a /var/log/configuration + su-exec "$USER" /usr/local/bin/gitea admin user create --admin --username "{{admin_username}}" --random-password --random-password-length 20 --email "{{admin_email}}" 2> /dev/null + sleep 1 +done + +# Ensure that LDAP authentication is enabled +until su-exec "$USER" /usr/local/bin/gitea admin auth list | grep "DataSafeHavenLDAP" > /dev/null 2>&1; do + echo "$(date -Iseconds) Attempting to register LDAP authentication..." | tee -a /var/log/configuration + su-exec "$USER" /usr/local/bin/gitea admin auth add-ldap \ + --name DataSafeHavenLDAP \ + --security-protocol "unencrypted" \ + --host "{{ldap_server_hostname}}" \ + --port "{{ldap_server_port}}" \ + --user-search-base "{{ldap_user_search_base}}" \ + --user-filter "(&{{{ldap_user_filter}}}({{ldap_username_attribute}}=%[1]s))" \ + --email-attribute "mail" + sleep 1 +done diff --git a/data_safe_haven/resources/gitea_external/gitea/entrypoint.sh b/data_safe_haven/resources/gitea_external/gitea/entrypoint.sh new file mode 100644 index 0000000000..ee58e49337 --- /dev/null +++ b/data_safe_haven/resources/gitea_external/gitea/entrypoint.sh @@ -0,0 +1,9 @@ +#! /usr/bin/env sh + +# Add configuration as an s6 target +mkdir -p /etc/s6/setup +rm /etc/s6/setup/run 2> /dev/null +ln -s /app/custom/configure.sh /etc/s6/setup/run + +# Run the usual entrypoint +/usr/bin/entrypoint From fa62af6be7efa4f91c79b39a44ad792b8b3333ef Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:51:14 +0000 Subject: [PATCH 03/44] Add gitea_external_mirror config flag --- data_safe_haven/config/config_sections.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data_safe_haven/config/config_sections.py b/data_safe_haven/config/config_sections.py index 0423605e10..4366d5e969 100644 --- a/data_safe_haven/config/config_sections.py +++ b/data_safe_haven/config/config_sections.py @@ -13,6 +13,7 @@ DatabaseSystem, EmailAddress, Fqdn, + GiteaServerAvailability, Guid, IpAddress, SafeString, @@ -53,6 +54,7 @@ class ConfigSectionSRE(BaseModel, validate_assignment=True): data_provider_ip_addresses: list[IpAddress] = Field( ..., default_factory=list[IpAddress] ) + gitea_external_mirror: GiteaServerAvailability = GiteaServerAvailability.NONE remote_desktop: ConfigSubsectionRemoteDesktopOpts = Field( ..., default_factory=ConfigSubsectionRemoteDesktopOpts ) From cb70be7e547de3e3d50721c401888cda1d75e48e Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:51:37 +0000 Subject: [PATCH 04/44] Create enum for GiteaServerAvailability --- data_safe_haven/types/enums.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/data_safe_haven/types/enums.py b/data_safe_haven/types/enums.py index 44339cd5f5..12ffc75b8d 100644 --- a/data_safe_haven/types/enums.py +++ b/data_safe_haven/types/enums.py @@ -183,3 +183,11 @@ class SoftwarePackageCategory(str, Enum): ANY = "any" PRE_APPROVED = "pre-approved" NONE = "none" + +@verify(UNIQUE) +class GiteaServerAvailability(str, Enum): + """Availability of Gitea server.""" + + EXTERNAL = "external" + INTERNAL = "internal" + NONE = "none" \ No newline at end of file From 73259e4d4181cbc2145022fddf891bf4f0430a24 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:52:10 +0000 Subject: [PATCH 05/44] Create GiteaServerAvailability type --- data_safe_haven/types/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data_safe_haven/types/__init__.py b/data_safe_haven/types/__init__.py index f304520c9c..19bc194c89 100644 --- a/data_safe_haven/types/__init__.py +++ b/data_safe_haven/types/__init__.py @@ -17,6 +17,7 @@ DatabaseSystem, FirewallPriorities, ForbiddenDomains, + GiteaServerAvailability, NetworkingPriorities, PermittedDomains, Ports, @@ -36,6 +37,7 @@ "FirewallPriorities", "ForbiddenDomains", "Fqdn", + "GiteaServerAvailability", "Guid", "IpAddress", "NetworkingPriorities", From 34b25cdeea243ee5ced33ca08d3ab8e0a7e4452f Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 12:55:25 +0000 Subject: [PATCH 06/44] Add gitea_external_mirror to config template --- data_safe_haven/config/sre_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_safe_haven/config/sre_config.py b/data_safe_haven/config/sre_config.py index 2093190c1c..8280a777bb 100644 --- a/data_safe_haven/config/sre_config.py +++ b/data_safe_haven/config/sre_config.py @@ -64,6 +64,7 @@ def template(cls: type[Self]) -> SREConfig: data_provider_ip_addresses=[ "List of IP addresses belonging to data providers" ], + gitea_external_mirror="external/internal/none: whether to create a mirror for external Gitea repositories", remote_desktop=ConfigSubsectionRemoteDesktopOpts.model_construct( allow_copy="True/False: whether to allow copying text out of the environment", allow_paste="True/False: whether to allow pasting text into the environment", From b21849962674303a6bbfbe69ce9b9068eda37fd9 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:16:10 +0000 Subject: [PATCH 07/44] add gitea_server field to GiteaServerProps --- data_safe_haven/infrastructure/programs/sre/gitea_server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/sre/gitea_server.py b/data_safe_haven/infrastructure/programs/sre/gitea_server.py index e7b956a866..fdb55725d7 100644 --- a/data_safe_haven/infrastructure/programs/sre/gitea_server.py +++ b/data_safe_haven/infrastructure/programs/sre/gitea_server.py @@ -16,6 +16,7 @@ PostgresqlDatabaseProps, ) from data_safe_haven.resources import resources_path +from data_safe_haven.types import GiteaServerAvailability from data_safe_haven.utility import FileReader @@ -29,6 +30,7 @@ def __init__( database_subnet_id: Input[str], dns_server_ip: Input[str], dockerhub_credentials: DockerHubCredentials, + gitea_server: GiteaServerAvailability, ldap_server_hostname: Input[str], ldap_server_port: Input[int], ldap_username_attribute: Input[str], @@ -49,6 +51,7 @@ def __init__( ) self.dns_server_ip = dns_server_ip self.dockerhub_credentials = dockerhub_credentials + self.gitea_server = gitea_server self.ldap_server_hostname = ldap_server_hostname self.ldap_server_port = ldap_server_port self.ldap_username_attribute = ldap_username_attribute From 15dcbf06dbfda26a7981aded2cfa8a24213b6f70 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:19:53 +0000 Subject: [PATCH 08/44] change gitea_external_mirror field entry to list --- data_safe_haven/config/sre_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/config/sre_config.py b/data_safe_haven/config/sre_config.py index 8280a777bb..fd5f1df25e 100644 --- a/data_safe_haven/config/sre_config.py +++ b/data_safe_haven/config/sre_config.py @@ -64,7 +64,7 @@ def template(cls: type[Self]) -> SREConfig: data_provider_ip_addresses=[ "List of IP addresses belonging to data providers" ], - gitea_external_mirror="external/internal/none: whether to create a mirror for external Gitea repositories", + gitea_external_mirror=["List of Gitea servers to deploy"], remote_desktop=ConfigSubsectionRemoteDesktopOpts.model_construct( allow_copy="True/False: whether to allow copying text out of the environment", allow_paste="True/False: whether to allow pasting text into the environment", From 7f5c04c04941976ccdef3b555153558166919925 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:20:21 +0000 Subject: [PATCH 09/44] add gitea_servers field to SREUserServiceProps --- data_safe_haven/infrastructure/programs/declarative_sre.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 1698be923b..54bdeab92c 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -306,6 +306,7 @@ def __call__(self) -> None: dns_server_ip=dns.ip_address, dockerhub_credentials=dockerhub_credentials, gitea_database_password=data.password_gitea_database_admin, + gitea_servers=self.config.sre.gitea_external_mirrors, hedgedoc_database_password=data.password_hedgedoc_database_admin, ldap_server_hostname=identity.hostname, ldap_server_port=identity.server_port, From 2acba926021b863596eee26d060c5ec10a3e5fdf Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:38:45 +0000 Subject: [PATCH 10/44] change gitea enum name and possible responses --- data_safe_haven/types/enums.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/data_safe_haven/types/enums.py b/data_safe_haven/types/enums.py index 12ffc75b8d..f9564f37c7 100644 --- a/data_safe_haven/types/enums.py +++ b/data_safe_haven/types/enums.py @@ -185,9 +185,8 @@ class SoftwarePackageCategory(str, Enum): NONE = "none" @verify(UNIQUE) -class GiteaServerAvailability(str, Enum): +class GiteaServers(str, Enum): """Availability of Gitea server.""" EXTERNAL = "external" - INTERNAL = "internal" - NONE = "none" \ No newline at end of file + INTERNAL = "internal" \ No newline at end of file From 39845fe956ad06bd3a3868be49599fbc2f500e70 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:39:03 +0000 Subject: [PATCH 11/44] change GiteaServer type --- data_safe_haven/types/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/types/__init__.py b/data_safe_haven/types/__init__.py index 19bc194c89..e0c88652f4 100644 --- a/data_safe_haven/types/__init__.py +++ b/data_safe_haven/types/__init__.py @@ -17,7 +17,7 @@ DatabaseSystem, FirewallPriorities, ForbiddenDomains, - GiteaServerAvailability, + GiteaServers, NetworkingPriorities, PermittedDomains, Ports, @@ -37,7 +37,7 @@ "FirewallPriorities", "ForbiddenDomains", "Fqdn", - "GiteaServerAvailability", + "GiteaServers", "Guid", "IpAddress", "NetworkingPriorities", From 700f6f2256955e14312be445e0048e7a35389925 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 14:10:08 +0000 Subject: [PATCH 12/44] Change use of GiteaServers enum --- data_safe_haven/config/config_sections.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/config/config_sections.py b/data_safe_haven/config/config_sections.py index 4366d5e969..b13866f240 100644 --- a/data_safe_haven/config/config_sections.py +++ b/data_safe_haven/config/config_sections.py @@ -13,7 +13,7 @@ DatabaseSystem, EmailAddress, Fqdn, - GiteaServerAvailability, + GiteaServers, Guid, IpAddress, SafeString, @@ -54,7 +54,7 @@ class ConfigSectionSRE(BaseModel, validate_assignment=True): data_provider_ip_addresses: list[IpAddress] = Field( ..., default_factory=list[IpAddress] ) - gitea_external_mirror: GiteaServerAvailability = GiteaServerAvailability.NONE + gitea_servers: GiteaServers = GiteaServers.INTERNAL remote_desktop: ConfigSubsectionRemoteDesktopOpts = Field( ..., default_factory=ConfigSubsectionRemoteDesktopOpts ) From 4042d1e855cc89f8720431fec11b69c19acccb45 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 14:10:25 +0000 Subject: [PATCH 13/44] Change GiteaServer Enum possible values --- data_safe_haven/types/enums.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/types/enums.py b/data_safe_haven/types/enums.py index f9564f37c7..07bffb5760 100644 --- a/data_safe_haven/types/enums.py +++ b/data_safe_haven/types/enums.py @@ -188,5 +188,5 @@ class SoftwarePackageCategory(str, Enum): class GiteaServers(str, Enum): """Availability of Gitea server.""" - EXTERNAL = "external" + BOTH = "both" INTERNAL = "internal" \ No newline at end of file From 27f0c00d2c3ab63702312b7067dbc85105c99b29 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 14:11:20 +0000 Subject: [PATCH 14/44] Iterate over gitea_servers to deploy multiple types --- .../programs/sre/user_services.py | 58 +++++++++++-------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/user_services.py b/data_safe_haven/infrastructure/programs/sre/user_services.py index 90e2319f6f..0d35db3211 100644 --- a/data_safe_haven/infrastructure/programs/sre/user_services.py +++ b/data_safe_haven/infrastructure/programs/sre/user_services.py @@ -7,7 +7,7 @@ DockerHubCredentials, get_id_from_subnet, ) -from data_safe_haven.types import DatabaseSystem, SoftwarePackageCategory +from data_safe_haven.types import DatabaseSystem, GiteaServers, SoftwarePackageCategory from .database_servers import SREDatabaseServerComponent, SREDatabaseServerProps from .gitea_server import SREGiteaServerComponent, SREGiteaServerProps @@ -28,6 +28,7 @@ def __init__( dns_server_ip: Input[str], dockerhub_credentials: DockerHubCredentials, gitea_database_password: Input[str], + gitea_servers: GiteaServers, hedgedoc_database_password: Input[str], ldap_server_hostname: Input[str], ldap_server_port: Input[int], @@ -51,6 +52,7 @@ def __init__( self.dns_server_ip = dns_server_ip self.dockerhub_credentials = dockerhub_credentials self.gitea_database_password = gitea_database_password + self.gitea_servers = gitea_servers self.hedgedoc_database_password = hedgedoc_database_password self.ldap_server_hostname = ldap_server_hostname self.ldap_server_port = ldap_server_port @@ -93,30 +95,36 @@ def __init__( child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) child_tags = tags if tags else {} - # Deploy the Gitea server - SREGiteaServerComponent( - "sre_gitea_server", - stack_name, - SREGiteaServerProps( - containers_subnet_id=props.subnet_containers_id, - database_subnet_id=props.subnet_containers_support_id, - database_password=props.gitea_database_password, - dns_server_ip=props.dns_server_ip, - dockerhub_credentials=props.dockerhub_credentials, - ldap_server_hostname=props.ldap_server_hostname, - ldap_server_port=props.ldap_server_port, - ldap_username_attribute=props.ldap_username_attribute, - ldap_user_filter=props.ldap_user_filter, - ldap_user_search_base=props.ldap_user_search_base, - location=props.location, - resource_group_name=props.resource_group_name, - sre_fqdn=props.sre_fqdn, - storage_account_key=props.storage_account_key, - storage_account_name=props.storage_account_name, - ), - opts=child_opts, - tags=child_tags, - ) + # Deploy the Gitea servers + if props.gitea_servers == GiteaServers.BOTH: + gitea_servers = ["external", "internal"] + else: + gitea_servers = ["internal"] + for gitea_server in gitea_servers: + SREGiteaServerComponent( + f"sre_{gitea_server}_gitea_server", + stack_name, + SREGiteaServerProps( + containers_subnet_id=props.subnet_containers_id, + database_subnet_id=props.subnet_containers_support_id, + database_password=props.gitea_database_password, + dns_server_ip=props.dns_server_ip, + dockerhub_credentials=props.dockerhub_credentials, + gitea_server=gitea_server, + ldap_server_hostname=props.ldap_server_hostname, + ldap_server_port=props.ldap_server_port, + ldap_username_attribute=props.ldap_username_attribute, + ldap_user_filter=props.ldap_user_filter, + ldap_user_search_base=props.ldap_user_search_base, + location=props.location, + resource_group_name=props.resource_group_name, + sre_fqdn=props.sre_fqdn, + storage_account_key=props.storage_account_key, + storage_account_name=props.storage_account_name, + ), + opts=child_opts, + tags=child_tags, + ) # Deploy the HedgeDoc server SREHedgeDocServerComponent( From 26b5f47c915a3b70e48fac09bb8dadf50e34e76e Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 14:11:35 +0000 Subject: [PATCH 15/44] Modify config template --- data_safe_haven/config/sre_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/config/sre_config.py b/data_safe_haven/config/sre_config.py index fd5f1df25e..fe62790f23 100644 --- a/data_safe_haven/config/sre_config.py +++ b/data_safe_haven/config/sre_config.py @@ -64,7 +64,7 @@ def template(cls: type[Self]) -> SREConfig: data_provider_ip_addresses=[ "List of IP addresses belonging to data providers" ], - gitea_external_mirror=["List of Gitea servers to deploy"], + gitea_servers="both/internal: whether to deploy both external and internal Gitea servers, or only an internal server", remote_desktop=ConfigSubsectionRemoteDesktopOpts.model_construct( allow_copy="True/False: whether to allow copying text out of the environment", allow_paste="True/False: whether to allow pasting text into the environment", From ebcc5021ed991c47be55981ca0bce35d97186428 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:42:39 +0000 Subject: [PATCH 16/44] Remove seperate gitea component for external mirror --- .../programs/sre/gitea_external_server.py | 349 ------------------ 1 file changed, 349 deletions(-) delete mode 100644 data_safe_haven/infrastructure/programs/sre/gitea_external_server.py diff --git a/data_safe_haven/infrastructure/programs/sre/gitea_external_server.py b/data_safe_haven/infrastructure/programs/sre/gitea_external_server.py deleted file mode 100644 index 138fe5d24a..0000000000 --- a/data_safe_haven/infrastructure/programs/sre/gitea_external_server.py +++ /dev/null @@ -1,349 +0,0 @@ -from collections.abc import Mapping - -from pulumi import ComponentResource, Input, Output, ResourceOptions -from pulumi_azure_native import containerinstance, storage - -from data_safe_haven.infrastructure.common import ( - DockerHubCredentials, - get_ip_address_from_container_group, -) -from data_safe_haven.infrastructure.components import ( - FileShareFile, - FileShareFileProps, - LocalDnsRecordComponent, - LocalDnsRecordProps, - PostgresqlDatabaseComponent, - PostgresqlDatabaseProps, -) -from data_safe_haven.resources import resources_path -from data_safe_haven.utility import FileReader - - -class SREGiteaExternalServerProps: - """Properties for SREGiteaExternalServerComponent""" - - def __init__( - self, - containers_subnet_id: Input[str], - database_password: Input[str], - database_subnet_id: Input[str], - dns_server_ip: Input[str], - dockerhub_credentials: DockerHubCredentials, - ldap_server_hostname: Input[str], - ldap_server_port: Input[int], - ldap_username_attribute: Input[str], - ldap_user_filter: Input[str], - ldap_user_search_base: Input[str], - location: Input[str], - resource_group_name: Input[str], - sre_fqdn: Input[str], - storage_account_key: Input[str], - storage_account_name: Input[str], - database_username: Input[str] | None = None, - ) -> None: - self.containers_subnet_id = containers_subnet_id - self.database_password = database_password - self.database_subnet_id = database_subnet_id - self.database_username = ( - database_username if database_username else "postgresadmin" - ) - self.dns_server_ip = dns_server_ip - self.dockerhub_credentials = dockerhub_credentials - self.ldap_server_hostname = ldap_server_hostname - self.ldap_server_port = ldap_server_port - self.ldap_username_attribute = ldap_username_attribute - self.ldap_user_filter = ldap_user_filter - self.ldap_user_search_base = ldap_user_search_base - self.location = location - self.resource_group_name = resource_group_name - self.sre_fqdn = sre_fqdn - self.storage_account_key = storage_account_key - self.storage_account_name = storage_account_name - - -class SREGiteaExternalServerComponent(ComponentResource): - """Deploy Gitea server with Pulumi""" - - def __init__( - self, - name: str, - stack_name: str, - props: SREGiteaExternalServerProps, - opts: ResourceOptions | None = None, - tags: Input[Mapping[str, Input[str]]] | None = None, - ) -> None: - super().__init__("dsh:sre:GiteaExternalServerComponent", name, {}, opts) - child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) - child_tags = tags if tags else {} - - # Define configuration file shares - file_share_gitea_caddy = storage.FileShare( - f"{self._name}_file_share_gitea_caddy", - access_tier=storage.ShareAccessTier.COOL, - account_name=props.storage_account_name, - resource_group_name=props.resource_group_name, - share_name="gitea-caddy", - share_quota=1, - signed_identifiers=[], - opts=child_opts, - ) - file_share_gitea_gitea = storage.FileShare( - f"{self._name}_file_share_gitea_gitea", - access_tier=storage.ShareAccessTier.COOL, - account_name=props.storage_account_name, - resource_group_name=props.resource_group_name, - share_name="gitea-gitea", - share_quota=1, - signed_identifiers=[], - opts=child_opts, - ) - - # Upload caddy file - caddy_caddyfile_reader = FileReader( - resources_path / "gitea" / "caddy" / "Caddyfile" - ) - file_share_gitea_caddy_caddyfile = FileShareFile( - f"{self._name}_file_share_gitea_caddy_caddyfile", - FileShareFileProps( - destination_path=caddy_caddyfile_reader.name, - share_name=file_share_gitea_caddy.name, - file_contents=Output.secret(caddy_caddyfile_reader.file_contents()), - storage_account_key=props.storage_account_key, - storage_account_name=props.storage_account_name, - ), - opts=ResourceOptions.merge( - child_opts, ResourceOptions(parent=file_share_gitea_caddy) - ), - ) - - # Upload Gitea configuration script - gitea_configure_sh_reader = FileReader( - resources_path / "gitea" / "gitea" / "configure.mustache.sh" - ) - gitea_configure_sh = Output.all( - admin_email="dshadmin@example.com", - admin_username="dshadmin", - ldap_username_attribute=props.ldap_username_attribute, - ldap_user_filter=props.ldap_user_filter, - ldap_server_hostname=props.ldap_server_hostname, - ldap_server_port=props.ldap_server_port, - ldap_user_search_base=props.ldap_user_search_base, - ).apply( - lambda mustache_values: gitea_configure_sh_reader.file_contents( - mustache_values - ) - ) - file_share_gitea_gitea_configure_sh = FileShareFile( - f"{self._name}_file_share_gitea_gitea_configure_sh", - FileShareFileProps( - destination_path=gitea_configure_sh_reader.name, - share_name=file_share_gitea_gitea.name, - file_contents=Output.secret(gitea_configure_sh), - storage_account_key=props.storage_account_key, - storage_account_name=props.storage_account_name, - ), - opts=ResourceOptions.merge( - child_opts, ResourceOptions(parent=file_share_gitea_gitea) - ), - ) - # Upload Gitea entrypoint script - gitea_entrypoint_sh_reader = FileReader( - resources_path / "gitea" / "gitea" / "entrypoint.sh" - ) - file_share_gitea_gitea_entrypoint_sh = FileShareFile( - f"{self._name}_file_share_gitea_gitea_entrypoint_sh", - FileShareFileProps( - destination_path=gitea_entrypoint_sh_reader.name, - share_name=file_share_gitea_gitea.name, - file_contents=Output.secret(gitea_entrypoint_sh_reader.file_contents()), - storage_account_key=props.storage_account_key, - storage_account_name=props.storage_account_name, - ), - opts=ResourceOptions.merge( - child_opts, ResourceOptions(parent=file_share_gitea_gitea) - ), - ) - - # Define a PostgreSQL server and default database - db_gitea_repository_name = "gitea" - db_server_gitea = PostgresqlDatabaseComponent( - f"{self._name}_db_gitea", - PostgresqlDatabaseProps( - database_names=[db_gitea_repository_name], - database_password=props.database_password, - database_resource_group_name=props.resource_group_name, - database_server_name=f"{stack_name}-db-server-gitea", - database_subnet_id=props.database_subnet_id, - database_username=props.database_username, - location=props.location, - ), - opts=child_opts, - tags=child_tags, - ) - - # Define the container group with guacd, guacamole and caddy - container_group = containerinstance.ContainerGroup( - f"{self._name}_container_group", - container_group_name=f"{stack_name}-container-group-gitea", - containers=[ - containerinstance.ContainerArgs( - image="caddy:2.8.4", - name="caddy"[:63], - ports=[ - containerinstance.ContainerPortArgs( - port=80, - protocol=containerinstance.ContainerGroupNetworkProtocol.TCP, - ), - ], - resources=containerinstance.ResourceRequirementsArgs( - requests=containerinstance.ResourceRequestsArgs( - cpu=0.5, - memory_in_gb=0.5, - ), - ), - volume_mounts=[ - containerinstance.VolumeMountArgs( - mount_path="/etc/caddy", - name="caddy-etc-caddy", - read_only=True, - ), - ], - ), - containerinstance.ContainerArgs( - image="gitea/gitea:1.22.1", - name="gitea"[:63], - command=["/app/custom/entrypoint.sh"], - environment_variables=[ - containerinstance.EnvironmentVariableArgs( - name="APP_NAME", value="Data Safe Haven Git server" - ), - containerinstance.EnvironmentVariableArgs( - name="RUN_MODE", value="dev" - ), - containerinstance.EnvironmentVariableArgs( - name="GITEA__database__DB_TYPE", value="postgres" - ), - containerinstance.EnvironmentVariableArgs( - name="GITEA__database__HOST", - value=db_server_gitea.private_ip_address, - ), - containerinstance.EnvironmentVariableArgs( - name="GITEA__database__NAME", value=db_gitea_repository_name - ), - containerinstance.EnvironmentVariableArgs( - name="GITEA__database__USER", - value=props.database_username, - ), - containerinstance.EnvironmentVariableArgs( - name="GITEA__database__PASSWD", - secure_value=props.database_password, - ), - containerinstance.EnvironmentVariableArgs( - name="GITEA__database__SSL_MODE", value="require" - ), - containerinstance.EnvironmentVariableArgs( - name="GITEA__log__LEVEL", - # Options are: "Trace", "Debug", "Info" [default], "Warn", "Error", "Critical" or "None". - value="Debug", - ), - containerinstance.EnvironmentVariableArgs( - name="GITEA__security__INSTALL_LOCK", value="true" - ), - ], - ports=[ - containerinstance.ContainerPortArgs( - port=22, - protocol=containerinstance.ContainerGroupNetworkProtocol.TCP, - ), - ], - resources=containerinstance.ResourceRequirementsArgs( - requests=containerinstance.ResourceRequestsArgs( - cpu=2, - memory_in_gb=2, - ), - ), - volume_mounts=[ - containerinstance.VolumeMountArgs( - mount_path="/app/custom", - name="gitea-app-custom", - read_only=True, - ), - ], - ), - ], - dns_config=containerinstance.DnsConfigurationArgs( - name_servers=[props.dns_server_ip], - ), - # Required due to DockerHub rate-limit: https://docs.docker.com/docker-hub/download-rate-limit/ - image_registry_credentials=[ - { - "password": Output.secret(props.dockerhub_credentials.access_token), - "server": props.dockerhub_credentials.server, - "username": props.dockerhub_credentials.username, - } - ], - ip_address=containerinstance.IpAddressArgs( - ports=[ - containerinstance.PortArgs( - port=80, - protocol=containerinstance.ContainerGroupNetworkProtocol.TCP, - ) - ], - type=containerinstance.ContainerGroupIpAddressType.PRIVATE, - ), - location=props.location, - os_type=containerinstance.OperatingSystemTypes.LINUX, - resource_group_name=props.resource_group_name, - restart_policy=containerinstance.ContainerGroupRestartPolicy.ALWAYS, - sku=containerinstance.ContainerGroupSku.STANDARD, - subnet_ids=[ - containerinstance.ContainerGroupSubnetIdArgs( - id=props.containers_subnet_id - ) - ], - volumes=[ - containerinstance.VolumeArgs( - azure_file=containerinstance.AzureFileVolumeArgs( - share_name=file_share_gitea_caddy.name, - storage_account_key=props.storage_account_key, - storage_account_name=props.storage_account_name, - ), - name="caddy-etc-caddy", - ), - containerinstance.VolumeArgs( - azure_file=containerinstance.AzureFileVolumeArgs( - share_name=file_share_gitea_gitea.name, - storage_account_key=props.storage_account_key, - storage_account_name=props.storage_account_name, - ), - name="gitea-app-custom", - ), - ], - opts=ResourceOptions.merge( - child_opts, - ResourceOptions( - delete_before_replace=True, - depends_on=[ - file_share_gitea_caddy_caddyfile, - file_share_gitea_gitea_configure_sh, - file_share_gitea_gitea_entrypoint_sh, - ], - replace_on_changes=["containers"], - ), - ), - tags=child_tags, - ) - - # Register the container group in the SRE DNS zone - LocalDnsRecordComponent( - f"{self._name}_gitea_dns_record_set", - LocalDnsRecordProps( - base_fqdn=props.sre_fqdn, - private_ip_address=get_ip_address_from_container_group(container_group), - record_name="gitea", - resource_group_name=props.resource_group_name, - ), - opts=ResourceOptions.merge( - child_opts, ResourceOptions(parent=container_group) - ), - ) From a9643f1e14c8658936c3c12e518a3d3913d69b12 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 1 Aug 2024 15:52:00 +0000 Subject: [PATCH 17/44] Switch to boolean toggle for external mirror --- data_safe_haven/config/sre_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/config/sre_config.py b/data_safe_haven/config/sre_config.py index fe62790f23..aaec15d26d 100644 --- a/data_safe_haven/config/sre_config.py +++ b/data_safe_haven/config/sre_config.py @@ -64,7 +64,7 @@ def template(cls: type[Self]) -> SREConfig: data_provider_ip_addresses=[ "List of IP addresses belonging to data providers" ], - gitea_servers="both/internal: whether to deploy both external and internal Gitea servers, or only an internal server", + external_git_mirror="True/False: whether to deploy an external mirror git server (True), or only an internal server", remote_desktop=ConfigSubsectionRemoteDesktopOpts.model_construct( allow_copy="True/False: whether to allow copying text out of the environment", allow_paste="True/False: whether to allow pasting text into the environment", From 5fa097b6d1eed0996967b230f4d558f9819f7579 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 08:59:17 +0000 Subject: [PATCH 18/44] Remove GiteaServers enum --- data_safe_haven/types/enums.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/data_safe_haven/types/enums.py b/data_safe_haven/types/enums.py index 07bffb5760..18494ce3d9 100644 --- a/data_safe_haven/types/enums.py +++ b/data_safe_haven/types/enums.py @@ -184,9 +184,3 @@ class SoftwarePackageCategory(str, Enum): PRE_APPROVED = "pre-approved" NONE = "none" -@verify(UNIQUE) -class GiteaServers(str, Enum): - """Availability of Gitea server.""" - - BOTH = "both" - INTERNAL = "internal" \ No newline at end of file From 3adc1368b3d8a484c8b408746a4225164b364444 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 09:01:22 +0000 Subject: [PATCH 19/44] Change gitea flag to boolean --- data_safe_haven/config/config_sections.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/data_safe_haven/config/config_sections.py b/data_safe_haven/config/config_sections.py index b13866f240..d6741bdf91 100644 --- a/data_safe_haven/config/config_sections.py +++ b/data_safe_haven/config/config_sections.py @@ -13,7 +13,6 @@ DatabaseSystem, EmailAddress, Fqdn, - GiteaServers, Guid, IpAddress, SafeString, @@ -54,7 +53,7 @@ class ConfigSectionSRE(BaseModel, validate_assignment=True): data_provider_ip_addresses: list[IpAddress] = Field( ..., default_factory=list[IpAddress] ) - gitea_servers: GiteaServers = GiteaServers.INTERNAL + external_git_mirror: bool = False remote_desktop: ConfigSubsectionRemoteDesktopOpts = Field( ..., default_factory=ConfigSubsectionRemoteDesktopOpts ) From 3a20cc8ef2c886184c128cacb31d214e512e4218 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 09:01:38 +0000 Subject: [PATCH 20/44] Use new fieldname --- data_safe_haven/infrastructure/programs/declarative_sre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 54bdeab92c..5c2175ee1f 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -306,7 +306,7 @@ def __call__(self) -> None: dns_server_ip=dns.ip_address, dockerhub_credentials=dockerhub_credentials, gitea_database_password=data.password_gitea_database_admin, - gitea_servers=self.config.sre.gitea_external_mirrors, + git_mirror=self.config.sre.external_git_mirror, hedgedoc_database_password=data.password_hedgedoc_database_admin, ldap_server_hostname=identity.hostname, ldap_server_port=identity.server_port, From dfbaf54c36f8365e56cebb0a45763725db10ea43 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 12:11:33 +0000 Subject: [PATCH 21/44] Switch to boolean toggle for external mirror --- .../infrastructure/programs/sre/gitea_server.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/gitea_server.py b/data_safe_haven/infrastructure/programs/sre/gitea_server.py index fdb55725d7..111d745537 100644 --- a/data_safe_haven/infrastructure/programs/sre/gitea_server.py +++ b/data_safe_haven/infrastructure/programs/sre/gitea_server.py @@ -16,7 +16,6 @@ PostgresqlDatabaseProps, ) from data_safe_haven.resources import resources_path -from data_safe_haven.types import GiteaServerAvailability from data_safe_haven.utility import FileReader @@ -30,7 +29,7 @@ def __init__( database_subnet_id: Input[str], dns_server_ip: Input[str], dockerhub_credentials: DockerHubCredentials, - gitea_server: GiteaServerAvailability, + external_git_mirror: Input[bool], ldap_server_hostname: Input[str], ldap_server_port: Input[int], ldap_username_attribute: Input[str], @@ -51,7 +50,7 @@ def __init__( ) self.dns_server_ip = dns_server_ip self.dockerhub_credentials = dockerhub_credentials - self.gitea_server = gitea_server + self.external_git_mirror = external_git_mirror self.ldap_server_hostname = ldap_server_hostname self.ldap_server_port = ldap_server_port self.ldap_username_attribute = ldap_username_attribute @@ -170,12 +169,12 @@ def __init__( # Define a PostgreSQL server and default database db_gitea_repository_name = "gitea" db_server_gitea = PostgresqlDatabaseComponent( - f"{self._name}_db_gitea", + f"{self._name}_db_gitea_{props.gitea_server}", PostgresqlDatabaseProps( database_names=[db_gitea_repository_name], database_password=props.database_password, database_resource_group_name=props.resource_group_name, - database_server_name=f"{stack_name}-db-server-gitea", + database_server_name=f"{stack_name}-db-server-gitea-{props.gitea_server}", database_subnet_id=props.database_subnet_id, database_username=props.database_username, location=props.location, @@ -187,7 +186,7 @@ def __init__( # Define the container group with guacd, guacamole and caddy container_group = containerinstance.ContainerGroup( f"{self._name}_container_group", - container_group_name=f"{stack_name}-container-group-gitea", + container_group_name=f"{stack_name}-container-group-gitea-{self.props.gitea_server}", containers=[ containerinstance.ContainerArgs( image="caddy:2.8.4", From d9067f9db74b915595871b985144d04c2fe27e63 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 12:11:57 +0000 Subject: [PATCH 22/44] Run lint:fmt --- data_safe_haven/types/enums.py | 1 - 1 file changed, 1 deletion(-) diff --git a/data_safe_haven/types/enums.py b/data_safe_haven/types/enums.py index 18494ce3d9..44339cd5f5 100644 --- a/data_safe_haven/types/enums.py +++ b/data_safe_haven/types/enums.py @@ -183,4 +183,3 @@ class SoftwarePackageCategory(str, Enum): ANY = "any" PRE_APPROVED = "pre-approved" NONE = "none" - From 4705e22101834e57458ab4e6da4f30ffeaccf5a8 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 12:12:28 +0000 Subject: [PATCH 23/44] Switch to boolean toggle and change logic for deploying multiple servers --- .../infrastructure/programs/sre/user_services.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/user_services.py b/data_safe_haven/infrastructure/programs/sre/user_services.py index 0d35db3211..456cce120d 100644 --- a/data_safe_haven/infrastructure/programs/sre/user_services.py +++ b/data_safe_haven/infrastructure/programs/sre/user_services.py @@ -7,7 +7,7 @@ DockerHubCredentials, get_id_from_subnet, ) -from data_safe_haven.types import DatabaseSystem, GiteaServers, SoftwarePackageCategory +from data_safe_haven.types import DatabaseSystem, SoftwarePackageCategory from .database_servers import SREDatabaseServerComponent, SREDatabaseServerProps from .gitea_server import SREGiteaServerComponent, SREGiteaServerProps @@ -28,7 +28,7 @@ def __init__( dns_server_ip: Input[str], dockerhub_credentials: DockerHubCredentials, gitea_database_password: Input[str], - gitea_servers: GiteaServers, + external_git_mirror: Input[bool], hedgedoc_database_password: Input[str], ldap_server_hostname: Input[str], ldap_server_port: Input[int], @@ -52,7 +52,7 @@ def __init__( self.dns_server_ip = dns_server_ip self.dockerhub_credentials = dockerhub_credentials self.gitea_database_password = gitea_database_password - self.gitea_servers = gitea_servers + self.external_git_mirror = external_git_mirror self.hedgedoc_database_password = hedgedoc_database_password self.ldap_server_hostname = ldap_server_hostname self.ldap_server_port = ldap_server_port @@ -96,10 +96,9 @@ def __init__( child_tags = tags if tags else {} # Deploy the Gitea servers - if props.gitea_servers == GiteaServers.BOTH: - gitea_servers = ["external", "internal"] - else: - gitea_servers = ["internal"] + gitea_servers = ( + ["external", "internal"] if props.external_git_mirror else ["internal"] + ) for gitea_server in gitea_servers: SREGiteaServerComponent( f"sre_{gitea_server}_gitea_server", From b6f3a50cc4eb72932bc118c49dd63a356a9324c8 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 12:31:35 +0000 Subject: [PATCH 24/44] remove deprecated enum --- data_safe_haven/types/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/data_safe_haven/types/__init__.py b/data_safe_haven/types/__init__.py index e0c88652f4..f304520c9c 100644 --- a/data_safe_haven/types/__init__.py +++ b/data_safe_haven/types/__init__.py @@ -17,7 +17,6 @@ DatabaseSystem, FirewallPriorities, ForbiddenDomains, - GiteaServers, NetworkingPriorities, PermittedDomains, Ports, @@ -37,7 +36,6 @@ "FirewallPriorities", "ForbiddenDomains", "Fqdn", - "GiteaServers", "Guid", "IpAddress", "NetworkingPriorities", From d8e7ee5b1050c5484aeb9fb82f98372f74a87cbe Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:11:05 +0000 Subject: [PATCH 25/44] switch to external_git_mirror --- data_safe_haven/infrastructure/programs/declarative_sre.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index a4ea169894..61dabd53a4 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -312,7 +312,7 @@ def __call__(self) -> None: dns_server_ip=dns.ip_address, dockerhub_credentials=dockerhub_credentials, gitea_database_password=data.password_gitea_database_admin, - git_mirror=self.config.sre.external_git_mirror, + external_git_mirror=self.config.sre.external_git_mirror, hedgedoc_database_password=data.password_hedgedoc_database_admin, ldap_server_hostname=identity.hostname, ldap_server_port=identity.server_port, @@ -329,6 +329,7 @@ def __call__(self) -> None: subnet_containers=networking.subnet_user_services_containers, subnet_containers_support=networking.subnet_user_services_containers_support, subnet_databases=networking.subnet_user_services_databases, + # subnet_external_git_mirror=networking.subnet_user_services_external_git_mirror, subnet_software_repositories=networking.subnet_user_services_software_repositories, ), tags=self.tags, From 486261e745326c3da5970f8bb14a2e046eaa5a20 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:11:23 +0000 Subject: [PATCH 26/44] rename gitea property --- .../infrastructure/programs/sre/gitea_server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/gitea_server.py b/data_safe_haven/infrastructure/programs/sre/gitea_server.py index 5a4fd8d409..f6192633f9 100644 --- a/data_safe_haven/infrastructure/programs/sre/gitea_server.py +++ b/data_safe_haven/infrastructure/programs/sre/gitea_server.py @@ -29,7 +29,7 @@ def __init__( database_subnet_id: Input[str], dns_server_ip: Input[str], dockerhub_credentials: DockerHubCredentials, - external_git_mirror: Input[bool], + gitea_server: Input[str], ldap_server_hostname: Input[str], ldap_server_port: Input[int], ldap_username_attribute: Input[str], @@ -50,7 +50,7 @@ def __init__( ) self.dns_server_ip = dns_server_ip self.dockerhub_credentials = dockerhub_credentials - self.external_git_mirror = external_git_mirror + self.gitea_server = gitea_server self.ldap_server_hostname = ldap_server_hostname self.ldap_server_port = ldap_server_port self.ldap_username_attribute = ldap_username_attribute @@ -184,10 +184,10 @@ def __init__( tags=child_tags, ) - # Define the container group with guacd, guacamole and caddy + # Define the container group with gitea and caddy container_group = containerinstance.ContainerGroup( f"{self._name}_container_group", - container_group_name=f"{stack_name}-container-group-gitea-{self.props.gitea_server}", + container_group_name=f"{stack_name}-container-group-gitea-{props.gitea_server}", containers=[ containerinstance.ContainerArgs( image="caddy:2.8.4", @@ -343,7 +343,7 @@ def __init__( LocalDnsRecordProps( base_fqdn=props.sre_fqdn, private_ip_address=get_ip_address_from_container_group(container_group), - record_name="gitea", + record_name=f"{props.gitea_server}-gitea", resource_group_name=props.resource_group_name, ), opts=ResourceOptions.merge( From da2b231a9f6c2f509621dbcb87a7c8abf9a21bf2 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:11:55 +0000 Subject: [PATCH 27/44] comment out external subnet while testing --- data_safe_haven/infrastructure/programs/sre/networking.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_safe_haven/infrastructure/programs/sre/networking.py b/data_safe_haven/infrastructure/programs/sre/networking.py index 1f33c52082..5419021472 100644 --- a/data_safe_haven/infrastructure/programs/sre/networking.py +++ b/data_safe_haven/infrastructure/programs/sre/networking.py @@ -1454,6 +1454,7 @@ def __init__( subnet_data_configuration_name = "DataConfigurationSubnet" subnet_data_desired_state_name = "DataDesiredStateSubnet" subnet_data_private_name = "DataPrivateSubnet" + # subnet_external_git_mirror_name = "ExternalGitMirrorSubnet" subnet_firewall_name = "AzureFirewallSubnet" subnet_firewall_management_name = "AzureFirewallManagementSubnet" subnet_guacamole_containers_name = "GuacamoleContainersSubnet" From cb242ace1ef413cfa28cfc911a0ef8dccb8dc9eb Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:12:06 +0000 Subject: [PATCH 28/44] comment out external subnet while testing --- .../infrastructure/programs/sre/user_services.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/user_services.py b/data_safe_haven/infrastructure/programs/sre/user_services.py index c123e00b48..5b3a1f0dcf 100644 --- a/data_safe_haven/infrastructure/programs/sre/user_services.py +++ b/data_safe_haven/infrastructure/programs/sre/user_services.py @@ -45,6 +45,7 @@ def __init__( subnet_containers_support: Input[network.GetSubnetResult], subnet_containers: Input[network.GetSubnetResult], subnet_databases: Input[network.GetSubnetResult], + # subnet_external_git_mirror: Input[network.GetSubnetResult], subnet_software_repositories: Input[network.GetSubnetResult], ) -> None: self.database_service_admin_password = database_service_admin_password @@ -75,6 +76,9 @@ def __init__( self.subnet_databases_id = Output.from_input(subnet_databases).apply( get_id_from_subnet ) + # self.subnet_external_git_mirror_id = Output.from_input( + # subnet_external_git_mirror + # ).apply(get_id_from_subnet) self.subnet_software_repositories_id = Output.from_input( subnet_software_repositories ).apply(get_id_from_subnet) @@ -96,15 +100,19 @@ def __init__( child_tags = {"component": "user services"} | (tags if tags else {}) # Deploy the Gitea servers - gitea_servers = ( - ["external", "internal"] if props.external_git_mirror else ["internal"] - ) + if props.external_git_mirror: + gitea_servers = ["external", "internal"] + subnet = [props.subnet_containers_id, props.subnet_containers_id] + else: + gitea_servers = ["internal"] + subnet = [props.subnet_containers_id] + for gitea_server in gitea_servers: SREGiteaServerComponent( f"sre_{gitea_server}_gitea_server", stack_name, SREGiteaServerProps( - containers_subnet_id=props.subnet_containers_id, + containers_subnet_id=subnet[0], database_subnet_id=props.subnet_containers_support_id, database_password=props.gitea_database_password, dns_server_ip=props.dns_server_ip, From d499684154d499f3b7e23d8b61464ed12d211a49 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:07:39 +0000 Subject: [PATCH 29/44] use separate fileshares for external and internal gitea servers --- .../infrastructure/programs/sre/gitea_server.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/gitea_server.py b/data_safe_haven/infrastructure/programs/sre/gitea_server.py index f6192633f9..8506b22e28 100644 --- a/data_safe_haven/infrastructure/programs/sre/gitea_server.py +++ b/data_safe_haven/infrastructure/programs/sre/gitea_server.py @@ -84,7 +84,7 @@ def __init__( access_tier=storage.ShareAccessTier.COOL, account_name=props.storage_account_name, resource_group_name=props.resource_group_name, - share_name="gitea-caddy", + share_name=f"{props.gitea_server}-gitea-caddy", share_quota=1, signed_identifiers=[], opts=child_opts, @@ -94,7 +94,7 @@ def __init__( access_tier=storage.ShareAccessTier.COOL, account_name=props.storage_account_name, resource_group_name=props.resource_group_name, - share_name="gitea-gitea", + share_name=f"{props.gitea_server}-gitea-gitea", share_quota=1, signed_identifiers=[], opts=child_opts, @@ -119,9 +119,14 @@ def __init__( ) # Upload Gitea configuration script - gitea_configure_sh_reader = FileReader( - resources_path / "gitea" / "gitea" / "configure.mustache.sh" - ) + if props.gitea_server == "external": + gitea_configure_sh_reader = FileReader( + resources_path / "gitea_external" / "gitea" / "configure.mustache.sh" + ) + else: + gitea_configure_sh_reader = FileReader( + resources_path / "gitea" / "gitea" / "configure.mustache.sh" + ) gitea_configure_sh = Output.all( admin_email="dshadmin@example.com", admin_username="dshadmin", From 599f771a0b06ae2e6ce15b3e5113908b982066f8 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:07:54 +0000 Subject: [PATCH 30/44] don't setup ldap on external gitea server --- .../gitea_external/gitea/configure.mustache.sh | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/data_safe_haven/resources/gitea_external/gitea/configure.mustache.sh b/data_safe_haven/resources/gitea_external/gitea/configure.mustache.sh index 4108c5c9dd..eaa4d39399 100644 --- a/data_safe_haven/resources/gitea_external/gitea/configure.mustache.sh +++ b/data_safe_haven/resources/gitea_external/gitea/configure.mustache.sh @@ -6,17 +6,3 @@ until su-exec "$USER" /usr/local/bin/gitea admin user list --admin | grep "{{adm su-exec "$USER" /usr/local/bin/gitea admin user create --admin --username "{{admin_username}}" --random-password --random-password-length 20 --email "{{admin_email}}" 2> /dev/null sleep 1 done - -# Ensure that LDAP authentication is enabled -until su-exec "$USER" /usr/local/bin/gitea admin auth list | grep "DataSafeHavenLDAP" > /dev/null 2>&1; do - echo "$(date -Iseconds) Attempting to register LDAP authentication..." | tee -a /var/log/configuration - su-exec "$USER" /usr/local/bin/gitea admin auth add-ldap \ - --name DataSafeHavenLDAP \ - --security-protocol "unencrypted" \ - --host "{{ldap_server_hostname}}" \ - --port "{{ldap_server_port}}" \ - --user-search-base "{{ldap_user_search_base}}" \ - --user-filter "(&{{{ldap_user_filter}}}({{ldap_username_attribute}}=%[1]s))" \ - --email-attribute "mail" - sleep 1 -done From 0d3233905e992b383cd9bc4c57857dd6947c9eb3 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 13 Sep 2024 11:54:03 +0000 Subject: [PATCH 31/44] fix linting error --- data_safe_haven/config/sre_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/config/sre_config.py b/data_safe_haven/config/sre_config.py index 211535d3bc..89756fd1b9 100644 --- a/data_safe_haven/config/sre_config.py +++ b/data_safe_haven/config/sre_config.py @@ -91,7 +91,7 @@ def template(cls: type[Self], tier: int | None = None) -> SREConfig: data_provider_ip_addresses=[ "List of IP addresses belonging to data providers" ], - external_git_mirror="True/False: whether to deploy an external mirror git server (True), or only an internal server", + external_git_mirror="True/False: whether to deploy an external mirror git server (True), or only an internal server", # type: ignore remote_desktop=ConfigSubsectionRemoteDesktopOpts.model_construct( allow_copy=remote_desktop_allow_copy, allow_paste=remote_desktop_allow_paste, From 426c6d60472e2842ea7ad0bcc79d38f1f087e2f1 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 13 Sep 2024 14:30:32 +0000 Subject: [PATCH 32/44] Add ip range for external git mirror --- data_safe_haven/infrastructure/common/ip_ranges.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_safe_haven/infrastructure/common/ip_ranges.py b/data_safe_haven/infrastructure/common/ip_ranges.py index f0613e577a..c0b1411550 100644 --- a/data_safe_haven/infrastructure/common/ip_ranges.py +++ b/data_safe_haven/infrastructure/common/ip_ranges.py @@ -16,6 +16,7 @@ class SREIpRanges: data_configuration = vnet.next_subnet(8) data_desired_state = vnet.next_subnet(8) data_private = vnet.next_subnet(8) + external_git_mirror = vnet.next_subnet(8) firewall = vnet.next_subnet(64) # 64 address minimum firewall_management = vnet.next_subnet(64) # 64 address minimum guacamole_containers = vnet.next_subnet(8) From 424b18b7b0977c91eb82e7ba31d68bdaaa84ac03 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 13 Sep 2024 14:31:20 +0000 Subject: [PATCH 33/44] add subnet and nsg for external git mirror --- .../infrastructure/programs/sre/networking.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/networking.py b/data_safe_haven/infrastructure/programs/sre/networking.py index 7ff07643a7..be1d8a1edd 100644 --- a/data_safe_haven/infrastructure/programs/sre/networking.py +++ b/data_safe_haven/infrastructure/programs/sre/networking.py @@ -81,6 +81,29 @@ def __init__( ) # Define NSGs + nsg_external_git_mirror = network.NetworkSecurityGroup( + f"{self._name}_nsg_external_git_mirror", + location=props.location, + network_security_group_name=f"{stack_name}-nsg-external-git-mirror", + resource_group_name=props.resource_group_name, + security_rules=[ + # Inbound + network.SecurityRuleArgs( + access=network.SecurityRuleAccess.DENY, + description="Deny all other inbound traffic.", + destination_address_prefix="*", + destination_port_range="*", + direction=network.SecurityRuleDirection.INBOUND, + name="DenyAllOtherInbound", + priority=NetworkingPriorities.ALL_OTHER, + protocol=network.SecurityRuleProtocol.ASTERISK, + source_address_prefix="*", + source_port_range="*", + ), + ], + opts=child_opts, + tags=child_tags, + ) nsg_application_gateway = network.NetworkSecurityGroup( f"{self._name}_nsg_application_gateway", location=props.location, @@ -1560,7 +1583,7 @@ def __init__( subnet_data_configuration_name = "DataConfigurationSubnet" subnet_data_desired_state_name = "DataDesiredStateSubnet" subnet_data_private_name = "DataPrivateSubnet" - # subnet_external_git_mirror_name = "ExternalGitMirrorSubnet" + subnet_external_git_mirror_name = "ExternalGitMirrorSubnet" subnet_firewall_name = "AzureFirewallSubnet" subnet_firewall_management_name = "AzureFirewallManagementSubnet" subnet_guacamole_containers_name = "GuacamoleContainersSubnet" @@ -1751,6 +1774,15 @@ def __init__( ), route_table=network.RouteTableArgs(id=route_table.id), ), + # User services external git mirror + network.SubnetArgs( + address_prefix=SREIpRanges.external_git_mirror.prefix, + name=subnet_external_git_mirror_name, + network_security_group=network.NetworkSecurityGroupArgs( + id=nsg_external_git_mirror.id + ), + route_table=network.RouteTableArgs(id=route_table.id), + ), # User services containers support network.SubnetArgs( address_prefix=SREIpRanges.user_services_containers_support.prefix, @@ -1982,6 +2014,11 @@ def __init__( resource_group_name=props.resource_group_name, virtual_network_name=sre_virtual_network.name, ) + self.subnet_external_git_mirror = network.get_subnet_output( + subnet_name=subnet_external_git_mirror_name, + resource_group_name=props.resource_group_name, + virtual_network_name=sre_virtual_network.name, + ) self.subnet_firewall = network.get_subnet_output( subnet_name=subnet_firewall_name, resource_group_name=props.resource_group_name, From c86beec4e7700d00671aaed86e523da4f3ce23a7 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 16 Sep 2024 09:59:43 +0000 Subject: [PATCH 34/44] add enum for git mirror nsg rule --- data_safe_haven/types/enums.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_safe_haven/types/enums.py b/data_safe_haven/types/enums.py index 170cbba4a0..7c86218a21 100644 --- a/data_safe_haven/types/enums.py +++ b/data_safe_haven/types/enums.py @@ -85,6 +85,7 @@ class NetworkingPriorities(int, Enum): INTERNAL_SRE_DATA_CONFIGURATION = 1900 INTERNAL_SRE_DATA_DESIRED_STATE = 1910 INTERNAL_SRE_DATA_PRIVATE = 1920 + INTERNAL_SRE_EXTERNAL_GIT_MIRROR = 1930 INTERNAL_SRE_GUACAMOLE_CONTAINERS = 2000 INTERNAL_SRE_GUACAMOLE_CONTAINERS_SUPPORT = 2100 INTERNAL_SRE_IDENTITY_CONTAINERS = 2200 From 52badd4d273ff451506503962bce59656c9a613a Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:00:33 +0000 Subject: [PATCH 35/44] differentiate between internal and external gitea hosts --- data_safe_haven/infrastructure/programs/sre/workspaces.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/workspaces.py b/data_safe_haven/infrastructure/programs/sre/workspaces.py index b48de97668..3c57a18530 100644 --- a/data_safe_haven/infrastructure/programs/sre/workspaces.py +++ b/data_safe_haven/infrastructure/programs/sre/workspaces.py @@ -28,7 +28,7 @@ def __init__( data_collection_endpoint_id: Input[str], data_collection_rule_id: Input[str], database_service_admin_password: Input[str], - gitea_hostname: Input[str], + internal_gitea_hostname: Input[str], hedgedoc_hostname: Input[str], ldap_group_filter: Input[str], ldap_group_search_base: Input[str], @@ -48,6 +48,7 @@ def __init__( subscription_name: Input[str], virtual_network: Input[network.VirtualNetwork], vm_details: list[tuple[int, str]], # this must *not* be passed as an Input[T] + external_gitea_hostname: Input[str] | None = None, ) -> None: self.admin_password = Output.secret(admin_password) self.admin_username = "dshadmin" @@ -56,7 +57,8 @@ def __init__( self.data_collection_rule_id = data_collection_rule_id self.data_collection_endpoint_id = data_collection_endpoint_id self.database_service_admin_password = database_service_admin_password - self.gitea_hostname = gitea_hostname + self.internal_gitea_hostname = internal_gitea_hostname + self.external_gitea_hostname = external_gitea_hostname self.hedgedoc_hostname = hedgedoc_hostname self.ldap_group_filter = ldap_group_filter self.ldap_group_search_base = ldap_group_search_base @@ -120,7 +122,7 @@ def __init__( apt_proxy_server_hostname=props.apt_proxy_server_hostname, clamav_mirror_hostname=props.clamav_mirror_hostname, database_service_admin_password=props.database_service_admin_password, - gitea_hostname=props.gitea_hostname, + gitea_hostname=props.internal_gitea_hostname, hedgedoc_hostname=props.hedgedoc_hostname, ldap_group_filter=props.ldap_group_filter, ldap_group_search_base=props.ldap_group_search_base, From 92669fcda3a779c322591e6314dfaf7d91d7cea5 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:01:35 +0000 Subject: [PATCH 36/44] add some test NSG rules --- .../infrastructure/programs/sre/networking.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/sre/networking.py b/data_safe_haven/infrastructure/programs/sre/networking.py index be1d8a1edd..9cc14aef4b 100644 --- a/data_safe_haven/infrastructure/programs/sre/networking.py +++ b/data_safe_haven/infrastructure/programs/sre/networking.py @@ -100,6 +100,18 @@ def __init__( source_address_prefix="*", source_port_range="*", ), + network.SecurityRuleArgs( + access=network.SecurityRuleAccess.ALLOW, + description="Allow inbound connections from user container services.", + destination_address_prefix=SREIpRanges.external_git_mirror.prefix, + destination_port_range="*", + direction=network.SecurityRuleDirection.INBOUND, + name="AllowGitServersInbound", + priority=NetworkingPriorities.INTERNAL_SRE_EXTERNAL_GIT_MIRROR, + protocol=network.SecurityRuleProtocol.ASTERISK, + source_address_prefix=SREIpRanges.user_services_containers.prefix, + source_port_range="*", + ), ], opts=child_opts, tags=child_tags, @@ -1777,6 +1789,13 @@ def __init__( # User services external git mirror network.SubnetArgs( address_prefix=SREIpRanges.external_git_mirror.prefix, + delegations=[ + network.DelegationArgs( + name="SubnetDelegationContainerGroups", + service_name="Microsoft.ContainerInstance/containerGroups", + type="Microsoft.Network/virtualNetworks/subnets/delegations", + ), + ], name=subnet_external_git_mirror_name, network_security_group=network.NetworkSecurityGroupArgs( id=nsg_external_git_mirror.id From 343b20c9865db5405b47007c713d0372783c1910 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:02:04 +0000 Subject: [PATCH 37/44] differentiate between internal and external gitea servers --- data_safe_haven/infrastructure/programs/declarative_sre.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index ccfa066963..3cdb442375 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -320,7 +320,7 @@ def __call__(self) -> None: subnet_containers=networking.subnet_user_services_containers, subnet_containers_support=networking.subnet_user_services_containers_support, subnet_databases=networking.subnet_user_services_databases, - # subnet_external_git_mirror=networking.subnet_user_services_external_git_mirror, + subnet_external_git_mirror=networking.subnet_external_git_mirror, subnet_software_repositories=networking.subnet_user_services_software_repositories, ), tags=self.tags, @@ -351,7 +351,8 @@ def __call__(self) -> None: data_collection_rule_id=monitoring.data_collection_rule_vms.id, data_collection_endpoint_id=monitoring.data_collection_endpoint.id, database_service_admin_password=data.password_database_service_admin, - gitea_hostname=user_services.gitea_server.hostname, + internal_gitea_hostname=user_services.gitea_server[1].hostname, + external_gitea_hostname=user_services.gitea_server[0].hostname, hedgedoc_hostname=user_services.hedgedoc_server.hostname, ldap_group_filter=ldap_group_filter, ldap_group_search_base=ldap_group_search_base, From 3b7e1f958ad4f767a77068d5f91a87150403ae39 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:02:22 +0000 Subject: [PATCH 38/44] create list of gitea servers --- .../programs/sre/user_services.py | 62 ++++++++++--------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/user_services.py b/data_safe_haven/infrastructure/programs/sre/user_services.py index 16c2168db5..086e231cf1 100644 --- a/data_safe_haven/infrastructure/programs/sre/user_services.py +++ b/data_safe_haven/infrastructure/programs/sre/user_services.py @@ -45,7 +45,7 @@ def __init__( subnet_containers: Input[network.GetSubnetResult], subnet_containers_support: Input[network.GetSubnetResult], subnet_databases: Input[network.GetSubnetResult], - # subnet_external_git_mirror: Input[network.GetSubnetResult], + subnet_external_git_mirror: Input[network.GetSubnetResult], subnet_software_repositories: Input[network.GetSubnetResult], ) -> None: self.database_service_admin_password = database_service_admin_password @@ -76,9 +76,9 @@ def __init__( self.subnet_databases_id = Output.from_input(subnet_databases).apply( get_id_from_subnet ) - # self.subnet_external_git_mirror_id = Output.from_input( - # subnet_external_git_mirror - # ).apply(get_id_from_subnet) + self.subnet_external_git_mirror_id = Output.from_input( + subnet_external_git_mirror + ).apply(get_id_from_subnet) self.subnet_software_repositories_id = Output.from_input( subnet_software_repositories ).apply(get_id_from_subnet) @@ -100,37 +100,41 @@ def __init__( child_tags = {"component": "user services"} | (tags if tags else {}) # Deploy the Gitea servers + self.gitea_server = [] if props.external_git_mirror: gitea_servers = ["external", "internal"] - subnet = [props.subnet_containers_id, props.subnet_containers_id] + subnet = [props.subnet_containers_id, props.subnet_external_git_mirror_id] else: gitea_servers = ["internal"] subnet = [props.subnet_containers_id] + self.gitea_server.append(None) - for gitea_server in gitea_servers: - SREGiteaServerComponent( - f"sre_{gitea_server}_gitea_server", - stack_name, - SREGiteaServerProps( - containers_subnet_id=subnet[0], - database_subnet_id=props.subnet_containers_support_id, - database_password=props.gitea_database_password, - dns_server_ip=props.dns_server_ip, - dockerhub_credentials=props.dockerhub_credentials, - gitea_server=gitea_server, - ldap_server_hostname=props.ldap_server_hostname, - ldap_server_port=props.ldap_server_port, - ldap_username_attribute=props.ldap_username_attribute, - ldap_user_filter=props.ldap_user_filter, - ldap_user_search_base=props.ldap_user_search_base, - location=props.location, - resource_group_name=props.resource_group_name, - sre_fqdn=props.sre_fqdn, - storage_account_key=props.storage_account_key, - storage_account_name=props.storage_account_name, - ), - opts=child_opts, - tags=child_tags, + for index, gitea_server in enumerate(gitea_servers): + self.gitea_server.append( + SREGiteaServerComponent( + f"sre_{gitea_server}_gitea_server", + stack_name, + SREGiteaServerProps( + containers_subnet_id=subnet[index], + database_subnet_id=props.subnet_containers_support_id, + database_password=props.gitea_database_password, + dns_server_ip=props.dns_server_ip, + dockerhub_credentials=props.dockerhub_credentials, + gitea_server=gitea_server, + ldap_server_hostname=props.ldap_server_hostname, + ldap_server_port=props.ldap_server_port, + ldap_username_attribute=props.ldap_username_attribute, + ldap_user_filter=props.ldap_user_filter, + ldap_user_search_base=props.ldap_user_search_base, + location=props.location, + resource_group_name=props.resource_group_name, + sre_fqdn=props.sre_fqdn, + storage_account_key=props.storage_account_key, + storage_account_name=props.storage_account_name, + ), + opts=child_opts, + tags=child_tags, + ) ) # Deploy the HedgeDoc server From e8d3488817cd8ac6dd44dbf9c8d0881c111a7c80 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:16:49 +0000 Subject: [PATCH 39/44] add type hint --- data_safe_haven/infrastructure/programs/sre/user_services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/user_services.py b/data_safe_haven/infrastructure/programs/sre/user_services.py index 086e231cf1..01242c9c0f 100644 --- a/data_safe_haven/infrastructure/programs/sre/user_services.py +++ b/data_safe_haven/infrastructure/programs/sre/user_services.py @@ -100,9 +100,9 @@ def __init__( child_tags = {"component": "user services"} | (tags if tags else {}) # Deploy the Gitea servers - self.gitea_server = [] + self.gitea_server = [] # type: list[str] if props.external_git_mirror: - gitea_servers = ["external", "internal"] + gitea_servers = ["internal", "external"] subnet = [props.subnet_containers_id, props.subnet_external_git_mirror_id] else: gitea_servers = ["internal"] From 72e7cc168937651c7bf8dc655ed7dda527904579 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 16 Sep 2024 10:17:00 +0000 Subject: [PATCH 40/44] swap gitea over --- data_safe_haven/infrastructure/programs/declarative_sre.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 3cdb442375..b3470240ff 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -351,8 +351,8 @@ def __call__(self) -> None: data_collection_rule_id=monitoring.data_collection_rule_vms.id, data_collection_endpoint_id=monitoring.data_collection_endpoint.id, database_service_admin_password=data.password_database_service_admin, - internal_gitea_hostname=user_services.gitea_server[1].hostname, - external_gitea_hostname=user_services.gitea_server[0].hostname, + internal_gitea_hostname=user_services.gitea_server[0].hostname, + external_gitea_hostname=user_services.gitea_server[1].hostname, hedgedoc_hostname=user_services.hedgedoc_server.hostname, ldap_group_filter=ldap_group_filter, ldap_group_search_base=ldap_group_search_base, From cf31e765374802852509e20225526feb5c68cb0f Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:52:34 +0000 Subject: [PATCH 41/44] Capture gitea hostnames correctly --- data_safe_haven/infrastructure/programs/sre/desired_state.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/desired_state.py b/data_safe_haven/infrastructure/programs/sre/desired_state.py index 78e2c22c7f..a73593ceb7 100644 --- a/data_safe_haven/infrastructure/programs/sre/desired_state.py +++ b/data_safe_haven/infrastructure/programs/sre/desired_state.py @@ -65,7 +65,8 @@ def __init__( self.clamav_mirror_hostname = clamav_mirror_hostname self.database_service_admin_password = database_service_admin_password self.dns_private_zones = dns_private_zones - self.gitea_hostname = internal_gitea_hostname + self.internal_gitea_hostname = internal_gitea_hostname + self.external_gitea_hostname = external_gitea_hostname self.hedgedoc_hostname = hedgedoc_hostname self.ldap_group_filter = ldap_group_filter self.ldap_group_search_base = ldap_group_search_base From dfb6d7e3a5ae55e3d24d7ce4eda9296402d8866f Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 23 Sep 2024 15:43:18 +0000 Subject: [PATCH 42/44] Use dicts rather than lists for gitea server related variables --- .../programs/sre/user_services.py | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/user_services.py b/data_safe_haven/infrastructure/programs/sre/user_services.py index 01242c9c0f..66ccdae544 100644 --- a/data_safe_haven/infrastructure/programs/sre/user_services.py +++ b/data_safe_haven/infrastructure/programs/sre/user_services.py @@ -100,41 +100,41 @@ def __init__( child_tags = {"component": "user services"} | (tags if tags else {}) # Deploy the Gitea servers - self.gitea_server = [] # type: list[str] + self.gitea_server = {} # type: dict[str, SREGiteaServerComponent] if props.external_git_mirror: gitea_servers = ["internal", "external"] - subnet = [props.subnet_containers_id, props.subnet_external_git_mirror_id] + subnet = { + "internal": props.subnet_containers_id, + "external": props.subnet_external_git_mirror_id, + } else: gitea_servers = ["internal"] - subnet = [props.subnet_containers_id] - self.gitea_server.append(None) + subnet = {"internal": props.subnet_containers_id} - for index, gitea_server in enumerate(gitea_servers): - self.gitea_server.append( - SREGiteaServerComponent( - f"sre_{gitea_server}_gitea_server", - stack_name, - SREGiteaServerProps( - containers_subnet_id=subnet[index], - database_subnet_id=props.subnet_containers_support_id, - database_password=props.gitea_database_password, - dns_server_ip=props.dns_server_ip, - dockerhub_credentials=props.dockerhub_credentials, - gitea_server=gitea_server, - ldap_server_hostname=props.ldap_server_hostname, - ldap_server_port=props.ldap_server_port, - ldap_username_attribute=props.ldap_username_attribute, - ldap_user_filter=props.ldap_user_filter, - ldap_user_search_base=props.ldap_user_search_base, - location=props.location, - resource_group_name=props.resource_group_name, - sre_fqdn=props.sre_fqdn, - storage_account_key=props.storage_account_key, - storage_account_name=props.storage_account_name, - ), - opts=child_opts, - tags=child_tags, - ) + for gitea_server in gitea_servers: + self.gitea_server[gitea_server] = SREGiteaServerComponent( + f"sre_{gitea_server}_gitea_server", + stack_name, + SREGiteaServerProps( + containers_subnet_id=subnet[gitea_server], + database_subnet_id=props.subnet_containers_support_id, + database_password=props.gitea_database_password, + dns_server_ip=props.dns_server_ip, + dockerhub_credentials=props.dockerhub_credentials, + gitea_server=gitea_server, + ldap_server_hostname=props.ldap_server_hostname, + ldap_server_port=props.ldap_server_port, + ldap_username_attribute=props.ldap_username_attribute, + ldap_user_filter=props.ldap_user_filter, + ldap_user_search_base=props.ldap_user_search_base, + location=props.location, + resource_group_name=props.resource_group_name, + sre_fqdn=props.sre_fqdn, + storage_account_key=props.storage_account_key, + storage_account_name=props.storage_account_name, + ), + opts=child_opts, + tags=child_tags, ) # Deploy the HedgeDoc server From b53ac701f8f1efc11107cb58c0c43e71d322555d Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 24 Sep 2024 09:01:16 +0000 Subject: [PATCH 43/44] Fix setting hostnames for external and internal gitea --- .../infrastructure/programs/sre/desired_state.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/desired_state.py b/data_safe_haven/infrastructure/programs/sre/desired_state.py index a73593ceb7..b8d68ef890 100644 --- a/data_safe_haven/infrastructure/programs/sre/desired_state.py +++ b/data_safe_haven/infrastructure/programs/sre/desired_state.py @@ -46,7 +46,7 @@ def __init__( clamav_mirror_hostname: Input[str], database_service_admin_password: Input[str], dns_private_zones: Input[dict[str, network.PrivateZone]], - internal_gitea_hostname: Input[str], + gitea_hostnames: Input[dict[str, str]], hedgedoc_hostname: Input[str], ldap_group_filter: Input[str], ldap_group_search_base: Input[str], @@ -59,14 +59,13 @@ def __init__( software_repository_hostname: Input[str], subscription_name: Input[str], subnet_desired_state: Input[network.GetSubnetResult], - external_gitea_hostname: Input[str] | None = None, ) -> None: self.admin_ip_addresses = admin_ip_addresses self.clamav_mirror_hostname = clamav_mirror_hostname self.database_service_admin_password = database_service_admin_password self.dns_private_zones = dns_private_zones - self.internal_gitea_hostname = internal_gitea_hostname - self.external_gitea_hostname = external_gitea_hostname + self.internal_gitea_hostname = gitea_hostnames.get("internal") + self.external_gitea_hostname = gitea_hostnames.get("external") self.hedgedoc_hostname = hedgedoc_hostname self.ldap_group_filter = ldap_group_filter self.ldap_group_search_base = ldap_group_search_base From 36a8cd6cc13533ba93847246372b40ab5b19acd8 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 24 Sep 2024 09:01:58 +0000 Subject: [PATCH 44/44] populate gitea_hostnames --- data_safe_haven/infrastructure/programs/declarative_sre.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 573d787788..79a9dcba74 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -349,8 +349,10 @@ def __call__(self) -> None: clamav_mirror_hostname=clamav_mirror.hostname, database_service_admin_password=data.password_database_service_admin, dns_private_zones=dns.private_zones, - internal_gitea_hostname=user_services.gitea_server[0].hostname, - external_gitea_hostname=user_services.gitea_server[1].hostname, + gitea_hostnames={ + availability: server.hostname + for (availability, server) in user_services.gitea_server.items() + }, hedgedoc_hostname=user_services.hedgedoc_server.hostname, ldap_group_filter=ldap_group_filter, ldap_group_search_base=ldap_group_search_base,