From 36fa31675674211c2942c08a1a7ffa8343b5bb20 Mon Sep 17 00:00:00 2001 From: David Martos Date: Mon, 1 Jul 2024 16:42:27 +0200 Subject: [PATCH] Upstream changes e518919d - fix(cli): Use an IPV6-enabled network in CLI's compose file (#1299) 2024-07-01 09:46:10 +0000 - Oleksii Sholik https://github.com/electric-sql/electric/commit/e518919d --- .../assets/docker/compose-with-postgres.yaml | 63 ------------------- .../assets/docker/compose.hostnet.yaml | 13 ++++ .../assets/docker/compose.ip6net.yaml | 55 ++++++++++++++++ .../assets/docker/compose.yaml | 14 +++++ .../docker_commands/command_start.dart | 7 +-- .../docker_commands/docker_utils.dart | 48 ++++++++++++-- .../lib/src/config_options.dart | 9 ++- .../test/src/cli/config_options_test.dart | 1 + 8 files changed, 137 insertions(+), 73 deletions(-) delete mode 100644 packages/electricsql_cli/assets/docker/compose-with-postgres.yaml create mode 100644 packages/electricsql_cli/assets/docker/compose.hostnet.yaml create mode 100644 packages/electricsql_cli/assets/docker/compose.ip6net.yaml diff --git a/packages/electricsql_cli/assets/docker/compose-with-postgres.yaml b/packages/electricsql_cli/assets/docker/compose-with-postgres.yaml deleted file mode 100644 index 9ce7aeef..00000000 --- a/packages/electricsql_cli/assets/docker/compose-with-postgres.yaml +++ /dev/null @@ -1,63 +0,0 @@ -configs: - postgres_config: - file: './postgres.conf' - -volumes: - pg_data: - -services: - postgres: - profiles: ['with-postgres'] - image: '${POSTGRESQL_IMAGE:-postgres:14-alpine}' - environment: - POSTGRES_DB: ${DATABASE_NAME:-electric} - POSTGRES_USER: ${DATABASE_USER:-postgres} - POSTGRES_PASSWORD: ${DATABASE_PASSWORD:-db_password} - command: - - -c - - config_file=/etc/postgresql.conf - - -p - - ${DATABASE_PORT:-5432} - configs: - - source: postgres_config - target: /etc/postgresql.conf - healthcheck: - test: - [ - 'CMD-SHELL', - 'pg_isready -U ${DATABASE_USER:-postgres} -p ${DATABASE_PORT:-5432}', - ] - extra_hosts: - - 'host.docker.internal:host-gateway' - ports: - - ${DATABASE_PORT:-5432}:${DATABASE_PORT:-5432} - volumes: - - pg_data:/var/lib/postgresql/data - - electric: - image: '${ELECTRIC_IMAGE:-electricsql/electric:latest}' - init: true - stop_signal: SIGINT # use SIGINT as the more speedy alternative to SIGTERM - ports: - - ${HTTP_PORT:-5133}:${HTTP_PORT:-5133} - - ${PG_PROXY_PORT_PARSED:-65432}:${PG_PROXY_PORT_PARSED:-65432} - environment: - DATABASE_REQUIRE_SSL: ${DATABASE_REQUIRE_SSL:-} - DATABASE_URL: ${DATABASE_URL:-} - DATABASE_USE_IPV6: ${DATABASE_USE_IPV6:-} - ELECTRIC_USE_IPV6: ${ELECTRIC_USE_IPV6:-} - HTTP_PORT: ${HTTP_PORT:-5133} - ELECTRIC_WRITE_TO_PG_MODE: ${ELECTRIC_WRITE_TO_PG_MODE:-} - LOGICAL_PUBLISHER_HOST: ${LOGICAL_PUBLISHER_HOST:-} - LOGICAL_PUBLISHER_PORT: ${LOGICAL_PUBLISHER_PORT:-5433} - PG_PROXY_PASSWORD: ${PG_PROXY_PASSWORD:-proxy_password} - PG_PROXY_PORT: ${PG_PROXY_PORT:-65432} - AUTH_MODE: ${AUTH_MODE:-insecure} - AUTH_JWT_ALG: ${AUTH_JWT_ALG:-} - AUTH_JWT_AUD: ${AUTH_JWT_AUD:-} - AUTH_JWT_ISS: ${AUTH_JWT_ISS:-} - AUTH_JWT_KEY: ${AUTH_JWT_KEY:-} - AUTH_JWT_NAMESPACE: ${AUTH_JWT_NAMESPACE:-} - ELECTRIC_FEATURES: ${ELECTRIC_FEATURES:-} - depends_on: - - postgres diff --git a/packages/electricsql_cli/assets/docker/compose.hostnet.yaml b/packages/electricsql_cli/assets/docker/compose.hostnet.yaml new file mode 100644 index 00000000..5101861d --- /dev/null +++ b/packages/electricsql_cli/assets/docker/compose.hostnet.yaml @@ -0,0 +1,13 @@ +# This extra compose file is meant to be used together with the main compose.yaml file when +# running Docker Compose commands. +# +# In this file we configure both services to use the host network mode, side-stepping Docker +# networking entirely. + +services: + electric: + network_mode: host + + postgres: + # Postgres must be on the same network as the sync service. + network_mode: host diff --git a/packages/electricsql_cli/assets/docker/compose.ip6net.yaml b/packages/electricsql_cli/assets/docker/compose.ip6net.yaml new file mode 100644 index 00000000..acd7cac5 --- /dev/null +++ b/packages/electricsql_cli/assets/docker/compose.ip6net.yaml @@ -0,0 +1,55 @@ +# This extra compose file is meant to be used together with the main compose.yaml file when +# running Docker Compose commands. +# +# In this file we define an IPv6-enabled network and assign it to both "electric" and +# "postgres" services. + +networks: + # An IPv6-enabled network is necessary for the sync service to connect to + # databases that are only reachable at IPv6 addresses. IPv4 can also be used as usual with + # this network. + ip6net: + enable_ipv6: true + # These two options provide an escape hatch that allows users to define their own Docker + # network using `docker network create` and use that for the services defined in this + # compose file. + # + # These environment variables are set by the CLI and are derived from the single + # user-facing `ELECTRIC_DOCKER_NETWORK_USE_EXTERNAL` variable. + external: ${ELECTRIC_COMPOSE_NETWORK_IS_EXTERNAL} + name: '${ELECTRIC_COMPOSE_EXTERNAL_NETWORK_NAME}' + ipam: + config: + # Subnet definition isn't required if the Docker daemon has a default address pool for + # IPv6 addresses configured. However, since that's not the case for Docker + # out-of-the-box (at least until version 27.0.1) and since we want to free our users + # from additional manual configuration when possible, we include a default subnet + # definition here that should work with any Docker daemon configuration. + # + # The fd00:: prefix is part of the address space reserved for Unique Local Addresses, + # i.e. private networks. This is analogous to 192.168.x.x in the IPv4 land. + # + # There is a possibility that this subnet overlaps with another network configured + # separately for the same Docker daemon. That situation would result in `docker compose + # up` failing with the error message + # + # Error response from daemon: Pool overlaps with other one on this address space + # + # To resolve this conflict, an external Docker network can be used by setting + # `ELECTRIC_DOCKER_NETWORK_USE_EXTERNAL` to its name. + # + # The default subnet here has 2 randomly generated bytes in the 2nd and the 6th groups, + # it can accommodate 4 addresses which is small enough to further reduce any chance of conflicts. + - subnet: 'fd00:56f0::4acd:f0fc/126' + +services: + electric: + networks: + # Despite the name, assigning this network to the sync service does not preclude the use + # of IPv4. + - ip6net + + postgres: + networks: + # Postgres must be on the same network as the sync service. + - ip6net diff --git a/packages/electricsql_cli/assets/docker/compose.yaml b/packages/electricsql_cli/assets/docker/compose.yaml index 8691aeec..cb3c0391 100644 --- a/packages/electricsql_cli/assets/docker/compose.yaml +++ b/packages/electricsql_cli/assets/docker/compose.yaml @@ -1,3 +1,17 @@ +# This compose file is used by the CLI `start` command to run the sync service and, optionally, +# a Postgres database inside Docker containers. +# +# When a user wants to start only the sync service by executing `electric-sql start`, the CLI +# will execute `docker compose up ... electric`, omitting the "postgres" service, so only the +# "electric" service will be running. +# +# When `electric-sql start --with-postgres` is invoked, the CLI will execute `docker compose up +# ... postgres electric`. In this case, the sync service may spend some time in a reconnection loop +# until the Postgres database is ready to accept connections. This is a normal mode of operation. +# +# Docker Compose should be invoked with this file and one of the two extra +# files, compose.hostnest.yaml or compose.ip6net.yaml, depending on the chosen network mode. + configs: postgres_config: file: './postgres.conf' diff --git a/packages/electricsql_cli/lib/src/commands/docker_commands/command_start.dart b/packages/electricsql_cli/lib/src/commands/docker_commands/command_start.dart index 0d847050..bf05360a 100644 --- a/packages/electricsql_cli/lib/src/commands/docker_commands/command_start.dart +++ b/packages/electricsql_cli/lib/src/commands/docker_commands/command_start.dart @@ -103,11 +103,8 @@ Future runStartCommand({ finalLogger.info('Docker compose config:'); printConfig(finalLogger, Config(dockerConfig)); - final proc = await dockerCompose( - 'up', - [ - ...(detach == true ? ['--detach'] : []), - ], + final proc = await dockerComposeUp( + userArgs: detach == true ? ['--detach'] : [], containerName: config.read('CONTAINER_NAME'), env: dockerConfig, ); diff --git a/packages/electricsql_cli/lib/src/commands/docker_commands/docker_utils.dart b/packages/electricsql_cli/lib/src/commands/docker_commands/docker_utils.dart index a5ebbd3c..564832b0 100644 --- a/packages/electricsql_cli/lib/src/commands/docker_commands/docker_utils.dart +++ b/packages/electricsql_cli/lib/src/commands/docker_commands/docker_utils.dart @@ -5,6 +5,17 @@ import 'package:electricsql_cli/src/commands/docker_commands/precheck.dart'; import 'package:electricsql_cli/src/util/util.dart'; import 'package:path/path.dart' as path; +bool _useExternalDockerNetwork(String? networkName) { + return networkName != null && networkName.isNotEmpty; +} + +// Derive network name from the current working directory, matching Docker Compose's default +// naming. +String _deriveNetworkName(String? networkName) { + if (networkName != null && networkName.isNotEmpty) return networkName; + return '${path.basename(Directory.current.path)}_ip6net'; +} + Future dockerCompose( String command, List userArgs, { @@ -15,16 +26,21 @@ Future dockerCompose( final assetsDir = await getElectricCLIAssetsDir(); final composeFile = File(path.join(assetsDir.path, 'docker/compose.yaml')); - final composeFileWithPostgres = - File(path.join(assetsDir.path, 'docker/compose-with-postgres.yaml')); - final withPostgres = env['COMPOSE_PROFILES'] == 'with-postgres'; + final extraComposeFileName = env['DOCKER_NETWORK_USE_EXTERNAL'] == 'host' + ? 'compose.hostnet.yaml' + : 'compose.ip6net.yaml'; + final extraComposeFile = + File(path.join(assetsDir.path, 'docker', extraComposeFileName)); + final args = [ 'compose', '--ansi', 'always', '-f', - (withPostgres ? composeFileWithPostgres : composeFile).absolute.path, + composeFile.absolute.path, + '-f', + extraComposeFile.absolute.path, command, ...userArgs, ]; @@ -34,6 +50,11 @@ Future dockerCompose( args, mode: ProcessStartMode.inheritStdio, environment: { + 'ELECTRIC_COMPOSE_NETWORK_IS_EXTERNAL': + _useExternalDockerNetwork(env['DOCKER_NETWORK_USE_EXTERNAL']) + .toString(), + 'ELECTRIC_COMPOSE_EXTERNAL_NETWORK_NAME': + _deriveNetworkName(env['DOCKER_NETWORK_USE_EXTERNAL']), ...Platform.environment, if (containerName != null) 'COMPOSE_PROJECT_NAME': containerName, ...env, @@ -43,6 +64,25 @@ Future dockerCompose( return proc; } +Future dockerComposeUp({ + List userArgs = const [], + String? containerName, + Map env = const {}, +}) { + // We use the same compose.yaml file for `electric-sql start` and `electric-sql start + // --with-postgres` and vary the services started by passing them as arguments to `docker + // compose up`. + final List services = env['COMPOSE_PROFILES'] == 'with-postgres' + ? ['postgres', 'electric'] + : ['electric']; + return dockerCompose( + 'up', + [...userArgs, ...services], + containerName: containerName, + env: env, + ); +} + Future _assertValidDocker() async { // Check that a valid Docker is installed final res = await checkValidDockerVersion(); diff --git a/packages/electricsql_cli/lib/src/config_options.dart b/packages/electricsql_cli/lib/src/config_options.dart index 5afad5e3..fcd80d5b 100644 --- a/packages/electricsql_cli/lib/src/config_options.dart +++ b/packages/electricsql_cli/lib/src/config_options.dart @@ -302,7 +302,14 @@ final _kConfigOptions = >{ 'ELECTRIC_FEATURES': ConfigOption( valueTypeName: 'features', defaultValue: '', - doc: 'Flags to enable experimental features', + doc: 'Flags to enable experimental features.', + groups: ['electric'], + ), + 'DOCKER_NETWORK_USE_EXTERNAL': ConfigOption( + valueTypeName: "'host' | name", + defaultValue: '', + doc: + "Name of an existing Docker network to use or 'host' to run the container using the host OS networking.", groups: ['electric'], ), }; diff --git a/packages/electricsql_cli/test/src/cli/config_options_test.dart b/packages/electricsql_cli/test/src/cli/config_options_test.dart index 5f86daa5..7df8d170 100644 --- a/packages/electricsql_cli/test/src/cli/config_options_test.dart +++ b/packages/electricsql_cli/test/src/cli/config_options_test.dart @@ -35,6 +35,7 @@ const expectedEnvVars = [ 'ELECTRIC_IMAGE', 'CONTAINER_NAME', 'ELECTRIC_FEATURES', + 'DOCKER_NETWORK_USE_EXTERNAL', ]; void main() {