Upstream changes

e518919d - fix(cli): Use an IPV6-enabled network in CLI's compose file (#1299)
2024-07-01 09:46:10 +0000 - Oleksii Sholik
electric-sql/electric@e518919d

packages/electricsql_cli/assets/docker/compose.hostnet.yaml

@@ -0,0 +1,13 @@
+# This extra compose file is meant to be used together with the main compose.yaml file when
+# running Docker Compose commands.
+#
+# In this file we configure both services to use the host network mode, side-stepping Docker
+# networking entirely.
+
+services:
+  electric:
+    network_mode: host
+
+  postgres:
+    # Postgres must be on the same network as the sync service.
+    network_mode: host

packages/electricsql_cli/assets/docker/compose.ip6net.yaml

@@ -0,0 +1,55 @@
+# This extra compose file is meant to be used together with the main compose.yaml file when
+# running Docker Compose commands.
+#
+# In this file we define an IPv6-enabled network and assign it to both "electric" and
+# "postgres" services.
+
+networks:
+  # An IPv6-enabled network is necessary for the sync service to connect to
+  # databases that are only reachable at IPv6 addresses. IPv4 can also be used as usual with
+  # this network.
+  ip6net:
+    enable_ipv6: true
+    # These two options provide an escape hatch that allows users to define their own Docker
+    # network using `docker network create` and use that for the services defined in this
+    # compose file.
+    #
+    # These environment variables are set by the CLI and are derived from the single
+    # user-facing `ELECTRIC_DOCKER_NETWORK_USE_EXTERNAL` variable.
+    external: ${ELECTRIC_COMPOSE_NETWORK_IS_EXTERNAL}
+    name: '${ELECTRIC_COMPOSE_EXTERNAL_NETWORK_NAME}'
+    ipam:
+      config:
+        # Subnet definition isn't required if the Docker daemon has a default address pool for
+        # IPv6 addresses configured. However, since that's not the case for Docker
+        # out-of-the-box (at least until version 27.0.1) and since we want to free our users
+        # from additional manual configuration when possible, we include a default subnet
+        # definition here that should work with any Docker daemon configuration.
+        #
+        # The fd00:: prefix is part of the address space reserved for Unique Local Addresses,
+        # i.e. private networks. This is analogous to 192.168.x.x in the IPv4 land.
+        #
+        # There is a possibility that this subnet overlaps with another network configured
+        # separately for the same Docker daemon. That situation would result in `docker compose
+        # up` failing with the error message
+        #
+        #     Error response from daemon: Pool overlaps with other one on this address space
+        #
+        # To resolve this conflict, an external Docker network can be used by setting
+        # `ELECTRIC_DOCKER_NETWORK_USE_EXTERNAL` to its name.
+        #
+        # The default subnet here has 2 randomly generated bytes in the 2nd and the 6th groups,
+        # it can accommodate 4 addresses which is small enough to further reduce any chance of conflicts.
+        - subnet: 'fd00:56f0::4acd:f0fc/126'
+
+services:
+  electric:
+    networks:
+      # Despite the name, assigning this network to the sync service does not preclude the use
+      # of IPv4.
+      - ip6net
+
+  postgres:
+    networks:
+      # Postgres must be on the same network as the sync service.
+      - ip6net

packages/electricsql_cli/assets/docker/compose.yaml

@@ -1,3 +1,17 @@
+# This compose file is used by the CLI `start` command to run the sync service and, optionally,
+# a Postgres database inside Docker containers.
+#
+# When a user wants to start only the sync service by executing `electric-sql start`, the CLI
+# will execute `docker compose up ... electric`, omitting the "postgres" service, so only the
+# "electric" service will be running.
+#
+# When `electric-sql start --with-postgres` is invoked, the CLI will execute `docker compose up
+# ... postgres electric`. In this case, the sync service may spend some time in a reconnection loop
+# until the Postgres database is ready to accept connections. This is a normal mode of operation.
+#
+# Docker Compose should be invoked with this file and one of the two extra
+# files, compose.hostnest.yaml or compose.ip6net.yaml, depending on the chosen network mode.
+
 configs:
   postgres_config:
     file: './postgres.conf'

packages/electricsql_cli/lib/src/commands/docker_commands/command_start.dart

@@ -103,11 +103,8 @@ Future<void> runStartCommand({
   finalLogger.info('Docker compose config:');
   printConfig(finalLogger, Config(dockerConfig));
 
-  final proc = await dockerCompose(
-    'up',
-    [
-      ...(detach == true ? ['--detach'] : []),
-    ],
+  final proc = await dockerComposeUp(
+    userArgs: detach == true ? ['--detach'] : [],
     containerName: config.read<String>('CONTAINER_NAME'),
     env: dockerConfig,
   );

packages/electricsql_cli/lib/src/commands/docker_commands/docker_utils.dart

@@ -5,6 +5,17 @@ import 'package:electricsql_cli/src/commands/docker_commands/precheck.dart';
 import 'package:electricsql_cli/src/util/util.dart';
 import 'package:path/path.dart' as path;
 
+bool _useExternalDockerNetwork(String? networkName) {
+  return networkName != null && networkName.isNotEmpty;
+}
+
+// Derive network name from the current working directory, matching Docker Compose's default
+// naming.
+String _deriveNetworkName(String? networkName) {
+  if (networkName != null && networkName.isNotEmpty) return networkName;
+  return '${path.basename(Directory.current.path)}_ip6net';
+}
+
 Future<Process> dockerCompose(
   String command,
   List<String> userArgs, {
@@ -15,16 +26,21 @@ Future<Process> dockerCompose(
 
   final assetsDir = await getElectricCLIAssetsDir();
   final composeFile = File(path.join(assetsDir.path, 'docker/compose.yaml'));
-  final composeFileWithPostgres =
-      File(path.join(assetsDir.path, 'docker/compose-with-postgres.yaml'));
 
-  final withPostgres = env['COMPOSE_PROFILES'] == 'with-postgres';
+  final extraComposeFileName = env['DOCKER_NETWORK_USE_EXTERNAL'] == 'host'
+      ? 'compose.hostnet.yaml'
+      : 'compose.ip6net.yaml';
+  final extraComposeFile =
+      File(path.join(assetsDir.path, 'docker', extraComposeFileName));
+
   final args = <String>[
     'compose',
     '--ansi',
     'always',
     '-f',
-    (withPostgres ? composeFileWithPostgres : composeFile).absolute.path,
+    composeFile.absolute.path,
+    '-f',
+    extraComposeFile.absolute.path,
     command,
     ...userArgs,
   ];
@@ -34,6 +50,11 @@ Future<Process> dockerCompose(
     args,
     mode: ProcessStartMode.inheritStdio,
     environment: {
+      'ELECTRIC_COMPOSE_NETWORK_IS_EXTERNAL':
+          _useExternalDockerNetwork(env['DOCKER_NETWORK_USE_EXTERNAL'])
+              .toString(),
+      'ELECTRIC_COMPOSE_EXTERNAL_NETWORK_NAME':
+          _deriveNetworkName(env['DOCKER_NETWORK_USE_EXTERNAL']),
       ...Platform.environment,
       if (containerName != null) 'COMPOSE_PROJECT_NAME': containerName,
       ...env,
@@ -43,6 +64,25 @@ Future<Process> dockerCompose(
   return proc;
 }
 
+Future<Process> dockerComposeUp({
+  List<String> userArgs = const [],
+  String? containerName,
+  Map<String, String> env = const {},
+}) {
+  // We use the same compose.yaml file for `electric-sql start` and `electric-sql start
+  // --with-postgres` and vary the services started by passing them as arguments to `docker
+  // compose up`.
+  final List<String> services = env['COMPOSE_PROFILES'] == 'with-postgres'
+      ? ['postgres', 'electric']
+      : ['electric'];
+  return dockerCompose(
+    'up',
+    [...userArgs, ...services],
+    containerName: containerName,
+    env: env,
+  );
+}
+
 Future<void> _assertValidDocker() async {
   // Check that a valid Docker is installed
   final res = await checkValidDockerVersion();

packages/electricsql_cli/lib/src/config_options.dart

@@ -302,7 +302,14 @@ final _kConfigOptions = <String, ConfigOption<Object>>{
   'ELECTRIC_FEATURES': ConfigOption<String>(
     valueTypeName: 'features',
     defaultValue: '',
-    doc: 'Flags to enable experimental features',
+    doc: 'Flags to enable experimental features.',
+    groups: ['electric'],
+  ),
+  'DOCKER_NETWORK_USE_EXTERNAL': ConfigOption<String>(
+    valueTypeName: "'host' | name",
+    defaultValue: '',
+    doc:
+        "Name of an existing Docker network to use or 'host' to run the container using the host OS networking.",
     groups: ['electric'],
   ),
 };

packages/electricsql_cli/test/src/cli/config_options_test.dart

@@ -35,6 +35,7 @@ const expectedEnvVars = <String>[
   'ELECTRIC_IMAGE',
   'CONTAINER_NAME',
   'ELECTRIC_FEATURES',
+  'DOCKER_NETWORK_USE_EXTERNAL',
 ];
 
 void main() {