Upstream changes

febb710e - feat(electric): Internal API for distributed tracing using OpenTelemetry (#1370)
2024-06-27 10:19:43 +0000 - Oleksii Sholik
electric-sql/electric@febb710e

31ffde3b - Minor fixes to client Makefile (#1408)
2024-06-27 15:32:51 +0000 - Rob A'Court
electric-sql/electric@31ffde3b

d46d148b - fix(ci): Upgrade GitHub Action versions to resolve Node 16 deprecation warnings and fix pnpm lockfile compatibility (#1384)
2024-06-27 16:09:28 +0000 - Oleksii Sholik
electric-sql/electric@d46d148b

31d8418f - fix (toolbar): modify drivers dependency to be a workspace dependency (#1411)
2024-06-28 09:44:32 +0000 - Kevin
electric-sql/electric@31d8418f

ab3e89a5 - chore (client): add .tool-versions to gitignore  (#1410)
2024-06-28 09:47:18 +0000 - Kevin
electric-sql/electric@ab3e89a5

.github/workflows/e2e.yml

@@ -62,9 +62,9 @@ jobs:
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
       - name: Inject variables for `docker buildx` github actions caching
-        uses: crazy-max/ghaction-github-runtime@v2
+        uses: crazy-max/ghaction-github-runtime@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Log in to the Container registry
         uses: docker/login-action@v3
         with:
@@ -93,7 +93,7 @@ jobs:
         working-directory: ${{env.ELECTRIC_REPO}}/components/electric
 
       - name: Cache built lux
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             e2e/lux/bin

README.md

@@ -25,7 +25,7 @@ Client based on the Typescript client from the `clients/typescript` subfolder fr
 
 * [NPM package](https://www.npmjs.com/package/electric-sql).
 * Version `v0.12.1-dev`
-* Commit: `837ce928e3a887ee8a48966efd029224a585e6f1`
+* Commit: `ab3e89a5309c90b29460ccef791b5b02d5ba8e2b`
 
 
 ### What is ElectricSQL?

e2e/Makefile

@@ -1,19 +1,12 @@
 include common.mk
 
-deps: lux
-	make clone_electric
-	make -C satellite_client build
-	make -C elixir_client build
-	make -C prisma_example build
-
-test_sqlite_and_pg: test_only
-	make -C tests test_pg
+export DOCKER_COMPOSE_FILE=compose.yaml
 
 test_only:
-	${LUX} --junit tests
+	${LUX} tests/*.lux
 
 test-no-satellite:
-	find tests -type f -maxdepth 1 -name "*.lux" -and -not -name "03.*.lux" | sort -h | xargs ${LUX} --junit
+	find tests -maxdepth 1 -type f -name "*.lux" -and -not -name "03.*.lux" | sort -h | xargs ${LUX} --junit
 
 test-satellite-only:
 	${LUX} --junit tests/03.*.lux
@@ -23,11 +16,11 @@ test_only_custom:
 
 test: deps pull test_only
 
-test-no-satellite:
-	find tests -type f -maxdepth 1 -name "*.lux" -and -not -name "03.*.lux" | sort -h | xargs ${LUX} --junit
-
-test-satellite-only:
-	${LUX} --junit tests/03.*.lux
+deps: lux
+	make clone_electric
+	make -C satellite_client build
+	make -C elixir_client build
+	make -C prisma_example build
 
 pull:
 	docker compose -f services_templates.yaml pull \

e2e/common.mk

@@ -41,6 +41,8 @@ else
 	export ELECTRIC_CLIENT_IMAGE=${ELECTRIC_CLIENT_IMAGE_NAME}:${ELECTRIC_IMAGE_TAG}
 endif
 
+export DISABLE_TZDATA_AUTOUPDATE=true
+
 export ELECTRIC_COMMIT:=$(shell cd $(PROJECT_ROOT) && tool/extract_electric_commit.sh)
 
 export ELECTRIC_REPO := $(abspath $(E2E_ROOT)/electric_repo)
@@ -91,8 +93,12 @@ stop_dev_env:
 	if [ -n "`docker ps --filter name=sysbench_run --format '{{.Names}}'`" ]; then \
 		docker ps --filter name=sysbench_run --format '{{.Names}}' | xargs docker kill; \
 	fi
-	docker compose -f ${DOCKER_COMPOSE_FILE} stop --timeout 1
-	docker compose -f ${DOCKER_COMPOSE_FILE} down
+	# The timeout should allow for any remaining OT batches to be exported prior to server shutdown.
+	docker compose -f ${DOCKER_COMPOSE_FILE} stop --timeout 5 electric_1 pg_1
+	docker compose -f ${DOCKER_COMPOSE_FILE} down --timeout 5
+
+start_otel_collector:
+	docker compose -f ${DOCKER_COMPOSE_FILE} up otel-collector
 
 start_sysbench:
 	docker compose -f ${DOCKER_COMPOSE_FILE} run \

e2e/prisma_example/Dockerfile

@@ -1,6 +1,6 @@
 FROM node:18-alpine AS workspace
 
-RUN corepack enable && corepack prepare pnpm@8.6.0 --activate
+RUN corepack enable && corepack prepare pnpm@9.4.0 --activate
 RUN mkdir /app
 WORKDIR /app
 

e2e/services_templates.yaml

@@ -9,8 +9,6 @@ services:
       PG_PROXY_PASSWORD:
       PG_PROXY_PORT:
       ELECTRIC_FEATURES:
-    init: true
-    stop_signal: SIGINT  # use SIGINT as the more speedy alternative to SIGTERM
 
   sysbench:
     image: "${SYSBENCH_IMAGE}"

e2e/tests/Makefile

@@ -1,12 +1,4 @@
 include ../common.mk
 
-DOCKER_COMPOSE_FILE=compose.yaml
-
-test:
-	${LUX} *.lux
-
-test_pg:
-	DIALECT=Postgres ${LUX} 03.*.lux
-
 docker-prisma:
-	DOCKER_COMPOSE_FILE=${DOCKER_COMPOSE_FILE} ../prisma_example/run.sh $(ARGS)
+	DOCKER_COMPOSE_FILE=compose.yaml ../prisma_example/run.sh $(ARGS)

e2e/tests/_shared.luxinc

@@ -5,6 +5,10 @@
 [newshell log]
 
 [macro setup]
+    [newshell otel_collector]
+        !make start_otel_collector
+        ??Everything is ready. Begin running and processing data.
+
     [newshell postgres_logs]
         [progress start development environment]
         !make start_dev_env
@@ -18,6 +22,7 @@
     [newshell electric]
         [invoke start_electric 1]
         ??Successfully initialized Postgres connector "postgres_1"
+        -$fail_pattern
         [progress setup finished]
 
     [shell postgres_logs]
@@ -36,7 +41,6 @@
 
 [macro start_electric n]
     !make start_electric_${n}
-    -$fail_pattern
 [endmacro]
 
 [macro stop_electric n]

e2e/tests/compose.yaml

@@ -19,6 +19,9 @@ services:
       ELECTRIC_WRITE_TO_PG_MODE: "${ELECTRIC_WRITE_TO_PG_MODE:-logical_replication}"
       LOGICAL_PUBLISHER_HOST: electric_1
       PG_PROXY_LOG_LEVEL: info
+      LOG_LEVEL: "${LOG_LEVEL:-debug}"
+      OTEL_EXPORT: "${OTEL_EXPORT:-otlp}"
+      OTLP_ENDPOINT: "http://otel-collector:4318"
     ports:
       - "5133:5133"
       # proxy access
@@ -65,3 +68,12 @@ services:
       PROXY_URL: "postgresql://postgres:${PG_PROXY_PASSWORD}@electric_1:${PG_PROXY_PORT}/postgres"
     volumes:
       - ../prisma_example/prisma:/app/prisma
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib
+    environment:
+      HONEYCOMB_API_KEY: "${HONEYCOMB_API_KEY}"
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
+    ports:
+      - 4318:4318 # OTLP http receiver

e2e/tests/otel-collector-config.yaml

@@ -0,0 +1,27 @@
+receivers:
+  otlp:
+    protocols:
+      http:
+        endpoint: "0.0.0.0:4318"
+        cors:
+          allowed_origins:
+            - "*"
+
+processors:
+  batch:
+
+exporters:
+  otlp/honeycomb:
+    endpoint: https://api.honeycomb.io:443
+    headers:
+      "x-honeycomb-team": ${env:HONEYCOMB_API_KEY}
+
+  debug:
+    verbosity: detailed
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/honeycomb]

packages/electricsql_cli/lib/src/config.dart

@@ -134,7 +134,8 @@ T? getOptionalConfigValue<T>(
   // Then check if the option was passed as an environment variable
   final envName = name.startsWith('ELECTRIC_') ? name : 'ELECTRIC_$name';
   final envVal = programEnv[envName];
-  if (configOptions[name]! is ConfigOption<bool>) {
+  if (programEnv.isDefined(envName) &&
+      configOptions[name]! is ConfigOption<bool>) {
     final bool b = notBlank(envVal) &&
         !['f', 'false', '0', '', 'no'].contains(envVal!.toLowerCase());
     return b as T;

packages/electricsql_cli/lib/src/config_options.dart

@@ -1,9 +1,20 @@
 import 'package:electricsql/electricsql.dart';
 import 'package:electricsql_cli/src/config.dart';
 import 'package:electricsql_cli/src/util/util.dart';
+import 'package:meta/meta.dart';
 import 'package:path/path.dart';
 
-final configOptions = <String, ConfigOption<Object>>{
+Map<String, ConfigOption<Object>>? _debugConfigOptions;
+
+@visibleForTesting
+void setDebugConfigOptions(Map<String, ConfigOption<Object>>? options) {
+  _debugConfigOptions = options;
+}
+
+Map<String, ConfigOption<Object>> get configOptions =>
+    _debugConfigOptions ?? _kConfigOptions;
+
+final _kConfigOptions = <String, ConfigOption<Object>>{
   // *** Client options ***
   'SERVICE': ConfigOption<String>(
     valueTypeName: 'url',

packages/electricsql_cli/lib/src/env.dart

@@ -1,6 +1,7 @@
 import 'dart:io';
 
 import 'package:dotenv/dotenv.dart';
+import 'package:meta/meta.dart';
 
 DotEnv? _cachedProgramEnv;
 
@@ -12,6 +13,11 @@ DotEnv get programEnv {
   return _cachedProgramEnv!;
 }
 
+@visibleForTesting
+void setProgramEnv(DotEnv? env) {
+  _cachedProgramEnv = env;
+}
+
 DotEnv _loadEnv() {
   final envFiles = <String>[
     '.env',

packages/electricsql_cli/pubspec.yaml

@@ -18,6 +18,7 @@ dependencies:
   equatable: ^2.0.5
   http: ^1.1.0
   mason_logger: ^0.2.5
+  meta: ^1.0.0
   package_config: ^2.1.0
   path: ^1.8.3
   recase: ^4.1.0

packages/electricsql_cli/test/src/cli/config_test.dart

@@ -1,10 +1,58 @@
+import 'dart:collection';
+
+import 'package:dotenv/dotenv.dart';
 import 'package:electricsql_cli/src/config.dart';
+import 'package:electricsql_cli/src/config_options.dart';
+import 'package:electricsql_cli/src/env.dart';
 import 'package:electricsql_cli/src/util/util.dart';
 import 'package:test/test.dart';
 
 void main() {
   setDebugMockParsedPubspecLockInfo();
 
+  final origProgramEnv = programEnv;
+  final origConfigOptions =
+      UnmodifiableMapView<String, ConfigOption<Object>>(configOptions);
+
+  setUp(() {
+    final dotEnv = DotEnv()..addAll(origProgramEnv.map);
+    setProgramEnv(dotEnv);
+    setDebugConfigOptions({...origConfigOptions});
+  });
+
+  tearDown(() {
+    // cleanup
+    setProgramEnv(origProgramEnv);
+    setDebugConfigOptions(null);
+  });
+
+  test('getConfigValue respects boolean flag defaults', () async {
+    const flagWithTrueDefault = '_MOCK_TRUE_DEFAULT';
+    const flagWithFalseDefault = '_MOCK_FALSE_DEFAULT';
+
+    configOptions[flagWithTrueDefault] = ConfigOption<bool>(
+      doc: '',
+      defaultValue: true,
+    );
+
+    configOptions[flagWithFalseDefault] = ConfigOption<bool>(
+      doc: '',
+      defaultValue: false,
+    );
+
+    expect(getConfigValue<bool>(flagWithTrueDefault), true);
+    expect(getConfigValue<bool>(flagWithFalseDefault), false);
+
+    // ensure environment overrides default
+    programEnv.addAll({
+      'ELECTRIC_$flagWithTrueDefault': 'false',
+      'ELECTRIC_$flagWithFalseDefault': 'true',
+    });
+
+    expect(getConfigValue<bool>(flagWithTrueDefault), false);
+    expect(getConfigValue<bool>(flagWithFalseDefault), true);
+  });
+
   test('getConfigValue can capture `ELECTRIC_` prefixed CLI opitons', () {
     final image =
         getConfigValue<String>('ELECTRIC_IMAGE', {'image': 'electric:test'});

patch/e2e.patch

@@ -1,9 +1,9 @@
 diff -x lux -x satellite_client -x lux_logs -ur electric/common.mk dart/common.mk
 --- electric/common.mk	2024-01-01 00:00:00.000000000 +0000
 +++ dart/common.mk	2024-01-01 00:00:00.000000000 +0000
-@@ -41,6 +41,15 @@
- 	export ELECTRIC_CLIENT_IMAGE=${ELECTRIC_CLIENT_IMAGE_NAME}:${ELECTRIC_IMAGE_TAG}
- endif
+@@ -43,6 +43,16 @@
+
+ export DISABLE_TZDATA_AUTOUPDATE=true
 
 +export ELECTRIC_COMMIT:=$(shell cd $(PROJECT_ROOT) && tool/extract_electric_commit.sh)
 +
@@ -14,9 +14,10 @@ diff -x lux -x satellite_client -x lux_logs -ur electric/common.mk dart/common.m
 +	cd ${ELECTRIC_REPO} && \
 +	git fetch origin && \
 +	git checkout --force ${ELECTRIC_COMMIT}
- 
++
  lux: ${LUX_PATH}
 
+ ${LUX}:
 Only in dart: electric_repo
 diff -x lux -x satellite_client -x lux_logs -ur electric/elixir_client/Makefile dart/elixir_client/Makefile
 --- electric/elixir_client/Makefile	2024-01-01 00:00:00.000000000 +0000
@@ -42,35 +43,25 @@ diff -x lux -x satellite_client -x lux_logs -ur electric/.gitignore dart/.gitign
 diff -x lux -x satellite_client -x lux_logs -ur electric/Makefile dart/Makefile
 --- electric/Makefile	2024-01-01 00:00:00.000000000 +0000
 +++ dart/Makefile	2024-01-01 00:00:00.000000000 +0000
-@@ -1,6 +1,7 @@
- include common.mk
-
- deps: lux
-+	make clone_electric
- 	make -C satellite_client build
- 	make -C elixir_client build
- 	make -C prisma_example build
-@@ -11,6 +12,15 @@
- test_only:
- 	${LUX} --junit tests
+@@ -11,9 +11,13 @@
+ test-satellite-only:
+ 	${LUX} --junit tests/03.*.lux
 
-+test-no-satellite:
-+	find tests -type f -maxdepth 1 -name "*.lux" -and -not -name "03.*.lux" | sort -h | xargs ${LUX} --junit
-+
-+test-satellite-only:
-+	${LUX} --junit tests/03.*.lux
-+
 +test_only_custom:
 +	${LUX} --junit ${TEST}
 +
  test: deps pull test_only
 
- test-no-satellite:
+ deps: lux
++	make clone_electric
+ 	make -C satellite_client build
+ 	make -C elixir_client build
+ 	make -C prisma_example build
 Only in dart: run_client_e2e_with_retries.sh
 diff -x lux -x satellite_client -x lux_logs -ur electric/services_templates.yaml dart/services_templates.yaml
 --- electric/services_templates.yaml	2024-01-01 00:00:00.000000000 +0000
 +++ dart/services_templates.yaml	2024-01-01 00:00:00.000000000 +0000
-@@ -42,7 +42,7 @@
+@@ -40,7 +40,7 @@
        - "host.docker.internal:host-gateway"
 
    satellite_client: