diff --git a/profiles/default/package.json b/profiles/default/package.json index 74d6acf44..feae798b2 100644 --- a/profiles/default/package.json +++ b/profiles/default/package.json @@ -2,73 +2,9 @@ "version": "1", "enabled-engines": ["vertx", "postgres", "kafka", "flink"], "compile" : { - "sqrl-version": "0.5.8", "flink-build-image": "gradle:8.6-jdk11", "sqrl-vertx-image": "datasqrl/sqrl-server:v0.5.8" }, - "engines" : { - "flink" : { - "connectors": { - "postgres": { - "connector": "jdbc-sqrl", - "password": "${JDBC_PASSWORD}", - "driver": "org.postgresql.Driver", - "username": "${JDBC_USERNAME}", - "url": "${JDBC_URL}", - "table-name": "${sqrl:table}" - }, - "kafka": { - "connector" : "kafka", - "format" : "flexible-json", - "properties.bootstrap.servers": "${PROPERTIES_BOOTSTRAP_SERVERS}", - "properties.group.id": "${PROPERTIES_GROUP_ID}", - "scan.startup.mode" : "group-offsets", - "properties.auto.offset.reset" : "earliest", - "topic" : "${sqrl:topic}" - }, - "localfile": { - "connector" : "filesystem", - "format" : "${sqrl:format}", - "path" : "${sqrl:filepath}", - "source.monitor-interval" : 10000 - }, - "iceberg": { - "connector": "iceberg", - "catalog-table": "${sqrl:table}" - }, - "postgres_log-source": { - "connector": "postgres-cdc", - "hostname": "${PGHOST}", - "port": "5432", - "username": "${JDBC_USERNAME}", - "password": "${JDBC_PASSWORD}", - "database-name": "datasqrl", - "schema-name": "public", - "table-name": "${sqrl:table}", - "slot.name": "flink_slot", - "decoding.plugin.name": "pgoutput", - "debezium.slot.drop_on_stop": "false" - }, - "postgres_log-sink": { - "connector": "jdbc-sqrl", - "password": "${JDBC_PASSWORD}", - "driver": "org.postgresql.Driver", - "username": "${JDBC_USERNAME}", - "url": "${JDBC_URL}", - "table-name": "${sqrl:table}" - } - } - }, - "snowflake" : { - "schema-type": "aws-glue", - "catalog-name": "${SNOWFLAKE_CATALOG_NAME}", - "external-volume": "${SNOWFLAKE_EXTERNAL_VOLUME}", - "url": "jdbc:snowflake://${SNOWFLAKE_ID}.snowflakecomputing.com/?user=${SNOWFLAKE_USER}&password=${SNOWFLAKE_PASSWORD}&warehouse=COMPUTE_WH&db=MYSNOWFLAKEDB&schema=public&disableSslHostnameVerification=true" - } - }, - "test-runner": { - "delay-sec": 30 - }, "package": { "name": "datasqrl.profile.default", "version": "0.5.8", diff --git a/sqrl-testing/sqrl-integration-tests/src/test/java/com/datasqrl/FullUsecasesIT.java b/sqrl-testing/sqrl-integration-tests/src/test/java/com/datasqrl/FullUsecasesIT.java index 0c3890427..39900ddac 100644 --- a/sqrl-testing/sqrl-integration-tests/src/test/java/com/datasqrl/FullUsecasesIT.java +++ b/sqrl-testing/sqrl-integration-tests/src/test/java/com/datasqrl/FullUsecasesIT.java @@ -1,5 +1,6 @@ package com.datasqrl; +import static com.datasqrl.packager.Packager.PACKAGE_JSON; import static org.junit.jupiter.api.Assertions.fail; import com.datasqrl.cmd.AssertStatusHook; @@ -17,12 +18,12 @@ import com.datasqrl.tests.UseCaseTestExtensions; import com.datasqrl.util.FlinkOperatorStatusChecker; import com.datasqrl.util.SnapshotTest.Snapshot; +import com.google.common.io.Resources; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -33,11 +34,8 @@ import lombok.ToString; import lombok.Value; import lombok.extern.slf4j.Slf4j; -import org.apache.flink.table.api.ResultKind; import org.apache.flink.table.api.TableResult; import org.apache.flink.test.junit5.MiniClusterExtension; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -161,7 +159,7 @@ public void testUseCase(UseCaseTestParameter param, TestInfo testInfo) { executor.execute(new AssertStatusHook()); PackageJson packageJson = SqrlConfigCommons.fromFilesPackageJson(ErrorCollector.root(), - List.of(rootDir.resolve("build").resolve("package.json"))); + Resources.getResource(PACKAGE_JSON), List.of(rootDir.resolve("build").resolve("package.json"))); try { diff --git a/sqrl-testing/sqrl-integration-tests/src/test/java/com/datasqrl/discovery/FlexibleSchemaInferencePreprocessorTest.java b/sqrl-testing/sqrl-integration-tests/src/test/java/com/datasqrl/discovery/FlexibleSchemaInferencePreprocessorTest.java index b2f1740f1..09edb95de 100644 --- a/sqrl-testing/sqrl-integration-tests/src/test/java/com/datasqrl/discovery/FlexibleSchemaInferencePreprocessorTest.java +++ b/sqrl-testing/sqrl-integration-tests/src/test/java/com/datasqrl/discovery/FlexibleSchemaInferencePreprocessorTest.java @@ -1,5 +1,6 @@ package com.datasqrl.discovery; +import static com.datasqrl.packager.Packager.PACKAGE_JSON; import static org.junit.jupiter.api.Assertions.assertTrue; import com.datasqrl.AbstractAssetSnapshotTest; @@ -15,6 +16,7 @@ import com.datasqrl.packager.preprocess.Preprocessor.ProcessorContext; import com.datasqrl.plan.validate.ExecutionGoal; import com.datasqrl.util.SnapshotTest.Snapshot; +import com.google.common.io.Resources; import com.google.inject.Guice; import com.google.inject.Injector; import java.nio.file.Files; @@ -42,7 +44,7 @@ protected FlexibleSchemaInferencePreprocessorTest() { super(FILES_DIR.resolve("output")); try { packageJson = SqrlConfigCommons.fromFilesPackageJson(errors, - List.of(Path.of("../../profiles/default/package.json"))); + Resources.getResource(PACKAGE_JSON), List.of(Path.of("../../profiles/default/package.json"))); } catch (Exception e) { System.out.println(ErrorPrinter.prettyPrint(errors)); throw e; diff --git a/sqrl-tools/sqrl-config/src/main/java/com/datasqrl/config/SqrlConfigCommons.java b/sqrl-tools/sqrl-config/src/main/java/com/datasqrl/config/SqrlConfigCommons.java index 7c67a86b4..63154b8db 100644 --- a/sqrl-tools/sqrl-config/src/main/java/com/datasqrl/config/SqrlConfigCommons.java +++ b/sqrl-tools/sqrl-config/src/main/java/com/datasqrl/config/SqrlConfigCommons.java @@ -9,6 +9,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.io.Resources; import com.networknt.schema.JsonSchema; import com.networknt.schema.JsonSchemaFactory; import com.networknt.schema.SpecVersion.VersionFlag; @@ -40,6 +41,7 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.NonNull; +import lombok.SneakyThrows; import org.apache.commons.configuration2.BaseHierarchicalConfiguration; import org.apache.commons.configuration2.CombinedConfiguration; import org.apache.commons.configuration2.Configuration; @@ -349,15 +351,21 @@ public static SqrlConfig create(ErrorCollector errors, int version) { public static TableConfigImpl fromFilesTableConfig(@NonNull Name name, ErrorCollector errors, @NonNull List files) { - return new TableConfigImpl(name, fromFiles(errors, "/jsonSchema/tableConfig.json", files)); + return new TableConfigImpl(name, fromFiles(errors, "/jsonSchema/tableConfig.json", Optional.empty(), files)); } - public static PackageJson fromFilesPackageJson(ErrorCollector errors, @NonNull List files) { - return new PackageJsonImpl(fromFiles(errors, "/jsonSchema/packageSchema.json", files)); + public static PackageJson fromFilesPackageJson(ErrorCollector errors, URL base, @NonNull List files) { + return new PackageJsonImpl(fromFiles(errors, "/jsonSchema/packageSchema.json", Optional.of(base), files)); } public static PackageJson fromFilesPublishPackageJson(ErrorCollector errors, @NonNull List files) { - return new PackageJsonImpl(fromFiles(errors, "/jsonSchema/publishPackageSchema.json", files)); + return new PackageJsonImpl(fromFiles(errors, "/jsonSchema/publishPackageSchema.json", Optional.empty(), + files)); + } + + public static PackageJson fromBaseResource(ErrorCollector errors) { + URL resource = Resources.getResource("package.json"); + return new PackageJsonImpl(fromURL(errors, resource)); } public static boolean validateJsonFile(Path jsonFilePath, String schemaResourcePath, @@ -398,16 +406,22 @@ public static boolean validateJsonFile(Path jsonFilePath, String schemaResourceP } public static SqrlConfig fromFiles(ErrorCollector errors, @NonNull Path firstFile) { - return fromFiles(errors, null, List.of(firstFile)); + return fromFiles(errors, null, Optional.empty(), List.of(firstFile)); } - public static SqrlConfig fromFiles(ErrorCollector errors, String jsonSchemaResource, @NonNull List files) { + @SneakyThrows + public static SqrlConfig fromFiles(ErrorCollector errors, String jsonSchemaResource, Optional base, @NonNull List files) { Preconditions.checkArgument(files!=null && !files.isEmpty(),"Need to provide at least one configuration file"); Configurations configs = new Configurations(); boolean isValid = true; NodeCombiner combiner = new OverrideCombiner(); CombinedConfiguration combinedConfiguration = new CombinedConfiguration(combiner); + if (base.isPresent()) { + Configuration nextconfig = configs.fileBased(JSONConfiguration.class, base.get()); + combinedConfiguration.addConfiguration(nextconfig); + } + for (int i = files.size()-1; i >= 0; i--) { //iterate backwards so last file takes precedence Path file = files.get(i); ErrorCollector localErrors = errors.withConfig(file); diff --git a/sqrl-tools/sqrl-config/src/main/resources/package.json b/sqrl-tools/sqrl-config/src/main/resources/package.json new file mode 100644 index 000000000..475cf7ab7 --- /dev/null +++ b/sqrl-tools/sqrl-config/src/main/resources/package.json @@ -0,0 +1,70 @@ +{ + "version": "1", + "enabled-engines": ["vertx", "postgres", "kafka", "flink"], + "compile" : { + "sqrl-version": "0.5.8" + }, + "engines" : { + "flink" : { + "connectors": { + "postgres": { + "connector": "jdbc-sqrl", + "password": "${JDBC_PASSWORD}", + "driver": "org.postgresql.Driver", + "username": "${JDBC_USERNAME}", + "url": "${JDBC_URL}", + "table-name": "${sqrl:table}" + }, + "kafka": { + "connector" : "kafka", + "format" : "flexible-json", + "properties.bootstrap.servers": "${PROPERTIES_BOOTSTRAP_SERVERS}", + "properties.group.id": "${PROPERTIES_GROUP_ID}", + "scan.startup.mode" : "group-offsets", + "properties.auto.offset.reset" : "earliest", + "topic" : "${sqrl:topic}" + }, + "localfile": { + "connector" : "filesystem", + "format" : "${sqrl:format}", + "path" : "${sqrl:filepath}", + "source.monitor-interval" : 10000 + }, + "iceberg": { + "connector": "iceberg", + "catalog-table": "${sqrl:table}" + }, + "postgres_log-source": { + "connector": "postgres-cdc", + "hostname": "${PGHOST}", + "port": "5432", + "username": "${JDBC_USERNAME}", + "password": "${JDBC_PASSWORD}", + "database-name": "datasqrl", + "schema-name": "public", + "table-name": "${sqrl:table}", + "slot.name": "flink_slot", + "decoding.plugin.name": "pgoutput", + "debezium.slot.drop_on_stop": "false" + }, + "postgres_log-sink": { + "connector": "jdbc-sqrl", + "password": "${JDBC_PASSWORD}", + "driver": "org.postgresql.Driver", + "username": "${JDBC_USERNAME}", + "url": "${JDBC_URL}", + "table-name": "${sqrl:table}" + } + } + }, + "snowflake" : { + "schema-type": "aws-glue", + "catalog-name": "${SNOWFLAKE_CATALOG_NAME}", + "external-volume": "${SNOWFLAKE_EXTERNAL_VOLUME}", + "url": "jdbc:snowflake://${SNOWFLAKE_ID}.snowflakecomputing.com/?user=${SNOWFLAKE_USER}&password=${SNOWFLAKE_PASSWORD}&warehouse=COMPUTE_WH&db=MYSNOWFLAKEDB&schema=public&disableSslHostnameVerification=true" + } + }, + "test-runner": { + "delay-sec": 30 + } +} \ No newline at end of file diff --git a/sqrl-tools/sqrl-config/src/test/java/com/datasqrl/config/PackageJsonSchemaTest.java b/sqrl-tools/sqrl-config/src/test/java/com/datasqrl/config/PackageJsonSchemaTest.java index 994cf6965..bc86ed940 100644 --- a/sqrl-tools/sqrl-config/src/test/java/com/datasqrl/config/PackageJsonSchemaTest.java +++ b/sqrl-tools/sqrl-config/src/test/java/com/datasqrl/config/PackageJsonSchemaTest.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.fail; import com.datasqrl.error.ErrorCollector; +import com.google.common.io.Resources; import java.nio.file.Path; import java.util.List; import org.junit.jupiter.params.ParameterizedTest; @@ -27,7 +28,7 @@ public class PackageJsonSchemaTest { public void testValidConfigFile(String configFileName) { ErrorCollector errors = ErrorCollector.root(); try { - SqrlConfigCommons.fromFilesPackageJson(errors, List.of(TEST_CASES.resolve(configFileName))); + SqrlConfigCommons.fromFilesPackageJson(errors, Resources.getResource("package.json"), List.of(TEST_CASES.resolve(configFileName))); } catch (Exception e) { fail("Unexpected error: " + errors.getErrors().toString()); } @@ -48,6 +49,6 @@ public void testValidConfigFile(String configFileName) { }) public void testInvalidConfigFile(String configFileName) { testForErrors(errors -> SqrlConfigCommons.fromFilesPackageJson(errors, - List.of(CONFIG_DIR.resolve(configFileName)))); + Resources.getResource("package.json"), List.of(CONFIG_DIR.resolve(configFileName)))); } } diff --git a/sqrl-tools/sqrl-packager/src/main/java/com/datasqrl/cmd/PackageBootstrap.java b/sqrl-tools/sqrl-packager/src/main/java/com/datasqrl/cmd/PackageBootstrap.java index 531425ef8..d0bce0862 100644 --- a/sqrl-tools/sqrl-packager/src/main/java/com/datasqrl/cmd/PackageBootstrap.java +++ b/sqrl-tools/sqrl-packager/src/main/java/com/datasqrl/cmd/PackageBootstrap.java @@ -9,12 +9,12 @@ import com.datasqrl.config.Dependency; import com.datasqrl.config.PackageJson; import com.datasqrl.config.PackageJson.ScriptConfig; -import com.datasqrl.config.PackageJsonImpl; import com.datasqrl.config.SqrlConfigCommons; import com.datasqrl.error.ErrorCollector; import com.datasqrl.error.ErrorPrefix; import com.datasqrl.packager.Packager; import com.datasqrl.packager.repository.Repository; +import com.google.common.io.Resources; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -47,7 +47,7 @@ public PackageJson bootstrap(Path rootDir, List packageFiles, Optional> existingPackage = Packager.findPackageFile(rootDir, packageFiles); Optional existingConfig; existingConfig = existingPackage.map( - paths -> SqrlConfigCommons.fromFilesPackageJson(errors, paths)); + paths -> SqrlConfigCommons.fromFilesPackageJson(errors, Resources.getResource(PACKAGE_JSON), paths)); Map dependencies = new HashMap<>(); // Check if 'profiles' key is set, replace if existing @@ -117,7 +117,8 @@ public PackageJson bootstrap(Path rootDir, List packageFiles, } // Merge all configurations - PackageJson packageJson = SqrlConfigCommons.fromFilesPackageJson(errors, configFiles); + PackageJson packageJson = SqrlConfigCommons.fromFilesPackageJson(errors, + Resources.getResource(PACKAGE_JSON), configFiles); packageJson.setProfiles(profiles); //Add dependencies of discovered profiles @@ -156,7 +157,7 @@ public PackageJson bootstrap(Path rootDir, List packageFiles, } public PackageJson createDefaultConfig(ErrorCollector errors) { - PackageJson packageJson = new PackageJsonImpl(); + PackageJson packageJson = SqrlConfigCommons.fromBaseResource(errors); return setDefaultConfig(packageJson); }