From 525d8e266d9d5f38bb56ed7e5c97a74d67ad1f4e Mon Sep 17 00:00:00 2001 From: TaoZex <2633363995@qq.com> Date: Wed, 16 Nov 2022 20:17:04 +0800 Subject: [PATCH 1/2] [Feature][Connector-V2][OneSignal]Add OneSignal source conector --- docs/en/connector-v2/source/OneSignal.md | 182 ++++++++++++++++++ plugin-mapping.properties | 1 + .../connector-http-onesignal/pom.xml | 40 ++++ .../onesignal/source/OneSignalSource.java | 61 ++++++ .../source/OneSignalSourceFactory.java | 55 ++++++ .../source/config/OneSignalSourceConfig.java | 34 ++++ .../config/OneSignalSourceParameter.java | 35 ++++ .../connector-http/pom.xml | 1 + seatunnel-dist/pom.xml | 6 + .../connector-http-e2e/pom.xml | 6 + .../e2e/connector/http/HttpOneSignalIT.java | 76 ++++++++ .../mockserver-onesignal-config.json | 174 +++++++++++++++++ .../resources/onesignal_json_to_assert.conf | 112 +++++++++++ 13 files changed, 783 insertions(+) create mode 100644 docs/en/connector-v2/source/OneSignal.md create mode 100644 seatunnel-connectors-v2/connector-http/connector-http-onesignal/pom.xml create mode 100644 seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSource.java create mode 100644 seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSourceFactory.java create mode 100644 seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceConfig.java create mode 100644 seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceParameter.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpOneSignalIT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-onesignal-config.json create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/onesignal_json_to_assert.conf diff --git a/docs/en/connector-v2/source/OneSignal.md b/docs/en/connector-v2/source/OneSignal.md new file mode 100644 index 00000000000..9e0286537eb --- /dev/null +++ b/docs/en/connector-v2/source/OneSignal.md @@ -0,0 +1,182 @@ +# OneSignal + +> OneSignal source connector + +## Description + +Used to read data from OneSignal. + +## Key features + +- [x] [batch](../../concept/connector-v2-features.md) +- [ ] [stream](../../concept/connector-v2-features.md) +- [ ] [exactly-once](../../concept/connector-v2-features.md) +- [x] [schema projection](../../concept/connector-v2-features.md) +- [ ] [parallelism](../../concept/connector-v2-features.md) +- [ ] [support user-defined split](../../concept/connector-v2-features.md) + +## Options + +| name | type | required | default value | +| --------------------------- | ------ | -------- | ------------- | +| url | String | Yes | - | +| password | String | Yes | - | +| method | String | No | get | +| schema | Config | No | - | +| schema.fields | Config | No | - | +| format | String | No | json | +| params | Map | No | - | +| body | String | No | - | +| poll_interval_ms | int | No | - | +| retry | int | No | - | +| retry_backoff_multiplier_ms | int | No | 100 | +| retry_backoff_max_ms | int | No | 10000 | +| common-options | config | No | - | + +### url [String] + +http request url + +### password [String] + +Auth key for login, you can get more detail at this link: + +https://documentation.onesignal.com/docs/accounts-and-keys#user-auth-key + +### method [String] + +http request method, only supports GET, POST method + +### params [Map] + +http params + +### body [String] + +http body + +### poll_interval_ms [int] + +request http api interval(millis) in stream mode + +### retry [int] + +The max retry times if request http return to `IOException` + +### retry_backoff_multiplier_ms [int] + +The retry-backoff times(millis) multiplier if request http failed + +### retry_backoff_max_ms [int] + +The maximum retry-backoff times(millis) if request http failed + +### format [String] + +the format of upstream data, now only support `json` `text`, default `json`. + +when you assign format is `json`, you should also assign schema option, for example: + +upstream data is the following: + +```json + +{"code": 200, "data": "get success", "success": true} + +``` + +you should assign schema as the following: + +```hocon + +schema { + fields { + code = int + data = string + success = boolean + } +} + +``` + +connector will generate data as the following: + +| code | data | success | +|------|-------------|---------| +| 200 | get success | true | + +when you assign format is `text`, connector will do nothing for upstream data, for example: + +upstream data is the following: + +```json + +{"code": 200, "data": "get success", "success": true} + +``` + +connector will generate data as the following: + +| content | +|---------| +| {"code": 200, "data": "get success", "success": true} | + +### schema [Config] + +#### fields [Config] + +the schema fields of upstream data + +### common options + +Source plugin common parameters, please refer to [Source Common Options](common-options.md) for details + +## Example + +```hocon + +OneSignal { + url = "https://onesignal.com/api/v1/apps" + password = "Seatunnel-test" + schema = { + fields { + id = string + name = string + gcm_key = string + chrome_key = string + chrome_web_key = string + chrome_web_origin = string + chrome_web_gcm_sender_id = string + chrome_web_default_notification_icon = string + chrome_web_sub_domain = string + apns_env = string + apns_certificates = string + apns_p8 = string + apns_team_id = string + apns_key_id = string + apns_bundle_id = string + safari_apns_certificate = string + safari_site_origin = string + safari_push_id = string + safari_icon_16_16 = string + safari_icon_32_32 = string + safari_icon_64_64 = string + safari_icon_128_128 = string + safari_icon_256_256 = string + site_name = string + created_at = string + updated_at = string + players = int + messageable_players = int + basic_auth_key = string + additional_data_is_root_payload = string + } + } +} +``` + +## Changelog + +### next version + +- Add OneSignal Source Connector diff --git a/plugin-mapping.properties b/plugin-mapping.properties index 90971b5f9ff..05a5fe7d36c 100644 --- a/plugin-mapping.properties +++ b/plugin-mapping.properties @@ -147,3 +147,4 @@ seatunnel.sink.InfluxDB = connector-influxdb seatunnel.source.GoogleSheets = connector-google-sheets seatunnel.sink.Tablestore = connector-tablestore seatunnel.source.Lemlist = connector-http-lemlist +seatunnel.source.OneSignal = connector-http-onesignal diff --git a/seatunnel-connectors-v2/connector-http/connector-http-onesignal/pom.xml b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/pom.xml new file mode 100644 index 00000000000..e87ad5accae --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/pom.xml @@ -0,0 +1,40 @@ + + + + + connector-http + org.apache.seatunnel + ${revision} + + 4.0.0 + + connector-http-onesignal + + + + org.apache.seatunnel + connector-http-base + ${project.version} + + + + \ No newline at end of file diff --git a/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSource.java b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSource.java new file mode 100644 index 00000000000..c59687d752e --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSource.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.onesignal.source; + +import org.apache.seatunnel.api.common.PrepareFailException; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.common.config.CheckConfigUtil; +import org.apache.seatunnel.common.config.CheckResult; +import org.apache.seatunnel.common.constants.PluginType; +import org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitReader; +import org.apache.seatunnel.connectors.seatunnel.common.source.SingleSplitReaderContext; +import org.apache.seatunnel.connectors.seatunnel.http.source.HttpSource; +import org.apache.seatunnel.connectors.seatunnel.http.source.HttpSourceReader; +import org.apache.seatunnel.connectors.seatunnel.onesignal.source.config.OneSignalSourceConfig; +import org.apache.seatunnel.connectors.seatunnel.onesignal.source.config.OneSignalSourceParameter; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import com.google.auto.service.AutoService; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@AutoService(SeaTunnelSource.class) +public class OneSignalSource extends HttpSource { + private final OneSignalSourceParameter oneSignalSourceParameter = new OneSignalSourceParameter(); + @Override + public String getPluginName() { + return "OneSignal"; + } + + @Override + public void prepare(Config pluginConfig) throws PrepareFailException { + CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, OneSignalSourceConfig.URL.key(), OneSignalSourceConfig.PASSWORD.key()); + if (!result.isSuccess()) { + throw new PrepareFailException(getPluginName(), PluginType.SOURCE, result.getMsg()); + } + oneSignalSourceParameter.buildWithConfig(pluginConfig); + buildSchemaWithConfig(pluginConfig); + } + + @Override + public AbstractSingleSplitReader createReader(SingleSplitReaderContext readerContext) throws Exception { + return new HttpSourceReader(this.oneSignalSourceParameter, readerContext, this.deserializationSchema); + } +} diff --git a/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSourceFactory.java b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSourceFactory.java new file mode 100644 index 00000000000..ff2b40b23f7 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/OneSignalSourceFactory.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.onesignal.source; + +import org.apache.seatunnel.api.configuration.util.Condition; +import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.api.table.factory.Factory; +import org.apache.seatunnel.api.table.factory.TableSourceFactory; +import org.apache.seatunnel.connectors.seatunnel.common.schema.SeaTunnelSchema; +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpConfig; +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpRequestMethod; +import org.apache.seatunnel.connectors.seatunnel.onesignal.source.config.OneSignalSourceConfig; + +import com.google.auto.service.AutoService; + +@AutoService(Factory.class) +public class OneSignalSourceFactory implements TableSourceFactory { + @Override + public String factoryIdentifier() { + return "OneSignal"; + } + + @Override + public OptionRule optionRule() { + return OptionRule.builder() + .required(OneSignalSourceConfig.URL) + .required(OneSignalSourceConfig.PASSWORD) + .optional(OneSignalSourceConfig.METHOD) + .optional(OneSignalSourceConfig.HEADERS) + .optional(OneSignalSourceConfig.PARAMS) + .conditional(Condition.of(HttpConfig.METHOD, HttpRequestMethod.POST), OneSignalSourceConfig.BODY) + .conditional(Condition.of(HttpConfig.FORMAT, "json"), SeaTunnelSchema.SCHEMA) + .optional(OneSignalSourceConfig.FORMAT) + .optional(OneSignalSourceConfig.POLL_INTERVAL_MILLS) + .optional(OneSignalSourceConfig.RETRY) + .optional(OneSignalSourceConfig.RETRY_BACKOFF_MAX_MS) + .optional(OneSignalSourceConfig.RETRY_BACKOFF_MULTIPLIER_MS) + .build(); + } +} diff --git a/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceConfig.java b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceConfig.java new file mode 100644 index 00000000000..551a05023c6 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceConfig.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.onesignal.source.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpConfig; + +public class OneSignalSourceConfig extends HttpConfig { + public static final String AUTHORIZATION = "Authorization"; + public static final String CONTENT_TYPE = "Content-Type"; + public static final String APPLICATION_JSON = "application/json"; + public static final String BASIC = "Basic"; + + public static final Option PASSWORD = Options.key("password") + .stringType() + .noDefaultValue() + .withDescription("OneSignal login auth key"); +} diff --git a/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceParameter.java b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceParameter.java new file mode 100644 index 00000000000..4fa79cce177 --- /dev/null +++ b/seatunnel-connectors-v2/connector-http/connector-http-onesignal/src/main/java/org/apache/seatunnel/connectors/seatunnel/onesignal/source/config/OneSignalSourceParameter.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.onesignal.source.config; + +import org.apache.seatunnel.connectors.seatunnel.http.config.HttpParameter; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import java.util.HashMap; + +public class OneSignalSourceParameter extends HttpParameter { + public void buildWithConfig(Config pluginConfig) { + super.buildWithConfig(pluginConfig); + // put authorization in headers + this.headers = this.getHeaders() == null ? new HashMap<>() : this.getHeaders(); + this.headers.put(OneSignalSourceConfig.CONTENT_TYPE, OneSignalSourceConfig.APPLICATION_JSON); + this.headers.put(OneSignalSourceConfig.AUTHORIZATION, OneSignalSourceConfig.BASIC + " " + pluginConfig.getString(OneSignalSourceConfig.PASSWORD.key())); + this.setHeaders(this.headers); + } +} diff --git a/seatunnel-connectors-v2/connector-http/pom.xml b/seatunnel-connectors-v2/connector-http/pom.xml index e9228fdef01..233b86c2267 100644 --- a/seatunnel-connectors-v2/connector-http/pom.xml +++ b/seatunnel-connectors-v2/connector-http/pom.xml @@ -35,6 +35,7 @@ connector-http-wechat connector-http-myhours connector-http-lemlist + connector-http-onesignal \ No newline at end of file diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index ab07d4a08d5..70d88fba927 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -165,6 +165,12 @@ ${project.version} provided + + org.apache.seatunnel + connector-http-onesignal + ${project.version} + provided + org.apache.seatunnel connector-jdbc diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml index fc577553c2e..edcd501d3ef 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/pom.xml @@ -50,6 +50,12 @@ ${project.version} test + + org.apache.seatunnel + connector-http-onesignal + ${project.version} + test + \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpOneSignalIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpOneSignalIT.java new file mode 100644 index 00000000000..0e0a446b2ec --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpOneSignalIT.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.connector.http; + +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.TestContainer; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; +import org.testcontainers.utility.MountableFile; + +import java.io.File; +import java.io.IOException; +import java.util.stream.Stream; + +public class HttpOneSignalIT extends TestSuiteBase implements TestResource { + + private static final String IMAGE = "mockserver/mockserver:5.14.0"; + + private GenericContainer mockserverContainer; + + @BeforeAll + @Override + public void startUp() { + this.mockserverContainer = new GenericContainer<>(DockerImageName.parse(IMAGE)) + .withNetwork(NETWORK) + .withNetworkAliases("mockserver") + .withExposedPorts(1080) + .withCopyFileToContainer(MountableFile.forHostPath(new File(HttpIT.class.getResource( + "/mockserver-onesignal-config.json").getPath()).getAbsolutePath()), + "/tmp/mockserver-onesignal-config.json") + .withEnv("MOCKSERVER_INITIALIZATION_JSON_PATH", "/tmp/mockserver-onesignal-config.json") + .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))) + .waitingFor(new HttpWaitStrategy().forPath("/").forStatusCode(404)); + Startables.deepStart(Stream.of(mockserverContainer)).join(); + } + + @AfterAll + @Override + public void tearDown() { + if (mockserverContainer != null) { + mockserverContainer.stop(); + } + } + + @TestTemplate + public void testHttpOneSignalSourceToAssertSink(TestContainer container) throws IOException, InterruptedException { + Container.ExecResult execResult = container.executeJob("/onesignal_json_to_assert.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-onesignal-config.json b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-onesignal-config.json new file mode 100644 index 00000000000..c7dac39363a --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-onesignal-config.json @@ -0,0 +1,174 @@ +// https://www.mock-server.com/mock_server/getting_started.html#request_matchers + +[ + { + "httpRequest": { + "method" : "GET", + "path": "/api/v1/apps" + }, + "httpResponse": { + "body": [ + { + "id": "8151f1be-63f2-4c8c-9348-7ad6fda73b3d", + "name": "enjoy life", + "gcm_key": null, + "chrome_key": null, + "chrome_web_key": null, + "chrome_web_origin": null, + "chrome_web_gcm_sender_id": null, + "chrome_web_default_notification_icon": null, + "chrome_web_sub_domain": null, + "apns_env": null, + "apns_certificates": null, + "apns_p8": null, + "apns_team_id": null, + "apns_key_id": null, + "apns_bundle_id": null, + "safari_apns_certificate": null, + "safari_site_origin": null, + "safari_push_id": null, + "safari_icon_16_16": "public/safari_packages/8151f1be-63f2-4c8c-9348-7ad6fda73b3d/icons/16x16.png", + "safari_icon_32_32": "public/safari_packages/8151f1be-63f2-4c8c-9348-7ad6fda73b3d/icons/16x16@2x.png", + "safari_icon_64_64": "public/safari_packages/8151f1be-63f2-4c8c-9348-7ad6fda73b3d/icons/32x32@2x.png", + "safari_icon_128_128": "public/safari_packages/8151f1be-63f2-4c8c-9348-7ad6fda73b3d/icons/128x128.png", + "safari_icon_256_256": "public/safari_packages/8151f1be-63f2-4c8c-9348-7ad6fda73b3d/icons/128x128@2x.png", + "site_name": null, + "created_at": "2022-10-30T14:48:14.688Z", + "updated_at": "2022-10-30T14:48:14.953Z", + "players": 100, + "messageable_players": 0, + "basic_auth_key": "Y2EyZjI5NzgtMzU1NC00NTU3LWIwNWItXGQ0MzQ4MzQ2ZjY2", + "additional_data_is_root_payload": false + }, + { + "id": "6386d2ea-d98f-468b-8f01-116d920a1e42", + "name": "test", + "gcm_key": null, + "chrome_key": null, + "chrome_web_key": null, + "chrome_web_origin": null, + "chrome_web_gcm_sender_id": null, + "chrome_web_default_notification_icon": null, + "chrome_web_sub_domain": null, + "apns_env": null, + "apns_certificates": null, + "apns_p8": null, + "apns_team_id": null, + "apns_key_id": null, + "apns_bundle_id": null, + "safari_apns_certificate": null, + "safari_site_origin": null, + "safari_push_id": null, + "safari_icon_16_16": "public/safari_packages/6386d2ea-d98f-468b-8f01-116d920a1e42/icons/16x16.png", + "safari_icon_32_32": "public/safari_packages/6386d2ea-d98f-468b-8f01-116d920a1e42/icons/16x16@2x.png", + "safari_icon_64_64": "public/safari_packages/6386d2ea-d98f-468b-8f01-116d920a1e42/icons/32x32@2x.png", + "safari_icon_128_128": "public/safari_packages/6386d2ea-d98f-468b-8f01-116d920a1e42/icons/128x128.png", + "safari_icon_256_256": "public/safari_packages/6386d2ea-d98f-468b-8f01-116d920a1e42/icons/128x128@2x.png", + "site_name": null, + "created_at": "2022-10-30T14:50:24.711Z", + "updated_at": "2022-10-30T14:50:24.849Z", + "players": 1, + "messageable_players": 0, + "basic_auth_key": "ODFiZWJiZDItZWYwZC00ODYzLWE4YmUtYTRmY2ZjNTU5NTVi", + "additional_data_is_root_payload": false + }, + { + "id": "63844g53-d98f-468b-8f01-11632320a1e42", + "name": "game", + "gcm_key": null, + "chrome_key": null, + "chrome_web_key": null, + "chrome_web_origin": null, + "chrome_web_gcm_sender_id": null, + "chrome_web_default_notification_icon": null, + "chrome_web_sub_domain": null, + "apns_env": null, + "apns_certificates": null, + "apns_p8": null, + "apns_team_id": null, + "apns_key_id": null, + "apns_bundle_id": null, + "safari_apns_certificate": null, + "safari_site_origin": null, + "safari_push_id": null, + "safari_icon_16_16": "public/safari_packages/63844g53-d98f-468b-8f01-11632320a1e42/icons/16x16.png", + "safari_icon_32_32": "public/safari_packages/63844g53-d98f-468b-8f01-11632320a1e42/icons/16x16@2x.png", + "safari_icon_64_64": "public/safari_packages/63844g53-d98f-468b-8f01-11632320a1e42/icons/32x32@2x.png", + "safari_icon_128_128": "public/safari_packages/63844g53-d98f-468b-8f01-11632320a1e42/icons/128x128.png", + "safari_icon_256_256": "public/safari_packages/63844g53-d98f-468b-8f01-11632320a1e42/icons/128x128@2x.png", + "site_name": null, + "created_at": "2022-10-30T14:50:24.711Z", + "updated_at": "2022-10-30T14:50:24.849Z", + "players": 16, + "messageable_players": 0, + "basic_auth_key": "ODFiZWJiZDItZWYwZC00ODYzLWE4MmUtYTRmY2ZjNTU5NTVi", + "additional_data_is_root_payload": false + }, + { + "id": "632332a-d9f-43238b-8f2301-11a1e42", + "name": "metting", + "gcm_key": null, + "chrome_key": null, + "chrome_web_key": null, + "chrome_web_origin": null, + "chrome_web_gcm_sender_id": null, + "chrome_web_default_notification_icon": null, + "chrome_web_sub_domain": null, + "apns_env": null, + "apns_certificates": null, + "apns_p8": null, + "apns_team_id": null, + "apns_key_id": null, + "apns_bundle_id": null, + "safari_apns_certificate": null, + "safari_site_origin": null, + "safari_push_id": null, + "safari_icon_16_16": "public/safari_packages/632332a-d9f-43238b-8f2301-11a1e42/icons/16x16.png", + "safari_icon_32_32": "public/safari_packages/632332a-d9f-43238b-8f2301-11a1e42/icons/16x16@2x.png", + "safari_icon_64_64": "public/safari_packages/632332a-d9f-43238b-8f2301-11a1e42/icons/32x32@2x.png", + "safari_icon_128_128": "public/safari_packages/632332a-d9f-43238b-8f2301-11a1e42/icons/128x128.png", + "safari_icon_256_256": "public/safari_packages/632332a-d9f-43238b-8f2301-11a1e42/icons/128x128@2x.png", + "site_name": null, + "created_at": "2022-10-30T14:50:24.711Z", + "updated_at": "2022-10-30T14:50:24.849Z", + "players": 0, + "messageable_players": 0, + "basic_auth_key": "ODFiZWJiZDItZWYwZgbvODYzLWE4MmUtYTRmY2ZjNTU5NTVi", + "additional_data_is_root_payload": false + }, + { + "id": "6386d2ea-d98f-468b-8f01-116d23r0a1e42", + "name": "app test", + "gcm_key": null, + "chrome_key": null, + "chrome_web_key": null, + "chrome_web_origin": null, + "chrome_web_gcm_sender_id": null, + "chrome_web_default_notification_icon": null, + "chrome_web_sub_domain": null, + "apns_env": null, + "apns_certificates": null, + "apns_p8": null, + "apns_team_id": null, + "apns_key_id": null, + "apns_bundle_id": null, + "safari_apns_certificate": null, + "safari_site_origin": null, + "safari_push_id": null, + "safari_icon_16_16": "public/safari_packages/6386d2ea-d98f-468b-8f01-116d23r0a1e42/icons/16x16.png", + "safari_icon_32_32": "public/safari_packages/6386d2ea-d98f-468b-8f01-116d23r0a1e42/icons/16x16@2x.png", + "safari_icon_64_64": "public/safari_packages/6386d2ea-d98f-468b-8f01-116d23r0a1e42/icons/32x32@2x.png", + "safari_icon_128_128": "public/safari_packages/6386d2ea-d98f-468b-8f01-116d23r0a1e42/icons/128x128.png", + "safari_icon_256_256": "public/safari_packages/6386d2ea-d98f-468b-8f01-116d23r0a1e42/icons/128x128@2x.png", + "site_name": null, + "created_at": "2022-10-30T14:50:24.711Z", + "updated_at": "2022-10-30T14:50:24.849Z", + "players": 2, + "messageable_players": 0, + "basic_auth_key": "ODFiZWJiZDItZWYwZC00ODYzLWE4MmUtYgreY2ZjNTU5NTVi", + "additional_data_is_root_payload": false + } + ] + } + } +] diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/onesignal_json_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/onesignal_json_to_assert.conf new file mode 100644 index 00000000000..1da25e22b02 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/onesignal_json_to_assert.conf @@ -0,0 +1,112 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + execution.parallelism = 1 + job.mode = "BATCH" +} + +source { + OneSignal { + url = "http://mockserver:1080/api/v1/apps" + password = "Seatunnel-test" + method = "GET" + format = "json" + schema = { + fields { + id = string + name = string + gcm_key = string + chrome_key = string + chrome_web_key = string + chrome_web_origin = string + chrome_web_gcm_sender_id = string + chrome_web_default_notification_icon = string + chrome_web_sub_domain = string + apns_env = string + apns_certificates = string + apns_p8 = string + apns_team_id = string + apns_key_id = string + apns_bundle_id = string + safari_apns_certificate = string + safari_site_origin = string + safari_push_id = string + safari_icon_16_16 = string + safari_icon_32_32 = string + safari_icon_64_64 = string + safari_icon_128_128 = string + safari_icon_256_256 = string + site_name = string + created_at = string + updated_at = string + players = int + messageable_players = int + basic_auth_key = string + additional_data_is_root_payload = string + } + } + } +} + +sink { + Console {} + Assert { + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 5 + }, + { + rule_type = MIN_ROW + rule_value = 5 + } + ], + + field_rules = [ + { + field_name = id + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = name + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = players + field_type = int + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} \ No newline at end of file From 2e97612fee3d6461f3f1778d365cee53aacdc804 Mon Sep 17 00:00:00 2001 From: TaoZex <45089228+TaoZex@users.noreply.github.com> Date: Fri, 18 Nov 2022 18:01:13 +0800 Subject: [PATCH 2/2] update pom --- seatunnel-dist/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index 4c81b790d4e..61bbe3a494a 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -173,7 +173,7 @@ org.apache.seatunnel - >connector-http-onesignal + connector-http-onesignal ${project.version} provided