From 729e5c18b8544c6b836cb8ff1fd06018cfdeaf67 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Mon, 20 Nov 2023 09:24:12 +0800 Subject: [PATCH 1/5] Fix sync script and sync docs. --- .../v2.5.1/io-activemq-source.md | 161 +++++++++++++ .../{v3.1.0.5 => v2.10.5.9}/amqp-1-0-sink.md | 8 +- .../{v2.10.5.8 => v3.1.0.6}/amqp-1-0-sink.md | 8 +- .../amqp-1-0-source.md | 8 +- .../amqp-1-0-source.md | 8 +- .../aws-eventbridge-sink.md | 10 +- .../aws-eventbridge-sink.md | 8 +- .../aws-lambda-sink.md | 43 ++-- .../{v3.1.0.5 => v3.1.0.6}/aws-lambda-sink.md | 43 ++-- .../aws-s3-sink/v3.1.0.6/aws-s3-sink.md | 225 ++++++++++++++++++ .../v3.1.0.6/azure-blob-storage-sink.md | 222 +++++++++++++++++ .../cloud-storage-sink.md | 24 +- .../cloud-storage-sink.md | 24 +- .../google-bigquery-sink.md | 10 +- .../google-bigquery-sink.md | 10 +- .../google-bigquery-source.md | 10 +- .../google-bigquery-source.md | 10 +- .../v3.1.0.6/google-cloud-storage-sink.md | 220 +++++++++++++++++ .../google-pubsub-sink.md | 30 +-- .../google-pubsub-sink.md | 30 +-- .../google-pubsub-source.md | 30 +-- .../google-pubsub-source.md | 30 +-- .../lakehouse-sink.md | 26 +- .../{v3.1.0.5 => v3.1.0.6}/lakehouse-sink.md | 26 +- .../lakehouse-source.md | 18 +- .../lakehouse-source.md | 18 +- .../snowflake-sink.md | 26 +- .../{v3.1.0.5 => v3.1.0.6}/snowflake-sink.md | 26 +- .../{v3.1.0.5 => v2.10.5.9}/sqs-sink.md | 14 +- .../{v2.10.5.8 => v3.1.0.6}/sqs-sink.md | 14 +- .../{v3.1.0.5 => v2.10.5.9}/sqs-source.md | 14 +- .../{v2.10.5.8 => v3.1.0.6}/sqs-source.md | 14 +- sync.js | 6 +- 33 files changed, 1102 insertions(+), 272 deletions(-) create mode 100644 connectors/activemq-source/v2.5.1/io-activemq-source.md rename connectors/amqp-1-0-sink/{v3.1.0.5 => v2.10.5.9}/amqp-1-0-sink.md (97%) rename connectors/amqp-1-0-sink/{v2.10.5.8 => v3.1.0.6}/amqp-1-0-sink.md (97%) rename connectors/amqp-1-0-source/{v3.1.0.5 => v2.10.5.9}/amqp-1-0-source.md (97%) rename connectors/amqp-1-0-source/{v2.10.5.8 => v3.1.0.6}/amqp-1-0-source.md (97%) rename connectors/aws-eventbridge-sink/{v2.10.5.8 => v2.10.5.9}/aws-eventbridge-sink.md (99%) rename connectors/aws-eventbridge-sink/{v3.1.0.5 => v3.1.0.6}/aws-eventbridge-sink.md (98%) rename connectors/aws-lambda-sink/{v2.10.5.8 => v2.10.5.9}/aws-lambda-sink.md (91%) rename connectors/aws-lambda-sink/{v3.1.0.5 => v3.1.0.6}/aws-lambda-sink.md (91%) create mode 100644 connectors/aws-s3-sink/v3.1.0.6/aws-s3-sink.md create mode 100644 connectors/azure-blob-storage-sink/v3.1.0.6/azure-blob-storage-sink.md rename connectors/cloud-storage-sink/{v2.10.5.8 => v2.10.5.9}/cloud-storage-sink.md (98%) rename connectors/cloud-storage-sink/{v3.1.0.5 => v3.1.0.6}/cloud-storage-sink.md (98%) rename connectors/google-bigquery-sink/{v2.10.5.8 => v2.10.5.9}/google-bigquery-sink.md (99%) rename connectors/google-bigquery-sink/{v3.1.0.5 => v3.1.0.6}/google-bigquery-sink.md (99%) rename connectors/google-bigquery-source/{v3.1.0.5 => v2.10.5.9}/google-bigquery-source.md (98%) rename connectors/google-bigquery-source/{v2.10.5.8 => v3.1.0.6}/google-bigquery-source.md (98%) create mode 100644 connectors/google-cloud-storage-sink/v3.1.0.6/google-cloud-storage-sink.md rename connectors/google-pubsub-sink/{v2.10.5.8 => v2.10.5.9}/google-pubsub-sink.md (93%) rename connectors/google-pubsub-sink/{v3.1.0.5 => v3.1.0.6}/google-pubsub-sink.md (93%) rename connectors/google-pubsub-source/{v2.10.5.8 => v2.10.5.9}/google-pubsub-source.md (93%) rename connectors/google-pubsub-source/{v3.1.0.5 => v3.1.0.6}/google-pubsub-source.md (93%) rename connectors/lakehouse-sink/{v2.10.5.8 => v2.10.5.9}/lakehouse-sink.md (95%) rename connectors/lakehouse-sink/{v3.1.0.5 => v3.1.0.6}/lakehouse-sink.md (95%) rename connectors/lakehouse-source/{v2.10.5.8 => v2.10.5.9}/lakehouse-source.md (93%) rename connectors/lakehouse-source/{v3.1.0.5 => v3.1.0.6}/lakehouse-source.md (94%) rename connectors/snowflake-sink/{v2.10.5.8 => v2.10.5.9}/snowflake-sink.md (96%) rename connectors/snowflake-sink/{v3.1.0.5 => v3.1.0.6}/snowflake-sink.md (96%) rename connectors/sqs-sink/{v3.1.0.5 => v2.10.5.9}/sqs-sink.md (97%) rename connectors/sqs-sink/{v2.10.5.8 => v3.1.0.6}/sqs-sink.md (97%) rename connectors/sqs-source/{v3.1.0.5 => v2.10.5.9}/sqs-source.md (97%) rename connectors/sqs-source/{v2.10.5.8 => v3.1.0.6}/sqs-source.md (97%) diff --git a/connectors/activemq-source/v2.5.1/io-activemq-source.md b/connectors/activemq-source/v2.5.1/io-activemq-source.md new file mode 100644 index 000000000..0447c6837 --- /dev/null +++ b/connectors/activemq-source/v2.5.1/io-activemq-source.md @@ -0,0 +1,161 @@ +--- +description: ActiveMQ Connector integrates Apache Pulsar with Apache ActiveMQ. +author: gaoran10,dependabot[bot],shibd,sijie +contributors: gaoran10,dependabot[bot],shibd,sijie +language: Java,Shell,Dockerfile +document: +source: https://github.com/streamnative/pulsar-io-activemq +license: Apache License 2.0 +license_link: "https://github.com/streamnative/pulsar-io-activemq/blob/master/LICENSE" +tags: apache-pulsar,pulsar-io,source-connector,sink-connector,apache-activemq +alias: ActiveMQ Sink +features: ["ActiveMQ Connector integrates Apache Pulsar with Apache ActiveMQ. "] +icon: "/images/connectors/activemq_logo_white_vertical.jpg" +download: https://api.github.com/repos/streamnative/pulsar-io-activemq/tarball/refs/tags/v2.5.1 +support: streamnative +support_link: https://github.com/streamnative/pulsar-io-activemq +support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" +owner_name: "streamnative" +owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" +dockerfile: +sn_available: "" +id: "io-activemq-source" +--- + + +The ActiveMQ source connector receives messages from ActiveMQ clusters and writes messages to Pulsar topics. + +# Installation + +``` +git clone https://github.com/streamnative/pulsar-io-activemq.git +cd pulsar-io-activemq/ +mvn clean install -DskipTests +cp target/pulsar-io-activemq-0.0.1.nar $PULSAR_HOME/pulsar-io-activemq-0.0.1.nar +``` + +# Configuration + +The configuration of the ActiveMQ source connector has the following properties. + +## ActiveMQ source connector configuration + +| Name | Type|Required | Default | Description +|------|----------|----------|---------|-------------| +| `protocol` |String| true | "tcp" | The ActiveMQ protocol. | +| `host` | String| true | " " (empty string) | The ActiveMQ host. | +| `port` | int |true | 5672 | The ActiveMQ port. | +| `username` | String|false | " " (empty string) | The username used to authenticate to ActiveMQ. | +| `password` | String|false | " " (empty string) | The password used to authenticate to ActiveMQ. | +| `queueName` | String|false | " " (empty string) | The ActiveMQ queue name that messages should be read from or written to. | +| `topicName` | String|false | " " (empty string) | The ActiveMQ topic name that messages should be read from or written to. | + +## Configure ActiveMQ source connector + +Before using the ActiveMQ source connector, you need to create a configuration file through one of the following methods. + +* JSON + + ```json + { + "tenant": "public", + "namespace": "default", + "name": "activemq-source", + "topicName": "user-op-queue-topic", + "archive": "connectors/pulsar-io-activemq-0.0.1.nar", + "parallelism": 1, + "configs": { + "protocol": "tcp", + "host": "localhost", + "port": "61616", + "username": "admin", + "password": "admin", + "queueName": "user-op-queue" + } + } + ``` + +* YAML + + ```yaml + tenant: "public" + namespace: "default" + name: "activemq-source" + topicName: "user-op-queue-topic" + archive: "connectors/pulsar-io-activemq-0.0.1.nar" + parallelism: 1 + + configs: + protocol: "tcp" + host: "localhost" + port: "61616" + username: "admin" + password: "admin" + queueName: "user-op-queue" + ``` + +1. Prepare ActiveMQ service. + + ``` + docker pull rmohr/activemq + docker run -p 61616:61616 -p 8161:8161 rmohr/activemq + ``` + +2. Put the `pulsar-io-activemq-0.0.1.nar` in the pulsar connectors catalog. + + ``` + cp pulsar-io-activemq-0.0.1.nar $PULSAR_HOME/connectors/pulsar-io-activemq-0.0.1.nar + ``` + +3. Start Pulsar in standalone mode. + + ``` + $PULSAR_HOME/bin/pulsar standalone + ``` + +4. Run ActiveMQ source locally. + + ``` + $PULSAR_HOME/bin/pulsar-admin source localrun --source-config-file activemq-source-config.yaml + ``` + +5. Consume Pulsar messages. + + ``` + bin/pulsar-client consume -s "sub-products" public/default/user-op-queue-topic -n 0 + ``` + +6. Send ActiveMQ messages. + + Use the test method `sendMessage` of the `class org.apache.pulsar.ecosystem.io.activemq.ActiveMQDemo` +to send ActiveMQ messages. + + ``` + @Test + private void sendMessage() throws JMSException { + + ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("tcp://localhost:61616"); + + @Cleanup + Connection connection = connectionFactory.createConnection(); + connection.start(); + + @Cleanup + Session session = connection.createSession(false, Session.CLIENT_ACKNOWLEDGE); + + Destination destination = session.createQueue("user-op-queue"); + + @Cleanup + MessageProducer producer = session.createProducer(destination); + producer.setDeliveryMode(DeliveryMode.NON_PERSISTENT); + + for (int i = 0; i < 10; i++) { + String msgContent = "Hello ActiveMQ - " + i; + ActiveMQTextMessage message = new ActiveMQTextMessage(); + message.setText(msgContent); + producer.send(message); + } + } + ``` + + diff --git a/connectors/amqp-1-0-sink/v3.1.0.5/amqp-1-0-sink.md b/connectors/amqp-1-0-sink/v2.10.5.9/amqp-1-0-sink.md similarity index 97% rename from connectors/amqp-1-0-sink/v3.1.0.5/amqp-1-0-sink.md rename to connectors/amqp-1-0-sink/v2.10.5.9/amqp-1-0-sink.md index 75ce4f079..33ba86a2a 100644 --- a/connectors/amqp-1-0-sink/v3.1.0.5/amqp-1-0-sink.md +++ b/connectors/amqp-1-0-sink/v2.10.5.9/amqp-1-0-sink.md @@ -4,20 +4,20 @@ author: gaoran10,Anonymitaet,dependabot[bot],freeznet contributors: gaoran10,Anonymitaet,dependabot[bot],freeznet language: Java,Shell,Dockerfile,Python document: -source: "https://github.com/streamnative/pulsar-io-amqp-1-0" +source: https://github.com/streamnative/pulsar-io-amqp-1-0 license: Apache License 2.0 license_link: "https://github.com/streamnative/pulsar-io-amqp-1-0/blob/master/LICENSE" tags: alias: AMQP 1.0 Sink Connector features: ["support sink/source for AMQP version 1.0.0"] icon: "/images/connectors/amqp-logo.png" -download: "https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.1.0.5" +download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v2.10.5.9 support: streamnative support_link: https://github.com/streamnative/pulsar-io-amqp-1-0 support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "https://hub.docker.com/r/streamnative/pulsar-io-amqp-1-0" +dockerfile: https://hub.docker.com/r/streamnative/pulsar-io-amqp-1-0 sn_available: "true" id: "amqp-1-0-sink" --- @@ -27,7 +27,7 @@ id: "amqp-1-0-sink" The AMQP 1.0 sink connector pulls messages from Pulsar topics and persists messages to [AMQP 1.0](https://www.amqp.org/). -![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.1.0.5/docs/amqp-1-0-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v2.10.5.9/docs/amqp-1-0-sink.png) ## Quick start diff --git a/connectors/amqp-1-0-sink/v2.10.5.8/amqp-1-0-sink.md b/connectors/amqp-1-0-sink/v3.1.0.6/amqp-1-0-sink.md similarity index 97% rename from connectors/amqp-1-0-sink/v2.10.5.8/amqp-1-0-sink.md rename to connectors/amqp-1-0-sink/v3.1.0.6/amqp-1-0-sink.md index 5899af5c4..985a5ebe6 100644 --- a/connectors/amqp-1-0-sink/v2.10.5.8/amqp-1-0-sink.md +++ b/connectors/amqp-1-0-sink/v3.1.0.6/amqp-1-0-sink.md @@ -4,20 +4,20 @@ author: gaoran10,Anonymitaet,dependabot[bot],freeznet contributors: gaoran10,Anonymitaet,dependabot[bot],freeznet language: Java,Shell,Dockerfile,Python document: -source: "https://github.com/streamnative/pulsar-io-amqp-1-0" +source: https://github.com/streamnative/pulsar-io-amqp-1-0 license: Apache License 2.0 license_link: "https://github.com/streamnative/pulsar-io-amqp-1-0/blob/master/LICENSE" tags: alias: AMQP 1.0 Sink Connector features: ["support sink/source for AMQP version 1.0.0"] icon: "/images/connectors/amqp-logo.png" -download: "https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v2.10.5.8" +download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.1.0.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-amqp-1-0 support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "https://hub.docker.com/r/streamnative/pulsar-io-amqp-1-0" +dockerfile: https://hub.docker.com/r/streamnative/pulsar-io-amqp-1-0 sn_available: "true" id: "amqp-1-0-sink" --- @@ -27,7 +27,7 @@ id: "amqp-1-0-sink" The AMQP 1.0 sink connector pulls messages from Pulsar topics and persists messages to [AMQP 1.0](https://www.amqp.org/). -![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v2.10.5.8/docs/amqp-1-0-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.1.0.6/docs/amqp-1-0-sink.png) ## Quick start diff --git a/connectors/amqp-1-0-source/v3.1.0.5/amqp-1-0-source.md b/connectors/amqp-1-0-source/v2.10.5.9/amqp-1-0-source.md similarity index 97% rename from connectors/amqp-1-0-source/v3.1.0.5/amqp-1-0-source.md rename to connectors/amqp-1-0-source/v2.10.5.9/amqp-1-0-source.md index 08264d24c..72f13cd2f 100644 --- a/connectors/amqp-1-0-source/v3.1.0.5/amqp-1-0-source.md +++ b/connectors/amqp-1-0-source/v2.10.5.9/amqp-1-0-source.md @@ -4,20 +4,20 @@ author: gaoran10,Anonymitaet,dependabot[bot],freeznet contributors: gaoran10,Anonymitaet,dependabot[bot],freeznet language: Java,Shell,Dockerfile,Python document: -source: "https://github.com/streamnative/pulsar-io-amqp-1-0" +source: https://github.com/streamnative/pulsar-io-amqp-1-0 license: Apache License 2.0 license_link: "https://github.com/streamnative/pulsar-io-amqp-1-0/blob/master/LICENSE" tags: alias: AMQP 1.0 Source Connector features: ["support sink/source for AMQP version 1.0.0"] icon: "/images/connectors/amqp-logo.png" -download: "https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.1.0.5" +download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v2.10.5.9 support: streamnative support_link: https://github.com/streamnative/pulsar-io-amqp-1-0 support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "https://hub.docker.com/r/streamnative/pulsar-io-amqp-1-0" +dockerfile: https://hub.docker.com/r/streamnative/pulsar-io-amqp-1-0 sn_available: "true" id: "amqp-1-0-source" --- @@ -27,7 +27,7 @@ id: "amqp-1-0-source" The AMQP 1.0 source connector receives messages from [AMQP 1.0](https://www.amqp.org/) and writes messages to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.1.0.5/docs/amqp-1-0-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v2.10.5.9/docs/amqp-1-0-source.png) ## Quick start diff --git a/connectors/amqp-1-0-source/v2.10.5.8/amqp-1-0-source.md b/connectors/amqp-1-0-source/v3.1.0.6/amqp-1-0-source.md similarity index 97% rename from connectors/amqp-1-0-source/v2.10.5.8/amqp-1-0-source.md rename to connectors/amqp-1-0-source/v3.1.0.6/amqp-1-0-source.md index 196306005..22f916b96 100644 --- a/connectors/amqp-1-0-source/v2.10.5.8/amqp-1-0-source.md +++ b/connectors/amqp-1-0-source/v3.1.0.6/amqp-1-0-source.md @@ -4,20 +4,20 @@ author: gaoran10,Anonymitaet,dependabot[bot],freeznet contributors: gaoran10,Anonymitaet,dependabot[bot],freeznet language: Java,Shell,Dockerfile,Python document: -source: "https://github.com/streamnative/pulsar-io-amqp-1-0" +source: https://github.com/streamnative/pulsar-io-amqp-1-0 license: Apache License 2.0 license_link: "https://github.com/streamnative/pulsar-io-amqp-1-0/blob/master/LICENSE" tags: alias: AMQP 1.0 Source Connector features: ["support sink/source for AMQP version 1.0.0"] icon: "/images/connectors/amqp-logo.png" -download: "https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v2.10.5.8" +download: https://api.github.com/repos/streamnative/pulsar-io-amqp-1-0/tarball/refs/tags/v3.1.0.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-amqp-1-0 support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "https://hub.docker.com/r/streamnative/pulsar-io-amqp-1-0" +dockerfile: https://hub.docker.com/r/streamnative/pulsar-io-amqp-1-0 sn_available: "true" id: "amqp-1-0-source" --- @@ -27,7 +27,7 @@ id: "amqp-1-0-source" The AMQP 1.0 source connector receives messages from [AMQP 1.0](https://www.amqp.org/) and writes messages to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v2.10.5.8/docs/amqp-1-0-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-amqp-1-0/v3.1.0.6/docs/amqp-1-0-source.png) ## Quick start diff --git a/connectors/aws-eventbridge-sink/v2.10.5.8/aws-eventbridge-sink.md b/connectors/aws-eventbridge-sink/v2.10.5.9/aws-eventbridge-sink.md similarity index 99% rename from connectors/aws-eventbridge-sink/v2.10.5.8/aws-eventbridge-sink.md rename to connectors/aws-eventbridge-sink/v2.10.5.9/aws-eventbridge-sink.md index c32ed86ae..3b8054229 100644 --- a/connectors/aws-eventbridge-sink/v2.10.5.8/aws-eventbridge-sink.md +++ b/connectors/aws-eventbridge-sink/v2.10.5.9/aws-eventbridge-sink.md @@ -6,18 +6,18 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-aws-eventbridge/blob/master/LICENSE" +license_link: "" tags: alias: AWS EventBridge Sink Connector features: [""] icon: "/images/connectors/aws-eventbridge.png" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "true" id: "aws-eventbridge-sink" --- @@ -26,7 +26,7 @@ id: "aws-eventbridge-sink" The [Amazon EventBridge](https://aws.amazon.com/eventbridge/) sink connector pulls data from Pulsar topics and persists data to Amazon EventBridge. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-eventbridge/v2.10.5.8/docs/aws-eventbridge-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-eventbridge/v2.10.5.9/docs/aws-eventbridge-sink.png) ## Quick start diff --git a/connectors/aws-eventbridge-sink/v3.1.0.5/aws-eventbridge-sink.md b/connectors/aws-eventbridge-sink/v3.1.0.6/aws-eventbridge-sink.md similarity index 98% rename from connectors/aws-eventbridge-sink/v3.1.0.5/aws-eventbridge-sink.md rename to connectors/aws-eventbridge-sink/v3.1.0.6/aws-eventbridge-sink.md index 1fbd8001d..26a441de2 100644 --- a/connectors/aws-eventbridge-sink/v3.1.0.5/aws-eventbridge-sink.md +++ b/connectors/aws-eventbridge-sink/v3.1.0.6/aws-eventbridge-sink.md @@ -6,14 +6,14 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-aws-eventbridge/blob/master/LICENSE" +license_link: "" tags: alias: AWS EventBridge Sink Connector features: [""] icon: "/images/connectors/aws-eventbridge.png" -download: Business License +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" @@ -26,7 +26,7 @@ id: "aws-eventbridge-sink" The [Amazon EventBridge](https://aws.amazon.com/eventbridge/) sink connector pulls data from Pulsar topics and persists data to Amazon EventBridge. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-eventbridge/v3.1.0.5/docs/aws-eventbridge-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-eventbridge/v3.1.0.6/docs/aws-eventbridge-sink.png) ## Quick start diff --git a/connectors/aws-lambda-sink/v2.10.5.8/aws-lambda-sink.md b/connectors/aws-lambda-sink/v2.10.5.9/aws-lambda-sink.md similarity index 91% rename from connectors/aws-lambda-sink/v2.10.5.8/aws-lambda-sink.md rename to connectors/aws-lambda-sink/v2.10.5.9/aws-lambda-sink.md index 43a3b8fc2..ebd923d79 100644 --- a/connectors/aws-lambda-sink/v2.10.5.8/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v2.10.5.9/aws-lambda-sink.md @@ -1,23 +1,23 @@ --- description: -author: freeznet,nlu90,Anonymitaet,Technoboy- -contributors: freeznet,nlu90,Anonymitaet,Technoboy- +author: freeznet,RobertIndie,nlu90,Anonymitaet +contributors: freeznet,RobertIndie,nlu90,Anonymitaet language: Java,Shell,Python,Dockerfile document: -source: "Private source" +source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-aws-lambda/blob/master/LICENSE" +license_link: "" tags: alias: AWS Lambda Sink features: [""] icon: "/images/connectors/lambda-logo.png" -download: "Business License" +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "" +dockerfile: sn_available: "true" id: "aws-lambda-sink" --- @@ -25,7 +25,7 @@ id: "aws-lambda-sink" The [AWS Lambda](https://aws.amazon.com/lambda/) sink connector is a [Pulsar IO connector](http://pulsar.apache.org/docs/en/next/io-overview/) for pulling messages from Pulsar topics to AWS Lambda to invoke Lambda functions. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-lambda/v2.10.5.8/docs/lambda-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-lambda/v2.10.5.9/docs/lambda-sink.png) # How to get @@ -35,7 +35,7 @@ This section describes how to build the AWS Lambda sink connector. You can get the AWS Lambda sink connector using one of the following methods if you use [Pulsar Function Worker](https://pulsar.apache.org/docs/en/functions-worker/) to run connectors in a cluster. -- Download the NAR package from [the download page](https://github.com/streamnative/pulsar-io-aws-lambda/releases/download/v2.10.5.8/pulsar-io-google-pubsub-2.10.5.8.nar). +- Download the NAR package from [the download page](https://github.com/streamnative/pulsar-io-aws-lambda/releases/download/v2.10.5.9/pulsar-io-google-pubsub-2.10.5.9.nar). - Build it from the source code. @@ -57,7 +57,7 @@ To build the AWS Lambda sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-aws-lambda-2.10.5.8.nar + pulsar-io-aws-lambda-2.10.5.9.nar ``` ## Work with Function Mesh @@ -93,7 +93,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-aws-lambda-topic" ], - "archive": "connectors/pulsar-io-aws-lambda-2.10.5.8.nar", + "archive": "connectors/pulsar-io-aws-lambda-2.10.5.9.nar", "parallelism": 1, "configs": { @@ -115,7 +115,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you name: "aws-lambda-sink" inputs: - "test-aws-lambda-topic" - archive: "connectors/pulsar-io-aws-lambda-2.10.5.8.nar" + archive: "connectors/pulsar-io-aws-lambda-2.10.5.9.nar" parallelism: 1 configs: @@ -139,7 +139,7 @@ kind: Sink metadata: name: aws-lambda-sink-sample spec: - image: streamnative/pulsar-io-aws-lambda:2.10.5.8 + image: streamnative/pulsar-io-aws-lambda:2.10.5.9 className: org.apache.pulsar.ecosystem.io.aws.lambda.AWSLambdaBytesSink replicas: 1 input: @@ -163,7 +163,7 @@ spec: cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-aws-lambda-2.10.5.8.nar + jar: connectors/pulsar-io-aws-lambda-2.10.5.9.nar clusterName: test-pulsar autoAck: true ``` @@ -186,7 +186,7 @@ This example shows how to create an AWS Lambda sink connector on a Pulsar cluste ```bash PULSAR_HOME/bin/pulsar-admin sinks create \ ---archive pulsar-io-aws-lambda-2.10.5.8.nar \ +--archive pulsar-io-aws-lambda-2.10.5.9.nar \ --sink-config-file aws-lambda-sink-config.yaml \ --classname org.apache.pulsar.ecosystem.io.aws.lambda.AWSLambdaBytesSink \ --name aws-lambda-sink @@ -209,8 +209,8 @@ This example describes how to use the AWS Lambda sink connector to pull messages 2. Copy the NAR package of the AWS Lambda connector to the Pulsar connectors directory. ``` - cp pulsar-io-aws-lambda-2.10.5.8.nar - PULSAR_HOME/connectors/pulsar-io-aws-lambda-2.10.5.8.nar + cp pulsar-io-aws-lambda-2.10.5.9.nar + PULSAR_HOME/connectors/pulsar-io-aws-lambda-2.10.5.9.nar ``` 3. Start Pulsar in standalone mode. @@ -244,8 +244,8 @@ This example explains how to create an AWS Lambda sink connector in an on-premis 1. Copy the NAR package of the AWS Lambda connector to the Pulsar connectors directory. ``` - cp pulsar-io-aws-lambda-2.10.5.8.nar - PULSAR_HOME/connectors/pulsar-io-aws-lambda-2.10.5.8.nar + cp pulsar-io-aws-lambda-2.10.5.9.nar + PULSAR_HOME/connectors/pulsar-io-aws-lambda-2.10.5.9.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). @@ -297,7 +297,7 @@ This example describes how to create an AWS Lambda sink connector for a Kuberbet metadata: name: aws-lambda-sink-sample spec: - image: streamnative/pulsar-io-aws-lambda:2.10.5.8 + image: streamnative/pulsar-io-aws-lambda:2.10.5.9 className: org.apache.pulsar.ecosystem.io.aws.lambda.AWSLambdaBytesSink replicas: 1 input: @@ -321,7 +321,7 @@ This example describes how to create an AWS Lambda sink connector for a Kuberbet cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-aws-lambda-2.10.5.8.nar + jar: connectors/pulsar-io-aws-lambda-2.10.5.9.nar clusterName: test-pulsar autoAck: true ``` @@ -357,3 +357,4 @@ This example describes how to create an AWS Lambda sink connector for a Kuberbet After that, you can use the AWS Lambda sink connector to export Pulsar messages to AWS Lambda to invoke Lambda functions. + diff --git a/connectors/aws-lambda-sink/v3.1.0.5/aws-lambda-sink.md b/connectors/aws-lambda-sink/v3.1.0.6/aws-lambda-sink.md similarity index 91% rename from connectors/aws-lambda-sink/v3.1.0.5/aws-lambda-sink.md rename to connectors/aws-lambda-sink/v3.1.0.6/aws-lambda-sink.md index 9cc8bf443..6b0ee68dc 100644 --- a/connectors/aws-lambda-sink/v3.1.0.5/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v3.1.0.6/aws-lambda-sink.md @@ -1,23 +1,23 @@ --- description: -author: freeznet,nlu90,Anonymitaet,Technoboy- -contributors: freeznet,nlu90,Anonymitaet,Technoboy- +author: freeznet,RobertIndie,nlu90,Anonymitaet +contributors: freeznet,RobertIndie,nlu90,Anonymitaet language: Java,Shell,Python,Dockerfile document: -source: "Private source" +source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-aws-lambda/blob/master/LICENSE" +license_link: "" tags: alias: AWS Lambda Sink features: [""] icon: "/images/connectors/lambda-logo.png" -download: "Business License" +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "" +dockerfile: sn_available: "true" id: "aws-lambda-sink" --- @@ -25,7 +25,7 @@ id: "aws-lambda-sink" The [AWS Lambda](https://aws.amazon.com/lambda/) sink connector is a [Pulsar IO connector](http://pulsar.apache.org/docs/en/next/io-overview/) for pulling messages from Pulsar topics to AWS Lambda to invoke Lambda functions. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-lambda/v3.1.0.5/docs/lambda-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-aws-lambda/v3.1.0.6/docs/lambda-sink.png) # How to get @@ -35,7 +35,7 @@ This section describes how to build the AWS Lambda sink connector. You can get the AWS Lambda sink connector using one of the following methods if you use [Pulsar Function Worker](https://pulsar.apache.org/docs/en/functions-worker/) to run connectors in a cluster. -- Download the NAR package from [the download page](https://github.com/streamnative/pulsar-io-aws-lambda/releases/download/v3.1.0.5/pulsar-io-google-pubsub-3.1.0.5.nar). +- Download the NAR package from [the download page](https://github.com/streamnative/pulsar-io-aws-lambda/releases/download/v3.1.0.6/pulsar-io-google-pubsub-3.1.0.6.nar). - Build it from the source code. @@ -57,7 +57,7 @@ To build the AWS Lambda sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-aws-lambda-3.1.0.5.nar + pulsar-io-aws-lambda-3.1.0.6.nar ``` ## Work with Function Mesh @@ -93,7 +93,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-aws-lambda-topic" ], - "archive": "connectors/pulsar-io-aws-lambda-3.1.0.5.nar", + "archive": "connectors/pulsar-io-aws-lambda-3.1.0.6.nar", "parallelism": 1, "configs": { @@ -115,7 +115,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you name: "aws-lambda-sink" inputs: - "test-aws-lambda-topic" - archive: "connectors/pulsar-io-aws-lambda-3.1.0.5.nar" + archive: "connectors/pulsar-io-aws-lambda-3.1.0.6.nar" parallelism: 1 configs: @@ -139,7 +139,7 @@ kind: Sink metadata: name: aws-lambda-sink-sample spec: - image: streamnative/pulsar-io-aws-lambda:3.1.0.5 + image: streamnative/pulsar-io-aws-lambda:3.1.0.6 className: org.apache.pulsar.ecosystem.io.aws.lambda.AWSLambdaBytesSink replicas: 1 input: @@ -163,7 +163,7 @@ spec: cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-aws-lambda-3.1.0.5.nar + jar: connectors/pulsar-io-aws-lambda-3.1.0.6.nar clusterName: test-pulsar autoAck: true ``` @@ -186,7 +186,7 @@ This example shows how to create an AWS Lambda sink connector on a Pulsar cluste ```bash PULSAR_HOME/bin/pulsar-admin sinks create \ ---archive pulsar-io-aws-lambda-3.1.0.5.nar \ +--archive pulsar-io-aws-lambda-3.1.0.6.nar \ --sink-config-file aws-lambda-sink-config.yaml \ --classname org.apache.pulsar.ecosystem.io.aws.lambda.AWSLambdaBytesSink \ --name aws-lambda-sink @@ -209,8 +209,8 @@ This example describes how to use the AWS Lambda sink connector to pull messages 2. Copy the NAR package of the AWS Lambda connector to the Pulsar connectors directory. ``` - cp pulsar-io-aws-lambda-3.1.0.5.nar - PULSAR_HOME/connectors/pulsar-io-aws-lambda-3.1.0.5.nar + cp pulsar-io-aws-lambda-3.1.0.6.nar + PULSAR_HOME/connectors/pulsar-io-aws-lambda-3.1.0.6.nar ``` 3. Start Pulsar in standalone mode. @@ -244,8 +244,8 @@ This example explains how to create an AWS Lambda sink connector in an on-premis 1. Copy the NAR package of the AWS Lambda connector to the Pulsar connectors directory. ``` - cp pulsar-io-aws-lambda-3.1.0.5.nar - PULSAR_HOME/connectors/pulsar-io-aws-lambda-3.1.0.5.nar + cp pulsar-io-aws-lambda-3.1.0.6.nar + PULSAR_HOME/connectors/pulsar-io-aws-lambda-3.1.0.6.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). @@ -297,7 +297,7 @@ This example describes how to create an AWS Lambda sink connector for a Kuberbet metadata: name: aws-lambda-sink-sample spec: - image: streamnative/pulsar-io-aws-lambda:3.1.0.5 + image: streamnative/pulsar-io-aws-lambda:3.1.0.6 className: org.apache.pulsar.ecosystem.io.aws.lambda.AWSLambdaBytesSink replicas: 1 input: @@ -321,7 +321,7 @@ This example describes how to create an AWS Lambda sink connector for a Kuberbet cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-aws-lambda-3.1.0.5.nar + jar: connectors/pulsar-io-aws-lambda-3.1.0.6.nar clusterName: test-pulsar autoAck: true ``` @@ -357,3 +357,4 @@ This example describes how to create an AWS Lambda sink connector for a Kuberbet After that, you can use the AWS Lambda sink connector to export Pulsar messages to AWS Lambda to invoke Lambda functions. + diff --git a/connectors/aws-s3-sink/v3.1.0.6/aws-s3-sink.md b/connectors/aws-s3-sink/v3.1.0.6/aws-s3-sink.md new file mode 100644 index 000000000..2866a4e5b --- /dev/null +++ b/connectors/aws-s3-sink/v3.1.0.6/aws-s3-sink.md @@ -0,0 +1,225 @@ +--- +description: Cloud Storage Connector integrates Apache Pulsar with cloud storage. +author: freeznet,jianyun8023,nicoloboschi,Huanli-Meng +contributors: freeznet,jianyun8023,nicoloboschi,Huanli-Meng +language: Java,Shell,Dockerfile +document: +source: https://github.com/streamnative/pulsar-io-cloud-storage +license: Apache License 2.0 +license_link: "https://github.com/streamnative/pulsar-io-cloud-storage/blob/master/LICENSE" +tags: +alias: AWS S3 Sink Connector +features: ["Cloud Storage Connector integrates Apache Pulsar with cloud storage."] +icon: "/images/pulsar-hub.svg" +download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.1.0.6 +support: streamnative +support_link: https://github.com/streamnative/pulsar-io-cloud-storage +support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" +owner_name: "streamnative" +owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" +dockerfile: https://hub.docker.com/r/streamnative/pulsar-io-cloud-storage +sn_available: "true" +id: "aws-s3-sink" +--- + + +The [AWS S3](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html) sink connector pulls data from Pulsar topics and persists data to AWS S3 buckets. + +![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.1.0.6/docs/aws-s3-sink.png) + +## Quick start + +### Prerequisites + +The prerequisites for connecting an AWS S3 sink connector to external systems include: + +1. Create S3 buckets in AWS. +2. Create the [AWS User](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html) and create `AccessKey`(Please record `AccessKey` and `SecretAccessKey`). +3. Assign permissions to AWS User, and ensure they have the following permissions to the AWS S3. +```text +- s3:AbortMultipartUpload +- s3:GetObject* +- s3:PutObject* +- s3:List* +``` + +### 1. Create a connector + +Depending on the environment, there are several ways to create an AWS S3 sink connector: + +- [Create Connector on StreamNative Cloud](https://docs.streamnative.io/docs/connector-create). +- [Create Connector with Function worker](https://pulsar.apache.org/docs/3.0.x/io-quickstart/). + Using this way requires you to download a **JAR** package to create a built-in or non-built-in connector. You can download the version you need from [here](https://github.com/streamnative/pulsar-io-cloud-storage/releases). +- [Create Connector with Function mesh](https://functionmesh.io/docs/connectors/run-connector). + Using this way requires you to set the docker image. You can choose the version you want to launch from [here](https://hub.docker.com/r/streamnative/pulsar-io-cloud-storage/tags) + +No matter how you create an AWS S3 sink connector, the minimum configuration contains the following parameters. + +```yaml + configs: + accessKeyId: "{{Your access access key}}" + secretAccessKey: "{{Your secret access key}}" + provider: "s3v2" + bucket: "{{Your bucket name}}" + region: "{{Your AWS S3 region}}" + formatType: "json" + partitionerType: "PARTITION" +``` + +> * The configuration structure varies depending on how you create the AWS S3 sink connector. +> For example, some are **JSON**, some are **YAML**, and some are **Kubernetes YAML**. You need to adapt the configs to the corresponding format. +> +> * If you want to configure more parameters, see [Configuration Properties](#configuration-properties) for reference. + +### 2. Send messages to the topic + +{% callout title="Note" type="note" %} +If your connector is created on StreamNative Cloud, you need to authenticate your clients. See [Build applications using Pulsar clients](https://docs.streamnative.io/docs/qs-connect#jumpstart-for-beginners) for more information. +{% /callout %} + +``` java + public static void main(String[] args) throws Exception { + PulsarClient client = PulsarClient.builder() + .serviceUrl("{{Your Pulsar URL}}") + .build(); + + + Producer producer = client.newProducer(Schema.STRING) + .topic("{{Your topic name}}") + .create(); + + for (int i = 0; i < 10; i++) { + // JSON string containing a single character + String message = "{\"test-message\": \"test-value\"}"; + producer.send(message); + } + + producer.close(); + client.close(); + } +``` + +### 3. Display data on AWS S3 console + +You can see the object at public/default/{{Your topic name}}-partition-0/xxxx.json on the AWS S3 console. Download and open it, the content is: + +```text +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +``` + +## Configuration Properties + +Before using the AWS S3 sink connector, you need to configure it. This table outlines the properties and the descriptions. + +| Name | Type | Required | Default | Description | +|---------------------------------|---------|----------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `provider` | String | True | null | The AWS S3 client type, such as `aws-s3`,`s3v2`(`s3v2` uses the AWS client but not the JCloud client). | +| `accessKeyId` | String | True | null | The AWS access key ID. It requires permission to write objects. | +| `secretAccessKey` | String | True | null | The AWS secret access key. | +| `bucket` | String | True | null | The AWS S3 bucket. | +| `formatType` | String | True | null | The data format type. Available options are JSON, Avro, Bytes, or Parquet. By default, it is set to JSON. | +| `partitionerType` | String | True | null | The partitioning type. It can be configured by topic partitions or by time. By default, the partition type is configured by topic partitions. | +| `region` | String | False | null | The AWS S3 region. Either the endpoint or region must be set. | +| `endpoint` | String | False | null | The AWS S3 endpoint. Either the endpoint or region must be set. | +| `role` | String | False | null | The AWS role. | +| `roleSessionName` | String | False | null | The AWS role session name. | +| `timePartitionPattern` | String | False | "yyyy-MM-dd" | The format pattern of the time-based partitioning. For details, refer to the Java date and time format. | +| `timePartitionDuration` | String | False | "86400000" | The time interval for time-based partitioning. Support formatted interval string, such as `30d`, `24h`, `30m`, `10s`, and also support number in milliseconds precision, such as `86400000` refers to `24h` or `1d`. | +| `partitionerUseIndexAsOffset` | Boolean | False | false | Whether to use the Pulsar's message index as offset or the record sequence. It's recommended if the incoming messages may be batched. The brokers may or not expose the index metadata and, if it's not present on the record, the sequence will be used. See [PIP-70](https://github.com/apache/pulsar/wiki/PIP-70%3A-Introduce-lightweight-broker-entry-metadata) for more details. | +| `batchSize` | int | False | 10 | The number of records submitted in batch. | +| `batchTimeMs` | long | False | 1000 | The interval for batch submission. | +| `maxBatchBytes` | long | False | 10000000 | The maximum number of bytes in a batch. | +| `sliceTopicPartitionPath` | Boolean | False | false | When it is set to `true`, split the partitioned topic name into separate folders in the bucket path. | +| `withMetadata` | Boolean | False | false | Save message attributes to metadata. | +| `useHumanReadableMessageId` | Boolean | False | false | Use a human-readable format string for messageId in message metadata. The messageId is in a format like `ledgerId:entryId:partitionIndex:batchIndex`. Otherwise, the messageId is a Hex-encoded string. | +| `withTopicPartitionNumber` | Boolean | False | true | When it is set to `true`, include the topic partition number to the object path. | +| `bytesFormatTypeSeparator` | String | False | "0x10" | It is inserted between records for the `formatType` of bytes. By default, it is set to '0x10'. An input record that contains the line separator looks like multiple records in the output object. | +| `pendingQueueSize` | int | False | 10 | The number of records buffered in queue. By default, it is equal to `batchSize`. You can set it manually. | +| `useHumanReadableSchemaVersion` | Boolean | False | false | Use a human-readable format string for the schema version in the message metadata. If it is set to `true`, the schema version is in plain string format. Otherwise, the schema version is in hex-encoded string format. | +| `skipFailedMessages` | Boolean | False | false | Configure whether to skip a message which it fails to be processed. If it is set to `true`, the connector will skip the failed messages by `ack` it. Otherwise, the connector will `fail` the message. | +| `pathPrefix` | String | False | false | If it is set, the output files are stored in a folder under the given bucket path. The `pathPrefix` must be in the format of `xx/xxx/`. | +| `avroCodec` | String | False | snappy | Compression codec used when formatType=`avro`. Available compression types are: null (no compression), deflate, bzip2, xz, zstandard, snappy. | +| `parquetCodec` | String | False | gzip | Compression codec used when formatType=`parquet`. Available compression types are: null (no compression), snappy, gzip, lzo, brotli, lz4, zstd. | +| `jsonAllowNaN` | Boolean | False | false | Recognize 'NaN', 'INF', '-INF' as legal floating number values when formatType=`json`. Since JSON specification does not allow such values this is a non-standard feature and disabled by default. | + +## Advanced features + +### Data format types + +AWS S3 Sink Connector provides multiple output format options, including JSON, Avro, Bytes, or Parquet. The default format is JSON. +With current implementation, there are some limitations for different formats: + +This table lists the Pulsar Schema types supported by the writers. + +| Pulsar Schema | Writer: Avro | Writer: JSON | Writer: Parquet | Writer: Bytes | +|----------------|--------------|--------------|-----------------|---------------| +| Primitive | ✗ | ✔ * | ✗ | ✔ | +| Avro | ✔ | ✔ | ✔ | ✔ | +| Json | ✔ | ✔ | ✔ | ✔ | +| Protobuf ** | ✔ | ✔ | ✔ | ✔ | +| ProtobufNative | ✔ *** | ✗ | ✔ | ✔ | + +> *: The JSON writer will try to convert the data with a `String` or `Bytes` schema to JSON-format data if convertable. +> +> **: The Protobuf schema is based on the Avro schema. It uses Avro as an intermediate format, so it may not provide the best effort conversion. +> +> ***: The ProtobufNative record holds the Protobuf descriptor and the message. When writing to Avro format, the connector uses [avro-protobuf](https://github.com/apache/avro/tree/master/lang/java/protobuf) to do the conversion. + +This table lists the support of `withMetadata` configurations for different writer formats: + +| Writer Format | `withMetadata` | +|---------------|----------------| +| Avro | ✔ | +| JSON | ✔ | +| Parquet | ✔ * | +| Bytes | ✗ | + +> *: When using `Parquet` with `PROTOBUF_NATIVE` format, the connector will write the messages with `DynamicMessage` format. When `withMetadata` is set to `true`, the connector will add `__message_metadata__` to the messages with `PulsarIOCSCProtobufMessageMetadata` format. +> +> For example, if a message `User` has the following schema: +> ```protobuf +> syntax = "proto3"; +> message User { +> string name = 1; +> int32 age = 2; +> } +> ``` +> +> When `withMetadata` is set to `true`, the connector will write the message `DynamicMessage` with the following schema: +> ```protobuf +> syntax = "proto3"; +> message PulsarIOCSCProtobufMessageMetadata { +> map properties = 1; +> string schema_version = 2; +> string message_id = 3; +> } +> message User { +> string name = 1; +> int32 age = 2; +> PulsarIOCSCProtobufMessageMetadata __message_metadata__ = 3; +> } +> ``` +> + + +### Dead-letter topics + +To use a dead-letter topic, you need to set `skipFailedMessages` to `false`, and set `--max-redeliver-count` and `--dead-letter-topic` when submit the connector with the `pulsar-admin` CLI tool. For more info about dead-letter topics, see the [Pulsar documentation](https://pulsar.apache.org/docs/en/concepts-messaging/#dead-letter-topic). +If a message fails to be sent to the AWS S3 and there is a dead-letter topic, the connector will send the message to the dead-letter topic. + +### Sink flushing only after batchTimeMs elapses + +There is a scenario where the sink is only flushing whenever the `batchTimeMs` has elapsed, even though there are many messages waiting to be processed. +The reason for this is that the sink will only acknowledge messages after they are flushed to AWS S3 but the broker stops sending messages when it reaches a certain limit of unacknowledged messages. +If this limit is lower or close to `batchSize`, the sink never receives enough messages to trigger a flush based on the amount of messages. +In this case please ensure the `maxUnackedMessagesPerConsumer` set in the broker configuration is sufficiently larger than the `batchSize` setting of the sink. + diff --git a/connectors/azure-blob-storage-sink/v3.1.0.6/azure-blob-storage-sink.md b/connectors/azure-blob-storage-sink/v3.1.0.6/azure-blob-storage-sink.md new file mode 100644 index 000000000..56c5d6b1c --- /dev/null +++ b/connectors/azure-blob-storage-sink/v3.1.0.6/azure-blob-storage-sink.md @@ -0,0 +1,222 @@ +--- +description: Cloud Storage Connector integrates Apache Pulsar with cloud storage. +author: freeznet,jianyun8023,nicoloboschi,Huanli-Meng +contributors: freeznet,jianyun8023,nicoloboschi,Huanli-Meng +language: Java,Shell,Dockerfile +document: +source: https://github.com/streamnative/pulsar-io-cloud-storage +license: Apache License 2.0 +license_link: "https://github.com/streamnative/pulsar-io-cloud-storage/blob/master/LICENSE" +tags: +alias: Azure Blob Storage Sink Connector +features: ["Cloud Storage Connector integrates Apache Pulsar with cloud storage."] +icon: "/images/pulsar-hub.svg" +download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.1.0.6 +support: streamnative +support_link: https://github.com/streamnative/pulsar-io-cloud-storage +support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" +owner_name: "streamnative" +owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" +dockerfile: https://hub.docker.com/r/streamnative/pulsar-io-cloud-storage +sn_available: "true" +id: "azure-blob-storage-sink" +--- + + +The [Azure Blob Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-overview) sink connector pulls data from Pulsar topics and persists data to Azure Blob Storage containers. + +![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.1.0.6/docs/azure-blob-storage-sink.png) + +## Quick start + +### Prerequisites + +The prerequisites for connecting an Azure Blob Storage sink connector to external systems include: + +1. Create Blob Storage container in Azure Cloud. +2. Get Storage account `Connection string`. + + +### 1. Create a connector + +Depending on the environment, there are several ways to create an Azure Blob Storage sink connector: + +- [Create Connector on StreamNative Cloud](https://docs.streamnative.io/docs/connector-create). +- [Create Connector with Function worker](https://pulsar.apache.org/docs/3.0.x/io-quickstart/). + Using this way requires you to download a **NAR** package to create a built-in or non-built-in connector. You can download the version you need from [here](https://github.com/streamnative/pulsar-io-cloud-storage/releases). +- [Create Connector with Function mesh](https://functionmesh.io/docs/connectors/run-connector). + Using this way requires you to set the docker image. You can choose the version you want to launch from [here](https://hub.docker.com/r/streamnative/pulsar-io-cloud-storage/tags) + +No matter how you create an Azure Blob Storage sink connector, the minimum configuration contains the following parameters. + +```yaml + configs: + provider: "azure-blob-storage" + azureStorageAccountConnectionString: {{Your azure blob storage account connection string}} + bucket: {{Your container name}} + formatType: "json" + partitionerType: "PARTITION" +``` + +> * The configuration structure varies depending on how you create the Azure Blob Storage sink connector. +> For example, some are **JSON**, some are **YAML**, and some are **Kubernetes YAML**. You need to adapt the configs to the corresponding format. +> +> * If you want to configure more parameters, see [Configuration Properties](#configuration-properties) for reference. + +### 2. Send messages to the topic + +{% callout title="Note" type="note" %} +If your connector is created on StreamNative Cloud, you need to authenticate your clients. See [Build applications using Pulsar clients](https://docs.streamnative.io/docs/qs-connect#jumpstart-for-beginners) for more information. +{% /callout %} + +``` java + public static void main(String[] args) throws Exception { + PulsarClient client = PulsarClient.builder() + .serviceUrl("{{Your Pulsar URL}}") + .build(); + + + Producer producer = client.newProducer(Schema.STRING) + .topic("{{Your topic name}}") + .create(); + + for (int i = 0; i < 10; i++) { + // JSON string containing a single character + String message = "{\"test-message\": \"test-value\"}"; + producer.send(message); + } + + producer.close(); + client.close(); + } +``` + +### 3. Display data on Azure Blob Storage console + +You can see the object at public/default/{{Your topic name}}-partition-0/xxxx.json on the Azure Blob Storage console. Download and open it, the content is: + +```text +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +``` + +## Configuration Properties + +Before using the Azure Blob Storage sink connector, you need to configure it. This table outlines the properties and the descriptions. + +| Name | Type | Required | Default | Description | +|---------------------------------------|---------|----------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `provider` | String | True | null | The Cloud Storage type, Azure Blob Storage only supports the `azure-blob-storage` provider. | +| `bucket` | String | True | null | The Azure Blob Storage container name. | +| `formatType` | String | True | null | The data format type. Available options are JSON, Avro, Bytes, or Parquet. By default, it is set to JSON. | +| `partitionerType` | String | True | null | The partitioning type. It can be configured by topic partitions or by time. By default, the partition type is configured by topic partitions. | +| `azureStorageAccountConnectionString` | String | False | "" | The Azure Blob Storage connection string. Required when authenticating via connection string. | +| `azureStorageAccountSASToken` | String | False | "" | The Azure Blob Storage account SAS token. Required when authenticating via SAS token. | +| `azureStorageAccountName` | String | False | "" | The Azure Blob Storage account name. Required when authenticating via account name and account key. | +| `azureStorageAccountKey` | String | False | "" | The Azure Blob Storage account key. Required when authenticating via account name and account key. | +| `endpoint` | String | False | null | The Azure Blob Storage endpoint. Required when authenticating via account name or SAS token. | +| `timePartitionPattern` | String | False | "yyyy-MM-dd" | The format pattern of the time-based partitioning. For details, refer to the Java date and time format. | +| `timePartitionDuration` | String | False | "86400000" | The time interval for time-based partitioning. Support formatted interval string, such as `30d`, `24h`, `30m`, `10s`, and also support number in milliseconds precision, such as `86400000` refers to `24h` or `1d`. | +| `partitionerUseIndexAsOffset` | Boolean | False | false | Whether to use the Pulsar's message index as offset or the record sequence. It's recommended if the incoming messages may be batched. The brokers may or not expose the index metadata and, if it's not present on the record, the sequence will be used. See [PIP-70](https://github.com/apache/pulsar/wiki/PIP-70%3A-Introduce-lightweight-broker-entry-metadata) for more details. | +| `batchSize` | int | False | 10 | The number of records submitted in batch. | +| `batchTimeMs` | long | False | 1000 | The interval for batch submission. | +| `maxBatchBytes` | long | False | 10000000 | The maximum number of bytes in a batch. | +| `sliceTopicPartitionPath` | Boolean | False | false | When it is set to `true`, split the partitioned topic name into separate folders in the bucket path. | +| `withMetadata` | Boolean | False | false | Save message attributes to metadata. | +| `useHumanReadableMessageId` | Boolean | False | false | Use a human-readable format string for messageId in message metadata. The messageId is in a format like `ledgerId:entryId:partitionIndex:batchIndex`. Otherwise, the messageId is a Hex-encoded string. | +| `withTopicPartitionNumber` | Boolean | False | true | When it is set to `true`, include the topic partition number to the object path. | +| `bytesFormatTypeSeparator` | String | False | "0x10" | It is inserted between records for the `formatType` of bytes. By default, it is set to '0x10'. An input record that contains the line separator looks like multiple records in the output object. | +| `pendingQueueSize` | int | False | 10 | The number of records buffered in queue. By default, it is equal to `batchSize`. You can set it manually. | +| `useHumanReadableSchemaVersion` | Boolean | False | false | Use a human-readable format string for the schema version in the message metadata. If it is set to `true`, the schema version is in plain string format. Otherwise, the schema version is in hex-encoded string format. | +| `skipFailedMessages` | Boolean | False | false | Configure whether to skip a message which it fails to be processed. If it is set to `true`, the connector will skip the failed messages by `ack` it. Otherwise, the connector will `fail` the message. | +| `pathPrefix` | String | False | false | If it is set, the output files are stored in a folder under the given bucket path. The `pathPrefix` must be in the format of `xx/xxx/`. | +| `avroCodec` | String | False | snappy | Compression codec used when formatType=`avro`. Available compression types are: null (no compression), deflate, bzip2, xz, zstandard, snappy. | +| `parquetCodec` | String | False | gzip | Compression codec used when formatType=`parquet`. Available compression types are: null (no compression), snappy, gzip, lzo, brotli, lz4, zstd. | +| `jsonAllowNaN` | Boolean | False | false | Recognize 'NaN', 'INF', '-INF' as legal floating number values when formatType=`json`. Since JSON specification does not allow such values this is a non-standard feature and disabled by default. | + +There are three methods to authenticate with Azure Blob Storage: +1. `azureStorageAccountConnectionString`: This method involves using the Azure Blob Storage connection string for authentication. It's the simplest method as it only requires the connection string. +2. `azureStorageAccountSASToken`: This method uses a Shared Access Signature (SAS) token for the Azure Blob Storage account. When using this method, you must also set the `endpoint`. +3. `azureStorageAccountName` and `azureStorageAccountKey`: This method uses the Azure Blob Storage account name and account key for authentication. Similar to the SAS token method, you must also set the `endpoint` when using this method. + + +## Advanced features + +### Data format types + +Azure Blob Storage Sink Connector provides multiple output format options, including JSON, Avro, Bytes, or Parquet. The default format is JSON. +With current implementation, there are some limitations for different formats: + +This table lists the Pulsar Schema types supported by the writers. + +| Pulsar Schema | Writer: Avro | Writer: JSON | Writer: Parquet | Writer: Bytes | +|----------------|--------------|--------------|-----------------|---------------| +| Primitive | ✗ | ✔ * | ✗ | ✔ | +| Avro | ✔ | ✔ | ✔ | ✔ | +| Json | ✔ | ✔ | ✔ | ✔ | +| Protobuf ** | ✔ | ✔ | ✔ | ✔ | +| ProtobufNative | ✔ *** | ✗ | ✔ | ✔ | + +> *: The JSON writer will try to convert the data with a `String` or `Bytes` schema to JSON-format data if convertable. +> +> **: The Protobuf schema is based on the Avro schema. It uses Avro as an intermediate format, so it may not provide the best effort conversion. +> +> ***: The ProtobufNative record holds the Protobuf descriptor and the message. When writing to Avro format, the connector uses [avro-protobuf](https://github.com/apache/avro/tree/master/lang/java/protobuf) to do the conversion. + +This table lists the support of `withMetadata` configurations for different writer formats: + +| Writer Format | `withMetadata` | +|---------------|----------------| +| Avro | ✔ | +| JSON | ✔ | +| Parquet | ✔ * | +| Bytes | ✗ | + +> *: When using `Parquet` with `PROTOBUF_NATIVE` format, the connector will write the messages with `DynamicMessage` format. When `withMetadata` is set to `true`, the connector will add `__message_metadata__` to the messages with `PulsarIOCSCProtobufMessageMetadata` format. +> +> For example, if a message `User` has the following schema: +> ```protobuf +> syntax = "proto3"; +> message User { +> string name = 1; +> int32 age = 2; +> } +> ``` +> +> When `withMetadata` is set to `true`, the connector will write the message `DynamicMessage` with the following schema: +> ```protobuf +> syntax = "proto3"; +> message PulsarIOCSCProtobufMessageMetadata { +> map properties = 1; +> string schema_version = 2; +> string message_id = 3; +> } +> message User { +> string name = 1; +> int32 age = 2; +> PulsarIOCSCProtobufMessageMetadata __message_metadata__ = 3; +> } +> ``` +> + + +### Dead-letter topics + +To use a dead-letter topic, you need to set `skipFailedMessages` to `false`, and set `--max-redeliver-count` and `--dead-letter-topic` when submit the connector with the `pulsar-admin` CLI tool. For more info about dead-letter topics, see the [Pulsar documentation](https://pulsar.apache.org/docs/en/concepts-messaging/#dead-letter-topic). +If a message fails to be sent to the Azure Blob Storage and there is a dead-letter topic, the connector will send the message to the dead-letter topic. + +### Sink flushing only after batchTimeMs elapses + +There is a scenario where the sink is only flushing whenever the `batchTimeMs` has elapsed, even though there are many messages waiting to be processed. +The reason for this is that the sink will only acknowledge messages after they are flushed to the Azure Blob Storage but the broker stops sending messages when it reaches a certain limit of unacknowledged messages. +If this limit is lower or close to `batchSize`, the sink never receives enough messages to trigger a flush based on the amount of messages. +In this case please ensure the `maxUnackedMessagesPerConsumer` set in the broker configuration is sufficiently larger than the `batchSize` setting of the sink. + diff --git a/connectors/cloud-storage-sink/v2.10.5.8/cloud-storage-sink.md b/connectors/cloud-storage-sink/v2.10.5.9/cloud-storage-sink.md similarity index 98% rename from connectors/cloud-storage-sink/v2.10.5.8/cloud-storage-sink.md rename to connectors/cloud-storage-sink/v2.10.5.9/cloud-storage-sink.md index 7b5825fc5..be6b59d11 100644 --- a/connectors/cloud-storage-sink/v2.10.5.8/cloud-storage-sink.md +++ b/connectors/cloud-storage-sink/v2.10.5.9/cloud-storage-sink.md @@ -4,20 +4,20 @@ author: freeznet,jianyun8023,nicoloboschi,Huanli-Meng contributors: freeznet,jianyun8023,nicoloboschi,Huanli-Meng language: Java,Shell,Dockerfile document: -source: "https://github.com/streamnative/pulsar-io-cloud-storage" +source: https://github.com/streamnative/pulsar-io-cloud-storage license: Apache License 2.0 license_link: "https://github.com/streamnative/pulsar-io-cloud-storage/blob/master/LICENSE" tags: alias: Cloud Storage Sink Connector features: ["Cloud Storage Connector integrates Apache Pulsar with cloud storage."] icon: "/images/pulsar-hub.svg" -download: "https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v2.10.5.8" +download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v2.10.5.9 support: streamnative support_link: https://github.com/streamnative/pulsar-io-cloud-storage support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "https://hub.docker.com/r/streamnative/pulsar-io-cloud-storage" +dockerfile: https://hub.docker.com/r/streamnative/pulsar-io-cloud-storage sn_available: "true" id: "cloud-storage-sink" --- @@ -25,7 +25,7 @@ id: "cloud-storage-sink" The Cloud Storage sink connector supports exporting data from Pulsar topics to cloud storage (such as AWS S3 and Google GCS) either in Avro, JSON, Parquet or other formats. According to your environment, the Cloud Storage sink connector can guarantee exactly-once support for exporting data to cloud storage. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v2.10.5.8/docs/cloud-storage-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v2.10.5.9/docs/cloud-storage-sink.png) # How to get @@ -33,7 +33,7 @@ You can get the Cloud Storage sink connector using one of the following methods: ## Use it with Function Worker -- Download the NAR package from [here](https://github.com/streamnative/pulsar-io-cloud-storage/releases/download/v2.10.5.8/pulsar-io-cloud-storage-2.10.5.8.nar). +- Download the NAR package from [here](https://github.com/streamnative/pulsar-io-cloud-storage/releases/download/v2.10.5.9/pulsar-io-cloud-storage-2.10.5.9.nar). - Build it from the source code. @@ -53,7 +53,7 @@ You can get the Cloud Storage sink connector using one of the following methods: ```bash ls target - pulsar-io-cloud-storage-2.10.5.8.nar + pulsar-io-cloud-storage-2.10.5.9.nar ``` ## Use it with Function Mesh @@ -301,7 +301,7 @@ kind: Sink metadata: name: cloud-storage-sink-sample spec: - image: streamnative/pulsar-io-cloud-storage:2.10.5.8 + image: streamnative/pulsar-io-cloud-storage:2.10.5.9 className: org.apache.pulsar.io.jcloud.sink.CloudStorageGenericRecordSink replicas: 1 maxReplicas: 1 @@ -333,7 +333,7 @@ spec: cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-cloud-storage-2.10.5.8.nar + jar: connectors/pulsar-io-cloud-storage-2.10.5.9.nar clusterName: test-pulsar ``` @@ -353,7 +353,7 @@ This example shows how to create an Cloud Storage sink connector on a Pulsar clu ``` PULSAR_HOME/bin/pulsar-admin sinks create \ ---archive pulsar-io-cloud-storage-2.10.5.8.nar \ +--archive pulsar-io-cloud-storage-2.10.5.9.nar \ --sink-config-file cloud-storage-sink-config.yaml \ --classname org.apache.pulsar.io.jcloud.sink.CloudStorageGenericRecordSink \ --name cloud-storage-sink @@ -452,7 +452,7 @@ This example explains how to create a Cloud Storage sink connector in an on-prem 1. Copy the NAR package of the Cloud Storage connector to the Pulsar connectors directory. ``` - cp pulsar-io-cloud-storage-2.10.5.8.nar $PULSAR_HOME/connectors/pulsar-io-cloud-storage-2.10.5.8.nar + cp pulsar-io-cloud-storage-2.10.5.9.nar $PULSAR_HOME/connectors/pulsar-io-cloud-storage-2.10.5.9.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). @@ -500,7 +500,7 @@ This example demonstrates how to create Cloud Storage sink connector through Fun metadata: name: cloud-storage-sink-sample spec: - image: streamnative/pulsar-io-cloud-storage:2.10.5.8 + image: streamnative/pulsar-io-cloud-storage:2.10.5.9 className: org.apache.pulsar.io.jcloud.sink.CloudStorageGenericRecordSink replicas: 1 maxReplicas: 1 @@ -532,7 +532,7 @@ This example demonstrates how to create Cloud Storage sink connector through Fun cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-cloud-storage-2.10.5.8.nar + jar: connectors/pulsar-io-cloud-storage-2.10.5.9.nar clusterName: test-pulsar ``` diff --git a/connectors/cloud-storage-sink/v3.1.0.5/cloud-storage-sink.md b/connectors/cloud-storage-sink/v3.1.0.6/cloud-storage-sink.md similarity index 98% rename from connectors/cloud-storage-sink/v3.1.0.5/cloud-storage-sink.md rename to connectors/cloud-storage-sink/v3.1.0.6/cloud-storage-sink.md index 0506177e9..a3d8a3e44 100644 --- a/connectors/cloud-storage-sink/v3.1.0.5/cloud-storage-sink.md +++ b/connectors/cloud-storage-sink/v3.1.0.6/cloud-storage-sink.md @@ -4,20 +4,20 @@ author: freeznet,jianyun8023,nicoloboschi,Huanli-Meng contributors: freeznet,jianyun8023,nicoloboschi,Huanli-Meng language: Java,Shell,Dockerfile document: -source: "https://github.com/streamnative/pulsar-io-cloud-storage" +source: https://github.com/streamnative/pulsar-io-cloud-storage license: Apache License 2.0 license_link: "https://github.com/streamnative/pulsar-io-cloud-storage/blob/master/LICENSE" tags: alias: Cloud Storage Sink Connector features: ["Cloud Storage Connector integrates Apache Pulsar with cloud storage."] icon: "/images/pulsar-hub.svg" -download: "https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.1.0.5" +download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.1.0.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-cloud-storage support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "https://hub.docker.com/r/streamnative/pulsar-io-cloud-storage" +dockerfile: https://hub.docker.com/r/streamnative/pulsar-io-cloud-storage sn_available: "true" id: "cloud-storage-sink" --- @@ -25,7 +25,7 @@ id: "cloud-storage-sink" The Cloud Storage sink connector supports exporting data from Pulsar topics to cloud storage (such as AWS S3 and Google GCS) either in Avro, JSON, Parquet or other formats. According to your environment, the Cloud Storage sink connector can guarantee exactly-once support for exporting data to cloud storage. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.1.0.5/docs/cloud-storage-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.1.0.6/docs/cloud-storage-sink.png) # How to get @@ -33,7 +33,7 @@ You can get the Cloud Storage sink connector using one of the following methods: ## Use it with Function Worker -- Download the NAR package from [here](https://github.com/streamnative/pulsar-io-cloud-storage/releases/download/v3.1.0.5/pulsar-io-cloud-storage-3.1.0.5.nar). +- Download the NAR package from [here](https://github.com/streamnative/pulsar-io-cloud-storage/releases/download/v3.1.0.6/pulsar-io-cloud-storage-3.1.0.6.nar). - Build it from the source code. @@ -53,7 +53,7 @@ You can get the Cloud Storage sink connector using one of the following methods: ```bash ls target - pulsar-io-cloud-storage-3.1.0.5.nar + pulsar-io-cloud-storage-3.1.0.6.nar ``` ## Use it with Function Mesh @@ -301,7 +301,7 @@ kind: Sink metadata: name: cloud-storage-sink-sample spec: - image: streamnative/pulsar-io-cloud-storage:3.1.0.5 + image: streamnative/pulsar-io-cloud-storage:3.1.0.6 className: org.apache.pulsar.io.jcloud.sink.CloudStorageGenericRecordSink replicas: 1 maxReplicas: 1 @@ -333,7 +333,7 @@ spec: cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-cloud-storage-3.1.0.5.nar + jar: connectors/pulsar-io-cloud-storage-3.1.0.6.nar clusterName: test-pulsar ``` @@ -353,7 +353,7 @@ This example shows how to create an Cloud Storage sink connector on a Pulsar clu ``` PULSAR_HOME/bin/pulsar-admin sinks create \ ---archive pulsar-io-cloud-storage-3.1.0.5.nar \ +--archive pulsar-io-cloud-storage-3.1.0.6.nar \ --sink-config-file cloud-storage-sink-config.yaml \ --classname org.apache.pulsar.io.jcloud.sink.CloudStorageGenericRecordSink \ --name cloud-storage-sink @@ -452,7 +452,7 @@ This example explains how to create a Cloud Storage sink connector in an on-prem 1. Copy the NAR package of the Cloud Storage connector to the Pulsar connectors directory. ``` - cp pulsar-io-cloud-storage-3.1.0.5.nar $PULSAR_HOME/connectors/pulsar-io-cloud-storage-3.1.0.5.nar + cp pulsar-io-cloud-storage-3.1.0.6.nar $PULSAR_HOME/connectors/pulsar-io-cloud-storage-3.1.0.6.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). @@ -500,7 +500,7 @@ This example demonstrates how to create Cloud Storage sink connector through Fun metadata: name: cloud-storage-sink-sample spec: - image: streamnative/pulsar-io-cloud-storage:3.1.0.5 + image: streamnative/pulsar-io-cloud-storage:3.1.0.6 className: org.apache.pulsar.io.jcloud.sink.CloudStorageGenericRecordSink replicas: 1 maxReplicas: 1 @@ -532,7 +532,7 @@ This example demonstrates how to create Cloud Storage sink connector through Fun cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-cloud-storage-3.1.0.5.nar + jar: connectors/pulsar-io-cloud-storage-3.1.0.6.nar clusterName: test-pulsar ``` diff --git a/connectors/google-bigquery-sink/v2.10.5.8/google-bigquery-sink.md b/connectors/google-bigquery-sink/v2.10.5.9/google-bigquery-sink.md similarity index 99% rename from connectors/google-bigquery-sink/v2.10.5.8/google-bigquery-sink.md rename to connectors/google-bigquery-sink/v2.10.5.9/google-bigquery-sink.md index d7ebdbd68..6cf8c8644 100644 --- a/connectors/google-bigquery-sink/v2.10.5.8/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v2.10.5.9/google-bigquery-sink.md @@ -6,18 +6,18 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-bigquery/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] icon: "/images/connectors/google-bigquery-logo.png" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "true" id: "google-bigquery-sink" --- @@ -25,7 +25,7 @@ id: "google-bigquery-sink" The [Google Cloud BigQuery](https://cloud.google.com/bigquery) sink connector pulls data from Pulsar topics and persists data to Google Cloud BigQuery tables. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v2.10.5.8/docs/google-bigquery-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v2.10.5.9/docs/google-bigquery-sink.png) ## Quick start diff --git a/connectors/google-bigquery-sink/v3.1.0.5/google-bigquery-sink.md b/connectors/google-bigquery-sink/v3.1.0.6/google-bigquery-sink.md similarity index 99% rename from connectors/google-bigquery-sink/v3.1.0.5/google-bigquery-sink.md rename to connectors/google-bigquery-sink/v3.1.0.6/google-bigquery-sink.md index 11842ae2e..99a31ee15 100644 --- a/connectors/google-bigquery-sink/v3.1.0.5/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v3.1.0.6/google-bigquery-sink.md @@ -6,18 +6,18 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-bigquery/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] icon: "/images/connectors/google-bigquery-logo.png" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "true" id: "google-bigquery-sink" --- @@ -25,7 +25,7 @@ id: "google-bigquery-sink" The [Google Cloud BigQuery](https://cloud.google.com/bigquery) sink connector pulls data from Pulsar topics and persists data to Google Cloud BigQuery tables. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.1.0.5/docs/google-bigquery-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.1.0.6/docs/google-bigquery-sink.png) ## Quick start diff --git a/connectors/google-bigquery-source/v3.1.0.5/google-bigquery-source.md b/connectors/google-bigquery-source/v2.10.5.9/google-bigquery-source.md similarity index 98% rename from connectors/google-bigquery-source/v3.1.0.5/google-bigquery-source.md rename to connectors/google-bigquery-source/v2.10.5.9/google-bigquery-source.md index 4358697ef..8b038f00c 100644 --- a/connectors/google-bigquery-source/v3.1.0.5/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v2.10.5.9/google-bigquery-source.md @@ -6,18 +6,18 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-bigquery/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud BigQuery Source Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] icon: "/images/connectors/google-bigquery-logo.png" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "true" id: "google-bigquery-source" --- @@ -25,7 +25,7 @@ id: "google-bigquery-source" The [Google Cloud BigQuery](https://cloud.google.com/bigquery) Source Connector feeds data from Google Cloud BigQuery tables and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.1.0.5/docs/google-bigquery-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v2.10.5.9/docs/google-bigquery-source.png) ## Quick start diff --git a/connectors/google-bigquery-source/v2.10.5.8/google-bigquery-source.md b/connectors/google-bigquery-source/v3.1.0.6/google-bigquery-source.md similarity index 98% rename from connectors/google-bigquery-source/v2.10.5.8/google-bigquery-source.md rename to connectors/google-bigquery-source/v3.1.0.6/google-bigquery-source.md index d128fe293..3fdd50493 100644 --- a/connectors/google-bigquery-source/v2.10.5.8/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v3.1.0.6/google-bigquery-source.md @@ -6,18 +6,18 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-bigquery/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud BigQuery Source Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] icon: "/images/connectors/google-bigquery-logo.png" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "true" id: "google-bigquery-source" --- @@ -25,7 +25,7 @@ id: "google-bigquery-source" The [Google Cloud BigQuery](https://cloud.google.com/bigquery) Source Connector feeds data from Google Cloud BigQuery tables and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v2.10.5.8/docs/google-bigquery-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-bigquery/v3.1.0.6/docs/google-bigquery-source.png) ## Quick start diff --git a/connectors/google-cloud-storage-sink/v3.1.0.6/google-cloud-storage-sink.md b/connectors/google-cloud-storage-sink/v3.1.0.6/google-cloud-storage-sink.md new file mode 100644 index 000000000..03e2f0a67 --- /dev/null +++ b/connectors/google-cloud-storage-sink/v3.1.0.6/google-cloud-storage-sink.md @@ -0,0 +1,220 @@ +--- +description: Cloud Storage Connector integrates Apache Pulsar with cloud storage. +author: freeznet,jianyun8023,nicoloboschi,Huanli-Meng +contributors: freeznet,jianyun8023,nicoloboschi,Huanli-Meng +language: Java,Shell,Dockerfile +document: +source: https://github.com/streamnative/pulsar-io-cloud-storage +license: Apache License 2.0 +license_link: "https://github.com/streamnative/pulsar-io-cloud-storage/blob/master/LICENSE" +tags: +alias: Google Cloud Storage Sink Connector +features: ["Cloud Storage Connector integrates Apache Pulsar with cloud storage."] +icon: "/images/pulsar-hub.svg" +download: https://api.github.com/repos/streamnative/pulsar-io-cloud-storage/tarball/refs/tags/v3.1.0.6 +support: streamnative +support_link: https://github.com/streamnative/pulsar-io-cloud-storage +support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" +owner_name: "streamnative" +owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" +dockerfile: https://hub.docker.com/r/streamnative/pulsar-io-cloud-storage +sn_available: "true" +id: "google-cloud-storage-sink" +--- + + +The [Google Cloud Storage](https://cloud.google.com/storage/docs) sink connector pulls data from Pulsar topics and persists data to Google Cloud Storage buckets. + +![](https://raw.githubusercontent.com/streamnative/pulsar-io-cloud-storage/v3.1.0.6/docs/google-cloud-storage-sink.png) + +## Quick start + +### Prerequisites + +The prerequisites for connecting an Google Cloud Storage sink connector to external systems include: + +1. Create Cloud Storage buckets in Google Cloud. +2. Create the [Google cloud ServiceAccount](https://cloud.google.com/iam/docs/service-accounts-create) and create a public key certificate. +3. Create the [Google cloud Role](https://cloud.google.com/iam/docs/creating-custom-roles), ensure the Google Cloud role have the following permissions: +```text +- storage.buckets.get +- storage.buckets.list +- storage.objects.create +``` +4. Grant the `ServiceAccount` the above `Role`. + + +### 1. Create a connector + +Depending on the environment, there are several ways to create an Google Cloud Storage sink connector: + +- [Create Connector on StreamNative Cloud](https://docs.streamnative.io/docs/connector-create). +- [Create Connector with Function worker](https://pulsar.apache.org/docs/3.0.x/io-quickstart/). + Using this way requires you to download a **NAR** package to create a built-in or non-built-in connector. You can download the version you need from [here](https://github.com/streamnative/pulsar-io-cloud-storage/releases). +- [Create Connector with Function mesh](https://functionmesh.io/docs/connectors/run-connector). + Using this way requires you to set the docker image. You can choose the version you want to launch from [here](https://hub.docker.com/r/streamnative/pulsar-io-cloud-storage/tags) + +No matter how you create an Google Cloud Storage sink connector, the minimum configuration contains the following parameters. + +```yaml + configs: + provider: "google-cloud-storage" + gcsServiceAccountKeyFileContent: {{Public key certificate you created above}} + bucket: {{Your bucket name}} + formatType: "json" + partitionerType: "PARTITION" +``` + +> * The configuration structure varies depending on how you create the Google Cloud Storage sink connector. +> For example, some are **JSON**, some are **YAML**, and some are **Kubernetes YAML**. You need to adapt the configs to the corresponding format. +> +> * If you want to configure more parameters, see [Configuration Properties](#configuration-properties) for reference. + +### 2. Send messages to the topic + +{% callout title="Note" type="note" %} +If your connector is created on StreamNative Cloud, you need to authenticate your clients. See [Build applications using Pulsar clients](https://docs.streamnative.io/docs/qs-connect#jumpstart-for-beginners) for more information. +{% /callout %} + +``` java + public static void main(String[] args) throws Exception { + PulsarClient client = PulsarClient.builder() + .serviceUrl("{{Your Pulsar URL}}") + .build(); + + + Producer producer = client.newProducer(Schema.STRING) + .topic("{{Your topic name}}") + .create(); + + for (int i = 0; i < 10; i++) { + // JSON string containing a single character + String message = "{\"test-message\": \"test-value\"}"; + producer.send(message); + } + + producer.close(); + client.close(); + } +``` + +### 3. Display data on Google Cloud Storage console + +You can see the object at public/default/{{Your topic name}}-partition-0/xxxx.json on the Google Cloud Storage console. Download and open it, the content is: + +```text +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +{"test-message":"test-value"} +``` + +## Configuration Properties + +Before using the Google Cloud Storage sink connector, you need to configure it. This table outlines the properties and the descriptions. + +| Name | Type | Required | Default | Description | +|-----------------------------------|---------|----------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `provider` | String | True | null | The Cloud Storage type, google cloud storage only supports the `google-cloud-storage` provider. | +| `bucket` | String | True | null | The Cloud Storage bucket. | +| `formatType` | String | True | null | The data format type. Available options are JSON, Avro, Bytes, or Parquet. By default, it is set to JSON. | +| `partitionerType` | String | True | null | The partitioning type. It can be configured by topic partitions or by time. By default, the partition type is configured by topic partitions. | +| `gcsServiceAccountKeyFileContent` | String | False | "" | The contents of the JSON service key file. If empty, credentials are read from `gcsServiceAccountKeyFilePath` file. | +| `gcsServiceAccountKeyFilePath` | String | False | "" | Path to the GCS credentials file. If empty, the credentials file will be read from the `GOOGLE_APPLICATION_CREDENTIALS` environment variable. | +| `timePartitionPattern` | String | False | "yyyy-MM-dd" | The format pattern of the time-based partitioning. For details, refer to the Java date and time format. | +| `timePartitionDuration` | String | False | "86400000" | The time interval for time-based partitioning. Support formatted interval string, such as `30d`, `24h`, `30m`, `10s`, and also support number in milliseconds precision, such as `86400000` refers to `24h` or `1d`. | +| `partitionerUseIndexAsOffset` | Boolean | False | false | Whether to use the Pulsar's message index as offset or the record sequence. It's recommended if the incoming messages may be batched. The brokers may or not expose the index metadata and, if it's not present on the record, the sequence will be used. See [PIP-70](https://github.com/apache/pulsar/wiki/PIP-70%3A-Introduce-lightweight-broker-entry-metadata) for more details. | +| `batchSize` | int | False | 10 | The number of records submitted in batch. | +| `batchTimeMs` | long | False | 1000 | The interval for batch submission. | +| `maxBatchBytes` | long | False | 10000000 | The maximum number of bytes in a batch. | +| `sliceTopicPartitionPath` | Boolean | False | false | When it is set to `true`, split the partitioned topic name into separate folders in the bucket path. | +| `withMetadata` | Boolean | False | false | Save message attributes to metadata. | +| `useHumanReadableMessageId` | Boolean | False | false | Use a human-readable format string for messageId in message metadata. The messageId is in a format like `ledgerId:entryId:partitionIndex:batchIndex`. Otherwise, the messageId is a Hex-encoded string. | +| `withTopicPartitionNumber` | Boolean | False | true | When it is set to `true`, include the topic partition number to the object path. | +| `bytesFormatTypeSeparator` | String | False | "0x10" | It is inserted between records for the `formatType` of bytes. By default, it is set to '0x10'. An input record that contains the line separator looks like multiple records in the output object. | +| `pendingQueueSize` | int | False | 10 | The number of records buffered in queue. By default, it is equal to `batchSize`. You can set it manually. | +| `useHumanReadableSchemaVersion` | Boolean | False | false | Use a human-readable format string for the schema version in the message metadata. If it is set to `true`, the schema version is in plain string format. Otherwise, the schema version is in hex-encoded string format. | +| `skipFailedMessages` | Boolean | False | false | Configure whether to skip a message which it fails to be processed. If it is set to `true`, the connector will skip the failed messages by `ack` it. Otherwise, the connector will `fail` the message. | +| `pathPrefix` | String | False | false | If it is set, the output files are stored in a folder under the given bucket path. The `pathPrefix` must be in the format of `xx/xxx/`. | +| `avroCodec` | String | False | snappy | Compression codec used when formatType=`avro`. Available compression types are: null (no compression), deflate, bzip2, xz, zstandard, snappy. | +| `parquetCodec` | String | False | gzip | Compression codec used when formatType=`parquet`. Available compression types are: null (no compression), snappy, gzip, lzo, brotli, lz4, zstd. | +| `jsonAllowNaN` | Boolean | False | false | Recognize 'NaN', 'INF', '-INF' as legal floating number values when formatType=`json`. Since JSON specification does not allow such values this is a non-standard feature and disabled by default. | + +## Advanced features + +### Data format types + +Cloud Storage Sink Connector provides multiple output format options, including JSON, Avro, Bytes, or Parquet. The default format is JSON. +With current implementation, there are some limitations for different formats: + +This table lists the Pulsar Schema types supported by the writers. + +| Pulsar Schema | Writer: Avro | Writer: JSON | Writer: Parquet | Writer: Bytes | +|----------------|--------------|--------------|-----------------|---------------| +| Primitive | ✗ | ✔ * | ✗ | ✔ | +| Avro | ✔ | ✔ | ✔ | ✔ | +| Json | ✔ | ✔ | ✔ | ✔ | +| Protobuf ** | ✔ | ✔ | ✔ | ✔ | +| ProtobufNative | ✔ *** | ✗ | ✔ | ✔ | + +> *: The JSON writer will try to convert the data with a `String` or `Bytes` schema to JSON-format data if convertable. +> +> **: The Protobuf schema is based on the Avro schema. It uses Avro as an intermediate format, so it may not provide the best effort conversion. +> +> ***: The ProtobufNative record holds the Protobuf descriptor and the message. When writing to Avro format, the connector uses [avro-protobuf](https://github.com/apache/avro/tree/master/lang/java/protobuf) to do the conversion. + +This table lists the support of `withMetadata` configurations for different writer formats: + +| Writer Format | `withMetadata` | +|---------------|----------------| +| Avro | ✔ | +| JSON | ✔ | +| Parquet | ✔ * | +| Bytes | ✗ | + +> *: When using `Parquet` with `PROTOBUF_NATIVE` format, the connector will write the messages with `DynamicMessage` format. When `withMetadata` is set to `true`, the connector will add `__message_metadata__` to the messages with `PulsarIOCSCProtobufMessageMetadata` format. +> +> For example, if a message `User` has the following schema: +> ```protobuf +> syntax = "proto3"; +> message User { +> string name = 1; +> int32 age = 2; +> } +> ``` +> +> When `withMetadata` is set to `true`, the connector will write the message `DynamicMessage` with the following schema: +> ```protobuf +> syntax = "proto3"; +> message PulsarIOCSCProtobufMessageMetadata { +> map properties = 1; +> string schema_version = 2; +> string message_id = 3; +> } +> message User { +> string name = 1; +> int32 age = 2; +> PulsarIOCSCProtobufMessageMetadata __message_metadata__ = 3; +> } +> ``` +> + + +### Dead-letter topics + +To use a dead-letter topic, you need to set `skipFailedMessages` to `false`, and set `--max-redeliver-count` and `--dead-letter-topic` when submit the connector with the `pulsar-admin` CLI tool. For more info about dead-letter topics, see the [Pulsar documentation](https://pulsar.apache.org/docs/en/concepts-messaging/#dead-letter-topic). +If a message fails to be sent to the Cloud Storage and there is a dead-letter topic, the connector will send the message to the dead-letter topic. + +### Sink flushing only after batchTimeMs elapses + +There is a scenario where the sink is only flushing whenever the `batchTimeMs` has elapsed, even though there are many messages waiting to be processed. +The reason for this is that the sink will only acknowledge messages after they are flushed to cloud storage but the broker stops sending messages when it reaches a certain limit of unacknowledged messages. +If this limit is lower or close to `batchSize`, the sink never receives enough messages to trigger a flush based on the amount of messages. +In this case please ensure the `maxUnackedMessagesPerConsumer` set in the broker configuration is sufficiently larger than the `batchSize` setting of the sink. + diff --git a/connectors/google-pubsub-sink/v2.10.5.8/google-pubsub-sink.md b/connectors/google-pubsub-sink/v2.10.5.9/google-pubsub-sink.md similarity index 93% rename from connectors/google-pubsub-sink/v2.10.5.8/google-pubsub-sink.md rename to connectors/google-pubsub-sink/v2.10.5.9/google-pubsub-sink.md index d77e04a04..2811b5ee5 100644 --- a/connectors/google-pubsub-sink/v2.10.5.8/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v2.10.5.9/google-pubsub-sink.md @@ -6,18 +6,18 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-google-pubsub/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud Pub/Sub Sink Connector features: [""] icon: "/images/connectors/google-pubsub.svg" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "/images/connectors/streamnative.png" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "" id: "google-pubsub-sink" --- @@ -25,7 +25,7 @@ id: "google-pubsub-sink" The [Google Cloud Pub/Sub](https://cloud.google.com/pubsub) sink connector pulls data from Pulsar topics and persists data to Google Cloud Pub/Sub topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v2.10.5.8/docs/google-pubsub-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v2.10.5.9/docs/google-pubsub-sink.png) # How to get @@ -35,7 +35,7 @@ This section describes how to build the Google Cloud Pub/Sub sink connector. You can get the Google Cloud Pub/Sub sink connector using one of the following methods if you use [Pulsar Function Worker](https://pulsar.apache.org/docs/en/functions-worker/) to run connectors in a cluster. -- Download the NAR package from [the download page](https://github.com/streamnative/pulsar-io-google-pubsub/releases/download/v2.10.5.8/pulsar-io-google-pubsub-2.10.5.8.nar). +- Download the NAR package from [the download page](https://github.com/streamnative/pulsar-io-google-pubsub/releases/download/v2.10.5.9/pulsar-io-google-pubsub-2.10.5.9.nar). - Build it from the source code. @@ -57,7 +57,7 @@ To build the Google Cloud Pub/Sub sink connector from the source code, follow th ```bash ls target - pulsar-io-google-pubsub-2.10.5.8.nar + pulsar-io-google-pubsub-2.10.5.9.nar ``` ## Work with Function Mesh @@ -105,7 +105,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-google-pubsub-pulsar" ], - "archive": "connectors/pulsar-io-google-pubsub-2.10.5.8.nar", + "archive": "connectors/pulsar-io-google-pubsub-2.10.5.9.nar", "parallelism": 1, "configs": { "pubsubCredential": "SECRETS", @@ -123,7 +123,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you name: google-pubsub-sink inputs: - test-google-pubsub-pulsar - archive: connectors/pulsar-io-google-pubsub-2.10.5.8.nar + archive: connectors/pulsar-io-google-pubsub-2.10.5.9.nar parallelism: 1 configs: pubsubCredential: 'SECRETS' @@ -143,7 +143,7 @@ kind: Sink metadata: name: google-pubsub-sink-sample spec: - image: streamnative/pulsar-io-google-pubsub:2.10.5.8 + image: streamnative/pulsar-io-google-pubsub:2.10.5.9 className: org.apache.pulsar.ecosystem.io.pubsub.PubsubSink replicas: 1 maxReplicas: 1 @@ -164,7 +164,7 @@ spec: cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-google-pubsub-2.10.5.8.nar + jar: connectors/pulsar-io-google-pubsub-2.10.5.9.nar clusterName: test-pulsar autoAck: true ``` @@ -214,7 +214,7 @@ This example describes how to use the Google Cloud Pub/Sub sink connector to pul 2. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-google-pubsub-2.10.5.8.nar PULSAR_HOME/connectors/pulsar-io-google-pubsub-2.10.5.8.nar + cp pulsar-io-google-pubsub-2.10.5.9.nar PULSAR_HOME/connectors/pulsar-io-google-pubsub-2.10.5.9.nar ``` 3. Start Pulsar in standalone mode. @@ -253,7 +253,7 @@ This example explains how to create a Google Cloud Pub/Sub sink connector in an 1. Copy the NAR package of the Google Cloud Pub/Sub sink connector to the Pulsar connectors directory. ``` - cp pulsar-io-google-pubsub-2.10.5.8.nar $PULSAR_HOME/connectors/pulsar-io-google-pubsub-2.10.5.8.nar + cp pulsar-io-google-pubsub-2.10.5.9.nar $PULSAR_HOME/connectors/pulsar-io-google-pubsub-2.10.5.9.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). @@ -305,7 +305,7 @@ This example describes how to create a Google Cloud Pub/Sub sink connector for a metadata: name: google-pubsub-sink-sample spec: - image: streamnative/pulsar-io-google-pubsub:2.10.5.8 + image: streamnative/pulsar-io-google-pubsub:2.10.5.9 className: org.apache.pulsar.ecosystem.io.pubsub.PubsubSink replicas: 1 maxReplicas: 1 @@ -327,7 +327,7 @@ This example describes how to create a Google Cloud Pub/Sub sink connector for a cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-google-pubsub-2.10.5.8.nar + jar: connectors/pulsar-io-google-pubsub-2.10.5.9.nar clusterName: test-pulsar autoAck: true ``` diff --git a/connectors/google-pubsub-sink/v3.1.0.5/google-pubsub-sink.md b/connectors/google-pubsub-sink/v3.1.0.6/google-pubsub-sink.md similarity index 93% rename from connectors/google-pubsub-sink/v3.1.0.5/google-pubsub-sink.md rename to connectors/google-pubsub-sink/v3.1.0.6/google-pubsub-sink.md index 7a6d8aa7a..db9fc8580 100644 --- a/connectors/google-pubsub-sink/v3.1.0.5/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v3.1.0.6/google-pubsub-sink.md @@ -6,18 +6,18 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-google-pubsub/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud Pub/Sub Sink Connector features: [""] icon: "/images/connectors/google-pubsub.svg" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "/images/connectors/streamnative.png" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "" id: "google-pubsub-sink" --- @@ -25,7 +25,7 @@ id: "google-pubsub-sink" The [Google Cloud Pub/Sub](https://cloud.google.com/pubsub) sink connector pulls data from Pulsar topics and persists data to Google Cloud Pub/Sub topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.1.0.5/docs/google-pubsub-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.1.0.6/docs/google-pubsub-sink.png) # How to get @@ -35,7 +35,7 @@ This section describes how to build the Google Cloud Pub/Sub sink connector. You can get the Google Cloud Pub/Sub sink connector using one of the following methods if you use [Pulsar Function Worker](https://pulsar.apache.org/docs/en/functions-worker/) to run connectors in a cluster. -- Download the NAR package from [the download page](https://github.com/streamnative/pulsar-io-google-pubsub/releases/download/v3.1.0.5/pulsar-io-google-pubsub-3.1.0.5.nar). +- Download the NAR package from [the download page](https://github.com/streamnative/pulsar-io-google-pubsub/releases/download/v3.1.0.6/pulsar-io-google-pubsub-3.1.0.6.nar). - Build it from the source code. @@ -57,7 +57,7 @@ To build the Google Cloud Pub/Sub sink connector from the source code, follow th ```bash ls target - pulsar-io-google-pubsub-3.1.0.5.nar + pulsar-io-google-pubsub-3.1.0.6.nar ``` ## Work with Function Mesh @@ -105,7 +105,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-google-pubsub-pulsar" ], - "archive": "connectors/pulsar-io-google-pubsub-3.1.0.5.nar", + "archive": "connectors/pulsar-io-google-pubsub-3.1.0.6.nar", "parallelism": 1, "configs": { "pubsubCredential": "SECRETS", @@ -123,7 +123,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you name: google-pubsub-sink inputs: - test-google-pubsub-pulsar - archive: connectors/pulsar-io-google-pubsub-3.1.0.5.nar + archive: connectors/pulsar-io-google-pubsub-3.1.0.6.nar parallelism: 1 configs: pubsubCredential: 'SECRETS' @@ -143,7 +143,7 @@ kind: Sink metadata: name: google-pubsub-sink-sample spec: - image: streamnative/pulsar-io-google-pubsub:3.1.0.5 + image: streamnative/pulsar-io-google-pubsub:3.1.0.6 className: org.apache.pulsar.ecosystem.io.pubsub.PubsubSink replicas: 1 maxReplicas: 1 @@ -164,7 +164,7 @@ spec: cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-google-pubsub-3.1.0.5.nar + jar: connectors/pulsar-io-google-pubsub-3.1.0.6.nar clusterName: test-pulsar autoAck: true ``` @@ -214,7 +214,7 @@ This example describes how to use the Google Cloud Pub/Sub sink connector to pul 2. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-google-pubsub-3.1.0.5.nar PULSAR_HOME/connectors/pulsar-io-google-pubsub-3.1.0.5.nar + cp pulsar-io-google-pubsub-3.1.0.6.nar PULSAR_HOME/connectors/pulsar-io-google-pubsub-3.1.0.6.nar ``` 3. Start Pulsar in standalone mode. @@ -253,7 +253,7 @@ This example explains how to create a Google Cloud Pub/Sub sink connector in an 1. Copy the NAR package of the Google Cloud Pub/Sub sink connector to the Pulsar connectors directory. ``` - cp pulsar-io-google-pubsub-3.1.0.5.nar $PULSAR_HOME/connectors/pulsar-io-google-pubsub-3.1.0.5.nar + cp pulsar-io-google-pubsub-3.1.0.6.nar $PULSAR_HOME/connectors/pulsar-io-google-pubsub-3.1.0.6.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). @@ -305,7 +305,7 @@ This example describes how to create a Google Cloud Pub/Sub sink connector for a metadata: name: google-pubsub-sink-sample spec: - image: streamnative/pulsar-io-google-pubsub:3.1.0.5 + image: streamnative/pulsar-io-google-pubsub:3.1.0.6 className: org.apache.pulsar.ecosystem.io.pubsub.PubsubSink replicas: 1 maxReplicas: 1 @@ -327,7 +327,7 @@ This example describes how to create a Google Cloud Pub/Sub sink connector for a cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-google-pubsub-3.1.0.5.nar + jar: connectors/pulsar-io-google-pubsub-3.1.0.6.nar clusterName: test-pulsar autoAck: true ``` diff --git a/connectors/google-pubsub-source/v2.10.5.8/google-pubsub-source.md b/connectors/google-pubsub-source/v2.10.5.9/google-pubsub-source.md similarity index 93% rename from connectors/google-pubsub-source/v2.10.5.8/google-pubsub-source.md rename to connectors/google-pubsub-source/v2.10.5.9/google-pubsub-source.md index 495364e4e..06d098691 100644 --- a/connectors/google-pubsub-source/v2.10.5.8/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v2.10.5.9/google-pubsub-source.md @@ -6,18 +6,18 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-google-pubsub/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud Pub/Sub Source Connector features: [""] icon: "/images/connectors/google-pubsub.svg" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "" id: "google-pubsub-source" --- @@ -25,7 +25,7 @@ id: "google-pubsub-source" The [Google Cloud Pub/Sub](https://cloud.google.com/pubsub) source connector feeds data from Google Cloud Pub/Sub topics and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v2.10.5.8/docs/google-pubsub-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v2.10.5.9/docs/google-pubsub-source.png) # How to get @@ -35,7 +35,7 @@ This section describes how to build the Google Cloud Pub/Sub source connector. You can get the Google Cloud Pub/Sub source connector using one of the following methods if you use [Pulsar Function Worker](https://pulsar.apache.org/docs/en/functions-worker/) to run connectors in a cluster. -- Download the NAR package of the connector from the [download page](https://github.com/streamnative/pulsar-io-google-pubsub/releases/download/v2.10.5.8/pulsar-io-google-pubsub-2.10.5.8.nar). +- Download the NAR package of the connector from the [download page](https://github.com/streamnative/pulsar-io-google-pubsub/releases/download/v2.10.5.9/pulsar-io-google-pubsub-2.10.5.9.nar). - Build the connector from the source code. @@ -57,7 +57,7 @@ To build the Google Cloud Pub/Sub source connector from the source code, follow ```bash ls target - pulsar-io-google-pubsub-2.10.5.8.nar + pulsar-io-google-pubsub-2.10.5.9.nar ``` ## Work with Function Mesh @@ -104,7 +104,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "namespace": "default", "name": "google-pubsub-source", "topicName": "test-google-pubsub-pulsar", - "archive": "connectors/pulsar-io-google-pubsub-2.10.5.8.nar", + "archive": "connectors/pulsar-io-google-pubsub-2.10.5.9.nar", "parallelism": 1, "configs": { @@ -121,7 +121,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you namespace: default name: google-pubsub-source topicName: test-google-pubsub-pulsar - archive: connectors/pulsar-io-google-pubsub-2.10.5.8.nar + archive: connectors/pulsar-io-google-pubsub-2.10.5.9.nar parallelism: 1 configs: pubsubProjectId: pulsar-io-google-pubsub @@ -140,7 +140,7 @@ kind: Source metadata: name: google-pubsub-source-sample spec: - image: streamnative/pulsar-io-google-pubsub:2.10.5.8 + image: streamnative/pulsar-io-google-pubsub:2.10.5.9 className: org.apache.pulsar.ecosystem.io.pubsub.PubsubSource replicas: 1 maxReplicas: 1 @@ -161,7 +161,7 @@ spec: cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-google-pubsub-2.10.5.8.nar + jar: connectors/pulsar-io-google-pubsub-2.10.5.9.nar clusterName: test-pulsar ``` @@ -210,7 +210,7 @@ This example describes how to use the Google Cloud Pub/Sub source connector to f 2. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-google-pubsub-2.10.5.8.nar PULSAR_HOME/connectors/pulsar-io-google-pubsub-2.10.5.8.nar + cp pulsar-io-google-pubsub-2.10.5.9.nar PULSAR_HOME/connectors/pulsar-io-google-pubsub-2.10.5.9.nar ``` 3. Start Pulsar in standalone mode. @@ -249,7 +249,7 @@ This example explains how to create a Google Cloud Pub/Sub source connector in a 1. Copy the NAR package of the Google Cloud Pub/Sub connector to the Pulsar connectors directory. ``` - cp pulsar-io-google-pubsub-2.10.5.8.nar $PULSAR_HOME/connectors/pulsar-io-google-pubsub-2.10.5.8.nar + cp pulsar-io-google-pubsub-2.10.5.9.nar $PULSAR_HOME/connectors/pulsar-io-google-pubsub-2.10.5.9.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). @@ -297,7 +297,7 @@ This example describes how to create a Google Cloud Pub/Sub source connector for metadata: name: google-pubsub-source-sample spec: - image: streamnative/pulsar-io-google-pubsub:2.10.5.8 + image: streamnative/pulsar-io-google-pubsub:2.10.5.9 className: org.apache.pulsar.ecosystem.io.pubsub.PubsubSource replicas: 1 maxReplicas: 1 @@ -318,7 +318,7 @@ This example describes how to create a Google Cloud Pub/Sub source connector for cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-google-pubsub-2.10.5.8.nar + jar: connectors/pulsar-io-google-pubsub-2.10.5.9.nar clusterName: test-pulsar ``` diff --git a/connectors/google-pubsub-source/v3.1.0.5/google-pubsub-source.md b/connectors/google-pubsub-source/v3.1.0.6/google-pubsub-source.md similarity index 93% rename from connectors/google-pubsub-source/v3.1.0.5/google-pubsub-source.md rename to connectors/google-pubsub-source/v3.1.0.6/google-pubsub-source.md index f7c39a109..e35e5ca4f 100644 --- a/connectors/google-pubsub-source/v3.1.0.5/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v3.1.0.6/google-pubsub-source.md @@ -6,18 +6,18 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-google-pubsub/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud Pub/Sub Source Connector features: [""] icon: "/images/connectors/google-pubsub.svg" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "" id: "google-pubsub-source" --- @@ -25,7 +25,7 @@ id: "google-pubsub-source" The [Google Cloud Pub/Sub](https://cloud.google.com/pubsub) source connector feeds data from Google Cloud Pub/Sub topics and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.1.0.5/docs/google-pubsub-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-google-pubsub/v3.1.0.6/docs/google-pubsub-source.png) # How to get @@ -35,7 +35,7 @@ This section describes how to build the Google Cloud Pub/Sub source connector. You can get the Google Cloud Pub/Sub source connector using one of the following methods if you use [Pulsar Function Worker](https://pulsar.apache.org/docs/en/functions-worker/) to run connectors in a cluster. -- Download the NAR package of the connector from the [download page](https://github.com/streamnative/pulsar-io-google-pubsub/releases/download/v3.1.0.5/pulsar-io-google-pubsub-3.1.0.5.nar). +- Download the NAR package of the connector from the [download page](https://github.com/streamnative/pulsar-io-google-pubsub/releases/download/v3.1.0.6/pulsar-io-google-pubsub-3.1.0.6.nar). - Build the connector from the source code. @@ -57,7 +57,7 @@ To build the Google Cloud Pub/Sub source connector from the source code, follow ```bash ls target - pulsar-io-google-pubsub-3.1.0.5.nar + pulsar-io-google-pubsub-3.1.0.6.nar ``` ## Work with Function Mesh @@ -104,7 +104,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "namespace": "default", "name": "google-pubsub-source", "topicName": "test-google-pubsub-pulsar", - "archive": "connectors/pulsar-io-google-pubsub-3.1.0.5.nar", + "archive": "connectors/pulsar-io-google-pubsub-3.1.0.6.nar", "parallelism": 1, "configs": { @@ -121,7 +121,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you namespace: default name: google-pubsub-source topicName: test-google-pubsub-pulsar - archive: connectors/pulsar-io-google-pubsub-3.1.0.5.nar + archive: connectors/pulsar-io-google-pubsub-3.1.0.6.nar parallelism: 1 configs: pubsubProjectId: pulsar-io-google-pubsub @@ -140,7 +140,7 @@ kind: Source metadata: name: google-pubsub-source-sample spec: - image: streamnative/pulsar-io-google-pubsub:3.1.0.5 + image: streamnative/pulsar-io-google-pubsub:3.1.0.6 className: org.apache.pulsar.ecosystem.io.pubsub.PubsubSource replicas: 1 maxReplicas: 1 @@ -161,7 +161,7 @@ spec: cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-google-pubsub-3.1.0.5.nar + jar: connectors/pulsar-io-google-pubsub-3.1.0.6.nar clusterName: test-pulsar ``` @@ -210,7 +210,7 @@ This example describes how to use the Google Cloud Pub/Sub source connector to f 2. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-google-pubsub-3.1.0.5.nar PULSAR_HOME/connectors/pulsar-io-google-pubsub-3.1.0.5.nar + cp pulsar-io-google-pubsub-3.1.0.6.nar PULSAR_HOME/connectors/pulsar-io-google-pubsub-3.1.0.6.nar ``` 3. Start Pulsar in standalone mode. @@ -249,7 +249,7 @@ This example explains how to create a Google Cloud Pub/Sub source connector in a 1. Copy the NAR package of the Google Cloud Pub/Sub connector to the Pulsar connectors directory. ``` - cp pulsar-io-google-pubsub-3.1.0.5.nar $PULSAR_HOME/connectors/pulsar-io-google-pubsub-3.1.0.5.nar + cp pulsar-io-google-pubsub-3.1.0.6.nar $PULSAR_HOME/connectors/pulsar-io-google-pubsub-3.1.0.6.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). @@ -297,7 +297,7 @@ This example describes how to create a Google Cloud Pub/Sub source connector for metadata: name: google-pubsub-source-sample spec: - image: streamnative/pulsar-io-google-pubsub:3.1.0.5 + image: streamnative/pulsar-io-google-pubsub:3.1.0.6 className: org.apache.pulsar.ecosystem.io.pubsub.PubsubSource replicas: 1 maxReplicas: 1 @@ -318,7 +318,7 @@ This example describes how to create a Google Cloud Pub/Sub source connector for cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-google-pubsub-3.1.0.5.nar + jar: connectors/pulsar-io-google-pubsub-3.1.0.6.nar clusterName: test-pulsar ``` diff --git a/connectors/lakehouse-sink/v2.10.5.8/lakehouse-sink.md b/connectors/lakehouse-sink/v2.10.5.9/lakehouse-sink.md similarity index 95% rename from connectors/lakehouse-sink/v2.10.5.8/lakehouse-sink.md rename to connectors/lakehouse-sink/v2.10.5.9/lakehouse-sink.md index e9046d518..2db20cdbe 100644 --- a/connectors/lakehouse-sink/v2.10.5.8/lakehouse-sink.md +++ b/connectors/lakehouse-sink/v2.10.5.9/lakehouse-sink.md @@ -4,20 +4,20 @@ author: hangc0276,zymap,Huanli-Meng,horizonzy contributors: hangc0276,zymap,Huanli-Meng,horizonzy language: Java,Shell,Python document: -source: "https://github.com/streamnative/pulsar-io-lakehouse" +source: https://github.com/streamnative/pulsar-io-lakehouse license: Apache License 2.0 license_link: "https://github.com/streamnative/pulsar-io-lakehouse/blob/master/LICENSE" tags: alias: Lakehouse Sink Connector features: ["pulsar lakehouse connector"] icon: "/images/pulsar-hub.svg" -download: "https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v2.10.5.8" +download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v2.10.5.9 support: streamnative support_link: https://github.com/streamnative/pulsar-io-lakehouse support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "" +dockerfile: sn_available: "" id: "lakehouse-sink" --- @@ -25,7 +25,7 @@ id: "lakehouse-sink" The Lakehouse sink connector (including the [Hudi](https://hudi.apache.org), [Iceberg](https://iceberg.apache.org/), and [Delta Lake](https://delta.io/) sink connectors) fetches data from a Pulsar topic and saves data to the Lakehouse tables. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v2.10.5.8/docs/lakehouse-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v2.10.5.9/docs/lakehouse-sink.png) # How to get @@ -62,7 +62,7 @@ To build the Lakehouse sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-lakehouse-2.10.5.8.nar + pulsar-io-lakehouse-2.10.5.9.nar ``` # How to configure @@ -153,7 +153,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-2.10.5.8.nar", + "archive": "connectors/pulsar-io-hudi-2.10.5.9.nar", "parallelism": 1, "configs": { "type": "hudi", @@ -176,7 +176,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-2.10.5.8.nar", + "archive": "connectors/pulsar-io-hudi-2.10.5.9.nar", "parallelism": 1, "configs": { "type": "hudi", @@ -205,7 +205,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-2.10.5.8.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.9.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -233,7 +233,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-2.10.5.8.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.9.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -266,7 +266,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-2.10.5.8.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.9.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -288,7 +288,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-2.10.5.8.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.9.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -359,7 +359,7 @@ This example describes how to use the Lakehouse sink connector to fetch data fro 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-2.10.5.8.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.8.nar + cp pulsar-io-lakehouse-2.10.5.9.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.9.nar ``` 2. Start Pulsar in standalone mode. @@ -392,7 +392,7 @@ This example explains how to create a Lakehouse sink connector in an on-premises 1. Copy the NAR package of the Lakehouse sink connector to the Pulsar connectors directory. ```bash - cp pulsar-io-lakehouse-2.10.5.8.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.8.nar + cp pulsar-io-lakehouse-2.10.5.9.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.9.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/connectors/lakehouse-sink/v3.1.0.5/lakehouse-sink.md b/connectors/lakehouse-sink/v3.1.0.6/lakehouse-sink.md similarity index 95% rename from connectors/lakehouse-sink/v3.1.0.5/lakehouse-sink.md rename to connectors/lakehouse-sink/v3.1.0.6/lakehouse-sink.md index 3c15bfb4c..b593c4c1a 100644 --- a/connectors/lakehouse-sink/v3.1.0.5/lakehouse-sink.md +++ b/connectors/lakehouse-sink/v3.1.0.6/lakehouse-sink.md @@ -4,20 +4,20 @@ author: hangc0276,zymap,Huanli-Meng,horizonzy contributors: hangc0276,zymap,Huanli-Meng,horizonzy language: Java,Shell,Python document: -source: "https://github.com/streamnative/pulsar-io-lakehouse" +source: https://github.com/streamnative/pulsar-io-lakehouse license: Apache License 2.0 license_link: "https://github.com/streamnative/pulsar-io-lakehouse/blob/master/LICENSE" tags: alias: Lakehouse Sink Connector features: ["pulsar lakehouse connector"] icon: "/images/pulsar-hub.svg" -download: "https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.1.0.5" +download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.1.0.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-lakehouse support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "" +dockerfile: sn_available: "" id: "lakehouse-sink" --- @@ -25,7 +25,7 @@ id: "lakehouse-sink" The Lakehouse sink connector (including the [Hudi](https://hudi.apache.org), [Iceberg](https://iceberg.apache.org/), and [Delta Lake](https://delta.io/) sink connectors) fetches data from a Pulsar topic and saves data to the Lakehouse tables. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.1.0.5/docs/lakehouse-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.1.0.6/docs/lakehouse-sink.png) # How to get @@ -62,7 +62,7 @@ To build the Lakehouse sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-lakehouse-3.1.0.5.nar + pulsar-io-lakehouse-3.1.0.6.nar ``` # How to configure @@ -147,7 +147,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-3.1.0.5.nar", + "archive": "connectors/pulsar-io-hudi-3.1.0.6.nar", "processingGuarantees": "EFFECTIVELY_ONCE", "parallelism": 1, "configs": { @@ -171,7 +171,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-3.1.0.5-cloud.nar", + "archive": "connectors/pulsar-io-hudi-3.1.0.6-cloud.nar", "parallelism": 1, "processingGuarantees": "EFFECTIVELY_ONCE", "configs": { @@ -199,7 +199,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.1.0.5.nar", + "archive": "connectors/pulsar-io-lakehouse-3.1.0.6.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -227,7 +227,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.1.0.5-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.1.0.6-cloud.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -258,7 +258,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.1.0.5.nar", + "archive": "connectors/pulsar-io-lakehouse-3.1.0.6.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -280,7 +280,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-3.1.0.5-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.1.0.6-cloud.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -350,7 +350,7 @@ This example describes how to use the Lakehouse sink connector to fetch data fro 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-3.1.0.5.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.1.0.5.nar + cp pulsar-io-lakehouse-3.1.0.6.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.1.0.6.nar ``` 2. Start Pulsar in standalone mode. @@ -383,7 +383,7 @@ This example explains how to create a Lakehouse sink connector in an on-premises 1. Copy the NAR package of the Lakehouse sink connector to the Pulsar connectors directory. ```bash - cp pulsar-io-lakehouse-3.1.0.5.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.1.0.5.nar + cp pulsar-io-lakehouse-3.1.0.6.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.1.0.6.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/connectors/lakehouse-source/v2.10.5.8/lakehouse-source.md b/connectors/lakehouse-source/v2.10.5.9/lakehouse-source.md similarity index 93% rename from connectors/lakehouse-source/v2.10.5.8/lakehouse-source.md rename to connectors/lakehouse-source/v2.10.5.9/lakehouse-source.md index 0d0ab3606..ed01b682e 100644 --- a/connectors/lakehouse-source/v2.10.5.8/lakehouse-source.md +++ b/connectors/lakehouse-source/v2.10.5.9/lakehouse-source.md @@ -4,20 +4,20 @@ author: hangc0276,zymap,Huanli-Meng,horizonzy contributors: hangc0276,zymap,Huanli-Meng,horizonzy language: Java,Shell,Python document: -source: "https://github.com/streamnative/pulsar-io-lakehouse" +source: https://github.com/streamnative/pulsar-io-lakehouse license: Apache License 2.0 license_link: "https://github.com/streamnative/pulsar-io-lakehouse/blob/master/LICENSE" tags: alias: Lakehouse Source Connector features: ["pulsar lakehouse connector"] icon: "/images/pulsar-hub.svg" -download: "https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v2.10.5.8" +download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v2.10.5.9 support: streamnative support_link: https://github.com/streamnative/pulsar-io-lakehouse support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "" +dockerfile: sn_available: "" id: "lakehouse-source" --- @@ -25,7 +25,7 @@ id: "lakehouse-source" The Lakehouse source connector (currently only including the [Delta Lake](https://delta.io/) source connector) fetches the Lakehouse table's changelog and saves changelogs into a Pulsar topic. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v2.10.5.8/docs/lakehouse-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v2.10.5.9/docs/lakehouse-source.png) # How to get @@ -62,7 +62,7 @@ To build the Lakehouse source connector from the source code, follow these steps ```bash ls target - pulsar-io-lakehouse-2.10.5.8.nar + pulsar-io-lakehouse-2.10.5.9.nar ``` # How to configure @@ -112,7 +112,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-2.10.5.8.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.9.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -137,7 +137,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-2.10.5.8.nar", + "archive": "connectors/pulsar-io-lakehouse-2.10.5.9.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -198,7 +198,7 @@ This example describes how to use the Lakehouse source connector to fetch data f 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-2.10.5.8.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.8.nar + cp pulsar-io-lakehouse-2.10.5.9.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.9.nar ``` 2. Start Pulsar in standalone mode. @@ -229,7 +229,7 @@ This example explains how to create a Lakehouse source connector in an on-premis 1. Copy the NAR package of the Lakehouse source connector to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-2.10.5.8.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.8.nar + cp pulsar-io-lakehouse-2.10.5.9.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-2.10.5.9.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/connectors/lakehouse-source/v3.1.0.5/lakehouse-source.md b/connectors/lakehouse-source/v3.1.0.6/lakehouse-source.md similarity index 94% rename from connectors/lakehouse-source/v3.1.0.5/lakehouse-source.md rename to connectors/lakehouse-source/v3.1.0.6/lakehouse-source.md index cdf0e94ca..52b328d0b 100644 --- a/connectors/lakehouse-source/v3.1.0.5/lakehouse-source.md +++ b/connectors/lakehouse-source/v3.1.0.6/lakehouse-source.md @@ -4,20 +4,20 @@ author: hangc0276,zymap,Huanli-Meng,horizonzy contributors: hangc0276,zymap,Huanli-Meng,horizonzy language: Java,Shell,Python document: -source: "https://github.com/streamnative/pulsar-io-lakehouse" +source: https://github.com/streamnative/pulsar-io-lakehouse license: Apache License 2.0 license_link: "https://github.com/streamnative/pulsar-io-lakehouse/blob/master/LICENSE" tags: alias: Lakehouse Source Connector features: ["pulsar lakehouse connector"] icon: "/images/pulsar-hub.svg" -download: "https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.1.0.5" +download: https://api.github.com/repos/streamnative/pulsar-io-lakehouse/tarball/refs/tags/v3.1.0.6 support: streamnative support_link: https://github.com/streamnative/pulsar-io-lakehouse support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: "" +dockerfile: sn_available: "" id: "lakehouse-source" --- @@ -25,7 +25,7 @@ id: "lakehouse-source" The Lakehouse source connector (currently only including the [Delta Lake](https://delta.io/) source connector) fetches the Lakehouse table's changelog and saves changelogs into a Pulsar topic. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.1.0.5/docs/lakehouse-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-lakehouse/v3.1.0.6/docs/lakehouse-source.png) # How to get @@ -62,7 +62,7 @@ To build the Lakehouse source connector from the source code, follow these steps ```bash ls target - pulsar-io-lakehouse-3.1.0.5.nar + pulsar-io-lakehouse-3.1.0.6.nar ``` # How to configure @@ -110,7 +110,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-3.1.0.5.nar", + "archive": "connectors/pulsar-io-lakehouse-3.1.0.6.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -135,7 +135,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-3.1.0.5-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-3.1.0.6-cloud.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -195,7 +195,7 @@ This example describes how to use the Lakehouse source connector to fetch data f 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-3.1.0.5.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.1.0.5.nar + cp pulsar-io-lakehouse-3.1.0.6.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-3.1.0.6.nar ``` 2. Start Pulsar in standalone mode. @@ -226,7 +226,7 @@ This example explains how to create a Lakehouse source connector in an on-premis 1. Copy the NAR package of the Lakehouse source connector to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-3.1.0.5.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.1.0.5.nar + cp pulsar-io-lakehouse-3.1.0.6.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-3.1.0.6.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/connectors/snowflake-sink/v2.10.5.8/snowflake-sink.md b/connectors/snowflake-sink/v2.10.5.9/snowflake-sink.md similarity index 96% rename from connectors/snowflake-sink/v2.10.5.8/snowflake-sink.md rename to connectors/snowflake-sink/v2.10.5.9/snowflake-sink.md index 09e85754d..d39b1c2b9 100644 --- a/connectors/snowflake-sink/v2.10.5.8/snowflake-sink.md +++ b/connectors/snowflake-sink/v2.10.5.9/snowflake-sink.md @@ -6,18 +6,18 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-snowflake/blob/master/LICENSE" +license_link: "" tags: alias: Snowflake Sink Connector features: [""] icon: "/images/connectors/snowflake-logo.png" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "true" id: "snowflake-sink" --- @@ -89,7 +89,7 @@ You can get the Snowflake sink connector using one of the following methods if y use [Pulsar Function Worker](https://pulsar.apache.org/docs/en/functions-worker/) to run the connector in a cluster. - Download the NAR package - from [the download page](https://github.com/streamnative/pulsar-io-snowflake/releases/download/v2.10.5.8/pulsar-io-snowflake-2.10.5.8.nar) + from [the download page](https://github.com/streamnative/pulsar-io-snowflake/releases/download/v2.10.5.9/pulsar-io-snowflake-2.10.5.9.nar) . - Build it from the source code. @@ -112,7 +112,7 @@ To build the Snowflake sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-snowflake-2.10.5.8.nar + pulsar-io-snowflake-2.10.5.9.nar ``` ## Work with Function Mesh @@ -153,7 +153,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "tenant": "public", "namespace": "default", "name": "snowflake-sink", - "archive": "connectors/pulsar-io-snowflake-2.10.5.8.nar", + "archive": "connectors/pulsar-io-snowflake-2.10.5.9.nar", "inputs": [ "test-snowflake-pulsar" ], @@ -181,7 +181,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you parallelism: 1 inputs: - test-snowflake-pulsar - archive: connectors/pulsar-io-snowflake-2.10.5.8.nar + archive: connectors/pulsar-io-snowflake-2.10.5.9.nar sourceSubscriptionName: sf_sink_sub retainOrdering: true processingGuarantees: ATLEAST_ONCE @@ -205,7 +205,7 @@ kind: Sink metadata: name: snowflake-sink-sample spec: - image: streamnative/pulsar-io-snowflake:2.10.5.8 + image: streamnative/pulsar-io-snowflake:2.10.5.9 replicas: 1 maxReplicas: 1 retainOrdering: true @@ -229,7 +229,7 @@ spec: cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-snowflake-2.10.5.8.nar + jar: connectors/pulsar-io-snowflake-2.10.5.9.nar clusterName: test-pulsar autoAck: false ``` @@ -268,7 +268,7 @@ This example describes how to use the Snowflake sink connector to fetch data fro 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-snowflake-2.10.5.8.nar PULSAR_HOME/connectors/pulsar-io-snowflake-2.10.5.8.nar + cp pulsar-io-snowflake-2.10.5.9.nar PULSAR_HOME/connectors/pulsar-io-snowflake-2.10.5.9.nar ``` 2. Start a Pulsar cluster in standalone mode. @@ -325,7 +325,7 @@ This example describes how to create a Snowflake sink connector for a Kubernetes metadata: name: snowflake-sink-sample spec: - image: streamnative/pulsar-io-snowflake:2.10.5.8 + image: streamnative/pulsar-io-snowflake:2.10.5.9 className: org.apache.pulsar.ecosystem.io.snowflake.SnowflakeSinkConnector replicas: 1 maxReplicas: 1 @@ -350,7 +350,7 @@ This example describes how to create a Snowflake sink connector for a Kubernetes cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-snowflake-2.10.5.8.nar + jar: connectors/pulsar-io-snowflake-2.10.5.9.nar clusterName: test-pulsar autoAck: false ``` diff --git a/connectors/snowflake-sink/v3.1.0.5/snowflake-sink.md b/connectors/snowflake-sink/v3.1.0.6/snowflake-sink.md similarity index 96% rename from connectors/snowflake-sink/v3.1.0.5/snowflake-sink.md rename to connectors/snowflake-sink/v3.1.0.6/snowflake-sink.md index 9967cfa6a..dd3cc8444 100644 --- a/connectors/snowflake-sink/v3.1.0.5/snowflake-sink.md +++ b/connectors/snowflake-sink/v3.1.0.6/snowflake-sink.md @@ -6,18 +6,18 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-snowflake/blob/master/LICENSE" +license_link: "" tags: alias: Snowflake Sink Connector features: [""] icon: "/images/connectors/snowflake-logo.png" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "true" id: "snowflake-sink" --- @@ -89,7 +89,7 @@ You can get the Snowflake sink connector using one of the following methods if y use [Pulsar Function Worker](https://pulsar.apache.org/docs/en/functions-worker/) to run the connector in a cluster. - Download the NAR package - from [the download page](https://github.com/streamnative/pulsar-io-snowflake/releases/download/v3.1.0.5/pulsar-io-snowflake-3.1.0.5.nar) + from [the download page](https://github.com/streamnative/pulsar-io-snowflake/releases/download/v3.1.0.6/pulsar-io-snowflake-3.1.0.6.nar) . - Build it from the source code. @@ -112,7 +112,7 @@ To build the Snowflake sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-snowflake-3.1.0.5.nar + pulsar-io-snowflake-3.1.0.6.nar ``` ## Work with Function Mesh @@ -153,7 +153,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "tenant": "public", "namespace": "default", "name": "snowflake-sink", - "archive": "connectors/pulsar-io-snowflake-3.1.0.5.nar", + "archive": "connectors/pulsar-io-snowflake-3.1.0.6.nar", "inputs": [ "test-snowflake-pulsar" ], @@ -181,7 +181,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you parallelism: 1 inputs: - test-snowflake-pulsar - archive: connectors/pulsar-io-snowflake-3.1.0.5.nar + archive: connectors/pulsar-io-snowflake-3.1.0.6.nar sourceSubscriptionName: sf_sink_sub retainOrdering: true processingGuarantees: ATLEAST_ONCE @@ -205,7 +205,7 @@ kind: Sink metadata: name: snowflake-sink-sample spec: - image: streamnative/pulsar-io-snowflake:3.1.0.5 + image: streamnative/pulsar-io-snowflake:3.1.0.6 replicas: 1 maxReplicas: 1 retainOrdering: true @@ -229,7 +229,7 @@ spec: cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-snowflake-3.1.0.5.nar + jar: connectors/pulsar-io-snowflake-3.1.0.6.nar clusterName: test-pulsar autoAck: false ``` @@ -268,7 +268,7 @@ This example describes how to use the Snowflake sink connector to fetch data fro 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-snowflake-3.1.0.5.nar PULSAR_HOME/connectors/pulsar-io-snowflake-3.1.0.5.nar + cp pulsar-io-snowflake-3.1.0.6.nar PULSAR_HOME/connectors/pulsar-io-snowflake-3.1.0.6.nar ``` 2. Start a Pulsar cluster in standalone mode. @@ -325,7 +325,7 @@ This example describes how to create a Snowflake sink connector for a Kubernetes metadata: name: snowflake-sink-sample spec: - image: streamnative/pulsar-io-snowflake:3.1.0.5 + image: streamnative/pulsar-io-snowflake:3.1.0.6 className: org.apache.pulsar.ecosystem.io.snowflake.SnowflakeSinkConnector replicas: 1 maxReplicas: 1 @@ -350,7 +350,7 @@ This example describes how to create a Snowflake sink connector for a Kubernetes cpu: "0.1" memory: 1G java: - jar: connectors/pulsar-io-snowflake-3.1.0.5.nar + jar: connectors/pulsar-io-snowflake-3.1.0.6.nar clusterName: test-pulsar autoAck: false ``` diff --git a/connectors/sqs-sink/v3.1.0.5/sqs-sink.md b/connectors/sqs-sink/v2.10.5.9/sqs-sink.md similarity index 97% rename from connectors/sqs-sink/v3.1.0.5/sqs-sink.md rename to connectors/sqs-sink/v2.10.5.9/sqs-sink.md index ab21206dc..53bd40c67 100644 --- a/connectors/sqs-sink/v3.1.0.5/sqs-sink.md +++ b/connectors/sqs-sink/v2.10.5.9/sqs-sink.md @@ -1,23 +1,23 @@ --- description: -author: freeznet,Anonymitaet,nlu90,shibd -contributors: freeznet,Anonymitaet,nlu90,shibd +author: freeznet,Anonymitaet,shibd,nlu90 +contributors: freeznet,Anonymitaet,shibd,nlu90 language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Sink Connector features: [""] icon: "/images/connectors/sqs-logo.png" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "true" id: "sqs-sink" --- @@ -25,7 +25,7 @@ id: "sqs-sink" The [AWS Simple Queue Service (SQS)](https://aws.amazon.com/sqs/?nc1=h_ls) sink connector pulls data from Pulsar topics and persists data to AWS SQS. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.1.0.5/docs/sqs-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v2.10.5.9/docs/sqs-sink.png) ## Quick start diff --git a/connectors/sqs-sink/v2.10.5.8/sqs-sink.md b/connectors/sqs-sink/v3.1.0.6/sqs-sink.md similarity index 97% rename from connectors/sqs-sink/v2.10.5.8/sqs-sink.md rename to connectors/sqs-sink/v3.1.0.6/sqs-sink.md index 50cd0d8a4..b5074790d 100644 --- a/connectors/sqs-sink/v2.10.5.8/sqs-sink.md +++ b/connectors/sqs-sink/v3.1.0.6/sqs-sink.md @@ -1,23 +1,23 @@ --- description: -author: freeznet,Anonymitaet,nlu90,shibd -contributors: freeznet,Anonymitaet,nlu90,shibd +author: freeznet,Anonymitaet,shibd,nlu90 +contributors: freeznet,Anonymitaet,shibd,nlu90 language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Sink Connector features: [""] icon: "/images/connectors/sqs-logo.png" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "true" id: "sqs-sink" --- @@ -25,7 +25,7 @@ id: "sqs-sink" The [AWS Simple Queue Service (SQS)](https://aws.amazon.com/sqs/?nc1=h_ls) sink connector pulls data from Pulsar topics and persists data to AWS SQS. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v2.10.5.8/docs/sqs-sink.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.1.0.6/docs/sqs-sink.png) ## Quick start diff --git a/connectors/sqs-source/v3.1.0.5/sqs-source.md b/connectors/sqs-source/v2.10.5.9/sqs-source.md similarity index 97% rename from connectors/sqs-source/v3.1.0.5/sqs-source.md rename to connectors/sqs-source/v2.10.5.9/sqs-source.md index e209c370d..e42046b94 100644 --- a/connectors/sqs-source/v3.1.0.5/sqs-source.md +++ b/connectors/sqs-source/v2.10.5.9/sqs-source.md @@ -1,23 +1,23 @@ --- description: -author: freeznet,Anonymitaet,nlu90,shibd -contributors: freeznet,Anonymitaet,nlu90,shibd +author: freeznet,Anonymitaet,shibd,nlu90 +contributors: freeznet,Anonymitaet,shibd,nlu90 language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Source Connector features: [""] icon: "/images/connectors/sqs-logo.png" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "true" id: "sqs-source" --- @@ -25,7 +25,7 @@ id: "sqs-source" The [AWS Simple Queue Service (SQS)](https://aws.amazon.com/sqs/?nc1=h_ls) source connector feeds data from Amazon AWS SQS and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.1.0.5/docs/sqs-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v2.10.5.9/docs/sqs-source.png) ## Quick start diff --git a/connectors/sqs-source/v2.10.5.8/sqs-source.md b/connectors/sqs-source/v3.1.0.6/sqs-source.md similarity index 97% rename from connectors/sqs-source/v2.10.5.8/sqs-source.md rename to connectors/sqs-source/v3.1.0.6/sqs-source.md index d3c44cd28..4ae74e7e7 100644 --- a/connectors/sqs-source/v2.10.5.8/sqs-source.md +++ b/connectors/sqs-source/v3.1.0.6/sqs-source.md @@ -1,23 +1,23 @@ --- description: -author: freeznet,Anonymitaet,nlu90,shibd -contributors: freeznet,Anonymitaet,nlu90,shibd +author: freeznet,Anonymitaet,shibd,nlu90 +contributors: freeznet,Anonymitaet,shibd,nlu90 language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Source Connector features: [""] icon: "/images/connectors/sqs-logo.png" -download: +download: support: streamnative -support_link: +support_link: https://streamnative.io support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" owner_name: "streamnative" owner_img: "https://avatars.githubusercontent.com/u/44651383?v=4" -dockerfile: +dockerfile: sn_available: "true" id: "sqs-source" --- @@ -25,7 +25,7 @@ id: "sqs-source" The [AWS Simple Queue Service (SQS)](https://aws.amazon.com/sqs/?nc1=h_ls) source connector feeds data from Amazon AWS SQS and writes data to Pulsar topics. -![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v2.10.5.8/docs/sqs-source.png) +![](https://raw.githubusercontent.com/streamnative/pulsar-io-sqs/v3.1.0.6/docs/sqs-source.png) ## Quick start diff --git a/sync.js b/sync.js index f8454d4bb..d1ba78c29 100644 --- a/sync.js +++ b/sync.js @@ -282,14 +282,14 @@ async function syncDoc(tag, pathPrefix, fileName, organization, repository, proj document: "", source: project.private_source ? "Private source" : (project.source || source), license: project.private_source ? "Business License" : (project.license || license), - licenseLink: project.license_link || licenseLink, + licenseLink: project.private_source ? "" : (project.license_link || licenseLink), tags: project.tags || topics["names"], alias: project.alias || alias, features: project.description || description, icon: project.icon || icon, download: project.private_source ? "" : (project.download || download), - support: project.support || support, - supportLink: project.private_source ? "" : (project.support_link || source), + support: project.private_source ? "streamnative" : (project.support || support), + supportLink: project.private_source ? "https://streamnative.io" : (project.support_link || source), supportImg: project.support_img || ownerImg, ownerName: project.owner_name || ownerName, ownerImg: project.owner_img || ownerImg, From 027744834ccf8e72f4421fa52689943fbb389309 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Mon, 20 Nov 2023 09:36:06 +0800 Subject: [PATCH 2/5] Fix license link for private source connector. --- .../aws-eventbridge-sink/v2.11.2.12/aws-eventbridge-sink.md | 2 +- .../aws-eventbridge-sink/v3.0.1.6/aws-eventbridge-sink.md | 2 +- connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md | 2 +- connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md | 2 +- connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md | 2 +- .../google-bigquery-sink/v2.11.2.12/google-bigquery-sink.md | 2 +- .../google-bigquery-sink/v2.8.4.2/google-bigquery-sink.md | 2 +- .../google-bigquery-sink/v2.9.5.4/google-bigquery-sink.md | 2 +- .../google-bigquery-sink/v3.0.1.6/google-bigquery-sink.md | 2 +- .../google-bigquery-source/v2.11.2.12/google-bigquery-source.md | 2 +- .../google-bigquery-source/v2.8.4.2/google-bigquery-source.md | 2 +- .../google-bigquery-source/v3.0.1.6/google-bigquery-source.md | 2 +- connectors/google-pubsub-sink/v2.11.2.12/google-pubsub-sink.md | 2 +- connectors/google-pubsub-sink/v2.8.4.2/google-pubsub-sink.md | 2 +- connectors/google-pubsub-sink/v3.0.1.6/google-pubsub-sink.md | 2 +- .../google-pubsub-source/v2.11.2.12/google-pubsub-source.md | 2 +- .../google-pubsub-source/v2.8.4.2/google-pubsub-source.md | 2 +- .../google-pubsub-source/v3.0.1.6/google-pubsub-source.md | 2 +- connectors/sqs-sink/v2.11.2.12/sqs-sink.md | 2 +- connectors/sqs-sink/v2.7.4.8/sqs-sink.md | 2 +- connectors/sqs-sink/v2.8.4.2/sqs-sink.md | 2 +- connectors/sqs-sink/v2.9.5.4/sqs-sink.md | 2 +- connectors/sqs-sink/v3.0.1.6/sqs-sink.md | 2 +- connectors/sqs-source/v2.11.2.12/sqs-source.md | 2 +- connectors/sqs-source/v2.7.4.8/sqs-source.md | 2 +- connectors/sqs-source/v2.8.4.2/sqs-source.md | 2 +- connectors/sqs-source/v2.9.5.4/sqs-source.md | 2 +- connectors/sqs-source/v3.0.1.6/sqs-source.md | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/connectors/aws-eventbridge-sink/v2.11.2.12/aws-eventbridge-sink.md b/connectors/aws-eventbridge-sink/v2.11.2.12/aws-eventbridge-sink.md index 0df9c3583..0dcaa75e1 100644 --- a/connectors/aws-eventbridge-sink/v2.11.2.12/aws-eventbridge-sink.md +++ b/connectors/aws-eventbridge-sink/v2.11.2.12/aws-eventbridge-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-aws-eventbridge/blob/master/LICENSE" +license_link: "" tags: alias: AWS EventBridge Sink Connector features: [""] diff --git a/connectors/aws-eventbridge-sink/v3.0.1.6/aws-eventbridge-sink.md b/connectors/aws-eventbridge-sink/v3.0.1.6/aws-eventbridge-sink.md index 432adc6b6..c63ec826e 100644 --- a/connectors/aws-eventbridge-sink/v3.0.1.6/aws-eventbridge-sink.md +++ b/connectors/aws-eventbridge-sink/v3.0.1.6/aws-eventbridge-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-aws-eventbridge/blob/master/LICENSE" +license_link: "" tags: alias: AWS EventBridge Sink Connector features: [""] diff --git a/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md b/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md index 7709bb330..fa2017831 100644 --- a/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: "Private source" license: Business License -license_link: "https://github.com/streamnative/pulsar-io-aws-lambda/blob/master/LICENSE" +license_link: "" tags: alias: AWS Lambda Sink features: [""] diff --git a/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md b/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md index 242ab1681..dc287c300 100644 --- a/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: "Private source" license: Business License -license_link: "https://github.com/streamnative/pulsar-io-aws-lambda/blob/master/LICENSE" +license_link: "" tags: alias: AWS Lambda Sink features: [""] diff --git a/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md b/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md index 5fe70f8ec..31b5c01cd 100644 --- a/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: "Private source" license: Business License -license_link: "https://github.com/streamnative/pulsar-io-aws-lambda/blob/master/LICENSE" +license_link: "" tags: alias: AWS Lambda Sink features: [""] diff --git a/connectors/google-bigquery-sink/v2.11.2.12/google-bigquery-sink.md b/connectors/google-bigquery-sink/v2.11.2.12/google-bigquery-sink.md index ca7852cab..c6c235374 100644 --- a/connectors/google-bigquery-sink/v2.11.2.12/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v2.11.2.12/google-bigquery-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-bigquery/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-sink/v2.8.4.2/google-bigquery-sink.md b/connectors/google-bigquery-sink/v2.8.4.2/google-bigquery-sink.md index 315d00f8a..edf27b57f 100644 --- a/connectors/google-bigquery-sink/v2.8.4.2/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v2.8.4.2/google-bigquery-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-bigquery/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-sink/v2.9.5.4/google-bigquery-sink.md b/connectors/google-bigquery-sink/v2.9.5.4/google-bigquery-sink.md index 175da660e..a0922b7a0 100644 --- a/connectors/google-bigquery-sink/v2.9.5.4/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v2.9.5.4/google-bigquery-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-bigquery/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-sink/v3.0.1.6/google-bigquery-sink.md b/connectors/google-bigquery-sink/v3.0.1.6/google-bigquery-sink.md index 69ca873da..602c2dc98 100644 --- a/connectors/google-bigquery-sink/v3.0.1.6/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v3.0.1.6/google-bigquery-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-bigquery/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-source/v2.11.2.12/google-bigquery-source.md b/connectors/google-bigquery-source/v2.11.2.12/google-bigquery-source.md index 869d2e3b8..07e161e8c 100644 --- a/connectors/google-bigquery-source/v2.11.2.12/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v2.11.2.12/google-bigquery-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-bigquery/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud BigQuery Source Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-source/v2.8.4.2/google-bigquery-source.md b/connectors/google-bigquery-source/v2.8.4.2/google-bigquery-source.md index 0cb5ef084..1617663a4 100644 --- a/connectors/google-bigquery-source/v2.8.4.2/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v2.8.4.2/google-bigquery-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-bigquery/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud BigQuery Source Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-source/v3.0.1.6/google-bigquery-source.md b/connectors/google-bigquery-source/v3.0.1.6/google-bigquery-source.md index 0b63207ab..3243974d1 100644 --- a/connectors/google-bigquery-source/v3.0.1.6/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v3.0.1.6/google-bigquery-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-bigquery/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud BigQuery Source Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-pubsub-sink/v2.11.2.12/google-pubsub-sink.md b/connectors/google-pubsub-sink/v2.11.2.12/google-pubsub-sink.md index b5c36a097..948764221 100644 --- a/connectors/google-pubsub-sink/v2.11.2.12/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v2.11.2.12/google-pubsub-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-google-pubsub/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud Pub/Sub Sink Connector features: [""] diff --git a/connectors/google-pubsub-sink/v2.8.4.2/google-pubsub-sink.md b/connectors/google-pubsub-sink/v2.8.4.2/google-pubsub-sink.md index a0dac27ab..35b7bf618 100644 --- a/connectors/google-pubsub-sink/v2.8.4.2/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v2.8.4.2/google-pubsub-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-google-pubsub/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud Pub/Sub Sink Connector features: [""] diff --git a/connectors/google-pubsub-sink/v3.0.1.6/google-pubsub-sink.md b/connectors/google-pubsub-sink/v3.0.1.6/google-pubsub-sink.md index 9920033d6..3142c2aa4 100644 --- a/connectors/google-pubsub-sink/v3.0.1.6/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v3.0.1.6/google-pubsub-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-google-pubsub/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud Pub/Sub Sink Connector features: [""] diff --git a/connectors/google-pubsub-source/v2.11.2.12/google-pubsub-source.md b/connectors/google-pubsub-source/v2.11.2.12/google-pubsub-source.md index c238ccdbc..1558b8280 100644 --- a/connectors/google-pubsub-source/v2.11.2.12/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v2.11.2.12/google-pubsub-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-google-pubsub/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud Pub/Sub Source Connector features: [""] diff --git a/connectors/google-pubsub-source/v2.8.4.2/google-pubsub-source.md b/connectors/google-pubsub-source/v2.8.4.2/google-pubsub-source.md index 9f8ab0a7f..2854bf25a 100644 --- a/connectors/google-pubsub-source/v2.8.4.2/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v2.8.4.2/google-pubsub-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-google-pubsub/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud Pub/Sub Source Connector features: [""] diff --git a/connectors/google-pubsub-source/v3.0.1.6/google-pubsub-source.md b/connectors/google-pubsub-source/v3.0.1.6/google-pubsub-source.md index 7fc5e6605..796335cd6 100644 --- a/connectors/google-pubsub-source/v3.0.1.6/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v3.0.1.6/google-pubsub-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-google-pubsub/blob/master/LICENSE" +license_link: "" tags: alias: Google Cloud Pub/Sub Source Connector features: [""] diff --git a/connectors/sqs-sink/v2.11.2.12/sqs-sink.md b/connectors/sqs-sink/v2.11.2.12/sqs-sink.md index 42945f2f7..aef6151f4 100644 --- a/connectors/sqs-sink/v2.11.2.12/sqs-sink.md +++ b/connectors/sqs-sink/v2.11.2.12/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-sink/v2.7.4.8/sqs-sink.md b/connectors/sqs-sink/v2.7.4.8/sqs-sink.md index 69783f14a..c8c7b7d71 100644 --- a/connectors/sqs-sink/v2.7.4.8/sqs-sink.md +++ b/connectors/sqs-sink/v2.7.4.8/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-sink/v2.8.4.2/sqs-sink.md b/connectors/sqs-sink/v2.8.4.2/sqs-sink.md index dc1fb29e8..5ed020533 100644 --- a/connectors/sqs-sink/v2.8.4.2/sqs-sink.md +++ b/connectors/sqs-sink/v2.8.4.2/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-sink/v2.9.5.4/sqs-sink.md b/connectors/sqs-sink/v2.9.5.4/sqs-sink.md index b3b15887e..3e46b1c7d 100644 --- a/connectors/sqs-sink/v2.9.5.4/sqs-sink.md +++ b/connectors/sqs-sink/v2.9.5.4/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-sink/v3.0.1.6/sqs-sink.md b/connectors/sqs-sink/v3.0.1.6/sqs-sink.md index 59b383383..8f20bb293 100644 --- a/connectors/sqs-sink/v3.0.1.6/sqs-sink.md +++ b/connectors/sqs-sink/v3.0.1.6/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-source/v2.11.2.12/sqs-source.md b/connectors/sqs-source/v2.11.2.12/sqs-source.md index bbbe7934d..787ed2e7b 100644 --- a/connectors/sqs-source/v2.11.2.12/sqs-source.md +++ b/connectors/sqs-source/v2.11.2.12/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Source Connector features: [""] diff --git a/connectors/sqs-source/v2.7.4.8/sqs-source.md b/connectors/sqs-source/v2.7.4.8/sqs-source.md index ec04f5d10..0f076878e 100644 --- a/connectors/sqs-source/v2.7.4.8/sqs-source.md +++ b/connectors/sqs-source/v2.7.4.8/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Source Connector features: [""] diff --git a/connectors/sqs-source/v2.8.4.2/sqs-source.md b/connectors/sqs-source/v2.8.4.2/sqs-source.md index 0b35a11e1..5de6b2868 100644 --- a/connectors/sqs-source/v2.8.4.2/sqs-source.md +++ b/connectors/sqs-source/v2.8.4.2/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Source Connector features: [""] diff --git a/connectors/sqs-source/v2.9.5.4/sqs-source.md b/connectors/sqs-source/v2.9.5.4/sqs-source.md index aeabd44de..112f08684 100644 --- a/connectors/sqs-source/v2.9.5.4/sqs-source.md +++ b/connectors/sqs-source/v2.9.5.4/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Source Connector features: [""] diff --git a/connectors/sqs-source/v3.0.1.6/sqs-source.md b/connectors/sqs-source/v3.0.1.6/sqs-source.md index 604b6e9cb..575c77208 100644 --- a/connectors/sqs-source/v3.0.1.6/sqs-source.md +++ b/connectors/sqs-source/v3.0.1.6/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "https://github.com/streamnative/pulsar-io-sqs/blob/master/LICENSE" +license_link: "" tags: alias: AWS SQS Source Connector features: [""] From 449300feadfbbf3d6cfb8a74302ffd6005c9879b Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Mon, 20 Nov 2023 09:37:56 +0800 Subject: [PATCH 3/5] fix incorrect download link. --- connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md | 2 +- connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md | 2 +- connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md b/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md index fa2017831..da1d6f603 100644 --- a/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md @@ -11,7 +11,7 @@ tags: alias: AWS Lambda Sink features: [""] icon: "/images/connectors/lambda-logo.png" -download: "Business License" +download: support: streamnative support_link: support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" diff --git a/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md b/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md index dc287c300..117b21a89 100644 --- a/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md @@ -11,7 +11,7 @@ tags: alias: AWS Lambda Sink features: [""] icon: "/images/connectors/lambda-logo.png" -download: "Business License" +download: support: streamnative support_link: support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" diff --git a/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md b/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md index 31b5c01cd..5d23e78dc 100644 --- a/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md @@ -11,7 +11,7 @@ tags: alias: AWS Lambda Sink features: [""] icon: "/images/connectors/lambda-logo.png" -download: "Business License" +download: support: streamnative support_link: support_img: "https://avatars.githubusercontent.com/u/44651383?v=4" From 84ac98110b6967fe527720d7fbfe8ab8b27157e3 Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Mon, 20 Nov 2023 09:51:18 +0800 Subject: [PATCH 4/5] fix incorrect license link. --- .../aws-eventbridge-sink/v2.10.5.9/aws-eventbridge-sink.md | 2 +- .../aws-eventbridge-sink/v2.11.2.12/aws-eventbridge-sink.md | 2 +- .../aws-eventbridge-sink/v3.0.1.6/aws-eventbridge-sink.md | 2 +- .../aws-eventbridge-sink/v3.1.0.6/aws-eventbridge-sink.md | 2 +- connectors/aws-lambda-sink/v2.10.5.9/aws-lambda-sink.md | 2 +- connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md | 2 +- connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md | 2 +- connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md | 2 +- connectors/aws-lambda-sink/v3.1.0.6/aws-lambda-sink.md | 2 +- .../google-bigquery-sink/v2.10.5.9/google-bigquery-sink.md | 2 +- .../google-bigquery-sink/v2.11.2.12/google-bigquery-sink.md | 2 +- .../google-bigquery-sink/v2.8.4.2/google-bigquery-sink.md | 2 +- .../google-bigquery-sink/v2.9.5.4/google-bigquery-sink.md | 2 +- .../google-bigquery-sink/v3.0.1.6/google-bigquery-sink.md | 2 +- .../google-bigquery-sink/v3.1.0.6/google-bigquery-sink.md | 2 +- .../google-bigquery-source/v2.10.5.9/google-bigquery-source.md | 2 +- .../google-bigquery-source/v2.11.2.12/google-bigquery-source.md | 2 +- .../google-bigquery-source/v2.8.4.2/google-bigquery-source.md | 2 +- .../google-bigquery-source/v3.0.1.6/google-bigquery-source.md | 2 +- .../google-bigquery-source/v3.1.0.6/google-bigquery-source.md | 2 +- connectors/google-pubsub-sink/v2.10.5.9/google-pubsub-sink.md | 2 +- connectors/google-pubsub-sink/v2.11.2.12/google-pubsub-sink.md | 2 +- connectors/google-pubsub-sink/v2.8.4.2/google-pubsub-sink.md | 2 +- connectors/google-pubsub-sink/v3.0.1.6/google-pubsub-sink.md | 2 +- connectors/google-pubsub-sink/v3.1.0.6/google-pubsub-sink.md | 2 +- .../google-pubsub-source/v2.10.5.9/google-pubsub-source.md | 2 +- .../google-pubsub-source/v2.11.2.12/google-pubsub-source.md | 2 +- .../google-pubsub-source/v2.8.4.2/google-pubsub-source.md | 2 +- .../google-pubsub-source/v3.0.1.6/google-pubsub-source.md | 2 +- .../google-pubsub-source/v3.1.0.6/google-pubsub-source.md | 2 +- connectors/snowflake-sink/v2.10.5.9/snowflake-sink.md | 2 +- connectors/snowflake-sink/v3.1.0.6/snowflake-sink.md | 2 +- connectors/sqs-sink/v2.10.5.9/sqs-sink.md | 2 +- connectors/sqs-sink/v2.11.2.12/sqs-sink.md | 2 +- connectors/sqs-sink/v2.7.4.8/sqs-sink.md | 2 +- connectors/sqs-sink/v2.8.4.2/sqs-sink.md | 2 +- connectors/sqs-sink/v2.9.5.4/sqs-sink.md | 2 +- connectors/sqs-sink/v3.0.1.6/sqs-sink.md | 2 +- connectors/sqs-sink/v3.1.0.6/sqs-sink.md | 2 +- connectors/sqs-source/v2.10.5.9/sqs-source.md | 2 +- connectors/sqs-source/v2.11.2.12/sqs-source.md | 2 +- connectors/sqs-source/v2.7.4.8/sqs-source.md | 2 +- connectors/sqs-source/v2.8.4.2/sqs-source.md | 2 +- connectors/sqs-source/v2.9.5.4/sqs-source.md | 2 +- connectors/sqs-source/v3.0.1.6/sqs-source.md | 2 +- connectors/sqs-source/v3.1.0.6/sqs-source.md | 2 +- hub.template | 2 +- 47 files changed, 47 insertions(+), 47 deletions(-) diff --git a/connectors/aws-eventbridge-sink/v2.10.5.9/aws-eventbridge-sink.md b/connectors/aws-eventbridge-sink/v2.10.5.9/aws-eventbridge-sink.md index 3b8054229..10751f0ee 100644 --- a/connectors/aws-eventbridge-sink/v2.10.5.9/aws-eventbridge-sink.md +++ b/connectors/aws-eventbridge-sink/v2.10.5.9/aws-eventbridge-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS EventBridge Sink Connector features: [""] diff --git a/connectors/aws-eventbridge-sink/v2.11.2.12/aws-eventbridge-sink.md b/connectors/aws-eventbridge-sink/v2.11.2.12/aws-eventbridge-sink.md index 0dcaa75e1..eb2d71126 100644 --- a/connectors/aws-eventbridge-sink/v2.11.2.12/aws-eventbridge-sink.md +++ b/connectors/aws-eventbridge-sink/v2.11.2.12/aws-eventbridge-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS EventBridge Sink Connector features: [""] diff --git a/connectors/aws-eventbridge-sink/v3.0.1.6/aws-eventbridge-sink.md b/connectors/aws-eventbridge-sink/v3.0.1.6/aws-eventbridge-sink.md index c63ec826e..2f9f70f32 100644 --- a/connectors/aws-eventbridge-sink/v3.0.1.6/aws-eventbridge-sink.md +++ b/connectors/aws-eventbridge-sink/v3.0.1.6/aws-eventbridge-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS EventBridge Sink Connector features: [""] diff --git a/connectors/aws-eventbridge-sink/v3.1.0.6/aws-eventbridge-sink.md b/connectors/aws-eventbridge-sink/v3.1.0.6/aws-eventbridge-sink.md index 26a441de2..98050037c 100644 --- a/connectors/aws-eventbridge-sink/v3.1.0.6/aws-eventbridge-sink.md +++ b/connectors/aws-eventbridge-sink/v3.1.0.6/aws-eventbridge-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS EventBridge Sink Connector features: [""] diff --git a/connectors/aws-lambda-sink/v2.10.5.9/aws-lambda-sink.md b/connectors/aws-lambda-sink/v2.10.5.9/aws-lambda-sink.md index ebd923d79..54c5faacf 100644 --- a/connectors/aws-lambda-sink/v2.10.5.9/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v2.10.5.9/aws-lambda-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS Lambda Sink features: [""] diff --git a/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md b/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md index da1d6f603..aa9db0119 100644 --- a/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: "Private source" license: Business License -license_link: "" +license_link: tags: alias: AWS Lambda Sink features: [""] diff --git a/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md b/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md index 117b21a89..2d0ec9bdf 100644 --- a/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: "Private source" license: Business License -license_link: "" +license_link: tags: alias: AWS Lambda Sink features: [""] diff --git a/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md b/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md index 5d23e78dc..fdf98f37e 100644 --- a/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: "Private source" license: Business License -license_link: "" +license_link: tags: alias: AWS Lambda Sink features: [""] diff --git a/connectors/aws-lambda-sink/v3.1.0.6/aws-lambda-sink.md b/connectors/aws-lambda-sink/v3.1.0.6/aws-lambda-sink.md index 6b0ee68dc..7ba51035b 100644 --- a/connectors/aws-lambda-sink/v3.1.0.6/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v3.1.0.6/aws-lambda-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS Lambda Sink features: [""] diff --git a/connectors/google-bigquery-sink/v2.10.5.9/google-bigquery-sink.md b/connectors/google-bigquery-sink/v2.10.5.9/google-bigquery-sink.md index 6cf8c8644..ecdac7f65 100644 --- a/connectors/google-bigquery-sink/v2.10.5.9/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v2.10.5.9/google-bigquery-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-sink/v2.11.2.12/google-bigquery-sink.md b/connectors/google-bigquery-sink/v2.11.2.12/google-bigquery-sink.md index c6c235374..cb75c0f30 100644 --- a/connectors/google-bigquery-sink/v2.11.2.12/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v2.11.2.12/google-bigquery-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-sink/v2.8.4.2/google-bigquery-sink.md b/connectors/google-bigquery-sink/v2.8.4.2/google-bigquery-sink.md index edf27b57f..adc0903d4 100644 --- a/connectors/google-bigquery-sink/v2.8.4.2/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v2.8.4.2/google-bigquery-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-sink/v2.9.5.4/google-bigquery-sink.md b/connectors/google-bigquery-sink/v2.9.5.4/google-bigquery-sink.md index a0922b7a0..a7b174780 100644 --- a/connectors/google-bigquery-sink/v2.9.5.4/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v2.9.5.4/google-bigquery-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-sink/v3.0.1.6/google-bigquery-sink.md b/connectors/google-bigquery-sink/v3.0.1.6/google-bigquery-sink.md index 602c2dc98..4788c457a 100644 --- a/connectors/google-bigquery-sink/v3.0.1.6/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v3.0.1.6/google-bigquery-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-sink/v3.1.0.6/google-bigquery-sink.md b/connectors/google-bigquery-sink/v3.1.0.6/google-bigquery-sink.md index 99a31ee15..472c2cf9f 100644 --- a/connectors/google-bigquery-sink/v3.1.0.6/google-bigquery-sink.md +++ b/connectors/google-bigquery-sink/v3.1.0.6/google-bigquery-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud BigQuery Sink Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-source/v2.10.5.9/google-bigquery-source.md b/connectors/google-bigquery-source/v2.10.5.9/google-bigquery-source.md index 8b038f00c..801805a6f 100644 --- a/connectors/google-bigquery-source/v2.10.5.9/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v2.10.5.9/google-bigquery-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud BigQuery Source Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-source/v2.11.2.12/google-bigquery-source.md b/connectors/google-bigquery-source/v2.11.2.12/google-bigquery-source.md index 07e161e8c..5053ce0be 100644 --- a/connectors/google-bigquery-source/v2.11.2.12/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v2.11.2.12/google-bigquery-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud BigQuery Source Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-source/v2.8.4.2/google-bigquery-source.md b/connectors/google-bigquery-source/v2.8.4.2/google-bigquery-source.md index 1617663a4..3443079e7 100644 --- a/connectors/google-bigquery-source/v2.8.4.2/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v2.8.4.2/google-bigquery-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud BigQuery Source Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-source/v3.0.1.6/google-bigquery-source.md b/connectors/google-bigquery-source/v3.0.1.6/google-bigquery-source.md index 3243974d1..35f4406bf 100644 --- a/connectors/google-bigquery-source/v3.0.1.6/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v3.0.1.6/google-bigquery-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud BigQuery Source Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-bigquery-source/v3.1.0.6/google-bigquery-source.md b/connectors/google-bigquery-source/v3.1.0.6/google-bigquery-source.md index 3fdd50493..8e1e05065 100644 --- a/connectors/google-bigquery-source/v3.1.0.6/google-bigquery-source.md +++ b/connectors/google-bigquery-source/v3.1.0.6/google-bigquery-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud BigQuery Source Connector features: ["BigQuery Connector integrates Apache Pulsar with Google BigQuery."] diff --git a/connectors/google-pubsub-sink/v2.10.5.9/google-pubsub-sink.md b/connectors/google-pubsub-sink/v2.10.5.9/google-pubsub-sink.md index 2811b5ee5..0e2933f7c 100644 --- a/connectors/google-pubsub-sink/v2.10.5.9/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v2.10.5.9/google-pubsub-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud Pub/Sub Sink Connector features: [""] diff --git a/connectors/google-pubsub-sink/v2.11.2.12/google-pubsub-sink.md b/connectors/google-pubsub-sink/v2.11.2.12/google-pubsub-sink.md index 948764221..810647774 100644 --- a/connectors/google-pubsub-sink/v2.11.2.12/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v2.11.2.12/google-pubsub-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud Pub/Sub Sink Connector features: [""] diff --git a/connectors/google-pubsub-sink/v2.8.4.2/google-pubsub-sink.md b/connectors/google-pubsub-sink/v2.8.4.2/google-pubsub-sink.md index 35b7bf618..cc864a258 100644 --- a/connectors/google-pubsub-sink/v2.8.4.2/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v2.8.4.2/google-pubsub-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud Pub/Sub Sink Connector features: [""] diff --git a/connectors/google-pubsub-sink/v3.0.1.6/google-pubsub-sink.md b/connectors/google-pubsub-sink/v3.0.1.6/google-pubsub-sink.md index 3142c2aa4..1e8e8711d 100644 --- a/connectors/google-pubsub-sink/v3.0.1.6/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v3.0.1.6/google-pubsub-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud Pub/Sub Sink Connector features: [""] diff --git a/connectors/google-pubsub-sink/v3.1.0.6/google-pubsub-sink.md b/connectors/google-pubsub-sink/v3.1.0.6/google-pubsub-sink.md index db9fc8580..d9a74d959 100644 --- a/connectors/google-pubsub-sink/v3.1.0.6/google-pubsub-sink.md +++ b/connectors/google-pubsub-sink/v3.1.0.6/google-pubsub-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud Pub/Sub Sink Connector features: [""] diff --git a/connectors/google-pubsub-source/v2.10.5.9/google-pubsub-source.md b/connectors/google-pubsub-source/v2.10.5.9/google-pubsub-source.md index 06d098691..223aba5ed 100644 --- a/connectors/google-pubsub-source/v2.10.5.9/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v2.10.5.9/google-pubsub-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud Pub/Sub Source Connector features: [""] diff --git a/connectors/google-pubsub-source/v2.11.2.12/google-pubsub-source.md b/connectors/google-pubsub-source/v2.11.2.12/google-pubsub-source.md index 1558b8280..3703d026f 100644 --- a/connectors/google-pubsub-source/v2.11.2.12/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v2.11.2.12/google-pubsub-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud Pub/Sub Source Connector features: [""] diff --git a/connectors/google-pubsub-source/v2.8.4.2/google-pubsub-source.md b/connectors/google-pubsub-source/v2.8.4.2/google-pubsub-source.md index 2854bf25a..b55a7618d 100644 --- a/connectors/google-pubsub-source/v2.8.4.2/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v2.8.4.2/google-pubsub-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud Pub/Sub Source Connector features: [""] diff --git a/connectors/google-pubsub-source/v3.0.1.6/google-pubsub-source.md b/connectors/google-pubsub-source/v3.0.1.6/google-pubsub-source.md index 796335cd6..6b6734bbe 100644 --- a/connectors/google-pubsub-source/v3.0.1.6/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v3.0.1.6/google-pubsub-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud Pub/Sub Source Connector features: [""] diff --git a/connectors/google-pubsub-source/v3.1.0.6/google-pubsub-source.md b/connectors/google-pubsub-source/v3.1.0.6/google-pubsub-source.md index e35e5ca4f..4f609a742 100644 --- a/connectors/google-pubsub-source/v3.1.0.6/google-pubsub-source.md +++ b/connectors/google-pubsub-source/v3.1.0.6/google-pubsub-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Google Cloud Pub/Sub Source Connector features: [""] diff --git a/connectors/snowflake-sink/v2.10.5.9/snowflake-sink.md b/connectors/snowflake-sink/v2.10.5.9/snowflake-sink.md index d39b1c2b9..238dee89b 100644 --- a/connectors/snowflake-sink/v2.10.5.9/snowflake-sink.md +++ b/connectors/snowflake-sink/v2.10.5.9/snowflake-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Snowflake Sink Connector features: [""] diff --git a/connectors/snowflake-sink/v3.1.0.6/snowflake-sink.md b/connectors/snowflake-sink/v3.1.0.6/snowflake-sink.md index dd3cc8444..d977c276f 100644 --- a/connectors/snowflake-sink/v3.1.0.6/snowflake-sink.md +++ b/connectors/snowflake-sink/v3.1.0.6/snowflake-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: Snowflake Sink Connector features: [""] diff --git a/connectors/sqs-sink/v2.10.5.9/sqs-sink.md b/connectors/sqs-sink/v2.10.5.9/sqs-sink.md index 53bd40c67..629b1c5e1 100644 --- a/connectors/sqs-sink/v2.10.5.9/sqs-sink.md +++ b/connectors/sqs-sink/v2.10.5.9/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-sink/v2.11.2.12/sqs-sink.md b/connectors/sqs-sink/v2.11.2.12/sqs-sink.md index aef6151f4..6b5ac72a8 100644 --- a/connectors/sqs-sink/v2.11.2.12/sqs-sink.md +++ b/connectors/sqs-sink/v2.11.2.12/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-sink/v2.7.4.8/sqs-sink.md b/connectors/sqs-sink/v2.7.4.8/sqs-sink.md index c8c7b7d71..ae33d9407 100644 --- a/connectors/sqs-sink/v2.7.4.8/sqs-sink.md +++ b/connectors/sqs-sink/v2.7.4.8/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-sink/v2.8.4.2/sqs-sink.md b/connectors/sqs-sink/v2.8.4.2/sqs-sink.md index 5ed020533..87916c856 100644 --- a/connectors/sqs-sink/v2.8.4.2/sqs-sink.md +++ b/connectors/sqs-sink/v2.8.4.2/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-sink/v2.9.5.4/sqs-sink.md b/connectors/sqs-sink/v2.9.5.4/sqs-sink.md index 3e46b1c7d..5f51c9789 100644 --- a/connectors/sqs-sink/v2.9.5.4/sqs-sink.md +++ b/connectors/sqs-sink/v2.9.5.4/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-sink/v3.0.1.6/sqs-sink.md b/connectors/sqs-sink/v3.0.1.6/sqs-sink.md index 8f20bb293..3151577fa 100644 --- a/connectors/sqs-sink/v3.0.1.6/sqs-sink.md +++ b/connectors/sqs-sink/v3.0.1.6/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-sink/v3.1.0.6/sqs-sink.md b/connectors/sqs-sink/v3.1.0.6/sqs-sink.md index b5074790d..a1cc36856 100644 --- a/connectors/sqs-sink/v3.1.0.6/sqs-sink.md +++ b/connectors/sqs-sink/v3.1.0.6/sqs-sink.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Sink Connector features: [""] diff --git a/connectors/sqs-source/v2.10.5.9/sqs-source.md b/connectors/sqs-source/v2.10.5.9/sqs-source.md index e42046b94..0aa8bb5a8 100644 --- a/connectors/sqs-source/v2.10.5.9/sqs-source.md +++ b/connectors/sqs-source/v2.10.5.9/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Source Connector features: [""] diff --git a/connectors/sqs-source/v2.11.2.12/sqs-source.md b/connectors/sqs-source/v2.11.2.12/sqs-source.md index 787ed2e7b..ed977d778 100644 --- a/connectors/sqs-source/v2.11.2.12/sqs-source.md +++ b/connectors/sqs-source/v2.11.2.12/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Source Connector features: [""] diff --git a/connectors/sqs-source/v2.7.4.8/sqs-source.md b/connectors/sqs-source/v2.7.4.8/sqs-source.md index 0f076878e..4eaeedc52 100644 --- a/connectors/sqs-source/v2.7.4.8/sqs-source.md +++ b/connectors/sqs-source/v2.7.4.8/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Source Connector features: [""] diff --git a/connectors/sqs-source/v2.8.4.2/sqs-source.md b/connectors/sqs-source/v2.8.4.2/sqs-source.md index 5de6b2868..5e0c634e2 100644 --- a/connectors/sqs-source/v2.8.4.2/sqs-source.md +++ b/connectors/sqs-source/v2.8.4.2/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Source Connector features: [""] diff --git a/connectors/sqs-source/v2.9.5.4/sqs-source.md b/connectors/sqs-source/v2.9.5.4/sqs-source.md index 112f08684..c2b5b8aea 100644 --- a/connectors/sqs-source/v2.9.5.4/sqs-source.md +++ b/connectors/sqs-source/v2.9.5.4/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Source Connector features: [""] diff --git a/connectors/sqs-source/v3.0.1.6/sqs-source.md b/connectors/sqs-source/v3.0.1.6/sqs-source.md index 575c77208..81986f429 100644 --- a/connectors/sqs-source/v3.0.1.6/sqs-source.md +++ b/connectors/sqs-source/v3.0.1.6/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Source Connector features: [""] diff --git a/connectors/sqs-source/v3.1.0.6/sqs-source.md b/connectors/sqs-source/v3.1.0.6/sqs-source.md index 4ae74e7e7..53996b749 100644 --- a/connectors/sqs-source/v3.1.0.6/sqs-source.md +++ b/connectors/sqs-source/v3.1.0.6/sqs-source.md @@ -6,7 +6,7 @@ language: Java,Shell,Python,Dockerfile document: source: Private source license: Business License -license_link: "" +license_link: tags: alias: AWS SQS Source Connector features: [""] diff --git a/hub.template b/hub.template index 8eeb2a040..1c71c8000 100644 --- a/hub.template +++ b/hub.template @@ -6,7 +6,7 @@ language: <%= languages %> document: source: <%= source %> license: <%= license %> -license_link: "<%= licenseLink %>" +license_link: <%= licenseLink %> tags: <%= tags %> alias: <%= alias %> features: ["<%= features %>"] From 59b747e40027fbacfe5ec9f9c079428698520fbe Mon Sep 17 00:00:00 2001 From: Baodi Shi Date: Mon, 20 Nov 2023 09:54:21 +0800 Subject: [PATCH 5/5] Fix private source. --- connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md | 2 +- connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md | 2 +- connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md b/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md index aa9db0119..6b8a9ad59 100644 --- a/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v2.11.2.12/aws-lambda-sink.md @@ -4,7 +4,7 @@ author: freeznet,nlu90,Anonymitaet,Technoboy- contributors: freeznet,nlu90,Anonymitaet,Technoboy- language: Java,Shell,Python,Dockerfile document: -source: "Private source" +source: Private source license: Business License license_link: tags: diff --git a/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md b/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md index 2d0ec9bdf..35159b70a 100644 --- a/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v2.9.5.4/aws-lambda-sink.md @@ -4,7 +4,7 @@ author: freeznet,nlu90,Anonymitaet,Technoboy- contributors: freeznet,nlu90,Anonymitaet,Technoboy- language: Java,Shell,Python,Dockerfile document: -source: "Private source" +source: Private source license: Business License license_link: tags: diff --git a/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md b/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md index fdf98f37e..34617751d 100644 --- a/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md +++ b/connectors/aws-lambda-sink/v3.0.1.6/aws-lambda-sink.md @@ -4,7 +4,7 @@ author: freeznet,nlu90,Anonymitaet,Technoboy- contributors: freeznet,nlu90,Anonymitaet,Technoboy- language: Java,Shell,Python,Dockerfile document: -source: "Private source" +source: Private source license: Business License license_link: tags: