[ISSUE #367] Refactor connector (#433)

* [ISSUE #367] Refactor connector
    1.support load plugin from eventMeshPluginDir
    2.remove connector plugin from runtime

* Add docs of plugin

* Remove connector-api in connector-plugin

build.gradle

@@ -188,45 +188,6 @@ subprojects {
         }
     }
 
-//    checkstyle {
-//        toolVersion = "8.32"
-//        ignoreFailures = true
-//        sourceSets = [sourceSets.main]
-//        configFile = '../style/codeStyle.xml' as File
-//        showViolations true
-//    }
-//
-//    tasks.withType(Checkstyle) {
-//        reports {
-//            xml.enabled false
-//            html.enabled true
-//        }
-//    }
-//
-//    sourceSets {
-//        main {
-//            java {
-//                srcDir 'src/main/java'
-//            }
-//
-//            resources {
-//                srcDir 'src/main/resources'
-//            }
-//
-//        }
-//
-//        test {
-//            java {
-//                srcDir 'src/test/java'
-//            }
-//
-//            resources {
-//                srcDir 'src/test/resources'
-//            }
-//
-//        }
-//    }
-
     spotbugs {
         //toolVersion = '4.2.3'
         ignoreFailures = true
@@ -271,14 +232,6 @@ subprojects {
         }
     }
 
-//        tasks.withType(Pmd) {
-//            reports {
-//                xml.enabled = false
-//                html.enabled = true
-//            }
-//        }
-
-
     pmd {
         consoleOutput = true
         toolVersion = "6.23.0"
@@ -314,6 +267,7 @@ subprojects {
                 from project.jar.getArchivePath()
                 exclude 'eventmesh-common*.jar'
                 exclude 'eventmesh-connector-api*.jar'
+                exclude 'eventmesh-connector-plugin*.jar'
                 exclude 'eventmesh-starter*.jar'
                 exclude 'eventmesh-test*.jar'
                 exclude 'eventmesh-sdk*.jar'

docs/cn/instructions/eventmesh-runtime-quickstart.zh-CN.md

@@ -20,7 +20,7 @@ Gradle至少为7.0, 推荐 7.0.*
 ```$ xslt
 unzip EventMesh-master.zip
 cd / *您的部署路径* /EventMesh-master
-gradle clean dist tar -x test
+gradle clean dist copyConnectorPlugin tar -x test
 ```
 
 您将在目录/ *您的部署路径* /EventMesh-master/eventmesh-runtime/dist中获得**eventmesh-runtime_1.0.0.tar.gz**
@@ -57,14 +57,20 @@ sh start.sh
 ![project-structure](../../images/project-structure.png)
 
 - eventmesh-common : eventmesh公共类与方法模块
-- eventmesh-connector-api : eventmesh插件接口定义模块
-- eventmesh-connector-rocketmq : eventmesh rocketmq插件模块
+- eventmesh-connector-api : eventmesh connector插件接口定义模块
+- eventmesh-connector-plugin : eventmesh connector插件模块
 - eventmesh-runtime : eventmesh运行时模块
 - eventmesh-sdk-java : eventmesh java客户端sdk
 - eventmesh-starter : eventmesh本地启动运行项目入口
 - eventmesh-spi : eventmesh SPI加载模块
 
-> 注：插件模块遵循eventmesh定义的spi机制，需要在对应模块中的/main/resources/META-INF/eventmesh 下配置相关接口与实现类的映射文件
+> 注：插件模块遵循eventmesh定义的SPI规范, 自定义的SPI接口需要使用注解@EventMeshSPI标识.
+> 插件实例需要在对应模块中的/main/resources/META-INF/eventmesh 下配置相关接口与实现类的映射文件,文件名为SPI接口全类名. 
+> 文件内容为插件实例名到插件实例的映射, 具体可以参考eventmesh-connector-rocketmq插件模块
+
+插件可以从classpath和插件目录下面加载. 在本地开发阶段可以将使用的插件在eventmesh-starter模块build.gradle中进行声明,或者执行gradle的copyConnectorPlugin任务
+将插件拷贝至dist/plugin目录下, eventmesh默认会加载项目下dist/plugin目录下的插件, 加载目录可以通过-DeventMeshPluginDir=your_plugin_directory来改变插件目录.
+运行时需要使用的插件实例可以在eventmesh.properties中进行配置
 
 **2.3.2 配置插件**
 

docs/en/instructions/eventmesh-runtime-quickstart.md

@@ -20,7 +20,7 @@ You will get **EventMesh-master.zip**
 ```$xslt
 unzip EventMesh-master.zip
 cd /*YOUR DEPLOY PATH*/EventMesh-master
-gradle clean dist tar -x test
+gradle clean dist copyConnectorPlugin tar -x test
 ```
 
 You will get **EventMesh_1.2.0.tar.gz** in directory /* YOUR DEPLOY PATH */EventMesh-master/build
@@ -58,14 +58,22 @@ Same with 1.2
 
 - eventmesh-common : eventmesh common classes and method module
 - eventmesh-connector-api : eventmesh connector api definition module
-- eventmesh-connector-rocketmq : eventmesh rocketmq connector module
+- eventmesh-connector-plugin : eventmesh connector plugin instance module
 - eventmesh-runtime : eventmesh runtime module
 - eventmesh-sdk-java : eventmesh java client sdk
 - eventmesh-starter : eventmesh project local start entry
 - eventmesh-spi : eventmesh SPI load module
 
-> ps: The loading of connector plugin follows the eventmesh SPI mechanism, it's necessary to configure the mapping file of
-related interface and implementation class under /main/resources/meta-inf/eventmesh in the corresponding module
+> ps: The plugin module follows the eventmesh SPI specification, custom SPI interface need to be identified with the @EventMeshSPI annotation.
+> The plugin instance needs to be configured in corresponding module under /main/resources/meta-inf/eventmesh with the mapping file of
+> related interface and implementation class. The content of the file is a mapping of plugin instance name to plugin instance, you can find more
+> detail in eventmesh-connector-rocketmq module
+
+The plugin can be loaded from classpath and plugin directory. In local develop, you can declare the used plugins in build.gradle of eventmesh-starter module,
+or execute copyConnectorPlugin task of gradle to copy the plugin instance jar to dist/plugin directory. By default, eventmesh will load the plugins in project's
+dist/plugin, this can be changed by add -DeventMeshPluginDir=your_plugin_directory.
+The plugin instance need to be used at runtime can be configured in eventmesh.properties.
+
 
 **2.3.2 Configure plugin**
 

eventmesh-connector-plugin/build.gradle

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+task copyConnectorPlugin(dependsOn: ['jar']) {
+    doFirst {
+        new File(projectDir, '../eventmesh-connector-plugin/dist/apps').mkdir()
+        new File(projectDir, '../dist/plugin/connector').mkdirs()
+    }
+    doLast {
+        copy {
+            into('../eventmesh-connector-plugin/dist/apps/')
+            from project.jar.getArchivePath()
+            exclude {
+                "eventmesh-connector-plugin-${version}.jar"
+                "eventmesh-connector-api-${version}.jar"
+            }
+        }
+        copy {
+            into '../dist/plugin/connector'
+            from "../eventmesh-connector-plugin/dist/apps/eventmesh-connector-rocketmq-${version}.jar"
+        }
+    }
+}

eventmesh-connector-api/build.gradle → eventmesh-connector-plugin/eventmesh-connector-api/build.gradle

@@ -20,6 +20,6 @@ List open_message = [
 ]
 
 dependencies {
-    implementation open_message,project(":eventmesh-common"), project(":eventmesh-spi")
+    api open_message, project(":eventmesh-common"), project(":eventmesh-spi")
     testImplementation open_message,project(":eventmesh-common"), project(":eventmesh-spi")
 }

eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/PluginFactory.java → eventmesh-connector-plugin/eventmesh-connector-api/src/main/java/org/apache/eventmesh/api/factory/ConnectorPluginFactory.java

@@ -17,18 +17,33 @@
  * under the License.
  */
 
-package org.apache.eventmesh.runtime.core.plugin;
+package org.apache.eventmesh.api.factory;
 
 import org.apache.eventmesh.api.consumer.MeshMQPushConsumer;
 import org.apache.eventmesh.api.producer.MeshMQProducer;
 import org.apache.eventmesh.spi.EventMeshExtensionFactory;
 
-public class PluginFactory {
+/**
+ * The factory to get connector {@link MeshMQProducer} and {@link MeshMQPushConsumer}
+ */
+public class ConnectorPluginFactory {
 
+    /**
+     * Get MeshMQProducer instance by plugin name
+     *
+     * @param connectorPluginName plugin name
+     * @return MeshMQProducer instance
+     */
     public static MeshMQProducer getMeshMQProducer(String connectorPluginName) {
         return EventMeshExtensionFactory.getExtension(MeshMQProducer.class, connectorPluginName);
     }
 
+    /**
+     * Get MeshMQPushConsumer instance by plugin name
+     *
+     * @param connectorPluginName plugin name
+     * @return MeshMQPushConsumer instance
+     */
     public static MeshMQPushConsumer getMeshMQPushConsumer(String connectorPluginName) {
         return EventMeshExtensionFactory.getExtension(MeshMQPushConsumer.class, connectorPluginName);
     }

eventmesh-connector-rocketmq/build.gradle → eventmesh-connector-plugin/eventmesh-connector-rocketmq/build.gradle

@@ -55,6 +55,6 @@ List open_message = [
 ]
 
 dependencies {
-    implementation rocketmq, metrics,open_message,project(":eventmesh-connector-api"),project(":eventmesh-common")
-    testImplementation rocketmq, metrics,open_message,project(":eventmesh-connector-api"),project(":eventmesh-common")
+    api rocketmq, metrics, open_message, project(":eventmesh-connector-plugin:eventmesh-connector-api")
+    testImplementation rocketmq, metrics, open_message, project(":eventmesh-connector-plugin:eventmesh-connector-api")
 }

eventmesh-connector-rocketmq/src/main/java/org/apache/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java → eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java

@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.rocketmq.client.impl.consumer;
+package org.apache.eventmesh.connector.rocketmq.client.impl.consumer;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -38,6 +38,9 @@
 import org.apache.rocketmq.client.consumer.listener.ConsumeReturnType;
 import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
 import org.apache.rocketmq.client.hook.ConsumeMessageContext;
+import org.apache.rocketmq.client.impl.consumer.ConsumeMessageService;
+import org.apache.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl;
+import org.apache.rocketmq.client.impl.consumer.ProcessQueue;
 import org.apache.rocketmq.client.log.ClientLogger;
 import org.apache.rocketmq.client.stat.ConsumerStatsManager;
 import org.apache.rocketmq.common.MixAll;

eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/config/ConfigurationWrapper.java → eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/config/ConfigurationWrapper.java

@@ -73,4 +73,4 @@ private void load() {
     public String getProp(String key) {
         return StringUtils.isEmpty(key) ? null : properties.getProperty(key, null);
     }
-}
+}

eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/consumer/RocketMQConsumerImpl.java → eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/consumer/RocketMQConsumerImpl.java

@@ -40,7 +40,7 @@
 import org.apache.eventmesh.connector.rocketmq.config.ConfigurationWrapper;
 import org.apache.eventmesh.connector.rocketmq.patch.EventMeshConsumeConcurrentlyContext;
 import org.apache.eventmesh.connector.rocketmq.utils.OMSUtil;
-import org.apache.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService;
+import org.apache.eventmesh.connector.rocketmq.client.impl.consumer.ConsumeMessageConcurrentlyService;
 import org.apache.rocketmq.client.impl.consumer.ConsumeMessageService;
 import org.apache.rocketmq.common.message.MessageExt;
 import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;

eventmesh-connector-rocketmq/src/test/java/org/apache/rocketmq/consumer/PushConsumerImplTest.java → eventmesh-connector-plugin/eventmesh-connector-rocketmq/src/test/java/org/apache/rocketmq/consumer/PushConsumerImplTest.java

@@ -23,7 +23,6 @@
 import java.util.Collections;
 import java.util.Properties;
 
-import io.openmessaging.api.Action;
 import io.openmessaging.api.AsyncConsumeContext;
 import io.openmessaging.api.AsyncMessageListener;
 import io.openmessaging.api.Consumer;

eventmesh-connector-rocketmq/src/main/resources/META-INF/services/org.apache.eventmesh.api.consumer.MeshMQPushConsumer → eventmesh-connector-plugin/gradle.properties

@@ -1,3 +1,4 @@
+#
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
@@ -12,5 +13,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-org.apache.eventmesh.connector.rocketmq.consumer.RocketMQConsumerImpl
+#
+group=org.apache.eventmesh
+version=1.2.0-SNAPSHOT
+jdk=1.8
+mavenUserName=
+mavenPassword=

eventmesh-runtime/build.gradle

@@ -31,6 +31,6 @@ List open_message = [
 
 
 dependencies {
-    implementation  metrics, open_message,project(":eventmesh-connector-api"),project(":eventmesh-common"),project(":eventmesh-spi")
-    testImplementation metrics,open_message,project(":eventmesh-common"),project(":eventmesh-connector-api"),project(":eventmesh-spi")
+    implementation metrics, open_message, project(":eventmesh-connector-plugin:eventmesh-connector-api")
+    testImplementation metrics, open_message, project(":eventmesh-connector-plugin:eventmesh-connector-api")
 }

eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/plugin/MQConsumerWrapper.java

@@ -19,13 +19,13 @@
 
 import java.util.List;
 import java.util.Properties;
-import java.util.ServiceLoader;
 
 import io.openmessaging.api.AsyncMessageListener;
 import io.openmessaging.api.Message;
 
 import org.apache.eventmesh.api.AbstractContext;
 import org.apache.eventmesh.api.consumer.MeshMQPushConsumer;
+import org.apache.eventmesh.api.factory.ConnectorPluginFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -36,7 +36,7 @@ public class MQConsumerWrapper extends MQWrapper {
     protected MeshMQPushConsumer meshMQPushConsumer;
 
     public MQConsumerWrapper(String connectorPluginType) {
-        this.meshMQPushConsumer = PluginFactory.getMeshMQPushConsumer(connectorPluginType);
+        this.meshMQPushConsumer = ConnectorPluginFactory.getMeshMQPushConsumer(connectorPluginType);
         if (meshMQPushConsumer == null) {
             logger.error("can't load the meshMQPushConsumer plugin, please check.");
             throw new RuntimeException("doesn't load the meshMQPushConsumer plugin, please check.");
@@ -52,24 +52,11 @@ public void unsubscribe(String topic) throws Exception {
     }
 
     public synchronized void init(Properties keyValue) throws Exception {
-        meshMQPushConsumer = getMeshMQPushConsumer();
-        if (meshMQPushConsumer == null) {
-            logger.error("can't load the meshMQPushConsumer plugin, please check.");
-            throw new RuntimeException("doesn't load the meshMQPushConsumer plugin, please check.");
-        }
 
         meshMQPushConsumer.init(keyValue);
         inited.compareAndSet(false, true);
     }
 
-    private MeshMQPushConsumer getMeshMQPushConsumer() {
-        ServiceLoader<MeshMQPushConsumer> meshMQPushConsumerServiceLoader = ServiceLoader.load(MeshMQPushConsumer.class);
-        if (meshMQPushConsumerServiceLoader.iterator().hasNext()) {
-            return meshMQPushConsumerServiceLoader.iterator().next();
-        }
-        return null;
-    }
-
     public synchronized void start() throws Exception {
         meshMQPushConsumer.start();
         started.compareAndSet(false, true);