[INLONG-10758][Manager] Support obtaining serialization configuration…

… when wrapType is raw (#10759)
apache · Aug 7, 2024 · 076f894 · 076f894
1 parent 0b2646a
commit 076f894
Show file tree

Hide file tree

Showing 6 changed files with 45 additions and 8 deletions.
diff --git a/inlong-common/src/main/java/org/apache/inlong/common/constant/DeserializationType.java b/inlong-common/src/main/java/org/apache/inlong/common/constant/DeserializationType.java
@@ -19,8 +19,9 @@
 
 public class DeserializationType {
 
-    public static final String INLONG_MSG = "INLONG_MSG";
-    public static final String INLONG_MSG_PB = "INLONG_MSG_PB";
-    public static final String CSV = "CSV";
-    public static final String KV = "KV";
+    public static final String INLONG_MSG = "inlong_msg";
+    public static final String INLONG_MSG_PB = "inlong_msg_pb";
+    public static final String RAW = "raw";
+    public static final String CSV = "csv";
+    public static final String KV = "kv";
 }
diff --git a/...va/org/apache/inlong/common/pojo/sort/dataflow/deserialization/DeserializationConfig.java b/...va/org/apache/inlong/common/pojo/sort/dataflow/deserialization/DeserializationConfig.java
@@ -28,6 +28,7 @@
 @JsonSubTypes({
         @JsonSubTypes.Type(value = InlongMsgDeserializationConfig.class, name = DeserializationType.INLONG_MSG),
         @JsonSubTypes.Type(value = InlongMsgPbDeserialiationConfig.class, name = DeserializationType.INLONG_MSG_PB),
+        @JsonSubTypes.Type(value = RawDeserializationConfig.class, name = DeserializationType.RAW),
 })
 public interface DeserializationConfig extends Serializable {
 

diff --git a/...org/apache/inlong/common/pojo/sort/dataflow/deserialization/RawDeserializationConfig.java b/...org/apache/inlong/common/pojo/sort/dataflow/deserialization/RawDeserializationConfig.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.common.pojo.sort.dataflow.deserialization;
+
+import lombok.Data;
+
+@Data
+public class RawDeserializationConfig implements DeserializationConfig {
+
+    private String streamId;
+}
diff --git a/...ce/src/main/java/org/apache/inlong/manager/service/message/RawMsgDeserializeOperator.java b/...ce/src/main/java/org/apache/inlong/manager/service/message/RawMsgDeserializeOperator.java
@@ -20,6 +20,8 @@
 import org.apache.inlong.common.enums.DataTypeEnum;
 import org.apache.inlong.common.enums.MessageWrapType;
 import org.apache.inlong.common.msg.AttributeConstants;
+import org.apache.inlong.common.pojo.sort.dataflow.deserialization.DeserializationConfig;
+import org.apache.inlong.common.pojo.sort.dataflow.deserialization.RawDeserializationConfig;
 import org.apache.inlong.manager.common.exceptions.BusinessException;
 import org.apache.inlong.manager.pojo.consume.BriefMQMessage;
 import org.apache.inlong.manager.pojo.consume.BriefMQMessage.FieldInfo;
@@ -82,4 +84,11 @@ public List<BriefMQMessage> decodeMsg(InlongStreamInfo streamInfo, List<BriefMQM
         }
     }
 
+    @Override
+    public DeserializationConfig getDeserializationConfig(InlongStreamInfo streamInfo) {
+        RawDeserializationConfig rawDeserializationConfig = new RawDeserializationConfig();
+        rawDeserializationConfig.setStreamId(streamInfo.getInlongStreamId());
+        return rawDeserializationConfig;
+    }
+
 }
diff --git a/inlong-sort-standalone/sort-standalone-source/src/test/java/SortConfig.conf b/inlong-sort-standalone/sort-standalone-source/src/test/java/SortConfig.conf
@@ -45,11 +45,11 @@
                                 "subscription": "sid_es_v3",
                                 "encodingType": "UTF-8",
                                 "deserializationConfig": {
-                                    "type": "INLONG_MSG",
+                                    "type": "inlong_msg",
                                     "streamId": "0fc00000046"
                                 },
                                 "dataTypeConfig": {
-                                    "type": "CSV",
+                                    "type": "csv",
                                     "delimiter": "|",
                                     "escapeChar": "\\"
                                 },

diff --git a/inlong-sort-standalone/sort-standalone-source/src/test/resources/SortConfig.conf b/inlong-sort-standalone/sort-standalone-source/src/test/resources/SortConfig.conf
@@ -45,11 +45,11 @@
                                 "subscription": "sid_es_v3",
                                 "encodingType": "UTF-8",
                                 "deserializationConfig": {
-                                    "type": "INLONG_MSG",
+                                    "type": "inlong_msg",
                                     "streamId": "0fc00000046"
                                 },
                                 "dataTypeConfig": {
-                                    "type": "CSV",
+                                    "type": "csv",
                                     "delimiter": "|",
                                     "escapeChar": "\\"
                                 },