(feature) (headless) tag api (#769)

tencentmusic · Feb 27, 2024 · 0a42932 · 0a42932
1 parent 3b4678d
commit 0a42932
Show file tree

Hide file tree

Showing 29 changed files with 1,017 additions and 37 deletions.
diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagDefineParams.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagDefineParams.java
@@ -0,0 +1,9 @@
+package com.tencent.supersonic.headless.api.pojo;
+
+import lombok.Data;
+
+@Data
+public class TagDefineParams {
+
+    private String expr;
+}
diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagDefineType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagDefineType.java
@@ -0,0 +1,8 @@
+package com.tencent.supersonic.headless.api.pojo.enums;
+
+public enum TagDefineType {
+
+    FIELD,
+    DIMENSION,
+    Tag
+}
diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagType.java
@@ -0,0 +1,27 @@
+package com.tencent.supersonic.headless.api.pojo.enums;
+
+import java.util.Objects;
+
+public enum TagType {
+    ATOMIC,
+    DERIVED;
+
+    public static TagType of(String src) {
+        for (TagType tagType : TagType.values()) {
+            if (Objects.nonNull(src) && src.equalsIgnoreCase(tagType.name())) {
+                return tagType;
+            }
+        }
+        return null;
+    }
+
+    public static Boolean isDerived(String src) {
+        TagType tagType = of(src);
+        return Objects.nonNull(tagType) && tagType.equals(DERIVED);
+    }
+
+    public static TagType getType(TagDefineType tagDefineType) {
+        return Objects.nonNull(tagDefineType) && TagDefineType.Tag.equals(tagDefineType) ? TagType.DERIVED
+                : TagType.ATOMIC;
+    }
+}
diff --git a/...ss/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/...ss/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java
@@ -1,5 +1,6 @@
 package com.tencent.supersonic.headless.api.pojo.request;
 
+
 import com.google.common.collect.Lists;
 import com.tencent.supersonic.common.pojo.Aggregator;
 import com.tencent.supersonic.common.pojo.Constants;

diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java
@@ -1,18 +1,64 @@
 package com.tencent.supersonic.headless.api.pojo.request;
 
+import com.google.common.collect.Lists;
+import com.tencent.supersonic.common.pojo.Aggregator;
+import com.tencent.supersonic.common.pojo.DateConf;
+import com.tencent.supersonic.common.pojo.Filter;
+import com.tencent.supersonic.common.pojo.Order;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 import lombok.Data;
 import lombok.ToString;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.util.CollectionUtils;
 
 @Data
+@Slf4j
 @ToString
-public class QueryTagReq {
+public class QueryTagReq extends SemanticQueryReq {
 
-    private Long domainId;
+    private List<String> groups = new ArrayList<>();
+    private List<Aggregator> aggregators = new ArrayList<>();
+    private List<Filter> tagFilters = new ArrayList<>();
+    private List<Order> orders = new ArrayList<>();
 
-    private List<Long> tagIds;
+    private Long limit = 20L;
+    private Long offset = 0L;
 
-    private List<String> tagNames;
+    private String tagFiltersDate;
+    private DateConf dateInfo;
 
-    private Long limit = 2000L;
-}
+    @Override
+    public String toCustomizedString() {
+        StringBuilder stringBuilder = new StringBuilder("{");
+        stringBuilder.append("\"viewId\":")
+                .append(viewId);
+        stringBuilder.append("\"modelIds\":")
+                .append(modelIds);
+        stringBuilder.append(",\"groups\":")
+                .append(groups);
+        stringBuilder.append(",\"aggregators\":")
+                .append(aggregators);
+        stringBuilder.append(",\"orders\":")
+                .append(orders);
+        stringBuilder.append(",\"tagFilters\":")
+                .append(tagFilters);
+        stringBuilder.append(",\"dateInfo\":")
+                .append(dateInfo);
+        stringBuilder.append(",\"params\":")
+                .append(params);
+        stringBuilder.append(",\"limit\":")
+                .append(limit);
+        stringBuilder.append('}');
+        return stringBuilder.toString();
+    }
+
+    public List<String> getMetrics() {
+        List<String> metrics = Lists.newArrayList();
+        if (!CollectionUtils.isEmpty(this.aggregators)) {
+            metrics = aggregators.stream().map(Aggregator::getColumn).collect(Collectors.toList());
+        }
+        return metrics;
+    }
+}
diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagReq.java
@@ -0,0 +1,33 @@
+package com.tencent.supersonic.headless.api.pojo.request;
+
+import com.alibaba.fastjson.JSONObject;
+import com.tencent.supersonic.headless.api.pojo.SchemaItem;
+import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
+import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
+import com.tencent.supersonic.headless.api.pojo.enums.TagType;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import lombok.Data;
+
+@Data
+public class TagReq extends SchemaItem {
+
+    private Long modelId;
+    private Map<String, Object> ext = new HashMap<>();
+    private TagDefineType tagDefineType;
+    private TagDefineParams tagDefineParams;
+
+    public String getTypeParamsJson() {
+        return JSONObject.toJSONString(tagDefineParams);
+    }
+
+    public String getExtJson() {
+        return Objects.nonNull(ext) && ext.size() > 0 ? JSONObject.toJSONString(ext) : "";
+    }
+
+    public TagType getType() {
+        return TagType.getType(tagDefineType);
+    }
+
+}
diff --git a/...i/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java b/...i/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java
@@ -22,6 +22,7 @@ public class SemanticSchemaResp {
     private SchemaType schemaType;
     private List<MetricSchemaResp> metrics = Lists.newArrayList();
     private List<DimSchemaResp> dimensions = Lists.newArrayList();
+    private List<TagResp> tags = Lists.newArrayList();
     private List<ModelRela> modelRelas = Lists.newArrayList();
     private List<ModelResp> modelResps = Lists.newArrayList();
     private ViewResp viewResp;

diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagResp.java
@@ -0,0 +1,29 @@
+package com.tencent.supersonic.headless.api.pojo.response;
+
+import com.tencent.supersonic.headless.api.pojo.SchemaItem;
+import com.tencent.supersonic.headless.api.pojo.TagDefineParams;
+import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
+import java.util.HashMap;
+import java.util.Map;
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString(callSuper = true)
+public class TagResp extends SchemaItem {
+
+    private Long modelId;
+
+    private String type;
+
+    private Map<String, Object> ext = new HashMap<>();
+
+    private TagDefineType tagDefineType = TagDefineType.FIELD;
+
+    private TagDefineParams tagDefineParams;
+
+    public String getExpr() {
+        return tagDefineParams.getExpr();
+    }
+
+}
diff --git a/...ss/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java b/...ss/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java
@@ -13,11 +13,19 @@
 import com.tencent.supersonic.headless.api.pojo.SchemaItem;
 import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq;
 import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq;
+import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq;
 import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq;
 import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
 import com.tencent.supersonic.headless.server.pojo.MetaFilter;
 import com.tencent.supersonic.headless.server.service.DimensionService;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.util.Strings;
@@ -29,14 +37,6 @@
 import org.springframework.stereotype.Component;
 import org.springframework.util.CollectionUtils;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
 @Aspect
 @Component
 @Slf4j
@@ -63,9 +63,17 @@ public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
         if (queryReq instanceof QuerySqlReq) {
             return handleSqlDimValue(joinPoint);
         }
+
+        if (queryReq instanceof QueryTagReq) {
+            return handleTagValue(joinPoint);
+        }
         throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq);
     }
 
+    public Object handleTagValue(ProceedingJoinPoint joinPoint) throws Throwable {
+        return (SemanticQueryResp) joinPoint.proceed();
+    }
+
     private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable {
         Object[] args = joinPoint.getArgs();
         QueryStructReq queryStructReq = (QueryStructReq) args[0];

diff --git a/...server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/...server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java
@@ -52,6 +52,7 @@ public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp,
             dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery());
         }
         dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields());
+        dataModelYamlTpl.setId(modelResp.getId());
         return dataModelYamlTpl;
     }
 

diff --git a/...r/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/...r/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java
@@ -5,6 +5,7 @@
 import com.tencent.supersonic.headless.api.pojo.Field;
 import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp;
 import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp;
+import com.tencent.supersonic.headless.api.pojo.response.TagResp;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource;
 import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType;
@@ -29,11 +30,6 @@
 import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl;
 import com.tencent.supersonic.headless.server.service.Catalog;
 import com.tencent.supersonic.headless.server.utils.DatabaseConverter;
-import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.tuple.Triple;
-import org.springframework.stereotype.Service;
-import org.springframework.util.CollectionUtils;
-
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -44,11 +40,16 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.tuple.Triple;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
 
 
 @Slf4j
 @Service
 public class SemanticSchemaManager {
+
     private final Catalog catalog;
 
     public SemanticSchemaManager(Catalog catalog) {
@@ -87,6 +88,54 @@ public SemanticModel getSemanticModel(SemanticSchemaResp semanticSchemaResp) {
         return semanticModel;
     }
 
+    public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception {
+        if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) {
+            throw new Exception("semanticSchemaResp tag is empty");
+        }
+        SemanticModel semanticModel = getSemanticModel(semanticSchemaResp);
+        //Map<String, List<Dimension>> dimensions = new HashMap<>();
+        Map<Long, List<TagResp>> tagMap = new HashMap<>();
+        for (TagResp tagResp : semanticSchemaResp.getTags()) {
+            if (!tagMap.containsKey(tagResp.getModelId())) {
+                tagMap.put(tagResp.getModelId(), new ArrayList<>());
+            }
+            tagMap.get(tagResp.getModelId()).add(tagResp);
+        }
+        if (Objects.nonNull(semanticModel.getDatasourceMap()) && !semanticModel.getDatasourceMap().isEmpty()) {
+            for (Map.Entry<String, DataSource> entry : semanticModel.getDatasourceMap().entrySet()) {
+                List<Dimension> dimensions = new ArrayList<>();
+                List<String> tagNames = new ArrayList<>();
+                if (tagMap.containsKey(entry.getValue().getId())) {
+                    for (TagResp tagResp : tagMap.get(entry.getValue().getId())) {
+                        tagNames.add(tagResp.getBizName());
+                        Dimension dimension = Dimension.builder().build();
+                        dimension.setType("");
+                        dimension.setExpr(tagResp.getExpr());
+                        dimension.setName(tagResp.getBizName());
+                        dimension.setOwners("");
+                        dimension.setBizName(tagResp.getBizName());
+                        if (Objects.isNull(dimension.getDataType())) {
+                            dimension.setDataType(DataType.UNKNOWN);
+                        }
+                        DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams();
+                        dimension.setDimensionTimeTypeParams(dimensionTimeTypeParams);
+                        dimensions.add(dimension);
+                    }
+                }
+                if (semanticModel.getDimensionMap().containsKey(entry.getKey())) {
+                    semanticModel.getDimensionMap().get(entry.getKey()).stream()
+                            .filter(d -> !tagNames.contains(d.getBizName())).forEach(d -> {
+                                dimensions.add(d);
+                            });
+                }
+                semanticModel.getDimensionMap().put(entry.getKey(), dimensions);
+            }
+        }
+        // metric ignored
+        semanticModel.setMetrics(new ArrayList<>());
+        return semanticModel;
+    }
+
     public static List<Metric> getMetrics(final List<MetricYamlTpl> t) {
         return getMetricsByMetricYamlTpl(t);
     }