From 0a42932db237c0f53f777503d3eaafa97425aad4 Mon Sep 17 00:00:00 2001 From: jipeli <54889677+jipeli@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:22:26 +0800 Subject: [PATCH] (feature) (headless) tag api (#769) --- .../headless/api/pojo/TagDefineParams.java | 9 + .../api/pojo/enums/TagDefineType.java | 8 + .../headless/api/pojo/enums/TagType.java | 27 +++ .../api/pojo/request/QueryStructReq.java | 1 + .../api/pojo/request/QueryTagReq.java | 58 +++++- .../headless/api/pojo/request/TagReq.java | 33 ++++ .../api/pojo/response/SemanticSchemaResp.java | 1 + .../headless/api/pojo/response/TagResp.java | 29 +++ .../server/aspect/DimValueAspect.java | 24 ++- .../server/manager/ModelYamlManager.java | 1 + .../server/manager/SemanticSchemaManager.java | 59 ++++++- .../server/persistence/dataobject/TagDO.java | 79 +++++++++ .../persistence/mapper/TagCustomMapper.java | 11 ++ .../server/persistence/mapper/TagMapper.java | 10 ++ .../persistence/repository/TagRepository.java | 18 ++ .../repository/impl/TagRepositoryImpl.java | 44 +++++ .../headless/server/pojo/TagFilter.java | 13 ++ .../headless/server/rest/TagController.java | 59 +++++++ .../rest/api/TagQueryApiController.java | 11 +- .../headless/server/service/TagService.java | 20 +++ .../server/service/impl/QueryServiceImpl.java | 24 ++- .../service/impl/SchemaServiceImpl.java | 22 ++- .../server/service/impl/TagServiceImpl.java | 165 ++++++++++++++++++ .../headless/server/utils/StatUtils.java | 54 +++++- .../server/utils/TagReqConverter.java | 97 ++++++++++ .../mapper/custom/TagCustomMapper.xml | 110 ++++++++++++ .../resources/config.update/sql-update.sql | 22 ++- .../src/main/resources/db/schema-h2.sql | 22 ++- .../src/main/resources/db/schema-mysql.sql | 23 ++- 29 files changed, 1017 insertions(+), 37 deletions(-) create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagDefineParams.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagDefineType.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagType.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagReq.java create mode 100644 headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagResp.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagDO.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagCustomMapper.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagMapper.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/TagRepository.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagRepositoryImpl.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagFilter.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagService.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java create mode 100644 headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java create mode 100644 headless/server/src/main/resources/mapper/custom/TagCustomMapper.xml diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagDefineParams.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagDefineParams.java new file mode 100644 index 0000000000..f6f3fb08c0 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/TagDefineParams.java @@ -0,0 +1,9 @@ +package com.tencent.supersonic.headless.api.pojo; + +import lombok.Data; + +@Data +public class TagDefineParams { + + private String expr; +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagDefineType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagDefineType.java new file mode 100644 index 0000000000..b876606b87 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagDefineType.java @@ -0,0 +1,8 @@ +package com.tencent.supersonic.headless.api.pojo.enums; + +public enum TagDefineType { + + FIELD, + DIMENSION, + Tag +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagType.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagType.java new file mode 100644 index 0000000000..fa8d0e7402 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/enums/TagType.java @@ -0,0 +1,27 @@ +package com.tencent.supersonic.headless.api.pojo.enums; + +import java.util.Objects; + +public enum TagType { + ATOMIC, + DERIVED; + + public static TagType of(String src) { + for (TagType tagType : TagType.values()) { + if (Objects.nonNull(src) && src.equalsIgnoreCase(tagType.name())) { + return tagType; + } + } + return null; + } + + public static Boolean isDerived(String src) { + TagType tagType = of(src); + return Objects.nonNull(tagType) && tagType.equals(DERIVED); + } + + public static TagType getType(TagDefineType tagDefineType) { + return Objects.nonNull(tagDefineType) && TagDefineType.Tag.equals(tagDefineType) ? TagType.DERIVED + : TagType.ATOMIC; + } +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java index 470a2afdf6..a122087a91 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryStructReq.java @@ -1,5 +1,6 @@ package com.tencent.supersonic.headless.api.pojo.request; + import com.google.common.collect.Lists; import com.tencent.supersonic.common.pojo.Aggregator; import com.tencent.supersonic.common.pojo.Constants; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java index d73c9eb93d..be1a6f3af0 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/QueryTagReq.java @@ -1,18 +1,64 @@ package com.tencent.supersonic.headless.api.pojo.request; +import com.google.common.collect.Lists; +import com.tencent.supersonic.common.pojo.Aggregator; +import com.tencent.supersonic.common.pojo.DateConf; +import com.tencent.supersonic.common.pojo.Filter; +import com.tencent.supersonic.common.pojo.Order; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import lombok.Data; import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; @Data +@Slf4j @ToString -public class QueryTagReq { +public class QueryTagReq extends SemanticQueryReq { - private Long domainId; + private List groups = new ArrayList<>(); + private List aggregators = new ArrayList<>(); + private List tagFilters = new ArrayList<>(); + private List orders = new ArrayList<>(); - private List tagIds; + private Long limit = 20L; + private Long offset = 0L; - private List tagNames; + private String tagFiltersDate; + private DateConf dateInfo; - private Long limit = 2000L; -} \ No newline at end of file + @Override + public String toCustomizedString() { + StringBuilder stringBuilder = new StringBuilder("{"); + stringBuilder.append("\"viewId\":") + .append(viewId); + stringBuilder.append("\"modelIds\":") + .append(modelIds); + stringBuilder.append(",\"groups\":") + .append(groups); + stringBuilder.append(",\"aggregators\":") + .append(aggregators); + stringBuilder.append(",\"orders\":") + .append(orders); + stringBuilder.append(",\"tagFilters\":") + .append(tagFilters); + stringBuilder.append(",\"dateInfo\":") + .append(dateInfo); + stringBuilder.append(",\"params\":") + .append(params); + stringBuilder.append(",\"limit\":") + .append(limit); + stringBuilder.append('}'); + return stringBuilder.toString(); + } + + public List getMetrics() { + List metrics = Lists.newArrayList(); + if (!CollectionUtils.isEmpty(this.aggregators)) { + metrics = aggregators.stream().map(Aggregator::getColumn).collect(Collectors.toList()); + } + return metrics; + } +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagReq.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagReq.java new file mode 100644 index 0000000000..e03a958904 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/request/TagReq.java @@ -0,0 +1,33 @@ +package com.tencent.supersonic.headless.api.pojo.request; + +import com.alibaba.fastjson.JSONObject; +import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.TagDefineParams; +import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; +import com.tencent.supersonic.headless.api.pojo.enums.TagType; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import lombok.Data; + +@Data +public class TagReq extends SchemaItem { + + private Long modelId; + private Map ext = new HashMap<>(); + private TagDefineType tagDefineType; + private TagDefineParams tagDefineParams; + + public String getTypeParamsJson() { + return JSONObject.toJSONString(tagDefineParams); + } + + public String getExtJson() { + return Objects.nonNull(ext) && ext.size() > 0 ? JSONObject.toJSONString(ext) : ""; + } + + public TagType getType() { + return TagType.getType(tagDefineType); + } + +} diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java index 543a071b3a..d333e51b46 100644 --- a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/SemanticSchemaResp.java @@ -22,6 +22,7 @@ public class SemanticSchemaResp { private SchemaType schemaType; private List metrics = Lists.newArrayList(); private List dimensions = Lists.newArrayList(); + private List tags = Lists.newArrayList(); private List modelRelas = Lists.newArrayList(); private List modelResps = Lists.newArrayList(); private ViewResp viewResp; diff --git a/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagResp.java b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagResp.java new file mode 100644 index 0000000000..6857330fa3 --- /dev/null +++ b/headless/api/src/main/java/com/tencent/supersonic/headless/api/pojo/response/TagResp.java @@ -0,0 +1,29 @@ +package com.tencent.supersonic.headless.api.pojo.response; + +import com.tencent.supersonic.headless.api.pojo.SchemaItem; +import com.tencent.supersonic.headless.api.pojo.TagDefineParams; +import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; +import java.util.HashMap; +import java.util.Map; +import lombok.Data; +import lombok.ToString; + +@Data +@ToString(callSuper = true) +public class TagResp extends SchemaItem { + + private Long modelId; + + private String type; + + private Map ext = new HashMap<>(); + + private TagDefineType tagDefineType = TagDefineType.FIELD; + + private TagDefineParams tagDefineParams; + + public String getExpr() { + return tagDefineParams.getExpr(); + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java index afc48d73d6..86be574a11 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/aspect/DimValueAspect.java @@ -13,11 +13,19 @@ import com.tencent.supersonic.headless.api.pojo.SchemaItem; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.DimensionResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; import com.tencent.supersonic.headless.server.service.DimensionService; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.util.Strings; @@ -29,14 +37,6 @@ import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; - @Aspect @Component @Slf4j @@ -63,9 +63,17 @@ public Object handleDimValue(ProceedingJoinPoint joinPoint) throws Throwable { if (queryReq instanceof QuerySqlReq) { return handleSqlDimValue(joinPoint); } + + if (queryReq instanceof QueryTagReq) { + return handleTagValue(joinPoint); + } throw new InvalidArgumentException("queryReq is not Invalid:" + queryReq); } + public Object handleTagValue(ProceedingJoinPoint joinPoint) throws Throwable { + return (SemanticQueryResp) joinPoint.proceed(); + } + private SemanticQueryResp handleStructDimValue(ProceedingJoinPoint joinPoint) throws Throwable { Object[] args = joinPoint.getArgs(); QueryStructReq queryStructReq = (QueryStructReq) args[0]; diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java index 333f608d46..975fae4ee5 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/ModelYamlManager.java @@ -52,6 +52,7 @@ public static synchronized DataModelYamlTpl convert2YamlObj(ModelResp modelResp, dataModelYamlTpl.setTableQuery(modelDetail.getTableQuery()); } dataModelYamlTpl.setFields(modelResp.getModelDetail().getFields()); + dataModelYamlTpl.setId(modelResp.getId()); return dataModelYamlTpl; } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java index d35cabff67..07953c41f3 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/manager/SemanticSchemaManager.java @@ -5,6 +5,7 @@ import com.tencent.supersonic.headless.api.pojo.Field; import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.Constants; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataSource; import com.tencent.supersonic.headless.core.parser.calcite.s2sql.DataType; @@ -29,11 +30,6 @@ import com.tencent.supersonic.headless.server.pojo.yaml.MetricYamlTpl; import com.tencent.supersonic.headless.server.service.Catalog; import com.tencent.supersonic.headless.server.utils.DatabaseConverter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Triple; -import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; - import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -44,11 +40,16 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Triple; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; @Slf4j @Service public class SemanticSchemaManager { + private final Catalog catalog; public SemanticSchemaManager(Catalog catalog) { @@ -87,6 +88,54 @@ public SemanticModel getSemanticModel(SemanticSchemaResp semanticSchemaResp) { return semanticModel; } + public SemanticModel getTagSemanticModel(SemanticSchemaResp semanticSchemaResp) throws Exception { + if (CollectionUtils.isEmpty(semanticSchemaResp.getTags())) { + throw new Exception("semanticSchemaResp tag is empty"); + } + SemanticModel semanticModel = getSemanticModel(semanticSchemaResp); + //Map> dimensions = new HashMap<>(); + Map> tagMap = new HashMap<>(); + for (TagResp tagResp : semanticSchemaResp.getTags()) { + if (!tagMap.containsKey(tagResp.getModelId())) { + tagMap.put(tagResp.getModelId(), new ArrayList<>()); + } + tagMap.get(tagResp.getModelId()).add(tagResp); + } + if (Objects.nonNull(semanticModel.getDatasourceMap()) && !semanticModel.getDatasourceMap().isEmpty()) { + for (Map.Entry entry : semanticModel.getDatasourceMap().entrySet()) { + List dimensions = new ArrayList<>(); + List tagNames = new ArrayList<>(); + if (tagMap.containsKey(entry.getValue().getId())) { + for (TagResp tagResp : tagMap.get(entry.getValue().getId())) { + tagNames.add(tagResp.getBizName()); + Dimension dimension = Dimension.builder().build(); + dimension.setType(""); + dimension.setExpr(tagResp.getExpr()); + dimension.setName(tagResp.getBizName()); + dimension.setOwners(""); + dimension.setBizName(tagResp.getBizName()); + if (Objects.isNull(dimension.getDataType())) { + dimension.setDataType(DataType.UNKNOWN); + } + DimensionTimeTypeParams dimensionTimeTypeParams = new DimensionTimeTypeParams(); + dimension.setDimensionTimeTypeParams(dimensionTimeTypeParams); + dimensions.add(dimension); + } + } + if (semanticModel.getDimensionMap().containsKey(entry.getKey())) { + semanticModel.getDimensionMap().get(entry.getKey()).stream() + .filter(d -> !tagNames.contains(d.getBizName())).forEach(d -> { + dimensions.add(d); + }); + } + semanticModel.getDimensionMap().put(entry.getKey(), dimensions); + } + } + // metric ignored + semanticModel.setMetrics(new ArrayList<>()); + return semanticModel; + } + public static List getMetrics(final List t) { return getMetricsByMetricYamlTpl(t); } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagDO.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagDO.java new file mode 100644 index 0000000000..e0863bc171 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/dataobject/TagDO.java @@ -0,0 +1,79 @@ +package com.tencent.supersonic.headless.server.persistence.dataobject; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import java.util.Date; +import lombok.Data; + +@Data +@TableName("s2_tag") +public class TagDO { + + @TableId(type = IdType.AUTO) + private Long id; + + /** + * 主体域ID + */ + private Long modelId; + + /** + * 指标名称 + */ + private String name; + + /** + * 字段名称 + */ + private String bizName; + + /** + * 描述 + */ + private String description; + + /** + * 指标状态,0正常,1下架,2删除 + */ + private Integer status; + + /** + * 敏感级别 + */ + private Integer sensitiveLevel; + + /** + * 类型 DERIVED,ATOMIC + */ + private String type; + + /** + * 创建时间 + */ + private Date createdAt; + + /** + * 创建人 + */ + private String createdBy; + + /** + * 更新时间 + */ + private Date updatedAt; + + /** + * 更新人 + */ + private String updatedBy; + + + /** + * 类型参数 + */ + private String defineType; + private String typeParams; + private String ext; + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagCustomMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagCustomMapper.java new file mode 100644 index 0000000000..f9344602ef --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagCustomMapper.java @@ -0,0 +1,11 @@ +package com.tencent.supersonic.headless.server.persistence.mapper; + +import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO; +import com.tencent.supersonic.headless.server.pojo.TagFilter; +import java.util.List; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface TagCustomMapper { + List query(TagFilter tagFilter); +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagMapper.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagMapper.java new file mode 100644 index 0000000000..1222762e35 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/mapper/TagMapper.java @@ -0,0 +1,10 @@ +package com.tencent.supersonic.headless.server.persistence.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface TagMapper extends BaseMapper { + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/TagRepository.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/TagRepository.java new file mode 100644 index 0000000000..99d27eeffb --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/TagRepository.java @@ -0,0 +1,18 @@ +package com.tencent.supersonic.headless.server.persistence.repository; + + +import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO; +import com.tencent.supersonic.headless.server.pojo.TagFilter; +import java.util.List; + + +public interface TagRepository { + + Long create(TagDO tagDO); + + void update(TagDO tagDO); + + TagDO getTagById(Long id); + + List query(TagFilter tagFilter); +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagRepositoryImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagRepositoryImpl.java new file mode 100644 index 0000000000..5445b74610 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/persistence/repository/impl/TagRepositoryImpl.java @@ -0,0 +1,44 @@ +package com.tencent.supersonic.headless.server.persistence.repository.impl; + +import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO; +import com.tencent.supersonic.headless.server.persistence.mapper.TagCustomMapper; +import com.tencent.supersonic.headless.server.persistence.mapper.TagMapper; +import com.tencent.supersonic.headless.server.persistence.repository.TagRepository; +import com.tencent.supersonic.headless.server.pojo.TagFilter; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Repository; + +@Slf4j +@Repository +public class TagRepositoryImpl implements TagRepository { + private final TagMapper mapper; + private final TagCustomMapper tagCustomMapper; + + public TagRepositoryImpl(TagMapper mapper, + TagCustomMapper tagCustomMapper) { + this.mapper = mapper; + this.tagCustomMapper = tagCustomMapper; + } + + @Override + public Long create(TagDO tagDO) { + mapper.insert(tagDO); + return tagDO.getId(); + } + + @Override + public void update(TagDO tagDO) { + mapper.updateById(tagDO); + } + + @Override + public TagDO getTagById(Long id) { + return mapper.selectById(id); + } + + @Override + public List query(TagFilter tagFilter) { + return tagCustomMapper.query(tagFilter); + } +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagFilter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagFilter.java new file mode 100644 index 0000000000..e37bd906e9 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/pojo/TagFilter.java @@ -0,0 +1,13 @@ +package com.tencent.supersonic.headless.server.pojo; + + +import java.util.List; +import lombok.Data; + +@Data +public class TagFilter extends MetaFilter { + + private String type; + private List statusList; + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java new file mode 100644 index 0000000000..4388b1c636 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/TagController.java @@ -0,0 +1,59 @@ +package com.tencent.supersonic.headless.server.rest; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; +import com.tencent.supersonic.headless.api.pojo.request.TagReq; +import com.tencent.supersonic.headless.api.pojo.response.TagResp; +import com.tencent.supersonic.headless.server.service.TagService; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/semantic/tag") +public class TagController { + + private final TagService tagService; + public TagController(TagService tagService) { + this.tagService = tagService; + } + + @PostMapping("/create") + public TagResp create(@RequestBody TagReq tagReq, + HttpServletRequest request, + HttpServletResponse response) throws Exception { + User user = UserHolder.findUser(request, response); + return tagService.create(tagReq, user); + } + + @PostMapping("/update") + public TagResp update(@RequestBody TagReq tagReq, + HttpServletRequest request, + HttpServletResponse response) throws Exception { + User user = UserHolder.findUser(request, response); + return tagService.update(tagReq, user); + } + + @DeleteMapping("delete/{id}") + public Boolean delete(@PathVariable("id") Long id, + HttpServletRequest request, + HttpServletResponse response) throws Exception { + User user = UserHolder.findUser(request, response); + tagService.delete(id, user); + return true; + } + + @GetMapping("getTag/{id}") + public TagResp getTag(@PathVariable("id") Long id, + HttpServletRequest request, + HttpServletResponse response) { + return tagService.getTag(id); + } + +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java index fa7ca8fefd..5e4d4cb360 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/rest/api/TagQueryApiController.java @@ -1,9 +1,13 @@ package com.tencent.supersonic.headless.server.rest.api; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.auth.api.authentication.utils.UserHolder; import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; +import com.tencent.supersonic.headless.server.service.QueryService; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -14,12 +18,15 @@ @Slf4j public class TagQueryApiController { + @Autowired + private QueryService queryService; + @PostMapping("/tag") public Object queryByTag(@RequestBody QueryTagReq queryTagReq, HttpServletRequest request, HttpServletResponse response) throws Exception { - //TODO - return null; + User user = UserHolder.findUser(request, response); + return queryService.queryByReq(queryTagReq, user); } } diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagService.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagService.java new file mode 100644 index 0000000000..5724530e99 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/TagService.java @@ -0,0 +1,20 @@ +package com.tencent.supersonic.headless.server.service; + +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.headless.api.pojo.request.TagReq; +import com.tencent.supersonic.headless.api.pojo.response.TagResp; +import com.tencent.supersonic.headless.server.pojo.TagFilter; +import java.util.List; + +public interface TagService { + + TagResp create(TagReq tagReq, User user) throws Exception; + + TagResp update(TagReq tagReq, User user) throws Exception; + + void delete(Long id, User user) throws Exception; + + TagResp getTag(Long id); + + List query(TagFilter tagFilter); +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java index a3bd5bdfb8..2971915f9f 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/QueryServiceImpl.java @@ -21,6 +21,7 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.api.pojo.request.SchemaFilterReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.AppDetailResp; @@ -49,6 +50,7 @@ import com.tencent.supersonic.headless.server.utils.QueryReqConverter; import com.tencent.supersonic.headless.server.utils.QueryUtils; import com.tencent.supersonic.headless.server.utils.StatUtils; +import com.tencent.supersonic.headless.server.utils.TagReqConverter; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -69,6 +71,7 @@ public class QueryServiceImpl implements QueryService { private StatUtils statUtils; private final QueryUtils queryUtils; private final QueryReqConverter queryReqConverter; + private final TagReqConverter tagReqConverter; private final Catalog catalog; private final AppService appService; private final QueryCache queryCache; @@ -80,7 +83,7 @@ public QueryServiceImpl( StatUtils statUtils, QueryUtils queryUtils, QueryReqConverter queryReqConverter, - Catalog catalog, + TagReqConverter tagReqConverter, Catalog catalog, AppService appService, QueryCache queryCache, SemanticSchemaManager semanticSchemaManager, @@ -89,6 +92,7 @@ public QueryServiceImpl( this.statUtils = statUtils; this.queryUtils = queryUtils; this.queryReqConverter = queryReqConverter; + this.tagReqConverter = tagReqConverter; this.catalog = catalog; this.appService = appService; this.queryCache = queryCache; @@ -157,6 +161,9 @@ private QueryStatement buildQueryStatement(SemanticQueryReq semanticQueryReq) th if (semanticQueryReq instanceof QueryMultiStructReq) { return buildMultiStructQueryStatement((QueryMultiStructReq) semanticQueryReq); } + if (semanticQueryReq instanceof QueryTagReq) { + return buildTagQueryStatement((QueryTagReq) semanticQueryReq); + } return null; } @@ -192,6 +199,21 @@ private QueryStatement buildMultiStructQueryStatement(QueryMultiStructReq queryM return queryUtils.sqlParserUnion(queryMultiStructReq, sqlParsers); } + private QueryStatement buildTagQueryStatement(QueryTagReq queryTagReq) + throws Exception { + SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); + SchemaFilterReq filter = buildSchemaFilterReq(queryTagReq); + schemaFilterReq.setModelIds(queryTagReq.getModelIds()); + SemanticSchemaResp semanticSchemaResp = catalog.fetchSemanticSchema(filter); + QueryStatement queryStatement = tagReqConverter.convert(queryTagReq, semanticSchemaResp); + queryStatement.setModelIds(queryTagReq.getModelIds()); + queryStatement.setEnableOptimize(queryUtils.enableOptimize()); + queryStatement.setSemanticSchemaResp(semanticSchemaResp); + SemanticModel semanticModel = semanticSchemaManager.getTagSemanticModel(semanticSchemaResp); + queryStatement.setSemanticModel(semanticModel); + return queryStatement; + } + private SchemaFilterReq buildSchemaFilterReq(SemanticQueryReq semanticQueryReq) { SchemaFilterReq schemaFilterReq = new SchemaFilterReq(); schemaFilterReq.setViewId(semanticQueryReq.getViewId()); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java index a1cf5160fa..d900d96c6c 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/SchemaServiceImpl.java @@ -29,15 +29,18 @@ import com.tencent.supersonic.headless.api.pojo.response.ModelResp; import com.tencent.supersonic.headless.api.pojo.response.ModelSchemaResp; import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.api.pojo.response.TagResp; import com.tencent.supersonic.headless.api.pojo.response.ViewResp; import com.tencent.supersonic.headless.api.pojo.response.ViewSchemaResp; import com.tencent.supersonic.headless.server.pojo.MetaFilter; +import com.tencent.supersonic.headless.server.pojo.TagFilter; import com.tencent.supersonic.headless.server.service.DimensionService; import com.tencent.supersonic.headless.server.service.DomainService; import com.tencent.supersonic.headless.server.service.MetricService; import com.tencent.supersonic.headless.server.service.ModelRelaService; import com.tencent.supersonic.headless.server.service.ModelService; import com.tencent.supersonic.headless.server.service.SchemaService; +import com.tencent.supersonic.headless.server.service.TagService; import com.tencent.supersonic.headless.server.service.ViewService; import com.tencent.supersonic.headless.server.utils.DimensionConverter; import com.tencent.supersonic.headless.server.utils.MetricConverter; @@ -78,14 +81,15 @@ public class SchemaServiceImpl implements SchemaService { private final DomainService domainService; private final ViewService viewService; private final ModelRelaService modelRelaService; + private final TagService tagService; public SchemaServiceImpl(ModelService modelService, - DimensionService dimensionService, - MetricService metricService, - DomainService domainService, - ViewService viewService, - ModelRelaService modelRelaService, - StatUtils statUtils) { + DimensionService dimensionService, + MetricService metricService, + DomainService domainService, + ViewService viewService, + ModelRelaService modelRelaService, + StatUtils statUtils, TagService tagService) { this.modelService = modelService; this.dimensionService = dimensionService; this.metricService = metricService; @@ -93,6 +97,7 @@ public SchemaServiceImpl(ModelService modelService, this.viewService = viewService; this.modelRelaService = modelRelaService; this.statUtils = statUtils; + this.tagService = tagService; } @SneakyThrows @@ -301,6 +306,11 @@ public SemanticSchemaResp buildSemanticSchema(SchemaFilterReq schemaFilterReq) { .flatMap(Collection::stream).collect(Collectors.toList())); semanticSchemaResp.setModelResps(modelSchemaResps.stream().map(this::convert).collect(Collectors.toList())); semanticSchemaResp.setSchemaType(SchemaType.MODEL); + // add tag info + TagFilter tagFilter = new TagFilter(); + tagFilter.setModelIds(schemaFilterReq.getModelIds()); + List tagResps = tagService.query(tagFilter); + semanticSchemaResp.setTags(tagResps); } if (!CollectionUtils.isEmpty(semanticSchemaResp.getModelIds())) { DatabaseResp databaseResp = modelService.getDatabaseByModelId(semanticSchemaResp.getModelIds().get(0)); diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java new file mode 100644 index 0000000000..7babbb66b7 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/service/impl/TagServiceImpl.java @@ -0,0 +1,165 @@ +package com.tencent.supersonic.headless.server.service.impl; + +import com.alibaba.fastjson.JSONObject; +import com.tencent.supersonic.auth.api.authentication.pojo.User; +import com.tencent.supersonic.common.pojo.enums.StatusEnum; +import com.tencent.supersonic.common.pojo.exception.InvalidArgumentException; +import com.tencent.supersonic.headless.api.pojo.TagDefineParams; +import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType; +import com.tencent.supersonic.headless.api.pojo.request.TagReq; +import com.tencent.supersonic.headless.api.pojo.response.TagResp; +import com.tencent.supersonic.headless.server.persistence.dataobject.TagDO; +import com.tencent.supersonic.headless.server.persistence.repository.TagRepository; +import com.tencent.supersonic.headless.server.pojo.TagFilter; +import com.tencent.supersonic.headless.server.service.TagService; +import com.tencent.supersonic.headless.server.utils.NameCheckUtils; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class TagServiceImpl implements TagService { + + private final TagRepository tagRepository; + + public TagServiceImpl(TagRepository tagRepository) { + this.tagRepository = tagRepository; + } + + @Override + public TagResp create(TagReq tagReq, User user) throws Exception { + checkParam(tagReq); + checkExit(tagReq); + TagDO tagDO = convert(tagReq); + tagDO.setCreatedBy(user.getName()); + tagDO.setCreatedAt(new Date()); + tagDO.setStatus(StatusEnum.ONLINE.getCode()); + tagRepository.create(tagDO); + return convert(tagDO); + } + + @Override + public TagResp update(TagReq tagReq, User user) throws Exception { + if (Objects.isNull(tagReq.getId()) || tagReq.getId() <= 0) { + throw new RuntimeException("id is empty"); + } + TagDO tagDO = tagRepository.getTagById(tagReq.getId()); + if (Objects.nonNull(tagDO) && tagDO.getId() > 0) { + if (Objects.nonNull(tagReq.getExt()) && !tagReq.getExt().isEmpty()) { + tagDO.setExt(tagReq.getExtJson()); + } + } + if (Objects.nonNull(tagReq.getTagDefineType())) { + tagDO.setDefineType(tagReq.getTagDefineType().name()); + } + if (Objects.nonNull(tagReq.getTagDefineParams()) && !StringUtils.isBlank( + tagReq.getTagDefineParams().getExpr())) { + tagDO.setTypeParams(tagReq.getTypeParamsJson()); + } + tagDO.setUpdatedBy(user.getName()); + tagDO.setUpdatedAt(new Date()); + tagRepository.update(tagDO); + return convert(tagDO); + } + + @Override + public void delete(Long id, User user) throws Exception { + TagDO tagDO = tagRepository.getTagById(id); + if (Objects.isNull(tagDO)) { + throw new RuntimeException("tag not found"); + } + tagDO.setStatus(StatusEnum.DELETED.getCode()); + tagDO.setUpdatedBy(user.getName()); + tagDO.setUpdatedAt(new Date()); + tagRepository.update(tagDO); + } + + @Override + public TagResp getTag(Long id) { + return convert(tagRepository.getTagById(id)); + } + + @Override + public List query(TagFilter tagFilter) { + List tagDOS = tagRepository.query(tagFilter); + if (!CollectionUtils.isEmpty(tagDOS)) { + return tagDOS.stream().map(tagDO -> convert(tagDO)).collect(Collectors.toList()); + } + return new ArrayList<>(); + } + + private void checkExit(TagReq tagReq) { + TagFilter tagFilter = new TagFilter(); + tagFilter.setModelIds(Arrays.asList(tagReq.getModelId())); + //tagFilter.setStatusList(Arrays.asList(StatusEnum.ONLINE.getCode(),StatusEnum.OFFLINE.getCode())); + List tagResps = query(tagFilter); + if (!CollectionUtils.isEmpty(tagResps)) { + Long bizNameSameCount = tagResps.stream().filter(tagResp -> !tagResp.getId().equals(tagReq.getId())) + .filter(tagResp -> tagResp.getBizName().equalsIgnoreCase(tagReq.getBizName())).count(); + if (bizNameSameCount > 0) { + throw new RuntimeException(String.format("the bizName %s is exit", tagReq.getBizName())); + } + Long nameSameCount = tagResps.stream().filter(tagResp -> !tagResp.getId().equals(tagReq.getId())) + .filter(tagResp -> tagResp.getName().equalsIgnoreCase(tagReq.getName())).count(); + if (nameSameCount > 0) { + throw new RuntimeException(String.format("the name %s is exit", tagReq.getName())); + } + } + } + + private void checkParam(TagReq tagReq) { + if (Objects.isNull(tagReq.getModelId()) || tagReq.getModelId() <= 0) { + throw new RuntimeException("the modelId is empty"); + } + if (Objects.isNull(tagReq.getBizName()) || tagReq.getBizName().isEmpty() || Objects.isNull(tagReq.getName()) + || tagReq.getName().isEmpty()) { + throw new RuntimeException("the bizName or name is empty"); + } + if (Objects.isNull(tagReq.getTagDefineType()) || Objects.isNull(tagReq.getTagDefineParams()) + || StringUtils.isBlank(tagReq.getTagDefineParams().getExpr())) { + throw new InvalidArgumentException("表达式不可为空"); + } + + if (NameCheckUtils.containsSpecialCharacters(tagReq.getBizName())) { + throw new InvalidArgumentException("名称包含特殊字符, 请修改"); + } + } + + private TagResp convert(TagDO tagDO) { + TagResp tagResp = new TagResp(); + BeanUtils.copyProperties(tagDO, tagResp); + if (Objects.nonNull(tagDO.getExt()) && !tagDO.getExt().isEmpty()) { + Map ext = JSONObject.parseObject(tagDO.getExt(), + Map.class); + tagResp.setExt(ext); + } + tagResp.setTagDefineType(TagDefineType.valueOf(tagDO.getDefineType())); + if (Objects.nonNull(tagDO.getTypeParams()) && !tagDO.getTypeParams().isEmpty()) { + TagDefineParams tagDefineParams = JSONObject.parseObject(tagDO.getTypeParams(), + TagDefineParams.class); + tagResp.setTagDefineParams(tagDefineParams); + } + + return tagResp; + } + + private TagDO convert(TagReq tagReq) { + TagDO tagDO = new TagDO(); + BeanUtils.copyProperties(tagReq, tagDO); + tagDO.setDefineType(tagReq.getTagDefineType().name()); + tagDO.setType(tagReq.getType().name()); + tagDO.setTypeParams(tagReq.getTypeParamsJson()); + tagDO.setExt(tagReq.getExtJson()); + return tagDO; + } +} diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java index 8f65682b60..25399c801d 100644 --- a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/StatUtils.java @@ -16,22 +16,22 @@ import com.tencent.supersonic.headless.api.pojo.request.QueryMultiStructReq; import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; import com.tencent.supersonic.headless.api.pojo.request.SemanticQueryReq; import com.tencent.supersonic.headless.api.pojo.response.ItemUseResp; import com.tencent.supersonic.headless.server.persistence.repository.StatRepository; import com.tencent.supersonic.headless.server.service.ModelService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.logging.log4j.util.Strings; -import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; - import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.logging.log4j.util.Strings; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; @Component @@ -96,6 +96,48 @@ public void initStatInfo(SemanticQueryReq semanticQueryReq, User facadeUser) { QueryStructReq queryStructCmd = ((QueryMultiStructReq) semanticQueryReq).getQueryStructReqs().get(0); initStructStatInfo(queryStructCmd, facadeUser); } + if (semanticQueryReq instanceof QueryTagReq) { + initTagStatInfo((QueryTagReq) semanticQueryReq, facadeUser); + } + } + + public void initTagStatInfo(QueryTagReq queryTagReq, User facadeUser) { + QueryStat queryStatInfo = new QueryStat(); + String traceId = ""; + List dimensions = queryTagReq.getGroups(); + + List metrics = new ArrayList<>(); + queryTagReq.getAggregators().stream().forEach(aggregator -> metrics.add(aggregator.getColumn())); + String user = getUserName(facadeUser); + + try { + queryStatInfo.setTraceId(traceId) + .setViewId(queryTagReq.getViewId()) + .setUser(user) + .setQueryType(QueryType.STRUCT.getValue()) + .setQueryTypeBack(QueryTypeBack.NORMAL.getState()) + .setQueryStructCmd(queryTagReq.toString()) + .setQueryStructCmdMd5(DigestUtils.md5Hex(queryTagReq.toString())) + .setStartTime(System.currentTimeMillis()) + .setNativeQuery(CollectionUtils.isEmpty(queryTagReq.getAggregators())) + .setGroupByCols(objectMapper.writeValueAsString(queryTagReq.getGroups())) + .setAggCols(objectMapper.writeValueAsString(queryTagReq.getAggregators())) + .setOrderByCols(objectMapper.writeValueAsString(queryTagReq.getOrders())) + .setFilterCols(objectMapper.writeValueAsString( + sqlFilterUtils.getFiltersCol(queryTagReq.getTagFilters()))) + .setUseResultCache(true) + .setUseSqlCache(true) + .setMetrics(objectMapper.writeValueAsString(metrics)) + .setDimensions(objectMapper.writeValueAsString(dimensions)) + .setQueryOptMode(QueryOptMode.NONE.name()); + if (!CollectionUtils.isEmpty(queryTagReq.getModelIds())) { + queryStatInfo.setModelId(queryTagReq.getModelIds().get(0)); + } + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + StatUtils.set(queryStatInfo); + } public void initSqlStatInfo(QuerySqlReq querySqlReq, User facadeUser) { diff --git a/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java new file mode 100644 index 0000000000..6fb8070a79 --- /dev/null +++ b/headless/server/src/main/java/com/tencent/supersonic/headless/server/utils/TagReqConverter.java @@ -0,0 +1,97 @@ +package com.tencent.supersonic.headless.server.utils; + +import com.tencent.supersonic.common.pojo.enums.QueryType; +import com.tencent.supersonic.common.util.jsqlparser.SqlSelectHelper; +import com.tencent.supersonic.headless.api.pojo.MetricTable; +import com.tencent.supersonic.headless.api.pojo.QueryParam; +import com.tencent.supersonic.headless.api.pojo.enums.AggOption; +import com.tencent.supersonic.headless.api.pojo.enums.EngineType; +import com.tencent.supersonic.headless.api.pojo.request.QuerySqlReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryStructReq; +import com.tencent.supersonic.headless.api.pojo.request.QueryTagReq; +import com.tencent.supersonic.headless.api.pojo.response.DatabaseResp; +import com.tencent.supersonic.headless.api.pojo.response.SemanticSchemaResp; +import com.tencent.supersonic.headless.core.pojo.QueryStatement; +import com.tencent.supersonic.headless.core.pojo.ViewQueryParam; +import com.tencent.supersonic.headless.core.utils.SqlGenerateUtils; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class TagReqConverter { + + @Value("${query.sql.limitWrapper:true}") + private Boolean limitWrapper; + + @Autowired + private QueryStructUtils queryStructUtils; + + @Autowired + private SqlGenerateUtils sqlGenerateUtils; + + public QueryStatement convert(QueryTagReq queryTagReq, + SemanticSchemaResp semanticSchemaResp) throws Exception { + QueryStatement queryStatement = new QueryStatement(); + // covert to QueryReqConverter + QueryStructReq queryStructReq = new QueryStructReq(); + BeanUtils.copyProperties(queryTagReq, queryStructReq); + if (!CollectionUtils.isEmpty(queryTagReq.getTagFilters())) { + queryStructReq.setDimensionFilters(queryTagReq.getTagFilters()); + } + QuerySqlReq querySqlReq = queryStructReq.convert(); + if (Objects.nonNull(querySqlReq)) { + log.info("convert to QuerySqlReq {}", querySqlReq); + String tableName = SqlSelectHelper.getTableName(querySqlReq.getSql()); + MetricTable metricTable = new MetricTable(); + metricTable.setMetrics(new ArrayList<>()); + metricTable.getMetrics().add(sqlGenerateUtils.generateInternalMetricName( + semanticSchemaResp.getModelResps().get(0).getBizName())); + metricTable.setAggOption(AggOption.NATIVE); + List allFields = SqlSelectHelper.getAllFields(querySqlReq.getSql()); + metricTable.setDimensions(allFields); + metricTable.setAlias(tableName.toLowerCase()); + List tables = new ArrayList<>(); + tables.add(metricTable); + //.build ParseSqlReq + ViewQueryParam result = new ViewQueryParam(); + BeanUtils.copyProperties(querySqlReq, result); + result.setTables(tables); + DatabaseResp database = semanticSchemaResp.getDatabaseResp(); + if (!sqlGenerateUtils.isSupportWith(EngineType.fromString(database.getType().toUpperCase()), + database.getVersion())) { + result.setSupportWith(false); + result.setWithAlias(false); + } + //.physicalSql by ParseSqlReq + queryStructReq.setDateInfo(queryStructUtils.getDateConfBySql(querySqlReq.getSql())); + queryStructReq.setViewId(querySqlReq.getViewId()); + queryStructReq.setQueryType(QueryType.TAG); + QueryParam queryParam = new QueryParam(); + convert(queryTagReq, queryParam); + queryStatement.setQueryParam(queryParam); + queryStatement.setViewQueryParam(result); + queryStatement.setIsS2SQL(true); + queryStatement.setMinMaxTime(queryStructUtils.getBeginEndTime(queryStructReq)); + queryStatement.setViewId(queryTagReq.getViewId()); + queryStatement.setEnableLimitWrapper(limitWrapper); + } + return queryStatement; + } + + public void convert(QueryTagReq queryTagReq, QueryParam queryParam) { + BeanUtils.copyProperties(queryTagReq, queryParam); + queryParam.setOrders(queryTagReq.getOrders()); + queryParam.setMetrics(queryTagReq.getMetrics()); + queryParam.setGroups(queryTagReq.getGroups()); + queryParam.setDimensionFilters(queryTagReq.getTagFilters()); + queryParam.setQueryType(QueryType.TAG); + } +} diff --git a/headless/server/src/main/resources/mapper/custom/TagCustomMapper.xml b/headless/server/src/main/resources/mapper/custom/TagCustomMapper.xml new file mode 100644 index 0000000000..339aefbb71 --- /dev/null +++ b/headless/server/src/main/resources/mapper/custom/TagCustomMapper.xml @@ -0,0 +1,110 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + and ${criterion.condition} + + + and ${criterion.condition} #{criterion.value} + + + and ${criterion.condition} #{criterion.value} and + #{criterion.secondValue} + + + and ${criterion.condition} + + #{listItem} + + + + + + + + + + + id, model_id, name, biz_name, description, status, sensitive_level, type, created_at, + created_by, updated_at, updated_by, define_type + + + type_params + + + + + + diff --git a/launchers/standalone/src/main/resources/config.update/sql-update.sql b/launchers/standalone/src/main/resources/config.update/sql-update.sql index 865e3735e5..3e5c8a9167 100644 --- a/launchers/standalone/src/main/resources/config.update/sql-update.sql +++ b/launchers/standalone/src/main/resources/config.update/sql-update.sql @@ -192,4 +192,24 @@ CREATE TABLE s2_view( alter table s2_plugin change column model `view` varchar(100); alter table s2_view_info rename to s2_canvas; -alter table s2_query_stat_info add column `view_id` bigint(20) DEFAULT NULL after `model_id`; \ No newline at end of file +alter table s2_query_stat_info add column `view_id` bigint(20) DEFAULT NULL after `model_id`; + +--20240221 +CREATE TABLE s2_tag( + `id` INT NOT NULL AUTO_INCREMENT, + `model_id` INT NOT NULL , + `name` varchar(255) NOT NULL , + `biz_name` varchar(255) NOT NULL , + `description` varchar(500) DEFAULT NULL , + `status` INT NOT NULL , + `sensitive_level` INT NOT NULL , + `type` varchar(50) NOT NULL , -- ATOMIC, DERIVED + `define_type` varchar(50) NOT NULL, -- FIELD, DIMENSION + `type_params` LONGVARCHAR DEFAULT NULL , + `created_at` TIMESTAMP NOT NULL , + `created_by` varchar(100) NOT NULL , + `updated_at` TIMESTAMP DEFAULT NULL , + `updated_by` varchar(100) DEFAULT NULL , + `ext` LONGVARCHAR DEFAULT NULL , + PRIMARY KEY (`id`) +)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/schema-h2.sql b/launchers/standalone/src/main/resources/db/schema-h2.sql index da180da0e3..21a4154f07 100644 --- a/launchers/standalone/src/main/resources/db/schema-h2.sql +++ b/launchers/standalone/src/main/resources/db/schema-h2.sql @@ -572,4 +572,24 @@ CREATE TABLE IF NOT EXISTS `s2_view` ( query_config VARCHAR(3000), `admin` varchar(3000) DEFAULT NULL, `admin_org` varchar(3000) DEFAULT NULL -); \ No newline at end of file +); + +CREATE TABLE IF NOT EXISTS `s2_tag` ( + `id` INT NOT NULL AUTO_INCREMENT, + `model_id` INT NOT NULL , + `name` varchar(255) NOT NULL , + `biz_name` varchar(255) NOT NULL , + `description` varchar(500) DEFAULT NULL , + `status` INT NOT NULL , + `sensitive_level` INT NOT NULL , + `type` varchar(50) NOT NULL , -- ATOMIC, DERIVED + `define_type` varchar(50) NOT NULL, -- FIELD, DIMENSION + `type_params` LONGVARCHAR DEFAULT NULL , + `created_at` TIMESTAMP NOT NULL , + `created_by` varchar(100) NOT NULL , + `updated_at` TIMESTAMP DEFAULT NULL , + `updated_by` varchar(100) DEFAULT NULL , + `ext` LONGVARCHAR DEFAULT NULL , + PRIMARY KEY (`id`) + ); +COMMENT ON TABLE s2_tag IS 'tag information'; \ No newline at end of file diff --git a/launchers/standalone/src/main/resources/db/schema-mysql.sql b/launchers/standalone/src/main/resources/db/schema-mysql.sql index 614d6c9a52..a8e38fad58 100644 --- a/launchers/standalone/src/main/resources/db/schema-mysql.sql +++ b/launchers/standalone/src/main/resources/db/schema-mysql.sql @@ -497,4 +497,25 @@ CREATE TABLE s2_view query_config VARCHAR(3000), `admin` varchar(3000) DEFAULT NULL, `admin_org` varchar(3000) DEFAULT NULL -)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; \ No newline at end of file +)ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +CREATE TABLE `s2_tag` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `model_id` bigint(20) DEFAULT NULL, + `name` varchar(255) NOT NULL COMMENT '名称', + `biz_name` varchar(255) NOT NULL COMMENT '英文名称', + `description` varchar(500) DEFAULT NULL COMMENT '描述', + `status` int(10) NOT NULL COMMENT '状态', + `sensitive_level` int(10) NOT NULL COMMENT '敏感级别', + `type` varchar(50) NOT NULL COMMENT '类型(DERIVED,ATOMIC)', + `define_type` varchar(50) DEFAULT NULL, -- FIELD, DIMENSION + `type_params` text NOT NULL COMMENT '类型参数', + `created_at` datetime NOT NULL COMMENT '创建时间', + `created_by` varchar(100) NOT NULL COMMENT '创建人', + `updated_at` datetime NULL COMMENT '更新时间', + `updated_by` varchar(100) NULL COMMENT '更新人', + `ext` text DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8 COMMENT ='标签表'; \ No newline at end of file