Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add TagTest and correctDateConf logic #799

Merged
merged 15 commits into from
Mar 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ public class MetricResp extends SchemaItem {

private MetricDefineByMetricParams metricDefineByMetricParams;

private int isTag;

public void setTag(String tag) {
if (StringUtils.isBlank(tag)) {
tags = Lists.newArrayList();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@


import java.util.List;

import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import lombok.Data;

@Data
public class TagFilter extends MetaFilter {

private String type;
private List<Integer> statusList;
private TagDefineType tagDefineType;

}
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
package com.tencent.supersonic.headless.server.pojo;


import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.request.PageSchemaItemReq;

import java.util.List;

public class TagFilterPage extends PageSchemaItemReq {
private String type;
private List<Integer> statusList;
private TagDefineType tagDefineType;
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.tencent.supersonic.common.util.ChatGptHelper;
import com.tencent.supersonic.headless.api.pojo.DimValueMap;
import com.tencent.supersonic.headless.api.pojo.ModelDetail;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.request.DimensionReq;
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.pojo.request.PageDimensionReq;
Expand All @@ -26,16 +27,19 @@
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.SemanticQueryResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.DimensionDO;
import com.tencent.supersonic.headless.server.persistence.repository.DimensionRepository;
import com.tencent.supersonic.headless.server.pojo.DimensionFilter;
import com.tencent.supersonic.headless.server.pojo.DimensionsFilter;
import com.tencent.supersonic.headless.server.pojo.MetaFilter;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DatabaseService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.ModelRelaService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.tencent.supersonic.headless.server.utils.DimensionConverter;
import com.tencent.supersonic.headless.server.utils.NameCheckUtils;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -50,6 +54,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;

@Service
Expand All @@ -69,22 +74,26 @@ public class DimensionServiceImpl implements DimensionService {

private DataSetService dataSetService;

private TagMetaService tagMetaService;

@Autowired
private ApplicationEventPublisher eventPublisher;


public DimensionServiceImpl(DimensionRepository dimensionRepository,
ModelService modelService,
ChatGptHelper chatGptHelper,
DatabaseService databaseService,
ModelRelaService modelRelaService,
DataSetService dataSetService) {
ModelService modelService,
ChatGptHelper chatGptHelper,
DatabaseService databaseService,
ModelRelaService modelRelaService,
DataSetService dataSetService,
TagMetaService tagMetaService) {
this.modelService = modelService;
this.dimensionRepository = dimensionRepository;
this.chatGptHelper = chatGptHelper;
this.databaseService = databaseService;
this.modelRelaService = modelRelaService;
this.dataSetService = dataSetService;
this.tagMetaService = tagMetaService;
}

@Override
Expand Down Expand Up @@ -115,7 +124,7 @@ public void createDimensionBatch(List<DimensionReq> dimensionReqs, User user) {
return;
}
List<DimensionDO> dimensionDOS = dimensionToInsert.stream().peek(dimension ->
dimension.createdBy(user.getName()))
dimension.createdBy(user.getName()))
.map(DimensionConverter::convert2DimensionDO)
.collect(Collectors.toList());
dimensionRepository.createDimensionBatch(dimensionDOS);
Expand Down Expand Up @@ -206,7 +215,7 @@ public PageInfo<DimensionResp> queryDimension(PageDimensionReq pageDimensionReq)
BeanUtils.copyProperties(pageDimensionReq, dimensionFilter);
dimensionFilter.setModelIds(pageDimensionReq.getModelIds());
PageInfo<DimensionDO> dimensionDOPageInfo = PageHelper.startPage(pageDimensionReq.getCurrent(),
pageDimensionReq.getPageSize())
pageDimensionReq.getPageSize())
.doSelectPageInfo(() -> queryDimension(dimensionFilter));
PageInfo<DimensionResp> pageInfo = new PageInfo<>();
BeanUtils.copyProperties(dimensionDOPageInfo, pageInfo);
Expand Down Expand Up @@ -272,17 +281,36 @@ public List<DimensionResp> getDimensionInModelCluster(Long modelId) {
}

private List<DimensionResp> convertList(List<DimensionDO> dimensionDOS,
Map<Long, ModelResp> modelRespMap) {
Map<Long, ModelResp> modelRespMap) {
List<DimensionResp> dimensionResps = Lists.newArrayList();
if (!CollectionUtils.isEmpty(dimensionDOS)) {
dimensionResps = dimensionDOS.stream()
.map(dimensionDO -> DimensionConverter
.convert2DimensionResp(dimensionDO, modelRespMap))
.collect(Collectors.toList());
}
fillTagInfo(dimensionResps);
return dimensionResps;
}

private void fillTagInfo(List<DimensionResp> dimensionResps) {
if (CollectionUtils.isEmpty(dimensionResps)) {
return;
}
TagFilter tagFilter = new TagFilter();
tagFilter.setTagDefineType(TagDefineType.DIMENSION);
Map<String, TagResp> keyAndTagMap = tagMetaService.getTags(tagFilter).stream()
.collect(Collectors.toMap(tag -> tag.getModelId() + "_" + tag.getBizName(), tag -> tag));
if (Objects.nonNull(keyAndTagMap)) {
dimensionResps.stream().forEach(dim -> {
String key = dim.getModelId() + "_" + dim.getBizName();
if (keyAndTagMap.containsKey(key)) {
dim.setIsTag(1);
}
});
}
}

@Override
public List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user) {
String mockAlias = chatGptHelper.mockAlias(mockType, dimensionReq.getName(), dimensionReq.getBizName(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.tencent.supersonic.headless.api.pojo.MetricQueryDefaultConfig;
import com.tencent.supersonic.headless.api.pojo.SchemaItem;
import com.tencent.supersonic.headless.api.pojo.enums.MetricDefineType;
import com.tencent.supersonic.headless.api.pojo.enums.TagDefineType;
import com.tencent.supersonic.headless.api.pojo.request.MetaBatchReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricBaseReq;
import com.tencent.supersonic.headless.api.pojo.request.MetricReq;
Expand All @@ -33,6 +34,7 @@
import com.tencent.supersonic.headless.api.pojo.response.DimensionResp;
import com.tencent.supersonic.headless.api.pojo.response.MetricResp;
import com.tencent.supersonic.headless.api.pojo.response.ModelResp;
import com.tencent.supersonic.headless.api.pojo.response.TagResp;
import com.tencent.supersonic.headless.server.persistence.dataobject.CollectDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricDO;
import com.tencent.supersonic.headless.server.persistence.dataobject.MetricQueryDefaultConfigDO;
Expand All @@ -42,11 +44,13 @@
import com.tencent.supersonic.headless.server.pojo.MetricFilter;
import com.tencent.supersonic.headless.server.pojo.MetricsFilter;
import com.tencent.supersonic.headless.server.pojo.ModelCluster;
import com.tencent.supersonic.headless.server.pojo.TagFilter;
import com.tencent.supersonic.headless.server.service.CollectService;
import com.tencent.supersonic.headless.server.service.DataSetService;
import com.tencent.supersonic.headless.server.service.DimensionService;
import com.tencent.supersonic.headless.server.service.MetricService;
import com.tencent.supersonic.headless.server.service.ModelService;
import com.tencent.supersonic.headless.server.service.TagMetaService;
import com.tencent.supersonic.headless.server.utils.MetricCheckUtils;
import com.tencent.supersonic.headless.server.utils.MetricConverter;
import com.tencent.supersonic.headless.server.utils.ModelClusterBuilder;
Expand All @@ -64,6 +68,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;

Expand All @@ -85,20 +90,24 @@ public class MetricServiceImpl implements MetricService {

private ApplicationEventPublisher eventPublisher;

private TagMetaService tagMetaService;

public MetricServiceImpl(MetricRepository metricRepository,
ModelService modelService,
ChatGptHelper chatGptHelper,
CollectService collectService,
DataSetService dataSetService,
ApplicationEventPublisher eventPublisher,
DimensionService dimensionService) {
ModelService modelService,
ChatGptHelper chatGptHelper,
CollectService collectService,
DataSetService dataSetService,
ApplicationEventPublisher eventPublisher,
DimensionService dimensionService,
TagMetaService tagMetaService) {
this.metricRepository = metricRepository;
this.modelService = modelService;
this.chatGptHelper = chatGptHelper;
this.eventPublisher = eventPublisher;
this.collectService = collectService;
this.dataSetService = dataSetService;
this.dimensionService = dimensionService;
this.tagMetaService = tagMetaService;
}

@Override
Expand Down Expand Up @@ -211,12 +220,13 @@ public PageInfo<MetricResp> queryMetric(PageMetricReq pageMetricReq, User user)
}
}
PageInfo<MetricDO> metricDOPageInfo = PageHelper.startPage(pageMetricReq.getCurrent(),
pageMetricReq.getPageSize())
pageMetricReq.getPageSize())
.doSelectPageInfo(() -> queryMetric(metricFilter));
PageInfo<MetricResp> pageInfo = new PageInfo<>();
BeanUtils.copyProperties(metricDOPageInfo, pageInfo);
List<MetricResp> metricResps = convertList(metricDOPageInfo.getList(), collectIds);
fillAdminRes(metricResps, user);
fillTagInfo(metricResps);
pageInfo.setList(metricResps);
return pageInfo;
}
Expand All @@ -240,6 +250,24 @@ public List<MetricResp> getMetrics(MetaFilter metaFilter) {
return metricResps;
}

private void fillTagInfo(List<MetricResp> metricRespList) {
if (CollectionUtils.isEmpty(metricRespList)) {
return;
}
TagFilter tagFilter = new TagFilter();
tagFilter.setTagDefineType(TagDefineType.METRIC);
Map<String, TagResp> keyAndTagMap = tagMetaService.getTags(tagFilter).stream()
.collect(Collectors.toMap(tag -> tag.getModelId() + "_" + tag.getBizName(), tag -> tag));
if (Objects.nonNull(keyAndTagMap)) {
metricRespList.stream().forEach(metric -> {
String key = metric.getModelId() + "_" + metric.getBizName();
if (keyAndTagMap.containsKey(key)) {
metric.setIsTag(1);
}
});
}
}

private List<MetricResp> filterByField(List<MetricResp> metricResps, List<String> fields) {
Set<MetricResp> metricRespFiltered = Sets.newHashSet();
for (MetricResp metricResp : metricResps) {
Expand All @@ -249,7 +277,7 @@ private List<MetricResp> filterByField(List<MetricResp> metricResps, List<String
}

private boolean filterByField(List<MetricResp> metricResps, MetricResp metricResp,
List<String> fields, Set<MetricResp> metricRespFiltered) {
List<String> fields, Set<MetricResp> metricRespFiltered) {
if (MetricDefineType.METRIC.equals(metricResp.getMetricDefineType())) {
List<Long> ids = metricResp.getMetricDefineByMetricParams().getMetrics()
.stream().map(MetricParam::getId).collect(Collectors.toList());
Expand Down Expand Up @@ -287,8 +315,8 @@ public List<MetricResp> getMetricsToCreateNewMetric(Long modelId) {
metricFilter.setModelIds(Lists.newArrayList(modelId));
List<MetricResp> metricResps = getMetrics(metricFilter);
return metricResps.stream().filter(metricResp ->
MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())
|| MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType()))
MetricDefineType.FIELD.equals(metricResp.getMetricDefineType())
|| MetricDefineType.MEASURE.equals(metricResp.getMetricDefineType()))
.collect(Collectors.toList());
}

Expand Down Expand Up @@ -450,6 +478,7 @@ private List<MetricResp> convertList(List<MetricDO> metricDOS) {

private List<MetricResp> convertList(List<MetricDO> metricDOS, List<Long> collect) {
List<MetricResp> metricResps = Lists.newArrayList();

Map<Long, ModelResp> modelMap = modelService.getModelMap();
if (!CollectionUtils.isEmpty(metricDOS)) {
metricResps = metricDOS.stream()
Expand Down Expand Up @@ -560,7 +589,7 @@ private ModelCluster getModelCluster(List<MetricResp> metricResps, Set<Long> mod
}

private Set<Long> getModelIds(Set<Long> modelIdsByDomainId, List<MetricResp> metricResps,
List<DimensionResp> dimensionResps) {
List<DimensionResp> dimensionResps) {
Set<Long> result = new HashSet<>();
if (org.apache.commons.collections.CollectionUtils.isNotEmpty(modelIdsByDomainId)) {
result.addAll(modelIdsByDomainId);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@
<if test="type != null and type != ''">
and type = #{type}
</if>
<if test="tagDefineType != null">
and define_type = #{tagDefineType}
</if>
<if test="key != null and key != ''">
and ( id like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
name like CONCAT('%',#{key , jdbcType=VARCHAR},'%') or
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,9 @@ private MetricService mockMetricService(MetricRepository metricRepository,
ApplicationEventPublisher eventPublisher = Mockito.mock(ApplicationEventPublisher.class);
DataSetService dataSetService = Mockito.mock(DataSetServiceImpl.class);
DimensionService dimensionService = Mockito.mock(DimensionService.class);
TagMetaService tagMetaService = Mockito.mock(TagMetaService.class);
return new MetricServiceImpl(metricRepository, modelService, chatGptHelper, collectService, dataSetService,
eventPublisher, dimensionService);
eventPublisher, dimensionService, tagMetaService);
}

private MetricReq buildMetricReq() {
Expand Down
Loading