Skip to content

Commit

Permalink
[#10704] improved batch to work on a multi-table basis
Browse files Browse the repository at this point in the history
  • Loading branch information
@minwoo-jung
minwoo-jung committed May 14, 2024
1 parent 88a361b commit 1a66dbc
Show file tree
Hide file tree
Showing 5 changed files with 112 additions and 85 deletions.
148 changes: 79 additions & 69 deletions batch/src/main/java/com/navercorp/pinpoint/batch/alarm/dao/model/BatchQueryParameter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,99 +29,96 @@
* @author minwoo-jung
*/
public class BatchQueryParameter {
private final String EMPTY_STRING = "";

private final static String DEFAULT_AGENT_ID = "agentId";
private final static String DEFAULT_FIELD_NAME = "fieldName";
private final String tableName;
private final String applicationName;
private final String agentId;
private final String metricName;
private final String sortKey;
private final List<String> fieldList;

private final List<Tag> tagList;
private final String fieldName;
private final Range range;

public BatchQueryParameter(String tableName,
String applicationName,
String agentId,
String metricName,
String fieldName,
List<Tag> tagList,
Range range) {
this(tableName, applicationName, agentId, metricName, fieldName, Collections.EMPTY_LIST, tagList, range);
}

public BatchQueryParameter(String applicationName, String metricName, List<String> fieldList, Range range) {
StringPrecondition.requireHasLength(applicationName, "applicationName");
this.applicationName = applicationName;
this.agentId = EMPTY_STRING;
StringPrecondition.requireHasLength(metricName, "metricName");
this.metricName = metricName;
this.sortKey = SortKeyUtils.generateKeyForAgentStat(applicationName, agentId, metricName);
this.fieldList = Objects.requireNonNull(fieldList, "fieldList");
this.tagList = Collections.EMPTY_LIST;
this.fieldName = EMPTY_STRING;
this.range = Objects.requireNonNull(range, "range");
public BatchQueryParameter(String tableName,
String applicationName,
String agentId,
String metricName,
List<String> fieldList,
List<Tag> tagList,
Range range) {
this(tableName, applicationName, agentId, metricName, DEFAULT_FIELD_NAME, fieldList, tagList, range);
}

public BatchQueryParameter(String applicationName, String metricName, String fieldName, Range range) {
StringPrecondition.requireHasLength(applicationName, "applicationName");
this.applicationName = applicationName;
this.agentId = EMPTY_STRING;
StringPrecondition.requireHasLength(metricName, "metricName");
this.metricName = metricName;
this.sortKey = SortKeyUtils.generateKeyForAgentStat(applicationName, agentId, metricName);
this.fieldList = Collections.EMPTY_LIST;
this.tagList = Collections.EMPTY_LIST;
this.fieldName = fieldName;
this.range = Objects.requireNonNull(range, "range");
public BatchQueryParameter(String tableName,
String applicationName,
String agentId,
String metricName,
String fieldName,
Range range) {
this(tableName, applicationName, agentId, metricName, fieldName, Collections.EMPTY_LIST, Collections.EMPTY_LIST, range);
}

public BatchQueryParameter(String applicationName, String metricName, List<String> fieldList, List<Tag> tagList, Range range) {
StringPrecondition.requireHasLength(applicationName, "applicationName");
this.applicationName = applicationName;
this.agentId = EMPTY_STRING;
StringPrecondition.requireHasLength(metricName, "metricName");
this.metricName = metricName;
this.sortKey = SortKeyUtils.generateKeyForAgentStat(applicationName, agentId, metricName);
this.fieldList = Objects.requireNonNull(fieldList, "fieldList");
this.tagList = Objects.requireNonNull(tagList, "tagList");
this.fieldName = EMPTY_STRING;
this.range = Objects.requireNonNull(range, "range");
public BatchQueryParameter(String tableName,
String applicationName,
String metricName,
List<String> fieldList,
List<Tag> tagList,
Range range) {
this(tableName, applicationName, DEFAULT_AGENT_ID, metricName, DEFAULT_FIELD_NAME, fieldList, tagList, range);
}

public BatchQueryParameter(String applicationName, String agentId, String metricName, String fieldName, Range range) {
StringPrecondition.requireHasLength(applicationName, "applicationName");
this.applicationName = applicationName;
StringPrecondition.requireHasLength(agentId, "agentId");
this.agentId = agentId;
StringPrecondition.requireHasLength(metricName, "metricName");
this.metricName = metricName;
this.sortKey = SortKeyUtils.generateKeyForAgentStat(applicationName, agentId, metricName);
this.fieldList = Collections.EMPTY_LIST;
this.tagList = Collections.EMPTY_LIST;
this.fieldName = fieldName;
this.range = Objects.requireNonNull(range, "range");
public BatchQueryParameter(String tableName,
String applicationName,
String metricName,
String fieldName,
Range range) {
this(tableName, applicationName, DEFAULT_AGENT_ID, metricName, fieldName, Collections.EMPTY_LIST, Collections.EMPTY_LIST, range);
}

public BatchQueryParameter(String applicationName, String agentId, String metricName, List<String> fieldList, List<Tag> tagList, Range range) {
StringPrecondition.requireHasLength(applicationName, "applicationName");
this.applicationName = applicationName;
StringPrecondition.requireHasLength(agentId, "agentId");
this.agentId = agentId;
StringPrecondition.requireHasLength(metricName, "metricName");
this.metricName = metricName;
this.sortKey = SortKeyUtils.generateKeyForAgentStat(applicationName, agentId, metricName);
this.fieldName = EMPTY_STRING;
this.fieldList = Objects.requireNonNull(fieldList, "fieldList");;
this.tagList = Objects.requireNonNull(tagList, "tagList");
this.range = Objects.requireNonNull(range, "range");
public BatchQueryParameter(String tableName,
String applicationName,
String metricName,
List<String> fieldList,
Range range) {
this(tableName, applicationName, DEFAULT_AGENT_ID, metricName, DEFAULT_FIELD_NAME, fieldList, Collections.EMPTY_LIST, range);
}

public BatchQueryParameter(String applicationName, String agentId, String metricName, String fieldName, List<Tag> tagList, Range range) {
StringPrecondition.requireHasLength(applicationName, "applicationName");
this.applicationName = applicationName;
StringPrecondition.requireHasLength(agentId, "agentId");
this.agentId = agentId;
StringPrecondition.requireHasLength(metricName, "metricName");
this.metricName = metricName;
private BatchQueryParameter(String tableName,
String applicationName,
String agentId,
String metricName,
String fieldName,
List<String> fieldList,
List<Tag> tagList,
Range range) {
this.tableName = StringPrecondition.requireHasLength(tableName, "tableName");
this.applicationName = StringPrecondition.requireHasLength(applicationName, "applicationName");
this.agentId = StringPrecondition.requireHasLength(agentId, "agentId");
this.metricName = StringPrecondition.requireHasLength(metricName, "metricName");
this.sortKey = SortKeyUtils.generateKeyForAgentStat(applicationName, agentId, metricName);
this.fieldName = fieldName;
this.fieldList = Collections.EMPTY_LIST;
this.fieldName = StringPrecondition.requireHasLength(fieldName, "fieldName");
this.fieldList = Objects.requireNonNull(fieldList, "fieldList");
this.tagList = Objects.requireNonNull(tagList, "tagList");
this.range = Objects.requireNonNull(range, "range");
}

public String getTableName() {
return tableName;
}

public String getApplicationName() {
return applicationName;
}
Expand Down Expand Up @@ -154,13 +151,26 @@ public String getSortKey() {
return sortKey;
}

public BatchQueryParameter(String tableName, String applicationName, String agentId, String metricName, String sortKey, List<String> fieldList, List<Tag> tagList, String fieldName, Range range) {
this.tableName = tableName;
this.applicationName = applicationName;
this.agentId = agentId;
this.metricName = metricName;
this.sortKey = sortKey;
this.fieldList = fieldList;
this.tagList = tagList;
this.fieldName = fieldName;
this.range = range;
}

@Override
public String toString() {
return "BatchQueryParameter{" +
"EMPTY_STRING='" + EMPTY_STRING + '\'' +
"tableName='" + tableName + '\'' +
", applicationName='" + applicationName + '\'' +
", agentId='" + agentId + '\'' +
", metricName='" + metricName + '\'' +
", sortKey='" + sortKey + '\'' +
", fieldList=" + fieldList +
", tagList=" + tagList +
", fieldName='" + fieldName + '\'' +
Expand Down
23 changes: 16 additions & 7 deletions batch/src/main/java/com/navercorp/pinpoint/batch/alarm/dao/pinot/PinotAlarmDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import com.navercorp.pinpoint.batch.alarm.vo.AgentFieldUsage;
import com.navercorp.pinpoint.batch.alarm.vo.AgentUsage;
import com.navercorp.pinpoint.batch.alarm.vo.AgentUsageCount;
import com.navercorp.pinpoint.batch.common.BatchProperties;
import com.navercorp.pinpoint.common.dao.pinot.AgentStatTopicAndTableNameManager;
import com.navercorp.pinpoint.common.model.TagInformation;
import com.navercorp.pinpoint.common.server.util.time.Range;
import com.navercorp.pinpoint.metric.common.model.Tag;
Expand All @@ -43,45 +45,52 @@ public class PinotAlarmDao implements AlarmDao {

private final PinotAsyncTemplate asyncTemplate;
private final SqlSessionTemplate syncTemplate;
private final int tableCount;

public PinotAlarmDao(@Qualifier("batchPinotAsyncTemplate") PinotAsyncTemplate asyncTemplate, @Qualifier("batchPinotTemplate") SqlSessionTemplate syncTemplate) {
public PinotAlarmDao(@Qualifier("batchPinotAsyncTemplate") PinotAsyncTemplate asyncTemplate, @Qualifier("batchPinotTemplate") SqlSessionTemplate syncTemplate, BatchProperties batchProperties) {
this.asyncTemplate = Objects.requireNonNull(asyncTemplate, "asyncTemplate");
this.syncTemplate = Objects.requireNonNull(syncTemplate, "syncTemplate");
Objects.requireNonNull(batchProperties, "batchProperties");
this.tableCount = batchProperties.getAlarmAgentInspectorStatTableCount();
}

@Override
public List<AgentFieldUsage> selectSumGroupByField(String applicationName, String metricName, List<String> fieldList, Range range) {
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(applicationName, metricName, fieldList, range);
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(getTableName(applicationName), applicationName, metricName, fieldList, range);
return syncTemplate.selectList(NAMESPACE + "selectSumGroupByField", batchQueryParameter);
}

@Override
public CompletableFuture<List<AgentFieldUsage>> selectAvgGroupByField(String applicationName, String agentId, String metricName, List<String> fieldList, List<Tag> tagList, Range range) {
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(applicationName, agentId, metricName, fieldList, tagList, range);
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(getTableName(applicationName), applicationName, agentId, metricName, fieldList, tagList, range);
return asyncTemplate.selectList(NAMESPACE + "selectAvgGroupByField", batchQueryParameter);
}

@Override
public List<AgentUsageCount> selectSumCount(String applicationName, String metricName, String fieldName, Range range) {
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(applicationName, metricName, fieldName, range);
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(getTableName(applicationName), applicationName, metricName, fieldName, range);
return syncTemplate.selectList(NAMESPACE + "selectSumCount", batchQueryParameter);
}

@Override
public List<AgentUsage> selectAvg(String applicationName, String metricName, String fieldName, Range range) {
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(applicationName, metricName, fieldName, range);
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(getTableName(applicationName), applicationName, metricName, fieldName, range);
return syncTemplate.selectList(NAMESPACE + "selectAvg", batchQueryParameter);
}

@Override
public CompletableFuture<List<Tag>> selectTagInfo(String applicationName, String agentId, String metricName, String fieldName, Range range) {
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(applicationName, agentId, metricName, fieldName, range);
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(getTableName(applicationName), applicationName, agentId, metricName, fieldName, range);
return asyncTemplate.selectList(NAMESPACE + "selectTagInfo", batchQueryParameter);
}

@Override
public CompletableFuture<List<TagInformation>> getTagInfoContainedSpecificTag(String applicationName, String agentId, String metricName, String fieldName, List<Tag> tagList, Range range) {
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(applicationName, agentId, metricName, fieldName, tagList, range);
BatchQueryParameter batchQueryParameter = new BatchQueryParameter(getTableName(applicationName), applicationName, agentId, metricName, fieldName, tagList, range);
return asyncTemplate.selectList(NAMESPACE + "selectTagInfoContainedSpecificTag", batchQueryParameter);
}

private String getTableName(String applicationName) {
return AgentStatTopicAndTableNameManager.getAgentStatTableName(applicationName, tableCount);
}
}
13 changes: 10 additions & 3 deletions batch/src/main/java/com/navercorp/pinpoint/batch/common/BatchProperties.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ public class BatchProperties {
@Value("${job.alarm.cron}")
private String alarmJobCron;

@Value("${job.alarm.agent.inspector.stat.table.count}")
private int alarmAgentInspectorStatTableCount;

@Value("${job.agent.count.enable:true}")
private boolean agentCountJobEnable;

Expand Down Expand Up @@ -177,15 +180,19 @@ public int getCollectorVersion() {
return collectorVersion;
}

public int getAlarmAgentInspectorStatTableCount() {
return alarmAgentInspectorStatTableCount;
}

@Override
public String toString() {
return "BatchProperties{" +
"logger=" + logger +
", batchEnv='" + batchEnv + '\'' +
"batchEnv='" + batchEnv + '\'' +
", flinkServerList=" + Arrays.toString(flinkServerList) +
", flinkRestPort=" + flinkRestPort +
", alarmJobEnable=" + alarmJobEnable +
", alarmJobCron='" + alarmJobCron + '\'' +
", alarmAgentInspectorStatTableCount=" + alarmAgentInspectorStatTableCount +
", agentCountJobEnable=" + agentCountJobEnable +
", agentCountJobCron='" + agentCountJobCron + '\'' +
", flinkCheckJobEnable=" + flinkCheckJobEnable +
Expand All @@ -197,7 +204,7 @@ public String toString() {
", cleanupInactiveAgentsDurationDays=" + cleanupInactiveAgentsDurationDays +
", cleanupInactiveApplicationsJobEnable=" + cleanupInactiveApplicationsJobEnable +
", cleanupInactiveApplicationsJobCron='" + cleanupInactiveApplicationsJobCron + '\'' +
", collectorVersion='" + collectorVersion + '\'' +
", collectorVersion=" + collectorVersion +
'}';
}
}
1 change: 1 addition & 0 deletions batch/src/main/resources/batch-root.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ batch.flink.rest.port=8081
###########################################################

job.alarm.cron=0 0/3 * * * *
job.alarm.agent.inspector.stat.table.count=1

job.agent.count.cron=0 0 2 * * *

Expand Down
12 changes: 6 additions & 6 deletions batch/src/main/resources/mapper/batch/BatchAlarmForPinotMapper.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

<select id="selectSumGroupByField" parameterType="BatchQueryParameter" resultType="AgentFieldUsage">
SELECT agentId, fieldName, SUM(fieldValue) AS sumValue
FROM inspectorStat
FROM ${tableName}
WHERE
applicationName = #{applicationName}
AND metricName = #{metricName}
Expand All @@ -30,7 +30,7 @@

<select id="selectAvgGroupByField" parameterType="BatchQueryParameter" resultType="AgentFieldUsage">
SELECT 'agentId', fieldName, AVG(fieldValue) AS sumValue
FROM inspectorStat
FROM ${tableName}
WHERE
sortKey = #{sortKey}
AND fieldName IN
Expand All @@ -47,7 +47,7 @@

<select id="selectSumCount" parameterType="BatchQueryParameter" resultType="AgentUsageCount">
SELECT agentId, SUM(fieldValue) AS sumValue, COUNT(*) AS countValue
FROM inspectorStat
FROM ${tableName}
WHERE
applicationName = #{applicationName}
AND metricName = #{metricName}
Expand All @@ -58,7 +58,7 @@

<select id="selectAvg" parameterType="BatchQueryParameter" resultType="AgentUsage">
SELECT agentId, AVG(fieldValue) AS avgValue
FROM inspectorStat
FROM ${tableName}
WHERE
applicationName = #{applicationName}
AND metricName = #{metricName}
Expand All @@ -69,7 +69,7 @@

<select id="selectTagInfo" parameterType="BatchQueryParameter" resultType="Tag">
SELECT DISTINCT(tags)
FROM inspectorStat
FROM ${tableName}
WHERE
sortKey = #{sortKey}
AND fieldName = #{fieldName}
Expand All @@ -78,7 +78,7 @@

<select id="selectTagInfoContainedSpecificTag" parameterType="BatchQueryParameter" resultMap="tagInfoMap">
SELECT applicationName, agentId, metricName, fieldName, tags
FROM inspectorStat
FROM ${tableName}
WHERE
sortKey = #{sortKey}
AND fieldName = #{fieldName}
Expand Down

0 comments on commit 1a66dbc

Please sign in to comment.