Skip to content

Commit

Permalink
[pinpoint-apm#9631] Add API to retrieve CLP replaced tokens with coun…
Browse files Browse the repository at this point in the history
…t and ordering
  • Loading branch information
intr3p1d committed Aug 23, 2024
1 parent c9233f7 commit d1d2fd2
Show file tree
Hide file tree
Showing 17 changed files with 513 additions and 40 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright 2024 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.navercorp.pinpoint.exceptiontrace.common.pinot;

/**
* @author intr3p1d
*/
public enum PinotFunctions {
ARRAY_SLICE_INT("arraySliceInt"),
ARRAY_SLICE_STRING("arraySliceString");

private final String name;

PinotFunctions(String name) {
this.name = name;
}

public String getName() {
return name;
}
}
9 changes: 9 additions & 0 deletions exceptiontrace/exceptiontrace-web/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,15 @@
<groupId>com.navercorp.pinpoint</groupId>
<artifactId>pinpoint-exceptiontrace-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-text</artifactId>
<version>1.10.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.mapstruct</groupId>
<artifactId>mapstruct</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@
import com.navercorp.pinpoint.exceptiontrace.web.entity.ErrorSummaryEntity;
import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionChartValueViewEntity;
import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionGroupSummaryEntity;
import com.navercorp.pinpoint.exceptiontrace.web.entity.ClpConvertedEntity;
import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionMetaDataEntity;
import com.navercorp.pinpoint.exceptiontrace.web.entity.GroupedFieldNameEntity;
import com.navercorp.pinpoint.exceptiontrace.web.util.ExceptionTraceQueryParameter;
import com.navercorp.pinpoint.exceptiontrace.web.query.ClpQueryParameter;
import com.navercorp.pinpoint.exceptiontrace.web.query.ExceptionTraceQueryParameter;
import com.navercorp.pinpoint.mybatis.MyBatisRegistryHandler;
import org.apache.ibatis.type.TypeAliasRegistry;
import org.apache.ibatis.type.TypeHandlerRegistry;
Expand All @@ -36,7 +38,9 @@ public void registerTypeAlias(TypeAliasRegistry typeAliasRegistry) {
typeAliasRegistry.registerAlias(ErrorSummaryEntity.class);
typeAliasRegistry.registerAlias(ExceptionGroupSummaryEntity.class);
typeAliasRegistry.registerAlias(ExceptionChartValueViewEntity.class);
typeAliasRegistry.registerAlias(ClpConvertedEntity.class);
typeAliasRegistry.registerAlias(ExceptionTraceQueryParameter.class);
typeAliasRegistry.registerAlias(ClpQueryParameter.class);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@
import com.navercorp.pinpoint.common.server.util.time.RangeValidator;
import com.navercorp.pinpoint.exceptiontrace.web.model.ErrorSummary;
import com.navercorp.pinpoint.exceptiontrace.web.model.ExceptionGroupSummary;
import com.navercorp.pinpoint.exceptiontrace.web.query.ExceptionTraceQueryParameter;
import com.navercorp.pinpoint.exceptiontrace.web.view.ExceptionChartValueView;
import com.navercorp.pinpoint.exceptiontrace.web.model.ClpConverted;
import com.navercorp.pinpoint.exceptiontrace.web.query.ClpQueryParameter;
import com.navercorp.pinpoint.exceptiontrace.web.service.ExceptionTraceService;
import com.navercorp.pinpoint.exceptiontrace.web.util.ExceptionTraceQueryParameter;
import com.navercorp.pinpoint.exceptiontrace.web.util.GroupByAttributes;
import com.navercorp.pinpoint.exceptiontrace.web.view.ErrorSummaryView;
import com.navercorp.pinpoint.exceptiontrace.web.view.ExceptionDetailView;
Expand Down Expand Up @@ -197,6 +199,37 @@ public List<ExceptionGroupSummary> getListOfExceptionMetaDataWithDynamicGroupBy(
);
}

@GetMapping("/replacedTokens")
public List<ClpConverted> getReplacedVariables(
@RequestParam("applicationName") @NotBlank String applicationName,
@RequestParam(value = "agentId", required = false) String agentId,
@RequestParam("from") @PositiveOrZero long from,
@RequestParam("to") @PositiveOrZero long to,

@RequestParam("logType") String logType,
@RequestParam("targetColumn") String targetColumn,
@RequestParam("targetIndex") int targetIndex
) {
Range range = Range.between(from, to);
rangeValidator.validate(range);

ClpQueryParameter queryParameter = new ClpQueryParameter.Builder()
.setTableName(tableName)
.setTenantId(tenantProvider.getTenantId())
.setApplicationName(applicationName)
.setAgentId(agentId)
.setRange(Range.between(from, to))
.setTimePrecision(DETAILED_TIME_PRECISION)
.setLogType(logType)
.setTargetColumn(targetColumn)
.setTargetIndex(targetIndex)
.build();

return exceptionTraceService.getReplacedVariables(
queryParameter
);
}

@GetMapping("/chart")
public ExceptionChartView getCollectedExceptionMetaDataByGivenRange(
@RequestParam("applicationName") @NotBlank String applicationName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@
import com.navercorp.pinpoint.exceptiontrace.common.model.ExceptionMetaData;
import com.navercorp.pinpoint.exceptiontrace.web.model.ErrorSummary;
import com.navercorp.pinpoint.exceptiontrace.web.model.ExceptionGroupSummary;
import com.navercorp.pinpoint.exceptiontrace.web.query.ExceptionTraceQueryParameter;
import com.navercorp.pinpoint.exceptiontrace.web.view.ExceptionChartValueView;
import com.navercorp.pinpoint.exceptiontrace.web.util.ExceptionTraceQueryParameter;
import com.navercorp.pinpoint.exceptiontrace.web.view.ExceptionDetailView;
import com.navercorp.pinpoint.exceptiontrace.web.model.ClpConverted;
import com.navercorp.pinpoint.exceptiontrace.web.query.ClpQueryParameter;

import java.util.List;

Expand All @@ -36,4 +38,5 @@ public interface ExceptionTraceDao {
List<ExceptionGroupSummary> getGroupSummaries(ExceptionTraceQueryParameter exceptionTraceQueryParameter);
List<ExceptionChartValueView> getChartValueViews(ExceptionTraceQueryParameter exceptionTraceQueryParameter);
List<ErrorSummary> getErrorSummaries(ExceptionTraceQueryParameter exceptionTraceQueryParameter);
List<ClpConverted> getReplacedVariables(ClpQueryParameter queryParameter);
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,17 @@
import com.navercorp.pinpoint.exceptiontrace.common.model.ExceptionMetaData;
import com.navercorp.pinpoint.exceptiontrace.web.entity.ErrorSummaryEntity;
import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionGroupSummaryEntity;
import com.navercorp.pinpoint.exceptiontrace.web.entity.ClpConvertedEntity;
import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionMetaDataEntity;
import com.navercorp.pinpoint.exceptiontrace.web.model.ErrorSummary;
import com.navercorp.pinpoint.exceptiontrace.web.entity.ExceptionChartValueViewEntity;
import com.navercorp.pinpoint.exceptiontrace.web.mapper.ExceptionEntityMapper;
import com.navercorp.pinpoint.exceptiontrace.web.model.ExceptionGroupSummary;
import com.navercorp.pinpoint.exceptiontrace.web.util.ExceptionTraceQueryParameter;
import com.navercorp.pinpoint.exceptiontrace.web.query.ExceptionTraceQueryParameter;
import com.navercorp.pinpoint.exceptiontrace.web.view.ExceptionChartValueView;
import com.navercorp.pinpoint.exceptiontrace.web.view.ExceptionDetailView;
import com.navercorp.pinpoint.exceptiontrace.web.model.ClpConverted;
import com.navercorp.pinpoint.exceptiontrace.web.query.ClpQueryParameter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.mybatis.spring.SqlSessionTemplate;
Expand All @@ -52,6 +55,7 @@ public class PinotExceptionTraceDao implements ExceptionTraceDao {
private static final String SELECT_GROUP_SUMMARIES_QUERY = "selectGroupSummaries";
private static final String SELECT_CHART_QUERY = "selectChartValueViews";
private static final String SELECT_ERROR_SUMMARIES_QUERY = "selectErrorSummaries";
private static final String SELECT_CLP_VARIABLES_QUERY = "selectClpVariables";

private final SqlSessionTemplate sqlPinotSessionTemplate;

Expand Down Expand Up @@ -124,4 +128,13 @@ public List<ErrorSummary> getErrorSummaries(ExceptionTraceQueryParameter excepti
)
).collect(Collectors.toList());
}

@Override
public List<ClpConverted> getReplacedVariables(ClpQueryParameter queryParameter) {
List<ClpConvertedEntity> clpConvertedEntities = this.sqlPinotSessionTemplate.selectList(NAMESPACE + SELECT_CLP_VARIABLES_QUERY, queryParameter);

return clpConvertedEntities.stream()
.map(mapper::toClpConverted)
.collect(Collectors.toList());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
* Copyright 2024 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.navercorp.pinpoint.exceptiontrace.web.entity;

/**
* @author intr3p1d
*/
public class ClpConvertedEntity {

private String replacedToken;
private int count;

public ClpConvertedEntity() {
}

public String getReplacedToken() {
return replacedToken;
}

public void setReplacedToken(String replacedToken) {
this.replacedToken = replacedToken;
}

public int getCount() {
return count;
}

public void setCount(int count) {
this.count = count;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,13 @@
*/
package com.navercorp.pinpoint.exceptiontrace.web.mapper;


import org.apache.commons.text.StringEscapeUtils;

import java.nio.charset.StandardCharsets;
import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
* @author intr3p1d
Expand All @@ -26,17 +32,76 @@ public class CLPMapper {
public final static char DICTIONARY_VARIABLE_VALUE = '\u0011';
public final static char NON_DICTIONARY_VALUE = '\u0012';

public final static String DICTIONARY_REPLACEMENT = "▨▨▨";
public final static String NON_DICTIONARY_REPLACEMENT = "▧▧▧";
public final static String DICTIONARY_REPLACEMENT = "<ClpPlaceHolder dict %d/>";
public final static String NON_DICTIONARY_REPLACEMENT = "<ClpPlaceHolder non-dict %d/>";

public final static String SIMPLE_REPLACEMENT = "▨▨▨";

static String fixAndEscapeLogType(String encodedLogType) {
return replaceDictPlaceHolder(replaceNonDictPlaceHolder(
escapeXml(correctEncoding(
encodedLogType
))
));
}

static String processEncodedLogType(String encodedLogType) {
return replaceSimple(correctEncoding(
encodedLogType
));
}

static String makeReadableString(String encodedLogType) {
byte[] encodedLogTypeBytes = encodedLogType.getBytes(StandardCharsets.ISO_8859_1);
static String correctEncoding(String isoString) {
byte[] encodedLogTypeBytes = isoString.getBytes(StandardCharsets.ISO_8859_1);
return new String(encodedLogTypeBytes, StandardCharsets.UTF_8);
}

static String replacePlaceHolders(String encodedLogType) {
return encodedLogType
.replaceAll(String.valueOf(DICTIONARY_VARIABLE_VALUE), DICTIONARY_REPLACEMENT)
.replaceAll(String.valueOf(NON_DICTIONARY_VALUE), NON_DICTIONARY_REPLACEMENT);
static String escapeXml(String rawString) {
return StringEscapeUtils.escapeHtml4(rawString);
}

static String replaceNonDictPlaceHolder(String encodedLogType) {
return replaceHolder(
encodedLogType,
String.valueOf(NON_DICTIONARY_VALUE),
x -> stringFunction(NON_DICTIONARY_REPLACEMENT, x)
);
}

static String replaceDictPlaceHolder(String encodedLogType) {
return replaceHolder(
encodedLogType,
String.valueOf(DICTIONARY_VARIABLE_VALUE),
x -> stringFunction(DICTIONARY_REPLACEMENT, x)
);
}

static String replaceSimple(String encodedLogType) {
return replaceHolder(replaceHolder(
encodedLogType,
String.valueOf((DICTIONARY_VARIABLE_VALUE)),
x -> stringFunction(SIMPLE_REPLACEMENT, x)
),
String.valueOf(NON_DICTIONARY_VALUE),
x -> stringFunction(SIMPLE_REPLACEMENT, x)
);
}

static String stringFunction(String format, int index) {
return String.format(format, index);
}

static String replaceHolder(String encodedLogType, String replacedCharacter, Function<Integer, String> replacement) {
Pattern pattern = Pattern.compile(replacedCharacter);
Matcher matcher = pattern.matcher(encodedLogType);
StringBuilder result = new StringBuilder();
int wordCount = 0;

while (matcher.find()) {
matcher.appendReplacement(result, replacement.apply(wordCount++));
}
matcher.appendTail(result);

return result.toString();
}
}
Loading

0 comments on commit d1d2fd2

Please sign in to comment.