[pinpoint-apm#9636] Added log livetail feature

smilu97 · Aug 24, 2023 · d2e067f · d2e067f
1 parent 79fbc87
commit d2e067f
Show file tree

Hide file tree

Showing 78 changed files with 3,172 additions and 43 deletions.
diff --git a/grpc/grpc-idl b/grpc/grpc-idl
diff --git a/log/log-collector/pom.xml b/log/log-collector/pom.xml
@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>pinpoint-log-module</artifactId>
+        <groupId>com.navercorp.pinpoint</groupId>
+        <version>2.6.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>pinpoint-log-collector</artifactId>
+    <name>pinpoint-log-collector</name>
+    <packaging>jar</packaging>
+
+    <properties>
+        <jdk.version>11</jdk.version>
+        <jdk.home>${env.JAVA_11_HOME}</jdk.home>
+
+        <log4j.version>2.19.0</log4j.version>
+
+        <!-- for development option -->
+        <skip.agent.distribution>false</skip.agent.distribution>
+
+        <pinpoint.log.collector.executable.name>
+            ${project.artifactId}-boot-${project.version}
+        </pinpoint.log.collector.executable.name>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>javax.annotation</groupId>
+            <artifactId>javax.annotation-api</artifactId>
+            <version>1.3.2</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-stub</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>io.grpc</groupId>
+            <artifactId>grpc-protobuf</artifactId>
+            <version>${grpc.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <version>${log4j.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+            <version>${log4j.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.navercorp.pinpoint</groupId>
+            <artifactId>pinpoint-grpc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.navercorp.pinpoint</groupId>
+            <artifactId>pinpoint-collector</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.navercorp.pinpoint</groupId>
+            <artifactId>pinpoint-log-common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.kafka</groupId>
+            <artifactId>spring-kafka</artifactId>
+            <version>2.9.4</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.assertj</groupId>
+            <artifactId>assertj-core</artifactId>
+            <version>3.23.1</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>1.17.6</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>testcontainers</artifactId>
+            <version>${testcontainers.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <version>${spring.boot.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.springframework.boot</groupId>
+                    <artifactId>spring-boot-starter-logging</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <configuration>
+                    <mainClass>com.navercorp.pinpoint.log.collector.LogCollectorApp</mainClass>
+                    <outputDirectory>${project.build.directory}/deploy</outputDirectory>
+                    <executable>true</executable>
+                    <attach>false</attach>
+                    <finalName>${pinpoint.log.collector.executable.name}</finalName>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/log/log-collector/src/main/java/com/navercorp/pinpoint/log/collector/LogCollectorConfig.java b/log/log-collector/src/main/java/com/navercorp/pinpoint/log/collector/LogCollectorConfig.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2023 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.navercorp.pinpoint.log.collector;
+
+import com.codahale.metrics.MetricRegistry;
+import com.navercorp.pinpoint.log.collector.grpc.LogCollectorGrpcServerConfig;
+import com.navercorp.pinpoint.log.collector.redis.LogCollectorRedisServerConfig;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+
+/**
+ * @author youngjin.kim2
+ */
+@Configuration(proxyBeanMethods = false)
+@Import({ LogCollectorGrpcServerConfig.class, LogCollectorRedisServerConfig.class })
+public class LogCollectorConfig {
+
+    @Bean
+    MetricRegistry metricRegistry() {
+        return new MetricRegistry();
+    }
+
+}
diff --git a/log/log-collector/src/main/java/com/navercorp/pinpoint/log/collector/LogCollectorModule.java b/log/log-collector/src/main/java/com/navercorp/pinpoint/log/collector/LogCollectorModule.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2023 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.navercorp.pinpoint.log.collector;
+
+import com.navercorp.pinpoint.redis.RedisPropertySources;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+
+/**
+ * @author youngjin.kim2
+ */
+@Configuration(proxyBeanMethods = false)
+@Import({ LogCollectorConfig.class, RedisPropertySources.class, LogCollectorPropertySources.class })
+public class LogCollectorModule {
+}
diff --git a/...ector/src/main/java/com/navercorp/pinpoint/log/collector/LogCollectorPropertySources.java b/...ector/src/main/java/com/navercorp/pinpoint/log/collector/LogCollectorPropertySources.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2023 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.navercorp.pinpoint.log.collector;
+
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.context.annotation.PropertySources;
+
+import static com.navercorp.pinpoint.log.collector.LogCollectorPropertySources.COLLECTOR;
+import static com.navercorp.pinpoint.log.collector.LogCollectorPropertySources.GRPC_PROFILE;
+import static com.navercorp.pinpoint.log.collector.LogCollectorPropertySources.GRPC_ROOT;
+
+/**
+ * @author youngjin.kim2
+ */
+@PropertySources({
+        @PropertySource(name = "CollectorLogAppPropertySources-GRPC", value = { GRPC_ROOT, GRPC_PROFILE }),
+        @PropertySource(name = "CollectorLogAppPropertySources", value = { COLLECTOR })
+})
+public class LogCollectorPropertySources {
+
+    private static final String PROFILE = "classpath:log/profiles/${pinpoint.profiles.active:local}/";
+
+    public static final String GRPC_ROOT = "classpath:log/pinpoint-collector-log-grpc-root.properties";
+    public static final String GRPC_PROFILE = PROFILE + "pinpoint-collector-log-grpc.properties";
+
+    public static final String COLLECTOR = "classpath:log/pinpoint-collector-log-root.properties";
+
+}
diff --git a/...ollector/src/main/java/com/navercorp/pinpoint/log/collector/grpc/EmptyStreamObserver.java b/...ollector/src/main/java/com/navercorp/pinpoint/log/collector/grpc/EmptyStreamObserver.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2023 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.navercorp.pinpoint.log.collector.grpc;
+
+import com.google.protobuf.Empty;
+import io.grpc.stub.StreamObserver;
+
+/**
+ * @author youngjin.kim2
+ */
+public class EmptyStreamObserver implements StreamObserver<Empty> {
+    @Override public void onNext(Empty t) {}
+    @Override public void onError(Throwable throwable) {}
+    @Override public void onCompleted() {}
+
+    @SuppressWarnings("unchecked")
+    static <T> StreamObserver<T> create() {
+        return (StreamObserver<T>) new EmptyStreamObserver();
+    }
+
+}
diff --git a/...or/src/main/java/com/navercorp/pinpoint/log/collector/grpc/GrpcLogReceiverProperties.java b/...or/src/main/java/com/navercorp/pinpoint/log/collector/grpc/GrpcLogReceiverProperties.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2023 NAVER Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.navercorp.pinpoint.log.collector.grpc;
+
+import com.navercorp.pinpoint.collector.config.ExecutorProperties;
+import com.navercorp.pinpoint.collector.grpc.config.GrpcReceiverProperties;
+import com.navercorp.pinpoint.collector.receiver.BindAddress;
+import com.navercorp.pinpoint.grpc.server.ServerOption;
+
+/**
+ * @author youngjin.kim2
+ */
+final class GrpcLogReceiverProperties extends GrpcReceiverProperties {
+
+    GrpcLogReceiverProperties(
+            boolean enable,
+            BindAddress bindAddress,
+            ExecutorProperties serverExecutor,
+            ExecutorProperties serverCallExecutor,
+            ExecutorProperties workerExecutor,
+            ServerOption serverOption
+    ) {
+        super(enable, bindAddress, serverExecutor, serverCallExecutor, workerExecutor, serverOption);
+    }
+
+}