Dynamic logger ecs reconfig

apm-agent-core/pom.xml

@@ -47,7 +47,7 @@
         <dependency>
             <groupId>co.elastic.logging</groupId>
             <artifactId>log4j2-ecs-layout</artifactId>
-            <version>0.3.0</version>
+            <version>0.4.0</version>
         </dependency>
         <dependency>
             <groupId>com.blogspot.mydailyjava</groupId>

apm-agent-core/src/main/java/co/elastic/apm/agent/logging/LoggingConfiguration.java

@@ -131,7 +131,7 @@ public void assertValid(Boolean value) {
         .buildWithDefault(false);
 
     @SuppressWarnings("unused")
-    public ConfigurationOption<ByteValue> logFileMaxSize = ByteValueConverter.byteOption()
+    public ConfigurationOption<ByteValue> logFileSize = ByteValueConverter.byteOption()
         .key(LOG_FILE_SIZE_KEY)
         .configurationCategory(LOGGING_CATEGORY)
         .description("The size of the log file.\n" +
@@ -213,6 +213,10 @@ public boolean isLogCorrelationEnabled() {
         return logCorrelationEnabled.get();
     }
 
+    public long getLogFileSize() {
+        return logFileSize.get().getBytes();
+    }
+
     public boolean isShipAgentLogs() {
         return shipAgentLogs.get();
     }

apm-agent-plugins/apm-log-shader-plugin/pom.xml

@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <artifactId>apm-agent-plugins</artifactId>
+        <groupId>co.elastic.apm</groupId>
+        <version>1.17.1-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>apm-log-shader-plugin</artifactId>
+    <name>${project.groupId}:${project.artifactId}</name>
+
+    <properties>
+        <apm-agent-parent.base.dir>${project.basedir}/../..</apm-agent-parent.base.dir>
+        <version.ecs.logging>0.4.0</version.ecs.logging>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <version>1.2.3</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <version>2.10.0</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>co.elastic.logging</groupId>
+            <artifactId>logback-ecs-encoder</artifactId>
+            <version>${version.ecs.logging}</version>
+        </dependency>
+        <dependency>
+            <groupId>co.elastic.logging</groupId>
+            <artifactId>log4j-ecs-layout</artifactId>
+            <version>${version.ecs.logging}</version>
+        </dependency>
+        <dependency>
+            <groupId>co.elastic.logging</groupId>
+            <artifactId>log4j2-ecs-layout</artifactId>
+            <version>${version.ecs.logging}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <!-- Relocating ECS-logging and logging framework packages.
+         The co.elastic.logging package is relocated to a shaded package to be used only within this plugin.
+         The logging framework packages referenced by the plugin, as well as the ECS logging dependencies, are relocated
+         to temp packages so they can be relocated back to the original packages. If we wouldn't done that, the
+         framework packages would have been shaded during agent packaging. -->
+        <plugins>
+            <plugin>
+                <artifactId>maven-shade-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <shadedArtifactAttached>false</shadedArtifactAttached>
+                            <minimizeJar>true</minimizeJar>
+                            <artifactSet>
+                                <includes>
+                                    <include>co.elastic.apm:apm-log-shader-plugin</include>
+                                </includes>
+                                <includes>
+                                    <include>co.elastic.logging:*</include>
+                                </includes>
+                            </artifactSet>
+                            <relocations>
+                                <relocation>
+                                    <pattern>co.elastic.logging</pattern>
+                                    <shadedPattern>co.elastic.apm.agent.shaded.logging.shader</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.slf4j</pattern>
+                                    <shadedPattern>co.elastic.apm.agent.logshading.shaded.slf4j</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.apache.logging</pattern>
+                                    <shadedPattern>co.elastic.apm.agent.logshading.shaded.apache.logging</shadedPattern>
+                                </relocation>
+                            </relocations>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

apm-agent-plugins/apm-log-shader-plugin/src/main/java/co/elastic/apm/agent/log/shader/AbstractLogShadingHelper.java

@@ -0,0 +1,102 @@
+/*-
+ * #%L
+ * Elastic APM Java agent
+ * %%
+ * Copyright (C) 2018 - 2020 Elastic and contributors
+ * %%
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * #L%
+ */
+package co.elastic.apm.agent.log.shader;
+
+import co.elastic.apm.agent.configuration.CoreConfiguration;
+import co.elastic.apm.agent.impl.ElasticApmTracer;
+import co.elastic.apm.agent.logging.LoggingConfiguration;
+import com.blogspot.mydailyjava.weaklockfree.WeakConcurrentMap;
+
+import javax.annotation.Nullable;
+
+/**
+ * The abstract Log shading helper- loaded as part of the agent core (agent CL / bootstrap CL / System CL)
+ *
+ * @param <A> logging-framework-specific Appender type
+ */
+public abstract class AbstractLogShadingHelper<A> {
+
+    private static final Object NULL_APPENDER = new Object();
+
+    private final ElasticApmTracer tracer;
+    private final LoggingConfiguration loggingConfiguration;
+    private final String serviceName;
+
+    public AbstractLogShadingHelper(ElasticApmTracer tracer) {
+        this.tracer = tracer;
+        loggingConfiguration = tracer.getConfig(LoggingConfiguration.class);
+        serviceName = tracer.getConfig(CoreConfiguration.class).getServiceName();
+    }
+
+    private static final WeakConcurrentMap<Object, Object> appenderToShadeAppender = new WeakConcurrentMap.WithInlinedExpunction<Object, Object>();
+
+    @Nullable
+    public A getOrCreateShadeAppenderFor(A originalAppender) {
+        if (isShadingAppender(originalAppender)) {
+            return null;
+        }
+
+        Object shadeAppender = appenderToShadeAppender.get(originalAppender);
+        if (shadeAppender == null) {
+            synchronized (appenderToShadeAppender) {
+                if (!appenderToShadeAppender.containsKey(originalAppender)) {
+                    A createdAppender = createAndConfigureAppender(originalAppender);
+                    appenderToShadeAppender.put(originalAppender, createdAppender != null ? createdAppender : NULL_APPENDER);
+                }
+            }
+            shadeAppender = appenderToShadeAppender.get(originalAppender);
+        }
+        return shadeAppender != NULL_APPENDER ? (A) shadeAppender : null;
+    }
+
+    public void stopShading(A originalAppender) {
+        synchronized (appenderToShadeAppender) {
+            Object shadeAppender = appenderToShadeAppender.remove(originalAppender);
+            if (shadeAppender != null) {
+                closeShadeAppender((A) shadeAppender);
+            }
+        }
+    }
+
+    /**
+     * Checks whether the given appender is a shading appender, so to avoid recursive shading
+     * @return true if the provide appender is a shading appender; false otherwise
+     */
+    protected abstract boolean isShadingAppender(A appender);
+
+    @Nullable
+    protected abstract A createAndConfigureAppender(A originalAppender);
+
+    // todo: find more accurate service name
+    protected String getServiceName() {
+        return serviceName;
+    }
+
+    protected long getMaxLogFileSize() {
+        return loggingConfiguration.getLogFileSize();
+    }
+
+    protected abstract void closeShadeAppender(A shadeAppender);
+}

apm-agent-plugins/apm-log-shader-plugin/src/main/java/co/elastic/apm/agent/log/shader/AbstractLogShadingInstrumentation.java

@@ -0,0 +1,40 @@
+/*-
+ * #%L
+ * Elastic APM Java agent
+ * %%
+ * Copyright (C) 2018 - 2020 Elastic and contributors
+ * %%
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * #L%
+ */
+package co.elastic.apm.agent.log.shader;
+
+import co.elastic.apm.agent.bci.ElasticApmInstrumentation;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+public abstract class AbstractLogShadingInstrumentation extends ElasticApmInstrumentation {
+
+    @Override
+    public Collection<String> getInstrumentationGroupNames() {
+        Collection<String> ret = new ArrayList<>();
+        ret.add("logging");
+        return ret;
+    }
+}

apm-agent-plugins/apm-log-shader-plugin/src/main/java/co/elastic/apm/agent/log/shader/Utils.java

@@ -0,0 +1,52 @@
+/*-
+ * #%L
+ * Elastic APM Java agent
+ * %%
+ * Copyright (C) 2018 - 2020 Elastic and contributors
+ * %%
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * #L%
+ */
+package co.elastic.apm.agent.log.shader;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class Utils {
+
+    static final String SHADE_LOGS_DIR_NAME = "shade_logs";
+
+    /**
+     * Computes a shade log file path based on a given log file. The shade log file will have the same name as the
+     * original log file and will be located in a subdirectory of the original file's parent directory.
+     * @param originalLogFile the log file to which a shade file path is required
+     * @return the shade log file path
+     */
+    public static String computeShadeLogFilePath(String originalLogFile) {
+        Path originalFilePath = Paths.get(originalLogFile);
+        Path logFileName = originalFilePath.getFileName();
+        Path logFileDir = originalFilePath.getParent();
+        Path shadeDir;
+        if (logFileDir != null) {
+            shadeDir = logFileDir.resolve(SHADE_LOGS_DIR_NAME);
+        } else {
+            shadeDir = Paths.get(SHADE_LOGS_DIR_NAME);
+        }
+        return shadeDir.resolve(logFileName).toString();
+    }
+}