diff --git a/CHANGELOG.md b/CHANGELOG.md index 4beef7174fc..2d43b67e7b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Fixed - Fixed missing `commons-codec` dependency ([\#642](https://github.com/testcontainers/testcontainers-java/issues/642)) - Fixed `HostPortWaitStrategy` throws `NumberFormatException` when port is exposed but not mapped ([\#640](https://github.com/testcontainers/testcontainers-java/issues/640)) +- Fixed log processing: multibyte unicode, linebreaks and ASCII color codes. Color codes can be turned on with `withRemoveAnsiCodes(false)` ([PR \#643](https://github.com/testcontainers/testcontainers-java/pull/643)) ### Changed - Support multiple HTTP status codes for HttpWaitStrategy ([\#630](https://github.com/testcontainers/testcontainers-java/issues/630)) diff --git a/core/src/main/java/org/testcontainers/containers/output/BaseConsumer.java b/core/src/main/java/org/testcontainers/containers/output/BaseConsumer.java new file mode 100644 index 00000000000..2967eb4d33a --- /dev/null +++ b/core/src/main/java/org/testcontainers/containers/output/BaseConsumer.java @@ -0,0 +1,17 @@ +package org.testcontainers.containers.output; + +import lombok.Getter; +import lombok.Setter; + +import java.util.function.Consumer; + +public abstract class BaseConsumer> implements Consumer { + @Getter + @Setter + private boolean removeColorCodes = true; + + public SELF withRemoveAnsiCodes(boolean removeAnsiCodes) { + this.removeColorCodes = removeAnsiCodes; + return (SELF) this; + } +} diff --git a/core/src/main/java/org/testcontainers/containers/output/FrameConsumerResultCallback.java b/core/src/main/java/org/testcontainers/containers/output/FrameConsumerResultCallback.java index 6860563344b..6894b9ec4b8 100644 --- a/core/src/main/java/org/testcontainers/containers/output/FrameConsumerResultCallback.java +++ b/core/src/main/java/org/testcontainers/containers/output/FrameConsumerResultCallback.java @@ -2,28 +2,45 @@ import com.github.dockerjava.api.model.Frame; +import com.github.dockerjava.api.model.StreamType; import com.github.dockerjava.core.async.ResultCallbackTemplate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.function.Consumer; +import java.util.regex.Pattern; /** * This class can be used as a generic callback for docker-java commands that produce Frames. */ public class FrameConsumerResultCallback extends ResultCallbackTemplate { - private final static Logger LOGGER = LoggerFactory.getLogger(FrameConsumerResultCallback.class); + private static final Logger LOGGER = LoggerFactory.getLogger(FrameConsumerResultCallback.class); + + private static final byte[] EMPTY_LINE = new byte[0]; + + private static final Pattern ANSI_COLOR_PATTERN = Pattern.compile("\u001B\\[[0-9;]+m"); + + private static final String LINE_BREAK_REGEX = "((\\r?\\n)|(\\r))"; + + static final String LINE_BREAK_AT_END_REGEX = LINE_BREAK_REGEX + "$"; private Map> consumers; private CountDownLatch completionLatch = new CountDownLatch(1); + private StringBuilder logString = new StringBuilder(); + + private OutputFrame brokenFrame; + public FrameConsumerResultCallback() { consumers = new HashMap<>(); } @@ -45,9 +62,13 @@ public void onNext(Frame frame) { if (outputFrame != null) { Consumer consumer = consumers.get(outputFrame.getType()); if (consumer == null) { - LOGGER.error("got frame with type " + frame.getStreamType() + ", for which no handler is configured"); - } else { - consumer.accept(outputFrame); + LOGGER.error("got frame with type {}, for which no handler is configured", frame.getStreamType()); + } else if (outputFrame.getBytes() != null && outputFrame.getBytes().length > 0) { + if (frame.getStreamType() == StreamType.RAW) { + processRawFrame(outputFrame, consumer); + } else { + processOtherFrame(outputFrame, consumer); + } } } } @@ -63,8 +84,17 @@ public void onError(Throwable throwable) { @Override public void close() throws IOException { + OutputFrame lastLine = null; + + if (logString.length() > 0) { + lastLine = new OutputFrame(OutputFrame.OutputType.STDOUT, logString.toString().getBytes()); + } + // send an END frame to every consumer... but only once per consumer. for (Consumer consumer : new HashSet<>(consumers.values())) { + if (lastLine != null) { + consumer.accept(lastLine); + } consumer.accept(OutputFrame.END); } super.close(); @@ -78,4 +108,72 @@ public void close() throws IOException { public CountDownLatch getCompletionLatch() { return completionLatch; } + + private synchronized void processRawFrame(OutputFrame outputFrame, Consumer consumer) { + String utf8String = outputFrame.getUtf8String(); + byte[] bytes = outputFrame.getBytes(); + + // Merging the strings by bytes to solve the problem breaking non-latin unicode symbols. + if (brokenFrame != null) { + bytes = merge(brokenFrame.getBytes(), bytes); + utf8String = new String(bytes); + brokenFrame = null; + } + // Logger chunks can break the string in middle of multibyte unicode character. + // Backup the bytes to reconstruct proper char sequence with bytes from next frame. + int lastCharacterType = Character.getType(utf8String.charAt(utf8String.length() - 1)); + if (lastCharacterType == Character.OTHER_SYMBOL) { + brokenFrame = new OutputFrame(outputFrame.getType(), bytes); + return; + } + + utf8String = processAnsiColorCodes(utf8String, consumer); + normalizeLogLines(utf8String, consumer); + } + + private synchronized void processOtherFrame(OutputFrame outputFrame, Consumer consumer) { + String utf8String = outputFrame.getUtf8String(); + + utf8String = processAnsiColorCodes(utf8String, consumer); + consumer.accept(new OutputFrame(outputFrame.getType(), utf8String.getBytes())); + } + + private void normalizeLogLines(String utf8String, Consumer consumer) { + // Reformat strings to normalize new lines. + List lines = new ArrayList<>(Arrays.asList(utf8String.split(LINE_BREAK_REGEX))); + if (lines.isEmpty()) { + consumer.accept(new OutputFrame(OutputFrame.OutputType.STDOUT, EMPTY_LINE)); + return; + } + if (utf8String.startsWith("\n") || utf8String.startsWith("\r")) { + lines.add(0, ""); + } + if (utf8String.endsWith("\n") || utf8String.endsWith("\r")) { + lines.add(""); + } + for (int i = 0; i < lines.size() - 1; i++) { + String line = lines.get(i); + if (i == 0 && logString.length() > 0) { + line = logString.toString() + line; + logString.setLength(0); + } + consumer.accept(new OutputFrame(OutputFrame.OutputType.STDOUT, line.getBytes())); + } + logString.append(lines.get(lines.size() - 1)); + } + + private String processAnsiColorCodes(String utf8String, Consumer consumer) { + if (!(consumer instanceof BaseConsumer) || ((BaseConsumer) consumer).isRemoveColorCodes()) { + return ANSI_COLOR_PATTERN.matcher(utf8String).replaceAll(""); + } + return utf8String; + } + + + private byte[] merge(byte[] str1, byte[] str2) { + byte[] mergedString = new byte[str1.length + str2.length]; + System.arraycopy(str1, 0, mergedString, 0, str1.length); + System.arraycopy(str2, 0, mergedString, str1.length, str2.length); + return mergedString; + } } diff --git a/core/src/main/java/org/testcontainers/containers/output/Slf4jLogConsumer.java b/core/src/main/java/org/testcontainers/containers/output/Slf4jLogConsumer.java index 7b04007ef72..26b017d4a36 100644 --- a/core/src/main/java/org/testcontainers/containers/output/Slf4jLogConsumer.java +++ b/core/src/main/java/org/testcontainers/containers/output/Slf4jLogConsumer.java @@ -2,18 +2,13 @@ import org.slf4j.Logger; -import java.util.function.Consumer; -import java.util.regex.Pattern; - /** * A consumer for container output that logs output to an SLF4J logger. */ -public class Slf4jLogConsumer implements Consumer { +public class Slf4jLogConsumer extends BaseConsumer { private final Logger logger; private String prefix = ""; - private static final Pattern ANSI_CODE_PATTERN = Pattern.compile("\\[\\d[ABCD]"); - public Slf4jLogConsumer(Logger logger) { this.logger = logger; } @@ -25,28 +20,19 @@ public Slf4jLogConsumer withPrefix(String prefix) { @Override public void accept(OutputFrame outputFrame) { - if (outputFrame != null) { - String utf8String = outputFrame.getUtf8String(); - - if (utf8String != null) { - OutputFrame.OutputType outputType = outputFrame.getType(); - String message = utf8String.trim(); - - if (ANSI_CODE_PATTERN.matcher(message).matches()) { - return; - } - - switch (outputType) { - case END: - break; - case STDOUT: - case STDERR: - logger.info("{}{}: {}", prefix, outputType, message); - break; - default: - throw new IllegalArgumentException("Unexpected outputType " + outputType); - } - } + OutputFrame.OutputType outputType = outputFrame.getType(); + + String utf8String = outputFrame.getUtf8String(); + utf8String = utf8String.replaceAll(FrameConsumerResultCallback.LINE_BREAK_AT_END_REGEX, ""); + switch (outputType) { + case END: + break; + case STDOUT: + case STDERR: + logger.info("{}{}: {}", prefix, outputType, utf8String); + break; + default: + throw new IllegalArgumentException("Unexpected outputType " + outputType); } } } diff --git a/core/src/main/java/org/testcontainers/containers/output/ToStringConsumer.java b/core/src/main/java/org/testcontainers/containers/output/ToStringConsumer.java index f46e4b9a318..a12325865aa 100644 --- a/core/src/main/java/org/testcontainers/containers/output/ToStringConsumer.java +++ b/core/src/main/java/org/testcontainers/containers/output/ToStringConsumer.java @@ -5,21 +5,26 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.Charset; -import java.util.function.Consumer; /** * Created by rnorth on 26/03/2016. */ -public class ToStringConsumer implements Consumer { +public class ToStringConsumer extends BaseConsumer { + private static final byte[] NEW_LINE = "\n".getBytes(); + private boolean firstLine = true; private ByteArrayOutputStream stringBuffer = new ByteArrayOutputStream(); @Override public void accept(OutputFrame outputFrame) { try { if (outputFrame.getBytes() != null) { + if (!firstLine) { + stringBuffer.write(NEW_LINE); + } stringBuffer.write(outputFrame.getBytes()); stringBuffer.flush(); + firstLine = false; } } catch (IOException e) { throw new RuntimeException(e); diff --git a/core/src/main/java/org/testcontainers/containers/output/WaitingConsumer.java b/core/src/main/java/org/testcontainers/containers/output/WaitingConsumer.java index 3cd0a7cecca..367581528c2 100644 --- a/core/src/main/java/org/testcontainers/containers/output/WaitingConsumer.java +++ b/core/src/main/java/org/testcontainers/containers/output/WaitingConsumer.java @@ -6,14 +6,13 @@ import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.function.Consumer; import java.util.function.Predicate; /** * A consumer for container output that buffers lines in a {@link java.util.concurrent.BlockingDeque} and enables tests * to wait for a matching condition. */ -public class WaitingConsumer implements Consumer { +public class WaitingConsumer extends BaseConsumer { private static final Logger LOGGER = LoggerFactory.getLogger(WaitingConsumer.class); diff --git a/core/src/test/java/org/testcontainers/containers/output/FrameConsumerResultCallbackTest.java b/core/src/test/java/org/testcontainers/containers/output/FrameConsumerResultCallbackTest.java new file mode 100644 index 00000000000..88c6e22d6c9 --- /dev/null +++ b/core/src/test/java/org/testcontainers/containers/output/FrameConsumerResultCallbackTest.java @@ -0,0 +1,177 @@ +package org.testcontainers.containers.output; + +import com.github.dockerjava.api.model.Frame; +import com.github.dockerjava.api.model.StreamType; +import org.junit.Test; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.testcontainers.containers.output.OutputFrame.OutputType; + +public class FrameConsumerResultCallbackTest { + private static final String FRAME_PAYLOAD = "\u001B[0;32mТест1\u001B[0m\n\u001B[1;33mTest2\u001B[0m\n\u001B[0;31mTest3\u001B[0m"; + private static final String LOG_RESULT = "Тест1\nTest2\nTest3"; + + @Test + public void passStderrFrameWithoutColors() { + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + ToStringConsumer consumer = new ToStringConsumer(); + callback.addConsumer(OutputType.STDERR, consumer); + callback.onNext(new Frame(StreamType.STDERR, FRAME_PAYLOAD.getBytes())); + assertEquals(LOG_RESULT, consumer.toUtf8String()); + } + + @Test + public void passStderrFrameWithColors() { + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + ToStringConsumer consumer = new ToStringConsumer().withRemoveAnsiCodes(false); + callback.addConsumer(OutputType.STDERR, consumer); + callback.onNext(new Frame(StreamType.STDERR, FRAME_PAYLOAD.getBytes())); + assertEquals(FRAME_PAYLOAD, consumer.toUtf8String()); + } + + @Test + public void passStdoutFrameWithoutColors() { + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + ToStringConsumer consumer = new ToStringConsumer(); + callback.addConsumer(OutputType.STDOUT, consumer); + callback.onNext(new Frame(StreamType.STDOUT, FRAME_PAYLOAD.getBytes())); + assertEquals(LOG_RESULT, consumer.toUtf8String()); + } + + @Test + public void passStdoutFrameWithColors() { + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + ToStringConsumer consumer = new ToStringConsumer().withRemoveAnsiCodes(false); + callback.addConsumer(OutputType.STDOUT, consumer); + callback.onNext(new Frame(StreamType.STDOUT, FRAME_PAYLOAD.getBytes())); + assertEquals(FRAME_PAYLOAD, consumer.toUtf8String()); + } + + @Test + public void basicConsumer() { + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + BasicConsumer consumer = new BasicConsumer(); + callback.addConsumer(OutputType.STDOUT, consumer); + callback.onNext(new Frame(StreamType.STDOUT, FRAME_PAYLOAD.getBytes())); + assertEquals(LOG_RESULT, consumer.toString()); + } + + @Test + public void passStdoutNull() { + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + ToStringConsumer consumer = new ToStringConsumer().withRemoveAnsiCodes(false); + callback.addConsumer(OutputType.STDOUT, consumer); + callback.onNext(new Frame(StreamType.STDOUT, null)); + assertEquals("", consumer.toUtf8String()); + } + + @Test + public void passStdoutEmptyLine() { + String payload = ""; + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + ToStringConsumer consumer = new ToStringConsumer().withRemoveAnsiCodes(false); + callback.addConsumer(OutputType.STDOUT, consumer); + callback.onNext(new Frame(StreamType.STDOUT, payload.getBytes())); + assertEquals(payload, consumer.toUtf8String()); + } + + @Test + public void passStdoutSingleLine() { + String payload = "Test"; + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + ToStringConsumer consumer = new ToStringConsumer().withRemoveAnsiCodes(false); + callback.addConsumer(OutputType.STDOUT, consumer); + callback.onNext(new Frame(StreamType.STDOUT, payload.getBytes())); + assertEquals(payload, consumer.toUtf8String()); + } + + @Test + public void passStdoutSingleLineWithNewline() { + String payload = "Test\n"; + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + ToStringConsumer consumer = new ToStringConsumer().withRemoveAnsiCodes(false); + callback.addConsumer(OutputType.STDOUT, consumer); + callback.onNext(new Frame(StreamType.STDOUT, payload.getBytes())); + assertEquals(payload, consumer.toUtf8String()); + } + + @Test + public void passRawFrameWithoutColors() throws TimeoutException, IOException { + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + WaitingConsumer waitConsumer = new WaitingConsumer(); + callback.addConsumer(OutputType.STDOUT, waitConsumer); + callback.onNext(new Frame(StreamType.RAW, FRAME_PAYLOAD.getBytes())); + waitConsumer.waitUntil(frame -> frame.getType() == OutputType.STDOUT && frame.getUtf8String().equals("Test2"), 1, TimeUnit.SECONDS); + waitConsumer.waitUntil(frame -> frame.getType() == OutputType.STDOUT && frame.getUtf8String().equals("Тест1"), 1, TimeUnit.SECONDS); + Exception exception = null; + try { + waitConsumer.waitUntil(frame -> frame.getType() == OutputType.STDOUT && frame.getUtf8String().equals("Test3"), 1, TimeUnit.SECONDS); + } catch (Exception e) { + exception = e; + } + assertTrue(exception instanceof TimeoutException); + callback.close(); + waitConsumer.waitUntil(frame -> frame.getType() == OutputType.STDOUT && frame.getUtf8String().equals("Test3"), 1, TimeUnit.SECONDS); + } + + @Test + public void passRawFrameWithColors() throws TimeoutException, IOException { + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + WaitingConsumer waitConsumer = new WaitingConsumer().withRemoveAnsiCodes(false); + callback.addConsumer(OutputType.STDOUT, waitConsumer); + callback.onNext(new Frame(StreamType.RAW, FRAME_PAYLOAD.getBytes())); + waitConsumer.waitUntil(frame -> frame.getType() == OutputType.STDOUT && frame.getUtf8String().equals("\u001B[1;33mTest2\u001B[0m"), 1, TimeUnit.SECONDS); + waitConsumer.waitUntil(frame -> frame.getType() == OutputType.STDOUT && frame.getUtf8String().equals("\u001B[0;32mТест1\u001B[0m"), 1, TimeUnit.SECONDS); + Exception exception = null; + try { + waitConsumer.waitUntil(frame -> frame.getType() == OutputType.STDOUT && frame.getUtf8String().equals("\u001B[0;31mTest3\u001B[0m"), 1, TimeUnit.SECONDS); + } catch (Exception e) { + exception = e; + } + assertTrue(exception instanceof TimeoutException); + callback.close(); + waitConsumer.waitUntil(frame -> frame.getType() == OutputType.STDOUT && frame.getUtf8String().equals("\u001B[0;31mTest3\u001B[0m"), 1, TimeUnit.SECONDS); + } + + @Test + public void reconstructBreakedUnicode() throws IOException { + String payload = "Тест"; + byte[] payloadBytes = payload.getBytes(); + byte[] bytes1 = new byte[5]; + byte[] bytes2 = new byte[3]; + System.arraycopy(payloadBytes, 0, bytes1, 0, 5); + System.arraycopy(payloadBytes, 5, bytes2, 0, 3); + FrameConsumerResultCallback callback = new FrameConsumerResultCallback(); + ToStringConsumer consumer = new ToStringConsumer().withRemoveAnsiCodes(false); + callback.addConsumer(OutputType.STDOUT, consumer); + callback.onNext(new Frame(StreamType.RAW, bytes1)); + callback.onNext(new Frame(StreamType.RAW, bytes2)); + callback.close(); + assertEquals(payload, consumer.toUtf8String()); + } + + private static class BasicConsumer implements Consumer { + private boolean firstLine = true; + private StringBuilder input = new StringBuilder(); + + @Override + public void accept(OutputFrame outputFrame) { + if (!firstLine) { + input.append('\n'); + } + firstLine = false; + input.append(outputFrame.getUtf8String()); + } + + @Override + public String toString() { + return input.toString(); + } + } +}