From 44e8e9ab279cad2cd9d0c9fc5bb6e7799c409999 Mon Sep 17 00:00:00 2001 From: Richard North Date: Thu, 5 Nov 2020 19:19:08 +0000 Subject: [PATCH] Image substitution (#3102) * Refactor Testcontainers configuration to allow config by env var * Add Image substitution mechanism Builds upon #3021 and #3411: * adds a pluggable image substitution mechanism using ServiceLoader, enabling users to perform custom substitution/auditing of images being used by their tests * provides a default implementation that behaves similarly to legacy `TestcontainersConfiguration` approach (`testcontainers.properties`) Notes: * behaviour is similar but not quite identical to `TestcontainersConfiguration`: use of a configured custom image for, e.g. Kafka/Pulsar that does not have a tag specified causes the substitution to take effect for all usages. It seems very unlikely that people would be using a mix of the config file image overrides in some places _and_ specific images specified in code in others. * Duplication of default image names in modules vs `TestcontainersConfiguration` class is intentional: specifying image overrides in `testcontainers.properties` should be removed in the future. * ~Add log deprecation warnings when `testcontainers.properties` image overrides are used.~ Defer to a future release? * Remove extraneous change * Un-ignore docs example test by implementing a 'reversing' image name substitutor * Use configuration, not service loader, to select an ImageNameSubstitutor * Add check for order of config setting precedence * Extract classpath scanner and support finding of multiple resources * Introduce deterministic merging of classpath properties files * Update docs * Update docs * Remove service loader reference * Chain substitution through default and configured implementations * Small tweaks following review * Fix test compile error * Add UnstableAPI annotation * Move TestSpecificImageNameSubstitutor back to original package and remove duplicate use of default substitutor --- .../testcontainers/DockerClientFactory.java | 11 +- .../containers/DockerComposeContainer.java | 5 +- .../containers/GenericContainer.java | 6 +- .../containers/PortForwardingContainer.java | 6 +- .../containers/SocatContainer.java | 8 +- .../containers/VncRecordingContainer.java | 4 +- .../DockerClientProviderStrategy.java | 2 +- .../images/RemoteDockerImage.java | 10 +- .../utility/ClasspathScanner.java | 59 ++++++++ ...ConfigurationFileImageNameSubstitutor.java | 44 ++++++ .../utility/DefaultImageNameSubstitutor.java | 35 +++++ .../utility/ImageNameSubstitutor.java | 130 ++++++++++++++++++ .../utility/ResourceReaper.java | 4 +- .../utility/TestcontainersConfiguration.java | 68 +++------ .../DockerClientFactoryTest.java | 9 +- .../java/org/testcontainers/TestImages.java | 3 +- .../utility/AuthenticatedImagePullTest.java | 26 ++-- .../utility/ClasspathScannerTest.java | 114 +++++++++++++++ .../DefaultImageNameSubstitutorTest.java | 38 +++++ .../DockerImageNameCompatibilityTest.java | 3 +- .../utility/FakeImageSubstitutor.java | 13 ++ .../utility/ImageNameSubstitutorTest.java | 75 ++++++++++ .../TestcontainersConfigurationTest.java | 63 +++++++++ .../test/resources/expectedClasspathFile.txt | 1 + docs/examples/junit4/generic/build.gradle | 3 + .../generic/ExampleImageNameSubstitutor.java | 24 ++++ .../generic/ImageNameSubstitutionTest.java | 48 +++++++ .../TestSpecificImageNameSubstitutor.java | 26 ++++ .../src/test/resources/logback-test.xml | 14 ++ .../test/resources/testcontainers.properties | 1 + docs/features/configuration.md | 45 ++++-- docs/features/image_name_substitution.md | 117 ++++++++++++++++ docs/features/pull_rate_limiting.md | 19 +++ .../containers/KafkaContainer.java | 5 +- .../containers/KafkaContainerTest.java | 2 +- .../localstack/LocalStackContainer.java | 5 +- .../containers/OracleContainer.java | 3 +- .../containers/PulsarContainer.java | 5 +- .../spock/SpockTestImages.groovy | 5 +- 39 files changed, 936 insertions(+), 123 deletions(-) create mode 100644 core/src/main/java/org/testcontainers/utility/ClasspathScanner.java create mode 100644 core/src/main/java/org/testcontainers/utility/ConfigurationFileImageNameSubstitutor.java create mode 100644 core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java create mode 100644 core/src/main/java/org/testcontainers/utility/ImageNameSubstitutor.java create mode 100644 core/src/test/java/org/testcontainers/utility/ClasspathScannerTest.java create mode 100644 core/src/test/java/org/testcontainers/utility/DefaultImageNameSubstitutorTest.java create mode 100644 core/src/test/java/org/testcontainers/utility/FakeImageSubstitutor.java create mode 100644 core/src/test/java/org/testcontainers/utility/ImageNameSubstitutorTest.java create mode 100644 core/src/test/resources/expectedClasspathFile.txt create mode 100644 docs/examples/junit4/generic/src/test/java/generic/ExampleImageNameSubstitutor.java create mode 100644 docs/examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java create mode 100644 docs/examples/junit4/generic/src/test/java/generic/support/TestSpecificImageNameSubstitutor.java create mode 100644 docs/examples/junit4/generic/src/test/resources/logback-test.xml create mode 100644 docs/examples/junit4/generic/src/test/resources/testcontainers.properties create mode 100644 docs/features/image_name_substitution.md create mode 100644 docs/features/pull_rate_limiting.md diff --git a/core/src/main/java/org/testcontainers/DockerClientFactory.java b/core/src/main/java/org/testcontainers/DockerClientFactory.java index 571b18fbcce..d44cd3b5b6f 100644 --- a/core/src/main/java/org/testcontainers/DockerClientFactory.java +++ b/core/src/main/java/org/testcontainers/DockerClientFactory.java @@ -25,6 +25,8 @@ import org.testcontainers.dockerclient.TransportConfig; import org.testcontainers.images.TimeLimitedLoggedPullImageResultCallback; import org.testcontainers.utility.ComparableVersion; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.ImageNameSubstitutor; import org.testcontainers.utility.MountableFile; import org.testcontainers.utility.ResourceReaper; import org.testcontainers.utility.TestcontainersConfiguration; @@ -61,7 +63,7 @@ public class DockerClientFactory { TESTCONTAINERS_SESSION_ID_LABEL, SESSION_ID ); - private static final String TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString(); + private static final DockerImageName TINY_IMAGE = DockerImageName.parse("alpine:3.5"); private static DockerClientFactory instance; // Cached client configuration @@ -343,8 +345,11 @@ public T runInsideDocker(Consumer createContainerCmdCons } private T runInsideDocker(DockerClient client, Consumer createContainerCmdConsumer, BiFunction block) { - checkAndPullImage(client, TINY_IMAGE); - CreateContainerCmd createContainerCmd = client.createContainerCmd(TINY_IMAGE) + + final String tinyImage = ImageNameSubstitutor.instance().apply(TINY_IMAGE).asCanonicalNameString(); + + checkAndPullImage(client, tinyImage); + CreateContainerCmd createContainerCmd = client.createContainerCmd(tinyImage) .withLabels(DEFAULT_LABELS); createContainerCmdConsumer.accept(createContainerCmd); String id = createContainerCmd.exec().getId(); diff --git a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java index bd06083644d..7a67073e383 100644 --- a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java +++ b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java @@ -29,11 +29,11 @@ import org.testcontainers.utility.AuditLogger; import org.testcontainers.utility.Base58; import org.testcontainers.utility.CommandLine; +import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.DockerLoggerFactory; import org.testcontainers.utility.LogUtils; import org.testcontainers.utility.MountableFile; import org.testcontainers.utility.ResourceReaper; -import org.testcontainers.utility.TestcontainersConfiguration; import org.zeroturnaround.exec.InvalidExitValueException; import org.zeroturnaround.exec.ProcessExecutor; import org.zeroturnaround.exec.stream.slf4j.Slf4jStream; @@ -608,10 +608,11 @@ interface DockerCompose { class ContainerisedDockerCompose extends GenericContainer implements DockerCompose { public static final char UNIX_PATH_SEPERATOR = ':'; + public static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("docker/compose:1.24.1"); public ContainerisedDockerCompose(List composeFiles, String identifier) { - super(TestcontainersConfiguration.getInstance().getDockerComposeDockerImageName()); + super(DEFAULT_IMAGE_NAME); addEnv(ENV_PROJECT_NAME, identifier); // Map the docker compose file into the container diff --git a/core/src/main/java/org/testcontainers/containers/GenericContainer.java b/core/src/main/java/org/testcontainers/containers/GenericContainer.java index 95050d8f8ee..641096a0779 100644 --- a/core/src/main/java/org/testcontainers/containers/GenericContainer.java +++ b/core/src/main/java/org/testcontainers/containers/GenericContainer.java @@ -239,13 +239,9 @@ public GenericContainer(@NonNull final RemoteDockerImage image) { */ @Deprecated public GenericContainer() { - this(TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString()); + this(TestcontainersConfiguration.getInstance().getTinyImage()); } - /** - * @deprecated use {@link GenericContainer(DockerImageName)} instead - */ - @Deprecated public GenericContainer(@NonNull final String dockerImageName) { this.setDockerImageName(dockerImageName); } diff --git a/core/src/main/java/org/testcontainers/containers/PortForwardingContainer.java b/core/src/main/java/org/testcontainers/containers/PortForwardingContainer.java index e42f2681675..b3f020500a1 100644 --- a/core/src/main/java/org/testcontainers/containers/PortForwardingContainer.java +++ b/core/src/main/java/org/testcontainers/containers/PortForwardingContainer.java @@ -5,15 +5,15 @@ import lombok.AccessLevel; import lombok.Getter; import lombok.SneakyThrows; -import org.testcontainers.utility.TestcontainersConfiguration; +import org.testcontainers.utility.DockerImageName; import java.time.Duration; import java.util.AbstractMap; import java.util.Collections; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; -import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; public enum PortForwardingContainer { @@ -29,7 +29,7 @@ public enum PortForwardingContainer { @SneakyThrows private Connection createSSHSession() { String password = UUID.randomUUID().toString(); - container = new GenericContainer<>(TestcontainersConfiguration.getInstance().getSSHdDockerImageName()) + container = new GenericContainer<>(DockerImageName.parse("testcontainers/sshd:1.0.0")) .withExposedPorts(22) .withEnv("PASSWORD", password) .withCommand( diff --git a/core/src/main/java/org/testcontainers/containers/SocatContainer.java b/core/src/main/java/org/testcontainers/containers/SocatContainer.java index fbca73eb29d..7592d9f4d1f 100644 --- a/core/src/main/java/org/testcontainers/containers/SocatContainer.java +++ b/core/src/main/java/org/testcontainers/containers/SocatContainer.java @@ -1,12 +1,10 @@ package org.testcontainers.containers; -import org.testcontainers.utility.Base58; -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.TestcontainersConfiguration; - import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; +import org.testcontainers.utility.Base58; +import org.testcontainers.utility.DockerImageName; /** * A socat container is used as a TCP proxy, enabling any TCP port of another container to be exposed @@ -17,7 +15,7 @@ public class SocatContainer extends GenericContainer { private final Map targets = new HashMap<>(); public SocatContainer() { - this(TestcontainersConfiguration.getInstance().getSocatDockerImageName()); + this(DockerImageName.parse("alpine/socat:1.7.3.4-r0")); } public SocatContainer(final DockerImageName dockerImageName) { diff --git a/core/src/main/java/org/testcontainers/containers/VncRecordingContainer.java b/core/src/main/java/org/testcontainers/containers/VncRecordingContainer.java index 8ac060c2af6..de720110a51 100644 --- a/core/src/main/java/org/testcontainers/containers/VncRecordingContainer.java +++ b/core/src/main/java/org/testcontainers/containers/VncRecordingContainer.java @@ -6,7 +6,7 @@ import lombok.ToString; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; -import org.testcontainers.utility.TestcontainersConfiguration; +import org.testcontainers.utility.DockerImageName; import java.io.File; import java.io.InputStream; @@ -52,7 +52,7 @@ public VncRecordingContainer(@NonNull GenericContainer targetContainer) { * Create a sidekick container and attach it to another container. The VNC output of that container will be recorded. */ public VncRecordingContainer(@NonNull Network network, @NonNull String targetNetworkAlias) throws IllegalStateException { - super(TestcontainersConfiguration.getInstance().getVncDockerImageName()); + super(DockerImageName.parse("testcontainers/vnc-recorder:1.1.0")); this.targetNetworkAlias = targetNetworkAlias; withNetwork(network); diff --git a/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java b/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java index e21c75a581d..1bd51da1263 100644 --- a/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java +++ b/core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java @@ -171,7 +171,7 @@ public static DockerClientProviderStrategy getFirstValidStrategy(List imageFuture) { @@ -100,7 +101,10 @@ protected final String resolve() { } private DockerImageName getImageName() throws InterruptedException, ExecutionException { - return imageNameFuture.get(); + final DockerImageName specifiedImageName = imageNameFuture.get(); + + // Allow the image name to be substituted + return ImageNameSubstitutor.instance().apply(specifiedImageName); } @ToString.Include(name = "imageName", rank = 1) diff --git a/core/src/main/java/org/testcontainers/utility/ClasspathScanner.java b/core/src/main/java/org/testcontainers/utility/ClasspathScanner.java new file mode 100644 index 00000000000..76716cfdb7c --- /dev/null +++ b/core/src/main/java/org/testcontainers/utility/ClasspathScanner.java @@ -0,0 +1,59 @@ +package org.testcontainers.utility; + +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.Nullable; +import org.jetbrains.annotations.VisibleForTesting; + +import java.net.URL; +import java.util.Collections; +import java.util.Comparator; +import java.util.Objects; +import java.util.stream.Stream; + +/** + * Utility for identifying resource files on classloaders. + */ +@Slf4j +class ClasspathScanner { + + @VisibleForTesting + static Stream scanFor(final String name, ClassLoader... classLoaders) { + return Stream + .of(classLoaders) + .flatMap(classLoader -> getAllPropertyFilesOnClassloader(classLoader, name)) + .filter(Objects::nonNull) + .sorted( + Comparator + .comparing(ClasspathScanner::filesFileSchemeFirst) // resolve 'local' files first + .thenComparing(URL::toString) // sort alphabetically for the sake of determinism + ) + .distinct(); + } + + private static Integer filesFileSchemeFirst(final URL t) { + return t.getProtocol().equals("file") ? 0 : 1; + } + + /** + * @param name the resource name to search for + * @return distinct, ordered stream of resources found by searching this class' classloader and the current thread's + * context classloader. Results are currently alphabetically sorted. + */ + static Stream scanFor(final String name) { + return scanFor( + name, + ClasspathScanner.class.getClassLoader(), + Thread.currentThread().getContextClassLoader() + ); + } + + @Nullable + private static Stream getAllPropertyFilesOnClassloader(final ClassLoader it, final String s) { + try { + return Collections.list(it.getResources(s)).stream(); + } catch (Exception e) { + log.error("Unable to read configuration from classloader {} - this is probably a bug", it, e); + return Stream.empty(); + } + } +} diff --git a/core/src/main/java/org/testcontainers/utility/ConfigurationFileImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/ConfigurationFileImageNameSubstitutor.java new file mode 100644 index 00000000000..530d5a76118 --- /dev/null +++ b/core/src/main/java/org/testcontainers/utility/ConfigurationFileImageNameSubstitutor.java @@ -0,0 +1,44 @@ +package org.testcontainers.utility; + +import com.google.common.annotations.VisibleForTesting; +import lombok.extern.slf4j.Slf4j; + +/** + * {@link ImageNameSubstitutor} which takes replacement image names from configuration. + * See {@link TestcontainersConfiguration} for the subset of image names which can be substituted using this mechanism. + */ +@Slf4j +final class ConfigurationFileImageNameSubstitutor extends ImageNameSubstitutor { + + private final TestcontainersConfiguration configuration; + + public ConfigurationFileImageNameSubstitutor() { + this(TestcontainersConfiguration.getInstance()); + } + + @VisibleForTesting + ConfigurationFileImageNameSubstitutor(TestcontainersConfiguration configuration) { + this.configuration = configuration; + } + + @Override + public DockerImageName apply(final DockerImageName original) { + final DockerImageName result = configuration + .getConfiguredSubstituteImage(original) + .asCompatibleSubstituteFor(original); + + if (!result.equals(original)) { + log.warn("Image name {} was substituted by configuration to {}. This approach is deprecated and will be removed in the future", + original, + result + ); + } + + return result; + } + + @Override + protected String getDescription() { + return getClass().getSimpleName(); + } +} diff --git a/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java new file mode 100644 index 00000000000..bcc7c96ba18 --- /dev/null +++ b/core/src/main/java/org/testcontainers/utility/DefaultImageNameSubstitutor.java @@ -0,0 +1,35 @@ +package org.testcontainers.utility; + +import com.google.common.annotations.VisibleForTesting; +import lombok.extern.slf4j.Slf4j; + +/** + * Testcontainers' default implementation of {@link ImageNameSubstitutor}. + * Delegates to {@link ConfigurationFileImageNameSubstitutor}. + */ +@Slf4j +final class DefaultImageNameSubstitutor extends ImageNameSubstitutor { + + private final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor; + + public DefaultImageNameSubstitutor() { + configurationFileImageNameSubstitutor = new ConfigurationFileImageNameSubstitutor(); + } + + @VisibleForTesting + DefaultImageNameSubstitutor( + final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor + ) { + this.configurationFileImageNameSubstitutor = configurationFileImageNameSubstitutor; + } + + @Override + public DockerImageName apply(final DockerImageName original) { + return configurationFileImageNameSubstitutor.apply(original); + } + + @Override + protected String getDescription() { + return "DefaultImageNameSubstitutor (" + configurationFileImageNameSubstitutor.getDescription() + ")"; + } +} diff --git a/core/src/main/java/org/testcontainers/utility/ImageNameSubstitutor.java b/core/src/main/java/org/testcontainers/utility/ImageNameSubstitutor.java new file mode 100644 index 00000000000..645b717a562 --- /dev/null +++ b/core/src/main/java/org/testcontainers/utility/ImageNameSubstitutor.java @@ -0,0 +1,130 @@ +package org.testcontainers.utility; + +import com.google.common.annotations.VisibleForTesting; +import lombok.extern.slf4j.Slf4j; +import org.testcontainers.UnstableAPI; + +import java.util.function.Function; + +/** + * An image name substitutor converts a Docker image name, as may be specified in code, to an alternative name. + * This is intended to provide a way to override image names, for example to enforce pulling of images from a private + * registry. + *

+ * This is marked as @{@link UnstableAPI} as this API is new. While we do not think major changes will be required, we + * will react to feedback if necessary. + */ +@Slf4j +@UnstableAPI +public abstract class ImageNameSubstitutor implements Function { + + @VisibleForTesting + static ImageNameSubstitutor instance; + + @VisibleForTesting + static ImageNameSubstitutor defaultImplementation = new DefaultImageNameSubstitutor(); + + public synchronized static ImageNameSubstitutor instance() { + if (instance == null) { + final String configuredClassName = TestcontainersConfiguration.getInstance().getImageSubstitutorClassName(); + + if (configuredClassName != null) { + log.debug("Attempting to instantiate an ImageNameSubstitutor with class: {}", configuredClassName); + ImageNameSubstitutor configuredInstance; + try { + configuredInstance = (ImageNameSubstitutor) Class.forName(configuredClassName).getConstructor().newInstance(); + } catch (Exception e) { + throw new IllegalArgumentException("Configured Image Substitutor could not be loaded: " + configuredClassName, e); + } + + log.info("Found configured ImageNameSubstitutor: {}", configuredInstance.getDescription()); + + instance = new ChainedImageNameSubstitutor( + wrapWithLogging(defaultImplementation), + wrapWithLogging(configuredInstance) + ); + } else { + instance = wrapWithLogging(defaultImplementation); + } + + log.info("Image name substitution will be performed by: {}", instance.getDescription()); + } + + return instance; + } + + private static ImageNameSubstitutor wrapWithLogging(final ImageNameSubstitutor wrappedInstance) { + return new LogWrappedImageNameSubstitutor(wrappedInstance); + } + + /** + * Substitute a {@link DockerImageName} for another, for example to replace a generic Docker Hub image name with a + * private registry copy of the image. + * + * @param original original name to be replaced + * @return a replacement name, or the original, as appropriate + */ + public abstract DockerImageName apply(DockerImageName original); + + /** + * @return a human-readable description of the substitutor + */ + protected abstract String getDescription(); + + /** + * Wrapper substitutor which logs which substitutions have been performed. + */ + static class LogWrappedImageNameSubstitutor extends ImageNameSubstitutor { + @VisibleForTesting + final ImageNameSubstitutor wrappedInstance; + + public LogWrappedImageNameSubstitutor(final ImageNameSubstitutor wrappedInstance) { + this.wrappedInstance = wrappedInstance; + } + + @Override + public DockerImageName apply(final DockerImageName original) { + final DockerImageName replacementImage = wrappedInstance.apply(original); + + if (!replacementImage.equals(original)) { + log.info("Using {} as a substitute image for {} (using image substitutor: {})", replacementImage.asCanonicalNameString(), original.asCanonicalNameString(), wrappedInstance.getDescription()); + return replacementImage; + } else { + log.debug("Did not find a substitute image for {} (using image substitutor: {})", original.asCanonicalNameString(), wrappedInstance.getDescription()); + return original; + } + } + + @Override + protected String getDescription() { + return wrappedInstance.getDescription(); + } + } + + /** + * Wrapper substitutor that passes the original image name through a default substitutor and then the configured one + */ + static class ChainedImageNameSubstitutor extends ImageNameSubstitutor { + private final ImageNameSubstitutor defaultInstance; + private final ImageNameSubstitutor configuredInstance; + + public ChainedImageNameSubstitutor(ImageNameSubstitutor defaultInstance, ImageNameSubstitutor configuredInstance) { + this.defaultInstance = defaultInstance; + this.configuredInstance = configuredInstance; + } + + @Override + public DockerImageName apply(DockerImageName original) { + return defaultInstance.andThen(configuredInstance).apply(original); + } + + @Override + protected String getDescription() { + return String.format( + "Chained substitutor of '%s' and then '%s'", + defaultInstance.getDescription(), + configuredInstance.getDescription() + ); + } + } +} diff --git a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java index c37e6660654..3b0cfa1e289 100644 --- a/core/src/main/java/org/testcontainers/utility/ResourceReaper.java +++ b/core/src/main/java/org/testcontainers/utility/ResourceReaper.java @@ -72,7 +72,9 @@ private ResourceReaper() { @SneakyThrows(InterruptedException.class) public static String start(String hostIpAddress, DockerClient client) { - String ryukImage = TestcontainersConfiguration.getInstance().getRyukDockerImageName().asCanonicalNameString(); + String ryukImage = ImageNameSubstitutor.instance() + .apply(DockerImageName.parse("testcontainers/ryuk:0.3.0")) + .asCanonicalNameString(); DockerClientFactory.instance().checkAndPullImage(client, ryukImage); List binds = new ArrayList<>(); diff --git a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java index b3c8e88a328..987301fbfb9 100644 --- a/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java +++ b/core/src/main/java/org/testcontainers/utility/TestcontainersConfiguration.java @@ -23,7 +23,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Properties; import java.util.concurrent.atomic.AtomicReference; @@ -104,37 +103,21 @@ public String getSocatContainerImage() { return getImage(SOCAT_IMAGE).asCanonicalNameString(); } - public DockerImageName getSocatDockerImageName() { - return getImage(SOCAT_IMAGE); - } - @Deprecated public String getVncRecordedContainerImage() { return getImage(VNC_RECORDER_IMAGE).asCanonicalNameString(); } - public DockerImageName getVncDockerImageName() { - return getImage(VNC_RECORDER_IMAGE); - } - @Deprecated public String getDockerComposeContainerImage() { return getImage(COMPOSE_IMAGE).asCanonicalNameString(); } - public DockerImageName getDockerComposeDockerImageName() { - return getImage(COMPOSE_IMAGE); - } - @Deprecated public String getTinyImage() { return getImage(ALPINE_IMAGE).asCanonicalNameString(); } - public DockerImageName getTinyDockerImageName() { - return getImage(ALPINE_IMAGE); - } - public boolean isRyukPrivileged() { return Boolean .parseBoolean(getEnvVarOrProperty("ryuk.container.privileged", "false")); @@ -145,19 +128,11 @@ public String getRyukImage() { return getImage(RYUK_IMAGE).asCanonicalNameString(); } - public DockerImageName getRyukDockerImageName() { - return getImage(RYUK_IMAGE); - } - @Deprecated public String getSSHdImage() { return getImage(SSHD_IMAGE).asCanonicalNameString(); } - public DockerImageName getSSHdDockerImageName() { - return getImage(SSHD_IMAGE); - } - public Integer getRyukTimeout() { return Integer.parseInt(getEnvVarOrProperty("ryuk.container.timeout", "30")); } @@ -167,11 +142,6 @@ public String getKafkaImage() { return getImage(KAFKA_IMAGE).asCanonicalNameString(); } - public DockerImageName getKafkaDockerImageName() { - return getImage(KAFKA_IMAGE); - } - - @Deprecated public String getOracleImage() { return getEnvVarOrUserProperty("oracle.container.image", null); @@ -182,20 +152,11 @@ public String getPulsarImage() { return getImage(PULSAR_IMAGE).asCanonicalNameString(); } - public DockerImageName getPulsarDockerImageName() { - return getImage(PULSAR_IMAGE); - } - @Deprecated public String getLocalStackImage() { return getImage(LOCALSTACK_IMAGE).asCanonicalNameString(); } - public DockerImageName getLocalstackDockerImageName() { - return getImage(LOCALSTACK_IMAGE); - } - - public boolean isDisableChecks() { return Boolean.parseBoolean(getEnvVarOrUserProperty("checks.disable", "false")); } @@ -218,6 +179,10 @@ public Integer getImagePullPauseTimeout() { return Integer.parseInt(getEnvVarOrProperty("pull.pause.timeout", "30")); } + public String getImageSubstitutorClassName() { + return getEnvVarOrProperty("image.substitutor", null); + } + @Nullable @Contract("_, !null, _ -> !null") private String getConfigurable(@NotNull final String propertyName, @Nullable final String defaultValue, Properties... propertiesSources) { @@ -279,6 +244,13 @@ public String getUserProperty(@NotNull final String propertyName, @Nullable fina return getConfigurable(propertyName, defaultValue); } + /** + * @return properties values available from user properties and classpath properties. Values set by environment + * variable are NOT included. + * @deprecated usages should be removed ASAP. See {@link TestcontainersConfiguration#getEnvVarOrProperty(String, String)}, + * {@link TestcontainersConfiguration#getEnvVarOrUserProperty(String, String)} or {@link TestcontainersConfiguration#getUserProperty(String, String)} + * for suitable replacements. + */ @Deprecated public Properties getProperties() { return Stream.of(userProperties, classpathProperties) @@ -320,17 +292,13 @@ public boolean updateUserConfig(@NonNull String prop, @NonNull String value) { private static TestcontainersConfiguration loadConfiguration() { return new TestcontainersConfiguration( readProperties(USER_CONFIG_FILE.toURI().toURL()), - Stream - .of( - TestcontainersConfiguration.class.getClassLoader(), - Thread.currentThread().getContextClassLoader() - ) - .map(it -> it.getResource(PROPERTIES_FILE_NAME)) - .filter(Objects::nonNull) + ClasspathScanner.scanFor(PROPERTIES_FILE_NAME) .map(TestcontainersConfiguration::readProperties) .reduce(new Properties(), (a, b) -> { - a.putAll(b); - return a; + // first-write-wins merging - URLs appearing first on the classpath alphabetically will take priority. + // Note that this means that file: URLs will always take priority over jar: URLs. + b.putAll(a); + return b; }), System.getenv()); } @@ -341,9 +309,9 @@ private static Properties readProperties(URL url) { try (InputStream inputStream = url.openStream()) { properties.load(inputStream); } catch (FileNotFoundException e) { - log.warn("Testcontainers config override was found on {} but the file was not found. Exception message: {}", url, ExceptionUtils.getRootCauseMessage(e)); + log.warn("Attempted to read Testcontainers configuration file at {} but the file was not found. Exception message: {}", url, ExceptionUtils.getRootCauseMessage(e)); } catch (IOException e) { - log.warn("Testcontainers config override was found on {} but could not be loaded. Exception message: {}", url, ExceptionUtils.getRootCauseMessage(e)); + log.warn("Attempted to read Testcontainers configuration file at {} but could it not be loaded. Exception message: {}", url, ExceptionUtils.getRootCauseMessage(e)); } return properties; } diff --git a/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java b/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java index 030e4164f00..d484cc92eaf 100644 --- a/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java +++ b/core/src/test/java/org/testcontainers/DockerClientFactoryTest.java @@ -1,5 +1,9 @@ package org.testcontainers; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.testcontainers.TestImages.TINY_IMAGE; + import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.exception.DockerException; import com.github.dockerjava.api.exception.NotFoundException; @@ -12,9 +16,6 @@ import org.testcontainers.utility.MockTestcontainersConfigurationRule; import org.testcontainers.utility.TestcontainersConfiguration; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - /** * Test for {@link DockerClientFactory}. */ @@ -30,7 +31,7 @@ public void runCommandInsideDockerShouldNotFailIfImageDoesNotExistsLocally() { try { //remove tiny image, so it will be pulled during next command run dockFactory.client() - .removeImageCmd(TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString()) + .removeImageCmd(TINY_IMAGE.asCanonicalNameString()) .withForce(true).exec(); } catch (NotFoundException ignored) { // Do not fail if it's not pulled yet diff --git a/core/src/test/java/org/testcontainers/TestImages.java b/core/src/test/java/org/testcontainers/TestImages.java index 3f16e88d882..7052f7e8f9d 100644 --- a/core/src/test/java/org/testcontainers/TestImages.java +++ b/core/src/test/java/org/testcontainers/TestImages.java @@ -1,7 +1,6 @@ package org.testcontainers; import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.TestcontainersConfiguration; public interface TestImages { DockerImageName REDIS_IMAGE = DockerImageName.parse("redis:3.0.2"); @@ -9,5 +8,5 @@ public interface TestImages { DockerImageName MONGODB_IMAGE = DockerImageName.parse("mongo:3.1.5"); DockerImageName ALPINE_IMAGE = DockerImageName.parse("alpine:3.2"); DockerImageName DOCKER_REGISTRY_IMAGE = DockerImageName.parse("registry:2.7.0"); - DockerImageName TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyDockerImageName(); + DockerImageName TINY_IMAGE = DockerImageName.parse("alpine:3.5"); } diff --git a/core/src/test/java/org/testcontainers/utility/AuthenticatedImagePullTest.java b/core/src/test/java/org/testcontainers/utility/AuthenticatedImagePullTest.java index e49ff2880df..c99770c4fbf 100644 --- a/core/src/test/java/org/testcontainers/utility/AuthenticatedImagePullTest.java +++ b/core/src/test/java/org/testcontainers/utility/AuthenticatedImagePullTest.java @@ -1,9 +1,21 @@ package org.testcontainers.utility; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; +import static org.rnorth.visibleassertions.VisibleAssertions.assertTrue; +import static org.testcontainers.TestImages.DOCKER_REGISTRY_IMAGE; +import static org.testcontainers.TestImages.TINY_IMAGE; + import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.async.ResultCallback; import com.github.dockerjava.api.command.PullImageResultCallback; import com.github.dockerjava.api.model.AuthConfig; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; import org.intellij.lang.annotations.Language; import org.junit.AfterClass; import org.junit.Before; @@ -18,18 +30,6 @@ import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; import org.testcontainers.images.builder.ImageFromDockerfile; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; -import static org.rnorth.visibleassertions.VisibleAssertions.assertTrue; -import static org.testcontainers.TestImages.DOCKER_REGISTRY_IMAGE; - /** * This test checks the integration between Testcontainers and an authenticated registry, but uses * a mock instance of {@link RegistryAuthLocator} - the purpose of the test is solely to ensure that @@ -165,7 +165,7 @@ private Path getLocalTempFile(String s) throws IOException { private static void putImageInRegistry() throws InterruptedException { // It doesn't matter which image we use for this test, but use one that's likely to have been pulled already - final String dummySourceImage = TestcontainersConfiguration.getInstance().getRyukDockerImageName().asCanonicalNameString(); + final String dummySourceImage = TINY_IMAGE.asCanonicalNameString(); client.pullImageCmd(dummySourceImage) .exec(new PullImageResultCallback()) diff --git a/core/src/test/java/org/testcontainers/utility/ClasspathScannerTest.java b/core/src/test/java/org/testcontainers/utility/ClasspathScannerTest.java new file mode 100644 index 00000000000..bdd71b0db9f --- /dev/null +++ b/core/src/test/java/org/testcontainers/utility/ClasspathScannerTest.java @@ -0,0 +1,114 @@ +package org.testcontainers.utility; + +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.net.URL; +import java.util.Collections; +import java.util.List; + +import static java.util.Arrays.asList; +import static java.util.stream.Collectors.toList; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; + +public class ClasspathScannerTest { + + private static URL FILE_A; + private static URL FILE_B; + private static URL JAR_A; + private static URL JAR_B; + private static URL FILE_C; + + @BeforeClass + public static void setUp() throws Exception { + FILE_A = new URL("file:///a/someName"); + FILE_B = new URL("file:///b/someName"); + FILE_C = new URL("file:///c/someName"); + JAR_A = new URL("jar:file:a!/someName"); + JAR_B = new URL("jar:file:b!/someName"); + } + + @Test + public void realClassLoaderLookupOccurs() { + // look for a resource that we know exists only once + final List foundURLs = ClasspathScanner.scanFor("expectedClasspathFile.txt").collect(toList()); + + assertEquals("Exactly one resource was found", 1, foundURLs.size()); + } + + @Test + public void multipleResultsOnOneClassLoaderAreFound() throws IOException { + final ClassLoader firstMockClassLoader = mock(ClassLoader.class); + when(firstMockClassLoader.getResources(eq("someName"))).thenReturn( + Collections.enumeration( + asList( + FILE_A, + FILE_B + ) + ) + ); + + final List foundURLs = ClasspathScanner.scanFor("someName", firstMockClassLoader).collect(toList()); + assertEquals( + "The expected URLs are found", + asList(FILE_A, FILE_B), + foundURLs + ); + } + + @Test + public void orderIsAlphabeticalForDeterminism() throws IOException { + final ClassLoader firstMockClassLoader = mock(ClassLoader.class); + when(firstMockClassLoader.getResources(eq("someName"))).thenReturn( + Collections.enumeration( + asList( + FILE_B, + JAR_A, + JAR_B, + FILE_A + ) + ) + ); + + final List foundURLs = ClasspathScanner.scanFor("someName", firstMockClassLoader).collect(toList()); + assertEquals( + "The expected URLs are found in the expected order", + asList(FILE_A, FILE_B, JAR_A, JAR_B), + foundURLs + ); + } + + @Test + public void multipleClassLoadersAreQueried() throws IOException { + final ClassLoader firstMockClassLoader = mock(ClassLoader.class); + when(firstMockClassLoader.getResources(eq("someName"))).thenReturn( + Collections.enumeration( + asList( + FILE_A, + FILE_B + ) + ) + ); + final ClassLoader secondMockClassLoader = mock(ClassLoader.class); + when(secondMockClassLoader.getResources(eq("someName"))).thenReturn( + Collections.enumeration( + asList( + FILE_B, // duplicate + FILE_C + ) + ) + ); + + final List foundURLs = ClasspathScanner.scanFor("someName", firstMockClassLoader, secondMockClassLoader).collect(toList()); + + assertEquals( + "The expected URLs are found", + asList(FILE_A, FILE_B, FILE_C), + foundURLs + ); + } +} diff --git a/core/src/test/java/org/testcontainers/utility/DefaultImageNameSubstitutorTest.java b/core/src/test/java/org/testcontainers/utility/DefaultImageNameSubstitutorTest.java new file mode 100644 index 00000000000..a6f7813c3d3 --- /dev/null +++ b/core/src/test/java/org/testcontainers/utility/DefaultImageNameSubstitutorTest.java @@ -0,0 +1,38 @@ +package org.testcontainers.utility; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mockito; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.eq; + +public class DefaultImageNameSubstitutorTest { + + public static final DockerImageName ORIGINAL_IMAGE = DockerImageName.parse("foo"); + public static final DockerImageName SUBSTITUTE_IMAGE = DockerImageName.parse("bar"); + private ConfigurationFileImageNameSubstitutor underTest; + + @Rule + public MockTestcontainersConfigurationRule config = new MockTestcontainersConfigurationRule(); + + @Before + public void setUp() { + underTest = new ConfigurationFileImageNameSubstitutor(TestcontainersConfiguration.getInstance()); + } + + @Test + public void testConfigurationLookup() { + Mockito + .doReturn(SUBSTITUTE_IMAGE) + .when(TestcontainersConfiguration.getInstance()) + .getConfiguredSubstituteImage(eq(ORIGINAL_IMAGE)); + + final DockerImageName substitute = underTest.apply(ORIGINAL_IMAGE); + + assertEquals("match is found", SUBSTITUTE_IMAGE, substitute); + assertTrue("compatibility is automatically set", substitute.isCompatibleWith(ORIGINAL_IMAGE)); + } +} diff --git a/core/src/test/java/org/testcontainers/utility/DockerImageNameCompatibilityTest.java b/core/src/test/java/org/testcontainers/utility/DockerImageNameCompatibilityTest.java index 9e7136ea6d3..7c794814faf 100644 --- a/core/src/test/java/org/testcontainers/utility/DockerImageNameCompatibilityTest.java +++ b/core/src/test/java/org/testcontainers/utility/DockerImageNameCompatibilityTest.java @@ -26,9 +26,8 @@ public void testNoTagTreatedAsWildcard() { /* foo:1.2.3 != foo:4.5.6 foo:1.2.3 ~= foo - foo:1.2.3 ~= foo:latest - The test is effectively making sure that no tag and `latest` tag are equivalent + The test is effectively making sure that 'no tag' is treated as a wildcard */ assertFalse("foo:4.5.6 != foo:1.2.3", subject.isCompatibleWith(DockerImageName.parse("foo:1.2.3"))); assertTrue("foo:4.5.6 ~= foo", subject.isCompatibleWith(DockerImageName.parse("foo"))); diff --git a/core/src/test/java/org/testcontainers/utility/FakeImageSubstitutor.java b/core/src/test/java/org/testcontainers/utility/FakeImageSubstitutor.java new file mode 100644 index 00000000000..f74a6c942b3 --- /dev/null +++ b/core/src/test/java/org/testcontainers/utility/FakeImageSubstitutor.java @@ -0,0 +1,13 @@ +package org.testcontainers.utility; + +public class FakeImageSubstitutor extends ImageNameSubstitutor { + @Override + public DockerImageName apply(final DockerImageName original) { + return DockerImageName.parse("transformed-" + original.asCanonicalNameString()); + } + + @Override + protected String getDescription() { + return "test implementation"; + } +} diff --git a/core/src/test/java/org/testcontainers/utility/ImageNameSubstitutorTest.java b/core/src/test/java/org/testcontainers/utility/ImageNameSubstitutorTest.java new file mode 100644 index 00000000000..97ee0213f21 --- /dev/null +++ b/core/src/test/java/org/testcontainers/utility/ImageNameSubstitutorTest.java @@ -0,0 +1,75 @@ +package org.testcontainers.utility; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.Mockito; + +import static org.mockito.ArgumentMatchers.eq; +import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals; + +public class ImageNameSubstitutorTest { + + @Rule + public MockTestcontainersConfigurationRule config = new MockTestcontainersConfigurationRule(); + private ImageNameSubstitutor originalInstance; + private ImageNameSubstitutor originalDefaultImplementation; + + @Before + public void setUp() throws Exception { + originalInstance = ImageNameSubstitutor.instance; + originalDefaultImplementation = ImageNameSubstitutor.defaultImplementation; + ImageNameSubstitutor.instance = null; + ImageNameSubstitutor.defaultImplementation = Mockito.mock(ImageNameSubstitutor.class); + + Mockito + .doReturn(DockerImageName.parse("substituted-image")) + .when(ImageNameSubstitutor.defaultImplementation) + .apply(eq(DockerImageName.parse("original"))); + Mockito + .doReturn("default implementation") + .when(ImageNameSubstitutor.defaultImplementation) + .getDescription(); + } + + @After + public void tearDown() throws Exception { + ImageNameSubstitutor.instance = originalInstance; + ImageNameSubstitutor.defaultImplementation = originalDefaultImplementation; + } + + @Test + public void simpleConfigurationTest() { + Mockito + .doReturn(FakeImageSubstitutor.class.getCanonicalName()) + .when(TestcontainersConfiguration.getInstance()) + .getImageSubstitutorClassName(); + + final ImageNameSubstitutor imageNameSubstitutor = ImageNameSubstitutor.instance(); + + DockerImageName result = imageNameSubstitutor.apply(DockerImageName.parse("original")); + assertEquals( + "the image has been substituted by default then configured implementations", + "transformed-substituted-image:latest", + result.asCanonicalNameString() + ); + } + + @Test + public void testWorksWithoutConfiguredImplementation() { + Mockito + .doReturn(null) + .when(TestcontainersConfiguration.getInstance()) + .getImageSubstitutorClassName(); + + final ImageNameSubstitutor imageNameSubstitutor = ImageNameSubstitutor.instance(); + + DockerImageName result = imageNameSubstitutor.apply(DockerImageName.parse("original")); + assertEquals( + "the image has been substituted by default then configured implementations", + "substituted-image:latest", + result.asCanonicalNameString() + ); + } +} diff --git a/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java index e48a2a6fdf9..ae07e2d9131 100644 --- a/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java +++ b/core/src/test/java/org/testcontainers/utility/TestcontainersConfigurationTest.java @@ -25,6 +25,69 @@ public void setUp() { environment = new HashMap<>(); } + @Test + public void shouldSubstituteImageNamesFromClasspathProperties() { + classpathProperties.setProperty("ryuk.container.image", "foo:version"); + assertEquals( + "an image name can be pulled from classpath properties", + DockerImageName.parse("foo:version"), + newConfig().getConfiguredSubstituteImage(DockerImageName.parse("testcontainers/ryuk:any")) + ); + } + + @Test + public void shouldSubstituteImageNamesFromUserProperties() { + userProperties.setProperty("ryuk.container.image", "foo:version"); + assertEquals( + "an image name can be pulled from user properties", + DockerImageName.parse("foo:version"), + newConfig().getConfiguredSubstituteImage(DockerImageName.parse("testcontainers/ryuk:any")) + ); + } + + @Test + public void shouldSubstituteImageNamesFromEnvironmentVariables() { + environment.put("TESTCONTAINERS_RYUK_CONTAINER_IMAGE", "foo:version"); + assertEquals( + "an image name can be pulled from an environment variable", + DockerImageName.parse("foo:version"), + newConfig().getConfiguredSubstituteImage(DockerImageName.parse("testcontainers/ryuk:any")) + ); + } + + @Test + public void shouldApplySettingsInOrder() { + assertEquals( + "precedence order for multiple sources of the same value is correct", + "default", + newConfig().getEnvVarOrProperty("key", "default") + ); + + classpathProperties.setProperty("key", "foo"); + + assertEquals( + "precedence order for multiple sources of the same value is correct", + "foo", + newConfig().getEnvVarOrProperty("key", "default") + ); + + userProperties.setProperty("key", "bar"); + + assertEquals( + "precedence order for multiple sources of the same value is correct", + "bar", + newConfig().getEnvVarOrProperty("key", "default") + ); + + environment.put("TESTCONTAINERS_KEY", "baz"); + + assertEquals( + "precedence order for multiple sources of the same value is correct", + "baz", + newConfig().getEnvVarOrProperty("key", "default") + ); + } + @Test public void shouldNotReadChecksFromClasspathProperties() { assertFalse("checks enabled by default", newConfig().isDisableChecks()); diff --git a/core/src/test/resources/expectedClasspathFile.txt b/core/src/test/resources/expectedClasspathFile.txt new file mode 100644 index 00000000000..e13c06f078f --- /dev/null +++ b/core/src/test/resources/expectedClasspathFile.txt @@ -0,0 +1 @@ +This file exists for org.testcontainers.utility.ClasspathScannerTest diff --git a/docs/examples/junit4/generic/build.gradle b/docs/examples/junit4/generic/build.gradle index 5ce878c7ba2..ff1b11f142e 100644 --- a/docs/examples/junit4/generic/build.gradle +++ b/docs/examples/junit4/generic/build.gradle @@ -4,6 +4,9 @@ dependencies { testCompile "junit:junit:4.12" testCompile project(":testcontainers") testCompile project(":selenium") + testCompile project(":mysql") + + testCompile 'mysql:mysql-connector-java:8.0.21' testCompile "org.seleniumhq.selenium:selenium-api:3.141.59" testCompile 'org.assertj:assertj-core:3.15.0' } diff --git a/docs/examples/junit4/generic/src/test/java/generic/ExampleImageNameSubstitutor.java b/docs/examples/junit4/generic/src/test/java/generic/ExampleImageNameSubstitutor.java new file mode 100644 index 00000000000..f6355312b3f --- /dev/null +++ b/docs/examples/junit4/generic/src/test/java/generic/ExampleImageNameSubstitutor.java @@ -0,0 +1,24 @@ +package generic; + +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.ImageNameSubstitutor; + +public class ExampleImageNameSubstitutor extends ImageNameSubstitutor { + + @Override + public DockerImageName apply(DockerImageName original) { + // convert the original name to something appropriate for + // our build environment + return DockerImageName.parse( + // your code goes here - silly example of capitalising + // the original name is shown + original.asCanonicalNameString().toUpperCase() + ); + } + + @Override + protected String getDescription() { + // used in logs + return "example image name substitutor"; + } +} diff --git a/docs/examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java b/docs/examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java new file mode 100644 index 00000000000..edc622a4104 --- /dev/null +++ b/docs/examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java @@ -0,0 +1,48 @@ +package generic; + + +import generic.support.TestSpecificImageNameSubstitutor; +import org.junit.Test; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.utility.DockerImageName; + +public class ImageNameSubstitutionTest { + + @Test + public void simpleExample() { + try ( + // directDockerHubReference { + // Referring directly to an image on Docker Hub (mysql:8.0.22) + final MySQLContainer mysql = new MySQLContainer<>( + DockerImageName.parse("mysql:8.0.22") + ) + + // start the container and use it for testing + // } + ) { + mysql.start(); + } + } + + /** + * Note that this test uses a fake image name, which will only work because + * {@link TestSpecificImageNameSubstitutor} steps in to override the substitution for this exact + * image name. + */ + @Test + public void substitutedExample() { + try ( + // hardcodedMirror { + // Referring directly to an image on a private registry - image name will vary + final MySQLContainer mysql = new MySQLContainer<>( + DockerImageName.parse("registry.mycompany.com/mirror/mysql:8.0.22") + .asCompatibleSubstituteFor("mysql") + ) + + // start the container and use it for testing + // } + ) { + mysql.start(); + } + } +} diff --git a/docs/examples/junit4/generic/src/test/java/generic/support/TestSpecificImageNameSubstitutor.java b/docs/examples/junit4/generic/src/test/java/generic/support/TestSpecificImageNameSubstitutor.java new file mode 100644 index 00000000000..5f9e92c0add --- /dev/null +++ b/docs/examples/junit4/generic/src/test/java/generic/support/TestSpecificImageNameSubstitutor.java @@ -0,0 +1,26 @@ +package generic.support; + +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.ImageNameSubstitutor; + +/** + * An {@link ImageNameSubstitutor} which makes it possible to use fake image names in + * {@link generic.ImageNameSubstitutionTest}. This implementation simply reverses a fake image name when presented, and + * is hardcoded to act upon the specific fake name in that test. + */ +public class TestSpecificImageNameSubstitutor extends ImageNameSubstitutor { + + @Override + public DockerImageName apply(final DockerImageName original) { + if (original.equals(DockerImageName.parse("registry.mycompany.com/mirror/mysql:8.0.22"))) { + return DockerImageName.parse("mysql"); + } else { + return original; + } + } + + @Override + protected String getDescription() { + return TestSpecificImageNameSubstitutor.class.getSimpleName(); + } +} diff --git a/docs/examples/junit4/generic/src/test/resources/logback-test.xml b/docs/examples/junit4/generic/src/test/resources/logback-test.xml new file mode 100644 index 00000000000..1378a823a63 --- /dev/null +++ b/docs/examples/junit4/generic/src/test/resources/logback-test.xml @@ -0,0 +1,14 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n + + + + + + + + + + diff --git a/docs/examples/junit4/generic/src/test/resources/testcontainers.properties b/docs/examples/junit4/generic/src/test/resources/testcontainers.properties new file mode 100644 index 00000000000..19bf8288d33 --- /dev/null +++ b/docs/examples/junit4/generic/src/test/resources/testcontainers.properties @@ -0,0 +1 @@ +image.substitutor=generic.support.TestSpecificImageNameSubstitutor diff --git a/docs/features/configuration.md b/docs/features/configuration.md index e3ac0fb6ac3..b567a8f564f 100644 --- a/docs/features/configuration.md +++ b/docs/features/configuration.md @@ -2,14 +2,26 @@ You can override some default properties if your environment requires that. -## Configuration file location +## Configuration locations The configuration will be loaded from multiple locations. Properties are considered in the following order: -1. `.testcontainers.properties` in user's home folder. Example locations: +1. Environment variables +2. `.testcontainers.properties` in user's home folder. Example locations: **Linux:** `/home/myuser/.testcontainers.properties` **Windows:** `C:/Users/myuser/.testcontainers.properties` **macOS:** `/Users/myuser/.testcontainers.properties` -2. `testcontainers.properties` on classpath +3. `testcontainers.properties` on the classpath. + +Note that when using environment variables, configuration property names should be set in upper +case with underscore separators, preceded by `TESTCONTAINERS_` - e.g. `checks.disable` becomes +`TESTCONTAINERS_CHECKS_DISABLE`. + +The classpath `testcontainers.properties` file may exist within the local codebase (e.g. within the `src/test/resources` directory) or within library dependencies that you may have. +Any such configuration files will have their contents merged. +If any keys conflict, the value will be taken on the basis of the first value found in: + +* 'local' classpath (i.e. where the URL of the file on the classpath begins with `file:`), then +* other classpath locations (i.e. JAR files) - considered in _alphabetical order of path_ to provide deterministic ordering. ## Disabling the startup checks > **checks.disable = [true|false]** @@ -26,18 +38,30 @@ It takes a couple of seconds, but if you want to speed up your tests, you can di ## Customizing images +!!! note + This approach is discouraged and deprecated, but is documented for completeness. + Overriding individual image names via configuration may be removed in 2021. + See [Image Name Substitution](./image_name_substitution.md) for other strategies for substituting image names to pull from other registries. + + Testcontainers uses public Docker images to perform different actions like startup checks, VNC recording and others. Some companies disallow the usage of Docker Hub, but you can override `*.image` properties with your own images from your private registry to workaround that. +> **ryuk.container.image = testcontainers/ryuk:0.3.0** +> Performs fail-safe cleanup of containers, and always required (unless [Ryuk is disabled](#disabling-ryuk)) + > **tinyimage.container.image = alpine:3.5** -> Used by Testcontainers' core +> Used to check whether images can be pulled at startup, and always required (unless [startup checks are disabled](#disabling-the-startup-checks)) + +> **sshd.container.image = testcontainers/sshd:1.0.0** +> Required if [exposing host ports to containers](./networking.md#exposing-host-ports-to-the-container) > **vncrecorder.container.image = testcontainers/vnc-recorder:1.1.0** -> Used by VNC recorder in Testcontainers' Seleniun integration +> Used by VNC recorder in Testcontainers' Selenium integration -> **ambassador.container.image = richnorth/ambassador:latest** +> **socat.container.image = alpine/socat** > **compose.container.image = docker/compose:1.8.0** -> Used by Docker Compose integration +> Required if using [Docker Compose](../modules/docker_compose.md) > **kafka.container.image = confluentinc/cp-kafka** > Used by KafkaContainer @@ -45,11 +69,8 @@ Some companies disallow the usage of Docker Hub, but you can override `*.image` > **localstack.container.image = localstack/localstack** > Used by LocalStack -Another possibility is to set up a registry mirror in your environment so that all images are pulled from there and not directly from Docker Hub. -For more information, see the [official Docker documentation about "Registry as a pull through cache"](https://docs.docker.com/registry/recipes/mirror/). - -!!!tip - Registry mirror currently only works for Docker images with image name that has no registry specified (for example, for Docker image `mariadb:10.3.6`, it works, for Docker image `quay.io/something/else`, not). +> **pulsar.container.image = apachepulsar/pulsar:2.2.0** +> Used by Apache Pulsar ## Customizing Ryuk resource reaper diff --git a/docs/features/image_name_substitution.md b/docs/features/image_name_substitution.md new file mode 100644 index 00000000000..0cf2d4cf8f0 --- /dev/null +++ b/docs/features/image_name_substitution.md @@ -0,0 +1,117 @@ +# Image name substitution + +Testcontainers supports automatic substitution of Docker image names. + +This allows replacement of an image name specified in test code with an alternative name - for example, to replace the +name of a Docker Hub image dependency with an alternative hosted on a private image registry. + +This is advisable to avoid [Docker Hub rate limiting](./pull_rate_limiting.md), and some companies will prefer this for policy reasons. + +This page describes four approaches for image name substitution: + +* [Manual substitution](#manual-substitution) - not relying upon an automated approach +* Using an Image Name Substitutor: + * [Developing a custom function for transforming image names on the fly](#developing-a-custom-function-for-transforming-image-names-on-the-fly) + * [Overriding image names individually in configuration](#overriding-image-names-individually-in-configuration) + +It is assumed that you have already set up a private registry hosting [all the Docker images your build requires](./pull_rate_limiting.md#which-images-are-used-by-testcontainers). + + + + +## Manual substitution + +Consider this if: + +* You use only a few images and updating code is not a chore +* All developers and CI machines in your organisation have access to a common registry server +* You also use one of the automated mechanisms to substitute [the images that Testcontainers itself requires](./pull_rate_limiting.md#which-images-are-used-by-testcontainers) + +This approach simply entails modifying test code manually, e.g. changing: + +For example, you may have a test that uses the `mysql` container image from Docker Hub: + + +[Direct Docker Hub image name](../examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java) inside_block:directDockerHubReference + + +to: + + +[Private registry image name](../examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java) inside_block:hardcodedMirror + + + + + + + + + + +## Developing a custom function for transforming image names on the fly + +Consider this if: + +* You have complex rules about which private registry images should be used as substitutes, e.g.: + * non-deterministic mapping of names meaning that a [name prefix](#adding-a-registry-url-prefix-to-image-names-automatically) cannot be used + * rules depending upon developer identity or location +* or you wish to add audit logging of images used in the build +* or you wish to prevent accidental usage of images that are not on an approved list + +In this case, image name references in code are **unchanged**. +i.e. you would leave as-is: + + +[Unchanged direct Docker Hub image name](../examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java) inside_block:directDockerHubReference + + +You can implement a custom image name substitutor by: + +* subclassing `org.testcontainers.utility.ImageNameSubstitutor` +* configuring Testcontainers to use your custom implementation + +The following is an example image substitutor implementation: + + +[Example Image Substitutor](../examples/junit4/generic/src/test/java/generic/ExampleImageNameSubstitutor.java) block:ExampleImageNameSubstitutor + + +Testcontainers can be configured to find it at runtime via configuration. +To do this, create or modify a file on the classpath named `testcontainers.properties`. + +For example: + +```text tab="src/test/resources/testcontainers.properties" +image.substitutor=com.mycompany.testcontainers.ExampleImageNameSubstitutor +``` + +Note that it is also possible to provide this same configuration property: + +* in a `testcontainers.properties` file at the root of a library JAR file (useful if you wish to distribute a drop-in image substitutor JAR within an organization) +* in a properties file in the user's home directory (`~/.testcontainers.properties`; note the leading `.`) +* or as an environment variable (e.g. `TESTCONTAINERS_IMAGE_SUBSTITUTOR=com.mycompany.testcontainers.ExampleImageNameSubstitutor`). + +Please see [the documentation on configuration mechanisms](./configuration.md) for more information. + + +## Overriding image names individually in configuration + +!!! note + This approach is discouraged and deprecated, but is documented for completeness. + Please consider one of the other approaches outlined in this page instead. + Overriding individual image names via configuration may be removed in 2021. + +Consider this if: + +* You have many references to image names in code and changing them is impractical, and +* None of the other options are practical for you + +In this case, image name references in code are left **unchanged**. +i.e. you would leave as-is: + + +[Unchanged direct Docker Hub image name](../examples/junit4/generic/src/test/java/generic/ImageNameSubstitutionTest.java) inside_block:directDockerHubReference + + +You can force Testcontainers to substitute in a different image [using a configuration file](./configuration.md), which allows some (but not all) container names to be substituted. diff --git a/docs/features/pull_rate_limiting.md b/docs/features/pull_rate_limiting.md new file mode 100644 index 00000000000..be266545afb --- /dev/null +++ b/docs/features/pull_rate_limiting.md @@ -0,0 +1,19 @@ +# Image Registry rate limiting + +As of November 2020 Docker Hub pulls are rate limited. +As Testcontainers uses Docker Hub for standard images, some users may hit these rate limits and should mitigate accordingly. + +Suggested mitigations are noted in [this issue](https://github.com/testcontainers/testcontainers-java/issues/3099) at present. + +## Which images are used by Testcontainers? + +As of the current version of Testcontainers ({{latest_version}}): + +* every image directly used by your tests +* images pulled by Testcontainers itself to support functionality: + * [`testcontainers/ryuk`](https://hub.docker.com/r/testcontainers/ryuk) - performs fail-safe cleanup of containers, and always required (unless [Ryuk is disabled](./configuration.md#disabling-ryuk)) + * [`alpine`](https://hub.docker.com/r/_/alpine) - used to check whether images can be pulled at startup, and always required (unless [startup checks are disabled](./configuration.md#disabling-the-startup-checks)) + * [`testcontainers/sshd`](https://hub.docker.com/r/testcontainers/sshd) - required if [exposing host ports to containers](./networking.md#exposing-host-ports-to-the-container) + * [`testcontainers/vnc-recorder`](https://hub.docker.com/r/testcontainers/vnc-recorder) - required if using [Webdriver containers](../modules/webdriver_containers.md) and using the screen recording feature + * [`docker/compose`](https://hub.docker.com/r/docker/compose) - required if using [Docker Compose](../modules/docker_compose.md) + * [`alpine/socat`](https://hub.docker.com/r/alpine/socat) - required if using [Docker Compose](../modules/docker_compose.md) diff --git a/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java b/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java index 37f99bf970b..12867239251 100644 --- a/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java +++ b/modules/kafka/src/main/java/org/testcontainers/containers/KafkaContainer.java @@ -4,7 +4,6 @@ import lombok.SneakyThrows; import org.testcontainers.images.builder.Transferable; import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.TestcontainersConfiguration; import java.nio.charset.StandardCharsets; import java.util.stream.Collectors; @@ -36,7 +35,7 @@ public class KafkaContainer extends GenericContainer { */ @Deprecated public KafkaContainer() { - this(TestcontainersConfiguration.getInstance().getKafkaDockerImageName().withTag(DEFAULT_TAG)); + this(DEFAULT_IMAGE_NAME.withTag(DEFAULT_TAG)); } /** @@ -44,7 +43,7 @@ public KafkaContainer() { */ @Deprecated public KafkaContainer(String confluentPlatformVersion) { - this(TestcontainersConfiguration.getInstance().getKafkaDockerImageName().withTag(confluentPlatformVersion)); + this(DEFAULT_IMAGE_NAME.withTag(confluentPlatformVersion)); } public KafkaContainer(final DockerImageName dockerImageName) { diff --git a/modules/kafka/src/test/java/org/testcontainers/containers/KafkaContainerTest.java b/modules/kafka/src/test/java/org/testcontainers/containers/KafkaContainerTest.java index 4f721424bb6..263653822e6 100644 --- a/modules/kafka/src/test/java/org/testcontainers/containers/KafkaContainerTest.java +++ b/modules/kafka/src/test/java/org/testcontainers/containers/KafkaContainerTest.java @@ -61,7 +61,7 @@ public void testUsageWithSpecificImage() throws Exception { @Test public void testUsageWithVersion() throws Exception { try ( - KafkaContainer kafka = new KafkaContainer("5.4.3") + KafkaContainer kafka = new KafkaContainer("5.5.1") ) { kafka.start(); testKafkaFunctionality(kafka.getBootstrapServers()); diff --git a/modules/localstack/src/main/java/org/testcontainers/containers/localstack/LocalStackContainer.java b/modules/localstack/src/main/java/org/testcontainers/containers/localstack/LocalStackContainer.java index 7532ebe0b3d..b8416afb1ed 100644 --- a/modules/localstack/src/main/java/org/testcontainers/containers/localstack/LocalStackContainer.java +++ b/modules/localstack/src/main/java/org/testcontainers/containers/localstack/LocalStackContainer.java @@ -14,7 +14,6 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.utility.ComparableVersion; import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.TestcontainersConfiguration; import java.net.InetAddress; import java.net.URI; @@ -65,7 +64,7 @@ public class LocalStackContainer extends GenericContainer { */ @Deprecated public LocalStackContainer() { - this(TestcontainersConfiguration.getInstance().getLocalstackDockerImageName().withTag(DEFAULT_TAG)); + this(DEFAULT_IMAGE_NAME.withTag(DEFAULT_TAG)); } /** @@ -73,7 +72,7 @@ public LocalStackContainer() { */ @Deprecated public LocalStackContainer(String version) { - this(TestcontainersConfiguration.getInstance().getLocalstackDockerImageName().withTag(version)); + this(DEFAULT_IMAGE_NAME.withTag(version)); } /** diff --git a/modules/oracle-xe/src/main/java/org/testcontainers/containers/OracleContainer.java b/modules/oracle-xe/src/main/java/org/testcontainers/containers/OracleContainer.java index b5700c1e84e..0a66ad6fd27 100644 --- a/modules/oracle-xe/src/main/java/org/testcontainers/containers/OracleContainer.java +++ b/modules/oracle-xe/src/main/java/org/testcontainers/containers/OracleContainer.java @@ -22,8 +22,7 @@ public class OracleContainer extends JdbcDatabaseContainer { private String password = "oracle"; private static String resolveImageName() { - String image = TestcontainersConfiguration.getInstance() - .getProperties().getProperty("oracle.container.image"); + String image = TestcontainersConfiguration.getInstance().getOracleImage(); if (image == null) { throw new IllegalStateException("An image to use for Oracle containers must be configured. " + diff --git a/modules/pulsar/src/main/java/org/testcontainers/containers/PulsarContainer.java b/modules/pulsar/src/main/java/org/testcontainers/containers/PulsarContainer.java index bceff8927d7..b3c836b56a5 100644 --- a/modules/pulsar/src/main/java/org/testcontainers/containers/PulsarContainer.java +++ b/modules/pulsar/src/main/java/org/testcontainers/containers/PulsarContainer.java @@ -3,7 +3,6 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.containers.wait.strategy.WaitAllStrategy; import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.TestcontainersConfiguration; /** * This container wraps Apache Pulsar running in standalone mode @@ -25,7 +24,7 @@ public class PulsarContainer extends GenericContainer { */ @Deprecated public PulsarContainer() { - this(TestcontainersConfiguration.getInstance().getPulsarDockerImageName().withTag(DEFAULT_TAG)); + this(DEFAULT_IMAGE_NAME.withTag(DEFAULT_TAG)); } /** @@ -33,7 +32,7 @@ public PulsarContainer() { */ @Deprecated public PulsarContainer(String pulsarVersion) { - this(TestcontainersConfiguration.getInstance().getPulsarDockerImageName().withTag(pulsarVersion)); + this(DEFAULT_IMAGE_NAME.withTag(pulsarVersion)); } public PulsarContainer(final DockerImageName dockerImageName) { diff --git a/modules/spock/src/test/groovy/org/testcontainers/spock/SpockTestImages.groovy b/modules/spock/src/test/groovy/org/testcontainers/spock/SpockTestImages.groovy index 526c1fe5c42..9698e7d433a 100644 --- a/modules/spock/src/test/groovy/org/testcontainers/spock/SpockTestImages.groovy +++ b/modules/spock/src/test/groovy/org/testcontainers/spock/SpockTestImages.groovy @@ -1,11 +1,10 @@ -package org.testcontainers.spock; +package org.testcontainers.spock import org.testcontainers.utility.DockerImageName -import org.testcontainers.utility.TestcontainersConfiguration; interface SpockTestImages { DockerImageName MYSQL_IMAGE = DockerImageName.parse("mysql:5.7.22") DockerImageName POSTGRES_TEST_IMAGE = DockerImageName.parse("postgres:9.6.12") DockerImageName HTTPD_IMAGE = DockerImageName.parse("httpd:2.4-alpine") - DockerImageName TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyDockerImageName() + DockerImageName TINY_IMAGE = DockerImageName.parse("alpine:3.5") }