Skip to content

Commit

Permalink
Image substitution (#3102)
Browse files Browse the repository at this point in the history
* Refactor Testcontainers configuration to allow config by env var

* Add Image substitution mechanism

Builds upon #3021 and #3411:

* adds a pluggable image substitution mechanism using ServiceLoader, enabling users to perform custom substitution/auditing of images being used by their tests

* provides a default implementation that behaves similarly to legacy `TestcontainersConfiguration` approach (`testcontainers.properties`)

Notes:

* behaviour is similar but not quite identical to `TestcontainersConfiguration`: use of a configured custom image for, e.g. Kafka/Pulsar that does not have a tag specified causes the substitution to take effect for all usages. It seems very unlikely that people would be using a mix of the config file image overrides in some places _and_ specific images specified in code in others.

* Duplication of default image names in modules vs `TestcontainersConfiguration` class is intentional: specifying image overrides in `testcontainers.properties` should be removed in the future.

* ~Add log deprecation warnings when `testcontainers.properties` image overrides are used.~ Defer to a future release?

* Remove extraneous change

* Un-ignore docs example test by implementing a 'reversing' image name substitutor

* Use configuration, not service loader, to select an ImageNameSubstitutor

* Add check for order of config setting precedence

* Extract classpath scanner and support finding of multiple resources

* Introduce deterministic merging of classpath properties files

* Update docs

* Update docs

* Remove service loader reference

* Chain substitution through default and configured implementations

* Small tweaks following review

* Fix test compile error

* Add UnstableAPI annotation

* Move TestSpecificImageNameSubstitutor back to original package and remove duplicate use of default substitutor
  • Loading branch information
rnorth authored Nov 5, 2020
1 parent 8d1a723 commit 44e8e9a
Show file tree
Hide file tree
Showing 39 changed files with 936 additions and 123 deletions.
11 changes: 8 additions & 3 deletions core/src/main/java/org/testcontainers/DockerClientFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import org.testcontainers.dockerclient.TransportConfig;
import org.testcontainers.images.TimeLimitedLoggedPullImageResultCallback;
import org.testcontainers.utility.ComparableVersion;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.ImageNameSubstitutor;
import org.testcontainers.utility.MountableFile;
import org.testcontainers.utility.ResourceReaper;
import org.testcontainers.utility.TestcontainersConfiguration;
Expand Down Expand Up @@ -61,7 +63,7 @@ public class DockerClientFactory {
TESTCONTAINERS_SESSION_ID_LABEL, SESSION_ID
);

private static final String TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString();
private static final DockerImageName TINY_IMAGE = DockerImageName.parse("alpine:3.5");
private static DockerClientFactory instance;

// Cached client configuration
Expand Down Expand Up @@ -343,8 +345,11 @@ public <T> T runInsideDocker(Consumer<CreateContainerCmd> createContainerCmdCons
}

private <T> T runInsideDocker(DockerClient client, Consumer<CreateContainerCmd> createContainerCmdConsumer, BiFunction<DockerClient, String, T> block) {
checkAndPullImage(client, TINY_IMAGE);
CreateContainerCmd createContainerCmd = client.createContainerCmd(TINY_IMAGE)

final String tinyImage = ImageNameSubstitutor.instance().apply(TINY_IMAGE).asCanonicalNameString();

checkAndPullImage(client, tinyImage);
CreateContainerCmd createContainerCmd = client.createContainerCmd(tinyImage)
.withLabels(DEFAULT_LABELS);
createContainerCmdConsumer.accept(createContainerCmd);
String id = createContainerCmd.exec().getId();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@
import org.testcontainers.utility.AuditLogger;
import org.testcontainers.utility.Base58;
import org.testcontainers.utility.CommandLine;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.DockerLoggerFactory;
import org.testcontainers.utility.LogUtils;
import org.testcontainers.utility.MountableFile;
import org.testcontainers.utility.ResourceReaper;
import org.testcontainers.utility.TestcontainersConfiguration;
import org.zeroturnaround.exec.InvalidExitValueException;
import org.zeroturnaround.exec.ProcessExecutor;
import org.zeroturnaround.exec.stream.slf4j.Slf4jStream;
Expand Down Expand Up @@ -608,10 +608,11 @@ interface DockerCompose {
class ContainerisedDockerCompose extends GenericContainer<ContainerisedDockerCompose> implements DockerCompose {

public static final char UNIX_PATH_SEPERATOR = ':';
public static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("docker/compose:1.24.1");

public ContainerisedDockerCompose(List<File> composeFiles, String identifier) {

super(TestcontainersConfiguration.getInstance().getDockerComposeDockerImageName());
super(DEFAULT_IMAGE_NAME);
addEnv(ENV_PROJECT_NAME, identifier);

// Map the docker compose file into the container
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -239,13 +239,9 @@ public GenericContainer(@NonNull final RemoteDockerImage image) {
*/
@Deprecated
public GenericContainer() {
this(TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString());
this(TestcontainersConfiguration.getInstance().getTinyImage());
}

/**
* @deprecated use {@link GenericContainer(DockerImageName)} instead
*/
@Deprecated
public GenericContainer(@NonNull final String dockerImageName) {
this.setDockerImageName(dockerImageName);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@
import lombok.AccessLevel;
import lombok.Getter;
import lombok.SneakyThrows;
import org.testcontainers.utility.TestcontainersConfiguration;
import org.testcontainers.utility.DockerImageName;

import java.time.Duration;
import java.util.AbstractMap;
import java.util.Collections;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;

public enum PortForwardingContainer {
Expand All @@ -29,7 +29,7 @@ public enum PortForwardingContainer {
@SneakyThrows
private Connection createSSHSession() {
String password = UUID.randomUUID().toString();
container = new GenericContainer<>(TestcontainersConfiguration.getInstance().getSSHdDockerImageName())
container = new GenericContainer<>(DockerImageName.parse("testcontainers/sshd:1.0.0"))
.withExposedPorts(22)
.withEnv("PASSWORD", password)
.withCommand(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
package org.testcontainers.containers;

import org.testcontainers.utility.Base58;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.TestcontainersConfiguration;

import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
import org.testcontainers.utility.Base58;
import org.testcontainers.utility.DockerImageName;

/**
* A socat container is used as a TCP proxy, enabling any TCP port of another container to be exposed
Expand All @@ -17,7 +15,7 @@ public class SocatContainer extends GenericContainer<SocatContainer> {
private final Map<Integer, String> targets = new HashMap<>();

public SocatContainer() {
this(TestcontainersConfiguration.getInstance().getSocatDockerImageName());
this(DockerImageName.parse("alpine/socat:1.7.3.4-r0"));
}

public SocatContainer(final DockerImageName dockerImageName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import lombok.ToString;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
import org.testcontainers.utility.TestcontainersConfiguration;
import org.testcontainers.utility.DockerImageName;

import java.io.File;
import java.io.InputStream;
Expand Down Expand Up @@ -52,7 +52,7 @@ public VncRecordingContainer(@NonNull GenericContainer<?> targetContainer) {
* Create a sidekick container and attach it to another container. The VNC output of that container will be recorded.
*/
public VncRecordingContainer(@NonNull Network network, @NonNull String targetNetworkAlias) throws IllegalStateException {
super(TestcontainersConfiguration.getInstance().getVncDockerImageName());
super(DockerImageName.parse("testcontainers/vnc-recorder:1.1.0"));

this.targetNetworkAlias = targetNetworkAlias;
withNetwork(network);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ public static DockerClientProviderStrategy getFirstValidStrategy(List<DockerClie
}

if (strategy.isPersistable()) {
TestcontainersConfiguration.getInstance().updateGlobalConfig("docker.client.strategy", strategy.getClass().getName());
TestcontainersConfiguration.getInstance().updateUserConfig("docker.client.strategy", strategy.getClass().getName());
}

return Stream.of(strategy);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.testcontainers.containers.ContainerFetchException;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.DockerLoggerFactory;
import org.testcontainers.utility.ImageNameSubstitutor;
import org.testcontainers.utility.LazyFuture;

import java.time.Duration;
Expand Down Expand Up @@ -44,12 +45,12 @@ public RemoteDockerImage(DockerImageName dockerImageName) {

@Deprecated
public RemoteDockerImage(String dockerImageName) {
this.imageNameFuture = CompletableFuture.completedFuture(DockerImageName.parse(dockerImageName));
this(DockerImageName.parse(dockerImageName));
}

@Deprecated
public RemoteDockerImage(@NonNull String repository, @NonNull String tag) {
this.imageNameFuture = CompletableFuture.completedFuture(DockerImageName.parse(repository).withTag(tag));
this(DockerImageName.parse(repository).withTag(tag));
}

public RemoteDockerImage(@NonNull Future<String> imageFuture) {
Expand Down Expand Up @@ -100,7 +101,10 @@ protected final String resolve() {
}

private DockerImageName getImageName() throws InterruptedException, ExecutionException {
return imageNameFuture.get();
final DockerImageName specifiedImageName = imageNameFuture.get();

// Allow the image name to be substituted
return ImageNameSubstitutor.instance().apply(specifiedImageName);
}

@ToString.Include(name = "imageName", rank = 1)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package org.testcontainers.utility;

import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.VisibleForTesting;

import java.net.URL;
import java.util.Collections;
import java.util.Comparator;
import java.util.Objects;
import java.util.stream.Stream;

/**
* Utility for identifying resource files on classloaders.
*/
@Slf4j
class ClasspathScanner {

@VisibleForTesting
static Stream<URL> scanFor(final String name, ClassLoader... classLoaders) {
return Stream
.of(classLoaders)
.flatMap(classLoader -> getAllPropertyFilesOnClassloader(classLoader, name))
.filter(Objects::nonNull)
.sorted(
Comparator
.comparing(ClasspathScanner::filesFileSchemeFirst) // resolve 'local' files first
.thenComparing(URL::toString) // sort alphabetically for the sake of determinism
)
.distinct();
}

private static Integer filesFileSchemeFirst(final URL t) {
return t.getProtocol().equals("file") ? 0 : 1;
}

/**
* @param name the resource name to search for
* @return distinct, ordered stream of resources found by searching this class' classloader and the current thread's
* context classloader. Results are currently alphabetically sorted.
*/
static Stream<URL> scanFor(final String name) {
return scanFor(
name,
ClasspathScanner.class.getClassLoader(),
Thread.currentThread().getContextClassLoader()
);
}

@Nullable
private static Stream<URL> getAllPropertyFilesOnClassloader(final ClassLoader it, final String s) {
try {
return Collections.list(it.getResources(s)).stream();
} catch (Exception e) {
log.error("Unable to read configuration from classloader {} - this is probably a bug", it, e);
return Stream.empty();
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package org.testcontainers.utility;

import com.google.common.annotations.VisibleForTesting;
import lombok.extern.slf4j.Slf4j;

/**
* {@link ImageNameSubstitutor} which takes replacement image names from configuration.
* See {@link TestcontainersConfiguration} for the subset of image names which can be substituted using this mechanism.
*/
@Slf4j
final class ConfigurationFileImageNameSubstitutor extends ImageNameSubstitutor {

private final TestcontainersConfiguration configuration;

public ConfigurationFileImageNameSubstitutor() {
this(TestcontainersConfiguration.getInstance());
}

@VisibleForTesting
ConfigurationFileImageNameSubstitutor(TestcontainersConfiguration configuration) {
this.configuration = configuration;
}

@Override
public DockerImageName apply(final DockerImageName original) {
final DockerImageName result = configuration
.getConfiguredSubstituteImage(original)
.asCompatibleSubstituteFor(original);

if (!result.equals(original)) {
log.warn("Image name {} was substituted by configuration to {}. This approach is deprecated and will be removed in the future",
original,
result
);
}

return result;
}

@Override
protected String getDescription() {
return getClass().getSimpleName();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package org.testcontainers.utility;

import com.google.common.annotations.VisibleForTesting;
import lombok.extern.slf4j.Slf4j;

/**
* Testcontainers' default implementation of {@link ImageNameSubstitutor}.
* Delegates to {@link ConfigurationFileImageNameSubstitutor}.
*/
@Slf4j
final class DefaultImageNameSubstitutor extends ImageNameSubstitutor {

private final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor;

public DefaultImageNameSubstitutor() {
configurationFileImageNameSubstitutor = new ConfigurationFileImageNameSubstitutor();
}

@VisibleForTesting
DefaultImageNameSubstitutor(
final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor
) {
this.configurationFileImageNameSubstitutor = configurationFileImageNameSubstitutor;
}

@Override
public DockerImageName apply(final DockerImageName original) {
return configurationFileImageNameSubstitutor.apply(original);
}

@Override
protected String getDescription() {
return "DefaultImageNameSubstitutor (" + configurationFileImageNameSubstitutor.getDescription() + ")";
}
}
Loading

0 comments on commit 44e8e9a

Please sign in to comment.