From 2000f6654f13cd34ea247fbf717456642ab3af83 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen Date: Thu, 18 Feb 2021 16:58:20 +0800 Subject: [PATCH] Support Podman for mocking Lambda (#3702) * Support Podman for mocking Lambda Podman supports all Docker APIs used in moto since version 3.0. Note that Podman requires pulling the image before creating a container using a fully-qualified image name (e.g., "docker.io/library/busybox" instead of "busybox"). Test plan: $ podman system service -t 0 $ DOCKER_HOST="unix://$XDG_RUNTIME_DIR/podman/podman.sock" pytest Fixes https://github.com/spulec/moto/issues/3276 * Run black * Python 2 compatibility * Address review comments and improve parse_image_ref --- moto/awslambda/models.py | 9 ++++-- moto/batch/models.py | 4 ++- moto/settings.py | 1 + moto/utilities/docker_utilities.py | 26 ++++++++++++++++ tests/test_utilities/test_docker_utilities.py | 31 +++++++++++++++++++ 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 tests/test_utilities/test_docker_utilities.py diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index b6c941cd7d3c..bfc9052c40f5 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -51,7 +51,7 @@ from moto.dynamodb2 import dynamodb_backends2 from moto.dynamodbstreams import dynamodbstreams_backends from moto.core import ACCOUNT_ID -from moto.utilities.docker_utilities import DockerModel +from moto.utilities.docker_utilities import DockerModel, parse_image_ref logger = logging.getLogger(__name__) @@ -131,6 +131,9 @@ def __enter__(self): volumes = {self.name: {"bind": "/tmp/data", "mode": "rw"}} else: volumes = {self.name: "/tmp/data"} + self._lambda_func.docker_client.images.pull( + ":".join(parse_image_ref("alpine")) + ) container = self._lambda_func.docker_client.containers.run( "alpine", "sleep 100", volumes=volumes, detach=True ) @@ -574,8 +577,10 @@ def _invoke_lambda(self, code, event=None, context=None): if settings.TEST_SERVER_MODE else {} ) + image_ref = "lambci/lambda:{}".format(self.run_time) + self.docker_client.images.pull(":".join(parse_image_ref(image_ref))) container = self.docker_client.containers.run( - "lambci/lambda:{}".format(self.run_time), + image_ref, [self.handler, json.dumps(event)], remove=False, mem_limit="{}m".format(self.memory_size), diff --git a/moto/batch/models.py b/moto/batch/models.py index 110188aad2c0..729a83cd8b20 100644 --- a/moto/batch/models.py +++ b/moto/batch/models.py @@ -28,7 +28,7 @@ from moto.ec2.models import INSTANCE_TYPES as EC2_INSTANCE_TYPES from moto.iam.exceptions import IAMNotFoundException from moto.core import ACCOUNT_ID as DEFAULT_ACCOUNT_ID -from moto.utilities.docker_utilities import DockerModel +from moto.utilities.docker_utilities import DockerModel, parse_image_ref logger = logging.getLogger(__name__) COMPUTE_ENVIRONMENT_NAME_REGEX = re.compile( @@ -428,6 +428,8 @@ def run(self): self.job_started_at = datetime.datetime.now() self.job_state = "STARTING" log_config = docker.types.LogConfig(type=docker.types.LogConfig.types.JSON) + image_repository, image_tag = parse_image_ref(image) + self.docker_client.images.pull(image_repository, image_tag) container = self.docker_client.containers.run( image, cmd, diff --git a/moto/settings.py b/moto/settings.py index d3259e0ec211..5cb6cfcf863e 100644 --- a/moto/settings.py +++ b/moto/settings.py @@ -4,6 +4,7 @@ INITIAL_NO_AUTH_ACTION_COUNT = float( os.environ.get("INITIAL_NO_AUTH_ACTION_COUNT", float("inf")) ) +DEFAULT_CONTAINER_REGISTRY = os.environ.get("DEFAULT_CONTAINER_REGISTRY", "docker.io") def get_sf_execution_history_type(): diff --git a/moto/utilities/docker_utilities.py b/moto/utilities/docker_utilities.py index 576a9df1d8e8..ac059c6fe09b 100644 --- a/moto/utilities/docker_utilities.py +++ b/moto/utilities/docker_utilities.py @@ -2,6 +2,8 @@ import functools import requests.adapters +from moto import settings + _orig_adapter_send = requests.adapters.HTTPAdapter.send @@ -31,3 +33,27 @@ def replace_adapter_send(*args, **kwargs): self.docker_client.api.get_adapter = replace_adapter_send return self.__docker_client + + +def parse_image_ref(image_name): + # podman does not support short container image name out of box - try to make a full name + # See ParseDockerRef() in https://github.com/distribution/distribution/blob/main/reference/normalize.go + parts = image_name.split("/") + if len(parts) == 1 or ( + "." not in parts[0] and ":" not in parts[0] and parts[0] != "localhost" + ): + domain = settings.DEFAULT_CONTAINER_REGISTRY + remainder = parts + else: + domain = parts[0] + remainder = parts[1:] + # Special handling for docker.io + # https://github.com/containers/image/blob/master/docs/containers-registries.conf.5.md#normalization-of-dockerio-references + if domain == "docker.io" and len(remainder) == 1: + remainder = ["library"] + remainder + if ":" in remainder[-1]: + remainder[-1], image_tag = remainder[-1].split(":", 1) + else: + image_tag = "latest" + image_repository = "/".join([domain] + remainder) + return image_repository, image_tag diff --git a/tests/test_utilities/test_docker_utilities.py b/tests/test_utilities/test_docker_utilities.py new file mode 100644 index 000000000000..e5db048a43d0 --- /dev/null +++ b/tests/test_utilities/test_docker_utilities.py @@ -0,0 +1,31 @@ +import sure +import pytest + +from moto.utilities.docker_utilities import parse_image_ref + + +@pytest.mark.parametrize( + "image_name,expected", + [ + ("python", ("docker.io/library/python", "latest")), + ("python:3.9", ("docker.io/library/python", "3.9")), + ("docker.io/python", ("docker.io/library/python", "latest")), + ("localhost/foobar", ("localhost/foobar", "latest")), + ("lambci/lambda:python2.7", ("docker.io/lambci/lambda", "python2.7")), + ( + "gcr.io/google.com/cloudsdktool/cloud-sdk", + ("gcr.io/google.com/cloudsdktool/cloud-sdk", "latest"), + ), + ], +) +def test_parse_image_ref(image_name, expected): + expected.should.be.equal(parse_image_ref(image_name)) + + +def test_parse_image_ref_default_container_registry(monkeypatch): + import moto.settings + + monkeypatch.setattr(moto.settings, "DEFAULT_CONTAINER_REGISTRY", "quay.io") + ("quay.io/centos/centos", "latest").should.be.equal( + parse_image_ref("centos/centos") + )