[ES-8436] feat: Change Dockerfile base images to Wolfi

docker/Dockerfiles/Dockerfile-dev

@@ -1,51 +1,31 @@
-################################################################################
-# Build stage 0 `builder`:
-# Install Rally from source inside a virtualenv
-################################################################################
+FROM docker.elastic.co/wolfi/python:3.12.3-dev AS builder
 
-FROM python:3.8.13-slim-bullseye as builder
+USER root
 
-RUN apt-get -y update && \
-    apt-get install -y curl git gcc && \
-    apt-get -y upgrade && \
-    rm -rf /var/lib/apt/lists/*
+RUN apk update 
+RUN apk add curl git gcc pigz bash zstd bzip2 gzip
 
-RUN mkdir -p /rally/esrally
-COPY pyproject.toml /rally/
-COPY README.md /rally/
-COPY esrally/ /rally/esrally/
-
-RUN python3 -m venv /rally/venv
-ENV PATH="/rally/venv/bin:$PATH"
-
-WORKDIR /rally
-# Wipe away any lingering caches, copied over from the local machine
-RUN find /rally -name "__pycache__" -exec rm -rf -- \{\} \; 2>/dev/null || true
-RUN find /rally -name ".pyc" -exec rm -rf -- \{\} \; 2>/dev/null || true
-RUN pip3 install --upgrade hatch hatchling pip wheel
-RUN pip3 install /rally
+# pbzip2 doesn't have a package for wolfi, so we build it from source
+RUN apk add bzip2-dev make wget 
 
-################################################################################
-# Build stage 1 (the actual Rally image):
-# Copy Rally from stage 0 and fix permissions to support randomized UIDs
-# Define VOLUME for ~/.rally
-# Add entrypoint
-################################################################################
+RUN cd /tmp && \
+  wget -q https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz && \
+  tar -xzf pbzip2-1.1.13.tar.gz && \
+  cd pbzip2-1.1.13/ && \
+  make install && \
+  rm -r /tmp/pbzip2-1.1.13/
 
-FROM python:3.8.12-slim-bullseye
+FROM docker.elastic.co/wolfi/python:3.12.3-dev
 ARG RALLY_VERSION
 ARG RALLY_LICENSE
-ENV RALLY_RUNNING_IN_DOCKER True
 
-RUN apt-get -y update && \
-    apt-get install -y curl git pbzip2 pigz && \
-    apt-get -y upgrade && \
-    rm -rf /var/lib/apt/lists/*
+ENV RALLY_RUNNING_IN_DOCKER=True
 
-RUN groupadd --gid 1000 rally && \
-    useradd -d /rally -m -k /dev/null -g 1000 -N -u 1000 -l -s /bin/bash rally
+USER root
+COPY --from=builder /usr/bin/ /usr/bin/
 
-COPY --chown=1000:0 --from=builder /rally/venv /rally/venv
+RUN addgroup --gid 1000 rally && \
+    adduser --system --home /rally --ingroup rally --no-create-home --uid 1000 --shell /bin/bash rally
 
 WORKDIR /rally
 COPY --chown=1000:0 docker/bin/entrypoint.sh /entrypoint.sh
@@ -57,12 +37,24 @@ RUN chgrp 0 /entrypoint.sh && \
     chmod 0775 /entrypoint.sh
 
 RUN mkdir -p /rally/.rally && \
-    chown -R 1000:0 /rally/.rally
+    chown -R 1000:0 /rally/
 
 USER 1000
 
+RUN mkdir -p /rally/esrally
+COPY pyproject.toml /rally/
+COPY README.md /rally/
+COPY esrally/ /rally/esrally/
+
+RUN python3 -m venv /rally/venv
 ENV PATH=/rally/venv/bin:$PATH
 
+# Wipe away any lingering caches, copied over from the local machine
+RUN find /rally -name "__pycache__" -exec rm -rf -- \{\} \; 2>/dev/null || true
+RUN find /rally -name ".pyc" -exec rm -rf -- \{\} \; 2>/dev/null || true
+RUN python3 -m pip install --upgrade hatch hatchling pip wheel
+RUN python3 -m pip install /rally
+
 LABEL org.label-schema.schema-version="1.0" \
   org.label-schema.vendor="Elastic" \
   org.label-schema.name="rally" \

docker/Dockerfiles/Dockerfile-release

@@ -1,19 +1,34 @@
-FROM python:3.8.13-slim-bullseye
+FROM docker.elastic.co/wolfi/python:3.12.3-dev AS builder
+
+USER root
+
+RUN apk update 
+RUN apk add curl git gcc pigz bash zstd bzip2 gzip
+
+# pbzip2 doesn't have a package for wolfi, so we build it from source
+RUN apk add bzip2-dev make wget 
+
+
+RUN cd /tmp && \
+  wget -q https://launchpad.net/pbzip2/1.1/1.1.13/+download/pbzip2-1.1.13.tar.gz && \
+  tar -xzf pbzip2-1.1.13.tar.gz && \
+  cd pbzip2-1.1.13/ && \
+  make install && \
+  rm -r /tmp/pbzip2-1.1.13/
+
+FROM docker.elastic.co/wolfi/python:3.12.3-dev
 ARG RALLY_VERSION
 ARG RALLY_LICENSE
 
-ENV RALLY_RUNNING_IN_DOCKER True
+ENV RALLY_RUNNING_IN_DOCKER=True
 
-RUN apt-get -y update && \
-    apt-get install -y curl git gcc pbzip2 pigz && \
-    apt-get -y upgrade && \
-    rm -rf /var/lib/apt/lists/*
+USER root
+COPY --from=builder /usr/bin/ /usr/bin/
 
-RUN groupadd --gid 1000 rally && \
-    useradd -d /rally -m -k /dev/null -g 1000 -N -u 1000 -l -s /bin/bash rally
+RUN addgroup --gid 1000 rally && \
+    adduser --system --home /rally --ingroup rally --no-create-home --uid 1000 --shell /bin/bash rally
 
-RUN pip3 install --upgrade hatch hatchling pip wheel
-RUN pip3 install esrally==$RALLY_VERSION
+RUN mkdir /rally/
 
 WORKDIR /rally
 COPY --chown=1000:0 docker/bin/entrypoint.sh /entrypoint.sh
@@ -25,12 +40,17 @@ RUN chgrp 0 /entrypoint.sh && \
     chmod 0775 /entrypoint.sh
 
 RUN mkdir -p /rally/.rally && \
-    chown -R 1000:0 /rally/.rally
+    chown -R 1000:0 /rally/
 
 USER 1000
 
+RUN python3 -m venv /rally/venv
 ENV PATH=/rally/venv/bin:$PATH
 
+RUN python3 -m pip install --upgrade hatch hatchling pip wheel
+RUN python3 -m pip install esrally==$RALLY_VERSION
+
+
 LABEL org.label-schema.schema-version="1.0" \
   org.label-schema.vendor="Elastic" \
   org.label-schema.name="rally" \

docs/configuration.rst

@@ -6,6 +6,10 @@ Rally Configuration
 
 Rally stores its configuration in the file ``~/.rally/rally.ini`` which is automatically created the first time Rally is executed. It comprises the following sections.
 
+.. note:: 
+    The configuration file can use `${CONFIG_DIR}` to refer to the directory where Rally stores its configuration files. This is useful for configuring Rally in a portable way.
+    This defaults to `~/.rally`, but can be overridden by setting the `RALLY_HOME` environment variable in your shell.
+
 meta
 ~~~~
 
@@ -239,3 +243,44 @@ With the following configuration Rally will log all output to standard error::
         }
       }
     }
+
+Portability
+~~~~~~~~~~~
+
+You can also use ``${LOG_PATH}`` in the ``"filename"`` value of the handler you are configuring to make the log configuration more portable.
+Rally will substitute ``${LOG_PATH}`` with the path to the directory where Rally stores its log files. By default, this is ``~/.rally/logs``. 
+But this can be overridden by setting the ``RALLY_HOME`` environment variable in your shell, and logs will be stored in ``${RALLY_HOME}/logs``.
+
+NOTE:: This is only supported with the ``esrally.log.configure_file_handler`` and ``esrally.log.configure_profile_file_handler`` handlers.
+
+Here is an example of a logging configuration that uses ``${LOG_PATH}``::
+
+    {
+      "version": 1,
+      "formatters": {
+        "normal": {
+          "format": "%(asctime)s,%(msecs)d %(actorAddress)s/PID:%(process)d %(name)s %(levelname)s %(message)s",
+          "datefmt": "%Y-%m-%d %H:%M:%S",
+          "()": "esrally.log.configure_utc_formatter"
+        }
+      },
+      "handlers": {
+        "rally_log_handler": {
+          "()": "esrally.log.configure_file_handler", # <-- use configure_file_handler or configure_profile_file_handler
+          "filename": "${LOG_PATH}/rally.log", # <-- use ${LOG_PATH} here
+          "encoding": "UTF-8",
+          "formatter": "normal"
+        }
+      },
+      "root": {
+        "handlers": ["rally_log_handler"],
+        "level": "INFO"
+      },
+      "loggers": {
+        "elasticsearch": {
+          "handlers": ["rally_log_handler"],
+          "level": "WARNING",
+          "propagate": false
+        }
+      }
+    }

docs/docker.rst

@@ -70,6 +70,17 @@ To customize Rally you can create your own ``rally.ini`` and bind mount it using
 
     docker run -v /home/<myuser>/custom_rally.ini:/rally/.rally/rally.ini elastic/rally ...
 
+As a quality of life improvement, the ``rally.ini`` and ``logging.json`` files can use ``${CONFIG_DIR}`` in ``rally.ini`` and ``${LOG_PATH}`` in ``logging.json`` in order to
+make the files more portable. For example:
+
+* In ``rally.ini``, you can set ``root.dir = ${CONFIG_DIR}/benchmarks`` instead of hard-coding the path as ``/rally/.rally/benchmarks``
+* In ``logging.json``, you can set ``"filename": "${LOG_PATH}/rally.log"`` instead of hard-coding the path as ``"filename": "/rally/.rally/logs/rally.log"``
+
+These files can then be used with the docker image, with the entire local ``~/.rally`` directory mounted as follows::
+
+    docker run -v type=bind,source=$HOME/.rally,target=/rally/.rally elastic/rally ...
+
+
 Persistence
 -----------
 

esrally/config.py

@@ -52,7 +52,10 @@ def present(self):
 
     def load(self) -> configparser.ConfigParser:
         config = configparser.ConfigParser()
-        config.read(self.location, encoding="utf-8")
+        with open(self.location, encoding="utf-8") as src:
+            contents = src.read()
+        contents = Template(contents).substitute(CONFIG_DIR=self.config_dir)
+        config.read_string(contents, source=self.location)
         return config
 
     def store_default_config(self, template_path=None):
@@ -64,7 +67,7 @@ def store_default_config(self, template_path=None):
         with open(self.location, "w", encoding="utf-8") as target:
             with open(source_path, encoding="utf-8") as src:
                 contents = src.read()
-                target.write(Template(contents).substitute(CONFIG_DIR=self.config_dir))
+                target.write(contents)
 
     def store(self, config: configparser.ConfigParser):
         io.ensure_dir(self.config_dir)

esrally/log.py

@@ -95,16 +95,29 @@ def install_default_log_config():
         source_path = io.normalize_path(os.path.join(os.path.dirname(__file__), "resources", "logging.json"))
         with open(log_config, "w", encoding="UTF-8") as target:
             with open(source_path, encoding="UTF-8") as src:
-                # Ensure we have a trailing path separator as after LOG_PATH there will only be the file name
-                log_path = os.path.join(paths.logs(), "")
-                # the logging path might contain backslashes that we need to escape
-                log_path = io.escape_path(log_path)
-                contents = src.read().replace("${LOG_PATH}", log_path)
+                contents = src.read()
                 target.write(contents)
     add_missing_loggers_to_config()
     io.ensure_dir(paths.logs())
 
 
+# pylint: disable=unused-argument
+def configure_file_handler(*args, **kwargs) -> logging.Handler:
+    """
+    Configures the WatchedFileHandler supporting expansion of `~` and `${LOG_PATH}` to the user's home and the log path respectively.
+    """
+    filename = kwargs.pop("filename").replace("${LOG_PATH}", paths.logs())
+    return logging.handlers.WatchedFileHandler(filename=filename, encoding=kwargs["encoding"], delay=kwargs.get("delay", False))
+
+
+def configure_profile_file_handler(*args, **kwargs) -> logging.Handler:
+    """
+    Configures the FileHandler supporting expansion of `~` and `${LOG_PATH}` to the user's home and the log path respectively.
+    """
+    filename = kwargs.pop("filename").replace("${LOG_PATH}", paths.logs())
+    return logging.FileHandler(filename=filename, encoding=kwargs["encoding"], delay=kwargs.get("delay", False))
+
+
 def load_configuration():
     """
     Loads the logging configuration. This is a low-level method and usually

esrally/resources/logging.json

@@ -19,15 +19,15 @@
   },
   "handlers": {
     "rally_log_handler": {
-      "class": "logging.handlers.WatchedFileHandler",
-      "filename": "${LOG_PATH}rally.log",
+      "()": "esrally.log.configure_file_handler",
+      "filename": "${LOG_PATH}/rally.log",
       "encoding": "UTF-8",
       "formatter": "normal",
       "filters": ["isActorLog"]
     },
     "rally_profile_handler": {
-      "class": "logging.FileHandler",
-      "filename": "${LOG_PATH}profile.log",
+      "()": "esrally.log.configure_profile_file_handler",
+      "filename": "${LOG_PATH}/profile.log",
       "delay": true,
       "encoding": "UTF-8",
       "formatter": "profile"