Add Buildkite pipeline to push Docker image

.buildkite/release-docker/pipeline.yml

@@ -1,14 +1,28 @@
 steps:
   - input: "Build parameters"
     fields:
-      - text: "RELEASE_VERSION"
+      - text: "Release version"
         key: "RELEASE_VERSION"
         default: ""
-        hint: "The version to release e.g. '2.8.0'."
-
+        format: "\\d{1,}.\\d{1,}.\\d{1,}"
+        hint: "The version to release e.g. '8.10.0' (without the v prefix)."
+      - select: "Environment"
+        key: "ENVIRONMENT"
+        options:
+          - label: "Staging"
+            value: "staging"
+          - label: "Production"
+            value: "production"
   - wait
   - label: "Release Docker Artifacts for Eland"
-    command: docker -v  # TODO: Actually release Eland
+    command: |
+      set -eo pipefail
+
+      export RELEASE_VERSION=$(buildkite-agent meta-data get RELEASE_VERSION)
+      export ENVIRONMENT=$(buildkite-agent meta-data get ENVIRONMENT)
+      export BUILDKIT_PROGRESS=plain
+
+      bash .buildkite/release-docker/run.sh
     # Run on GCP to use `docker`
     agents:
       provider: gcp

.buildkite/release-docker/run.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+export LC_ALL=en_US.UTF-8
+
+echo "Publishing Eland $RELEASE_VERSION Docker image to $ENVIRONMENT"
+
+set +x
+# login to docker registry
+docker_registry=$(vault read -field registry "secret/ci/elastic-eland/container-library/eland-$ENVIRONMENT")
+docker_username=$(vault read -field username "secret/ci/elastic-eland/container-library/eland-$ENVIRONMENT")
+docker_password=$(vault read -field password "secret/ci/elastic-eland/container-library/eland-$ENVIRONMENT")
+
+echo "$docker_password" | docker login "$docker_registry" --username "$docker_username" --password-stdin
+unset docker_username docker_password
+set -x
+
+tmp_dir=$(mktemp --directory)
+pushd "$tmp_dir"
+git clone https://github.com/elastic/eland
+pushd eland
+git checkout "v${RELEASE_VERSION}"
+git --no-pager show
+
+docker build -t "$docker_registry/eland/eland:$RELEASE_VERSION" "$PWD"
+docker push "$docker_registry/eland/eland:$RELEASE_VERSION"
+
+docker tag "$docker_registry/eland/eland:$RELEASE_VERSION" "$docker_registry/eland/eland:latest"
+docker push "$docker_registry/eland/eland:latest"
+
+popd
+popd
+rm -rf "$tmp_dir"

.readthedocs.yml

@@ -1,9 +1,11 @@
 version: 2
-sphinx:
-  configuration: docs/sphinx/conf.py
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3"
 
 python:
-  version: 3.8
   install:
     - requirements: docs/requirements-docs.txt
     - path: .

Dockerfile

@@ -1,8 +1,7 @@
-FROM debian:bullseye-20230904
+FROM python:3.10-slim-bookworm@sha256:d364435d339ad318ac4c533b9fbe709739f9ba006b0721fd25e8592d0bb857cb
 
 RUN apt-get update && \
     apt-get install -y build-essential pkg-config cmake \
-                       python3-dev python3-pip python3-venv \
                        libzip-dev libjpeg-dev && \
     apt-get clean
 

eland/dataframe.py

@@ -983,7 +983,7 @@ def _sizeof_fmt(num: float, size_qualifier: str) -> str:
                 index=self._query_compiler._index_pattern, metric=["store"]
             )["_all"]["total"]["store"]["size_in_bytes"]
             lines.append(
-                f"Elasticsearch storage usage: {_sizeof_fmt(storage_usage,size_qualifier)}\n"
+                f"Elasticsearch storage usage: {_sizeof_fmt(storage_usage, size_qualifier)}\n"
             )
 
         fmt.buffer_put_lines(buf, lines)

eland/ml/_optional.py

@@ -15,11 +15,12 @@
 #  specific language governing permissions and limitations
 #  under the License.
 
-import distutils.version
 import importlib
 import warnings
 from typing import TYPE_CHECKING, Any, Optional
 
+import packaging.version
+
 if TYPE_CHECKING:
     from types import ModuleType
 
@@ -30,7 +31,7 @@
 # The license for this library can be found NOTICE.txt and the code can be
 # https://raw.githubusercontent.com/pandas-dev/pandas/v1.0.1/pandas/compat/_optional.py
 
-VERSIONS = {"xgboost": "0.90", "sklearn": "0.22.1"}
+VERSIONS = {"xgboost": "0.90", "sklearn": "1.3"}
 
 # Update install.rst when updating versions!
 
@@ -104,7 +105,9 @@ def import_optional_dependency(
     minimum_version = VERSIONS.get(name)
     if minimum_version:
         version = _get_version(module)
-        if distutils.version.LooseVersion(version) < minimum_version:
+        if packaging.version.parse(version) < packaging.version.Version(
+            minimum_version
+        ):
             assert on_version in {"warn", "raise", "ignore"}
             msg = version_message.format(
                 minimum_version=minimum_version, name=name, actual_version=version

setup.py

@@ -86,6 +86,7 @@
         "pandas>=1.5,<2",
         "matplotlib>=3.6",
         "numpy>=1.2.0,<1.24",
+        "packaging",
     ],
     entry_points={
         "console_scripts": "eland_import_hub_model=eland.cli.eland_import_hub_model:main"

utils/generate-supported-apis.py

@@ -84,7 +84,7 @@ def main():
 
 .. currentmodule:: eland
 
-.. automethod:: { attr.replace('eland.', '') }
+.. automethod:: {attr.replace('eland.', '')}
 """
                     )