Upgrade Python version to 3.11 (#2278)

* Upgrade Python version to 3.11 Signed-off-by: Yuki Iwai <yuki.iwai.tz@gmail.com> * Upgrade the numpy version to 1.25.2 Signed-off-by: Yuki Iwai <yuki.iwai.tz@gmail.com> * Increase resource requests for the ENAS suggestion service Signed-off-by: Yuki Iwai <yuki.iwai.tz@gmail.com> * Update pytest CI Signed-off-by: Yuki Iwai <yuki.iwai.tz@gmail.com> * Prepare dedicated pytest for skopt Signed-off-by: Yuki Iwai <yuki.iwai.tz@gmail.com> --------- Signed-off-by: Yuki Iwai <yuki.iwai.tz@gmail.com>
kubeflow · Mar 12, 2024 · 6f372f6 · 6f372f6
1 parent 5837b8a
commit 6f372f6
Show file tree

Hide file tree

Showing 26 changed files with 56 additions and 27 deletions.
diff --git a/.github/workflows/test-python.yaml b/.github/workflows/test-python.yaml
@@ -21,7 +21,27 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: 3.11
 
       - name: Run Python test
         run: make pytest
+
+  # The skopt service doesn't work appropriately with Python 3.11.
+  # So, we need to run the test with Python 3.9.
+  # TODO (tenzen-y): Once we stop to support skopt, we can remove this test.
+  # REF: https://github.com/kubeflow/katib/issues/2280
+  test-skopt:
+    name: Test Skopt
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Run Python test
+        run: make pytest-skopt
diff --git a/Makefile b/Makefile
@@ -157,7 +157,6 @@ update-boilerplate:
 prepare-pytest:
 	pip install --prefer-binary -r test/unit/v1beta1/requirements.txt
 	pip install --prefer-binary -r cmd/suggestion/hyperopt/v1beta1/requirements.txt
-	pip install --prefer-binary -r cmd/suggestion/skopt/v1beta1/requirements.txt
 	pip install --prefer-binary -r cmd/suggestion/optuna/v1beta1/requirements.txt
 	pip install --prefer-binary -r cmd/suggestion/hyperband/v1beta1/requirements.txt
 	pip install --prefer-binary -r cmd/suggestion/nas/enas/v1beta1/requirements.txt
@@ -176,6 +175,16 @@ ifeq ("$(wildcard $(TEST_TENSORFLOW_EVENT_FILE_PATH))", "")
 endif
 
 pytest: prepare-pytest prepare-pytest-testdata
-	PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/suggestion
+	PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/suggestion --ignore=./test/unit/v1beta1/suggestion/test_skopt_service.py
 	PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/earlystopping
 	PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/metricscollector
+
+# The skopt service doesn't work appropriately with Python 3.11.
+# So, we need to run the test with Python 3.9.
+# TODO (tenzen-y): Once we stop to support skopt, we can remove this test.
+# REF: https://github.com/kubeflow/katib/issues/2280
+pytest-skopt:
+	pip install six
+	pip install --prefer-binary -r test/unit/v1beta1/requirements.txt
+	pip install --prefer-binary -r cmd/suggestion/skopt/v1beta1/requirements.txt
+	PYTHONPATH=$(PYTHONPATH) pytest ./test/unit/v1beta1/suggestion/test_skopt_service.py
diff --git a/cmd/earlystopping/medianstop/v1beta1/Dockerfile b/cmd/earlystopping/medianstop/v1beta1/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ARG TARGETARCH
 ENV TARGET_DIR /opt/katib

diff --git a/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile b/cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ARG TARGETARCH
 ENV TARGET_DIR /opt/katib

diff --git a/cmd/suggestion/hyperband/v1beta1/Dockerfile b/cmd/suggestion/hyperband/v1beta1/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ARG TARGETARCH
 ENV TARGET_DIR /opt/katib

diff --git a/cmd/suggestion/hyperband/v1beta1/requirements.txt b/cmd/suggestion/hyperband/v1beta1/requirements.txt
@@ -1,6 +1,6 @@
 grpcio>=1.41.1
 cloudpickle==0.5.6
-numpy>=1.20.0
+numpy>=1.25.2
 scikit-learn>=0.24.0
 scipy>=1.5.4
 forestci==0.3

diff --git a/cmd/suggestion/hyperopt/v1beta1/Dockerfile b/cmd/suggestion/hyperopt/v1beta1/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ARG TARGETARCH
 ENV TARGET_DIR /opt/katib

diff --git a/cmd/suggestion/hyperopt/v1beta1/requirements.txt b/cmd/suggestion/hyperopt/v1beta1/requirements.txt
@@ -1,6 +1,6 @@
 grpcio>=1.41.1
 cloudpickle==0.5.6
-numpy>=1.20.0
+numpy>=1.25.2
 scikit-learn>=0.24.0
 scipy>=1.5.4
 forestci==0.3

diff --git a/cmd/suggestion/nas/darts/v1beta1/Dockerfile b/cmd/suggestion/nas/darts/v1beta1/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ARG TARGETARCH
 ENV TARGET_DIR /opt/katib

diff --git a/cmd/suggestion/nas/enas/v1beta1/Dockerfile b/cmd/suggestion/nas/enas/v1beta1/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ARG TARGETARCH
 ENV TARGET_DIR /opt/katib

diff --git a/cmd/suggestion/optuna/v1beta1/Dockerfile b/cmd/suggestion/optuna/v1beta1/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ARG TARGETARCH
 ENV TARGET_DIR /opt/katib

diff --git a/cmd/suggestion/pbt/v1beta1/Dockerfile b/cmd/suggestion/pbt/v1beta1/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ARG TARGETARCH
 ENV TARGET_DIR /opt/katib

diff --git a/cmd/suggestion/pbt/v1beta1/requirements.txt b/cmd/suggestion/pbt/v1beta1/requirements.txt
@@ -1,4 +1,4 @@
 grpcio>=1.41.1
 protobuf>=3.19.5, <=3.20.3
 googleapis-common-protos==1.53.0
-numpy==1.22.2
+numpy==1.25.2
diff --git a/docs/developer-guide.md b/docs/developer-guide.md
@@ -16,7 +16,7 @@ see the following user guides:
 - [Docker](https://docs.docker.com/) (20.10 or later)
 - [Docker Buildx](https://docs.docker.com/build/buildx/) (0.8.0 or later)
 - [Java](https://docs.oracle.com/javase/8/docs/technotes/guides/install/install_overview.html) (8 or later)
-- [Python](https://www.python.org/) (3.10 or later)
+- [Python](https://www.python.org/) (3.11 or later)
 - [kustomize](https://kustomize.io/) (4.0.5 or later)
 
 ## Build from source code

diff --git a/examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.cpu b/examples/v1beta1/trial-images/darts-cnn-cifar10/Dockerfile.cpu
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ENV TARGET_DIR /opt/darts-cnn-cifar10
 

diff --git a/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu b/examples/v1beta1/trial-images/enas-cnn-cifar10/Dockerfile.cpu
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ARG TARGETARCH
 ENV TARGET_DIR /opt/enas-cnn-cifar10

diff --git a/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.cpu b/examples/v1beta1/trial-images/pytorch-mnist/Dockerfile.cpu
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ADD examples/v1beta1/trial-images/pytorch-mnist /opt/pytorch-mnist
 

diff --git a/examples/v1beta1/trial-images/simple-pbt/Dockerfile b/examples/v1beta1/trial-images/simple-pbt/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ADD examples/v1beta1/trial-images/simple-pbt /opt/pbt
 

diff --git a/examples/v1beta1/trial-images/simple-pbt/requirements.txt b/examples/v1beta1/trial-images/simple-pbt/requirements.txt
@@ -1 +1 @@
-numpy==1.22.2
+numpy==1.25.2
diff --git a/examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile b/examples/v1beta1/trial-images/tf-mnist-with-summaries/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 ARG TARGETARCH
 

diff --git a/manifests/v1beta1/installs/katib-cert-manager/katib-config.yaml b/manifests/v1beta1/installs/katib-cert-manager/katib-config.yaml
@@ -43,7 +43,7 @@ runtime:
       image: docker.io/kubeflowkatib/suggestion-enas:latest
       resources:
         limits:
-          memory: 200Mi
+          memory: 400Mi
     - algorithmName: darts
       image: docker.io/kubeflowkatib/suggestion-darts:latest
     - algorithmName: pbt

diff --git a/manifests/v1beta1/installs/katib-external-db/katib-config.yaml b/manifests/v1beta1/installs/katib-external-db/katib-config.yaml
@@ -45,7 +45,7 @@ runtime:
       image: docker.io/kubeflowkatib/suggestion-enas:latest
       resources:
         limits:
-          memory: 200Mi
+          memory: 400Mi
     - algorithmName: darts
       image: docker.io/kubeflowkatib/suggestion-darts:latest
     - algorithmName: pbt

diff --git a/manifests/v1beta1/installs/katib-leader-election/katib-config.yaml b/manifests/v1beta1/installs/katib-leader-election/katib-config.yaml
@@ -46,7 +46,7 @@ runtime:
       image: docker.io/kubeflowkatib/suggestion-enas:latest
       resources:
         limits:
-          memory: 200Mi
+          memory: 400Mi
     - algorithmName: darts
       image: docker.io/kubeflowkatib/suggestion-darts:latest
     - algorithmName: pbt

diff --git a/manifests/v1beta1/installs/katib-openshift/katib-config.yaml b/manifests/v1beta1/installs/katib-openshift/katib-config.yaml
@@ -43,7 +43,7 @@ runtime:
       image: docker.io/kubeflowkatib/suggestion-enas:latest
       resources:
         limits:
-          memory: 200Mi
+          memory: 400Mi
     - algorithmName: darts
       image: docker.io/kubeflowkatib/suggestion-darts:latest
     - algorithmName: pbt

diff --git a/manifests/v1beta1/installs/katib-standalone-postgres/katib-config.yaml b/manifests/v1beta1/installs/katib-standalone-postgres/katib-config.yaml
@@ -45,7 +45,7 @@ runtime:
       image: docker.io/kubeflowkatib/suggestion-enas:latest
       resources:
         limits:
-          memory: 200Mi
+          memory: 400Mi
     - algorithmName: darts
       image: docker.io/kubeflowkatib/suggestion-darts:latest
     - algorithmName: pbt

diff --git a/manifests/v1beta1/installs/katib-standalone/katib-config.yaml b/manifests/v1beta1/installs/katib-standalone/katib-config.yaml
@@ -45,7 +45,7 @@ runtime:
       image: docker.io/kubeflowkatib/suggestion-enas:latest
       resources:
         limits:
-          memory: 200Mi
+          memory: 400Mi
     - algorithmName: darts
       image: docker.io/kubeflowkatib/suggestion-darts:latest
     - algorithmName: pbt