From 1bdeec0d90441a32889c85e4f19ecb40c718d7b6 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 12 Aug 2022 19:22:23 +0200 Subject: [PATCH] Help pip resolved to make better decisions on Pyarrow version The pip resolver in `eager upgrade` mode on Python 3.10 decides to downgrade Pyarrow to 5.0.0 which triggers numpy not being compatible with 3.10 - even if Pyarrow 6.* is perfectly fine. Adding the limit to eager upgrade directly helps the resolver to make better decisions on it. --- Dockerfile | 4 +++- Dockerfile.ci | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 273f8de939070..0dfade43b25a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1218,7 +1218,9 @@ ARG ADDITIONAL_PYTHON_DEPS="" # Those are additional constraints that are needed for some extras but we do not want to # Force them on the main Airflow package. # * dill<0.3.3 required by apache-beam -ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3" +# * pyarrow>=6.0.0 is because pip resolver decides for Python 3.10 to downgrade pyarrow to 5 even if it is OK +# for python 3.10 and other dependencies adding the limit helps resolver to make better decisions +ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 pyarrow>=6.0.0" ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ INSTALL_PACKAGES_FROM_CONTEXT=${INSTALL_PACKAGES_FROM_CONTEXT} \ diff --git a/Dockerfile.ci b/Dockerfile.ci index 76d8e5b730eb7..8b643dfa9223f 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1175,7 +1175,9 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}" # Those are additional constraints that are needed for some extras but we do not want to # force them on the main Airflow package. Those limitations are: # * dill<0.3.3 required by apache-beam -ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3" +# * pyarrow>=6.0.0 is because pip resolver decides for Python 3.10 to downgrade pyarrow to 5 even if it is OK +# for python 3.10 and other dependencies adding the limit helps resolver to make better decisions +ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 pyarrow>=6.0.0" ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \ UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}