chore: bump python deps for 3.10 and robot system updates (#13865)

his PR contains updates required to handle Opentrons/oe-core#110 and Opentrons/buildroot#213 which update the flex operating system and ot-2 operating system respectively to more recent versions of, well everything. Perhaps most significantly to us, both PRs update the python that runs on the machine to 3.10, which requires a lot of changes to a lot of our code. This PR makes those changes by updating the python versions used in our test harnesses to 3.10; setting those dependencies explicitly captured by the system builders to their new versions from the new versions of the system builders; and bumping other dependencies the minimum required to make it all work. Further work may increase dependency versions further. The big ones are updating pydantic to 1.9, which is something we've been putting off for a while and probably unlocks further updates. This update is because of the following, which is typical of the process: system builders updated numpy, which is locked in because it must be cross-compiled, to 1.17 numpy 1.17 includes type annotations via .pyi mypy must be updated to 0.981 because previous versions interpreted .pyi files as-if the python version was 3.7, but the .pyis had some >3.7-only features pydantic must be updated to 1.9 because 1.8 doesn't work with that updated mypy because of a change to internal mypy hooks a lot of server code and tests have to be slightly altered because pydantic 1.9 doesn't elide None keys, which is (IMO) good anyway This should hopefully free us to update mypy to the latest again, and hopefully the changes to explicit handling of none keys will let us update pydantic further. Anyway, there's a fair amount of "just run the robot" style testing to do here, and we have to decide what to do about loading pickles; handling user-installed dependencies; and further support of 3.7. But all the tests should now pass.
Opentrons · Jan 4, 2024 · 4f8a983 · 4f8a983
1 parent ee0e130
commit 4f8a983
Show file tree

Hide file tree

Showing 125 changed files with 6,644 additions and 6,461 deletions.
diff --git a/.github/actions/python/setup/action.yaml b/.github/actions/python/setup/action.yaml
@@ -27,6 +27,6 @@ runs:
     - shell: bash
       run: |
         npm install --global shx@0.3.3
-        $OT_PYTHON -m pip install pipenv==2021.5.29
+        $OT_PYTHON -m pip install pipenv==2023.11.15
     - shell: bash
       run: 'make -C ${{ inputs.project }} setup'
diff --git a/.github/workflows/api-test-lint-deploy.yaml b/.github/workflows/api-test-lint-deploy.yaml
@@ -59,7 +59,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - uses: './.github/actions/python/setup'
         with:
@@ -75,17 +75,13 @@ jobs:
         os: ['windows-2022', 'ubuntu-22.04', 'macos-latest']
         # TODO(mc, 2022-02-24): expand this matrix to 3.8 and 3.9,
         # preferably in a nightly cronjob on edge or something
-        python: ['3.7', '3.10']
+        python: ['3.10']
         with-ot-hardware: ['true', 'false']
         exclude:
           - os: 'windows-2022'
             with-ot-hardware: 'true'
           - os: 'macos-latest'
             with-ot-hardware: 'true'
-          - os: 'macos-latest'
-            python: '3.10'
-          - python: '3.10'
-            with-ot-hardware: 'true'
     runs-on: '${{ matrix.os }}'
     steps:
       - uses: 'actions/checkout@v3'
@@ -152,7 +148,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
       - name: 'set complex environment variables'
         uses: actions/github-script@v6
         with:

diff --git a/.github/workflows/docs-build.yaml b/.github/workflows/docs-build.yaml
@@ -54,7 +54,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v3'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
       - uses: './.github/actions/python/setup'
         with:
           project: 'api'

diff --git a/.github/workflows/g-code-confirm-tests.yaml b/.github/workflows/g-code-confirm-tests.yaml
@@ -49,7 +49,7 @@ jobs:
           node-version: '12'
       - uses: 'actions/setup-python@v3'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
       - uses: './.github/actions/python/setup'
         with:
           project: 'g-code-testing'

diff --git a/.github/workflows/g-code-testing-lint-test.yaml b/.github/workflows/g-code-testing-lint-test.yaml
@@ -59,7 +59,7 @@ jobs:
             buildComplexEnvVars(core, context)
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
       - uses: './.github/actions/python/setup'
         with:
           project: 'g-code-testing'

diff --git a/.github/workflows/hardware-lint-test.yaml b/.github/workflows/hardware-lint-test.yaml
@@ -55,7 +55,7 @@ jobs:
       - name: Setup Python
         uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - name: Setup Hardware Project
         uses: './.github/actions/python/setup'

diff --git a/.github/workflows/hardware-testing-protocols.yaml b/.github/workflows/hardware-testing-protocols.yaml
@@ -50,7 +50,7 @@ jobs:
       - name: Setup Python
         uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - name: Setup Project
         uses: './.github/actions/python/setup'

diff --git a/.github/workflows/hardware-testing.yaml b/.github/workflows/hardware-testing.yaml
@@ -52,7 +52,7 @@ jobs:
       - name: Setup Python
         uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - name: Setup Project
         uses: './.github/actions/python/setup'

diff --git a/.github/workflows/http-docs-build.yaml b/.github/workflows/http-docs-build.yaml
@@ -51,7 +51,7 @@ jobs:
           git checkout ${{ github.ref }}
       - uses: 'actions/setup-python@v3'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
       - uses: 'actions/setup-node@v3'
         with:
           node-version: '16'

diff --git a/.github/workflows/notify-server-lint-test.yaml b/.github/workflows/notify-server-lint-test.yaml
@@ -54,7 +54,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - uses: './.github/actions/python/setup'
         with:

diff --git a/.github/workflows/robot-server-lint-test.yaml b/.github/workflows/robot-server-lint-test.yaml
@@ -66,7 +66,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - uses: './.github/actions/python/setup'
         with:

diff --git a/.github/workflows/server-utils-lint-test.yaml b/.github/workflows/server-utils-lint-test.yaml
@@ -49,7 +49,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - uses: './.github/actions/python/setup'
         with:
@@ -70,7 +70,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
       - uses: './.github/actions/python/setup'
         with:
           project: 'server-utils'

diff --git a/.github/workflows/shared-data-test-lint-deploy.yaml b/.github/workflows/shared-data-test-lint-deploy.yaml
@@ -54,7 +54,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v3'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - uses: './.github/actions/python/setup'
         with:
@@ -71,10 +71,8 @@ jobs:
         os: ['windows-2022', 'ubuntu-22.04', 'macos-latest']
         # TODO(mc, 2022-02-24): expand this matrix to 3.8 and 3.9,
         # preferably in a nightly cronjob on edge or something
-        python: ['3.7', '3.10']
-        exclude:
-          - os: 'macos-latest'
-            python: '3.10'
+        python: ['3.10']
+
     runs-on: '${{ matrix.os }}'
     steps:
       - uses: 'actions/checkout@v3'
@@ -164,7 +162,7 @@ jobs:
         run: sudo apt-get update && sudo apt-get install libudev-dev
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
       - uses: './.github/actions/python/setup'
         with:
           project: 'shared-data/python'

diff --git a/.github/workflows/system-server-lint-test.yaml b/.github/workflows/system-server-lint-test.yaml
@@ -51,7 +51,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - uses: './.github/actions/python/setup'
         with:
@@ -72,7 +72,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
       - uses: './.github/actions/python/setup'
         with:
           project: 'system-server'

diff --git a/.github/workflows/update-server-lint-test.yaml b/.github/workflows/update-server-lint-test.yaml
@@ -49,7 +49,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - uses: './.github/actions/python/setup'
         with:
@@ -70,7 +70,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
       - uses: './.github/actions/python/setup'
         with:
           project: 'update-server'

diff --git a/.github/workflows/usb-bridge-lint-test.yaml b/.github/workflows/usb-bridge-lint-test.yaml
@@ -49,7 +49,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
 
       - uses: './.github/actions/python/setup'
         with:
@@ -70,7 +70,7 @@ jobs:
           node-version: '16'
       - uses: 'actions/setup-python@v4'
         with:
-          python-version: '3.7'
+          python-version: '3.10'
       - uses: './.github/actions/python/setup'
         with:
           project: 'usb-bridge'

diff --git a/Makefile b/Makefile
@@ -66,7 +66,7 @@ PYTHON_SETUP_TARGETS := $(addsuffix -py-setup, $(PYTHON_DIRS))
 
 .PHONY: setup-py
 setup-py:
-	$(OT_PYTHON) -m pip install pipenv==2021.5.29
+	$(OT_PYTHON) -m pip install pipenv==2023.11.15
 	$(MAKE) $(PYTHON_SETUP_TARGETS)
 
 
@@ -266,4 +266,4 @@ test-js-internal:
 
 .PHONY: test-js-%
 test-js-%: 
-	$(MAKE) test-js-internal tests="$(if $(tests),$(foreach test,$(tests),$*/$(test)),$*)" test_opts="$(test_opts)" cov_opts="$(cov_opts)"
+	$(MAKE) test-js-internal tests="$(if $(tests),$(foreach test,$(tests),$*/$(test)),$*)" test_opts="$(test_opts)" cov_opts="$(cov_opts)"
diff --git a/api/Pipfile b/api/Pipfile
@@ -7,24 +7,24 @@ name = "pypi"
 # atomicwrites and colorama are pytest dependencies on windows,
 # spec'd here to force lockfile inclusion
 # https://github.com/pypa/pipenv/issues/4408#issuecomment-668324177
-atomicwrites = { version = "==1.4.0", sys_platform = "== 'win32'" }
-colorama = { version = "==0.4.4", sys_platform = "== 'win32'" }
+atomicwrites = { version = "==1.4.0", markers="sys_platform=='win32'" }
+colorama = { version = "==0.4.4", markers="sys_platform=='win32'" }
 coverage = "==5.1"
-mypy = "==0.910"
+mypy = "==0.981"
 numpydoc = "==0.9.1"
 pytest = "==7.0.1"
-pytest-asyncio = "~=0.18"
+pytest-asyncio = "~=0.16"
 pytest-cov = "==2.10.1"
 pytest-lazy-fixture = "==0.6.3"
-pytest-xdist = "~=2.2.1"
+pytest-xdist = "~=2.5.0"
 sphinx = "==5.0.1"
-twine = "==4.0.2"
-wheel = "==0.30.0"
-typeguard = "==2.12.1"
+twine = "==4.0.0"
+wheel = "==0.37.0"
+typeguard = "==2.13.1"
 sphinx-substitution-extensions = "==2020.9.30.0"
 sphinxext-opengraph = "==0.8.1"
 sphinx-tabs = ">=3.4.1,<4"
-mock = "~=4.0.2"
+mock = "~=4.0.3"
 flake8 = "~=3.9.0"
 flake8-annotations = "~=2.6.2"
 flake8-docstrings = "~=1.6.0"
@@ -35,7 +35,7 @@ types-mock = "==4.0.1"
 types-setuptools = "==57.0.2"
 opentrons-shared-data = { editable = true, path = "../shared-data/python" }
 opentrons = { editable = true, path = "." }
-opentrons-hardware = { editable = true, path = "./../hardware", extras= ["FLEX"] }
+opentrons-hardware = { editable = true, path = "./../hardware", extras=["FLEX"] }
 # specify typing-extensions explicitly to force lockfile inclusion on Python >= 3.8
 typing-extensions = ">=4.0.0,<5"
 pytest-profiling = "~=1.7.0"