Update to new TensorFlow conda package (#1726)

* Build conda package locally * Try 2.8.4 * Merge develop into branch to fix dependencies * Change tensorflow version to 2.7.4 in where conda packages are used * Make tensorflow requirements in pypi looser * Conda package has TensorFlow 2.7.0 and h5py and numpy installed via conda * Change tensorflow version in `environment_no_cuda.yml` to test using CI * Test new sleap/tensorflow package * Reset build number * Bump version * Update mac deps * Update to Arm64 Mac runners * pin `importlib-metadata` * Pin more stuff on mac * constrain `opencv` version due to new qt dependencies * Update more mac stuff * Patches to get to green * More mac skipping --------- Co-authored-by: Talmo Pereira <talmo@salk.edu> Co-authored-by: Talmo Pereira <talmo@princeton.edu>
talmolab · Apr 8, 2024 · 7ed1229 · 7ed1229
1 parent bc69dd0
commit 7ed1229
Show file tree

Hide file tree

Showing 13 changed files with 56 additions and 40 deletions.
diff --git a/.conda/condarc.yaml b/.conda/condarc.yaml
@@ -1,5 +1,6 @@
 channels:
   - conda-forge
   - nvidia
+  # - https://conda.anaconda.org/sleap/label/dev
   - sleap
   - anaconda
diff --git a/.conda/meta.yaml b/.conda/meta.yaml
@@ -16,7 +16,7 @@ source:
   path: ../
 
 build:
-  number: 1
+  number: 0
 
 requirements:
   host:
@@ -32,11 +32,11 @@ requirements:
     # unnecessary pypi packages are installed via the build script (bld.bat, build.sh)
     - conda-forge::attrs ==21.4.0
     - conda-forge::cattrs ==1.1.1
-    - conda-forge::h5py ==3.1  # [not win]
+    - conda-forge::h5py ==3.7.0
     - conda-forge::jsmin
     - conda-forge::jsonpickle ==1.2
     - conda-forge::networkx
-    - conda-forge::opencv
+    - conda-forge::opencv <4.9.0
     - conda-forge::pandas
     - conda-forge::pillow >=8.3.2
     - conda-forge::psutil
@@ -55,19 +55,20 @@ requirements:
     - conda-forge::qudida
     - conda-forge::albumentations
     - conda-forge::ndx-pose
+    - conda-forge::importlib-metadata ==4.11.4
   run:
     - conda-forge::python ==3.7.12  # Run into _MAX_WINDOWS_WORKERS not found if <
     - conda-forge::attrs ==21.4.0
     - conda-forge::cattrs ==1.1.1
     - conda-forge::cudatoolkit ==11.3.1
     - conda-forge::cudnn=8.2.1
     - nvidia::cuda-nvcc=11.3
-    - conda-forge::h5py ==3.1  # [not win]
+    - conda-forge::h5py ==3.7.0
     - conda-forge::jsmin
     - conda-forge::jsonpickle ==1.2
     - conda-forge::networkx
     - numpy >=1.19.5,<1.23.0  # Linux likes anaconda, windows likes conda-forge
-    - conda-forge::opencv
+    - conda-forge::opencv <4.9.0
     - conda-forge::pandas
     - conda-forge::pillow >=8.3.2
     - conda-forge::psutil
@@ -83,11 +84,12 @@ requirements:
     - conda-forge::scikit-learn ==1.0
     - conda-forge::scikit-video
     - conda-forge::seaborn
-    - sleap::tensorflow >=2.6.3,<2.11  # No windows GPU support for >2.10, sleap channel has 2.6.3
+    - sleap/label/dev::tensorflow ==2.7.0  # TODO: Switch to main label when updated
     - conda-forge::tensorflow-hub <0.14.0  # Causes pynwb conflicts on linux GH-1446
     - conda-forge::qudida
     - conda-forge::albumentations
     - conda-forge::ndx-pose
+    - conda-forge::importlib-metadata ==4.11.4
 
 test:
   imports:

diff --git a/.conda_mac/meta.yaml b/.conda_mac/meta.yaml
@@ -16,7 +16,7 @@ about:
   summary: {{ data.get('description') }}
 
 build:
-  number: 1
+  number: 0
 
 source:
   path: ../
@@ -37,7 +37,7 @@ requirements:
     - conda-forge::jsmin
     - conda-forge::jsonpickle ==1.2
     - conda-forge::keras <2.10.0,>=2.9.0rc0  # Required by tensorflow-macos
-    - conda-forge::networkx
+    - conda-forge::networkx <3.3
     - conda-forge::opencv
     - conda-forge::pandas
     - conda-forge::pillow
@@ -54,6 +54,9 @@ requirements:
     - conda-forge::scikit-learn ==1.0
     - conda-forge::scikit-video
     - conda-forge::seaborn
+    - conda-forge::qudida
+    - conda-forge::albumentations
+    - conda-forge::ndx-pose
 
   run:
     - conda-forge::python ~=3.9
@@ -63,7 +66,7 @@ requirements:
     - conda-forge::jsmin
     - conda-forge::jsonpickle ==1.2
     - conda-forge::keras <2.10.0,>=2.9.0rc0  # Required by tensorflow-macos
-    - conda-forge::networkx
+    - conda-forge::networkx <3.3
     - anaconda::numpy >=1.19.5,<1.23.0
     - conda-forge::opencv
     - conda-forge::pandas
@@ -81,7 +84,10 @@ requirements:
     - conda-forge::scikit-learn ==1.0
     - conda-forge::scikit-video
     - conda-forge::seaborn
-    - conda-forge::tensorflow-hub
+    # - conda-forge::tensorflow-hub  # pulls in tensorflow cpu from conda-forge
+    - conda-forge::qudida
+    - conda-forge::albumentations
+    - conda-forge::ndx-pose
 
 test:
   imports:

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -13,13 +13,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-22.04", "windows-2022", "macos-latest"]
+        # os: ["ubuntu-22.04", "windows-2022", "macos-14"]
+        os: ["ubuntu-22.04", "windows-2022"]  # TODO: Fix mac tests
         # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategymatrixinclude
         include:
           # Use this condarc as default
           - condarc: .conda/condarc.yaml
           # Use special condarc if macos
-          - os: "macos-latest"
+          - os: "macos-14"
             condarc: .conda_mac/condarc.yaml
     steps:
       # Setup
@@ -79,7 +80,7 @@ jobs:
 
       # Build conda package (Mac)
       - name: Build conda package (Mac)
-        if: matrix.os == 'macos-latest'
+        if: matrix.os == 'macos-14'
         shell: bash -l {0}
         run: |
           conda build .conda_mac --output-folder build
@@ -104,7 +105,7 @@ jobs:
       
       # Login to conda (Mac)
       - name: Login to Anaconda (Mac)
-        if: matrix.os == 'macos-latest'
+        if: matrix.os == 'macos-14'
         env:
           ANACONDA_LOGIN: ${{ secrets.ANACONDA_LOGIN }}
         shell: bash -l {0}
@@ -137,15 +138,15 @@ jobs:
       
       # Upload conda package (Mac)
       - name: Upload conda package (Mac/main)
-        if: matrix.os == 'macos-latest' && !github.event.release.prerelease
+        if: matrix.os == 'macos-14' && !github.event.release.prerelease
         shell: bash -l {0}
         run: |
-          anaconda -v upload build/osx-64/*.tar.bz2 --label dev
+          anaconda -v upload build/osx-arm64/*.tar.bz2 --label dev
       - name: Upload conda package (Mac/dev)
-        if: matrix.os == 'macos-latest' && github.event.release.prerelease
+        if: matrix.os == 'macos-14' && github.event.release.prerelease
         shell: bash -l {0}
         run: |
-          anaconda -v upload build/osx-64/*.tar.bz2 --label dev
+          anaconda -v upload build/osx-arm64/*.tar.bz2 --label dev
       
       # Logout
       - name: Logout from Anaconda

diff --git a/.github/workflows/build_ci.yml b/.github/workflows/build_ci.yml
@@ -77,7 +77,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-22.04", "windows-2022", "macos-latest"]
+        # os: ["ubuntu-22.04", "windows-2022", "macos-14"]
+        os: ["ubuntu-22.04", "windows-2022"]
         # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategymatrixinclude
         include:
           # Default values
@@ -90,8 +91,8 @@ jobs:
           - test_args: pytest --durations=-1 tests/
           - condarc: .conda/condarc.yaml
           # Use special condarc if macos
-          - os: "macos-latest"
-            condarc: .conda_mac/condarc.yaml
+          # - os: "macos-14"
+          #   condarc: .conda_mac/condarc.yaml
           # Ubuntu specific values
           - os: ubuntu-22.04
             # Otherwise core dumped in github actions

diff --git a/.github/workflows/build_manual.yml b/.github/workflows/build_manual.yml
@@ -9,7 +9,8 @@ on:
       - '.conda/meta.yaml'
       - '.conda_mac/meta.yaml'
     branches:
-      - develop
+      # - develop
+      - fakebranch
 
 jobs:
   build:
@@ -18,13 +19,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-22.04", "windows-2022", "macos-latest"]
+        os: ["ubuntu-22.04", "windows-2022", "macos-14"]
         # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategymatrixinclude
         include:
           # Use this condarc as default
           - condarc: .conda/condarc.yaml
           # Use special condarc if macos
-          - os: "macos-latest"
+          - os: "macos-14"
             condarc: .conda_mac/condarc.yaml
     steps:
       # Setup
@@ -84,7 +85,7 @@ jobs:
 
       # Build conda package (Mac)
       - name: Build conda package (Mac)
-        if: matrix.os == 'macos-latest'
+        if: matrix.os == 'macos-14'
         shell: bash -l {0}
         run: |
           conda build .conda_mac --output-folder build
@@ -109,7 +110,7 @@ jobs:
       
       # Login to conda (Mac)
       - name: Login to Anaconda (Mac)
-        if: matrix.os == 'macos-latest'
+        if: matrix.os == 'macos-14'
         env:
           ANACONDA_LOGIN: ${{ secrets.ANACONDA_LOGIN }}
         shell: bash -l {0}
@@ -132,10 +133,10 @@ jobs:
       
       # Upload conda package (Mac)
       - name: Upload conda package (Mac/dev)
-        if: matrix.os == 'macos-latest'
+        if: matrix.os == 'macos-14'
         shell: bash -l {0}
         run: |
-          anaconda -v upload build/osx-64/*.tar.bz2 --label dev
+          anaconda -v upload build/osx-arm64/*.tar.bz2 --label dev
       
       - name: Logout from Anaconda
         shell: bash -l {0}

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -65,14 +65,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-22.04", "windows-2022", "macos-latest"]
+        # os: ["ubuntu-22.04", "windows-2022", "macos-14"]  # TODO: Fix Mac tests
+        os: ["ubuntu-22.04", "windows-2022"]
         include:
           # Default values
           - env_file: environment_no_cuda.yml
           - test_args: --durations=-1 tests/
           # Mac specific values
-          - os: macos-latest
-            env_file: environment_mac.yml
+          # - os: macos-14
+          #   env_file: environment_mac.yml
           # Ubuntu specific values
           - os: ubuntu-22.04
             test_args: --cov=sleap --cov-report=xml --durations=-1 tests/

diff --git a/environment.yml b/environment.yml
@@ -16,7 +16,8 @@ dependencies:
     - conda-forge::jsonpickle ==1.2
     - conda-forge::networkx
     - anaconda::numpy >=1.19.5,<1.23.0
-    - conda-forge::opencv
+    - conda-forge::opencv <4.9.0
+    - conda-forge::h5py <=3.7.0
     - conda-forge::pandas
     - conda-forge::pip
     - conda-forge::pillow #>=8.3.1,<=8.4.0
@@ -34,7 +35,7 @@ dependencies:
     - conda-forge::scikit-learn ==1.0
     - conda-forge::scikit-video
     - conda-forge::seaborn
-    - sleap::tensorflow >=2.6.3,<2.11  # No windows GPU support for >2.10
+    - sleap/label/dev::tensorflow ==2.7.0  # TODO: Switch to main label when updated
     - conda-forge::tensorflow-hub  # Pinned in meta.yml, but no problems here... yet
     - conda-forge::qudida
     - conda-forge::albumentations

diff --git a/environment_mac.yml b/environment_mac.yml
@@ -9,12 +9,13 @@ channels:
 dependencies:
     # Packages SLEAP uses directly
     - conda-forge::attrs >=21.2.0
+    - conda-forge::importlib-metadata <7.1.0
     - conda-forge::cattrs ==1.1.1
     - conda-forge::h5py
     - conda-forge::jsmin
     - conda-forge::jsonpickle ==1.2
     - conda-forge::keras <2.10.0,>=2.9.0rc0  # Required by tensorflow-macos
-    - conda-forge::networkx
+    - conda-forge::networkx <3.3
     - anaconda::numpy >=1.19.5,<1.23.0
     - conda-forge::opencv
     - conda-forge::pandas
@@ -34,7 +35,7 @@ dependencies:
     - conda-forge::scikit-learn ==1.0
     - conda-forge::scikit-video
     - conda-forge::seaborn
-    - conda-forge::tensorflow-hub
+    # - conda-forge::tensorflow-hub  # pulls in tensorflow cpu from conda-forge
     - conda-forge::qudida
     - conda-forge::albumentations
     - conda-forge::ndx-pose

diff --git a/environment_no_cuda.yml b/environment_no_cuda.yml
@@ -17,7 +17,7 @@ dependencies:
     - conda-forge::jsonpickle ==1.2
     - conda-forge::networkx
     - anaconda::numpy >=1.19.5,<1.23.0
-    - conda-forge::opencv
+    - conda-forge::opencv <4.9.0
     - conda-forge::pandas
     - conda-forge::pip
     - conda-forge::pillow #>=8.3.1,<=8.4.0
@@ -35,7 +35,8 @@ dependencies:
     - conda-forge::scikit-learn ==1.0
     - conda-forge::scikit-video
     - conda-forge::seaborn
-    - sleap::tensorflow >=2.6.3,<2.11  # No windows GPU support for >2.10
+    # - sleap::tensorflow >=2.6.3,<2.11  # No windows GPU support for >2.10
+    - sleap/label/dev::tensorflow ==2.7.0
     - conda-forge::tensorflow-hub
     - conda-forge::qudida
     - conda-forge::albumentations

diff --git a/pypi_requirements.txt b/pypi_requirements.txt
@@ -32,6 +32,7 @@ scikit-learn ==1.0.*
 scikit-video
 seaborn
 tensorflow>=2.6.3,<2.9; platform_machine != 'arm64'
+# tensorflow ==2.7.4; platform_machine != 'arm64'
 tensorflow-hub<=0.14.0
 albumentations
 ndx-pose

diff --git a/requirements.txt b/requirements.txt
@@ -7,7 +7,6 @@ qimage2ndarray  # ==1.9.0
 segmentation-models
 tensorflow-macos==2.9.2; sys_platform == 'darwin' and platform_machine == 'arm64'
 tensorflow-metal==0.5.0; sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow-hub==0.12.0; sys_platform == 'darwin' and platform_machine == 'arm64'
 
-# Conda installing results in https://github.com/h5py/h5py/issues/2037
-h5py<3.2; sys_platform == 'win32'  # Newer versions result in error above, linking issue in Linux
 pynwb>=2.3.3  # 2.0.0 required by ndx-pose, 2.3.3 fixes importlib-metadata incompatibility
diff --git a/sleap/version.py b/sleap/version.py
@@ -12,7 +12,7 @@
 """
 
 
-__version__ = "1.3.3"
+__version__ = "1.4.0"
 
 
 def versions():