docs: add pytorch integration guide. (#2711)

Co-authored-by: Tim de Jager <tdejager89@gmail.com> Co-authored-by: Hofer-Julian <30049909+Hofer-Julian@users.noreply.github.com>
prefix-dev · Dec 19, 2024 · c909168 · c909168
1 parent 8575de0
commit c909168
Show file tree

Hide file tree

Showing 24 changed files with 2,786 additions and 3,169 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -351,9 +351,9 @@ jobs:
   # Run integration tests on important platforms
   #
 
-  test-integration-windows-x86_64:
-    timeout-minutes: 30
-    name: pytest integration | windows x86_64
+  test-pytest-windows-x86_64:
+    timeout-minutes: 10
+    name: Pytest | windows x86_64
     runs-on: windows-latest
     needs: build-binary-windows-x86_64
     env:
@@ -372,48 +372,223 @@ jobs:
           name: pixi-windows-x86_64-${{ github.sha }}
           path: ${{ env.PIXI_WORKSPACE }}/${{ env.TARGET_RELEASE }}
       - name: Verify pixi installation
-        run: pixi --version
+        working-directory: ${{ env.PIXI_WORKSPACE }}
+        run: pixi info
 
-      - name: Install pixi
+      - name: Run pytests
         working-directory: ${{ env.PIXI_WORKSPACE }}
-        run: pixi install -v
+        run: pixi run --locked test-integration-ci
+
+  test-pytest-macos-aarch64:
+    timeout-minutes: 10
+    name: Pytest | macos aarch64
+    runs-on: macos-14
+    needs: build-binary-macos-aarch64
+    env:
+      TARGET_RELEASE: "${{ github.workspace }}/target/pixi/release"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download binary from build
+        uses: actions/download-artifact@v4
+        with:
+          name: pixi-macos-aarch64-${{ github.sha }}
+          path: ${{ env.TARGET_RELEASE }}
+      - name: Setup unix binary, add to github path
+        run: |
+          chmod a+x ${{ env.TARGET_RELEASE }}/pixi
+          echo "${{ env.TARGET_RELEASE }}" >> $GITHUB_PATH
+      - name: Verify pixi installation
+        run: pixi info
 
       - name: Run integration tests
-        working-directory: ${{ env.PIXI_WORKSPACE }}
         run: pixi run --locked test-integration-ci
 
+  test-pytest-linux-x86_64:
+    timeout-minutes: 10
+    name: Pytest | linux x86_64
+    runs-on: 8core_ubuntu_latest_runner
+    needs: build-binary-linux-x86_64
+    env:
+      TARGET_RELEASE: "${{ github.workspace }}/target/pixi/release"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download binary from build
+        uses: actions/download-artifact@v4
+        with:
+          name: pixi-linux-x86_64-${{ github.sha }}
+          path: ${{ env.TARGET_RELEASE }}
+      - name: Setup unix binary, add to github path
+        run: |
+          chmod a+x ${{ env.TARGET_RELEASE }}/pixi
+          echo "${{ env.TARGET_RELEASE }}" >> $GITHUB_PATH
+      - name: Verify pixi installation
+        run: pixi info
+
+      - name: Run integration tests
+        run: pixi run --locked test-integration-ci
+
+  test-integration-windows-x86_64:
+    timeout-minutes: 30
+    name: Integration tests | windows x86_64
+    runs-on: windows-latest
+    needs: build-binary-windows-x86_64
+    if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }}
+    env:
+      TARGET_RELEASE: "target/pixi/release"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create Dev Drive using ReFS
+        run: ${{ github.workspace }}/.github/workflows/setup-dev-drive.ps1
+      - name: Copy Git Repo to Dev Drive
+        run: |
+          Copy-Item -Path "${{ github.workspace }}" -Destination "${{ env.PIXI_WORKSPACE }}" -Recurse
+          echo "${{ env.PIXI_WORKSPACE }}/${{ env.TARGET_RELEASE }}" | Out-File -Append -Encoding utf8 -FilePath $env:GITHUB_PATH
+      - name: Download binary from build
+        uses: actions/download-artifact@v4
+        with:
+          name: pixi-windows-x86_64-${{ github.sha }}
+          path: ${{ env.PIXI_WORKSPACE }}/${{ env.TARGET_RELEASE }}
+
+      - name: Verify pixi installation
+        working-directory: ${{ env.PIXI_WORKSPACE }}
+        run: pixi info
+
+      - name: Run long running integration tests
+        working-directory: ${{ env.PIXI_WORKSPACE }}
+        run: pixi run --locked test-integration-extra-slow-ci
+
       - name: Test examples
         shell: bash
         working-directory: ${{ env.PIXI_WORKSPACE }}
         run: bash tests/scripts/test-examples.sh
 
-      - name: "Checkout Deltares/Ribasim"
+
+  test-integration-macos-aarch64:
+    timeout-minutes: 30
+    name: Integration tests | macos aarch64
+    runs-on: macos-14
+    needs: build-binary-macos-aarch64
+    if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }}
+    env:
+      TARGET_RELEASE: "${{ github.workspace }}/target/pixi/release"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download binary from build
+        uses: actions/download-artifact@v4
+        with:
+          name: pixi-macos-aarch64-${{ github.sha }}
+          path: ${{ env.TARGET_RELEASE }}
+      - name: Setup unix binary, add to github path
+        run: |
+          chmod a+x ${{ env.TARGET_RELEASE }}/pixi
+          echo "${{ env.TARGET_RELEASE }}" >> $GITHUB_PATH
+      - name: Verify pixi installation
+        run: pixi info
+
+      - name: Run long running integration tests
+        run: pixi run --locked test-integration-extra-slow-ci
+
+      - name: Run integration tests
+        run: pixi run --locked test-integration-ci
+
+      - name: Test examples
+        run: bash tests/scripts/test-examples.sh
+
+      - name: Test export
+        run: pixi run --locked test-export
+
+  test-integration-linux-x86_64:
+    timeout-minutes: 30
+    name: Integration tests | linux x86_64
+    runs-on: 8core_ubuntu_latest_runner
+    needs: build-binary-linux-x86_64
+    if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }}
+    env:
+      TARGET_RELEASE: "${{ github.workspace }}/target/pixi/release"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download binary from build
+        uses: actions/download-artifact@v4
+        with:
+          name: pixi-linux-x86_64-${{ github.sha }}
+          path: ${{ env.TARGET_RELEASE }}
+      - name: Setup unix binary, add to github path
+        run: |
+          chmod a+x ${{ env.TARGET_RELEASE }}/pixi
+          echo "${{ env.TARGET_RELEASE }}" >> $GITHUB_PATH
+      - name: Verify pixi installation
+        run: pixi info
+
+      - name: Run long running integration tests
+        if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }}
+        run: pixi run --locked test-integration-ci -m "extra_slow"
+
+      - name: Run integration tests
+        run: pixi run --locked test-integration-ci
+
+      - name: "Test examples"
+        run: bash tests/scripts/test-examples.sh
+
+      - name: "Test export"
+        run: pixi run --locked test-export
+
+  test-downstream-windows-x86_64:
+    timeout-minutes: 30
+    name: Downstream tests | windows x86_64
+    runs-on: windows-latest
+    needs: build-binary-windows-x86_64
+    if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }}
+    env:
+      TARGET_RELEASE: "target/pixi/release"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Create Dev Drive using ReFS
+        run: ${{ github.workspace }}/.github/workflows/setup-dev-drive.ps1
+      - name: Copy Git Repo to Dev Drive
+        run: |
+          Copy-Item -Path "${{ github.workspace }}" -Destination "${{ env.PIXI_WORKSPACE }}" -Recurse
+          echo "${{ env.PIXI_WORKSPACE }}/${{ env.TARGET_RELEASE }}" | Out-File -Append -Encoding utf8 -FilePath $env:GITHUB_PATH
+      - name: Download binary from build
+        uses: actions/download-artifact@v4
+        with:
+          name: pixi-windows-x86_64-${{ github.sha }}
+          path: ${{ env.PIXI_WORKSPACE }}/${{ env.TARGET_RELEASE }}
+      - name: Verify pixi installation
+        run: pixi info
+
+      - name: Install pixi's pixi project
+        working-directory: ${{ env.PIXI_WORKSPACE }}
+        run: pixi install -vv
+
+      - name: Checkout Deltares/Ribasim
         uses: actions/checkout@v4
         with:
           repository: Deltares/Ribasim
           path: ribasim
-      - name: "Copy Deltares/Ribasim to Dev Drive"
+      - name: Copy Deltares/Ribasim to Dev Drive
         run: Copy-Item -Path "${{ github.workspace }}/ribasim" -Destination "${{ env.PIXI_WORKSPACE }}/ribasim" -Recurse
-      - name: "Install Deltares/Ribasim"
+      - name: Install Deltares/Ribasim
         run: pixi install
         working-directory: ${{ env.PIXI_WORKSPACE }}/ribasim
 
-      - name: "Checkout quantco/polarify"
+      - name: Checkout quantco/polarify
         uses: actions/checkout@v4
         with:
           repository: quantco/polarify
           path: polarify
-      - name: "Copy quantco/polarify to Dev Drive"
+      - name: Copy quantco/polarify to Dev Drive
         run: Copy-Item -Path "${{ github.workspace }}/polarify" -Destination "${{ env.PIXI_WORKSPACE }}/polarify" -Recurse
-      - name: "Install quantco/polarify"
+      - name: Install quantco/polarify
         run: pixi install
         working-directory: ${{ env.PIXI_WORKSPACE }}/polarify
 
-  test-integration-macos-aarch64:
-    timeout-minutes: 30
-    name: pytest integration | macos aarch64
+
+  test-downstream-macos-aarch64:
+    timeout-minutes: 15
+    name: Downstream tests | macos aarch64
     runs-on: macos-14
     needs: build-binary-macos-aarch64
+    if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }}
     env:
       TARGET_RELEASE: "${{ github.workspace }}/target/pixi/release"
     steps:
@@ -428,20 +603,11 @@ jobs:
           chmod a+x ${{ env.TARGET_RELEASE }}/pixi
           echo "${{ env.TARGET_RELEASE }}" >> $GITHUB_PATH
       - name: Verify pixi installation
-        run: pixi --version
+        run: pixi info
 
       - name: Install pixi
         run: pixi install -v
 
-      - name: Run integration tests
-        run: pixi run --locked test-integration-ci
-
-      - name: Test examples
-        run: bash tests/scripts/test-examples.sh
-
-      - name: Test export
-        run: pixi run --locked test-export
-
       - name: "Checkout Deltares/Ribasim"
         uses: actions/checkout@v4
         with:
@@ -460,11 +626,12 @@ jobs:
         run: pixi install
         working-directory: polarify
 
-  test-integration-linux-x86_64:
-    timeout-minutes: 30
-    name: pytest integration | linux x86_64
+  test-downstream-linux-x86_64:
+    timeout-minutes: 15
+    name: Downstream tests | linux x86_64
     runs-on: 8core_ubuntu_latest_runner
     needs: build-binary-linux-x86_64
+    if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }}
     env:
       TARGET_RELEASE: "${{ github.workspace }}/target/pixi/release"
     steps:
@@ -479,20 +646,12 @@ jobs:
           chmod a+x ${{ env.TARGET_RELEASE }}/pixi
           echo "${{ env.TARGET_RELEASE }}" >> $GITHUB_PATH
       - name: Verify pixi installation
-        run: pixi --version
+        run: pixi info
 
       - name: Install pixi
+        working-directory: ${{ env.PIXI_WORKSPACE }}
         run: pixi install -v
 
-      - name: Run integration tests
-        run: pixi run --locked test-integration-ci
-
-      - name: "Test examples"
-        run: bash tests/scripts/test-examples.sh
-
-      - name: "Test export"
-        run: pixi run --locked test-export
-
       - name: "Checkout nerfstudio-project/nerfstudio"
         uses: actions/checkout@v4
         with:
@@ -519,13 +678,12 @@ jobs:
       - name: "Install quantco/polarify"
         run: pixi install
         working-directory: polarify
-
   #
   # Install a number of common wheels on some platforms
   #
 
   test-common-wheels-linux-x86_64:
-    name: "test wheel installation | linux x86_64"
+    name: Wheel Tests | linux x86_64
     needs:
       - build-binary-linux-x86_64
     uses: ./.github/workflows/test_common_wheels.yml
@@ -535,7 +693,7 @@ jobs:
         runs-on: 8core_ubuntu_latest_runner
 
   test-common-wheels-windows-x86_64:
-    name: "test wheel installation | windows x86_64"
+    name: Wheel Tests | windows x86_64
     needs:
       - build-binary-windows-x86_64
     uses: ./.github/workflows/test_common_wheels.yml
@@ -545,7 +703,7 @@ jobs:
       runs-on: windows-latest
 
   test-common-wheels-macos-aarch64:
-    name: "test wheel installation | macos aarch64"
+    name: Wheel Tests | macos aarch64
     needs:
       - build-binary-macos-aarch64
     uses: ./.github/workflows/test_common_wheels.yml