From 791aefc35d6c039731cdf3b5c2d135dcf099377b Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Sat, 14 Dec 2024 20:21:16 +0000
Subject: [PATCH 01/12] remove --privileged flag

---
 .github/workflows/gh-build-and-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml
index dac1ff48..e92d1371 100644
--- a/.github/workflows/gh-build-and-test.yml
+++ b/.github/workflows/gh-build-and-test.yml
@@ -87,7 +87,7 @@ jobs:
     # Our self-hosted runners require a container
     # TODO: use a different (nvidia?) container
     container:
-      options: -u root --security-opt seccomp=unconfined --privileged --shm-size 16g
+      options: -u root --security-opt seccomp=unconfined --shm-size 16g
       image: ubuntu:22.04
       env:
         NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }}

From b0492d7a6f44c03dbfc0178ee3c83aacc3ae577c Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Sun, 15 Dec 2024 19:28:33 +0000
Subject: [PATCH 02/12] make mini-ctk creator a standalone action

---
 .github/actions/build/action.yml        |  16 +++-
 .github/actions/fetch_ctk/action.yml    | 117 ++++++++++++++++++++++++
 .github/actions/setup/action.yml        |  95 -------------------
 .github/actions/test/action.yml         |  42 +++------
 .github/workflows/gh-build-and-test.yml |   3 +
 5 files changed, 148 insertions(+), 125 deletions(-)
 create mode 100644 .github/actions/fetch_ctk/action.yml

diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml
index 6b90dbd5..f74a7ef7 100644
--- a/.github/actions/build/action.yml
+++ b/.github/actions/build/action.yml
@@ -13,6 +13,9 @@ inputs:
   host-platform:
     required: true
     type: string
+  cuda-version:
+    required: true
+    type: string
   upload-enabled:
     required: true
     type: boolean
@@ -58,17 +61,26 @@ runs:
         if-no-files-found: error
         overwrite: 'true'
 
+    - name: Set up mini CTK
+      uses: ./.github/actions/fetch_ctk
+      continue-on-error: false
+      with:
+        host-platform: ${{ inputs.host-platform }}
+        cuda-version: ${{ inputs.cuda-version }}
+        fail-on-ctk-cache-miss: false
+
     - name: Build cuda.bindings wheel
       uses: pypa/cibuildwheel@v2.22.0
       env:
         CIBW_BUILD: ${{ env.CIBW_BUILD }}
         CIBW_ARCHS_LINUX: "native"
         CIBW_BUILD_VERBOSITY: 1
+        # CIBW mounts the host filesystem under /host
         CIBW_ENVIRONMENT_LINUX: >
-          CUDA_PATH="$(realpath ./cuda_toolkit)"
+          CUDA_PATH=/host/${{ env.CUDA_PATH }}
           PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
         CIBW_ENVIRONMENT_WINDOWS: >
-          CUDA_HOME="$(cygpath -w $(realpath ./cuda_toolkit))"
+          CUDA_HOME="$(cygpath -w ${{ env.CUDA_PATH }})"
         #  PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
         # # ensure Python.h & co can be found
         # CIBW_BEFORE_BUILD_WINDOWS: >
diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml
new file mode 100644
index 00000000..ab7c50d0
--- /dev/null
+++ b/.github/actions/fetch_ctk/action.yml
@@ -0,0 +1,117 @@
+name: Fetch mini CTK
+
+description: Fetch (or create) a mini CUDA Toolkit from cache
+
+inputs:
+  host-platform:
+    required: true
+    type: string
+  cuda-version:
+    required: true
+    type: string
+  fail-on-ctk-cache-miss:
+    required: true
+    type: boolean
+
+runs:
+  using: composite
+  steps:
+    - name: Set up CTK cache variable
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        echo "CTK_CACHE_KEY=mini-ctk-${{ inputs.cuda-version }}-${{ inputs.host-platform }}" >> $GITHUB_ENV
+        echo "CTK_CACHE_FILENAME=mini-ctk-${{ inputs.cuda-version }}-${{ inputs.host-platform }}.tar.gz" >> $GITHUB_ENV
+
+    - name: Download CTK cache
+      id: ctk-get-cache
+      uses: actions/cache/restore@v4
+      continue-on-error: true
+      with:
+        key: ${{ env.CTK_CACHE_KEY }}
+        path: ./${{ env.CTK_CACHE_FILENAME }}
+        fail-on-cache-miss: ${{ inputs.fail-on-ctk-cache-miss }}
+
+    - name: Get CUDA components
+      if: ${{ steps.ctk-get-cache.outputs.cache-hit != 'true' }}
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        CUDA_PATH="$(pwd)/cuda_toolkit"
+        mkdir $CUDA_PATH
+
+        # The binary archives (redist) are guaranteed to be updated as part of the release posting.
+        CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/"
+        CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json"
+        if [[ "${{ inputs.host-platform }}" == linux* ]]; then
+          if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then
+            CTK_SUBDIR="linux-x86_64"
+          elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
+            CTK_SUBDIR="linux-sbsa"
+          fi
+          function extract() {
+            tar -xvf $1 -C $CUDA_PATH --strip-components=1
+          }
+        elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then
+          CTK_SUBDIR="windows-x86_64"
+          function extract() {
+            _TEMP_DIR_=$(mktemp -d)
+            unzip $1 -d $_TEMP_DIR_
+            cp -r $_TEMP_DIR_/*/* $CUDA_PATH
+            rm -rf $_TEMP_DIR_
+          }
+        fi
+        function populate_cuda_path() {
+          # take the component name as a argument
+          function download() {
+            curl -kLSs $1 -o $2
+          }
+          CTK_COMPONENT=$1
+          CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL |
+              python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")"
+          CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}"
+          CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)"
+          download $CTK_COMPONENT_URL $CTK_COMPONENT_COMPONENT_FILENAME
+          extract $CTK_COMPONENT_COMPONENT_FILENAME
+          rm $CTK_COMPONENT_COMPONENT_FILENAME
+        }
+
+        # Get headers and shared libraries in place
+        # Note: the existing artifact would need to be manually deleted (ex: through web UI)
+        # if this list is changed, as the artifact actions do not offer any option for us to
+        # invalidate the artifact.
+        populate_cuda_path cuda_nvcc
+        populate_cuda_path cuda_cudart
+        populate_cuda_path cuda_nvrtc
+        populate_cuda_path cuda_profiler_api
+        populate_cuda_path libnvjitlink
+        ls -l $CUDA_PATH
+
+        # Prepare the cache
+        # Note: try to escape | and > ...
+        tar -czvf ${CTK_CACHE_FILENAME} ${CUDA_PATH}
+
+        # Note: the headers will be copied into the cibuildwheel manylinux container,
+        # so setting the CUDA_PATH env var here is meaningless.
+
+    - name: Upload CTK cache
+      if: ${{ always() &&
+              steps.ctk-get-cache.outputs.cache-hit != 'true' }}
+      uses: actions/cache/save@v4
+      with:
+        key: ${{ env.CTK_CACHE_KEY }}
+        path: ./${{ env.CTK_CACHE_FILENAME }}
+
+    - name: Restore CTK cache
+      if: ${{ steps.ctk-get-cache.outputs.cache-hit == 'true' }}
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        ls -l
+        CUDA_PATH="$(pwd)/cuda_toolkit"
+        tar -xzvf $CTK_CACHE_FILENAME
+        ls -l $CUDA_PATH
+        if [ ! -d "$CUDA_PATH/include" ]; then
+          exit 1
+        fi
+
+        echo "CUDA_PATH=${CUDA_PATH}" >> $GITHUB_ENV
+        echo "${CUDA_PATH}/bin" >> $GITHUB_PATH
+        echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CUDA_PATH}/lib" >> $GITHUB_ENV
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
index 084f4c2f..9eddec5f 100644
--- a/.github/actions/setup/action.yml
+++ b/.github/actions/setup/action.yml
@@ -47,101 +47,6 @@ runs:
       run: |
         env
 
-    - name: Set up CTK cache variable
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        echo "CTK_CACHE_KEY=mini-ctk-${{ inputs.cuda-version }}-${{ inputs.host-platform }}" >> $GITHUB_ENV
-        echo "CTK_CACHE_FILENAME=mini-ctk-${{ inputs.cuda-version }}-${{ inputs.host-platform }}.tar.gz" >> $GITHUB_ENV
-
-    - name: Download CTK cache
-      id: ctk-get-cache
-      uses: actions/cache/restore@v4
-      continue-on-error: true
-      with:
-        key: ${{ env.CTK_CACHE_KEY }}
-        path: ./${{ env.CTK_CACHE_FILENAME }}
-
-    - name: Get CUDA components
-      if: ${{ steps.ctk-get-cache.outputs.cache-hit != 'true' }}
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        CUDA_PATH="./cuda_toolkit"
-        mkdir $CUDA_PATH
-
-        # The binary archives (redist) are guaranteed to be updated as part of the release posting.
-        CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/"
-        CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json"
-        if [[ "${{ inputs.host-platform }}" == linux* ]]; then
-          if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then
-            CTK_SUBDIR="linux-x86_64"
-          elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
-            CTK_SUBDIR="linux-sbsa"
-          fi
-          function extract() {
-            tar -xvf $1 -C $CUDA_PATH --strip-components=1
-          }
-        elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then
-          CTK_SUBDIR="windows-x86_64"
-          function extract() {
-            _TEMP_DIR_=$(mktemp -d)
-            unzip $1 -d $_TEMP_DIR_
-            cp -r $_TEMP_DIR_/*/* $CUDA_PATH
-            rm -rf $_TEMP_DIR_
-          }
-        fi
-        function populate_cuda_path() {
-          # take the component name as a argument
-          function download() {
-            curl -kLSs $1 -o $2
-          }
-          CTK_COMPONENT=$1
-          CTK_COMPONENT_REL_PATH="$(curl -s $CTK_JSON_URL |
-              python -c "import sys, json; print(json.load(sys.stdin)['${CTK_COMPONENT}']['${CTK_SUBDIR}']['relative_path'])")"
-          CTK_COMPONENT_URL="${CTK_BASE_URL}/${CTK_COMPONENT_REL_PATH}"
-          CTK_COMPONENT_COMPONENT_FILENAME="$(basename $CTK_COMPONENT_REL_PATH)"
-          download $CTK_COMPONENT_URL $CTK_COMPONENT_COMPONENT_FILENAME
-          extract $CTK_COMPONENT_COMPONENT_FILENAME
-          rm $CTK_COMPONENT_COMPONENT_FILENAME
-        }
-
-        # Get headers and shared libraries in place
-        # Note: the existing artifact would need to be manually deleted (ex: through web UI)
-        # if this list is changed, as the artifact actions do not offer any option for us to
-        # invalidate the artifact.
-        populate_cuda_path cuda_nvcc
-        populate_cuda_path cuda_cudart
-        populate_cuda_path cuda_nvrtc
-        populate_cuda_path cuda_profiler_api
-        populate_cuda_path libnvjitlink
-        ls -l $CUDA_PATH
-
-        # Prepare the cache
-        # Note: try to escape | and > ...
-        tar -czvf ${CTK_CACHE_FILENAME} ${CUDA_PATH}
-
-        # Note: the headers will be copied into the cibuildwheel manylinux container,
-        # so setting the CUDA_PATH env var here is meaningless.
-
-    - name: Upload CTK cache
-      if: ${{ always() &&
-              steps.ctk-get-cache.outputs.cache-hit != 'true' }}
-      uses: actions/cache/save@v4
-      with:
-        key: ${{ env.CTK_CACHE_KEY }}
-        path: ./${{ env.CTK_CACHE_FILENAME }}
-
-    - name: Restore CTK cache
-      if: ${{ steps.ctk-get-cache.outputs.cache-hit == 'true' }}
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        ls -l
-        CUDA_PATH="./cuda_toolkit"
-        tar -xzvf $CTK_CACHE_FILENAME
-        ls -l $CUDA_PATH
-        if [ ! -d "$CUDA_PATH/include" ]; then
-          exit 1
-        fi
-
     - name: Set environment variables
       shell: bash --noprofile --norc -xeuo pipefail {0}
       run: |
diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
index 079dd039..d8748f7e 100644
--- a/.github/actions/test/action.yml
+++ b/.github/actions/test/action.yml
@@ -3,6 +3,12 @@ name: test
 description: Run tests in specified project
 
 inputs:
+  host-platform:
+    required: true
+    type: string
+  cuda-version:
+    required: true
+    type: string
   test-options:
     required: true
     type: string
@@ -50,39 +56,19 @@ runs:
       with:
         python-version: ${{ env.PYTHON_VERSION }}
 
-    - name: Set up CTK cache variable
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        echo "CTK_CACHE_KEY=mini-ctk-${CTK_BUILD_VER}-${HOST_PLATFORM}" >> $GITHUB_ENV
-        echo "CTK_CACHE_FILENAME=mini-ctk-${CTK_BUILD_VER}-${HOST_PLATFORM}.tar.gz" >> $GITHUB_ENV
-
-    - name: Download CTK cache
-      id: ctk-get-cache
-      uses: actions/cache/restore@v4
-      continue-on-error: true
+    - name: Set up mini CTK
+      uses: ./.github/actions/fetch_ctk
+      continue-on-error: false
       with:
-        key: ${{ env.CTK_CACHE_KEY }}
-        path: ./${{ env.CTK_CACHE_FILENAME }}
-        fail-on-cache-miss: true
-
-    - name: Restore CTK cache
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        ls -l
-        CUDA_PATH="$(pwd)/cuda_toolkit"
-        tar -xzvf $CTK_CACHE_FILENAME
-        ls -l $CUDA_PATH
-        if [ ! -d "$CUDA_PATH/include" ]; then
-          exit 1
-        fi
-
-        echo "CUDA_PATH=$CUDA_PATH" >> $GITHUB_ENV
-        echo "PATH=$PATH:$CUDA_PATH/bin" >> $GITHUB_ENV
-        echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CUDA_PATH/lib" >> $GITHUB_ENV
+        host-platform: ${{ inputs.host-platform }}
+        cuda-version: ${{ inputs.cuda-version }}
+        fail-on-ctk-cache-miss: true
 
     - name: Run test / analysis
       shell: bash --noprofile --norc -xeuo pipefail {0}
       run: |
+        ls $CUDA_PATH
+
         REPO_DIR=$(pwd)
 
         cd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml
index e92d1371..db19dbc5 100644
--- a/.github/workflows/gh-build-and-test.yml
+++ b/.github/workflows/gh-build-and-test.yml
@@ -63,6 +63,7 @@ jobs:
           build-type: ${{ inputs.build-type }}
           target-device: "${{ inputs.target-device }}"
           host-platform: ${{ inputs.host-platform }}
+          cuda-version: ${{ inputs.cuda-version }}
           upload-enabled: ${{ inputs.upload-enabled }}
 
       - name: Pass environment variables
@@ -116,6 +117,8 @@ jobs:
         uses: ./.github/actions/test
         with:
           test-options: ${{ inputs.build-type }}
+          host-platform: ${{ inputs.host-platform }}
+          cuda-version: ${{ inputs.cuda-version }}
         env:
           CUDA_CORE_ARTIFACT_NAME: ${{ needs.build.outputs.CUDA_CORE_ARTIFACT_NAME }}
           CUDA_CORE_ARTIFACTS_DIR: ${{ needs.build.outputs.CUDA_CORE_ARTIFACTS_DIR }}

From 243a46ff70e39521113317107201df030ef46c26 Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Sun, 15 Dec 2024 20:45:29 +0000
Subject: [PATCH 03/12] Clean up + Merge composite actions (setup/build/test)
 into a single workflow

Making them reusable workflows are not possible because they would
not be callable as a single job step (which was what composite actions
are for). But the steps in these actions are so tiny and problem-specific
that making them standalone is hard to maintain anyway, so a single,
moderate-sized workflow is acceptable.
---
 .github/actions/build/action.yml        | 115 -----------
 .github/actions/setup/action.yml        |  90 ---------
 .github/actions/test/action.yml         |  86 ---------
 .github/workflows/ci-gh.yml             |  11 +-
 .github/workflows/gh-build-and-test.yml | 245 +++++++++++++++++++-----
 5 files changed, 194 insertions(+), 353 deletions(-)
 delete mode 100644 .github/actions/build/action.yml
 delete mode 100644 .github/actions/setup/action.yml
 delete mode 100644 .github/actions/test/action.yml

diff --git a/.github/actions/build/action.yml b/.github/actions/build/action.yml
deleted file mode 100644
index f74a7ef7..00000000
--- a/.github/actions/build/action.yml
+++ /dev/null
@@ -1,115 +0,0 @@
-name: build
-
-description: Build specified project
-
-inputs:
-  build-type:
-    required: true
-    type: string
-    description: One of ci / release
-  target-device:
-    required: true
-    type: string
-  host-platform:
-    required: true
-    type: string
-  cuda-version:
-    required: true
-    type: string
-  upload-enabled:
-    required: true
-    type: boolean
-
-runs:
-  using: composite
-  steps:
-    - name: Build cuda.core wheel
-      uses: pypa/cibuildwheel@v2.22.0
-      env:
-        CIBW_BUILD: ${{ env.CIBW_BUILD }}
-        CIBW_ARCHS_LINUX: "native"
-        CIBW_BUILD_VERBOSITY: 1
-        # # ensure Python.h & co can be found
-        # CIBW_BEFORE_BUILD_WINDOWS: >
-        #   python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE
-      with:
-        package-dir: ./cuda_core/
-        output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
-
-    - name: List the cuda.core artifacts directory
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        if [[ "${{ inputs.host-platform }}" == win* ]]; then
-          export CHOWN=chown
-        else
-          export CHOWN="sudo chown"
-        fi
-        $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
-        ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
-
-    - name: Check cuda.core wheel
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        pip install twine
-        twine check ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl
-
-    - name: Upload cuda.core build artifacts
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
-        path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl
-        if-no-files-found: error
-        overwrite: 'true'
-
-    - name: Set up mini CTK
-      uses: ./.github/actions/fetch_ctk
-      continue-on-error: false
-      with:
-        host-platform: ${{ inputs.host-platform }}
-        cuda-version: ${{ inputs.cuda-version }}
-        fail-on-ctk-cache-miss: false
-
-    - name: Build cuda.bindings wheel
-      uses: pypa/cibuildwheel@v2.22.0
-      env:
-        CIBW_BUILD: ${{ env.CIBW_BUILD }}
-        CIBW_ARCHS_LINUX: "native"
-        CIBW_BUILD_VERBOSITY: 1
-        # CIBW mounts the host filesystem under /host
-        CIBW_ENVIRONMENT_LINUX: >
-          CUDA_PATH=/host/${{ env.CUDA_PATH }}
-          PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
-        CIBW_ENVIRONMENT_WINDOWS: >
-          CUDA_HOME="$(cygpath -w ${{ env.CUDA_PATH }})"
-        #  PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
-        # # ensure Python.h & co can be found
-        # CIBW_BEFORE_BUILD_WINDOWS: >
-        #   python -c "import sysconfig; print(sysconfig.get_path('include'))" >> $env:INCLUDE
-      with:
-        package-dir: ./cuda_bindings/
-        output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
-
-    - name: List the cuda.bindings artifacts directory
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        if [[ "${{ inputs.host-platform }}" == win* ]]; then
-          export CHOWN=chown
-        else
-          export CHOWN="sudo chown"
-        fi
-        $CHOWN -R $(whoami) ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
-        ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
-
-    # TODO: enable this after NVIDIA/cuda-python#297 is resolved
-    # - name: Check cuda.bindings wheel
-    #   shell: bash --noprofile --norc -xeuo pipefail {0}
-    #   run: |
-    #     twine check ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl
-
-    - name: Upload cuda.bindings build artifacts
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
-        path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl
-        if-no-files-found: error
-        overwrite: 'true'
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
deleted file mode 100644
index 9eddec5f..00000000
--- a/.github/actions/setup/action.yml
+++ /dev/null
@@ -1,90 +0,0 @@
-name: Common setup
-
-inputs:
-  client-repo:
-    required: true
-    type: string
-  build-type:
-    required: true
-    type: string
-  target-device:
-    required: true
-    type: string
-  host-platform:
-    required: true
-    type: string
-  build-mode:
-    required: true
-    type: string
-  upload-enabled:
-    required: true
-    type: boolean
-  python-version:
-    required: true
-    type: string
-  cuda-version:
-    required: true
-    type: string
-
-runs:
-  using: composite
-  steps:
-    # WAR: setup-python is not relocatable...
-    # see https://github.com/actions/setup-python/issues/871
-    - name: Set up Python ${{ inputs.python-version }}
-      if: ${{ startsWith(inputs.host-platform, 'linux') }}
-      id: setup-python
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.12"
-
-    - name: Set up MSVC
-      if: ${{ startsWith(inputs.host-platform, 'win') }}
-      uses: ilammy/msvc-dev-cmd@v1
-
-    - name: Dump environment
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        env
-
-    - name: Set environment variables
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        # TODO: just align host-platform names with TARGET_PLATFORM...
-        if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then
-          TARGET_PLATFORM='linux-64'
-        elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
-          TARGET_PLATFORM='linux-aarch64'
-        elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then
-          TARGET_PLATFORM='win-64'
-        fi
-
-        PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.')
-        if [[ "${{ inputs.host-platform }}" == linux* ]]; then
-          CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*"
-          REPO_DIR=$(pwd)
-        elif [[ "${{ inputs.host-platform }}" == win* ]]; then
-          CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64"
-          PWD=$(pwd)
-          REPO_DIR=$(cygpath -w $PWD)
-        fi
-
-        BUILD_MODE="${{ inputs.build-mode }}"
-        if [[ ("${BUILD_MODE}" == "") || ("${BUILD_MODE}" == "release") ]]; then
-          # We upload release versions in the default folder.
-          PKG_DIR="${TARGET_PLATFORM}"
-        else
-          PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}"
-        fi
-
-        echo "PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV
-        echo "REPO_DIR=$REPO_DIR" >> $GITHUB_ENV
-        echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV
-        echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV
-        echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV
-        echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ github.sha }}" >> $GITHUB_ENV
-        echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
-        echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled  == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV
-        echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
-        echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV
-        echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV
diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
deleted file mode 100644
index d8748f7e..00000000
--- a/.github/actions/test/action.yml
+++ /dev/null
@@ -1,86 +0,0 @@
-name: test
-
-description: Run tests in specified project
-
-inputs:
-  host-platform:
-    required: true
-    type: string
-  cuda-version:
-    required: true
-    type: string
-  test-options:
-    required: true
-    type: string
-
-runs:
-  using: composite
-  steps:
-    - name: Run nvidia-smi to make sure GPU is working
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: nvidia-smi
-
-    # The cache action needs this
-    - name: Install zstd
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        apt update
-        apt install zstd
-
-    - name: Download bindings build artifacts
-      uses: actions/download-artifact@v4
-      with:
-        name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
-        path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
-
-    - name: Display structure of downloaded bindings artifacts
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        pwd
-        ls -lahR $CUDA_BINDINGS_ARTIFACTS_DIR
-
-    - name: Download core build artifacts
-      uses: actions/download-artifact@v4
-      with:
-        name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
-        path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
-
-    - name: Display structure of downloaded core build artifacts
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        pwd
-        ls -lahR $CUDA_CORE_ARTIFACTS_DIR
-
-    - name: Set up Python ${{ env.PYTHON_VERSION }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ env.PYTHON_VERSION }}
-
-    - name: Set up mini CTK
-      uses: ./.github/actions/fetch_ctk
-      continue-on-error: false
-      with:
-        host-platform: ${{ inputs.host-platform }}
-        cuda-version: ${{ inputs.cuda-version }}
-        fail-on-ctk-cache-miss: true
-
-    - name: Run test / analysis
-      shell: bash --noprofile --norc -xeuo pipefail {0}
-      run: |
-        ls $CUDA_PATH
-
-        REPO_DIR=$(pwd)
-
-        cd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
-        pip install *.whl
-
-        cd "${CUDA_CORE_ARTIFACTS_DIR}"
-        pip install *.whl
-
-        cd "${REPO_DIR}/cuda_bindings"
-        pip install -r requirements.txt
-        pytest tests/
-        #pytest tests/cython
-
-        cd "${REPO_DIR}/cuda_core"
-        pytest -rxXs tests/
diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml
index 71c3cee9..b4646f0e 100644
--- a/.github/workflows/ci-gh.yml
+++ b/.github/workflows/ci-gh.yml
@@ -15,16 +15,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        # TODO: align host-platform names with conda convention
         host-platform:
           - linux-x64
           - linux-aarch64
           - win-x64
-        target-device:
-          - gpu
-        build-mode:
-          - release
-        upload-enabled:
-          - false
         python-version:
           - "3.13"
           - "3.12"
@@ -40,10 +35,6 @@ jobs:
       ./.github/workflows/gh-build-and-test.yml
     with:
       host-platform: ${{ matrix.host-platform }}
-      target-device: ${{ matrix.target-device }}
-      build-mode: ${{ matrix.build-mode }}
-      build-type: ci
-      upload-enabled: ${{ matrix.upload-enabled }}
       python-version: ${{ matrix.python-version }}
       cuda-version: ${{ matrix.cuda-version }}
     secrets: inherit
diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml
index db19dbc5..5b031a24 100644
--- a/.github/workflows/gh-build-and-test.yml
+++ b/.github/workflows/gh-build-and-test.yml
@@ -1,21 +1,9 @@
 on:
   workflow_call:
     inputs:
-      target-device:
-        type: string
-        required: true
-      build-type:
-        type: string
-        required: true
       host-platform:
         type: string
         required: true
-      build-mode:
-        type: string
-        required: true
-      upload-enabled:
-        type: boolean
-        required: true
       python-version:
         type: string
         required: true
@@ -45,28 +33,130 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up build environment
-        uses: ./.github/actions/setup
+      # WAR: setup-python is not relocatable...
+      # see https://github.com/actions/setup-python/issues/871
+      - name: Set up Python ${{ inputs.python-version }}
+        if: ${{ startsWith(inputs.host-platform, 'linux') }}
+        id: setup-python
+        uses: actions/setup-python@v5
         with:
-          client-repo: ${{ github.event.repository.name }}
-          build-type: ${{ inputs.build-type }}
-          target-device: "${{ inputs.target-device }}"
-          host-platform: ${{ inputs.host-platform }}
-          build-mode: ${{ inputs.build-mode }}
-          upload-enabled: ${{ inputs.upload-enabled }}
-          python-version: ${{ inputs.python-version }}
-          cuda-version: ${{ inputs.cuda-version }}
+          python-version: "3.12"
+  
+      - name: Set up MSVC
+        if: ${{ startsWith(inputs.host-platform, 'win') }}
+        uses: ilammy/msvc-dev-cmd@v1
+  
+      - name: Set environment variables
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.')
+          if [[ "${{ inputs.host-platform }}" == linux* ]]; then
+            CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*"
+            REPO_DIR=$(pwd)
+          elif [[ "${{ inputs.host-platform }}" == win* ]]; then
+            CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64"
+            PWD=$(pwd)
+            REPO_DIR=$(cygpath -w $PWD)
+          fi
+  
+          echo "PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV
+          echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
+          echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV
+          echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
+          echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
+          echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV
+  
+      - name: Dump environment
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          env
 
-      - name: Call build action
-        uses: ./.github/actions/build
+      - name: Build cuda.core wheel
+        uses: pypa/cibuildwheel@v2.22.0
+        env:
+          CIBW_BUILD: ${{ env.CIBW_BUILD }}
+          CIBW_ARCHS_LINUX: "native"
+          CIBW_BUILD_VERBOSITY: 1
+        with:
+          package-dir: ./cuda_core/
+          output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
+
+      - name: List the cuda.core artifacts directory
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          if [[ "${{ inputs.host-platform }}" == win* ]]; then
+            export CHOWN=chown
+          else
+            export CHOWN="sudo chown"
+          fi
+          $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
+          ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
+
+      - name: Check cuda.core wheel
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          pip install twine
+          twine check ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl
+
+      - name: Upload cuda.core build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
+          path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl
+          if-no-files-found: error
+          overwrite: 'true'
+
+      - name: Set up mini CTK
+        uses: ./.github/actions/fetch_ctk
+        continue-on-error: false
         with:
-          build-type: ${{ inputs.build-type }}
-          target-device: "${{ inputs.target-device }}"
           host-platform: ${{ inputs.host-platform }}
           cuda-version: ${{ inputs.cuda-version }}
-          upload-enabled: ${{ inputs.upload-enabled }}
+          fail-on-ctk-cache-miss: false
+
+      - name: Build cuda.bindings wheel
+        uses: pypa/cibuildwheel@v2.22.0
+        env:
+          CIBW_BUILD: ${{ env.CIBW_BUILD }}
+          CIBW_ARCHS_LINUX: "native"
+          CIBW_BUILD_VERBOSITY: 1
+          # CIBW mounts the host filesystem under /host
+          CIBW_ENVIRONMENT_LINUX: >
+            CUDA_PATH=/host/${{ env.CUDA_PATH }}
+            PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
+          CIBW_ENVIRONMENT_WINDOWS: >
+            CUDA_HOME="$(cygpath -w ${{ env.CUDA_PATH }})"
+          #  PARALLEL_LEVEL=${{ env.PARALLEL_LEVEL }}
+        with:
+          package-dir: ./cuda_bindings/
+          output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
+
+      - name: List the cuda.bindings artifacts directory
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          if [[ "${{ inputs.host-platform }}" == win* ]]; then
+            export CHOWN=chown
+          else
+            export CHOWN="sudo chown"
+          fi
+          $CHOWN -R $(whoami) ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
+          ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
+
+      # TODO: enable this after NVIDIA/cuda-python#297 is resolved
+      # - name: Check cuda.bindings wheel
+      #   shell: bash --noprofile --norc -xeuo pipefail {0}
+      #   run: |
+      #     twine check ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl
 
-      - name: Pass environment variables
+      - name: Upload cuda.bindings build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
+          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl
+          if-no-files-found: error
+          overwrite: 'true'
+
+      - name: Pass environment variables to the next runner
         id: pass_env
         run: |
           echo "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_NAME}" >> $GITHUB_OUTPUT
@@ -95,35 +185,86 @@ jobs:
     needs:
       - build
     steps:
+      - name: Run nvidia-smi to make sure GPU is working
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: nvidia-smi
+
       - name: Checkout ${{ github.event.repository.name }}
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      # TODO: we probably don't need this?
-      # - name: Setup
-      #   if: ${{ inputs.has-built != 'true' }}
-      #   uses: ./.github/actions/setup
-      #   with:
-      #     client-repo: ${{ github.event.repository.name }}
-      #     build-type: ${{ inputs.build-type }}
-      #     target-device: "${{ inputs.target-device }}"
-      #     host-platform: ${{ inputs.host-platform }}
-      #     build-mode: ${{ inputs.build-mode }}
-      #     upload-enabled: ${{ inputs.upload-enabled }}
-      #     python-version: ${{ inputs.python-version }}
-
-      - name: Call test action
-        uses: ./.github/actions/test
+      - name: Set up test environment
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          # make outputs from the previous job as env vars
+          echo "CUDA_CORE_ARTIFACT_NAME=${{ needs.build.outputs.CUDA_CORE_ARTIFACT_NAME }}" >> $GITHUB_ENV
+          echo "CUDA_CORE_ARTIFACTS_DIR=${{ needs.build.outputs.CUDA_CORE_ARTIFACTS_DIR }}" >> $GITHUB_ENV
+          echo "CUDA_BINDINGS_ARTIFACT_NAME=${{ needs.build.outputs.CUDA_BINDINGS_ARTIFACT_NAME }}" >> $GITHUB_ENV
+          echo "CUDA_BINDINGS_ARTIFACTS_DIR=${{ needs.build.outputs.CUDA_BINDINGS_ARTIFACTS_DIR }}" >> $GITHUB_ENV
+
+      - name: Download bindings build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
+          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
+
+      - name: Display structure of downloaded bindings artifacts
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          pwd
+          ls -lahR $CUDA_BINDINGS_ARTIFACTS_DIR
+
+      - name: Download core build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
+          path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
+
+      - name: Display structure of downloaded core build artifacts
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          pwd
+          ls -lahR $CUDA_CORE_ARTIFACTS_DIR
+
+      - name: Set up Python ${{ inputs.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      # The cache action needs this
+      - name: Install zstd
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          apt update
+          apt install zstd
+
+      - name: Set up mini CTK
+        uses: ./.github/actions/fetch_ctk
+        continue-on-error: false
         with:
-          test-options: ${{ inputs.build-type }}
           host-platform: ${{ inputs.host-platform }}
           cuda-version: ${{ inputs.cuda-version }}
-        env:
-          CUDA_CORE_ARTIFACT_NAME: ${{ needs.build.outputs.CUDA_CORE_ARTIFACT_NAME }}
-          CUDA_CORE_ARTIFACTS_DIR: ${{ needs.build.outputs.CUDA_CORE_ARTIFACTS_DIR }}
-          CUDA_BINDINGS_ARTIFACT_NAME: ${{ needs.build.outputs.CUDA_BINDINGS_ARTIFACT_NAME }}
-          CUDA_BINDINGS_ARTIFACTS_DIR: ${{ needs.build.outputs.CUDA_BINDINGS_ARTIFACTS_DIR }}
-          PYTHON_VERSION: ${{ inputs.python-version }}
-          CTK_BUILD_VER: ${{ inputs.cuda-version }}
-          HOST_PLATFORM: ${{ inputs.host-platform }}
+          fail-on-ctk-cache-miss: true
+
+      - name: Run test / analysis
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          ls $CUDA_PATH
+
+          REPO_DIR=$(pwd)
+
+          cd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
+          pip install *.whl
+
+          cd "${CUDA_CORE_ARTIFACTS_DIR}"
+          pip install *.whl
+
+          cd "${REPO_DIR}/cuda_bindings"
+          pip install -r requirements.txt
+          pytest -rxXs tests/
+          # TODO: enable cython tests
+          #pytest tests/cython
+
+          cd "${REPO_DIR}/cuda_core"
+          pytest -rxXs tests/

From 478acffacaa1fc5f752025237b112da1c0b88e93 Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Sun, 15 Dec 2024 22:11:22 +0000
Subject: [PATCH 04/12] separate test matrix from build matrix and expand it
 (retry)

---
 .github/workflows/ci-gh.yml             |  22 -----
 .github/workflows/gh-build-and-test.yml | 103 +++++++++++++++---------
 2 files changed, 63 insertions(+), 62 deletions(-)

diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml
index b4646f0e..795d6d0a 100644
--- a/.github/workflows/ci-gh.yml
+++ b/.github/workflows/ci-gh.yml
@@ -12,29 +12,7 @@ on:
 
 jobs:
   ci:
-    strategy:
-      fail-fast: false
-      matrix:
-        # TODO: align host-platform names with conda convention
-        host-platform:
-          - linux-x64
-          - linux-aarch64
-          - win-x64
-        python-version:
-          - "3.13"
-          - "3.12"
-          - "3.11"
-          - "3.10"
-          - "3.9"
-        cuda-version:
-          # Note: this is for build-time only; the test-time matrix needs to be
-          # defined separately.
-          - "12.6.2"
     name: "CI"
     uses:
       ./.github/workflows/gh-build-and-test.yml
-    with:
-      host-platform: ${{ matrix.host-platform }}
-      python-version: ${{ matrix.python-version }}
-      cuda-version: ${{ matrix.cuda-version }}
     secrets: inherit
diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml
index 5b031a24..f4399118 100644
--- a/.github/workflows/gh-build-and-test.yml
+++ b/.github/workflows/gh-build-and-test.yml
@@ -1,27 +1,33 @@
-on:
-  workflow_call:
-    inputs:
-      host-platform:
-        type: string
-        required: true
-      python-version:
-        type: string
-        required: true
-      cuda-version:
-        type: string
-        required: true
+on: workflow_call
 
 jobs:
   build:
-    name: Build (${{ inputs.host-platform }}, Python "${{ inputs.python-version }}")
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO: align host-platform names with conda convention
+        host-platform:
+          - linux-x64
+          - linux-aarch64
+          - win-x64
+        python-version:
+          - "3.13"
+          - "3.12"
+          - "3.11"
+          - "3.10"
+          - "3.9"
+        cuda-version:
+          # Note: this is for build-time only.
+          - "12.6.2"
+    name: Build (${{ matrix.host-platform }}, Python "${{ matrix.python-version }}")
     if: ${{ github.repository_owner == 'nvidia' }}
     permissions:
       id-token: write # This is required for configure-aws-credentials
       contents: read  # This is required for actions/checkout
-    runs-on: ${{ (inputs.host-platform == 'linux-x64' && 'linux-amd64-cpu8') ||
-                 (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') ||
-                 (inputs.host-platform == 'win-x64' && 'windows-2019') }}
-               #  (inputs.host-platform == 'win-x64' && 'windows-amd64-cpu8') }}
+    runs-on: ${{ (matrix.host-platform == 'linux-x64' && 'linux-amd64-cpu8') ||
+                 (matrix.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') ||
+                 (matrix.host-platform == 'win-x64' && 'windows-2019') }}
+               #  (matrix.host-platform == 'win-x64' && 'windows-amd64-cpu8') }}
     outputs:
       CUDA_CORE_ARTIFACT_NAME: ${{ steps.pass_env.outputs.CUDA_CORE_ARTIFACT_NAME }}
       CUDA_CORE_ARTIFACTS_DIR: ${{ steps.pass_env.outputs.CUDA_CORE_ARTIFACTS_DIR }}
@@ -35,34 +41,34 @@ jobs:
 
       # WAR: setup-python is not relocatable...
       # see https://github.com/actions/setup-python/issues/871
-      - name: Set up Python ${{ inputs.python-version }}
-        if: ${{ startsWith(inputs.host-platform, 'linux') }}
+      - name: Set up Python ${{ matrix.python-version }}
+        if: ${{ startsWith(matrix.host-platform, 'linux') }}
         id: setup-python
         uses: actions/setup-python@v5
         with:
           python-version: "3.12"
   
       - name: Set up MSVC
-        if: ${{ startsWith(inputs.host-platform, 'win') }}
+        if: ${{ startsWith(matrix.host-platform, 'win') }}
         uses: ilammy/msvc-dev-cmd@v1
   
       - name: Set environment variables
         shell: bash --noprofile --norc -xeuo pipefail {0}
         run: |
-          PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.')
-          if [[ "${{ inputs.host-platform }}" == linux* ]]; then
+          PYTHON_VERSION_FORMATTED=$(echo '${{ matrix.python-version }}' | tr -d '.')
+          if [[ "${{ matrix.host-platform }}" == linux* ]]; then
             CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-manylinux*"
             REPO_DIR=$(pwd)
-          elif [[ "${{ inputs.host-platform }}" == win* ]]; then
+          elif [[ "${{ matrix.host-platform }}" == win* ]]; then
             CIBW_BUILD="cp${PYTHON_VERSION_FORMATTED}-win_amd64"
             PWD=$(pwd)
             REPO_DIR=$(cygpath -w $PWD)
           fi
   
           echo "PARALLEL_LEVEL=$(nproc)" >> $GITHUB_ENV
-          echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
+          echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ matrix.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
           echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ inputs.cuda-version }}-${{ inputs.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
+          echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ matrix.cuda-version }}-${{ matrix.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
           echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
           echo "CIBW_BUILD=${CIBW_BUILD}" >> $GITHUB_ENV
   
@@ -84,7 +90,7 @@ jobs:
       - name: List the cuda.core artifacts directory
         shell: bash --noprofile --norc -xeuo pipefail {0}
         run: |
-          if [[ "${{ inputs.host-platform }}" == win* ]]; then
+          if [[ "${{ matrix.host-platform }}" == win* ]]; then
             export CHOWN=chown
           else
             export CHOWN="sudo chown"
@@ -110,8 +116,8 @@ jobs:
         uses: ./.github/actions/fetch_ctk
         continue-on-error: false
         with:
-          host-platform: ${{ inputs.host-platform }}
-          cuda-version: ${{ inputs.cuda-version }}
+          host-platform: ${{ matrix.host-platform }}
+          cuda-version: ${{ matrix.cuda-version }}
           fail-on-ctk-cache-miss: false
 
       - name: Build cuda.bindings wheel
@@ -134,7 +140,7 @@ jobs:
       - name: List the cuda.bindings artifacts directory
         shell: bash --noprofile --norc -xeuo pipefail {0}
         run: |
-          if [[ "${{ inputs.host-platform }}" == win* ]]; then
+          if [[ "${{ matrix.host-platform }}" == win* ]]; then
             export CHOWN=chown
           else
             export CHOWN="sudo chown"
@@ -165,16 +171,33 @@ jobs:
           echo "CUDA_BINDINGS_ARTIFACTS_DIR=${CUDA_BINDINGS_ARTIFACTS_DIR}" >> $GITHUB_OUTPUT
 
   test:
-    # TODO: improve the name once a separate test matrix is defined
-    name: Test (CUDA ${{ inputs.cuda-version }})
-    # TODO: enable testing once win-64 GPU runners are up
-    if: ${{ (github.repository_owner == 'nvidia') &&
-             startsWith(inputs.host-platform, 'linux') }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO: align host-platform names with conda convention
+        host-platform:
+          - linux-x64
+          - linux-aarch64
+          # TODO: enable testing once win-64 GPU runners are up
+          # - win-x64
+        python-version:
+          - "3.13"
+          - "3.12"
+          - "3.11"
+          - "3.10"
+          - "3.9"
+        cuda-version:
+          # Note: this is for test-time only.
+          - "12.6.2"
+          - "12.0.1"
+          - "11.8.0"
+    name: Test (${{ matrix.host-platform }}, CUDA ${{ matrix.cuda-version }}, Python "${{ matrix.python-version }}")
+    if: ${{ (github.repository_owner == 'nvidia') }}
     permissions:
       id-token: write # This is required for configure-aws-credentials
       contents: read  # This is required for actions/checkout
-    runs-on: ${{ (inputs.host-platform == 'linux-x64' && 'linux-amd64-gpu-v100-latest-1') ||
-                 (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-gpu-a100-latest-1') }}
+    runs-on: ${{ (matrix.host-platform == 'linux-x64' && 'linux-amd64-gpu-v100-latest-1') ||
+                 (matrix.host-platform == 'linux-aarch64' && 'linux-arm64-gpu-a100-latest-1') }}
     # Our self-hosted runners require a container
     # TODO: use a different (nvidia?) container
     container:
@@ -227,10 +250,10 @@ jobs:
           pwd
           ls -lahR $CUDA_CORE_ARTIFACTS_DIR
 
-      - name: Set up Python ${{ inputs.python-version }}
+      - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ inputs.python-version }}
+          python-version: ${{ matrix.python-version }}
 
       # The cache action needs this
       - name: Install zstd
@@ -243,8 +266,8 @@ jobs:
         uses: ./.github/actions/fetch_ctk
         continue-on-error: false
         with:
-          host-platform: ${{ inputs.host-platform }}
-          cuda-version: ${{ inputs.cuda-version }}
+          host-platform: ${{ matrix.host-platform }}
+          cuda-version: ${{ matrix.cuda-version }}
           fail-on-ctk-cache-miss: true
 
       - name: Run test / analysis

From 61813fa5af4c43f587dbf7d2793adc0dedb0b87c Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Mon, 16 Dec 2024 02:25:50 +0000
Subject: [PATCH 05/12] Recompute the job outputs in test instead of reusing

the build/test matrices now mismatch, so it's no longer safe to keep the one-to-one relation
---
 .github/actions/fetch_ctk/action.yml    |  5 +----
 .github/workflows/gh-build-and-test.yml | 30 ++++++++++++-------------
 2 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml
index ab7c50d0..3720a853 100644
--- a/.github/actions/fetch_ctk/action.yml
+++ b/.github/actions/fetch_ctk/action.yml
@@ -9,9 +9,6 @@ inputs:
   cuda-version:
     required: true
     type: string
-  fail-on-ctk-cache-miss:
-    required: true
-    type: boolean
 
 runs:
   using: composite
@@ -29,7 +26,7 @@ runs:
       with:
         key: ${{ env.CTK_CACHE_KEY }}
         path: ./${{ env.CTK_CACHE_FILENAME }}
-        fail-on-cache-miss: ${{ inputs.fail-on-ctk-cache-miss }}
+        fail-on-cache-miss: false
 
     - name: Get CUDA components
       if: ${{ steps.ctk-get-cache.outputs.cache-hit != 'true' }}
diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml
index f4399118..13c5feef 100644
--- a/.github/workflows/gh-build-and-test.yml
+++ b/.github/workflows/gh-build-and-test.yml
@@ -29,10 +29,7 @@ jobs:
                  (matrix.host-platform == 'win-x64' && 'windows-2019') }}
                #  (matrix.host-platform == 'win-x64' && 'windows-amd64-cpu8') }}
     outputs:
-      CUDA_CORE_ARTIFACT_NAME: ${{ steps.pass_env.outputs.CUDA_CORE_ARTIFACT_NAME }}
-      CUDA_CORE_ARTIFACTS_DIR: ${{ steps.pass_env.outputs.CUDA_CORE_ARTIFACTS_DIR }}
-      CUDA_BINDINGS_ARTIFACT_NAME: ${{ steps.pass_env.outputs.CUDA_BINDINGS_ARTIFACT_NAME }}
-      CUDA_BINDINGS_ARTIFACTS_DIR: ${{ steps.pass_env.outputs.CUDA_BINDINGS_ARTIFACTS_DIR }}
+      BUILD_CTK_VER: ${{ steps.pass_env.outputs.CUDA_VERSION }}
     steps:
       - name: Checkout ${{ github.event.repository.name }}
         uses: actions/checkout@v4
@@ -118,7 +115,6 @@ jobs:
         with:
           host-platform: ${{ matrix.host-platform }}
           cuda-version: ${{ matrix.cuda-version }}
-          fail-on-ctk-cache-miss: false
 
       - name: Build cuda.bindings wheel
         uses: pypa/cibuildwheel@v2.22.0
@@ -165,10 +161,7 @@ jobs:
       - name: Pass environment variables to the next runner
         id: pass_env
         run: |
-          echo "CUDA_CORE_ARTIFACT_NAME=${CUDA_CORE_ARTIFACT_NAME}" >> $GITHUB_OUTPUT
-          echo "CUDA_CORE_ARTIFACTS_DIR=${CUDA_CORE_ARTIFACTS_DIR}" >> $GITHUB_OUTPUT
-          echo "CUDA_BINDINGS_ARTIFACT_NAME=${CUDA_BINDINGS_ARTIFACT_NAME}" >> $GITHUB_OUTPUT
-          echo "CUDA_BINDINGS_ARTIFACTS_DIR=${CUDA_BINDINGS_ARTIFACTS_DIR}" >> $GITHUB_OUTPUT
+          echo "CUDA_VERSION=${{ matrix.cuda-version }}" >> $GITHUB_OUTPUT
 
   test:
     strategy:
@@ -217,14 +210,22 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up test environment
+      - name: Set environment variables
         shell: bash --noprofile --norc -xeuo pipefail {0}
         run: |
+          PYTHON_VERSION_FORMATTED=$(echo '${{ matrix.python-version }}' | tr -d '.')
+          if [[ "${{ matrix.host-platform }}" == linux* ]]; then
+            REPO_DIR=$(pwd)
+          elif [[ "${{ matrix.host-platform }}" == win* ]]; then
+            PWD=$(pwd)
+            REPO_DIR=$(cygpath -w $PWD)
+          fi
+
           # make outputs from the previous job as env vars
-          echo "CUDA_CORE_ARTIFACT_NAME=${{ needs.build.outputs.CUDA_CORE_ARTIFACT_NAME }}" >> $GITHUB_ENV
-          echo "CUDA_CORE_ARTIFACTS_DIR=${{ needs.build.outputs.CUDA_CORE_ARTIFACTS_DIR }}" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_ARTIFACT_NAME=${{ needs.build.outputs.CUDA_BINDINGS_ARTIFACT_NAME }}" >> $GITHUB_ENV
-          echo "CUDA_BINDINGS_ARTIFACTS_DIR=${{ needs.build.outputs.CUDA_BINDINGS_ARTIFACTS_DIR }}" >> $GITHUB_ENV
+          echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ matrix.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
+          echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV
+          echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ needs.build.outputs.BUILD_CTK_VER }}-${{ matrix.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
+          echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
 
       - name: Download bindings build artifacts
         uses: actions/download-artifact@v4
@@ -268,7 +269,6 @@ jobs:
         with:
           host-platform: ${{ matrix.host-platform }}
           cuda-version: ${{ matrix.cuda-version }}
-          fail-on-ctk-cache-miss: true
 
       - name: Run test / analysis
         shell: bash --noprofile --norc -xeuo pipefail {0}

From 3dca084cfc7a415d49c47c25ab6193bada1c098f Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Mon, 16 Dec 2024 06:01:32 +0000
Subject: [PATCH 06/12] name cleanup + add H100 runner + install curl/xz

---
 .github/actions/fetch_ctk/action.yml    | 40 +++++++++++++++++++-----
 .github/workflows/ci-gh.yml             |  3 +-
 .github/workflows/gh-build-and-test.yml | 41 +++++++++++++------------
 3 files changed, 54 insertions(+), 30 deletions(-)

diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml
index 3720a853..b4faa473 100644
--- a/.github/actions/fetch_ctk/action.yml
+++ b/.github/actions/fetch_ctk/action.yml
@@ -19,6 +19,27 @@ runs:
         echo "CTK_CACHE_KEY=mini-ctk-${{ inputs.cuda-version }}-${{ inputs.host-platform }}" >> $GITHUB_ENV
         echo "CTK_CACHE_FILENAME=mini-ctk-${{ inputs.cuda-version }}-${{ inputs.host-platform }}.tar.gz" >> $GITHUB_ENV
 
+    - name: Install dependencies
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        if (command -v curl 2>&1 >/dev/null) && (command -v zstd 2>&1 >/dev/null); then
+          echo "All dependencies are found. Do nothing."
+          exit 0
+        fi
+        if ! command -v sudo 2>&1 >/dev/null; then
+          if [[ $EUID == 0 ]]; then
+            alias SUDO=""
+          else
+            echo "The following oprations require root access."
+            exit 1
+          fi
+        else
+          alias SUDO="sudo"
+        fi
+        shopt -s expand_aliases
+        SUDO apt update
+        SUDO apt install -y zstd curl xz-utils
+
     - name: Download CTK cache
       id: ctk-get-cache
       uses: actions/cache/restore@v4
@@ -32,14 +53,14 @@ runs:
       if: ${{ steps.ctk-get-cache.outputs.cache-hit != 'true' }}
       shell: bash --noprofile --norc -xeuo pipefail {0}
       run: |
-        CUDA_PATH="$(pwd)/cuda_toolkit"
+        CUDA_PATH="./cuda_toolkit"
         mkdir $CUDA_PATH
 
         # The binary archives (redist) are guaranteed to be updated as part of the release posting.
         CTK_BASE_URL="https://developer.download.nvidia.com/compute/cuda/redist/"
         CTK_JSON_URL="$CTK_BASE_URL/redistrib_${{ inputs.cuda-version }}.json"
         if [[ "${{ inputs.host-platform }}" == linux* ]]; then
-          if [[ "${{ inputs.host-platform }}" == "linux-x64" ]]; then
+          if [[ "${{ inputs.host-platform }}" == "linux-64" ]]; then
             CTK_SUBDIR="linux-x86_64"
           elif [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
             CTK_SUBDIR="linux-sbsa"
@@ -47,7 +68,7 @@ runs:
           function extract() {
             tar -xvf $1 -C $CUDA_PATH --strip-components=1
           }
-        elif [[ "${{ inputs.host-platform }}" == "win-x64" ]]; then
+        elif [[ "${{ inputs.host-platform }}" == "win-64" ]]; then
           CTK_SUBDIR="windows-x86_64"
           function extract() {
             _TEMP_DIR_=$(mktemp -d)
@@ -79,16 +100,15 @@ runs:
         populate_cuda_path cuda_cudart
         populate_cuda_path cuda_nvrtc
         populate_cuda_path cuda_profiler_api
-        populate_cuda_path libnvjitlink
+        if [[ "$(cut -d '.' -f 1 <<< ${{ inputs.cuda-version }})" -ge 12 ]]; then
+          populate_cuda_path libnvjitlink
+        fi
         ls -l $CUDA_PATH
 
         # Prepare the cache
         # Note: try to escape | and > ...
         tar -czvf ${CTK_CACHE_FILENAME} ${CUDA_PATH}
 
-        # Note: the headers will be copied into the cibuildwheel manylinux container,
-        # so setting the CUDA_PATH env var here is meaningless.
-
     - name: Upload CTK cache
       if: ${{ always() &&
               steps.ctk-get-cache.outputs.cache-hit != 'true' }}
@@ -102,13 +122,17 @@ runs:
       shell: bash --noprofile --norc -xeuo pipefail {0}
       run: |
         ls -l
-        CUDA_PATH="$(pwd)/cuda_toolkit"
+        CUDA_PATH="./cuda_toolkit"
         tar -xzvf $CTK_CACHE_FILENAME
         ls -l $CUDA_PATH
         if [ ! -d "$CUDA_PATH/include" ]; then
           exit 1
         fi
 
+    - name: Set output environment variables
+      shell: bash --noprofile --norc -xeuo pipefail {0}
+      run: |
+        CUDA_PATH=$(realpath "./cuda_toolkit")
         echo "CUDA_PATH=${CUDA_PATH}" >> $GITHUB_ENV
         echo "${CUDA_PATH}/bin" >> $GITHUB_PATH
         echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CUDA_PATH}/lib" >> $GITHUB_ENV
diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml
index 795d6d0a..81cf6da3 100644
--- a/.github/workflows/ci-gh.yml
+++ b/.github/workflows/ci-gh.yml
@@ -1,4 +1,4 @@
-name: "CI"
+name: CI
 
 concurrency:
   group: ${{ startsWith(github.ref_name, 'main') && format('unique-{0}', github.run_id) || format('ci-build-and-test-on-{0}-from-{1}', github.event_name, github.ref_name) }}
@@ -12,7 +12,6 @@ on:
 
 jobs:
   ci:
-    name: "CI"
     uses:
       ./.github/workflows/gh-build-and-test.yml
     secrets: inherit
diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml
index 13c5feef..8cc20bf3 100644
--- a/.github/workflows/gh-build-and-test.yml
+++ b/.github/workflows/gh-build-and-test.yml
@@ -5,11 +5,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # TODO: align host-platform names with conda convention
         host-platform:
-          - linux-x64
+          - linux-64
           - linux-aarch64
-          - win-x64
+          - win-64
         python-version:
           - "3.13"
           - "3.12"
@@ -19,15 +18,15 @@ jobs:
         cuda-version:
           # Note: this is for build-time only.
           - "12.6.2"
-    name: Build (${{ matrix.host-platform }}, Python "${{ matrix.python-version }}")
+    name: Build (${{ matrix.host-platform }}, Python ${{ matrix.python-version }}, CUDA ${{ matrix.cuda-version }})
     if: ${{ github.repository_owner == 'nvidia' }}
     permissions:
       id-token: write # This is required for configure-aws-credentials
       contents: read  # This is required for actions/checkout
-    runs-on: ${{ (matrix.host-platform == 'linux-x64' && 'linux-amd64-cpu8') ||
+    runs-on: ${{ (matrix.host-platform == 'linux-64' && 'linux-amd64-cpu8') ||
                  (matrix.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') ||
-                 (matrix.host-platform == 'win-x64' && 'windows-2019') }}
-               #  (matrix.host-platform == 'win-x64' && 'windows-amd64-cpu8') }}
+                 (matrix.host-platform == 'win-64' && 'windows-2019') }}
+               #  (matrix.host-platform == 'win-64' && 'windows-amd64-cpu8') }}
     outputs:
       BUILD_CTK_VER: ${{ steps.pass_env.outputs.CUDA_VERSION }}
     steps:
@@ -166,13 +165,13 @@ jobs:
   test:
     strategy:
       fail-fast: false
+      # TODO: add driver version here
       matrix:
-        # TODO: align host-platform names with conda convention
         host-platform:
-          - linux-x64
+          - linux-64
           - linux-aarch64
           # TODO: enable testing once win-64 GPU runners are up
-          # - win-x64
+          # - win-64
         python-version:
           - "3.13"
           - "3.12"
@@ -183,14 +182,23 @@ jobs:
           # Note: this is for test-time only.
           - "12.6.2"
           - "12.0.1"
+          # FIXME: cannot run cuda.bindings 12.x with CTK 11.x
           - "11.8.0"
-    name: Test (${{ matrix.host-platform }}, CUDA ${{ matrix.cuda-version }}, Python "${{ matrix.python-version }}")
+        runner:
+          - default
+        include:
+          - host-platform: linux-64
+            python-version: "3.12"
+            cuda-version: "12.6.2"
+            runner: H100
+    name: Test (${{ matrix.host-platform }}, Python ${{ matrix.python-version }}, CUDA ${{ matrix.cuda-version }}, Runner ${{ matrix.runner }})
     if: ${{ (github.repository_owner == 'nvidia') }}
     permissions:
       id-token: write # This is required for configure-aws-credentials
       contents: read  # This is required for actions/checkout
-    runs-on: ${{ (matrix.host-platform == 'linux-x64' && 'linux-amd64-gpu-v100-latest-1') ||
-                 (matrix.host-platform == 'linux-aarch64' && 'linux-arm64-gpu-a100-latest-1') }}
+    runs-on: ${{ (matrix.runner == 'default' && matrix.host-platform == 'linux-64' && 'linux-amd64-gpu-v100-latest-1') ||
+                 (matrix.runner == 'default' && matrix.host-platform == 'linux-aarch64' && 'linux-arm64-gpu-a100-latest-1') ||
+                 (matrix.runner == 'H100' && 'linux-amd64-gpu-h100-latest-1-testing') }}
     # Our self-hosted runners require a container
     # TODO: use a different (nvidia?) container
     container:
@@ -256,13 +264,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      # The cache action needs this
-      - name: Install zstd
-        shell: bash --noprofile --norc -xeuo pipefail {0}
-        run: |
-          apt update
-          apt install zstd
-
       - name: Set up mini CTK
         uses: ./.github/actions/fetch_ctk
         continue-on-error: false

From 39ffefc723a6654107709de6b5b2778cc6b0ad88 Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Mon, 16 Dec 2024 01:34:58 -0500
Subject: [PATCH 07/12] Get CCCL headers for mini CTK

---
 .github/actions/fetch_ctk/action.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml
index b4faa473..81d5f8dc 100644
--- a/.github/actions/fetch_ctk/action.yml
+++ b/.github/actions/fetch_ctk/action.yml
@@ -100,6 +100,7 @@ runs:
         populate_cuda_path cuda_cudart
         populate_cuda_path cuda_nvrtc
         populate_cuda_path cuda_profiler_api
+        populate_cuda_path cuda_cccl
         if [[ "$(cut -d '.' -f 1 <<< ${{ inputs.cuda-version }})" -ge 12 ]]; then
           populate_cuda_path libnvjitlink
         fi

From 7fcb5ad3a40df27128e497edd185cca4fbf95f79 Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Mon, 16 Dec 2024 22:30:22 +0000
Subject: [PATCH 08/12] skip testing cuda.bindings if runtime CTK major version
 is older

---
 .github/workflows/gh-build-and-test.yml | 57 +++++++++++++++++++------
 1 file changed, 43 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml
index 8cc20bf3..c254fd1a 100644
--- a/.github/workflows/gh-build-and-test.yml
+++ b/.github/workflows/gh-build-and-test.yml
@@ -192,7 +192,8 @@ jobs:
             cuda-version: "12.6.2"
             runner: H100
     name: Test (${{ matrix.host-platform }}, Python ${{ matrix.python-version }}, CUDA ${{ matrix.cuda-version }}, Runner ${{ matrix.runner }})
-    if: ${{ (github.repository_owner == 'nvidia') }}
+    # The build stage could fail but we want the CI to keep moving.
+    if: ${{ (github.repository_owner == 'nvidia') && always() }}
     permissions:
       id-token: write # This is required for configure-aws-credentials
       contents: read  # This is required for actions/checkout
@@ -229,31 +230,40 @@ jobs:
             REPO_DIR=$(cygpath -w $PWD)
           fi
 
+          BUILD_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ needs.build.outputs.BUILD_CTK_VER }})"
+          TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ matrix.cuda-version }})"
+          if [[ $BUILD_CUDA_MAJOR -gt $TEST_CUDA_MAJOR ]]; then
+            SKIP_CUDA_BINDINGS_TEST=1
+          else
+            SKIP_CUDA_BINDINGS_TEST=0
+          fi
+
           # make outputs from the previous job as env vars
           echo "CUDA_CORE_ARTIFACT_NAME=cuda-core-python${PYTHON_VERSION_FORMATTED}-${{ matrix.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
           echo "CUDA_CORE_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_core/dist")" >> $GITHUB_ENV
           echo "CUDA_BINDINGS_ARTIFACT_NAME=cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda${{ needs.build.outputs.BUILD_CTK_VER }}-${{ matrix.host-platform }}-${{ github.sha }}" >> $GITHUB_ENV
           echo "CUDA_BINDINGS_ARTIFACTS_DIR=$(realpath "$REPO_DIR/cuda_bindings/dist")" >> $GITHUB_ENV
+          echo "SKIP_CUDA_BINDINGS_TEST=${SKIP_CUDA_BINDINGS_TEST}" >> $GITHUB_ENV
 
-      - name: Download bindings build artifacts
+      - name: Download cuda.bindings build artifacts
         uses: actions/download-artifact@v4
         with:
           name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
           path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
 
-      - name: Display structure of downloaded bindings artifacts
+      - name: Display structure of downloaded cuda.bindings artifacts
         shell: bash --noprofile --norc -xeuo pipefail {0}
         run: |
           pwd
           ls -lahR $CUDA_BINDINGS_ARTIFACTS_DIR
 
-      - name: Download core build artifacts
+      - name: Download cuda.core build artifacts
         uses: actions/download-artifact@v4
         with:
           name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
           path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
 
-      - name: Display structure of downloaded core build artifacts
+      - name: Display structure of downloaded cuda.core build artifacts
         shell: bash --noprofile --norc -xeuo pipefail {0}
         run: |
           pwd
@@ -271,24 +281,43 @@ jobs:
           host-platform: ${{ matrix.host-platform }}
           cuda-version: ${{ matrix.cuda-version }}
 
-      - name: Run test / analysis
+      - name: Run cuda.bindings tests
+        if: ${{ env.SKIP_CUDA_BINDINGS_TEST == '0' }}
         shell: bash --noprofile --norc -xeuo pipefail {0}
         run: |
           ls $CUDA_PATH
 
-          REPO_DIR=$(pwd)
-
-          cd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
-          pip install *.whl
-
-          cd "${CUDA_CORE_ARTIFACTS_DIR}"
+          pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
           pip install *.whl
+          popd
 
-          cd "${REPO_DIR}/cuda_bindings"
+          pushd ./cuda_bindings
           pip install -r requirements.txt
           pytest -rxXs tests/
           # TODO: enable cython tests
           #pytest tests/cython
+          popd
+
+      - name: Run cuda.core tests
+        shell: bash --noprofile --norc -xeuo pipefail {0}
+        run: |
+          if [[ $SKIP_CUDA_BINDINGS_TEST == 1 ]]; then
+            # TODO: remove this hack once cuda-python has a cp313 build
+            if [[ ${{ matrix.python-version }} == "3.13" ]]; then
+              echo "Python 3.13 + cuda-python ${{ matrix.cuda-version }} is not supported, skipping the test..."
+              exit 0
+            else
+              pip install "cuda-python==$(cut -d '.' -f 1,2 <<< ${{ matrix.cuda-version }}).*"
+            fi
+          fi
+
+          pushd "${CUDA_CORE_ARTIFACTS_DIR}"
+          pip install *.whl
+          popd
 
-          cd "${REPO_DIR}/cuda_core"
+          pushd ./cuda_core
+          # TODO: add requirements.txt for test deps?
+          # TODO: add cupy as an optional test dep?
+          pip install pytest
           pytest -rxXs tests/
+          popd

From 732078625f29dfe8ff88f2d9b310ee7d07fe05be Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Mon, 16 Dec 2024 22:30:48 +0000
Subject: [PATCH 09/12] ensure kernel has C linkage

---
 cuda_core/tests/test_module.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/cuda_core/tests/test_module.py b/cuda_core/tests/test_module.py
index b2519c85..61933d41 100644
--- a/cuda_core/tests/test_module.py
+++ b/cuda_core/tests/test_module.py
@@ -15,13 +15,9 @@
 
 @pytest.mark.xfail(not can_load_generated_ptx(), reason="PTX version too new")
 def test_get_kernel():
-    kernel = """
-extern __device__ int B();
-extern __device__ int C(int a, int b);
-__global__ void A() { int result = C(B(), 1);}
-"""
+    kernel = """extern "C" __global__ void ABC() { }"""
     object_code = Program(kernel, "c++").compile("ptx", options=("-rdc=true",))
     assert object_code._handle is None
-    kernel = object_code.get_kernel("A")
+    kernel = object_code.get_kernel("ABC")
     assert object_code._handle is not None
     assert kernel._handle is not None

From 36f47717805166a7531bb26da78a3729ad9e32bc Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Mon, 16 Dec 2024 22:37:15 +0000
Subject: [PATCH 10/12] nit: automate dep management

---
 .github/actions/fetch_ctk/action.yml | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/.github/actions/fetch_ctk/action.yml b/.github/actions/fetch_ctk/action.yml
index 81d5f8dc..05076f73 100644
--- a/.github/actions/fetch_ctk/action.yml
+++ b/.github/actions/fetch_ctk/action.yml
@@ -22,11 +22,21 @@ runs:
     - name: Install dependencies
       shell: bash --noprofile --norc -xeuo pipefail {0}
       run: |
-        if (command -v curl 2>&1 >/dev/null) && (command -v zstd 2>&1 >/dev/null); then
+        dependencies=(zstd curl xz-utils)
+        dependent_exes=(zstd curl xz)
+
+        not_found=0
+        for dep in ${dependent_exes[@]}; do
+          if ! (command -v curl 2>&1 >/dev/null); then
+            not_found=1
+            break
+          fi
+        done
+        if [[ $not_found == 0 ]]; then
           echo "All dependencies are found. Do nothing."
           exit 0
         fi
-        if ! command -v sudo 2>&1 >/dev/null; then
+        if ! (command -v sudo 2>&1 >/dev/null); then
           if [[ $EUID == 0 ]]; then
             alias SUDO=""
           else
@@ -38,7 +48,7 @@ runs:
         fi
         shopt -s expand_aliases
         SUDO apt update
-        SUDO apt install -y zstd curl xz-utils
+        SUDO apt install -y ${dependencies[@]}
 
     - name: Download CTK cache
       id: ctk-get-cache

From 8ee415d5bbad4169f3d3cc7fb3fa0ca89d5a68ad Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Mon, 16 Dec 2024 23:51:15 +0000
Subject: [PATCH 11/12] skip nvJitLink binding tests if the library is not
 usable

---
 cuda_bindings/tests/test_nvjitlink.py | 13 +++++++++++++
 cuda_core/docs/source/conf.py         |  4 +++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/cuda_bindings/tests/test_nvjitlink.py b/cuda_bindings/tests/test_nvjitlink.py
index 839c7be1..4a2c1a6b 100644
--- a/cuda_bindings/tests/test_nvjitlink.py
+++ b/cuda_bindings/tests/test_nvjitlink.py
@@ -52,6 +52,19 @@ def ptx_header(version, arch):
 ]
 
 
+def check_nvjitlink_usable():
+    from cuda.bindings._internal import nvjitlink as inner_nvjitlink
+
+    if inner_nvjitlink._inspect_function_pointer("__nvJitLinkVersion") == 0:
+        return False
+    return True
+
+
+pytestmark = pytest.mark.skipif(
+    not check_nvjitlink_usable(), reason="nvJitLink not usable, maybe not installed or too old (<12.3)"
+)
+
+
 # create a valid LTOIR input for testing
 @pytest.fixture
 def get_dummy_ltoir():
diff --git a/cuda_core/docs/source/conf.py b/cuda_core/docs/source/conf.py
index 4b3e17ae..ca59d588 100644
--- a/cuda_core/docs/source/conf.py
+++ b/cuda_core/docs/source/conf.py
@@ -94,9 +94,11 @@
 
 
 section_titles = ["Returns"]
+
+
 def autodoc_process_docstring(app, what, name, obj, options, lines):
     if name.startswith("cuda.core.experimental.system"):
-        # patch the docstring (in lines) *in-place*. Should docstrings include section titles other than "Returns", 
+        # patch the docstring (in lines) *in-place*. Should docstrings include section titles other than "Returns",
         # this will need to be modified to handle them.
         attr = name.split(".")[-1]
         from cuda.core.experimental._system import System

From 4494a276d44177e68333a2cba21a35797d28c013 Mon Sep 17 00:00:00 2001
From: Leo Fang <leof@nvidia.com>
Date: Tue, 17 Dec 2024 03:43:42 +0000
Subject: [PATCH 12/12] test cuda.core optional dep logic

---
 .github/workflows/gh-build-and-test.yml | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml
index c254fd1a..51f4bd87 100644
--- a/.github/workflows/gh-build-and-test.yml
+++ b/.github/workflows/gh-build-and-test.yml
@@ -182,7 +182,6 @@ jobs:
           # Note: this is for test-time only.
           - "12.6.2"
           - "12.0.1"
-          # FIXME: cannot run cuda.bindings 12.x with CTK 11.x
           - "11.8.0"
         runner:
           - default
@@ -232,7 +231,7 @@ jobs:
 
           BUILD_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ needs.build.outputs.BUILD_CTK_VER }})"
           TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ matrix.cuda-version }})"
-          if [[ $BUILD_CUDA_MAJOR -gt $TEST_CUDA_MAJOR ]]; then
+          if [[ $BUILD_CUDA_MAJOR != $TEST_CUDA_MAJOR ]]; then
             SKIP_CUDA_BINDINGS_TEST=1
           else
             SKIP_CUDA_BINDINGS_TEST=0
@@ -306,18 +305,20 @@ jobs:
             if [[ ${{ matrix.python-version }} == "3.13" ]]; then
               echo "Python 3.13 + cuda-python ${{ matrix.cuda-version }} is not supported, skipping the test..."
               exit 0
-            else
-              pip install "cuda-python==$(cut -d '.' -f 1,2 <<< ${{ matrix.cuda-version }}).*"
             fi
           fi
 
+          # If build/test majors match: cuda.bindings is installed in the previous step.
+          # If mismatch: cuda.bindings is installed from PyPI.
+          TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${{ matrix.cuda-version }})"
           pushd "${CUDA_CORE_ARTIFACTS_DIR}"
-          pip install *.whl
+          pip install $(ls *.whl)["cu${TEST_CUDA_MAJOR}"]
           popd
 
           pushd ./cuda_core
           # TODO: add requirements.txt for test deps?
-          # TODO: add cupy as an optional test dep?
           pip install pytest
+          # TODO: add CuPy to test deps (which would require cuRAND)
+          # pip install "cupy-cuda${TEST_CUDA_MAJOR}x"
           pytest -rxXs tests/
           popd