Fixing artifacts download - back to matrix due to pypa/pip#11664

djpugh · May 1, 2023 · 8829778 · 8829778
1 parent 18d9b60
commit 8829778
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 35 deletions.
diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
@@ -132,33 +132,49 @@ jobs:
 
   collate-packages:
     name: Collate Wheels
-    runs-on: windows-latest
+    runs-on: ${{ matrix.os}}-latest
     needs:
       - changes
     if: >
       (needs.changes.outputs.config == 'true' || needs.changes.outputs.workflows == 'true') ||
       (github.event_name == 'pull_request' && contains(github.base_ref, 'main')) ||
       (github.event_name == 'push' && (startsWith(github.ref, 'refs/tags') || contains(github.ref, 'main'))) ||
       (github.event_name == 'release')
+    strategy:
+      fail-fast: true
+      # needs to run as matricx due to https://github.com/pypa/pip/issues/11664
+      # dependencies aren't evaluated by the requested platform
+      matrix:
+        os:
+          - ubuntu
+          - windows
+          - macos
+        py:
+          - 3.7
+          - 3.8
+          - 3.9
+          - "3.10"
+          - 3.11
+
     steps:
     - uses: actions/checkout@v3
 
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: 3.9
+        python-version: ${{ matrix.py }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install tox
 
     - name: Populate platform wheels
-      run: tox -e populate -- --target-dir ./wheels
+      run: tox -e populate -- --target-dir ./wheels --py ${{matrix.py}}
 
     - name: Store collated wheels
       uses: actions/upload-artifact@v3
       with:
-        name: wheels
+        name: wheels-${{matrix.os}}-${{matrix.py}}
         path: |
           ./wheels/*
 
@@ -204,10 +220,11 @@ jobs:
     - name: Download collated wheels
       uses: actions/download-artifact@v3
       with:
-        name: wheels
-        path: src/azure_devops_artifacts_helpers/wheels/
-    # - shell: bash
-    #   run: cp -u wheels/*.whl src/azure_devops_artifacts_helpers/wheels/
+        path: wheels
+    - shell: bash
+      run: |
+        cp wheels/*/*.whl src/azure_devops_artifacts_helpers/wheels/
+        ls -l src/azure_devops_artifacts_helpers/wheels/
     - name: Setup Test Suite
       run: tox -e test-${{ matrix.tox_version }} -vv --notest
     - name: Run Test Suite
@@ -343,8 +360,9 @@ jobs:
           path: wheels
 
       - shell: bash
-        run: cp -u wheels/*.whl src/azure_devops_artifacts_helpers/wheels/
-
+        run: |
+          cp -u wheels/*/*.whl src/azure_devops_artifacts_helpers/wheels/
+          ls -l src/azure_devops_artifacts_helpers/wheels/
       - name: Build azure_devops_artifacts_helpers
         run: tox -e build -vvv
 
@@ -381,8 +399,9 @@ jobs:
           path: wheels
 
       - shell: bash
-        run: cp -u wheels/*.whl src/azure_devops_artifacts_helpers/wheels/
-
+        run: |
+          cp -u wheels/*/*.whl src/azure_devops_artifacts_helpers/wheels/
+          ls -l src/azure_devops_artifacts_helpers/wheels/
       - name: Build docs
         run: tox -e docs
         env:

diff --git a/tools/populate-wheels.py b/tools/populate-wheels.py
@@ -22,9 +22,8 @@
 @click.command()
 @click.option('--index-url', default=DOWNLOAD_INDEX_URL, help='Index URL to use')
 @click.option('--py', 'python_versions', default=None, multiple=True, help='Python versions to populate, defaults to values set in pyproject.toml', type=str)
-@click.option('--platform', 'platforms', default=None, multiple=True, help='Platforms to populate, defaults to all platforms', type=str)
 @click.option('--target-dir', default=str(WHEELS_DIR), help='Target directory to output to')
-def populate_wheels(index_url: str = DOWNLOAD_INDEX_URL, python_versions: List[str] = [], platforms: List[str] = [], target_dir: str = str(WHEELS_DIR)) -> None:
+def populate_wheels(index_url: str = DOWNLOAD_INDEX_URL, python_versions: List[str] = [], target_dir: str = str(WHEELS_DIR)) -> None:
     # Load from toml section
     with open('pyproject.toml') as f:
         pyproject_toml = toml.loads(f.read())
@@ -47,28 +46,15 @@ def populate_wheels(index_url: str = DOWNLOAD_INDEX_URL, python_versions: List[s
             max_version = Version(f'{_v.major}.{_v.minor-1}')
         python_versions = [f'{min_version.major}.{minor}' for minor in range(min_version.minor, max_version.minor+1)]
         print(f'Using versions: {python_versions} from {_min_version}:{_max_version}')
-    if not platforms:
-        platforms = ['win32', 'darwin', 'linux']
-    processed_platforms = []
-    for plat in platforms:
-        if 'ubuntu' in plat:
-            plat = 'linux'
-        elif 'windows' in plat:
-            plat = 'win32'
-        elif 'macos' in plat:
-            plat = 'darwin'
-        processed_platforms.append(plat)
     for py_version in python_versions:
-        for platform in processed_platforms:
-            print(f'PLATFORM: {platform} - PYTHON VERSION: {py_version}')
-            args = [sys.executable, '-m', 'pip', 'download',
-                    '--only-binary=:all:',
-                    f'--platform={platform}',
-                    f'--python-version={py_version}',
-                    '-d', target_dir,
-                    '--index-url', index_url] + packages
-            print(' '.join(args))
-            print(subprocess.check_output(args).decode())
+        print(f'PLATFORM: {sys.platform} - PYTHON VERSION: {py_version}')
+        args = [sys.executable, '-m', 'pip', 'download',
+                '--only-binary=:all:',
+                f'--python-version={py_version}',
+                '-d', target_dir,
+                '--index-url', index_url] + packages
+        print(' '.join(args))
+        print(subprocess.check_output(args).decode())
     print('Downloaded Packages to '+target_dir)
     print('\t- '+'\n\t- '.join([u for u in os.listdir(target_dir) if u.endswith('whl')] ))