Generate lockfiles for the constructor envs (#201)

* try new constructor PR

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* scm pretend

* find_executable -> shutil.which

* debug a bit

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update 'Collect lockfiles' step

* more lockfile updates

* clean a bit and retry

* Require constructor 3.11+

* Upload lockfile asset too

* do not zip lockfiles

* better artifact name

* patch local paths

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* patch f-string

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add a note about the viability of lockfiles in PRs
---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/make_bundle_conda.yml

@@ -187,6 +187,7 @@ jobs:
   prepare_matrix:
     # See this SO answer for details on conditional matrices
     # https://stackoverflow.com/a/65434401/3407590
+    name: Prepare matrix
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
@@ -256,7 +257,7 @@ jobs:
 
     outputs:
       licenses-artifact: ${{ steps.licenses.outputs.licenses_artifact }}
-      pkgs-list-artifact: ${{ steps.pkgs-list.outputs.pkgs_list_artifact }}
+      lockfile-artifact: ${{ steps.pkgs-list.outputs.lockfile_artifact }}
 
     steps:
       - name: Checkout code
@@ -455,21 +456,23 @@ jobs:
           path: ${{ env.LICENSES_ARTIFACT_PATH }}
           name: ${{ env.LICENSES_ARTIFACT_NAME }}
 
-      - name: Collect list of packages
+      - name: Collect lockfiles
         id: pkgs-list
         shell: bash -el {0}
         working-directory: napari-packaging
         run: |
-          pkgs_list_zip_path=$(python build_installers.py --pkgs-list)
-          echo "PKGS_LIST_ARTIFACT_PATH=$pkgs_list_zip_path" >> $GITHUB_ENV
-          echo "PKGS_LIST_ARTIFACT_NAME=$(basename ${pkgs_list_zip_path})" >> $GITHUB_ENV
-          echo "pkgs_list_artifact=${pkgs_list_zip_path}" >> $GITHUB_OUTPUT
+          lockfile_path=$(python build_installers.py --lockfile)
+          echo "LOCKFILE_ARTIFACT_PATH=$lockfile_path" >> $GITHUB_ENV
+          echo "lockfile_artifact=${lockfile_path}" >> $GITHUB_OUTPUT
 
-      - name: Upload list of packages artifact
+      - name: Upload lockfile artifact
         uses: actions/upload-artifact@v4
+        # NOTE: These lockfiles will only provide functional installations if the
+        # napari packages in the 'packages' job above has been uploaded to anaconda.org/napari
+        # and this only happens in scheduled jobs @ main and tagged releases.
         with:
-          path: ${{ env.PKGS_LIST_ARTIFACT_PATH }}
-          name: ${{ env.PKGS_LIST_ARTIFACT_NAME }}
+          path: ${{ env.LOCKFILE_ARTIFACT_PATH }}
+          name: napari-${{ env.version }}-${{ runner.os }}-${{ env.arch-suffix }}.lockfile.txt
 
       - name: Notarize & staple PKG Installer (macOS)
         # We only sign pushes to main, nightlies, RCs and final releases
@@ -533,7 +536,7 @@ jobs:
         id: get_release
         uses: bruceadams/get-release@v1.3.2
 
-      - name: Upload Release Asset
+      - name: Upload Release Asset (Installer)
         if: inputs.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
         uses: actions/upload-release-asset@v1
         with:
@@ -542,6 +545,15 @@ jobs:
           asset_name: napari-${{ env.version }}-${{ runner.os }}-${{ env.arch-suffix }}.${{ env.extension }}
           asset_content_type: application/octet-stream
 
+      - name: Upload Release Asset (Lockfile)
+        if: inputs.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+        uses: actions/upload-release-asset@v1
+        with:
+          upload_url: ${{ steps.get_release.outputs.upload_url }}
+          asset_path: ${{ env.LOCKFILE_ARTIFACT_PATH }}
+          asset_name: napari-${{ env.version }}-${{ runner.os }}-${{ env.arch-suffix }}.lockfile.txt
+          asset_content_type: application/octet-stream
+
       - name: Test installation (Linux)
         if: runner.os == 'Linux'
         working-directory: napari-packaging/_work

build_installers.py

@@ -47,9 +47,9 @@
 import sys
 import zipfile
 from argparse import ArgumentParser
-from distutils.spawn import find_executable
 from functools import lru_cache, partial
 from pathlib import Path
+from shutil import which
 from subprocess import check_call, check_output
 from tempfile import NamedTemporaryFile
 from textwrap import dedent, indent
@@ -285,7 +285,7 @@ def _definitions(version=_version(), extra_specs=None, napari_repo=HERE):
             {env_state: env_state_path},
         ],
         "build_outputs": [
-            {"pkgs_list": {"env": napari_env["name"]}},
+            {"lockfile": {"env": napari_env["name"]}},
             {"licenses": {"include_text": True, "text_errors": "replace"}},
         ],
     }
@@ -380,7 +380,7 @@ def _constructor(version=_version(), extra_specs=None, napari_repo=HERE):
     napari_repo: str
         location where the napari/napari repository was cloned
     """
-    constructor = find_executable("constructor")
+    constructor = which("constructor")
     if not constructor:
         raise RuntimeError("Constructor must be installed and in PATH.")
 
@@ -437,18 +437,32 @@ def licenses():
     return zipname.resolve()
 
 
-def packages_list():
-    txtfile = next(Path("_work").glob("pkg-list.napari-*.txt"), None)
+def lockfiles():
+    txtfile = next(Path("_work").glob("lockfile.napari-*.txt"), None)
     if not txtfile or not txtfile.is_file():
         sys.exit(
-            "!! pkg-list.napari-*.txt not found."
+            "!! lockfile.napari-*.txt not found. "
             "Ensure 'construct.yaml' has a 'build_outputs' "
-            "key configured with 'pkgs_list'.",
+            "key configured with 'lockfile'.",
         )
-    zipname = Path("_work") / f"pkg-list.{OS}-{ARCH}.zip"
-    with zipfile.ZipFile(zipname, mode="w", compression=zipfile.ZIP_DEFLATED) as ozip:
-        ozip.write(txtfile)
-    return zipname.resolve()
+    if _use_local():
+        # With local builds, the lockfile will have a file:// path that can't be used
+        # remotely. Fortunately, we have uploaded that package to anaconda.org/napari too.
+        from conda.base.context import context
+
+        if WINDOWS:
+            local_channel = context.croot.replace("\\", "/")
+            local_channel = f"file:///{local_channel}"
+        else:
+            local_channel = f"file://{context.croot}"
+        if not local_channel.endswith("/"):
+            local_channel += "/"
+        remote_channel = "https://conda.anaconda.org/napari/"
+        if "rc" in _version() or "dev" in _version():
+            remote_channel += "label/nightly/"
+        contents = txtfile.read_text().replace(local_channel, remote_channel)
+        txtfile.write_text(contents)
+    return txtfile.resolve()
 
 
 def main(extra_specs=None, napari_repo=HERE):
@@ -503,9 +517,9 @@ def cli(argv=None):
         "This must be run as a separate step.",
     )
     p.add_argument(
-        "--pkgs-list",
+        "--lockfile",
         action="store_true",
-        help="Generate the list of packages used to build the napari environment."
+        help="Collect the installer-equivalent lockfiles. Run AFTER building the installer. "
         "This must be run as a separate step.",
     )
     p.add_argument(
@@ -542,8 +556,8 @@ def cli(argv=None):
     if args.licenses:
         print(licenses())
         sys.exit()
-    if args.pkgs_list:
-        print(packages_list())
+    if args.lockfile:
+        print(lockfiles())
         sys.exit()
     if args.images:
         _generate_background_images(napari_repo=args.location)

environments/ci_installers_environment.yml

@@ -4,7 +4,7 @@ channels:
 dependencies:
   - python
   - pip
-  - constructor >=3.9.3
+  - constructor >=3.11.0
   - conda-build >=3.28
   - ruamel.yaml
   - conda-standalone >=24.7.1