Optimize environment provision times with pixi

.azure-pipelines/azure-pipelines-osx.yml

@@ -20,8 +20,7 @@ jobs:
     displayName: Run OSX build
     env:
       CI: azure
-      CONDA_BLD_PATH: /Users/runner/Miniforge3/conda-bld
-      MINIFORGE_HOME: /Users/runner/Miniforge3
+      CONDA_BLD_PATH: /Users/runner/bld
 
-  - publish: /Users/runner/Miniforge3/conda-bld/osx-64/
+  - publish: /Users/runner/bld/osx-64/
     artifact: conda_pkgs_osx

.ci_support/requirements.yaml

@@ -0,0 +1,13 @@
+name: staged-recipes
+channels:
+- conda-forge
+dependencies:
+- conda>=23.7.3
+- conda-libmamba-solver>=23.7.0
+- conda-build>=24.3
+- conda-index>=0.3.0
+- conda-forge-ci-setup>=4.9.3,<5.0
+- conda-forge-pinning
+- frozendict
+- networkx=2.4
+- rattler-build-conda-compat>=1.2.0,<2.0.0a0

.gitattributes

@@ -6,3 +6,5 @@
 *.yaml text eol=lf
 *.sh text eol=lf
 *.bat text eol=crlf
+# GitHub syntax highlighting
+pixi.lock linguist-language=YAML linguist-generated=true

.gitignore

@@ -7,6 +7,7 @@ build_artifacts
 # Compiled Python code
 __pycache__
 *.pyc
+*.egg-info
 
 # Editor files
 *.swp
@@ -19,5 +20,4 @@ __pycache__
 
 # pixi environments
 .pixi
-pixi.toml
 pixi.lock

.scripts/build_steps.sh

@@ -61,9 +61,24 @@ fi
 git ls-tree --name-only main -- . | xargs -I {} sh -c "rm -rf ~/staged-recipes-copy/recipes/{} && echo Removing recipe: {}"
 popd > /dev/null
 
-# Prevent permission errors in ~/.cache/conda
-export CONDA_NUMBER_CHANNEL_NOTICES=0
-conda install --quiet --file ${FEEDSTOCK_ROOT}/.ci_support/requirements.txt
+if ! command -v pixi >/dev/null 2>&1; then
+  echo "Installing pixi"
+  curl -fsSL https://pixi.sh/install.sh | bash
+  export PATH="~/.pixi/bin:$PATH"
+fi
+echo "Creating environment"
+pushd "${FEEDSTOCK_ROOT}"
+arch=$(uname -m)
+if [[ "$arch" == "x86_64" ]]; then
+  arch="64"
+fi
+sed -i.bak "s/platforms = .*/platforms = [\"linux-${arch}\"]/" pixi.toml
+PIXI_CACHE_DIR=/opt/conda pixi install
+pixi list
+mv pixi.toml.bak pixi.toml
+echo "Activating environment"
+eval "$(pixi shell-hook)"
+popd
 
 setup_conda_rc "${FEEDSTOCK_ROOT}" "/home/conda/staged-recipes-copy/recipes" "${CI_SUPPORT}/${CONFIG}.yaml"
 source run_conda_forge_build_setup

.scripts/run_osx_build.sh

@@ -4,32 +4,32 @@ set -x
 
 source .scripts/logging_utils.sh
 
-( startgroup "Provisioning base env with micromamba" ) 2> /dev/null
+( startgroup "Provisioning base env with pixi" ) 2> /dev/null
 
-MINIFORGE_HOME=${MINIFORGE_HOME:-${HOME}/miniforge3}
-MINIFORGE_HOME=${MINIFORGE_HOME%/} # remove trailing slash
+REPO_ROOT=$(dirname -- $(dirname -- "$(readlink -f -- "$BASH_SOURCE")"))
+MINIFORGE_HOME="${MINIFORGE_HOME:-${REPO_ROOT}/.pixi/envs/default}"
+CONDA_BLD_PATH="${CONDA_BLD_PATH:-${MINIFORGE_HOME}/conda-bld}"
 
-if [[ -d "${MINIFORGE_HOME}" ]]; then
-  echo "Miniforge already installed at ${MINIFORGE_HOME}."
-else
-  MICROMAMBA_VERSION="1.5.10-0"
-  if [[ "$(uname -m)" == "arm64" ]]; then
-    osx_arch="osx-arm64"
-  else
-    osx_arch="osx-64"
-  fi
-  MICROMAMBA_URL="https://github.com/mamba-org/micromamba-releases/releases/download/${MICROMAMBA_VERSION}/micromamba-${osx_arch}"
-  echo "Downloading micromamba ${MICROMAMBA_VERSION}"
-  micromamba_exe="$(mktemp -d)/micromamba"
-  curl -L -o "${micromamba_exe}" "${MICROMAMBA_URL}"
-  chmod +x "${micromamba_exe}"
-  echo "Creating environment"
-  "${micromamba_exe}" create --yes --root-prefix ~/.conda --prefix "${MINIFORGE_HOME}" \
-    --channel conda-forge \
-    --file .ci_support/requirements.txt
+if ! command -v pixi >/dev/null 2>&1; then
+  echo "Installing pixi"
+  curl -fsSL https://pixi.sh/install.sh | bash
+  export PATH="~/.pixi/bin:$PATH"
+fi
+echo "Creating environment"
+pushd "$REPO_ROOT"
+arch=$(uname -m)
+if [[ "$arch" == "x86_64" ]]; then
+  arch="64"
 fi
+sed -i.bak "s/platforms = .*/platforms = [\"osx-${arch}\"]/" pixi.toml
+pixi install
+pixi list
+mv pixi.toml.bak pixi.toml
+echo "Activating environment"
+eval "$(pixi shell-hook)"
+popd
 
-( endgroup "Provisioning base env with micromamba" ) 2> /dev/null
+( endgroup "Provisioning base env with pixi" ) 2> /dev/null
 
 ( startgroup "Configuring conda" ) 2> /dev/null
 
@@ -39,9 +39,6 @@ show_channel_urls: true
 solver: libmamba
 CONDARC
 
-source "${MINIFORGE_HOME}/etc/profile.d/conda.sh"
-conda activate base
-
 echo -e "\n\nSetting up the condarc and mangling the compiler."
 setup_conda_rc ./ ./recipes ./.ci_support/${CONFIG}.yaml
 if [[ "${CI:-}" != "" ]]; then
@@ -62,7 +59,7 @@ source run_conda_forge_build_setup
 set -e
 
 # make sure there is a package directory so that artifact publishing works
-mkdir -p "${MINIFORGE_HOME}/conda-bld/osx-64/" "${MINIFORGE_HOME}/conda-bld/noarch/"
+mkdir -p "${CONDA_BLD_PATH}/osx-64/" "${CONDA_BLD_PATH}/noarch/"
 
 # Find the recipes from main in this PR and remove them.
 echo ""

.scripts/run_win_build.bat

@@ -10,32 +10,52 @@
 
 setlocal enableextensions enabledelayedexpansion
 
-if "%MINIFORGE_HOME%"=="" set "MINIFORGE_HOME=%USERPROFILE%\Miniforge3"
+call :start_group "Ensuring conda"
+
+FOR %%A IN ("%~dp0.") DO SET "REPO_ROOT=%%~dpA"
+if "%MINIFORGE_HOME%"=="" (
+    set "MINIFORGE_HOME=%REPO_ROOT%\.pixi\envs\default"
+) else (
+    set "PIXI_CACHE_DIR=%MINIFORGE_HOME%"
+)
 :: Remove trailing backslash, if present
 if "%MINIFORGE_HOME:~-1%"=="\" set "MINIFORGE_HOME=%MINIFORGE_HOME:~0,-1%"
-call :start_group "Provisioning base env with micromamba"
-set "MAMBA_ROOT_PREFIX=%MINIFORGE_HOME%-micromamba-%RANDOM%"
-set "MICROMAMBA_VERSION=1.5.10-0"
-set "MICROMAMBA_URL=https://github.com/mamba-org/micromamba-releases/releases/download/%MICROMAMBA_VERSION%/micromamba-win-64"
-set "MICROMAMBA_TMPDIR=%TMP%\micromamba-%RANDOM%"
-set "MICROMAMBA_EXE=%MICROMAMBA_TMPDIR%\micromamba.exe"
-
-echo Downloading micromamba %MICROMAMBA_VERSION%
-if not exist "%MICROMAMBA_TMPDIR%" mkdir "%MICROMAMBA_TMPDIR%"
-certutil -urlcache -split -f "%MICROMAMBA_URL%" "%MICROMAMBA_EXE%"
-if !errorlevel! neq 0 exit /b !errorlevel!
+
+WHERE /Q pixi && set "_install_pixi=no" || set "_install_pixi=yes"
+if "%_install_pixi%"=="yes" (
+    echo Installing pixi
+    powershell -NoProfile -ExecutionPolicy unrestricted -Command "iwr -useb https://pixi.sh/install.ps1 | iex"
+    if !errorlevel! neq 0 exit /b !errorlevel!
+    set "PATH=%USERPROFILE%\.pixi\bin;%PATH%"
+)
 
 echo Creating environment
-call "%MICROMAMBA_EXE%" create --yes --root-prefix "%MAMBA_ROOT_PREFIX%" --prefix "%MINIFORGE_HOME%" ^
-    --channel conda-forge ^
-    --file .ci_support\requirements.txt
+if "%PIXI_CACHE_DIR%"=="%MINIFORGE_HOME%" (
+    mkdir "%MINIFORGE_HOME%"
+    copy /Y pixi.toml "%MINIFORGE_HOME%"
+    pushd "%MINIFORGE_HOME%"
+) else (
+    pushd "%REPO_ROOT%"
+)
+move /y pixi.toml pixi.toml.bak
+set "arch=64"
+if "%PROCESSOR_ARCHITECTURE%"=="ARM64" set "arch=arm64"
+powershell -NoProfile -ExecutionPolicy unrestricted -Command "(Get-Content pixi.toml.bak -Encoding UTF8) -replace 'platforms = .*', 'platforms = [''win-%arch%'']' | Out-File -Encoding UTF8 pixi.toml"
+pixi install
 if !errorlevel! neq 0 exit /b !errorlevel!
-echo Moving pkgs cache from %MAMBA_ROOT_PREFIX% to %MINIFORGE_HOME%
-move /Y "%MAMBA_ROOT_PREFIX%\pkgs" "%MINIFORGE_HOME%"
+echo Listing environment
+pixi list
 if !errorlevel! neq 0 exit /b !errorlevel!
-echo Removing %MAMBA_ROOT_PREFIX%
-del /S /Q "%MAMBA_ROOT_PREFIX%"
-del /S /Q "%MICROMAMBA_TMPDIR%"
+:: Activate the base conda environment
+echo Activating environment
+set "ACTIVATE_PIXI=%TMP%\pixi-activate-%RANDOM%.bat"
+pixi shell-hook > %ACTIVATE_PIXI%
+if !errorlevel! neq 0 exit /b !errorlevel!
+call %ACTIVATE_PIXI%
+if !errorlevel! neq 0 exit /b !errorlevel!
+move /y pixi.toml.bak pixi.toml
+popd
+
 call :end_group
 
 call :start_group "Configuring conda"
@@ -44,10 +64,6 @@ if "%CONDA_BLD_PATH%" == "" (
     set "CONDA_BLD_PATH=C:\bld"
 )
 
-:: Activate the base conda environment
-echo Activating "%MINIFORGE_HOME%"
-call "%MINIFORGE_HOME%\Scripts\activate"
-
 :: Set basic configuration
 echo Setting up configuration
 conda.exe config --set always_yes yes
@@ -65,8 +81,10 @@ call run_conda_forge_build_setup
 if !errorlevel! neq 0 exit /b !errorlevel!
 
 echo Force fetch origin/main
+set GIT_SSL_NO_VERIFY=true
 git fetch --force origin main:main
 if !errorlevel! neq 0 exit /b !errorlevel!
+set "GIT_SSL_NO_VERIFY="
 echo Removing recipes also present in main
 cd recipes
 for /f "tokens=*" %%a in ('git ls-tree --name-only main -- .') do rmdir /s /q %%a && echo Removing recipe: %%a

pixi.toml

@@ -0,0 +1,21 @@
+[project]
+name = "staged-recipes"
+description = "The entry point to conda-forge"
+version = "0.1.0"
+authors = ["@conda-forge/core"]
+channels = ["conda-forge"]
+platforms = ["linux-64", "osx-64", "osx-arm64", "win-64"]
+
+[tasks]
+build-locally = "python build-locally.py"
+
+[dependencies]
+conda = ">=23.7.3"
+conda-libmamba-solver = ">=23.7.0"
+conda-build = ">=24.3"
+conda-index = ">=0.3.0"
+conda-forge-ci-setup = ">=4.9.3,<5.0"
+conda-forge-pinning = "*"
+frozendict = "*"
+networkx = "2.4.*"
+rattler-build-conda-compat = ">=1.2.0,<2.0.0a0"