Cleaner CI conda env setup | Selective weekly dependabot updates

.github/actions/cleanup-conda/action.yml

@@ -6,6 +6,7 @@ runs:
   steps:
     - name: Remove extra conda environments
       run: |
+        conda env list
         CONDA_REMOVE_NAMES=$(conda env list | awk '{print $1}' | tail -n +3 | grep "${{ env.conda-env-name-no-time }}" || true)
         if [ -z "$CONDA_REMOVE_NAMES" ]; then
           echo "No matching conda environments for ${{ env.conda-env-name-no-time }}. Skip removal."

.github/actions/create-conda-env/action.yml

@@ -5,24 +5,20 @@ runs:
   steps:
     - name: Create conda environments
       run: |
+        conda env list
         if conda env list | grep -q "envs/${{ env.conda-env-name-no-time }}"; then
           echo "Using pre-existing conda environments with prefix ${{ env.conda-env-name-no-time }}"
         else
           echo "Creating a conda environment for each toolchain with the toolchain installed"
-          conda activate base
-          conda-lock install --conda $(which conda) -n ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools ./conda-reqs/conda-lock-reqs/conda-requirements-riscv-tools-linux-64.conda-lock.yml
-          conda deactivate
-
-          echo "Add extra toolchain collateral + CIRCT to RISC-V install area"
-          git submodule update --init ./tools/install-circt
-          conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools
-          ./scripts/build-toolchain-extra.sh riscv-tools -p $CONDA_PREFIX/riscv-tools
-          ./tools/install-circt/bin/download-release-or-nightly-circt.sh \
-              -f circt-full-shared-linux-x64.tar.gz \
-              -i $CONDA_PREFIX \
-              -v version-file \
-              -x ./conda-reqs/circt.json \
-              -g ${{ github.token }}
-          conda deactivate
+          ./build-setup.sh \
+            --conda-env-name ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d) \
+            --github-token ${{ github.token }} \
+            --verbose \
+            --skip-submodules \
+            --skip-ctags \
+            --skip-precompile \
+            --skip-firesim \
+            --skip-marshal \
+            --skip-clean
         fi
       shell: bash -leo pipefail {0}

.github/actions/prepare-rtl/action.yml

@@ -13,10 +13,6 @@ inputs:
     description: type of build
     required: false
     default: "sim"
-  toolchain:
-    description: toolchain to use
-    required: false
-    default: "riscv-tools"
 
 runs:
   using: "composite"
@@ -32,7 +28,7 @@ runs:
 
     - name: Run RTL build if not cached
       run: |
-        conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-${{ inputs.toolchain }}
+        conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)
         if [[ "${{ steps.rtl-build-id.outputs.cache-hit }}" != 'true' ]]; then
           echo "Cache miss on ${{ inputs.group-key }}-${{ github.sha }}"
           ./.github/scripts/${{ inputs.build-script }} ${{ inputs.group-key }} ${{ inputs.build-type }}

.github/actions/run-tests/action.yml

@@ -12,17 +12,13 @@ inputs:
     description: rtl build script to use
     required: false
     default: "run-tests.sh"
-  toolchain:
-    description: toolchain to use
-    required: false
-    default: "riscv-tools"
 
 runs:
   using: "composite"
   steps:
     - name: Init submodules (since only the RTL is cached)
       run: |
-        conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-${{ inputs.toolchain }}
+        conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)
         ./scripts/init-submodules-no-riscv-tools.sh
       shell: bash -leo pipefail {0}
 
@@ -31,10 +27,9 @@ runs:
       uses: ./.github/actions/prepare-rtl
       with:
         group-key: ${{ inputs.group-key }}
-        toolchain: ${{ inputs.toolchain }}
 
     - name: Run RTL tests
       run: |
-        conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-${{ inputs.toolchain }}
+        conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)
         ./.github/scripts/${{ inputs.run-script }} ${{ inputs.project-key }}
       shell: bash -leo pipefail {0}

.github/dependabot.yml

@@ -1,28 +1,31 @@
 version: 2
 
+# update weekly (to avoid spam)
 updates:
-  # submit a PR for bumping submodules daily (for selected submodules).
+  # bump submodules.
   # updates a submodule to the latest commit on the branch given in .gitmodules.
   # if branch not given, then it defaults to the main/master branch.
   - package-ecosystem: gitsubmodule
     schedule:
-      interval: "daily"
+      interval: "weekly"
     directory: /
     labels:
       - "dependencies"
       - "changelog:omit"
-    # ignore certain submodules
-    ignore:
-      - dependency-name: "generators/gemmini"
-      - dependency-name: "generators/rocket-chip"
-      - dependency-name: "generators/rocket-chip-blocks"
-      - dependency-name: "generators/rocket-chip-inclusive-cache"
+    # allow certain submodules
+    allow:
+      - dependency-name: "sims/firesim"
+      - dependency-name: "software/baremetal-ide"
+      - dependency-name: "software/firemarshal"
+      - dependency-name: "toolchains/riscv-tools/riscv-isa-sim"
+      - dependency-name: "toolchains/riscv-tools/riscv-openocd"
+      - dependency-name: "toolchains/riscv-tools/riscv-pk"
+      - dependency-name: "toolchains/riscv-tools/riscv-spike-devices"
       - dependency-name: "toolchains/riscv-tools/riscv-tests"
-      - dependency-name: "toolchains/riscv-tools/riscv-tools-feedstock"
   # submit a PR for bumping GH-A dependencies
   - package-ecosystem: github-actions
     schedule:
-      interval: "daily"
+      interval: "weekly"
     directory: /
     labels:
       - "dependencies"

.github/workflows/chipyard-run-tests.yml

@@ -114,7 +114,7 @@ jobs:
         uses: ./.github/actions/create-conda-env
       - name: Check commits of each submodule
         run: |
-            conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools
+            conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)
             .github/scripts/check-commit.sh
 
   documentation-check:
@@ -136,7 +136,7 @@ jobs:
         uses: ./.github/actions/create-conda-env
       - name: Check that documentation builds with no warnings/errors
         run: |
-          conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools
+          conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)
           make -C docs html
       - name: Show error log from sphinx if failed
         if: ${{ failure() }}
@@ -172,7 +172,7 @@ jobs:
           restore-keys: ${{ steps.genkey.outputs.extra-tests-cache-key }}
       - name: Build extra tests
         run: |
-            conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools
+            conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)
             .github/scripts/build-extra-tests.sh
 
   prepare-chipyard-cores:
@@ -351,14 +351,14 @@ jobs:
         uses: ./.github/actions/create-conda-env
       - name: Build Gemmini FireMarshal
         run: |
-          conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools
+          conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)
           cd ${{ github.workspace }} && ./scripts/init-submodules-no-riscv-tools.sh
           cd ${{ github.workspace }} && source ./scripts/fix-open-files.sh
           git submodule update --init software/firemarshal && cd software/firemarshal && ./init-submodules.sh
           cd ${{ github.workspace }}/generators/gemmini/software && ${{ github.workspace }}/software/firemarshal/marshal -v -d build gemmini-smoke.json
       - name: Running Gemmini FireMarshal smoke test
         run: |
-          conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)-riscv-tools
+          conda activate ${{ env.conda-env-name-no-time }}-$(date --date "${{ env.workflow-timestamp }}" +%Y%m%d)
           cd ${{ github.workspace }}/generators/gemmini/software && ${{ github.workspace }}/software/firemarshal/marshal -v -d launch -s gemmini-smoke.json
 
   chipyard-rocket-run-tests:

.github/workflows/update-circt.yml

@@ -1,10 +1,10 @@
 name: update-circt
 
-# run daily
+# TODO: update weekly (to avoid spam)
 on:
   schedule:
     - cron: 0,15,30,45 * * * *
-    #- cron: 0 8 * * *
+    #- cron: 0 0 * * 1
 
 defaults:
   run:
@@ -23,3 +23,4 @@ jobs:
           pr-reviewers: ''
           pr-labels: 'changelog:changed'
           circt-config: './conda-reqs/circt.json'
+          github-token: ${{ secrets.BARTENDER_PERSONAL_ACCESS_TOKEN }}

scripts/build-setup.sh

@@ -37,8 +37,10 @@ usage() {
     echo "  --help -h               : Display this message"
     echo "  --verbose -v            : Verbose printout"
     echo "  --use-unpinned-deps -ud : Use unpinned conda environment"
-    echo "  --use-lean-conda        : Install a leaner version of the repository (Smaller conda env, no FireSim, no FireMarshal)"
+    echo "  --use-lean-conda        : Install a leaner version of the repository (smaller conda env, no FireSim, no FireMarshal)"
     echo "  --build-circt           : Builds CIRCT from source, instead of downloading the precompiled binary"
+    echo "  --conda-env-name NAME   : Optionally use a global conda env name instead of storing env in Chipyard directory"
+    echo "  --github-token TOKEN    : Optionally use a Github token to download CIRCT"
 
     echo "  --skip -s N             : Skip step N in the list above. Use multiple times to skip multiple steps ('-s N -s M ...')."
     echo "  --skip-conda            : Skip Conda initialization (step 1)"
@@ -61,6 +63,8 @@ USE_UNPINNED_DEPS=false
 USE_LEAN_CONDA=false
 SKIP_LIST=()
 BUILD_CIRCT=false
+GLOBAL_ENV_NAME=""
+GITHUB_TOKEN="null"
 
 # getopts does not support long options, and is inflexible
 while [ "$1" != "" ];
@@ -78,6 +82,12 @@ do
             SKIP_LIST+=(4 6 7 8 9) ;;
         --build-circt)
             BUILD_CIRCT=true ;;
+        --conda-env-name)
+            shift
+            GLOBAL_ENV_NAME=${1} ;;
+        --github-token)
+            shift
+            GITHUB_TOKEN=${1} ;;
         -ud | --use-unpinned-deps )
             USE_UNPINNED_DEPS=true ;;
         --skip | -s)
@@ -154,24 +164,35 @@ if run_step "1"; then
 
     # create conda-lock only environment to be used in this section.
     # done with cloning base then installing conda lock to speed up dependency solving.
-    rm -rf $CYDIR/.conda-lock-env &&
-    conda create -y -p $CYDIR/.conda-lock-env --clone base &&
+    CONDA_LOCK_ENV_PATH=$CYDIR/.conda-lock-env
+    rm -rf $CONDA_LOCK_ENV_PATH &&
+    conda create -y -p $CONDA_LOCK_ENV_PATH --clone base &&
     source $(conda info --base)/etc/profile.d/conda.sh &&
-    conda activate $CYDIR/.conda-lock-env &&
-    conda install -y -c conda-forge -p $CYDIR/.conda-lock-env $(grep "conda-lock" $CONDA_REQS/chipyard-base.yaml | sed 's/^ \+-//')
+    conda activate $CONDA_LOCK_ENV_PATH  &&
+    conda install -y -c conda-forge -p $CONDA_LOCK_ENV_PATH $(grep "conda-lock" $CONDA_REQS/chipyard-base.yaml | sed 's/^ \+-//')
     exit_if_last_command_failed
 
     if [ "$USE_UNPINNED_DEPS" = true ]; then
         # auto-gen the lockfiles
         $CYDIR/scripts/generate-conda-lockfiles.sh
         exit_if_last_command_failed
     fi
-    echo "Using lockfile: $LOCKFILE"
+    echo "Using lockfile for conda: $LOCKFILE"
 
     # use conda-lock to create env
-    conda-lock install --conda $(which conda) -p $CYDIR/.conda-env $LOCKFILE &&
+    if [ -z "$GLOBAL_ENV_NAME" ] ; then
+        CONDA_ENV_PATH=$CYDIR/.conda-env
+        CONDA_ENV_ARG="-p $CONDA_ENV_PATH"
+        CONDA_ENV_NAME=$CONDA_ENV_PATH
+    else
+        CONDA_ENV_ARG="-n $GLOBAL_ENV_NAME"
+        CONDA_ENV_NAME=$GLOBAL_ENV_NAME
+    fi
+    echo "Storing main conda environment in $CONDA_ENV_NAME"
+
+    conda-lock install --conda $(which conda) $CONDA_ENV_ARG $LOCKFILE &&
     source $(conda info --base)/etc/profile.d/conda.sh &&
-    conda activate $CYDIR/.conda-env
+    conda activate $CONDA_ENV_NAME
     exit_if_last_command_failed
 
     # Conda Setup
@@ -191,7 +212,7 @@ END_CONDA_ACTIVATE
 
     replace_content env.sh build-setup-conda "# line auto-generated by $0
 $CONDA_ACTIVATE_PREAMBLE
-conda activate $CYDIR/.conda-env
+conda activate $CONDA_ENV_NAME
 source $CYDIR/scripts/fix-open-files.sh"
 
 fi
@@ -308,7 +329,7 @@ if run_step "10"; then
 		-i $PREFIX \
 		-v version-file \
 		-x $CYDIR/conda-reqs/circt.json \
-		-g null
+		-g $GITHUB_TOKEN
     fi
     exit_if_last_command_failed
 fi