Skip to content

Commit

Permalink
Use correct GHA workflow syntax in new cache-clearing action, be more…
Browse files Browse the repository at this point in the history 
… resilient (#20756)

In #20755, I had a typo: formatting the jobs as a list, with `name`s
keys... but that's the syntax for steps. Jobs need to be a dict with the
ID as the key.

This also squashes any errors from `rm -rf ....`: on Mac, it seems that
attempts to delete directories that the runner doesn't have permission
for (particularly system ones in `~/Library/Caches`). But that's fine,
we'll still delete all of the caches created by "day-to-day" actions
runs, which are the big ones.

(Why wasn't this caught in #20755? I attempted to run the workflow
before merging, and got errors, but incorrectly assumed this was because
the workflow didn't exist on `main`, but they were actually because it's
invalid.)

This has now successfully run and cleared hundreds of gigs over two
runs:

- https://github.com/pantsbuild/pants/actions/runs/8562624080 (required
the "squash `rm` errors fix)
-
https://github.com/pantsbuild/pants/actions/runs/8562947394/job/23467155450
(with that fix).
  • Loading branch information
@huonw
huonw authored Apr 5, 2024
1 parent 9897614 commit 15fd1e1
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 64 deletions.
116 changes: 58 additions & 58 deletions .github/workflows/clear_self_hosted_persistent_caches.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,64 +4,64 @@


jobs:
- name: clear_linux_arm64
runs-on:
- self-hosted
- Linux
- ARM64
steps:
- name: df before
run: df -h
- name: Deleting ~/Library/Caches
run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches
- name: Deleting ~/.cache
run: du -sh ~/.cache || true; rm -rf ~/.cache
- name: Deleting ~/.nce
run: du -sh ~/.nce || true; rm -rf ~/.nce
- name: Deleting ~/.rustup
run: du -sh ~/.rustup || true; rm -rf ~/.rustup
- name: Deleting ~/.pex
run: du -sh ~/.pex || true; rm -rf ~/.pex
- name: df after
run: df -h
- name: clear_macos10_15_x86_64
runs-on:
- self-hosted
- macOS-10.15-X64
steps:
- name: df before
run: df -h
- name: Deleting ~/Library/Caches
run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches
- name: Deleting ~/.cache
run: du -sh ~/.cache || true; rm -rf ~/.cache
- name: Deleting ~/.nce
run: du -sh ~/.nce || true; rm -rf ~/.nce
- name: Deleting ~/.rustup
run: du -sh ~/.rustup || true; rm -rf ~/.rustup
- name: Deleting ~/.pex
run: du -sh ~/.pex || true; rm -rf ~/.pex
- name: df after
run: df -h
- name: clear_macos11_arm64
runs-on:
- self-hosted
- macOS-11-ARM64
steps:
- name: df before
run: df -h
- name: Deleting ~/Library/Caches
run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches
- name: Deleting ~/.cache
run: du -sh ~/.cache || true; rm -rf ~/.cache
- name: Deleting ~/.nce
run: du -sh ~/.nce || true; rm -rf ~/.nce
- name: Deleting ~/.rustup
run: du -sh ~/.rustup || true; rm -rf ~/.rustup
- name: Deleting ~/.pex
run: du -sh ~/.pex || true; rm -rf ~/.pex
- name: df after
run: df -h
clean_linux_arm64:
runs-on:
- self-hosted
- Linux
- ARM64
steps:
- name: df before
run: df -h
- name: Deleting ~/Library/Caches
run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches || true
- name: Deleting ~/.cache
run: du -sh ~/.cache || true; rm -rf ~/.cache || true
- name: Deleting ~/.nce
run: du -sh ~/.nce || true; rm -rf ~/.nce || true
- name: Deleting ~/.rustup
run: du -sh ~/.rustup || true; rm -rf ~/.rustup || true
- name: Deleting ~/.pex
run: du -sh ~/.pex || true; rm -rf ~/.pex || true
- name: df after
run: df -h
clean_macos10_15_x86_64:
runs-on:
- self-hosted
- macOS-10.15-X64
steps:
- name: df before
run: df -h
- name: Deleting ~/Library/Caches
run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches || true
- name: Deleting ~/.cache
run: du -sh ~/.cache || true; rm -rf ~/.cache || true
- name: Deleting ~/.nce
run: du -sh ~/.nce || true; rm -rf ~/.nce || true
- name: Deleting ~/.rustup
run: du -sh ~/.rustup || true; rm -rf ~/.rustup || true
- name: Deleting ~/.pex
run: du -sh ~/.pex || true; rm -rf ~/.pex || true
- name: df after
run: df -h
clean_macos11_arm64:
runs-on:
- self-hosted
- macOS-11-ARM64
steps:
- name: df before
run: df -h
- name: Deleting ~/Library/Caches
run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches || true
- name: Deleting ~/.cache
run: du -sh ~/.cache || true; rm -rf ~/.cache || true
- name: Deleting ~/.nce
run: du -sh ~/.nce || true; rm -rf ~/.nce || true
- name: Deleting ~/.rustup
run: du -sh ~/.rustup || true; rm -rf ~/.rustup || true
- name: Deleting ~/.pex
run: du -sh ~/.pex || true; rm -rf ~/.pex || true
- name: df after
run: df -h
name: Clear persistent caches on long-lived self-hosted runners
'on':
workflow_dispatch: {}
16 changes: 10 additions & 6 deletions src/python/pants_release/generate_github_workflows.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1570,14 +1570,19 @@ def gen_goals(use_default_version: bool) -> Sequence[object]:
return PublicReposOutput(jobs=jobs, inputs=inputs, run_name=run_name)


def clear_self_hosted_persistent_caches_jobs() -> list[dict[str, Any]]:
def make_job(platform: Platform) -> dict[str, Any]:
def clear_self_hosted_persistent_caches_jobs() -> Jobs:
jobs = {}

for platform in sorted(SELF_HOSTED, key=lambda p: p.value):
helper = Helper(platform)

clear_steps = [
{
"name": f"Deleting {directory}",
"run": f"du -sh {directory} || true; rm -rf {directory}",
# squash all errors: this is a best effort thing, so, for instance, it's fine if
# there's directories hanging around that this workflow doesn't have permission to
# delete
"run": f"du -sh {directory} || true; rm -rf {directory} || true",
}
for directory in [
# not all of these will necessarily exist (e.g. ~/Library/Caches is macOS-specific),
Expand All @@ -1589,8 +1594,7 @@ def make_job(platform: Platform) -> dict[str, Any]:
"~/.pex",
]
]
return {
"name": helper.job_name("clear"),
jobs[helper.job_name("clean")] = {
"runs-on": helper.runs_on(),
"steps": [
{"name": "df before", "run": "df -h"},
Expand All @@ -1599,7 +1603,7 @@ def make_job(platform: Platform) -> dict[str, Any]:
],
}

return [make_job(platform) for platform in sorted(SELF_HOSTED, key=lambda p: p.value)]
return jobs


# ----------------------------------------------------------------------
Expand Down

0 comments on commit 15fd1e1

Please sign in to comment.