Merge pull request #965 from dholt/slurm-epi-tmp-fix

Improved cleanup in Slurm epilog
NVIDIA · Jun 16, 2021 · 3d3e0dd · 3d3e0dd
2 parents 291b212 + 22c570a
commit 3d3e0dd
Show file tree

Hide file tree

Showing 18 changed files with 20 additions and 20 deletions.
diff --git a/docs/slurm-cluster/README.md b/docs/slurm-cluster/README.md
@@ -70,6 +70,11 @@ Now that Slurm is installed, try a ["Hello World" example using MPI](../../workl
 
 Read through the [slurm usage guide](slurm-usage.md) and [Open OnDemand guide](ood.md) for more information.
 
+## Prolog and Epilog
+
+The default Slurm deployment includes a collection of prolog and epilog scripts that should be modified to suit a particular system.
+For more information, see the [prolog/epilog documentation](slurm-prolog-epilog/README.md).
+
 ## Node Health Check
 
 The default Slurm deployment includes setting up [Node Health Check](https://github.com/mej/nhc).

diff --git a/...slurm/files/slurm-prolog-epilog/README.md → ...urm-cluster/slurm-prolog-epilog/README.md b/...slurm/files/slurm-prolog-epilog/README.md → ...urm-cluster/slurm-prolog-epilog/README.md
diff --git a/...iles/slurm-prolog-epilog/epilog-dcgmstats → ...ster/slurm-prolog-epilog/epilog-dcgmstats b/...iles/slurm-prolog-epilog/epilog-dcgmstats → ...ster/slurm-prolog-epilog/epilog-dcgmstats
diff --git a/...lurm/files/slurm-prolog-epilog/epilog-ecc → ...rm-cluster/slurm-prolog-epilog/epilog-ecc b/...lurm/files/slurm-prolog-epilog/epilog-ecc → ...rm-cluster/slurm-prolog-epilog/epilog-ecc
diff --git a/...lurm/files/slurm-prolog-epilog/epilog-mps → ...rm-cluster/slurm-prolog-epilog/epilog-mps b/...lurm/files/slurm-prolog-epilog/epilog-mps → ...rm-cluster/slurm-prolog-epilog/epilog-mps
diff --git a/...les/slurm-prolog-epilog/hyperthreadingoff → ...ter/slurm-prolog-epilog/hyperthreadingoff b/...les/slurm-prolog-epilog/hyperthreadingoff → ...ter/slurm-prolog-epilog/hyperthreadingoff
diff --git a/...iles/slurm-prolog-epilog/hyperthreadingon → ...ster/slurm-prolog-epilog/hyperthreadingon b/...iles/slurm-prolog-epilog/hyperthreadingon → ...ster/slurm-prolog-epilog/hyperthreadingon
diff --git a/...es/slurm-prolog-epilog/prolog-checkmounts → ...er/slurm-prolog-epilog/prolog-checkmounts b/...es/slurm-prolog-epilog/prolog-checkmounts → ...er/slurm-prolog-epilog/prolog-checkmounts
diff --git a/...les/slurm-prolog-epilog/prolog-dcgmhealth → ...ter/slurm-prolog-epilog/prolog-dcgmhealth b/...les/slurm-prolog-epilog/prolog-dcgmhealth → ...ter/slurm-prolog-epilog/prolog-dcgmhealth
diff --git a/...iles/slurm-prolog-epilog/prolog-dcgmstats → ...ster/slurm-prolog-epilog/prolog-dcgmstats b/...iles/slurm-prolog-epilog/prolog-dcgmstats → ...ster/slurm-prolog-epilog/prolog-dcgmstats
diff --git a/...lurm/files/slurm-prolog-epilog/prolog-ecc → ...rm-cluster/slurm-prolog-epilog/prolog-ecc b/...lurm/files/slurm-prolog-epilog/prolog-ecc → ...rm-cluster/slurm-prolog-epilog/prolog-ecc
diff --git a/...files/slurm-prolog-epilog/prolog-gpureset → ...uster/slurm-prolog-epilog/prolog-gpureset b/...files/slurm-prolog-epilog/prolog-gpureset → ...uster/slurm-prolog-epilog/prolog-gpureset
diff --git a/...rm/files/slurm-prolog-epilog/prolog-lspci → ...-cluster/slurm-prolog-epilog/prolog-lspci b/...rm/files/slurm-prolog-epilog/prolog-lspci → ...-cluster/slurm-prolog-epilog/prolog-lspci
diff --git a/...lurm/files/slurm-prolog-epilog/prolog-mps → ...rm-cluster/slurm-prolog-epilog/prolog-mps b/...lurm/files/slurm-prolog-epilog/prolog-mps → ...rm-cluster/slurm-prolog-epilog/prolog-mps
diff --git a/...es/slurm-prolog-epilog/prolog-mps-per-gpu → ...er/slurm-prolog-epilog/prolog-mps-per-gpu b/...es/slurm-prolog-epilog/prolog-mps-per-gpu → ...er/slurm-prolog-epilog/prolog-mps-per-gpu
diff --git a/roles/slurm/templates/etc/slurm/epilog.d/42-lastuserjob-cleanup b/roles/slurm/templates/etc/slurm/epilog.d/42-lastuserjob-cleanup
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -ex
+
+if [[ -n "$SLURM_JOB_USER" && "$SLURM_JOB_USER" != "root" ]]; then
+    logger -s -t slurm-epilog 'Removed residual user files'
+    for dir in /tmp /dev/shm ; do
+	find "${dir}" -user "${SLURM_JOB_USER}" -print0 | xargs -0 -r rm -fr ||:
+    done
+fi
diff --git a/roles/slurm/templates/etc/slurm/epilog.d/80-exclusive-cleanup b/roles/slurm/templates/etc/slurm/epilog.d/80-exclusive-cleanup
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -ex
+
+# Cleanup caches
+sync
+echo 3 > /proc/sys/vm/drop_caches
diff --git a/roles/slurm/templates/etc/slurm/epilog.d/80-exclusive-tmpfiles b/roles/slurm/templates/etc/slurm/epilog.d/80-exclusive-tmpfiles