Allow to load/store root owned directories via action; add slug(local…

…-dir) based namespacing to the remote storage directory

Pull Request: #23 (main)

.github/workflows/ci.yml

@@ -55,7 +55,7 @@ jobs:
       - name: Check that dummy.txt was restored
         run: |
           set -e
-          ls -la ~/storage-dir/${{ github.repository }}
+          ls -la ~/storage-dir/${{ github.repository }}/_
           [[ "$(cat dummy.txt)" == "dummy" ]] || { echo "dummy.txt must be restored"; exit 1; }
       - name: Remove layer.txt file and dir/subdir hierarchy
         run: rm -rf dir
@@ -68,7 +68,7 @@ jobs:
       - name: Check that dir/subdir/layer.txt was restored, and dummy.txt still exists
         run: |
           set -e
-          ls -la ~/storage-dir/${{ github.repository }}.my-layer
+          ls -la ~/storage-dir/${{ github.repository }}/_.my-layer
           [[ "$(cat dummy.txt)" == "dummy" ]] || { echo "dummy.txt must be kept"; exit 1; }
           [[ "$(cat dir/subdir/layer.txt)" == "layer" ]] || { echo "layer.txt must be restored"; exit 1; }
 

README.md

@@ -53,41 +53,48 @@ ones, so rsync can run efficiently.
     # Default: the content of ~/ci-storage-host file.
     storage-host: ''
 
-    # Storage directory on the storage host.
-    # Default: /mnt/{owner}/{repo}.
+    # Storage directory on the remote host. Notice that, when building the final
+    # directory on the storage host, owner and repo are always appended, so the
+    # path will be {storage-dir}/{owner}/{repo}/{slug(local-dir)} or
+    # {storage-dir}/{owner}/{repo}/{slug(local-dir)}.{layer-name}.
+    # Default: /mnt
     storage-dir: ''
 
     # Remove slots created earlier than this many seconds ago.
-    # Default: 14400 (4 hours)
+    # Default: 14400 (4 hours).
     storage-max-age-sec: ''
 
     # Id of the slot to store to or load from; use "*" to load a smart-random
     # slot (e.g. most recent or best in terms of layer compatibility) and skip
     # if it does not exist.
-    # Default: $GITHUB_RUN_ID
+    # Default: $GITHUB_RUN_ID (which is friendly to "Re-run failed jobs").
     slot-id: ''
 
-    # Local directory path to store from or load to.
-    # Default: "." (the current work directory)
+    # Local directory path to store from or load to. The value namespaces the
+    # data stored, so different local-dir values correspond to different
+    # storages. If the owner of the directory is different from the current
+    # user, then ci-storage tool is run with sudo, and the binary is used not
+    # from the action directory, but from /usr/bin/ci-storage.
+    # Default: "." (current work directory).
     local-dir: ''
 
     # Newline separated exclude pattern(s) for rsync.
-    # Default: empty
+    # Default: empty.
     exclude: ''
 
     # If set, the final directory on the storage host will be
-    # {storage-dir}/{owner}/{repo}.{layer-name}, plus deletion will be turned
-    # off on load.
-    # Default: empty
+    # {storage-dir}/{owner}/{repo}/{slug(local-dir)}.{layer-name},
+    # plus deletion will be turned off on load.
+    # Default: empty.
     layer-name: ''
 
     # Newline-separated include pattern(s) for rsync. If set, only the files
     # matching the patterns will be transferred. Implies setting layer-name.
-    # Default: empty
+    # Default: empty.
     layer-include: ''
 
     # If set, prints the list of transferred files.
-    # Default: false
+    # Default: false.
     verbose: ''
 ```
 <!-- end usage -->

action.yml

@@ -8,31 +8,31 @@ inputs:
     description: "What to do (store or load)."
     required: true
   storage-host:
-    description: "Storage host in the format [user@]host[:port]; it must allow password-free SSH key based access. If not passed, tries to read it from ~/ci-storage-host file."
+    description: "Storage host in the format [user@]host[:port]; it must allow password-free SSH key based access. Default: the content of ~/ci-storage-host file."
     required: false
   storage-dir:
-    description: "Storage directory on the remote host. If not set, uses /mnt. Notice that owner and repo are always appended, so the final directory will be {storage-dir}/{owner}/{repo} or {storage-dir}/{owner}/{repo}.{storage-namespace}."
+    description: "Storage directory on the remote host. Notice that, when building the final directory on the storage host, owner and repo are always appended, so the path will be {storage-dir}/{owner}/{repo}/{slug(local-dir)} or {storage-dir}/{owner}/{repo}/{slug(local-dir)}.{layer-name}. Default: /mnt"
     required: false
   storage-max-age-sec:
-    description: "Remove slots created earlier than this many seconds ago. If not set, uses the ci-storage tool default 4 hours."
+    description: "Remove slots created earlier than this many seconds ago. Default: 14400 (4 hours)."
     required: false
   slot-id:
-    description: 'Id of the slot to store to or load from; use "*" to load a smart-random slot (e.g. most recent or best in terms of layer compatibility) and skip if it does not exist. If empty, uses $GITHUB_RUN_ID value (which is friendly to "Re-run failed jobs").'
+    description: 'Id of the slot to store to or load from; use "*" to load a smart-random slot (e.g. most recent or best in terms of layer compatibility) and skip if it does not exist. Default: $GITHUB_RUN_ID (which is friendly to "Re-run failed jobs").'
     required: false
   local-dir:
-    description: 'Local directory path to store from or load to. If not set, uses "." (the current work directory).'
+    description: 'Local directory path to store from or load to. The value namespaces the data stored, so different local-dir values correspond to different storages. If the owner of the directory is different from the current user, then ci-storage tool is run with sudo, and the binary is used not from the action directory, but from /usr/bin/ci-storage. Default: "." (current work directory).'
     required: false
   exclude:
-    description: "Newline separated exclude pattern(s) for rsync."
+    description: "Newline separated exclude pattern(s) for rsync. Default: empty."
     required: false
   layer-name:
-    description: "If set, the final directory on the storage host will be {storage-dir}/{owner}/{repo}.{layer-name}, plus deletion will be turned off on load."
+    description: "If set, the final directory on the storage host will be {storage-dir}/{owner}/{repo}/{slug(local-dir)}.{layer-name}, plus deletion will be turned off on load. Default: empty."
     required: false
   layer-include:
-    description: "Newline-separated include pattern(s) for rsync. If set, only the files matching the patterns will be transferred. Implies setting layer-name."
+    description: "Newline-separated include pattern(s) for rsync. If set, only the files matching the patterns will be transferred. Implies setting layer-name. Default: empty."
     required: false
   verbose:
-    description: "If set, prints the list of transferred files."
+    description: "If set, prints the list of transferred files. Default: false."
     required: false
 runs:
   using: "composite"
@@ -68,22 +68,39 @@ runs:
           slot_id="$default_run_hash $slot_id"
         fi
 
+        if [[ "$local_dir" == "/" ]]; then
+          echo "You cannot use / as local-dir."
+          exit 1
+        elif [[ "$local_dir" == "." ]]; then
+          storage_dir="$storage_dir/_"
+        else
+          storage_dir="$storage_dir/$(echo "$local_dir" | tr / _)"
+        fi
+
         if [[ "$layer_name" != "" ]]; then
           layer_include=${layer_include:-"*"}
           storage_dir="$storage_dir.$layer_name"
         fi
+
         if [[ "$layer_include" != "" && "$layer_name" == "" ]]; then
           echo "When layer-include is used, you must also pass layer-name."
           exit 1
         fi
 
-        "${{ github.action_path }}/ci-storage" \
-          --storage-host="$storage_host" \
-          --storage-dir="$storage_dir" \
-          --storage-max-age-sec="$storage_max_age_sec" \
-          --slot-id="$slot_id" \
-          --local-dir="$local_dir" \
-          --exclude="$exclude" \
-          --layer="$layer_include" \
-          $verbose \
+        args=(
+          --storage-host="$storage_host"
+          --storage-dir="$storage_dir"
+          --storage-max-age-sec="$storage_max_age_sec"
+          --slot-id="$slot_id"
+          --local-dir="$local_dir"
+          --exclude="$exclude"
+          --layer="$layer_include"
+          $verbose
           "$action"
+        )
+
+        if [[ "$(stat -c '%U' "$local_dir")" == "$(whoami)" ]]; then
+          "${{ github.action_path }}/ci-storage" "${args[@]}"
+        else
+          sudo /usr/bin/ci-storage "${args[@]}"
+        fi

docker/ci-runner/Dockerfile

@@ -19,7 +19,7 @@ RUN true \
     && apt-get update -y \
     && apt-get install -y --no-install-recommends \
       openssh-client \
-      build-essential haproxy rinetd \
+      build-essential haproxy rinetd sudo \
       awscli jq gh rsync python3 python3-yaml rsyslog systemctl tzdata gosu less mc git curl wget pv psmisc unzip zstd file vim nano telnet net-tools apt-transport-https ca-certificates locales gnupg lsb-release
 
 # Install & patch Docker-in-Docker service. Requires sysbox installed on the host. See also:
@@ -56,6 +56,7 @@ RUN true \
     && curl --no-progress-meter -L https://github.com/actions/runner/releases/download/v$runner_version/actions-runner-$arch-$runner_version.tar.gz | tar xz \
     && date > .updated_at
 
+# Install OS dependencies needed by the action runner.
 USER root
 RUN true \
     && ~guest/actions-runner/bin/installdependencies.sh \

docker/ci-runner/guest/entrypoint.01-ci-storage-load.sh

@@ -18,7 +18,7 @@ if [[ "$CI_STORAGE_HOST" != "" && -f ~/.ssh/id_rsa ]]; then
   say "Running the initial \"ci-storage load\" for $local_dir..."
   ci-storage load \
     --storage-host="$CI_STORAGE_HOST" \
-    --storage-dir="$WORK_DIR/$GH_REPOSITORY" \
+    --storage-dir="$WORK_DIR/$GH_REPOSITORY/_" \
     --slot-id="*" \
     --local-dir="$local_dir" \
     & export CI_STORAGE_PID=$!

docker/ci-runner/root/entrypoint.20-sudo-ci-storage.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+# Allows to run /usr/bin/ci-storage with sudo.
+#
+set -u -e
+
+echo "guest ALL=(ALL) NOPASSWD: /usr/bin/ci-storage" > /etc/sudoers.d/ci-storage
+chmod 440 /etc/sudoers.d/ci-storage