Simplify and cleanup entry points and tests

.github/workflows/ci.yml

@@ -16,19 +16,19 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Create dummy file
-        run: echo "dummy" > dummy.txt
+        run: |
+          echo "dummy" > dummy.txt
+          echo -n "" > ~/ci-storage-host
       - name: Test store
         uses: ./
         with:
           action: "store"
-          storage-host: ""
       - name: Remove dummy file
         run: rm dummy.txt
       - name: Test load
         uses: ./
         with:
           action: "load"
-          storage-host: ""
       - name: Check that dummy file was restored
         run: |
           set -e
@@ -48,20 +48,21 @@ jobs:
         uses: actions/checkout@v2
       - name: Start test Docker containers
         run: |
-          ssh-keygen -t ed25519 -qf /tmp/key -N ""
-          export CI_STORAGE_HOST_SSH_KEY="$(cat /tmp/key)"
-          set -o xtrace
+          exec 2>&1; set -o xtrace
           cd docker
-          # Boot ci-storage-host container. It will have a default empty slot.
-          docker compose up ci-storage-host -d --build
+          # Build all containers.
+          docker compose build --parallel
+          # Boot ci-storage-host container in background.
+          docker compose up ci-storage-host -d
           # Now boot self-hosted-runner container. It will connect to
-          # ci-storage-host container and load the empty slot from there, then
-          # register a GitHub self-hosted runner and remain waiting for jobs.
-          docker compose up self-hosted-runner --build
+          # ci-storage-host container and load a test (non-existent) ci-storage
+          # slot from there, then register a GitHub self-hosted runner and
+          # remain waiting for jobs.
+          docker compose up self-hosted-runner
         env:
           GH_REPOSITORY: ${{ github.repository }}
           GH_LABELS: ci-storage-test
-          GH_TOKEN: ${{ secrets.CI_PAT  }}
+          GH_TOKEN: ${{ secrets.CI_PAT }}
 
   # The test job with ci-storage-test tag which is initially queued, but then is
   # picked up by the self-hosted-runner container booted in the previous job. In
@@ -74,12 +75,9 @@ jobs:
         uses: actions/checkout@v2
       - name: Run test job inside the self-hosted runner
         run: echo "Hello, world!"
-      - name: Test store
+      - name: Test store using GitHub Action
         uses: ./
         with:
           action: "store"
-          storage-host: "ci-storage-host"
       - name: Kill self-hosted runner container
-        run: |
-          cd /home/ubuntu/actions-runner
-          kill -SIGINT $(cat runner.pid)
+        run: kill -SIGINT $(cat ~user/entrypoint.pid)

.vscode/extensions.json

@@ -0,0 +1,10 @@
+{
+  "recommendations": [
+    "GitHub.vscode-github-actions",
+    "jeff-hykin.better-dockerfile-syntax",
+    "mads-hartmann.bash-ide-vscode",
+    "ms-python.black-formatter",
+    "ms-python.python",
+    "timonwong.shellcheck",
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,20 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "git grok: push local commits as individual PRs",
+      "detail": "Install git-grok first: https://github.com/dimikot/git-grok",
+      "type": "shell",
+      "command": "git grok",
+      "problemMatcher": [],
+      "hide": false
+    },
+    {
+      "label": "git rebase --interactive",
+      "detail": "Opens a UI for interactive rebase (install \"Git rebase shortcuts\" extension).",
+      "type": "shell",
+      "command": "GIT_EDITOR=\"code --wait\" git rebase -i",
+      "problemMatcher": []
+    }
+  ]
+}

README.md

@@ -39,7 +39,7 @@ ones, so rsync can run efficiently.
 
     # Storage host in the format [user@]host; it must have password-free
     # SSH key access.
-    # Required.
+    # Default: the content of ~/ci-storage-host file.
     storage-host: ''
 
     # Storage directory on the storage host.

action.yml

@@ -8,16 +8,16 @@ inputs:
     description: "What to do (store or load)."
     required: true
   storage-host:
-    description: "Storage host in the format [user@]host; it must have password-free SSH key access."
-    required: true
+    description: "Storage host in the format [user@]host; it must have password-free SSH key access. If not passed, tries to read it from ~/ci-storage-host file."
+    required: false
   storage-dir:
     description: "Storage directory on the remote host. If not set, uses ~/ci-storage/{owner}/{repo}"
     required: false
   storage-max-age-sec:
     description: "Remove slots created earlier than this many seconds ago. If not set, uses the ci-storage tool default 4 hours."
     required: false
   slot-id:
-    description: 'Id of the slot to store to or load from; use "*" to load a random most recent slot. If empty, uses "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" value.'
+    description: 'Id of the slot to store to or load from; use "*" to load a random most recent slot; use "?" to load a random most recent slot and skip if it does not exist. If empty, uses "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" value.'
     required: false
   local-dir:
     description: 'Local directory path to store from or load to. If not set, uses "." (the current work directory).'
@@ -27,20 +27,25 @@ inputs:
     required: false
   verbose:
     description: "If set, prints the list of transferred files."
-    type: boolean
     required: false
 runs:
   using: "composite"
   steps:
     - name: Run ci-storage ${{ inputs.action }}
-      run: >
-        pwd && "${{ github.action_path }}/ci-storage"
-        --storage-host="${{ inputs.storage-host || '' }}"
-        --storage-dir="${{ inputs.storage-dir || format('~/ci-storage/{0}', github.repository) }}"
-        --storage-max-age-sec="${{ inputs.storage-max-age-sec || '' }}"
-        --slot-id="${{ inputs.slot-id || format('{0}-{1}', github.run_id, github.run_attempt) }}"
-        --local-dir="${{ inputs.local-dir || '.' }}"
-        --exclude="${{ inputs.exclude || '' }}"
-        ${{ inputs.verbose && '--verbose' || '' }}
-        ${{ inputs.action }}
+      run: |
+        exec 2>&1; set -e -o xtrace
+        pwd
+        storage_host="${{ inputs.storage-host || '' }}"
+        if [[ "$storage_host" == "" ]]; then
+          storage_host=$(cat ~/ci-storage-host)
+        fi
+        "${{ github.action_path }}/ci-storage" \
+          --storage-host="$storage_host" \
+          --storage-dir="${{ inputs.storage-dir || format('~/ci-storage/{0}', github.repository) }}" \
+          --storage-max-age-sec="${{ inputs.storage-max-age-sec || '' }}" \
+          --slot-id="${{ inputs.slot-id || format('{0}-{1}', github.run_id, github.run_attempt) }}" \
+          --local-dir="${{ inputs.local-dir || '.' }}" \
+          --exclude="${{ inputs.exclude || '' }}" \
+          ${{ inputs.verbose && '--verbose' || '' }} \
+          ${{ inputs.action }}
       shell: bash

ci-storage

@@ -74,7 +74,7 @@ def main():
         "--slot-id",
         type=str,
         required=True,
-        help='id of the slot to store to or load from; use "*" to load a random most recent slot',
+        help='id of the slot to store to or load from; use "*" to load a random most recent slot; use "?" to load a random most recent slot and skip if it does not exist',
     )
     parser.add_argument(
         "--local-dir",
@@ -162,8 +162,8 @@ def action_store(
     exclude: list[str],
     verbose: bool,
 ):
-    if slot_id == "*":
-        raise UserException('slot_id="*" is not allowed for "store" action')
+    if slot_id == "*" or slot_id == "?":
+        raise UserException(f'slot_id="{slot_id}" is not allowed for "store" action')
     slot_id = normalize_slot_id(slot_id)
     slot_ids_and_ages = list_slots(storage_host=storage_host, storage_dir=storage_dir)
     slot_id_recent = slot_ids_and_ages[0][0] if len(slot_ids_and_ages) else None
@@ -208,14 +208,18 @@ def action_load(
     exclude: list[str],
     verbose: bool,
 ):
-    if slot_id == "*":
+    if slot_id == "*" or slot_id == "?":
         slot_ids_and_ages = list_slots(
             storage_host=storage_host, storage_dir=storage_dir
         )
         if len(slot_ids_and_ages) == 0:
-            raise UserException(
-                'to use slot_id="*", there must be at least one slot in the storage.'
-            )
+            if slot_id == "?":
+                print(f'No slots found, and slot-id="{slot_id}", so skipping.')
+                return
+            else:
+                raise UserException(
+                    f'to use slot-id="{slot_id}", there must be at least one slot in the storage.'
+                )
         slot_id = slot_ids_and_ages[0][0]
     else:
         slot_id = normalize_slot_id(slot_id)

docker/.env

@@ -0,0 +1,11 @@
+# This key (as well as docker-compose.yml) is only used in tests, so it's safe
+# to have it here.
+CI_STORAGE_HOST_PRIVATE_KEY_TEST_ONLY="-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACBZgbuAsWHfeYshNJacifV30KxJVFKr4/B4WnvxO8x2jAAAAKgm2KyUJtis
+lAAAAAtzc2gtZWQyNTUxOQAAACBZgbuAsWHfeYshNJacifV30KxJVFKr4/B4WnvxO8x2jA
+AAAECRcPB4jRqJEgNBvFPA6+k5HPT5/ZbXnD2KUyE+oJFfA1mBu4CxYd95iyE0lpyJ9XfQ
+rElUUqvj8Hhae/E7zHaMAAAAHmRtaXRyeUBEbWl0cnktTWFjQm9vay1NMS5sb2NhbAECAw
+QFBgc=
+-----END OPENSSH PRIVATE KEY-----"
+CI_STORAGE_HOST_PUBLIC_KEY_TEST_ONLY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFmBu4CxYd95iyE0lpyJ9XfQrElUUqvj8Hhae/E7zHaM"

docker/ci-storage-host/Dockerfile

@@ -2,22 +2,19 @@ ARG BASE_IMAGE="ubuntu:22.04"
 
 FROM $BASE_IMAGE
 
-ENV GH_REPOSITORY=""
-ENV CI_STORAGE_HOST_SSH_KEY=""
+ENV CI_STORAGE_HOST_PUBLIC_KEY=""
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN true \
     && apt-get update -y \
-    && apt-get upgrade -y \
     && apt-get install -y --no-install-recommends \
-      awscli rsync openssh-server \
-      mc gcc git curl wget pv psmisc unzip vim nano telnet net-tools bash-completion \
-      libssl-dev apt-transport-https build-essential ca-certificates locales pkg-config \
+      openssh-server \
+      jq gh rsync python3 mc git curl wget pv psmisc unzip vim nano telnet net-tools apt-transport-https ca-certificates locales \
     && sed -i -e "s|#PermitRootLogin.*|PermitRootLogin no|" /etc/ssh/sshd_config \
-    && useradd -m ubuntu \
-    && mkdir -p /home/ubuntu/.ssh \
-    && chown -R ubuntu:ubuntu /home/ubuntu/.ssh \
-    && chmod 700 /home/ubuntu/.ssh
+    && useradd -m user \
+    && mkdir -p ~user/.ssh ~user/ci-storage \
+    && chown -R user:user ~user \
+    && chmod 700 ~user/.ssh
 
 COPY --chmod=755 entrypoint.sh /
 

docker/ci-storage-host/README.md

@@ -0,0 +1,9 @@
+# A Simple Container with SSH Server
+
+Build an image from this Dockerfile to launch a simple SSH server with rsync.
+
+- Pre-creates /home/user/ci-storage directory.
+- Copies public key in CI_STORAGE_HOST_PUBLIC_KEY to user's authorized_keys.
+
+One ci-storage-host contain may serve multiple GitHub repositories. Each of them
+will have own directory in /home/user/ci-storage (managed by ci-storage tool).

docker/ci-storage-host/entrypoint.sh

@@ -1,28 +1,20 @@
 #!/bin/bash
 #
-# A container which holds ci-storage saved slots. Its ~ubuntu/ci-storage should
-# be persistent across container restarts.
+# A container which holds ci-storage saved slots. Its ~user/ci-storage should be
+# persistent across container restarts (e.g. point to an AWS EBS volume).
 #
 set -u -e
 
-if [ "${CI_STORAGE_HOST_SSH_KEY:-}" = "" ]; then
-  echo "CI_STORAGE_HOST_SSH_KEY is not set, exiting..."
+if [[ "${CI_STORAGE_HOST_PUBLIC_KEY:=''}" == "" ]]; then
+  echo "CI_STORAGE_HOST_PUBLIC_KEY must be set to a valid SSH public key."
   exit 1
 fi
 
-cd /home/ubuntu
+authorized_keys=~user/.ssh/authorized_keys
 
-echo "$CI_STORAGE_HOST_SSH_KEY" > .ssh/id_ed25519
-chmod 600 .ssh/id_ed25519
-ssh-keygen -f .ssh/id_ed25519 -y > .ssh/authorized_keys
-chown -R ubuntu:ubuntu .ssh
-
-# This code is for simplifying the CI tests and allow self-hosted-runner to boot
-# in docker-compose. In real world, the 1st slot created should contain the real
-# files (e.g. a cloned git repo).
-if [ ! -e ci-storage -a "${GH_REPOSITORY:-}" != "" ]; then
-  mkdir -p ci-storage/$GH_REPOSITORY/initial
-  chown -R ubuntu:ubuntu ci-storage
+if [[ ! -f $authorized_keys ]] || ! grep -qF "$CI_STORAGE_HOST_PUBLIC_KEY" $authorized_keys; then
+  echo "$CI_STORAGE_HOST_PUBLIC_KEY" >> $authorized_keys
+  chown user:user $authorized_keys
 fi
 
 mkdir -p /var/run/sshd

docker/compose-up.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+echo "Booting containters on the local laptop for debugging purposes..."
+
+GH_REPOSITORY=$(gh repo view --json owner,name -q '.owner.login + "/" + .name') GH_TOKEN=$(gh auth token) docker compose up --build "$@"

docker/compose.yml

@@ -0,0 +1,30 @@
+version: "3.4"
+services:
+  ci-storage-host:
+    build:
+      context: ci-storage-host
+      dockerfile: Dockerfile
+    healthcheck:
+      test: ["CMD", "bash", "-c", "netstat -ltn | grep -c :22"]
+      interval: 1s
+      timeout: 3s
+      retries: 10
+    ports:
+      - 10022:22
+    environment:
+      - CI_STORAGE_HOST_PUBLIC_KEY=${CI_STORAGE_HOST_PUBLIC_KEY_TEST_ONLY?}
+  self-hosted-runner:
+    build:
+      context: self-hosted-runner
+      additional_contexts:
+        root: ..
+      dockerfile: Dockerfile
+    depends_on:
+      ci-storage-host:
+        condition: service_healthy
+    environment:
+      - GH_REPOSITORY=${GH_REPOSITORY:-dimikot/ci-storage}
+      - GH_LABELS=${GH_LABELS:-ci-storage}
+      - GH_TOKEN
+      - CI_STORAGE_HOST=${CI_STORAGE_HOST:-ci-storage-host}
+      - CI_STORAGE_HOST_PRIVATE_KEY=${CI_STORAGE_HOST_PRIVATE_KEY_TEST_ONLY?}