Simplify and cleanup entry points and tests

Pull Request: #7 (main)

.github/workflows/ci.yml

@@ -48,20 +48,21 @@ jobs:
         uses: actions/checkout@v2
       - name: Start test Docker containers
         run: |
-          ssh-keygen -t ed25519 -qf /tmp/key -N ""
-          export CI_STORAGE_HOST_SSH_KEY="$(cat /tmp/key)"
           set -o xtrace
           cd docker
-          # Boot ci-storage-host container. It will have a default empty slot.
-          docker compose up ci-storage-host -d --build
+          # Build all containers.
+          docker compose build
+          # Boot ci-storage-host container in background.
+          docker compose up ci-storage-host -d
           # Now boot self-hosted-runner container. It will connect to
-          # ci-storage-host container and load the empty slot from there, then
-          # register a GitHub self-hosted runner and remain waiting for jobs.
-          docker compose up self-hosted-runner --build
+          # ci-storage-host container and load a test (non-existent) ci-storage
+          # slot from there, then register a GitHub self-hosted runner and
+          # remain waiting for jobs.
+          docker compose up self-hosted-runner
         env:
           GH_REPOSITORY: ${{ github.repository }}
           GH_LABELS: ci-storage-test
-          GH_TOKEN: ${{ secrets.CI_PAT  }}
+          GH_TOKEN: ${{ secrets.CI_PAT }}
 
   # The test job with ci-storage-test tag which is initially queued, but then is
   # picked up by the self-hosted-runner container booted in the previous job. In
@@ -80,6 +81,4 @@ jobs:
           action: "store"
           storage-host: "ci-storage-host"
       - name: Kill self-hosted runner container
-        run: |
-          cd /home/ubuntu/actions-runner
-          kill -SIGINT $(cat runner.pid)
+        run: kill -SIGINT $(cat cd ~user/entrypoint.pid)

.vscode/extensions.json

@@ -0,0 +1,10 @@
+{
+  "recommendations": [
+    "GitHub.vscode-github-actions",
+    "jeff-hykin.better-dockerfile-syntax",
+    "mads-hartmann.bash-ide-vscode",
+    "ms-python.black-formatter",
+    "ms-python.python",
+    "timonwong.shellcheck",
+  ]
+}

.vscode/tasks.json

@@ -0,0 +1,20 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "label": "git grok: push local commits as individual PRs",
+      "detail": "Install git-grok first: https://github.com/dimikot/git-grok",
+      "type": "shell",
+      "command": "git grok",
+      "problemMatcher": [],
+      "hide": false
+    },
+    {
+      "label": "git rebase --interactive",
+      "detail": "Opens a UI for interactive rebase (install \"Git rebase shortcuts\" extension).",
+      "type": "shell",
+      "command": "GIT_EDITOR=\"code --wait\" git rebase -i",
+      "problemMatcher": []
+    }
+  ]
+}

action.yml

@@ -17,7 +17,7 @@ inputs:
     description: "Remove slots created earlier than this many seconds ago. If not set, uses the ci-storage tool default 4 hours."
     required: false
   slot-id:
-    description: 'Id of the slot to store to or load from; use "*" to load a random most recent slot. If empty, uses "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" value.'
+    description: 'Id of the slot to store to or load from; use "*" to load a random most recent slot; use "?" to load a random most recent slot and skip if it does not exist. If empty, uses "$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" value.'
     required: false
   local-dir:
     description: 'Local directory path to store from or load to. If not set, uses "." (the current work directory).'
@@ -27,7 +27,6 @@ inputs:
     required: false
   verbose:
     description: "If set, prints the list of transferred files."
-    type: boolean
     required: false
 runs:
   using: "composite"

ci-storage

@@ -74,7 +74,7 @@ def main():
         "--slot-id",
         type=str,
         required=True,
-        help='id of the slot to store to or load from; use "*" to load a random most recent slot',
+        help='id of the slot to store to or load from; use "*" to load a random most recent slot; use "?" to load a random most recent slot and skip if it does not exist',
     )
     parser.add_argument(
         "--local-dir",
@@ -162,8 +162,8 @@ def action_store(
     exclude: list[str],
     verbose: bool,
 ):
-    if slot_id == "*":
-        raise UserException('slot_id="*" is not allowed for "store" action')
+    if slot_id == "*" or slot_id == "?":
+        raise UserException(f'slot_id="{slot_id}" is not allowed for "store" action')
     slot_id = normalize_slot_id(slot_id)
     slot_ids_and_ages = list_slots(storage_host=storage_host, storage_dir=storage_dir)
     slot_id_recent = slot_ids_and_ages[0][0] if len(slot_ids_and_ages) else None
@@ -208,14 +208,18 @@ def action_load(
     exclude: list[str],
     verbose: bool,
 ):
-    if slot_id == "*":
+    if slot_id == "*" or slot_id == "?":
         slot_ids_and_ages = list_slots(
             storage_host=storage_host, storage_dir=storage_dir
         )
         if len(slot_ids_and_ages) == 0:
-            raise UserException(
-                'to use slot_id="*", there must be at least one slot in the storage.'
-            )
+            if slot_id == "?":
+                print(f'No slots found, and slot-id="{slot_id}", so skipping.')
+                return
+            else:
+                raise UserException(
+                    f'to use slot-id="{slot_id}", there must be at least one slot in the storage.'
+                )
         slot_id = slot_ids_and_ages[0][0]
     else:
         slot_id = normalize_slot_id(slot_id)

docker/.env

@@ -0,0 +1,11 @@
+# This key (as well as docker-compose.yml) is only used in tests, so it's safe
+# to have it here.
+CI_STORAGE_HOST_PRIVATE_KEY_TEST_ONLY="-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACBZgbuAsWHfeYshNJacifV30KxJVFKr4/B4WnvxO8x2jAAAAKgm2KyUJtis
+lAAAAAtzc2gtZWQyNTUxOQAAACBZgbuAsWHfeYshNJacifV30KxJVFKr4/B4WnvxO8x2jA
+AAAECRcPB4jRqJEgNBvFPA6+k5HPT5/ZbXnD2KUyE+oJFfA1mBu4CxYd95iyE0lpyJ9XfQ
+rElUUqvj8Hhae/E7zHaMAAAAHmRtaXRyeUBEbWl0cnktTWFjQm9vay1NMS5sb2NhbAECAw
+QFBgc=
+-----END OPENSSH PRIVATE KEY-----"
+CI_STORAGE_HOST_PUBLIC_KEY_TEST_ONLY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFmBu4CxYd95iyE0lpyJ9XfQrElUUqvj8Hhae/E7zHaM"

docker/ci-storage-host/Dockerfile

@@ -2,22 +2,19 @@ ARG BASE_IMAGE="ubuntu:22.04"
 
 FROM $BASE_IMAGE
 
-ENV GH_REPOSITORY=""
-ENV CI_STORAGE_HOST_SSH_KEY=""
+ENV CI_STORAGE_HOST_PUBLIC_KEY=""
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN true \
     && apt-get update -y \
-    && apt-get upgrade -y \
     && apt-get install -y --no-install-recommends \
-      awscli rsync openssh-server \
-      mc gcc git curl wget pv psmisc unzip vim nano telnet net-tools bash-completion \
-      libssl-dev apt-transport-https build-essential ca-certificates locales pkg-config \
+      openssh-server \
+      jq gh rsync python3 mc git curl wget pv psmisc unzip vim nano telnet net-tools apt-transport-https ca-certificates locales \
     && sed -i -e "s|#PermitRootLogin.*|PermitRootLogin no|" /etc/ssh/sshd_config \
-    && useradd -m ubuntu \
-    && mkdir -p /home/ubuntu/.ssh \
-    && chown -R ubuntu:ubuntu /home/ubuntu/.ssh \
-    && chmod 700 /home/ubuntu/.ssh
+    && useradd -m user \
+    && mkdir -p ~user/.ssh ~user/ci-storage \
+    && chown -R user:user ~user \
+    && chmod 700 ~user/.ssh
 
 COPY --chmod=755 entrypoint.sh /
 

docker/ci-storage-host/README.md

@@ -0,0 +1,9 @@
+# A Simple Container with SSH Server
+
+Build an image from this Dockerfile to launch a simple SSH server with rsync.
+
+- Pre-creates /home/user/ci-storage directory.
+- Copies public key in CI_STORAGE_HOST_PUBLIC_KEY to user's authorized_keys.
+
+One ci-storage-host contain may serve multiple GitHub repositories. Each of them
+will have own directory in /home/user/ci-storage (managed by ci-storage tool).

docker/ci-storage-host/entrypoint.sh

@@ -1,28 +1,20 @@
 #!/bin/bash
 #
-# A container which holds ci-storage saved slots. Its ~ubuntu/ci-storage should
-# be persistent across container restarts.
+# A container which holds ci-storage saved slots. Its ~user/ci-storage should be
+# persistent across container restarts (e.g. point to an AWS EBS volume).
 #
 set -u -e
 
-if [ "${CI_STORAGE_HOST_SSH_KEY:-}" = "" ]; then
-  echo "CI_STORAGE_HOST_SSH_KEY is not set, exiting..."
+if [[ "${CI_STORAGE_HOST_PUBLIC_KEY:=''}" == "" ]]; then
+  echo "CI_STORAGE_HOST_PUBLIC_KEY must be set to a valid SSH public key."
   exit 1
 fi
 
-cd /home/ubuntu
+authorized_keys=~user/.ssh/authorized_keys
 
-echo "$CI_STORAGE_HOST_SSH_KEY" > .ssh/id_ed25519
-chmod 600 .ssh/id_ed25519
-ssh-keygen -f .ssh/id_ed25519 -y > .ssh/authorized_keys
-chown -R ubuntu:ubuntu .ssh
-
-# This code is for simplifying the CI tests and allow self-hosted-runner to boot
-# in docker-compose. In real world, the 1st slot created should contain the real
-# files (e.g. a cloned git repo).
-if [ ! -e ci-storage -a "${GH_REPOSITORY:-}" != "" ]; then
-  mkdir -p ci-storage/$GH_REPOSITORY/initial
-  chown -R ubuntu:ubuntu ci-storage
+if [[ ! -f $authorized_keys ]] || ! grep -qF "$CI_STORAGE_HOST_PUBLIC_KEY" $authorized_keys; then
+  echo "$CI_STORAGE_HOST_PUBLIC_KEY" >> $authorized_keys
+  chown user:user $authorized_keys
 fi
 
 mkdir -p /var/run/sshd

docker/compose-up.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+echo "Booting containters on the local laptop for debugging purposes..."
+
+GH_REPOSITORY=$(gh repo view --json owner,name -q '.owner.login + "/" + .name') GH_TOKEN=$(gh auth token) docker compose up --build "$@"

docker/compose.yml

@@ -0,0 +1,30 @@
+version: "3.4"
+services:
+  ci-storage-host:
+    build:
+      context: ci-storage-host
+      dockerfile: Dockerfile
+    healthcheck:
+      test: ["CMD", "bash", "-c", "netstat -ltn | grep -c :22"]
+      interval: 1s
+      timeout: 3s
+      retries: 10
+    ports:
+      - 10022:22
+    environment:
+      - CI_STORAGE_HOST_PUBLIC_KEY=${CI_STORAGE_HOST_PUBLIC_KEY_TEST_ONLY?}
+  self-hosted-runner:
+    build:
+      context: self-hosted-runner
+      additional_contexts:
+        root: ..
+      dockerfile: Dockerfile
+    depends_on:
+      ci-storage-host:
+        condition: service_healthy
+    environment:
+      - GH_REPOSITORY=${GH_REPOSITORY:-dimikot/ci-storage}
+      - GH_LABELS=${GH_LABELS:-ci-storage}
+      - GH_TOKEN
+      - CI_STORAGE_HOST=${CI_STORAGE_HOST:-ci-storage-host}
+      - CI_STORAGE_HOST_PRIVATE_KEY=${CI_STORAGE_HOST_PRIVATE_KEY_TEST_ONLY?}

docker/self-hosted-runner/Dockerfile

@@ -1,5 +1,4 @@
 ARG BASE_IMAGE="ubuntu:22.04"
-
 FROM $BASE_IMAGE
 
 ARG RUNNER_VERSION="2.314.1"
@@ -8,24 +7,22 @@ ENV GH_REPOSITORY=""
 ENV GH_LABELS=""
 ENV GH_TOKEN=""
 ENV CI_STORAGE_HOST=""
-ENV CI_STORAGE_HOST_SSH_KEY=""
+ENV CI_STORAGE_HOST_PRIVATE_KEY=""
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN true \
     && apt-get update -y \
-    && apt-get upgrade -y \
     && apt-get install -y --no-install-recommends \
-      awscli jq gh rsync openssh-client \
-      mc gcc git curl wget pv psmisc unzip vim nano telnet net-tools bash-completion \
-      libssl-dev apt-transport-https build-essential ca-certificates locales pkg-config \
-    && useradd -m ubuntu
+      openssh-client \
+      jq gh rsync python3 mc git curl wget pv psmisc unzip vim nano telnet net-tools apt-transport-https ca-certificates locales \
+    && useradd -m user
 
-USER ubuntu
+USER user
 RUN true \
-    && mkdir -p /home/ubuntu/.ssh \
-    && chmod 700 /home/ubuntu/.ssh \
-    && mkdir /home/ubuntu/actions-runner \
-    && cd /home/ubuntu/actions-runner \
+    && mkdir -p ~user/.ssh \
+    && chmod 700 ~user/.ssh \
+    && mkdir ~user/actions-runner \
+    && cd ~user/actions-runner \
     && arch=$(dpkg --print-architecture) \
     && case "$arch" in \
       x86_64|amd64) arch=linux-x64 ;; \
@@ -35,19 +32,19 @@ RUN true \
     && curl --no-progress-meter -L https://github.com/actions/runner/releases/download/v$RUNNER_VERSION/actions-runner-$arch-$RUNNER_VERSION.tar.gz | tar xz
 
 USER root
-RUN /home/ubuntu/actions-runner/bin/installdependencies.sh \
-    apt-get autoremove \
+RUN ~user/actions-runner/bin/installdependencies.sh \
+    && apt-get autoremove \
     && apt-get clean \
     && apt-get autoclean \
     && rm -rf /var/lib/apt/lists/*
 
-USER ubuntu
-COPY --chmod=755 --chown=ubuntu:ubuntu entrypoint.sh /home/ubuntu
+USER user
+COPY --chmod=755 --chown=user:user entrypoint.sh /home/user
 COPY --chmod=755 --from=root ci-storage /usr/bin/ci-storage
 
-WORKDIR /home/ubuntu
+WORKDIR /home/user
 ENTRYPOINT ["./entrypoint.sh"]
 
-# If overridden in the derived image, evals this as "ubuntu" user as a shell
-# script after config.sh, but before run.sh.
+# If overridden in the derived image, evals this as a shell script after
+# config.sh, but before run.sh.
 CMD []

docker/self-hosted-runner/README.md

@@ -20,3 +20,7 @@ self-hosted runners as you want. An example scenario:
    artifacts of someone else. If the job then uses ci-storage GitHub action to
    restore the files from a slot, it will be very quick, because most of the
    files are already there.
+
+The container in this Dockerfile is serves only one particular GitHub repository
+(controlled by GH_REPOSITORY environment variable at boot time). To serve
+different repositories, boot different containers.