Allow K8s demos to use prebuilt images (#872)

On a fresh clone of the repository this change reduces the setup time
of the kubernetes demo from ~10 minutes to ~2 minutes.

To achieve this, the Tekton pipeline now runs in a branching pattern
that first tries to find a trustworthy image upstream and falls back to
a local build if no image was found. This way we preserve the
flexibility of the development setup but get a massive speedup when the
repository wasn't modified.

The new setup is fully portable to any arbitrary other K8s
infrastructure. The curl commands now only require a single parameter,
making it much easier to experiment with custom toolchains.

We now expect correct tags in the example deployments. The chromium and
kubernetes examples have been updated to accomodate for this and are now
merged to a more idiomatic base/overlays pattern.

deployment-examples/chromium/.gitignore

@@ -0,0 +1,2 @@
+# Generated by 02_application.yaml
+/kustomization.yaml

deployment-examples/chromium/01_operations.sh

@@ -7,24 +7,19 @@ set -xeuo pipefail
 
 SRC_ROOT=$(git rev-parse --show-toplevel)
 
-EVENTLISTENER=$(kubectl get gtw eventlistener -o=jsonpath='{.status.addresses[0].value}')
+EVENTLISTENER=$(kubectl get \
+    gtw eventlistener -o=jsonpath='{.status.addresses[0].value}')
 
 # The image for the scheduler and CAS.
 curl -v \
     -H 'content-Type: application/json' \
-    -d '{
-        "flakeOutput": "./src_root#image",
-        "imageTagOverride": "local"
-    }' \
+    -d '{"flakeOutput": "./src_root#image"}' \
     http://${EVENTLISTENER}:8080
 
 # Wrap it nativelink to turn it into a worker.
 curl -v \
     -H 'content-Type: application/json' \
-    -d '{
-        "flakeOutput": "./src_root#nativelink-worker-siso-chromium",
-        "imageTagOverride": "local"
-    }' \
+    -d '{"flakeOutput": "./src_root#nativelink-worker-siso-chromium"}' \
     http://${EVENTLISTENER}:8080
 
 # Wait for the pipelines to finish.

deployment-examples/chromium/02_application.sh

@@ -4,6 +4,23 @@
 
 KUSTOMIZE_DIR=$(git rev-parse --show-toplevel)/deployment-examples/chromium
 
+cat <<EOF > "$KUSTOMIZE_DIR"/kustomization.yaml
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+  - ../kubernetes/base
+
+resources:
+  - worker-chromium.yaml
+EOF
+
+cd "$KUSTOMIZE_DIR" && kustomize edit set image \
+    nativelink=localhost:5001/nativelink:$(\
+        nix eval .#image.imageTag --raw) \
+    nativelink-worker-chromium=localhost:5001/nativelink-worker-siso-chromium:$(\
+        nix eval .#nativelink-worker-siso-chromium.imageTag --raw)
+
 kubectl apply -k "$KUSTOMIZE_DIR"
 
 kubectl rollout status deploy/nativelink-cas

deployment-examples/chromium/worker-chromium.yaml

@@ -15,7 +15,8 @@ spec:
     spec:
       containers:
         - name: nativelink-worker-chromium
-          image: "localhost:5001/nativelink-worker-siso-chromium:local"
+          # This image will be edited by kustomize.
+          image: nativelink-worker-chromium
           env:
             - name: RUST_LOG
               value: info
@@ -24,12 +25,12 @@ spec:
             - name: SCHEDULER_ENDPOINT
               value: nativelink-scheduler
           volumeMounts:
-            - name: worker-chromium-config
-              mountPath: /worker-chromium.json
-              subPath: worker-chromium.json
+            - name: worker-config
+              mountPath: /worker.json
+              subPath: worker.json
           command: ["/bin/nativelink"]
-          args: ["/worker-chromium.json"]
+          args: ["/worker.json"]
       volumes:
-        - name: worker-chromium-config
+        - name: worker-config
           configMap:
-            name: worker-chromium
+            name: worker

deployment-examples/kubernetes/.gitignore

@@ -1,2 +1,2 @@
-worker-lre-cc.json
-worker-lre-java.json
+# Generated by 02_application.yaml
+/kustomization.yaml

deployment-examples/kubernetes/01_operations.sh

@@ -7,30 +7,17 @@ set -xeuo pipefail
 
 SRC_ROOT=$(git rev-parse --show-toplevel)
 
-EVENTLISTENER=$(kubectl get gtw eventlistener -o=jsonpath='{.status.addresses[0].value}')
+EVENTLISTENER=$(kubectl get \
+    gtw eventlistener -o=jsonpath='{.status.addresses[0].value}')
 
 curl -v \
     -H 'content-Type: application/json' \
-    -d '{
-        "flakeOutput": "./src_root#image",
-        "imageTagOverride": "local"
-    }' \
+    -d '{"flakeOutput": "./src_root#image"}' \
     http://${EVENTLISTENER}:8080
 
 curl -v \
     -H 'content-Type: application/json' \
-    -d '{
-        "flakeOutput": "./src_root#nativelink-worker-lre-cc",
-        "imageTagOverride": "local"
-    }' \
-    http://${EVENTLISTENER}:8080
-
-curl -v \
-    -H 'content-Type: application/json' \
-    -d '{
-        "flakeOutput": "./src_root#nativelink-worker-lre-java",
-        "imageTagOverride": "local"
-    }' \
+    -d '{"flakeOutput": "./src_root#nativelink-worker-lre-cc"}' \
     http://${EVENTLISTENER}:8080
 
 # Wait for the pipelines to finish.

deployment-examples/kubernetes/02_application.sh

@@ -4,9 +4,34 @@
 
 KUSTOMIZE_DIR=$(git rev-parse --show-toplevel)/deployment-examples/kubernetes
 
+cat <<EOF > "$KUSTOMIZE_DIR"/kustomization.yaml
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+  - base
+
+resources:
+  - worker-lre-cc.yaml
+  # TODO(aaronmondal): Fix java and add this:
+  # - worker-lre-java.yaml
+EOF
+
+cd "$KUSTOMIZE_DIR" && kustomize edit set image \
+    nativelink=localhost:5001/nativelink:$(\
+        nix eval .#image.imageTag --raw) \
+    nativelink-worker-lre-cc=localhost:5001/nativelink-worker-lre-cc:$(\
+        nix eval .#nativelink-worker-lre-cc.imageTag --raw) \
+
+# TODO(aaronmondal): Fix java and add this:
+#   nativelink-worker-lre-java=localhost:5001/nativelink-worker-lre-java:$(\
+#       nix eval .#nativelink-worker-lre-java.imageTag --raw)
+
 kubectl apply -k "$KUSTOMIZE_DIR"
 
 kubectl rollout status deploy/nativelink-cas
 kubectl rollout status deploy/nativelink-scheduler
 kubectl rollout status deploy/nativelink-worker-lre-cc
-kubectl rollout status deploy/nativelink-worker-lre-java
+
+# TODO(aaronmondal): Fix java and add this:
+# kubectl rollout status deploy/nativelink-worker-lre-java

deployment-examples/chromium/cas.yaml → deployment-examples/kubernetes/base/cas.yaml

@@ -15,7 +15,8 @@ spec:
     spec:
       containers:
         - name: nativelink-cas
-          image: "localhost:5001/nativelink:local"
+          # This image will be edited by kustomize.
+          image: nativelink
           env:
             - name: RUST_LOG
               value: info

deployment-examples/kubernetes/kustomization.yaml → deployment-examples/kubernetes/base/kustomization.yaml

@@ -1,9 +1,9 @@
 ---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
 resources:
   - cas.yaml
   - scheduler.yaml
-  - worker-lre-cc.yaml
-  - worker-lre-java.yaml
   - routes.yaml
 
 configMapGenerator:
@@ -17,6 +17,9 @@ configMapGenerator:
     files:
       - worker.json
 
+images:
+  - name: nativelink
+
 secretGenerator:
   - name: tls-secret
     files:

deployment-examples/chromium/scheduler.yaml → deployment-examples/kubernetes/base/scheduler.yaml

@@ -15,7 +15,8 @@ spec:
     spec:
       containers:
         - name: nativelink-scheduler
-          image: "localhost:5001/nativelink:local"
+          # This image will be edited by kustomize.
+          image: nativelink
           env:
             - name: RUST_LOG
               value: info

deployment-examples/kubernetes/worker.json → deployment-examples/kubernetes/base/worker.json

@@ -52,7 +52,16 @@
       "work_directory": "~/.cache/nativelink/work",
       "platform_properties": {
         "cpu_count": {
-          "query_cmd": "nproc"
+          "query_cmd": "nproc",
+        },
+        "memory_kb": {
+          "values": ["500000"],
+        },
+        "network_kbps": {
+          "values": ["100000"],
+        },
+        "cpu_arch": {
+          "values": ["x86_64"],
         },
         "OSFamily": {
           "values": ["Linux"]
@@ -63,11 +72,14 @@
             // that is matched by the scheduler against the value specified in
             // the `exec_properties` of the corresponding platform at
             // `local-remote-execution/generated-cc/config/BUILD`.
-            "${NATIVELINK_WORKER_PLATFORM:-error_undefined_platform}"
+            "${NATIVELINK_WORKER_PLATFORM:-undefined_platform}"
           ]
         }
       }
     }
   }],
-  "servers": []
+  "servers": [],
+  "global": {
+    "max_open_files": 524288
+  }
 }