feat: spartan iac (#8455)

Consolidates code used configure our "spartan" kubernetes cluster, and
make deployments of the aztec network and metrics into it.

.github/workflows/spartan-test.yml

@@ -136,11 +136,10 @@ jobs:
           tester_ttl: 40
           run: |
             set -eux
-            cd ./yarn-project/end-to-end/
-            ./scripts/setup_local_k8s.sh
+            ./spartan/scripts/setup_local_k8s.sh
             export FORCE_COLOR=1
             export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}"
-            ../../scripts/earthly-ci --exec-stats -P --no-output ./+network-transfer --values-file=${{ matrix.values_file }}
+            ./scripts/earthly-ci --exec-stats -P --no-output ./yarn-project/end-to-end/+network-transfer --values-file=${{ matrix.values_file }}
 
   success-check:
     runs-on: ubuntu-20.04

.gitignore

@@ -8,6 +8,7 @@ build/
 .idea
 cmake-build-debug
 .terraform*
+terraform.tfstate*
 .bootstrapped
 .tsbuildinfo
 

spartan/.gitignore

@@ -0,0 +1 @@
+*.tgz

helm-charts/aztec-network/templates/boot-node.stateful-set.yaml → spartan/aztec-network/templates/boot-node.stateful-set.yaml

@@ -61,6 +61,16 @@ spec:
               "-c",
               "source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe",
             ]
+          livenessProbe:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - curl -fSs http://127.0.0.1:{{ .Values.bootNode.service.nodePort }}/status
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 30
+            failureThreshold: 3
           volumeMounts:
             - name: shared-volume
               mountPath: /shared
@@ -79,6 +89,8 @@ spec:
               value: "{{ .Values.bootNode.service.nodePort }}"
             - name: LOG_LEVEL
               value: "{{ .Values.bootNode.logLevel }}"
+            - name: LOG_JSON
+              value: "1"
             - name: DEBUG
               value: "{{ .Values.bootNode.debug }}"
             - name: ETHEREUM_HOST

helm-charts/aztec-network/templates/prover-node.stateful-set.yaml → spartan/aztec-network/templates/prover-node.stateful-set.yaml

@@ -59,6 +59,8 @@ spec:
               value: "{{ .Values.proverNode.service.nodePort }}"
             - name: LOG_LEVEL
               value: "{{ .Values.proverNode.logLevel }}"
+            - name: LOG_JSON
+              value: "1"
             - name: DEBUG
               value: "{{ .Values.proverNode.debug }}"
             - name: ETHEREUM_HOST

helm-charts/aztec-network/templates/pxe.deployment.yaml → spartan/aztec-network/templates/pxe.deployment.yaml

@@ -29,6 +29,12 @@ spec:
               value: {{ include "aztec-network.ethereumHost" . | quote }}
             - name: AZTEC_NODE_URL
               value: {{ include "aztec-network.bootNodeUrl" . | quote }}
+            - name: LOG_JSON
+              value: "1"
+            - name: LOG_LEVEL
+              value: "{{ .Values.pxe.logLevel }}"
+            - name: DEBUG
+              value: "{{ .Values.pxe.debug }}"
           ports:
             - name: http
               containerPort: {{ .Values.pxe.service.port }}

helm-charts/aztec-network/templates/tests/run-tests.yaml → spartan/aztec-network/templates/tests/run-tests.yaml

@@ -17,3 +17,9 @@ spec:
           value: {{ .Values.scenario }}
         - name: PXE_URL
           value: {{ include "aztec-network.pxeUrl" . | quote }}
+        - name: DEBUG
+          value: "aztec:*"
+        - name: LOG_LEVEL
+          value: "debug"
+        - name: LOG_JSON
+          value: "1"

helm-charts/aztec-network/templates/validator.stateful-set.yaml → spartan/aztec-network/templates/validator.stateful-set.yaml

@@ -59,6 +59,8 @@ spec:
               value: "{{ .Values.validator.service.nodePort }}"
             - name: LOG_LEVEL
               value: "{{ .Values.validator.logLevel }}"
+            - name: LOG_JSON
+              value: "1"
             - name: DEBUG
               value: "{{ .Values.validator.debug }}"
             - name: ETHEREUM_HOST

helm-charts/aztec-network/values.yaml → spartan/aztec-network/values.yaml

@@ -67,6 +67,8 @@ proverNode:
   resources: {}
 
 pxe:
+  logLevel: "debug"
+  debug: "aztec:*"
   replicas: 1
   service:
     type: ClusterIP

spartan/metrics/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

spartan/metrics/Chart.lock

@@ -0,0 +1,9 @@
+dependencies:
+- name: opentelemetry-collector
+  repository: https://open-telemetry.github.io/opentelemetry-helm-charts
+  version: 0.104.0
+- name: eck-stack
+  repository: https://helm.elastic.co
+  version: 0.12.1
+digest: sha256:e95083de14387953eb4093eb2c6b98cf889e532cd097b21b15b1896cfa117e7c
+generated: "2024-09-06T19:31:29.086654335-04:00"

spartan/metrics/Chart.yaml

@@ -0,0 +1,32 @@
+apiVersion: v2
+name: metrics
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"
+
+dependencies:
+  - name: opentelemetry-collector
+    version: 0.104.0
+    repository: https://open-telemetry.github.io/opentelemetry-helm-charts
+  - name: eck-stack
+    version: 0.12.1
+    repository: https://helm.elastic.co

spartan/metrics/values.yaml

@@ -0,0 +1,76 @@
+opentelemetry-collector:
+  extraEnvs:
+    - name: ELASTICSEARCH_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: elasticsearch-es-elastic-user
+          key: elastic
+
+  mode: daemonset
+
+  image:
+    repository: "otel/opentelemetry-collector-contrib"
+
+  presets:
+    logsCollection:
+      enabled: true
+      includeCollectorLogs: true
+    kubernetesAttributes:
+      enabled: true
+  config:
+    exporters:
+      debug: {}
+      elasticsearch:
+        endpoint: "https://elasticsearch-es-http.metrics.svc:9200/"
+        tls:
+          insecure_skip_verify: true
+        auth:
+          authenticator: basicauth
+    extensions:
+      basicauth:
+        client_auth:
+          username: elastic
+          password: ${ELASTICSEARCH_PASSWORD}
+      # The health_check extension is mandatory for this chart.
+      # Without the health_check extension the collector will fail the readiness and liveliness probes.
+      # The health_check extension can be modified, but should never be removed.
+      health_check:
+        endpoint: ${env:MY_POD_IP}:13133
+    processors:
+      batch: {}
+      transform:
+        error_mode: ignore
+        log_statements:
+          - context: log
+            statements:
+              - merge_maps(attributes, ParseJSON(body), "upsert") where IsMatch(body, "^\\{")
+              # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl/contexts/ottllog
+              - set(severity_number, 1) where attributes["level"] == "trace"
+              - set(severity_number, 5) where attributes["level"] == "debug"
+              - set(severity_number, 9) where attributes["level"] == "info"
+              - set(severity_number, 13) where attributes["level"] == "warn"
+              - set(severity_number, 17) where attributes["level"] == "error"
+              - set(severity_number, 21) where attributes["level"] == "fatal"
+              - set(body, attributes["message"])
+              - set(attributes["level"], nil)
+              - set(attributes["message"], nil)
+    receivers:
+      otlp:
+        protocols:
+          http:
+            endpoint: ${env:MY_POD_IP}:4318
+    service:
+      extensions: [basicauth, health_check]
+      telemetry:
+        metrics:
+          address: ${env:MY_POD_IP}:8888
+      pipelines:
+        logs:
+          exporters:
+            - elasticsearch
+            - debug
+          processors:
+            - transform
+            - batch
+          receivers:
+            - otlp

spartan/scripts/install_eck_operator.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -ex
+
+version=2.14.0
+
+kubectl create -f https://download.elastic.co/downloads/eck/$version/crds.yaml
+kubectl apply -f https://download.elastic.co/downloads/eck/$version/operator.yaml

spartan/terraform/main.tf

@@ -0,0 +1,125 @@
+# Configure the AWS Provider
+provider "aws" {
+  region = "us-east-2"  # Change this to your preferred region
+}
+
+# Create VPC for EKS
+resource "aws_vpc" "spartan_vpc" {
+  cidr_block = "10.0.0.0/16"
+
+  tags = {
+    Name = "spartan-vpc"
+  }
+}
+
+# Create an internet gateway
+resource "aws_internet_gateway" "spartan_igw" {
+  vpc_id = aws_vpc.spartan_vpc.id
+
+  tags = {
+    Name = "spartan-igw"
+  }
+}
+
+# Create a subnet
+resource "aws_subnet" "spartan_subnet" {
+  vpc_id     = aws_vpc.spartan_vpc.id
+  cidr_block = "10.0.1.0/24"
+  availability_zone = "us-east-2a"  # Change this to match your region
+
+  tags = {
+    Name = "spartan-subnet"
+  }
+}
+
+# Create EKS Cluster
+resource "aws_eks_cluster" "spartan_cluster" {
+  name     = "spartan-cluster"
+  role_arn = aws_iam_role.spartan_cluster_role.arn
+
+  vpc_config {
+    subnet_ids = [aws_subnet.spartan_subnet.id]
+  }
+
+  depends_on = [aws_iam_role_policy_attachment.spartan_cluster_policy]
+}
+
+# Create IAM role for EKS Cluster
+resource "aws_iam_role" "spartan_cluster_role" {
+  name = "spartan-cluster-role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = "eks.amazonaws.com"
+        }
+      }
+    ]
+  })
+}
+
+# Attach necessary policies to the EKS Cluster role
+resource "aws_iam_role_policy_attachment" "spartan_cluster_policy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
+  role       = aws_iam_role.spartan_cluster_role.name
+}
+
+# Create EKS Node Group
+resource "aws_eks_node_group" "spartan_node_group" {
+  cluster_name    = aws_eks_cluster.spartan_cluster.name
+  node_group_name = "spartan-node-group"
+  node_role_arn   = aws_iam_role.spartan_node_role.arn
+  subnet_ids      = [aws_subnet.spartan_subnet.id]
+
+  scaling_config {
+    desired_size = 1
+    max_size     = 1
+    min_size     = 1
+  }
+
+  instance_types = ["t4g.2xlarge"]
+
+  depends_on = [
+    aws_iam_role_policy_attachment.spartan_worker_node_policy,
+    aws_iam_role_policy_attachment.spartan_cni_policy,
+    aws_iam_role_policy_attachment.spartan_ecr_policy,
+  ]
+}
+
+# Create IAM role for EKS Node Group
+resource "aws_iam_role" "spartan_node_role" {
+  name = "spartan-node-role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = "ec2.amazonaws.com"
+        }
+      }
+    ]
+  })
+}
+
+# Attach necessary policies to the EKS Node role
+resource "aws_iam_role_policy_attachment" "spartan_worker_node_policy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
+  role       = aws_iam_role.spartan_node_role.name
+}
+
+resource "aws_iam_role_policy_attachment" "spartan_cni_policy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
+  role       = aws_iam_role.spartan_node_role.name
+}
+
+resource "aws_iam_role_policy_attachment" "spartan_ecr_policy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
+  role       = aws_iam_role.spartan_node_role.name
+}

yarn-project/end-to-end/Earthfile

@@ -92,10 +92,10 @@ NETWORK_TEST:
     RUN kubectl delete namespace $namespace --ignore-not-found=true --wait=true --now --timeout=10m
   END
 
-  RUN helm install spartan ../../helm-charts/aztec-network \
+  RUN helm install spartan ../../spartan/aztec-network/ \
         --namespace $namespace \
         --create-namespace \
-        --values ../../helm-charts/aztec-network/values/$values_file \
+        --values ../../spartan/aztec-network/values/$values_file \
         --set images.test.image="aztecprotocol/end-to-end:$AZTEC_DOCKER_TAG" \
         --set images.aztec.image="aztecprotocol/aztec:$AZTEC_DOCKER_TAG" \
         --set test="$test" \