From 38f01571ebbc90174fcdc765bac84dfcb12bbc0c Mon Sep 17 00:00:00 2001 From: just-mitch <68168980+just-mitch@users.noreply.github.com> Date: Tue, 27 Aug 2024 13:33:51 -0400 Subject: [PATCH] feat: spartan token transfer (#8163) Adds prover node and metrics to the k8s helm chart. Adds a test that transfers tokens privately. Create a new CI action for creating a k8s cluster and running the test. --- .github/workflows/ci.yml | 29 +++ docker-compose.yml | 2 +- .../aztec/aztec-dashboard-all-in-one.json | 108 +++++------ .../files/grafana_dashboards}/default.yml | 0 .../aztec-network/templates/_helpers.tpl | 20 ++ .../templates/boot-node.stateful-set.yaml | 15 +- .../configure-validator-env.config-map.yaml | 2 +- .../templates/deploy-l2-contracts.job.yaml | 36 ++++ .../aztec-network/templates/metrics.yaml | 175 ++++++++++++++++++ .../templates/prover-node.service.yaml | 14 ++ .../templates/prover-node.stateful-set.yaml | 87 +++++++++ .../templates/pxe.deployment.yaml | 4 +- helm-charts/aztec-network/values.yaml | 44 ++++- .../aztec/src/cli/cmds/start_prover_node.ts | 2 +- yarn-project/end-to-end/Earthfile | 13 +- .../scripts/create_k8s_dashboard.sh | 31 ++++ .../end-to-end/scripts/setup_local_k8s.sh | 59 ++---- .../src/fixtures/snapshot_manager.ts | 3 +- .../end-to-end/src/spartan/smoke.test.ts | 2 +- .../end-to-end/src/spartan/transfer.test.ts | 142 ++++++++++++++ .../pxe/src/pxe_service/pxe_service.ts | 1 + yarn-project/telemetry-client/src/otel.ts | 2 +- 22 files changed, 680 insertions(+), 111 deletions(-) rename {grafana_dashboards => helm-charts/aztec-network/files/grafana_dashboards}/aztec/aztec-dashboard-all-in-one.json (96%) rename {grafana_dashboards => helm-charts/aztec-network/files/grafana_dashboards}/default.yml (100%) create mode 100644 helm-charts/aztec-network/templates/deploy-l2-contracts.job.yaml create mode 100644 helm-charts/aztec-network/templates/metrics.yaml create mode 100644 helm-charts/aztec-network/templates/prover-node.service.yaml create mode 100644 helm-charts/aztec-network/templates/prover-node.stateful-set.yaml create mode 100644 yarn-project/end-to-end/scripts/create_k8s_dashboard.sh create mode 100644 yarn-project/end-to-end/src/spartan/transfer.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7cab8933a8..580b46338be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,6 +161,35 @@ jobs: export FORCE_COLOR=1 ../../scripts/earthly-ci -P --no-output +${{ matrix.test }} + # all the network end-to-end tests for aztec (not required to merge) + network-e2e: + needs: [build, changes] + if: ${{ needs.changes.outputs.non-barretenberg-cpp == 'true' }} + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + with: { ref: "${{ env.GIT_COMMIT }}" } + - uses: ./.github/ci-setup-action + - name: Setup and Test + uses: ./.github/ensure-tester-with-images + timeout-minutes: 45 + with: + runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }} + builder_type: builder-x86 + # these are copied to the tester and expected by the earthly command below + # if they fail to copy, it will try to build them on the tester and fail + builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }} + # command to produce the images in case they don't exist + builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images + tester_ttl: 40 + run: | + set -eux + cd ./yarn-project/end-to-end/ + ./scripts/setup_local_k8s.sh + export FORCE_COLOR=1 + export EARTHLY_BUILD_ARGS="${{ env.EARTHLY_BUILD_ARGS }}" + ../../scripts/earthly-ci --exec-stats -P --no-output ./+network-transfer + # all the benchmarking end-to-end integration tests for aztec (not required to merge) bench-e2e: needs: [build, changes] diff --git a/docker-compose.yml b/docker-compose.yml index 2e44e214ba7..9106ee66811 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -139,7 +139,7 @@ services: profiles: - metrics volumes: - - ./grafana_dashboards:/etc/grafana/provisioning/dashboards + - ./helm-charts/aztec-network/files/grafana_dashboards:/etc/grafana/provisioning/dashboards - grafana:/var/lib/grafana configs: - source: grafana-sources diff --git a/grafana_dashboards/aztec/aztec-dashboard-all-in-one.json b/helm-charts/aztec-network/files/grafana_dashboards/aztec/aztec-dashboard-all-in-one.json similarity index 96% rename from grafana_dashboards/aztec/aztec-dashboard-all-in-one.json rename to helm-charts/aztec-network/files/grafana_dashboards/aztec/aztec-dashboard-all-in-one.json index 81395d023e0..8d02e6687d1 100644 --- a/grafana_dashboards/aztec/aztec-dashboard-all-in-one.json +++ b/helm-charts/aztec-network/files/grafana_dashboards/aztec/aztec-dashboard-all-in-one.json @@ -37,7 +37,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -119,7 +119,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -139,7 +139,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -221,7 +221,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -254,7 +254,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -302,7 +302,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -322,7 +322,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -373,7 +373,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -389,7 +389,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -410,7 +410,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -490,7 +490,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -511,7 +511,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -603,7 +603,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -623,7 +623,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "description": "", "fieldConfig": { @@ -673,7 +673,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -693,7 +693,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "description": "", "fieldConfig": { @@ -775,7 +775,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -795,7 +795,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -843,7 +843,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -877,7 +877,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -956,7 +956,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -978,7 +978,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -1029,7 +1029,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -1046,7 +1046,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -1068,7 +1068,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -1116,7 +1116,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -1160,7 +1160,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -1212,7 +1212,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -1310,7 +1310,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -1330,7 +1330,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -1410,7 +1410,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -1428,7 +1428,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -1445,7 +1445,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -1467,7 +1467,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -1542,7 +1542,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -1559,7 +1559,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -1577,7 +1577,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -1612,7 +1612,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -1692,7 +1692,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -1712,7 +1712,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -1759,7 +1759,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -1780,7 +1780,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -1859,7 +1859,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguyxik" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -1894,7 +1894,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -1970,7 +1970,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -1986,7 +1986,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "builder", @@ -2007,7 +2007,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -2086,7 +2086,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -2108,7 +2108,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "fieldConfig": { "defaults": { @@ -2187,7 +2187,7 @@ { "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "disableTextWrap": false, "editorMode": "code", @@ -2220,7 +2220,7 @@ }, "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "definition": "label_values(target_info,aztec_network_name)", "hide": 0, @@ -2248,7 +2248,7 @@ }, "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "definition": "label_values(target_info{aztec_network_name=\"$network\"},exported_instance)", "hide": 0, @@ -2307,7 +2307,7 @@ }, "datasource": { "type": "prometheus", - "uid": "-qqguYXIk" + "uid": "aztec-node-metrics" }, "definition": "label_values(aztec_circuit_protocol_circuit_name)", "hide": 0, @@ -2335,7 +2335,7 @@ }, "timepicker": {}, "timezone": "browser", - "title": "New new new dashboard", + "title": "Aztec Network Dashboard", "uid": "cdtxao66xa1ogc", "version": 31, "weekStart": "" diff --git a/grafana_dashboards/default.yml b/helm-charts/aztec-network/files/grafana_dashboards/default.yml similarity index 100% rename from grafana_dashboards/default.yml rename to helm-charts/aztec-network/files/grafana_dashboards/default.yml diff --git a/helm-charts/aztec-network/templates/_helpers.tpl b/helm-charts/aztec-network/templates/_helpers.tpl index 44febac82a7..dea62e55f6d 100644 --- a/helm-charts/aztec-network/templates/_helpers.tpl +++ b/helm-charts/aztec-network/templates/_helpers.tpl @@ -57,3 +57,23 @@ http://{{ include "aztec-network.fullname" . }}-ethereum.{{ .Release.Namespace } {{- define "aztec-network.pxeUrl" -}} http://{{ include "aztec-network.fullname" . }}-pxe.{{ .Release.Namespace }}:{{ .Values.pxe.service.port }} {{- end -}} + +{{- define "aztec-network.bootNodeUrl" -}} +http://{{ include "aztec-network.fullname" . }}-boot-node-0.{{ include "aztec-network.fullname" . }}-boot-node.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.bootNode.service.nodePort }} +{{- end -}} + +{{- define "aztec-network.metricsHost" -}} +http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }} +{{- end -}} + +{{- define "aztec-network.otelCollectorMetricsEndpoint" -}} +{{ include "aztec-network.metricsHost" . }}:{{ .Values.metrics.ports.otlp }}/v1/metrics +{{- end -}} + +{{- define "aztec-network.otelCollectorTracesEndpoint" -}} +{{ include "aztec-network.metricsHost" . }}:{{ .Values.metrics.ports.otlp }}/v1/traces +{{- end -}} + + + + diff --git a/helm-charts/aztec-network/templates/boot-node.stateful-set.yaml b/helm-charts/aztec-network/templates/boot-node.stateful-set.yaml index d3132d3a9e9..fce20afc60a 100644 --- a/helm-charts/aztec-network/templates/boot-node.stateful-set.yaml +++ b/helm-charts/aztec-network/templates/boot-node.stateful-set.yaml @@ -31,6 +31,11 @@ spec: sleep 5 done echo "Ethereum node is ready!" + until curl --head --silent {{ include "aztec-network.otelCollectorMetricsEndpoint" . }} > /dev/null; do + echo "Waiting for OpenTelemetry collector..." + sleep 5 + done + echo "OpenTelemetry collector is ready!" - name: deploy-contracts image: {{ .Values.images.aztec.image }} command: @@ -54,7 +59,7 @@ spec: [ "/bin/bash", "-c", - "source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer", + "source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe", ] volumeMounts: - name: shared-volume @@ -90,9 +95,13 @@ spec: value: "0.0.0.0:{{ .Values.bootNode.service.p2pPort }}" - name: VALIDATOR_PRIVATE_KEY value: "0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a" + - name: OTEL_EXPORTER_OTLP_METRICS_ENDPOINT + value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }} + - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }} ports: - - containerPort: "{{ .Values.bootNode.service.nodePort }}" - - containerPort: "{{ .Values.bootNode.service.p2pPort }}" + - containerPort: {{ .Values.bootNode.service.nodePort }} + - containerPort: {{ .Values.bootNode.service.p2pPort }} resources: {{- toYaml .Values.bootNode.resources | nindent 12 }} volumes: diff --git a/helm-charts/aztec-network/templates/configure-validator-env.config-map.yaml b/helm-charts/aztec-network/templates/configure-validator-env.config-map.yaml index 28904f5f5b4..47ddeba836c 100644 --- a/helm-charts/aztec-network/templates/configure-validator-env.config-map.yaml +++ b/helm-charts/aztec-network/templates/configure-validator-env.config-map.yaml @@ -10,7 +10,7 @@ data: set -e # Ask the bootnode for l1 contract addresses - output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js get-node-info -u http://{{ include "aztec-network.fullname" . }}-boot-node-0.{{ include "aztec-network.fullname" . }}-boot-node.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.bootNode.service.nodePort }}) + output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js get-node-info -u {{ include "aztec-network.bootNodeUrl" . }}) echo "$output" diff --git a/helm-charts/aztec-network/templates/deploy-l2-contracts.job.yaml b/helm-charts/aztec-network/templates/deploy-l2-contracts.job.yaml new file mode 100644 index 00000000000..56f2dd3b998 --- /dev/null +++ b/helm-charts/aztec-network/templates/deploy-l2-contracts.job.yaml @@ -0,0 +1,36 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "aztec-network.fullname" . }}-deploy-l2-contracts + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +spec: + template: + metadata: + labels: + {{- include "aztec-network.selectorLabels" . | nindent 8 }} + app: deploy-l2-contracts + spec: + restartPolicy: OnFailure + containers: + - name: deploy-l2-contracts + image: {{ .Values.images.aztec.image }} + command: + - /bin/bash + - -c + - | + until curl -s -X POST -H 'content-type: application/json' \ + -d '{"jsonrpc":"2.0","method":"pxe_getNodeInfo","params":[],"id":67}' \ + {{ include "aztec-network.pxeUrl" . }} | grep -q '"enr:-'; do + echo "Waiting for PXE service..." + sleep 5 + done + echo "PXE service is ready!" + node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-protocol-contracts + env: + - name: PXE_URL + value: {{ include "aztec-network.pxeUrl" . | quote }} + - name: DEBUG + value: "aztec:*" + - name: LOG_LEVEL + value: "debug" diff --git a/helm-charts/aztec-network/templates/metrics.yaml b/helm-charts/aztec-network/templates/metrics.yaml new file mode 100644 index 00000000000..8af3aa2f1ab --- /dev/null +++ b/helm-charts/aztec-network/templates/metrics.yaml @@ -0,0 +1,175 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "aztec-network.fullname" . }}-metrics + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "aztec-network.selectorLabels" . | nindent 6 }} + app: metrics + template: + metadata: + labels: + {{- include "aztec-network.selectorLabels" . | nindent 8 }} + app: metrics + spec: + containers: + - name: otel-collector + image: "{{ .Values.images.otelCollector.image }}" + imagePullPolicy: {{ .Values.images.otelCollector.pullPolicy }} + ports: + - containerPort: {{ .Values.metrics.ports.otlp }} + name: otlp + - containerPort: {{ .Values.metrics.ports.metrics }} + name: metrics + args: + - "--config=/etc/otel-collector-config.yaml" + volumeMounts: + - name: metrics-config + mountPath: /etc/otel-collector-config.yaml + subPath: otel-collector-config.yaml + + - name: prometheus + image: "{{ .Values.images.prometheus.image }}" + ports: + - containerPort: {{ .Values.metrics.ports.prometheus }} + volumeMounts: + - name: metrics-config + mountPath: /etc/prometheus/prometheus.yml + subPath: prometheus.yaml + + - name: grafana + image: "{{ .Values.images.grafana.image }}" + ports: + - containerPort: {{ .Values.metrics.ports.grafana }} + volumeMounts: + - name: metrics-config + mountPath: /etc/grafana/provisioning/dashboards/default.yml + subPath: grafana-default-dashboard.yml + - name: metrics-config + mountPath: /etc/grafana/provisioning/dashboards/aztec/aztec-dashboard.json + subPath: grafana-aztec-dashboard.json + - name: metrics-config + mountPath: /etc/grafana/provisioning/datasources/default.yaml + subPath: grafana-sources.yaml + - name: grafana-storage + mountPath: /var/lib/grafana + + - name: jaeger + image: "{{ .Values.images.jaeger.image }}" + ports: + - containerPort: {{ .Values.metrics.ports.jaeger }} + volumes: + - name: metrics-config + configMap: + name: {{ include "aztec-network.fullname" . }}-metrics + - name: grafana-storage + emptyDir: {} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "aztec-network.fullname" . }}-metrics + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +spec: + type: {{ .Values.metrics.service.type }} + ports: + - port: {{ .Values.metrics.ports.otlp }} + targetPort: otlp + protocol: TCP + name: otlp + - port: {{ .Values.metrics.ports.metrics }} + targetPort: metrics + protocol: TCP + name: metrics + - port: {{ .Values.metrics.ports.prometheus }} + targetPort: {{ .Values.metrics.ports.prometheus }} + protocol: TCP + name: prometheus + - port: {{ .Values.metrics.ports.grafana }} + targetPort: {{ .Values.metrics.ports.grafana }} + protocol: TCP + name: grafana + - port: {{ .Values.metrics.ports.jaeger }} + targetPort: {{ .Values.metrics.ports.jaeger }} + protocol: TCP + name: jaeger-ui + - port: 14250 + targetPort: 14250 + protocol: TCP + name: jaeger-collector + selector: + {{- include "aztec-network.selectorLabels" . | nindent 4 }} + app: metrics +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "aztec-network.fullname" . }}-metrics + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +data: + otel-collector-config.yaml: | + receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:{{ .Values.metrics.ports.otlp }}" + + processors: + batch: + + exporters: + prometheus: + endpoint: "0.0.0.0:8889" + metric_expiration: 5m + otlp/jaeger: + endpoint: "0.0.0.0:4317" + tls: + insecure: true + service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp/jaeger] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] + + prometheus.yaml: | + global: + evaluation_interval: 15s + scrape_interval: 15s + scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ['localhost:8888'] + - job_name: aztec + static_configs: + - targets: ['localhost:8889'] + + grafana-sources.yaml: | + apiVersion: 1 + datasources: + - name: Prometheus + uid: aztec-node-metrics + type: prometheus + url: http://localhost:{{ .Values.metrics.ports.prometheus }} + editable: false + isDefault: true + jsonData: + timeInterval: 10s + + grafana-aztec-dashboard.json: | + {{ .Files.Get "files/grafana_dashboards/aztec/aztec-dashboard-all-in-one.json" | nindent 4 }} + + grafana-default-dashboard.yml: | + {{ .Files.Get "files/grafana_dashboards/default.yml" | nindent 4 }} diff --git a/helm-charts/aztec-network/templates/prover-node.service.yaml b/helm-charts/aztec-network/templates/prover-node.service.yaml new file mode 100644 index 00000000000..1d20ba7cd3e --- /dev/null +++ b/helm-charts/aztec-network/templates/prover-node.service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "aztec-network.fullname" . }}-prover-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +spec: + clusterIP: None + selector: + {{- include "aztec-network.selectorLabels" . | nindent 4 }} + app: prover-node + ports: + - port: {{ .Values.proverNode.service.nodePort }} + name: node \ No newline at end of file diff --git a/helm-charts/aztec-network/templates/prover-node.stateful-set.yaml b/helm-charts/aztec-network/templates/prover-node.stateful-set.yaml new file mode 100644 index 00000000000..79a2da1f581 --- /dev/null +++ b/helm-charts/aztec-network/templates/prover-node.stateful-set.yaml @@ -0,0 +1,87 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "aztec-network.fullname" . }}-prover-node + labels: + {{- include "aztec-network.labels" . | nindent 4 }} +spec: + serviceName: {{ include "aztec-network.fullname" . }}-prover-node + replicas: {{ .Values.proverNode.replicas }} + selector: + matchLabels: + {{- include "aztec-network.selectorLabels" . | nindent 6 }} + app: prover-node + template: + metadata: + labels: + {{- include "aztec-network.selectorLabels" . | nindent 8 }} + app: prover-node + spec: + initContainers: + - name: configure-validator-env + image: "{{ .Values.images.aztec.image }}" + imagePullPolicy: {{ .Values.images.aztec.pullPolicy }} + command: + - "/bin/sh" + - "-c" + - "cp /scripts/configure-validator-env.sh /tmp/configure-validator-env.sh && chmod +x /tmp/configure-validator-env.sh && /tmp/configure-validator-env.sh" + volumeMounts: + - name: shared-volume + mountPath: /shared + - name: scripts + mountPath: /scripts + env: + - name: ETHEREUM_HOST + value: {{ include "aztec-network.ethereumHost" . | quote }} + containers: + - name: aztec + image: "{{ .Values.images.aztec.image }}" + imagePullPolicy: {{ .Values.images.aztec.pullPolicy }} + command: + - "/bin/bash" + - "-c" + - "source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --prover-node --prover --archiver" + volumeMounts: + - name: shared-volume + mountPath: /shared + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_DNS_NAME + value: "$(POD_NAME).{{ include "aztec-network.fullname" . }}-prover-node.$(POD_NAMESPACE).svc.cluster.local" + - name: PORT + value: "{{ .Values.proverNode.service.nodePort }}" + - name: LOG_LEVEL + value: "{{ .Values.proverNode.logLevel }}" + - name: DEBUG + value: "{{ .Values.proverNode.debug }}" + - name: ETHEREUM_HOST + value: {{ include "aztec-network.ethereumHost" . | quote }} + - name: PROVER_REAL_PROOFS + value: "{{ .Values.proverNode.realProofs }}" + - name: PROVER_AGENT_ENABLED + value: "{{ .Values.proverNode.proverAgentEnabled }}" + - name: PROVER_PUBLISHER_PRIVATE_KEY + value: "0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97" + # get private proofs from the boot node + - name: TX_PROVIDER_NODE_URL + value: {{ include "aztec-network.bootNodeUrl" . | quote }} + # prover agent gets jobs from itself + - name: AZTEC_NODE_URL + value: "$(POD_DNS_NAME):{{ .Values.proverNode.service.nodePort }}" + ports: + - containerPort: {{ .Values.proverNode.service.nodePort }} + resources: + {{- toYaml .Values.proverNode.resources | nindent 12 }} + volumes: + - name: shared-volume + emptyDir: {} + - name: scripts + configMap: + name: {{ include "aztec-network.fullname" . }}-configure-validator-env \ No newline at end of file diff --git a/helm-charts/aztec-network/templates/pxe.deployment.yaml b/helm-charts/aztec-network/templates/pxe.deployment.yaml index 1dc77d49df2..fefe94987c4 100644 --- a/helm-charts/aztec-network/templates/pxe.deployment.yaml +++ b/helm-charts/aztec-network/templates/pxe.deployment.yaml @@ -26,9 +26,9 @@ spec: node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --pxe env: - name: ETHEREUM_HOST - value: "http://{{ include "aztec-network.fullname" . }}-ethereum.{{ .Release.Namespace }}:{{ .Values.ethereum.service.port }}" + value: {{ include "aztec-network.ethereumHost" . | quote }} - name: AZTEC_NODE_URL - value: "http://{{ include "aztec-network.fullname" . }}-boot-node-0.{{ include "aztec-network.fullname" . }}-boot-node.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.bootNode.service.nodePort }}" + value: {{ include "aztec-network.bootNodeUrl" . | quote }} ports: - name: http containerPort: {{ .Values.pxe.service.port }} diff --git a/helm-charts/aztec-network/values.yaml b/helm-charts/aztec-network/values.yaml index 8bc6f9ef73e..7d2234296e4 100644 --- a/helm-charts/aztec-network/values.yaml +++ b/helm-charts/aztec-network/values.yaml @@ -11,6 +11,18 @@ images: foundry: image: ghcr.io/foundry-rs/foundry@sha256:29ba6e34379e79c342ec02d437beb7929c9e254261e8032b17e187be71a2609f pullPolicy: IfNotPresent + otelCollector: + image: otel/opentelemetry-collector-contrib + pullPolicy: IfNotPresent + prometheus: + image: prom/prometheus + pullPolicy: IfNotPresent + grafana: + image: grafana/grafana + pullPolicy: IfNotPresent + jaeger: + image: jaegertracing/all-in-one + pullPolicy: IfNotPresent bootNode: replicas: 1 @@ -18,22 +30,32 @@ bootNode: p2pPort: 40400 nodePort: 8080 logLevel: "debug" - debug: "discv5:*,aztec:*" + debug: "aztec:*" p2p: enabled: "true" resources: {} validator: - replicas: 1 + replicas: 0 service: p2pPort: 40400 nodePort: 8080 logLevel: "debug" - debug: "discv5:*,aztec:*" + debug: "aztec:*" p2p: enabled: "true" resources: {} +proverNode: + replicas: 1 + service: + nodePort: 8080 + logLevel: "debug" + debug: "aztec:*" + realProofs: false + proverAgentEnabled: true + resources: {} + pxe: replicas: 1 service: @@ -66,5 +88,21 @@ ethereum: failureThreshold: 3 resources: {} +metrics: + service: + type: ClusterIP + ports: + otlp: 4316 + metrics: 4315 + prometheus: 9090 + grafana: 3000 + jaeger: 16686 + readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + test: "sample" scenario: "default" diff --git a/yarn-project/aztec/src/cli/cmds/start_prover_node.ts b/yarn-project/aztec/src/cli/cmds/start_prover_node.ts index 21d8a54f3f9..d804a40ff79 100644 --- a/yarn-project/aztec/src/cli/cmds/start_prover_node.ts +++ b/yarn-project/aztec/src/cli/cmds/start_prover_node.ts @@ -38,7 +38,7 @@ export const startProverNode = async ( process.exit(1); } - if (options.prover) { + if (options.prover || options.proverAgentEnabled) { userLog(`Running prover node with local prover agent.`); proverConfig.proverAgentEnabled = true; } else { diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile index e66ba5a57b5..1b41fe3d9c6 100644 --- a/yarn-project/end-to-end/Earthfile +++ b/yarn-project/end-to-end/Earthfile @@ -73,12 +73,14 @@ NETWORK_TEST: RUN kubectl delete namespace $namespace --ignore-not-found=true --wait=true --now --timeout=10m END - RUN helm install spartan ../../helm-charts/aztec-network \ --namespace $namespace --create-namespace \ --set images.test.image="aztecprotocol/end-to-end:$AZTEC_DOCKER_TAG" \ --set images.aztec.image="aztecprotocol/aztec:$AZTEC_DOCKER_TAG" \ - --set test="$test" + --set test="$test" \ + --wait \ + --wait-for-jobs=true \ + --timeout=10m RUN kubectl wait pod -l app==pxe --for=condition=Ready -n $namespace --timeout=10m @@ -299,4 +301,9 @@ e2e-cli-wallet: DO +E2E_COMPOSE_TEST --test=e2e_cli_wallet --compose_file=scripts/docker-compose-wallet.yml network-smoke: - DO +NETWORK_TEST --fresh_install=true --namespace=smoke --test=./src/spartan/smoke.test.ts + ARG force_build + DO +NETWORK_TEST --force_build=$force_build --fresh_install=true --namespace=smoke --test=./src/spartan/smoke.test.ts + +network-transfer: + ARG force_build + DO +NETWORK_TEST --force_build=$force_build --fresh_install=true --namespace=transfer --test=./src/spartan/transfer.test.ts diff --git a/yarn-project/end-to-end/scripts/create_k8s_dashboard.sh b/yarn-project/end-to-end/scripts/create_k8s_dashboard.sh new file mode 100644 index 00000000000..795ae65fc7a --- /dev/null +++ b/yarn-project/end-to-end/scripts/create_k8s_dashboard.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Install the Kubernetes Dashboard +helm repo add kubernetes-dashboard https://kubernetes.github.io/dashboard/ +helm upgrade --install kubernetes-dashboard kubernetes-dashboard/kubernetes-dashboard --create-namespace --namespace kubernetes-dashboard + + +# Create the ServiceAccount for the Kubernetes Dashboard +kubectl apply -f - < /dev/null; then + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x kubectl + sudo mv kubectl /usr/local/bin/kubectl +fi -# Install kind -[ $(uname -m) = x86_64 ] && curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-$(uname)-amd64 -chmod +x ./kind -sudo mv ./kind /usr/local/bin/kind +# Install kind if it is not installed +if ! command -v kind &> /dev/null; then + curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.23.0/kind-$(uname)-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind +fi -# Install helm -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +# Install helm if it is not installed +if ! command -v helm &> /dev/null; then + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod +x get_helm.sh + sudo ./get_helm.sh + rm get_helm.sh +fi kind create cluster - -# Install the Kubernetes Dashboard -helm repo add kubernetes-dashboard https://kubernetes.github.io/dashboard/ -helm upgrade --install kubernetes-dashboard kubernetes-dashboard/kubernetes-dashboard --create-namespace --namespace kubernetes-dashboard - - -# Create the ServiceAccount for the Kubernetes Dashboard -kubectl apply -f - < - async ({ pxe }: SubsystemsContext) => { + async ({ pxe }: { pxe: PXE }) => { // Generate account keys. const accountKeys: [Fr, GrumpkinScalar][] = Array.from({ length: numberOfAccounts }).map(_ => [ Fr.random(), diff --git a/yarn-project/end-to-end/src/spartan/smoke.test.ts b/yarn-project/end-to-end/src/spartan/smoke.test.ts index b003b82ff24..f1a58fa1731 100644 --- a/yarn-project/end-to-end/src/spartan/smoke.test.ts +++ b/yarn-project/end-to-end/src/spartan/smoke.test.ts @@ -5,7 +5,7 @@ const { PXE_URL } = process.env; if (!PXE_URL) { throw new Error('PXE_URL env variable must be set'); } -const debugLogger = createDebugLogger('aztec:spartan-test'); +const debugLogger = createDebugLogger('aztec:spartan-test:smoke'); // const userLog = createConsoleLogger(); describe('sample test', () => { diff --git a/yarn-project/end-to-end/src/spartan/transfer.test.ts b/yarn-project/end-to-end/src/spartan/transfer.test.ts new file mode 100644 index 00000000000..6e13df33ffd --- /dev/null +++ b/yarn-project/end-to-end/src/spartan/transfer.test.ts @@ -0,0 +1,142 @@ +import { getSchnorrAccount } from '@aztec/accounts/schnorr'; +import { + type AccountWallet, + type AccountWalletWithSecretKey, + type AztecAddress, + type CompleteAddress, + ExtendedNote, + Fr, + Note, + type PXE, + type TxHash, + computeSecretHash, + createCompatibleClient, +} from '@aztec/aztec.js'; +import { createDebugLogger } from '@aztec/foundation/log'; +import { TokenContract } from '@aztec/noir-contracts.js'; + +import { addAccounts, publicDeployAccounts } from '../fixtures/snapshot_manager.js'; + +const { PXE_URL } = process.env; +if (!PXE_URL) { + throw new Error('PXE_URL env variable must be set'); +} + +const toString = ({ value }: { value: bigint }) => { + const vals: number[] = Array.from(new Fr(value).toBuffer()); + + let str = ''; + for (let i = 0; i < vals.length; i++) { + if (vals[i] != 0) { + str += String.fromCharCode(Number(vals[i])); + } + } + return str; +}; + +const addPendingShieldNoteToPXE = async (args: { + amount: bigint; + secretHash: Fr; + txHash: TxHash; + accountAddress: AztecAddress; + assetAddress: AztecAddress; + wallet: AccountWallet; +}) => { + const { accountAddress, assetAddress, amount, secretHash, txHash, wallet } = args; + const note = new Note([new Fr(amount), secretHash]); + const extendedNote = new ExtendedNote( + note, + accountAddress, + assetAddress, + TokenContract.storage.pending_shields.slot, + TokenContract.notes.TransparentNote.id, + txHash, + ); + await wallet.addNote(extendedNote); +}; + +describe('token transfer test', () => { + const logger = createDebugLogger(`aztec:spartan-test:transfer`); + const TOKEN_NAME = 'USDC'; + const TOKEN_SYMBOL = 'USD'; + const TOKEN_DECIMALS = 18n; + const MINT_AMOUNT = 1000000n; + let pxe: PXE; + let wallets: AccountWalletWithSecretKey[]; + let completeAddresses: CompleteAddress[]; + let tokenAddress: AztecAddress; + let tokenAtWallet0: TokenContract; + beforeAll(async () => { + pxe = await createCompatibleClient(PXE_URL, logger); + const { accountKeys } = await addAccounts(3, logger)({ pxe }); + const accountManagers = accountKeys.map(ak => getSchnorrAccount(pxe, ak[0], ak[1], 1)); + wallets = await Promise.all(accountManagers.map(a => a.getWallet())); + completeAddresses = await pxe.getRegisteredAccounts(); + wallets.forEach((w, i) => logger.verbose(`Wallet ${i} address: ${w.getAddress()}`)); + await publicDeployAccounts(wallets[0], completeAddresses.slice(0, 2)); + + logger.verbose(`Deploying TokenContract...`); + const tokenContract = await TokenContract.deploy( + wallets[0], + completeAddresses[0], + TOKEN_NAME, + TOKEN_SYMBOL, + TOKEN_DECIMALS, + ) + .send() + .deployed(); + + tokenAddress = tokenContract.address; + tokenAtWallet0 = await TokenContract.at(tokenAddress, wallets[0]); + + logger.verbose(`Minting ${MINT_AMOUNT} publicly...`); + await tokenAtWallet0.methods.mint_public(completeAddresses[0].address, MINT_AMOUNT).send().wait(); + + logger.verbose(`Minting ${MINT_AMOUNT} privately...`); + const secret = Fr.random(); + const secretHash = computeSecretHash(secret); + const receipt = await tokenAtWallet0.methods.mint_private(MINT_AMOUNT, secretHash).send().wait(); + + await addPendingShieldNoteToPXE({ + amount: MINT_AMOUNT, + secretHash, + txHash: receipt.txHash, + accountAddress: completeAddresses[0].address, + assetAddress: tokenAddress, + wallet: wallets[0], + }); + const txClaim = tokenAtWallet0.methods.redeem_shield(completeAddresses[0].address, MINT_AMOUNT, secret).send(); + await txClaim.wait({ debug: true }); + logger.verbose(`Minting complete.`); + }); + + it('can get info', async () => { + const name = toString(await tokenAtWallet0.methods.private_get_name().simulate()); + expect(name).toBe(TOKEN_NAME); + }); + + it('can transfer 1 publicly', async () => { + const transferAmount = 1n; + const balance0 = await tokenAtWallet0.methods.balance_of_public(completeAddresses[0].address).simulate(); + expect(balance0).toBeGreaterThanOrEqual(transferAmount); + await tokenAtWallet0.methods + .transfer_public(completeAddresses[0].address, completeAddresses[1].address, transferAmount, 0) + .send() + .wait(); + const balance0After = await tokenAtWallet0.methods.balance_of_public(completeAddresses[0].address).simulate(); + const balance1After = await tokenAtWallet0.methods.balance_of_public(completeAddresses[1].address).simulate(); + expect(balance0After).toBe(balance0 - transferAmount); + expect(balance1After).toBe(transferAmount); + }); + + it('can transfer 1 privately', async () => { + const transferAmount = 1n; + const balance0 = await tokenAtWallet0.methods.balance_of_private(completeAddresses[0].address).simulate(); + expect(balance0).toBeGreaterThanOrEqual(transferAmount); + await tokenAtWallet0.methods.transfer(completeAddresses[1].address, transferAmount).send().wait(); + const balance0After = await tokenAtWallet0.methods.balance_of_private(completeAddresses[0].address).simulate(); + const balance1After = await tokenAtWallet0.methods.balance_of_private(completeAddresses[1].address).simulate(); + expect(balance0After).toBe(balance0 - transferAmount); + expect(balance1After).toBe(transferAmount); + }); +}); diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index d35e27808d5..972ef5ce916 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -565,6 +565,7 @@ export class PXEService implements PXE { } this.log.info(`Sending transaction ${txHash}`); await this.node.sendTx(tx); + this.log.info(`Sent transaction ${txHash}`); return txHash; } diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts index d46a3c70ac4..1d168895a2f 100644 --- a/yarn-project/telemetry-client/src/otel.ts +++ b/yarn-project/telemetry-client/src/otel.ts @@ -59,7 +59,7 @@ export class OpenTelemetryClient implements TelemetryClient { // https://opentelemetry.io/docs/specs/otel/compatibility/prometheus_and_openmetrics/#resource-attributes // https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#supporting-target-metadata-in-both-push-based-and-pull-based-systems this.targetInfo = this.meterProvider.getMeter('target').createGauge('target_info', { - description: 'Target information', + description: 'Target metadata', }); this.targetInfo.record(1, this.resource.attributes);