Merge pull request #4053 from emissary-ingress/dcamata/agent-metrics-…

…stream

[v2.2] Stream Envoy metrics to the cloud

CHANGELOG.md

@@ -107,7 +107,10 @@ Please see the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest
 
 - Change: Emissary will now watch for ConfigMap or Secret resources specified by the
   `AGENT_CONFIG_RESOURCE_NAME` environment variable in order to allow all components (and not only
-  the Ambassador Agent) to authenticate requests to Ambassador Cloud.
+  the Ambassador Agent) to authenticate requests to Ambassador Cloud. Support for the Envoy V2 API
+  and the `AMBASSADOR_ENVOY_API_VERSION` environment
+
+- Feature: Support for streaming Envoy metrics about the clusters to Ambassador's cloud. ([4053])
 
 - Feature: The Emissary agent now receives commands to manipulate Rollouts (pause, continue, and
   abort are currently supported) via directives and executes them in the cluster.  A report is send
@@ -119,6 +122,7 @@ Please see the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest
 
 [3906]: https://github.com/emissary-ingress/emissary/issues/3906
 [3821]: https://github.com/emissary-ingress/emissary/issues/3821
+[4053]: https://github.com/emissary-ingress/emissary/pull/4053
 [4040]: https://github.com/emissary-ingress/emissary/pull/4040
 
 ## [2.1.2] January 25, 2022

api/agent/director.proto

@@ -8,19 +8,24 @@ syntax = "proto3";
 import "google/protobuf/duration.proto";
 import "google/protobuf/timestamp.proto";
 
+import "prometheus/metrics.proto";
+
 package agent;
 
 service Director {
-  // Report a consistent Snapshot of information to the CEPC.  This
+  // Report a consistent Snapshot of information to the DCP.  This
   // method is deprecated, you should call ReportStream instead.
   rpc Report(Snapshot) returns (SnapshotResponse) {
     option deprecated = true;
   }
 
-  // Report a consistent Snapshot of information to the CEPC.
+  // Report a consistent Snapshot of information to the DCP.
   rpc ReportStream(stream RawSnapshotChunk) returns (SnapshotResponse) {}
 
-  // Retrieve Directives from the CEPC
+  // Stream metrics to the DCP.
+  rpc StreamMetrics(stream StreamMetricsMessage) returns (StreamMetricsResponse) {}
+
+  // Retrieve Directives from the DCP
   rpc Retrieve(Identity) returns (stream Directive) {}
 
   // Reports the result of a command execution to the cloud
@@ -29,11 +34,11 @@ service Director {
   rpc RetrieveSnapshot(Identity) returns (stream RawSnapshotChunk) {}
 }
 
-// How Ambassador's Agent identifies itself to the CEPC
+// How Ambassador's Agent identifies itself to the DCP
 // This is the identity of the ambassador the agent is reporting on behalf of
 // no user account specific information should be contained in here
 message Identity {
-  // The account ID assigned by the CEPC
+  // The account ID assigned by the DCP
   string account_id = 1 [deprecated=true];
 
   // Ambassador version
@@ -53,7 +58,7 @@ message Identity {
 }
 
 // Information that Ambassador's Agent can send to the Director
-// component of the CEPC
+// component of the DCP
 message Snapshot {
   Identity identity = 1;
   string message = 2;
@@ -86,7 +91,7 @@ message SnapshotResponse {
   // an error. In the future this may contain additional information.
 }
 
-// Instructions that the CEPC can send to Ambassador
+// Instructions that the DCP can send to Ambassador
 message Directive {
   string ID = 1;
 
@@ -103,7 +108,7 @@ message Directive {
   repeated Command commands = 4;
 }
 
-// An individual instruction from the CEPC
+// An individual instruction from the DCP
 message Command {
   // Log this message if present
   string message = 1;
@@ -128,4 +133,15 @@ message CommandResult {
   string message = 3;
 }
 
-message CommandResultResponse {}
+message CommandResultResponse {
+}
+
+message StreamMetricsMessage {
+  Identity identity = 1;
+
+  // A list of metric entries
+  repeated io.prometheus.client.MetricFamily envoy_metrics = 2;
+}
+
+message StreamMetricsResponse {
+}

api/prometheus/metrics.proto

@@ -0,0 +1,92 @@
+// Copyright 2013 Prometheus Team
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package io.prometheus.client;
+option java_package = "io.prometheus.client";
+option go_package = "github.com/prometheus/client_model/go;io_prometheus_client";
+
+import "google/protobuf/timestamp.proto";
+
+message LabelPair {
+  optional string name  = 1;
+  optional string value = 2;
+}
+
+enum MetricType {
+  COUNTER    = 0;
+  GAUGE      = 1;
+  SUMMARY    = 2;
+  UNTYPED    = 3;
+  HISTOGRAM  = 4;
+}
+
+message Gauge {
+  optional double value = 1;
+}
+
+message Counter {
+  optional double   value    = 1;
+  optional Exemplar exemplar = 2;
+}
+
+message Quantile {
+  optional double quantile = 1;
+  optional double value    = 2;
+}
+
+message Summary {
+  optional uint64   sample_count = 1;
+  optional double   sample_sum   = 2;
+  repeated Quantile quantile     = 3;
+}
+
+message Untyped {
+  optional double value = 1;
+}
+
+message Histogram {
+  optional uint64 sample_count = 1;
+  optional double sample_sum   = 2;
+  repeated Bucket bucket       = 3; // Ordered in increasing order of upper_bound, +Inf bucket is optional.
+}
+
+message Bucket {
+  optional uint64   cumulative_count = 1; // Cumulative in increasing order.
+  optional double   upper_bound      = 2; // Inclusive.
+  optional Exemplar exemplar         = 3;
+}
+
+message Exemplar {
+  repeated LabelPair                 label     = 1;
+  optional double                    value     = 2;
+  optional google.protobuf.Timestamp timestamp = 3; // OpenMetrics-style.
+}
+
+message Metric {
+  repeated LabelPair label        = 1;
+  optional Gauge     gauge        = 2;
+  optional Counter   counter      = 3;
+  optional Summary   summary      = 4;
+  optional Untyped   untyped      = 5;
+  optional Histogram histogram    = 7;
+  optional int64     timestamp_ms = 6;
+}
+
+message MetricFamily {
+  optional string     name   = 1;
+  optional string     help   = 2;
+  optional MetricType type   = 3;
+  repeated Metric     metric = 4;
+}

builder/requirements-dev.txt

@@ -3,7 +3,7 @@ dpath
 gitpython
 mypy
 pexpect
-pytest
+pytest==6.2.5
 pytest-cov
 retry
 httpretty

charts/emissary-ingress/templates/ambassador-agent.yaml

@@ -226,6 +226,9 @@ spec:
         image: {{ include "ambassador.image" . }}
         imagePullPolicy: {{ .Values.image.pullPolicy }}
         command: [ "agent" ]
+        ports:
+          - containerPort: 8006
+            name: grpc
         env:
         - name: AGENT_NAMESPACE
           valueFrom:
@@ -242,5 +245,33 @@ spec:
   progressDeadlineSeconds: {{ .Values.progressDeadlines.agent }}
   {{- end }}
   {{- end }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "ambassador.fullname" . }}-agent
+  namespace: {{ include "ambassador.namespace" . }}
+  labels:
+    {{- if ne .Values.deploymentTool "getambassador.io" }}
+    app.kubernetes.io/name: {{ include "ambassador.name" . }}-agent
+    app.kubernetes.io/part-of: {{ .Release.Name }}
+    helm.sh/chart: {{ include "ambassador.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- if .Values.deploymentTool }}
+    app.kubernetes.io/managed-by: {{ .Values.deploymentTool }}
+    {{- else }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    {{- end }}
+    {{- end }}
+    product: aes
+spec:
+  ports:
+    - port: 8006
+      targetPort: grpc
+      protocol: TCP
+      name: grpc
+  selector:
+    app.kubernetes.io/name: {{ include "ambassador.fullname" . }}-agent
+    app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 {{- end }}

charts/emissary-ingress/templates/deployment.yaml

@@ -172,6 +172,8 @@ spec:
             - name: admin
               containerPort: {{ .Values.adminService.port }}
           env:
+            - name: AMBASSADOR_GRPC_METRICS_SINK
+              value: {{ include "ambassador.fullname" . }}:8006
             - name: HOST_IP
               valueFrom:
                 fieldRef:

cmd/agent/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"github.com/datawire/dlib/dgroup"
 	"os"
 
 	"github.com/spf13/cobra"
@@ -50,6 +51,17 @@ func run(cmd *cobra.Command, args []string) error {
 		snapshotURL = fmt.Sprintf(DefaultSnapshotURLFmt, entrypoint.ExternalSnapshotPort)
 	}
 
+	group := dgroup.NewGroup(ctx, dgroup.GroupConfig{})
+
+	group.Go("metrics-server", func(ctx context.Context) error {
+		metricsServer := agent.NewMetricsServer(ambAgent.MetricsRelayHandler)
+		if err := metricsServer.StartServer(ctx); err != nil {
+			dlog.Errorf(ctx, "metrics service failed to listen: %v", err)
+			return err
+		}
+		return nil
+	})
+
 	if err := ambAgent.Watch(ctx, snapshotURL); err != nil {
 		return err
 	}

cmd/example-envoy-metrics-sink/main.go

@@ -29,7 +29,7 @@ func main() {
 
 	dlog.Print(ctx, "starting...")
 
-	if err := sc.ListenAndServe(ctx, ":8123"); err != nil {
+	if err := sc.ListenAndServe(ctx, ":8006"); err != nil {
 		dlog.Errorf(ctx, "shut down with error: %v", err)
 		os.Exit(1)
 	}

docs/releaseNotes.yml

@@ -73,6 +73,15 @@ items:
           `AGENT_CONFIG_RESOURCE_NAME` environment variable in order to allow all
           components (and not only the Ambassador Agent) to authenticate requests to
           Ambassador Cloud.
+          Support for the Envoy V2 API and the `AMBASSADOR_ENVOY_API_VERSION` environment
+
+      - title: Stream metrics from Envoy to the Ambassador cloud
+        type: feature
+        body: >-
+          Support for streaming Envoy metrics about the clusters to Ambassador's cloud.
+        github:
+        - title: 4053
+          link: https://github.com/emissary-ingress/emissary/pull/4053
 
       - title: Support received commands to pause, continue and abort a Rollout via Agent directives
         type: feature

k8s-config/emissary-defaultns/require.yaml

@@ -3,6 +3,7 @@ _anchors:
 resources:
   - { kind: Service,            name: emissary-ingress-admin,              namespace: *namespace }
   - { kind: Service,            name: emissary-ingress,                    namespace: *namespace }
+  - { kind: Service,            name: emissary-ingress-agent,              namespace: *namespace }
   - { kind: ClusterRole,        name: emissary-ingress                                           }
   - { kind: ServiceAccount,     name: emissary-ingress,                    namespace: *namespace }
   - { kind: ClusterRoleBinding, name: emissary-ingress                                           }

k8s-config/emissary-emissaryns/require.yaml

@@ -3,6 +3,7 @@ _anchors:
 resources:
   - { kind: Service,            name: emissary-ingress-admin,              namespace: *namespace }
   - { kind: Service,            name: emissary-ingress,                    namespace: *namespace }
+  - { kind: Service,            name: emissary-ingress-agent,              namespace: *namespace }
   - { kind: ClusterRole,        name: emissary-ingress                                           }
   - { kind: ServiceAccount,     name: emissary-ingress,                    namespace: *namespace }
   - { kind: ClusterRoleBinding, name: emissary-ingress                                           }

manifests/emissary/emissary-defaultns-agent.yaml.in

@@ -260,4 +260,7 @@ spec:
         image: $imageRepo$:$version$
         imagePullPolicy: IfNotPresent
         name: agent
+        ports:
+        - containerPort: 8006
+          name: grpc
       serviceAccountName: emissary-ingress-agent

manifests/emissary/emissary-defaultns-migration.yaml.in

@@ -271,6 +271,8 @@ spec:
             weight: 100
       containers:
       - env:
+        - name: AMBASSADOR_GRPC_METRICS_SINK
+          value: emissary-ingress:8006
         - name: HOST_IP
           valueFrom:
             fieldRef:

manifests/emissary/emissary-defaultns.yaml.in

@@ -74,6 +74,23 @@ spec:
     profile: main
   type: LoadBalancer
 ---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    product: aes
+  name: emissary-ingress-agent
+  namespace: default
+spec:
+  ports:
+  - name: grpc
+    port: 8006
+    protocol: TCP
+    targetPort: grpc
+  selector:
+    app.kubernetes.io/instance: emissary-ingress
+    app.kubernetes.io/name: emissary-ingress-agent
+---
 aggregationRule:
   clusterRoleSelectors:
   - matchLabels:
@@ -271,6 +288,8 @@ spec:
             weight: 100
       containers:
       - env:
+        - name: AMBASSADOR_GRPC_METRICS_SINK
+          value: emissary-ingress:8006
         - name: HOST_IP
           valueFrom:
             fieldRef:
@@ -592,4 +611,7 @@ spec:
         image: $imageRepo$:$version$
         imagePullPolicy: IfNotPresent
         name: agent
+        ports:
+        - containerPort: 8006
+          name: grpc
       serviceAccountName: emissary-ingress-agent

manifests/emissary/emissary-emissaryns-agent.yaml.in

@@ -260,4 +260,7 @@ spec:
         image: $imageRepo$:$version$
         imagePullPolicy: IfNotPresent
         name: agent
+        ports:
+        - containerPort: 8006
+          name: grpc
       serviceAccountName: emissary-ingress-agent