test: Changes to add KPIs to e2e action (#1321)

Co-authored-by: Nick Tran <nichotr@amazon.com> Co-authored-by: Nick Tran <10810510+njtran@users.noreply.github.com>
kubernetes-sigs · Jul 1, 2024 · b445b7f · b445b7f
1 parent 89a81c3
commit b445b7f
Show file tree

Hide file tree

Showing 5 changed files with 141 additions and 10 deletions.
diff --git a/.github/actions/install-prometheus/action.yaml b/.github/actions/install-prometheus/action.yaml
@@ -22,4 +22,4 @@ runs:
         --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].targetLabel=metrics_path" \
         --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].action=replace" \
         --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].sourceLabels[0]=__metrics_path__" \
-        --wait
+        --wait
diff --git a/.github/actions/install-prometheus/values.yaml b/.github/actions/install-prometheus/values.yaml
@@ -44,9 +44,115 @@ prometheus:
     tolerations:
       - key: CriticalAddonsOnly
         operator: Exists
-    serviceMonitorSelector:
-      matchLabels:
-        scrape: enabled
-    serviceMonitorNamespaceSelector:
-      matchLabels:
-        scrape: enabled
+    serviceMonitorSelector: { }
+    serviceMonitorNamespaceSelector: { }
+    additionalScrapeConfigs:
+      - job_name: serviceMonitor/kube-system/karpenter/0
+        honor_timestamps: true
+        track_timestamps_staleness: false
+        scrape_interval: 30s
+        scrape_timeout: 10s
+        scrape_protocols:
+          - OpenMetricsText1.0.0
+          - OpenMetricsText0.0.1
+          - PrometheusText0.0.4
+        metrics_path: /metrics
+        scheme: http
+        enable_compression: true
+        follow_redirects: true
+        enable_http2: true
+        relabel_configs:
+          - source_labels: [ job ]
+            separator: ;
+            regex: (.*)
+            target_label: __tmp_prometheus_job_name
+            replacement: $1
+            action: replace
+          - source_labels: [ __meta_kubernetes_service_label_app_kubernetes_io_instance, __meta_kubernetes_service_labelpresent_app_kubernetes_io_instance ]
+            separator: ;
+            regex: (karpenter);true
+            replacement: $1
+            action: keep
+          - source_labels: [ __meta_kubernetes_service_label_app_kubernetes_io_name, __meta_kubernetes_service_labelpresent_app_kubernetes_io_name ]
+            separator: ;
+            regex: (karpenter);true
+            replacement: $1
+            action: keep
+          - source_labels: [ __meta_kubernetes_endpoint_port_name ]
+            separator: ;
+            regex: http-metrics
+            replacement: $1
+            action: keep
+          - source_labels: [ __meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name ]
+            separator: ;
+            regex: Node;(.*)
+            target_label: node
+            replacement: ${1}
+            action: replace
+          - source_labels: [ __meta_kubernetes_endpoint_address_target_kind, __meta_kubernetes_endpoint_address_target_name ]
+            separator: ;
+            regex: Pod;(.*)
+            target_label: pod
+            replacement: ${1}
+            action: replace
+          - source_labels: [ __meta_kubernetes_namespace ]
+            separator: ;
+            regex: (.*)
+            target_label: namespace
+            replacement: $1
+            action: replace
+          - source_labels: [ __meta_kubernetes_service_name ]
+            separator: ;
+            regex: (.*)
+            target_label: service
+            replacement: $1
+            action: replace
+          - source_labels: [ __meta_kubernetes_pod_name ]
+            separator: ;
+            regex: (.*)
+            target_label: pod
+            replacement: $1
+            action: replace
+          - source_labels: [ __meta_kubernetes_pod_container_name ]
+            separator: ;
+            regex: (.*)
+            target_label: container
+            replacement: $1
+            action: replace
+          - source_labels: [ __meta_kubernetes_pod_phase ]
+            separator: ;
+            regex: (Failed|Succeeded)
+            replacement: $1
+            action: drop
+          - source_labels: [ __meta_kubernetes_service_name ]
+            separator: ;
+            regex: (.*)
+            target_label: job
+            replacement: ${1}
+            action: replace
+          - separator: ;
+            regex: (.*)
+            target_label: endpoint
+            replacement: http-metrics
+            action: replace
+          - source_labels: [ __address__ ]
+            separator: ;
+            regex: (.*)
+            modulus: 1
+            target_label: __tmp_hash
+            replacement: $1
+            action: hashmod
+          - source_labels: [ __tmp_hash ]
+            separator: ;
+            regex: "0"
+            replacement: $1
+            action: keep
+        kubernetes_sd_configs:
+          - role: endpoints
+            kubeconfig_file: ""
+            follow_redirects: true
+            enable_http2: true
+            namespaces:
+              own_namespace: false
+              names:
+                - kube-system
diff --git a/.github/workflows/kind-e2e.yaml b/.github/workflows/kind-e2e.yaml
@@ -15,6 +15,10 @@ jobs:
         k8sVersion: ["1.23.x", "1.24.x", "1.25.x", "1.26.x", "1.27.x", "1.28.x", "1.29.x", "1.30.x"]
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+    - name: Set up Python 3.10
+      uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1
+      with:
+        python-version: "3.10"
     - uses: ./.github/actions/install-deps
       with:
           k8sVersion: ${{ matrix.k8sVersion }}
@@ -35,18 +39,38 @@ jobs:
       run: |
         make toolchain
         make install-kwok
-        KWOK_REPO=kind.local KIND_CLUSTER_NAME=chart-testing make apply-with-kind
+        export KWOK_REPO=kind.local
+        export KIND_CLUSTER_NAME=chart-testing
+        make apply-with-kind
     - name: ping cluster
       shell: bash
       run: | 
+        sleep 15
         kubectl get pods -n kube-system | grep karpenter 
         kubectl get nodepools
         kubectl get pods -A
         kubectl describe nodes
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      with:
+        repository: nathangeology/karpenter_evaluate
+        path: ./karpenter_eval/ # Installs to a folder in the Karpenter repo for the test
+        ref: "3f4cebb703bd136f5034c8b5bf3e6c32e97d1ae6"
+        fetch-depth: 0
+    - name: install KPI report dependencies
+      shell: bash
+      run: |
+        pip install pandas==2.2.2
+        pip install pyarrow==16.1.0 
+        pip install tabulate==0.9.0
+        pip install prometheus-api-client==0.5.5
+        pip install ./karpenter_eval/
     - name: run test suites
       shell: bash
-      run: | 
+      run: |
+        OUTPUT_DIR=$(mktemp -d)
+        export OUTPUT_DIR
         make e2etests
+        python ./karpenter_eval/main.py
     - name: cleanup 
       shell: bash
       run: | 

diff --git a/Makefile b/Makefile
@@ -49,7 +49,7 @@ e2etests: ## Run the e2e suite against your local cluster
 		--ginkgo.focus="${FOCUS}" \
 		--ginkgo.timeout=30m \
 		--ginkgo.grace-period=5m \
-		--ginkgo.vv	
+		--ginkgo.vv
 
 # Run make install-kwok to install the kwok controller in your cluster first
 # Webhooks are currently not supported in the kwok provider.

diff --git a/test/suites/perf/scheduling_test.go b/test/suites/perf/scheduling_test.go
@@ -147,4 +147,5 @@ var _ = Describe("Performance", func() {
 			env.TimeIntervalCollector.End("Drift")
 		})
 	})
+
 })