test PR

.github/ISSUE_TEMPLATE/bug.yaml

@@ -1,6 +1,6 @@
 name: Bug
 description: Report a bug
-labels: ['kind/bug', 'needs-triage']
+labels: ['kind/bug']
 body:
 - type: textarea
   attributes:

.github/ISSUE_TEMPLATE/feature.yaml

@@ -1,6 +1,6 @@
 name: Feature
 description: Suggest an idea for a new feature
-labels: ['kind/feature', 'needs-triage']
+labels: ['kind/feature']
 body:
 - type: textarea
   attributes:

.github/actions/install-deps/action.yaml

@@ -7,7 +7,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+    - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
       id: setup-go
       with:
         go-version-file: go.mod

.github/workflows/codeql-analysis.yaml

@@ -17,7 +17,7 @@ jobs:
       actions: read # github/codeql-action/init@v2
       security-events: write # github/codeql-action/init@v2
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
       - uses: ./.github/actions/install-deps
       - run: make vulncheck
       - uses: github/codeql-action/init@df32e399139a3050671466d7d9b3cbacc1cfd034 # v2.22.8
@@ -34,7 +34,7 @@ jobs:
       actions: read # github/codeql-action/init@v2
       security-events: write # github/codeql-action/init@v2
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
       - uses: github/codeql-action/init@df32e399139a3050671466d7d9b3cbacc1cfd034 # v2.22.8
         with:
           languages: javascript

.github/workflows/presubmit.yaml

@@ -15,7 +15,7 @@ jobs:
         matrix:
           k8sVersion: ["1.23.x", "1.24.x", "1.25.x", "1.26.x", "1.27.x", "1.28.x", "1.29.x"]
     steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+    - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
     - uses: ./.github/actions/install-deps
       with:
         k8sVersion: ${{ matrix.k8sVersion }}

.github/workflows/release.yaml

@@ -13,7 +13,7 @@ jobs:
       id-token: write # Needed for cosigning build attestation files with tejolote
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+      - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
         with:
           fetch-depth: 0
 
@@ -32,7 +32,7 @@ jobs:
           prerelease: false
 
       - name: Install tejolote
-        uses: kubernetes-sigs/release-actions/setup-tejolote@841d76a188a7c121231a863572e27012805715a2 # v0.1.4
+        uses: kubernetes-sigs/release-actions/setup-tejolote@2f8b9ec22aedc9ce15039b6c7716aa6c2907df1c # v0.2.0
       - name: Run tejolote
         run: |
           tejolote attest "github://kubernetes-sigs/karpenter/${{ github.run_id }}" --artifacts "github://kubernetes-sigs/karpenter/$TAG" --output karpenter.intoto.json --sign

.github/workflows/test-kind.yaml

@@ -0,0 +1,134 @@
+name: kind-ci
+on:
+  # enable running this on merges + presubmits when the action has tests that it will execute
+  # push:
+  #   branches: [main]
+  # pull_request:
+  workflow_dispatch:
+jobs:
+  test-kind:
+    permissions:
+      issues: write
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k8sVersion: ["1.23.x", "1.24.x", "1.25.x", "1.26.x", "1.27.x", "1.28.x", "1.29.x"]
+    steps:
+    - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5
+    - uses: ./.github/actions/install-deps
+      with:
+          k8sVersion: ${{ matrix.k8sVersion }}
+    - name: Kind Cluster
+      uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+    - name: check kind cluster
+      shell: bash
+      run: |
+        kubectl config current-context
+        kubectl get nodes
+    - name: Enable the actionlint matcher
+      shell: bash
+      run: echo "::add-matcher::.github/actionlint-matcher.json"
+    - name: install pyroscope
+      shell: bash
+      run: | 
+        curl -fL https://github.com/grafana/pyroscope/releases/download/v1.1.5/profilecli_1.1.5_linux_amd64.tar.gz | tar xz profilecli
+        chmod +x profilecli
+        sudo mv profilecli /usr/local/bin
+    - name: install kwok and controller
+      shell: bash
+      run: |
+        make toolchain
+        make install-kwok
+        KWOK_REPO=kind.local KIND_CLUSTER_NAME=chart-testing make apply-with-kind
+    - name: install prometheus and grafana
+      shell: bash
+      run: |
+        helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+        helm repo add grafana https://grafana.github.io/helm-charts
+        helm repo update
+        kubectl create namespace prometheus
+        helm upgrade --install prometheus prometheus-community/kube-prometheus-stack -n prometheus \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].targetLabel=metrics_path" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].action=replace" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].sourceLabels[0]=__metrics_path__" \
+          --wait
+        echo "Installed prometheus"
+        kubectl --namespace prometheus get pods -l "release=prometheus"
+    - name: enable profiling
+      shell: bash
+      run: |
+        kubectl annotate -n kube-system pods profiles.grafana.com/memory.scrape="true" -l app.kubernetes.io/name="karpenter"
+        kubectl annotate -n kube-system pods profiles.grafana.com/memory.port="8000" -l app.kubernetes.io/name="karpenter"
+
+        kubectl annotate -n kube-system pods profiles.grafana.com/cpu.scrape="true" -l app.kubernetes.io/name="karpenter"
+        kubectl annotate -n kube-system pods profiles.grafana.com/cpu.port="8000" -l app.kubernetes.io/name="karpenter"
+
+        kubectl annotate -n kube-system pods profiles.grafana.com/goroutine.scrape="true" -l app.kubernetes.io/name="karpenter"
+        kubectl annotate -n kube-system pods profiles.grafana.com/goroutine.port="8000" -l app.kubernetes.io/name="karpenter"
+        # Testing out pyroscope for profiling
+    - name: apply pyroscope 
+      shell: bash
+      run: |
+        helm -n kube-system install pyroscope grafana/pyroscope
+        #Setup Metrics Endpoint
+        cat <<EOF | envsubst | kubectl apply -f -
+        apiVersion: monitoring.coreos.com/v1
+        kind: ServiceMonitor
+        metadata:
+          name: karpenter
+          labels:
+            release: prometheus
+        spec:
+          selector:
+            matchLabels:
+              app.kubernetes.io/name: karpenter
+          namespaceSelector:
+            matchNames:
+              - kube-system
+          endpoints:
+            - port: "8000"
+              path: /metrics
+        EOF
+    - name: apply nodepool 
+      shell: bash
+      run: |
+        # Setup node pool
+        cat <<EOF | envsubst | kubectl apply -f -
+        apiVersion: karpenter.sh/v1beta1
+        kind: NodePool
+        metadata:
+          name: default
+        spec:
+          template:
+            spec:
+              requirements:
+                - key: kubernetes.io/arch
+                  operator: In
+                  values: ["amd64"]
+                - key: kubernetes.io/os
+                  operator: In
+                  values: ["linux"]
+                - key: karpenter.sh/capacity-type
+                  operator: In
+                  values: ["spot"]
+              nodeClassRef:
+                name: nil
+          limits:
+            cpu: 1500
+          disruption:
+            consolidationPolicy: WhenUnderutilized
+            expireAfter: 720h # 30 * 24h = 720h
+        EOF
+    - name: ping cluster
+      shell: bash
+      run: | 
+        kubectl get pods -n kube-system | grep karpenter 
+        kubectl get nodepools
+        kubectl get pods -A
+    - name: cleanup 
+      shell: bash
+      run: | 
+        kubectl delete nodepools --all 
+        make delete
+        make uninstall-kwok
+

.golangci.yaml

@@ -1,19 +1,9 @@
 # See https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml
 run:
   tests: true
-
   timeout: 5m
-
   build-tags:
     - test_performance
-
-  skip-dirs:
-    - tools
-    - website
-    - hack
-    - charts
-    - designs
-
 linters:
   enable:
     - asciicheck
@@ -34,12 +24,13 @@ linters:
     - nilerr
   disable:
     - prealloc
-
 linters-settings:
   gocyclo:
     min-complexity: 11
   govet:
-    check-shadowing: true
+    enable-all: true
+    disable:
+      - fieldalignment
   revive:
     rules:
       - name: dot-imports
@@ -71,6 +62,12 @@ linters-settings:
 issues:
   fix: true
   exclude: ['declaration of "(err|ctx)" shadows declaration at']
+  exclude-dirs:
+    - tools
+    - website
+    - hack
+    - charts
+    - designs
   exclude-rules:
   - linters:
     - goheader

Makefile

@@ -6,7 +6,7 @@ HELM_OPTS ?= --set logLevel=debug \
 			--set controller.resources.requests.cpu=1 \
 			--set controller.resources.requests.memory=1Gi \
 			--set controller.resources.limits.cpu=1 \
-			--set controller.resources.limits.memory=1Gi 
+			--set controller.resources.limits.memory=1Gi
 
 help: ## Display help
 	@awk 'BEGIN {FS = ":.*##"; printf "Usage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
@@ -19,12 +19,26 @@ install-kwok: ## Install kwok provider
 uninstall-kwok: ## Uninstall kwok provider
 	UNINSTALL_KWOK=true ./hack/install-kwok.sh
 
+build-with-kind: # build with kind assumes the image will be uploaded directly onto the kind control plane, without an image repository
+	$(eval CONTROLLER_IMG=$(shell $(WITH_GOFLAGS) KO_DOCKER_REPO="$(KWOK_REPO)" ko build sigs.k8s.io/karpenter/kwok))
+	$(eval IMG_REPOSITORY=$(shell echo $(CONTROLLER_IMG) | cut -d ":" -f 1))
+	$(eval IMG_TAG=latest) 
+
 build: ## Build the Karpenter KWOK controller images using ko build
 	$(eval CONTROLLER_IMG=$(shell $(WITH_GOFLAGS) KO_DOCKER_REPO="$(KWOK_REPO)" ko build -B sigs.k8s.io/karpenter/kwok))
 	$(eval IMG_REPOSITORY=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 1))
 	$(eval IMG_TAG=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 2 -s))
 	$(eval IMG_DIGEST=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 2))
 
+apply-with-kind: verify build-with-kind ## Deploy the kwok controller from the current state of your git repository into your ~/.kube/config cluster
+	hack/validation/kwok-requirements.sh
+	kubectl apply -f pkg/apis/crds
+	helm upgrade --install karpenter kwok/charts --namespace $(KARPENTER_NAMESPACE) --skip-crds \
+		$(HELM_OPTS) \
+		--set controller.image.repository=$(IMG_REPOSITORY) \
+		--set controller.image.tag=$(IMG_TAG) \
+		--set-string controller.env[0].name=ENABLE_PROFILING \
+		--set-string controller.env[0].value=true
 
 # Run make install-kwok to install the kwok controller in your cluster first
 # Webhooks are currently not supported in the kwok provider.
@@ -41,7 +55,7 @@ apply: verify build ## Deploy the kwok controller from the current state of your
 
 delete: ## Delete the controller from your ~/.kube/config cluster
 	helm uninstall karpenter --namespace $(KARPENTER_NAMESPACE)
-	
+
 test: ## Run tests
 	go test ./... \
 		-race \
@@ -75,6 +89,7 @@ verify: ## Verify code. Includes codegen, docgen, dependencies, linting, formatt
 	hack/validation/requirements.sh
 	hack/validation/labels.sh
 	hack/validation/resources.sh
+	hack/validation/status.sh
 	hack/dependabot.sh
 	@# Use perl instead of sed due to https://stackoverflow.com/questions/4247068/sed-command-with-i-option-failing-on-mac-but-works-on-linux
 	@# We need to do this "sed replace" until controller-tools fixes this parameterized types issue: https://github.com/kubernetes-sigs/controller-tools/issues/756
@@ -96,4 +111,7 @@ download: ## Recursively "go mod download" on all directories where go.mod exist
 toolchain: ## Install developer toolchain
 	./hack/toolchain.sh
 
-.PHONY: help presubmit install-kwok uninstall-kwok build apply delete test deflake vulncheck licenses verify download toolchain
+gen_instance_types:
+	go run kwok/tools/gen_instance_types.go > kwok/cloudprovider/instance_types.json
+
+.PHONY: help presubmit install-kwok uninstall-kwok build apply delete test deflake vulncheck licenses verify download toolchain gen_instance_types

README.md

@@ -15,8 +15,8 @@ Karpenter improves the efficiency and cost of running workloads on Kubernetes cl
 * **Provisioning** nodes that meet the requirements of the pods
 * **Removing** the nodes when the nodes are no longer needed
 
-## Supported CloudProviders 
-Karpenter is a multi-cloud project supported on the following cloud providers 
+## Karpenter Implementations
+Karpenter is a multi-cloud project with implementations by the following cloud providers:
 - [AWS](https://github.com/aws/karpenter-provider-aws)
 - [Azure](https://github.com/Azure/karpenter-provider-azure)