Introduce customized scheduler to resolve issues from tekton

scheduler/.ko.yaml

@@ -0,0 +1 @@
+defaultBaseImage: gcr.io/distroless/static:nonroot

scheduler/OWNERS

@@ -0,0 +1,8 @@
+# The OWNERS file is used by prow to automatically merge approved PRs.
+# See https://github.com/kubernetes/test-infra/tree/master/prow/plugins/approve/approvers.
+# It supports a list of approvers and reviewers and their github names must be provided.
+
+approvers:
+- ImJasonH
+- jlpettersson 
+- vincent-pli

scheduler/README.md

@@ -0,0 +1,23 @@
+# coscheduler-same-node
+The repo is for trying to implements such a scheduler:
+https://github.com/tektoncd/pipeline/issues/3052
+
+# Installation
+```
+ko apply -f config/
+```
+
+# Take a try
+`kubectl create -f examples/pods.yaml`
+
+# Description
+The `scheduler` is based on [Scheduler plugin framework](https://github.com/kubernetes/enhancements/blob/master/keps/sig-scheduling/20180409-scheduling-framework.md)
+
+Use special labels of `pod` to mark the group of pods.
+```
+labels:
+     pod-group.scheduling.sigs.k8s.io/name: test
+     pod-group.scheduling.sigs.k8s.io/total: "2"
+```
+
+The `pod`s in same `pod-group` will be scheduler to same node if the node can satisfy the resource requirement of whole group, or all pods in the group will `pending`.

scheduler/cmd/main.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2020 The Tekton Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"math/rand"
+	"os"
+	"time"
+
+	"k8s.io/kubernetes/cmd/kube-scheduler/app"
+
+	"github.com/tektoncd/experimental/scheduler/pkg/coscheduler"
+)
+
+func main() {
+	rand.Seed(time.Now().UnixNano())
+	// Register custom plugins to the scheduler framework.
+	// Later they can consist of scheduler profile(s) and hence
+	// used by various kinds of workloads.
+	command := app.NewSchedulerCommand(
+		app.WithPlugin(coscheduler.CoschedulerName, coscheduler.NewCoscheduler),
+	)
+	if err := command.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

scheduler/config/100-serviceaccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sample-sa
+  namespace: kube-system

scheduler/config/101-clusterrole.yaml

@@ -0,0 +1,134 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: sample-cr
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+      - events
+    verbs:
+      - create
+      - get
+      - update
+  - apiGroups:
+      - ""
+    resourceNames:
+      - scheduler-framework-sample
+    resources:
+      - endpoints
+    verbs:
+      - delete
+      - get
+      - patch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - delete
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - bindings
+      - pods/binding
+    verbs:
+      - create
+  - apiGroups:
+      - ""
+    resources:
+      - pods/status
+    verbs:
+      - patch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - replicationcontrollers
+      - services
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - apps
+      - extensions
+    resources:
+      - replicasets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - policy
+    resources:
+      - poddisruptionbudgets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumeclaims
+      - persistentvolumes
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "storage.k8s.io"
+    resources:
+      - storageclasses
+      - csinodes
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - "coordination.k8s.io"
+    resources:
+      - leases
+    verbs:
+      - create
+      - get
+      - list
+      - update
+  - apiGroups:
+      - "events.k8s.io"
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update

scheduler/config/102-clusterrolebinding.yaml

@@ -0,0 +1,13 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: sample-crb
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: sample-cr
+subjects:
+  - kind: ServiceAccount
+    name: sample-sa
+    namespace: kube-system

scheduler/config/201-configmap.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: scheduler-config
+  namespace: kube-system
+data:
+  scheduler-config.yaml: |
+    apiVersion: kubescheduler.config.k8s.io/v1alpha1
+    kind: KubeSchedulerConfiguration
+    schedulerName: scheduler-framework-sample
+    leaderElection:
+      leaderElect: true
+      lockObjectName: scheduler-framework-sample
+      lockObjectNamespace: kube-system
+    plugins:
+      queueSort:
+        enabled:
+        - name: CoschedulerSamenode
+        disabled:
+        - name: "*"
+      preFilter:
+        enabled:
+        - name: CoschedulerSamenode
+      filter:
+        enabled:
+        - name: CoschedulerSamenode
+      postBind:
+        enabled:
+        - name: CoschedulerSamenode

scheduler/config/500-deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: scheduler-framework-sample
+  namespace: kube-system
+  labels:
+    component: scheduler-framework-sample
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: scheduler-framework-sample
+  template:
+    metadata:
+      labels:
+        component: scheduler-framework-sample
+    spec:
+      serviceAccount: sample-sa
+      priorityClassName: system-cluster-critical
+      volumes:
+      - name: scheduler-config
+        configMap:
+          name: scheduler-config
+      containers:
+      - name: scheduler-ctrl
+        image: ko://github.com/tektoncd/experimental/scheduler/cmd
+        imagePullPolicy: Always
+        args:
+        - kube-scheduler
+        - --config=/scheduler/scheduler-config.yaml
+        - --v=3
+        resources:
+          requests:
+            cpu: "50m"
+        volumeMounts:
+        - name: scheduler-config
+          mountPath: /scheduler

scheduler/example/pods.yaml

@@ -0,0 +1,43 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: static-web
+  labels:
+    pod-group.scheduling.sigs.k8s.io/name: test
+    pod-group.scheduling.sigs.k8s.io/total: "2"
+spec:
+  schedulerName: scheduler-framework-sample
+  containers:
+    - name: web
+      image: nginx
+      resources:
+        requests:
+          cpu: "4000m"
+        limits:
+          cpu: "4000m"
+      ports:
+        - name: web
+          containerPort: 80
+          protocol: TCP
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: static-web-1
+  labels:
+    pod-group.scheduling.sigs.k8s.io/name: test
+    pod-group.scheduling.sigs.k8s.io/total: "2"
+spec:
+  schedulerName: scheduler-framework-sample
+  containers:
+    - name: web
+      image: nginx
+      resources:
+        requests:
+          cpu: "4000m"
+        limits:
+          cpu: "4000m"
+      ports:
+        - name: web
+          containerPort: 80
+          protocol: TCP

scheduler/go.mod

@@ -0,0 +1,36 @@
+module github.com/tektoncd/experimental/scheduler
+
+go 1.13
+
+require (
+	k8s.io/api v0.18.0
+	k8s.io/apimachinery v0.18.0
+	k8s.io/apiserver v0.18.0
+	k8s.io/client-go v0.18.0
+	k8s.io/klog v1.0.0
+	k8s.io/kubernetes v1.18.0
+)
+
+replace (
+	k8s.io/api => k8s.io/api v0.18.0
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.0
+	k8s.io/apimachinery => k8s.io/apimachinery v0.18.0
+	k8s.io/apiserver => k8s.io/apiserver v0.18.0
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.18.0
+	k8s.io/client-go => k8s.io/client-go v0.18.0
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.18.0
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.18.0
+	k8s.io/code-generator => k8s.io/code-generator v0.18.0
+	k8s.io/component-base => k8s.io/component-base v0.18.0
+	k8s.io/cri-api => k8s.io/cri-api v0.18.0
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.18.0
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.18.0
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.18.0
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.18.0
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.18.0
+	k8s.io/kubectl => k8s.io/kubectl v0.18.0
+	k8s.io/kubelet => k8s.io/kubelet v0.18.0
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.18.0
+	k8s.io/metrics => k8s.io/metrics v0.18.0
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.0
+)