Merge pull request #512 from sivanzcw/dev

Add queue controller about state

pkg/controllers/queue/queue_controller_action.go

@@ -0,0 +1,167 @@
+/*
+Copyright 2019 The Volcano Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package queue
+
+import (
+	"fmt"
+	"reflect"
+
+	schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2"
+	"volcano.sh/volcano/pkg/controllers/queue/state"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/cache"
+
+	"k8s.io/klog"
+)
+
+func (c *Controller) syncQueue(queue *schedulingv1alpha2.Queue, updateStateFn state.UpdateQueueStatusFn) error {
+	klog.V(4).Infof("Begin to sync queue %s.", queue.Name)
+
+	podGroups := c.getPodGroups(queue.Name)
+	queueStatus := schedulingv1alpha2.QueueStatus{}
+
+	for _, pgKey := range podGroups {
+		// Ignore error here, tt can not occur.
+		ns, name, _ := cache.SplitMetaNamespaceKey(pgKey)
+
+		// TODO: check NotFound error and sync local cache.
+		pg, err := c.pgLister.PodGroups(ns).Get(name)
+		if err != nil {
+			return err
+		}
+
+		switch pg.Status.Phase {
+		case schedulingv1alpha2.PodGroupPending:
+			queueStatus.Pending++
+		case schedulingv1alpha2.PodGroupRunning:
+			queueStatus.Running++
+		case schedulingv1alpha2.PodGroupUnknown:
+			queueStatus.Unknown++
+		case schedulingv1alpha2.PodGroupInqueue:
+			queueStatus.Inqueue++
+		}
+	}
+
+	if updateStateFn != nil {
+		updateStateFn(&queueStatus, podGroups)
+	} else {
+		queueStatus.State = queue.Status.State
+	}
+
+	// ignore update when status does not change
+	if reflect.DeepEqual(queueStatus, queue.Status) {
+		return nil
+	}
+
+	newQueue := queue.DeepCopy()
+	newQueue.Status = queueStatus
+	if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil {
+		klog.Errorf("Failed to update status of Queue %s: %v.", newQueue.Name, err)
+		return err
+	}
+
+	return nil
+}
+
+func (c *Controller) openQueue(queue *schedulingv1alpha2.Queue, updateStateFn state.UpdateQueueStatusFn) error {
+	klog.V(4).Infof("Begin to open queue %s.", queue.Name)
+
+	newQueue := queue.DeepCopy()
+	newQueue.Spec.State = schedulingv1alpha2.QueueStateOpen
+
+	if queue.Spec.State != newQueue.Spec.State {
+		if _, err := c.vcClient.SchedulingV1alpha2().Queues().Update(newQueue); err != nil {
+			c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.OpenQueueAction),
+				fmt.Sprintf("Open queue failed for %v", err))
+			return err
+		}
+
+		c.recorder.Event(newQueue, v1.EventTypeNormal, string(schedulingv1alpha2.OpenQueueAction),
+			fmt.Sprintf("Open queue succeed"))
+	} else {
+		return nil
+	}
+
+	q, err := c.vcClient.SchedulingV1alpha2().Queues().Get(newQueue.Name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	newQueue = q.DeepCopy()
+	if updateStateFn != nil {
+		updateStateFn(&newQueue.Status, nil)
+	} else {
+		return fmt.Errorf("internal error, update state function should be provided")
+	}
+
+	if queue.Status.State != newQueue.Status.State {
+		if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil {
+			c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.OpenQueueAction),
+				fmt.Sprintf("Update queue status from %s to %s failed for %v",
+					queue.Status.State, newQueue.Status.State, err))
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (c *Controller) closeQueue(queue *schedulingv1alpha2.Queue, updateStateFn state.UpdateQueueStatusFn) error {
+	klog.V(4).Infof("Begin to close queue %s.", queue.Name)
+
+	newQueue := queue.DeepCopy()
+	newQueue.Spec.State = schedulingv1alpha2.QueueStateClosed
+
+	if queue.Spec.State != newQueue.Spec.State {
+		if _, err := c.vcClient.SchedulingV1alpha2().Queues().Update(newQueue); err != nil {
+			c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.CloseQueueAction),
+				fmt.Sprintf("Close queue failed for %v", err))
+			return err
+		}
+
+		c.recorder.Event(newQueue, v1.EventTypeNormal, string(schedulingv1alpha2.CloseQueueAction),
+			fmt.Sprintf("Close queue succeed"))
+	} else {
+		return nil
+	}
+
+	q, err := c.vcClient.SchedulingV1alpha2().Queues().Get(newQueue.Name, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	newQueue = q.DeepCopy()
+	podGroups := c.getPodGroups(newQueue.Name)
+	if updateStateFn != nil {
+		updateStateFn(&newQueue.Status, podGroups)
+	} else {
+		return fmt.Errorf("internal error, update state function should be provided")
+	}
+
+	if queue.Status.State != newQueue.Status.State {
+		if _, err := c.vcClient.SchedulingV1alpha2().Queues().UpdateStatus(newQueue); err != nil {
+			c.recorder.Event(newQueue, v1.EventTypeWarning, string(schedulingv1alpha2.CloseQueueAction),
+				fmt.Sprintf("Update queue status from %s to %s failed for %v",
+					queue.Status.State, newQueue.Status.State, err))
+			return err
+		}
+	}
+
+	return nil
+}

pkg/controllers/queue/queue_controller_handler.go

@@ -0,0 +1,186 @@
+/*
+Copyright 2019 The Volcano Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package queue
+
+import (
+	busv1alpha1 "volcano.sh/volcano/pkg/apis/bus/v1alpha1"
+	schedulingv1alpha2 "volcano.sh/volcano/pkg/apis/scheduling/v1alpha2"
+
+	"k8s.io/client-go/tools/cache"
+
+	"k8s.io/klog"
+)
+
+func (c *Controller) enqueue(req *schedulingv1alpha2.QueueRequest) {
+	c.queue.Add(req)
+}
+
+func (c *Controller) addQueue(obj interface{}) {
+	queue := obj.(*schedulingv1alpha2.Queue)
+
+	req := &schedulingv1alpha2.QueueRequest{
+		Name: queue.Name,
+
+		Event:  schedulingv1alpha2.QueueOutOfSyncEvent,
+		Action: schedulingv1alpha2.SyncQueueAction,
+	}
+
+	c.enqueue(req)
+}
+
+func (c *Controller) deleteQueue(obj interface{}) {
+	queue, ok := obj.(*schedulingv1alpha2.Queue)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			klog.Errorf("Couldn't get object from tombstone %#v.", obj)
+			return
+		}
+		queue, ok = tombstone.Obj.(*schedulingv1alpha2.Queue)
+		if !ok {
+			klog.Errorf("Tombstone contained object that is not a Queue: %#v.", obj)
+			return
+		}
+	}
+
+	c.pgMutex.Lock()
+	defer c.pgMutex.Unlock()
+	delete(c.podGroups, queue.Name)
+}
+
+func (c *Controller) updateQueue(old, new interface{}) {
+	oldQueue, ok := old.(*schedulingv1alpha2.Queue)
+	if !ok {
+		klog.Errorf("Can not covert old object %v to queues.scheduling.sigs.dev.", old)
+		return
+	}
+
+	newQueue, ok := new.(*schedulingv1alpha2.Queue)
+	if !ok {
+		klog.Errorf("Can not covert new object %v to queues.scheduling.sigs.dev.", old)
+		return
+	}
+
+	if oldQueue.ResourceVersion == newQueue.ResourceVersion {
+		return
+	}
+
+	c.addQueue(newQueue)
+
+	return
+}
+
+func (c *Controller) addPodGroup(obj interface{}) {
+	pg := obj.(*schedulingv1alpha2.PodGroup)
+	key, _ := cache.MetaNamespaceKeyFunc(obj)
+
+	c.pgMutex.Lock()
+	defer c.pgMutex.Unlock()
+
+	if c.podGroups[pg.Spec.Queue] == nil {
+		c.podGroups[pg.Spec.Queue] = make(map[string]struct{})
+	}
+	c.podGroups[pg.Spec.Queue][key] = struct{}{}
+
+	req := &schedulingv1alpha2.QueueRequest{
+		Name: pg.Spec.Queue,
+
+		Event:  schedulingv1alpha2.QueueOutOfSyncEvent,
+		Action: schedulingv1alpha2.SyncQueueAction,
+	}
+
+	c.enqueue(req)
+}
+
+func (c *Controller) updatePodGroup(old, new interface{}) {
+	oldPG := old.(*schedulingv1alpha2.PodGroup)
+	newPG := new.(*schedulingv1alpha2.PodGroup)
+
+	// Note: we have no use case update PodGroup.Spec.Queue
+	// So do not consider it here.
+	if oldPG.Status.Phase != newPG.Status.Phase {
+		c.addPodGroup(newPG)
+	}
+}
+
+func (c *Controller) deletePodGroup(obj interface{}) {
+	pg, ok := obj.(*schedulingv1alpha2.PodGroup)
+	if !ok {
+		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			klog.Errorf("Couldn't get object from tombstone %#v.", obj)
+			return
+		}
+		pg, ok = tombstone.Obj.(*schedulingv1alpha2.PodGroup)
+		if !ok {
+			klog.Errorf("Tombstone contained object that is not a PodGroup: %#v.", obj)
+			return
+		}
+	}
+
+	key, _ := cache.MetaNamespaceKeyFunc(obj)
+
+	c.pgMutex.Lock()
+	defer c.pgMutex.Unlock()
+
+	delete(c.podGroups[pg.Spec.Queue], key)
+
+	req := &schedulingv1alpha2.QueueRequest{
+		Name: pg.Spec.Queue,
+
+		Event:  schedulingv1alpha2.QueueOutOfSyncEvent,
+		Action: schedulingv1alpha2.SyncQueueAction,
+	}
+
+	c.enqueue(req)
+}
+
+func (c *Controller) addCommand(obj interface{}) {
+	cmd, ok := obj.(*busv1alpha1.Command)
+	if !ok {
+		klog.Errorf("Obj %v is not command.", obj)
+		return
+	}
+
+	c.commandQueue.Add(cmd)
+}
+
+func (c *Controller) getPodGroups(key string) []string {
+	c.pgMutex.RLock()
+	defer c.pgMutex.RUnlock()
+
+	if c.podGroups[key] == nil {
+		return nil
+	}
+	podGroups := make([]string, 0, len(c.podGroups[key]))
+	for pgKey := range c.podGroups[key] {
+		podGroups = append(podGroups, pgKey)
+	}
+
+	return podGroups
+}
+
+func (c *Controller) recordEventsForQueue(name, eventType, reason, message string) {
+	queue, err := c.queueLister.Get(name)
+	if err != nil {
+		klog.Errorf("Get queue %s failed for %v.", name, err)
+		return
+	}
+
+	c.recorder.Event(queue, eventType, reason, message)
+	return
+}

pkg/controllers/queue/queue_controller_test.go

@@ -271,7 +271,7 @@ func TestSyncQueue(t *testing.T) {
 		c.queueInformer.Informer().GetIndexer().Add(testcase.queue)
 		c.vcClient.SchedulingV1alpha2().Queues().Create(testcase.queue)
 
-		err := c.syncQueue(testcase.queue.Name)
+		err := c.syncQueue(testcase.queue, nil)
 		item, _ := c.vcClient.SchedulingV1alpha2().Queues().Get(testcase.queue.Name, metav1.GetOptions{})
 		if err != nil && testcase.ExpectValue != item.Status.Pending {
 			t.Errorf("case %d (%s): expected: %v, got %v ", i, testcase.Name, testcase.ExpectValue, c.queue.Len())