From 53bfe7d7bb10e853ed842a4b523231bdc4f1553e Mon Sep 17 00:00:00 2001
From: Andrew Kerr <andrew.kerr@netapp.com>
Date: Tue, 16 Jun 2020 13:48:44 -0400
Subject: [PATCH] Reconcile Trident nodes with current state of Kubernetes
 cluster at startup

Closes #391
---
 frontend/csi/helpers/kubernetes/helper.go |  2 +-
 frontend/csi/helpers/kubernetes/plugin.go | 47 +++++++++++++++++++++++
 frontend/csi/helpers/plain/plugin.go      |  1 +
 frontend/csi/helpers/types.go             |  2 +-
 4 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/frontend/csi/helpers/kubernetes/helper.go b/frontend/csi/helpers/kubernetes/helper.go
index add04098e..49458a5dd 100644
--- a/frontend/csi/helpers/kubernetes/helper.go
+++ b/frontend/csi/helpers/kubernetes/helper.go
@@ -1,4 +1,4 @@
-// Copyright 2019 NetApp, Inc. All Rights Reserved.
+// Copyright 2020 NetApp, Inc. All Rights Reserved.
 package kubernetes
 
 import (
diff --git a/frontend/csi/helpers/kubernetes/plugin.go b/frontend/csi/helpers/kubernetes/plugin.go
index 89ed3db80..535815e1f 100644
--- a/frontend/csi/helpers/kubernetes/plugin.go
+++ b/frontend/csi/helpers/kubernetes/plugin.go
@@ -340,6 +340,7 @@ func (p *Plugin) Activate() error {
 	go p.pvController.Run(p.pvControllerStopChan)
 	go p.scController.Run(p.scControllerStopChan)
 	go p.nodeController.Run(p.nodeControllerStopChan)
+	go p.reconcileNodes()
 
 	// Configure telemetry
 	config.OrchestratorTelemetry.Platform = string(config.PlatformKubernetes)
@@ -372,6 +373,52 @@ func (p *Plugin) Version() string {
 	return p.kubeVersion.GitVersion
 }
 
+// listClusterNodes returns the list of worker node names as a map for kubernetes cluster
+func (p *Plugin) listClusterNodes() (map[string]bool, error) {
+	nodeNames := make(map[string]bool)
+	nodes, err := p.kubeClient.CoreV1().Nodes().List(ctx(), listOpts)
+	if err != nil {
+		err = fmt.Errorf("error reading kubernetes nodes; %v", err)
+		log.Error(err)
+		return nodeNames, err
+	}
+	for _, node := range nodes.Items {
+		nodeNames[node.Name] = true
+	}
+	return nodeNames, nil
+}
+
+// reconcileNodes will make sure that Trident's list of nodes does not include any unnecessary node
+func (p *Plugin) reconcileNodes() {
+	log.Debug("Performing node reconciliation.")
+	clusterNodes, err := p.listClusterNodes()
+	if err != nil {
+		log.WithField("err", err).Errorf("unable to list nodes in Kubernetes; aborting node reconciliation")
+		return
+	}
+	tridentNodes, err := p.orchestrator.ListNodes()
+	if err != nil {
+		log.WithField("err", err).Errorf("unable to list nodes in Trident; aborting node reconciliation")
+		return
+	}
+
+	for _, node := range tridentNodes {
+		if _, ok := clusterNodes[node.Name]; !ok {
+			// Trident node no longer exists in cluster, remove it
+			log.WithField("node", node.Name).
+				Debug("Node not found in Kubernetes; removing from Trident.")
+			err = p.orchestrator.DeleteNode(node.Name)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"node": node.Name,
+					"err":  err,
+				}).Error("error removing node from Trident")
+			}
+		}
+	}
+	log.Debug("Node reconciliation complete.")
+}
+
 // addPVC is the add handler for the PVC watcher.
 func (p *Plugin) addPVC(obj interface{}) {
 	switch pvc := obj.(type) {
diff --git a/frontend/csi/helpers/plain/plugin.go b/frontend/csi/helpers/plain/plugin.go
index 10054d080..c69f18994 100644
--- a/frontend/csi/helpers/plain/plugin.go
+++ b/frontend/csi/helpers/plain/plugin.go
@@ -1,3 +1,4 @@
+// Copyright 2020 NetApp, Inc. All Rights Reserved.
 package plain
 
 import (
diff --git a/frontend/csi/helpers/types.go b/frontend/csi/helpers/types.go
index d76a96741..5369b8dc4 100644
--- a/frontend/csi/helpers/types.go
+++ b/frontend/csi/helpers/types.go
@@ -1,4 +1,4 @@
-// Copyright 2019 NetApp, Inc. All Rights Reserved.
+// Copyright 2020 NetApp, Inc. All Rights Reserved.
 
 package helpers