Backport of Displays Consul version of each nodes in UI nodes section…

… into release/1.16.x (#18113)

## Backport

This PR is auto-generated from #17754 to be assessed for backporting due
to the inclusion of the label backport/1.16.


:rotating_light:
>**Warning** automatic cherry-pick of commits failed. If the first
commit failed,
you will see a blank no-op commit below. If at least one commit
succeeded, you
will see the cherry-picked commits up to, _not including_, the commit
where
the merge conflict occurred.

The person who merged in the original PR is:
@WenInCode
This person should manually cherry-pick the original PR into a new
backport PR,
and close this one when the manual backport PR is merged in.

> merge conflict error: unable to process merge commit:
"1c757b8a2c1160ad53421b7b8bd7f74b205c4b89", automatic backport requires
rebase workflow



The below text is copied from the body of the original PR.

---

fixes #17097 Consul version of each nodes in UI nodes section

@jkirschner-hashicorp @huikang @team @maintainers

Updated consul version in the request to register consul.
Added this as Node MetaData.
Fetching this new metadata in UI

<img width="1512" alt="Screenshot 2023-06-15 at 4 21 33 PM"
src="https://github.com/hashicorp/consul/assets/3139634/94f7cf6b-701f-4230-b9f7-d8c4342d0737">

Also made this backward compatible and tested.

Backward compatible in this context means - If consul binary with above
PR changes is deployed to one of node, and if UI is run from this node,
then the version of not only current (upgraded) node is displayed in UI
, but also of older nodes given that they are consul servers only.
For older (non-server or client) nodes the version is not added in
NodeMeta Data and hence the version will not be displayed for them.
If a old node is consul server, the version will be displayed. As the
endpoint - "v1/internal/ui/nodes?dc=dc1" was already returning version
in service meta. This is made use of in current UI changes.

<img width="1480" alt="Screenshot 2023-06-16 at 6 58 32 PM"
src="https://github.com/hashicorp/consul/assets/3139634/257942f4-fbed-437d-a492-37849d2bec4c">




---

<details>
<summary> Overview of commits </summary>

- 931fdfc -
b3e2ec1 -
8d0e9a5 -
04e5d88 -
28286a2 -
43e50ad -
0cf1b70 -
27f34ce -
2ac76d6 -
3d618df -
1c757b8 -
23ce82b -
4dc1c9b -
85a12a9 -
25d30a3 -
7f1d619 -
5174cbf

</details>

---------

Co-authored-by: Vijay Srinivas <vijayraghav22@gmail.com>
Co-authored-by: John Murret <john.murret@hashicorp.com>
Co-authored-by: Jared Kirschner <85913323+jkirschner-hashicorp@users.noreply.github.com>

.changelog/17754.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+ui: consul version is displayed in nodes list with filtering and sorting based on versions
+```

agent/agent.go

@@ -3999,6 +3999,7 @@ func (a *Agent) loadMetadata(conf *config.RuntimeConfig) error {
 		meta[k] = v
 	}
 	meta[structs.MetaSegmentKey] = conf.SegmentName
+	meta[structs.MetaConsulVersion] = conf.Version
 	return a.State.LoadMetadata(meta)
 }
 

agent/agent_endpoint_test.go

@@ -1508,7 +1508,8 @@ func TestAgent_Self(t *testing.T) {
 			require.NoError(t, err)
 			require.Equal(t, cs[a.config.SegmentName], val.Coord)
 
-			delete(val.Meta, structs.MetaSegmentKey) // Added later, not in config.
+			delete(val.Meta, structs.MetaSegmentKey)    // Added later, not in config.
+			delete(val.Meta, structs.MetaConsulVersion) // Added later, not in config.
 			require.Equal(t, a.config.NodeMeta, val.Meta)
 
 			if tc.expectXDS {

agent/consul/leader.go

@@ -1087,6 +1087,13 @@ AFTER_CHECK:
 		"partition", getSerfMemberEnterpriseMeta(member).PartitionOrDefault(),
 	)
 
+	// Get consul version from serf member
+	// add this as node meta in catalog register request
+	buildVersion, err := metadata.Build(&member)
+	if err != nil {
+		return err
+	}
+
 	// Register with the catalog.
 	req := structs.RegisterRequest{
 		Datacenter: s.config.Datacenter,
@@ -1102,6 +1109,9 @@ AFTER_CHECK:
 			Output:  structs.SerfCheckAliveOutput,
 		},
 		EnterpriseMeta: *nodeEntMeta,
+		NodeMeta: map[string]string{
+			structs.MetaConsulVersion: buildVersion.String(),
+		},
 	}
 	if node != nil {
 		req.TaggedAddresses = node.TaggedAddresses

agent/consul/state/catalog.go

@@ -3450,6 +3450,13 @@ func parseNodes(tx ReadTxn, ws memdb.WatchSet, idx uint64,
 		ws.AddWithLimit(watchLimit, services.WatchCh(), allServicesCh)
 		for service := services.Next(); service != nil; service = services.Next() {
 			ns := service.(*structs.ServiceNode).ToNodeService()
+			// If version isn't defined in node meta, set it from the Consul service meta
+			if _, ok := dump.Meta[structs.MetaConsulVersion]; !ok && ns.ID == "consul" && ns.Meta["version"] != "" {
+				if dump.Meta == nil {
+					dump.Meta = make(map[string]string)
+				}
+				dump.Meta[structs.MetaConsulVersion] = ns.Meta["version"]
+			}
 			dump.Services = append(dump.Services, ns)
 		}
 

agent/consul/state/catalog_test.go

@@ -4837,6 +4837,9 @@ func TestStateStore_NodeInfo_NodeDump(t *testing.T) {
 	}
 
 	// Register some nodes
+	// node1 is registered withOut any nodemeta, and a consul service with id
+	// 'consul' is added later with meta 'version'. The expected node must have
+	// meta 'consul-version' with same value
 	testRegisterNode(t, s, 0, "node1")
 	testRegisterNode(t, s, 1, "node2")
 
@@ -4845,6 +4848,8 @@ func TestStateStore_NodeInfo_NodeDump(t *testing.T) {
 	testRegisterService(t, s, 3, "node1", "service2")
 	testRegisterService(t, s, 4, "node2", "service1")
 	testRegisterService(t, s, 5, "node2", "service2")
+	// Register consul service with meta 'version' for node1
+	testRegisterServiceWithMeta(t, s, 10, "node1", "consul", map[string]string{"version": "1.17.0"})
 
 	// Register service-level checks
 	testRegisterCheck(t, s, 6, "node1", "service1", "check1", api.HealthPassing)
@@ -4894,6 +4899,19 @@ func TestStateStore_NodeInfo_NodeDump(t *testing.T) {
 				},
 			},
 			Services: []*structs.NodeService{
+				{
+					ID:      "consul",
+					Service: "consul",
+					Address: "1.1.1.1",
+					Meta:    map[string]string{"version": "1.17.0"},
+					Port:    1111,
+					Weights: &structs.Weights{Passing: 1, Warning: 1},
+					RaftIndex: structs.RaftIndex{
+						CreateIndex: 10,
+						ModifyIndex: 10,
+					},
+					EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(),
+				},
 				{
 					ID:      "service1",
 					Service: "service1",
@@ -4921,6 +4939,7 @@ func TestStateStore_NodeInfo_NodeDump(t *testing.T) {
 					EnterpriseMeta: *structs.DefaultEnterpriseMetaInDefaultPartition(),
 				},
 			},
+			Meta: map[string]string{"consul-version": "1.17.0"},
 		},
 		&structs.NodeInfo{
 			Node:      "node2",
@@ -4988,7 +5007,7 @@ func TestStateStore_NodeInfo_NodeDump(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
-	if idx != 9 {
+	if idx != 10 {
 		t.Fatalf("bad index: %d", idx)
 	}
 	require.Len(t, dump, 1)
@@ -4999,8 +5018,8 @@ func TestStateStore_NodeInfo_NodeDump(t *testing.T) {
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
-	if idx != 9 {
-		t.Fatalf("bad index: %d", 9)
+	if idx != 10 {
+		t.Fatalf("bad index: %d", idx)
 	}
 	if !reflect.DeepEqual(dump, expect) {
 		t.Fatalf("bad: %#v", dump[0].Services[0])

agent/consul/state/state_store_test.go

@@ -189,6 +189,37 @@ func testRegisterServiceWithChangeOpts(t *testing.T, s *Store, idx uint64, nodeI
 	return svc
 }
 
+// testRegisterServiceWithMeta registers service with Meta passed as arg.
+func testRegisterServiceWithMeta(t *testing.T, s *Store, idx uint64, nodeID, serviceID string, meta map[string]string, opts ...func(service *structs.NodeService)) *structs.NodeService {
+	svc := &structs.NodeService{
+		ID:      serviceID,
+		Service: serviceID,
+		Address: "1.1.1.1",
+		Port:    1111,
+		Meta:    meta,
+	}
+	for _, o := range opts {
+		o(svc)
+	}
+
+	if err := s.EnsureService(idx, nodeID, svc); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	tx := s.db.Txn(false)
+	defer tx.Abort()
+	service, err := tx.First(tableServices, indexID, NodeServiceQuery{Node: nodeID, Service: serviceID, PeerName: svc.PeerName})
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if result, ok := service.(*structs.ServiceNode); !ok ||
+		result.Node != nodeID ||
+		result.ServiceID != serviceID {
+		t.Fatalf("bad service: %#v", result)
+	}
+	return svc
+}
+
 // testRegisterService register a service with given transaction idx
 // If the service already exists, transaction number might not be increased
 // Use `testRegisterServiceWithChange()` if you want perform a registration that

agent/local/state_test.go

@@ -189,7 +189,8 @@ func TestAgentAntiEntropy_Services(t *testing.T) {
 	id := services.NodeServices.Node.ID
 	addrs := services.NodeServices.Node.TaggedAddresses
 	meta := services.NodeServices.Node.Meta
-	delete(meta, structs.MetaSegmentKey) // Added later, not in config.
+	delete(meta, structs.MetaSegmentKey)    // Added later, not in config.
+	delete(meta, structs.MetaConsulVersion) // Added later, not in config.
 	assert.Equal(t, a.Config.NodeID, id)
 	assert.Equal(t, a.Config.TaggedAddresses, addrs)
 	assert.Equal(t, unNilMap(a.Config.NodeMeta), meta)
@@ -1355,7 +1356,8 @@ func TestAgentAntiEntropy_Checks(t *testing.T) {
 			id := services.NodeServices.Node.ID
 			addrs := services.NodeServices.Node.TaggedAddresses
 			meta := services.NodeServices.Node.Meta
-			delete(meta, structs.MetaSegmentKey) // Added later, not in config.
+			delete(meta, structs.MetaSegmentKey)    // Added later, not in config.
+			delete(meta, structs.MetaConsulVersion) // Added later, not in config.
 			assert.Equal(r, a.Config.NodeID, id)
 			assert.Equal(r, a.Config.TaggedAddresses, addrs)
 			assert.Equal(r, unNilMap(a.Config.NodeMeta), meta)
@@ -2016,7 +2018,8 @@ func TestAgentAntiEntropy_NodeInfo(t *testing.T) {
 	addrs := services.NodeServices.Node.TaggedAddresses
 	meta := services.NodeServices.Node.Meta
 	nodeLocality := services.NodeServices.Node.Locality
-	delete(meta, structs.MetaSegmentKey) // Added later, not in config.
+	delete(meta, structs.MetaSegmentKey)    // Added later, not in config.
+	delete(meta, structs.MetaConsulVersion) // Added later, not in config.
 	require.Equal(t, a.Config.NodeID, id)
 	require.Equal(t, a.Config.TaggedAddresses, addrs)
 	require.Equal(t, a.Config.StructLocality(), nodeLocality)
@@ -2041,7 +2044,8 @@ func TestAgentAntiEntropy_NodeInfo(t *testing.T) {
 		addrs := services.NodeServices.Node.TaggedAddresses
 		meta := services.NodeServices.Node.Meta
 		nodeLocality := services.NodeServices.Node.Locality
-		delete(meta, structs.MetaSegmentKey) // Added later, not in config.
+		delete(meta, structs.MetaSegmentKey)    // Added later, not in config.
+		delete(meta, structs.MetaConsulVersion) // Added later, not in config.
 		require.Equal(t, nodeID, id)
 		require.Equal(t, a.Config.TaggedAddresses, addrs)
 		require.Equal(t, a.Config.StructLocality(), nodeLocality)

agent/structs/structs.go

@@ -220,6 +220,9 @@ const (
 	// WildcardSpecifier is the string which should be used for specifying a wildcard
 	// The exact semantics of the wildcard is left up to the code where its used.
 	WildcardSpecifier = "*"
+
+	// MetaConsulVersion is the node metadata key used to store the node's consul version
+	MetaConsulVersion = "consul-version"
 )
 
 var allowedConsulMetaKeysForMeshGateway = map[string]struct{}{MetaWANFederationKey: {}}

agent/ui_endpoint.go

@@ -13,9 +13,12 @@ import (
 	"strings"
 
 	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/serf/serf"
 
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/config"
+	"github.com/hashicorp/consul/agent/consul"
+	"github.com/hashicorp/consul/agent/metadata"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/consul/logging"
@@ -110,32 +113,104 @@ RPC:
 		return nil, err
 	}
 
+	// Get version info for all serf members into a map of key-address,value-version.
+	// This logic of calling 'AgentMembersMapAddrVer()' and inserting version info in this func
+	// can be discarded in future releases ( may be after 3 or 4 minor releases),
+	// when all the nodes are registered with consul-version in nodemeta.
+	var err error
+	mapAddrVer, err := AgentMembersMapAddrVer(s, req)
+	if err != nil {
+		return nil, err
+	}
+
 	// Use empty list instead of nil
+	// Also check if consul-version exists in Meta, else add it
 	for _, info := range out.Dump {
 		if info.Services == nil {
 			info.Services = make([]*structs.NodeService, 0)
 		}
 		if info.Checks == nil {
 			info.Checks = make([]*structs.HealthCheck, 0)
 		}
+		// Check if Node Meta - 'consul-version' already exists by virtue of adding
+		// 'consul-version' during node registration itself.
+		// If not, get it from mapAddrVer.
+		if _, ok := info.Meta[structs.MetaConsulVersion]; !ok {
+			if _, okver := mapAddrVer[info.Address]; okver {
+				if info.Meta == nil {
+					info.Meta = make(map[string]string)
+				}
+				info.Meta[structs.MetaConsulVersion] = mapAddrVer[info.Address]
+			}
+		}
 	}
 	if out.Dump == nil {
 		out.Dump = make(structs.NodeDump, 0)
 	}
 
 	// Use empty list instead of nil
+	// Also check if consul-version exists in Meta, else add it
 	for _, info := range out.ImportedDump {
 		if info.Services == nil {
 			info.Services = make([]*structs.NodeService, 0)
 		}
 		if info.Checks == nil {
 			info.Checks = make([]*structs.HealthCheck, 0)
 		}
+		// Check if Node Meta - 'consul-version' already exists by virtue of adding
+		// 'consul-version' during node registration itself.
+		// If not, get it from mapAddrVer.
+		if _, ok := info.Meta[structs.MetaConsulVersion]; !ok {
+			if _, okver := mapAddrVer[info.Address]; okver {
+				if info.Meta == nil {
+					info.Meta = make(map[string]string)
+				}
+				info.Meta[structs.MetaConsulVersion] = mapAddrVer[info.Address]
+			}
+		}
 	}
 
 	return append(out.Dump, out.ImportedDump...), nil
 }
 
+// AgentMembersMapAddrVer is used to get version info from  all serf members into a
+// map of key-address,value-version.
+func AgentMembersMapAddrVer(s *HTTPHandlers, req *http.Request) (map[string]string, error) {
+	var members []serf.Member
+
+	//Get WAN Members
+	wanMembers := s.agent.WANMembers()
+
+	//Get LAN Members
+	//Get the request partition and default to that of the agent.
+	entMeta := s.agent.AgentEnterpriseMeta()
+	if err := s.parseEntMetaPartition(req, entMeta); err != nil {
+		return nil, err
+	}
+	filter := consul.LANMemberFilter{
+		Partition: entMeta.PartitionOrDefault(),
+	}
+	filter.AllSegments = true
+	lanMembers, err := s.agent.delegate.LANMembers(filter)
+	if err != nil {
+		return nil, err
+	}
+
+	//aggregate members
+	members = append(wanMembers, lanMembers...)
+
+	//create a map with key as IPv4 address and value as consul-version
+	mapAddrVer := make(map[string]string, len(members))
+	for i := range members {
+		buildVersion, err := metadata.Build(&members[i])
+		if err == nil {
+			mapAddrVer[members[i].Addr.String()] = buildVersion.String()
+		}
+	}
+
+	return mapAddrVer, nil
+}
+
 // UINodeInfo is used to get info on a single node in a given datacenter. We return a
 // NodeInfo which provides overview information for the node
 func (s *HTTPHandlers) UINodeInfo(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
@@ -172,6 +247,16 @@ RPC:
 		return nil, err
 	}
 
+	// Get version info for all serf members into a map of key-address,value-version.
+	// This logic of calling 'AgentMembersMapAddrVer()' and inserting version info in this func
+	// can be discarded in future releases ( may be after 3 or 4 minor releases),
+	// when all the nodes are registered with consul-version in nodemeta.
+	var err error
+	mapAddrVer, err := AgentMembersMapAddrVer(s, req)
+	if err != nil {
+		return nil, err
+	}
+
 	// Return only the first entry
 	if len(out.Dump) > 0 {
 		info := out.Dump[0]
@@ -181,6 +266,17 @@ RPC:
 		if info.Checks == nil {
 			info.Checks = make([]*structs.HealthCheck, 0)
 		}
+		// Check if Node Meta - 'consul-version' already exists by virtue of adding
+		// 'consul-version' during node registration itself.
+		// If not, get it from mapAddrVer.
+		if _, ok := info.Meta[structs.MetaConsulVersion]; !ok {
+			if _, okver := mapAddrVer[info.Address]; okver {
+				if info.Meta == nil {
+					info.Meta = make(map[string]string)
+				}
+				info.Meta[structs.MetaConsulVersion] = mapAddrVer[info.Address]
+			}
+		}
 		return info, nil
 	}
 

agent/ui_endpoint_test.go

@@ -162,6 +162,9 @@ func TestUINodes(t *testing.T) {
 	require.Len(t, nodes[2].Services, 0)
 	require.NotNil(t, nodes[1].Checks)
 	require.Len(t, nodes[2].Services, 0)
+
+	// check for consul-version in node meta
+	require.Equal(t, nodes[0].Meta[structs.MetaConsulVersion], a.Config.Version)
 }
 
 func TestUINodes_Filter(t *testing.T) {
@@ -260,6 +263,9 @@ func TestUINodeInfo(t *testing.T) {
 		node.Checks == nil || len(node.Checks) != 0 {
 		t.Fatalf("bad: %v", node)
 	}
+
+	// check for consul-version in node meta
+	require.Equal(t, node.Meta[structs.MetaConsulVersion], a.Config.Version)
 }
 
 func TestUIServices(t *testing.T) {

api/catalog_test.go

@@ -65,6 +65,7 @@ func TestAPI_CatalogNodes(t *testing.T) {
 			},
 			Meta: map[string]string{
 				"consul-network-segment": "",
+				"consul-version":         s.Config.Version,
 			},
 		}
 		require.Equal(r, want, got)